• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15"""configure script to get build parameters from user."""
16
17from __future__ import absolute_import
18from __future__ import division
19from __future__ import print_function
20
21import argparse
22import errno
23import glob
24import os
25import platform
26import re
27import subprocess
28import sys
29
30# pylint: disable=g-import-not-at-top
31try:
32  from shutil import which
33except ImportError:
34  from distutils.spawn import find_executable as which
35# pylint: enable=g-import-not-at-top
36
37_DEFAULT_CUDA_VERSION = '10'
38_DEFAULT_CUDNN_VERSION = '7'
39_DEFAULT_TENSORRT_VERSION = '6'
40_DEFAULT_CUDA_COMPUTE_CAPABILITIES = '3.5,7.0'
41
42_SUPPORTED_ANDROID_NDK_VERSIONS = [
43    10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21
44]
45
46_DEFAULT_PROMPT_ASK_ATTEMPTS = 10
47
48_TF_BAZELRC_FILENAME = '.tf_configure.bazelrc'
49_TF_WORKSPACE_ROOT = ''
50_TF_BAZELRC = ''
51_TF_CURRENT_BAZEL_VERSION = None
52_TF_MIN_BAZEL_VERSION = '3.7.2'
53_TF_MAX_BAZEL_VERSION = '4.99.0'
54
55NCCL_LIB_PATHS = [
56    'lib64/', 'lib/powerpc64le-linux-gnu/', 'lib/x86_64-linux-gnu/', ''
57]
58
59# List of files to configure when building Bazel on Apple platforms.
60APPLE_BAZEL_FILES = [
61    'tensorflow/lite/ios/BUILD', 'tensorflow/lite/objc/BUILD',
62    'tensorflow/lite/swift/BUILD',
63    'tensorflow/lite/tools/benchmark/experimental/ios/BUILD'
64]
65
66# List of files to move when building for iOS.
67IOS_FILES = [
68    'tensorflow/lite/objc/TensorFlowLiteObjC.podspec',
69    'tensorflow/lite/swift/TensorFlowLiteSwift.podspec',
70]
71
72
73class UserInputError(Exception):
74  pass
75
76
77def is_windows():
78  return platform.system() == 'Windows'
79
80
81def is_linux():
82  return platform.system() == 'Linux'
83
84
85def is_macos():
86  return platform.system() == 'Darwin'
87
88
89def is_ppc64le():
90  return platform.machine() == 'ppc64le'
91
92
93def is_cygwin():
94  return platform.system().startswith('CYGWIN_NT')
95
96
97def get_input(question):
98  try:
99    try:
100      answer = raw_input(question)
101    except NameError:
102      answer = input(question)  # pylint: disable=bad-builtin
103  except EOFError:
104    answer = ''
105  return answer
106
107
108def symlink_force(target, link_name):
109  """Force symlink, equivalent of 'ln -sf'.
110
111  Args:
112    target: items to link to.
113    link_name: name of the link.
114  """
115  try:
116    os.symlink(target, link_name)
117  except OSError as e:
118    if e.errno == errno.EEXIST:
119      os.remove(link_name)
120      os.symlink(target, link_name)
121    else:
122      raise e
123
124
125def sed_in_place(filename, old, new):
126  """Replace old string with new string in file.
127
128  Args:
129    filename: string for filename.
130    old: string to replace.
131    new: new string to replace to.
132  """
133  with open(filename, 'r') as f:
134    filedata = f.read()
135  newdata = filedata.replace(old, new)
136  with open(filename, 'w') as f:
137    f.write(newdata)
138
139
140def write_to_bazelrc(line):
141  with open(_TF_BAZELRC, 'a') as f:
142    f.write(line + '\n')
143
144
145def write_action_env_to_bazelrc(var_name, var):
146  write_to_bazelrc('build --action_env {}="{}"'.format(var_name, str(var)))
147
148
149def run_shell(cmd, allow_non_zero=False, stderr=None):
150  if stderr is None:
151    stderr = sys.stdout
152  if allow_non_zero:
153    try:
154      output = subprocess.check_output(cmd, stderr=stderr)
155    except subprocess.CalledProcessError as e:
156      output = e.output
157  else:
158    output = subprocess.check_output(cmd, stderr=stderr)
159  return output.decode('UTF-8').strip()
160
161
162def cygpath(path):
163  """Convert path from posix to windows."""
164  return os.path.abspath(path).replace('\\', '/')
165
166
167def get_python_path(environ_cp, python_bin_path):
168  """Get the python site package paths."""
169  python_paths = []
170  if environ_cp.get('PYTHONPATH'):
171    python_paths = environ_cp.get('PYTHONPATH').split(':')
172  try:
173    stderr = open(os.devnull, 'wb')
174    library_paths = run_shell([
175        python_bin_path, '-c',
176        'import site; print("\\n".join(site.getsitepackages()))'
177    ],
178                              stderr=stderr).split('\n')
179  except subprocess.CalledProcessError:
180    library_paths = [
181        run_shell([
182            python_bin_path, '-c',
183            'from distutils.sysconfig import get_python_lib;'
184            'print(get_python_lib())'
185        ])
186    ]
187
188  all_paths = set(python_paths + library_paths)
189  # Sort set so order is deterministic
190  all_paths = sorted(all_paths)
191
192  paths = []
193  for path in all_paths:
194    if os.path.isdir(path):
195      paths.append(path)
196  return paths
197
198
199def get_python_major_version(python_bin_path):
200  """Get the python major version."""
201  return run_shell([python_bin_path, '-c', 'import sys; print(sys.version[0])'])
202
203
204def setup_python(environ_cp):
205  """Setup python related env variables."""
206  # Get PYTHON_BIN_PATH, default is the current running python.
207  default_python_bin_path = sys.executable
208  ask_python_bin_path = ('Please specify the location of python. [Default is '
209                         '{}]: ').format(default_python_bin_path)
210  while True:
211    python_bin_path = get_from_env_or_user_or_default(environ_cp,
212                                                      'PYTHON_BIN_PATH',
213                                                      ask_python_bin_path,
214                                                      default_python_bin_path)
215    # Check if the path is valid
216    if os.path.isfile(python_bin_path) and os.access(python_bin_path, os.X_OK):
217      break
218    elif not os.path.exists(python_bin_path):
219      print('Invalid python path: {} cannot be found.'.format(python_bin_path))
220    else:
221      print('{} is not executable.  Is it the python binary?'.format(
222          python_bin_path))
223    environ_cp['PYTHON_BIN_PATH'] = ''
224
225  # Convert python path to Windows style before checking lib and version
226  if is_windows() or is_cygwin():
227    python_bin_path = cygpath(python_bin_path)
228
229  # Get PYTHON_LIB_PATH
230  python_lib_path = environ_cp.get('PYTHON_LIB_PATH')
231  if not python_lib_path:
232    python_lib_paths = get_python_path(environ_cp, python_bin_path)
233    if environ_cp.get('USE_DEFAULT_PYTHON_LIB_PATH') == '1':
234      python_lib_path = python_lib_paths[0]
235    else:
236      print('Found possible Python library paths:\n  %s' %
237            '\n  '.join(python_lib_paths))
238      default_python_lib_path = python_lib_paths[0]
239      python_lib_path = get_input(
240          'Please input the desired Python library path to use.  '
241          'Default is [{}]\n'.format(python_lib_paths[0]))
242      if not python_lib_path:
243        python_lib_path = default_python_lib_path
244    environ_cp['PYTHON_LIB_PATH'] = python_lib_path
245
246  python_major_version = get_python_major_version(python_bin_path)
247  if python_major_version == '2':
248    write_to_bazelrc('build --host_force_python=PY2')
249
250  # Convert python path to Windows style before writing into bazel.rc
251  if is_windows() or is_cygwin():
252    python_lib_path = cygpath(python_lib_path)
253
254  # Set-up env variables used by python_configure.bzl
255  write_action_env_to_bazelrc('PYTHON_BIN_PATH', python_bin_path)
256  write_action_env_to_bazelrc('PYTHON_LIB_PATH', python_lib_path)
257  write_to_bazelrc('build --python_path=\"{}"'.format(python_bin_path))
258  environ_cp['PYTHON_BIN_PATH'] = python_bin_path
259
260  # If choosen python_lib_path is from a path specified in the PYTHONPATH
261  # variable, need to tell bazel to include PYTHONPATH
262  if environ_cp.get('PYTHONPATH'):
263    python_paths = environ_cp.get('PYTHONPATH').split(':')
264    if python_lib_path in python_paths:
265      write_action_env_to_bazelrc('PYTHONPATH', environ_cp.get('PYTHONPATH'))
266
267  # Write tools/python_bin_path.sh
268  with open(
269      os.path.join(_TF_WORKSPACE_ROOT, 'tools', 'python_bin_path.sh'),
270      'w') as f:
271    f.write('export PYTHON_BIN_PATH="{}"'.format(python_bin_path))
272
273
274def reset_tf_configure_bazelrc():
275  """Reset file that contains customized config settings."""
276  open(_TF_BAZELRC, 'w').close()
277
278
279def cleanup_makefile():
280  """Delete any leftover BUILD files from the Makefile build.
281
282  These files could interfere with Bazel parsing.
283  """
284  makefile_download_dir = os.path.join(_TF_WORKSPACE_ROOT, 'tensorflow',
285                                       'contrib', 'makefile', 'downloads')
286  if os.path.isdir(makefile_download_dir):
287    for root, _, filenames in os.walk(makefile_download_dir):
288      for f in filenames:
289        if f.endswith('BUILD'):
290          os.remove(os.path.join(root, f))
291
292
293def get_var(environ_cp,
294            var_name,
295            query_item,
296            enabled_by_default,
297            question=None,
298            yes_reply=None,
299            no_reply=None):
300  """Get boolean input from user.
301
302  If var_name is not set in env, ask user to enable query_item or not. If the
303  response is empty, use the default.
304
305  Args:
306    environ_cp: copy of the os.environ.
307    var_name: string for name of environment variable, e.g. "TF_NEED_CUDA".
308    query_item: string for feature related to the variable, e.g. "CUDA for
309      Nvidia GPUs".
310    enabled_by_default: boolean for default behavior.
311    question: optional string for how to ask for user input.
312    yes_reply: optional string for reply when feature is enabled.
313    no_reply: optional string for reply when feature is disabled.
314
315  Returns:
316    boolean value of the variable.
317
318  Raises:
319    UserInputError: if an environment variable is set, but it cannot be
320      interpreted as a boolean indicator, assume that the user has made a
321      scripting error, and will continue to provide invalid input.
322      Raise the error to avoid infinitely looping.
323  """
324  if not question:
325    question = 'Do you wish to build TensorFlow with {} support?'.format(
326        query_item)
327  if not yes_reply:
328    yes_reply = '{} support will be enabled for TensorFlow.'.format(query_item)
329  if not no_reply:
330    no_reply = 'No {}'.format(yes_reply)
331
332  yes_reply += '\n'
333  no_reply += '\n'
334
335  if enabled_by_default:
336    question += ' [Y/n]: '
337  else:
338    question += ' [y/N]: '
339
340  var = environ_cp.get(var_name)
341  if var is not None:
342    var_content = var.strip().lower()
343    true_strings = ('1', 't', 'true', 'y', 'yes')
344    false_strings = ('0', 'f', 'false', 'n', 'no')
345    if var_content in true_strings:
346      var = True
347    elif var_content in false_strings:
348      var = False
349    else:
350      raise UserInputError(
351          'Environment variable %s must be set as a boolean indicator.\n'
352          'The following are accepted as TRUE : %s.\n'
353          'The following are accepted as FALSE: %s.\n'
354          'Current value is %s.' %
355          (var_name, ', '.join(true_strings), ', '.join(false_strings), var))
356
357  while var is None:
358    user_input_origin = get_input(question)
359    user_input = user_input_origin.strip().lower()
360    if user_input == 'y':
361      print(yes_reply)
362      var = True
363    elif user_input == 'n':
364      print(no_reply)
365      var = False
366    elif not user_input:
367      if enabled_by_default:
368        print(yes_reply)
369        var = True
370      else:
371        print(no_reply)
372        var = False
373    else:
374      print('Invalid selection: {}'.format(user_input_origin))
375  return var
376
377
378def set_build_var(environ_cp,
379                  var_name,
380                  query_item,
381                  option_name,
382                  enabled_by_default,
383                  bazel_config_name=None):
384  """Set if query_item will be enabled for the build.
385
386  Ask user if query_item will be enabled. Default is used if no input is given.
387  Set subprocess environment variable and write to .bazelrc if enabled.
388
389  Args:
390    environ_cp: copy of the os.environ.
391    var_name: string for name of environment variable, e.g. "TF_NEED_CUDA".
392    query_item: string for feature related to the variable, e.g. "CUDA for
393      Nvidia GPUs".
394    option_name: string for option to define in .bazelrc.
395    enabled_by_default: boolean for default behavior.
396    bazel_config_name: Name for Bazel --config argument to enable build feature.
397  """
398
399  var = str(int(get_var(environ_cp, var_name, query_item, enabled_by_default)))
400  environ_cp[var_name] = var
401  if var == '1':
402    write_to_bazelrc('build:%s --define %s=true' %
403                     (bazel_config_name, option_name))
404    write_to_bazelrc('build --config=%s' % bazel_config_name)
405  elif bazel_config_name is not None:
406    # TODO(mikecase): Migrate all users of configure.py to use --config Bazel
407    # options and not to set build configs through environment variables.
408    write_to_bazelrc('build:%s --define %s=true' %
409                     (bazel_config_name, option_name))
410
411
412def set_action_env_var(environ_cp,
413                       var_name,
414                       query_item,
415                       enabled_by_default,
416                       question=None,
417                       yes_reply=None,
418                       no_reply=None,
419                       bazel_config_name=None):
420  """Set boolean action_env variable.
421
422  Ask user if query_item will be enabled. Default is used if no input is given.
423  Set environment variable and write to .bazelrc.
424
425  Args:
426    environ_cp: copy of the os.environ.
427    var_name: string for name of environment variable, e.g. "TF_NEED_CUDA".
428    query_item: string for feature related to the variable, e.g. "CUDA for
429      Nvidia GPUs".
430    enabled_by_default: boolean for default behavior.
431    question: optional string for how to ask for user input.
432    yes_reply: optional string for reply when feature is enabled.
433    no_reply: optional string for reply when feature is disabled.
434    bazel_config_name: adding config to .bazelrc instead of action_env.
435  """
436  var = int(
437      get_var(environ_cp, var_name, query_item, enabled_by_default, question,
438              yes_reply, no_reply))
439
440  if not bazel_config_name:
441    write_action_env_to_bazelrc(var_name, var)
442  elif var:
443    write_to_bazelrc('build --config=%s' % bazel_config_name)
444  environ_cp[var_name] = str(var)
445
446
447def convert_version_to_int(version):
448  """Convert a version number to a integer that can be used to compare.
449
450  Version strings of the form X.YZ and X.Y.Z-xxxxx are supported. The
451  'xxxxx' part, for instance 'homebrew' on OS/X, is ignored.
452
453  Args:
454    version: a version to be converted
455
456  Returns:
457    An integer if converted successfully, otherwise return None.
458  """
459  version = version.split('-')[0]
460  version_segments = version.split('.')
461  # Treat "0.24" as "0.24.0"
462  if len(version_segments) == 2:
463    version_segments.append('0')
464  for seg in version_segments:
465    if not seg.isdigit():
466      return None
467
468  version_str = ''.join(['%03d' % int(seg) for seg in version_segments])
469  return int(version_str)
470
471
472def check_bazel_version(min_version, max_version):
473  """Check installed bazel version is between min_version and max_version.
474
475  Args:
476    min_version: string for minimum bazel version (must exist!).
477    max_version: string for maximum bazel version (must exist!).
478
479  Returns:
480    The bazel version detected.
481  """
482  bazel_executable = which('bazel')
483  if bazel_executable is None:
484    print('Cannot find bazel. Please install bazel.')
485    sys.exit(1)
486
487  stderr = open(os.devnull, 'wb')
488  curr_version = run_shell([bazel_executable, '--version'],
489                           allow_non_zero=True,
490                           stderr=stderr)
491  if curr_version.startswith('bazel '):
492    curr_version = curr_version.split('bazel ')[1]
493
494  min_version_int = convert_version_to_int(min_version)
495  curr_version_int = convert_version_to_int(curr_version)
496  max_version_int = convert_version_to_int(max_version)
497
498  # Check if current bazel version can be detected properly.
499  if not curr_version_int:
500    print('WARNING: current bazel installation is not a release version.')
501    print('Make sure you are running at least bazel %s' % min_version)
502    return curr_version
503
504  print('You have bazel %s installed.' % curr_version)
505
506  if curr_version_int < min_version_int:
507    print('Please upgrade your bazel installation to version %s or higher to '
508          'build TensorFlow!' % min_version)
509    sys.exit(1)
510  if (curr_version_int > max_version_int and
511      'TF_IGNORE_MAX_BAZEL_VERSION' not in os.environ):
512    print('Please downgrade your bazel installation to version %s or lower to '
513          'build TensorFlow! To downgrade: download the installer for the old '
514          'version (from https://github.com/bazelbuild/bazel/releases) then '
515          'run the installer.' % max_version)
516    sys.exit(1)
517  return curr_version
518
519
520def set_cc_opt_flags(environ_cp):
521  """Set up architecture-dependent optimization flags.
522
523  Also append CC optimization flags to bazel.rc..
524
525  Args:
526    environ_cp: copy of the os.environ.
527  """
528  if is_ppc64le():
529    # gcc on ppc64le does not support -march, use mcpu instead
530    default_cc_opt_flags = '-mcpu=native'
531  elif is_windows():
532    default_cc_opt_flags = '/arch:AVX'
533  else:
534    # On all other platforms, no longer use `-march=native` as this can result
535    # in instructions that are too modern being generated. Users that want
536    # maximum performance should compile TF in their environment and can pass
537    # `-march=native` there.
538    # See https://github.com/tensorflow/tensorflow/issues/45744 and duplicates
539    default_cc_opt_flags = '-Wno-sign-compare'
540  question = ('Please specify optimization flags to use during compilation when'
541              ' bazel option "--config=opt" is specified [Default is %s]: '
542             ) % default_cc_opt_flags
543  cc_opt_flags = get_from_env_or_user_or_default(environ_cp, 'CC_OPT_FLAGS',
544                                                 question, default_cc_opt_flags)
545  for opt in cc_opt_flags.split():
546    write_to_bazelrc('build:opt --copt=%s' % opt)
547    write_to_bazelrc('build:opt --host_copt=%s' % opt)
548
549
550def set_tf_cuda_clang(environ_cp):
551  """set TF_CUDA_CLANG action_env.
552
553  Args:
554    environ_cp: copy of the os.environ.
555  """
556  question = 'Do you want to use clang as CUDA compiler?'
557  yes_reply = 'Clang will be used as CUDA compiler.'
558  no_reply = 'nvcc will be used as CUDA compiler.'
559  set_action_env_var(
560      environ_cp,
561      'TF_CUDA_CLANG',
562      None,
563      False,
564      question=question,
565      yes_reply=yes_reply,
566      no_reply=no_reply,
567      bazel_config_name='cuda_clang')
568
569
570def set_tf_download_clang(environ_cp):
571  """Set TF_DOWNLOAD_CLANG action_env."""
572  question = 'Do you wish to download a fresh release of clang? (Experimental)'
573  yes_reply = 'Clang will be downloaded and used to compile tensorflow.'
574  no_reply = 'Clang will not be downloaded.'
575  set_action_env_var(
576      environ_cp,
577      'TF_DOWNLOAD_CLANG',
578      None,
579      False,
580      question=question,
581      yes_reply=yes_reply,
582      no_reply=no_reply,
583      bazel_config_name='download_clang')
584
585
586def get_from_env_or_user_or_default(environ_cp, var_name, ask_for_var,
587                                    var_default):
588  """Get var_name either from env, or user or default.
589
590  If var_name has been set as environment variable, use the preset value, else
591  ask for user input. If no input is provided, the default is used.
592
593  Args:
594    environ_cp: copy of the os.environ.
595    var_name: string for name of environment variable, e.g. "TF_NEED_CUDA".
596    ask_for_var: string for how to ask for user input.
597    var_default: default value string.
598
599  Returns:
600    string value for var_name
601  """
602  var = environ_cp.get(var_name)
603  if not var:
604    var = get_input(ask_for_var)
605    print('\n')
606  if not var:
607    var = var_default
608  return var
609
610
611def set_clang_cuda_compiler_path(environ_cp):
612  """Set CLANG_CUDA_COMPILER_PATH."""
613  default_clang_path = which('clang') or ''
614  ask_clang_path = ('Please specify which clang should be used as device and '
615                    'host compiler. [Default is %s]: ') % default_clang_path
616
617  while True:
618    clang_cuda_compiler_path = get_from_env_or_user_or_default(
619        environ_cp, 'CLANG_CUDA_COMPILER_PATH', ask_clang_path,
620        default_clang_path)
621    if os.path.exists(clang_cuda_compiler_path):
622      break
623
624    # Reset and retry
625    print('Invalid clang path: %s cannot be found.' % clang_cuda_compiler_path)
626    environ_cp['CLANG_CUDA_COMPILER_PATH'] = ''
627
628  # Set CLANG_CUDA_COMPILER_PATH
629  environ_cp['CLANG_CUDA_COMPILER_PATH'] = clang_cuda_compiler_path
630  write_action_env_to_bazelrc('CLANG_CUDA_COMPILER_PATH',
631                              clang_cuda_compiler_path)
632
633
634def prompt_loop_or_load_from_env(environ_cp,
635                                 var_name,
636                                 var_default,
637                                 ask_for_var,
638                                 check_success,
639                                 error_msg,
640                                 suppress_default_error=False,
641                                 resolve_symlinks=False,
642                                 n_ask_attempts=_DEFAULT_PROMPT_ASK_ATTEMPTS):
643  """Loop over user prompts for an ENV param until receiving a valid response.
644
645  For the env param var_name, read from the environment or verify user input
646  until receiving valid input. When done, set var_name in the environ_cp to its
647  new value.
648
649  Args:
650    environ_cp: (Dict) copy of the os.environ.
651    var_name: (String) string for name of environment variable, e.g. "TF_MYVAR".
652    var_default: (String) default value string.
653    ask_for_var: (String) string for how to ask for user input.
654    check_success: (Function) function that takes one argument and returns a
655      boolean. Should return True if the value provided is considered valid. May
656      contain a complex error message if error_msg does not provide enough
657      information. In that case, set suppress_default_error to True.
658    error_msg: (String) String with one and only one '%s'. Formatted with each
659      invalid response upon check_success(input) failure.
660    suppress_default_error: (Bool) Suppress the above error message in favor of
661      one from the check_success function.
662    resolve_symlinks: (Bool) Translate symbolic links into the real filepath.
663    n_ask_attempts: (Integer) Number of times to query for valid input before
664      raising an error and quitting.
665
666  Returns:
667    [String] The value of var_name after querying for input.
668
669  Raises:
670    UserInputError: if a query has been attempted n_ask_attempts times without
671      success, assume that the user has made a scripting error, and will
672      continue to provide invalid input. Raise the error to avoid infinitely
673      looping.
674  """
675  default = environ_cp.get(var_name) or var_default
676  full_query = '%s [Default is %s]: ' % (
677      ask_for_var,
678      default,
679  )
680
681  for _ in range(n_ask_attempts):
682    val = get_from_env_or_user_or_default(environ_cp, var_name, full_query,
683                                          default)
684    if check_success(val):
685      break
686    if not suppress_default_error:
687      print(error_msg % val)
688    environ_cp[var_name] = ''
689  else:
690    raise UserInputError('Invalid %s setting was provided %d times in a row. '
691                         'Assuming to be a scripting mistake.' %
692                         (var_name, n_ask_attempts))
693
694  if resolve_symlinks and os.path.islink(val):
695    val = os.path.realpath(val)
696  environ_cp[var_name] = val
697  return val
698
699
700def create_android_ndk_rule(environ_cp):
701  """Set ANDROID_NDK_HOME and write Android NDK WORKSPACE rule."""
702  if is_windows() or is_cygwin():
703    default_ndk_path = cygpath('%s/Android/Sdk/ndk-bundle' %
704                               environ_cp['APPDATA'])
705  elif is_macos():
706    default_ndk_path = '%s/library/Android/Sdk/ndk-bundle' % environ_cp['HOME']
707  else:
708    default_ndk_path = '%s/Android/Sdk/ndk-bundle' % environ_cp['HOME']
709
710  def valid_ndk_path(path):
711    return (os.path.exists(path) and
712            os.path.exists(os.path.join(path, 'source.properties')))
713
714  android_ndk_home_path = prompt_loop_or_load_from_env(
715      environ_cp,
716      var_name='ANDROID_NDK_HOME',
717      var_default=default_ndk_path,
718      ask_for_var='Please specify the home path of the Android NDK to use.',
719      check_success=valid_ndk_path,
720      error_msg=('The path %s or its child file "source.properties" '
721                 'does not exist.'))
722  write_action_env_to_bazelrc('ANDROID_NDK_HOME', android_ndk_home_path)
723  write_action_env_to_bazelrc(
724      'ANDROID_NDK_API_LEVEL',
725      get_ndk_api_level(environ_cp, android_ndk_home_path))
726
727
728def create_android_sdk_rule(environ_cp):
729  """Set Android variables and write Android SDK WORKSPACE rule."""
730  if is_windows() or is_cygwin():
731    default_sdk_path = cygpath('%s/Android/Sdk' % environ_cp['APPDATA'])
732  elif is_macos():
733    default_sdk_path = '%s/library/Android/Sdk' % environ_cp['HOME']
734  else:
735    default_sdk_path = '%s/Android/Sdk' % environ_cp['HOME']
736
737  def valid_sdk_path(path):
738    return (os.path.exists(path) and
739            os.path.exists(os.path.join(path, 'platforms')) and
740            os.path.exists(os.path.join(path, 'build-tools')))
741
742  android_sdk_home_path = prompt_loop_or_load_from_env(
743      environ_cp,
744      var_name='ANDROID_SDK_HOME',
745      var_default=default_sdk_path,
746      ask_for_var='Please specify the home path of the Android SDK to use.',
747      check_success=valid_sdk_path,
748      error_msg=('Either %s does not exist, or it does not contain the '
749                 'subdirectories "platforms" and "build-tools".'))
750
751  platforms = os.path.join(android_sdk_home_path, 'platforms')
752  api_levels = sorted(os.listdir(platforms))
753  api_levels = [x.replace('android-', '') for x in api_levels]
754
755  def valid_api_level(api_level):
756    return os.path.exists(
757        os.path.join(android_sdk_home_path, 'platforms',
758                     'android-' + api_level))
759
760  android_api_level = prompt_loop_or_load_from_env(
761      environ_cp,
762      var_name='ANDROID_API_LEVEL',
763      var_default=api_levels[-1],
764      ask_for_var=('Please specify the Android SDK API level to use. '
765                   '[Available levels: %s]') % api_levels,
766      check_success=valid_api_level,
767      error_msg='Android-%s is not present in the SDK path.')
768
769  build_tools = os.path.join(android_sdk_home_path, 'build-tools')
770  versions = sorted(os.listdir(build_tools))
771
772  def valid_build_tools(version):
773    return os.path.exists(
774        os.path.join(android_sdk_home_path, 'build-tools', version))
775
776  android_build_tools_version = prompt_loop_or_load_from_env(
777      environ_cp,
778      var_name='ANDROID_BUILD_TOOLS_VERSION',
779      var_default=versions[-1],
780      ask_for_var=('Please specify an Android build tools version to use. '
781                   '[Available versions: %s]') % versions,
782      check_success=valid_build_tools,
783      error_msg=('The selected SDK does not have build-tools version %s '
784                 'available.'))
785
786  write_action_env_to_bazelrc('ANDROID_BUILD_TOOLS_VERSION',
787                              android_build_tools_version)
788  write_action_env_to_bazelrc('ANDROID_SDK_API_LEVEL', android_api_level)
789  write_action_env_to_bazelrc('ANDROID_SDK_HOME', android_sdk_home_path)
790
791
792def get_ndk_api_level(environ_cp, android_ndk_home_path):
793  """Gets the appropriate NDK API level to use for the provided Android NDK path."""
794
795  # First check to see if we're using a blessed version of the NDK.
796  properties_path = '%s/source.properties' % android_ndk_home_path
797  if is_windows() or is_cygwin():
798    properties_path = cygpath(properties_path)
799  with open(properties_path, 'r') as f:
800    filedata = f.read()
801
802  revision = re.search(r'Pkg.Revision = (\d+)', filedata)
803  if revision:
804    ndk_version = revision.group(1)
805  else:
806    raise Exception('Unable to parse NDK revision.')
807  if int(ndk_version) not in _SUPPORTED_ANDROID_NDK_VERSIONS:
808    print('WARNING: The NDK version in %s is %s, which is not '
809          'supported by Bazel (officially supported versions: %s). Please use '
810          'another version. Compiling Android targets may result in confusing '
811          'errors.\n' %
812          (android_ndk_home_path, ndk_version, _SUPPORTED_ANDROID_NDK_VERSIONS))
813
814  # Now grab the NDK API level to use. Note that this is different from the
815  # SDK API level, as the NDK API level is effectively the *min* target SDK
816  # version.
817  platforms = os.path.join(android_ndk_home_path, 'platforms')
818  api_levels = sorted(os.listdir(platforms))
819  api_levels = [
820      x.replace('android-', '') for x in api_levels if 'android-' in x
821  ]
822
823  def valid_api_level(api_level):
824    return os.path.exists(
825        os.path.join(android_ndk_home_path, 'platforms',
826                     'android-' + api_level))
827
828  android_ndk_api_level = prompt_loop_or_load_from_env(
829      environ_cp,
830      var_name='ANDROID_NDK_API_LEVEL',
831      var_default='21',  # 21 is required for ARM64 support.
832      ask_for_var=('Please specify the (min) Android NDK API level to use. '
833                   '[Available levels: %s]') % api_levels,
834      check_success=valid_api_level,
835      error_msg='Android-%s is not present in the NDK path.')
836
837  return android_ndk_api_level
838
839
840def set_gcc_host_compiler_path(environ_cp):
841  """Set GCC_HOST_COMPILER_PATH."""
842  default_gcc_host_compiler_path = which('gcc') or ''
843  cuda_bin_symlink = '%s/bin/gcc' % environ_cp.get('CUDA_TOOLKIT_PATH')
844
845  if os.path.islink(cuda_bin_symlink):
846    # os.readlink is only available in linux
847    default_gcc_host_compiler_path = os.path.realpath(cuda_bin_symlink)
848
849  gcc_host_compiler_path = prompt_loop_or_load_from_env(
850      environ_cp,
851      var_name='GCC_HOST_COMPILER_PATH',
852      var_default=default_gcc_host_compiler_path,
853      ask_for_var='Please specify which gcc should be used by nvcc as the host '
854      'compiler.',
855      check_success=os.path.exists,
856      resolve_symlinks=True,
857      error_msg='Invalid gcc path. %s cannot be found.',
858  )
859
860  write_action_env_to_bazelrc('GCC_HOST_COMPILER_PATH', gcc_host_compiler_path)
861
862
863def reformat_version_sequence(version_str, sequence_count):
864  """Reformat the version string to have the given number of sequences.
865
866  For example:
867  Given (7, 2) -> 7.0
868        (7.0.1, 2) -> 7.0
869        (5, 1) -> 5
870        (5.0.3.2, 1) -> 5
871
872  Args:
873      version_str: String, the version string.
874      sequence_count: int, an integer.
875
876  Returns:
877      string, reformatted version string.
878  """
879  v = version_str.split('.')
880  if len(v) < sequence_count:
881    v = v + (['0'] * (sequence_count - len(v)))
882
883  return '.'.join(v[:sequence_count])
884
885
886def set_tf_cuda_paths(environ_cp):
887  """Set TF_CUDA_PATHS."""
888  ask_cuda_paths = (
889      'Please specify the comma-separated list of base paths to look for CUDA '
890      'libraries and headers. [Leave empty to use the default]: ')
891  tf_cuda_paths = get_from_env_or_user_or_default(environ_cp, 'TF_CUDA_PATHS',
892                                                  ask_cuda_paths, '')
893  if tf_cuda_paths:
894    environ_cp['TF_CUDA_PATHS'] = tf_cuda_paths
895
896
897def set_tf_cuda_version(environ_cp):
898  """Set TF_CUDA_VERSION."""
899  ask_cuda_version = (
900      'Please specify the CUDA SDK version you want to use. '
901      '[Leave empty to default to CUDA %s]: ') % _DEFAULT_CUDA_VERSION
902  tf_cuda_version = get_from_env_or_user_or_default(environ_cp,
903                                                    'TF_CUDA_VERSION',
904                                                    ask_cuda_version,
905                                                    _DEFAULT_CUDA_VERSION)
906  environ_cp['TF_CUDA_VERSION'] = tf_cuda_version
907
908
909def set_tf_cudnn_version(environ_cp):
910  """Set TF_CUDNN_VERSION."""
911  ask_cudnn_version = (
912      'Please specify the cuDNN version you want to use. '
913      '[Leave empty to default to cuDNN %s]: ') % _DEFAULT_CUDNN_VERSION
914  tf_cudnn_version = get_from_env_or_user_or_default(environ_cp,
915                                                     'TF_CUDNN_VERSION',
916                                                     ask_cudnn_version,
917                                                     _DEFAULT_CUDNN_VERSION)
918  environ_cp['TF_CUDNN_VERSION'] = tf_cudnn_version
919
920
921def is_cuda_compatible(lib, cuda_ver, cudnn_ver):
922  """Check compatibility between given library and cudnn/cudart libraries."""
923  ldd_bin = which('ldd') or '/usr/bin/ldd'
924  ldd_out = run_shell([ldd_bin, lib], True)
925  ldd_out = ldd_out.split(os.linesep)
926  cudnn_pattern = re.compile('.*libcudnn.so\\.?(.*) =>.*$')
927  cuda_pattern = re.compile('.*libcudart.so\\.?(.*) =>.*$')
928  cudnn = None
929  cudart = None
930  cudnn_ok = True  # assume no cudnn dependency by default
931  cuda_ok = True  # assume no cuda dependency by default
932  for line in ldd_out:
933    if 'libcudnn.so' in line:
934      cudnn = cudnn_pattern.search(line)
935      cudnn_ok = False
936    elif 'libcudart.so' in line:
937      cudart = cuda_pattern.search(line)
938      cuda_ok = False
939  if cudnn and len(cudnn.group(1)):
940    cudnn = convert_version_to_int(cudnn.group(1))
941  if cudart and len(cudart.group(1)):
942    cudart = convert_version_to_int(cudart.group(1))
943  if cudnn is not None:
944    cudnn_ok = (cudnn == cudnn_ver)
945  if cudart is not None:
946    cuda_ok = (cudart == cuda_ver)
947  return cudnn_ok and cuda_ok
948
949
950def set_tf_tensorrt_version(environ_cp):
951  """Set TF_TENSORRT_VERSION."""
952  if not is_linux():
953    raise ValueError('Currently TensorRT is only supported on Linux platform.')
954
955  if not int(environ_cp.get('TF_NEED_TENSORRT', False)):
956    return
957
958  ask_tensorrt_version = (
959      'Please specify the TensorRT version you want to use. '
960      '[Leave empty to default to TensorRT %s]: ') % _DEFAULT_TENSORRT_VERSION
961  tf_tensorrt_version = get_from_env_or_user_or_default(
962      environ_cp, 'TF_TENSORRT_VERSION', ask_tensorrt_version,
963      _DEFAULT_TENSORRT_VERSION)
964  environ_cp['TF_TENSORRT_VERSION'] = tf_tensorrt_version
965
966
967def set_tf_nccl_version(environ_cp):
968  """Set TF_NCCL_VERSION."""
969  if not is_linux():
970    raise ValueError('Currently NCCL is only supported on Linux platform.')
971
972  if 'TF_NCCL_VERSION' in environ_cp:
973    return
974
975  ask_nccl_version = (
976      'Please specify the locally installed NCCL version you want to use. '
977      '[Leave empty to use http://github.com/nvidia/nccl]: ')
978  tf_nccl_version = get_from_env_or_user_or_default(environ_cp,
979                                                    'TF_NCCL_VERSION',
980                                                    ask_nccl_version, '')
981  environ_cp['TF_NCCL_VERSION'] = tf_nccl_version
982
983
984def get_native_cuda_compute_capabilities(environ_cp):
985  """Get native cuda compute capabilities.
986
987  Args:
988    environ_cp: copy of the os.environ.
989
990  Returns:
991    string of native cuda compute capabilities, separated by comma.
992  """
993  device_query_bin = os.path.join(
994      environ_cp.get('CUDA_TOOLKIT_PATH'), 'extras/demo_suite/deviceQuery')
995  if os.path.isfile(device_query_bin) and os.access(device_query_bin, os.X_OK):
996    try:
997      output = run_shell(device_query_bin).split('\n')
998      pattern = re.compile('[0-9]*\\.[0-9]*')
999      output = [pattern.search(x) for x in output if 'Capability' in x]
1000      output = ','.join(x.group() for x in output if x is not None)
1001    except subprocess.CalledProcessError:
1002      output = ''
1003  else:
1004    output = ''
1005  return output
1006
1007
1008def set_tf_cuda_compute_capabilities(environ_cp):
1009  """Set TF_CUDA_COMPUTE_CAPABILITIES."""
1010  while True:
1011    native_cuda_compute_capabilities = get_native_cuda_compute_capabilities(
1012        environ_cp)
1013    if not native_cuda_compute_capabilities:
1014      default_cuda_compute_capabilities = _DEFAULT_CUDA_COMPUTE_CAPABILITIES
1015    else:
1016      default_cuda_compute_capabilities = native_cuda_compute_capabilities
1017
1018    ask_cuda_compute_capabilities = (
1019        'Please specify a list of comma-separated CUDA compute capabilities '
1020        'you want to build with.\nYou can find the compute capability of your '
1021        'device at: https://developer.nvidia.com/cuda-gpus. Each capability '
1022        'can be specified as "x.y" or "compute_xy" to include both virtual and'
1023        ' binary GPU code, or as "sm_xy" to only include the binary '
1024        'code.\nPlease note that each additional compute capability '
1025        'significantly increases your build time and binary size, and that '
1026        'TensorFlow only supports compute capabilities >= 3.5 [Default is: '
1027        '%s]: ' % default_cuda_compute_capabilities)
1028    tf_cuda_compute_capabilities = get_from_env_or_user_or_default(
1029        environ_cp, 'TF_CUDA_COMPUTE_CAPABILITIES',
1030        ask_cuda_compute_capabilities, default_cuda_compute_capabilities)
1031    # Check whether all capabilities from the input is valid
1032    all_valid = True
1033    # Remove all whitespace characters before splitting the string
1034    # that users may insert by accident, as this will result in error
1035    tf_cuda_compute_capabilities = ''.join(tf_cuda_compute_capabilities.split())
1036    for compute_capability in tf_cuda_compute_capabilities.split(','):
1037      m = re.match('[0-9]+.[0-9]+', compute_capability)
1038      if not m:
1039        # We now support sm_35,sm_50,sm_60,compute_70.
1040        sm_compute_match = re.match('(sm|compute)_?([0-9]+[0-9]+)',
1041                                    compute_capability)
1042        if not sm_compute_match:
1043          print('Invalid compute capability: %s' % compute_capability)
1044          all_valid = False
1045        else:
1046          ver = int(sm_compute_match.group(2))
1047          if ver < 30:
1048            print(
1049                'ERROR: TensorFlow only supports small CUDA compute'
1050                ' capabilities of sm_30 and higher. Please re-specify the list'
1051                ' of compute capabilities excluding version %s.' % ver)
1052            all_valid = False
1053          if ver < 35:
1054            print('WARNING: XLA does not support CUDA compute capabilities '
1055                  'lower than sm_35. Disable XLA when running on older GPUs.')
1056      else:
1057        ver = float(m.group(0))
1058        if ver < 3.0:
1059          print('ERROR: TensorFlow only supports CUDA compute capabilities 3.0 '
1060                'and higher. Please re-specify the list of compute '
1061                'capabilities excluding version %s.' % ver)
1062          all_valid = False
1063        if ver < 3.5:
1064          print('WARNING: XLA does not support CUDA compute capabilities '
1065                'lower than 3.5. Disable XLA when running on older GPUs.')
1066
1067    if all_valid:
1068      break
1069
1070    # Reset and Retry
1071    environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = ''
1072
1073  # Set TF_CUDA_COMPUTE_CAPABILITIES
1074  environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = tf_cuda_compute_capabilities
1075  write_action_env_to_bazelrc('TF_CUDA_COMPUTE_CAPABILITIES',
1076                              tf_cuda_compute_capabilities)
1077
1078
1079def set_other_cuda_vars(environ_cp):
1080  """Set other CUDA related variables."""
1081  # If CUDA is enabled, always use GPU during build and test.
1082  if environ_cp.get('TF_CUDA_CLANG') == '1':
1083    write_to_bazelrc('build --config=cuda_clang')
1084  else:
1085    write_to_bazelrc('build --config=cuda')
1086
1087
1088def set_host_cxx_compiler(environ_cp):
1089  """Set HOST_CXX_COMPILER."""
1090  default_cxx_host_compiler = which('g++') or ''
1091
1092  host_cxx_compiler = prompt_loop_or_load_from_env(
1093      environ_cp,
1094      var_name='HOST_CXX_COMPILER',
1095      var_default=default_cxx_host_compiler,
1096      ask_for_var=('Please specify which C++ compiler should be used as the '
1097                   'host C++ compiler.'),
1098      check_success=os.path.exists,
1099      error_msg='Invalid C++ compiler path. %s cannot be found.',
1100  )
1101
1102  write_action_env_to_bazelrc('HOST_CXX_COMPILER', host_cxx_compiler)
1103
1104
1105def set_host_c_compiler(environ_cp):
1106  """Set HOST_C_COMPILER."""
1107  default_c_host_compiler = which('gcc') or ''
1108
1109  host_c_compiler = prompt_loop_or_load_from_env(
1110      environ_cp,
1111      var_name='HOST_C_COMPILER',
1112      var_default=default_c_host_compiler,
1113      ask_for_var=('Please specify which C compiler should be used as the host '
1114                   'C compiler.'),
1115      check_success=os.path.exists,
1116      error_msg='Invalid C compiler path. %s cannot be found.',
1117  )
1118
1119  write_action_env_to_bazelrc('HOST_C_COMPILER', host_c_compiler)
1120
1121
1122def system_specific_test_config(environ_cp):
1123  """Add default build and test flags required for TF tests to bazelrc."""
1124  write_to_bazelrc('test --flaky_test_attempts=3')
1125  write_to_bazelrc('test --test_size_filters=small,medium')
1126
1127  # Each instance of --test_tag_filters or --build_tag_filters overrides all
1128  # previous instances, so we need to build up a complete list and write a
1129  # single list of filters for the .bazelrc file.
1130
1131  # Filters to use with both --test_tag_filters and --build_tag_filters
1132  test_and_build_filters = ['-benchmark-test', '-no_oss']
1133  # Additional filters for --test_tag_filters beyond those in
1134  # test_and_build_filters
1135  test_only_filters = ['-oss_serial']
1136  if is_windows():
1137    test_and_build_filters.append('-no_windows')
1138    if ((environ_cp.get('TF_NEED_CUDA', None) == '1') or
1139        (environ_cp.get('TF_NEED_ROCM', None) == '1')):
1140      test_and_build_filters += ['-no_windows_gpu', '-no_gpu']
1141    else:
1142      test_and_build_filters.append('-gpu')
1143  elif is_macos():
1144    test_and_build_filters += ['-gpu', '-nomac', '-no_mac']
1145  elif is_linux():
1146    if ((environ_cp.get('TF_NEED_CUDA', None) == '1') or
1147        (environ_cp.get('TF_NEED_ROCM', None) == '1')):
1148      test_and_build_filters.append('-no_gpu')
1149      write_to_bazelrc('test --test_env=LD_LIBRARY_PATH')
1150    else:
1151      test_and_build_filters.append('-gpu')
1152
1153  # Disable tests with "v1only" tag in "v2" Bazel config, but not in "v1" config
1154  write_to_bazelrc('test:v1 --test_tag_filters=%s' %
1155                   ','.join(test_and_build_filters + test_only_filters))
1156  write_to_bazelrc('test:v1 --build_tag_filters=%s' %
1157                   ','.join(test_and_build_filters))
1158  write_to_bazelrc(
1159      'test:v2 --test_tag_filters=%s' %
1160      ','.join(test_and_build_filters + test_only_filters + ['-v1only']))
1161  write_to_bazelrc('test:v2 --build_tag_filters=%s' %
1162                   ','.join(test_and_build_filters + ['-v1only']))
1163
1164
1165def set_system_libs_flag(environ_cp):
1166  syslibs = environ_cp.get('TF_SYSTEM_LIBS', '')
1167  if syslibs:
1168    if ',' in syslibs:
1169      syslibs = ','.join(sorted(syslibs.split(',')))
1170    else:
1171      syslibs = ','.join(sorted(syslibs.split()))
1172    write_action_env_to_bazelrc('TF_SYSTEM_LIBS', syslibs)
1173
1174  for varname in ('PREFIX', 'LIBDIR', 'INCLUDEDIR', 'PROTOBUF_INCLUDE_PATH'):
1175    if varname in environ_cp:
1176      write_to_bazelrc('build --define=%s=%s' % (varname, environ_cp[varname]))
1177
1178
1179def set_windows_build_flags(environ_cp):
1180  """Set Windows specific build options."""
1181
1182  # First available in VS 16.4. Speeds up Windows compile times by a lot. See
1183  # https://groups.google.com/a/tensorflow.org/d/topic/build/SsW98Eo7l3o/discussion
1184  # pylint: disable=line-too-long
1185  write_to_bazelrc(
1186      'build --copt=/d2ReducedOptimizeHugeFunctions --host_copt=/d2ReducedOptimizeHugeFunctions'
1187  )
1188
1189  if get_var(
1190      environ_cp, 'TF_OVERRIDE_EIGEN_STRONG_INLINE', 'Eigen strong inline',
1191      True, ('Would you like to override eigen strong inline for some C++ '
1192             'compilation to reduce the compilation time?'),
1193      'Eigen strong inline overridden.', 'Not overriding eigen strong inline, '
1194      'some compilations could take more than 20 mins.'):
1195    # Due to a known MSVC compiler issue
1196    # https://github.com/tensorflow/tensorflow/issues/10521
1197    # Overriding eigen strong inline speeds up the compiling of
1198    # conv_grad_ops_3d.cc and conv_ops_3d.cc by 20 minutes,
1199    # but this also hurts the performance. Let users decide what they want.
1200    write_to_bazelrc('build --define=override_eigen_strong_inline=true')
1201
1202
1203def config_info_line(name, help_text):
1204  """Helper function to print formatted help text for Bazel config options."""
1205  print('\t--config=%-12s\t# %s' % (name, help_text))
1206
1207
1208def configure_ios(environ_cp):
1209  """Configures TensorFlow for iOS builds."""
1210  if not is_macos():
1211    return
1212  if not get_var(environ_cp, 'TF_CONFIGURE_IOS', 'iOS', False):
1213    return
1214  for filepath in APPLE_BAZEL_FILES:
1215    existing_filepath = os.path.join(_TF_WORKSPACE_ROOT, filepath + '.apple')
1216    renamed_filepath = os.path.join(_TF_WORKSPACE_ROOT, filepath)
1217    symlink_force(existing_filepath, renamed_filepath)
1218  for filepath in IOS_FILES:
1219    filename = os.path.basename(filepath)
1220    new_filepath = os.path.join(_TF_WORKSPACE_ROOT, filename)
1221    symlink_force(filepath, new_filepath)
1222
1223
1224def validate_cuda_config(environ_cp):
1225  """Run find_cuda_config.py and return cuda_toolkit_path, or None."""
1226
1227  def maybe_encode_env(env):
1228    """Encodes unicode in env to str on Windows python 2.x."""
1229    if not is_windows() or sys.version_info[0] != 2:
1230      return env
1231    for k, v in env.items():
1232      if isinstance(k, unicode):
1233        k = k.encode('ascii')
1234      if isinstance(v, unicode):
1235        v = v.encode('ascii')
1236      env[k] = v
1237    return env
1238
1239  cuda_libraries = ['cuda', 'cudnn']
1240  if is_linux():
1241    if int(environ_cp.get('TF_NEED_TENSORRT', False)):
1242      cuda_libraries.append('tensorrt')
1243    if environ_cp.get('TF_NCCL_VERSION', None):
1244      cuda_libraries.append('nccl')
1245
1246  paths = glob.glob('**/third_party/gpus/find_cuda_config.py', recursive=True)
1247  if not paths:
1248    raise FileNotFoundError(
1249        "Can't find 'find_cuda_config.py' script inside working directory")
1250  proc = subprocess.Popen(
1251      [environ_cp['PYTHON_BIN_PATH'], paths[0]] + cuda_libraries,
1252      stdout=subprocess.PIPE,
1253      env=maybe_encode_env(environ_cp))
1254
1255  if proc.wait():
1256    # Errors from find_cuda_config.py were sent to stderr.
1257    print('Asking for detailed CUDA configuration...\n')
1258    return False
1259
1260  config = dict(
1261      tuple(line.decode('ascii').rstrip().split(': ')) for line in proc.stdout)
1262
1263  print('Found CUDA %s in:' % config['cuda_version'])
1264  print('    %s' % config['cuda_library_dir'])
1265  print('    %s' % config['cuda_include_dir'])
1266
1267  print('Found cuDNN %s in:' % config['cudnn_version'])
1268  print('    %s' % config['cudnn_library_dir'])
1269  print('    %s' % config['cudnn_include_dir'])
1270
1271  if 'tensorrt_version' in config:
1272    print('Found TensorRT %s in:' % config['tensorrt_version'])
1273    print('    %s' % config['tensorrt_library_dir'])
1274    print('    %s' % config['tensorrt_include_dir'])
1275
1276  if config.get('nccl_version', None):
1277    print('Found NCCL %s in:' % config['nccl_version'])
1278    print('    %s' % config['nccl_library_dir'])
1279    print('    %s' % config['nccl_include_dir'])
1280
1281  print('\n')
1282
1283  environ_cp['CUDA_TOOLKIT_PATH'] = config['cuda_toolkit_path']
1284  return True
1285
1286
1287def main():
1288  global _TF_WORKSPACE_ROOT
1289  global _TF_BAZELRC
1290  global _TF_CURRENT_BAZEL_VERSION
1291
1292  parser = argparse.ArgumentParser()
1293  parser.add_argument(
1294      '--workspace',
1295      type=str,
1296      default=os.path.abspath(os.path.dirname(__file__)),
1297      help='The absolute path to your active Bazel workspace.')
1298  args = parser.parse_args()
1299
1300  _TF_WORKSPACE_ROOT = args.workspace
1301  _TF_BAZELRC = os.path.join(_TF_WORKSPACE_ROOT, _TF_BAZELRC_FILENAME)
1302
1303  # Make a copy of os.environ to be clear when functions and getting and setting
1304  # environment variables.
1305  environ_cp = dict(os.environ)
1306
1307  try:
1308    current_bazel_version = check_bazel_version(_TF_MIN_BAZEL_VERSION,
1309                                                _TF_MAX_BAZEL_VERSION)
1310  except subprocess.CalledProcessError as e:
1311    print('Error checking bazel version: ', e.output.decode('UTF-8').strip())
1312    raise e
1313
1314  _TF_CURRENT_BAZEL_VERSION = convert_version_to_int(current_bazel_version)
1315
1316  reset_tf_configure_bazelrc()
1317
1318  cleanup_makefile()
1319  setup_python(environ_cp)
1320
1321  if is_windows():
1322    environ_cp['TF_NEED_OPENCL'] = '0'
1323    environ_cp['TF_CUDA_CLANG'] = '0'
1324    environ_cp['TF_NEED_TENSORRT'] = '0'
1325    # TODO(ibiryukov): Investigate using clang as a cpu or cuda compiler on
1326    # Windows.
1327    environ_cp['TF_DOWNLOAD_CLANG'] = '0'
1328    environ_cp['TF_NEED_MPI'] = '0'
1329
1330  if is_macos():
1331    environ_cp['TF_NEED_TENSORRT'] = '0'
1332
1333  with_xla_support = environ_cp.get('TF_ENABLE_XLA', None)
1334  if with_xla_support is not None:
1335    write_to_bazelrc('build --define=with_xla_support=%s' %
1336                     ('true' if int(with_xla_support) else 'false'))
1337
1338  set_action_env_var(
1339      environ_cp, 'TF_NEED_ROCM', 'ROCm', False, bazel_config_name='rocm')
1340  if (environ_cp.get('TF_NEED_ROCM') == '1' and
1341      'LD_LIBRARY_PATH' in environ_cp and
1342      environ_cp.get('LD_LIBRARY_PATH') != '1'):
1343    write_action_env_to_bazelrc('LD_LIBRARY_PATH',
1344                                environ_cp.get('LD_LIBRARY_PATH'))
1345
1346  if (environ_cp.get('TF_NEED_ROCM') == '1' and environ_cp.get('ROCM_PATH')):
1347    write_action_env_to_bazelrc('ROCM_PATH', environ_cp.get('ROCM_PATH'))
1348    write_action_env_to_bazelrc('ROCBLAS_TENSILE_LIBPATH',
1349                                environ_cp.get('ROCM_PATH') + '/lib/library')
1350
1351  environ_cp['TF_NEED_CUDA'] = str(
1352      int(get_var(environ_cp, 'TF_NEED_CUDA', 'CUDA', False)))
1353  if (environ_cp.get('TF_NEED_CUDA') == '1' and
1354      'TF_CUDA_CONFIG_REPO' not in environ_cp):
1355
1356    set_action_env_var(
1357        environ_cp,
1358        'TF_NEED_TENSORRT',
1359        'TensorRT',
1360        False,
1361        bazel_config_name='tensorrt')
1362
1363    environ_save = dict(environ_cp)
1364    for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS):
1365
1366      if validate_cuda_config(environ_cp):
1367        cuda_env_names = [
1368            'TF_CUDA_VERSION',
1369            'TF_CUBLAS_VERSION',
1370            'TF_CUDNN_VERSION',
1371            'TF_TENSORRT_VERSION',
1372            'TF_NCCL_VERSION',
1373            'TF_CUDA_PATHS',
1374            # Items below are for backwards compatibility when not using
1375            # TF_CUDA_PATHS.
1376            'CUDA_TOOLKIT_PATH',
1377            'CUDNN_INSTALL_PATH',
1378            'NCCL_INSTALL_PATH',
1379            'NCCL_HDR_PATH',
1380            'TENSORRT_INSTALL_PATH'
1381        ]
1382        # Note: set_action_env_var above already writes to bazelrc.
1383        for name in cuda_env_names:
1384          if name in environ_cp:
1385            write_action_env_to_bazelrc(name, environ_cp[name])
1386        break
1387
1388      # Restore settings changed below if CUDA config could not be validated.
1389      environ_cp = dict(environ_save)
1390
1391      set_tf_cuda_version(environ_cp)
1392      set_tf_cudnn_version(environ_cp)
1393      if is_linux():
1394        set_tf_tensorrt_version(environ_cp)
1395        set_tf_nccl_version(environ_cp)
1396
1397      set_tf_cuda_paths(environ_cp)
1398
1399    else:
1400      raise UserInputError(
1401          'Invalid CUDA setting were provided %d '
1402          'times in a row. Assuming to be a scripting mistake.' %
1403          _DEFAULT_PROMPT_ASK_ATTEMPTS)
1404
1405    set_tf_cuda_compute_capabilities(environ_cp)
1406    if 'LD_LIBRARY_PATH' in environ_cp and environ_cp.get(
1407        'LD_LIBRARY_PATH') != '1':
1408      write_action_env_to_bazelrc('LD_LIBRARY_PATH',
1409                                  environ_cp.get('LD_LIBRARY_PATH'))
1410
1411    set_tf_cuda_clang(environ_cp)
1412    if environ_cp.get('TF_CUDA_CLANG') == '1':
1413      # Ask whether we should download the clang toolchain.
1414      set_tf_download_clang(environ_cp)
1415      if environ_cp.get('TF_DOWNLOAD_CLANG') != '1':
1416        # Set up which clang we should use as the cuda / host compiler.
1417        set_clang_cuda_compiler_path(environ_cp)
1418      else:
1419        # Use downloaded LLD for linking.
1420        write_to_bazelrc('build:cuda_clang --config=download_clang_use_lld')
1421    else:
1422      # Set up which gcc nvcc should use as the host compiler
1423      # No need to set this on Windows
1424      if not is_windows():
1425        set_gcc_host_compiler_path(environ_cp)
1426    set_other_cuda_vars(environ_cp)
1427  else:
1428    # CUDA not required. Ask whether we should download the clang toolchain and
1429    # use it for the CPU build.
1430    set_tf_download_clang(environ_cp)
1431
1432  # ROCm / CUDA are mutually exclusive.
1433  # At most 1 GPU platform can be configured.
1434  gpu_platform_count = 0
1435  if environ_cp.get('TF_NEED_ROCM') == '1':
1436    gpu_platform_count += 1
1437  if environ_cp.get('TF_NEED_CUDA') == '1':
1438    gpu_platform_count += 1
1439  if gpu_platform_count >= 2:
1440    raise UserInputError('CUDA / ROCm are mututally exclusive. '
1441                         'At most 1 GPU platform can be configured.')
1442
1443  set_cc_opt_flags(environ_cp)
1444  set_system_libs_flag(environ_cp)
1445  if is_windows():
1446    set_windows_build_flags(environ_cp)
1447
1448  if get_var(environ_cp, 'TF_SET_ANDROID_WORKSPACE', 'android workspace', False,
1449             ('Would you like to interactively configure ./WORKSPACE for '
1450              'Android builds?'), 'Searching for NDK and SDK installations.',
1451             'Not configuring the WORKSPACE for Android builds.'):
1452    create_android_ndk_rule(environ_cp)
1453    create_android_sdk_rule(environ_cp)
1454
1455  system_specific_test_config(environ_cp)
1456
1457  configure_ios(environ_cp)
1458
1459  print('Preconfigured Bazel build configs. You can use any of the below by '
1460        'adding "--config=<>" to your build command. See .bazelrc for more '
1461        'details.')
1462  config_info_line('mkl', 'Build with MKL support.')
1463  config_info_line(
1464      'mkl_aarch64',
1465      'Build with oneDNN and Compute Library for the Arm Architecture (ACL).')
1466  config_info_line('monolithic', 'Config for mostly static monolithic build.')
1467  config_info_line('numa', 'Build with NUMA support.')
1468  config_info_line(
1469      'dynamic_kernels',
1470      '(Experimental) Build kernels into separate shared objects.')
1471  config_info_line('v1', 'Build with TensorFlow 1 API instead of TF 2 API.')
1472
1473  print('Preconfigured Bazel build configs to DISABLE default on features:')
1474  config_info_line('nogcp', 'Disable GCP support.')
1475  config_info_line('nonccl', 'Disable NVIDIA NCCL support.')
1476
1477
1478if __name__ == '__main__':
1479  main()
1480