• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4"""Utilities for toolchain build."""
5
6from __future__ import print_function
7
8__author__ = 'asharif@google.com (Ahmad Sharif)'
9
10from contextlib import contextmanager
11import os
12import re
13import shutil
14import sys
15import traceback
16
17import command_executer
18import logger
19
20CHROMEOS_SCRIPTS_DIR = '~/trunk/src/scripts'
21TOOLCHAIN_UTILS_PATH = ('/mnt/host/source/src/third_party/toolchain-utils/'
22                        'cros_utils/toolchain_utils.sh')
23
24
25def GetChromeOSVersionFromLSBVersion(lsb_version):
26  """Get Chromeos version from Lsb version."""
27  ce = command_executer.GetCommandExecuter()
28  command = ('git ls-remote '
29             'https://chromium.googlesource.com/chromiumos/manifest.git')
30  ret, out, _ = ce.RunCommandWOutput(command, print_to_console=False)
31  assert ret == 0, 'Command %s failed' % command
32  lower = []
33  for line in out.splitlines():
34    mo = re.search(r'refs/heads/release-R(\d+)-(\d+)\.B', line)
35    if mo:
36      revision = int(mo.group(1))
37      build = int(mo.group(2))
38      lsb_build = int(lsb_version.split('.')[0])
39      if lsb_build > build:
40        lower.append(revision)
41  lower = sorted(lower)
42  if lower:
43    return 'R%d-%s' % (lower[-1] + 1, lsb_version)
44  else:
45    return 'Unknown'
46
47
48def ApplySubs(string, *substitutions):
49  for pattern, replacement in substitutions:
50    string = re.sub(pattern, replacement, string)
51  return string
52
53
54def UnitToNumber(unit_num, base=1000):
55  """Convert a number with unit to float."""
56  unit_dict = {'kilo': base, 'mega': base**2, 'giga': base**3}
57  unit_num = unit_num.lower()
58  mo = re.search(r'(\d*)(.+)?', unit_num)
59  number = mo.group(1)
60  unit = mo.group(2)
61  if not unit:
62    return float(number)
63  for k, v in unit_dict.items():
64    if k.startswith(unit):
65      return float(number) * v
66  raise RuntimeError('Unit: %s not found in byte: %s!' % (unit, unit_num))
67
68
69def GetFilenameFromString(string):
70  return ApplySubs(
71      string,
72      (r'/', '__'),
73      (r'\s', '_'),
74      (r'[\\$="?^]', ''),)
75
76
77def GetRoot(scr_name):
78  """Break up pathname into (dir+name)."""
79  abs_path = os.path.abspath(scr_name)
80  return (os.path.dirname(abs_path), os.path.basename(abs_path))
81
82
83def GetChromeOSKeyFile(chromeos_root):
84  return os.path.join(chromeos_root, 'src', 'scripts', 'mod_for_test_scripts',
85                      'ssh_keys', 'testing_rsa')
86
87
88def GetChrootPath(chromeos_root):
89  return os.path.join(chromeos_root, 'chroot')
90
91
92def GetInsideChrootPath(chromeos_root, file_path):
93  if not file_path.startswith(GetChrootPath(chromeos_root)):
94    raise RuntimeError("File: %s doesn't seem to be in the chroot: %s" %
95                       (file_path, chromeos_root))
96  return file_path[len(GetChrootPath(chromeos_root)):]
97
98
99def GetOutsideChrootPath(chromeos_root, file_path):
100  return os.path.join(GetChrootPath(chromeos_root), file_path.lstrip('/'))
101
102
103def FormatQuotedCommand(command):
104  return ApplySubs(command, ('"', r'\"'))
105
106
107def FormatCommands(commands):
108  return ApplySubs(
109      str(commands), ('&&', '&&\n'), (';', ';\n'), (r'\n+\s*', '\n'))
110
111
112def GetImageDir(chromeos_root, board):
113  return os.path.join(chromeos_root, 'src', 'build', 'images', board)
114
115
116def LabelLatestImage(chromeos_root, board, label, vanilla_path=None):
117  image_dir = GetImageDir(chromeos_root, board)
118  latest_image_dir = os.path.join(image_dir, 'latest')
119  latest_image_dir = os.path.realpath(latest_image_dir)
120  latest_image_dir = os.path.basename(latest_image_dir)
121  retval = 0
122  with WorkingDirectory(image_dir):
123    command = 'ln -sf -T %s %s' % (latest_image_dir, label)
124    ce = command_executer.GetCommandExecuter()
125    retval = ce.RunCommand(command)
126    if retval:
127      return retval
128    if vanilla_path:
129      command = 'ln -sf -T %s %s' % (vanilla_path, 'vanilla')
130      retval2 = ce.RunCommand(command)
131      return retval2
132  return retval
133
134
135def DoesLabelExist(chromeos_root, board, label):
136  image_label = os.path.join(GetImageDir(chromeos_root, board), label)
137  return os.path.exists(image_label)
138
139
140def GetBuildPackagesCommand(board, usepkg=False, debug=False):
141  if usepkg:
142    usepkg_flag = '--usepkg'
143  else:
144    usepkg_flag = '--nousepkg'
145  if debug:
146    withdebug_flag = '--withdebug'
147  else:
148    withdebug_flag = '--nowithdebug'
149  return ('%s/build_packages %s --withdev --withtest --withautotest '
150          '--skip_toolchain_update %s --board=%s '
151          '--accept_licenses=@CHROMEOS' % (CHROMEOS_SCRIPTS_DIR, usepkg_flag,
152                                           withdebug_flag, board))
153
154
155def GetBuildImageCommand(board, dev=False):
156  dev_args = ''
157  if dev:
158    dev_args = '--noenable_rootfs_verification --disk_layout=2gb-rootfs'
159  return ('%s/build_image --board=%s %s test' % (CHROMEOS_SCRIPTS_DIR, board,
160                                                 dev_args))
161
162
163def GetSetupBoardCommand(board,
164                         gcc_version=None,
165                         binutils_version=None,
166                         usepkg=None,
167                         force=None):
168  """Get setup_board command."""
169  options = []
170
171  if gcc_version:
172    options.append('--gcc_version=%s' % gcc_version)
173
174  if binutils_version:
175    options.append('--binutils_version=%s' % binutils_version)
176
177  if usepkg:
178    options.append('--usepkg')
179  else:
180    options.append('--nousepkg')
181
182  if force:
183    options.append('--force')
184
185  options.append('--accept_licenses=@CHROMEOS')
186
187  return ('%s/setup_board --board=%s %s' % (CHROMEOS_SCRIPTS_DIR, board,
188                                            ' '.join(options)))
189
190
191def CanonicalizePath(path):
192  path = os.path.expanduser(path)
193  path = os.path.realpath(path)
194  return path
195
196
197def GetCtargetFromBoard(board, chromeos_root):
198  """Get Ctarget from board."""
199  base_board = board.split('_')[0]
200  command = ('source %s; get_ctarget_from_board %s' % (TOOLCHAIN_UTILS_PATH,
201                                                       base_board))
202  ce = command_executer.GetCommandExecuter()
203  ret, out, _ = ce.ChrootRunCommandWOutput(chromeos_root, command)
204  if ret != 0:
205    raise ValueError('Board %s is invalid!' % board)
206  # Remove ANSI escape sequences.
207  out = StripANSIEscapeSequences(out)
208  return out.strip()
209
210
211def GetArchFromBoard(board, chromeos_root):
212  """Get Arch from board."""
213  base_board = board.split('_')[0]
214  command = ('source %s; get_board_arch %s' % (TOOLCHAIN_UTILS_PATH,
215                                               base_board))
216  ce = command_executer.GetCommandExecuter()
217  ret, out, _ = ce.ChrootRunCommandWOutput(chromeos_root, command)
218  if ret != 0:
219    raise ValueError('Board %s is invalid!' % board)
220  # Remove ANSI escape sequences.
221  out = StripANSIEscapeSequences(out)
222  return out.strip()
223
224
225def GetGccLibsDestForBoard(board, chromeos_root):
226  """Get gcc libs destination from board."""
227  arch = GetArchFromBoard(board, chromeos_root)
228  if arch == 'x86':
229    return '/build/%s/usr/lib/gcc/' % board
230  if arch == 'amd64':
231    return '/build/%s/usr/lib64/gcc/' % board
232  if arch == 'arm':
233    return '/build/%s/usr/lib/gcc/' % board
234  if arch == 'arm64':
235    return '/build/%s/usr/lib/gcc/' % board
236  raise ValueError('Arch %s is invalid!' % arch)
237
238
239def StripANSIEscapeSequences(string):
240  string = re.sub(r'\x1b\[[0-9]*[a-zA-Z]', '', string)
241  return string
242
243
244def GetChromeSrcDir():
245  return 'var/cache/distfiles/target/chrome-src/src'
246
247
248def GetEnvStringFromDict(env_dict):
249  return ' '.join(["%s=\"%s\"" % var for var in env_dict.items()])
250
251
252def MergeEnvStringWithDict(env_string, env_dict, prepend=True):
253  """Merge env string with dict."""
254  if not env_string.strip():
255    return GetEnvStringFromDict(env_dict)
256  override_env_list = []
257  ce = command_executer.GetCommandExecuter()
258  for k, v in env_dict.items():
259    v = v.strip("\"'")
260    if prepend:
261      new_env = "%s=\"%s $%s\"" % (k, v, k)
262    else:
263      new_env = "%s=\"$%s %s\"" % (k, k, v)
264    command = '; '.join([env_string, new_env, 'echo $%s' % k])
265    ret, out, _ = ce.RunCommandWOutput(command)
266    override_env_list.append('%s=%r' % (k, out.strip()))
267  ret = env_string + ' ' + ' '.join(override_env_list)
268  return ret.strip()
269
270
271def GetAllImages(chromeos_root, board):
272  ce = command_executer.GetCommandExecuter()
273  command = ('find %s/src/build/images/%s -name chromiumos_test_image.bin' %
274             (chromeos_root, board))
275  ret, out, _ = ce.RunCommandWOutput(command)
276  assert ret == 0, 'Could not run command: %s' % command
277  return out.splitlines()
278
279
280def IsFloat(text):
281  if text is None:
282    return False
283  try:
284    float(text)
285    return True
286  except ValueError:
287    return False
288
289
290def RemoveChromeBrowserObjectFiles(chromeos_root, board):
291  """Remove any object files from all the posible locations."""
292  out_dir = os.path.join(
293      GetChrootPath(chromeos_root),
294      'var/cache/chromeos-chrome/chrome-src/src/out_%s' % board)
295  if os.path.exists(out_dir):
296    shutil.rmtree(out_dir)
297    logger.GetLogger().LogCmd('rm -rf %s' % out_dir)
298  out_dir = os.path.join(
299      GetChrootPath(chromeos_root),
300      'var/cache/chromeos-chrome/chrome-src-internal/src/out_%s' % board)
301  if os.path.exists(out_dir):
302    shutil.rmtree(out_dir)
303    logger.GetLogger().LogCmd('rm -rf %s' % out_dir)
304
305
306@contextmanager
307def WorkingDirectory(new_dir):
308  """Get the working directory."""
309  old_dir = os.getcwd()
310  if old_dir != new_dir:
311    msg = 'cd %s' % new_dir
312    logger.GetLogger().LogCmd(msg)
313  os.chdir(new_dir)
314  yield new_dir
315  if old_dir != new_dir:
316    msg = 'cd %s' % old_dir
317    logger.GetLogger().LogCmd(msg)
318  os.chdir(old_dir)
319
320
321def HasGitStagedChanges(git_dir):
322  """Return True if git repository has staged changes."""
323  command = 'cd {0} && git diff --quiet --cached --exit-code HEAD'.format(
324      git_dir)
325  return command_executer.GetCommandExecuter().RunCommand(
326      command, print_to_console=False)
327
328
329def HasGitUnstagedChanges(git_dir):
330  """Return True if git repository has un-staged changes."""
331  command = 'cd {0} && git diff --quiet --exit-code HEAD'.format(git_dir)
332  return command_executer.GetCommandExecuter().RunCommand(
333      command, print_to_console=False)
334
335
336def HasGitUntrackedChanges(git_dir):
337  """Return True if git repository has un-tracked changes."""
338  command = ('cd {0} && test -z '
339             '$(git ls-files --exclude-standard --others)').format(git_dir)
340  return command_executer.GetCommandExecuter().RunCommand(
341      command, print_to_console=False)
342
343
344def GitGetCommitHash(git_dir, commit_symbolic_name):
345  """Return githash for the symbolic git commit.
346
347  For example, commit_symbolic_name could be
348  "cros/gcc.gnu.org/branches/gcc/gcc-4_8-mobile, this function returns the git
349  hash for this symbolic name.
350
351  Args:
352    git_dir: a git working tree.
353    commit_symbolic_name: a symbolic name for a particular git commit.
354
355  Returns:
356    The git hash for the symbolic name or None if fails.
357  """
358
359  command = ('cd {0} && git log -n 1 --pretty="format:%H" {1}').format(
360      git_dir, commit_symbolic_name)
361  rv, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
362      command, print_to_console=False)
363  if rv == 0:
364    return out.strip()
365  return None
366
367
368def IsGitTreeClean(git_dir):
369  """Test if git tree has no local changes.
370
371  Args:
372    git_dir: git tree directory.
373
374  Returns:
375    True if git dir is clean.
376  """
377  if HasGitStagedChanges(git_dir):
378    logger.GetLogger().LogWarning('Git tree has staged changes.')
379    return False
380  if HasGitUnstagedChanges(git_dir):
381    logger.GetLogger().LogWarning('Git tree has unstaged changes.')
382    return False
383  if HasGitUntrackedChanges(git_dir):
384    logger.GetLogger().LogWarning('Git tree has un-tracked changes.')
385    return False
386  return True
387
388
389def GetGitChangesAsList(git_dir, path=None, staged=False):
390  """Get changed files as a list.
391
392  Args:
393    git_dir: git tree directory.
394    path: a relative path that is part of the tree directory, could be null.
395    staged: whether to include staged files as well.
396
397  Returns:
398    A list containing all the changed files.
399  """
400  command = 'cd {0} && git diff --name-only'.format(git_dir)
401  if staged:
402    command += ' --cached'
403  if path:
404    command += ' -- ' + path
405  _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
406      command, print_to_console=False)
407  rv = []
408  for line in out.splitlines():
409    rv.append(line)
410  return rv
411
412
413def IsChromeOsTree(chromeos_root):
414  return (os.path.isdir(
415      os.path.join(chromeos_root, 'src/third_party/chromiumos-overlay')) and
416          os.path.isdir(os.path.join(chromeos_root, 'manifest')))
417
418
419def DeleteChromeOsTree(chromeos_root, dry_run=False):
420  """Delete a ChromeOs tree *safely*.
421
422  Args:
423    chromeos_root: dir of the tree, could be a relative one (but be careful)
424    dry_run: only prints out the command if True
425
426  Returns:
427    True if everything is ok.
428  """
429  if not IsChromeOsTree(chromeos_root):
430    logger.GetLogger().LogWarning(
431        '"{0}" does not seem to be a valid chromeos tree, do nothing.'.format(
432            chromeos_root))
433    return False
434  cmd0 = 'cd {0} && cros_sdk --delete'.format(chromeos_root)
435  if dry_run:
436    print(cmd0)
437  else:
438    if command_executer.GetCommandExecuter().RunCommand(
439        cmd0, print_to_console=True) != 0:
440      return False
441
442  cmd1 = ('export CHROMEOSDIRNAME="$(dirname $(cd {0} && pwd))" && '
443          'export CHROMEOSBASENAME="$(basename $(cd {0} && pwd))" && '
444          'cd $CHROMEOSDIRNAME && sudo rm -fr $CHROMEOSBASENAME'
445         ).format(chromeos_root)
446  if dry_run:
447    print(cmd1)
448    return True
449
450  return command_executer.GetCommandExecuter().RunCommand(
451      cmd1, print_to_console=True) == 0
452
453
454def ApplyGerritPatches(chromeos_root, gerrit_patch_string,
455                       branch='cros/master'):
456  """Apply gerrit patches on a chromeos tree.
457
458  Args:
459    chromeos_root: chromeos tree path
460    gerrit_patch_string: a patch string just like the one gives to cbuildbot,
461    'id1 id2 *id3 ... idn'. A prefix of '* means this is an internal patch.
462    branch: the tree based on which to apply the patches.
463
464  Returns:
465    True if success.
466  """
467
468  ### First of all, we need chromite libs
469  sys.path.append(os.path.join(chromeos_root, 'chromite'))
470  # Imports below are ok after modifying path to add chromite.
471  # Pylint cannot detect that and complains.
472  # pylint: disable=import-error
473  from lib import git
474  from lib import gerrit
475  manifest = git.ManifestCheckout(chromeos_root)
476  patch_list = gerrit_patch_string.split(' ')
477  ### This takes time, print log information.
478  logger.GetLogger().LogOutput('Retrieving patch information from server ...')
479  patch_info_list = gerrit.GetGerritPatchInfo(patch_list)
480  for pi in patch_info_list:
481    project_checkout = manifest.FindCheckout(pi.project, strict=False)
482    if not project_checkout:
483      logger.GetLogger().LogError(
484          'Failed to find patch project "{project}" in manifest.'.format(
485              project=pi.project))
486      return False
487
488    pi_str = '{project}:{ref}'.format(project=pi.project, ref=pi.ref)
489    try:
490      project_git_path = project_checkout.GetPath(absolute=True)
491      logger.GetLogger().LogOutput(
492          'Applying patch "{0}" in "{1}" ...'.format(pi_str, project_git_path))
493      pi.Apply(project_git_path, branch, trivial=False)
494    except Exception:
495      traceback.print_exc(file=sys.stdout)
496      logger.GetLogger().LogError('Failed to apply patch "{0}"'.format(pi_str))
497      return False
498  return True
499
500
501def BooleanPrompt(prompt='Do you want to continue?',
502                  default=True,
503                  true_value='yes',
504                  false_value='no',
505                  prolog=None):
506  """Helper function for processing boolean choice prompts.
507
508  Args:
509    prompt: The question to present to the user.
510    default: Boolean to return if the user just presses enter.
511    true_value: The text to display that represents a True returned.
512    false_value: The text to display that represents a False returned.
513    prolog: The text to display before prompt.
514
515  Returns:
516    True or False.
517  """
518  true_value, false_value = true_value.lower(), false_value.lower()
519  true_text, false_text = true_value, false_value
520  if true_value == false_value:
521    raise ValueError(
522        'true_value and false_value must differ: got %r' % true_value)
523
524  if default:
525    true_text = true_text[0].upper() + true_text[1:]
526  else:
527    false_text = false_text[0].upper() + false_text[1:]
528
529  prompt = ('\n%s (%s/%s)? ' % (prompt, true_text, false_text))
530
531  if prolog:
532    prompt = ('\n%s\n%s' % (prolog, prompt))
533
534  while True:
535    try:
536      response = raw_input(prompt).lower()
537    except EOFError:
538      # If the user hits CTRL+D, or stdin is disabled, use the default.
539      print()
540      response = None
541    except KeyboardInterrupt:
542      # If the user hits CTRL+C, just exit the process.
543      print()
544      print('CTRL+C detected; exiting')
545      sys.exit()
546
547    if not response:
548      return default
549    if true_value.startswith(response):
550      if not false_value.startswith(response):
551        return True
552      # common prefix between the two...
553    elif false_value.startswith(response):
554      return False
555
556
557# pylint: disable=unused-argument
558def rgb2short(r, g, b):
559  """Converts RGB values to xterm-256 color."""
560
561  redcolor = [255, 124, 160, 196, 9]
562  greencolor = [255, 118, 82, 46, 10]
563
564  if g == 0:
565    return redcolor[r / 52]
566  if r == 0:
567    return greencolor[g / 52]
568  return 4
569