• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# -*- coding: utf-8 -*-
2# Copyright 2013 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Utilities for toolchain build."""
7
8from __future__ import division
9from __future__ import print_function
10
11__author__ = 'asharif@google.com (Ahmad Sharif)'
12
13from contextlib import contextmanager
14import os
15import re
16import shutil
17import sys
18import traceback
19
20from cros_utils import command_executer
21from cros_utils import logger
22
23CHROMEOS_SCRIPTS_DIR = '/mnt/host/source/src/scripts'
24TOOLCHAIN_UTILS_PATH = ('/mnt/host/source/src/third_party/toolchain-utils/'
25                        'cros_utils/toolchain_utils.sh')
26
27CROS_MAIN_BRANCH = 'cros/master'
28
29
30def GetChromeOSVersionFromLSBVersion(lsb_version):
31  """Get Chromeos version from Lsb version."""
32  ce = command_executer.GetCommandExecuter()
33  command = ('git ls-remote '
34             'https://chromium.googlesource.com/chromiumos/manifest.git '
35             'refs/heads/release-R*')
36  ret, out, _ = ce.RunCommandWOutput(command, print_to_console=False)
37  assert ret == 0, 'Command %s failed' % command
38  lower = []
39  for line in out.splitlines():
40    mo = re.search(r'refs/heads/release-R(\d+)-(\d+)\.B', line)
41    if mo:
42      revision = int(mo.group(1))
43      build = int(mo.group(2))
44      lsb_build = int(lsb_version.split('.')[0])
45      if lsb_build > build:
46        lower.append(revision)
47  lower = sorted(lower)
48  if lower:
49    return 'R%d-%s' % (lower[-1] + 1, lsb_version)
50  else:
51    return 'Unknown'
52
53
54def ApplySubs(string, *substitutions):
55  for pattern, replacement in substitutions:
56    string = re.sub(pattern, replacement, string)
57  return string
58
59
60def UnitToNumber(unit_num, base=1000):
61  """Convert a number with unit to float."""
62  unit_dict = {'kilo': base, 'mega': base**2, 'giga': base**3}
63  unit_num = unit_num.lower()
64  mo = re.search(r'(\d*)(.+)?', unit_num)
65  number = mo.group(1)
66  unit = mo.group(2)
67  if not unit:
68    return float(number)
69  for k, v in unit_dict.items():
70    if k.startswith(unit):
71      return float(number) * v
72  raise RuntimeError('Unit: %s not found in byte: %s!' % (unit, unit_num))
73
74
75def GetFilenameFromString(string):
76  return ApplySubs(
77      string,
78      (r'/', '__'),
79      (r'\s', '_'),
80      (r'[\\$="?^]', ''),
81  )
82
83
84def GetRoot(scr_name):
85  """Break up pathname into (dir+name)."""
86  abs_path = os.path.abspath(scr_name)
87  return (os.path.dirname(abs_path), os.path.basename(abs_path))
88
89
90def GetChromeOSKeyFile(chromeos_root):
91  return os.path.join(chromeos_root, 'src', 'scripts', 'mod_for_test_scripts',
92                      'ssh_keys', 'testing_rsa')
93
94
95def GetChrootPath(chromeos_root):
96  return os.path.join(chromeos_root, 'chroot')
97
98
99def GetInsideChrootPath(chromeos_root, file_path):
100  if not file_path.startswith(GetChrootPath(chromeos_root)):
101    raise RuntimeError("File: %s doesn't seem to be in the chroot: %s" %
102                       (file_path, chromeos_root))
103  return file_path[len(GetChrootPath(chromeos_root)):]
104
105
106def GetOutsideChrootPath(chromeos_root, file_path):
107  return os.path.join(GetChrootPath(chromeos_root), file_path.lstrip('/'))
108
109
110def FormatQuotedCommand(command):
111  return ApplySubs(command, ('"', r'\"'))
112
113
114def FormatCommands(commands):
115  return ApplySubs(
116      str(commands), ('&&', '&&\n'), (';', ';\n'), (r'\n+\s*', '\n'))
117
118
119def GetImageDir(chromeos_root, board):
120  return os.path.join(chromeos_root, 'src', 'build', 'images', board)
121
122
123def LabelLatestImage(chromeos_root, board, label, vanilla_path=None):
124  image_dir = GetImageDir(chromeos_root, board)
125  latest_image_dir = os.path.join(image_dir, 'latest')
126  latest_image_dir = os.path.realpath(latest_image_dir)
127  latest_image_dir = os.path.basename(latest_image_dir)
128  retval = 0
129  with WorkingDirectory(image_dir):
130    command = 'ln -sf -T %s %s' % (latest_image_dir, label)
131    ce = command_executer.GetCommandExecuter()
132    retval = ce.RunCommand(command)
133    if retval:
134      return retval
135    if vanilla_path:
136      command = 'ln -sf -T %s %s' % (vanilla_path, 'vanilla')
137      retval2 = ce.RunCommand(command)
138      return retval2
139  return retval
140
141
142def DoesLabelExist(chromeos_root, board, label):
143  image_label = os.path.join(GetImageDir(chromeos_root, board), label)
144  return os.path.exists(image_label)
145
146
147def GetBuildPackagesCommand(board, usepkg=False, debug=False):
148  if usepkg:
149    usepkg_flag = '--usepkg'
150  else:
151    usepkg_flag = '--nousepkg'
152  if debug:
153    withdebug_flag = '--withdebug'
154  else:
155    withdebug_flag = '--nowithdebug'
156  return ('%s/build_packages %s --withdev --withtest --withautotest '
157          '--skip_toolchain_update %s --board=%s '
158          '--accept_licenses=@CHROMEOS' % (CHROMEOS_SCRIPTS_DIR, usepkg_flag,
159                                           withdebug_flag, board))
160
161
162def GetBuildImageCommand(board, dev=False):
163  dev_args = ''
164  if dev:
165    dev_args = '--noenable_rootfs_verification --disk_layout=2gb-rootfs'
166  return ('%s/build_image --board=%s %s test' % (CHROMEOS_SCRIPTS_DIR, board,
167                                                 dev_args))
168
169
170def GetSetupBoardCommand(board, usepkg=None, force=None):
171  """Get setup_board command."""
172  options = []
173
174  if usepkg:
175    options.append('--usepkg')
176  else:
177    options.append('--nousepkg')
178
179  if force:
180    options.append('--force')
181
182  options.append('--accept-licenses=@CHROMEOS')
183
184  return 'setup_board --board=%s %s' % (board, ' '.join(options))
185
186
187def CanonicalizePath(path):
188  path = os.path.expanduser(path)
189  path = os.path.realpath(path)
190  return path
191
192
193def GetCtargetFromBoard(board, chromeos_root):
194  """Get Ctarget from board."""
195  base_board = board.split('_')[0]
196  command = ('source %s; get_ctarget_from_board %s' % (TOOLCHAIN_UTILS_PATH,
197                                                       base_board))
198  ce = command_executer.GetCommandExecuter()
199  ret, out, _ = ce.ChrootRunCommandWOutput(chromeos_root, command)
200  if ret != 0:
201    raise ValueError('Board %s is invalid!' % board)
202  # Remove ANSI escape sequences.
203  out = StripANSIEscapeSequences(out)
204  return out.strip()
205
206
207def GetArchFromBoard(board, chromeos_root):
208  """Get Arch from board."""
209  base_board = board.split('_')[0]
210  command = (
211      'source %s; get_board_arch %s' % (TOOLCHAIN_UTILS_PATH, base_board))
212  ce = command_executer.GetCommandExecuter()
213  ret, out, _ = ce.ChrootRunCommandWOutput(chromeos_root, command)
214  if ret != 0:
215    raise ValueError('Board %s is invalid!' % board)
216  # Remove ANSI escape sequences.
217  out = StripANSIEscapeSequences(out)
218  return out.strip()
219
220
221def GetGccLibsDestForBoard(board, chromeos_root):
222  """Get gcc libs destination from board."""
223  arch = GetArchFromBoard(board, chromeos_root)
224  if arch == 'x86':
225    return '/build/%s/usr/lib/gcc/' % board
226  if arch == 'amd64':
227    return '/build/%s/usr/lib64/gcc/' % board
228  if arch == 'arm':
229    return '/build/%s/usr/lib/gcc/' % board
230  if arch == 'arm64':
231    return '/build/%s/usr/lib/gcc/' % board
232  raise ValueError('Arch %s is invalid!' % arch)
233
234
235def StripANSIEscapeSequences(string):
236  string = re.sub(r'\x1b\[[0-9]*[a-zA-Z]', '', string)
237  return string
238
239
240def GetChromeSrcDir():
241  return 'var/cache/distfiles/target/chrome-src/src'
242
243
244def GetEnvStringFromDict(env_dict):
245  return ' '.join(['%s="%s"' % var for var in env_dict.items()])
246
247
248def MergeEnvStringWithDict(env_string, env_dict, prepend=True):
249  """Merge env string with dict."""
250  if not env_string.strip():
251    return GetEnvStringFromDict(env_dict)
252  override_env_list = []
253  ce = command_executer.GetCommandExecuter()
254  for k, v in env_dict.items():
255    v = v.strip('"\'')
256    if prepend:
257      new_env = '%s="%s $%s"' % (k, v, k)
258    else:
259      new_env = '%s="$%s %s"' % (k, k, v)
260    command = '; '.join([env_string, new_env, 'echo $%s' % k])
261    ret, out, _ = ce.RunCommandWOutput(command)
262    override_env_list.append('%s=%r' % (k, out.strip()))
263  ret = env_string + ' ' + ' '.join(override_env_list)
264  return ret.strip()
265
266
267def GetAllImages(chromeos_root, board):
268  ce = command_executer.GetCommandExecuter()
269  command = ('find %s/src/build/images/%s -name chromiumos_test_image.bin' %
270             (chromeos_root, board))
271  ret, out, _ = ce.RunCommandWOutput(command)
272  assert ret == 0, 'Could not run command: %s' % command
273  return out.splitlines()
274
275
276def IsFloat(text):
277  if text is None:
278    return False
279  try:
280    float(text)
281    return True
282  except ValueError:
283    return False
284
285
286def RemoveChromeBrowserObjectFiles(chromeos_root, board):
287  """Remove any object files from all the posible locations."""
288  out_dir = os.path.join(
289      GetChrootPath(chromeos_root),
290      'var/cache/chromeos-chrome/chrome-src/src/out_%s' % board)
291  if os.path.exists(out_dir):
292    shutil.rmtree(out_dir)
293    logger.GetLogger().LogCmd('rm -rf %s' % out_dir)
294  out_dir = os.path.join(
295      GetChrootPath(chromeos_root),
296      'var/cache/chromeos-chrome/chrome-src-internal/src/out_%s' % board)
297  if os.path.exists(out_dir):
298    shutil.rmtree(out_dir)
299    logger.GetLogger().LogCmd('rm -rf %s' % out_dir)
300
301
302@contextmanager
303def WorkingDirectory(new_dir):
304  """Get the working directory."""
305  old_dir = os.getcwd()
306  if old_dir != new_dir:
307    msg = 'cd %s' % new_dir
308    logger.GetLogger().LogCmd(msg)
309  os.chdir(new_dir)
310  yield new_dir
311  if old_dir != new_dir:
312    msg = 'cd %s' % old_dir
313    logger.GetLogger().LogCmd(msg)
314  os.chdir(old_dir)
315
316
317def HasGitStagedChanges(git_dir):
318  """Return True if git repository has staged changes."""
319  command = 'cd {0} && git diff --quiet --cached --exit-code HEAD'.format(
320      git_dir)
321  return command_executer.GetCommandExecuter().RunCommand(
322      command, print_to_console=False)
323
324
325def HasGitUnstagedChanges(git_dir):
326  """Return True if git repository has un-staged changes."""
327  command = 'cd {0} && git diff --quiet --exit-code HEAD'.format(git_dir)
328  return command_executer.GetCommandExecuter().RunCommand(
329      command, print_to_console=False)
330
331
332def HasGitUntrackedChanges(git_dir):
333  """Return True if git repository has un-tracked changes."""
334  command = ('cd {0} && test -z '
335             '$(git ls-files --exclude-standard --others)').format(git_dir)
336  return command_executer.GetCommandExecuter().RunCommand(
337      command, print_to_console=False)
338
339
340def GitGetCommitHash(git_dir, commit_symbolic_name):
341  """Return githash for the symbolic git commit.
342
343  For example, commit_symbolic_name could be
344  "cros/gcc.gnu.org/branches/gcc/gcc-4_8-mobile, this function returns the git
345  hash for this symbolic name.
346
347  Args:
348    git_dir: a git working tree.
349    commit_symbolic_name: a symbolic name for a particular git commit.
350
351  Returns:
352    The git hash for the symbolic name or None if fails.
353  """
354
355  command = ('cd {0} && git log -n 1 --pretty="format:%H" {1}').format(
356      git_dir, commit_symbolic_name)
357  rv, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
358      command, print_to_console=False)
359  if rv == 0:
360    return out.strip()
361  return None
362
363
364def IsGitTreeClean(git_dir):
365  """Test if git tree has no local changes.
366
367  Args:
368    git_dir: git tree directory.
369
370  Returns:
371    True if git dir is clean.
372  """
373  if HasGitStagedChanges(git_dir):
374    logger.GetLogger().LogWarning('Git tree has staged changes.')
375    return False
376  if HasGitUnstagedChanges(git_dir):
377    logger.GetLogger().LogWarning('Git tree has unstaged changes.')
378    return False
379  if HasGitUntrackedChanges(git_dir):
380    logger.GetLogger().LogWarning('Git tree has un-tracked changes.')
381    return False
382  return True
383
384
385def GetGitChangesAsList(git_dir, path=None, staged=False):
386  """Get changed files as a list.
387
388  Args:
389    git_dir: git tree directory.
390    path: a relative path that is part of the tree directory, could be null.
391    staged: whether to include staged files as well.
392
393  Returns:
394    A list containing all the changed files.
395  """
396  command = 'cd {0} && git diff --name-only'.format(git_dir)
397  if staged:
398    command += ' --cached'
399  if path:
400    command += ' -- ' + path
401  _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
402      command, print_to_console=False)
403  rv = []
404  for line in out.splitlines():
405    rv.append(line)
406  return rv
407
408
409def IsChromeOsTree(chromeos_root):
410  return (os.path.isdir(
411      os.path.join(chromeos_root, 'src/third_party/chromiumos-overlay')) and
412          os.path.isdir(os.path.join(chromeos_root, 'manifest')))
413
414
415def DeleteChromeOsTree(chromeos_root, dry_run=False):
416  """Delete a ChromeOs tree *safely*.
417
418  Args:
419    chromeos_root: dir of the tree, could be a relative one (but be careful)
420    dry_run: only prints out the command if True
421
422  Returns:
423    True if everything is ok.
424  """
425  if not IsChromeOsTree(chromeos_root):
426    logger.GetLogger().LogWarning(
427        '"{0}" does not seem to be a valid chromeos tree, do nothing.'.format(
428            chromeos_root))
429    return False
430  cmd0 = 'cd {0} && cros_sdk --delete'.format(chromeos_root)
431  if dry_run:
432    print(cmd0)
433  else:
434    if command_executer.GetCommandExecuter().RunCommand(
435        cmd0, print_to_console=True) != 0:
436      return False
437
438  cmd1 = ('export CHROMEOSDIRNAME="$(dirname $(cd {0} && pwd))" && '
439          'export CHROMEOSBASENAME="$(basename $(cd {0} && pwd))" && '
440          'cd $CHROMEOSDIRNAME && sudo rm -fr $CHROMEOSBASENAME'
441         ).format(chromeos_root)
442  if dry_run:
443    print(cmd1)
444    return True
445
446  return command_executer.GetCommandExecuter().RunCommand(
447      cmd1, print_to_console=True) == 0
448
449
450def ApplyGerritPatches(chromeos_root,
451                       gerrit_patch_string,
452                       branch=CROS_MAIN_BRANCH):
453  """Apply gerrit patches on a chromeos tree.
454
455  Args:
456    chromeos_root: chromeos tree path
457    gerrit_patch_string: a patch string just like the one gives to cbuildbot,
458    'id1 id2 *id3 ... idn'. A prefix of '* means this is an internal patch.
459    branch: the tree based on which to apply the patches.
460
461  Returns:
462    True if success.
463  """
464
465  ### First of all, we need chromite libs
466  sys.path.append(os.path.join(chromeos_root, 'chromite'))
467  # Imports below are ok after modifying path to add chromite.
468  # Pylint cannot detect that and complains.
469  # pylint: disable=import-error, import-outside-toplevel
470  from lib import git
471  from lib import gerrit
472  manifest = git.ManifestCheckout(chromeos_root)
473  patch_list = gerrit_patch_string.split(' ')
474  ### This takes time, print log information.
475  logger.GetLogger().LogOutput('Retrieving patch information from server ...')
476  patch_info_list = gerrit.GetGerritPatchInfo(patch_list)
477  for pi in patch_info_list:
478    project_checkout = manifest.FindCheckout(pi.project, strict=False)
479    if not project_checkout:
480      logger.GetLogger().LogError(
481          'Failed to find patch project "{project}" in manifest.'.format(
482              project=pi.project))
483      return False
484
485    pi_str = '{project}:{ref}'.format(project=pi.project, ref=pi.ref)
486    try:
487      project_git_path = project_checkout.GetPath(absolute=True)
488      logger.GetLogger().LogOutput('Applying patch "{0}" in "{1}" ...'.format(
489          pi_str, project_git_path))
490      pi.Apply(project_git_path, branch, trivial=False)
491    except Exception:
492      traceback.print_exc(file=sys.stdout)
493      logger.GetLogger().LogError('Failed to apply patch "{0}"'.format(pi_str))
494      return False
495  return True
496
497
498def BooleanPrompt(prompt='Do you want to continue?',
499                  default=True,
500                  true_value='yes',
501                  false_value='no',
502                  prolog=None):
503  """Helper function for processing boolean choice prompts.
504
505  Args:
506    prompt: The question to present to the user.
507    default: Boolean to return if the user just presses enter.
508    true_value: The text to display that represents a True returned.
509    false_value: The text to display that represents a False returned.
510    prolog: The text to display before prompt.
511
512  Returns:
513    True or False.
514  """
515  true_value, false_value = true_value.lower(), false_value.lower()
516  true_text, false_text = true_value, false_value
517  if true_value == false_value:
518    raise ValueError(
519        'true_value and false_value must differ: got %r' % true_value)
520
521  if default:
522    true_text = true_text[0].upper() + true_text[1:]
523  else:
524    false_text = false_text[0].upper() + false_text[1:]
525
526  prompt = ('\n%s (%s/%s)? ' % (prompt, true_text, false_text))
527
528  if prolog:
529    prompt = ('\n%s\n%s' % (prolog, prompt))
530
531  while True:
532    try:
533      # pylint: disable=input-builtin, bad-builtin
534      response = input(prompt).lower()
535    except EOFError:
536      # If the user hits CTRL+D, or stdin is disabled, use the default.
537      print()
538      response = None
539    except KeyboardInterrupt:
540      # If the user hits CTRL+C, just exit the process.
541      print()
542      print('CTRL+C detected; exiting')
543      sys.exit()
544
545    if not response:
546      return default
547    if true_value.startswith(response):
548      if not false_value.startswith(response):
549        return True
550      # common prefix between the two...
551    elif false_value.startswith(response):
552      return False
553
554
555# pylint: disable=unused-argument
556def rgb2short(r, g, b):
557  """Converts RGB values to xterm-256 color."""
558
559  redcolor = [255, 124, 160, 196, 9]
560  greencolor = [255, 118, 82, 46, 10]
561
562  if g == 0:
563    return redcolor[r // 52]
564  if r == 0:
565    return greencolor[g // 52]
566  return 4
567