• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# -*- coding: utf-8 -*-
2# Copyright 2013 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Utilities for toolchain build."""
7
8from __future__ import division
9from __future__ import print_function
10
11__author__ = 'asharif@google.com (Ahmad Sharif)'
12
13from contextlib import contextmanager
14import os
15import re
16import shutil
17import sys
18import traceback
19
20from cros_utils import command_executer
21from cros_utils import logger
22
23CHROMEOS_SCRIPTS_DIR = '/mnt/host/source/src/scripts'
24TOOLCHAIN_UTILS_PATH = ('/mnt/host/source/src/third_party/toolchain-utils/'
25                        'cros_utils/toolchain_utils.sh')
26
27
28def GetChromeOSVersionFromLSBVersion(lsb_version):
29  """Get Chromeos version from Lsb version."""
30  ce = command_executer.GetCommandExecuter()
31  command = ('git ls-remote '
32             'https://chromium.googlesource.com/chromiumos/manifest.git')
33  ret, out, _ = ce.RunCommandWOutput(command, print_to_console=False)
34  assert ret == 0, 'Command %s failed' % command
35  lower = []
36  for line in out.splitlines():
37    mo = re.search(r'refs/heads/release-R(\d+)-(\d+)\.B', line)
38    if mo:
39      revision = int(mo.group(1))
40      build = int(mo.group(2))
41      lsb_build = int(lsb_version.split('.')[0])
42      if lsb_build > build:
43        lower.append(revision)
44  lower = sorted(lower)
45  if lower:
46    return 'R%d-%s' % (lower[-1] + 1, lsb_version)
47  else:
48    return 'Unknown'
49
50
51def ApplySubs(string, *substitutions):
52  for pattern, replacement in substitutions:
53    string = re.sub(pattern, replacement, string)
54  return string
55
56
57def UnitToNumber(unit_num, base=1000):
58  """Convert a number with unit to float."""
59  unit_dict = {'kilo': base, 'mega': base**2, 'giga': base**3}
60  unit_num = unit_num.lower()
61  mo = re.search(r'(\d*)(.+)?', unit_num)
62  number = mo.group(1)
63  unit = mo.group(2)
64  if not unit:
65    return float(number)
66  for k, v in unit_dict.items():
67    if k.startswith(unit):
68      return float(number) * v
69  raise RuntimeError('Unit: %s not found in byte: %s!' % (unit, unit_num))
70
71
72def GetFilenameFromString(string):
73  return ApplySubs(
74      string,
75      (r'/', '__'),
76      (r'\s', '_'),
77      (r'[\\$="?^]', ''),
78  )
79
80
81def GetRoot(scr_name):
82  """Break up pathname into (dir+name)."""
83  abs_path = os.path.abspath(scr_name)
84  return (os.path.dirname(abs_path), os.path.basename(abs_path))
85
86
87def GetChromeOSKeyFile(chromeos_root):
88  return os.path.join(chromeos_root, 'src', 'scripts', 'mod_for_test_scripts',
89                      'ssh_keys', 'testing_rsa')
90
91
92def GetChrootPath(chromeos_root):
93  return os.path.join(chromeos_root, 'chroot')
94
95
96def GetInsideChrootPath(chromeos_root, file_path):
97  if not file_path.startswith(GetChrootPath(chromeos_root)):
98    raise RuntimeError("File: %s doesn't seem to be in the chroot: %s" %
99                       (file_path, chromeos_root))
100  return file_path[len(GetChrootPath(chromeos_root)):]
101
102
103def GetOutsideChrootPath(chromeos_root, file_path):
104  return os.path.join(GetChrootPath(chromeos_root), file_path.lstrip('/'))
105
106
107def FormatQuotedCommand(command):
108  return ApplySubs(command, ('"', r'\"'))
109
110
111def FormatCommands(commands):
112  return ApplySubs(
113      str(commands), ('&&', '&&\n'), (';', ';\n'), (r'\n+\s*', '\n'))
114
115
116def GetImageDir(chromeos_root, board):
117  return os.path.join(chromeos_root, 'src', 'build', 'images', board)
118
119
120def LabelLatestImage(chromeos_root, board, label, vanilla_path=None):
121  image_dir = GetImageDir(chromeos_root, board)
122  latest_image_dir = os.path.join(image_dir, 'latest')
123  latest_image_dir = os.path.realpath(latest_image_dir)
124  latest_image_dir = os.path.basename(latest_image_dir)
125  retval = 0
126  with WorkingDirectory(image_dir):
127    command = 'ln -sf -T %s %s' % (latest_image_dir, label)
128    ce = command_executer.GetCommandExecuter()
129    retval = ce.RunCommand(command)
130    if retval:
131      return retval
132    if vanilla_path:
133      command = 'ln -sf -T %s %s' % (vanilla_path, 'vanilla')
134      retval2 = ce.RunCommand(command)
135      return retval2
136  return retval
137
138
139def DoesLabelExist(chromeos_root, board, label):
140  image_label = os.path.join(GetImageDir(chromeos_root, board), label)
141  return os.path.exists(image_label)
142
143
144def GetBuildPackagesCommand(board, usepkg=False, debug=False):
145  if usepkg:
146    usepkg_flag = '--usepkg'
147  else:
148    usepkg_flag = '--nousepkg'
149  if debug:
150    withdebug_flag = '--withdebug'
151  else:
152    withdebug_flag = '--nowithdebug'
153  return ('%s/build_packages %s --withdev --withtest --withautotest '
154          '--skip_toolchain_update %s --board=%s '
155          '--accept_licenses=@CHROMEOS' % (CHROMEOS_SCRIPTS_DIR, usepkg_flag,
156                                           withdebug_flag, board))
157
158
159def GetBuildImageCommand(board, dev=False):
160  dev_args = ''
161  if dev:
162    dev_args = '--noenable_rootfs_verification --disk_layout=2gb-rootfs'
163  return ('%s/build_image --board=%s %s test' % (CHROMEOS_SCRIPTS_DIR, board,
164                                                 dev_args))
165
166
167def GetSetupBoardCommand(board, usepkg=None, force=None):
168  """Get setup_board command."""
169  options = []
170
171  if usepkg:
172    options.append('--usepkg')
173  else:
174    options.append('--nousepkg')
175
176  if force:
177    options.append('--force')
178
179  options.append('--accept-licenses=@CHROMEOS')
180
181  return 'setup_board --board=%s %s' % (board, ' '.join(options))
182
183
184def CanonicalizePath(path):
185  path = os.path.expanduser(path)
186  path = os.path.realpath(path)
187  return path
188
189
190def GetCtargetFromBoard(board, chromeos_root):
191  """Get Ctarget from board."""
192  base_board = board.split('_')[0]
193  command = ('source %s; get_ctarget_from_board %s' % (TOOLCHAIN_UTILS_PATH,
194                                                       base_board))
195  ce = command_executer.GetCommandExecuter()
196  ret, out, _ = ce.ChrootRunCommandWOutput(chromeos_root, command)
197  if ret != 0:
198    raise ValueError('Board %s is invalid!' % board)
199  # Remove ANSI escape sequences.
200  out = StripANSIEscapeSequences(out)
201  return out.strip()
202
203
204def GetArchFromBoard(board, chromeos_root):
205  """Get Arch from board."""
206  base_board = board.split('_')[0]
207  command = (
208      'source %s; get_board_arch %s' % (TOOLCHAIN_UTILS_PATH, base_board))
209  ce = command_executer.GetCommandExecuter()
210  ret, out, _ = ce.ChrootRunCommandWOutput(chromeos_root, command)
211  if ret != 0:
212    raise ValueError('Board %s is invalid!' % board)
213  # Remove ANSI escape sequences.
214  out = StripANSIEscapeSequences(out)
215  return out.strip()
216
217
218def GetGccLibsDestForBoard(board, chromeos_root):
219  """Get gcc libs destination from board."""
220  arch = GetArchFromBoard(board, chromeos_root)
221  if arch == 'x86':
222    return '/build/%s/usr/lib/gcc/' % board
223  if arch == 'amd64':
224    return '/build/%s/usr/lib64/gcc/' % board
225  if arch == 'arm':
226    return '/build/%s/usr/lib/gcc/' % board
227  if arch == 'arm64':
228    return '/build/%s/usr/lib/gcc/' % board
229  raise ValueError('Arch %s is invalid!' % arch)
230
231
232def StripANSIEscapeSequences(string):
233  string = re.sub(r'\x1b\[[0-9]*[a-zA-Z]', '', string)
234  return string
235
236
237def GetChromeSrcDir():
238  return 'var/cache/distfiles/target/chrome-src/src'
239
240
241def GetEnvStringFromDict(env_dict):
242  return ' '.join(['%s="%s"' % var for var in env_dict.items()])
243
244
245def MergeEnvStringWithDict(env_string, env_dict, prepend=True):
246  """Merge env string with dict."""
247  if not env_string.strip():
248    return GetEnvStringFromDict(env_dict)
249  override_env_list = []
250  ce = command_executer.GetCommandExecuter()
251  for k, v in env_dict.items():
252    v = v.strip('"\'')
253    if prepend:
254      new_env = '%s="%s $%s"' % (k, v, k)
255    else:
256      new_env = '%s="$%s %s"' % (k, k, v)
257    command = '; '.join([env_string, new_env, 'echo $%s' % k])
258    ret, out, _ = ce.RunCommandWOutput(command)
259    override_env_list.append('%s=%r' % (k, out.strip()))
260  ret = env_string + ' ' + ' '.join(override_env_list)
261  return ret.strip()
262
263
264def GetAllImages(chromeos_root, board):
265  ce = command_executer.GetCommandExecuter()
266  command = ('find %s/src/build/images/%s -name chromiumos_test_image.bin' %
267             (chromeos_root, board))
268  ret, out, _ = ce.RunCommandWOutput(command)
269  assert ret == 0, 'Could not run command: %s' % command
270  return out.splitlines()
271
272
273def IsFloat(text):
274  if text is None:
275    return False
276  try:
277    float(text)
278    return True
279  except ValueError:
280    return False
281
282
283def RemoveChromeBrowserObjectFiles(chromeos_root, board):
284  """Remove any object files from all the posible locations."""
285  out_dir = os.path.join(
286      GetChrootPath(chromeos_root),
287      'var/cache/chromeos-chrome/chrome-src/src/out_%s' % board)
288  if os.path.exists(out_dir):
289    shutil.rmtree(out_dir)
290    logger.GetLogger().LogCmd('rm -rf %s' % out_dir)
291  out_dir = os.path.join(
292      GetChrootPath(chromeos_root),
293      'var/cache/chromeos-chrome/chrome-src-internal/src/out_%s' % board)
294  if os.path.exists(out_dir):
295    shutil.rmtree(out_dir)
296    logger.GetLogger().LogCmd('rm -rf %s' % out_dir)
297
298
299@contextmanager
300def WorkingDirectory(new_dir):
301  """Get the working directory."""
302  old_dir = os.getcwd()
303  if old_dir != new_dir:
304    msg = 'cd %s' % new_dir
305    logger.GetLogger().LogCmd(msg)
306  os.chdir(new_dir)
307  yield new_dir
308  if old_dir != new_dir:
309    msg = 'cd %s' % old_dir
310    logger.GetLogger().LogCmd(msg)
311  os.chdir(old_dir)
312
313
314def HasGitStagedChanges(git_dir):
315  """Return True if git repository has staged changes."""
316  command = 'cd {0} && git diff --quiet --cached --exit-code HEAD'.format(
317      git_dir)
318  return command_executer.GetCommandExecuter().RunCommand(
319      command, print_to_console=False)
320
321
322def HasGitUnstagedChanges(git_dir):
323  """Return True if git repository has un-staged changes."""
324  command = 'cd {0} && git diff --quiet --exit-code HEAD'.format(git_dir)
325  return command_executer.GetCommandExecuter().RunCommand(
326      command, print_to_console=False)
327
328
329def HasGitUntrackedChanges(git_dir):
330  """Return True if git repository has un-tracked changes."""
331  command = ('cd {0} && test -z '
332             '$(git ls-files --exclude-standard --others)').format(git_dir)
333  return command_executer.GetCommandExecuter().RunCommand(
334      command, print_to_console=False)
335
336
337def GitGetCommitHash(git_dir, commit_symbolic_name):
338  """Return githash for the symbolic git commit.
339
340  For example, commit_symbolic_name could be
341  "cros/gcc.gnu.org/branches/gcc/gcc-4_8-mobile, this function returns the git
342  hash for this symbolic name.
343
344  Args:
345    git_dir: a git working tree.
346    commit_symbolic_name: a symbolic name for a particular git commit.
347
348  Returns:
349    The git hash for the symbolic name or None if fails.
350  """
351
352  command = ('cd {0} && git log -n 1 --pretty="format:%H" {1}').format(
353      git_dir, commit_symbolic_name)
354  rv, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
355      command, print_to_console=False)
356  if rv == 0:
357    return out.strip()
358  return None
359
360
361def IsGitTreeClean(git_dir):
362  """Test if git tree has no local changes.
363
364  Args:
365    git_dir: git tree directory.
366
367  Returns:
368    True if git dir is clean.
369  """
370  if HasGitStagedChanges(git_dir):
371    logger.GetLogger().LogWarning('Git tree has staged changes.')
372    return False
373  if HasGitUnstagedChanges(git_dir):
374    logger.GetLogger().LogWarning('Git tree has unstaged changes.')
375    return False
376  if HasGitUntrackedChanges(git_dir):
377    logger.GetLogger().LogWarning('Git tree has un-tracked changes.')
378    return False
379  return True
380
381
382def GetGitChangesAsList(git_dir, path=None, staged=False):
383  """Get changed files as a list.
384
385  Args:
386    git_dir: git tree directory.
387    path: a relative path that is part of the tree directory, could be null.
388    staged: whether to include staged files as well.
389
390  Returns:
391    A list containing all the changed files.
392  """
393  command = 'cd {0} && git diff --name-only'.format(git_dir)
394  if staged:
395    command += ' --cached'
396  if path:
397    command += ' -- ' + path
398  _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
399      command, print_to_console=False)
400  rv = []
401  for line in out.splitlines():
402    rv.append(line)
403  return rv
404
405
406def IsChromeOsTree(chromeos_root):
407  return (os.path.isdir(
408      os.path.join(chromeos_root, 'src/third_party/chromiumos-overlay')) and
409          os.path.isdir(os.path.join(chromeos_root, 'manifest')))
410
411
412def DeleteChromeOsTree(chromeos_root, dry_run=False):
413  """Delete a ChromeOs tree *safely*.
414
415  Args:
416    chromeos_root: dir of the tree, could be a relative one (but be careful)
417    dry_run: only prints out the command if True
418
419  Returns:
420    True if everything is ok.
421  """
422  if not IsChromeOsTree(chromeos_root):
423    logger.GetLogger().LogWarning(
424        '"{0}" does not seem to be a valid chromeos tree, do nothing.'.format(
425            chromeos_root))
426    return False
427  cmd0 = 'cd {0} && cros_sdk --delete'.format(chromeos_root)
428  if dry_run:
429    print(cmd0)
430  else:
431    if command_executer.GetCommandExecuter().RunCommand(
432        cmd0, print_to_console=True) != 0:
433      return False
434
435  cmd1 = ('export CHROMEOSDIRNAME="$(dirname $(cd {0} && pwd))" && '
436          'export CHROMEOSBASENAME="$(basename $(cd {0} && pwd))" && '
437          'cd $CHROMEOSDIRNAME && sudo rm -fr $CHROMEOSBASENAME'
438         ).format(chromeos_root)
439  if dry_run:
440    print(cmd1)
441    return True
442
443  return command_executer.GetCommandExecuter().RunCommand(
444      cmd1, print_to_console=True) == 0
445
446
447def ApplyGerritPatches(chromeos_root, gerrit_patch_string,
448                       branch='cros/master'):
449  """Apply gerrit patches on a chromeos tree.
450
451  Args:
452    chromeos_root: chromeos tree path
453    gerrit_patch_string: a patch string just like the one gives to cbuildbot,
454    'id1 id2 *id3 ... idn'. A prefix of '* means this is an internal patch.
455    branch: the tree based on which to apply the patches.
456
457  Returns:
458    True if success.
459  """
460
461  ### First of all, we need chromite libs
462  sys.path.append(os.path.join(chromeos_root, 'chromite'))
463  # Imports below are ok after modifying path to add chromite.
464  # Pylint cannot detect that and complains.
465  # pylint: disable=import-error
466  from lib import git
467  from lib import gerrit
468  manifest = git.ManifestCheckout(chromeos_root)
469  patch_list = gerrit_patch_string.split(' ')
470  ### This takes time, print log information.
471  logger.GetLogger().LogOutput('Retrieving patch information from server ...')
472  patch_info_list = gerrit.GetGerritPatchInfo(patch_list)
473  for pi in patch_info_list:
474    project_checkout = manifest.FindCheckout(pi.project, strict=False)
475    if not project_checkout:
476      logger.GetLogger().LogError(
477          'Failed to find patch project "{project}" in manifest.'.format(
478              project=pi.project))
479      return False
480
481    pi_str = '{project}:{ref}'.format(project=pi.project, ref=pi.ref)
482    try:
483      project_git_path = project_checkout.GetPath(absolute=True)
484      logger.GetLogger().LogOutput('Applying patch "{0}" in "{1}" ...'.format(
485          pi_str, project_git_path))
486      pi.Apply(project_git_path, branch, trivial=False)
487    except Exception:
488      traceback.print_exc(file=sys.stdout)
489      logger.GetLogger().LogError('Failed to apply patch "{0}"'.format(pi_str))
490      return False
491  return True
492
493
494def BooleanPrompt(prompt='Do you want to continue?',
495                  default=True,
496                  true_value='yes',
497                  false_value='no',
498                  prolog=None):
499  """Helper function for processing boolean choice prompts.
500
501  Args:
502    prompt: The question to present to the user.
503    default: Boolean to return if the user just presses enter.
504    true_value: The text to display that represents a True returned.
505    false_value: The text to display that represents a False returned.
506    prolog: The text to display before prompt.
507
508  Returns:
509    True or False.
510  """
511  true_value, false_value = true_value.lower(), false_value.lower()
512  true_text, false_text = true_value, false_value
513  if true_value == false_value:
514    raise ValueError(
515        'true_value and false_value must differ: got %r' % true_value)
516
517  if default:
518    true_text = true_text[0].upper() + true_text[1:]
519  else:
520    false_text = false_text[0].upper() + false_text[1:]
521
522  prompt = ('\n%s (%s/%s)? ' % (prompt, true_text, false_text))
523
524  if prolog:
525    prompt = ('\n%s\n%s' % (prolog, prompt))
526
527  while True:
528    try:
529      # pylint: disable=input-builtin, bad-builtin
530      response = input(prompt).lower()
531    except EOFError:
532      # If the user hits CTRL+D, or stdin is disabled, use the default.
533      print()
534      response = None
535    except KeyboardInterrupt:
536      # If the user hits CTRL+C, just exit the process.
537      print()
538      print('CTRL+C detected; exiting')
539      sys.exit()
540
541    if not response:
542      return default
543    if true_value.startswith(response):
544      if not false_value.startswith(response):
545        return True
546      # common prefix between the two...
547    elif false_value.startswith(response):
548      return False
549
550
551# pylint: disable=unused-argument
552def rgb2short(r, g, b):
553  """Converts RGB values to xterm-256 color."""
554
555  redcolor = [255, 124, 160, 196, 9]
556  greencolor = [255, 118, 82, 46, 10]
557
558  if g == 0:
559    return redcolor[r // 52]
560  if r == 0:
561    return greencolor[g // 52]
562  return 4
563