1# -*- coding: utf-8 -*- 2# Copyright 2013 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Utilities for toolchain build.""" 7 8from __future__ import division 9from __future__ import print_function 10 11__author__ = 'asharif@google.com (Ahmad Sharif)' 12 13from contextlib import contextmanager 14import os 15import re 16import shutil 17import sys 18import traceback 19 20from cros_utils import command_executer 21from cros_utils import logger 22 23CHROMEOS_SCRIPTS_DIR = '/mnt/host/source/src/scripts' 24TOOLCHAIN_UTILS_PATH = ('/mnt/host/source/src/third_party/toolchain-utils/' 25 'cros_utils/toolchain_utils.sh') 26 27CROS_MAIN_BRANCH = 'cros/master' 28 29 30def GetChromeOSVersionFromLSBVersion(lsb_version): 31 """Get Chromeos version from Lsb version.""" 32 ce = command_executer.GetCommandExecuter() 33 command = ('git ls-remote ' 34 'https://chromium.googlesource.com/chromiumos/manifest.git ' 35 'refs/heads/release-R*') 36 ret, out, _ = ce.RunCommandWOutput(command, print_to_console=False) 37 assert ret == 0, 'Command %s failed' % command 38 lower = [] 39 for line in out.splitlines(): 40 mo = re.search(r'refs/heads/release-R(\d+)-(\d+)\.B', line) 41 if mo: 42 revision = int(mo.group(1)) 43 build = int(mo.group(2)) 44 lsb_build = int(lsb_version.split('.')[0]) 45 if lsb_build > build: 46 lower.append(revision) 47 lower = sorted(lower) 48 if lower: 49 return 'R%d-%s' % (lower[-1] + 1, lsb_version) 50 else: 51 return 'Unknown' 52 53 54def ApplySubs(string, *substitutions): 55 for pattern, replacement in substitutions: 56 string = re.sub(pattern, replacement, string) 57 return string 58 59 60def UnitToNumber(unit_num, base=1000): 61 """Convert a number with unit to float.""" 62 unit_dict = {'kilo': base, 'mega': base**2, 'giga': base**3} 63 unit_num = unit_num.lower() 64 mo = re.search(r'(\d*)(.+)?', unit_num) 65 number = mo.group(1) 66 unit = mo.group(2) 67 if not unit: 68 return float(number) 69 for k, v in unit_dict.items(): 70 if k.startswith(unit): 71 return float(number) * v 72 raise RuntimeError('Unit: %s not found in byte: %s!' % (unit, unit_num)) 73 74 75def GetFilenameFromString(string): 76 return ApplySubs( 77 string, 78 (r'/', '__'), 79 (r'\s', '_'), 80 (r'[\\$="?^]', ''), 81 ) 82 83 84def GetRoot(scr_name): 85 """Break up pathname into (dir+name).""" 86 abs_path = os.path.abspath(scr_name) 87 return (os.path.dirname(abs_path), os.path.basename(abs_path)) 88 89 90def GetChromeOSKeyFile(chromeos_root): 91 return os.path.join(chromeos_root, 'src', 'scripts', 'mod_for_test_scripts', 92 'ssh_keys', 'testing_rsa') 93 94 95def GetChrootPath(chromeos_root): 96 return os.path.join(chromeos_root, 'chroot') 97 98 99def GetInsideChrootPath(chromeos_root, file_path): 100 if not file_path.startswith(GetChrootPath(chromeos_root)): 101 raise RuntimeError("File: %s doesn't seem to be in the chroot: %s" % 102 (file_path, chromeos_root)) 103 return file_path[len(GetChrootPath(chromeos_root)):] 104 105 106def GetOutsideChrootPath(chromeos_root, file_path): 107 return os.path.join(GetChrootPath(chromeos_root), file_path.lstrip('/')) 108 109 110def FormatQuotedCommand(command): 111 return ApplySubs(command, ('"', r'\"')) 112 113 114def FormatCommands(commands): 115 return ApplySubs( 116 str(commands), ('&&', '&&\n'), (';', ';\n'), (r'\n+\s*', '\n')) 117 118 119def GetImageDir(chromeos_root, board): 120 return os.path.join(chromeos_root, 'src', 'build', 'images', board) 121 122 123def LabelLatestImage(chromeos_root, board, label, vanilla_path=None): 124 image_dir = GetImageDir(chromeos_root, board) 125 latest_image_dir = os.path.join(image_dir, 'latest') 126 latest_image_dir = os.path.realpath(latest_image_dir) 127 latest_image_dir = os.path.basename(latest_image_dir) 128 retval = 0 129 with WorkingDirectory(image_dir): 130 command = 'ln -sf -T %s %s' % (latest_image_dir, label) 131 ce = command_executer.GetCommandExecuter() 132 retval = ce.RunCommand(command) 133 if retval: 134 return retval 135 if vanilla_path: 136 command = 'ln -sf -T %s %s' % (vanilla_path, 'vanilla') 137 retval2 = ce.RunCommand(command) 138 return retval2 139 return retval 140 141 142def DoesLabelExist(chromeos_root, board, label): 143 image_label = os.path.join(GetImageDir(chromeos_root, board), label) 144 return os.path.exists(image_label) 145 146 147def GetBuildPackagesCommand(board, usepkg=False, debug=False): 148 if usepkg: 149 usepkg_flag = '--usepkg' 150 else: 151 usepkg_flag = '--nousepkg' 152 if debug: 153 withdebug_flag = '--withdebug' 154 else: 155 withdebug_flag = '--nowithdebug' 156 return ('%s/build_packages %s --withdev --withtest --withautotest ' 157 '--skip_toolchain_update %s --board=%s ' 158 '--accept_licenses=@CHROMEOS' % (CHROMEOS_SCRIPTS_DIR, usepkg_flag, 159 withdebug_flag, board)) 160 161 162def GetBuildImageCommand(board, dev=False): 163 dev_args = '' 164 if dev: 165 dev_args = '--noenable_rootfs_verification --disk_layout=2gb-rootfs' 166 return ('%s/build_image --board=%s %s test' % (CHROMEOS_SCRIPTS_DIR, board, 167 dev_args)) 168 169 170def GetSetupBoardCommand(board, usepkg=None, force=None): 171 """Get setup_board command.""" 172 options = [] 173 174 if usepkg: 175 options.append('--usepkg') 176 else: 177 options.append('--nousepkg') 178 179 if force: 180 options.append('--force') 181 182 options.append('--accept-licenses=@CHROMEOS') 183 184 return 'setup_board --board=%s %s' % (board, ' '.join(options)) 185 186 187def CanonicalizePath(path): 188 path = os.path.expanduser(path) 189 path = os.path.realpath(path) 190 return path 191 192 193def GetCtargetFromBoard(board, chromeos_root): 194 """Get Ctarget from board.""" 195 base_board = board.split('_')[0] 196 command = ('source %s; get_ctarget_from_board %s' % (TOOLCHAIN_UTILS_PATH, 197 base_board)) 198 ce = command_executer.GetCommandExecuter() 199 ret, out, _ = ce.ChrootRunCommandWOutput(chromeos_root, command) 200 if ret != 0: 201 raise ValueError('Board %s is invalid!' % board) 202 # Remove ANSI escape sequences. 203 out = StripANSIEscapeSequences(out) 204 return out.strip() 205 206 207def GetArchFromBoard(board, chromeos_root): 208 """Get Arch from board.""" 209 base_board = board.split('_')[0] 210 command = ( 211 'source %s; get_board_arch %s' % (TOOLCHAIN_UTILS_PATH, base_board)) 212 ce = command_executer.GetCommandExecuter() 213 ret, out, _ = ce.ChrootRunCommandWOutput(chromeos_root, command) 214 if ret != 0: 215 raise ValueError('Board %s is invalid!' % board) 216 # Remove ANSI escape sequences. 217 out = StripANSIEscapeSequences(out) 218 return out.strip() 219 220 221def GetGccLibsDestForBoard(board, chromeos_root): 222 """Get gcc libs destination from board.""" 223 arch = GetArchFromBoard(board, chromeos_root) 224 if arch == 'x86': 225 return '/build/%s/usr/lib/gcc/' % board 226 if arch == 'amd64': 227 return '/build/%s/usr/lib64/gcc/' % board 228 if arch == 'arm': 229 return '/build/%s/usr/lib/gcc/' % board 230 if arch == 'arm64': 231 return '/build/%s/usr/lib/gcc/' % board 232 raise ValueError('Arch %s is invalid!' % arch) 233 234 235def StripANSIEscapeSequences(string): 236 string = re.sub(r'\x1b\[[0-9]*[a-zA-Z]', '', string) 237 return string 238 239 240def GetChromeSrcDir(): 241 return 'var/cache/distfiles/target/chrome-src/src' 242 243 244def GetEnvStringFromDict(env_dict): 245 return ' '.join(['%s="%s"' % var for var in env_dict.items()]) 246 247 248def MergeEnvStringWithDict(env_string, env_dict, prepend=True): 249 """Merge env string with dict.""" 250 if not env_string.strip(): 251 return GetEnvStringFromDict(env_dict) 252 override_env_list = [] 253 ce = command_executer.GetCommandExecuter() 254 for k, v in env_dict.items(): 255 v = v.strip('"\'') 256 if prepend: 257 new_env = '%s="%s $%s"' % (k, v, k) 258 else: 259 new_env = '%s="$%s %s"' % (k, k, v) 260 command = '; '.join([env_string, new_env, 'echo $%s' % k]) 261 ret, out, _ = ce.RunCommandWOutput(command) 262 override_env_list.append('%s=%r' % (k, out.strip())) 263 ret = env_string + ' ' + ' '.join(override_env_list) 264 return ret.strip() 265 266 267def GetAllImages(chromeos_root, board): 268 ce = command_executer.GetCommandExecuter() 269 command = ('find %s/src/build/images/%s -name chromiumos_test_image.bin' % 270 (chromeos_root, board)) 271 ret, out, _ = ce.RunCommandWOutput(command) 272 assert ret == 0, 'Could not run command: %s' % command 273 return out.splitlines() 274 275 276def IsFloat(text): 277 if text is None: 278 return False 279 try: 280 float(text) 281 return True 282 except ValueError: 283 return False 284 285 286def RemoveChromeBrowserObjectFiles(chromeos_root, board): 287 """Remove any object files from all the posible locations.""" 288 out_dir = os.path.join( 289 GetChrootPath(chromeos_root), 290 'var/cache/chromeos-chrome/chrome-src/src/out_%s' % board) 291 if os.path.exists(out_dir): 292 shutil.rmtree(out_dir) 293 logger.GetLogger().LogCmd('rm -rf %s' % out_dir) 294 out_dir = os.path.join( 295 GetChrootPath(chromeos_root), 296 'var/cache/chromeos-chrome/chrome-src-internal/src/out_%s' % board) 297 if os.path.exists(out_dir): 298 shutil.rmtree(out_dir) 299 logger.GetLogger().LogCmd('rm -rf %s' % out_dir) 300 301 302@contextmanager 303def WorkingDirectory(new_dir): 304 """Get the working directory.""" 305 old_dir = os.getcwd() 306 if old_dir != new_dir: 307 msg = 'cd %s' % new_dir 308 logger.GetLogger().LogCmd(msg) 309 os.chdir(new_dir) 310 yield new_dir 311 if old_dir != new_dir: 312 msg = 'cd %s' % old_dir 313 logger.GetLogger().LogCmd(msg) 314 os.chdir(old_dir) 315 316 317def HasGitStagedChanges(git_dir): 318 """Return True if git repository has staged changes.""" 319 command = 'cd {0} && git diff --quiet --cached --exit-code HEAD'.format( 320 git_dir) 321 return command_executer.GetCommandExecuter().RunCommand( 322 command, print_to_console=False) 323 324 325def HasGitUnstagedChanges(git_dir): 326 """Return True if git repository has un-staged changes.""" 327 command = 'cd {0} && git diff --quiet --exit-code HEAD'.format(git_dir) 328 return command_executer.GetCommandExecuter().RunCommand( 329 command, print_to_console=False) 330 331 332def HasGitUntrackedChanges(git_dir): 333 """Return True if git repository has un-tracked changes.""" 334 command = ('cd {0} && test -z ' 335 '$(git ls-files --exclude-standard --others)').format(git_dir) 336 return command_executer.GetCommandExecuter().RunCommand( 337 command, print_to_console=False) 338 339 340def GitGetCommitHash(git_dir, commit_symbolic_name): 341 """Return githash for the symbolic git commit. 342 343 For example, commit_symbolic_name could be 344 "cros/gcc.gnu.org/branches/gcc/gcc-4_8-mobile, this function returns the git 345 hash for this symbolic name. 346 347 Args: 348 git_dir: a git working tree. 349 commit_symbolic_name: a symbolic name for a particular git commit. 350 351 Returns: 352 The git hash for the symbolic name or None if fails. 353 """ 354 355 command = ('cd {0} && git log -n 1 --pretty="format:%H" {1}').format( 356 git_dir, commit_symbolic_name) 357 rv, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( 358 command, print_to_console=False) 359 if rv == 0: 360 return out.strip() 361 return None 362 363 364def IsGitTreeClean(git_dir): 365 """Test if git tree has no local changes. 366 367 Args: 368 git_dir: git tree directory. 369 370 Returns: 371 True if git dir is clean. 372 """ 373 if HasGitStagedChanges(git_dir): 374 logger.GetLogger().LogWarning('Git tree has staged changes.') 375 return False 376 if HasGitUnstagedChanges(git_dir): 377 logger.GetLogger().LogWarning('Git tree has unstaged changes.') 378 return False 379 if HasGitUntrackedChanges(git_dir): 380 logger.GetLogger().LogWarning('Git tree has un-tracked changes.') 381 return False 382 return True 383 384 385def GetGitChangesAsList(git_dir, path=None, staged=False): 386 """Get changed files as a list. 387 388 Args: 389 git_dir: git tree directory. 390 path: a relative path that is part of the tree directory, could be null. 391 staged: whether to include staged files as well. 392 393 Returns: 394 A list containing all the changed files. 395 """ 396 command = 'cd {0} && git diff --name-only'.format(git_dir) 397 if staged: 398 command += ' --cached' 399 if path: 400 command += ' -- ' + path 401 _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( 402 command, print_to_console=False) 403 rv = [] 404 for line in out.splitlines(): 405 rv.append(line) 406 return rv 407 408 409def IsChromeOsTree(chromeos_root): 410 return (os.path.isdir( 411 os.path.join(chromeos_root, 'src/third_party/chromiumos-overlay')) and 412 os.path.isdir(os.path.join(chromeos_root, 'manifest'))) 413 414 415def DeleteChromeOsTree(chromeos_root, dry_run=False): 416 """Delete a ChromeOs tree *safely*. 417 418 Args: 419 chromeos_root: dir of the tree, could be a relative one (but be careful) 420 dry_run: only prints out the command if True 421 422 Returns: 423 True if everything is ok. 424 """ 425 if not IsChromeOsTree(chromeos_root): 426 logger.GetLogger().LogWarning( 427 '"{0}" does not seem to be a valid chromeos tree, do nothing.'.format( 428 chromeos_root)) 429 return False 430 cmd0 = 'cd {0} && cros_sdk --delete'.format(chromeos_root) 431 if dry_run: 432 print(cmd0) 433 else: 434 if command_executer.GetCommandExecuter().RunCommand( 435 cmd0, print_to_console=True) != 0: 436 return False 437 438 cmd1 = ('export CHROMEOSDIRNAME="$(dirname $(cd {0} && pwd))" && ' 439 'export CHROMEOSBASENAME="$(basename $(cd {0} && pwd))" && ' 440 'cd $CHROMEOSDIRNAME && sudo rm -fr $CHROMEOSBASENAME' 441 ).format(chromeos_root) 442 if dry_run: 443 print(cmd1) 444 return True 445 446 return command_executer.GetCommandExecuter().RunCommand( 447 cmd1, print_to_console=True) == 0 448 449 450def ApplyGerritPatches(chromeos_root, 451 gerrit_patch_string, 452 branch=CROS_MAIN_BRANCH): 453 """Apply gerrit patches on a chromeos tree. 454 455 Args: 456 chromeos_root: chromeos tree path 457 gerrit_patch_string: a patch string just like the one gives to cbuildbot, 458 'id1 id2 *id3 ... idn'. A prefix of '* means this is an internal patch. 459 branch: the tree based on which to apply the patches. 460 461 Returns: 462 True if success. 463 """ 464 465 ### First of all, we need chromite libs 466 sys.path.append(os.path.join(chromeos_root, 'chromite')) 467 # Imports below are ok after modifying path to add chromite. 468 # Pylint cannot detect that and complains. 469 # pylint: disable=import-error, import-outside-toplevel 470 from lib import git 471 from lib import gerrit 472 manifest = git.ManifestCheckout(chromeos_root) 473 patch_list = gerrit_patch_string.split(' ') 474 ### This takes time, print log information. 475 logger.GetLogger().LogOutput('Retrieving patch information from server ...') 476 patch_info_list = gerrit.GetGerritPatchInfo(patch_list) 477 for pi in patch_info_list: 478 project_checkout = manifest.FindCheckout(pi.project, strict=False) 479 if not project_checkout: 480 logger.GetLogger().LogError( 481 'Failed to find patch project "{project}" in manifest.'.format( 482 project=pi.project)) 483 return False 484 485 pi_str = '{project}:{ref}'.format(project=pi.project, ref=pi.ref) 486 try: 487 project_git_path = project_checkout.GetPath(absolute=True) 488 logger.GetLogger().LogOutput('Applying patch "{0}" in "{1}" ...'.format( 489 pi_str, project_git_path)) 490 pi.Apply(project_git_path, branch, trivial=False) 491 except Exception: 492 traceback.print_exc(file=sys.stdout) 493 logger.GetLogger().LogError('Failed to apply patch "{0}"'.format(pi_str)) 494 return False 495 return True 496 497 498def BooleanPrompt(prompt='Do you want to continue?', 499 default=True, 500 true_value='yes', 501 false_value='no', 502 prolog=None): 503 """Helper function for processing boolean choice prompts. 504 505 Args: 506 prompt: The question to present to the user. 507 default: Boolean to return if the user just presses enter. 508 true_value: The text to display that represents a True returned. 509 false_value: The text to display that represents a False returned. 510 prolog: The text to display before prompt. 511 512 Returns: 513 True or False. 514 """ 515 true_value, false_value = true_value.lower(), false_value.lower() 516 true_text, false_text = true_value, false_value 517 if true_value == false_value: 518 raise ValueError( 519 'true_value and false_value must differ: got %r' % true_value) 520 521 if default: 522 true_text = true_text[0].upper() + true_text[1:] 523 else: 524 false_text = false_text[0].upper() + false_text[1:] 525 526 prompt = ('\n%s (%s/%s)? ' % (prompt, true_text, false_text)) 527 528 if prolog: 529 prompt = ('\n%s\n%s' % (prolog, prompt)) 530 531 while True: 532 try: 533 # pylint: disable=input-builtin, bad-builtin 534 response = input(prompt).lower() 535 except EOFError: 536 # If the user hits CTRL+D, or stdin is disabled, use the default. 537 print() 538 response = None 539 except KeyboardInterrupt: 540 # If the user hits CTRL+C, just exit the process. 541 print() 542 print('CTRL+C detected; exiting') 543 sys.exit() 544 545 if not response: 546 return default 547 if true_value.startswith(response): 548 if not false_value.startswith(response): 549 return True 550 # common prefix between the two... 551 elif false_value.startswith(response): 552 return False 553 554 555# pylint: disable=unused-argument 556def rgb2short(r, g, b): 557 """Converts RGB values to xterm-256 color.""" 558 559 redcolor = [255, 124, 160, 196, 9] 560 greencolor = [255, 118, 82, 46, 10] 561 562 if g == 0: 563 return redcolor[r // 52] 564 if r == 0: 565 return greencolor[g // 52] 566 return 4 567