• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# -*- coding: utf-8 -*-
2# Copyright 2013 The ChromiumOS Authors
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Utilities for toolchain build."""
7
8
9__author__ = "asharif@google.com (Ahmad Sharif)"
10
11from contextlib import contextmanager
12import os
13import re
14import shutil
15import sys
16
17from cros_utils import command_executer
18from cros_utils import logger
19
20
21CHROMEOS_SCRIPTS_DIR = "/mnt/host/source/src/scripts"
22TOOLCHAIN_UTILS_PATH = (
23    "/mnt/host/source/src/third_party/toolchain-utils/"
24    "cros_utils/toolchain_utils.sh"
25)
26
27
28def GetChromeOSVersionFromLSBVersion(lsb_version):
29    """Get Chromeos version from Lsb version."""
30    ce = command_executer.GetCommandExecuter()
31    command = (
32        "git ls-remote "
33        "https://chromium.googlesource.com/chromiumos/manifest.git "
34        "refs/heads/release-R*"
35    )
36    ret, out, _ = ce.RunCommandWOutput(command, print_to_console=False)
37    assert ret == 0, "Command %s failed" % command
38    lower = []
39    for line in out.splitlines():
40        mo = re.search(r"refs/heads/release-R(\d+)-(\d+)\.B", line)
41        if mo:
42            revision = int(mo.group(1))
43            build = int(mo.group(2))
44            lsb_build = int(lsb_version.split(".")[0])
45            if lsb_build > build:
46                lower.append(revision)
47    lower = sorted(lower)
48    if lower:
49        return "R%d-%s" % (lower[-1] + 1, lsb_version)
50    else:
51        return "Unknown"
52
53
54def ApplySubs(string, *substitutions):
55    for pattern, replacement in substitutions:
56        string = re.sub(pattern, replacement, string)
57    return string
58
59
60def UnitToNumber(unit_num, base=1000):
61    """Convert a number with unit to float."""
62    unit_dict = {"kilo": base, "mega": base ** 2, "giga": base ** 3}
63    unit_num = unit_num.lower()
64    mo = re.search(r"(\d*)(.+)?", unit_num)
65    number = mo.group(1)
66    unit = mo.group(2)
67    if not unit:
68        return float(number)
69    for k, v in unit_dict.items():
70        if k.startswith(unit):
71            return float(number) * v
72    raise RuntimeError("Unit: %s not found in byte: %s!" % (unit, unit_num))
73
74
75def GetFilenameFromString(string):
76    return ApplySubs(
77        string,
78        (r"/", "__"),
79        (r"\s", "_"),
80        (r'[\\$="?^]', ""),
81    )
82
83
84def GetRoot(scr_name):
85    """Break up pathname into (dir+name)."""
86    abs_path = os.path.abspath(scr_name)
87    return (os.path.dirname(abs_path), os.path.basename(abs_path))
88
89
90def GetChromeOSKeyFile(chromeos_root):
91    return os.path.join(
92        chromeos_root,
93        "src",
94        "scripts",
95        "mod_for_test_scripts",
96        "ssh_keys",
97        "testing_rsa",
98    )
99
100
101def GetChrootPath(chromeos_root):
102    return os.path.join(chromeos_root, "chroot")
103
104
105def GetInsideChrootPath(chromeos_root, file_path):
106    if not file_path.startswith(GetChrootPath(chromeos_root)):
107        raise RuntimeError(
108            "File: %s doesn't seem to be in the chroot: %s"
109            % (file_path, chromeos_root)
110        )
111    return file_path[len(GetChrootPath(chromeos_root)) :]
112
113
114def GetOutsideChrootPath(chromeos_root, file_path):
115    return os.path.join(GetChrootPath(chromeos_root), file_path.lstrip("/"))
116
117
118def FormatQuotedCommand(command):
119    return ApplySubs(command, ('"', r"\""))
120
121
122def FormatCommands(commands):
123    return ApplySubs(
124        str(commands), ("&&", "&&\n"), (";", ";\n"), (r"\n+\s*", "\n")
125    )
126
127
128def GetImageDir(chromeos_root, board):
129    return os.path.join(chromeos_root, "src", "build", "images", board)
130
131
132def LabelLatestImage(chromeos_root, board, label, vanilla_path=None):
133    image_dir = GetImageDir(chromeos_root, board)
134    latest_image_dir = os.path.join(image_dir, "latest")
135    latest_image_dir = os.path.realpath(latest_image_dir)
136    latest_image_dir = os.path.basename(latest_image_dir)
137    retval = 0
138    with WorkingDirectory(image_dir):
139        command = "ln -sf -T %s %s" % (latest_image_dir, label)
140        ce = command_executer.GetCommandExecuter()
141        retval = ce.RunCommand(command)
142        if retval:
143            return retval
144        if vanilla_path:
145            command = "ln -sf -T %s %s" % (vanilla_path, "vanilla")
146            retval2 = ce.RunCommand(command)
147            return retval2
148    return retval
149
150
151def DoesLabelExist(chromeos_root, board, label):
152    image_label = os.path.join(GetImageDir(chromeos_root, board), label)
153    return os.path.exists(image_label)
154
155
156def GetBuildPackagesCommand(board, usepkg=False, debug=False):
157    if usepkg:
158        usepkg_flag = "--usepkg"
159    else:
160        usepkg_flag = "--nousepkg"
161    if debug:
162        withdebug_flag = "--withdebug"
163    else:
164        withdebug_flag = "--nowithdebug"
165    return (
166        "%s/build_packages %s --withdev --withtest --withautotest "
167        "--skip_toolchain_update %s --board=%s "
168        "--accept_licenses=@CHROMEOS"
169        % (CHROMEOS_SCRIPTS_DIR, usepkg_flag, withdebug_flag, board)
170    )
171
172
173def GetBuildImageCommand(board, dev=False):
174    dev_args = ""
175    if dev:
176        dev_args = "--noenable_rootfs_verification --disk_layout=2gb-rootfs"
177    return "%s/build_image --board=%s %s test" % (
178        CHROMEOS_SCRIPTS_DIR,
179        board,
180        dev_args,
181    )
182
183
184def GetSetupBoardCommand(board, usepkg=None, force=None):
185    """Get setup_board command."""
186    options = []
187
188    if usepkg:
189        options.append("--usepkg")
190    else:
191        options.append("--nousepkg")
192
193    if force:
194        options.append("--force")
195
196    options.append("--accept-licenses=@CHROMEOS")
197
198    return "setup_board --board=%s %s" % (board, " ".join(options))
199
200
201def CanonicalizePath(path):
202    path = os.path.expanduser(path)
203    path = os.path.realpath(path)
204    return path
205
206
207def GetCtargetFromBoard(board, chromeos_root):
208    """Get Ctarget from board."""
209    base_board = board.split("_")[0]
210    command = "source %s; get_ctarget_from_board %s" % (
211        TOOLCHAIN_UTILS_PATH,
212        base_board,
213    )
214    ce = command_executer.GetCommandExecuter()
215    ret, out, _ = ce.ChrootRunCommandWOutput(chromeos_root, command)
216    if ret != 0:
217        raise ValueError("Board %s is invalid!" % board)
218    # Remove ANSI escape sequences.
219    out = StripANSIEscapeSequences(out)
220    return out.strip()
221
222
223def GetArchFromBoard(board, chromeos_root):
224    """Get Arch from board."""
225    base_board = board.split("_")[0]
226    command = "source %s; get_board_arch %s" % (
227        TOOLCHAIN_UTILS_PATH,
228        base_board,
229    )
230    ce = command_executer.GetCommandExecuter()
231    ret, out, _ = ce.ChrootRunCommandWOutput(chromeos_root, command)
232    if ret != 0:
233        raise ValueError("Board %s is invalid!" % board)
234    # Remove ANSI escape sequences.
235    out = StripANSIEscapeSequences(out)
236    return out.strip()
237
238
239def GetGccLibsDestForBoard(board, chromeos_root):
240    """Get gcc libs destination from board."""
241    arch = GetArchFromBoard(board, chromeos_root)
242    if arch == "x86":
243        return "/build/%s/usr/lib/gcc/" % board
244    if arch == "amd64":
245        return "/build/%s/usr/lib64/gcc/" % board
246    if arch == "arm":
247        return "/build/%s/usr/lib/gcc/" % board
248    if arch == "arm64":
249        return "/build/%s/usr/lib/gcc/" % board
250    raise ValueError("Arch %s is invalid!" % arch)
251
252
253def StripANSIEscapeSequences(string):
254    string = re.sub(r"\x1b\[[0-9]*[a-zA-Z]", "", string)
255    return string
256
257
258def GetChromeSrcDir():
259    return "var/cache/distfiles/target/chrome-src/src"
260
261
262def GetEnvStringFromDict(env_dict):
263    return " ".join(['%s="%s"' % var for var in env_dict.items()])
264
265
266def MergeEnvStringWithDict(env_string, env_dict, prepend=True):
267    """Merge env string with dict."""
268    if not env_string.strip():
269        return GetEnvStringFromDict(env_dict)
270    override_env_list = []
271    ce = command_executer.GetCommandExecuter()
272    for k, v in env_dict.items():
273        v = v.strip("\"'")
274        if prepend:
275            new_env = '%s="%s $%s"' % (k, v, k)
276        else:
277            new_env = '%s="$%s %s"' % (k, k, v)
278        command = "; ".join([env_string, new_env, "echo $%s" % k])
279        ret, out, _ = ce.RunCommandWOutput(command)
280        override_env_list.append("%s=%r" % (k, out.strip()))
281    ret = env_string + " " + " ".join(override_env_list)
282    return ret.strip()
283
284
285def GetAllImages(chromeos_root, board):
286    ce = command_executer.GetCommandExecuter()
287    command = "find %s/src/build/images/%s -name chromiumos_test_image.bin" % (
288        chromeos_root,
289        board,
290    )
291    ret, out, _ = ce.RunCommandWOutput(command)
292    assert ret == 0, "Could not run command: %s" % command
293    return out.splitlines()
294
295
296def IsFloat(text):
297    if text is None:
298        return False
299    try:
300        float(text)
301        return True
302    except ValueError:
303        return False
304
305
306def RemoveChromeBrowserObjectFiles(chromeos_root, board):
307    """Remove any object files from all the posible locations."""
308    out_dir = os.path.join(
309        GetChrootPath(chromeos_root),
310        "var/cache/chromeos-chrome/chrome-src/src/out_%s" % board,
311    )
312    if os.path.exists(out_dir):
313        shutil.rmtree(out_dir)
314        logger.GetLogger().LogCmd("rm -rf %s" % out_dir)
315    out_dir = os.path.join(
316        GetChrootPath(chromeos_root),
317        "var/cache/chromeos-chrome/chrome-src-internal/src/out_%s" % board,
318    )
319    if os.path.exists(out_dir):
320        shutil.rmtree(out_dir)
321        logger.GetLogger().LogCmd("rm -rf %s" % out_dir)
322
323
324@contextmanager
325def WorkingDirectory(new_dir):
326    """Get the working directory."""
327    old_dir = os.getcwd()
328    if old_dir != new_dir:
329        msg = "cd %s" % new_dir
330        logger.GetLogger().LogCmd(msg)
331    os.chdir(new_dir)
332    yield new_dir
333    if old_dir != new_dir:
334        msg = "cd %s" % old_dir
335        logger.GetLogger().LogCmd(msg)
336    os.chdir(old_dir)
337
338
339def HasGitStagedChanges(git_dir):
340    """Return True if git repository has staged changes."""
341    command = f"cd {git_dir} && git diff --quiet --cached --exit-code HEAD"
342    return command_executer.GetCommandExecuter().RunCommand(
343        command, print_to_console=False
344    )
345
346
347def HasGitUnstagedChanges(git_dir):
348    """Return True if git repository has un-staged changes."""
349    command = f"cd {git_dir} && git diff --quiet --exit-code HEAD"
350    return command_executer.GetCommandExecuter().RunCommand(
351        command, print_to_console=False
352    )
353
354
355def HasGitUntrackedChanges(git_dir):
356    """Return True if git repository has un-tracked changes."""
357    command = (
358        f"cd {git_dir} && test -z "
359        "$(git ls-files --exclude-standard --others)"
360    )
361    return command_executer.GetCommandExecuter().RunCommand(
362        command, print_to_console=False
363    )
364
365
366def GitGetCommitHash(git_dir, commit_symbolic_name):
367    """Return githash for the symbolic git commit.
368
369    For example, commit_symbolic_name could be
370    "cros/gcc.gnu.org/branches/gcc/gcc-4_8-mobile, this function returns the git
371    hash for this symbolic name.
372
373    Args:
374      git_dir: a git working tree.
375      commit_symbolic_name: a symbolic name for a particular git commit.
376
377    Returns:
378      The git hash for the symbolic name or None if fails.
379    """
380
381    command = (
382        f"cd {git_dir} && git log -n 1"
383        f' --pretty="format:%H" {commit_symbolic_name}'
384    )
385    rv, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
386        command, print_to_console=False
387    )
388    if rv == 0:
389        return out.strip()
390    return None
391
392
393def IsGitTreeClean(git_dir):
394    """Test if git tree has no local changes.
395
396    Args:
397      git_dir: git tree directory.
398
399    Returns:
400      True if git dir is clean.
401    """
402    if HasGitStagedChanges(git_dir):
403        logger.GetLogger().LogWarning("Git tree has staged changes.")
404        return False
405    if HasGitUnstagedChanges(git_dir):
406        logger.GetLogger().LogWarning("Git tree has unstaged changes.")
407        return False
408    if HasGitUntrackedChanges(git_dir):
409        logger.GetLogger().LogWarning("Git tree has un-tracked changes.")
410        return False
411    return True
412
413
414def GetGitChangesAsList(git_dir, path=None, staged=False):
415    """Get changed files as a list.
416
417    Args:
418      git_dir: git tree directory.
419      path: a relative path that is part of the tree directory, could be null.
420      staged: whether to include staged files as well.
421
422    Returns:
423      A list containing all the changed files.
424    """
425    command = f"cd {git_dir} && git diff --name-only"
426    if staged:
427        command += " --cached"
428    if path:
429        command += " -- " + path
430    _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
431        command, print_to_console=False
432    )
433    rv = []
434    for line in out.splitlines():
435        rv.append(line)
436    return rv
437
438
439def IsChromeOsTree(chromeos_root):
440    return os.path.isdir(
441        os.path.join(chromeos_root, "src/third_party/chromiumos-overlay")
442    ) and os.path.isdir(os.path.join(chromeos_root, "manifest"))
443
444
445def DeleteChromeOsTree(chromeos_root, dry_run=False):
446    """Delete a ChromeOs tree *safely*.
447
448    Args:
449      chromeos_root: dir of the tree, could be a relative one (but be careful)
450      dry_run: only prints out the command if True
451
452    Returns:
453      True if everything is ok.
454    """
455    if not IsChromeOsTree(chromeos_root):
456        logger.GetLogger().LogWarning(
457            f'"{chromeos_root}" does not seem to be a'
458            " valid chromeos tree, do nothing."
459        )
460        return False
461    cmd0 = f"cd {chromeos_root} && cros_sdk --delete"
462    if dry_run:
463        print(cmd0)
464    else:
465        if (
466            command_executer.GetCommandExecuter().RunCommand(
467                cmd0, print_to_console=True
468            )
469            != 0
470        ):
471            return False
472
473    cmd1 = (
474        f'export CHROMEOSDIRNAME="$(dirname $(cd {chromeos_root} && pwd))" && '
475        f'export CHROMEOSBASENAME="$(basename $(cd {chromeos_root} && pwd))" && '
476        "cd $CHROMEOSDIRNAME && sudo rm -fr $CHROMEOSBASENAME"
477    )
478    if dry_run:
479        print(cmd1)
480        return True
481
482    return (
483        command_executer.GetCommandExecuter().RunCommand(
484            cmd1, print_to_console=True
485        )
486        == 0
487    )
488
489
490def BooleanPrompt(
491    prompt="Do you want to continue?",
492    default=True,
493    true_value="yes",
494    false_value="no",
495    prolog=None,
496):
497    """Helper function for processing boolean choice prompts.
498
499    Args:
500      prompt: The question to present to the user.
501      default: Boolean to return if the user just presses enter.
502      true_value: The text to display that represents a True returned.
503      false_value: The text to display that represents a False returned.
504      prolog: The text to display before prompt.
505
506    Returns:
507      True or False.
508    """
509    true_value, false_value = true_value.lower(), false_value.lower()
510    true_text, false_text = true_value, false_value
511    if true_value == false_value:
512        raise ValueError(
513            "true_value and false_value must differ: got %r" % true_value
514        )
515
516    if default:
517        true_text = true_text[0].upper() + true_text[1:]
518    else:
519        false_text = false_text[0].upper() + false_text[1:]
520
521    prompt = "\n%s (%s/%s)? " % (prompt, true_text, false_text)
522
523    if prolog:
524        prompt = "\n%s\n%s" % (prolog, prompt)
525
526    while True:
527        try:
528            # pylint: disable=input-builtin, bad-builtin
529            response = input(prompt).lower()
530        except EOFError:
531            # If the user hits CTRL+D, or stdin is disabled, use the default.
532            print()
533            response = None
534        except KeyboardInterrupt:
535            # If the user hits CTRL+C, just exit the process.
536            print()
537            print("CTRL+C detected; exiting")
538            sys.exit()
539
540        if not response:
541            return default
542        if true_value.startswith(response):
543            if not false_value.startswith(response):
544                return True
545            # common prefix between the two...
546        elif false_value.startswith(response):
547            return False
548
549
550# pylint: disable=unused-argument
551def rgb2short(r, g, b):
552    """Converts RGB values to xterm-256 color."""
553
554    redcolor = [255, 124, 160, 196, 9]
555    greencolor = [255, 118, 82, 46, 10]
556
557    if g == 0:
558        return redcolor[r // 52]
559    if r == 0:
560        return greencolor[g // 52]
561    return 4
562