• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# Copyright 2021 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6import argparse
7import functools
8import json
9import os
10import random
11import subprocess
12import sys
13from multiprocessing import Pool
14from pathlib import Path
15from typing import Dict, Iterable, List, NamedTuple
16import typing
17
18import test_target
19from test_target import TestTarget
20import testvm
21from test_config import CRATE_OPTIONS, TestOption, BUILD_FEATURES
22from check_code_hygiene import (
23    has_platform_dependent_code,
24    has_crlf_line_endings,
25)
26
27USAGE = """\
28Runs tests for crosvm locally, in a vm or on a remote device.
29
30To build and run all tests locally:
31
32    $ ./tools/run_tests --target=host
33
34To cross-compile tests for aarch64 and run them on a built-in VM:
35
36    $ ./tools/run_tests --target=vm:aarch64
37
38The VM will be automatically set up and booted. It will remain running between
39test runs and can be managed with `./tools/aarch64vm`.
40
41Tests can also be run on a remote device via SSH. However it is your
42responsiblity that runtime dependencies of crosvm are provided.
43
44    $ ./tools/run_tests --target=ssh:hostname
45
46The default test target can be managed with `./tools/set_test_target`
47
48To see full build and test output, add the `-v` or `--verbose` flag.
49"""
50
51Arch = test_target.Arch
52
53# Print debug info. Overriden by -v
54VERBOSE = False
55
56# Timeouts for tests to prevent them from running too long.
57TEST_TIMEOUT_SECS = 60
58LARGE_TEST_TIMEOUT_SECS = 120
59
60# Double the timeout if the test is running in an emulation environment, which will be
61# significantly slower than native environments.
62EMULATION_TIMEOUT_MULTIPLIER = 2
63
64# Number of parallel processes for executing tests.
65PARALLELISM = 4
66
67CROSVM_ROOT = Path(__file__).parent.parent.parent.resolve()
68COMMON_ROOT = CROSVM_ROOT / "common"
69
70
71class ExecutableResults(object):
72    """Container for results of a test executable."""
73
74    def __init__(self, name: str, success: bool, test_log: str):
75        self.name = name
76        self.success = success
77        self.test_log = test_log
78
79
80class Executable(NamedTuple):
81    """Container for info about an executable generated by cargo build/test."""
82
83    binary_path: Path
84    crate_name: str
85    cargo_target: str
86    kind: str
87    is_test: bool
88    is_fresh: bool
89    arch: Arch
90
91    @property
92    def name(self):
93        return f"{self.crate_name}:{self.cargo_target}"
94
95
96class Crate(NamedTuple):
97    """Container for info about crate."""
98
99    name: str
100    path: Path
101
102
103def get_workspace_excludes(target_arch: Arch):
104    for crate, options in CRATE_OPTIONS.items():
105        if TestOption.DO_NOT_BUILD in options:
106            yield crate
107        elif TestOption.DO_NOT_BUILD_X86_64 in options and target_arch == "x86_64":
108            yield crate
109        elif TestOption.DO_NOT_BUILD_AARCH64 in options and target_arch == "aarch64":
110            yield crate
111        elif TestOption.DO_NOT_BUILD_ARMHF in options and target_arch == "armhf":
112            yield crate
113        elif TestOption.DO_NOT_BUILD_WIN64 in options and target_arch == "win64":
114            yield crate
115
116
117def should_run_executable(executable: Executable, target_arch: Arch):
118    options = CRATE_OPTIONS.get(executable.crate_name, [])
119    if TestOption.DO_NOT_RUN in options:
120        return False
121    if TestOption.DO_NOT_RUN_X86_64 in options and target_arch == "x86_64":
122        return False
123    if TestOption.DO_NOT_RUN_AARCH64 in options and target_arch == "aarch64":
124        return False
125    if TestOption.DO_NOT_RUN_ARMHF in options and target_arch == "armhf":
126        return False
127    if TestOption.DO_NOT_RUN_ON_FOREIGN_KERNEL in options and target_arch != executable.arch:
128        return False
129    return True
130
131
132def list_common_crates(target_arch: Arch):
133    excluded_crates = list(get_workspace_excludes(target_arch))
134    for path in COMMON_ROOT.glob("**/Cargo.toml"):
135        if not path.parent.name in excluded_crates:
136            yield Crate(name=path.parent.name, path=path.parent)
137
138
139def exclude_crosvm(target_arch: Arch):
140    return "crosvm" in get_workspace_excludes(target_arch)
141
142
143def cargo(
144    cargo_command: str, cwd: Path, flags: list[str], env: dict[str, str], build_arch: Arch
145) -> Iterable[Executable]:
146    """
147    Executes a cargo command and returns the list of test binaries generated.
148
149    The build log will be hidden by default and only printed if the build
150    fails. In VERBOSE mode the output will be streamed directly.
151
152    Note: Exits the program if the build fails.
153    """
154    cmd = [
155        "cargo",
156        cargo_command,
157        "--message-format=json-diagnostic-rendered-ansi",
158        *flags,
159    ]
160    if VERBOSE:
161        print("$", " ".join(cmd))
162    process = subprocess.Popen(
163        cmd,
164        cwd=cwd,
165        stdout=subprocess.PIPE,
166        stderr=subprocess.STDOUT,
167        text=True,
168        env=env,
169    )
170
171    messages: List[str] = []
172
173    # Read messages as cargo is running.
174    assert process.stdout
175    for line in iter(process.stdout.readline, ""):
176        # any non-json line is a message to print
177        if not line.startswith("{"):
178            if VERBOSE:
179                print(line.rstrip())
180            messages.append(line.rstrip())
181            continue
182        json_line = json.loads(line)
183
184        # 'message' type lines will be printed
185        if json_line.get("message"):
186            message = json_line.get("message").get("rendered")
187            if VERBOSE:
188                print(message)
189            messages.append(message)
190
191        # Collect info about test executables produced
192        elif json_line.get("executable"):
193            yield Executable(
194                Path(json_line.get("executable")),
195                crate_name=json_line.get("package_id", "").split(" ")[0],
196                cargo_target=json_line.get("target").get("name"),
197                kind=json_line.get("target").get("kind")[0],
198                is_test=json_line.get("profile", {}).get("test", False),
199                is_fresh=json_line.get("fresh", False),
200                arch=build_arch,
201            )
202
203    if process.wait() != 0:
204        if not VERBOSE:
205            for message in messages:
206                print(message)
207        sys.exit(-1)
208
209
210def cargo_build_executables(
211    flags: list[str],
212    build_arch: Arch,
213    cwd: Path = Path("."),
214    env: Dict[str, str] = {},
215) -> Iterable[Executable]:
216    """Build all test binaries for the given list of crates."""
217    # Run build first, to make sure compiler errors of building non-test
218    # binaries are caught.
219    yield from cargo("build", cwd, flags, env, build_arch)
220
221    # Build all tests and return the collected executables
222    yield from cargo("test", cwd, ["--no-run", *flags], env, build_arch)
223
224
225def build_common_crate(build_env: dict[str, str], build_arch: Arch, crate: Crate):
226    print(f"Building tests for: common/{crate.name}")
227    return list(cargo_build_executables([], build_arch, env=build_env, cwd=crate.path))
228
229
230def build_all_binaries(target: TestTarget, build_arch: Arch):
231    """Discover all crates and build them."""
232    build_env = os.environ.copy()
233    build_env.update(test_target.get_cargo_env(target, build_arch))
234
235    print("Building crosvm workspace")
236    yield from cargo_build_executables(
237        [
238            "--features=" + BUILD_FEATURES[build_arch],
239            "--verbose",
240            "--workspace",
241            *[f"--exclude={crate}" for crate in get_workspace_excludes(build_arch)],
242        ],
243        build_arch,
244        cwd=CROSVM_ROOT,
245        env=build_env,
246    )
247
248    with Pool(PARALLELISM) as pool:
249        for executables in pool.imap(
250            functools.partial(build_common_crate, build_env, build_arch),
251            list_common_crates(build_arch),
252        ):
253            yield from executables
254
255
256def is_emulated(target: TestTarget, executable: Executable) -> bool:
257    if target.is_host:
258        # User-space emulation can run foreing-arch executables on the host.
259        return executable.arch != target.arch
260    elif target.vm:
261        return target.vm == "aarch64"
262    return False
263
264
265def get_test_timeout(target: TestTarget, executable: Executable):
266    large = TestOption.LARGE in CRATE_OPTIONS.get(executable.crate_name, [])
267    timeout = LARGE_TEST_TIMEOUT_SECS if large else TEST_TIMEOUT_SECS
268    if is_emulated(target, executable):
269        return timeout * EMULATION_TIMEOUT_MULTIPLIER
270    else:
271        return timeout
272
273
274def execute_test(target: TestTarget, executable: Executable):
275    """
276    Executes a single test on the given test targed
277
278    Note: This function is run in a multiprocessing.Pool.
279
280    Test output is hidden unless the test fails or VERBOSE mode is enabled.
281    """
282    options = CRATE_OPTIONS.get(executable.crate_name, [])
283    args: list[str] = []
284    if TestOption.SINGLE_THREADED in options:
285        args += ["--test-threads=1"]
286
287    binary_path = executable.binary_path
288
289    if executable.arch == "win64" and executable.kind != "proc-macro" and os.name != "nt":
290        args.insert(0, binary_path)
291        binary_path = "wine64"
292
293
294    # proc-macros and their tests are executed on the host.
295    if executable.kind == "proc-macro":
296        target = TestTarget("host")
297
298    if VERBOSE:
299        print(f"Running test {executable.name} on {target}...")
300    try:
301        # Pipe stdout/err to be printed in the main process if needed.
302        test_process = test_target.exec_file_on_target(
303            target,
304            binary_path,
305            args=args,
306            timeout=get_test_timeout(target, executable),
307            stdout=subprocess.PIPE,
308            stderr=subprocess.STDOUT,
309        )
310        return ExecutableResults(
311            executable.name,
312            test_process.returncode == 0,
313            test_process.stdout,
314        )
315    except subprocess.TimeoutExpired as e:
316        # Append a note about the timeout to the stdout of the process.
317        msg = f"\n\nProcess timed out after {e.timeout}s\n"
318        return ExecutableResults(
319            executable.name,
320            False,
321            e.stdout.decode("utf-8") + msg,
322        )
323
324
325def execute_all(
326    executables: list[Executable],
327    target: test_target.TestTarget,
328    repeat: int,
329):
330    """Executes all tests in the `executables` list in parallel."""
331    executables = [e for e in executables if should_run_executable(e, target.arch)]
332    if repeat > 1:
333        executables = executables * repeat
334        random.shuffle(executables)
335
336    sys.stdout.write(f"Running {len(executables)} test binaries on {target}")
337    sys.stdout.flush()
338    with Pool(PARALLELISM) as pool:
339        for result in pool.imap(functools.partial(execute_test, target), executables):
340            if not result.success or VERBOSE:
341                msg = "passed" if result.success else "failed"
342                print()
343                print("--------------------------------")
344                print("-", result.name, msg)
345                print("--------------------------------")
346                print(result.test_log)
347            else:
348                sys.stdout.write(".")
349                sys.stdout.flush()
350            yield result
351    print()
352
353
354def find_crosvm_binary(executables: list[Executable]):
355    for executable in executables:
356        if not executable.is_test and executable.cargo_target == "crosvm":
357            return executable
358    raise Exception("Cannot find crosvm executable")
359
360
361def main():
362    parser = argparse.ArgumentParser(usage=USAGE)
363    parser.add_argument(
364        "--verbose",
365        "-v",
366        action="store_true",
367        default=False,
368        help="Print all test output.",
369    )
370    parser.add_argument(
371        "--target",
372        help="Execute tests on the selected target. See ./tools/set_test_target",
373    )
374    parser.add_argument(
375        "--arch",
376        choices=typing.get_args(Arch),
377        help="Target architecture to build for.",
378    )
379    parser.add_argument(
380        "--build-only",
381        action="store_true",
382    )
383    parser.add_argument(
384        "--repeat",
385        type=int,
386        default=1,
387        help="Repeat each test N times to check for flakes.",
388    )
389    args = parser.parse_args()
390
391    global VERBOSE
392    VERBOSE = args.verbose  # type: ignore
393    os.environ["RUST_BACKTRACE"] = "1"
394
395    target = (
396        test_target.TestTarget(args.target) if args.target else test_target.TestTarget.default()
397    )
398    print("Test target:", target)
399
400    build_arch = args.arch or target.arch
401    print("Building for architecture:", build_arch)
402
403    # Start booting VM while we build
404    if target.vm:
405        testvm.build_if_needed(target.vm)
406        testvm.up(target.vm)
407
408    hygiene, error = has_platform_dependent_code(Path("common/sys_util_core"))
409    if not hygiene:
410        print("Error: Platform dependent code not allowed in sys_util_core crate.")
411        print("Offending line: " + error)
412        sys.exit(-1)
413
414    crlf_endings = has_crlf_line_endings()
415    if crlf_endings:
416        print("Error: Following files have crlf(dos) line encodings")
417        print(*crlf_endings)
418        sys.exit(-1)
419
420    executables = list(build_all_binaries(target, build_arch))
421
422    if args.build_only:
423        print("Not running tests as requested.")
424        sys.exit(0)
425
426    # Upload dependencies plus the main crosvm binary for integration tests if the
427    # crosvm binary is not excluded from testing.
428    extra_files = (
429        [find_crosvm_binary(executables).binary_path] if not exclude_crosvm(build_arch) else []
430    )
431
432    test_target.prepare_target(target, extra_files=extra_files)
433
434    # Execute all test binaries
435    test_executables = [e for e in executables if e.is_test]
436    all_results = list(execute_all(test_executables, target, repeat=args.repeat))
437
438    failed = [r for r in all_results if not r.success]
439    if len(failed) == 0:
440        print("All tests passed.")
441        sys.exit(0)
442    else:
443        print(f"{len(failed)} of {len(all_results)} tests failed:")
444        for result in failed:
445            print(f"  {result.name}")
446        sys.exit(-1)
447
448
449if __name__ == "__main__":
450    try:
451        main()
452    except subprocess.CalledProcessError as e:
453        print("Command failed:", e.cmd)
454        print(e.stdout)
455        print(e.stderr)
456        sys.exit(-1)
457