1#!/usr/bin/env python3 2"""Run Trusty under QEMU in different configurations""" 3import enum 4import errno 5import fcntl 6import json 7import logging 8import os 9from textwrap import dedent 10import re 11import select 12import shlex 13import socket 14import subprocess 15import shutil 16import sys 17import tempfile 18import time 19import threading 20 21from typing import Optional, List, IO 22 23import alloc_adb_ports 24import qemu_options 25from qemu_error import AdbFailure, ConfigError, RunnerGenericError, Timeout 26 27logger = logging.getLogger(__name__) 28 29# ADB expects its first console on 5554, and control on 5555 30ADB_BASE_PORT = 5554 31 32_TRUSTY_PRODUCT_PATH = "target/product/trusty" 33_ANDROID_HOST_PATH = "host/linux-x86" 34 35def find_android_build_dir(android): 36 if os.path.exists(os.path.join(android, _TRUSTY_PRODUCT_PATH)): 37 return android 38 if os.path.exists(os.path.join(android, "out", _TRUSTY_PRODUCT_PATH)): 39 return os.path.join(android, "out") 40 41 print(f"{android} not an Android source or build directory") 42 sys.exit(1) 43 44 45def find_android_image_dir(android): 46 return os.path.join(find_android_build_dir(android), _TRUSTY_PRODUCT_PATH) 47 48def find_adb_path(android): 49 return os.path.join(find_android_build_dir(android), _ANDROID_HOST_PATH, "bin/adb") 50 51 52class Config(object): 53 """Stores a QEMU configuration for use with the runner 54 55 Attributes: 56 boot_android: Boolean indicating to boot Android. Setting the 57 "android" config option to a path containing a 58 built Android tree or prebuilt will implicitly 59 set this attribute to true 60 android_image_dir: Path to directory containing Android images. 61 Can be set by providing a built Android tree 62 or prebuilt with the "android" config option. 63 Implies booting android. 64 linux: Path to a built Linux kernel tree or prebuilt 65 kernel image. 66 linux_arch: Architecture of Linux kernel. 67 initrd: Path to a built ramdisk.img or other initrd, 68 if different from the one in the Android images. 69 atf: Path to the ATF build to use. 70 qemu: Path to the emulator to use. 71 arch: Architecture definition. 72 rpmbd: Path to the rpmb daemon to use. 73 adb: Path to adb host tool. 74 extra_linux_args: Extra arguments to pass to Linux kernel. 75 extra_qemu_flags: Extra flags to pass to QEMU. 76 Setting android or linux to a false value will result in a QEMU which starts 77 without those components. Only one of android and android_image_dir may be provided 78 in the config. 79 """ 80 81 def __init__(self, config=None): 82 """Qemu Configuration 83 84 If config is passed in, it should be a file containing a json 85 specification fields described in the docs. 86 Unspecified fields will be defaulted. 87 88 If you do not pass in a config, you will almost always need to 89 override these values; the default is not especially useful. 90 """ 91 config_dict = {} 92 if config: 93 config_dict = json.load(config) 94 95 self.script_dir = os.path.dirname(os.path.realpath(__file__)) 96 97 def abspath(config_key, default_value=None): 98 if config_value := config_dict.get(config_key, default_value): 99 return os.path.join(self.script_dir, config_value) 100 return None 101 102 if config_dict.get("android") and config_dict.get("android_image_dir"): 103 raise ConfigError("Config may only have one of android and android_image_dir") 104 105 self.adb = abspath("adb") 106 if android_path := abspath("android"): 107 logger.error("`android` config setting is deprecated. Please replace with " 108 "`android_image_dir` and `adb` config entries.") 109 android_out = find_android_build_dir(android_path) 110 self.android_image_dir = os.path.join(android_out, _TRUSTY_PRODUCT_PATH) 111 if self.adb is None: 112 self.adb = os.path.join(android_out, _ANDROID_HOST_PATH, "bin/adb") 113 else: 114 self.android_image_dir = abspath("android_image_dir") 115 self.boot_android = self.android_image_dir is not None 116 self.linux = abspath("linux") 117 self.linux_arch = config_dict.get("linux_arch") 118 self.initrd = abspath("initrd") 119 self.atf = abspath("atf") 120 self.qemu = abspath("qemu", "qemu-system-aarch64") 121 self.rpmbd = abspath("rpmbd") 122 self.arch = config_dict.get("arch") 123 self.extra_linux_args = config_dict.get("extra_linux_args", []) 124 self.extra_qemu_flags = config_dict.get("extra_qemu_flags", []) 125 126 def check_config(self, interactive: bool, boot_tests=(), 127 android_tests=()): 128 """Checks the runner/qemu config to make sure they are compatible""" 129 # If we have any android tests, we need a linux dir and android dir 130 if android_tests: 131 if not self.linux: 132 raise ConfigError("Need Linux to run android tests") 133 if not self.boot_android: 134 raise ConfigError("Need Android to run android tests") 135 136 # For now, we can't run boot tests and android tests at the same time, 137 # because test-runner reports its exit code by terminating the 138 # emulator. 139 if android_tests: 140 if boot_tests: 141 raise ConfigError("Cannot run Android tests and boot" 142 " tests from same runner") 143 144 # Since boot test utilizes virtio serial console port for communication 145 # between QEMU guest and current process, it is not compatible with 146 # interactive mode. 147 if boot_tests: 148 if interactive: 149 raise ConfigError("Cannot run boot tests interactively") 150 151 if self.boot_android: 152 if not self.linux: 153 raise ConfigError("Cannot run Android without Linux") 154 155 if not self.android_image_dir: 156 raise ConfigError("Missing android_image_dir for Android") 157 158 if not self.adb: 159 raise ConfigError("Missing adb tool for Android") 160 161 162def forward_ports(ports): 163 """Generates arguments to forward ports in QEMU on a virtio network""" 164 forwards = "" 165 remap_port = ADB_BASE_PORT 166 for port in ports: 167 forwards += f",hostfwd=tcp::{port}-:{remap_port}" 168 remap_port = remap_port + 1 169 return [ 170 "-device", "virtio-net,netdev=adbnet0", "-netdev", 171 "user,id=adbnet0" + forwards 172 ] 173 174 175class QEMUCommandPipe(object): 176 """Communicate with QEMU.""" 177 178 def __init__(self): 179 """Produces pipes for talking to QEMU and args to enable them.""" 180 self.command_dir = tempfile.mkdtemp() 181 os.mkfifo(f"{self.command_dir}/com.in") 182 os.mkfifo(f"{self.command_dir}/com.out") 183 self.command_args = [ 184 "-chardev", 185 f"pipe,id=command0,path={self.command_dir}/com", "-mon", 186 "chardev=command0,mode=control" 187 ] 188 self.com_pipe_in: Optional[IO] = None 189 self.com_pipe_out: Optional[IO] = None 190 191 def open(self): 192 # pylint: disable=consider-using-with 193 self.com_pipe_in = open(f"{self.command_dir}/com.in", "w", 194 encoding="utf-8") 195 self.com_pipe_out = open(f"{self.command_dir}/com.out", "r", 196 encoding="utf-8") 197 self.qmp_command({"execute": "qmp_capabilities"}) 198 199 def close(self): 200 """Close and clean up command pipes.""" 201 202 def try_close(pipe): 203 try: 204 pipe.close() 205 except IOError as e: 206 print("close error ignored", e) 207 208 try_close(self.com_pipe_in) 209 try_close(self.com_pipe_out) 210 211 # Onerror callback function to handle errors when we try to remove 212 # command pipe directory, since we sleep one second if QEMU doesn't 213 # die immediately, command pipe directory might has been removed 214 # already during sleep period. 215 def cb_handle_error(func, path, exc_info): 216 if not os.access(path, os.F_OK): 217 # Command pipe directory already removed, this case is 218 # expected, pass this case. 219 pass 220 else: 221 raise RunnerGenericError("Failed to clean up command pipe.") 222 223 # Clean up our command pipe 224 shutil.rmtree(self.command_dir, onexc=cb_handle_error) 225 226 def qmp_command(self, qmp_command): 227 """Send a qmp command and return result.""" 228 assert self.com_pipe_in 229 assert self.com_pipe_out 230 try: 231 json.dump(qmp_command, self.com_pipe_in) 232 self.com_pipe_in.flush() 233 for line in iter(self.com_pipe_out.readline, ""): 234 res = json.loads(line) 235 236 if err := res.get("error"): 237 sys.stderr.write(f"Command {qmp_command} failed: {err}\n") 238 return res 239 240 if "return" in res: 241 return res 242 243 if "QMP" not in res and "event" not in res: 244 # Print unexpected extra lines 245 sys.stderr.write("ignored:" + line) 246 except IOError as e: 247 print("qmp_command error ignored", e) 248 249 return None 250 251 def qmp_execute(self, execute, arguments=None): 252 """Send a qmp execute command and return result.""" 253 cmp_command = {"execute": execute} 254 if arguments: 255 cmp_command["arguments"] = arguments 256 return self.qmp_command(cmp_command) 257 258 def monitor_command(self, monitor_command, log_return=True): 259 """Send a monitor command and write result to stderr.""" 260 261 res = self.qmp_execute("human-monitor-command", 262 {"command-line": monitor_command}) 263 if log_return and res and "return" in res: 264 sys.stderr.write(res["return"]) 265 266 267def qemu_handle_error(command_pipe, debug_on_error): 268 """Dump registers and/or wait for debugger.""" 269 270 sys.stdout.flush() 271 272 sys.stderr.write("QEMU register dump:\n") 273 command_pipe.monitor_command("info registers -a") 274 sys.stderr.write("\n") 275 276 if debug_on_error: 277 command_pipe.monitor_command("gdbserver") 278 print("Connect gdb, press enter when done ") 279 select.select([sys.stdin], [], []) 280 input("\n") # pylint: disable=bad-builtin 281 282 283def qemu_exit(command_pipe, qemu_proc, has_error, debug_on_error): 284 """Ensures QEMU is terminated. Tries to write to drive image files.""" 285 unclean_exit = False 286 287 if command_pipe: 288 # Ask QEMU to quit 289 if qemu_proc and (qemu_proc.poll() is None): 290 try: 291 if has_error: 292 qemu_handle_error(command_pipe=command_pipe, 293 debug_on_error=debug_on_error) 294 command_pipe.qmp_execute("quit") 295 except OSError: 296 pass 297 298 # If it doesn't die immediately, wait a second 299 if qemu_proc.poll() is None: 300 time.sleep(1) 301 # If it's still not dead, take it out 302 if qemu_proc.poll() is None: 303 qemu_proc.kill() 304 print("QEMU refused quit") 305 unclean_exit = True 306 qemu_proc.wait() 307 308 command_pipe.close() 309 310 elif qemu_proc and (qemu_proc.poll() is None): 311 # This was an interactive run or a boot test 312 # QEMU should not be running at this point 313 print("QEMU still running with no command channel") 314 qemu_proc.kill() 315 qemu_proc.wait() 316 unclean_exit = True 317 return unclean_exit 318 319class RunnerSession: 320 """Hold shared state between runner launch and shutdown.""" 321 322 def __init__(self): 323 self.has_error = False 324 self.command_pipe = None 325 self.qemu_proc = None 326 self.ports: Optional[List[int]] = None 327 # stores the arguments used to start qemu iff performing a boot test 328 self.args = [] 329 self.temp_files = [] 330 331 def get_qemu_arg_temp_file(self): 332 """Returns a temp file that will be deleted after qemu exits.""" 333 tmp = tempfile.NamedTemporaryFile(delete=False) # pylint: disable=consider-using-with 334 self.temp_files.append(tmp.name) 335 return tmp 336 337 338class RunnerState(enum.Enum): 339 OFF = 0 340 BOOTLOADER = 1 341 ANDROID = 2 342 ERROR = 3 343 344 345class Runner(object): 346 """Executes tests in QEMU""" 347 348 def __init__(self, 349 config, 350 instance_dir: os.PathLike, 351 interactive=False, 352 verbose=False, 353 rpmb=True, 354 debug=False, 355 debug_on_error=False): 356 """Initializes the runner with provided settings. 357 358 See .run() for the meanings of these. 359 """ 360 self.config = config 361 self.interactive = interactive 362 self.debug = debug 363 self.verbose = verbose 364 self.adb_transport: Optional[int] = None 365 self.use_rpmb = rpmb 366 self.rpmb_proc: Optional[subprocess.Popen[bytes]] = None 367 self.rpmb_sock_dir: Optional[str] = None 368 self.msg_sock: Optional[socket.socket] = None 369 self.msg_sock_conn: Optional[socket.socket] = None 370 self.msg_sock_dir: Optional[str] = None 371 self.debug_on_error = debug_on_error 372 self.dump_stdout_on_error = False 373 self.default_timeout = 60 * 10 # 10 Minutes 374 self.session: Optional[RunnerSession] = None 375 self.state = RunnerState.OFF 376 self.instance_dir = instance_dir 377 378 # If we're not verbose or interactive, squelch command output 379 if verbose or self.interactive: 380 self.stdout: Optional[IO] = None 381 self.stderr: Optional[int] = None 382 else: 383 self.stdout = tempfile.TemporaryFile() # pylint: disable=consider-using-with 384 self.stderr = subprocess.STDOUT 385 self.dump_stdout_on_error = True 386 387 # If we're interactive connect stdin to the user 388 if self.interactive: 389 self.stdin = None 390 else: 391 self.stdin = subprocess.DEVNULL 392 393 if self.config.arch in ("arm64", "arm"): 394 self.qemu_arch_options = qemu_options.QemuArm64Options( 395 self.config, self.instance_dir) 396 elif self.config.arch == "x86_64": 397 # pylint: disable=no-member 398 self.qemu_arch_options = qemu_options.QemuX86_64Options(self.config) # type: ignore[attr-defined] 399 else: 400 raise ConfigError("Architecture unspecified or unsupported!") 401 402 def error_dump_output(self): 403 if self.dump_stdout_on_error: 404 assert self.stdout 405 assert self.session 406 sys.stdout.flush() 407 sys.stderr.write("System log:\n") 408 self.stdout.seek(0) 409 sys.stderr.buffer.write(self.stdout.read()) 410 411 def get_qemu_arg_temp_file(self): 412 """Returns a temp file that will be deleted after qemu exits.""" 413 assert self.session 414 # pylint: disable=consider-using-with 415 tmp = tempfile.NamedTemporaryFile(delete=False) 416 self.session.temp_files.append(tmp.name) 417 return tmp 418 419 def rpmb_up(self): 420 """Brings up the rpmb daemon, returning QEMU args to connect""" 421 rpmb_data = self.qemu_arch_options.rpmb_data_path() 422 423 self.rpmb_sock_dir = tempfile.mkdtemp() 424 rpmb_sock = f"{self.rpmb_sock_dir}/rpmb" 425 # pylint: disable=consider-using-with 426 rpmb_proc = subprocess.Popen([self.config.rpmbd, 427 "-d", rpmb_data, 428 "--sock", rpmb_sock]) 429 self.rpmb_proc = rpmb_proc 430 431 # Wait for RPMB socket to appear to avoid a race with QEMU 432 test_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 433 tries = 0 434 max_tries = 10 435 while True: 436 tries += 1 437 try: 438 test_sock.connect(rpmb_sock) 439 break 440 except socket.error as exn: 441 if tries >= max_tries: 442 raise exn 443 time.sleep(1) 444 445 return self.qemu_arch_options.rpmb_options(rpmb_sock) 446 447 def rpmb_down(self): 448 """Kills the running rpmb daemon, cleaning up its socket directory""" 449 if self.rpmb_proc: 450 self.rpmb_proc.kill() 451 self.rpmb_proc = None 452 if self.rpmb_sock_dir: 453 shutil.rmtree(self.rpmb_sock_dir) 454 self.rpmb_sock_dir = None 455 456 def msg_channel_up(self): 457 """Create message channel between host and QEMU guest 458 459 Virtual serial console port 'testrunner0' is introduced as socket 460 communication channel for QEMU guest and current process. Testrunner 461 enumerates this port, reads test case which to be executed from 462 testrunner0 port, sends output log message and test result to 463 testrunner0 port. 464 """ 465 466 self.msg_sock_dir = tempfile.mkdtemp() 467 msg_sock_file = f"{self.msg_sock_dir}/msg" 468 self.msg_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 469 self.msg_sock.bind(msg_sock_file) 470 471 # Listen on message socket 472 self.msg_sock.listen(1) 473 474 return ["-device", 475 "virtserialport,chardev=testrunner0,name=testrunner0", 476 "-chardev", f"socket,id=testrunner0,path={msg_sock_file}"] 477 478 def msg_channel_down(self): 479 if self.msg_sock_conn: 480 self.msg_sock_conn.close() 481 self.msg_sock_conn = None 482 if self.msg_sock_dir: 483 shutil.rmtree(self.msg_sock_dir) 484 self.msg_sock_dir = None 485 486 def msg_channel_wait_for_connection(self): 487 """wait for testrunner to connect.""" 488 assert self.msg_sock 489 # Accept testrunner's connection request 490 self.msg_sock_conn, _ = self.msg_sock.accept() 491 492 def msg_channel_send_msg(self, msg): 493 """Send message to testrunner via testrunner0 port 494 495 Testrunner tries to connect port while message with following format 496 "boottest your.port.here". Currently, we utilize this format to execute 497 cases in boot test. 498 If message does not comply above format, testrunner starts to launch 499 secondary OS. 500 501 """ 502 if self.msg_sock_conn: 503 self.msg_sock_conn.send(msg.encode()) 504 else: 505 sys.stderr.write("Connection has not been established yet!") 506 507 def msg_channel_recv(self): 508 if self.msg_sock_conn: 509 return self.msg_sock_conn.recv(64) 510 511 # error cases: channel not yet initialized or channel torn down 512 return bytes() 513 514 def msg_channel_close(self): 515 if self.msg_sock_conn: 516 self.msg_sock_conn.close() 517 518 def boottest_run(self, boot_tests, timeout=60 * 2): 519 """Run boot test cases""" 520 assert self.session 521 args = self.session.args 522 has_error = False 523 result = 2 524 525 if self.debug: 526 warning = """\ 527 Warning: Test selection does not work when --debug is set. 528 To run a test in test runner, run in GDB: 529 530 target remote :1234 531 break host_get_cmdline 532 c 533 next 6 534 set cmdline="boottest your.port.here" 535 set cmdline_len=sizeof("boottest your.port.here")-1 536 c 537 """ 538 print(dedent(warning)) 539 540 if self.interactive: 541 args = ["-serial", "mon:stdio"] + args 542 else: 543 # This still leaves stdin connected, but doesn't connect a monitor 544 args = ["-serial", "stdio", "-monitor", "none"] + args 545 546 # Create command channel which used to quit QEMU after case execution 547 command_pipe = QEMUCommandPipe() 548 args += command_pipe.command_args 549 cmd = [self.config.qemu] + args 550 551 # pylint: disable=consider-using-with 552 qemu_proc = subprocess.Popen(cmd, cwd=self.config.atf, 553 stdin=self.stdin, 554 stdout=self.stdout, 555 stderr=self.stderr) 556 557 command_pipe.open() 558 self.msg_channel_wait_for_connection() 559 560 def kill_testrunner(): 561 self.msg_channel_down() 562 qemu_exit(command_pipe, qemu_proc, has_error=True, 563 debug_on_error=self.debug_on_error) 564 raise Timeout("Wait for boottest to complete", timeout) 565 566 kill_timer = threading.Timer(timeout, kill_testrunner) 567 if not self.debug: 568 kill_timer.start() 569 570 testcase = "boottest " + "".join(boot_tests) 571 try: 572 self.msg_channel_send_msg(testcase) 573 574 while True: 575 ret = self.msg_channel_recv() 576 577 # If connection is disconnected accidently by peer, for 578 # instance child QEMU process crashed, a message with length 579 # 0 would be received. We should drop this message, and 580 # indicate test framework that something abnormal happened. 581 if len(ret) == 0: 582 has_error = True 583 break 584 585 # Print message to STDOUT. Since we might meet EAGAIN IOError 586 # when writting to STDOUT, use try except loop to catch EAGAIN 587 # and waiting STDOUT to be available, then try to write again. 588 def print_msg(msg): 589 while True: 590 try: 591 sys.stdout.write(msg) 592 break 593 except IOError as e: 594 if e.errno != errno.EAGAIN: 595 raise RunnerGenericError( 596 "Failed to print message") from e 597 select.select([], [sys.stdout], []) 598 599 # Please align message structure definition in testrunner. 600 if ret[0] == 0: 601 msg_len = ret[1] 602 msg = ret[2 : 2 + msg_len].decode() 603 print_msg(msg) 604 elif ret[0] == 1: 605 result = ret[1] 606 break 607 else: 608 # Unexpected type, return test result:TEST_FAILED 609 has_error = True 610 result = 1 611 break 612 finally: 613 kill_timer.cancel() 614 self.msg_channel_down() 615 self.session.has_error = has_error or result != 0 616 unclean_exit = qemu_exit(command_pipe, qemu_proc, 617 has_error=has_error, 618 debug_on_error=self.debug_on_error) 619 620 if unclean_exit: 621 raise RunnerGenericError("QEMU did not exit cleanly") 622 623 return result 624 625 def androidtest_run(self, cmd, test_timeout=None): 626 """Run android test cases""" 627 assert self.session, "No session; call launch before running any tests." 628 session: RunnerSession = self.session 629 630 try: 631 if not test_timeout: 632 test_timeout = self.default_timeout 633 634 def on_adb_timeout(): 635 print(f"adb Timed out ({test_timeout} s)") 636 qemu_handle_error(command_pipe=session.command_pipe, 637 debug_on_error=self.debug_on_error) 638 639 test_result = self.adb(["shell"] + cmd, timeout=test_timeout, 640 on_timeout=on_adb_timeout, force_output=True) 641 if test_result: 642 session.has_error = True 643 644 # If android reboots, adb starts failing before QEMU exits. 645 if self.adb(["get-state"]): 646 # Set self.state to RunnerState.ERROR if adb is no longer 647 # functional so the caller can reboot (if it needs to get back 648 # to a functional RunnerState.ANDROID state). 649 print("adb get-state failed after androidtest") 650 self.state = RunnerState.ERROR 651 652 # Give QEMU some time to exit. 653 self.session.qemu_proc.wait(10) 654 655 if self.session.qemu_proc.poll() is not None: 656 # Set self.state to RunnerState.ERROR if QEMU is no longer 657 # running so the caller can reboot (if it needs to get back to 658 # a functional RunnerState.ANDROID state). 659 print("QEMU is no longer running after androidtest") 660 self.state = RunnerState.ERROR 661 662 return test_result 663 except: 664 session.has_error = True 665 raise 666 667 def hostcommandtest_run(self, hostcmd: str, test_timeout=None): 668 """ 669 Run host command which interacts with a running Android device over adb. 670 671 This differs from android tests which run entirely on-device and boot tests 672 which runs without a full Android environment. 673 """ 674 assert self.session, "No session; call launch before running any tests." 675 assert self.session.ports, "No ports; did the QEMU session launch cleanly?" 676 677 env = os.environ.copy() 678 env["ADB_PORT"] = str(self.session.ports[1]) 679 env["PROJECT_ROOT"] = self.config.script_dir 680 cmd_status = 1 681 match shlex.split(hostcmd): 682 case []: # parse error 683 raise RunnerGenericError(f"Invalid host cmd: {hostcmd}") 684 case [exe, *args]: 685 exe_path = [f"{self.config.script_dir}/host_commands/{exe}"] 686 try: 687 subproc = subprocess.run( 688 exe_path + args,timeout=test_timeout, env=env 689 ) 690 cmd_status = subproc.returncode 691 except subprocess.TimeoutExpired: 692 print( 693 f"Host command test timed out ({test_timeout} s): {hostcmd}" 694 ) 695 qemu_handle_error( 696 command_pipe=self.session.command_pipe, 697 debug_on_error=self.debug_on_error 698 ) 699 case _: # this shouldn't happen 700 raise RunnerGenericError(f"Unknown error") 701 702 return cmd_status 703 704 def adb_bin(self): 705 """Returns location of adb""" 706 return self.config.adb 707 708 def adb(self, 709 args, 710 timeout=60, 711 on_timeout=lambda timeout: print(f"Timed out ({timeout} s)"), 712 need_qemu_running=False, 713 force_output=False): 714 """Runs an adb command 715 716 If self.adb_transport is set, specializes the command to that 717 transport to allow for multiple simultaneous tests. 718 719 Timeout specifies a timeout for the command in seconds. 720 721 If need_qemu_running is set to True the command will not be sent if 722 QEMU is not running. Set this to True when using "wait-for-device" 723 since "wait-for-device" can't tell the difference between adbd had not 724 started yet and adbd will never start since the emulator is no longer 725 running. Also, since this option currently only checks if QEMU is 726 running before sending the command, use this with short timeout values. 727 728 If force_output is set true, will send results to stdout and 729 stderr regardless of the runner's preferences. 730 """ 731 732 if need_qemu_running and self.session.qemu_proc.poll() is not None: 733 # Check if qemu is still running to avoid wasting time waiting for 734 # a command that can never succeed. Qemu should not normally exit 735 # before or while we are waiting for an adb command to complete. 736 # If it does the emulated system has crashed and adb 737 # wait-for-device will never succeed. Normal adb shell commands 738 # return 255 if QEMU exits while the command is running, so return 739 # the same status value here. 740 return 255 741 742 if self.adb_transport: 743 args = ["-t", str(self.adb_transport)] + args 744 745 if force_output: 746 stdout = None 747 stderr = None 748 else: 749 stdout = self.stdout 750 stderr = self.stderr 751 752 adb_proc = subprocess.Popen( # pylint: disable=consider-using-with 753 [self.adb_bin()] + args, stdin=self.stdin, stdout=stdout, 754 stderr=stderr) 755 756 status = 1 757 try: 758 status = adb_proc.wait(timeout) 759 except subprocess.TimeoutExpired: 760 if on_timeout: 761 on_timeout() 762 finally: 763 # Make sure we don't return with adp_proc still running. 764 # Since adb_proc.kill() is a NOP if adb_proc.wait() completed we 765 # can call this unconditionally. 766 try: 767 adb_proc.kill() 768 except OSError: 769 pass 770 771 return status 772 773 774 def check_adb(self, args, **kwargs): 775 """As .adb(), but throws an exception if the command fails""" 776 code = self.adb(args, **kwargs) 777 if code != 0: 778 raise AdbFailure(args, code) 779 780 def adb_root(self): 781 """Restarts adbd with root permissions and waits until it's back up""" 782 max_tries = 10 783 num_tries = 0 784 785 # Ensure device is up else adb root can fail 786 self.adb(["wait-for-device"], need_qemu_running=True) 787 self.check_adb(["root"]) 788 789 while True: 790 # adbd might not be down by this point yet 791 self.adb(["wait-for-device"], need_qemu_running=True) 792 793 # Check that adbd is up and running with root permissions 794 code = self.adb(["shell", 795 "if [[ $(id -u) -ne 0 ]] ; then exit 1; fi"]) 796 if code == 0: 797 return 798 799 num_tries += 1 800 if num_tries >= max_tries: 801 raise AdbFailure(["root"], code) 802 time.sleep(1) 803 804 def scan_transport(self, port, expect_none=False): 805 """Given a port and `adb devices -l`, find the transport id""" 806 output = subprocess.check_output([self.adb_bin(), "devices", "-l"], 807 universal_newlines=True) 808 match = re.search(fr"localhost:{port}.*transport_id:(\d+)", output) 809 if not match: 810 if expect_none: 811 self.adb_transport = None 812 return 813 raise RunnerGenericError( 814 f"Failed to find transport for port {port} in \n{output}") 815 self.adb_transport = int(match.group(1)) 816 817 def adb_up(self, port): 818 """Ensures adb is connected to adbd on the selected port""" 819 # Wait until we can connect to the target port 820 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 821 connect_max_tries = 15 822 connect_tries = 0 823 while True: 824 try: 825 sock.connect(("localhost", port)) 826 break 827 except IOError as ioe: 828 connect_tries += 1 829 if connect_tries >= connect_max_tries: 830 raise Timeout("Wait for adbd socket", 831 connect_max_tries) from ioe 832 time.sleep(1) 833 sock.close() 834 self.check_adb(["connect", f"localhost:{port}"]) 835 self.scan_transport(port) 836 837 # Sometimes adb can get stuck and will never connect. Using multiple 838 # shorter timeouts works better than one longer timeout in such cases. 839 adb_exception: Optional[AdbFailure] = None 840 for _ in range(10): 841 try: 842 self.check_adb(["wait-for-device"], timeout=30, 843 on_timeout=None, need_qemu_running=True) 844 break 845 except AdbFailure as e: 846 adb_exception = e 847 continue 848 else: 849 print("'adb wait-for-device' Timed out") 850 assert adb_exception 851 raise adb_exception 852 853 self.adb_root() 854 855 # Files put onto the data partition in the Android build will not 856 # actually be populated into userdata.img when make dist is used. 857 # To work around this, we manually update /data once the device is 858 # booted by pushing it the files that would have been there. 859 userdata = self.qemu_arch_options.android_trusty_user_data() 860 self.check_adb(["push", userdata, "/"]) 861 862 def adb_down(self, port): 863 """Cleans up after adb connection to adbd on selected port""" 864 self.check_adb(["disconnect", f"localhost:{port}"]) 865 866 # Wait until QEMU's forward has expired 867 connect_max_tries = 300 868 connect_tries = 0 869 while True: 870 try: 871 self.scan_transport(port, expect_none=True) 872 if not self.adb_transport: 873 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 874 sock.connect(("localhost", port)) 875 sock.close() 876 connect_tries += 1 877 if connect_tries >= connect_max_tries: 878 raise Timeout("Wait for port forward to go away", 879 connect_max_tries) 880 time.sleep(1) 881 except IOError: 882 break 883 884 def universal_args(self): 885 """Generates arguments used in all qemu invocations""" 886 args = self.qemu_arch_options.basic_options() 887 args += self.qemu_arch_options.bios_options() 888 889 if self.config.linux: 890 args += self.qemu_arch_options.linux_options() 891 if len(self.config.extra_linux_args) > 0: 892 args[-1] += " " + " ".join(self.config.extra_linux_args) 893 894 if self.config.boot_android: 895 args += self.qemu_arch_options.android_drives_args() 896 897 # Append configured extra flags 898 args += self.config.extra_qemu_flags 899 900 return args 901 902 def launch(self, target_state): 903 """Launches the QEMU execution. 904 905 If interactive is specified, it will leave the user connected 906 to the serial console/monitor, and they are responsible for 907 terminating execution. 908 909 If debug is on, the main QEMU instance will be launched with -S and 910 -s, which pause the CPU rather than booting, and starts a gdb server 911 on port 1234 respectively. 912 913 It is the responsibility of callers to ensure that shutdown gets called 914 after launch - regardless of whether the launch succeeded or not. 915 916 Limitations: 917 If the adb port range is already in use, port forwarding may fail. 918 919 TODO: For boot tests, the emulator isn't actually launched here but in 920 boottest_run. Eventually, we want to unify the way boot tests and 921 android tests are launched. Specifically, we might stop execution 922 in the bootloader and have it wait for a command to boot android. 923 """ 924 assert self.state == RunnerState.OFF 925 assert target_state in [RunnerState.BOOTLOADER, 926 RunnerState.ANDROID], target_state 927 928 # Set self.state to RunnerState.ERROR in case we get an exception 929 # before self.state is set to target_state. This tells shutdown that 930 # cleanup might be needed. 931 self.state = RunnerState.ERROR 932 933 self.session = RunnerSession() 934 args = self.universal_args() 935 936 try: 937 if self.use_rpmb: 938 self.qemu_arch_options.create_rpmb_data() 939 args += self.rpmb_up() 940 941 if self.config.boot_android: 942 self.qemu_arch_options.create_drives_data() 943 944 if self.config.linux: 945 args += self.qemu_arch_options.gen_dtb( 946 args, 947 self.session.get_qemu_arg_temp_file()) 948 949 # Prepend the machine since we don't need to edit it as in gen_dtb 950 args = self.qemu_arch_options.machine_options() + args 951 952 if self.debug: 953 args += ["-s", "-S"] 954 955 # Create socket for communication channel 956 args += self.msg_channel_up() 957 958 if target_state == RunnerState.BOOTLOADER: 959 self.session.args = args 960 self.state = target_state 961 return 962 963 # Logging and terminal monitor 964 # Prepend so that it is the *first* serial port and avoid 965 # conflicting with rpmb0. 966 args = ["-serial", "mon:stdio"] + args 967 968 # If we're noninteractive (e.g. testing) we need a command channel 969 # to tell the guest to exit 970 if not self.interactive: 971 self.session.command_pipe = QEMUCommandPipe() 972 args += self.session.command_pipe.command_args 973 974 # Reserve ADB ports - adb uses ports in pairs 975 self.session.ports = alloc_adb_ports.alloc_adb_ports() 976 assert self.session.ports 977 978 # Write expected serial number (as given in adb) to stdout. 979 sys.stdout.write( 980 f"DEVICE_SERIAL: localhost:{self.session.ports[1]}\n") 981 sys.stdout.flush() 982 983 # Forward ADB ports in qemu 984 args += forward_ports(self.session.ports) 985 986 qemu_cmd = [self.config.qemu] + args 987 logger.info("qemu command: %s", qemu_cmd) 988 self.session.qemu_proc = subprocess.Popen( # pylint: disable=consider-using-with 989 qemu_cmd, 990 cwd=self.config.atf, 991 stdin=self.stdin, 992 stdout=self.stdout, 993 stderr=self.stderr) 994 995 if self.session.command_pipe: 996 self.session.command_pipe.open() 997 self.msg_channel_wait_for_connection() 998 999 if self.debug: 1000 script_dir = self.config.script_dir 1001 if script_dir.endswith("/build-qemu-generic-arm64-test-debug"): 1002 print(f"Debug with: lldb --source {script_dir}/lldbinit") 1003 else: 1004 print("Debug with: lldb --one-line 'gdb-remote 1234'") 1005 1006 # Send request to boot secondary OS 1007 self.msg_channel_send_msg("Boot Secondary OS") 1008 1009 # Bring ADB up talking to the command port 1010 self.adb_up(self.session.ports[1]) 1011 1012 self.state = target_state 1013 except: 1014 self.session.has_error = True 1015 raise 1016 1017 def shutdown(self, factory_reset: bool, full_wipe: bool): 1018 """Shut down emulator after test cases have run 1019 1020 The launch and shutdown methods store shared state in a session object. 1021 Calls to launch and shutdown must be correctly paired no matter whether 1022 the launch steps and calls to adb succeed or fail. 1023 """ 1024 if self.state == RunnerState.OFF: 1025 return 1026 1027 assert self.session 1028 1029 # Clean up generated device tree 1030 for temp_file in self.session.temp_files: 1031 try: 1032 os.remove(temp_file) 1033 except OSError: 1034 pass 1035 1036 if self.session.has_error: 1037 self.error_dump_output() 1038 1039 unclean_exit = qemu_exit(self.session.command_pipe, 1040 self.session.qemu_proc, 1041 has_error=self.session.has_error, 1042 debug_on_error=self.debug_on_error) 1043 1044 fcntl.fcntl(0, fcntl.F_SETFL, 1045 fcntl.fcntl(0, fcntl.F_GETFL) & ~os.O_NONBLOCK) 1046 1047 self.rpmb_down() 1048 1049 self.msg_channel_down() 1050 1051 if self.adb_transport: 1052 # Disconnect ADB and wait for our port to be released by qemu 1053 assert self.session.ports 1054 self.adb_down(self.session.ports[1]) 1055 1056 # Ideally, we'd clear on launch instead, but it doesn't know whether a 1057 # clear should happen. We can't change launch to take factory_reset and 1058 # full_wipe args because TrustyRebootCommand doesn't call launch, only 1059 # shutdown. (The next test that runs after a reboot will re-launch when 1060 # it notices the runner is down, but that test doesn't have the 1061 # RebootMode info from the reboot.) 1062 if factory_reset: 1063 self.qemu_arch_options.delete_drives_data() 1064 if full_wipe: 1065 assert factory_reset, ( 1066 "Cannot perform a full wipe without factory resetting.") 1067 self.qemu_arch_options.delete_rpmb_data() 1068 1069 self.session = None 1070 self.state = RunnerState.OFF 1071 1072 if unclean_exit: 1073 raise RunnerGenericError("QEMU did not exit cleanly") 1074 1075 def reboot(self, target_state, factory_reset: bool, full_wipe: bool): 1076 self.shutdown(factory_reset, full_wipe) 1077 1078 try: 1079 self.launch(target_state) 1080 except: 1081 self.shutdown(factory_reset, full_wipe) 1082 raise 1083 1084 def run(self, boot_tests: Optional[List] = None, 1085 android_tests: Optional[List] = None, 1086 hostcommand_tests: Optional[List] = None, 1087 timeout: Optional[int] = None) -> List[int]: 1088 """Run boot or android tests. 1089 1090 Runs boot_tests through test_runner, android_tests through ADB, 1091 hostcommand_tests on host after starting ADB, returning 1092 aggregated test return codes in a list. 1093 1094 Returns: 1095 A list of return codes for the provided tests. 1096 A negative return code indicates an internal tool failure. 1097 1098 Limitations: 1099 Until test_runner is updated, only one of android_tests, 1100 hostcommand_tests, or boot_tests may be provided. 1101 Similarly, while boot_tests is a list, test_runner only knows how to 1102 correctly run a single test at a time. 1103 Again due to test_runner's current state, if boot_tests are 1104 specified, interactive will be ignored since the machine will 1105 terminate itself. 1106 1107 If android_tests or hostcommand_tests is provided, a Linux and 1108 Android dir must be provided in the config. 1109 """ 1110 assert self.state == RunnerState.OFF 1111 self.config.check_config(self.interactive, boot_tests, android_tests) 1112 1113 if boot_tests and (android_tests or hostcommand_tests): 1114 raise RunnerGenericError( 1115 "Cannot run boot tests and android or hostcommand tests" 1116 " in the same QEMU instance") 1117 1118 if boot_tests and len(boot_tests) > 1: 1119 raise RunnerGenericError( 1120 "Can only run a single boot test at a time") 1121 1122 timeout = timeout if timeout else self.default_timeout 1123 try: 1124 self.launch(RunnerState.BOOTLOADER if boot_tests else 1125 RunnerState.ANDROID) 1126 test_results = [] 1127 1128 if boot_tests: 1129 test_results.append(self.boottest_run(boot_tests, timeout)) 1130 1131 if android_tests: 1132 for android_test in android_tests: 1133 test_result = self.androidtest_run([android_test], timeout) 1134 test_results.append(test_result) 1135 if test_result: 1136 break 1137 1138 if hostcommand_tests: 1139 for hostcommand_test in hostcommand_tests: 1140 test_result = self.hostcommandtest_run(hostcommand_test, timeout) 1141 test_results.append(test_result) 1142 if test_result: 1143 break 1144 1145 return test_results 1146 finally: 1147 # The wait on QEMU is done here to ensure that ADB failures do not 1148 # take away the user's serial console in interactive mode. 1149 if self.interactive and self.session: 1150 # The user is responsible for quitting QEMU 1151 assert self.session.qemu_proc 1152 self.session.qemu_proc.wait() 1153 self.shutdown(factory_reset=True, full_wipe=False) 1154