1#!/bin/sh 2# 3# Copyright (C) 2018 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17# 18# This test script to be used by the build server. 19# It is supposed to be executed from trusty root directory 20# and expects the following environment variables: 21# 22""":" # Shell script (in docstring to appease pylint) 23 24# Find and invoke hermetic python3 interpreter 25. "`dirname $0`/envsetup.sh"; exec "$PY3" "$0" "$@" 26# Shell script end 27Run tests for a project. 28""" 29 30import argparse 31from enum import Enum 32import importlib 33import os 34import re 35import subprocess 36import sys 37import time 38from typing import Optional 39 40from trusty_build_config import PortType, TrustyCompositeTest, TrustyTest 41from trusty_build_config import TrustyAndroidTest, TrustyBuildConfig 42from trusty_build_config import TrustyHostTest, TrustyRebootCommand 43from trusty_build_config import TrustyPrintCommand 44from trusty_build_config import TrustyHostcommandTest 45 46 47TEST_STATUS = Enum("TEST_STATUS", ["PASSED", "FAILED", "SKIPPED"]) 48 49class TestResult: 50 """Stores results for a single test. 51 52 Attributes: 53 test: Name of the test. 54 status: Test's integer return code, or None if this test was skipped. 55 retried: True if this test was retried. 56 """ 57 test: str 58 status: Optional[int] 59 retried: bool 60 61 def __init__(self, test: str, status: Optional[int], retried: bool): 62 self.test = test 63 self.status = status 64 self.retried = retried 65 66 def test_status(self) -> TEST_STATUS: 67 if self.status is None: 68 return TEST_STATUS.SKIPPED 69 return TEST_STATUS.PASSED if self.status == 0 else TEST_STATUS.FAILED 70 71 def failed(self) -> bool: 72 return self.test_status() == TEST_STATUS.FAILED 73 74 def __format__(self, _format_spec: str) -> str: 75 return f"{self.test:s} returned {self.status:d}" 76 77 78class TestResults(object): 79 """Stores test results. 80 81 Attributes: 82 project: Name of project that tests were run on. 83 passed: True if all tests passed, False if one or more tests failed. 84 passed_count: Number of tests passed. 85 failed_count: Number of tests failed. 86 flaked_count: Number of tests that failed then passed on second try. 87 retried_count: Number of tests that were given a second try. 88 test_results: List of tuples storing test name an status. 89 """ 90 91 def __init__(self, project): 92 """Inits TestResults with project name and empty test results.""" 93 self.project = project 94 self.passed = True 95 self.passed_count = 0 96 self.failed_count = 0 97 self.skipped_count = 0 98 self.flaked_count = 0 99 self.retried_count = 0 100 self.test_results = [] 101 102 def add_result(self, result: TestResult): 103 """Add a test result.""" 104 self.test_results.append(result) 105 if result.test_status() == TEST_STATUS.PASSED: 106 self.passed_count += 1 107 if result.retried: 108 self.flaked_count += 1 109 elif result.test_status() == TEST_STATUS.FAILED: 110 self.failed_count += 1 111 self.passed = False 112 elif result.test_status() == TEST_STATUS.SKIPPED: 113 self.skipped_count += 1 114 115 if result.retried: 116 self.retried_count += 1 117 118 def print_results(self, print_failed_only=False): 119 """Print test results.""" 120 if print_failed_only: 121 if self.passed: 122 return 123 sys.stdout.flush() 124 out = sys.stderr 125 else: 126 out = sys.stdout 127 test_count = self.passed_count + self.failed_count + self.skipped_count 128 test_attempted = self.passed_count + self.failed_count 129 out.write( 130 "\n" 131 f"There were {test_count} defined for project {self.project}.\n" 132 f"{test_attempted} ran and {self.skipped_count} were skipped." 133 ) 134 if test_count: 135 for result in self.test_results: 136 match (result.test_status(), result.retried, print_failed_only): 137 case (TEST_STATUS.FAILED, _, _): 138 out.write(f"[ FAILED ] {result.test}\n") 139 case (TEST_STATUS.SKIPPED, _, False): 140 out.write(f"[ SKIPPED ] {result.test}\n") 141 case (TEST_STATUS.PASSED, retried, False): 142 out.write(f"[ OK ] {result.test}\n") 143 if retried: 144 out.write( 145 f"WARNING: {result.test} was re-run and " 146 "passed on second try; it may be flaky\n" 147 ) 148 149 out.write( 150 f"[==========] {test_count} tests ran for project " 151 f"{self.project}.\n" 152 ) 153 if self.passed_count and not print_failed_only: 154 out.write(f"[ PASSED ] {self.passed_count} tests.\n") 155 if self.failed_count: 156 out.write(f"[ FAILED ] {self.failed_count} tests.\n") 157 if self.skipped_count: 158 out.write(f"[ SKIPPED ] {self.skipped_count} tests.\n") 159 if self.flaked_count > 0: 160 out.write( 161 f"WARNING: {self.flaked_count} tests passed when " 162 "re-run which indicates that they may be flaky.\n" 163 ) 164 if self.retried_count == MAX_RETRIES: 165 out.write( 166 f"WARNING: hit MAX_RETRIES({MAX_RETRIES}) during " 167 "testing after which point, no tests were retried.\n" 168 ) 169 170 171class MultiProjectTestResults: 172 """Stores results from testing multiple projects. 173 174 Attributes: 175 test_results: List containing the results for each project. 176 failed_projects: List of projects with test failures. 177 tests_passed: Count of test passes across all projects. 178 tests_failed: Count of test failures across all projects. 179 had_passes: Count of all projects with any test passes. 180 had_failures: Count of all projects with any test failures. 181 """ 182 183 def __init__(self, test_results: list[TestResults]): 184 self.test_results = test_results 185 self.failed_projects = [] 186 self.tests_passed = 0 187 self.tests_failed = 0 188 self.tests_skipped = 0 189 self.had_passes = 0 190 self.had_failures = 0 191 self.had_skip = 0 192 193 for result in self.test_results: 194 if not result.passed: 195 self.failed_projects.append(result.project) 196 self.tests_passed += result.passed_count 197 self.tests_failed += result.failed_count 198 self.tests_skipped += result.skipped_count 199 if result.passed_count: 200 self.had_passes += 1 201 if result.failed_count: 202 self.had_failures += 1 203 if result.skipped_count: 204 self.had_skip += 1 205 206 def print_results(self): 207 """Prints the test results to stdout and stderr.""" 208 for test_result in self.test_results: 209 test_result.print_results() 210 211 sys.stdout.write("\n") 212 if self.had_passes: 213 sys.stdout.write( 214 f"[ PASSED ] {self.tests_passed} tests in " 215 f"{self.had_passes} projects.\n" 216 ) 217 if self.had_failures: 218 sys.stdout.write( 219 f"[ FAILED ] {self.tests_failed} tests in " 220 f"{self.had_failures} projects.\n" 221 ) 222 sys.stdout.flush() 223 if self.had_skip: 224 sys.stdout.write( 225 f"[ SKIPPED ] {self.tests_skipped} tests in " 226 f"{self.had_skip} projects.\n" 227 ) 228 sys.stdout.flush() 229 230 # Print the failed tests again to stderr as the build server will 231 # store this in a separate file with a direct link from the build 232 # status page. The full build long page on the build server, buffers 233 # stdout and stderr and interleaves them at random. By printing 234 # the summary to both stderr and stdout, we get at least one of them 235 # at the bottom of that file. 236 for test_result in self.test_results: 237 test_result.print_results(print_failed_only=True) 238 sys.stderr.write( 239 f"[ FAILED ] {self.tests_failed,} tests in " 240 f"{self.had_failures} projects.\n" 241 ) 242 243 244def test_should_run(testname: str, test_filters: Optional[list[re.Pattern]]): 245 """Check if test should run. 246 247 Args: 248 testname: Name of test to check. 249 test_filters: Regex list that limits the tests to run. 250 251 Returns: 252 True if test_filters list is empty or None, True if testname matches any 253 regex in test_filters, False otherwise. 254 """ 255 if not test_filters: 256 return True 257 for r in test_filters: 258 if r.search(testname): 259 return True 260 return False 261 262 263def projects_to_test( 264 build_config: TrustyBuildConfig, 265 projects: list[str], 266 test_filters: list[re.Pattern], 267 run_disabled_tests: bool = False, 268) -> list[str]: 269 """Checks which projects have any of the specified tests. 270 271 Args: 272 build_config: TrustyBuildConfig object. 273 projects: Names of the projects to search for active tests. 274 test_filters: List that limits the tests to run. Projects without any 275 tests that match a filter will be skipped. 276 run_disabled_tests: Also run disabled tests from config file. 277 278 Returns: 279 A list of projects with tests that should be run 280 """ 281 282 def has_test(name: str): 283 project = build_config.get_project(name) 284 for test in project.tests: 285 if not test.enabled and not run_disabled_tests: 286 continue 287 if test_should_run(test.name, test_filters): 288 return True 289 return False 290 291 return [project for project in projects if has_test(project)] 292 293 294# Put a global cap on the number of retries to detect flaky tests such that we 295# do not risk increasing the time to try all tests substantially. This should be 296# fine since *most* tests are not flaky. 297# TODO: would it be better to put a cap on the time spent retrying tests? We may 298# not want to retry long running tests. 299MAX_RETRIES = 10 300 301 302def run_tests( 303 build_config: TrustyBuildConfig, 304 root: os.PathLike, 305 project: str, 306 qemu_instance_id: Optional[str], 307 run_disabled_tests: bool = False, 308 test_filters: Optional[list[re.Pattern]] = None, 309 verbose: bool = False, 310 debug_on_error: bool = False, 311 emulator: bool = True, 312) -> TestResults: 313 """Run tests for a project. 314 315 Args: 316 build_config: TrustyBuildConfig object. 317 root: Trusty build root output directory. 318 project: Project name. 319 qemu_instance_id: name of the QEmu instance to use. If the instance 320 doesn't already exist, a new fresh instance will be created. If 321 None, use the default instance. 322 run_disabled_tests: Also run disabled tests from config file. 323 test_filters: Optional list that limits the tests to run. 324 verbose: Enable debug output. 325 debug_on_error: Wait for debugger connection on errors. 326 327 Returns: 328 TestResults object listing overall and detailed test results. 329 """ 330 project_config = build_config.get_project(project=project) 331 project_root = f"{root}/build-{project}" 332 333 test_results = TestResults(project) 334 test_env = None 335 test_runner = None 336 337 if not qemu_instance_id: 338 qemu_instance_id = "default" 339 qemu_instance_dir = f"{project_root}/qemu-instances/{qemu_instance_id}" 340 341 def load_test_environment(): 342 sys.path.append(project_root) 343 try: 344 if run := sys.modules.get("run"): 345 if not run.__file__.startswith(project_root): 346 # Reload qemu and its dependencies because run.py uses them 347 # We do this in topological sort order 348 if qemu_error := sys.modules.get("qemu_error"): 349 importlib.reload(qemu_error) 350 if qemu_options := sys.modules.get("qemu_options"): 351 importlib.reload(qemu_options) 352 if qemu := sys.modules.get("qemu"): 353 importlib.reload(qemu) 354 355 # run module was imported for another project and needs 356 # to be replaced with the one for the current project. 357 run = importlib.reload(run) 358 else: 359 # first import in this interpreter instance, we use importlib 360 # rather than a regular import statement since it avoids 361 # linter warnings. 362 run = importlib.import_module("run") 363 sys.path.pop() 364 except ImportError: 365 return None 366 367 return run 368 369 def print_test_command(name, cmd: Optional[list[str]] = None): 370 print() 371 print("Running", name, "on", test_results.project) 372 if cmd: 373 print( 374 "Command line:", " ".join([s.replace(" ", "\\ ") for s in cmd]) 375 ) 376 sys.stdout.flush() 377 378 def run_test( 379 test, parent_test: Optional[TrustyCompositeTest] = None, retry=True 380 ) -> Optional[TestResult]: 381 """Execute a single test and print out helpful information 382 383 Returns: 384 The results of running this test, or None for non-tests, like 385 reboots or tests that don't work in this environment. 386 """ 387 nonlocal test_env, test_runner 388 cmd = test.command[1:] 389 disable_rpmb = True if "--disable_rpmb" in cmd else None 390 391 test_start_time = time.time() 392 393 if not emulator and not isinstance(test, TrustyHostTest): 394 return None 395 396 match test: 397 case TrustyHostTest(): 398 # append nice and expand path to command 399 cmd = ["nice", f"{project_root}/{test.command[0]}"] + cmd 400 print_test_command(test.name, cmd) 401 cmd_status = subprocess.call(cmd) 402 result = TestResult(test.name, cmd_status, False) 403 case TrustyCompositeTest(): 404 status_code: Optional[int] = 0 405 for subtest in test.sequence: 406 subtest_result = run_test(subtest, test, retry) 407 if subtest_result and subtest_result.failed(): 408 status_code = subtest_result.status 409 # fail the composite test with the same status code as 410 # the first failing subtest 411 break 412 result = TestResult(test.name, status_code, False) 413 414 case TrustyTest(): 415 # Benchmark runs on QEMU are meaningless and take a lot of 416 # CI time. One can still run the bootport test manually 417 # if desired 418 if test.port_type == PortType.BENCHMARK: 419 return TestResult(test.name, None, False) 420 else: 421 if isinstance(test, TrustyAndroidTest): 422 print_test_command(test.name, [test.shell_command]) 423 elif isinstance(test, TrustyHostcommandTest): 424 print_test_command(test.name, test.command[3:]) 425 else: 426 # port tests are identified by their port name, 427 # no command 428 print_test_command(test.name) 429 430 if not test_env: 431 test_env = load_test_environment() 432 if test_env: 433 if not test_runner: 434 test_runner = test_env.init( 435 android=build_config.android, 436 instance_dir=qemu_instance_dir, 437 disable_rpmb=disable_rpmb, 438 verbose=verbose, 439 debug_on_error=debug_on_error, 440 ) 441 cmd_status = test_env.run_test(test_runner, cmd) 442 result = TestResult(test.name, cmd_status, False) 443 else: 444 return TestResult(test.name, None, False) 445 case TrustyRebootCommand() if parent_test: 446 assert isinstance(parent_test, TrustyCompositeTest) 447 if test_env: 448 test_env.shutdown(test_runner, test.mode.factory_reset(), 449 full_wipe=test.mode.full_wipe()) 450 test_runner = None 451 print(f"Shutting down to {test.mode} test environment on " 452 f"{test_results.project}") 453 # return early so we do not report the time to reboot or try to 454 # add the reboot command to test results. 455 return None 456 case TrustyRebootCommand(): 457 raise RuntimeError( 458 "Reboot may only be used inside compositetest" 459 ) 460 case TrustyPrintCommand() if parent_test: 461 print(test.msg()) 462 return None 463 case TrustyPrintCommand(): 464 raise RuntimeError( 465 "Print may only be used inside compositetest" 466 ) 467 case _: 468 raise NotImplementedError(f"Don't know how to run {test.name}") 469 470 elapsed = time.time() - test_start_time 471 print( f"{result} after {elapsed:.3f} seconds") 472 473 can_retry = retry and test_results.retried_count < MAX_RETRIES 474 if result and result.failed() and can_retry: 475 print( 476 f"retrying potentially flaky test {test.name} on", 477 test_results.project, 478 ) 479 # TODO: first retry the test without restarting the test 480 # environment and if that fails, restart and then 481 # retry if < MAX_RETRIES. 482 if test_env: 483 test_env.shutdown(test_runner) 484 test_runner = None 485 retried_result = run_test(test, parent_test, retry=False) 486 # Know this is the kind of test that returns a status b/c it failed 487 assert retried_result is not None 488 retried_result.retried = True 489 return retried_result 490 else: 491 # Test passed, was skipped, or we're not retrying it. 492 return result 493 494 # the retry mechanism is intended to allow a batch run of all tests to pass 495 # even if a small handful of tests exhibit flaky behavior. If a test filter 496 # was provided or debug on error is set, we are most likely not doing a 497 # batch run (as is the case for presubmit testing) meaning that it is 498 # not all that helpful to retry failing tests vs. finishing the run faster. 499 retry = test_filters is None and not debug_on_error 500 try: 501 for test in project_config.tests: 502 if not test.enabled and not run_disabled_tests: 503 continue 504 if not test_should_run(test.name, test_filters): 505 continue 506 507 if result := run_test(test, None, retry): 508 test_results.add_result(result) 509 finally: 510 # finally is used here to make sure that we attempt to shutdown the 511 # test environment no matter whether an exception was raised or not 512 # and no matter what kind of test caused an exception to be raised. 513 if test_env: 514 test_env.shutdown(test_runner) 515 # any saved exception from the try block will be re-raised here 516 517 return test_results 518 519 520def test_projects( 521 build_config: TrustyBuildConfig, 522 root: os.PathLike, 523 projects: list[str], 524 qemu_instance_id: Optional[str] = None, 525 run_disabled_tests: bool = False, 526 test_filters: Optional[list[re.Pattern]] = None, 527 verbose: bool = False, 528 debug_on_error: bool = False, 529 emulator: bool = True, 530) -> MultiProjectTestResults: 531 """Run tests for multiple project. 532 533 Args: 534 build_config: TrustyBuildConfig object. 535 root: Trusty build root output directory. 536 projects: Names of the projects to run tests for. 537 qemu_instance_id: name of the QEmu instance to use. If the instance 538 doesn't already exist, a new fresh instance will be created. If 539 None, use the default instance. 540 run_disabled_tests: Also run disabled tests from config file. 541 test_filters: Optional list that limits the tests to run. Projects 542 without any tests that match a filter will be skipped. 543 verbose: Enable debug output. 544 debug_on_error: Wait for debugger connection on errors. 545 546 Returns: 547 MultiProjectTestResults listing overall and detailed test results. 548 """ 549 if test_filters: 550 projects = projects_to_test( 551 build_config, 552 projects, 553 test_filters, 554 run_disabled_tests=run_disabled_tests, 555 ) 556 557 results = [] 558 for project in projects: 559 results.append( 560 run_tests( 561 build_config, 562 root, 563 project, 564 qemu_instance_id=qemu_instance_id, 565 run_disabled_tests=run_disabled_tests, 566 test_filters=test_filters, 567 verbose=verbose, 568 debug_on_error=debug_on_error, 569 emulator=emulator, 570 ) 571 ) 572 return MultiProjectTestResults(results) 573 574 575def default_root() -> str: 576 script_dir = os.path.dirname(os.path.abspath(__file__)) 577 top = os.path.abspath(os.path.join(script_dir, "../../../../..")) 578 return os.path.join(top, "build-root") 579 580 581def main(): 582 parser = argparse.ArgumentParser() 583 parser.add_argument( 584 "project", type=str, nargs="+", help="Project(s) to test." 585 ) 586 parser.add_argument( 587 "--instance-id", 588 type=str, 589 default=None, 590 help=("ID of a QEmu instance to use for the tests. A fresh instance " 591 "will be created if no instance with this ID already exists." 592 "'default' will be used if no value is provided.") 593 ) 594 parser.add_argument( 595 "--build-root", 596 type=str, 597 default=default_root(), 598 help="Root of intermediate build directory.", 599 ) 600 parser.add_argument( 601 "--run_disabled_tests", 602 help="Also run disabled tests from config file.", 603 action="store_true", 604 ) 605 parser.add_argument( 606 "--test", 607 type=str, 608 action="append", 609 help="Only run tests that match the provided regexes.", 610 ) 611 parser.add_argument( 612 "--verbose", help="Enable debug output.", action="store_true" 613 ) 614 parser.add_argument( 615 "--debug_on_error", 616 help="Wait for debugger connection on errors.", 617 action="store_true", 618 ) 619 parser.add_argument( 620 "--android", 621 type=str, 622 help="Path to an Android build to run tests against.", 623 ) 624 args = parser.parse_args() 625 626 build_config = TrustyBuildConfig(android=args.android) 627 628 test_filters = ( 629 [re.compile(test) for test in args.test] if args.test else None 630 ) 631 test_results = test_projects( 632 build_config, 633 args.build_root, 634 args.project, 635 qemu_instance_id=args.instance_id, 636 run_disabled_tests=args.run_disabled_tests, 637 test_filters=test_filters, 638 verbose=args.verbose, 639 debug_on_error=args.debug_on_error, 640 ) 641 test_results.print_results() 642 643 if test_results.failed_projects: 644 sys.exit(1) 645 646 647if __name__ == "__main__": 648 main() 649