1# Copyright 2015 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import errno 6import os 7import re 8import shutil 9import signal 10import stat 11import subprocess 12import sys 13import tempfile 14import threading 15 16import logging 17# Turn the logging level to INFO before importing other autotest 18# code, to avoid having failed import logging messages confuse the 19# test_that user. 20logging.basicConfig(level=logging.INFO) 21 22import common 23from autotest_lib.client.common_lib.cros import dev_server, retry 24from autotest_lib.client.common_lib import logging_manager 25from autotest_lib.server.cros.dynamic_suite import suite, constants 26from autotest_lib.server.cros import provision 27from autotest_lib.server.hosts import factory 28from autotest_lib.server.hosts import file_store 29from autotest_lib.server.hosts import host_info 30from autotest_lib.server import autoserv_utils 31from autotest_lib.server import server_logging_config 32from autotest_lib.server import utils 33from autotest_lib.utils import labellib 34 35 36_autoserv_proc = None 37_sigint_handler_lock = threading.Lock() 38 39_AUTOSERV_SIGINT_TIMEOUT_SECONDS = 5 40NO_BOARD = 'ad_hoc_board' 41NO_BUILD = 'ad_hoc_build' 42_SUITE_REGEX = r'suite:(.*)' 43 44_TEST_KEY_FILENAME = 'testing_rsa' 45TEST_KEY_PATH = ('/mnt/host/source/src/scripts/mod_for_test_scripts/' 46 'ssh_keys/%s' % _TEST_KEY_FILENAME) 47 48_LATEST_RESULTS_DIRECTORY = '/tmp/test_that_latest' 49_HOST_INFO_SUBDIR = 'host_info_store' 50 51 52class TestThatRunError(Exception): 53 """Raised if test_that encounters something unexpected while running.""" 54 55 56class TestThatProvisioningError(Exception): 57 """Raised when it fails to provision the DUT to the requested build.""" 58 59 60def add_common_args(parser): 61 """ 62 Add common arguments for both test_that and test_droid to their parser. 63 64 @param parser: argparse.ArgumentParser object to add arguments to. 65 """ 66 parser.add_argument('tests', nargs='+', metavar='TEST', 67 help='Run given test(s). Use suite:SUITE to specify ' 68 'test suite. Use e:[NAME_PATTERN] to specify a ' 69 'NAME-matching regular expression. Use ' 70 'f:[FILE_PATTERN] to specify a filename matching ' 71 'regular expression. Specified regular ' 72 'expressions will be implicitly wrapped in ' 73 '^ and $.') 74 parser.add_argument('--fast', action='store_true', dest='fast_mode', 75 default=False, 76 help='Enable fast mode. This will cause test_droid ' 77 'to skip time consuming steps like sysinfo and ' 78 'collecting crash information.') 79 parser.add_argument('--args', metavar='ARGS', 80 help='Whitespace separated argument string to pass ' 81 'through to test. Only supported for runs ' 82 'against a local DUT. ' 83 "e.g. --args='foo=bar cat=\"in a hat\"'.") 84 parser.add_argument('--results_dir', metavar='RESULTS_DIR', default=None, 85 help='Instead of storing results in a new subdirectory' 86 ' of /tmp , store results in RESULTS_DIR. If ' 87 'RESULTS_DIR already exists, it will be deleted.') 88 parser.add_argument('--pretend', action='store_true', default=False, 89 help='Print autoserv commands that would be run, ' 90 'rather than running them.') 91 parser.add_argument('--no-experimental', action='store_true', 92 default=False, dest='no_experimental', 93 help='When scheduling a suite, skip any tests marked ' 94 'as experimental. Applies only to tests scheduled' 95 ' via suite:[SUITE].') 96 parser.add_argument('--enforce-deps', action='store_true', 97 default=False, dest='enforce_deps', 98 help='Skip tests whose DEPENDENCIES can not ' 99 'be satisfied.') 100 parser.add_argument('--debug', action='store_true', 101 help='Include DEBUG level messages in stdout. Note: ' 102 'these messages will be included in output log ' 103 'file regardless. In addition, turn on autoserv ' 104 'verbosity.') 105 parser.add_argument('--iterations', action='store', type=int, default=1, 106 help='Number of times to run the tests specified.') 107 parser.add_argument('--ssh_verbosity', action='store', type=int, 108 choices=[0, 1, 2, 3], default=0, 109 help='Verbosity level for ssh, between 0 and 3 ' 110 'inclusive.') 111 parser.add_argument('--ssh_options', action='store', default=None, 112 help='A string giving additional options to be ' 113 'added to ssh commands.') 114 115 116class LocalSuite(suite.Suite): 117 """Subclass of Suite with methods for running locally""" 118 119 def handle_local_result(self, job_id, results_dir, record): 120 """ 121 Handle recording and/or retrying a completed job run locally. 122 123 @param job_id: int ID of job 124 @param results_dir: absolute path where test results were stored. 125 @param record: callable that records job status 126 127 @returns: new job_id if a job was scheduled for retry, None otherwise. 128 """ 129 logging.debug('Parsing test results for job %s',job_id) 130 code = generate_report(results_dir, just_status_code=True) 131 if not self._retry_handler: 132 return None 133 logging.debug('Handling result of job %s',job_id) 134 logging.debug(self._retry_handler._retry_map) 135 if code == 0: 136 logging.debug('All tests for job %s succeeded, no retry', job_id) 137 if self._retry_handler.job_present(job_id): 138 self._retry_handler.set_attempted(job_id) 139 return None 140 141 new_job_id = None 142 go_ahead = (self._job_retry and 143 self._retry_handler._should_retry_local_job(job_id)) 144 if go_ahead: 145 new_job_id = self._retry_local_result(job_id, record) 146 return new_job_id 147 148 def _retry_local_result(self, job_id, record): 149 """ 150 Retry a test job by id. 151 152 @param job_id: int ID of job 153 @param record: callable that records job status. 154 prototype: 155 record(base_job.status_log_entry) 156 157 @returns: new job_id if a job was scheduled for retry, None otherwise. 158 """ 159 test = self._jobs_to_tests[job_id] 160 logging.debug('Attempting to retry job %s, test %s', job_id, test.name) 161 test.fast = False 162 new_job = self._schedule_test( 163 record=record, test=test, retry_for=job_id) 164 if new_job: 165 return new_job.id 166 return None 167 168 def test_name_from_job(self, job_id): 169 """Find the name of the test run by a job with a given job ID.""" 170 if self._jobs_to_tests[job_id]: 171 return self._jobs_to_tests[job_id].name 172 173 174 175def fetch_local_suite(autotest_path, suite_predicate, afe, test_arg, remote, 176 build=NO_BUILD, board=NO_BOARD, 177 results_directory=None, no_experimental=False, 178 ignore_deps=True, job_retry=True): 179 """Create a suite from the given suite predicate. 180 181 Satisfaction of dependencies is enforced by Suite.schedule() if 182 ignore_deps is False. Note that this method assumes only one host, 183 i.e. |remote|, was added to afe. Suite.schedule() will not 184 schedule a job if none of the hosts in the afe (in our case, 185 just one host |remote|) has a label that matches a requested 186 test dependency. 187 188 @param autotest_path: Absolute path to autotest (in sysroot or 189 custom autotest directory set by --autotest_dir). 190 @param suite_predicate: callable that takes ControlData objects, and 191 returns True on those that should be in suite 192 @param afe: afe object to schedule against (typically a directAFE) 193 @param test_arg: String. An individual TEST command line argument, e.g. 194 'login_CryptohomeMounted' or 'suite:smoke'. 195 @param remote: String representing the IP of the remote host. 196 @param build: Build to schedule suite for. 197 @param board: Board to schedule suite for. 198 @param results_directory: Absolute path of directory to store results in. 199 (results will be stored in subdirectory of this). 200 @param no_experimental: Skip experimental tests when scheduling a suite. 201 @param ignore_deps: If True, test dependencies will be ignored. 202 @param job_retry: If False, tests will not be retried at all. 203 204 @returns: A LocalSuite object. 205 206 """ 207 fs_getter = suite.create_fs_getter(autotest_path) 208 devserver = dev_server.ImageServer('') 209 my_suite = LocalSuite.create_from_predicates( 210 [suite_predicate], 211 {provision.CROS_VERSION_PREFIX: build}, 212 constants.BOARD_PREFIX + board, 213 devserver, fs_getter, afe=afe, 214 ignore_deps=ignore_deps, 215 results_dir=results_directory, 216 forgiving_parser=False, 217 job_retry=job_retry 218 ) 219 if len(my_suite.tests) == 0: 220 (similarity_predicate, similarity_description) = ( 221 get_predicate_for_possible_test_arg(test_arg)) 222 logging.error('No test found, searching for possible tests with %s', 223 similarity_description) 224 possible_tests = suite.find_possible_tests(fs_getter, 225 similarity_predicate) 226 raise ValueError('Found no tests. Check your suite name, test name, ' 227 'or test matching wildcard.\nDid you mean any of ' 228 'following tests?\n %s' % '\n '.join(possible_tests)) 229 230 if not ignore_deps: 231 # Log tests whose dependencies can't be satisfied. 232 labels = [label.name for label in 233 afe.get_labels(host__hostname=remote)] 234 for test in my_suite.tests: 235 if test.experimental and no_experimental: 236 continue 237 unsatisfiable_deps = set(test.dependencies).difference(labels) 238 if unsatisfiable_deps: 239 logging.warning('%s will be skipped, unsatisfiable ' 240 'test dependencies: %s', test.name, 241 unsatisfiable_deps) 242 return my_suite 243 244 245def _run_autoserv(command, pretend=False): 246 """Run autoserv command. 247 248 Run the autoserv command and wait on it. Log the stdout. 249 Ensure that SIGINT signals are passed along to autoserv. 250 251 @param command: the autoserv command to run. 252 @returns: exit code of the command. 253 254 """ 255 if not pretend: 256 logging.debug('Running autoserv command: %s', command) 257 global _autoserv_proc 258 _autoserv_proc = subprocess.Popen(command, 259 stdout=subprocess.PIPE, 260 stderr=subprocess.STDOUT) 261 # This incantation forces unbuffered reading from stdout, 262 # so that autoserv output can be displayed to the user 263 # immediately. 264 for message in iter(_autoserv_proc.stdout.readline, b''): 265 logging.info('autoserv| %s', message.rstrip()) 266 267 _autoserv_proc.wait() 268 returncode = _autoserv_proc.returncode 269 _autoserv_proc = None 270 else: 271 logging.info('Pretend mode. Would run autoserv command: %s', 272 command) 273 returncode = 0 274 return returncode 275 276 277def run_provisioning_job(provision_label, host, info, autotest_path, 278 results_directory, fast_mode, 279 ssh_verbosity=0, ssh_options=None, 280 pretend=False, autoserv_verbose=False): 281 """Shell out to autoserv to run provisioning job. 282 283 @param provision_label: Label to provision the machine to. 284 @param host: Hostname of DUT. 285 @param info: A host_info.HostInfo for the remote host. 286 @param autotest_path: Absolute path of autotest directory. 287 @param results_directory: Absolute path of directory to store results in. 288 (results will be stored in subdirectory of this). 289 @param fast_mode: bool to use fast mode (disables slow autotest features). 290 @param ssh_verbosity: SSH verbosity level, passed along to autoserv_utils 291 @param ssh_options: Additional ssh options to be passed to autoserv_utils 292 @param pretend: If True, will print out autoserv commands rather than 293 running them. 294 @param autoserv_verbose: If true, pass the --verbose flag to autoserv. 295 296 @returns: Absolute path of directory where results were stored. 297 298 """ 299 # TODO(fdeng): When running against a local DUT, autoserv 300 # is still hitting the AFE in the lab. 301 # provision_AutoUpdate checks the current build of DUT by 302 # retrieving build info from AFE. crosbug.com/295178 303 results_directory = os.path.join(results_directory, 'results-provision') 304 _write_host_info(results_directory, _HOST_INFO_SUBDIR, host, info) 305 command = autoserv_utils.autoserv_run_job_command( 306 os.path.join(autotest_path, 'server'), 307 machines=host, job=None, verbose=autoserv_verbose, 308 results_directory=results_directory, 309 fast_mode=fast_mode, ssh_verbosity=ssh_verbosity, 310 ssh_options=ssh_options, 311 extra_args=['--provision', '--job-labels', provision_label], 312 no_console_prefix=True, 313 host_info_subdir=_HOST_INFO_SUBDIR) 314 if _run_autoserv(command, pretend) != 0: 315 raise TestThatProvisioningError('Command returns non-zero code: %s ' % 316 command) 317 return results_directory 318 319 320def run_job(job, host, info, autotest_path, results_directory, fast_mode, 321 id_digits=1, ssh_verbosity=0, ssh_options=None, 322 args=None, pretend=False, 323 autoserv_verbose=False): 324 """ 325 Shell out to autoserv to run an individual test job. 326 327 @param job: A Job object containing the control file contents and other 328 relevent metadata for this test. 329 @param host: Hostname of DUT to run test against. 330 @param info: a host_info.HostInfo for the remote host. 331 @param autotest_path: Absolute path of autotest directory. 332 @param results_directory: Absolute path of directory to store results in. 333 (results will be stored in subdirectory of this). 334 @param fast_mode: bool to use fast mode (disables slow autotest features). 335 @param id_digits: The minimum number of digits that job ids should be 336 0-padded to when formatting as a string for results 337 directory. 338 @param ssh_verbosity: SSH verbosity level, passed along to autoserv_utils 339 @param ssh_options: Additional ssh options to be passed to autoserv_utils 340 @param args: String that should be passed as args parameter to autoserv, 341 and then ultimitely to test itself. 342 @param pretend: If True, will print out autoserv commands rather than 343 running them. 344 @param autoserv_verbose: If true, pass the --verbose flag to autoserv. 345 346 @returns: a tuple, return code of the job and absolute path of directory 347 where results were stored. 348 """ 349 with tempfile.NamedTemporaryFile() as temp_file: 350 temp_file.write(job.control_file) 351 temp_file.flush() 352 name_tail = job.name.split('/')[-1] 353 results_directory = os.path.join(results_directory, 354 'results-%0*d-%s' % (id_digits, job.id, 355 name_tail)) 356 # Drop experimental keyval in the keval file in the job result folder. 357 os.makedirs(results_directory) 358 utils.write_keyval(results_directory, 359 {constants.JOB_EXPERIMENTAL_KEY: job.keyvals[ 360 constants.JOB_EXPERIMENTAL_KEY]}) 361 _write_host_info(results_directory, _HOST_INFO_SUBDIR, host, info) 362 extra_args = [temp_file.name] 363 if args: 364 extra_args.extend(['--args', args]) 365 366 command = autoserv_utils.autoserv_run_job_command( 367 os.path.join(autotest_path, 'server'), 368 machines=host, job=job, verbose=autoserv_verbose, 369 results_directory=results_directory, 370 fast_mode=fast_mode, ssh_verbosity=ssh_verbosity, 371 ssh_options=ssh_options, 372 extra_args=extra_args, 373 no_console_prefix=True, 374 use_packaging=False, 375 host_attributes=info.attributes, 376 host_info_subdir=_HOST_INFO_SUBDIR) 377 378 code = _run_autoserv(command, pretend) 379 return code, results_directory 380 381 382def setup_local_afe(): 383 """ 384 Setup a local afe database and return a direct_afe object to access it. 385 386 @returns: A autotest_lib.frontend.afe.direct_afe instance. 387 """ 388 # This import statement is delayed until now rather than running at 389 # module load time, because it kicks off a local sqlite :memory: backed 390 # database, and we don't need that unless we are doing a local run. 391 from autotest_lib.frontend import setup_django_lite_environment 392 from autotest_lib.frontend.afe import direct_afe 393 return direct_afe.directAFE() 394 395 396def get_predicate_for_test_arg(test): 397 """ 398 Gets a suite predicte function for a given command-line argument. 399 400 @param test: String. An individual TEST command line argument, e.g. 401 'login_CryptohomeMounted' or 'suite:smoke' 402 @returns: A (predicate, string) tuple with the necessary suite 403 predicate, and a description string of the suite that 404 this predicate will produce. 405 """ 406 suitematch = re.match(_SUITE_REGEX, test) 407 name_pattern_match = re.match(r'e:(.*)', test) 408 file_pattern_match = re.match(r'f:(.*)', test) 409 if suitematch: 410 suitename = suitematch.group(1) 411 return (suite.name_in_tag_predicate(suitename), 412 'suite named %s' % suitename) 413 if name_pattern_match: 414 pattern = '^%s$' % name_pattern_match.group(1) 415 return (suite.test_name_matches_pattern_predicate(pattern), 416 'suite to match name pattern %s' % pattern) 417 if file_pattern_match: 418 pattern = '^%s$' % file_pattern_match.group(1) 419 return (suite.test_file_matches_pattern_predicate(pattern), 420 'suite to match file name pattern %s' % pattern) 421 return (suite.test_name_equals_predicate(test), 422 'job named %s' % test) 423 424 425def get_predicate_for_possible_test_arg(test): 426 """ 427 Gets a suite predicte function to calculate the similarity of given test 428 and possible tests. 429 430 @param test: String. An individual TEST command line argument, e.g. 431 'login_CryptohomeMounted' or 'suite:smoke' 432 @returns: A (predicate, string) tuple with the necessary suite 433 predicate, and a description string of the suite that 434 this predicate will produce. 435 """ 436 suitematch = re.match(_SUITE_REGEX, test) 437 name_pattern_match = re.match(r'e:(.*)', test) 438 file_pattern_match = re.match(r'f:(.*)', test) 439 if suitematch: 440 suitename = suitematch.group(1) 441 return (suite.name_in_tag_similarity_predicate(suitename), 442 'suite name similar to %s' % suitename) 443 if name_pattern_match: 444 pattern = '^%s$' % name_pattern_match.group(1) 445 return (suite.test_name_similarity_predicate(pattern), 446 'job name similar to %s' % pattern) 447 if file_pattern_match: 448 pattern = '^%s$' % file_pattern_match.group(1) 449 return (suite.test_file_similarity_predicate(pattern), 450 'suite to match file name similar to %s' % pattern) 451 return (suite.test_name_similarity_predicate(test), 452 'job name similar to %s' % test) 453 454 455def add_ssh_identity(temp_directory, ssh_private_key=TEST_KEY_PATH): 456 """Add an ssh identity to the agent. 457 458 TODO (sbasi) b/26186193: Add support for test_droid and make TEST_KEY_PATH 459 not Chrome OS specific. 460 461 @param temp_directory: A directory to copy the |private key| into. 462 @param ssh_private_key: Path to the ssh private key to use for testing. 463 """ 464 # Add the testing key to the current ssh agent. 465 if os.environ.has_key('SSH_AGENT_PID'): 466 # Copy the testing key to the temp directory and make it NOT 467 # world-readable. Otherwise, ssh-add complains. 468 shutil.copy(ssh_private_key, temp_directory) 469 key_copy_path = os.path.join(temp_directory, 470 os.path.basename(ssh_private_key)) 471 os.chmod(key_copy_path, stat.S_IRUSR | stat.S_IWUSR) 472 p = subprocess.Popen(['ssh-add', key_copy_path], 473 stderr=subprocess.STDOUT, stdout=subprocess.PIPE) 474 p_out, _ = p.communicate() 475 for line in p_out.splitlines(): 476 logging.info(line) 477 else: 478 logging.warning('There appears to be no running ssh-agent. Attempting ' 479 'to continue without running ssh-add, but ssh commands ' 480 'may fail.') 481 482 483def _auto_detect_labels(afe, remote): 484 """Automatically detect host labels and add them to the host in afe. 485 486 Note that the label of board will not be auto-detected. 487 This method assumes the host |remote| has already been added to afe. 488 489 @param afe: A direct_afe object used to interact with local afe database. 490 @param remote: The hostname of the remote device. 491 492 @returns: the detected labels as a list of strings. 493 """ 494 cros_host = factory.create_host(remote) 495 labels_to_create = [label for label in cros_host.get_labels() 496 if not label.startswith(constants.BOARD_PREFIX)] 497 labels_to_add_to_afe_host = [] 498 for label in labels_to_create: 499 new_label = afe.create_label(label) 500 labels_to_add_to_afe_host.append(new_label.name) 501 hosts = afe.get_hosts(hostname=remote) 502 if not hosts: 503 raise TestThatRunError('Unexpected error: %s has not ' 504 'been added to afe.' % remote) 505 afe_host = hosts[0] 506 afe_host.add_labels(labels_to_add_to_afe_host) 507 return labels_to_add_to_afe_host 508 509 510def perform_local_run(afe, autotest_path, tests, remote, fast_mode, 511 build=NO_BUILD, board=NO_BOARD, args=None, 512 pretend=False, no_experimental=False, 513 ignore_deps=True, 514 results_directory=None, ssh_verbosity=0, 515 ssh_options=None, 516 autoserv_verbose=False, 517 iterations=1, 518 host_attributes={}, 519 job_retry=True): 520 """Perform local run of tests. 521 522 This method enforces satisfaction of test dependencies for tests that are 523 run as a part of a suite. 524 525 @param afe: A direct_afe object used to interact with local afe database. 526 @param autotest_path: Absolute path of autotest installed in sysroot or 527 custom autotest path set by --autotest_dir. 528 @param tests: List of strings naming tests and suites to run. Suite strings 529 should be formed like "suite:smoke". 530 @param remote: Remote hostname. 531 @param fast_mode: bool to use fast mode (disables slow autotest features). 532 @param build: String specifying build for local run. 533 @param board: String specifyinb board for local run. 534 @param args: String that should be passed as args parameter to autoserv, 535 and then ultimitely to test itself. 536 @param pretend: If True, will print out autoserv commands rather than 537 running them. 538 @param no_experimental: Skip experimental tests when scheduling a suite. 539 @param ignore_deps: If True, test dependencies will be ignored. 540 @param results_directory: Directory to store results in. Defaults to None, 541 in which case results will be stored in a new 542 subdirectory of /tmp 543 @param ssh_verbosity: SSH verbosity level, passed through to 544 autoserv_utils. 545 @param ssh_options: Additional ssh options to be passed to autoserv_utils 546 @param autoserv_verbose: If true, pass the --verbose flag to autoserv. 547 @param iterations: int number of times to schedule tests. 548 @param host_attributes: Dict of host attributes to pass into autoserv. 549 @param job_retry: If False, tests will not be retried at all. 550 551 @returns: A list of return codes each job that has run. Or [1] if 552 provision failed prior to running any jobs. 553 """ 554 args = _set_default_servo_args(args) 555 # Create host in afe, add board and build labels. 556 cros_version_label = labellib.format_keyval_label( 557 labellib.KeyvalLabel(labellib.Key.CROS_VERSION, build)) 558 559 build_label = afe.create_label(cros_version_label) 560 board_label = afe.create_label(constants.BOARD_PREFIX + board) 561 labels = [build_label.name, board_label.name] 562 563 new_host = afe.create_host(remote) 564 new_host.add_labels(labels) 565 if not ignore_deps: 566 logging.info('Auto-detecting labels for %s', remote) 567 labels += _auto_detect_labels(afe, remote) 568 # Auto-detected labels may duplicate explicitly set ones. 569 labels = list(set(labels)) 570 571 info = host_info.HostInfo(labels, host_attributes) 572 573 # Provision the host to |build|. 574 if build != NO_BUILD: 575 logging.info('Provisioning %s...', cros_version_label) 576 try: 577 run_provisioning_job( 578 cros_version_label, 579 remote, 580 info, 581 autotest_path, 582 results_directory, 583 fast_mode, 584 ssh_verbosity, 585 ssh_options, 586 pretend, 587 autoserv_verbose, 588 ) 589 except TestThatProvisioningError as e: 590 logging.error('Provisioning %s to %s failed, tests are aborted, ' 591 'failure reason: %s', 592 remote, cros_version_label, e) 593 return [1] 594 595 # Create suites that will be scheduled. 596 suites_and_descriptions = [] 597 for test in tests: 598 (predicate, description) = get_predicate_for_test_arg(test) 599 logging.info('Fetching suite for %s...', description) 600 suite = fetch_local_suite(autotest_path, predicate, afe, test_arg=test, 601 remote=remote, 602 build=build, board=board, 603 results_directory=results_directory, 604 no_experimental=no_experimental, 605 ignore_deps=ignore_deps, 606 job_retry=job_retry) 607 suites_and_descriptions.append((suite, description)) 608 609 jobs_to_suites = {} 610 null_logger = lambda log_entry, log_in_subdir=False: None 611 # Schedule the suites, looping over iterations if necessary. 612 for iteration in range(iterations): 613 if iteration > 0: 614 logging.info('Repeating scheduling for iteration %d:', iteration) 615 616 for suite, description in suites_and_descriptions: 617 logging.info('Scheduling suite for %s...', description) 618 ntests = suite.schedule(null_logger) 619 logging.debug('jobs: %s nonzero job_retries: %s', 620 len(suite._jobs_to_tests), 621 len([True for (job_id, test) in 622 suite._jobs_to_tests.items()])) 623 logging.info('... scheduled %s job(s).', ntests) 624 for job in suite.jobs: 625 jobs_to_suites[job.id] = suite 626 627 if not afe.get_jobs(): 628 logging.info('No jobs scheduled. End of local run.') 629 return [] 630 631 last_job_id = afe.get_jobs()[-1].id 632 job_id_digits = len(str(last_job_id)) 633 codes = [] 634 job_queue = afe.get_jobs() 635 completed_job_ids = set() 636 while job_queue: 637 logging.info('%s jobs in job queue', len(job_queue)) 638 for job in job_queue: 639 suite = jobs_to_suites.get(job.id) 640 if not suite: 641 logging.error('Job %s not run, no associated suite.', job.id) 642 else: 643 logging.debug('Running job %s of test %s', 644 job.id, suite.test_name_from_job(job.id)) 645 code, abs_dir = run_job( 646 job, 647 remote, 648 info, 649 autotest_path, 650 results_directory, 651 fast_mode, 652 job_id_digits, 653 ssh_verbosity, 654 ssh_options, 655 args, 656 pretend, 657 autoserv_verbose, 658 ) 659 codes.append(code) 660 logging.debug("Code: %s, Results in %s", code, abs_dir) 661 new_id = suite.handle_local_result(job.id, abs_dir, null_logger) 662 if new_id: 663 jobs_to_suites[new_id] = jobs_to_suites[job.id] 664 completed_job_ids.add(job.id) 665 all_jobs = afe.get_jobs(not_yet_run=True, running=True) 666 new_jobs = set(job for job in all_jobs if job.id not in completed_job_ids) 667 logging.debug('%s incomplete jobs, %s jobs total', 668 len(new_jobs), len(all_jobs)) 669 job_queue = list(new_jobs) 670 return codes 671 672 673def _set_default_servo_args(args): 674 """Add default servo arguments for backward compatibitlity. 675 676 See crbug.com/881006 for context. Some servo related defaults were baked 677 into the autotest ServoHost code. These have now been deleted. A side effect 678 was that users of test_that relied on these defaults for some tests to work 679 magically in the chroot environment. 680 681 Current plan is to add back these defaults to test_that invocations for 682 backwards compatibility of these use cases. There is no planned removal date 683 for this hack. 684 685 @return modified args str. 686 """ 687 # args is a str with whitespace separated key=value arguments. 688 # Avoid parsing args here (to avoid adding another implicit constraint on 689 # the exact args format) by adding defaults only in the obvious cases where 690 # relevant keys are entirely missing. 691 if args is None: 692 args = '' 693 if 'servo_host' not in args: 694 args += ' servo_host=localhost' 695 if 'servo_port' not in args: 696 args += ' servo_port=9999' 697 return args 698 699 700def sigint_handler(signum, stack_frame): 701 #pylint: disable-msg=C0111 702 """Handle SIGINT or SIGTERM to a local test_that run. 703 704 This handler sends a SIGINT to the running autoserv process, 705 if one is running, giving it up to 5 seconds to clean up and exit. After 706 the timeout elapses, autoserv is killed. In either case, after autoserv 707 exits then this process exits with status 1. 708 """ 709 # If multiple signals arrive before handler is unset, ignore duplicates 710 if not _sigint_handler_lock.acquire(False): 711 return 712 try: 713 # Ignore future signals by unsetting handler. 714 signal.signal(signal.SIGINT, signal.SIG_IGN) 715 signal.signal(signal.SIGTERM, signal.SIG_IGN) 716 717 logging.warning('Received SIGINT or SIGTERM. Cleaning up and exiting.') 718 if _autoserv_proc: 719 logging.warning('Sending SIGINT to autoserv process. Waiting up ' 720 'to %s seconds for cleanup.', 721 _AUTOSERV_SIGINT_TIMEOUT_SECONDS) 722 _autoserv_proc.send_signal(signal.SIGINT) 723 timed_out, _ = retry.timeout(_autoserv_proc.wait, 724 timeout_sec=_AUTOSERV_SIGINT_TIMEOUT_SECONDS) 725 if timed_out: 726 _autoserv_proc.kill() 727 logging.warning('Timed out waiting for autoserv to handle ' 728 'SIGINT. Killed autoserv.') 729 finally: 730 _sigint_handler_lock.release() # this is not really necessary? 731 sys.exit(1) 732 733 734def create_results_directory(results_directory=None, board_name=None): 735 """Create a results directory. 736 737 If no directory is specified this method will create and return a 738 temp directory to hold results. If a directory name is specified this 739 method will create a directory at the given path, provided it doesn't 740 already exist. 741 742 @param results_directory: The path to the results_directory to create. 743 744 @return results_directory: A path to the results_directory, ready for use. 745 """ 746 if results_directory is None: 747 # Create a results_directory as subdir of /tmp 748 dirname_prefix='test_that_results_' 749 if board_name is not None: 750 dirname_prefix += (board_name + '_') 751 results_directory = tempfile.mkdtemp(prefix=dirname_prefix) 752 else: 753 # Delete results_directory if it already exists. 754 try: 755 shutil.rmtree(results_directory) 756 except OSError as e: 757 if e.errno != errno.ENOENT: 758 raise 759 760 # Create results_directory if it does not exist 761 try: 762 os.makedirs(results_directory) 763 except OSError as e: 764 if e.errno != errno.EEXIST: 765 raise 766 return results_directory 767 768def generate_report(directory, 769 whitelist_chrome_crashes=False, 770 just_status_code=False, html_report=False): 771 """Parse the test result files in the given directory into a report 772 773 @param directory: string, the absolute path of the directory to look in 774 @param whitelist_chrome_crashes: boolean, ignore Chrome crashes in the 775 report. Default: False, report Chrome crashes. 776 @param just_status_code: boolean, skip the report and only parse the files 777 to determine whether there were failures. Default: False, generate report. 778 """ 779 test_report_command = [os.path.join(os.path.dirname(__file__), 780 'generate_test_report')] 781 # Experimental test results do not influence the exit code. 782 test_report_command.append('--ignore_experimental_tests') 783 if html_report: 784 test_report_command.append('--html') 785 test_report_command.append('--html-report-dir=%s' % directory) 786 if whitelist_chrome_crashes: 787 test_report_command.append('--whitelist_chrome_crashes') 788 if just_status_code: 789 test_report_command.append('--just_status_code') 790 test_report_command.append(directory) 791 status_code = subprocess.call(test_report_command) 792 if not just_status_code: 793 with open(os.path.join(directory, 'test_report.log'), 794 'w') as report_log: 795 subprocess.call(test_report_command, stdout=report_log) 796 return status_code 797 798 799def perform_run_from_autotest_root(autotest_path, argv, tests, remote, 800 build=NO_BUILD, board=NO_BOARD, args=None, 801 pretend=False, no_experimental=False, 802 ignore_deps=True, 803 results_directory=None, ssh_verbosity=0, 804 ssh_options=None, 805 iterations=1, fast_mode=False, debug=False, 806 whitelist_chrome_crashes=False, 807 host_attributes={}, job_retry=True): 808 """ 809 Perform a test_that run, from the |autotest_path|. 810 811 This function is to be called from test_that/test_droid's main() script, 812 when tests are executed from the |autotest_path|. It handles all stages 813 of a test run that come after the bootstrap into |autotest_path|. 814 815 @param autotest_path: Full absolute path to the autotest root directory. 816 @param argv: The arguments list, as passed to main(...) 817 @param tests: List of strings naming tests and suites to run. Suite strings 818 should be formed like "suite:smoke". 819 @param remote: Remote hostname. 820 @param build: String specifying build for local run. 821 @param board: String specifying board for local run. 822 @param args: String that should be passed as args parameter to autoserv, 823 and then ultimitely to test itself. 824 @param pretend: If True, will print out autoserv commands rather than 825 running them. 826 @param no_experimental: Skip experimental tests when scheduling a suite. 827 @param ignore_deps: If True, test dependencies will be ignored. 828 @param results_directory: Directory to store results in. Defaults to None, 829 in which case results will be stored in a new 830 subdirectory of /tmp 831 @param ssh_verbosity: SSH verbosity level, passed through to 832 autoserv_utils. 833 @param ssh_options: Additional ssh options to be passed to autoserv_utils 834 @param autoserv_verbose: If true, pass the --verbose flag to autoserv. 835 @param iterations: int number of times to schedule tests. 836 @param fast_mode: bool to use fast mode (disables slow autotest features). 837 @param debug: Logging and autoserv verbosity. 838 @param whitelist_chrome_crashes: If True, whitelist chrome crashes. 839 @param host_attributes: Dict of host attributes to pass into autoserv. 840 @param job_retry: If False, tests will not be retried at all. 841 842 @return: A return code that test_that should exit with. 843 """ 844 if results_directory is None or not os.path.exists(results_directory): 845 raise ValueError('Expected valid results directory, got %s' % 846 results_directory) 847 848 logging_manager.configure_logging( 849 server_logging_config.ServerLoggingConfig(), 850 results_dir=results_directory, 851 use_console=True, 852 verbose=debug, 853 debug_log_name='test_that') 854 logging.info('Began logging to %s', results_directory) 855 856 logging.debug('test_that command line was: %s', argv) 857 858 signal.signal(signal.SIGINT, sigint_handler) 859 signal.signal(signal.SIGTERM, sigint_handler) 860 861 afe = setup_local_afe() 862 codes = perform_local_run(afe, autotest_path, tests, remote, fast_mode, 863 build, board, 864 args=args, 865 pretend=pretend, 866 no_experimental=no_experimental, 867 ignore_deps=ignore_deps, 868 results_directory=results_directory, 869 ssh_verbosity=ssh_verbosity, 870 ssh_options=ssh_options, 871 autoserv_verbose=debug, 872 iterations=iterations, 873 host_attributes=host_attributes, 874 job_retry=job_retry) 875 if pretend: 876 logging.info('Finished pretend run. Exiting.') 877 return 0 878 879 final_result = generate_report( 880 results_directory, 881 whitelist_chrome_crashes=whitelist_chrome_crashes, html_report=True) 882 try: 883 os.unlink(_LATEST_RESULTS_DIRECTORY) 884 except OSError: 885 pass 886 link_target = os.path.relpath(results_directory, 887 os.path.dirname(_LATEST_RESULTS_DIRECTORY)) 888 if any(codes): 889 logging.error('Autoserv encountered unexpected errors ' 890 'when executing jobs.') 891 final_result = final_result or 1 892 os.symlink(link_target, _LATEST_RESULTS_DIRECTORY) 893 logging.info('Finished running tests. Results can be found in %s or %s', 894 results_directory, _LATEST_RESULTS_DIRECTORY) 895 return final_result 896 897 898def _write_host_info(results_dir, host_info_subdir, hostname, info): 899 """ Write HostInfo to a FileStore to be used by autoserv. 900 901 @param results_dir: Path to he results directory. 902 @param host_info_subdir: Subdirectory of results directory for host info. 903 @param hostname: Hostname passed into autoserv. 904 @param info: hosts.HostInfo to write. 905 """ 906 d = os.path.join(results_dir, host_info_subdir) 907 os.makedirs(d) 908 store = file_store.FileStore(os.path.join(d, '%s.store' % hostname)) 909 store.commit(info)