• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2015 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import errno
6import os
7import re
8import shutil
9import signal
10import stat
11import subprocess
12import sys
13import tempfile
14import threading
15
16import logging
17# Turn the logging level to INFO before importing other autotest
18# code, to avoid having failed import logging messages confuse the
19# test_that user.
20logging.basicConfig(level=logging.INFO)
21
22import common
23from autotest_lib.client.common_lib.cros import dev_server, retry
24from autotest_lib.client.common_lib import logging_manager
25from autotest_lib.server.cros.dynamic_suite import suite, constants
26from autotest_lib.server.cros import provision
27from autotest_lib.server.hosts import factory
28from autotest_lib.server.hosts import file_store
29from autotest_lib.server.hosts import host_info
30from autotest_lib.server import autoserv_utils
31from autotest_lib.server import server_logging_config
32from autotest_lib.server import utils
33from autotest_lib.utils import labellib
34
35
36_autoserv_proc = None
37_sigint_handler_lock = threading.Lock()
38
39_AUTOSERV_SIGINT_TIMEOUT_SECONDS = 5
40NO_BOARD = 'ad_hoc_board'
41NO_BUILD = 'ad_hoc_build'
42_SUITE_REGEX = r'suite:(.*)'
43
44_TEST_KEY_FILENAME = 'testing_rsa'
45TEST_KEY_PATH = ('/mnt/host/source/src/scripts/mod_for_test_scripts/'
46                  'ssh_keys/%s' % _TEST_KEY_FILENAME)
47
48_LATEST_RESULTS_DIRECTORY = '/tmp/test_that_latest'
49_HOST_INFO_SUBDIR = 'host_info_store'
50
51
52class TestThatRunError(Exception):
53    """Raised if test_that encounters something unexpected while running."""
54
55
56class TestThatProvisioningError(Exception):
57    """Raised when it fails to provision the DUT to the requested build."""
58
59
60def add_common_args(parser):
61    """
62    Add common arguments for both test_that and test_droid to their parser.
63
64    @param parser: argparse.ArgumentParser object to add arguments to.
65    """
66    parser.add_argument('tests', nargs='+', metavar='TEST',
67                        help='Run given test(s). Use suite:SUITE to specify '
68                             'test suite. Use e:[NAME_PATTERN] to specify a '
69                             'NAME-matching regular expression. Use '
70                             'f:[FILE_PATTERN] to specify a filename matching '
71                             'regular expression. Specified regular '
72                             'expressions will be implicitly wrapped in '
73                             '^ and $.')
74    parser.add_argument('--fast', action='store_true', dest='fast_mode',
75                        default=False,
76                        help='Enable fast mode.  This will cause test_droid '
77                             'to skip time consuming steps like sysinfo and '
78                             'collecting crash information.')
79    parser.add_argument('--args', metavar='ARGS',
80                        help='Whitespace separated argument string to pass '
81                             'through to test. Only supported for runs '
82                             'against a local DUT. '
83                             "e.g. --args='foo=bar cat=\"in a hat\"'.")
84    parser.add_argument('--results_dir', metavar='RESULTS_DIR', default=None,
85                        help='Instead of storing results in a new subdirectory'
86                             ' of /tmp , store results in RESULTS_DIR. If '
87                             'RESULTS_DIR already exists, it will be deleted.')
88    parser.add_argument('--pretend', action='store_true', default=False,
89                        help='Print autoserv commands that would be run, '
90                             'rather than running them.')
91    parser.add_argument('--no-experimental', action='store_true',
92                        default=False, dest='no_experimental',
93                        help='When scheduling a suite, skip any tests marked '
94                             'as experimental. Applies only to tests scheduled'
95                             ' via suite:[SUITE].')
96    parser.add_argument('--enforce-deps', action='store_true',
97                        default=False, dest='enforce_deps',
98                        help='Skip tests whose DEPENDENCIES can not '
99                             'be satisfied.')
100    parser.add_argument('--debug', action='store_true',
101                        help='Include DEBUG level messages in stdout. Note: '
102                             'these messages will be included in output log '
103                             'file regardless. In addition, turn on autoserv '
104                             'verbosity.')
105    parser.add_argument('--iterations', action='store', type=int, default=1,
106                        help='Number of times to run the tests specified.')
107    parser.add_argument('--ssh_verbosity', action='store', type=int,
108                        choices=[0, 1, 2, 3], default=0,
109                        help='Verbosity level for ssh, between 0 and 3 '
110                             'inclusive.')
111    parser.add_argument('--ssh_options', action='store', default=None,
112                        help='A string giving additional options to be '
113                        'added to ssh commands.')
114
115
116class LocalSuite(suite.Suite):
117    """Subclass of Suite with methods for running locally"""
118
119    def handle_local_result(self, job_id, results_dir, record):
120        """
121        Handle recording and/or retrying a completed job run locally.
122
123        @param job_id: int ID of job
124        @param results_dir: absolute path where test results were stored.
125        @param record: callable that records job status
126
127        @returns: new job_id if a job was scheduled for retry, None otherwise.
128        """
129        logging.debug('Parsing test results for job %s',job_id)
130        code = generate_report(results_dir, just_status_code=True)
131        if not self._retry_handler:
132            return None
133        logging.debug('Handling result of job %s',job_id)
134        logging.debug(self._retry_handler._retry_map)
135        if code == 0:
136            logging.debug('All tests for job %s succeeded, no retry', job_id)
137            if self._retry_handler.job_present(job_id):
138                self._retry_handler.set_attempted(job_id)
139            return None
140
141        new_job_id = None
142        go_ahead = (self._job_retry and
143                    self._retry_handler._should_retry_local_job(job_id))
144        if go_ahead:
145            new_job_id = self._retry_local_result(job_id, record)
146        return new_job_id
147
148    def _retry_local_result(self, job_id, record):
149        """
150        Retry a test job by id.
151
152        @param job_id: int ID of job
153        @param record: callable that records job status.
154                 prototype:
155                   record(base_job.status_log_entry)
156
157        @returns: new job_id if a job was scheduled for retry, None otherwise.
158        """
159        test = self._jobs_to_tests[job_id]
160        logging.debug('Attempting to retry job %s, test %s', job_id, test.name)
161        test.fast = False
162        new_job = self._schedule_test(
163                record=record, test=test, retry_for=job_id)
164        if new_job:
165            return new_job.id
166        return None
167
168    def test_name_from_job(self, job_id):
169        """Find the name of the test run by a job with a given job ID."""
170        if self._jobs_to_tests[job_id]:
171            return self._jobs_to_tests[job_id].name
172
173
174
175def fetch_local_suite(autotest_path, suite_predicate, afe, test_arg, remote,
176                      build=NO_BUILD, board=NO_BOARD,
177                      results_directory=None, no_experimental=False,
178                      ignore_deps=True, job_retry=True):
179    """Create a suite from the given suite predicate.
180
181    Satisfaction of dependencies is enforced by Suite.schedule() if
182    ignore_deps is False. Note that this method assumes only one host,
183    i.e. |remote|, was added to afe. Suite.schedule() will not
184    schedule a job if none of the hosts in the afe (in our case,
185    just one host |remote|) has a label that matches a requested
186    test dependency.
187
188    @param autotest_path: Absolute path to autotest (in sysroot or
189                          custom autotest directory set by --autotest_dir).
190    @param suite_predicate: callable that takes ControlData objects, and
191                            returns True on those that should be in suite
192    @param afe: afe object to schedule against (typically a directAFE)
193    @param test_arg: String. An individual TEST command line argument, e.g.
194                     'login_CryptohomeMounted' or 'suite:smoke'.
195    @param remote: String representing the IP of the remote host.
196    @param build: Build to schedule suite for.
197    @param board: Board to schedule suite for.
198    @param results_directory: Absolute path of directory to store results in.
199                              (results will be stored in subdirectory of this).
200    @param no_experimental: Skip experimental tests when scheduling a suite.
201    @param ignore_deps: If True, test dependencies will be ignored.
202    @param job_retry: If False, tests will not be retried at all.
203
204    @returns: A LocalSuite object.
205
206    """
207    fs_getter = suite.create_fs_getter(autotest_path)
208    devserver = dev_server.ImageServer('')
209    my_suite = LocalSuite.create_from_predicates(
210        [suite_predicate],
211        {provision.CROS_VERSION_PREFIX: build},
212        constants.BOARD_PREFIX + board,
213        devserver, fs_getter, afe=afe,
214        ignore_deps=ignore_deps,
215        results_dir=results_directory,
216        forgiving_parser=False,
217        job_retry=job_retry
218    )
219    if len(my_suite.tests) == 0:
220        (similarity_predicate, similarity_description) = (
221                get_predicate_for_possible_test_arg(test_arg))
222        logging.error('No test found, searching for possible tests with %s',
223                      similarity_description)
224        possible_tests = suite.find_possible_tests(fs_getter,
225                                                         similarity_predicate)
226        raise ValueError('Found no tests. Check your suite name, test name, '
227                         'or test matching wildcard.\nDid you mean any of '
228                         'following tests?\n  %s' % '\n  '.join(possible_tests))
229
230    if not ignore_deps:
231        # Log tests whose dependencies can't be satisfied.
232        labels = [label.name for label in
233                  afe.get_labels(host__hostname=remote)]
234        for test in my_suite.tests:
235            if test.experimental and no_experimental:
236                continue
237            unsatisfiable_deps = set(test.dependencies).difference(labels)
238            if unsatisfiable_deps:
239                logging.warning('%s will be skipped, unsatisfiable '
240                             'test dependencies: %s', test.name,
241                             unsatisfiable_deps)
242    return my_suite
243
244
245def _run_autoserv(command, pretend=False):
246    """Run autoserv command.
247
248    Run the autoserv command and wait on it. Log the stdout.
249    Ensure that SIGINT signals are passed along to autoserv.
250
251    @param command: the autoserv command to run.
252    @returns: exit code of the command.
253
254    """
255    if not pretend:
256        logging.debug('Running autoserv command: %s', command)
257        global _autoserv_proc
258        _autoserv_proc = subprocess.Popen(command,
259                                          stdout=subprocess.PIPE,
260                                          stderr=subprocess.STDOUT)
261        # This incantation forces unbuffered reading from stdout,
262        # so that autoserv output can be displayed to the user
263        # immediately.
264        for message in iter(_autoserv_proc.stdout.readline, b''):
265            logging.info('autoserv| %s', message.rstrip())
266
267        _autoserv_proc.wait()
268        returncode = _autoserv_proc.returncode
269        _autoserv_proc = None
270    else:
271        logging.info('Pretend mode. Would run autoserv command: %s',
272                     command)
273        returncode = 0
274    return returncode
275
276
277def run_provisioning_job(provision_label, host, info, autotest_path,
278                         results_directory, fast_mode,
279                         ssh_verbosity=0, ssh_options=None,
280                         pretend=False, autoserv_verbose=False):
281    """Shell out to autoserv to run provisioning job.
282
283    @param provision_label: Label to provision the machine to.
284    @param host: Hostname of DUT.
285    @param info: A host_info.HostInfo for the remote host.
286    @param autotest_path: Absolute path of autotest directory.
287    @param results_directory: Absolute path of directory to store results in.
288                              (results will be stored in subdirectory of this).
289    @param fast_mode: bool to use fast mode (disables slow autotest features).
290    @param ssh_verbosity: SSH verbosity level, passed along to autoserv_utils
291    @param ssh_options: Additional ssh options to be passed to autoserv_utils
292    @param pretend: If True, will print out autoserv commands rather than
293                    running them.
294    @param autoserv_verbose: If true, pass the --verbose flag to autoserv.
295
296    @returns: Absolute path of directory where results were stored.
297
298    """
299    # TODO(fdeng): When running against a local DUT, autoserv
300    # is still hitting the AFE in the lab.
301    # provision_AutoUpdate checks the current build of DUT by
302    # retrieving build info from AFE. crosbug.com/295178
303    results_directory = os.path.join(results_directory, 'results-provision')
304    _write_host_info(results_directory, _HOST_INFO_SUBDIR, host, info)
305    command = autoserv_utils.autoserv_run_job_command(
306            os.path.join(autotest_path, 'server'),
307            machines=host, job=None, verbose=autoserv_verbose,
308            results_directory=results_directory,
309            fast_mode=fast_mode, ssh_verbosity=ssh_verbosity,
310            ssh_options=ssh_options,
311            extra_args=['--provision', '--job-labels', provision_label],
312            no_console_prefix=True,
313            host_info_subdir=_HOST_INFO_SUBDIR)
314    if _run_autoserv(command, pretend) != 0:
315        raise TestThatProvisioningError('Command returns non-zero code: %s ' %
316                                        command)
317    return results_directory
318
319
320def run_job(job, host, info, autotest_path, results_directory, fast_mode,
321            id_digits=1, ssh_verbosity=0, ssh_options=None,
322            args=None, pretend=False,
323            autoserv_verbose=False):
324    """
325    Shell out to autoserv to run an individual test job.
326
327    @param job: A Job object containing the control file contents and other
328                relevent metadata for this test.
329    @param host: Hostname of DUT to run test against.
330    @param info: a host_info.HostInfo for the remote host.
331    @param autotest_path: Absolute path of autotest directory.
332    @param results_directory: Absolute path of directory to store results in.
333                              (results will be stored in subdirectory of this).
334    @param fast_mode: bool to use fast mode (disables slow autotest features).
335    @param id_digits: The minimum number of digits that job ids should be
336                      0-padded to when formatting as a string for results
337                      directory.
338    @param ssh_verbosity: SSH verbosity level, passed along to autoserv_utils
339    @param ssh_options: Additional ssh options to be passed to autoserv_utils
340    @param args: String that should be passed as args parameter to autoserv,
341                 and then ultimitely to test itself.
342    @param pretend: If True, will print out autoserv commands rather than
343                    running them.
344    @param autoserv_verbose: If true, pass the --verbose flag to autoserv.
345
346    @returns: a tuple, return code of the job and absolute path of directory
347              where results were stored.
348    """
349    with tempfile.NamedTemporaryFile() as temp_file:
350        temp_file.write(job.control_file)
351        temp_file.flush()
352        name_tail = job.name.split('/')[-1]
353        results_directory = os.path.join(results_directory,
354                                         'results-%0*d-%s' % (id_digits, job.id,
355                                                              name_tail))
356        # Drop experimental keyval in the keval file in the job result folder.
357        os.makedirs(results_directory)
358        utils.write_keyval(results_directory,
359                           {constants.JOB_EXPERIMENTAL_KEY: job.keyvals[
360                                   constants.JOB_EXPERIMENTAL_KEY]})
361        _write_host_info(results_directory, _HOST_INFO_SUBDIR, host, info)
362        extra_args = [temp_file.name]
363        if args:
364            extra_args.extend(['--args', args])
365
366        command = autoserv_utils.autoserv_run_job_command(
367                os.path.join(autotest_path, 'server'),
368                machines=host, job=job, verbose=autoserv_verbose,
369                results_directory=results_directory,
370                fast_mode=fast_mode, ssh_verbosity=ssh_verbosity,
371                ssh_options=ssh_options,
372                extra_args=extra_args,
373                no_console_prefix=True,
374                use_packaging=False,
375                host_attributes=info.attributes,
376                host_info_subdir=_HOST_INFO_SUBDIR)
377
378        code = _run_autoserv(command, pretend)
379        return code, results_directory
380
381
382def setup_local_afe():
383    """
384    Setup a local afe database and return a direct_afe object to access it.
385
386    @returns: A autotest_lib.frontend.afe.direct_afe instance.
387    """
388    # This import statement is delayed until now rather than running at
389    # module load time, because it kicks off a local sqlite :memory: backed
390    # database, and we don't need that unless we are doing a local run.
391    from autotest_lib.frontend import setup_django_lite_environment
392    from autotest_lib.frontend.afe import direct_afe
393    return direct_afe.directAFE()
394
395
396def get_predicate_for_test_arg(test):
397    """
398    Gets a suite predicte function for a given command-line argument.
399
400    @param test: String. An individual TEST command line argument, e.g.
401                         'login_CryptohomeMounted' or 'suite:smoke'
402    @returns: A (predicate, string) tuple with the necessary suite
403              predicate, and a description string of the suite that
404              this predicate will produce.
405    """
406    suitematch = re.match(_SUITE_REGEX, test)
407    name_pattern_match = re.match(r'e:(.*)', test)
408    file_pattern_match = re.match(r'f:(.*)', test)
409    if suitematch:
410        suitename = suitematch.group(1)
411        return (suite.name_in_tag_predicate(suitename),
412                'suite named %s' % suitename)
413    if name_pattern_match:
414        pattern = '^%s$' % name_pattern_match.group(1)
415        return (suite.test_name_matches_pattern_predicate(pattern),
416                'suite to match name pattern %s' % pattern)
417    if file_pattern_match:
418        pattern = '^%s$' % file_pattern_match.group(1)
419        return (suite.test_file_matches_pattern_predicate(pattern),
420                'suite to match file name pattern %s' % pattern)
421    return (suite.test_name_equals_predicate(test),
422            'job named %s' % test)
423
424
425def get_predicate_for_possible_test_arg(test):
426    """
427    Gets a suite predicte function to calculate the similarity of given test
428    and possible tests.
429
430    @param test: String. An individual TEST command line argument, e.g.
431                         'login_CryptohomeMounted' or 'suite:smoke'
432    @returns: A (predicate, string) tuple with the necessary suite
433              predicate, and a description string of the suite that
434              this predicate will produce.
435    """
436    suitematch = re.match(_SUITE_REGEX, test)
437    name_pattern_match = re.match(r'e:(.*)', test)
438    file_pattern_match = re.match(r'f:(.*)', test)
439    if suitematch:
440        suitename = suitematch.group(1)
441        return (suite.name_in_tag_similarity_predicate(suitename),
442                'suite name similar to %s' % suitename)
443    if name_pattern_match:
444        pattern = '^%s$' % name_pattern_match.group(1)
445        return (suite.test_name_similarity_predicate(pattern),
446                'job name similar to %s' % pattern)
447    if file_pattern_match:
448        pattern = '^%s$' % file_pattern_match.group(1)
449        return (suite.test_file_similarity_predicate(pattern),
450                'suite to match file name similar to %s' % pattern)
451    return (suite.test_name_similarity_predicate(test),
452            'job name similar to %s' % test)
453
454
455def add_ssh_identity(temp_directory, ssh_private_key=TEST_KEY_PATH):
456    """Add an ssh identity to the agent.
457
458    TODO (sbasi) b/26186193: Add support for test_droid and make TEST_KEY_PATH
459    not Chrome OS specific.
460
461    @param temp_directory: A directory to copy the |private key| into.
462    @param ssh_private_key: Path to the ssh private key to use for testing.
463    """
464    # Add the testing key to the current ssh agent.
465    if os.environ.has_key('SSH_AGENT_PID'):
466        # Copy the testing key to the temp directory and make it NOT
467        # world-readable. Otherwise, ssh-add complains.
468        shutil.copy(ssh_private_key, temp_directory)
469        key_copy_path = os.path.join(temp_directory,
470                                     os.path.basename(ssh_private_key))
471        os.chmod(key_copy_path, stat.S_IRUSR | stat.S_IWUSR)
472        p = subprocess.Popen(['ssh-add', key_copy_path],
473                             stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
474        p_out, _ = p.communicate()
475        for line in p_out.splitlines():
476            logging.info(line)
477    else:
478        logging.warning('There appears to be no running ssh-agent. Attempting '
479                        'to continue without running ssh-add, but ssh commands '
480                        'may fail.')
481
482
483def _auto_detect_labels(afe, remote):
484    """Automatically detect host labels and add them to the host in afe.
485
486    Note that the label of board will not be auto-detected.
487    This method assumes the host |remote| has already been added to afe.
488
489    @param afe: A direct_afe object used to interact with local afe database.
490    @param remote: The hostname of the remote device.
491
492    @returns: the detected labels as a list of strings.
493    """
494    cros_host = factory.create_host(remote)
495    labels_to_create = [label for label in cros_host.get_labels()
496                        if not label.startswith(constants.BOARD_PREFIX)]
497    labels_to_add_to_afe_host = []
498    for label in labels_to_create:
499        new_label = afe.create_label(label)
500        labels_to_add_to_afe_host.append(new_label.name)
501    hosts = afe.get_hosts(hostname=remote)
502    if not hosts:
503        raise TestThatRunError('Unexpected error: %s has not '
504                               'been added to afe.' % remote)
505    afe_host = hosts[0]
506    afe_host.add_labels(labels_to_add_to_afe_host)
507    return labels_to_add_to_afe_host
508
509
510def perform_local_run(afe, autotest_path, tests, remote, fast_mode,
511                      build=NO_BUILD, board=NO_BOARD, args=None,
512                      pretend=False, no_experimental=False,
513                      ignore_deps=True,
514                      results_directory=None, ssh_verbosity=0,
515                      ssh_options=None,
516                      autoserv_verbose=False,
517                      iterations=1,
518                      host_attributes={},
519                      job_retry=True):
520    """Perform local run of tests.
521
522    This method enforces satisfaction of test dependencies for tests that are
523    run as a part of a suite.
524
525    @param afe: A direct_afe object used to interact with local afe database.
526    @param autotest_path: Absolute path of autotest installed in sysroot or
527                          custom autotest path set by --autotest_dir.
528    @param tests: List of strings naming tests and suites to run. Suite strings
529                  should be formed like "suite:smoke".
530    @param remote: Remote hostname.
531    @param fast_mode: bool to use fast mode (disables slow autotest features).
532    @param build: String specifying build for local run.
533    @param board: String specifyinb board for local run.
534    @param args: String that should be passed as args parameter to autoserv,
535                 and then ultimitely to test itself.
536    @param pretend: If True, will print out autoserv commands rather than
537                    running them.
538    @param no_experimental: Skip experimental tests when scheduling a suite.
539    @param ignore_deps: If True, test dependencies will be ignored.
540    @param results_directory: Directory to store results in. Defaults to None,
541                              in which case results will be stored in a new
542                              subdirectory of /tmp
543    @param ssh_verbosity: SSH verbosity level, passed through to
544                          autoserv_utils.
545    @param ssh_options: Additional ssh options to be passed to autoserv_utils
546    @param autoserv_verbose: If true, pass the --verbose flag to autoserv.
547    @param iterations: int number of times to schedule tests.
548    @param host_attributes: Dict of host attributes to pass into autoserv.
549    @param job_retry: If False, tests will not be retried at all.
550
551    @returns: A list of return codes each job that has run. Or [1] if
552              provision failed prior to running any jobs.
553    """
554    args = _set_default_servo_args(args)
555    # Create host in afe, add board and build labels.
556    cros_version_label = labellib.format_keyval_label(
557        labellib.KeyvalLabel(labellib.Key.CROS_VERSION, build))
558
559    build_label = afe.create_label(cros_version_label)
560    board_label = afe.create_label(constants.BOARD_PREFIX + board)
561    labels = [build_label.name, board_label.name]
562
563    new_host = afe.create_host(remote)
564    new_host.add_labels(labels)
565    if not ignore_deps:
566        logging.info('Auto-detecting labels for %s', remote)
567        labels += _auto_detect_labels(afe, remote)
568        # Auto-detected labels may duplicate explicitly set ones.
569        labels = list(set(labels))
570
571    info = host_info.HostInfo(labels, host_attributes)
572
573    # Provision the host to |build|.
574    if build != NO_BUILD:
575        logging.info('Provisioning %s...', cros_version_label)
576        try:
577            run_provisioning_job(
578                cros_version_label,
579                remote,
580                info,
581                autotest_path,
582                results_directory,
583                fast_mode,
584                ssh_verbosity,
585                ssh_options,
586                pretend,
587                autoserv_verbose,
588            )
589        except TestThatProvisioningError as e:
590            logging.error('Provisioning %s to %s failed, tests are aborted, '
591                          'failure reason: %s',
592                          remote, cros_version_label, e)
593            return [1]
594
595    # Create suites that will be scheduled.
596    suites_and_descriptions = []
597    for test in tests:
598        (predicate, description) = get_predicate_for_test_arg(test)
599        logging.info('Fetching suite for %s...', description)
600        suite = fetch_local_suite(autotest_path, predicate, afe, test_arg=test,
601                                  remote=remote,
602                                  build=build, board=board,
603                                  results_directory=results_directory,
604                                  no_experimental=no_experimental,
605                                  ignore_deps=ignore_deps,
606                                  job_retry=job_retry)
607        suites_and_descriptions.append((suite, description))
608
609    jobs_to_suites = {}
610    null_logger = lambda log_entry, log_in_subdir=False: None
611    # Schedule the suites, looping over iterations if necessary.
612    for iteration in range(iterations):
613        if iteration > 0:
614            logging.info('Repeating scheduling for iteration %d:', iteration)
615
616        for suite, description in suites_and_descriptions:
617            logging.info('Scheduling suite for %s...', description)
618            ntests = suite.schedule(null_logger)
619            logging.debug('jobs: %s nonzero job_retries: %s',
620                          len(suite._jobs_to_tests),
621                          len([True for (job_id, test) in
622                               suite._jobs_to_tests.items()]))
623            logging.info('... scheduled %s job(s).', ntests)
624            for job in suite.jobs:
625                jobs_to_suites[job.id] = suite
626
627    if not afe.get_jobs():
628        logging.info('No jobs scheduled. End of local run.')
629        return []
630
631    last_job_id = afe.get_jobs()[-1].id
632    job_id_digits = len(str(last_job_id))
633    codes = []
634    job_queue = afe.get_jobs()
635    completed_job_ids = set()
636    while job_queue:
637      logging.info('%s jobs in job queue', len(job_queue))
638      for job in job_queue:
639          suite = jobs_to_suites.get(job.id)
640          if not suite:
641              logging.error('Job %s not run, no associated suite.', job.id)
642          else:
643              logging.debug('Running job %s of test %s',
644                            job.id, suite.test_name_from_job(job.id))
645              code, abs_dir = run_job(
646                  job,
647                  remote,
648                  info,
649                  autotest_path,
650                  results_directory,
651                  fast_mode,
652                  job_id_digits,
653                  ssh_verbosity,
654                  ssh_options,
655                  args,
656                  pretend,
657                  autoserv_verbose,
658              )
659              codes.append(code)
660              logging.debug("Code: %s, Results in %s", code, abs_dir)
661              new_id = suite.handle_local_result(job.id, abs_dir, null_logger)
662              if new_id:
663                  jobs_to_suites[new_id] = jobs_to_suites[job.id]
664          completed_job_ids.add(job.id)
665      all_jobs = afe.get_jobs(not_yet_run=True, running=True)
666      new_jobs = set(job for job in all_jobs if job.id not in completed_job_ids)
667      logging.debug('%s incomplete jobs, %s jobs total',
668                    len(new_jobs), len(all_jobs))
669      job_queue = list(new_jobs)
670    return codes
671
672
673def _set_default_servo_args(args):
674    """Add default servo arguments for backward compatibitlity.
675
676    See crbug.com/881006 for context.  Some servo related defaults were baked
677    into the autotest ServoHost code. These have now been deleted. A side effect
678    was that users of test_that relied on these defaults for some tests to work
679    magically in the chroot environment.
680
681    Current plan is to add back these defaults to test_that invocations for
682    backwards compatibility of these use cases. There is no planned removal date
683    for this hack.
684
685    @return modified args str.
686    """
687    # args is a str with whitespace separated key=value arguments.
688    # Avoid parsing args here (to avoid adding another implicit constraint on
689    # the exact args format) by adding defaults only in the obvious cases where
690    # relevant keys are entirely missing.
691    if args is None:
692        args = ''
693    if 'servo_host' not in args:
694        args += ' servo_host=localhost'
695    if 'servo_port' not in args:
696        args += ' servo_port=9999'
697    return args
698
699
700def sigint_handler(signum, stack_frame):
701    #pylint: disable-msg=C0111
702    """Handle SIGINT or SIGTERM to a local test_that run.
703
704    This handler sends a SIGINT to the running autoserv process,
705    if one is running, giving it up to 5 seconds to clean up and exit. After
706    the timeout elapses, autoserv is killed. In either case, after autoserv
707    exits then this process exits with status 1.
708    """
709    # If multiple signals arrive before handler is unset, ignore duplicates
710    if not _sigint_handler_lock.acquire(False):
711        return
712    try:
713        # Ignore future signals by unsetting handler.
714        signal.signal(signal.SIGINT, signal.SIG_IGN)
715        signal.signal(signal.SIGTERM, signal.SIG_IGN)
716
717        logging.warning('Received SIGINT or SIGTERM. Cleaning up and exiting.')
718        if _autoserv_proc:
719            logging.warning('Sending SIGINT to autoserv process. Waiting up '
720                            'to %s seconds for cleanup.',
721                            _AUTOSERV_SIGINT_TIMEOUT_SECONDS)
722            _autoserv_proc.send_signal(signal.SIGINT)
723            timed_out, _ = retry.timeout(_autoserv_proc.wait,
724                    timeout_sec=_AUTOSERV_SIGINT_TIMEOUT_SECONDS)
725            if timed_out:
726                _autoserv_proc.kill()
727                logging.warning('Timed out waiting for autoserv to handle '
728                                'SIGINT. Killed autoserv.')
729    finally:
730        _sigint_handler_lock.release() # this is not really necessary?
731        sys.exit(1)
732
733
734def create_results_directory(results_directory=None, board_name=None):
735    """Create a results directory.
736
737    If no directory is specified this method will create and return a
738    temp directory to hold results. If a directory name is specified this
739    method will create a directory at the given path, provided it doesn't
740    already exist.
741
742    @param results_directory: The path to the results_directory to create.
743
744    @return results_directory: A path to the results_directory, ready for use.
745    """
746    if results_directory is None:
747        # Create a results_directory as subdir of /tmp
748        dirname_prefix='test_that_results_'
749        if board_name is not None:
750            dirname_prefix += (board_name + '_')
751        results_directory = tempfile.mkdtemp(prefix=dirname_prefix)
752    else:
753        # Delete results_directory if it already exists.
754        try:
755            shutil.rmtree(results_directory)
756        except OSError as e:
757            if e.errno != errno.ENOENT:
758                raise
759
760        # Create results_directory if it does not exist
761        try:
762            os.makedirs(results_directory)
763        except OSError as e:
764            if e.errno != errno.EEXIST:
765                raise
766    return results_directory
767
768def generate_report(directory,
769                    whitelist_chrome_crashes=False,
770                    just_status_code=False, html_report=False):
771    """Parse the test result files in the given directory into a report
772
773    @param directory: string, the absolute path of the directory to look in
774    @param whitelist_chrome_crashes: boolean, ignore Chrome crashes in the
775    report. Default: False, report Chrome crashes.
776    @param just_status_code: boolean, skip the report and only parse the files
777    to determine whether there were failures. Default: False, generate report.
778    """
779    test_report_command = [os.path.join(os.path.dirname(__file__),
780                                        'generate_test_report')]
781    # Experimental test results do not influence the exit code.
782    test_report_command.append('--ignore_experimental_tests')
783    if html_report:
784        test_report_command.append('--html')
785        test_report_command.append('--html-report-dir=%s' % directory)
786    if whitelist_chrome_crashes:
787        test_report_command.append('--whitelist_chrome_crashes')
788    if just_status_code:
789        test_report_command.append('--just_status_code')
790    test_report_command.append(directory)
791    status_code = subprocess.call(test_report_command)
792    if not just_status_code:
793        with open(os.path.join(directory, 'test_report.log'),
794                  'w') as report_log:
795            subprocess.call(test_report_command, stdout=report_log)
796    return status_code
797
798
799def perform_run_from_autotest_root(autotest_path, argv, tests, remote,
800                                   build=NO_BUILD, board=NO_BOARD, args=None,
801                                   pretend=False, no_experimental=False,
802                                   ignore_deps=True,
803                                   results_directory=None, ssh_verbosity=0,
804                                   ssh_options=None,
805                                   iterations=1, fast_mode=False, debug=False,
806                                   whitelist_chrome_crashes=False,
807                                   host_attributes={}, job_retry=True):
808    """
809    Perform a test_that run, from the |autotest_path|.
810
811    This function is to be called from test_that/test_droid's main() script,
812    when tests are executed from the |autotest_path|. It handles all stages
813    of a test run that come after the bootstrap into |autotest_path|.
814
815    @param autotest_path: Full absolute path to the autotest root directory.
816    @param argv: The arguments list, as passed to main(...)
817    @param tests: List of strings naming tests and suites to run. Suite strings
818                  should be formed like "suite:smoke".
819    @param remote: Remote hostname.
820    @param build: String specifying build for local run.
821    @param board: String specifying board for local run.
822    @param args: String that should be passed as args parameter to autoserv,
823                 and then ultimitely to test itself.
824    @param pretend: If True, will print out autoserv commands rather than
825                    running them.
826    @param no_experimental: Skip experimental tests when scheduling a suite.
827    @param ignore_deps: If True, test dependencies will be ignored.
828    @param results_directory: Directory to store results in. Defaults to None,
829                              in which case results will be stored in a new
830                              subdirectory of /tmp
831    @param ssh_verbosity: SSH verbosity level, passed through to
832                          autoserv_utils.
833    @param ssh_options: Additional ssh options to be passed to autoserv_utils
834    @param autoserv_verbose: If true, pass the --verbose flag to autoserv.
835    @param iterations: int number of times to schedule tests.
836    @param fast_mode: bool to use fast mode (disables slow autotest features).
837    @param debug: Logging and autoserv verbosity.
838    @param whitelist_chrome_crashes: If True, whitelist chrome crashes.
839    @param host_attributes: Dict of host attributes to pass into autoserv.
840    @param job_retry: If False, tests will not be retried at all.
841
842    @return: A return code that test_that should exit with.
843    """
844    if results_directory is None or not os.path.exists(results_directory):
845        raise ValueError('Expected valid results directory, got %s' %
846                          results_directory)
847
848    logging_manager.configure_logging(
849            server_logging_config.ServerLoggingConfig(),
850            results_dir=results_directory,
851            use_console=True,
852            verbose=debug,
853            debug_log_name='test_that')
854    logging.info('Began logging to %s', results_directory)
855
856    logging.debug('test_that command line was: %s', argv)
857
858    signal.signal(signal.SIGINT, sigint_handler)
859    signal.signal(signal.SIGTERM, sigint_handler)
860
861    afe = setup_local_afe()
862    codes = perform_local_run(afe, autotest_path, tests, remote, fast_mode,
863                      build, board,
864                      args=args,
865                      pretend=pretend,
866                      no_experimental=no_experimental,
867                      ignore_deps=ignore_deps,
868                      results_directory=results_directory,
869                      ssh_verbosity=ssh_verbosity,
870                      ssh_options=ssh_options,
871                      autoserv_verbose=debug,
872                      iterations=iterations,
873                      host_attributes=host_attributes,
874                      job_retry=job_retry)
875    if pretend:
876        logging.info('Finished pretend run. Exiting.')
877        return 0
878
879    final_result = generate_report(
880        results_directory,
881        whitelist_chrome_crashes=whitelist_chrome_crashes, html_report=True)
882    try:
883        os.unlink(_LATEST_RESULTS_DIRECTORY)
884    except OSError:
885        pass
886    link_target = os.path.relpath(results_directory,
887                                  os.path.dirname(_LATEST_RESULTS_DIRECTORY))
888    if any(codes):
889        logging.error('Autoserv encountered unexpected errors '
890                      'when executing jobs.')
891        final_result = final_result or 1
892    os.symlink(link_target, _LATEST_RESULTS_DIRECTORY)
893    logging.info('Finished running tests. Results can be found in %s or %s',
894                 results_directory, _LATEST_RESULTS_DIRECTORY)
895    return final_result
896
897
898def _write_host_info(results_dir, host_info_subdir, hostname, info):
899    """ Write HostInfo to a FileStore to be used by autoserv.
900
901    @param results_dir: Path to he results directory.
902    @param host_info_subdir: Subdirectory of results directory for host info.
903    @param hostname: Hostname passed into autoserv.
904    @param info: hosts.HostInfo to write.
905    """
906    d = os.path.join(results_dir, host_info_subdir)
907    os.makedirs(d)
908    store = file_store.FileStore(os.path.join(d, '%s.store' % hostname))
909    store.commit(info)