• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import logging
6from autotest_lib.client.bin import utils
7from autotest_lib.client.common_lib import error
8from autotest_lib.server import test, autotest
9
10class desktopui_CrashyRebootServer(test.test):
11    """Validate logic for mitigating too-crashy UI.
12
13    If the UI crashes too much too fast, the device will eventually
14    reboot to attempt to mitigate the problem. If the device
15    determines that it's already tried that once, it will shut down
16    the UI and remain up.
17
18    This test deploys the client test desktopui_CrashyReboot in order
19    to drive the device into the desired states.
20    """
21    version = 1
22
23    CRASHY_DEVICE_TIMEOUT_SECONDS = 120
24    CLIENT_TEST = 'desktopui_CrashyReboot'
25
26    def run_once(self, host=None):
27        host.run('rm -f /var/lib/ui/reboot-timestamps')
28        boot_id = host.get_boot_id()
29
30        # Run a client-side test that crashes the UI a bunch, and
31        # expect a reboot.  We need to run this test in the background in
32        # order to prevent the reboot from causing autotest to auto-fail
33        # the entire test. This means we also need to handle collecting
34        # and parsing results manually if it doesn't work.
35        logging.info('CrashyRebootServer: start client test')
36        tag = 'reboot'
37        client_at = autotest.Autotest(host)
38        client_at.run_test(self.CLIENT_TEST, expect_reboot=True, tag='reboot',
39                           background=True)
40
41        logging.info('Client test now running in background.')
42        # Prepare for result gathering.
43        collector = autotest.log_collector(host, None, '.')
44        host.job.add_client_log(host.hostname,
45                                collector.client_results_dir,
46                                collector.server_results_dir)
47        job_record_context = host.job.get_record_context()
48
49        logging.info('Waiting for host to go down.')
50        if not host.wait_down(timeout=self.CRASHY_DEVICE_TIMEOUT_SECONDS,
51                              old_boot_id=boot_id):
52            # Gather results to determine why device didn't reboot.
53            collector.collect_client_job_results()
54            collector.remove_redundant_client_logs()
55            host.job.remove_client_log(host.hostname,
56                                       collector.client_results_dir,
57                                       collector.server_results_dir)
58            job_record_context.restore()
59            raise error.TestError('Host should have rebooted!')
60
61        logging.info('Waiting for host to come back up.')
62        try:
63            # wait_up() issues an ssh connection attempt and then spends
64            # the entire given timeout waiting for it to succeed. If it
65            # does this before the device is ready to accept ssh
66            # connections, it will decide that the device never came up,
67            # even if it is ready and waiting. To combat this, loop with
68            # a short timeout.
69            utils.poll_for_condition(lambda: host.wait_up(5),
70                                     timeout=self.CRASHY_DEVICE_TIMEOUT_SECONDS)
71        except utils.TimeoutError:
72            raise error.TestError('Host never came back!')
73
74        # NB: If we change the reboot-attempt threshold in
75        # /etc/init/ui-respawn.conf to be >1, this will start failing
76        # and need to be updated.
77        client_at.run_test(self.CLIENT_TEST, expect_reboot=False)
78