• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright 2016 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""A script to recover devices in a known bad state."""
7
8import argparse
9import logging
10import os
11import psutil
12import signal
13import sys
14
15if __name__ == '__main__':
16  sys.path.append(
17      os.path.abspath(os.path.join(os.path.dirname(__file__),
18                                   '..', '..', '..')))
19from devil import devil_env
20from devil.android import device_blacklist
21from devil.android import device_errors
22from devil.android import device_utils
23from devil.android.tools import device_status
24from devil.utils import lsusb
25# TODO(jbudorick): Resolve this after experimenting w/ disabling the USB reset.
26from devil.utils import reset_usb  # pylint: disable=unused-import
27from devil.utils import run_tests_helper
28
29
30def KillAllAdb():
31  def get_all_adb():
32    for p in psutil.process_iter():
33      try:
34        if 'adb' in p.name:
35          yield p
36      except (psutil.NoSuchProcess, psutil.AccessDenied):
37        pass
38
39  for sig in [signal.SIGTERM, signal.SIGQUIT, signal.SIGKILL]:
40    for p in get_all_adb():
41      try:
42        logging.info('kill %d %d (%s [%s])', sig, p.pid, p.name,
43                     ' '.join(p.cmdline))
44        p.send_signal(sig)
45      except (psutil.NoSuchProcess, psutil.AccessDenied):
46        pass
47  for p in get_all_adb():
48    try:
49      logging.error('Unable to kill %d (%s [%s])', p.pid, p.name,
50                    ' '.join(p.cmdline))
51    except (psutil.NoSuchProcess, psutil.AccessDenied):
52      pass
53
54
55def RecoverDevice(device, blacklist, should_reboot=lambda device: True):
56  if device_status.IsBlacklisted(device.adb.GetDeviceSerial(),
57                                 blacklist):
58    logging.debug('%s is blacklisted, skipping recovery.', str(device))
59    return
60
61  if should_reboot(device):
62    try:
63      device.WaitUntilFullyBooted(retries=0)
64    except (device_errors.CommandTimeoutError,
65            device_errors.CommandFailedError):
66      logging.exception('Failure while waiting for %s. '
67                        'Attempting to recover.', str(device))
68    try:
69      try:
70        device.Reboot(block=False, timeout=5, retries=0)
71      except device_errors.CommandTimeoutError:
72        logging.warning('Timed out while attempting to reboot %s normally.'
73                        'Attempting alternative reboot.', str(device))
74        # The device drops offline before we can grab the exit code, so
75        # we don't check for status.
76        device.adb.Root()
77        device.adb.Shell('echo b > /proc/sysrq-trigger', expect_status=None,
78                         timeout=5, retries=0)
79    except device_errors.CommandFailedError:
80      logging.exception('Failed to reboot %s.', str(device))
81      if blacklist:
82        blacklist.Extend([device.adb.GetDeviceSerial()],
83                         reason='reboot_failure')
84    except device_errors.CommandTimeoutError:
85      logging.exception('Timed out while rebooting %s.', str(device))
86      if blacklist:
87        blacklist.Extend([device.adb.GetDeviceSerial()],
88                         reason='reboot_timeout')
89
90    try:
91      device.WaitUntilFullyBooted(
92          retries=0, timeout=device.REBOOT_DEFAULT_TIMEOUT)
93    except device_errors.CommandFailedError:
94      logging.exception('Failure while waiting for %s.', str(device))
95      if blacklist:
96        blacklist.Extend([device.adb.GetDeviceSerial()],
97                         reason='reboot_failure')
98    except device_errors.CommandTimeoutError:
99      logging.exception('Timed out while waiting for %s.', str(device))
100      if blacklist:
101        blacklist.Extend([device.adb.GetDeviceSerial()],
102                         reason='reboot_timeout')
103
104
105def RecoverDevices(devices, blacklist, enable_usb_reset=False):
106  """Attempts to recover any inoperable devices in the provided list.
107
108  Args:
109    devices: The list of devices to attempt to recover.
110    blacklist: The current device blacklist, which will be used then
111      reset.
112  """
113
114  statuses = device_status.DeviceStatus(devices, blacklist)
115
116  should_restart_usb = set(
117      status['serial'] for status in statuses
118      if (not status['usb_status']
119          or status['adb_status'] in ('offline', 'missing')))
120  should_restart_adb = should_restart_usb.union(set(
121      status['serial'] for status in statuses
122      if status['adb_status'] == 'unauthorized'))
123  should_reboot_device = should_restart_adb.union(set(
124      status['serial'] for status in statuses
125      if status['blacklisted']))
126
127  logging.debug('Should restart USB for:')
128  for d in should_restart_usb:
129    logging.debug('  %s', d)
130  logging.debug('Should restart ADB for:')
131  for d in should_restart_adb:
132    logging.debug('  %s', d)
133  logging.debug('Should reboot:')
134  for d in should_reboot_device:
135    logging.debug('  %s', d)
136
137  if blacklist:
138    blacklist.Reset()
139
140  if should_restart_adb:
141    KillAllAdb()
142  for serial in should_restart_usb:
143    try:
144      # TODO(crbug.com/642194): Resetting may be causing more harm
145      # (specifically, kernel panics) than it does good.
146      if enable_usb_reset:
147        reset_usb.reset_android_usb(serial)
148      else:
149        logging.warning('USB reset disabled for %s (crbug.com/642914)',
150                        serial)
151    except IOError:
152      logging.exception('Unable to reset USB for %s.', serial)
153      if blacklist:
154        blacklist.Extend([serial], reason='USB failure')
155    except device_errors.DeviceUnreachableError:
156      logging.exception('Unable to reset USB for %s.', serial)
157      if blacklist:
158        blacklist.Extend([serial], reason='offline')
159
160  device_utils.DeviceUtils.parallel(devices).pMap(
161      RecoverDevice, blacklist,
162      should_reboot=lambda device: device in should_reboot_device)
163
164
165def main():
166  parser = argparse.ArgumentParser()
167  parser.add_argument('--adb-path',
168                      help='Absolute path to the adb binary to use.')
169  parser.add_argument('--blacklist-file', help='Device blacklist JSON file.')
170  parser.add_argument('--known-devices-file', action='append', default=[],
171                      dest='known_devices_files',
172                      help='Path to known device lists.')
173  parser.add_argument('--enable-usb-reset', action='store_true',
174                      help='Reset USB if necessary.')
175  parser.add_argument('-v', '--verbose', action='count', default=1,
176                      help='Log more information.')
177
178  args = parser.parse_args()
179  run_tests_helper.SetLogLevel(args.verbose)
180
181  devil_dynamic_config = devil_env.EmptyConfig()
182  if args.adb_path:
183    devil_dynamic_config['dependencies'].update(
184        devil_env.LocalConfigItem(
185            'adb', devil_env.GetPlatform(), args.adb_path))
186  devil_env.config.Initialize(configs=[devil_dynamic_config])
187
188  blacklist = (device_blacklist.Blacklist(args.blacklist_file)
189               if args.blacklist_file
190               else None)
191
192  expected_devices = device_status.GetExpectedDevices(args.known_devices_files)
193  usb_devices = set(lsusb.get_android_devices())
194  devices = [device_utils.DeviceUtils(s)
195             for s in expected_devices.union(usb_devices)]
196
197  RecoverDevices(devices, blacklist, enable_usb_reset=args.enable_usb_reset)
198
199
200if __name__ == '__main__':
201  sys.exit(main())
202