• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env vpython
2# Copyright 2016 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""A script to recover devices in a known bad state."""
7
8import argparse
9import glob
10import logging
11import os
12import signal
13import sys
14
15import psutil
16
17if __name__ == '__main__':
18  sys.path.append(
19      os.path.abspath(os.path.join(os.path.dirname(__file__),
20                                   '..', '..', '..')))
21from devil.android import device_blacklist
22from devil.android import device_errors
23from devil.android import device_utils
24from devil.android.sdk import adb_wrapper
25from devil.android.tools import device_status
26from devil.android.tools import script_common
27from devil.utils import logging_common
28from devil.utils import lsusb
29# TODO(jbudorick): Resolve this after experimenting w/ disabling the USB reset.
30from devil.utils import reset_usb  # pylint: disable=unused-import
31
32logger = logging.getLogger(__name__)
33
34from py_utils import modules_util
35
36
37# Script depends on features from psutil version 2.0 or higher.
38modules_util.RequireVersion(psutil, '2.0')
39
40
41def KillAllAdb():
42  def get_all_adb():
43    for p in psutil.process_iter():
44      try:
45        # Retrieve all required process infos at once.
46        pinfo = p.as_dict(attrs=['pid', 'name', 'cmdline'])
47        if pinfo['name'] == 'adb':
48          pinfo['cmdline'] = ' '.join(pinfo['cmdline'])
49          yield p, pinfo
50      except (psutil.NoSuchProcess, psutil.AccessDenied):
51        pass
52
53  for sig in [signal.SIGTERM, signal.SIGQUIT, signal.SIGKILL]:
54    for p, pinfo in get_all_adb():
55      try:
56        pinfo['signal'] = sig
57        logger.info('kill %(signal)s %(pid)s (%(name)s [%(cmdline)s])', pinfo)
58        p.send_signal(sig)
59      except (psutil.NoSuchProcess, psutil.AccessDenied):
60        pass
61  for _, pinfo in get_all_adb():
62    try:
63      logger.error('Unable to kill %(pid)s (%(name)s [%(cmdline)s])', pinfo)
64    except (psutil.NoSuchProcess, psutil.AccessDenied):
65      pass
66
67
68def TryAuth(device):
69  """Uses anything in ~/.android/ that looks like a key to auth with the device.
70
71  Args:
72    device: The DeviceUtils device to attempt to auth.
73
74  Returns:
75    True if device successfully authed.
76  """
77  possible_keys = glob.glob(os.path.join(adb_wrapper.ADB_HOST_KEYS_DIR, '*key'))
78  if len(possible_keys) <= 1:
79    logger.warning(
80        'Only %d ADB keys available. Not forcing auth.', len(possible_keys))
81    return False
82
83  KillAllAdb()
84  adb_wrapper.AdbWrapper.StartServer(keys=possible_keys)
85  new_state = device.adb.GetState()
86  if new_state != 'device':
87    logger.error(
88        'Auth failed. Device %s still stuck in %s.', str(device), new_state)
89    return False
90
91  # It worked! Now register the host's default ADB key on the device so we don't
92  # have to do all that again.
93  pub_key = os.path.join(adb_wrapper.ADB_HOST_KEYS_DIR, 'adbkey.pub')
94  if not os.path.exists(pub_key):  # This really shouldn't happen.
95    logger.error('Default ADB key not available at %s.', pub_key)
96    return False
97
98  with open(pub_key) as f:
99    pub_key_contents = f.read()
100  try:
101    device.WriteFile(adb_wrapper.ADB_KEYS_FILE, pub_key_contents, as_root=True)
102  except (device_errors.CommandTimeoutError,
103          device_errors.CommandFailedError,
104          device_errors.DeviceUnreachableError):
105    logger.exception('Unable to write default ADB key to %s.', str(device))
106    return False
107  return True
108
109
110def RecoverDevice(device, blacklist, should_reboot=lambda device: True):
111  if device_status.IsBlacklisted(device.adb.GetDeviceSerial(),
112                                 blacklist):
113    logger.debug('%s is blacklisted, skipping recovery.', str(device))
114    return
115
116  if device.adb.GetState() == 'unauthorized' and TryAuth(device):
117    logger.info('Successfully authed device %s!', str(device))
118    return
119
120  if should_reboot(device):
121    try:
122      device.WaitUntilFullyBooted(retries=0)
123    except (device_errors.CommandTimeoutError,
124            device_errors.CommandFailedError,
125            device_errors.DeviceUnreachableError):
126      logger.exception('Failure while waiting for %s. '
127                       'Attempting to recover.', str(device))
128    try:
129      try:
130        device.Reboot(block=False, timeout=5, retries=0)
131      except device_errors.CommandTimeoutError:
132        logger.warning('Timed out while attempting to reboot %s normally.'
133                       'Attempting alternative reboot.', str(device))
134        # The device drops offline before we can grab the exit code, so
135        # we don't check for status.
136        try:
137          device.adb.Root()
138        finally:
139          # We are already in a failure mode, attempt to reboot regardless of
140          # what device.adb.Root() returns. If the sysrq reboot fails an
141          # exception willbe thrown at that level.
142          device.adb.Shell('echo b > /proc/sysrq-trigger', expect_status=None,
143                           timeout=5, retries=0)
144    except (device_errors.CommandFailedError,
145            device_errors.DeviceUnreachableError):
146      logger.exception('Failed to reboot %s.', str(device))
147      if blacklist:
148        blacklist.Extend([device.adb.GetDeviceSerial()],
149                         reason='reboot_failure')
150    except device_errors.CommandTimeoutError:
151      logger.exception('Timed out while rebooting %s.', str(device))
152      if blacklist:
153        blacklist.Extend([device.adb.GetDeviceSerial()],
154                         reason='reboot_timeout')
155
156    try:
157      device.WaitUntilFullyBooted(
158          retries=0, timeout=device.REBOOT_DEFAULT_TIMEOUT)
159    except (device_errors.CommandFailedError,
160            device_errors.DeviceUnreachableError):
161      logger.exception('Failure while waiting for %s.', str(device))
162      if blacklist:
163        blacklist.Extend([device.adb.GetDeviceSerial()],
164                         reason='reboot_failure')
165    except device_errors.CommandTimeoutError:
166      logger.exception('Timed out while waiting for %s.', str(device))
167      if blacklist:
168        blacklist.Extend([device.adb.GetDeviceSerial()],
169                         reason='reboot_timeout')
170
171
172def RecoverDevices(devices, blacklist, enable_usb_reset=False):
173  """Attempts to recover any inoperable devices in the provided list.
174
175  Args:
176    devices: The list of devices to attempt to recover.
177    blacklist: The current device blacklist, which will be used then
178      reset.
179  """
180
181  statuses = device_status.DeviceStatus(devices, blacklist)
182
183  should_restart_usb = set(
184      status['serial'] for status in statuses
185      if (not status['usb_status']
186          or status['adb_status'] in ('offline', 'missing')))
187  should_restart_adb = should_restart_usb.union(set(
188      status['serial'] for status in statuses
189      if status['adb_status'] == 'unauthorized'))
190  should_reboot_device = should_restart_usb.union(set(
191      status['serial'] for status in statuses
192      if status['blacklisted']))
193
194  logger.debug('Should restart USB for:')
195  for d in should_restart_usb:
196    logger.debug('  %s', d)
197  logger.debug('Should restart ADB for:')
198  for d in should_restart_adb:
199    logger.debug('  %s', d)
200  logger.debug('Should reboot:')
201  for d in should_reboot_device:
202    logger.debug('  %s', d)
203
204  if blacklist:
205    blacklist.Reset()
206
207  if should_restart_adb:
208    KillAllAdb()
209    adb_wrapper.AdbWrapper.StartServer()
210
211  for serial in should_restart_usb:
212    try:
213      # TODO(crbug.com/642194): Resetting may be causing more harm
214      # (specifically, kernel panics) than it does good.
215      if enable_usb_reset:
216        reset_usb.reset_android_usb(serial)
217      else:
218        logger.warning('USB reset disabled for %s (crbug.com/642914)',
219                       serial)
220    except IOError:
221      logger.exception('Unable to reset USB for %s.', serial)
222      if blacklist:
223        blacklist.Extend([serial], reason='USB failure')
224    except device_errors.DeviceUnreachableError:
225      logger.exception('Unable to reset USB for %s.', serial)
226      if blacklist:
227        blacklist.Extend([serial], reason='offline')
228
229  device_utils.DeviceUtils.parallel(devices).pMap(
230      RecoverDevice, blacklist,
231      should_reboot=lambda device: device.serial in should_reboot_device)
232
233
234def main():
235  parser = argparse.ArgumentParser()
236  logging_common.AddLoggingArguments(parser)
237  script_common.AddEnvironmentArguments(parser)
238  parser.add_argument('--blacklist-file', help='Device blacklist JSON file.')
239  parser.add_argument('--known-devices-file', action='append', default=[],
240                      dest='known_devices_files',
241                      help='Path to known device lists.')
242  parser.add_argument('--enable-usb-reset', action='store_true',
243                      help='Reset USB if necessary.')
244
245  args = parser.parse_args()
246  logging_common.InitializeLogging(args)
247  script_common.InitializeEnvironment(args)
248
249  blacklist = (device_blacklist.Blacklist(args.blacklist_file)
250               if args.blacklist_file
251               else None)
252
253  expected_devices = device_status.GetExpectedDevices(args.known_devices_files)
254  usb_devices = set(lsusb.get_android_devices())
255  devices = [device_utils.DeviceUtils(s)
256             for s in expected_devices.union(usb_devices)]
257
258  RecoverDevices(devices, blacklist, enable_usb_reset=args.enable_usb_reset)
259
260
261if __name__ == '__main__':
262  sys.exit(main())
263