• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2#
3# Copyright (c) 2013 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Provisions Android devices with settings required for bots.
8
9Usage:
10  ./provision_devices.py [-d <device serial number>]
11"""
12
13import argparse
14import datetime
15import json
16import logging
17import os
18import posixpath
19import re
20import sys
21import time
22
23# Import _strptime before threaded code. datetime.datetime.strptime is
24# threadsafe except for the initial import of the _strptime module.
25# See crbug.com/584730 and https://bugs.python.org/issue7980.
26import _strptime  # pylint: disable=unused-import
27
28if __name__ == '__main__':
29  sys.path.append(
30      os.path.abspath(os.path.join(os.path.dirname(__file__),
31                                   '..', '..', '..')))
32
33from devil import devil_env
34from devil.android import battery_utils
35from devil.android import device_blacklist
36from devil.android import device_errors
37from devil.android import device_temp_file
38from devil.android import device_utils
39from devil.android import settings
40from devil.android.constants import chrome
41from devil.android.sdk import adb_wrapper
42from devil.android.sdk import intent
43from devil.android.sdk import keyevent
44from devil.android.sdk import version_codes
45from devil.android.tools import script_common
46from devil.constants import exit_codes
47from devil.utils import run_tests_helper
48from devil.utils import timeout_retry
49
50logger = logging.getLogger(__name__)
51
52_SYSTEM_APP_DIRECTORIES = ['/system/app/', '/system/priv-app/']
53_SYSTEM_WEBVIEW_NAMES = ['webview', 'WebViewGoogle']
54_CHROME_PACKAGE_REGEX = re.compile('.*chrom.*')
55_TOMBSTONE_REGEX = re.compile('tombstone.*')
56
57
58class _DEFAULT_TIMEOUTS(object):
59  # L can take a while to reboot after a wipe.
60  LOLLIPOP = 600
61  PRE_LOLLIPOP = 180
62
63  HELP_TEXT = '{}s on L, {}s on pre-L'.format(LOLLIPOP, PRE_LOLLIPOP)
64
65
66class ProvisionStep(object):
67
68  def __init__(self, cmd, reboot=False):
69    self.cmd = cmd
70    self.reboot = reboot
71
72
73def ProvisionDevices(
74    devices,
75    blacklist_file,
76    adb_key_files=None,
77    disable_location=False,
78    disable_mock_location=False,
79    disable_network=False,
80    disable_system_chrome=False,
81    emulators=False,
82    enable_java_debug=False,
83    max_battery_temp=None,
84    min_battery_level=None,
85    output_device_blacklist=None,
86    reboot_timeout=None,
87    remove_system_webview=False,
88    system_app_remove_list=None,
89    wipe=True):
90  blacklist = (device_blacklist.Blacklist(blacklist_file)
91               if blacklist_file
92               else None)
93  system_app_remove_list = system_app_remove_list or []
94  try:
95    devices = script_common.GetDevices(devices, blacklist)
96  except device_errors.NoDevicesError:
97    logging.error('No available devices to provision.')
98    if blacklist:
99      logging.error('Local device blacklist: %s', blacklist.Read())
100    raise
101  devices = [d for d in devices
102             if not emulators or d.adb.is_emulator]
103  parallel_devices = device_utils.DeviceUtils.parallel(devices)
104
105  steps = []
106  if wipe:
107    steps += [ProvisionStep(lambda d: Wipe(d, adb_key_files), reboot=True)]
108  steps += [ProvisionStep(
109      lambda d: SetProperties(d, enable_java_debug, disable_location,
110                              disable_mock_location),
111      reboot=not emulators)]
112
113  if disable_network:
114    steps.append(ProvisionStep(DisableNetwork))
115
116  if disable_system_chrome:
117    steps.append(ProvisionStep(DisableSystemChrome))
118
119  if max_battery_temp:
120    steps.append(ProvisionStep(
121        lambda d: WaitForTemperature(d, max_battery_temp)))
122
123  if min_battery_level:
124    steps.append(ProvisionStep(
125        lambda d: WaitForCharge(d, min_battery_level)))
126
127  if remove_system_webview:
128    system_app_remove_list.extend(_SYSTEM_WEBVIEW_NAMES)
129
130  if system_app_remove_list:
131    steps.append(ProvisionStep(
132        lambda d: RemoveSystemApps(d, system_app_remove_list)))
133
134  steps.append(ProvisionStep(SetDate))
135  steps.append(ProvisionStep(CheckExternalStorage))
136
137  parallel_devices.pMap(ProvisionDevice, steps, blacklist, reboot_timeout)
138
139  blacklisted_devices = blacklist.Read() if blacklist else []
140  if output_device_blacklist:
141    with open(output_device_blacklist, 'w') as f:
142      json.dump(blacklisted_devices, f)
143  if all(d in blacklisted_devices for d in devices):
144    raise device_errors.NoDevicesError
145  return 0
146
147
148def ProvisionDevice(device, steps, blacklist, reboot_timeout=None):
149  try:
150    if not reboot_timeout:
151      if device.build_version_sdk >= version_codes.LOLLIPOP:
152        reboot_timeout = _DEFAULT_TIMEOUTS.LOLLIPOP
153      else:
154        reboot_timeout = _DEFAULT_TIMEOUTS.PRE_LOLLIPOP
155
156    for step in steps:
157      try:
158        device.WaitUntilFullyBooted(timeout=reboot_timeout, retries=0)
159      except device_errors.CommandTimeoutError:
160        logger.error('Device did not finish booting. Will try to reboot.')
161        device.Reboot(timeout=reboot_timeout)
162      step.cmd(device)
163      if step.reboot:
164        device.Reboot(False, retries=0)
165        device.adb.WaitForDevice()
166
167  except device_errors.CommandTimeoutError:
168    logger.exception('Timed out waiting for device %s. Adding to blacklist.',
169                     str(device))
170    if blacklist:
171      blacklist.Extend([str(device)], reason='provision_timeout')
172
173  except device_errors.CommandFailedError:
174    logger.exception('Failed to provision device %s. Adding to blacklist.',
175                     str(device))
176    if blacklist:
177      blacklist.Extend([str(device)], reason='provision_failure')
178
179
180def Wipe(device, adb_key_files=None):
181  if (device.IsUserBuild() or
182      device.build_version_sdk >= version_codes.MARSHMALLOW):
183    WipeChromeData(device)
184
185    package = "com.google.android.gms"
186    version_name = device.GetApplicationVersion(package)
187    logger.info("Version name for %s is %s", package, version_name)
188  else:
189    WipeDevice(device, adb_key_files)
190
191
192def WipeChromeData(device):
193  """Wipes chrome specific data from device
194
195  (1) uninstall any app whose name matches *chrom*, except
196      com.android.chrome, which is the chrome stable package. Doing so also
197      removes the corresponding dirs under /data/data/ and /data/app/
198  (2) remove any dir under /data/app-lib/ whose name matches *chrom*
199  (3) remove any files under /data/tombstones/ whose name matches "tombstone*"
200  (4) remove /data/local.prop if there is any
201  (5) remove /data/local/chrome-command-line if there is any
202  (6) remove anything under /data/local/.config/ if the dir exists
203      (this is telemetry related)
204  (7) remove anything under /data/local/tmp/
205
206  Arguments:
207    device: the device to wipe
208  """
209  try:
210    if device.IsUserBuild():
211      _UninstallIfMatch(device, _CHROME_PACKAGE_REGEX,
212                        chrome.PACKAGE_INFO['chrome_stable'].package)
213      device.RunShellCommand('rm -rf %s/*' % device.GetExternalStoragePath(),
214                             shell=True, check_return=True)
215      device.RunShellCommand('rm -rf /data/local/tmp/*',
216                             shell=True, check_return=True)
217    else:
218      device.EnableRoot()
219      _UninstallIfMatch(device, _CHROME_PACKAGE_REGEX,
220                        chrome.PACKAGE_INFO['chrome_stable'].package)
221      _WipeUnderDirIfMatch(device, '/data/app-lib/', _CHROME_PACKAGE_REGEX)
222      _WipeUnderDirIfMatch(device, '/data/tombstones/', _TOMBSTONE_REGEX)
223
224      _WipeFileOrDir(device, '/data/local.prop')
225      _WipeFileOrDir(device, '/data/local/chrome-command-line')
226      _WipeFileOrDir(device, '/data/local/.config/')
227      _WipeFileOrDir(device, '/data/local/tmp/')
228      device.RunShellCommand('rm -rf %s/*' % device.GetExternalStoragePath(),
229                             shell=True, check_return=True)
230  except device_errors.CommandFailedError:
231    logger.exception('Possible failure while wiping the device. '
232                     'Attempting to continue.')
233
234
235def _UninstallIfMatch(device, pattern, app_to_keep):
236  installed_packages = device.RunShellCommand(
237      ['pm', 'list', 'packages'], check_return=True)
238  installed_system_packages = [
239      pkg.split(':')[1] for pkg in device.RunShellCommand(
240          ['pm', 'list', 'packages', '-s'], check_return=True)]
241  for package_output in installed_packages:
242    package = package_output.split(":")[1]
243    if pattern.match(package) and not package == app_to_keep:
244      if not device.IsUserBuild() or package not in installed_system_packages:
245        device.Uninstall(package)
246
247
248def _WipeUnderDirIfMatch(device, path, pattern):
249  for filename in device.ListDirectory(path):
250    if pattern.match(filename):
251      _WipeFileOrDir(device, posixpath.join(path, filename))
252
253
254def _WipeFileOrDir(device, path):
255  if device.PathExists(path):
256    device.RunShellCommand(['rm', '-rf', path], check_return=True)
257
258
259def WipeDevice(device, adb_key_files):
260  """Wipes data from device, keeping only the adb_keys for authorization.
261
262  After wiping data on a device that has been authorized, adb can still
263  communicate with the device, but after reboot the device will need to be
264  re-authorized because the adb keys file is stored in /data/misc/adb/.
265  Thus, adb_keys file is rewritten so the device does not need to be
266  re-authorized.
267
268  Arguments:
269    device: the device to wipe
270  """
271  try:
272    device.EnableRoot()
273    device_authorized = device.FileExists(adb_wrapper.ADB_KEYS_FILE)
274    if device_authorized:
275      adb_keys = device.ReadFile(adb_wrapper.ADB_KEYS_FILE,
276                                 as_root=True).splitlines()
277    device.RunShellCommand(['wipe', 'data'],
278                           as_root=True, check_return=True)
279    device.adb.WaitForDevice()
280
281    if device_authorized:
282      adb_keys_set = set(adb_keys)
283      for adb_key_file in adb_key_files or []:
284        try:
285          with open(adb_key_file, 'r') as f:
286            adb_public_keys = f.readlines()
287          adb_keys_set.update(adb_public_keys)
288        except IOError:
289          logger.warning('Unable to find adb keys file %s.', adb_key_file)
290      _WriteAdbKeysFile(device, '\n'.join(adb_keys_set))
291  except device_errors.CommandFailedError:
292    logger.exception('Possible failure while wiping the device. '
293                     'Attempting to continue.')
294
295
296def _WriteAdbKeysFile(device, adb_keys_string):
297  dir_path = posixpath.dirname(adb_wrapper.ADB_KEYS_FILE)
298  device.RunShellCommand(['mkdir', '-p', dir_path],
299                         as_root=True, check_return=True)
300  device.RunShellCommand(['restorecon', dir_path],
301                         as_root=True, check_return=True)
302  device.WriteFile(adb_wrapper.ADB_KEYS_FILE, adb_keys_string, as_root=True)
303  device.RunShellCommand(['restorecon', adb_wrapper.ADB_KEYS_FILE],
304                         as_root=True, check_return=True)
305
306
307def SetProperties(device, enable_java_debug, disable_location,
308                  disable_mock_location):
309  try:
310    device.EnableRoot()
311  except device_errors.CommandFailedError as e:
312    logger.warning(str(e))
313
314  if not device.IsUserBuild():
315    _ConfigureLocalProperties(device, enable_java_debug)
316  else:
317    logger.warning('Cannot configure properties in user builds.')
318  settings.ConfigureContentSettings(
319      device, settings.DETERMINISTIC_DEVICE_SETTINGS)
320  if disable_location:
321    settings.ConfigureContentSettings(
322        device, settings.DISABLE_LOCATION_SETTINGS)
323  else:
324    settings.ConfigureContentSettings(
325        device, settings.ENABLE_LOCATION_SETTINGS)
326
327  if disable_mock_location:
328    settings.ConfigureContentSettings(
329        device, settings.DISABLE_MOCK_LOCATION_SETTINGS)
330  else:
331    settings.ConfigureContentSettings(
332        device, settings.ENABLE_MOCK_LOCATION_SETTINGS)
333
334  settings.SetLockScreenSettings(device)
335
336  # Some device types can momentarily disappear after setting properties.
337  device.adb.WaitForDevice()
338
339
340def DisableNetwork(device):
341  settings.ConfigureContentSettings(
342      device, settings.NETWORK_DISABLED_SETTINGS)
343  if device.build_version_sdk >= version_codes.MARSHMALLOW:
344    # Ensure that NFC is also switched off.
345    device.RunShellCommand(['svc', 'nfc', 'disable'],
346                           as_root=True, check_return=True)
347
348
349def DisableSystemChrome(device):
350  # The system chrome version on the device interferes with some tests.
351  device.RunShellCommand(['pm', 'disable', 'com.android.chrome'],
352                         check_return=True)
353
354
355def _RemoveSystemApp(device, system_app):
356  found_paths = []
357  for directory in _SYSTEM_APP_DIRECTORIES:
358    path = os.path.join(directory, system_app)
359    if device.PathExists(path):
360      found_paths.append(path)
361  if not found_paths:
362    logger.warning('Could not find install location for system app %s',
363                   system_app)
364  device.RemovePath(found_paths, force=True, recursive=True)
365
366def RemoveSystemApps(device, system_app_remove_list):
367  """Attempts to remove the provided system apps from the given device.
368
369  Arguments:
370    device: The device to remove the system apps from.
371    system_app_remove_list: A list of app names to remove, e.g.
372        ['WebViewGoogle', 'GoogleVrCore']
373  """
374  device.EnableRoot()
375  if device.HasRoot():
376    # Disable Marshmallow's Verity security feature
377    if device.build_version_sdk >= version_codes.MARSHMALLOW:
378      logger.info('Disabling Verity on %s', device.serial)
379      device.adb.DisableVerity()
380      device.Reboot()
381      device.WaitUntilFullyBooted()
382      device.EnableRoot()
383
384    device.adb.Remount()
385    device.RunShellCommand(['stop'], check_return=True)
386    for system_app in system_app_remove_list:
387      _RemoveSystemApp(device, system_app)
388    device.RunShellCommand(['start'], check_return=True)
389  else:
390    raise device_errors.CommandFailedError(
391        'Failed to remove system apps from non-rooted device', str(device))
392
393
394def _ConfigureLocalProperties(device, java_debug=True):
395  """Set standard readonly testing device properties prior to reboot."""
396  local_props = [
397      'persist.sys.usb.config=adb',
398      'ro.monkey=1',
399      'ro.test_harness=1',
400      'ro.audio.silent=1',
401      'ro.setupwizard.mode=DISABLED',
402      ]
403  if java_debug:
404    local_props.append(
405        '%s=all' % device_utils.DeviceUtils.JAVA_ASSERT_PROPERTY)
406    local_props.append('debug.checkjni=1')
407  try:
408    device.WriteFile(
409        device.LOCAL_PROPERTIES_PATH,
410        '\n'.join(local_props), as_root=True)
411    # Android will not respect the local props file if it is world writable.
412    device.RunShellCommand(
413        ['chmod', '644', device.LOCAL_PROPERTIES_PATH],
414        as_root=True, check_return=True)
415  except device_errors.CommandFailedError:
416    logger.exception('Failed to configure local properties.')
417
418
419def FinishProvisioning(device):
420  # The lockscreen can't be disabled on user builds, so send a keyevent
421  # to unlock it.
422  if device.IsUserBuild():
423    device.SendKeyEvent(keyevent.KEYCODE_MENU)
424
425
426def WaitForCharge(device, min_battery_level):
427  battery = battery_utils.BatteryUtils(device)
428  try:
429    battery.ChargeDeviceToLevel(min_battery_level)
430  except device_errors.DeviceChargingError:
431    device.Reboot()
432    battery.ChargeDeviceToLevel(min_battery_level)
433
434
435def WaitForTemperature(device, max_battery_temp):
436  try:
437    battery = battery_utils.BatteryUtils(device)
438    battery.LetBatteryCoolToTemperature(max_battery_temp)
439  except device_errors.CommandFailedError:
440    logger.exception('Unable to let battery cool to specified temperature.')
441
442
443def SetDate(device):
444  def _set_and_verify_date():
445    if device.build_version_sdk >= version_codes.MARSHMALLOW:
446      date_format = '%m%d%H%M%Y.%S'
447      set_date_command = ['date', '-u']
448      get_date_command = ['date', '-u']
449    else:
450      date_format = '%Y%m%d.%H%M%S'
451      set_date_command = ['date', '-s']
452      get_date_command = ['date']
453
454    # TODO(jbudorick): This is wrong on pre-M devices -- get/set are
455    # dealing in local time, but we're setting based on GMT.
456    strgmtime = time.strftime(date_format, time.gmtime())
457    set_date_command.append(strgmtime)
458    device.RunShellCommand(set_date_command, as_root=True, check_return=True)
459
460    get_date_command.append('+"%Y%m%d.%H%M%S"')
461    device_time = device.RunShellCommand(
462        get_date_command, check_return=True,
463        as_root=True, single_line=True).replace('"', '')
464    device_time = datetime.datetime.strptime(device_time, "%Y%m%d.%H%M%S")
465    correct_time = datetime.datetime.strptime(strgmtime, date_format)
466    tdelta = (correct_time - device_time).seconds
467    if tdelta <= 1:
468      logger.info('Date/time successfully set on %s', device)
469      return True
470    else:
471      logger.error('Date mismatch. Device: %s Correct: %s',
472                   device_time.isoformat(), correct_time.isoformat())
473      return False
474
475  # Sometimes the date is not set correctly on the devices. Retry on failure.
476  if device.IsUserBuild():
477    # TODO(bpastene): Figure out how to set the date & time on user builds.
478    pass
479  else:
480    if not timeout_retry.WaitFor(
481        _set_and_verify_date, wait_period=1, max_tries=2):
482      raise device_errors.CommandFailedError(
483          'Failed to set date & time.', device_serial=str(device))
484    device.EnableRoot()
485    device.BroadcastIntent(
486        intent.Intent(action='android.intent.action.TIME_SET'))
487
488
489def LogDeviceProperties(device):
490  props = device.RunShellCommand(['getprop'], check_return=True)
491  for prop in props:
492    logger.info('  %s', prop)
493
494
495def CheckExternalStorage(device):
496  """Checks that storage is writable and if not makes it writable.
497
498  Arguments:
499    device: The device to check.
500  """
501  try:
502    with device_temp_file.DeviceTempFile(
503        device.adb, suffix='.sh', dir=device.GetExternalStoragePath()) as f:
504      device.WriteFile(f.name, 'test')
505  except device_errors.CommandFailedError:
506    logger.info('External storage not writable. Remounting / as RW')
507    device.RunShellCommand(['mount', '-o', 'remount,rw', '/'],
508                           check_return=True, as_root=True)
509    device.EnableRoot()
510    with device_temp_file.DeviceTempFile(
511        device.adb, suffix='.sh', dir=device.GetExternalStoragePath()) as f:
512      device.WriteFile(f.name, 'test')
513
514
515def main(raw_args):
516  # Recommended options on perf bots:
517  # --disable-network
518  #     TODO(tonyg): We eventually want network on. However, currently radios
519  #     can cause perfbots to drain faster than they charge.
520  # --min-battery-level 95
521  #     Some perf bots run benchmarks with USB charging disabled which leads
522  #     to gradual draining of the battery. We must wait for a full charge
523  #     before starting a run in order to keep the devices online.
524
525  parser = argparse.ArgumentParser(
526      description='Provision Android devices with settings required for bots.')
527  parser.add_argument(
528      '--adb-key-files', type=str, nargs='+',
529      help='list of adb keys to push to device')
530  parser.add_argument(
531      '--adb-path',
532      help='Absolute path to the adb binary to use.')
533  parser.add_argument('--blacklist-file', help='Device blacklist JSON file.')
534  parser.add_argument(
535      '-d', '--device', metavar='SERIAL', action='append', dest='devices',
536      help='the serial number of the device to be provisioned '
537           '(the default is to provision all devices attached)')
538  parser.add_argument(
539      '--disable-location', action='store_true',
540      help='disable Google location services on devices')
541  parser.add_argument(
542      '--disable-mock-location', action='store_true', default=False,
543      help='Set ALLOW_MOCK_LOCATION to false')
544  parser.add_argument(
545      '--disable-network', action='store_true',
546      help='disable network access on devices')
547  parser.add_argument(
548      '--disable-java-debug', action='store_false',
549      dest='enable_java_debug', default=True,
550      help='disable Java property asserts and JNI checking')
551  parser.add_argument(
552      '--disable-system-chrome', action='store_true',
553      help='Disable the system chrome from devices.')
554  parser.add_argument(
555      '--emulators', action='store_true',
556      help='provision only emulators and ignore usb devices '
557           '(this will not wipe emulators)')
558  parser.add_argument(
559      '--max-battery-temp', type=int, metavar='NUM',
560      help='Wait for the battery to have this temp or lower.')
561  parser.add_argument(
562      '--min-battery-level', type=int, metavar='NUM',
563      help='wait for the device to reach this minimum battery'
564           ' level before trying to continue')
565  parser.add_argument(
566      '--output-device-blacklist',
567      help='Json file to output the device blacklist.')
568  parser.add_argument(
569      '--reboot-timeout', metavar='SECS', type=int,
570      help='when wiping the device, max number of seconds to'
571           ' wait after each reboot '
572           '(default: %s)' % _DEFAULT_TIMEOUTS.HELP_TEXT)
573  parser.add_argument(
574      '--remove-system-apps', nargs='*', dest='system_app_remove_list',
575      help='the names of system apps to remove')
576  parser.add_argument(
577      '--remove-system-webview', action='store_true',
578      help='Remove the system webview from devices.')
579  parser.add_argument(
580      '--skip-wipe', action='store_true', default=False,
581      help='do not wipe device data during provisioning')
582  parser.add_argument(
583      '-v', '--verbose', action='count', default=1,
584      help='Log more information.')
585
586  # No-op arguments for compatibility with build/android/provision_devices.py.
587  # TODO(jbudorick): Remove these once all callers have stopped using them.
588  parser.add_argument(
589      '--chrome-specific-wipe', action='store_true',
590      help=argparse.SUPPRESS)
591  parser.add_argument(
592      '--phase', action='append',
593      help=argparse.SUPPRESS)
594  parser.add_argument(
595      '-r', '--auto-reconnect', action='store_true',
596      help=argparse.SUPPRESS)
597  parser.add_argument(
598      '-t', '--target',
599      help=argparse.SUPPRESS)
600
601  args = parser.parse_args(raw_args)
602
603  run_tests_helper.SetLogLevel(args.verbose)
604
605  devil_dynamic_config = devil_env.EmptyConfig()
606  if args.adb_path:
607    devil_dynamic_config['dependencies'].update(
608        devil_env.LocalConfigItem(
609            'adb', devil_env.GetPlatform(), args.adb_path))
610
611  devil_env.config.Initialize(configs=[devil_dynamic_config])
612
613  try:
614    return ProvisionDevices(
615        args.devices,
616        args.blacklist_file,
617        adb_key_files=args.adb_key_files,
618        disable_location=args.disable_location,
619        disable_mock_location=args.disable_mock_location,
620        disable_network=args.disable_network,
621        disable_system_chrome=args.disable_system_chrome,
622        emulators=args.emulators,
623        enable_java_debug=args.enable_java_debug,
624        max_battery_temp=args.max_battery_temp,
625        min_battery_level=args.min_battery_level,
626        output_device_blacklist=args.output_device_blacklist,
627        reboot_timeout=args.reboot_timeout,
628        remove_system_webview=args.remove_system_webview,
629        system_app_remove_list=args.system_app_remove_list,
630        wipe=not args.skip_wipe and not args.emulators)
631  except (device_errors.DeviceUnreachableError, device_errors.NoDevicesError):
632    logging.exception('Unable to provision local devices.')
633    return exit_codes.INFRA
634
635
636if __name__ == '__main__':
637  sys.exit(main(sys.argv[1:]))
638