• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2#
3# Copyright (c) 2013 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Provisions Android devices with settings required for bots.
8
9Usage:
10  ./provision_devices.py [-d <device serial number>]
11"""
12
13import argparse
14import datetime
15import json
16import logging
17import os
18import posixpath
19import re
20import sys
21import time
22
23# Import _strptime before threaded code. datetime.datetime.strptime is
24# threadsafe except for the initial import of the _strptime module.
25# See crbug.com/584730 and https://bugs.python.org/issue7980.
26import _strptime  # pylint: disable=unused-import
27
28if __name__ == '__main__':
29  sys.path.append(
30      os.path.abspath(os.path.join(os.path.dirname(__file__),
31                                   '..', '..', '..')))
32
33from devil import devil_env
34from devil.android import battery_utils
35from devil.android import device_blacklist
36from devil.android import device_errors
37from devil.android import device_temp_file
38from devil.android import device_utils
39from devil.android import settings
40from devil.android.constants import chrome
41from devil.android.sdk import adb_wrapper
42from devil.android.sdk import keyevent
43from devil.android.sdk import version_codes
44from devil.android.tools import script_common
45from devil.constants import exit_codes
46from devil.utils import run_tests_helper
47from devil.utils import timeout_retry
48
49_SYSTEM_WEBVIEW_PATHS = ['/system/app/webview', '/system/app/WebViewGoogle']
50_CHROME_PACKAGE_REGEX = re.compile('.*chrom.*')
51_TOMBSTONE_REGEX = re.compile('tombstone.*')
52
53
54class _DEFAULT_TIMEOUTS(object):
55  # L can take a while to reboot after a wipe.
56  LOLLIPOP = 600
57  PRE_LOLLIPOP = 180
58
59  HELP_TEXT = '{}s on L, {}s on pre-L'.format(LOLLIPOP, PRE_LOLLIPOP)
60
61
62class ProvisionStep(object):
63
64  def __init__(self, cmd, reboot=False):
65    self.cmd = cmd
66    self.reboot = reboot
67
68
69def ProvisionDevices(
70    devices,
71    blacklist_file,
72    adb_key_files=None,
73    disable_location=False,
74    disable_mock_location=False,
75    disable_network=False,
76    disable_system_chrome=False,
77    emulators=False,
78    enable_java_debug=False,
79    max_battery_temp=None,
80    min_battery_level=None,
81    output_device_blacklist=None,
82    reboot_timeout=None,
83    remove_system_webview=False,
84    wipe=True):
85  blacklist = (device_blacklist.Blacklist(blacklist_file)
86               if blacklist_file
87               else None)
88  devices = script_common.GetDevices(devices, blacklist)
89  devices = [d for d in devices
90             if not emulators or d.adb.is_emulator]
91  parallel_devices = device_utils.DeviceUtils.parallel(devices)
92
93  steps = []
94  if wipe:
95    steps += [ProvisionStep(lambda d: Wipe(d, adb_key_files), reboot=True)]
96  steps += [ProvisionStep(
97      lambda d: SetProperties(d, enable_java_debug, disable_location,
98                              disable_mock_location),
99      reboot=not emulators)]
100
101  if disable_network:
102    steps.append(ProvisionStep(DisableNetwork))
103
104  if disable_system_chrome:
105    steps.append(ProvisionStep(DisableSystemChrome))
106
107  if max_battery_temp:
108    steps.append(ProvisionStep(
109        lambda d: WaitForTemperature(d, max_battery_temp)))
110
111  if min_battery_level:
112    steps.append(ProvisionStep(
113        lambda d: WaitForCharge(d, min_battery_level)))
114
115  if remove_system_webview:
116    steps.append(ProvisionStep(RemoveSystemWebView))
117
118  steps.append(ProvisionStep(SetDate))
119  steps.append(ProvisionStep(CheckExternalStorage))
120
121  parallel_devices.pMap(ProvisionDevice, steps, blacklist, reboot_timeout)
122
123  blacklisted_devices = blacklist.Read() if blacklist else []
124  if output_device_blacklist:
125    with open(output_device_blacklist, 'w') as f:
126      json.dump(blacklisted_devices, f)
127  if all(d in blacklisted_devices for d in devices):
128    raise device_errors.NoDevicesError
129  return 0
130
131
132def ProvisionDevice(device, steps, blacklist, reboot_timeout=None):
133  try:
134    if not reboot_timeout:
135      if device.build_version_sdk >= version_codes.LOLLIPOP:
136        reboot_timeout = _DEFAULT_TIMEOUTS.LOLLIPOP
137      else:
138        reboot_timeout = _DEFAULT_TIMEOUTS.PRE_LOLLIPOP
139
140    for step in steps:
141      try:
142        device.WaitUntilFullyBooted(timeout=reboot_timeout, retries=0)
143      except device_errors.CommandTimeoutError:
144        logging.error('Device did not finish booting. Will try to reboot.')
145        device.Reboot(timeout=reboot_timeout)
146      step.cmd(device)
147      if step.reboot:
148        device.Reboot(False, retries=0)
149        device.adb.WaitForDevice()
150
151  except device_errors.CommandTimeoutError:
152    logging.exception('Timed out waiting for device %s. Adding to blacklist.',
153                      str(device))
154    if blacklist:
155      blacklist.Extend([str(device)], reason='provision_timeout')
156
157  except device_errors.CommandFailedError:
158    logging.exception('Failed to provision device %s. Adding to blacklist.',
159                      str(device))
160    if blacklist:
161      blacklist.Extend([str(device)], reason='provision_failure')
162
163
164def Wipe(device, adb_key_files=None):
165  if (device.IsUserBuild() or
166      device.build_version_sdk >= version_codes.MARSHMALLOW):
167    WipeChromeData(device)
168
169    package = "com.google.android.gms"
170    version_name = device.GetApplicationVersion(package)
171    logging.info("Version name for %s is %s", package, version_name)
172  else:
173    WipeDevice(device, adb_key_files)
174
175
176def WipeChromeData(device):
177  """Wipes chrome specific data from device
178
179  (1) uninstall any app whose name matches *chrom*, except
180      com.android.chrome, which is the chrome stable package. Doing so also
181      removes the corresponding dirs under /data/data/ and /data/app/
182  (2) remove any dir under /data/app-lib/ whose name matches *chrom*
183  (3) remove any files under /data/tombstones/ whose name matches "tombstone*"
184  (4) remove /data/local.prop if there is any
185  (5) remove /data/local/chrome-command-line if there is any
186  (6) remove anything under /data/local/.config/ if the dir exists
187      (this is telemetry related)
188  (7) remove anything under /data/local/tmp/
189
190  Arguments:
191    device: the device to wipe
192  """
193  try:
194    if device.IsUserBuild():
195      _UninstallIfMatch(device, _CHROME_PACKAGE_REGEX,
196                        chrome.PACKAGE_INFO['chrome_stable'].package)
197      device.RunShellCommand('rm -rf %s/*' % device.GetExternalStoragePath(),
198                             check_return=True)
199      device.RunShellCommand('rm -rf /data/local/tmp/*', check_return=True)
200    else:
201      device.EnableRoot()
202      _UninstallIfMatch(device, _CHROME_PACKAGE_REGEX,
203                        chrome.PACKAGE_INFO['chrome_stable'].package)
204      _WipeUnderDirIfMatch(device, '/data/app-lib/', _CHROME_PACKAGE_REGEX)
205      _WipeUnderDirIfMatch(device, '/data/tombstones/', _TOMBSTONE_REGEX)
206
207      _WipeFileOrDir(device, '/data/local.prop')
208      _WipeFileOrDir(device, '/data/local/chrome-command-line')
209      _WipeFileOrDir(device, '/data/local/.config/')
210      _WipeFileOrDir(device, '/data/local/tmp/')
211      device.RunShellCommand('rm -rf %s/*' % device.GetExternalStoragePath(),
212                             check_return=True)
213  except device_errors.CommandFailedError:
214    logging.exception('Possible failure while wiping the device. '
215                      'Attempting to continue.')
216
217
218def _UninstallIfMatch(device, pattern, app_to_keep):
219  installed_packages = device.RunShellCommand(['pm', 'list', 'packages'])
220  installed_system_packages = [
221      pkg.split(':')[1] for pkg in device.RunShellCommand(['pm', 'list',
222                                                           'packages', '-s'])]
223  for package_output in installed_packages:
224    package = package_output.split(":")[1]
225    if pattern.match(package) and not package == app_to_keep:
226      if not device.IsUserBuild() or package not in installed_system_packages:
227        device.Uninstall(package)
228
229
230def _WipeUnderDirIfMatch(device, path, pattern):
231  for filename in device.ListDirectory(path):
232    if pattern.match(filename):
233      _WipeFileOrDir(device, posixpath.join(path, filename))
234
235
236def _WipeFileOrDir(device, path):
237  if device.PathExists(path):
238    device.RunShellCommand(['rm', '-rf', path], check_return=True)
239
240
241def WipeDevice(device, adb_key_files):
242  """Wipes data from device, keeping only the adb_keys for authorization.
243
244  After wiping data on a device that has been authorized, adb can still
245  communicate with the device, but after reboot the device will need to be
246  re-authorized because the adb keys file is stored in /data/misc/adb/.
247  Thus, adb_keys file is rewritten so the device does not need to be
248  re-authorized.
249
250  Arguments:
251    device: the device to wipe
252  """
253  try:
254    device.EnableRoot()
255    device_authorized = device.FileExists(adb_wrapper.ADB_KEYS_FILE)
256    if device_authorized:
257      adb_keys = device.ReadFile(adb_wrapper.ADB_KEYS_FILE,
258                                 as_root=True).splitlines()
259    device.RunShellCommand(['wipe', 'data'],
260                           as_root=True, check_return=True)
261    device.adb.WaitForDevice()
262
263    if device_authorized:
264      adb_keys_set = set(adb_keys)
265      for adb_key_file in adb_key_files or []:
266        try:
267          with open(adb_key_file, 'r') as f:
268            adb_public_keys = f.readlines()
269          adb_keys_set.update(adb_public_keys)
270        except IOError:
271          logging.warning('Unable to find adb keys file %s.', adb_key_file)
272      _WriteAdbKeysFile(device, '\n'.join(adb_keys_set))
273  except device_errors.CommandFailedError:
274    logging.exception('Possible failure while wiping the device. '
275                      'Attempting to continue.')
276
277
278def _WriteAdbKeysFile(device, adb_keys_string):
279  dir_path = posixpath.dirname(adb_wrapper.ADB_KEYS_FILE)
280  device.RunShellCommand(['mkdir', '-p', dir_path],
281                         as_root=True, check_return=True)
282  device.RunShellCommand(['restorecon', dir_path],
283                         as_root=True, check_return=True)
284  device.WriteFile(adb_wrapper.ADB_KEYS_FILE, adb_keys_string, as_root=True)
285  device.RunShellCommand(['restorecon', adb_wrapper.ADB_KEYS_FILE],
286                         as_root=True, check_return=True)
287
288
289def SetProperties(device, enable_java_debug, disable_location,
290                  disable_mock_location):
291  try:
292    device.EnableRoot()
293  except device_errors.CommandFailedError as e:
294    logging.warning(str(e))
295
296  if not device.IsUserBuild():
297    _ConfigureLocalProperties(device, enable_java_debug)
298  else:
299    logging.warning('Cannot configure properties in user builds.')
300  settings.ConfigureContentSettings(
301      device, settings.DETERMINISTIC_DEVICE_SETTINGS)
302  if disable_location:
303    settings.ConfigureContentSettings(
304        device, settings.DISABLE_LOCATION_SETTINGS)
305  else:
306    settings.ConfigureContentSettings(
307        device, settings.ENABLE_LOCATION_SETTINGS)
308
309  if disable_mock_location:
310    settings.ConfigureContentSettings(
311        device, settings.DISABLE_MOCK_LOCATION_SETTINGS)
312  else:
313    settings.ConfigureContentSettings(
314        device, settings.ENABLE_MOCK_LOCATION_SETTINGS)
315
316  settings.SetLockScreenSettings(device)
317
318  # Some device types can momentarily disappear after setting properties.
319  device.adb.WaitForDevice()
320
321
322def DisableNetwork(device):
323  settings.ConfigureContentSettings(
324      device, settings.NETWORK_DISABLED_SETTINGS)
325  if device.build_version_sdk >= version_codes.MARSHMALLOW:
326    # Ensure that NFC is also switched off.
327    device.RunShellCommand(['svc', 'nfc', 'disable'],
328                           as_root=True, check_return=True)
329
330
331def DisableSystemChrome(device):
332  # The system chrome version on the device interferes with some tests.
333  device.RunShellCommand(['pm', 'disable', 'com.android.chrome'],
334                         check_return=True)
335
336
337def RemoveSystemWebView(device):
338  if any(device.PathExists(p) for p in _SYSTEM_WEBVIEW_PATHS):
339    logging.info('System WebView exists and needs to be removed')
340    if device.HasRoot():
341      # Disabled Marshmallow's Verity security feature
342      if device.build_version_sdk >= version_codes.MARSHMALLOW:
343        device.adb.DisableVerity()
344        device.Reboot()
345        device.WaitUntilFullyBooted()
346        device.EnableRoot()
347
348      # This is required, e.g., to replace the system webview on a device.
349      device.adb.Remount()
350      device.RunShellCommand(['stop'], check_return=True)
351      device.RunShellCommand(['rm', '-rf'] + _SYSTEM_WEBVIEW_PATHS,
352                             check_return=True)
353      device.RunShellCommand(['start'], check_return=True)
354    else:
355      logging.warning('Cannot remove system webview from a non-rooted device')
356  else:
357    logging.info('System WebView already removed')
358
359
360
361def _ConfigureLocalProperties(device, java_debug=True):
362  """Set standard readonly testing device properties prior to reboot."""
363  local_props = [
364      'persist.sys.usb.config=adb',
365      'ro.monkey=1',
366      'ro.test_harness=1',
367      'ro.audio.silent=1',
368      'ro.setupwizard.mode=DISABLED',
369      ]
370  if java_debug:
371    local_props.append(
372        '%s=all' % device_utils.DeviceUtils.JAVA_ASSERT_PROPERTY)
373    local_props.append('debug.checkjni=1')
374  try:
375    device.WriteFile(
376        device.LOCAL_PROPERTIES_PATH,
377        '\n'.join(local_props), as_root=True)
378    # Android will not respect the local props file if it is world writable.
379    device.RunShellCommand(
380        ['chmod', '644', device.LOCAL_PROPERTIES_PATH],
381        as_root=True, check_return=True)
382  except device_errors.CommandFailedError:
383    logging.exception('Failed to configure local properties.')
384
385
386def FinishProvisioning(device):
387  # The lockscreen can't be disabled on user builds, so send a keyevent
388  # to unlock it.
389  if device.IsUserBuild():
390    device.SendKeyEvent(keyevent.KEYCODE_MENU)
391
392
393def WaitForCharge(device, min_battery_level):
394  battery = battery_utils.BatteryUtils(device)
395  try:
396    battery.ChargeDeviceToLevel(min_battery_level)
397  except device_errors.DeviceChargingError:
398    device.Reboot()
399    battery.ChargeDeviceToLevel(min_battery_level)
400
401
402def WaitForTemperature(device, max_battery_temp):
403  try:
404    battery = battery_utils.BatteryUtils(device)
405    battery.LetBatteryCoolToTemperature(max_battery_temp)
406  except device_errors.CommandFailedError:
407    logging.exception('Unable to let battery cool to specified temperature.')
408
409
410def SetDate(device):
411  def _set_and_verify_date():
412    if device.build_version_sdk >= version_codes.MARSHMALLOW:
413      date_format = '%m%d%H%M%Y.%S'
414      set_date_command = ['date', '-u']
415      get_date_command = ['date', '-u']
416    else:
417      date_format = '%Y%m%d.%H%M%S'
418      set_date_command = ['date', '-s']
419      get_date_command = ['date']
420
421    # TODO(jbudorick): This is wrong on pre-M devices -- get/set are
422    # dealing in local time, but we're setting based on GMT.
423    strgmtime = time.strftime(date_format, time.gmtime())
424    set_date_command.append(strgmtime)
425    device.RunShellCommand(set_date_command, as_root=True, check_return=True)
426
427    get_date_command.append('+"%Y%m%d.%H%M%S"')
428    device_time = device.RunShellCommand(
429        get_date_command, as_root=True, single_line=True).replace('"', '')
430    device_time = datetime.datetime.strptime(device_time, "%Y%m%d.%H%M%S")
431    correct_time = datetime.datetime.strptime(strgmtime, date_format)
432    tdelta = (correct_time - device_time).seconds
433    if tdelta <= 1:
434      logging.info('Date/time successfully set on %s', device)
435      return True
436    else:
437      logging.error('Date mismatch. Device: %s Correct: %s',
438                    device_time.isoformat(), correct_time.isoformat())
439      return False
440
441  # Sometimes the date is not set correctly on the devices. Retry on failure.
442  if device.IsUserBuild():
443    # TODO(bpastene): Figure out how to set the date & time on user builds.
444    pass
445  else:
446    if not timeout_retry.WaitFor(
447        _set_and_verify_date, wait_period=1, max_tries=2):
448      raise device_errors.CommandFailedError(
449          'Failed to set date & time.', device_serial=str(device))
450
451
452def LogDeviceProperties(device):
453  props = device.RunShellCommand('getprop', check_return=True)
454  for prop in props:
455    logging.info('  %s', prop)
456
457
458def CheckExternalStorage(device):
459  """Checks that storage is writable and if not makes it writable.
460
461  Arguments:
462    device: The device to check.
463  """
464  try:
465    with device_temp_file.DeviceTempFile(
466        device.adb, suffix='.sh', dir=device.GetExternalStoragePath()) as f:
467      device.WriteFile(f.name, 'test')
468  except device_errors.CommandFailedError:
469    logging.info('External storage not writable. Remounting / as RW')
470    device.RunShellCommand(['mount', '-o', 'remount,rw', '/'],
471                           check_return=True, as_root=True)
472    device.EnableRoot()
473    with device_temp_file.DeviceTempFile(
474        device.adb, suffix='.sh', dir=device.GetExternalStoragePath()) as f:
475      device.WriteFile(f.name, 'test')
476
477
478def main(raw_args):
479  # Recommended options on perf bots:
480  # --disable-network
481  #     TODO(tonyg): We eventually want network on. However, currently radios
482  #     can cause perfbots to drain faster than they charge.
483  # --min-battery-level 95
484  #     Some perf bots run benchmarks with USB charging disabled which leads
485  #     to gradual draining of the battery. We must wait for a full charge
486  #     before starting a run in order to keep the devices online.
487
488  parser = argparse.ArgumentParser(
489      description='Provision Android devices with settings required for bots.')
490  parser.add_argument(
491      '-d', '--device', metavar='SERIAL', action='append', dest='devices',
492      help='the serial number of the device to be provisioned '
493           '(the default is to provision all devices attached)')
494  parser.add_argument(
495      '--adb-path',
496      help='Absolute path to the adb binary to use.')
497  parser.add_argument('--blacklist-file', help='Device blacklist JSON file.')
498  parser.add_argument(
499      '--skip-wipe', action='store_true', default=False,
500      help="don't wipe device data during provisioning")
501  parser.add_argument(
502      '--reboot-timeout', metavar='SECS', type=int,
503      help='when wiping the device, max number of seconds to'
504           ' wait after each reboot '
505           '(default: %s)' % _DEFAULT_TIMEOUTS.HELP_TEXT)
506  parser.add_argument(
507      '--min-battery-level', type=int, metavar='NUM',
508      help='wait for the device to reach this minimum battery'
509           ' level before trying to continue')
510  parser.add_argument(
511      '--disable-location', action='store_true',
512      help='disable Google location services on devices')
513  parser.add_argument(
514      '--disable-mock-location', action='store_true', default=False,
515      help='Set ALLOW_MOCK_LOCATION to false')
516  parser.add_argument(
517      '--disable-network', action='store_true',
518      help='disable network access on devices')
519  parser.add_argument(
520      '--disable-java-debug', action='store_false',
521      dest='enable_java_debug', default=True,
522      help='disable Java property asserts and JNI checking')
523  parser.add_argument(
524      '--disable-system-chrome', action='store_true',
525      help='Disable the system chrome from devices.')
526  parser.add_argument(
527      '--remove-system-webview', action='store_true',
528      help='Remove the system webview from devices.')
529  parser.add_argument(
530      '--adb-key-files', type=str, nargs='+',
531      help='list of adb keys to push to device')
532  parser.add_argument(
533      '-v', '--verbose', action='count', default=1,
534      help='Log more information.')
535  parser.add_argument(
536      '--max-battery-temp', type=int, metavar='NUM',
537      help='Wait for the battery to have this temp or lower.')
538  parser.add_argument(
539      '--output-device-blacklist',
540      help='Json file to output the device blacklist.')
541  parser.add_argument(
542      '--emulators', action='store_true',
543      help='provision only emulators and ignore usb devices')
544  args = parser.parse_args(raw_args)
545
546  run_tests_helper.SetLogLevel(args.verbose)
547
548  devil_dynamic_config = devil_env.EmptyConfig()
549  if args.adb_path:
550    devil_dynamic_config['dependencies'].update(
551        devil_env.LocalConfigItem(
552            'adb', devil_env.GetPlatform(), args.adb_path))
553
554  devil_env.config.Initialize(configs=[devil_dynamic_config])
555
556  try:
557    return ProvisionDevices(
558        args.devices,
559        args.blacklist_file,
560        adb_key_files=args.adb_key_files,
561        disable_location=args.disable_location,
562        disable_mock_location=args.disable_mock_location,
563        disable_network=args.disable_network,
564        disable_system_chrome=args.disable_system_chrome,
565        emulators=args.emulators,
566        enable_java_debug=args.enable_java_debug,
567        max_battery_temp=args.max_battery_temp,
568        min_battery_level=args.min_battery_level,
569        output_device_blacklist=args.output_device_blacklist,
570        reboot_timeout=args.reboot_timeout,
571        remove_system_webview=args.remove_system_webview,
572        wipe=not args.skip_wipe)
573  except (device_errors.DeviceUnreachableError, device_errors.NoDevicesError):
574    return exit_codes.INFRA
575
576
577if __name__ == '__main__':
578  sys.exit(main(sys.argv[1:]))
579