• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Provides a variety of device interactions based on adb.
6
7Eventually, this will be based on adb_wrapper.
8"""
9# pylint: disable=unused-argument
10
11import calendar
12import collections
13import fnmatch
14import json
15import logging
16import math
17import os
18import posixpath
19import pprint
20import random
21import re
22import shutil
23import stat
24import tempfile
25import time
26import threading
27import uuid
28
29from devil import base_error
30from devil import devil_env
31from devil.utils import cmd_helper
32from devil.android import apk_helper
33from devil.android import device_signal
34from devil.android import decorators
35from devil.android import device_errors
36from devil.android import device_temp_file
37from devil.android import install_commands
38from devil.android import logcat_monitor
39from devil.android import md5sum
40from devil.android.sdk import adb_wrapper
41from devil.android.sdk import intent
42from devil.android.sdk import keyevent
43from devil.android.sdk import split_select
44from devil.android.sdk import version_codes
45from devil.utils import host_utils
46from devil.utils import parallelizer
47from devil.utils import reraiser_thread
48from devil.utils import timeout_retry
49from devil.utils import zip_utils
50
51from py_utils import tempfile_ext
52
53logger = logging.getLogger(__name__)
54
55_DEFAULT_TIMEOUT = 30
56_DEFAULT_RETRIES = 3
57
58# A sentinel object for default values
59# TODO(jbudorick,perezju): revisit how default values are handled by
60# the timeout_retry decorators.
61DEFAULT = object()
62
63# A sentinel object to require that calls to RunShellCommand force running the
64# command with su even if the device has been rooted. To use, pass into the
65# as_root param.
66_FORCE_SU = object()
67
68_RECURSIVE_DIRECTORY_LIST_SCRIPT = """
69  function list_subdirs() {
70    for f in "$1"/* ;
71    do
72      if [ -d "$f" ] ;
73      then
74        if [ "$f" == "." ] || [ "$f" == ".." ] ;
75        then
76          continue ;
77        fi ;
78        echo "$f" ;
79        list_subdirs "$f" ;
80      fi ;
81    done ;
82  } ;
83  list_subdirs %s
84"""
85
86_RESTART_ADBD_SCRIPT = """
87  trap '' HUP
88  trap '' TERM
89  trap '' PIPE
90  function restart() {
91    stop adbd
92    start adbd
93  }
94  restart &
95"""
96
97# Not all permissions can be set.
98_PERMISSIONS_BLACKLIST_RE = re.compile('|'.join(fnmatch.translate(p) for p in [
99    'android.permission.ACCESS_LOCATION_EXTRA_COMMANDS',
100    'android.permission.ACCESS_MOCK_LOCATION',
101    'android.permission.ACCESS_NETWORK_STATE',
102    'android.permission.ACCESS_NOTIFICATION_POLICY',
103    'android.permission.ACCESS_VR_STATE',
104    'android.permission.ACCESS_WIFI_STATE',
105    'android.permission.AUTHENTICATE_ACCOUNTS',
106    'android.permission.BLUETOOTH',
107    'android.permission.BLUETOOTH_ADMIN',
108    'android.permission.BROADCAST_STICKY',
109    'android.permission.CHANGE_NETWORK_STATE',
110    'android.permission.CHANGE_WIFI_MULTICAST_STATE',
111    'android.permission.CHANGE_WIFI_STATE',
112    'android.permission.DISABLE_KEYGUARD',
113    'android.permission.DOWNLOAD_WITHOUT_NOTIFICATION',
114    'android.permission.EXPAND_STATUS_BAR',
115    'android.permission.FOREGROUND_SERVICE',
116    'android.permission.GET_PACKAGE_SIZE',
117    'android.permission.INSTALL_SHORTCUT',
118    'android.permission.INJECT_EVENTS',
119    'android.permission.INTERNET',
120    'android.permission.KILL_BACKGROUND_PROCESSES',
121    'android.permission.MANAGE_ACCOUNTS',
122    'android.permission.MODIFY_AUDIO_SETTINGS',
123    'android.permission.NFC',
124    'android.permission.READ_SYNC_SETTINGS',
125    'android.permission.READ_SYNC_STATS',
126    'android.permission.RECEIVE_BOOT_COMPLETED',
127    'android.permission.RECORD_VIDEO',
128    'android.permission.REORDER_TASKS',
129    'android.permission.REQUEST_INSTALL_PACKAGES',
130    'android.permission.RESTRICTED_VR_ACCESS',
131    'android.permission.RUN_INSTRUMENTATION',
132    'android.permission.SET_ALARM',
133    'android.permission.SET_TIME_ZONE',
134    'android.permission.SET_WALLPAPER',
135    'android.permission.SET_WALLPAPER_HINTS',
136    'android.permission.TRANSMIT_IR',
137    'android.permission.USE_CREDENTIALS',
138    'android.permission.USE_FINGERPRINT',
139    'android.permission.VIBRATE',
140    'android.permission.WAKE_LOCK',
141    'android.permission.WRITE_SYNC_SETTINGS',
142    'com.android.browser.permission.READ_HISTORY_BOOKMARKS',
143    'com.android.browser.permission.WRITE_HISTORY_BOOKMARKS',
144    'com.android.launcher.permission.INSTALL_SHORTCUT',
145    'com.chrome.permission.DEVICE_EXTRAS',
146    'com.google.android.apps.now.CURRENT_ACCOUNT_ACCESS',
147    'com.google.android.c2dm.permission.RECEIVE',
148    'com.google.android.providers.gsf.permission.READ_GSERVICES',
149    'com.google.vr.vrcore.permission.VRCORE_INTERNAL',
150    'com.sec.enterprise.knox.MDM_CONTENT_PROVIDER',
151    '*.permission.C2D_MESSAGE',
152    '*.permission.READ_WRITE_BOOKMARK_FOLDERS',
153    '*.TOS_ACKED',
154]))
155_SHELL_OUTPUT_SEPARATOR = '~X~'
156_PERMISSIONS_EXCEPTION_RE = re.compile(
157    r'java\.lang\.\w+Exception: .*$', re.MULTILINE)
158
159_CURRENT_FOCUS_CRASH_RE = re.compile(
160    r'\s*mCurrentFocus.*Application (Error|Not Responding): (\S+)}')
161
162_GETPROP_RE = re.compile(r'\[(.*?)\]: \[(.*?)\]')
163
164# Regex to parse the long (-l) output of 'ls' command, c.f.
165# https://github.com/landley/toybox/blob/master/toys/posix/ls.c#L446
166_LONG_LS_OUTPUT_RE = re.compile(
167    r'(?P<st_mode>[\w-]{10})\s+'                  # File permissions
168    r'(?:(?P<st_nlink>\d+)\s+)?'                  # Number of links (optional)
169    r'(?P<st_owner>\w+)\s+'                       # Name of owner
170    r'(?P<st_group>\w+)\s+'                       # Group of owner
171    r'(?:'                                        # Either ...
172      r'(?P<st_rdev_major>\d+),\s+'                 # Device major, and
173      r'(?P<st_rdev_minor>\d+)\s+'                  # Device minor
174    r'|'                                          # .. or
175      r'(?P<st_size>\d+)\s+'                        # Size in bytes
176    r')?'                                         # .. or nothing
177    r'(?P<st_mtime>\d{4}-\d\d-\d\d \d\d:\d\d)\s+' # Modification date/time
178    r'(?P<filename>.+?)'                          # File name
179    r'(?: -> (?P<symbolic_link_to>.+))?'          # Symbolic link (optional)
180    r'$'                                          # End of string
181)
182_LS_DATE_FORMAT = '%Y-%m-%d %H:%M'
183_FILE_MODE_RE = re.compile(r'[dbclps-](?:[r-][w-][xSs-]){2}[r-][w-][xTt-]$')
184_FILE_MODE_KIND = {
185    'd': stat.S_IFDIR, 'b': stat.S_IFBLK, 'c': stat.S_IFCHR,
186    'l': stat.S_IFLNK, 'p': stat.S_IFIFO, 's': stat.S_IFSOCK,
187    '-': stat.S_IFREG}
188_FILE_MODE_PERMS = [
189    stat.S_IRUSR, stat.S_IWUSR, stat.S_IXUSR,
190    stat.S_IRGRP, stat.S_IWGRP, stat.S_IXGRP,
191    stat.S_IROTH, stat.S_IWOTH, stat.S_IXOTH,
192]
193_FILE_MODE_SPECIAL = [
194    ('s', stat.S_ISUID),
195    ('s', stat.S_ISGID),
196    ('t', stat.S_ISVTX),
197]
198_PS_COLUMNS = {
199  'pid': 1,
200  'ppid': 2,
201  'name': -1
202}
203_SELINUX_MODE = {
204    'enforcing': True,
205    'permissive': False,
206    'disabled': None
207}
208# Some devices require different logic for checking if root is necessary
209_SPECIAL_ROOT_DEVICE_LIST = [
210    'marlin', # Pixel XL
211    'sailfish', # Pixel
212    'taimen', # Pixel 2 XL
213    'vega', # Lenovo Mirage Solo
214    'walleye', # Pixel 2
215]
216_IMEI_RE = re.compile(r'  Device ID = (.+)$')
217# The following regex is used to match result parcels like:
218"""
219Result: Parcel(
220  0x00000000: 00000000 0000000f 00350033 00360033 '........3.5.3.6.'
221  0x00000010: 00360032 00370030 00300032 00300039 '2.6.0.7.2.0.9.0.'
222  0x00000020: 00380033 00000039                   '3.8.9...        ')
223"""
224_PARCEL_RESULT_RE = re.compile(
225    r'0x[0-9a-f]{8}\: (?:[0-9a-f]{8}\s+){1,4}\'(.{16})\'')
226_EBUSY_RE = re.compile(
227    r'mkdir failed for ([^,]*), Device or resource busy')
228
229PS_COLUMNS = ('name', 'pid', 'ppid')
230ProcessInfo = collections.namedtuple('ProcessInfo', PS_COLUMNS)
231
232
233@decorators.WithExplicitTimeoutAndRetries(
234    _DEFAULT_TIMEOUT, _DEFAULT_RETRIES)
235def GetAVDs():
236  """Returns a list of Android Virtual Devices.
237
238  Returns:
239    A list containing the configured AVDs.
240  """
241  lines = cmd_helper.GetCmdOutput([
242      os.path.join(devil_env.config.LocalPath('android_sdk'),
243                   'tools', 'android'),
244      'list', 'avd']).splitlines()
245  avds = []
246  for line in lines:
247    if 'Name:' not in line:
248      continue
249    key, value = (s.strip() for s in line.split(':', 1))
250    if key == 'Name':
251      avds.append(value)
252  return avds
253
254
255@decorators.WithExplicitTimeoutAndRetries(
256    _DEFAULT_TIMEOUT, _DEFAULT_RETRIES)
257def RestartServer():
258  """Restarts the adb server.
259
260  Raises:
261    CommandFailedError if we fail to kill or restart the server.
262  """
263  def adb_killed():
264    return not adb_wrapper.AdbWrapper.IsServerOnline()
265
266  def adb_started():
267    return adb_wrapper.AdbWrapper.IsServerOnline()
268
269  adb_wrapper.AdbWrapper.KillServer()
270  if not timeout_retry.WaitFor(adb_killed, wait_period=1, max_tries=5):
271    # TODO(perezju): raise an exception after fixng http://crbug.com/442319
272    logger.warning('Failed to kill adb server')
273  adb_wrapper.AdbWrapper.StartServer()
274  if not timeout_retry.WaitFor(adb_started, wait_period=1, max_tries=5):
275    raise device_errors.CommandFailedError('Failed to start adb server')
276
277
278def _ParseModeString(mode_str):
279  """Parse a mode string, e.g. 'drwxrwxrwx', into a st_mode value.
280
281  Effectively the reverse of |mode_to_string| in, e.g.:
282  https://github.com/landley/toybox/blob/master/lib/lib.c#L896
283  """
284  if not _FILE_MODE_RE.match(mode_str):
285    raise ValueError('Unexpected file mode %r', mode_str)
286  mode = _FILE_MODE_KIND[mode_str[0]]
287  for c, flag in zip(mode_str[1:], _FILE_MODE_PERMS):
288    if c != '-' and c.islower():
289      mode |= flag
290  for c, (t, flag) in zip(mode_str[3::3], _FILE_MODE_SPECIAL):
291    if c.lower() == t:
292      mode |= flag
293  return mode
294
295
296def _GetTimeStamp():
297  """Return a basic ISO 8601 time stamp with the current local time."""
298  return time.strftime('%Y%m%dT%H%M%S', time.localtime())
299
300
301def _JoinLines(lines):
302  # makes sure that the last line is also terminated, and is more memory
303  # efficient than first appending an end-line to each line and then joining
304  # all of them together.
305  return ''.join(s for line in lines for s in (line, '\n'))
306
307
308def _CreateAdbWrapper(device):
309  if isinstance(device, adb_wrapper.AdbWrapper):
310    return device
311  else:
312    return adb_wrapper.AdbWrapper(device)
313
314
315def _FormatPartialOutputError(output):
316  lines = output.splitlines() if isinstance(output, basestring) else output
317  message = ['Partial output found:']
318  if len(lines) > 11:
319    message.extend('- %s' % line for line in lines[:5])
320    message.extend('<snip>')
321    message.extend('- %s' % line for line in lines[-5:])
322  else:
323    message.extend('- %s' % line for line in lines)
324  return '\n'.join(message)
325
326
327class DeviceUtils(object):
328
329  _MAX_ADB_COMMAND_LENGTH = 512
330  _MAX_ADB_OUTPUT_LENGTH = 32768
331  _LAUNCHER_FOCUSED_RE = re.compile(
332      r'\s*mCurrentFocus.*(Launcher|launcher).*')
333  _VALID_SHELL_VARIABLE = re.compile('^[a-zA-Z_][a-zA-Z0-9_]*$')
334
335  LOCAL_PROPERTIES_PATH = posixpath.join('/', 'data', 'local.prop')
336
337  # Property in /data/local.prop that controls Java assertions.
338  JAVA_ASSERT_PROPERTY = 'dalvik.vm.enableassertions'
339
340  def __init__(self, device, enable_device_files_cache=False,
341               default_timeout=_DEFAULT_TIMEOUT,
342               default_retries=_DEFAULT_RETRIES):
343    """DeviceUtils constructor.
344
345    Args:
346      device: Either a device serial, an existing AdbWrapper instance, or an
347        an existing AndroidCommands instance.
348      enable_device_files_cache: For PushChangedFiles(), cache checksums of
349        pushed files rather than recomputing them on a subsequent call.
350      default_timeout: An integer containing the default number of seconds to
351        wait for an operation to complete if no explicit value is provided.
352      default_retries: An integer containing the default number or times an
353        operation should be retried on failure if no explicit value is provided.
354    """
355    self.adb = None
356    if isinstance(device, basestring):
357      self.adb = _CreateAdbWrapper(device)
358    elif isinstance(device, adb_wrapper.AdbWrapper):
359      self.adb = device
360    else:
361      raise ValueError('Unsupported device value: %r' % device)
362    self._commands_installed = None
363    self._default_timeout = default_timeout
364    self._default_retries = default_retries
365    self._enable_device_files_cache = enable_device_files_cache
366    self._cache = {}
367    self._client_caches = {}
368    self._cache_lock = threading.RLock()
369    assert hasattr(self, decorators.DEFAULT_TIMEOUT_ATTR)
370    assert hasattr(self, decorators.DEFAULT_RETRIES_ATTR)
371
372    self._ClearCache()
373
374  @property
375  def serial(self):
376    """Returns the device serial."""
377    return self.adb.GetDeviceSerial()
378
379  def __eq__(self, other):
380    """Checks whether |other| refers to the same device as |self|.
381
382    Args:
383      other: The object to compare to. This can be a basestring, an instance
384        of adb_wrapper.AdbWrapper, or an instance of DeviceUtils.
385    Returns:
386      Whether |other| refers to the same device as |self|.
387    """
388    return self.serial == str(other)
389
390  def __lt__(self, other):
391    """Compares two instances of DeviceUtils.
392
393    This merely compares their serial numbers.
394
395    Args:
396      other: The instance of DeviceUtils to compare to.
397    Returns:
398      Whether |self| is less than |other|.
399    """
400    return self.serial < other.serial
401
402  def __str__(self):
403    """Returns the device serial."""
404    return self.serial
405
406  @decorators.WithTimeoutAndRetriesFromInstance()
407  def IsOnline(self, timeout=None, retries=None):
408    """Checks whether the device is online.
409
410    Args:
411      timeout: timeout in seconds
412      retries: number of retries
413
414    Returns:
415      True if the device is online, False otherwise.
416
417    Raises:
418      CommandTimeoutError on timeout.
419    """
420    try:
421      return self.adb.GetState() == 'device'
422    except base_error.BaseError as exc:
423      logger.info('Failed to get state: %s', exc)
424      return False
425
426  @decorators.WithTimeoutAndRetriesFromInstance()
427  def HasRoot(self, timeout=None, retries=None):
428    """Checks whether or not adbd has root privileges.
429
430    A device is considered to have root if all commands are implicitly run
431    with elevated privileges, i.e. without having to use "su" to run them.
432
433    Note that some devices do not allow this implicit privilige elevation,
434    but _can_ run commands as root just fine when done explicitly with "su".
435    To check if your device can run commands with elevated privileges at all
436    use:
437
438      device.HasRoot() or device.NeedsSU()
439
440    Luckily, for the most part you don't need to worry about this and using
441    RunShellCommand(cmd, as_root=True) will figure out for you the right
442    command incantation to run with elevated privileges.
443
444    Args:
445      timeout: timeout in seconds
446      retries: number of retries
447
448    Returns:
449      True if adbd has root privileges, False otherwise.
450
451    Raises:
452      CommandTimeoutError on timeout.
453      DeviceUnreachableError on missing device.
454    """
455    try:
456      if self.product_name in _SPECIAL_ROOT_DEVICE_LIST:
457        return self.GetProp('service.adb.root') == '1'
458      self.RunShellCommand(['ls', '/root'], check_return=True)
459      return True
460    except device_errors.AdbCommandFailedError:
461      return False
462
463  def NeedsSU(self, timeout=DEFAULT, retries=DEFAULT):
464    """Checks whether 'su' is needed to access protected resources.
465
466    Args:
467      timeout: timeout in seconds
468      retries: number of retries
469
470    Returns:
471      True if 'su' is available on the device and is needed to to access
472        protected resources; False otherwise if either 'su' is not available
473        (e.g. because the device has a user build), or not needed (because adbd
474        already has root privileges).
475
476    Raises:
477      CommandTimeoutError on timeout.
478      DeviceUnreachableError on missing device.
479    """
480    if 'needs_su' not in self._cache:
481      cmd = '%s && ! ls /root' % self._Su('ls /root')
482      if self.product_name in _SPECIAL_ROOT_DEVICE_LIST:
483        if self.HasRoot():
484          self._cache['needs_su'] = False
485          return False
486        cmd = 'which which && which su'
487      try:
488        self.RunShellCommand(cmd, shell=True, check_return=True,
489            timeout=self._default_timeout if timeout is DEFAULT else timeout,
490            retries=self._default_retries if retries is DEFAULT else retries)
491        self._cache['needs_su'] = True
492      except device_errors.AdbCommandFailedError:
493        self._cache['needs_su'] = False
494    return self._cache['needs_su']
495
496
497  def _Su(self, command):
498    if self.build_version_sdk >= version_codes.MARSHMALLOW:
499      return 'su 0 %s' % command
500    return 'su -c %s' % command
501
502  @decorators.WithTimeoutAndRetriesFromInstance()
503  def EnableRoot(self, timeout=None, retries=None):
504    """Restarts adbd with root privileges.
505
506    Args:
507      timeout: timeout in seconds
508      retries: number of retries
509
510    Raises:
511      CommandFailedError if root could not be enabled.
512      CommandTimeoutError on timeout.
513    """
514    if 'needs_su' in self._cache:
515      del self._cache['needs_su']
516
517    try:
518      self.adb.Root()
519    except device_errors.AdbCommandFailedError:
520      if self.IsUserBuild():
521        raise device_errors.CommandFailedError(
522            'Unable to root device with user build.', str(self))
523      else:
524        raise  # Failed probably due to some other reason.
525
526    def device_online_with_root():
527      try:
528        self.adb.WaitForDevice()
529        return self.GetProp('service.adb.root', cache=False) == '1'
530      except (device_errors.AdbCommandFailedError,
531              device_errors.DeviceUnreachableError):
532        return False
533
534    timeout_retry.WaitFor(device_online_with_root, wait_period=1)
535
536  @decorators.WithTimeoutAndRetriesFromInstance()
537  def IsUserBuild(self, timeout=None, retries=None):
538    """Checks whether or not the device is running a user build.
539
540    Args:
541      timeout: timeout in seconds
542      retries: number of retries
543
544    Returns:
545      True if the device is running a user build, False otherwise (i.e. if
546        it's running a userdebug build).
547
548    Raises:
549      CommandTimeoutError on timeout.
550      DeviceUnreachableError on missing device.
551    """
552    return self.build_type == 'user'
553
554  @decorators.WithTimeoutAndRetriesFromInstance()
555  def GetExternalStoragePath(self, timeout=None, retries=None):
556    """Get the device's path to its SD card.
557
558    Args:
559      timeout: timeout in seconds
560      retries: number of retries
561
562    Returns:
563      The device's path to its SD card.
564
565    Raises:
566      CommandFailedError if the external storage path could not be determined.
567      CommandTimeoutError on timeout.
568      DeviceUnreachableError on missing device.
569    """
570    self._EnsureCacheInitialized()
571    if not self._cache['external_storage']:
572      raise device_errors.CommandFailedError('$EXTERNAL_STORAGE is not set',
573                                             str(self))
574    return self._cache['external_storage']
575
576  @decorators.WithTimeoutAndRetriesFromInstance()
577  def GetIMEI(self, timeout=None, retries=None):
578    """Get the device's IMEI.
579
580    Args:
581      timeout: timeout in seconds
582      retries: number of retries
583
584    Returns:
585      The device's IMEI.
586
587    Raises:
588      AdbCommandFailedError on error
589    """
590    if self._cache.get('imei') is not None:
591      return self._cache.get('imei')
592
593    if self.build_version_sdk < 21:
594      out = self.RunShellCommand(['dumpsys', 'iphonesubinfo'],
595                                 raw_output=True, check_return=True)
596      if out:
597        match = re.search(_IMEI_RE, out)
598        if match:
599          self._cache['imei'] = match.group(1)
600          return self._cache['imei']
601    else:
602      out = self.RunShellCommand(['service', 'call', 'iphonesubinfo', '1'],
603                                 check_return=True)
604      if out:
605        imei = ''
606        for line in out:
607          match = re.search(_PARCEL_RESULT_RE, line)
608          if match:
609            imei = imei + match.group(1)
610        imei = imei.replace('.', '').strip()
611        if imei:
612          self._cache['imei'] = imei
613          return self._cache['imei']
614
615    raise device_errors.CommandFailedError('Unable to fetch IMEI.')
616
617  @decorators.WithTimeoutAndRetriesFromInstance()
618  def GetApplicationPaths(self, package, timeout=None, retries=None):
619    """Get the paths of the installed apks on the device for the given package.
620
621    Args:
622      package: Name of the package.
623
624    Returns:
625      List of paths to the apks on the device for the given package.
626    """
627    return self._GetApplicationPathsInternal(package)
628
629  def _GetApplicationPathsInternal(self, package, skip_cache=False):
630    cached_result = self._cache['package_apk_paths'].get(package)
631    if cached_result is not None and not skip_cache:
632      if package in self._cache['package_apk_paths_to_verify']:
633        self._cache['package_apk_paths_to_verify'].remove(package)
634        # Don't verify an app that is not thought to be installed. We are
635        # concerned only with apps we think are installed having been
636        # uninstalled manually.
637        if cached_result and not self.PathExists(cached_result):
638          cached_result = None
639          self._cache['package_apk_checksums'].pop(package, 0)
640      if cached_result is not None:
641        return list(cached_result)
642    # 'pm path' is liable to incorrectly exit with a nonzero number starting
643    # in Lollipop.
644    # TODO(jbudorick): Check if this is fixed as new Android versions are
645    # released to put an upper bound on this.
646    should_check_return = (self.build_version_sdk < version_codes.LOLLIPOP)
647    output = self.RunShellCommand(
648        ['pm', 'path', package], check_return=should_check_return)
649    apks = []
650    bad_output = False
651    for line in output:
652      if line.startswith('package:'):
653        apks.append(line[len('package:'):])
654      elif line.startswith('WARNING:'):
655        continue
656      else:
657        bad_output = True  # Unexpected line in output.
658    if not apks and output:
659      if bad_output:
660        raise device_errors.CommandFailedError(
661            'Unexpected pm path output: %r' % '\n'.join(output), str(self))
662      else:
663        logger.warning('pm returned no paths but the following warnings:')
664        for line in output:
665          logger.warning('- %s', line)
666    self._cache['package_apk_paths'][package] = list(apks)
667    return apks
668
669  @decorators.WithTimeoutAndRetriesFromInstance()
670  def GetApplicationVersion(self, package, timeout=None, retries=None):
671    """Get the version name of a package installed on the device.
672
673    Args:
674      package: Name of the package.
675
676    Returns:
677      A string with the version name or None if the package is not found
678      on the device.
679    """
680    output = self.RunShellCommand(
681        ['dumpsys', 'package', package], check_return=True)
682    if not output:
683      return None
684    for line in output:
685      line = line.strip()
686      if line.startswith('versionName='):
687        return line[len('versionName='):]
688    raise device_errors.CommandFailedError(
689        'Version name for %s not found on dumpsys output' % package, str(self))
690
691  @decorators.WithTimeoutAndRetriesFromInstance()
692  def GetPackageArchitecture(self, package, timeout=None, retries=None):
693    """Get the architecture of a package installed on the device.
694
695    Args:
696      package: Name of the package.
697
698    Returns:
699      A string with the architecture, or None if the package is missing.
700    """
701    lines = self._GetDumpsysOutput(['package', package], 'primaryCpuAbi')
702    if lines:
703      _, _, package_arch = lines[-1].partition('=')
704      return package_arch.strip()
705    return None
706
707  @decorators.WithTimeoutAndRetriesFromInstance()
708  def GetApplicationDataDirectory(self, package, timeout=None, retries=None):
709    """Get the data directory on the device for the given package.
710
711    Args:
712      package: Name of the package.
713
714    Returns:
715      The package's data directory.
716    Raises:
717      CommandFailedError if the package's data directory can't be found,
718        whether because it's not installed or otherwise.
719    """
720    output = self._RunPipedShellCommand(
721        'pm dump %s | grep dataDir=' % cmd_helper.SingleQuote(package))
722    for line in output:
723      _, _, dataDir = line.partition('dataDir=')
724      if dataDir:
725        return dataDir
726    raise device_errors.CommandFailedError(
727        'Could not find data directory for %s', package)
728
729  @decorators.WithTimeoutAndRetriesFromInstance()
730  def GetSecurityContextForPackage(self, package, encrypted=False, timeout=None,
731      retries=None):
732    """Gets the SELinux security context for the given package.
733
734    Args:
735      package: Name of the package.
736      encrypted: Whether to check in the encrypted data directory
737          (/data/user_de/0/) or the unencrypted data directory (/data/data/).
738
739    Returns:
740      The package's security context as a string, or None if not found.
741    """
742    directory = '/data/user_de/0/' if encrypted else '/data/data/'
743    for line in self.RunShellCommand(['ls', '-Z', directory],
744                                     as_root=True, check_return=True):
745      split_line = line.split()
746      # ls -Z output differs between Android versions, but the package is
747      # always last and the context always starts with "u:object"
748      if split_line[-1] == package:
749        for column in split_line:
750          if column.startswith('u:object'):
751            return column
752    return None
753
754  def TakeBugReport(self, path, timeout=60*5, retries=None):
755    """Takes a bug report and dumps it to the specified path.
756
757    This doesn't use adb's bugreport option since its behavior is dependent on
758    both adb version and device OS version. To make it simpler, this directly
759    runs the bugreport command on the device itself and dumps the stdout to a
760    file.
761
762    Args:
763      path: Path on the host to drop the bug report.
764      timeout: (optional) Timeout per try in seconds.
765      retries: (optional) Number of retries to attempt.
766    """
767    with device_temp_file.DeviceTempFile(self.adb) as device_tmp_file:
768      cmd = '( bugreport )>%s 2>&1' % device_tmp_file.name
769      self.RunShellCommand(
770          cmd, check_return=True, shell=True, timeout=timeout, retries=retries)
771      self.PullFile(device_tmp_file.name, path)
772
773  @decorators.WithTimeoutAndRetriesFromInstance()
774  def WaitUntilFullyBooted(self, wifi=False, timeout=None, retries=None):
775    """Wait for the device to fully boot.
776
777    This means waiting for the device to boot, the package manager to be
778    available, and the SD card to be ready. It can optionally mean waiting
779    for wifi to come up, too.
780
781    Args:
782      wifi: A boolean indicating if we should wait for wifi to come up or not.
783      timeout: timeout in seconds
784      retries: number of retries
785
786    Raises:
787      CommandFailedError on failure.
788      CommandTimeoutError if one of the component waits times out.
789      DeviceUnreachableError if the device becomes unresponsive.
790    """
791    def sd_card_ready():
792      try:
793        self.RunShellCommand(['test', '-d', self.GetExternalStoragePath()],
794                             check_return=True)
795        return True
796      except device_errors.AdbCommandFailedError:
797        return False
798
799    def pm_ready():
800      try:
801        return self._GetApplicationPathsInternal('android', skip_cache=True)
802      except device_errors.CommandFailedError:
803        return False
804
805    def boot_completed():
806      try:
807        return self.GetProp('sys.boot_completed', cache=False) == '1'
808      except device_errors.CommandFailedError:
809        return False
810
811    def wifi_enabled():
812      return 'Wi-Fi is enabled' in self.RunShellCommand(['dumpsys', 'wifi'],
813                                                        check_return=False)
814
815    self.adb.WaitForDevice()
816    timeout_retry.WaitFor(sd_card_ready)
817    timeout_retry.WaitFor(pm_ready)
818    timeout_retry.WaitFor(boot_completed)
819    if wifi:
820      timeout_retry.WaitFor(wifi_enabled)
821
822  REBOOT_DEFAULT_TIMEOUT = 10 * _DEFAULT_TIMEOUT
823
824  @decorators.WithTimeoutAndRetriesFromInstance(
825      min_default_timeout=REBOOT_DEFAULT_TIMEOUT)
826  def Reboot(self, block=True, wifi=False, timeout=None, retries=None):
827    """Reboot the device.
828
829    Args:
830      block: A boolean indicating if we should wait for the reboot to complete.
831      wifi: A boolean indicating if we should wait for wifi to be enabled after
832        the reboot. The option has no effect unless |block| is also True.
833      timeout: timeout in seconds
834      retries: number of retries
835
836    Raises:
837      CommandTimeoutError on timeout.
838      DeviceUnreachableError on missing device.
839    """
840    def device_offline():
841      return not self.IsOnline()
842
843    self.adb.Reboot()
844    self._ClearCache()
845    timeout_retry.WaitFor(device_offline, wait_period=1)
846    if block:
847      self.WaitUntilFullyBooted(wifi=wifi)
848
849  INSTALL_DEFAULT_TIMEOUT = 4 * _DEFAULT_TIMEOUT
850
851  @decorators.WithTimeoutAndRetriesFromInstance(
852      min_default_timeout=INSTALL_DEFAULT_TIMEOUT)
853  def Install(self, apk, allow_downgrade=False, reinstall=False,
854              permissions=None, timeout=None, retries=None):
855    """Install an APK.
856
857    Noop if an identical APK is already installed.
858
859    Args:
860      apk: An ApkHelper instance or string containing the path to the APK.
861      allow_downgrade: A boolean indicating if we should allow downgrades.
862      reinstall: A boolean indicating if we should keep any existing app data.
863      permissions: Set of permissions to set. If not set, finds permissions with
864          apk helper. To set no permissions, pass [].
865      timeout: timeout in seconds
866      retries: number of retries
867
868    Raises:
869      CommandFailedError if the installation fails.
870      CommandTimeoutError if the installation times out.
871      DeviceUnreachableError on missing device.
872    """
873    self._InstallInternal(apk, None, allow_downgrade=allow_downgrade,
874                          reinstall=reinstall, permissions=permissions)
875
876  @decorators.WithTimeoutAndRetriesFromInstance(
877      min_default_timeout=INSTALL_DEFAULT_TIMEOUT)
878  def InstallSplitApk(self, base_apk, split_apks, allow_downgrade=False,
879                      reinstall=False, allow_cached_props=False,
880                      permissions=None, timeout=None, retries=None):
881    """Install a split APK.
882
883    Noop if all of the APK splits are already installed.
884
885    Args:
886      base_apk: An ApkHelper instance or string containing the path to the base
887          APK.
888      split_apks: A list of strings of paths of all of the APK splits.
889      allow_downgrade: A boolean indicating if we should allow downgrades.
890      reinstall: A boolean indicating if we should keep any existing app data.
891      allow_cached_props: Whether to use cached values for device properties.
892      permissions: Set of permissions to set. If not set, finds permissions with
893          apk helper. To set no permissions, pass [].
894      timeout: timeout in seconds
895      retries: number of retries
896
897    Raises:
898      CommandFailedError if the installation fails.
899      CommandTimeoutError if the installation times out.
900      DeviceUnreachableError on missing device.
901      DeviceVersionError if device SDK is less than Android L.
902    """
903    self._InstallInternal(base_apk, split_apks, reinstall=reinstall,
904                          allow_cached_props=allow_cached_props,
905                          permissions=permissions,
906                          allow_downgrade=allow_downgrade)
907
908  def _InstallInternal(self, base_apk, split_apks, allow_downgrade=False,
909                       reinstall=False, allow_cached_props=False,
910                       permissions=None):
911    if split_apks:
912      self._CheckSdkLevel(version_codes.LOLLIPOP)
913
914    base_apk = apk_helper.ToHelper(base_apk)
915
916    all_apks = [base_apk.path]
917    if split_apks:
918      all_apks += split_select.SelectSplits(
919        self, base_apk.path, split_apks, allow_cached_props=allow_cached_props)
920      if len(all_apks) == 1:
921        logger.warning('split-select did not select any from %s', split_apks)
922
923    missing_apks = [apk for apk in all_apks if not os.path.exists(apk)]
924    if missing_apks:
925      raise device_errors.CommandFailedError(
926          'Attempted to install non-existent apks: %s'
927              % pprint.pformat(missing_apks))
928
929    package_name = base_apk.GetPackageName()
930    device_apk_paths = self._GetApplicationPathsInternal(package_name)
931
932    apks_to_install = None
933    host_checksums = None
934    if not device_apk_paths:
935      apks_to_install = all_apks
936    elif len(device_apk_paths) > 1 and not split_apks:
937      logger.warning(
938          'Installing non-split APK when split APK was previously installed')
939      apks_to_install = all_apks
940    elif len(device_apk_paths) == 1 and split_apks:
941      logger.warning(
942          'Installing split APK when non-split APK was previously installed')
943      apks_to_install = all_apks
944    else:
945      try:
946        apks_to_install, host_checksums = (
947            self._ComputeStaleApks(package_name, all_apks))
948      except EnvironmentError as e:
949        logger.warning('Error calculating md5: %s', e)
950        apks_to_install, host_checksums = all_apks, None
951      if apks_to_install and not reinstall:
952        self.Uninstall(package_name)
953        apks_to_install = all_apks
954
955    if apks_to_install:
956      # Assume that we won't know the resulting device state.
957      self._cache['package_apk_paths'].pop(package_name, 0)
958      self._cache['package_apk_checksums'].pop(package_name, 0)
959      if split_apks:
960        partial = package_name if len(apks_to_install) < len(all_apks) else None
961        self.adb.InstallMultiple(
962            apks_to_install, partial=partial, reinstall=reinstall,
963            allow_downgrade=allow_downgrade)
964      else:
965        self.adb.Install(
966            base_apk.path, reinstall=reinstall, allow_downgrade=allow_downgrade)
967    else:
968      # Running adb install terminates running instances of the app, so to be
969      # consistent, we explicitly terminate it when skipping the install.
970      self.ForceStop(package_name)
971
972    if (permissions is None
973        and self.build_version_sdk >= version_codes.MARSHMALLOW):
974      permissions = base_apk.GetPermissions()
975    self.GrantPermissions(package_name, permissions)
976    # Upon success, we know the device checksums, but not their paths.
977    if host_checksums is not None:
978      self._cache['package_apk_checksums'][package_name] = host_checksums
979
980  @decorators.WithTimeoutAndRetriesFromInstance()
981  def Uninstall(self, package_name, keep_data=False, timeout=None,
982                retries=None):
983    """Remove the app |package_name| from the device.
984
985    This is a no-op if the app is not already installed.
986
987    Args:
988      package_name: The package to uninstall.
989      keep_data: (optional) Whether to keep the data and cache directories.
990      timeout: Timeout in seconds.
991      retries: Number of retries.
992
993    Raises:
994      CommandFailedError if the uninstallation fails.
995      CommandTimeoutError if the uninstallation times out.
996      DeviceUnreachableError on missing device.
997    """
998    installed = self._GetApplicationPathsInternal(package_name)
999    if not installed:
1000      return
1001    try:
1002      self.adb.Uninstall(package_name, keep_data)
1003      self._cache['package_apk_paths'][package_name] = []
1004      self._cache['package_apk_checksums'][package_name] = set()
1005    except:
1006      # Clear cache since we can't be sure of the state.
1007      self._cache['package_apk_paths'].pop(package_name, 0)
1008      self._cache['package_apk_checksums'].pop(package_name, 0)
1009      raise
1010
1011  def _CheckSdkLevel(self, required_sdk_level):
1012    """Raises an exception if the device does not have the required SDK level.
1013    """
1014    if self.build_version_sdk < required_sdk_level:
1015      raise device_errors.DeviceVersionError(
1016          ('Requires SDK level %s, device is SDK level %s' %
1017           (required_sdk_level, self.build_version_sdk)),
1018           device_serial=self.serial)
1019
1020  @decorators.WithTimeoutAndRetriesFromInstance()
1021  def RunShellCommand(self, cmd, shell=False, check_return=False, cwd=None,
1022                      env=None, run_as=None, as_root=False, single_line=False,
1023                      large_output=False, raw_output=False,
1024                      ensure_logs_on_timeout=False, timeout=None, retries=None):
1025    """Run an ADB shell command.
1026
1027    The command to run |cmd| should be a sequence of program arguments
1028    (preferred) or a single string with a shell script to run.
1029
1030    When |cmd| is a sequence, it is assumed to contain the name of the command
1031    to run followed by its arguments. In this case, arguments are passed to the
1032    command exactly as given, preventing any further processing by the shell.
1033    This allows callers to easily pass arguments with spaces or special
1034    characters without having to worry about quoting rules. Whenever possible,
1035    it is recomended to pass |cmd| as a sequence.
1036
1037    When |cmd| is passed as a single string, |shell| should be set to True.
1038    The command will be interpreted and run by the shell on the device,
1039    allowing the use of shell features such as pipes, wildcards, or variables.
1040    Failing to set shell=True will issue a warning, but this will be changed
1041    to a hard failure in the future (see: catapult:#3242).
1042
1043    This behaviour is consistent with that of command runners in cmd_helper as
1044    well as Python's own subprocess.Popen.
1045
1046    TODO(perezju) Change the default of |check_return| to True when callers
1047      have switched to the new behaviour.
1048
1049    Args:
1050      cmd: A sequence containing the command to run and its arguments, or a
1051        string with a shell script to run (should also set shell=True).
1052      shell: A boolean indicating whether shell features may be used in |cmd|.
1053      check_return: A boolean indicating whether or not the return code should
1054        be checked.
1055      cwd: The device directory in which the command should be run.
1056      env: The environment variables with which the command should be run.
1057      run_as: A string containing the package as which the command should be
1058        run.
1059      as_root: A boolean indicating whether the shell command should be run
1060        with root privileges.
1061      single_line: A boolean indicating if only a single line of output is
1062        expected.
1063      large_output: Uses a work-around for large shell command output. Without
1064        this large output will be truncated.
1065      raw_output: Whether to only return the raw output
1066          (no splitting into lines).
1067      ensure_logs_on_timeout: If True, will use a slightly smaller timeout for
1068          the internal adb command, which allows to retrive logs on timeout.
1069          Note that that logs are not guaranteed to be produced with this option
1070          as adb command may still hang and fail to respect the reduced timeout.
1071      timeout: timeout in seconds
1072      retries: number of retries
1073
1074    Returns:
1075      If single_line is False, the output of the command as a list of lines,
1076      otherwise, a string with the unique line of output emmited by the command
1077      (with the optional newline at the end stripped).
1078
1079    Raises:
1080      AdbCommandFailedError if check_return is True and the exit code of
1081        the command run on the device is non-zero.
1082      CommandFailedError if single_line is True but the output contains two or
1083        more lines.
1084      CommandTimeoutError on timeout.
1085      DeviceUnreachableError on missing device.
1086    """
1087    def env_quote(key, value):
1088      if not DeviceUtils._VALID_SHELL_VARIABLE.match(key):
1089        raise KeyError('Invalid shell variable name %r' % key)
1090      # using double quotes here to allow interpolation of shell variables
1091      return '%s=%s' % (key, cmd_helper.DoubleQuote(value))
1092
1093    def run(cmd):
1094      return self.adb.Shell(cmd, ensure_logs_on_timeout=ensure_logs_on_timeout)
1095
1096    def handle_check_return(cmd):
1097      try:
1098        return run(cmd)
1099      except device_errors.AdbCommandFailedError as exc:
1100        if check_return:
1101          raise
1102        else:
1103          return exc.output
1104
1105    def handle_large_command(cmd):
1106      if len(cmd) < self._MAX_ADB_COMMAND_LENGTH:
1107        return handle_check_return(cmd)
1108      else:
1109        with device_temp_file.DeviceTempFile(self.adb, suffix='.sh') as script:
1110          self._WriteFileWithPush(script.name, cmd)
1111          logger.info('Large shell command will be run from file: %s ...',
1112                      cmd[:self._MAX_ADB_COMMAND_LENGTH])
1113          return handle_check_return('sh %s' % script.name_quoted)
1114
1115    def handle_large_output(cmd, large_output_mode):
1116      if large_output_mode:
1117        with device_temp_file.DeviceTempFile(self.adb) as large_output_file:
1118          cmd = '( %s )>%s 2>&1' % (cmd, large_output_file.name)
1119          logger.debug('Large output mode enabled. Will write output to '
1120                       'device and read results from file.')
1121          handle_large_command(cmd)
1122          return self.ReadFile(large_output_file.name, force_pull=True)
1123      else:
1124        try:
1125          return handle_large_command(cmd)
1126        except device_errors.AdbCommandFailedError as exc:
1127          if exc.status is None:
1128            logger.error(_FormatPartialOutputError(exc.output))
1129            logger.warning('Attempting to run in large_output mode.')
1130            logger.warning('Use RunShellCommand(..., large_output=True) for '
1131                           'shell commands that expect a lot of output.')
1132            return handle_large_output(cmd, True)
1133          else:
1134            raise
1135
1136    if isinstance(cmd, basestring):
1137      if not shell:
1138        logger.warning(
1139            'The command to run should preferably be passed as a sequence of'
1140            ' args. If shell features are needed (pipes, wildcards, variables)'
1141            ' clients should explicitly set shell=True.')
1142    else:
1143      cmd = ' '.join(cmd_helper.SingleQuote(s) for s in cmd)
1144    if env:
1145      env = ' '.join(env_quote(k, v) for k, v in env.iteritems())
1146      cmd = '%s %s' % (env, cmd)
1147    if cwd:
1148      cmd = 'cd %s && %s' % (cmd_helper.SingleQuote(cwd), cmd)
1149    if run_as:
1150      cmd = 'run-as %s sh -c %s' % (cmd_helper.SingleQuote(run_as),
1151                                    cmd_helper.SingleQuote(cmd))
1152    if (as_root is _FORCE_SU) or (as_root and self.NeedsSU()):
1153      # "su -c sh -c" allows using shell features in |cmd|
1154      cmd = self._Su('sh -c %s' % cmd_helper.SingleQuote(cmd))
1155
1156    output = handle_large_output(cmd, large_output)
1157
1158    if raw_output:
1159      return output
1160
1161    output = output.splitlines()
1162    if single_line:
1163      if not output:
1164        return ''
1165      elif len(output) == 1:
1166        return output[0]
1167      else:
1168        msg = 'one line of output was expected, but got: %s'
1169        raise device_errors.CommandFailedError(msg % output, str(self))
1170    else:
1171      return output
1172
1173  def _RunPipedShellCommand(self, script, **kwargs):
1174    PIPESTATUS_LEADER = 'PIPESTATUS: '
1175
1176    script += '; echo "%s${PIPESTATUS[@]}"' % PIPESTATUS_LEADER
1177    kwargs.update(shell=True, check_return=True)
1178    output = self.RunShellCommand(script, **kwargs)
1179    pipestatus_line = output[-1]
1180
1181    if not pipestatus_line.startswith(PIPESTATUS_LEADER):
1182      logger.error('Pipe exit statuses of shell script missing.')
1183      raise device_errors.AdbShellCommandFailedError(
1184          script, output, status=None,
1185          device_serial=self.serial)
1186
1187    output = output[:-1]
1188    statuses = [
1189        int(s) for s in pipestatus_line[len(PIPESTATUS_LEADER):].split()]
1190    if any(statuses):
1191      raise device_errors.AdbShellCommandFailedError(
1192          script, output, status=statuses,
1193          device_serial=self.serial)
1194    return output
1195
1196  @decorators.WithTimeoutAndRetriesFromInstance()
1197  def KillAll(self, process_name, exact=False, signum=device_signal.SIGKILL,
1198              as_root=False, blocking=False, quiet=False,
1199              timeout=None, retries=None):
1200    """Kill all processes with the given name on the device.
1201
1202    Args:
1203      process_name: A string containing the name of the process to kill.
1204      exact: A boolean indicating whether to kill all processes matching
1205             the string |process_name| exactly, or all of those which contain
1206             |process_name| as a substring. Defaults to False.
1207      signum: An integer containing the signal number to send to kill. Defaults
1208              to SIGKILL (9).
1209      as_root: A boolean indicating whether the kill should be executed with
1210               root privileges.
1211      blocking: A boolean indicating whether we should wait until all processes
1212                with the given |process_name| are dead.
1213      quiet: A boolean indicating whether to ignore the fact that no processes
1214             to kill were found.
1215      timeout: timeout in seconds
1216      retries: number of retries
1217
1218    Returns:
1219      The number of processes attempted to kill.
1220
1221    Raises:
1222      CommandFailedError if no process was killed and |quiet| is False.
1223      CommandTimeoutError on timeout.
1224      DeviceUnreachableError on missing device.
1225    """
1226    processes = self.ListProcesses(process_name)
1227    if exact:
1228      processes = [p for p in processes if p.name == process_name]
1229    if not processes:
1230      if quiet:
1231        return 0
1232      else:
1233        raise device_errors.CommandFailedError(
1234            'No processes matching %r (exact=%r)' % (process_name, exact),
1235            str(self))
1236
1237    logger.info(
1238        'KillAll(%r, ...) attempting to kill the following:', process_name)
1239    for p in processes:
1240      logger.info('  %05d %s', p.pid, p.name)
1241
1242    pids = set(p.pid for p in processes)
1243    cmd = ['kill', '-%d' % signum] + sorted(str(p) for p in pids)
1244    self.RunShellCommand(cmd, as_root=as_root, check_return=True)
1245
1246    def all_pids_killed():
1247      pids_left = (p.pid for p in self.ListProcesses(process_name))
1248      return not pids.intersection(pids_left)
1249
1250    if blocking:
1251      timeout_retry.WaitFor(all_pids_killed, wait_period=0.1)
1252
1253    return len(pids)
1254
1255  @decorators.WithTimeoutAndRetriesFromInstance()
1256  def StartActivity(self, intent_obj, blocking=False, trace_file_name=None,
1257                    force_stop=False, timeout=None, retries=None):
1258    """Start package's activity on the device.
1259
1260    Args:
1261      intent_obj: An Intent object to send.
1262      blocking: A boolean indicating whether we should wait for the activity to
1263                finish launching.
1264      trace_file_name: If present, a string that both indicates that we want to
1265                       profile the activity and contains the path to which the
1266                       trace should be saved.
1267      force_stop: A boolean indicating whether we should stop the activity
1268                  before starting it.
1269      timeout: timeout in seconds
1270      retries: number of retries
1271
1272    Raises:
1273      CommandFailedError if the activity could not be started.
1274      CommandTimeoutError on timeout.
1275      DeviceUnreachableError on missing device.
1276    """
1277    cmd = ['am', 'start']
1278    if blocking:
1279      cmd.append('-W')
1280    if trace_file_name:
1281      cmd.extend(['--start-profiler', trace_file_name])
1282    if force_stop:
1283      cmd.append('-S')
1284    cmd.extend(intent_obj.am_args)
1285    for line in self.RunShellCommand(cmd, check_return=True):
1286      if line.startswith('Error:'):
1287        raise device_errors.CommandFailedError(line, str(self))
1288
1289  @decorators.WithTimeoutAndRetriesFromInstance()
1290  def StartService(self, intent_obj, user_id=None, timeout=None, retries=None):
1291    """Start a service on the device.
1292
1293    Args:
1294      intent_obj: An Intent object to send describing the service to start.
1295      user_id: A specific user to start the service as, defaults to current.
1296      timeout: Timeout in seconds.
1297      retries: Number of retries
1298
1299    Raises:
1300      CommandFailedError if the service could not be started.
1301      CommandTimeoutError on timeout.
1302      DeviceUnreachableError on missing device.
1303    """
1304    # For whatever reason, startservice was changed to start-service on O and
1305    # above.
1306    cmd = ['am', 'startservice']
1307    if self.build_version_sdk >= version_codes.OREO:
1308      cmd[1] = 'start-service'
1309    if user_id:
1310      cmd.extend(['--user', str(user_id)])
1311    cmd.extend(intent_obj.am_args)
1312    for line in self.RunShellCommand(cmd, check_return=True):
1313      if line.startswith('Error:'):
1314        raise device_errors.CommandFailedError(line, str(self))
1315
1316  @decorators.WithTimeoutAndRetriesFromInstance()
1317  def StartInstrumentation(self, component, finish=True, raw=False,
1318                           extras=None, timeout=None, retries=None):
1319    if extras is None:
1320      extras = {}
1321
1322    cmd = ['am', 'instrument']
1323    if finish:
1324      cmd.append('-w')
1325    if raw:
1326      cmd.append('-r')
1327    for k, v in extras.iteritems():
1328      cmd.extend(['-e', str(k), str(v)])
1329    cmd.append(component)
1330
1331    # Store the package name in a shell variable to help the command stay under
1332    # the _MAX_ADB_COMMAND_LENGTH limit.
1333    package = component.split('/')[0]
1334    shell_snippet = 'p=%s;%s' % (package,
1335                                 cmd_helper.ShrinkToSnippet(cmd, 'p', package))
1336    return self.RunShellCommand(shell_snippet, shell=True, check_return=True,
1337                                large_output=True)
1338
1339  @decorators.WithTimeoutAndRetriesFromInstance()
1340  def BroadcastIntent(self, intent_obj, timeout=None, retries=None):
1341    """Send a broadcast intent.
1342
1343    Args:
1344      intent: An Intent to broadcast.
1345      timeout: timeout in seconds
1346      retries: number of retries
1347
1348    Raises:
1349      CommandTimeoutError on timeout.
1350      DeviceUnreachableError on missing device.
1351    """
1352    cmd = ['am', 'broadcast'] + intent_obj.am_args
1353    self.RunShellCommand(cmd, check_return=True)
1354
1355  @decorators.WithTimeoutAndRetriesFromInstance()
1356  def GoHome(self, timeout=None, retries=None):
1357    """Return to the home screen and obtain launcher focus.
1358
1359    This command launches the home screen and attempts to obtain
1360    launcher focus until the timeout is reached.
1361
1362    Args:
1363      timeout: timeout in seconds
1364      retries: number of retries
1365
1366    Raises:
1367      CommandTimeoutError on timeout.
1368      DeviceUnreachableError on missing device.
1369    """
1370    def is_launcher_focused():
1371      output = self.RunShellCommand(['dumpsys', 'window', 'windows'],
1372                                    check_return=True, large_output=True)
1373      return any(self._LAUNCHER_FOCUSED_RE.match(l) for l in output)
1374
1375    def dismiss_popups():
1376      # There is a dialog present; attempt to get rid of it.
1377      # Not all dialogs can be dismissed with back.
1378      self.SendKeyEvent(keyevent.KEYCODE_ENTER)
1379      self.SendKeyEvent(keyevent.KEYCODE_BACK)
1380      return is_launcher_focused()
1381
1382    # If Home is already focused, return early to avoid unnecessary work.
1383    if is_launcher_focused():
1384      return
1385
1386    self.StartActivity(
1387        intent.Intent(action='android.intent.action.MAIN',
1388                      category='android.intent.category.HOME'),
1389        blocking=True)
1390
1391    if not is_launcher_focused():
1392      timeout_retry.WaitFor(dismiss_popups, wait_period=1)
1393
1394  @decorators.WithTimeoutAndRetriesFromInstance()
1395  def ForceStop(self, package, timeout=None, retries=None):
1396    """Close the application.
1397
1398    Args:
1399      package: A string containing the name of the package to stop.
1400      timeout: timeout in seconds
1401      retries: number of retries
1402
1403    Raises:
1404      CommandTimeoutError on timeout.
1405      DeviceUnreachableError on missing device.
1406    """
1407    if self.GetApplicationPids(package):
1408      self.RunShellCommand(['am', 'force-stop', package], check_return=True)
1409
1410  @decorators.WithTimeoutAndRetriesFromInstance()
1411  def ClearApplicationState(
1412      self, package, permissions=None, timeout=None, retries=None):
1413    """Clear all state for the given package.
1414
1415    Args:
1416      package: A string containing the name of the package to stop.
1417      permissions: List of permissions to set after clearing data.
1418      timeout: timeout in seconds
1419      retries: number of retries
1420
1421    Raises:
1422      CommandTimeoutError on timeout.
1423      DeviceUnreachableError on missing device.
1424    """
1425    # Check that the package exists before clearing it for android builds below
1426    # JB MR2. Necessary because calling pm clear on a package that doesn't exist
1427    # may never return.
1428    if ((self.build_version_sdk >= version_codes.JELLY_BEAN_MR2)
1429        or self._GetApplicationPathsInternal(package)):
1430      self.RunShellCommand(['pm', 'clear', package], check_return=True)
1431      self.GrantPermissions(package, permissions)
1432
1433  @decorators.WithTimeoutAndRetriesFromInstance()
1434  def SendKeyEvent(self, keycode, timeout=None, retries=None):
1435    """Sends a keycode to the device.
1436
1437    See the devil.android.sdk.keyevent module for suitable keycode values.
1438
1439    Args:
1440      keycode: A integer keycode to send to the device.
1441      timeout: timeout in seconds
1442      retries: number of retries
1443
1444    Raises:
1445      CommandTimeoutError on timeout.
1446      DeviceUnreachableError on missing device.
1447    """
1448    self.RunShellCommand(['input', 'keyevent', format(keycode, 'd')],
1449                         check_return=True)
1450
1451  PUSH_CHANGED_FILES_DEFAULT_TIMEOUT = 10 * _DEFAULT_TIMEOUT
1452
1453  @decorators.WithTimeoutAndRetriesFromInstance(
1454      min_default_timeout=PUSH_CHANGED_FILES_DEFAULT_TIMEOUT)
1455  def PushChangedFiles(self, host_device_tuples, timeout=None,
1456                       retries=None, delete_device_stale=False):
1457    """Push files to the device, skipping files that don't need updating.
1458
1459    When a directory is pushed, it is traversed recursively on the host and
1460    all files in it are pushed to the device as needed.
1461    Additionally, if delete_device_stale option is True,
1462    files that exist on the device but don't exist on the host are deleted.
1463
1464    Args:
1465      host_device_tuples: A list of (host_path, device_path) tuples, where
1466        |host_path| is an absolute path of a file or directory on the host
1467        that should be minimially pushed to the device, and |device_path| is
1468        an absolute path of the destination on the device.
1469      timeout: timeout in seconds
1470      retries: number of retries
1471      delete_device_stale: option to delete stale files on device
1472
1473    Raises:
1474      CommandFailedError on failure.
1475      CommandTimeoutError on timeout.
1476      DeviceUnreachableError on missing device.
1477    """
1478
1479    all_changed_files = []
1480    all_stale_files = []
1481    missing_dirs = set()
1482    cache_commit_funcs = []
1483    for h, d in host_device_tuples:
1484      assert os.path.isabs(h) and posixpath.isabs(d)
1485      h = os.path.realpath(h)
1486      changed_files, up_to_date_files, stale_files, cache_commit_func = (
1487          self._GetChangedAndStaleFiles(h, d, delete_device_stale))
1488      all_changed_files += changed_files
1489      all_stale_files += stale_files
1490      cache_commit_funcs.append(cache_commit_func)
1491      if changed_files and not up_to_date_files and not stale_files:
1492        if os.path.isdir(h):
1493          missing_dirs.add(d)
1494        else:
1495          missing_dirs.add(posixpath.dirname(d))
1496
1497    if delete_device_stale and all_stale_files:
1498      self.RemovePath(all_stale_files, force=True, recursive=True)
1499
1500    if all_changed_files:
1501      if missing_dirs:
1502        try:
1503          self.RunShellCommand(['mkdir', '-p'] + list(missing_dirs),
1504                               check_return=True)
1505        except device_errors.AdbShellCommandFailedError as e:
1506          # TODO(crbug.com/739899): This is attempting to diagnose flaky EBUSY
1507          # errors that have been popping up in single-device scenarios.
1508          # Remove it once we've figured out what's causing them and how best
1509          # to handle them.
1510          m = _EBUSY_RE.search(e.output)
1511          if m:
1512            logging.error(
1513                'Hit EBUSY while attempting to make missing directories.')
1514            logging.error('lsof output:')
1515            # Don't check for return below since grep exits with a non-zero when
1516            # no match is found.
1517            for l in self.RunShellCommand(
1518                'lsof | grep %s' % cmd_helper.SingleQuote(m.group(1)),
1519                check_return=False):
1520              logging.error('  %s', l)
1521          raise
1522      self._PushFilesImpl(host_device_tuples, all_changed_files)
1523    for func in cache_commit_funcs:
1524      func()
1525
1526  def _GetChangedAndStaleFiles(self, host_path, device_path, track_stale=False):
1527    """Get files to push and delete
1528
1529    Args:
1530      host_path: an absolute path of a file or directory on the host
1531      device_path: an absolute path of a file or directory on the device
1532      track_stale: whether to bother looking for stale files (slower)
1533
1534    Returns:
1535      a four-element tuple
1536      1st element: a list of (host_files_path, device_files_path) tuples to push
1537      2nd element: a list of host_files_path that are up-to-date
1538      3rd element: a list of stale files under device_path, or [] when
1539        track_stale == False
1540      4th element: a cache commit function.
1541    """
1542    try:
1543      # Length calculations below assume no trailing /.
1544      host_path = host_path.rstrip('/')
1545      device_path = device_path.rstrip('/')
1546
1547      specific_device_paths = [device_path]
1548      ignore_other_files = not track_stale and os.path.isdir(host_path)
1549      if ignore_other_files:
1550        specific_device_paths = []
1551        for root, _, filenames in os.walk(host_path):
1552          relative_dir = root[len(host_path) + 1:]
1553          specific_device_paths.extend(
1554              posixpath.join(device_path, relative_dir, f) for f in filenames)
1555
1556      def calculate_host_checksums():
1557        return md5sum.CalculateHostMd5Sums([host_path])
1558
1559      def calculate_device_checksums():
1560        if self._enable_device_files_cache:
1561          cache_entry = self._cache['device_path_checksums'].get(device_path)
1562          if cache_entry and cache_entry[0] == ignore_other_files:
1563            return dict(cache_entry[1])
1564
1565        sums = md5sum.CalculateDeviceMd5Sums(specific_device_paths, self)
1566
1567        cache_entry = [ignore_other_files, sums]
1568        self._cache['device_path_checksums'][device_path] = cache_entry
1569        return dict(sums)
1570
1571      host_checksums, device_checksums = reraiser_thread.RunAsync((
1572          calculate_host_checksums,
1573          calculate_device_checksums))
1574    except EnvironmentError as e:
1575      logger.warning('Error calculating md5: %s', e)
1576      return ([(host_path, device_path)], [], [], lambda: 0)
1577
1578    to_push = []
1579    up_to_date = []
1580    to_delete = []
1581    if os.path.isfile(host_path):
1582      host_checksum = host_checksums.get(host_path)
1583      device_checksum = device_checksums.get(device_path)
1584      if host_checksum == device_checksum:
1585        up_to_date.append(host_path)
1586      else:
1587        to_push.append((host_path, device_path))
1588    else:
1589      for host_abs_path, host_checksum in host_checksums.iteritems():
1590        device_abs_path = posixpath.join(
1591            device_path, os.path.relpath(host_abs_path, host_path))
1592        device_checksum = device_checksums.pop(device_abs_path, None)
1593        if device_checksum == host_checksum:
1594          up_to_date.append(host_abs_path)
1595        else:
1596          to_push.append((host_abs_path, device_abs_path))
1597      to_delete = device_checksums.keys()
1598    # We can't rely solely on the checksum approach since it does not catch
1599    # stale directories, which can result in empty directories that cause issues
1600    # during copying in efficient_android_directory_copy.sh. So, find any stale
1601    # directories here so they can be removed in addition to stale files.
1602    if track_stale:
1603      to_delete.extend(self._GetStaleDirectories(host_path, device_path))
1604
1605    def cache_commit_func():
1606      # When host_path is a not a directory, the path.join() call below would
1607      # have an '' as the second argument, causing an unwanted / to be appended.
1608      if os.path.isfile(host_path):
1609        assert len(host_checksums) == 1
1610        new_sums = {device_path: host_checksums[host_path]}
1611      else:
1612        new_sums = {posixpath.join(device_path, path[len(host_path) + 1:]): val
1613                    for path, val in host_checksums.iteritems()}
1614      cache_entry = [ignore_other_files, new_sums]
1615      self._cache['device_path_checksums'][device_path] = cache_entry
1616
1617    return (to_push, up_to_date, to_delete, cache_commit_func)
1618
1619  def _GetStaleDirectories(self, host_path, device_path):
1620    """Gets a list of stale directories on the device.
1621
1622    Args:
1623      host_path: an absolute path of a directory on the host
1624      device_path: an absolute path of a directory on the device
1625
1626    Returns:
1627      A list containing absolute paths to directories on the device that are
1628      considered stale.
1629    """
1630    def get_device_dirs(path):
1631      directories = set()
1632      command = _RECURSIVE_DIRECTORY_LIST_SCRIPT % cmd_helper.SingleQuote(path)
1633      # We use shell=True to evaluate the command as a script through the shell,
1634      # otherwise RunShellCommand tries to interpret it as the name of a (non
1635      # existent) command to run.
1636      for line in self.RunShellCommand(
1637          command, shell=True, check_return=True):
1638        directories.add(posixpath.relpath(posixpath.normpath(line), path))
1639      return directories
1640
1641    def get_host_dirs(path):
1642      directories = set()
1643      if not os.path.isdir(path):
1644        return directories
1645      for root, _, _ in os.walk(path):
1646        if root != path:
1647          # Strip off the top level directory so we can compare the device and
1648          # host.
1649          directories.add(
1650              os.path.relpath(root, path).replace(os.sep, posixpath.sep))
1651      return directories
1652
1653    host_dirs = get_host_dirs(host_path)
1654    device_dirs = get_device_dirs(device_path)
1655    stale_dirs = device_dirs - host_dirs
1656    return [posixpath.join(device_path, d) for d in stale_dirs]
1657
1658  def _ComputeDeviceChecksumsForApks(self, package_name):
1659    ret = self._cache['package_apk_checksums'].get(package_name)
1660    if ret is None:
1661      device_paths = self._GetApplicationPathsInternal(package_name)
1662      file_to_checksums = md5sum.CalculateDeviceMd5Sums(device_paths, self)
1663      ret = set(file_to_checksums.values())
1664      self._cache['package_apk_checksums'][package_name] = ret
1665    return ret
1666
1667  def _ComputeStaleApks(self, package_name, host_apk_paths):
1668    def calculate_host_checksums():
1669      return md5sum.CalculateHostMd5Sums(host_apk_paths)
1670
1671    def calculate_device_checksums():
1672      return self._ComputeDeviceChecksumsForApks(package_name)
1673
1674    host_checksums, device_checksums = reraiser_thread.RunAsync((
1675        calculate_host_checksums, calculate_device_checksums))
1676    stale_apks = [k for (k, v) in host_checksums.iteritems()
1677                  if v not in device_checksums]
1678    return stale_apks, set(host_checksums.values())
1679
1680  def _PushFilesImpl(self, host_device_tuples, files):
1681    if not files:
1682      return
1683
1684    size = sum(host_utils.GetRecursiveDiskUsage(h) for h, _ in files)
1685    file_count = len(files)
1686    dir_size = sum(host_utils.GetRecursiveDiskUsage(h)
1687                   for h, _ in host_device_tuples)
1688    dir_file_count = 0
1689    for h, _ in host_device_tuples:
1690      if os.path.isdir(h):
1691        dir_file_count += sum(len(f) for _r, _d, f in os.walk(h))
1692      else:
1693        dir_file_count += 1
1694
1695    push_duration = self._ApproximateDuration(
1696        file_count, file_count, size, False)
1697    dir_push_duration = self._ApproximateDuration(
1698        len(host_device_tuples), dir_file_count, dir_size, False)
1699    zip_duration = self._ApproximateDuration(1, 1, size, True)
1700
1701    if (dir_push_duration < push_duration and dir_push_duration < zip_duration
1702        # TODO(jbudorick): Resume directory pushing once clients have switched
1703        # to 1.0.36-compatible syntax.
1704        and False):
1705      self._PushChangedFilesIndividually(host_device_tuples)
1706    elif push_duration < zip_duration:
1707      self._PushChangedFilesIndividually(files)
1708    elif self._commands_installed is False:
1709      # Already tried and failed to install unzip command.
1710      self._PushChangedFilesIndividually(files)
1711    elif not self._PushChangedFilesZipped(
1712        files, [d for _, d in host_device_tuples]):
1713      self._PushChangedFilesIndividually(files)
1714
1715  def _MaybeInstallCommands(self):
1716    if self._commands_installed is None:
1717      try:
1718        if not install_commands.Installed(self):
1719          install_commands.InstallCommands(self)
1720        self._commands_installed = True
1721      except device_errors.CommandFailedError as e:
1722        logger.warning('unzip not available: %s', str(e))
1723        self._commands_installed = False
1724    return self._commands_installed
1725
1726  @staticmethod
1727  def _ApproximateDuration(adb_calls, file_count, byte_count, is_zipping):
1728    # We approximate the time to push a set of files to a device as:
1729    #   t = c1 * a + c2 * f + c3 + b / c4 + b / (c5 * c6), where
1730    #     t: total time (sec)
1731    #     c1: adb call time delay (sec)
1732    #     a: number of times adb is called (unitless)
1733    #     c2: push time delay (sec)
1734    #     f: number of files pushed via adb (unitless)
1735    #     c3: zip time delay (sec)
1736    #     c4: zip rate (bytes/sec)
1737    #     b: total number of bytes (bytes)
1738    #     c5: transfer rate (bytes/sec)
1739    #     c6: compression ratio (unitless)
1740
1741    # All of these are approximations.
1742    ADB_CALL_PENALTY = 0.1  # seconds
1743    ADB_PUSH_PENALTY = 0.01  # seconds
1744    ZIP_PENALTY = 2.0  # seconds
1745    ZIP_RATE = 10000000.0  # bytes / second
1746    TRANSFER_RATE = 2000000.0  # bytes / second
1747    COMPRESSION_RATIO = 2.0  # unitless
1748
1749    adb_call_time = ADB_CALL_PENALTY * adb_calls
1750    adb_push_setup_time = ADB_PUSH_PENALTY * file_count
1751    if is_zipping:
1752      zip_time = ZIP_PENALTY + byte_count / ZIP_RATE
1753      transfer_time = byte_count / (TRANSFER_RATE * COMPRESSION_RATIO)
1754    else:
1755      zip_time = 0
1756      transfer_time = byte_count / TRANSFER_RATE
1757    return adb_call_time + adb_push_setup_time + zip_time + transfer_time
1758
1759  def _PushChangedFilesIndividually(self, files):
1760    for h, d in files:
1761      self.adb.Push(h, d)
1762
1763  def _PushChangedFilesZipped(self, files, dirs):
1764    if not self._MaybeInstallCommands():
1765      return False
1766
1767    with tempfile_ext.NamedTemporaryDirectory() as working_dir:
1768      zip_path = os.path.join(working_dir, 'tmp.zip')
1769      try:
1770        zip_utils.WriteZipFile(zip_path, files)
1771      except zip_utils.ZipFailedError:
1772        return False
1773
1774      logger.info('Pushing %d files via .zip of size %d', len(files),
1775                  os.path.getsize(zip_path))
1776      self.NeedsSU()
1777      with device_temp_file.DeviceTempFile(
1778          self.adb, suffix='.zip') as device_temp:
1779        self.adb.Push(zip_path, device_temp.name)
1780
1781        quoted_dirs = ' '.join(cmd_helper.SingleQuote(d) for d in dirs)
1782        self.RunShellCommand(
1783            'unzip %s&&chmod -R 777 %s' % (device_temp.name, quoted_dirs),
1784            shell=True, as_root=True,
1785            env={'PATH': '%s:$PATH' % install_commands.BIN_DIR},
1786            check_return=True)
1787
1788    return True
1789
1790  # TODO(nednguyen): remove this and migrate the callsite to PathExists().
1791  @decorators.WithTimeoutAndRetriesFromInstance()
1792  def FileExists(self, device_path, timeout=None, retries=None):
1793    """Checks whether the given file exists on the device.
1794
1795    Arguments are the same as PathExists.
1796    """
1797    return self.PathExists(device_path, timeout=timeout, retries=retries)
1798
1799  @decorators.WithTimeoutAndRetriesFromInstance()
1800  def PathExists(self, device_paths, as_root=False, timeout=None, retries=None):
1801    """Checks whether the given path(s) exists on the device.
1802
1803    Args:
1804      device_path: A string containing the absolute path to the file on the
1805                   device, or an iterable of paths to check.
1806      as_root: Whether root permissions should be use to check for the existence
1807               of the given path(s).
1808      timeout: timeout in seconds
1809      retries: number of retries
1810
1811    Returns:
1812      True if the all given paths exist on the device, False otherwise.
1813
1814    Raises:
1815      CommandTimeoutError on timeout.
1816      DeviceUnreachableError on missing device.
1817    """
1818    paths = device_paths
1819    if isinstance(paths, basestring):
1820      paths = (paths,)
1821    if not paths:
1822      return True
1823    cmd = ['test', '-e', paths[0]]
1824    for p in paths[1:]:
1825      cmd.extend(['-a', '-e', p])
1826    try:
1827      self.RunShellCommand(cmd, as_root=as_root, check_return=True,
1828                           timeout=timeout, retries=retries)
1829      return True
1830    except device_errors.CommandFailedError:
1831      return False
1832
1833  @decorators.WithTimeoutAndRetriesFromInstance()
1834  def RemovePath(self, device_path, force=False, recursive=False,
1835                 as_root=False, rename=False, timeout=None, retries=None):
1836    """Removes the given path(s) from the device.
1837
1838    Args:
1839      device_path: A string containing the absolute path to the file on the
1840                   device, or an iterable of paths to check.
1841      force: Whether to remove the path(s) with force (-f).
1842      recursive: Whether to remove any directories in the path(s) recursively.
1843      as_root: Whether root permissions should be use to remove the given
1844               path(s).
1845      rename: Whether to rename the path(s) before removing to help avoid
1846            filesystem errors. See https://stackoverflow.com/questions/11539657
1847      timeout: timeout in seconds
1848      retries: number of retries
1849    """
1850    def _RenamePath(path):
1851      random_suffix = hex(random.randint(2 ** 12, 2 ** 16 - 1))[2:]
1852      dest = '%s-%s' % (path, random_suffix)
1853      try:
1854        self.RunShellCommand(
1855            ['mv', path, dest], as_root=as_root, check_return=True)
1856        return dest
1857      except device_errors.AdbShellCommandFailedError:
1858        # If it couldn't be moved, just try rm'ing the original path instead.
1859        return path
1860    args = ['rm']
1861    if force:
1862      args.append('-f')
1863    if recursive:
1864      args.append('-r')
1865    if isinstance(device_path, basestring):
1866      args.append(device_path if not rename else _RenamePath(device_path))
1867    else:
1868      args.extend(
1869          device_path if not rename else [_RenamePath(p) for p in device_path])
1870    self.RunShellCommand(args, as_root=as_root, check_return=True)
1871
1872  @decorators.WithTimeoutAndRetriesFromInstance()
1873  def PullFile(self, device_path, host_path, timeout=None, retries=None):
1874    """Pull a file from the device.
1875
1876    Args:
1877      device_path: A string containing the absolute path of the file to pull
1878                   from the device.
1879      host_path: A string containing the absolute path of the destination on
1880                 the host.
1881      timeout: timeout in seconds
1882      retries: number of retries
1883
1884    Raises:
1885      CommandFailedError on failure.
1886      CommandTimeoutError on timeout.
1887    """
1888    # Create the base dir if it doesn't exist already
1889    dirname = os.path.dirname(host_path)
1890    if dirname and not os.path.exists(dirname):
1891      os.makedirs(dirname)
1892    self.adb.Pull(device_path, host_path)
1893
1894  def _ReadFileWithPull(self, device_path):
1895    try:
1896      d = tempfile.mkdtemp()
1897      host_temp_path = os.path.join(d, 'tmp_ReadFileWithPull')
1898      self.adb.Pull(device_path, host_temp_path)
1899      with open(host_temp_path, 'r') as host_temp:
1900        return host_temp.read()
1901    finally:
1902      if os.path.exists(d):
1903        shutil.rmtree(d)
1904
1905  @decorators.WithTimeoutAndRetriesFromInstance()
1906  def ReadFile(self, device_path, as_root=False, force_pull=False,
1907               timeout=None, retries=None):
1908    """Reads the contents of a file from the device.
1909
1910    Args:
1911      device_path: A string containing the absolute path of the file to read
1912                   from the device.
1913      as_root: A boolean indicating whether the read should be executed with
1914               root privileges.
1915      force_pull: A boolean indicating whether to force the operation to be
1916          performed by pulling a file from the device. The default is, when the
1917          contents are short, to retrieve the contents using cat instead.
1918      timeout: timeout in seconds
1919      retries: number of retries
1920
1921    Returns:
1922      The contents of |device_path| as a string. Contents are intepreted using
1923      universal newlines, so the caller will see them encoded as '\n'. Also,
1924      all lines will be terminated.
1925
1926    Raises:
1927      AdbCommandFailedError if the file can't be read.
1928      CommandTimeoutError on timeout.
1929      DeviceUnreachableError on missing device.
1930    """
1931    def get_size(path):
1932      return self.FileSize(path, as_root=as_root)
1933
1934    if (not force_pull
1935        and 0 < get_size(device_path) <= self._MAX_ADB_OUTPUT_LENGTH):
1936      return _JoinLines(self.RunShellCommand(
1937          ['cat', device_path], as_root=as_root, check_return=True))
1938    elif as_root and self.NeedsSU():
1939      with device_temp_file.DeviceTempFile(self.adb) as device_temp:
1940        cmd = 'SRC=%s DEST=%s;cp "$SRC" "$DEST" && chmod 666 "$DEST"' % (
1941            cmd_helper.SingleQuote(device_path),
1942            cmd_helper.SingleQuote(device_temp.name))
1943        self.RunShellCommand(cmd, shell=True, as_root=True, check_return=True)
1944        return self._ReadFileWithPull(device_temp.name)
1945    else:
1946      return self._ReadFileWithPull(device_path)
1947
1948  def _WriteFileWithPush(self, device_path, contents):
1949    with tempfile.NamedTemporaryFile() as host_temp:
1950      host_temp.write(contents)
1951      host_temp.flush()
1952      self.adb.Push(host_temp.name, device_path)
1953
1954  @decorators.WithTimeoutAndRetriesFromInstance()
1955  def WriteFile(self, device_path, contents, as_root=False, force_push=False,
1956                timeout=None, retries=None):
1957    """Writes |contents| to a file on the device.
1958
1959    Args:
1960      device_path: A string containing the absolute path to the file to write
1961          on the device.
1962      contents: A string containing the data to write to the device.
1963      as_root: A boolean indicating whether the write should be executed with
1964          root privileges (if available).
1965      force_push: A boolean indicating whether to force the operation to be
1966          performed by pushing a file to the device. The default is, when the
1967          contents are short, to pass the contents using a shell script instead.
1968      timeout: timeout in seconds
1969      retries: number of retries
1970
1971    Raises:
1972      CommandFailedError if the file could not be written on the device.
1973      CommandTimeoutError on timeout.
1974      DeviceUnreachableError on missing device.
1975    """
1976    if not force_push and len(contents) < self._MAX_ADB_COMMAND_LENGTH:
1977      # If the contents are small, for efficieny we write the contents with
1978      # a shell command rather than pushing a file.
1979      cmd = 'echo -n %s > %s' % (cmd_helper.SingleQuote(contents),
1980                                 cmd_helper.SingleQuote(device_path))
1981      self.RunShellCommand(cmd, shell=True, as_root=as_root, check_return=True)
1982    elif as_root and self.NeedsSU():
1983      # Adb does not allow to "push with su", so we first push to a temp file
1984      # on a safe location, and then copy it to the desired location with su.
1985      with device_temp_file.DeviceTempFile(self.adb) as device_temp:
1986        self._WriteFileWithPush(device_temp.name, contents)
1987        # Here we need 'cp' rather than 'mv' because the temp and
1988        # destination files might be on different file systems (e.g.
1989        # on internal storage and an external sd card).
1990        self.RunShellCommand(['cp', device_temp.name, device_path],
1991                             as_root=True, check_return=True)
1992    else:
1993      # If root is not needed, we can push directly to the desired location.
1994      self._WriteFileWithPush(device_path, contents)
1995
1996  def _ParseLongLsOutput(self, device_path, as_root=False, **kwargs):
1997    """Run and scrape the output of 'ls -a -l' on a device directory."""
1998    device_path = posixpath.join(device_path, '')  # Force trailing '/'.
1999    output = self.RunShellCommand(
2000        ['ls', '-a', '-l', device_path], as_root=as_root,
2001        check_return=True, env={'TZ': 'utc'}, **kwargs)
2002    if output and output[0].startswith('total '):
2003      output.pop(0) # pylint: disable=maybe-no-member
2004
2005    entries = []
2006    for line in output:
2007      m = _LONG_LS_OUTPUT_RE.match(line)
2008      if m:
2009        if m.group('filename') not in ['.', '..']:
2010          item = m.groupdict()
2011          # A change in toybox is causing recent Android versions to escape
2012          # spaces in file names. Here we just unquote those spaces. If we
2013          # later find more essoteric characters in file names, a more careful
2014          # unquoting mechanism may be needed. But hopefully not.
2015          # See: https://goo.gl/JAebZj
2016          item['filename'] = item['filename'].replace('\\ ', ' ')
2017          entries.append(item)
2018      else:
2019        logger.info('Skipping: %s', line)
2020
2021    return entries
2022
2023  def ListDirectory(self, device_path, as_root=False, **kwargs):
2024    """List all files on a device directory.
2025
2026    Mirroring os.listdir (and most client expectations) the resulting list
2027    does not include the special entries '.' and '..' even if they are present
2028    in the directory.
2029
2030    Args:
2031      device_path: A string containing the path of the directory on the device
2032                   to list.
2033      as_root: A boolean indicating whether the to use root privileges to list
2034               the directory contents.
2035      timeout: timeout in seconds
2036      retries: number of retries
2037
2038    Returns:
2039      A list of filenames for all entries contained in the directory.
2040
2041    Raises:
2042      AdbCommandFailedError if |device_path| does not specify a valid and
2043          accessible directory in the device.
2044      CommandTimeoutError on timeout.
2045      DeviceUnreachableError on missing device.
2046    """
2047    entries = self._ParseLongLsOutput(device_path, as_root=as_root, **kwargs)
2048    return [d['filename'] for d in entries]
2049
2050  def StatDirectory(self, device_path, as_root=False, **kwargs):
2051    """List file and stat info for all entries on a device directory.
2052
2053    Implementation notes: this is currently implemented by parsing the output
2054    of 'ls -a -l' on the device. Whether possible and convenient, we attempt to
2055    make parsing strict and return values mirroring those of the standard |os|
2056    and |stat| Python modules.
2057
2058    Mirroring os.listdir (and most client expectations) the resulting list
2059    does not include the special entries '.' and '..' even if they are present
2060    in the directory.
2061
2062    Args:
2063      device_path: A string containing the path of the directory on the device
2064                   to list.
2065      as_root: A boolean indicating whether the to use root privileges to list
2066               the directory contents.
2067      timeout: timeout in seconds
2068      retries: number of retries
2069
2070    Returns:
2071      A list of dictionaries, each containing the following keys:
2072        filename: A string with the file name.
2073        st_mode: File permissions, use the stat module to interpret these.
2074        st_nlink: Number of hard links (may be missing).
2075        st_owner: A string with the user name of the owner.
2076        st_group: A string with the group name of the owner.
2077        st_rdev_pair: Device type as (major, minior) (only if inode device).
2078        st_size: Size of file, in bytes (may be missing for non-regular files).
2079        st_mtime: Time of most recent modification, in seconds since epoch
2080          (although resolution is in minutes).
2081        symbolic_link_to: If entry is a symbolic link, path where it points to;
2082          missing otherwise.
2083
2084    Raises:
2085      AdbCommandFailedError if |device_path| does not specify a valid and
2086          accessible directory in the device.
2087      CommandTimeoutError on timeout.
2088      DeviceUnreachableError on missing device.
2089    """
2090    entries = self._ParseLongLsOutput(device_path, as_root=as_root, **kwargs)
2091    for d in entries:
2092      for key, value in d.items():
2093        if value is None:
2094          del d[key]  # Remove missing fields.
2095      d['st_mode'] = _ParseModeString(d['st_mode'])
2096      d['st_mtime'] = calendar.timegm(
2097          time.strptime(d['st_mtime'], _LS_DATE_FORMAT))
2098      for key in ['st_nlink', 'st_size', 'st_rdev_major', 'st_rdev_minor']:
2099        if key in d:
2100          d[key] = int(d[key])
2101      if 'st_rdev_major' in d and 'st_rdev_minor' in d:
2102        d['st_rdev_pair'] = (d.pop('st_rdev_major'), d.pop('st_rdev_minor'))
2103    return entries
2104
2105  def StatPath(self, device_path, as_root=False, **kwargs):
2106    """Get the stat attributes of a file or directory on the device.
2107
2108    Args:
2109      device_path: A string containing the path of a file or directory from
2110                   which to get attributes.
2111      as_root: A boolean indicating whether the to use root privileges to
2112               access the file information.
2113      timeout: timeout in seconds
2114      retries: number of retries
2115
2116    Returns:
2117      A dictionary with the stat info collected; see StatDirectory for details.
2118
2119    Raises:
2120      CommandFailedError if device_path cannot be found on the device.
2121      CommandTimeoutError on timeout.
2122      DeviceUnreachableError on missing device.
2123    """
2124    dirname, filename = posixpath.split(posixpath.normpath(device_path))
2125    for entry in self.StatDirectory(dirname, as_root=as_root, **kwargs):
2126      if entry['filename'] == filename:
2127        return entry
2128    raise device_errors.CommandFailedError(
2129        'Cannot find file or directory: %r' % device_path, str(self))
2130
2131  def FileSize(self, device_path, as_root=False, **kwargs):
2132    """Get the size of a file on the device.
2133
2134    Note: This is implemented by parsing the output of the 'ls' command on
2135    the device. On some Android versions, when passing a directory or special
2136    file, the size is *not* reported and this function will throw an exception.
2137
2138    Args:
2139      device_path: A string containing the path of a file on the device.
2140      as_root: A boolean indicating whether the to use root privileges to
2141               access the file information.
2142      timeout: timeout in seconds
2143      retries: number of retries
2144
2145    Returns:
2146      The size of the file in bytes.
2147
2148    Raises:
2149      CommandFailedError if device_path cannot be found on the device, or
2150        its size cannot be determited for some reason.
2151      CommandTimeoutError on timeout.
2152      DeviceUnreachableError on missing device.
2153    """
2154    entry = self.StatPath(device_path, as_root=as_root, **kwargs)
2155    try:
2156      return entry['st_size']
2157    except KeyError:
2158      raise device_errors.CommandFailedError(
2159          'Could not determine the size of: %s' % device_path, str(self))
2160
2161  @decorators.WithTimeoutAndRetriesFromInstance()
2162  def SetJavaAsserts(self, enabled, timeout=None, retries=None):
2163    """Enables or disables Java asserts.
2164
2165    Args:
2166      enabled: A boolean indicating whether Java asserts should be enabled
2167               or disabled.
2168      timeout: timeout in seconds
2169      retries: number of retries
2170
2171    Returns:
2172      True if the device-side property changed and a restart is required as a
2173      result, False otherwise.
2174
2175    Raises:
2176      CommandTimeoutError on timeout.
2177    """
2178    def find_property(lines, property_name):
2179      for index, line in enumerate(lines):
2180        if line.strip() == '':
2181          continue
2182        key_value = tuple(s.strip() for s in line.split('=', 1))
2183        if len(key_value) != 2:
2184          continue
2185        key, value = key_value
2186        if key == property_name:
2187          return index, value
2188      return None, ''
2189
2190    new_value = 'all' if enabled else ''
2191
2192    # First ensure the desired property is persisted.
2193    try:
2194      properties = self.ReadFile(self.LOCAL_PROPERTIES_PATH).splitlines()
2195    except device_errors.CommandFailedError:
2196      properties = []
2197    index, value = find_property(properties, self.JAVA_ASSERT_PROPERTY)
2198    if new_value != value:
2199      if new_value:
2200        new_line = '%s=%s' % (self.JAVA_ASSERT_PROPERTY, new_value)
2201        if index is None:
2202          properties.append(new_line)
2203        else:
2204          properties[index] = new_line
2205      else:
2206        assert index is not None  # since new_value == '' and new_value != value
2207        properties.pop(index)
2208      self.WriteFile(self.LOCAL_PROPERTIES_PATH, _JoinLines(properties))
2209
2210    # Next, check the current runtime value is what we need, and
2211    # if not, set it and report that a reboot is required.
2212    value = self.GetProp(self.JAVA_ASSERT_PROPERTY)
2213    if new_value != value:
2214      self.SetProp(self.JAVA_ASSERT_PROPERTY, new_value)
2215      return True
2216    else:
2217      return False
2218
2219  def GetLanguage(self, cache=False):
2220    """Returns the language setting on the device.
2221    Args:
2222      cache: Whether to use cached properties when available.
2223    """
2224    return self.GetProp('persist.sys.language', cache=cache)
2225
2226  def GetCountry(self, cache=False):
2227    """Returns the country setting on the device.
2228
2229    Args:
2230      cache: Whether to use cached properties when available.
2231    """
2232    return self.GetProp('persist.sys.country', cache=cache)
2233
2234  @property
2235  def screen_density(self):
2236    """Returns the screen density of the device."""
2237    DPI_TO_DENSITY = {
2238      120: 'ldpi',
2239      160: 'mdpi',
2240      240: 'hdpi',
2241      320: 'xhdpi',
2242      480: 'xxhdpi',
2243      640: 'xxxhdpi',
2244    }
2245    return DPI_TO_DENSITY.get(self.pixel_density, 'tvdpi')
2246
2247  @property
2248  def pixel_density(self):
2249    return int(self.GetProp('ro.sf.lcd_density', cache=True))
2250
2251  @property
2252  def build_description(self):
2253    """Returns the build description of the system.
2254
2255    For example:
2256      nakasi-user 4.4.4 KTU84P 1227136 release-keys
2257    """
2258    return self.GetProp('ro.build.description', cache=True)
2259
2260  @property
2261  def build_fingerprint(self):
2262    """Returns the build fingerprint of the system.
2263
2264    For example:
2265      google/nakasi/grouper:4.4.4/KTU84P/1227136:user/release-keys
2266    """
2267    return self.GetProp('ro.build.fingerprint', cache=True)
2268
2269  @property
2270  def build_id(self):
2271    """Returns the build ID of the system (e.g. 'KTU84P')."""
2272    return self.GetProp('ro.build.id', cache=True)
2273
2274  @property
2275  def build_product(self):
2276    """Returns the build product of the system (e.g. 'grouper')."""
2277    return self.GetProp('ro.build.product', cache=True)
2278
2279  @property
2280  def build_type(self):
2281    """Returns the build type of the system (e.g. 'user')."""
2282    return self.GetProp('ro.build.type', cache=True)
2283
2284  @property
2285  def build_version_sdk(self):
2286    """Returns the build version sdk of the system as a number (e.g. 19).
2287
2288    For version code numbers see:
2289    http://developer.android.com/reference/android/os/Build.VERSION_CODES.html
2290
2291    For named constants see devil.android.sdk.version_codes
2292
2293    Raises:
2294      CommandFailedError if the build version sdk is not a number.
2295    """
2296    value = self.GetProp('ro.build.version.sdk', cache=True)
2297    try:
2298      return int(value)
2299    except ValueError:
2300      raise device_errors.CommandFailedError(
2301          'Invalid build version sdk: %r' % value)
2302
2303  @property
2304  def product_cpu_abi(self):
2305    """Returns the product cpu abi of the device (e.g. 'armeabi-v7a')."""
2306    return self.GetProp('ro.product.cpu.abi', cache=True)
2307
2308  @property
2309  def product_model(self):
2310    """Returns the name of the product model (e.g. 'Nexus 7')."""
2311    return self.GetProp('ro.product.model', cache=True)
2312
2313  @property
2314  def product_name(self):
2315    """Returns the product name of the device (e.g. 'nakasi')."""
2316    return self.GetProp('ro.product.name', cache=True)
2317
2318  @property
2319  def product_board(self):
2320    """Returns the product board name of the device (e.g. 'shamu')."""
2321    return self.GetProp('ro.product.board', cache=True)
2322
2323  def _EnsureCacheInitialized(self):
2324    """Populates cache token, runs getprop and fetches $EXTERNAL_STORAGE."""
2325    if self._cache['token']:
2326      return
2327    with self._cache_lock:
2328      if self._cache['token']:
2329        return
2330      # Change the token every time to ensure that it will match only the
2331      # previously dumped cache.
2332      token = str(uuid.uuid1())
2333      cmd = (
2334          'c=/data/local/tmp/cache_token;'
2335          'echo $EXTERNAL_STORAGE;'
2336          'cat $c 2>/dev/null||echo;'
2337          'echo "%s">$c &&' % token +
2338          'getprop'
2339      )
2340      output = self.RunShellCommand(
2341          cmd, shell=True, check_return=True, large_output=True)
2342      # Error-checking for this existing is done in GetExternalStoragePath().
2343      self._cache['external_storage'] = output[0]
2344      self._cache['prev_token'] = output[1]
2345      output = output[2:]
2346
2347      prop_cache = self._cache['getprop']
2348      prop_cache.clear()
2349      for key, value in _GETPROP_RE.findall(''.join(output)):
2350        prop_cache[key] = value
2351      self._cache['token'] = token
2352
2353  @decorators.WithTimeoutAndRetriesFromInstance()
2354  def GetProp(self, property_name, cache=False, timeout=None, retries=None):
2355    """Gets a property from the device.
2356
2357    Args:
2358      property_name: A string containing the name of the property to get from
2359                     the device.
2360      cache: Whether to use cached properties when available.
2361      timeout: timeout in seconds
2362      retries: number of retries
2363
2364    Returns:
2365      The value of the device's |property_name| property.
2366
2367    Raises:
2368      CommandTimeoutError on timeout.
2369    """
2370    assert isinstance(property_name, basestring), (
2371        "property_name is not a string: %r" % property_name)
2372
2373    if cache:
2374      # It takes ~120ms to query a single property, and ~130ms to query all
2375      # properties. So, when caching we always query all properties.
2376      self._EnsureCacheInitialized()
2377    else:
2378      # timeout and retries are handled down at run shell, because we don't
2379      # want to apply them in the other branch when reading from the cache
2380      value = self.RunShellCommand(
2381          ['getprop', property_name], single_line=True, check_return=True,
2382          timeout=timeout, retries=retries)
2383      self._cache['getprop'][property_name] = value
2384    # Non-existent properties are treated as empty strings by getprop.
2385    return self._cache['getprop'].get(property_name, '')
2386
2387  @decorators.WithTimeoutAndRetriesFromInstance()
2388  def SetProp(self, property_name, value, check=False, timeout=None,
2389              retries=None):
2390    """Sets a property on the device.
2391
2392    Args:
2393      property_name: A string containing the name of the property to set on
2394                     the device.
2395      value: A string containing the value to set to the property on the
2396             device.
2397      check: A boolean indicating whether to check that the property was
2398             successfully set on the device.
2399      timeout: timeout in seconds
2400      retries: number of retries
2401
2402    Raises:
2403      CommandFailedError if check is true and the property was not correctly
2404        set on the device (e.g. because it is not rooted).
2405      CommandTimeoutError on timeout.
2406    """
2407    assert isinstance(property_name, basestring), (
2408        "property_name is not a string: %r" % property_name)
2409    assert isinstance(value, basestring), "value is not a string: %r" % value
2410
2411    self.RunShellCommand(['setprop', property_name, value], check_return=True)
2412    prop_cache = self._cache['getprop']
2413    if property_name in prop_cache:
2414      del prop_cache[property_name]
2415    # TODO(perezju) remove the option and make the check mandatory, but using a
2416    # single shell script to both set- and getprop.
2417    if check and value != self.GetProp(property_name, cache=False):
2418      raise device_errors.CommandFailedError(
2419          'Unable to set property %r on the device to %r'
2420          % (property_name, value), str(self))
2421
2422  @decorators.WithTimeoutAndRetriesFromInstance()
2423  def GetABI(self, timeout=None, retries=None):
2424    """Gets the device main ABI.
2425
2426    Args:
2427      timeout: timeout in seconds
2428      retries: number of retries
2429
2430    Returns:
2431      The device's main ABI name.
2432
2433    Raises:
2434      CommandTimeoutError on timeout.
2435    """
2436    return self.GetProp('ro.product.cpu.abi', cache=True)
2437
2438  def _GetPsOutput(self, pattern):
2439    """Runs |ps| command on the device and returns its output,
2440
2441    This private method abstracts away differences between Android verions for
2442    calling |ps|, and implements support for filtering the output by a given
2443    |pattern|, but does not do any output parsing.
2444    """
2445    try:
2446      ps_cmd = 'ps'
2447      # ps behavior was changed in Android O and above, http://crbug.com/686716
2448      if self.build_version_sdk >= version_codes.OREO:
2449        ps_cmd = 'ps -e'
2450      if pattern:
2451        return self._RunPipedShellCommand(
2452            '%s | grep -F %s' % (ps_cmd, cmd_helper.SingleQuote(pattern)))
2453      else:
2454        return self.RunShellCommand(
2455            ps_cmd.split(), check_return=True, large_output=True)
2456    except device_errors.AdbShellCommandFailedError as e:
2457      if e.status and isinstance(e.status, list) and not e.status[0]:
2458        # If ps succeeded but grep failed, there were no processes with the
2459        # given name.
2460        return []
2461      else:
2462        raise
2463
2464  @decorators.WithTimeoutAndRetriesFromInstance()
2465  def ListProcesses(self, process_name=None, timeout=None, retries=None):
2466    """Returns a list of tuples with info about processes on the device.
2467
2468    This essentially parses the output of the |ps| command into convenient
2469    ProcessInfo tuples.
2470
2471    Args:
2472      process_name: A string used to filter the returned processes. If given,
2473                    only processes whose name have this value as a substring
2474                    will be returned.
2475      timeout: timeout in seconds
2476      retries: number of retries
2477
2478    Returns:
2479      A list of ProcessInfo tuples with |name|, |pid|, and |ppid| fields.
2480    """
2481    process_name = process_name or ''
2482    processes = []
2483    for line in self._GetPsOutput(process_name):
2484      row = line.split()
2485      try:
2486        row = {k: row[i] for k, i in _PS_COLUMNS.iteritems()}
2487        if row['pid'] == 'PID' or process_name not in row['name']:
2488          # Skip over header and non-matching processes.
2489          continue
2490        row['pid'] = int(row['pid'])
2491        row['ppid'] = int(row['ppid'])
2492      except StandardError:  # e.g. IndexError, TypeError, ValueError.
2493        logging.warning('failed to parse ps line: %r', line)
2494        continue
2495      processes.append(ProcessInfo(**row))
2496    return processes
2497
2498  def _GetDumpsysOutput(self, extra_args, pattern=None):
2499    """Runs |dumpsys| command on the device and returns its output.
2500
2501    This private method implements support for filtering the output by a given
2502    |pattern|, but does not do any output parsing.
2503    """
2504    try:
2505      cmd = ['dumpsys'] + extra_args
2506      if pattern:
2507        cmd = ' '.join(cmd_helper.SingleQuote(s) for s in cmd)
2508        return self._RunPipedShellCommand(
2509            '%s | grep -F %s' % (cmd, cmd_helper.SingleQuote(pattern)))
2510      else:
2511        cmd = ['dumpsys'] + extra_args
2512        return self.RunShellCommand(cmd, check_return=True, large_output=True)
2513    except device_errors.AdbShellCommandFailedError as e:
2514      if e.status and isinstance(e.status, list) and not e.status[0]:
2515        # If dumpsys succeeded but grep failed, there were no lines matching
2516        # the given pattern.
2517        return []
2518      else:
2519        raise
2520
2521  # TODO(#4103): Remove after migrating clients to ListProcesses.
2522  @decorators.WithTimeoutAndRetriesFromInstance()
2523  def GetPids(self, process_name=None, timeout=None, retries=None):
2524    """Returns the PIDs of processes containing the given name as substring.
2525
2526    DEPRECATED
2527
2528    Note that the |process_name| is often the package name.
2529
2530    Args:
2531      process_name: A string containing the process name to get the PIDs for.
2532                    If missing returns PIDs for all processes.
2533      timeout: timeout in seconds
2534      retries: number of retries
2535
2536    Returns:
2537      A dict mapping process name to a list of PIDs for each process that
2538      contained the provided |process_name|.
2539
2540    Raises:
2541      CommandTimeoutError on timeout.
2542      DeviceUnreachableError on missing device.
2543    """
2544    procs_pids = collections.defaultdict(list)
2545    for p in self.ListProcesses(process_name):
2546      procs_pids[p.name].append(str(p.pid))
2547    return procs_pids
2548
2549  @decorators.WithTimeoutAndRetriesFromInstance()
2550  def GetApplicationPids(self, process_name, at_most_one=False,
2551                         timeout=None, retries=None):
2552    """Returns the PID or PIDs of a given process name.
2553
2554    Note that the |process_name|, often the package name, must match exactly.
2555
2556    Args:
2557      process_name: A string containing the process name to get the PIDs for.
2558      at_most_one: A boolean indicating that at most one PID is expected to
2559                   be found.
2560      timeout: timeout in seconds
2561      retries: number of retries
2562
2563    Returns:
2564      A list of the PIDs for the named process. If at_most_one=True returns
2565      the single PID found or None otherwise.
2566
2567    Raises:
2568      CommandFailedError if at_most_one=True and more than one PID is found
2569          for the named process.
2570      CommandTimeoutError on timeout.
2571      DeviceUnreachableError on missing device.
2572    """
2573    pids = [p.pid for p in self.ListProcesses(process_name)
2574            if p.name == process_name]
2575    if at_most_one:
2576      if len(pids) > 1:
2577        raise device_errors.CommandFailedError(
2578            'Expected a single PID for %r but found: %r.' % (
2579                process_name, pids),
2580            device_serial=str(self))
2581      return pids[0] if pids else None
2582    else:
2583      return pids
2584
2585  @decorators.WithTimeoutAndRetriesFromInstance()
2586  def GetEnforce(self, timeout=None, retries=None):
2587    """Get the current mode of SELinux.
2588
2589    Args:
2590      timeout: timeout in seconds
2591      retries: number of retries
2592
2593    Returns:
2594      True (enforcing), False (permissive), or None (disabled).
2595
2596    Raises:
2597      CommandFailedError on failure.
2598      CommandTimeoutError on timeout.
2599      DeviceUnreachableError on missing device.
2600    """
2601    output = self.RunShellCommand(
2602        ['getenforce'], check_return=True, single_line=True).lower()
2603    if output not in _SELINUX_MODE:
2604      raise device_errors.CommandFailedError(
2605          'Unexpected getenforce output: %s' % output)
2606    return _SELINUX_MODE[output]
2607
2608  @decorators.WithTimeoutAndRetriesFromInstance()
2609  def SetEnforce(self, enabled, timeout=None, retries=None):
2610    """Modify the mode SELinux is running in.
2611
2612    Args:
2613      enabled: a boolean indicating whether to put SELinux in encorcing mode
2614               (if True), or permissive mode (otherwise).
2615      timeout: timeout in seconds
2616      retries: number of retries
2617
2618    Raises:
2619      CommandFailedError on failure.
2620      CommandTimeoutError on timeout.
2621      DeviceUnreachableError on missing device.
2622    """
2623    self.RunShellCommand(
2624        ['setenforce', '1' if int(enabled) else '0'], as_root=True,
2625        check_return=True)
2626
2627  @decorators.WithTimeoutAndRetriesFromInstance()
2628  def SetWebViewImplementation(self, package_name, timeout=None, retries=None):
2629    """Select the WebView implementation to the specified package.
2630
2631    Args:
2632      package_name: The package name of a WebView implementation. The package
2633        must be already installed on the device.
2634      timeout: timeout in seconds
2635      retries: number of retries
2636
2637    Raises:
2638      CommandFailedError on failure.
2639      CommandTimeoutError on timeout.
2640      DeviceUnreachableError on missing device.
2641    """
2642    output = self.RunShellCommand(
2643        ['cmd', 'webviewupdate', 'set-webview-implementation', package_name],
2644        single_line=True, check_return=True)
2645    if output == 'Success':
2646      logging.info('WebView provider set to: %s', package_name)
2647    else:
2648      raise device_errors.CommandFailedError(
2649          'Error setting WebView provider: %s' % output, str(self))
2650
2651  @decorators.WithTimeoutAndRetriesFromInstance()
2652  def TakeScreenshot(self, host_path=None, timeout=None, retries=None):
2653    """Takes a screenshot of the device.
2654
2655    Args:
2656      host_path: A string containing the path on the host to save the
2657                 screenshot to. If None, a file name in the current
2658                 directory will be generated.
2659      timeout: timeout in seconds
2660      retries: number of retries
2661
2662    Returns:
2663      The name of the file on the host to which the screenshot was saved.
2664
2665    Raises:
2666      CommandFailedError on failure.
2667      CommandTimeoutError on timeout.
2668      DeviceUnreachableError on missing device.
2669    """
2670    if not host_path:
2671      host_path = os.path.abspath('screenshot-%s-%s.png' % (
2672          self.serial, _GetTimeStamp()))
2673    with device_temp_file.DeviceTempFile(self.adb, suffix='.png') as device_tmp:
2674      self.RunShellCommand(['/system/bin/screencap', '-p', device_tmp.name],
2675                           check_return=True)
2676      self.PullFile(device_tmp.name, host_path)
2677    return host_path
2678
2679  @decorators.WithTimeoutAndRetriesFromInstance()
2680  def DismissCrashDialogIfNeeded(self, timeout=None, retries=None):
2681    """Dismiss the error/ANR dialog if present.
2682
2683    Returns: Name of the crashed package if a dialog is focused,
2684             None otherwise.
2685    """
2686    def _FindFocusedWindow():
2687      match = None
2688      # TODO(jbudorick): Try to grep the output on the device instead of using
2689      # large_output if/when DeviceUtils exposes a public interface for piped
2690      # shell command handling.
2691      for line in self.RunShellCommand(['dumpsys', 'window', 'windows'],
2692                                       check_return=True, large_output=True):
2693        match = re.match(_CURRENT_FOCUS_CRASH_RE, line)
2694        if match:
2695          break
2696      return match
2697
2698    match = _FindFocusedWindow()
2699    if not match:
2700      return None
2701    package = match.group(2)
2702    logger.warning('Trying to dismiss %s dialog for %s', *match.groups())
2703    self.SendKeyEvent(keyevent.KEYCODE_DPAD_RIGHT)
2704    self.SendKeyEvent(keyevent.KEYCODE_DPAD_RIGHT)
2705    self.SendKeyEvent(keyevent.KEYCODE_ENTER)
2706    match = _FindFocusedWindow()
2707    if match:
2708      logger.error('Still showing a %s dialog for %s', *match.groups())
2709    return package
2710
2711  def GetLogcatMonitor(self, *args, **kwargs):
2712    """Returns a new LogcatMonitor associated with this device.
2713
2714    Parameters passed to this function are passed directly to
2715    |logcat_monitor.LogcatMonitor| and are documented there.
2716    """
2717    return logcat_monitor.LogcatMonitor(self.adb, *args, **kwargs)
2718
2719  def GetClientCache(self, client_name):
2720    """Returns client cache."""
2721    if client_name not in self._client_caches:
2722      self._client_caches[client_name] = {}
2723    return self._client_caches[client_name]
2724
2725  def _ClearCache(self):
2726    """Clears all caches."""
2727    for client in self._client_caches:
2728      self._client_caches[client].clear()
2729    self._cache = {
2730        # Map of packageId -> list of on-device .apk paths
2731        'package_apk_paths': {},
2732        # Set of packageId that were loaded from LoadCacheData and not yet
2733        # verified.
2734        'package_apk_paths_to_verify': set(),
2735        # Map of packageId -> set of on-device .apk checksums
2736        'package_apk_checksums': {},
2737        # Map of property_name -> value
2738        'getprop': {},
2739        # Map of device_path -> [ignore_other_files, map of path->checksum]
2740        'device_path_checksums': {},
2741        # Location of sdcard ($EXTERNAL_STORAGE).
2742        'external_storage': None,
2743        # Token used to detect when LoadCacheData is stale.
2744        'token': None,
2745        'prev_token': None,
2746    }
2747
2748  @decorators.WithTimeoutAndRetriesFromInstance()
2749  def LoadCacheData(self, data, timeout=None, retries=None):
2750    """Initializes the cache from data created using DumpCacheData.
2751
2752    The cache is used only if its token matches the one found on the device.
2753    This prevents a stale cache from being used (which can happen when sharing
2754    devices).
2755
2756    Args:
2757      data: A previously serialized cache (string).
2758      timeout: timeout in seconds
2759      retries: number of retries
2760
2761    Returns:
2762      Whether the cache was loaded.
2763    """
2764    obj = json.loads(data)
2765    self._EnsureCacheInitialized()
2766    given_token = obj.get('token')
2767    if not given_token or self._cache['prev_token'] != given_token:
2768      logger.warning('Stale cache detected. Not using it.')
2769      return False
2770
2771    self._cache['package_apk_paths'] = obj.get('package_apk_paths', {})
2772    # When using a cache across script invokations, verify that apps have
2773    # not been uninstalled.
2774    self._cache['package_apk_paths_to_verify'] = set(
2775        self._cache['package_apk_paths'].iterkeys())
2776
2777    package_apk_checksums = obj.get('package_apk_checksums', {})
2778    for k, v in package_apk_checksums.iteritems():
2779      package_apk_checksums[k] = set(v)
2780    self._cache['package_apk_checksums'] = package_apk_checksums
2781    device_path_checksums = obj.get('device_path_checksums', {})
2782    self._cache['device_path_checksums'] = device_path_checksums
2783    return True
2784
2785  @decorators.WithTimeoutAndRetriesFromInstance()
2786  def DumpCacheData(self, timeout=None, retries=None):
2787    """Dumps the current cache state to a string.
2788
2789    Args:
2790      timeout: timeout in seconds
2791      retries: number of retries
2792
2793    Returns:
2794      A serialized cache as a string.
2795    """
2796    self._EnsureCacheInitialized()
2797    obj = {}
2798    obj['token'] = self._cache['token']
2799    obj['package_apk_paths'] = self._cache['package_apk_paths']
2800    obj['package_apk_checksums'] = self._cache['package_apk_checksums']
2801    # JSON can't handle sets.
2802    for k, v in obj['package_apk_checksums'].iteritems():
2803      obj['package_apk_checksums'][k] = list(v)
2804    obj['device_path_checksums'] = self._cache['device_path_checksums']
2805    return json.dumps(obj, separators=(',', ':'))
2806
2807  @classmethod
2808  def parallel(cls, devices, async=False):
2809    """Creates a Parallelizer to operate over the provided list of devices.
2810
2811    Args:
2812      devices: A list of either DeviceUtils instances or objects from
2813               from which DeviceUtils instances can be constructed. If None,
2814               all attached devices will be used.
2815      async: If true, returns a Parallelizer that runs operations
2816             asynchronously.
2817
2818    Returns:
2819      A Parallelizer operating over |devices|.
2820    """
2821    devices = [d if isinstance(d, cls) else cls(d) for d in devices]
2822    if async:
2823      return parallelizer.Parallelizer(devices)
2824    else:
2825      return parallelizer.SyncParallelizer(devices)
2826
2827  @classmethod
2828  def HealthyDevices(cls, blacklist=None, device_arg='default', retries=1,
2829                     abis=None, **kwargs):
2830    """Returns a list of DeviceUtils instances.
2831
2832    Returns a list of DeviceUtils instances that are attached, not blacklisted,
2833    and optionally filtered by --device flags or ANDROID_SERIAL environment
2834    variable.
2835
2836    Args:
2837      blacklist: A DeviceBlacklist instance (optional). Device serials in this
2838          blacklist will never be returned, but a warning will be logged if they
2839          otherwise would have been.
2840      device_arg: The value of the --device flag. This can be:
2841          'default' -> Same as [], but returns an empty list rather than raise a
2842              NoDevicesError.
2843          [] -> Returns all devices, unless $ANDROID_SERIAL is set.
2844          None -> Use $ANDROID_SERIAL if set, otherwise looks for a single
2845              attached device. Raises an exception if multiple devices are
2846              attached.
2847          'serial' -> Returns an instance for the given serial, if not
2848              blacklisted.
2849          ['A', 'B', ...] -> Returns instances for the subset that is not
2850              blacklisted.
2851      retries: Number of times to restart adb server and query it again if no
2852          devices are found on the previous attempts, with exponential backoffs
2853          up to 60s between each retry.
2854      abis: A list of ABIs for which the device needs to support at least one of
2855          (optional).
2856      A device serial, or a list of device serials (optional).
2857
2858    Returns:
2859      A list of DeviceUtils instances.
2860
2861    Raises:
2862      NoDevicesError: Raised when no non-blacklisted devices exist and
2863          device_arg is passed.
2864      MultipleDevicesError: Raise when multiple devices exist, but |device_arg|
2865          is None.
2866    """
2867    allow_no_devices = False
2868    if device_arg == 'default':
2869      allow_no_devices = True
2870      device_arg = ()
2871
2872    select_multiple = True
2873    if not (isinstance(device_arg, tuple) or isinstance(device_arg, list)):
2874      select_multiple = False
2875      if device_arg:
2876        device_arg = (device_arg,)
2877
2878    blacklisted_devices = blacklist.Read() if blacklist else []
2879
2880    # adb looks for ANDROID_SERIAL, so support it as well.
2881    android_serial = os.environ.get('ANDROID_SERIAL')
2882    if not device_arg and android_serial:
2883      device_arg = (android_serial,)
2884
2885    def blacklisted(serial):
2886      if serial in blacklisted_devices:
2887        logger.warning('Device %s is blacklisted.', serial)
2888        return True
2889      return False
2890
2891    def supports_abi(abi, serial):
2892      if abis and abi not in abis:
2893        logger.warning("Device %s doesn't support required ABIs.", serial)
2894        return False
2895      return True
2896
2897    def _get_devices():
2898      if device_arg:
2899        devices = [cls(x, **kwargs) for x in device_arg if not blacklisted(x)]
2900      else:
2901        devices = []
2902        for adb in adb_wrapper.AdbWrapper.Devices():
2903          serial = adb.GetDeviceSerial()
2904          if not blacklisted(serial):
2905            device = cls(_CreateAdbWrapper(adb), **kwargs)
2906            if supports_abi(device.GetABI(), serial):
2907              devices.append(device)
2908
2909      if len(devices) == 0 and not allow_no_devices:
2910        raise device_errors.NoDevicesError()
2911      if len(devices) > 1 and not select_multiple:
2912        raise device_errors.MultipleDevicesError(devices)
2913      return sorted(devices)
2914
2915    for attempt in xrange(retries+1):
2916      try:
2917        return _get_devices()
2918      except device_errors.NoDevicesError:
2919        if attempt == retries:
2920          logging.error('No devices found after exhausting all retries.')
2921          raise
2922        # math.pow returns floats, so cast to int for easier testing
2923        sleep_s = min(int(math.pow(2, attempt + 1)), 60)
2924        logger.warning(
2925            'No devices found. Will try again after restarting adb server '
2926            'and a short nap of %d s.', sleep_s)
2927        time.sleep(sleep_s)
2928        RestartServer()
2929
2930  @decorators.WithTimeoutAndRetriesFromInstance()
2931  def RestartAdbd(self, timeout=None, retries=None):
2932    logger.info('Restarting adbd on device.')
2933    with device_temp_file.DeviceTempFile(self.adb, suffix='.sh') as script:
2934      self.WriteFile(script.name, _RESTART_ADBD_SCRIPT)
2935      self.RunShellCommand(
2936          ['source', script.name], check_return=True, as_root=True)
2937      self.adb.WaitForDevice()
2938
2939  @decorators.WithTimeoutAndRetriesFromInstance()
2940  def GrantPermissions(self, package, permissions, timeout=None, retries=None):
2941    # Permissions only need to be set on M and above because of the changes to
2942    # the permission model.
2943    if not permissions or self.build_version_sdk < version_codes.MARSHMALLOW:
2944      return
2945
2946    permissions = set(
2947        p for p in permissions if not _PERMISSIONS_BLACKLIST_RE.match(p))
2948
2949    if ('android.permission.WRITE_EXTERNAL_STORAGE' in permissions
2950        and 'android.permission.READ_EXTERNAL_STORAGE' not in permissions):
2951      permissions.add('android.permission.READ_EXTERNAL_STORAGE')
2952
2953    script = ';'.join([
2954      'p={package}',
2955      'for q in {permissions}',
2956      'do pm grant "$p" "$q"',
2957      'echo "{sep}$q{sep}$?{sep}"',
2958      'done'
2959    ]).format(
2960        package=cmd_helper.SingleQuote(package),
2961        permissions=' '.join(
2962            cmd_helper.SingleQuote(p) for p in sorted(permissions)),
2963        sep=_SHELL_OUTPUT_SEPARATOR)
2964
2965    logger.info('Setting permissions for %s.', package)
2966    res = self.RunShellCommand(
2967        script, shell=True, raw_output=True, large_output=True,
2968        check_return=True)
2969    res = res.split(_SHELL_OUTPUT_SEPARATOR)
2970    failures = [
2971      (permission, output.strip())
2972      for permission, status, output in zip(res[1::3], res[2::3], res[0::3])
2973      if int(status)]
2974
2975    if failures:
2976      logger.warning(
2977          'Failed to grant some permissions. Blacklist may need to be updated?')
2978      for permission, output in failures:
2979        # Try to grab the relevant error message from the output.
2980        m = _PERMISSIONS_EXCEPTION_RE.search(output)
2981        if m:
2982          error_msg = m.group(0)
2983        elif len(output) > 200:
2984          error_msg = repr(output[:200]) + ' (truncated)'
2985        else:
2986          error_msg = repr(output)
2987        logger.warning('- %s: %s', permission, error_msg)
2988
2989  @decorators.WithTimeoutAndRetriesFromInstance()
2990  def IsScreenOn(self, timeout=None, retries=None):
2991    """Determines if screen is on.
2992
2993    Dumpsys input_method exposes screen on/off state. Below is an explination of
2994    the states.
2995
2996    Pre-L:
2997      On: mScreenOn=true
2998      Off: mScreenOn=false
2999    L+:
3000      On: mInteractive=true
3001      Off: mInteractive=false
3002
3003    Returns:
3004      True if screen is on, false if it is off.
3005
3006    Raises:
3007      device_errors.CommandFailedError: If screen state cannot be found.
3008    """
3009    if self.build_version_sdk < version_codes.LOLLIPOP:
3010      input_check = 'mScreenOn'
3011      check_value = 'mScreenOn=true'
3012    else:
3013      input_check = 'mInteractive'
3014      check_value = 'mInteractive=true'
3015    dumpsys_out = self._RunPipedShellCommand(
3016        'dumpsys input_method | grep %s' % input_check)
3017    if not dumpsys_out:
3018      raise device_errors.CommandFailedError(
3019          'Unable to detect screen state', str(self))
3020    return check_value in dumpsys_out[0]
3021
3022  @decorators.WithTimeoutAndRetriesFromInstance()
3023  def SetScreen(self, on, timeout=None, retries=None):
3024    """Turns screen on and off.
3025
3026    Args:
3027      on: bool to decide state to switch to. True = on False = off.
3028    """
3029    def screen_test():
3030      return self.IsScreenOn() == on
3031
3032    if screen_test():
3033      logger.info('Screen already in expected state.')
3034      return
3035    self.SendKeyEvent(keyevent.KEYCODE_POWER)
3036    timeout_retry.WaitFor(screen_test, wait_period=1)
3037
3038  @decorators.WithTimeoutAndRetriesFromInstance()
3039  def ChangeOwner(self, owner_group, paths, timeout=None, retries=None):
3040    """Changes file system ownership for permissions.
3041
3042    Args:
3043      owner_group: New owner and group to assign. Note that this should be a
3044        string in the form user[.group] where the group is option.
3045      paths: Paths to change ownership of.
3046
3047      Note that the -R recursive option is not supported by all Android
3048      versions.
3049    """
3050    if not paths:
3051      return
3052    self.RunShellCommand(['chown', owner_group] + paths, check_return=True)
3053
3054  @decorators.WithTimeoutAndRetriesFromInstance()
3055  def ChangeSecurityContext(self, security_context, paths, timeout=None,
3056                            retries=None):
3057    """Changes the SELinux security context for files.
3058
3059    Args:
3060      security_context: The new security context as a string
3061      paths: Paths to change the security context of.
3062
3063      Note that the -R recursive option is not supported by all Android
3064      versions.
3065    """
3066    if not paths:
3067      return
3068    command = ['chcon', security_context] + paths
3069
3070    # Note, need to force su because chcon can fail with permission errors even
3071    # if the device is rooted.
3072    self.RunShellCommand(command, as_root=_FORCE_SU, check_return=True)
3073