• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (C) 2012 Google Inc. All rights reserved.
2#
3# Redistribution and use in source and binary forms, with or without
4# modification, are permitted provided that the following conditions are
5# met:
6#
7#     * Redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer.
9#     * Redistributions in binary form must reproduce the above
10# copyright notice, this list of conditions and the following disclaimer
11# in the documentation and/or other materials provided with the
12# distribution.
13#     * Neither the name of Google Inc. nor the names of its
14# contributors may be used to endorse or promote products derived from
15# this software without specific prior written permission.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29import copy
30import logging
31import os
32import re
33import signal
34import sys
35import subprocess
36import threading
37import time
38
39from multiprocessing.pool import ThreadPool
40
41from webkitpy.common.system.executive import ScriptError
42from webkitpy.layout_tests.breakpad.dump_reader_multipart import DumpReaderAndroid
43from webkitpy.layout_tests.models import test_run_results
44from webkitpy.layout_tests.port import base
45from webkitpy.layout_tests.port import linux
46from webkitpy.layout_tests.port import driver
47from webkitpy.layout_tests.port import factory
48from webkitpy.layout_tests.port import server_process
49from webkitpy.common.system.profiler import SingleFileOutputProfiler
50
51_log = logging.getLogger(__name__)
52
53# The root directory for test resources, which has the same structure as the
54# source root directory of Chromium.
55# This path is defined in Chromium's base/test/test_support_android.cc.
56DEVICE_SOURCE_ROOT_DIR = '/data/local/tmp/'
57
58# The layout tests directory on device, which has two usages:
59# 1. as a virtual path in file urls that will be bridged to HTTP.
60# 2. pointing to some files that are pushed to the device for tests that
61# don't work on file-over-http (e.g. blob protocol tests).
62DEVICE_WEBKIT_BASE_DIR = DEVICE_SOURCE_ROOT_DIR + 'third_party/WebKit/'
63DEVICE_LAYOUT_TESTS_DIR = DEVICE_WEBKIT_BASE_DIR + 'LayoutTests/'
64
65SCALING_GOVERNORS_PATTERN = "/sys/devices/system/cpu/cpu*/cpufreq/scaling_governor"
66KPTR_RESTRICT_PATH = "/proc/sys/kernel/kptr_restrict"
67
68# All the test cases are still served to the test runner through file protocol,
69# but we use a file-to-http feature to bridge the file request to host's http
70# server to get the real test files and corresponding resources.
71# See webkit/support/platform_support_android.cc for the other side of this bridge.
72PERF_TEST_PATH_PREFIX = '/all-perf-tests'
73LAYOUT_TEST_PATH_PREFIX = '/all-tests'
74
75# All ports the Android forwarder to forward.
76# 8000, 8080 and 8443 are for http/https tests.
77# 8880 and 9323 are for websocket tests
78# (see http_server.py, apache_http_server.py and websocket_server.py).
79FORWARD_PORTS = '8000 8080 8443 8880 9323'
80
81MS_TRUETYPE_FONTS_DIR = '/usr/share/fonts/truetype/msttcorefonts/'
82MS_TRUETYPE_FONTS_PACKAGE = 'ttf-mscorefonts-installer'
83
84# Timeout in seconds to wait for starting/stopping the driver.
85DRIVER_START_STOP_TIMEOUT_SECS = 10
86
87HOST_FONT_FILES = [
88    [[MS_TRUETYPE_FONTS_DIR], 'Arial.ttf', MS_TRUETYPE_FONTS_PACKAGE],
89    [[MS_TRUETYPE_FONTS_DIR], 'Arial_Bold.ttf', MS_TRUETYPE_FONTS_PACKAGE],
90    [[MS_TRUETYPE_FONTS_DIR], 'Arial_Bold_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE],
91    [[MS_TRUETYPE_FONTS_DIR], 'Arial_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE],
92    [[MS_TRUETYPE_FONTS_DIR], 'Comic_Sans_MS.ttf', MS_TRUETYPE_FONTS_PACKAGE],
93    [[MS_TRUETYPE_FONTS_DIR], 'Comic_Sans_MS_Bold.ttf', MS_TRUETYPE_FONTS_PACKAGE],
94    [[MS_TRUETYPE_FONTS_DIR], 'Courier_New.ttf', MS_TRUETYPE_FONTS_PACKAGE],
95    [[MS_TRUETYPE_FONTS_DIR], 'Courier_New_Bold.ttf', MS_TRUETYPE_FONTS_PACKAGE],
96    [[MS_TRUETYPE_FONTS_DIR], 'Courier_New_Bold_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE],
97    [[MS_TRUETYPE_FONTS_DIR], 'Courier_New_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE],
98    [[MS_TRUETYPE_FONTS_DIR], 'Georgia.ttf', MS_TRUETYPE_FONTS_PACKAGE],
99    [[MS_TRUETYPE_FONTS_DIR], 'Georgia_Bold.ttf', MS_TRUETYPE_FONTS_PACKAGE],
100    [[MS_TRUETYPE_FONTS_DIR], 'Georgia_Bold_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE],
101    [[MS_TRUETYPE_FONTS_DIR], 'Georgia_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE],
102    [[MS_TRUETYPE_FONTS_DIR], 'Impact.ttf', MS_TRUETYPE_FONTS_PACKAGE],
103    [[MS_TRUETYPE_FONTS_DIR], 'Trebuchet_MS.ttf', MS_TRUETYPE_FONTS_PACKAGE],
104    [[MS_TRUETYPE_FONTS_DIR], 'Trebuchet_MS_Bold.ttf', MS_TRUETYPE_FONTS_PACKAGE],
105    [[MS_TRUETYPE_FONTS_DIR], 'Trebuchet_MS_Bold_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE],
106    [[MS_TRUETYPE_FONTS_DIR], 'Trebuchet_MS_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE],
107    [[MS_TRUETYPE_FONTS_DIR], 'Times_New_Roman.ttf', MS_TRUETYPE_FONTS_PACKAGE],
108    [[MS_TRUETYPE_FONTS_DIR], 'Times_New_Roman_Bold.ttf', MS_TRUETYPE_FONTS_PACKAGE],
109    [[MS_TRUETYPE_FONTS_DIR], 'Times_New_Roman_Bold_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE],
110    [[MS_TRUETYPE_FONTS_DIR], 'Times_New_Roman_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE],
111    [[MS_TRUETYPE_FONTS_DIR], 'Verdana.ttf', MS_TRUETYPE_FONTS_PACKAGE],
112    [[MS_TRUETYPE_FONTS_DIR], 'Verdana_Bold.ttf', MS_TRUETYPE_FONTS_PACKAGE],
113    [[MS_TRUETYPE_FONTS_DIR], 'Verdana_Bold_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE],
114    [[MS_TRUETYPE_FONTS_DIR], 'Verdana_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE],
115    # The Microsoft font EULA
116    [['/usr/share/doc/ttf-mscorefonts-installer/'], 'READ_ME!.gz', MS_TRUETYPE_FONTS_PACKAGE],
117    # Other fonts: Arabic, CJK, Indic, Thai, etc.
118    [['/usr/share/fonts/truetype/ttf-dejavu/'], 'DejaVuSans.ttf', 'ttf-dejavu'],
119    [['/usr/share/fonts/truetype/kochi/'], 'kochi-mincho.ttf', 'ttf-kochi-mincho'],
120    [['/usr/share/fonts/truetype/ttf-indic-fonts-core/'], 'lohit_hi.ttf', 'ttf-indic-fonts-core'],
121    [['/usr/share/fonts/truetype/ttf-indic-fonts-core/'], 'lohit_ta.ttf', 'ttf-indic-fonts-core'],
122    [['/usr/share/fonts/truetype/ttf-indic-fonts-core/'], 'MuktiNarrow.ttf', 'ttf-indic-fonts-core'],
123    [['/usr/share/fonts/truetype/thai/', '/usr/share/fonts/truetype/tlwg/'], 'Garuda.ttf', 'fonts-tlwg-garuda'],
124    [['/usr/share/fonts/truetype/ttf-indic-fonts-core/', '/usr/share/fonts/truetype/ttf-punjabi-fonts/'], 'lohit_pa.ttf', 'ttf-indic-fonts-core'],
125]
126
127# Test resources that need to be accessed as files directly.
128# Each item can be the relative path of a directory or a file.
129TEST_RESOURCES_TO_PUSH = [
130    # Blob tests need to access files directly.
131    'editing/pasteboard/resources',
132    'fast/files/resources',
133    'http/tests/local/resources',
134    'http/tests/local/formdata/resources',
135    # User style URLs are accessed as local files in webkit_support.
136    'http/tests/security/resources/cssStyle.css',
137    # Media tests need to access audio/video as files.
138    'media/content',
139    'compositing/resources/video.mp4',
140]
141
142MD5SUM_DEVICE_FILE_NAME = 'md5sum_bin'
143MD5SUM_HOST_FILE_NAME = 'md5sum_bin_host'
144MD5SUM_DEVICE_PATH = '/data/local/tmp/' + MD5SUM_DEVICE_FILE_NAME
145
146
147# Information required when running layout tests using content_shell as the test runner.
148class ContentShellDriverDetails():
149    def device_cache_directory(self):
150        return self.device_directory() + 'cache/'
151
152    def device_fonts_directory(self):
153        return self.device_directory() + 'fonts/'
154
155    def device_forwarder_path(self):
156        return self.device_directory() + 'forwarder'
157
158    def device_fifo_directory(self):
159        return '/data/data/' + self.package_name() + '/files/'
160
161    def apk_name(self):
162        return 'apks/ContentShell.apk'
163
164    def package_name(self):
165        return 'org.chromium.content_shell_apk'
166
167    def activity_name(self):
168        return self.package_name() + '/.ContentShellActivity'
169
170    def library_name(self):
171        return 'libcontent_shell_content_view.so'
172
173    def additional_resources(self):
174        return ['content_resources.pak', 'content_shell.pak', 'shell_resources.pak']
175
176    def command_line_file(self):
177        return '/data/local/tmp/content-shell-command-line'
178
179    def device_crash_dumps_directory(self):
180        return '/data/local/tmp/content-shell-crash-dumps'
181
182    def additional_command_line_flags(self, use_breakpad):
183        flags = ['--dump-render-tree', '--encode-binary']
184        if use_breakpad:
185            flags.extend(['--enable-crash-reporter', '--crash-dumps-dir=%s' % self.device_crash_dumps_directory()])
186        return flags
187
188    def device_directory(self):
189        return DEVICE_SOURCE_ROOT_DIR + 'content_shell/'
190
191
192# The AndroidCommands class encapsulates commands to communicate with an attached device.
193class AndroidCommands(object):
194    _adb_command_path = None
195    _adb_command_path_options = []
196
197    def __init__(self, executive, device_serial, debug_logging):
198        self._executive = executive
199        self._device_serial = device_serial
200        self._debug_logging = debug_logging
201
202    # Local public methods.
203
204    def file_exists(self, full_path):
205        assert full_path.startswith('/')
206        return self.run(['shell', 'ls', '-d', full_path]).strip() == full_path
207
208    def push(self, host_path, device_path, ignore_error=False):
209        return self.run(['push', host_path, device_path], ignore_error=ignore_error)
210
211    def pull(self, device_path, host_path, ignore_error=False):
212        return self.run(['pull', device_path, host_path], ignore_error=ignore_error)
213
214    def mkdir(self, device_path, chmod=None):
215        self.run(['shell', 'mkdir', '-p', device_path])
216        if chmod:
217            self.run(['shell', 'chmod', chmod, device_path])
218
219    def restart_adb(self):
220        pids = self.extract_pids('adbd')
221        if pids:
222            output = self.run(['shell', 'kill', '-' + str(signal.SIGTERM)] + pids)
223        self.run(['wait-for-device'])
224
225    def restart_as_root(self):
226        output = self.run(['root'])
227        if 'adbd is already running as root' in output:
228            return
229
230        elif not 'restarting adbd as root' in output:
231            self._log_error('Unrecognized output from adb root: %s' % output)
232
233        self.run(['wait-for-device'])
234
235    def extract_pids(self, process_name):
236        pids = []
237        output = self.run(['shell', 'ps'])
238        for line in output.splitlines():
239            data = line.split()
240            try:
241                if process_name in data[-1]:  # name is in the last column
242                    if process_name == data[-1]:
243                        pids.insert(0, data[1])  # PID is in the second column
244                    else:
245                        pids.append(data[1])
246            except IndexError:
247                pass
248        return pids
249
250    def run(self, command, ignore_error=False):
251        self._log_debug('Run adb command: ' + str(command))
252        if ignore_error:
253            error_handler = self._executive.ignore_error
254        else:
255            error_handler = None
256
257        result = self._executive.run_command(self.adb_command() + command, error_handler=error_handler, debug_logging=self._debug_logging)
258
259        # We limit the length to avoid outputting too verbose commands, such as "adb logcat".
260        # Also make sure that the output is ascii-encoded to avoid confusing other parts of
261        # the system.
262        self._log_debug('Run adb result: ' + result[:80].encode('ascii', errors='replace'))
263        return result
264
265    def get_serial(self):
266        return self._device_serial
267
268    def adb_command(self):
269        return [AndroidCommands.adb_command_path(self._executive, self._debug_logging), '-s', self._device_serial]
270
271    @staticmethod
272    def set_adb_command_path_options(paths):
273        AndroidCommands._adb_command_path_options = paths
274
275    @staticmethod
276    def adb_command_path(executive, debug_logging):
277        if AndroidCommands._adb_command_path:
278            return AndroidCommands._adb_command_path
279
280        assert AndroidCommands._adb_command_path_options, 'No commands paths have been set to look for the "adb" command.'
281
282        command_path = None
283        command_version = None
284        for path_option in AndroidCommands._adb_command_path_options:
285            path_version = AndroidCommands._determine_adb_version(path_option, executive, debug_logging)
286            if not path_version:
287                continue
288            if command_version != None and path_version < command_version:
289                continue
290
291            command_path = path_option
292            command_version = path_version
293
294        assert command_path, 'Unable to locate the "adb" command. Are you using an Android checkout of Chromium?'
295
296        AndroidCommands._adb_command_path = command_path
297        return command_path
298
299    # Local private methods.
300
301    def _log_error(self, message):
302        _log.error('[%s] %s' % (self._device_serial, message))
303
304    def _log_info(self, message):
305        _log.info('[%s] %s' % (self._device_serial, message))
306
307    def _log_debug(self, message):
308        if self._debug_logging:
309            _log.debug('[%s] %s' % (self._device_serial, message))
310
311    @staticmethod
312    def _determine_adb_version(adb_command_path, executive, debug_logging):
313        re_version = re.compile('^.*version ([\d\.]+)$')
314        try:
315            output = executive.run_command([adb_command_path, 'version'], error_handler=executive.ignore_error,
316                                           debug_logging=debug_logging)
317        except OSError:
318            return None
319
320        result = re_version.match(output)
321        if not output or not result:
322            return None
323
324        return [int(n) for n in result.group(1).split('.')]
325
326
327# A class to encapsulate device status and information, such as the AndroidCommands
328# instances and whether the device has been set up.
329class AndroidDevices(object):
330    # Percentage of battery a device needs to have in order for it to be considered
331    # to participate in running the layout tests.
332    MINIMUM_BATTERY_PERCENTAGE = 30
333
334    def __init__(self, executive, default_device=None, debug_logging=False):
335        self._usable_devices = []
336        self._default_device = default_device
337        self._prepared_devices = []
338        self._debug_logging = debug_logging
339
340    def prepared_devices(self):
341        return self._prepared_devices
342
343    def usable_devices(self, executive):
344        if self._usable_devices:
345            return self._usable_devices
346
347        if self._default_device:
348            self._usable_devices = [AndroidCommands(executive, self._default_device, self._debug_logging)]
349            return self._usable_devices
350
351        # Example "adb devices" command output:
352        #   List of devices attached
353        #   0123456789ABCDEF        device
354        re_device = re.compile('^([a-zA-Z0-9_:.-]+)\tdevice$', re.MULTILINE)
355
356        result = executive.run_command([AndroidCommands.adb_command_path(executive, debug_logging=self._debug_logging), 'devices'],
357                                       error_handler=executive.ignore_error, debug_logging=self._debug_logging)
358        devices = re_device.findall(result)
359        if not devices:
360            return []
361
362        for device_serial in sorted(devices):
363            commands = AndroidCommands(executive, device_serial, self._debug_logging)
364            if self._battery_level_for_device(commands) < AndroidDevices.MINIMUM_BATTERY_PERCENTAGE:
365                _log.warning('Device with serial "%s" skipped because it has less than %d percent battery.'
366                    % (commands.get_serial(), AndroidDevices.MINIMUM_BATTERY_PERCENTAGE))
367                continue
368
369            if not self._is_device_screen_on(commands):
370                _log.warning('Device with serial "%s" skipped because the screen must be on.' % commands.get_serial())
371                continue
372
373            self._usable_devices.append(commands)
374
375        return self._usable_devices
376
377    def get_device(self, executive, device_index):
378        devices = self.usable_devices(executive)
379        if device_index >= len(devices):
380            raise AssertionError('Device index exceeds number of usable devices.')
381
382        return devices[device_index]
383
384    def is_device_prepared(self, device_serial):
385        return device_serial in self._prepared_devices
386
387    def set_device_prepared(self, device_serial):
388        self._prepared_devices.append(device_serial)
389
390    # Private methods
391    def _battery_level_for_device(self, commands):
392        battery_status = commands.run(['shell', 'dumpsys', 'battery'])
393        if 'Error' in battery_status or "Can't find service: battery" in battery_status:
394            _log.warning('Unable to read the battery level from device with serial "%s".' % commands.get_serial())
395            return 0
396
397        return int(re.findall('level: (\d+)', battery_status)[0])
398
399    def _is_device_screen_on(self, commands):
400        power_status = commands.run(['shell', 'dumpsys', 'power'])
401        return 'mScreenOn=true' in power_status or 'mScreenOn=SCREEN_ON_BIT' in power_status or 'Display Power: state=ON' in power_status
402
403
404class AndroidPort(base.Port):
405    port_name = 'android'
406
407    # Avoid initializing the adb path [worker count]+1 times by storing it as a static member.
408    _adb_path = None
409
410    SUPPORTED_VERSIONS = ('android')
411
412    FALLBACK_PATHS = {'icecreamsandwich': ['android'] + linux.LinuxPort.latest_platform_fallback_path()}
413
414    # Android has aac and mp3 codecs built in.
415    PORT_HAS_AUDIO_CODECS_BUILT_IN = True
416
417    BUILD_REQUIREMENTS_URL = 'https://code.google.com/p/chromium/wiki/AndroidBuildInstructions'
418
419    def __init__(self, host, port_name, **kwargs):
420        super(AndroidPort, self).__init__(host, port_name, **kwargs)
421
422        self._operating_system = 'android'
423        self._version = 'icecreamsandwich'
424
425        self._host_port = factory.PortFactory(host).get('chromium', **kwargs)
426        self._server_process_constructor = self._android_server_process_constructor
427
428        if not self.get_option('disable_breakpad'):
429            self._dump_reader = DumpReaderAndroid(host, self._build_path())
430
431        if self.driver_name() != self.CONTENT_SHELL_NAME:
432            raise AssertionError('Layout tests on Android only support content_shell as the driver.')
433
434        self._driver_details = ContentShellDriverDetails()
435
436        # Initialize the AndroidDevices class which tracks available devices.
437        default_device = None
438        if hasattr(self._options, 'adb_device') and len(self._options.adb_device):
439            default_device = self._options.adb_device
440
441        self._debug_logging = self.get_option('android_logging')
442        self._devices = AndroidDevices(self._executive, default_device, self._debug_logging)
443
444        # Tell AndroidCommands where to search for the "adb" command.
445        AndroidCommands.set_adb_command_path_options(['adb',
446            self.path_from_chromium_base('third_party', 'android_tools', 'sdk', 'platform-tools', 'adb')])
447
448        prepared_devices = self.get_option('prepared_devices', [])
449        for serial in prepared_devices:
450            self._devices.set_device_prepared(serial)
451
452    def default_smoke_test_only(self):
453        return True
454
455    # Local public methods.
456    def path_to_forwarder(self):
457        return self._build_path('forwarder')
458
459    def path_to_md5sum(self):
460        return self._build_path(MD5SUM_DEVICE_FILE_NAME)
461
462    def path_to_md5sum_host(self):
463        return self._build_path(MD5SUM_HOST_FILE_NAME)
464
465    def additional_drt_flag(self):
466        return self._driver_details.additional_command_line_flags(use_breakpad=not self.get_option('disable_breakpad'))
467
468    def default_timeout_ms(self):
469        # Android platform has less computing power than desktop platforms.
470        # Using 10 seconds allows us to pass most slow tests which are not
471        # marked as slow tests on desktop platforms.
472        return 10 * 1000
473
474    def driver_stop_timeout(self):
475        # The driver doesn't respond to closing stdin, so we might as well stop the driver immediately.
476        return 0.0
477
478    def default_child_processes(self):
479        usable_devices = self._devices.usable_devices(self._executive)
480        if not usable_devices:
481            raise test_run_results.TestRunException(test_run_results.NO_DEVICES_EXIT_STATUS, "Unable to find any attached Android devices.")
482        return len(usable_devices)
483
484    def check_wdiff(self, logging=True):
485        return self._host_port.check_wdiff(logging)
486
487    def check_build(self, needs_http, printer):
488        exit_status = super(AndroidPort, self).check_build(needs_http, printer)
489        if exit_status:
490            return exit_status
491
492        result = self._check_file_exists(self.path_to_md5sum(), 'md5sum utility')
493        result = self._check_file_exists(self.path_to_md5sum_host(), 'md5sum host utility') and result
494        result = self._check_file_exists(self.path_to_forwarder(), 'forwarder utility') and result
495
496        if not result:
497            # There is a race condition in adb at least <= 4.3 on Linux that causes it to go offline periodically
498            # We set the processor affinity for any running adb process to attempt to work around this.
499            # See crbug.com/268450
500            if self.host.platform.is_linux():
501                pids = self._executive.running_pids(lambda name: 'adb' in name)
502                if not pids:
503                    # Apparently adb is not running, which is unusual. Running any adb command should start it.
504                    self._executive.run_command(['adb', 'devices'])
505                    pids = self._executive.running_pids(lambda name: 'adb' in name)
506                if not pids:
507                    _log.error("The adb daemon does not appear to be running.")
508                    return False
509
510                for pid in pids:
511                    self._executive.run_command(['taskset', '-p', '-c', '0', str(pid)])
512
513        if not result:
514            _log.error('For complete Android build requirements, please see:')
515            _log.error('')
516            _log.error('    http://code.google.com/p/chromium/wiki/AndroidBuildInstructions')
517            return test_run_results.UNEXPECTED_ERROR_EXIT_STATUS
518
519        return self._check_devices(printer)
520
521    def _check_devices(self, printer):
522        # Printer objects aren't threadsafe, so we need to protect calls to them.
523        lock = threading.Lock()
524        pool = None
525
526        # Push the executables and other files to the devices; doing this now
527        # means we can do this in parallel in the manager process and not mix
528        # this in with starting and stopping workers.
529        def setup_device(worker_number):
530            d = self.create_driver(worker_number)
531            serial = d._android_commands.get_serial()
532
533            def log_safely(msg, throttled=True):
534                if throttled:
535                    callback = printer.write_throttled_update
536                else:
537                    callback = printer.write_update
538                lock.acquire()
539                try:
540                    callback("[%s] %s" % (serial, msg))
541                finally:
542                    lock.release()
543
544            log_safely("preparing device", throttled=False)
545            try:
546                d._setup_test(log_safely)
547                log_safely("device prepared", throttled=False)
548            except (ScriptError, driver.DeviceFailure) as e:
549                lock.acquire()
550                _log.warning("[%s] failed to prepare_device: %s" % (serial, str(e)))
551                lock.release()
552            except KeyboardInterrupt:
553                if pool:
554                    pool.terminate()
555
556        # FIXME: It would be nice if we knew how many workers we needed.
557        num_workers = self.default_child_processes()
558        num_child_processes = int(self.get_option('child_processes'))
559        if num_child_processes:
560            num_workers = min(num_workers, num_child_processes)
561        if num_workers > 1:
562            pool = ThreadPool(num_workers)
563            try:
564                pool.map(setup_device, range(num_workers))
565            except KeyboardInterrupt:
566                pool.terminate()
567                raise
568        else:
569            setup_device(0)
570
571        if not self._devices.prepared_devices():
572            _log.error('Could not prepare any devices for testing.')
573            return test_run_results.NO_DEVICES_EXIT_STATUS
574        return test_run_results.OK_EXIT_STATUS
575
576    def setup_test_run(self):
577        super(AndroidPort, self).setup_test_run()
578
579        # By setting this on the options object, we can propagate the list
580        # of prepared devices to the workers (it is read in __init__()).
581        if self._devices._prepared_devices:
582            self._options.prepared_devices = self._devices.prepared_devices()
583        else:
584            # We were called with --no-build, so assume the devices are up to date.
585            self._options.prepared_devices = [d.get_serial() for d in self._devices.usable_devices(self.host.executive)]
586
587    def num_workers(self, requested_num_workers):
588        return min(len(self._options.prepared_devices), requested_num_workers)
589
590    def check_sys_deps(self, needs_http):
591        for (font_dirs, font_file, package) in HOST_FONT_FILES:
592            exists = False
593            for font_dir in font_dirs:
594                font_path = font_dir + font_file
595                if self._check_file_exists(font_path, '', logging=False):
596                    exists = True
597                    break
598            if not exists:
599                _log.error('You are missing %s under %s. Try installing %s. See build instructions.' % (font_file, font_dirs, package))
600                return test_run_results.SYS_DEPS_EXIT_STATUS
601        return test_run_results.OK_EXIT_STATUS
602
603    def requires_http_server(self):
604        """Chromium Android runs tests on devices, and uses the HTTP server to
605        serve the actual layout tests to the test driver."""
606        return True
607
608    def start_http_server(self, additional_dirs, number_of_drivers):
609        additional_dirs[PERF_TEST_PATH_PREFIX] = self.perf_tests_dir()
610        additional_dirs[LAYOUT_TEST_PATH_PREFIX] = self.layout_tests_dir()
611        super(AndroidPort, self).start_http_server(additional_dirs, number_of_drivers)
612
613    def create_driver(self, worker_number, no_timeout=False):
614        return ChromiumAndroidDriver(self, worker_number, pixel_tests=self.get_option('pixel_tests'),
615                                     driver_details=self._driver_details,
616                                     android_devices=self._devices,
617                                     # Force no timeout to avoid test driver timeouts before NRWT.
618                                     no_timeout=True)
619
620    def driver_cmd_line(self):
621        # Override to return the actual test driver's command line.
622        return self.create_driver(0)._android_driver_cmd_line(self.get_option('pixel_tests'), [])
623
624    def clobber_old_port_specific_results(self):
625        if not self.get_option('disable_breakpad'):
626            self._dump_reader.clobber_old_results()
627
628    # Overridden protected methods.
629
630    def _build_path(self, *comps):
631        return self._host_port._build_path(*comps)
632
633    def _build_path_with_configuration(self, configuration, *comps):
634        return self._host_port._build_path_with_configuration(configuration, *comps)
635
636    def path_to_apache(self):
637        return self._host_port.path_to_apache()
638
639    def path_to_apache_config_file(self):
640        return self._host_port.path_to_apache_config_file()
641
642    def _path_to_driver(self, configuration=None):
643        return self._build_path_with_configuration(configuration, self._driver_details.apk_name())
644
645    def _path_to_helper(self):
646        return None
647
648    def _path_to_image_diff(self):
649        return self._host_port._path_to_image_diff()
650
651    def _path_to_wdiff(self):
652        return self._host_port._path_to_wdiff()
653
654    def _shut_down_http_server(self, pid):
655        return self._host_port._shut_down_http_server(pid)
656
657    def _driver_class(self):
658        return ChromiumAndroidDriver
659
660    # Local private methods.
661
662    @staticmethod
663    def _android_server_process_constructor(port, server_name, cmd_line, env=None, logging=False):
664        return server_process.ServerProcess(port, server_name, cmd_line, env,
665                                            universal_newlines=True, treat_no_data_as_crash=True, logging=logging)
666
667
668class AndroidPerf(SingleFileOutputProfiler):
669    _cached_perf_host_path = None
670    _have_searched_for_perf_host = False
671
672    def __init__(self, host, executable_path, output_dir, android_commands, symfs_path, kallsyms_path, identifier=None):
673        super(AndroidPerf, self).__init__(host, executable_path, output_dir, "data", identifier)
674        self._android_commands = android_commands
675        self._perf_process = None
676        self._symfs_path = symfs_path
677        self._kallsyms_path = kallsyms_path
678
679    def check_configuration(self):
680        # Check that perf is installed
681        if not self._android_commands.file_exists('/system/bin/perf'):
682            print "Cannot find /system/bin/perf on device %s" % self._android_commands.get_serial()
683            return False
684
685        # Check that the device is a userdebug build (or at least has the necessary libraries).
686        if self._android_commands.run(['shell', 'getprop', 'ro.build.type']).strip() != 'userdebug':
687            print "Device %s is not flashed with a userdebug build of Android" % self._android_commands.get_serial()
688            return False
689
690        # FIXME: Check that the binary actually is perf-able (has stackframe pointers)?
691        # objdump -s a function and make sure it modifies the fp?
692        # Instruct users to rebuild after export GYP_DEFINES="profiling=1 $GYP_DEFINES"
693        return True
694
695    def print_setup_instructions(self):
696        print """
697perf on android requires a 'userdebug' build of Android, see:
698http://source.android.com/source/building-devices.html"
699
700The perf command can be built from:
701https://android.googlesource.com/platform/external/linux-tools-perf/
702and requires libefl, libebl, libdw, and libdwfl available in:
703https://android.googlesource.com/platform/external/elfutils/
704
705The test driver must be built with profiling=1, make sure you've done:
706export GYP_DEFINES="profiling=1 $GYP_DEFINES"
707update-webkit --chromium-android
708build-webkit --chromium-android
709
710Googlers should read:
711http://goto.google.com/cr-android-perf-howto
712"""
713
714    def attach_to_pid(self, pid):
715        assert(pid)
716        assert(self._perf_process == None)
717        # FIXME: This can't be a fixed timeout!
718        cmd = self._android_commands.adb_command() + ['shell', 'perf', 'record', '-g', '-p', pid, 'sleep', 30]
719        self._perf_process = self._host.executive.popen(cmd)
720
721    def _perf_version_string(self, perf_path):
722        try:
723            return self._host.executive.run_command([perf_path, '--version'])
724        except:
725            return None
726
727    def _find_perfhost_binary(self):
728        perfhost_version = self._perf_version_string('perfhost_linux')
729        if perfhost_version:
730            return 'perfhost_linux'
731        perf_version = self._perf_version_string('perf')
732        if perf_version:
733            return 'perf'
734        return None
735
736    def _perfhost_path(self):
737        if self._have_searched_for_perf_host:
738            return self._cached_perf_host_path
739        self._have_searched_for_perf_host = True
740        self._cached_perf_host_path = self._find_perfhost_binary()
741        return self._cached_perf_host_path
742
743    def _first_ten_lines_of_profile(self, perf_output):
744        match = re.search("^#[^\n]*\n((?: [^\n]*\n){1,10})", perf_output, re.MULTILINE)
745        return match.group(1) if match else None
746
747    def profile_after_exit(self):
748        perf_exitcode = self._perf_process.wait()
749        if perf_exitcode != 0:
750            print "Perf failed (exit code: %i), can't process results." % perf_exitcode
751            return
752
753        self._android_commands.pull('/data/perf.data', self._output_path)
754
755        perfhost_path = self._perfhost_path()
756        perfhost_report_command = [
757            'report',
758            '--input', self._output_path,
759            '--symfs', self._symfs_path,
760            '--kallsyms', self._kallsyms_path,
761        ]
762        if perfhost_path:
763            perfhost_args = [perfhost_path] + perfhost_report_command + ['--call-graph', 'none']
764            perf_output = self._host.executive.run_command(perfhost_args)
765            # We could save off the full -g report to a file if users found that useful.
766            print self._first_ten_lines_of_profile(perf_output)
767        else:
768            print """
769Failed to find perfhost_linux binary, can't process samples from the device.
770
771perfhost_linux can be built from:
772https://android.googlesource.com/platform/external/linux-tools-perf/
773also, modern versions of perf (available from apt-get install goobuntu-kernel-tools-common)
774may also be able to process the perf.data files from the device.
775
776Googlers should read:
777http://goto.google.com/cr-android-perf-howto
778for instructions on installing pre-built copies of perfhost_linux
779http://crbug.com/165250 discusses making these pre-built binaries externally available.
780"""
781
782        perfhost_display_patch = perfhost_path if perfhost_path else 'perfhost_linux'
783        print "To view the full profile, run:"
784        print ' '.join([perfhost_display_patch] + perfhost_report_command)
785
786
787class ChromiumAndroidDriver(driver.Driver):
788    def __init__(self, port, worker_number, pixel_tests, driver_details, android_devices, no_timeout=False):
789        super(ChromiumAndroidDriver, self).__init__(port, worker_number, pixel_tests, no_timeout)
790        self._in_fifo_path = driver_details.device_fifo_directory() + 'stdin.fifo'
791        self._out_fifo_path = driver_details.device_fifo_directory() + 'test.fifo'
792        self._err_fifo_path = driver_details.device_fifo_directory() + 'stderr.fifo'
793        self._read_stdout_process = None
794        self._read_stderr_process = None
795        self._forwarder_process = None
796        self._original_governors = {}
797        self._original_kptr_restrict = None
798
799        self._android_devices = android_devices
800        self._android_commands = android_devices.get_device(port._executive, worker_number)
801        self._driver_details = driver_details
802        self._debug_logging = self._port._debug_logging
803        self._created_cmd_line = False
804        self._device_failed = False
805
806        # FIXME: If we taught ProfileFactory about "target" devices we could
807        # just use the logic in Driver instead of duplicating it here.
808        if self._port.get_option("profile"):
809            # FIXME: This should be done once, instead of per-driver!
810            symfs_path = self._find_or_create_symfs()
811            kallsyms_path = self._update_kallsyms_cache(symfs_path)
812            # FIXME: We should pass this some sort of "Bridge" object abstraction around ADB instead of a path/device pair.
813            self._profiler = AndroidPerf(self._port.host, self._port._path_to_driver(), self._port.results_directory(),
814                self._android_commands, symfs_path, kallsyms_path)
815            # FIXME: This is a layering violation and should be moved to Port.check_sys_deps
816            # once we have an abstraction around an adb_path/device_serial pair to make it
817            # easy to make these class methods on AndroidPerf.
818            if not self._profiler.check_configuration():
819                self._profiler.print_setup_instructions()
820                sys.exit(1)
821        else:
822            self._profiler = None
823
824    def __del__(self):
825        self._teardown_performance()
826        self._clean_up_cmd_line()
827        super(ChromiumAndroidDriver, self).__del__()
828
829    def _update_kallsyms_cache(self, output_dir):
830        kallsyms_name = "%s-kallsyms" % self._android_commands.get_serial()
831        kallsyms_cache_path = self._port.host.filesystem.join(output_dir, kallsyms_name)
832
833        self._android_commands.restart_as_root()
834
835        saved_kptr_restrict = self._android_commands.run(['shell', 'cat', KPTR_RESTRICT_PATH]).strip()
836        self._android_commands.run(['shell', 'echo', '0', '>', KPTR_RESTRICT_PATH])
837
838        print "Updating kallsyms file (%s) from device" % kallsyms_cache_path
839        self._android_commands.pull("/proc/kallsyms", kallsyms_cache_path)
840
841        self._android_commands.run(['shell', 'echo', saved_kptr_restrict, '>', KPTR_RESTRICT_PATH])
842
843        return kallsyms_cache_path
844
845    def _find_or_create_symfs(self):
846        environment = self._port.host.copy_current_environment()
847        env = environment.to_dictionary()
848        fs = self._port.host.filesystem
849
850        if 'ANDROID_SYMFS' in env:
851            symfs_path = env['ANDROID_SYMFS']
852        else:
853            symfs_path = fs.join(self._port.results_directory(), 'symfs')
854            print "ANDROID_SYMFS not set, using %s" % symfs_path
855
856        # find the installed path, and the path of the symboled built library
857        # FIXME: We should get the install path from the device!
858        symfs_library_path = fs.join(symfs_path, "data/app-lib/%s-1/%s" % (self._driver_details.package_name(), self._driver_details.library_name()))
859        built_library_path = self._port._build_path('lib', self._driver_details.library_name())
860        assert(fs.exists(built_library_path))
861
862        # FIXME: Ideally we'd check the sha1's first and make a soft-link instead of copying (since we probably never care about windows).
863        print "Updating symfs libary (%s) from built copy (%s)" % (symfs_library_path, built_library_path)
864        fs.maybe_make_directory(fs.dirname(symfs_library_path))
865        fs.copyfile(built_library_path, symfs_library_path)
866
867        return symfs_path
868
869    def _setup_md5sum_and_push_data_if_needed(self, log_callback):
870        self._md5sum_path = self._port.path_to_md5sum()
871        if not self._android_commands.file_exists(MD5SUM_DEVICE_PATH):
872            if not self._android_commands.push(self._md5sum_path, MD5SUM_DEVICE_PATH):
873                self._abort('Could not push md5sum to device')
874
875        self._push_executable(log_callback)
876        self._push_fonts(log_callback)
877        self._push_test_resources(log_callback)
878
879    def _setup_test(self, log_callback):
880        # FIXME: Move this routine and its subroutines off of the AndroidDriver
881        # class and onto AndroidCommands or some other helper class, so that we
882        # can initialize the device without needing to create a driver.
883
884        if self._android_devices.is_device_prepared(self._android_commands.get_serial()):
885            return
886
887        self._android_commands.restart_adb()
888        self._android_commands.restart_as_root()
889        self._setup_md5sum_and_push_data_if_needed(log_callback)
890        self._setup_performance()
891
892        # Required by webkit_support::GetWebKitRootDirFilePath().
893        # Other directories will be created automatically by adb push.
894        self._android_commands.mkdir(DEVICE_SOURCE_ROOT_DIR + 'chrome')
895
896        # Allow the test driver to get full read and write access to the directory on the device,
897        # as well as for the FIFOs. We'll need a world writable directory.
898        self._android_commands.mkdir(self._driver_details.device_directory(), chmod='777')
899        self._android_commands.mkdir(self._driver_details.device_fifo_directory(), chmod='777')
900
901        # Make sure that the disk cache on the device resets to a clean state.
902        self._android_commands.run(['shell', 'rm', '-r', self._driver_details.device_cache_directory()])
903
904        # Mark this device as having been set up.
905        self._android_devices.set_device_prepared(self._android_commands.get_serial())
906
907    def _log_error(self, message):
908        _log.error('[%s] %s' % (self._android_commands.get_serial(), message))
909
910    def _log_warning(self, message):
911        _log.warning('[%s] %s' % (self._android_commands.get_serial(), message))
912
913    def _log_debug(self, message):
914        if self._debug_logging:
915            _log.debug('[%s] %s' % (self._android_commands.get_serial(), message))
916
917    def _abort(self, message):
918        self._device_failed = True
919        raise driver.DeviceFailure('[%s] %s' % (self._android_commands.get_serial(), message))
920
921    @staticmethod
922    def _extract_hashes_from_md5sum_output(md5sum_output):
923        assert md5sum_output
924        return [line.split('  ')[0] for line in md5sum_output]
925
926    def _files_match(self, host_file, device_file):
927        assert self._port.host.filesystem.exists(host_file)
928        device_hashes = self._extract_hashes_from_md5sum_output(
929                self._port.host.executive.popen(self._android_commands.adb_command() + ['shell', MD5SUM_DEVICE_PATH, device_file],
930                                                stdout=subprocess.PIPE).stdout)
931        host_hashes = self._extract_hashes_from_md5sum_output(
932                self._port.host.executive.popen(args=['%s_host' % self._md5sum_path, host_file],
933                                                stdout=subprocess.PIPE).stdout)
934        return host_hashes and device_hashes == host_hashes
935
936    def _push_file_if_needed(self, host_file, device_file, log_callback):
937        basename = self._port.host.filesystem.basename(host_file)
938        log_callback("checking %s" % basename)
939        if not self._files_match(host_file, device_file):
940            log_callback("pushing %s" % basename)
941            self._android_commands.push(host_file, device_file)
942
943    def _push_executable(self, log_callback):
944        self._push_file_if_needed(self._port.path_to_forwarder(), self._driver_details.device_forwarder_path(), log_callback)
945        for resource in self._driver_details.additional_resources():
946            self._push_file_if_needed(self._port._build_path(resource), self._driver_details.device_directory() + resource, log_callback)
947
948        self._push_file_if_needed(self._port._build_path('android_main_fonts.xml'), self._driver_details.device_directory() + 'android_main_fonts.xml', log_callback)
949        self._push_file_if_needed(self._port._build_path('android_fallback_fonts.xml'), self._driver_details.device_directory() + 'android_fallback_fonts.xml', log_callback)
950
951        log_callback("checking apk")
952        if self._files_match(self._port._build_path('apks', 'ContentShell.apk'),
953                             '/data/app/org.chromium.content_shell_apk-1.apk'):
954            return
955
956        log_callback("uninstalling apk")
957        self._android_commands.run(['uninstall', self._driver_details.package_name()])
958        driver_host_path = self._port._path_to_driver()
959        log_callback("installing apk")
960        install_result = self._android_commands.run(['install', driver_host_path])
961        if install_result.find('Success') == -1:
962            self._abort('Failed to install %s onto device: %s' % (driver_host_path, install_result))
963
964    def _push_fonts(self, log_callback):
965        path_to_ahem_font = self._port._build_path('AHEM____.TTF')
966        self._push_file_if_needed(path_to_ahem_font, self._driver_details.device_fonts_directory() + 'AHEM____.TTF', log_callback)
967        for (host_dirs, font_file, package) in HOST_FONT_FILES:
968            for host_dir in host_dirs:
969                host_font_path = host_dir + font_file
970                if self._port._check_file_exists(host_font_path, '', logging=False):
971                    self._push_file_if_needed(host_font_path, self._driver_details.device_fonts_directory() + font_file, log_callback)
972
973    def _push_test_resources(self, log_callback):
974        for resource in TEST_RESOURCES_TO_PUSH:
975            self._push_file_if_needed(self._port.layout_tests_dir() + '/' + resource, DEVICE_LAYOUT_TESTS_DIR + resource, log_callback)
976
977    def _get_last_stacktrace(self):
978        tombstones = self._android_commands.run(['shell', 'ls', '-n', '/data/tombstones/tombstone_*'])
979        if not tombstones or tombstones.startswith('/data/tombstones/tombstone_*: No such file or directory'):
980            self._log_error('The driver crashed, but no tombstone found!')
981            return ''
982
983        if tombstones.startswith('/data/tombstones/tombstone_*: Permission denied'):
984            # FIXME: crbug.com/321489 ... figure out why this happens.
985            self._log_error('The driver crashed, but we could not read the tombstones!')
986            return ''
987
988        tombstones = tombstones.rstrip().split('\n')
989        last_tombstone = None
990        for tombstone in tombstones:
991            # Format of fields:
992            # 0          1      2      3     4          5     6
993            # permission uid    gid    size  date       time  filename
994            # -rw------- 1000   1000   45859 2011-04-13 06:00 tombstone_00
995            fields = tombstone.split()
996            if len(fields) != 7:
997                self._log_warning("unexpected line in tombstone output, skipping: '%s'" % tombstone)
998                continue
999
1000            if not last_tombstone or fields[4] + fields[5] >= last_tombstone[4] + last_tombstone[5]:
1001                last_tombstone = fields
1002            else:
1003                break
1004
1005        if not last_tombstone:
1006            self._log_error('The driver crashed, but we could not find any valid tombstone!')
1007            return ''
1008
1009        # Use Android tool vendor/google/tools/stack to convert the raw
1010        # stack trace into a human readable format, if needed.
1011        # It takes a long time, so don't do it here.
1012        return '%s\n%s' % (' '.join(last_tombstone),
1013                           self._android_commands.run(['shell', 'cat', '/data/tombstones/' + last_tombstone[6]]))
1014
1015    def _get_logcat(self):
1016        return self._android_commands.run(['logcat', '-d', '-v', 'threadtime'])
1017
1018    def _setup_performance(self):
1019        # Disable CPU scaling and drop ram cache to reduce noise in tests
1020        if not self._original_governors:
1021            governor_files = self._android_commands.run(['shell', 'ls', SCALING_GOVERNORS_PATTERN])
1022            if governor_files.find('No such file or directory') == -1:
1023                for file in governor_files.split():
1024                    self._original_governors[file] = self._android_commands.run(['shell', 'cat', file]).strip()
1025                    self._android_commands.run(['shell', 'echo', 'performance', '>', file])
1026
1027    def _teardown_performance(self):
1028        for file, original_content in self._original_governors.items():
1029            self._android_commands.run(['shell', 'echo', original_content, '>', file])
1030        self._original_governors = {}
1031
1032    def _get_crash_log(self, stdout, stderr, newer_than):
1033        if not stdout:
1034            stdout = ''
1035        stdout += '********* [%s] Logcat:\n%s' % (self._android_commands.get_serial(), self._get_logcat())
1036        if not stderr:
1037            stderr = ''
1038        stderr += '********* [%s] Tombstone file:\n%s' % (self._android_commands.get_serial(), self._get_last_stacktrace())
1039
1040        if not self._port.get_option('disable_breakpad'):
1041            crashes = self._pull_crash_dumps_from_device()
1042            for crash in crashes:
1043                stderr += '********* [%s] breakpad minidump %s:\n%s' % (self._port.host.filesystem.basename(crash), self._android_commands.get_serial(), self._port._dump_reader._get_stack_from_dump(crash))
1044
1045        return super(ChromiumAndroidDriver, self)._get_crash_log(stdout, stderr, newer_than)
1046
1047    def cmd_line(self, pixel_tests, per_test_args):
1048        # The returned command line is used to start _server_process. In our case, it's an interactive 'adb shell'.
1049        # The command line passed to the driver process is returned by _driver_cmd_line() instead.
1050        return self._android_commands.adb_command() + ['shell']
1051
1052    def _android_driver_cmd_line(self, pixel_tests, per_test_args):
1053        return driver.Driver.cmd_line(self, pixel_tests, per_test_args)
1054
1055    @staticmethod
1056    def _loop_with_timeout(condition, timeout_secs):
1057        deadline = time.time() + timeout_secs
1058        while time.time() < deadline:
1059            if condition():
1060                return True
1061        return False
1062
1063    def _all_pipes_created(self):
1064        return (self._android_commands.file_exists(self._in_fifo_path) and
1065                self._android_commands.file_exists(self._out_fifo_path) and
1066                self._android_commands.file_exists(self._err_fifo_path))
1067
1068    def _remove_all_pipes(self):
1069        for file in [self._in_fifo_path, self._out_fifo_path, self._err_fifo_path]:
1070            self._android_commands.run(['shell', 'rm', file])
1071
1072        return (not self._android_commands.file_exists(self._in_fifo_path) and
1073                not self._android_commands.file_exists(self._out_fifo_path) and
1074                not self._android_commands.file_exists(self._err_fifo_path))
1075
1076    def start(self, pixel_tests, per_test_args, deadline):
1077        # We override the default start() so that we can call _android_driver_cmd_line()
1078        # instead of cmd_line().
1079        new_cmd_line = self._android_driver_cmd_line(pixel_tests, per_test_args)
1080
1081        # Since _android_driver_cmd_line() is different than cmd_line() we need to provide
1082        # our own mechanism for detecting when the process should be stopped.
1083        if self._current_cmd_line is None:
1084            self._current_android_cmd_line = None
1085        if new_cmd_line != self._current_android_cmd_line:
1086            self.stop()
1087        self._current_android_cmd_line = new_cmd_line
1088
1089        super(ChromiumAndroidDriver, self).start(pixel_tests, per_test_args, deadline)
1090
1091    def _start(self, pixel_tests, per_test_args):
1092        if not self._android_devices.is_device_prepared(self._android_commands.get_serial()):
1093            raise driver.DeviceFailure("%s is not prepared in _start()" % self._android_commands.get_serial())
1094
1095        for retries in range(3):
1096            try:
1097                if self._start_once(pixel_tests, per_test_args):
1098                    return
1099            except ScriptError as e:
1100                self._abort('ScriptError("%s") in _start()' % str(e))
1101
1102            self._log_error('Failed to start the content_shell application. Retries=%d. Log:%s' % (retries, self._get_logcat()))
1103            self.stop()
1104            time.sleep(2)
1105        self._abort('Failed to start the content_shell application multiple times. Giving up.')
1106
1107    def _start_once(self, pixel_tests, per_test_args):
1108        super(ChromiumAndroidDriver, self)._start(pixel_tests, per_test_args, wait_for_ready=False)
1109
1110        self._log_debug('Starting forwarder')
1111        self._forwarder_process = self._port._server_process_constructor(
1112            self._port, 'Forwarder', self._android_commands.adb_command() + ['shell', '%s -D %s' % (self._driver_details.device_forwarder_path(), FORWARD_PORTS)])
1113        self._forwarder_process.start()
1114
1115        deadline = time.time() + DRIVER_START_STOP_TIMEOUT_SECS
1116        if not self._wait_for_server_process_output(self._forwarder_process, deadline, 'Forwarding device port'):
1117            return False
1118
1119        self._android_commands.run(['logcat', '-c'])
1120
1121        cmd_line_file_path = self._driver_details.command_line_file()
1122        original_cmd_line_file_path = cmd_line_file_path + '.orig'
1123        if self._android_commands.file_exists(cmd_line_file_path) and not self._android_commands.file_exists(original_cmd_line_file_path):
1124            # We check for both the normal path and the backup because we do not want to step
1125            # on the backup. Otherwise, we'd clobber the backup whenever we changed the
1126            # command line during the run.
1127            self._android_commands.run(['shell', 'mv', cmd_line_file_path, original_cmd_line_file_path])
1128
1129        self._android_commands.run(['shell', 'echo'] + self._android_driver_cmd_line(pixel_tests, per_test_args) + ['>', self._driver_details.command_line_file()])
1130        self._created_cmd_line = True
1131
1132        self._android_commands.run(['shell', 'rm', '-rf', self._driver_details.device_crash_dumps_directory()])
1133        self._android_commands.mkdir(self._driver_details.device_crash_dumps_directory(), chmod='777')
1134
1135        start_result = self._android_commands.run(['shell', 'am', 'start', '-e', 'RunInSubThread', '-n', self._driver_details.activity_name()])
1136        if start_result.find('Exception') != -1:
1137            self._log_error('Failed to start the content_shell application. Exception:\n' + start_result)
1138            return False
1139
1140        if not ChromiumAndroidDriver._loop_with_timeout(self._all_pipes_created, DRIVER_START_STOP_TIMEOUT_SECS):
1141            return False
1142
1143        # Read back the shell prompt to ensure adb shell ready.
1144        deadline = time.time() + DRIVER_START_STOP_TIMEOUT_SECS
1145        self._server_process.start()
1146        self._read_prompt(deadline)
1147        self._log_debug('Interactive shell started')
1148
1149        # Start a process to read from the stdout fifo of the test driver and print to stdout.
1150        self._log_debug('Redirecting stdout to ' + self._out_fifo_path)
1151        self._read_stdout_process = self._port._server_process_constructor(
1152            self._port, 'ReadStdout', self._android_commands.adb_command() + ['shell', 'cat', self._out_fifo_path])
1153        self._read_stdout_process.start()
1154
1155        # Start a process to read from the stderr fifo of the test driver and print to stdout.
1156        self._log_debug('Redirecting stderr to ' + self._err_fifo_path)
1157        self._read_stderr_process = self._port._server_process_constructor(
1158            self._port, 'ReadStderr', self._android_commands.adb_command() + ['shell', 'cat', self._err_fifo_path])
1159        self._read_stderr_process.start()
1160
1161        self._log_debug('Redirecting stdin to ' + self._in_fifo_path)
1162        self._server_process.write('cat >%s\n' % self._in_fifo_path)
1163
1164        # Combine the stdout and stderr pipes into self._server_process.
1165        self._server_process.replace_outputs(self._read_stdout_process._proc.stdout, self._read_stderr_process._proc.stdout)
1166
1167        def deadlock_detector(processes, normal_startup_event):
1168            if not ChromiumAndroidDriver._loop_with_timeout(lambda: normal_startup_event.is_set(), DRIVER_START_STOP_TIMEOUT_SECS):
1169                # If normal_startup_event is not set in time, the main thread must be blocked at
1170                # reading/writing the fifo. Kill the fifo reading/writing processes to let the
1171                # main thread escape from the deadlocked state. After that, the main thread will
1172                # treat this as a crash.
1173                self._log_error('Deadlock detected. Processes killed.')
1174                for i in processes:
1175                    i.kill()
1176
1177        # Start a thread to kill the pipe reading/writing processes on deadlock of the fifos during startup.
1178        normal_startup_event = threading.Event()
1179        threading.Thread(name='DeadlockDetector', target=deadlock_detector,
1180                         args=([self._server_process, self._read_stdout_process, self._read_stderr_process], normal_startup_event)).start()
1181
1182        # The test driver might crash during startup or when the deadlock detector hits
1183        # a deadlock and kills the fifo reading/writing processes.
1184        if not self._wait_for_server_process_output(self._server_process, deadline, '#READY'):
1185            return False
1186
1187        # Inform the deadlock detector that the startup is successful without deadlock.
1188        normal_startup_event.set()
1189        self._log_debug("content_shell is ready")
1190        return True
1191
1192    def _pid_from_android_ps_output(self, ps_output, package_name):
1193        # ps output seems to be fixed width, we only care about the name and the pid
1194        # u0_a72    21630 125   947920 59364 ffffffff 400beee4 S org.chromium.native_test
1195        for line in ps_output.split('\n'):
1196            if line.find(self._driver_details.package_name()) != -1:
1197                match = re.match(r'\S+\s+(\d+)', line)
1198                return int(match.group(1))
1199
1200    def _pid_on_target(self):
1201        # FIXME: There must be a better way to do this than grepping ps output!
1202        ps_output = self._android_commands.run(['shell', 'ps'])
1203        return self._pid_from_android_ps_output(ps_output, self._driver_details.package_name())
1204
1205    def stop(self):
1206        if not self._device_failed:
1207            # Do not try to stop the application if there's something wrong with the device; adb may hang.
1208            # FIXME: crbug.com/305040. Figure out if it's really hanging (and why).
1209            self._android_commands.run(['shell', 'am', 'force-stop', self._driver_details.package_name()])
1210
1211        if self._read_stdout_process:
1212            self._read_stdout_process.kill()
1213            self._read_stdout_process = None
1214
1215        if self._read_stderr_process:
1216            self._read_stderr_process.kill()
1217            self._read_stderr_process = None
1218
1219        super(ChromiumAndroidDriver, self).stop()
1220
1221        if self._forwarder_process:
1222            self._forwarder_process.kill()
1223            self._forwarder_process = None
1224
1225        if self._android_devices.is_device_prepared(self._android_commands.get_serial()):
1226            if not ChromiumAndroidDriver._loop_with_timeout(self._remove_all_pipes, DRIVER_START_STOP_TIMEOUT_SECS):
1227                self._abort('Failed to remove fifo files. May be locked.')
1228
1229        self._clean_up_cmd_line()
1230
1231    def _pull_crash_dumps_from_device(self):
1232        result = []
1233        if not self._android_commands.file_exists(self._driver_details.device_crash_dumps_directory()):
1234            return result
1235        dumps = self._android_commands.run(['shell', 'ls', self._driver_details.device_crash_dumps_directory()])
1236        for dump in dumps.splitlines():
1237            device_dump = '%s/%s' % (self._driver_details.device_crash_dumps_directory(), dump)
1238            local_dump = self._port._filesystem.join(self._port._dump_reader.crash_dumps_directory(), dump)
1239
1240            # FIXME: crbug.com/321489. Figure out why these commands would fail ...
1241            err = self._android_commands.run(['shell', 'chmod', '777', device_dump])
1242            if not err:
1243                self._android_commands.pull(device_dump, local_dump)
1244            if not err:
1245                self._android_commands.run(['shell', 'rm', '-f', device_dump])
1246
1247            if self._port._filesystem.exists(local_dump):
1248                result.append(local_dump)
1249        return result
1250
1251    def _clean_up_cmd_line(self):
1252        if not self._created_cmd_line:
1253            return
1254
1255        cmd_line_file_path = self._driver_details.command_line_file()
1256        original_cmd_line_file_path = cmd_line_file_path + '.orig'
1257        if self._android_commands.file_exists(original_cmd_line_file_path):
1258            self._android_commands.run(['shell', 'mv', original_cmd_line_file_path, cmd_line_file_path])
1259        elif self._android_commands.file_exists(cmd_line_file_path):
1260            self._android_commands.run(['shell', 'rm', cmd_line_file_path])
1261        self._created_cmd_line = False
1262
1263    def _command_from_driver_input(self, driver_input):
1264        command = super(ChromiumAndroidDriver, self)._command_from_driver_input(driver_input)
1265        if command.startswith('/'):
1266            fs = self._port._filesystem
1267            # FIXME: what happens if command lies outside of the layout_tests_dir on the host?
1268            relative_test_filename = fs.relpath(command, fs.dirname(self._port.layout_tests_dir()))
1269            command = DEVICE_WEBKIT_BASE_DIR + relative_test_filename
1270        return command
1271
1272    def _read_prompt(self, deadline):
1273        last_char = ''
1274        while True:
1275            current_char = self._server_process.read_stdout(deadline, 1)
1276            if current_char == ' ':
1277                if last_char in ('#', '$'):
1278                    return
1279            last_char = current_char
1280