• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2#
3# Copyright (C) 2015 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17"""Runs all NDK tests.
18
19TODO: Handle default ABI case.
20The old test script would build and test all ABIs listed in APP_ABI in a single
21invocation if an explicit ABI was not given. Currently this will fall down over
22that case. I've written most of the code to handle this, but I decided to
23factor that for-each up several levels because it was going to make the device
24tests rather messy and I haven't finished doing that yet.
25
26TODO: Handle explicit test lists from command line.
27The old test runner allowed specifying an exact list of tests to run with
28--tests. That seems like a useful thing to keep around, but I haven't ported it
29yet.
30"""
31from __future__ import print_function
32
33import argparse
34import atexit
35import distutils.spawn
36import inspect
37import os
38import re
39import shutil
40import subprocess
41import sys
42import tempfile
43
44import adb
45import filters
46import ndk
47import printers
48import tests
49
50from tests import AwkTest, BuildTest, DeviceTest
51
52
53SUPPORTED_ABIS = (
54    'armeabi',
55    'armeabi-v7a',
56    'armeabi-v7a-hard',
57    'arm64-v8a',
58    'mips',
59    'mips64',
60    'x86',
61    'x86_64',
62)
63
64
65# TODO(danalbert): How much time do we actually save by not running these?
66LONG_TESTS = (
67    'prebuild-stlport',
68    'test-stlport',
69    'test-gnustl-full',
70    'test-stlport_shared-exception',
71    'test-stlport_static-exception',
72    'test-gnustl_shared-exception-full',
73    'test-gnustl_static-exception-full',
74    'test-googletest-full',
75    'test-libc++-shared-full',
76    'test-libc++-static-full',
77)
78
79
80def get_test_device():
81    if distutils.spawn.find_executable('adb') is None:
82        raise RuntimeError('Could not find adb.')
83
84    p = subprocess.Popen(['adb', 'devices'], stdout=subprocess.PIPE)
85    out, _ = p.communicate()
86    if p.returncode != 0:
87        raise RuntimeError('Failed to get list of devices from adb.')
88
89    # The first line of `adb devices` just says "List of attached devices", so
90    # skip that.
91    devices = []
92    for line in out.split('\n')[1:]:
93        if not line.strip():
94            continue
95        if 'offline' in line:
96            continue
97
98        serial, _ = re.split(r'\s+', line, maxsplit=1)
99        devices.append(serial)
100
101    if len(devices) == 0:
102        raise RuntimeError('No devices detected.')
103
104    device = os.getenv('ANDROID_SERIAL')
105    if device is None and len(devices) == 1:
106        device = devices[0]
107
108    if device is not None and device not in devices:
109        raise RuntimeError('Device {} is not available.'.format(device))
110
111    # TODO(danalbert): Handle running against multiple devices in one pass.
112    if len(devices) > 1 and device is None:
113        raise RuntimeError('Multiple devices detected and ANDROID_SERIAL not '
114                           'set. Cannot continue.')
115
116    return device
117
118
119def get_device_abis():
120    # 64-bit devices list their ABIs differently than 32-bit devices. Check all
121    # the possible places for stashing ABI info and merge them.
122    abi_properties = [
123        'ro.product.cpu.abi',
124        'ro.product.cpu.abi2',
125        'ro.product.cpu.abilist',
126    ]
127    abis = set()
128    for abi_prop in abi_properties:
129        prop = adb.get_prop(abi_prop)
130        if prop is not None:
131            abis.update(prop.split(','))
132
133    if 'armeabi-v7a' in abis:
134        abis.add('armeabi-v7a-hard')
135    return sorted(list(abis))
136
137
138def check_adb_works_or_die(abi):
139    # TODO(danalbert): Check that we can do anything with the device.
140    try:
141        device = get_test_device()
142    except RuntimeError as ex:
143        sys.exit('Error: {}'.format(ex))
144
145    supported_abis = get_device_abis()
146    if abi is not None and abi not in supported_abis:
147        msg = ('The test device ({}) does not support the requested ABI '
148               '({}).\nSupported ABIs: {}'.format(device, abi,
149                                                  ', '.join(supported_abis)))
150        sys.exit(msg)
151
152
153def can_use_asan(abi, api, toolchain):
154    # ASAN is currently only supported for 32-bit ARM and x86...
155    if not abi.startswith('armeabi') and not abi == 'x86':
156        return False
157
158    # On KitKat and newer...
159    if api < 19:
160        return False
161
162    # When using clang...
163    if toolchain != 'clang':
164        return False
165
166    # On rooted devices.
167    if int(adb.get_prop('ro.debuggable')) == 0:
168        return False
169
170    return True
171
172
173def asan_device_setup():
174    path = os.path.join(
175        os.environ['NDK'], 'toolchains', 'llvm', 'prebuilt',
176        ndk.get_host_tag(), 'bin', 'asan_device_setup')
177    subprocess.check_call([path])
178
179
180def is_valid_platform_version(version_string):
181    match = re.match(r'^android-(\d+)$', version_string)
182    if not match:
183        return False
184
185    # We don't support anything before Gingerbread.
186    version = int(match.group(1))
187    return version >= 9
188
189
190def android_platform_version(version_string):
191    if is_valid_platform_version(version_string):
192        return version_string
193    else:
194        raise argparse.ArgumentTypeError(
195            'Platform version must match the format "android-VERSION", where '
196            'VERSION >= 9.')
197
198
199class ArgParser(argparse.ArgumentParser):
200    def __init__(self):
201        super(ArgParser, self).__init__(
202            description=inspect.getdoc(sys.modules[__name__]))
203
204        self.add_argument(
205            '--abi', default=None, choices=SUPPORTED_ABIS,
206            help=('Run tests against the specified ABI. Defaults to the '
207                  'contents of APP_ABI in jni/Application.mk'))
208        self.add_argument(
209            '--platform', default=None, type=android_platform_version,
210            help=('Run tests against the specified platform version. Defaults '
211                  'to the contents of APP_PLATFORM in jni/Application.mk'))
212        self.add_argument(
213            '--toolchain', default='clang', choices=('4.9', 'clang'),
214            help='Toolchain for building tests. Defaults to clang.')
215
216        self.add_argument(
217            '--show-commands', action='store_true',
218            help='Show build commands for each test.')
219        self.add_argument(
220            '--suite', default=None,
221            choices=('awk', 'build', 'device'),
222            help=('Run only the chosen test suite.'))
223
224        self.add_argument(
225            '--filter', help='Only run tests that match the given patterns.')
226        self.add_argument(
227            '--quick', action='store_true', help='Skip long running tests.')
228        self.add_argument(
229            '--show-all', action='store_true',
230            help='Show all test results, not just failures.')
231
232        self.add_argument(
233            '--out-dir',
234            help='Path to build tests to. Will not be removed upon exit.')
235
236
237class ResultStats(object):
238    def __init__(self, suites, results):
239        self.num_tests = sum(len(s) for s in results.values())
240
241        zero_stats = {'pass': 0, 'skip': 0, 'fail': 0}
242        self.global_stats = dict(zero_stats)
243        self.suite_stats = {suite: dict(zero_stats) for suite in suites}
244        self._analyze_results(results)
245
246    def _analyze_results(self, results):
247        for suite, test_results in results.items():
248            for result in test_results:
249                if result.failed():
250                    self.suite_stats[suite]['fail'] += 1
251                    self.global_stats['fail'] += 1
252                elif result.passed():
253                    self.suite_stats[suite]['pass'] += 1
254                    self.global_stats['pass'] += 1
255                else:
256                    self.suite_stats[suite]['skip'] += 1
257                    self.global_stats['skip'] += 1
258
259
260def main():
261    orig_cwd = os.getcwd()
262    os.chdir(os.path.dirname(os.path.realpath(__file__)))
263
264    # Defining _NDK_TESTING_ALL_=yes to put armeabi-v7a-hard in its own
265    # libs/armeabi-v7a-hard directory and tested separately from armeabi-v7a.
266    # Some tests are now compiled with both APP_ABI=armeabi-v7a and
267    # APP_ABI=armeabi-v7a-hard. Without _NDK_TESTING_ALL_=yes, tests may fail
268    # to install due to race condition on the same libs/armeabi-v7a
269    if '_NDK_TESTING_ALL_' not in os.environ:
270        os.environ['_NDK_TESTING_ALL_'] = 'all'
271
272    if 'NDK' not in os.environ:
273        os.environ['NDK'] = os.path.dirname(os.getcwd())
274
275    args = ArgParser().parse_args()
276    ndk_build_flags = []
277    if args.abi is not None:
278        ndk_build_flags.append('APP_ABI={}'.format(args.abi))
279    if args.platform is not None:
280        ndk_build_flags.append('APP_PLATFORM={}'.format(args.platform))
281    if args.show_commands:
282        ndk_build_flags.append('V=1')
283
284    if not os.path.exists(os.path.join('../build/tools/prebuilt-common.sh')):
285        sys.exit('Error: Not run from a valid NDK.')
286
287    out_dir = args.out_dir
288    if out_dir is not None:
289        if not os.path.isabs(out_dir):
290            out_dir = os.path.join(orig_cwd, out_dir)
291        if os.path.exists(out_dir):
292            shutil.rmtree(out_dir)
293        os.makedirs(out_dir)
294    else:
295        out_dir = tempfile.mkdtemp()
296        atexit.register(lambda: shutil.rmtree(out_dir))
297
298    suites = ['awk', 'build', 'device']
299    if args.suite:
300        suites = [args.suite]
301
302    # Do this early so we find any device issues now rather than after we've
303    # run all the build tests.
304    if 'device' in suites:
305        check_adb_works_or_die(args.abi)
306        api_level = int(adb.get_prop('ro.build.version.sdk'))
307
308        # PIE is required in L. All of the device tests are written toward the
309        # ndk-build defaults, so we need to inform the build that we need PIE
310        # if we're running on a newer device.
311        if api_level >= 21:
312            ndk_build_flags.append('APP_PIE=true')
313
314        os.environ['ANDROID_SERIAL'] = get_test_device()
315
316        if can_use_asan(args.abi, api_level, args.toolchain):
317            asan_device_setup()
318
319        # Do this as part of initialization rather than with a `mkdir -p` later
320        # because Gingerbread didn't actually support -p :(
321        adb.shell('rm -r /data/local/tmp/ndk-tests')
322        adb.shell('mkdir /data/local/tmp/ndk-tests')
323
324    runner = tests.TestRunner()
325    if 'awk' in suites:
326        runner.add_suite('awk', 'awk', AwkTest)
327    if 'build' in suites:
328        runner.add_suite('build', 'build', BuildTest, args.abi, args.platform,
329                         args.toolchain, ndk_build_flags)
330    if 'device' in suites:
331        runner.add_suite('device', 'device', DeviceTest, args.abi,
332                         args.platform, api_level, args.toolchain,
333                         ndk_build_flags)
334
335    test_filters = filters.TestFilter.from_string(args.filter)
336    results = runner.run(out_dir, test_filters)
337
338    stats = ResultStats(suites, results)
339
340    use_color = sys.stdin.isatty() and os.name != 'nt'
341    printer = printers.StdoutPrinter(use_color=use_color,
342                                     show_all=args.show_all)
343    printer.print_results(results, stats)
344    sys.exit(stats.global_stats['fail'] > 0)
345
346
347if __name__ == '__main__':
348    main()
349