1# Copyright 2015 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5from collections import namedtuple 6import fnmatch 7import logging 8import os 9 10import common 11from autotest_lib.client.common_lib import error, gtest_parser 12from autotest_lib.client.common_lib.cros import dev_server 13from autotest_lib.server import afe_utils 14from autotest_lib.server import test 15 16 17NATIVE_TESTS_PATH = '/data/nativetest' 18WHITELIST_FILE = '/data/nativetest/tests.txt' 19ANDROID_NATIVE_TESTS_FILE_FMT = ( 20 '%(build_target)s-continuous_native_tests-%(build_id)s.zip') 21BRILLO_NATIVE_TESTS_FILE_FMT = '%(build_target)s-brillo-tests-%(build_id)s.zip' 22LIST_TEST_BINARIES_TEMPLATE = ( 23 'find %(path)s -type f -mindepth 2 -maxdepth 2 ' 24 '\( -perm -100 -o -perm -010 -o -perm -001 \)') 25NATIVE_ONLY_BOARDS = ['dragonboard'] 26 27GtestSuite = namedtuple('GtestSuite', ['name', 'path', 'run_as_root', 'args']) 28 29class brillo_Gtests(test.test): 30 """Run one or more native gTest Suites.""" 31 version = 1 32 33 34 def initialize(self, host=None, gtest_suites=None, use_whitelist=False, 35 filter_tests=None, native_tests=None): 36 if not afe_utils.host_in_lab(host): 37 return 38 # TODO(ralphnathan): Remove this once we can determine this in another 39 # way (b/29185385). 40 if host.get_board_name() in NATIVE_ONLY_BOARDS: 41 self._install_nativetests( 42 host, BRILLO_NATIVE_TESTS_FILE_FMT, 'nativetests') 43 else: 44 self._install_nativetests(host, 45 ANDROID_NATIVE_TESTS_FILE_FMT, 46 'continuous_native_tests') 47 48 49 def _install_nativetests(self, host, test_file_format, artifact): 50 """Install the nativetests zip onto the DUT. 51 52 Device images built by the Android Build System do not have the 53 native gTests installed. This method requests the devserver to 54 download the nativetests package into the lab, the test station 55 will download/unzip the package, and finally install it onto the DUT. 56 57 @param host: host object to install the nativetests onto. 58 @param test_file_format: Format of the zip file containing the tests. 59 @param artifact: Devserver artifact to stage. 60 """ 61 info = host.host_info_store.get() 62 ds = dev_server.AndroidBuildServer.resolve(info.build, host.hostname) 63 ds.stage_artifacts(image=info.build, artifacts=[artifact]) 64 build_url = os.path.join(ds.url(), 'static', info.build) 65 nativetests_file = (test_file_format % 66 host.get_build_info_from_build_url(build_url)) 67 tmp_dir = host.teststation.get_tmp_dir() 68 host.download_file(build_url, nativetests_file, tmp_dir, unzip=True) 69 host.adb_run('push %s %s' % (os.path.join(tmp_dir, 'DATA', 70 'nativetest'), 71 NATIVE_TESTS_PATH)) 72 73 74 def _get_whitelisted_tests(self, whitelist_path): 75 """Return the list of whitelisted tests. 76 77 The whitelist is expected to be a three column CSV file containing: 78 * the test name 79 * "yes" or "no" whether the test should be run as root or not. 80 * optional command line arguments to be passed to the test. 81 82 Anything after a # on a line is considered to be a comment and ignored. 83 84 @param whitelist_path: Path to the whitelist. 85 86 @return a map of test name to GtestSuite tuple. 87 """ 88 suite_map = dict() 89 for line in self.host.run_output( 90 'cat %s' % whitelist_path).splitlines(): 91 # Remove anything after the first # (comments). 92 line = line.split('#')[0] 93 if line.strip() == '': 94 continue 95 96 parts = line.split(',') 97 if len(parts) < 2: 98 logging.error('badly formatted line in %s: %s', whitelist_path, 99 line) 100 continue 101 102 name = parts[0].strip() 103 extra_args = parts[2].strip() if len(parts) > 2 else '' 104 path = '' # Path will be updated if the test is present on the DUT. 105 suite_map[name] = GtestSuite(name, path, parts[1].strip() == 'yes', 106 extra_args) 107 return suite_map 108 109 110 def _find_all_gtestsuites(self, use_whitelist=False, filter_tests=None): 111 """Find all the gTest Suites installed on the DUT. 112 113 @param use_whitelist: Only whitelisted tests found on the system will 114 be used. 115 @param filter_tests: Only tests that match these globs will be used. 116 """ 117 list_cmd = LIST_TEST_BINARIES_TEMPLATE % {'path': NATIVE_TESTS_PATH} 118 gtest_suites_path = self.host.run_output(list_cmd).splitlines() 119 gtest_suites = [GtestSuite(os.path.basename(path), path, True, '') 120 for path in gtest_suites_path] 121 122 if use_whitelist: 123 try: 124 whitelisted = self._get_whitelisted_tests(WHITELIST_FILE) 125 suites_to_run = [] 126 for suite in gtest_suites: 127 if whitelisted.get(suite.name): 128 whitelisted_suite = whitelisted.get(suite.name) 129 # Get the name and path from the suites on the DUT and 130 # get the other args from the whitelist map. 131 suites_to_run.append(GtestSuite( 132 suite.name, suite.path, 133 whitelisted_suite.run_as_root, 134 whitelisted_suite.args)) 135 gtest_suites = suites_to_run 136 if (len(suites_to_run) != len(whitelisted)): 137 whitelist_test_names = set(whitelisted.keys()) 138 found_test_names = set([t.name for t in suites_to_run]) 139 diff_tests = list(whitelist_test_names - found_test_names) 140 for t in diff_tests: 141 logging.warning('Could not find %s', t); 142 raise error.TestWarn( 143 'Not all whitelisted tests found on the DUT. ' 144 'Expected %i tests but only found %i' % 145 (len(whitelisted), len(suites_to_run))) 146 except error.GenericHostRunError: 147 logging.error('Failed to read whitelist %s', WHITELIST_FILE) 148 149 if filter_tests: 150 gtest_suites = [t for t in gtest_suites 151 if any(fnmatch.fnmatch(t.path, n) 152 for n in filter_tests)] 153 logging.info('Running tests:\n %s', 154 '\n '.join(t.path for t in gtest_suites)) 155 156 if not gtest_suites: 157 raise error.TestWarn('No test executables found on the DUT') 158 logging.debug('Test executables found:\n%s', 159 '\n'.join([str(t) for t in gtest_suites])) 160 return gtest_suites 161 162 163 def run_gtestsuite(self, gtestSuite): 164 """Run a gTest Suite. 165 166 @param gtestSuite: GtestSuite tuple. 167 168 @return True if the all the tests in the gTest Suite pass. False 169 otherwise. 170 """ 171 # Make sure the gTest Suite exists. 172 result = self.host.run('test -e %s' % gtestSuite.path, 173 ignore_status=True) 174 if not result.exit_status == 0: 175 logging.error('Unable to find %s', gtestSuite.path) 176 return False 177 178 result = self.host.run('test -x %s' % gtestSuite.path, 179 ignore_status=True) 180 if not result.exit_status == 0: 181 self.host.run('chmod +x %s' % gtestSuite.path) 182 183 logging.debug('Running: %s', gtestSuite) 184 command = '%s %s' % (gtestSuite.path, gtestSuite.args) 185 if not gtestSuite.run_as_root: 186 command = 'su shell %s' % command 187 188 # host.run() will print the stdout/stderr output in the debug logs 189 # properly interleaved. 190 result = self.host.run(command, ignore_status=True) 191 192 parser = gtest_parser.gtest_parser() 193 for line in result.stdout.splitlines(): 194 parser.ProcessLogLine(line) 195 passed_tests = parser.PassedTests() 196 if passed_tests: 197 logging.debug('Passed Tests: %s', passed_tests) 198 failed_tests = parser.FailedTests(include_fails=True, 199 include_flaky=True) 200 if failed_tests: 201 logging.error('Failed Tests: %s', failed_tests) 202 for test in failed_tests: 203 logging.error('Test %s failed:\n%s', test, 204 parser.FailureDescription(test)) 205 return False 206 if result.exit_status != 0: 207 logging.error('%s exited with exit code: %s', 208 gtestSuite, result.exit_status) 209 return False 210 return True 211 212 213 def run_once(self, host=None, gtest_suites=None, use_whitelist=False, 214 filter_tests=None, native_tests=None): 215 """Run gTest Suites on the DUT. 216 217 @param host: host object representing the device under test. 218 @param gtest_suites: List of gTest suites to run. Default is to run 219 every gTest suite on the host. 220 @param use_whitelist: If gTestSuites is not passed in and use_whitelist 221 is true, only whitelisted tests found on the 222 system will be used. 223 @param filter_tests: If gTestSuites is not passed in, search for tests 224 that match these globs to run instead. 225 @param native_tests: Execute these specific tests. 226 227 @raise TestFail: The test failed. 228 """ 229 self.host = host 230 if not gtest_suites and native_tests: 231 gtest_suites = [GtestSuite('', t, True, '') for t in native_tests] 232 if not gtest_suites: 233 gtest_suites = self._find_all_gtestsuites( 234 use_whitelist=use_whitelist, filter_tests=filter_tests) 235 236 failed_gtest_suites = [] 237 for gtestSuite in gtest_suites: 238 if not self.run_gtestsuite(gtestSuite): 239 failed_gtest_suites.append(gtestSuite) 240 241 if failed_gtest_suites: 242 logging.error( 243 'The following gTest Suites failed: \n %s', 244 '\n'.join([str(t) for t in failed_gtest_suites])) 245 raise error.TestFail( 246 'Not all gTest Suites completed successfully. ' 247 '%s out of %s suites failed. ' 248 'Failed Suites: %s' 249 % (len(failed_gtest_suites), 250 len(gtest_suites), 251 failed_gtest_suites)) 252