1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.testtype; 18 19 import static com.android.tradefed.testtype.coverage.CoverageOptions.Toolchain.CLANG; 20 import static com.android.tradefed.util.EnvironmentVariableUtil.buildMinimalLdLibraryPath; 21 22 import com.android.ddmlib.IShellOutputReceiver; 23 import com.android.tradefed.build.BuildInfoKey.BuildInfoFileKey; 24 import com.android.tradefed.build.DeviceBuildInfo; 25 import com.android.tradefed.build.IBuildInfo; 26 import com.android.tradefed.cache.ICacheClient; 27 import com.android.tradefed.config.Option; 28 import com.android.tradefed.config.OptionClass; 29 import com.android.tradefed.device.DeviceNotAvailableException; 30 import com.android.tradefed.error.HarnessRuntimeException; 31 import com.android.tradefed.invoker.TestInformation; 32 import com.android.tradefed.log.ITestLogger; 33 import com.android.tradefed.log.LogUtil.CLog; 34 import com.android.tradefed.result.FileInputStreamSource; 35 import com.android.tradefed.result.ITestInvocationListener; 36 import com.android.tradefed.result.LogDataType; 37 import com.android.tradefed.result.error.TestErrorIdentifier; 38 import com.android.tradefed.util.CacheClientFactory; 39 import com.android.tradefed.util.ClangProfileIndexer; 40 import com.android.tradefed.util.CommandResult; 41 import com.android.tradefed.util.CommandStatus; 42 import com.android.tradefed.util.FileUtil; 43 import com.android.tradefed.util.IRunUtil.EnvPriority; 44 import com.android.tradefed.util.RunUtil; 45 import com.android.tradefed.util.ShellOutputReceiverStream; 46 import com.android.tradefed.util.TestRunnerUtil; 47 48 import org.json.JSONException; 49 import org.json.JSONObject; 50 51 import java.io.File; 52 import java.io.FileOutputStream; 53 import java.io.IOException; 54 import java.util.ArrayList; 55 import java.util.Arrays; 56 import java.util.LinkedHashMap; 57 import java.util.LinkedHashSet; 58 import java.util.List; 59 import java.util.Map; 60 import java.util.Set; 61 import java.util.stream.Collectors; 62 63 /** A Test that runs a native test package. */ 64 @OptionClass(alias = "hostgtest") 65 public class HostGTest extends GTestBase implements IBuildReceiver { 66 private static final long DEFAULT_HOST_COMMAND_TIMEOUT_MS = 2 * 60 * 1000; 67 68 private IBuildInfo mBuildInfo = null; 69 70 @Option( 71 name = "use-updated-shard-retry", 72 description = "Whether to use the updated logic for retry with sharding.") 73 private boolean mUseUpdatedShardRetry = true; 74 75 @Option( 76 name = "inherit-env-vars", 77 description = 78 "Whether the subprocess should inherit environment variables from the main" 79 + " process.") 80 private boolean mInheritEnvVars = true; 81 82 @Option( 83 name = "use-minimal-shared-libs", 84 description = "Whether use the shared libs in per module folder.") 85 private boolean mUseMinimalSharedLibs = false; 86 87 /** Whether any incomplete test is found in the current run. */ 88 private boolean mIncompleteTestFound = false; 89 90 /** List of tests that failed in the current test run when test run was complete. */ 91 private Set<String> mCurFailedTests = new LinkedHashSet<>(); 92 93 @Override setBuild(IBuildInfo buildInfo)94 public void setBuild(IBuildInfo buildInfo) { 95 this.mBuildInfo = buildInfo; 96 } 97 98 /** 99 * @param cmd command that want to execute in host 100 * @return the {@link CommandResult} of command 101 */ executeHostCommand(String cmd)102 public CommandResult executeHostCommand(String cmd) { 103 return executeHostCommand(cmd, DEFAULT_HOST_COMMAND_TIMEOUT_MS); 104 } 105 106 /** 107 * @param cmd command that want to execute in host 108 * @param timeoutMs timeout for command in milliseconds 109 * @return the {@link CommandResult} of command 110 */ executeHostCommand(String cmd, long timeoutMs)111 public CommandResult executeHostCommand(String cmd, long timeoutMs) { 112 String[] cmds = cmd.split("\\s+"); 113 return RunUtil.getDefault().runTimedCmd(timeoutMs, cmds); 114 } 115 116 /** 117 * @param gtestFile file pointing to the binary to be executed 118 * @param cmd command that want to execute in host 119 * @param timeoutMs timeout for command in milliseconds 120 * @param receiver the result parser 121 * @return the {@link CommandResult} of command 122 */ executeHostGTestCommand( File gtestFile, String cmd, long timeoutMs, IShellOutputReceiver receiver, ITestLogger logger)123 private CommandResult executeHostGTestCommand( 124 File gtestFile, 125 String cmd, 126 long timeoutMs, 127 IShellOutputReceiver receiver, 128 ITestLogger logger) { 129 RunUtil runUtil = new RunUtil(mInheritEnvVars); 130 String[] cmds = cmd.split("\\s+"); 131 132 if (getShardCount() > 0) { 133 if (isCollectTestsOnly()) { 134 CLog.w( 135 "--collect-tests-only option ignores sharding parameters, and will cause " 136 + "each shard to collect all tests."); 137 } 138 runUtil.setEnvVariable("GTEST_SHARD_INDEX", Integer.toString(getShardIndex())); 139 runUtil.setEnvVariable("GTEST_TOTAL_SHARDS", Integer.toString(getShardCount())); 140 } 141 142 // Set the RunUtil to combine stderr with stdout so that they are interleaved correctly. 143 runUtil.setRedirectStderrToStdout(true); 144 // Set the working dir to the folder containing the binary to execute from the same path. 145 runUtil.setWorkingDir(gtestFile.getParentFile()); 146 147 String separator = System.getProperty("path.separator"); 148 List<String> paths = new ArrayList<>(); 149 paths.add("/usr/bin"); 150 paths.add("/usr/sbin"); 151 paths.add("."); 152 String path = paths.stream().distinct().collect(Collectors.joining(separator)); 153 CLog.d("Using updated $PATH: %s", path); 154 runUtil.setEnvVariablePriority(EnvPriority.SET); 155 runUtil.setEnvVariable("PATH", path); 156 157 // Update LD_LIBRARY_PATH 158 String ldLibraryPath = 159 mUseMinimalSharedLibs 160 ? buildMinimalLdLibraryPath( 161 gtestFile.getParentFile(), Arrays.asList("shared_libs")) 162 : TestRunnerUtil.getLdLibraryPath(gtestFile); 163 if (ldLibraryPath != null) { 164 runUtil.setEnvVariable("LD_LIBRARY_PATH", ldLibraryPath); 165 } 166 167 // Set LLVM_PROFILE_FILE for coverage. 168 File coverageDir = null; 169 if (isClangCoverageEnabled()) { 170 try { 171 coverageDir = FileUtil.createTempDir("clang"); 172 } catch (IOException e) { 173 throw new RuntimeException(e); 174 } 175 runUtil.setEnvVariable( 176 "LLVM_PROFILE_FILE", coverageDir.getAbsolutePath() + "/clang-%m.profraw"); 177 } 178 179 // If there's a shell output receiver to pass results along to, then 180 // ShellOutputReceiverStream will write that into the IShellOutputReceiver. If not, the 181 // command output will just be ignored. 182 CommandResult result = null; 183 File stdout = null; 184 try { 185 stdout = 186 FileUtil.createTempFile( 187 String.format("%s-output", gtestFile.getName()), ".txt"); 188 try (ShellOutputReceiverStream stream = 189 new ShellOutputReceiverStream(receiver, new FileOutputStream(stdout))) { 190 result = runUtil.runTimedCmdWithOutputMonitor(timeoutMs, 0, stream, null, cmds); 191 } catch (IOException e) { 192 throw new RuntimeException( 193 "Should never happen, ShellOutputReceiverStream.close is a no-op", e); 194 } 195 } catch (IOException e) { 196 throw new RuntimeException(e); 197 } finally { 198 // Flush before the log to ensure order of events 199 receiver.flush(); 200 try { 201 // Add a small extra log to the output for verification sake. 202 FileUtil.writeToFile( 203 String.format( 204 "\nBinary '%s' still exists: %s", gtestFile, gtestFile.exists()), 205 stdout, 206 true); 207 } catch (IOException e) { 208 // Ignore 209 } 210 if (stdout != null && stdout.length() > 0L) { 211 try (FileInputStreamSource source = new FileInputStreamSource(stdout, true)) { 212 logger.testLog( 213 String.format("%s-output", gtestFile.getName()), 214 LogDataType.TEXT, 215 source); 216 } 217 } 218 FileUtil.deleteFile(stdout); 219 220 if (isClangCoverageEnabled()) { 221 File profdata = null; 222 try { 223 Set<String> profraws = FileUtil.findFiles(coverageDir, ".*\\.profraw"); 224 ClangProfileIndexer indexer = 225 new ClangProfileIndexer( 226 getConfiguration().getCoverageOptions().getLlvmProfdataPath()); 227 profdata = FileUtil.createTempFile(gtestFile.getName(), ".profdata"); 228 indexer.index(profraws, profdata); 229 230 try (FileInputStreamSource source = new FileInputStreamSource(profdata, true)) { 231 logger.testLog(gtestFile.getName(), LogDataType.CLANG_COVERAGE, source); 232 } 233 } catch (IOException e) { 234 throw new RuntimeException(e); 235 } finally { 236 FileUtil.deleteFile(profdata); 237 FileUtil.recursiveDelete(coverageDir); 238 } 239 } 240 } 241 return result; 242 } 243 244 /** {@inheritDoc} */ 245 @Override loadFilter(String binaryOnHost)246 public String loadFilter(String binaryOnHost) { 247 try { 248 CLog.i("Loading filter from file for key: '%s'", getTestFilterKey()); 249 String filterFileName = String.format("%s%s", binaryOnHost, FILTER_EXTENSION); 250 File filterFile = new File(filterFileName); 251 if (filterFile.exists()) { 252 CommandResult cmdResult = 253 executeHostCommand(String.format("cat %s", filterFileName)); 254 String content = cmdResult.getStdout(); 255 if (content != null && !content.isEmpty()) { 256 JSONObject filter = new JSONObject(content); 257 String key = getTestFilterKey(); 258 JSONObject filterObject = filter.getJSONObject(key); 259 return filterObject.getString("filter"); 260 } 261 CLog.e("Error with content of the filter file %s: %s", filterFile, content); 262 } else { 263 CLog.e("Filter file %s not found", filterFile); 264 } 265 } catch (JSONException e) { 266 CLog.e(e); 267 } 268 return null; 269 } 270 271 /** 272 * Run the given gtest binary 273 * 274 * @param resultParser the test run output parser 275 * @param gtestFile file pointing to gtest binary 276 * @param flags gtest execution flags 277 */ runTest( final IShellOutputReceiver resultParser, final File gtestFile, final String flags, ITestLogger logger)278 private void runTest( 279 final IShellOutputReceiver resultParser, 280 final File gtestFile, 281 final String flags, 282 ITestLogger logger) { 283 for (String cmd : getBeforeTestCmd()) { 284 CommandResult result = executeHostCommand(cmd); 285 if (!result.getStatus().equals(CommandStatus.SUCCESS)) { 286 throw new RuntimeException("'Before test' command failed: " + result.getStderr()); 287 } 288 } 289 290 long maxTestTimeMs = getMaxTestTimeMs(); 291 String cmd = getGTestCmdLine(gtestFile.getAbsolutePath(), flags); 292 CommandResult testResult = 293 executeHostGTestCommand(gtestFile, cmd, maxTestTimeMs, resultParser, logger); 294 // TODO: Switch throwing exceptions to use ITestInvocation.testRunFailed 295 switch (testResult.getStatus()) { 296 case TIMED_OUT: 297 throw new HarnessRuntimeException( 298 String.format("Command run timed out after %d ms", maxTestTimeMs), 299 TestErrorIdentifier.TEST_BINARY_TIMED_OUT); 300 case EXCEPTION: 301 throw new RuntimeException("Command run failed with exception"); 302 case FAILED: 303 // Check the command exit code. If it's 1, then this is just a red herring; 304 // gtest returns 1 when a test fails. 305 final Integer exitCode = testResult.getExitCode(); 306 if (exitCode == null || exitCode != 1) { 307 // No need to handle it as the parser would have reported it already. 308 CLog.e("Command run failed with exit code %s", exitCode); 309 } 310 break; 311 default: 312 break; 313 } 314 // Execute the host command if nothing failed badly before. 315 for (String afterCmd : getAfterTestCmd()) { 316 CommandResult result = executeHostCommand(afterCmd); 317 if (!result.getStatus().equals(CommandStatus.SUCCESS)) { 318 throw new RuntimeException("'After test' command failed: " + result.getStderr()); 319 } 320 } 321 } 322 323 /** {@inheritDoc} */ 324 @Override run(TestInformation testInfo, ITestInvocationListener listener)325 public void run(TestInformation testInfo, ITestInvocationListener listener) 326 throws DeviceNotAvailableException { // DNAE is part of IRemoteTest. 327 try { 328 // Reset flags that are used to track results of current test run. 329 mIncompleteTestFound = false; 330 mCurFailedTests = new LinkedHashSet<>(); 331 332 // Get testcases directory using the key HOST_LINKED_DIR first. 333 // If the directory is null, then get testcase directory from getTestDir() since *TS 334 // will invoke setTestDir(). 335 List<File> scanDirs = new ArrayList<>(); 336 File hostLinkedDir = mBuildInfo.getFile(BuildInfoFileKey.HOST_LINKED_DIR); 337 if (hostLinkedDir != null) { 338 scanDirs.add(hostLinkedDir); 339 } 340 File testsDir = ((DeviceBuildInfo) mBuildInfo).getTestsDir(); 341 if (testsDir != null) { 342 scanDirs.add(testsDir); 343 } 344 345 String moduleName = getTestModule(); 346 Set<File> gTestFiles; 347 try { 348 gTestFiles = 349 FileUtil.findFiles( 350 moduleName, getAbi(), false, scanDirs.toArray(new File[] {})); 351 gTestFiles = applyFileExclusionFilters(gTestFiles); 352 } catch (IOException e) { 353 throw new RuntimeException(e); 354 } 355 if (gTestFiles == null || gTestFiles.isEmpty()) { 356 // If we ended up here we most likely failed to find the proper file as is, so we 357 // search for it with a potential suffix (which is allowed). 358 try { 359 gTestFiles = 360 FileUtil.findFiles( 361 moduleName + ".*", 362 getAbi(), 363 false, 364 scanDirs.toArray(new File[] {})); 365 gTestFiles = applyFileExclusionFilters(gTestFiles); 366 } catch (IOException e) { 367 throw new RuntimeException(e); 368 } 369 } 370 371 if (gTestFiles == null || gTestFiles.isEmpty()) { 372 throw new RuntimeException( 373 String.format( 374 "Fail to find native test %s in directory %s.", 375 moduleName, scanDirs)); 376 } 377 // Since we searched files in multiple directories, it is possible that we may have the 378 // same file in different source directories. Exclude duplicates. 379 gTestFiles = excludeDuplicateFiles(gTestFiles); 380 for (File gTestFile : gTestFiles) { 381 if (!gTestFile.canExecute()) { 382 CLog.i("%s is not executable! Skipping.", gTestFile.getAbsolutePath()); 383 continue; 384 } 385 386 listener = getGTestListener(listener); 387 // TODO: Need to support XML test output based on isEnableXmlOutput 388 IShellOutputReceiver resultParser = 389 createResultParser(gTestFile.getName(), listener); 390 String flags = getAllGTestFlags(gTestFile.getName()); 391 CLog.i("Running gtest %s %s", gTestFile.getName(), flags); 392 try { 393 runTest(resultParser, gTestFile, flags, listener); 394 } finally { 395 if (resultParser instanceof GTestResultParser) { 396 if (((GTestResultParser) resultParser).isTestRunIncomplete()) { 397 mIncompleteTestFound = true; 398 } else { 399 // if test run is complete, collect the failed tests so that they can be 400 // retried 401 mCurFailedTests.addAll( 402 ((GTestResultParser) resultParser).getFailedTests()); 403 } 404 } 405 } 406 } 407 } catch (Throwable t) { 408 // if we encounter any errors, count it as test Incomplete so that retry attempts 409 // during sharding uses a full retry. 410 mIncompleteTestFound = true; 411 throw t; 412 } finally { 413 if (mUseUpdatedShardRetry) { 414 // notify of test execution will enable the new sharding retry behavior since Gtest 415 // will be aware of retries. 416 notifyTestExecution(mIncompleteTestFound, mCurFailedTests); 417 } 418 } 419 } 420 421 /** 422 * Apply exclusion filters and return the remaining files. 423 * 424 * @param filesToFilterFrom a set of files which need to be filtered. 425 * @return a set of files 426 */ applyFileExclusionFilters(Set<File> filesToFilterFrom)427 private Set<File> applyFileExclusionFilters(Set<File> filesToFilterFrom) { 428 Set<File> retFiles = new LinkedHashSet<>(); 429 Set<String> fileExclusionFilterRegex = getFileExclusionFilterRegex(); 430 for (File file : filesToFilterFrom) { 431 boolean matchedRegex = false; 432 for (String regex : fileExclusionFilterRegex) { 433 if (file.getPath().matches(regex)) { 434 CLog.i( 435 "File %s matches exclusion file regex %s, skipping", 436 file.getPath(), regex); 437 matchedRegex = true; 438 break; 439 } 440 } 441 if (!matchedRegex) { 442 retFiles.add(file); 443 } 444 } 445 return retFiles; 446 } 447 448 /** exclude files with same names */ excludeDuplicateFiles(Set<File> files)449 private Set<File> excludeDuplicateFiles(Set<File> files) { 450 Map<String, File> seen = new LinkedHashMap<>(); 451 for (File file : files) { 452 if (seen.containsKey(file.getName())) { 453 CLog.i( 454 "File %s already exists in location %s. skipping %s.", 455 file.getName(), 456 seen.get(file.getName()).getAbsolutePath(), 457 file.getAbsolutePath()); 458 } else { 459 seen.put(file.getName(), file); 460 } 461 } 462 return new LinkedHashSet<>(seen.values()); 463 } 464 465 /** Returns whether Clang code coverage is enabled. */ isClangCoverageEnabled()466 private boolean isClangCoverageEnabled() { 467 return getConfiguration().getCoverageOptions().isCoverageEnabled() 468 && getConfiguration().getCoverageOptions().getCoverageToolchains().contains(CLANG); 469 } 470 getCacheClient(File workFolder, String instanceName)471 ICacheClient getCacheClient(File workFolder, String instanceName) { 472 return CacheClientFactory.createCacheClient(workFolder, instanceName); 473 } 474 } 475