1 /* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.compatibility.common.tradefed.testtype; 18 19 import com.android.compatibility.SuiteInfo; 20 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 21 import com.android.compatibility.common.tradefed.targetprep.NetworkConnectivityChecker; 22 import com.android.compatibility.common.tradefed.targetprep.SystemStatusChecker; 23 import com.android.compatibility.common.tradefed.util.OptionHelper; 24 import com.android.compatibility.common.util.AbiUtils; 25 import com.android.compatibility.common.util.ICaseResult; 26 import com.android.compatibility.common.util.IInvocationResult; 27 import com.android.compatibility.common.util.IModuleResult; 28 import com.android.compatibility.common.util.ITestResult; 29 import com.android.compatibility.common.util.ResultHandler; 30 import com.android.compatibility.common.util.TestFilter; 31 import com.android.compatibility.common.util.TestStatus; 32 import com.android.ddmlib.Log.LogLevel; 33 import com.android.tradefed.build.IBuildInfo; 34 import com.android.tradefed.config.ArgsOptionParser; 35 import com.android.tradefed.config.ConfigurationException; 36 import com.android.tradefed.config.ConfigurationFactory; 37 import com.android.tradefed.config.IConfiguration; 38 import com.android.tradefed.config.IConfigurationFactory; 39 import com.android.tradefed.config.Option; 40 import com.android.tradefed.config.Option.Importance; 41 import com.android.tradefed.config.OptionClass; 42 import com.android.tradefed.config.OptionCopier; 43 import com.android.tradefed.device.DeviceNotAvailableException; 44 import com.android.tradefed.device.DeviceUnresponsiveException; 45 import com.android.tradefed.device.ITestDevice; 46 import com.android.tradefed.log.ITestLogger; 47 import com.android.tradefed.log.LogUtil.CLog; 48 import com.android.tradefed.result.ITestInvocationListener; 49 import com.android.tradefed.result.InputStreamSource; 50 import com.android.tradefed.result.LogDataType; 51 import com.android.tradefed.targetprep.ITargetPreparer; 52 import com.android.tradefed.testtype.IAbi; 53 import com.android.tradefed.testtype.IBuildReceiver; 54 import com.android.tradefed.testtype.IDeviceTest; 55 import com.android.tradefed.testtype.IRemoteTest; 56 import com.android.tradefed.testtype.IShardableTest; 57 import com.android.tradefed.util.AbiFormatter; 58 import com.android.tradefed.util.ArrayUtil; 59 import com.android.tradefed.util.TimeUtil; 60 61 import java.io.ByteArrayOutputStream; 62 import java.io.FileNotFoundException; 63 import java.io.IOException; 64 import java.io.PrintWriter; 65 import java.util.ArrayList; 66 import java.util.Collection; 67 import java.util.HashSet; 68 import java.util.LinkedList; 69 import java.util.List; 70 import java.util.Set; 71 72 /** 73 * A Test for running Compatibility Suites 74 */ 75 @OptionClass(alias = "compatibility") 76 public class CompatibilityTest implements IDeviceTest, IShardableTest, IBuildReceiver { 77 78 public static final String INCLUDE_FILTER_OPTION = "include-filter"; 79 public static final String EXCLUDE_FILTER_OPTION = "exclude-filter"; 80 private static final String PLAN_OPTION = "plan"; 81 private static final String MODULE_OPTION = "module"; 82 private static final String TEST_OPTION = "test"; 83 private static final String MODULE_ARG_OPTION = "module-arg"; 84 private static final String TEST_ARG_OPTION = "test-arg"; 85 public static final String RETRY_OPTION = "retry"; 86 private static final String ABI_OPTION = "abi"; 87 private static final String SHARD_OPTION = "shards"; 88 public static final String SKIP_DEVICE_INFO_OPTION = "skip-device-info"; 89 public static final String SKIP_PRECONDITIONS_OPTION = "skip-preconditions"; 90 public static final String PRIMARY_ABI_RUN = "primary-abi-only"; 91 public static final String DEVICE_TOKEN_OPTION = "device-token"; 92 private static final String URL = "dynamic-config-url"; 93 94 /* API Key for compatibility test project, used for dynamic configuration */ 95 private static final String API_KEY = "AIzaSyAbwX5JRlmsLeygY2WWihpIJPXFLueOQ3U"; 96 97 98 @Option(name = PLAN_OPTION, 99 description = "the test suite plan to run, such as \"everything\" or \"cts\"", 100 importance = Importance.ALWAYS) 101 private String mSuitePlan; 102 103 @Option(name = INCLUDE_FILTER_OPTION, 104 description = "the include module filters to apply.", 105 importance = Importance.ALWAYS) 106 private List<String> mIncludeFilters = new ArrayList<>(); 107 108 @Option(name = EXCLUDE_FILTER_OPTION, 109 description = "the exclude module filters to apply.", 110 importance = Importance.ALWAYS) 111 private List<String> mExcludeFilters = new ArrayList<>(); 112 113 @Option(name = MODULE_OPTION, 114 shortName = 'm', 115 description = "the test module to run.", 116 importance = Importance.IF_UNSET) 117 private String mModuleName = null; 118 119 @Option(name = TEST_OPTION, 120 shortName = 't', 121 description = "the test run.", 122 importance = Importance.IF_UNSET) 123 private String mTestName = null; 124 125 @Option(name = MODULE_ARG_OPTION, 126 description = "the arguments to pass to a module. The expected format is" 127 + "\"<module-name>:<arg-name>:<arg-value>\"", 128 importance = Importance.ALWAYS) 129 private List<String> mModuleArgs = new ArrayList<>(); 130 131 @Option(name = TEST_ARG_OPTION, 132 description = "the arguments to pass to a test. The expected format is" 133 + "\"<test-class>:<arg-name>:<arg-value>\"", 134 importance = Importance.ALWAYS) 135 private List<String> mTestArgs = new ArrayList<>(); 136 137 @Option(name = RETRY_OPTION, 138 shortName = 'r', 139 description = "retry a previous session.", 140 importance = Importance.IF_UNSET) 141 private Integer mRetrySessionId = null; 142 143 @Option(name = ABI_OPTION, 144 shortName = 'a', 145 description = "the abi to test.", 146 importance = Importance.IF_UNSET) 147 private String mAbiName = null; 148 149 @Option(name = SHARD_OPTION, 150 description = "split the modules up to run on multiple devices concurrently.") 151 private int mShards = 1; 152 153 @Option(name = URL, 154 description = "Specify the url for override config") 155 private String mURL = "https://androidpartner.googleapis.com/v1/dynamicconfig/" 156 + "suites/{suite-name}/modules/{module}/version/{version}?key=" + API_KEY; 157 158 @Option(name = SKIP_DEVICE_INFO_OPTION, 159 shortName = 'd', 160 description = "Whether device info collection should be skipped") 161 private boolean mSkipDeviceInfo = false; 162 163 @Option(name = SKIP_PRECONDITIONS_OPTION, 164 shortName = 'o', 165 description = "Whether preconditions should be skipped") 166 private boolean mSkipPreconditions = false; 167 168 @Option(name = PRIMARY_ABI_RUN, 169 description = "Whether to run tests with only the device primary abi. " 170 + "This override the --abi option.") 171 private boolean mPrimaryAbiRun = false; 172 173 @Option(name = DEVICE_TOKEN_OPTION, 174 description = "Holds the devices' tokens, used when scheduling tests that have" 175 + "prerequisites such as requiring a SIM card. Format is <serial>:<token>", 176 importance = Importance.ALWAYS) 177 private List<String> mDeviceTokens = new ArrayList<>(); 178 179 @Option(name = "bugreport-on-failure", 180 description = "Take a bugreport on every test failure. " + 181 "Warning: can potentially use a lot of disk space.") 182 private boolean mBugReportOnFailure = false; 183 184 @Option(name = "logcat-on-failure", 185 description = "Take a logcat snapshot on every test failure.") 186 private boolean mLogcatOnFailure = false; 187 188 @Option(name = "screenshot-on-failure", 189 description = "Take a screenshot on every test failure.") 190 private boolean mScreenshotOnFailure = false; 191 192 @Option(name = "reboot-before-test", 193 description = "Reboot the device before the test suite starts.") 194 private boolean mRebootBeforeTest = false; 195 196 @Option(name = "reboot-on-failure", 197 description = "Reboot the device after every test failure.") 198 private boolean mRebootOnFailure = false; 199 200 @Option(name = "reboot-per-module", 201 description = "Reboot the device before every module run.") 202 private boolean mRebootPerModule = false; 203 204 @Option(name = "skip-connectivity-check", 205 description = "Don't verify device connectivity between module execution.") 206 private boolean mSkipConnectivityCheck = false; 207 208 @Option(name = "preparer-whitelist", 209 description = "Only run specific preparers." 210 + "Specify zero or more ITargetPreparers as canonical class names. " 211 + "e.g. \"com.android.compatibility.common.tradefed.targetprep.ApkInstaller\" " 212 + "If not specified, all configured preparers are run.") 213 private Set<String> mPreparerWhitelist = new HashSet<>(); 214 215 @Option(name = "skip-all-system-status-check", 216 description = "Whether all system status check between modules should be skipped") 217 private boolean mSkipAllSystemStatusCheck = false; 218 219 @Option(name = "skip-system-status-check", 220 description = "Disable specific system status checkers." 221 + "Specify zero or more SystemStatusChecker as canonical class names. e.g. " 222 + "\"com.android.compatibility.common.tradefed.targetprep.NetworkConnectivityChecker\" " 223 + "If not specified, all configured or whitelisted system status checkers are run.") 224 private Set<String> mSystemStatusCheckBlacklist = new HashSet<>(); 225 226 @Option(name = "system-status-check-whitelist", 227 description = "Only run specific system status checkers." 228 + "Specify zero or more SystemStatusChecker as canonical class names. e.g. " 229 + "\"com.android.compatibility.common.tradefed.targetprep.NetworkConnectivityChecker\" " 230 + "If not specified, all configured system status checkers are run.") 231 private Set<String> mSystemStatusCheckWhitelist = new HashSet<>(); 232 233 @Option(name = "system-status-checker-config", description = "Configuration file for system " 234 + "status checkers invoked between module execution.") 235 private String mSystemStatusCheckerConfig = "system-status-checkers"; 236 237 private int mTotalShards; 238 private IModuleRepo mModuleRepo; 239 private ITestDevice mDevice; 240 private CompatibilityBuildHelper mBuildHelper; 241 242 /** 243 * Create a new {@link CompatibilityTest} that will run the default list of 244 * modules. 245 */ CompatibilityTest()246 public CompatibilityTest() { 247 this(1 /* totalShards */, new ModuleRepo()); 248 } 249 250 /** 251 * Create a new {@link CompatibilityTest} that will run a sublist of 252 * modules. 253 */ CompatibilityTest(int totalShards, IModuleRepo moduleRepo)254 public CompatibilityTest(int totalShards, IModuleRepo moduleRepo) { 255 if (totalShards < 1) { 256 throw new IllegalArgumentException( 257 "Must be at least 1 shard. Given:" + totalShards); 258 } 259 mTotalShards = totalShards; 260 mModuleRepo = moduleRepo; 261 } 262 263 /** 264 * {@inheritDoc} 265 */ 266 @Override getDevice()267 public ITestDevice getDevice() { 268 return mDevice; 269 } 270 271 /** 272 * {@inheritDoc} 273 */ 274 @Override setDevice(ITestDevice device)275 public void setDevice(ITestDevice device) { 276 mDevice = device; 277 } 278 279 /** 280 * {@inheritDoc} 281 */ 282 @Override setBuild(IBuildInfo buildInfo)283 public void setBuild(IBuildInfo buildInfo) { 284 mBuildHelper = new CompatibilityBuildHelper(buildInfo); 285 // Initializing the mBuildHelper also updates properties in buildInfo. 286 // TODO(nicksauer): Keeping invocation properties around via buildInfo 287 // is confusing and would be better done in an "InvocationInfo". 288 // Note, the current time is used to generated the result directory. 289 mBuildHelper.init(mSuitePlan, mURL, System.currentTimeMillis()); 290 } 291 292 /** 293 * {@inheritDoc} 294 */ 295 @Override run(ITestInvocationListener listener)296 public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 297 try { 298 // Synchronized so only one shard enters and sets up the moduleRepo. When the other 299 // shards enter after this, moduleRepo is already initialized so they dont do anything 300 synchronized (mModuleRepo) { 301 if (!mModuleRepo.isInitialized()) { 302 setupFilters(); 303 // Initialize the repository, {@link CompatibilityBuildHelper#getTestsDir} can 304 // throw a {@link FileNotFoundException} 305 mModuleRepo.initialize(mTotalShards, mBuildHelper.getTestsDir(), getAbis(), 306 mDeviceTokens, mTestArgs, mModuleArgs, mIncludeFilters, 307 mExcludeFilters, mBuildHelper.getBuildInfo()); 308 309 // Add the entire list of modules to the CompatibilityBuildHelper for reporting 310 mBuildHelper.setModuleIds(mModuleRepo.getModuleIds()); 311 } 312 313 } 314 // Get the tests to run in this shard 315 List<IModuleDef> modules = mModuleRepo.getModules(getDevice().getSerialNumber()); 316 317 listener = new FailureListener(listener, getDevice(), mBugReportOnFailure, 318 mLogcatOnFailure, mScreenshotOnFailure, mRebootOnFailure); 319 int moduleCount = modules.size(); 320 CLog.logAndDisplay(LogLevel.INFO, "Starting %d module%s on %s", moduleCount, 321 (moduleCount > 1) ? "s" : "", mDevice.getSerialNumber()); 322 if (mRebootBeforeTest) { 323 CLog.d("Rebooting device before test starts as requested."); 324 mDevice.reboot(); 325 } 326 327 if (mSkipConnectivityCheck) { 328 String clazz = NetworkConnectivityChecker.class.getCanonicalName(); 329 CLog.logAndDisplay(LogLevel.INFO, "\"--skip-connectivity-check\" is deprecated, " 330 + "please use \"--skip-system-status-check %s\" instead", clazz); 331 mSystemStatusCheckBlacklist.add(clazz); 332 } 333 334 // Get system status checkers 335 List<SystemStatusChecker> checkers = null; 336 if (!mSkipAllSystemStatusCheck) { 337 try { 338 checkers = initSystemStatusCheckers(); 339 } catch (ConfigurationException ce) { 340 throw new RuntimeException("failed to load system status checker config", ce); 341 } 342 } 343 344 // Set values and run preconditions 345 for (int i = 0; i < moduleCount; i++) { 346 IModuleDef module = modules.get(i); 347 module.setBuild(mBuildHelper.getBuildInfo()); 348 module.setDevice(mDevice); 349 module.setPreparerWhitelist(mPreparerWhitelist); 350 module.prepare(mSkipPreconditions); 351 } 352 // Run the tests 353 for (int i = 0; i < moduleCount; i++) { 354 IModuleDef module = modules.get(i); 355 long start = System.currentTimeMillis(); 356 357 if (mRebootPerModule) { 358 if ("user".equals(mDevice.getProperty("ro.build.type"))) { 359 CLog.e("reboot-per-module should only be used during development, " 360 + "this is a\" user\" build device"); 361 } else { 362 CLog.logAndDisplay(LogLevel.INFO, "Rebooting device before starting next " 363 + "module"); 364 mDevice.reboot(); 365 } 366 } 367 368 // execute pre module execution checker 369 if (checkers != null && !checkers.isEmpty()) { 370 runPreModuleCheck(module.getName(), checkers, mDevice, listener); 371 } 372 try { 373 module.run(listener); 374 } catch (DeviceUnresponsiveException due) { 375 // being able to catch a DeviceUnresponsiveException here implies that recovery 376 // was successful, and test execution should proceed to next module 377 ByteArrayOutputStream stack = new ByteArrayOutputStream(); 378 due.printStackTrace(new PrintWriter(stack, true)); 379 try { 380 stack.close(); 381 } catch (IOException ioe) { 382 // won't happen on BAOS 383 } 384 CLog.w("Ignored DeviceUnresponsiveException because recovery was successful, " 385 + "proceeding with next module. Stack trace: %s", 386 stack.toString()); 387 CLog.w("This may be due to incorrect timeout setting on module %s", 388 module.getName()); 389 } 390 long duration = System.currentTimeMillis() - start; 391 long expected = module.getRuntimeHint(); 392 long delta = Math.abs(duration - expected); 393 // Show warning if delta is more than 10% of expected 394 if (expected > 0 && ((float)delta / (float)expected) > 0.1f) { 395 CLog.logAndDisplay(LogLevel.WARN, 396 "Inaccurate runtime hint for %s, expected %s was %s", 397 module.getId(), 398 TimeUtil.formatElapsedTime(expected), 399 TimeUtil.formatElapsedTime(duration)); 400 } 401 if (checkers != null && !checkers.isEmpty()) { 402 runPostModuleCheck(module.getName(), checkers, mDevice, listener); 403 } 404 } 405 } catch (FileNotFoundException fnfe) { 406 throw new RuntimeException("Failed to initialize modules", fnfe); 407 } 408 } 409 410 /** 411 * Gets the set of ABIs supported by both Compatibility and the device under test 412 * 413 * @return The set of ABIs to run the tests on 414 * @throws DeviceNotAvailableException 415 */ getAbis()416 Set<IAbi> getAbis() throws DeviceNotAvailableException { 417 Set<IAbi> abis = new HashSet<>(); 418 Set<String> archAbis = AbiUtils.getAbisForArch(SuiteInfo.TARGET_ARCH); 419 if (mPrimaryAbiRun) { 420 if (mAbiName == null) { 421 // Get the primary from the device and make it the --abi to run. 422 mAbiName = mDevice.getProperty("ro.product.cpu.abi").trim(); 423 } else { 424 CLog.d("Option --%s supersedes the option --%s, using abi: %s", ABI_OPTION, 425 PRIMARY_ABI_RUN, mAbiName); 426 } 427 } 428 for (String abi : AbiFormatter.getSupportedAbis(mDevice, "")) { 429 // Only test against ABIs supported by Compatibility, and if the 430 // --abi option was given, it must match. 431 if (AbiUtils.isAbiSupportedByCompatibility(abi) && archAbis.contains(abi) 432 && (mAbiName == null || mAbiName.equals(abi))) { 433 abis.add(new Abi(abi, AbiUtils.getBitness(abi))); 434 } 435 } 436 if (abis.isEmpty()) { 437 if (mAbiName == null) { 438 throw new IllegalArgumentException("Could not get device's ABIs"); 439 } else { 440 throw new IllegalArgumentException(String.format( 441 "Device %s doesn't support %s", mDevice.getSerialNumber(), mAbiName)); 442 } 443 } 444 return abis; 445 } 446 initSystemStatusCheckers()447 private List<SystemStatusChecker> initSystemStatusCheckers() throws ConfigurationException { 448 IConfigurationFactory cf = ConfigurationFactory.getInstance(); 449 IConfiguration config = cf.createConfigurationFromArgs( 450 new String[]{mSystemStatusCheckerConfig}); 451 // only checks the target preparers from the config 452 List<ITargetPreparer> preparers = config.getTargetPreparers(); 453 List<SystemStatusChecker> checkers = new ArrayList<>(); 454 for (ITargetPreparer p : preparers) { 455 if (p instanceof SystemStatusChecker) { 456 SystemStatusChecker s = (SystemStatusChecker)p; 457 if (shouldIncludeSystemStatusChecker(s)) { 458 checkers.add(s); 459 } else { 460 CLog.i("%s skipped because it's not whitelisted.", 461 s.getClass().getCanonicalName()); 462 } 463 } else { 464 CLog.w("Preparer %s does not have type %s, ignored ", 465 p.getClass().getCanonicalName(), 466 SystemStatusChecker.class.getCanonicalName()); 467 } 468 } 469 return checkers; 470 } 471 472 /** 473 * Resolve the inclusion and exclusion logic of system status checkers 474 * 475 * @param s the {@link SystemStatusChecker} to perform filtering logic on 476 * @return 477 */ shouldIncludeSystemStatusChecker(SystemStatusChecker s)478 private boolean shouldIncludeSystemStatusChecker(SystemStatusChecker s) { 479 String clazz = s.getClass().getCanonicalName(); 480 boolean shouldInclude = mSystemStatusCheckWhitelist.isEmpty() 481 || mSystemStatusCheckWhitelist.contains(clazz); 482 boolean shouldExclude = !mSystemStatusCheckBlacklist.isEmpty() 483 && mSystemStatusCheckBlacklist.contains(clazz); 484 return shouldInclude && !shouldExclude; 485 } 486 runPreModuleCheck(String moduleName, List<SystemStatusChecker> checkers, ITestDevice device, ITestLogger logger)487 private void runPreModuleCheck(String moduleName, List<SystemStatusChecker> checkers, 488 ITestDevice device, ITestLogger logger) throws DeviceNotAvailableException { 489 CLog.i("Running system status checker before module execution: %s", moduleName); 490 List<String> failures = new ArrayList<>(); 491 for (SystemStatusChecker checker : checkers) { 492 boolean result = checker.preExecutionCheck(device); 493 if (!result) { 494 failures.add(checker.getClass().getCanonicalName()); 495 CLog.w("System status checker [%s] failed with message: %s", 496 checker.getClass().getCanonicalName(), checker.getFailureMessage()); 497 } 498 } 499 if (!failures.isEmpty()) { 500 CLog.w("There are failed system status checkers: %s capturing a bugreport", 501 failures.toString()); 502 InputStreamSource bugSource = device.getBugreport(); 503 logger.testLog(String.format("bugreport-checker-pre-module-%s", moduleName), 504 LogDataType.TEXT, bugSource); 505 bugSource.cancel(); 506 } 507 } 508 runPostModuleCheck(String moduleName, List<SystemStatusChecker> checkers, ITestDevice device, ITestLogger logger)509 private void runPostModuleCheck(String moduleName, List<SystemStatusChecker> checkers, 510 ITestDevice device, ITestLogger logger) throws DeviceNotAvailableException { 511 CLog.i("Running system status checker after module execution: %s", moduleName); 512 List<String> failures = new ArrayList<>(); 513 for (SystemStatusChecker checker : checkers) { 514 boolean result = checker.postExecutionCheck(device); 515 if (!result) { 516 failures.add(checker.getClass().getCanonicalName()); 517 CLog.w("System status checker [%s] failed with message: %s", 518 checker.getClass().getCanonicalName(), checker.getFailureMessage()); 519 } 520 } 521 if (!failures.isEmpty()) { 522 CLog.w("There are failed system status checkers: %s capturing a bugreport", 523 failures.toString()); 524 InputStreamSource bugSource = device.getBugreport(); 525 logger.testLog(String.format("bugreport-checker-post-module-%s", moduleName), 526 LogDataType.TEXT, bugSource); 527 bugSource.cancel(); 528 } 529 } 530 531 /** 532 * Sets the include/exclude filters up based on if a module name was given or whether this is a 533 * retry run. 534 */ setupFilters()535 void setupFilters() throws DeviceNotAvailableException { 536 if (mRetrySessionId != null) { 537 // We're retrying so clear -m and -t options 538 // eventually reset these options with values given in the previous session 539 mModuleName = null; 540 mTestName = null; 541 // Load the invocation result 542 IInvocationResult result = null; 543 try { 544 result = ResultHandler.findResult(mBuildHelper.getResultsDir(), mRetrySessionId); 545 } catch (FileNotFoundException e) { 546 throw new RuntimeException(e); 547 } 548 if (result == null) { 549 throw new IllegalArgumentException(String.format( 550 "Could not find session with id %d", mRetrySessionId)); 551 } 552 553 String oldBuildFingerprint = result.getBuildFingerprint(); 554 String currentBuildFingerprint = mDevice.getProperty("ro.build.fingerprint"); 555 if (oldBuildFingerprint.equals(currentBuildFingerprint)) { 556 CLog.logAndDisplay(LogLevel.INFO, "Retrying session from: %s", 557 CompatibilityBuildHelper.getDirSuffix(result.getStartTime())); 558 } else { 559 throw new IllegalArgumentException(String.format( 560 "Device build fingerprint must match %s to retry session %d", 561 oldBuildFingerprint, mRetrySessionId)); 562 } 563 564 String retryCommandLineArgs = result.getCommandLineArgs(); 565 if (retryCommandLineArgs != null) { 566 // Copy the original command into the build helper so it can be serialized later 567 mBuildHelper.setRetryCommandLineArgs(retryCommandLineArgs); 568 try { 569 // parse the command-line string from the result file and set options 570 ArgsOptionParser parser = new ArgsOptionParser(this); 571 parser.parse(OptionHelper.getValidCliArgs(retryCommandLineArgs, this)); 572 } catch (ConfigurationException e) { 573 throw new RuntimeException(e); 574 } 575 } 576 // Append each test that failed or was not executed to the filters 577 for (IModuleResult module : result.getModules()) { 578 if (module.isPassed()) { 579 // Whole module passed, exclude entire module 580 TestFilter filter = new TestFilter(module.getAbi(), module.getName(), null); 581 mExcludeFilters.add(filter.toString()); 582 } else { 583 for (ICaseResult testResultList : module.getResults()) { 584 for (ITestResult testResult : testResultList.getResults(TestStatus.PASS)) { 585 // Test passed, exclude it for retry 586 TestFilter filter = new TestFilter( 587 module.getAbi(), module.getName(), testResult.getFullName()); 588 mExcludeFilters.add(filter.toString()); 589 } 590 } 591 } 592 } 593 } 594 if (mModuleName != null) { 595 mIncludeFilters.clear(); 596 try { 597 List<String> modules = ModuleRepo.getModuleNamesMatching( 598 mBuildHelper.getTestsDir(), mModuleName); 599 if (modules.size() == 0) { 600 throw new IllegalArgumentException( 601 String.format("No modules found matching %s", mModuleName)); 602 } else if (modules.size() > 1) { 603 throw new IllegalArgumentException(String.format( 604 "Multiple modules found matching %s:\n%s\nWhich one did you mean?\n", 605 mModuleName, ArrayUtil.join("\n", modules))); 606 } else { 607 String module = modules.get(0); 608 mIncludeFilters.add(new TestFilter(mAbiName, module, mTestName).toString()); 609 // We will run this module with previous exclusions, 610 // unless they exclude the whole module. 611 List<String> excludeFilters = new ArrayList<>(); 612 for (String excludeFilter : mExcludeFilters) { 613 TestFilter filter = TestFilter.createFrom(excludeFilter); 614 String name = filter.getName(); 615 // Add the filter if it applies to this module 616 if (module.equals(name)) { 617 excludeFilters.add(excludeFilter); 618 } 619 } 620 mExcludeFilters = excludeFilters; 621 } 622 } catch (FileNotFoundException e) { 623 e.printStackTrace(); 624 } 625 } else if (mTestName != null) { 626 throw new IllegalArgumentException( 627 "Test name given without module name. Add --module <module-name>"); 628 } else { 629 // If a module has an arg, assume it's included 630 for (String arg : mModuleArgs) { 631 mIncludeFilters.add(arg.split(":")[0]); 632 } 633 } 634 } 635 636 /** 637 * {@inheritDoc} 638 */ 639 @Override split()640 public Collection<IRemoteTest> split() { 641 if (mShards <= 1) { 642 return null; 643 } 644 645 List<IRemoteTest> shardQueue = new LinkedList<>(); 646 for (int i = 0; i < mShards; i++) { 647 CompatibilityTest test = new CompatibilityTest(mShards, mModuleRepo); 648 OptionCopier.copyOptionsNoThrow(this, test); 649 // Set the shard count because the copy option on the previous line 650 // copies over the mShard value 651 test.mShards = 0; 652 shardQueue.add(test); 653 } 654 655 return shardQueue; 656 } 657 658 } 659