1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.testtype.suite; 18 19 import com.android.tradefed.config.IConfiguration; 20 import com.android.tradefed.device.DeviceNotAvailableException; 21 import com.android.tradefed.device.DeviceUnresponsiveException; 22 import com.android.tradefed.device.ITestDevice; 23 import com.android.tradefed.device.StubDevice; 24 import com.android.tradefed.device.metric.CollectorHelper; 25 import com.android.tradefed.device.metric.IMetricCollector; 26 import com.android.tradefed.device.metric.IMetricCollectorReceiver; 27 import com.android.tradefed.invoker.IInvocationContext; 28 import com.android.tradefed.log.LogUtil.CLog; 29 import com.android.tradefed.result.ILogSaver; 30 import com.android.tradefed.result.ITestInvocationListener; 31 import com.android.tradefed.result.LogSaverResultForwarder; 32 import com.android.tradefed.result.MergeStrategy; 33 import com.android.tradefed.result.TestDescription; 34 import com.android.tradefed.result.TestRunResult; 35 import com.android.tradefed.testtype.IRemoteTest; 36 import com.android.tradefed.testtype.ITestCollector; 37 import com.android.tradefed.testtype.ITestFilterReceiver; 38 import com.android.tradefed.testtype.suite.ITestSuite.RetryStrategy; 39 40 import com.google.common.annotations.VisibleForTesting; 41 import com.google.common.collect.Sets; 42 43 import java.util.ArrayList; 44 import java.util.HashMap; 45 import java.util.HashSet; 46 import java.util.LinkedHashMap; 47 import java.util.LinkedHashSet; 48 import java.util.List; 49 import java.util.Map; 50 import java.util.Set; 51 52 /** 53 * A wrapper class works on the {@link IRemoteTest} to granulate the IRemoteTest in testcase level. 54 * An IRemoteTest can contain multiple testcases. Previously, these testcases are treated as a 55 * whole: When IRemoteTest runs, all testcases will run. Some IRemoteTest (The ones that implements 56 * ITestFilterReceiver) can accept a whitelist of testcases and only run those testcases. This class 57 * takes advantage of the existing feature and provides a more flexible way to run test suite. 58 * 59 * <ul> 60 * <li> Single testcase can be retried multiple times (within the same IRemoteTest run) to reduce 61 * the non-test-error failure rates. 62 * <li> The retried testcases are dynamically collected from previous run failures. 63 * </ul> 64 * 65 * <p>Note: 66 * 67 * <ul> 68 * <li> The prerequisite to run a subset of test cases is that the test type should implement the 69 * interface {@link ITestFilterReceiver}. 70 * <li> X is customized max retry number. 71 * </ul> 72 */ 73 public class GranularRetriableTestWrapper implements IRemoteTest, ITestCollector { 74 75 private IRemoteTest mTest; 76 private List<IMetricCollector> mRunMetricCollectors; 77 private TestFailureListener mFailureListener; 78 private IInvocationContext mModuleInvocationContext; 79 private IConfiguration mModuleConfiguration; 80 private ModuleListener mMainGranularRunListener; 81 private RetryLogSaverResultForwarder mRetryAttemptForwarder; 82 private List<ITestInvocationListener> mModuleLevelListeners; 83 private ILogSaver mLogSaver; 84 private String mModuleId; 85 private int mMaxRunLimit; 86 87 private boolean mCollectTestsOnly = false; 88 89 // Tracking of the metrics 90 /** How much time are we spending doing the retry attempts */ 91 private long mRetryTime = 0L; 92 /** The number of test cases that passed after a failed attempt */ 93 private long mSuccessRetried = 0L; 94 /** The number of test cases that remained failed after all retry attempts */ 95 private long mFailedRetried = 0L; 96 /** Store the test that successfully re-run and at which attempt they passed */ 97 private Map<String, Integer> mAttemptSuccess = new HashMap<>(); 98 99 private RetryStrategy mRetryStrategy = RetryStrategy.RETRY_TEST_CASE_FAILURE; 100 private boolean mRebootAtLastRetry = false; 101 GranularRetriableTestWrapper( IRemoteTest test, ITestInvocationListener mainListener, TestFailureListener failureListener, List<ITestInvocationListener> moduleLevelListeners, int maxRunLimit)102 public GranularRetriableTestWrapper( 103 IRemoteTest test, 104 ITestInvocationListener mainListener, 105 TestFailureListener failureListener, 106 List<ITestInvocationListener> moduleLevelListeners, 107 int maxRunLimit) { 108 mTest = test; 109 mMainGranularRunListener = new ModuleListener(mainListener); 110 mFailureListener = failureListener; 111 mModuleLevelListeners = moduleLevelListeners; 112 mMaxRunLimit = maxRunLimit; 113 } 114 115 /** 116 * Set the {@link ModuleDefinition} name as a {@link GranularRetriableTestWrapper} attribute. 117 * 118 * @param moduleId the name of the moduleDefinition. 119 */ setModuleId(String moduleId)120 public void setModuleId(String moduleId) { 121 mModuleId = moduleId; 122 } 123 124 /** 125 * Set the {@link ModuleDefinition} RunStrategy as a {@link GranularRetriableTestWrapper} 126 * attribute. 127 * 128 * @param skipTestCases whether the testcases should be skipped. 129 */ setMarkTestsSkipped(boolean skipTestCases)130 public void setMarkTestsSkipped(boolean skipTestCases) { 131 mMainGranularRunListener.setMarkTestsSkipped(skipTestCases); 132 } 133 134 /** 135 * Set the {@link ModuleDefinition}'s runMetricCollector as a {@link 136 * GranularRetriableTestWrapper} attribute. 137 * 138 * @param runMetricCollectors A list of MetricCollector for the module. 139 */ setMetricCollectors(List<IMetricCollector> runMetricCollectors)140 public void setMetricCollectors(List<IMetricCollector> runMetricCollectors) { 141 mRunMetricCollectors = runMetricCollectors; 142 } 143 144 /** 145 * Set the {@link ModuleDefinition}'s ModuleConfig as a {@link GranularRetriableTestWrapper} 146 * attribute. 147 * 148 * @param moduleConfiguration Provide the module metrics. 149 */ setModuleConfig(IConfiguration moduleConfiguration)150 public void setModuleConfig(IConfiguration moduleConfiguration) { 151 mModuleConfiguration = moduleConfiguration; 152 } 153 154 /** 155 * Set the {@link IInvocationContext} as a {@link GranularRetriableTestWrapper} attribute. 156 * 157 * @param moduleInvocationContext The wrapper uses the InvocationContext to initialize the 158 * MetricCollector when necessary. 159 */ setInvocationContext(IInvocationContext moduleInvocationContext)160 public void setInvocationContext(IInvocationContext moduleInvocationContext) { 161 mModuleInvocationContext = moduleInvocationContext; 162 } 163 164 /** 165 * Set the Module's {@link ILogSaver} as a {@link GranularRetriableTestWrapper} attribute. 166 * 167 * @param logSaver The listeners for each test run should save the logs. 168 */ setLogSaver(ILogSaver logSaver)169 public void setLogSaver(ILogSaver logSaver) { 170 mLogSaver = logSaver; 171 } 172 173 /** Sets the {@link RetryStrategy} to be used when retrying. */ setRetryStrategy(RetryStrategy retryStrategy)174 public final void setRetryStrategy(RetryStrategy retryStrategy) { 175 mRetryStrategy = retryStrategy; 176 } 177 178 /** Sets the flag to reboot devices at the last intra-module retry. */ setRebootAtLastRetry(boolean rebootAtLastRetry)179 public final void setRebootAtLastRetry(boolean rebootAtLastRetry) { 180 mRebootAtLastRetry = rebootAtLastRetry; 181 } 182 183 /** 184 * Initialize a new {@link ModuleListener} for each test run. 185 * 186 * @return a {@link ITestInvocationListener} listener which contains the new {@link 187 * ModuleListener}, the main {@link ITestInvocationListener} and main {@link 188 * TestFailureListener}, and wrapped by RunMetricsCollector and Module MetricCollector (if 189 * not initialized). 190 */ initializeListeners()191 private ITestInvocationListener initializeListeners() { 192 List<ITestInvocationListener> currentTestListener = new ArrayList<>(); 193 // Add all the module level listeners, including TestFailureListener 194 if (mModuleLevelListeners != null) { 195 currentTestListener.addAll(mModuleLevelListeners); 196 } 197 currentTestListener.add(mMainGranularRunListener); 198 199 mRetryAttemptForwarder = new RetryLogSaverResultForwarder(mLogSaver, currentTestListener); 200 ITestInvocationListener runListener = mRetryAttemptForwarder; 201 if (mFailureListener != null) { 202 mFailureListener.setLogger(mRetryAttemptForwarder); 203 currentTestListener.add(mFailureListener); 204 } 205 206 // The module collectors itself are added: this list will be very limited. 207 for (IMetricCollector collector : mModuleConfiguration.getMetricCollectors()) { 208 if (collector.isDisabled()) { 209 CLog.d("%s has been disabled. Skipping.", collector); 210 } else { 211 runListener = collector.init(mModuleInvocationContext, runListener); 212 } 213 } 214 215 return runListener; 216 } 217 218 /** 219 * Schedule a series of {@link IRemoteTest#run(ITestInvocationListener)}. 220 * 221 * @param listener The ResultForwarder listener which contains a new moduleListener for each 222 * run. 223 */ 224 @Override run(ITestInvocationListener listener)225 public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 226 mMainGranularRunListener.setCollectTestsOnly(mCollectTestsOnly); 227 ITestInvocationListener allListeners = initializeListeners(); 228 // First do the regular run, not retried. 229 intraModuleRun(allListeners); 230 231 if (mMaxRunLimit <= 1) { 232 return; 233 } 234 235 // If the very first attempt failed, then don't proceed. 236 if (RetryStrategy.RERUN_UNTIL_FAILURE.equals(mRetryStrategy)) { 237 Set<TestDescription> lastRun = getFailedTestCases(0); 238 // If we encountered a failure 239 if (!lastRun.isEmpty() || mMainGranularRunListener.hasRunCrashedAtAttempt(0)) { 240 CLog.w("%s failed after the first run. Stopping.", lastRun); 241 return; 242 } 243 } 244 245 // Deal with retried attempted 246 long startTime = System.currentTimeMillis(); 247 Set<TestDescription> previousFailedTests = null; 248 Set<String> originalFilters = new HashSet<>(); 249 250 // TODO(b/77548917): Right now we only support ITestFilterReceiver. We should expect to 251 // support ITestFile*Filter*Receiver in the future. 252 if (mTest instanceof ITestFilterReceiver) { 253 ITestFilterReceiver test = (ITestFilterReceiver) mTest; 254 originalFilters = new LinkedHashSet<>(test.getIncludeFilters()); 255 } else if (!shouldHandleFailure(mRetryStrategy)) { 256 // TODO: improve this for test run failures, since they rerun the full run we should 257 // be able to rerun even non-ITestFilterReceiver 258 CLog.d("RetryStrategy does not involved moving filters proceeding with retry."); 259 } else { 260 CLog.d( 261 "%s does not implement ITestFilterReceiver, thus cannot work with " 262 + "intra-module retry.", 263 mTest); 264 return; 265 } 266 267 try { 268 CLog.d("Starting intra-module retry."); 269 for (int attemptNumber = 1; attemptNumber < mMaxRunLimit; attemptNumber++) { 270 CLog.d("Retry attempt number %s", attemptNumber); 271 // Reset the filters to original. 272 if (mTest instanceof ITestFilterReceiver) { 273 ((ITestFilterReceiver) mTest).clearIncludeFilters(); 274 ((ITestFilterReceiver) mTest).addAllIncludeFilters(originalFilters); 275 } 276 // TODO: sort out the collection of metrics for each strategy 277 if (shouldHandleFailure(mRetryStrategy)) { 278 boolean shouldContinue = false; 279 // In case of test run failure and we should retry test runs 280 if (RetryStrategy.RETRY_TEST_RUN_FAILURE.equals(mRetryStrategy) 281 || RetryStrategy.RETRY_ANY_FAILURE.equals(mRetryStrategy)) { 282 if (mMainGranularRunListener.hasRunCrashedAtAttempt(attemptNumber - 1)) { 283 CLog.d("Retrying the run failure."); 284 shouldContinue = true; 285 } 286 } 287 288 if (RetryStrategy.RETRY_TEST_CASE_FAILURE.equals(mRetryStrategy) 289 || RetryStrategy.RETRY_ANY_FAILURE.equals(mRetryStrategy)) { 290 // In case of test case failure, we retry with filters. 291 previousFailedTests = getFailedTestCases(attemptNumber - 1); 292 if (previousFailedTests.size() > 0 && !shouldContinue) { 293 CLog.d("Retrying the test case failure."); 294 shouldContinue = true; 295 addRetriedTestsToIncludeFilters(mTest, previousFailedTests); 296 } 297 } 298 299 if (!shouldContinue) { 300 CLog.d("No test run or test case failures. No need to retry."); 301 break; 302 } 303 } 304 // Reboot device at the last intra-module retry if reboot-at-last-retry is set. 305 if (mRebootAtLastRetry && (attemptNumber == (mMaxRunLimit-1))) { 306 for (ITestDevice device : mModuleInvocationContext.getDevices()) { 307 if (!(device.getIDevice() instanceof StubDevice)) { 308 CLog.i("Rebooting device: %s at the last intra-module retry.", 309 device.getSerialNumber()); 310 device.reboot(); 311 } 312 } 313 } 314 // Run the tests again 315 intraModuleRun(allListeners); 316 317 Set<TestDescription> lastRun = getFailedTestCases(attemptNumber); 318 if (shouldHandleFailure(mRetryStrategy)) { 319 // Evaluate success from what we just ran 320 if (previousFailedTests != null) { 321 Set<TestDescription> diff = Sets.difference(previousFailedTests, lastRun); 322 mSuccessRetried += diff.size(); 323 final int currentAttempt = attemptNumber; 324 diff.forEach( 325 (desc) -> mAttemptSuccess.put(desc.toString(), currentAttempt)); 326 previousFailedTests = lastRun; 327 } 328 } 329 330 if (RetryStrategy.RERUN_UNTIL_FAILURE.equals(mRetryStrategy)) { 331 // If we encountered a failure do not proceed 332 if (!lastRun.isEmpty() 333 || mMainGranularRunListener.hasRunCrashedAtAttempt(attemptNumber)) { 334 CLog.w("%s failed at iteration %s. Stopping.", lastRun, attemptNumber); 335 break; 336 } 337 } 338 } 339 } finally { 340 if (previousFailedTests != null) { 341 mFailedRetried += previousFailedTests.size(); 342 } 343 // Track how long we spend in retry 344 mRetryTime = System.currentTimeMillis() - startTime; 345 } 346 } 347 348 /** 349 * If the strategy needs to handle some failures return True. If it needs to retry no matter 350 * what like {@link RetryStrategy#ITERATIONS} returns False. 351 */ shouldHandleFailure(RetryStrategy retryStrategy)352 private boolean shouldHandleFailure(RetryStrategy retryStrategy) { 353 return RetryStrategy.RETRY_ANY_FAILURE.equals(retryStrategy) 354 || RetryStrategy.RETRY_TEST_RUN_FAILURE.equals(retryStrategy) 355 || RetryStrategy.RETRY_TEST_CASE_FAILURE.equals(retryStrategy); 356 } 357 358 /** 359 * Collect failed test cases from listener. 360 * 361 * @param attemptNumber the 0-indexed integer indicating which attempt to gather failed cases. 362 */ getFailedTestCases(int attemptNumber)363 private Set<TestDescription> getFailedTestCases(int attemptNumber) { 364 Set<TestDescription> failedTestCases = new HashSet<TestDescription>(); 365 for (String runName : mMainGranularRunListener.getTestRunNames()) { 366 TestRunResult run = 367 mMainGranularRunListener.getTestRunAtAttempt(runName, attemptNumber); 368 if (run != null) { 369 failedTestCases.addAll(run.getFailedTests()); 370 } 371 } 372 return failedTestCases; 373 } 374 375 /** 376 * Update the arguments of {@link IRemoteTest} to only run failed tests. This arguments/logic is 377 * implemented differently for each IRemoteTest testtype in the overridden 378 * ITestFilterReceiver.addIncludeFilter method. 379 * 380 * @param test The {@link IRemoteTest} to evaluate as ITestFilterReceiver. 381 * @param testDescriptions The set of failed testDescriptions to retry. 382 */ addRetriedTestsToIncludeFilters( IRemoteTest test, Set<TestDescription> testDescriptions)383 private void addRetriedTestsToIncludeFilters( 384 IRemoteTest test, Set<TestDescription> testDescriptions) { 385 if (test instanceof ITestFilterReceiver) { 386 for (TestDescription testCase : testDescriptions) { 387 String filter = testCase.toString(); 388 ((ITestFilterReceiver) test).addIncludeFilter(filter); 389 } 390 } 391 } 392 393 /** The workflow for each individual {@link IRemoteTest} run. */ intraModuleRun(ITestInvocationListener runListener)394 private final void intraModuleRun(ITestInvocationListener runListener) 395 throws DeviceNotAvailableException { 396 try { 397 List<IMetricCollector> clonedCollectors = cloneCollectors(mRunMetricCollectors); 398 if (mTest instanceof IMetricCollectorReceiver) { 399 ((IMetricCollectorReceiver) mTest).setMetricCollectors(clonedCollectors); 400 // If test can receive collectors then let it handle how to set them up 401 mTest.run(runListener); 402 } else { 403 // Module only init the collectors here to avoid triggering the collectors when 404 // replaying the cached events at the end. This ensures metrics are capture at 405 // the proper time in the invocation. 406 for (IMetricCollector collector : clonedCollectors) { 407 if (collector.isDisabled()) { 408 CLog.d("%s has been disabled. Skipping.", collector); 409 } else { 410 runListener = collector.init(mModuleInvocationContext, runListener); 411 } 412 } 413 mTest.run(runListener); 414 } 415 } catch (RuntimeException | AssertionError re) { 416 CLog.e("Module '%s' - test '%s' threw exception:", mModuleId, mTest.getClass()); 417 CLog.e(re); 418 CLog.e("Proceeding to the next test."); 419 runListener.testRunFailed(re.getMessage()); 420 } catch (DeviceUnresponsiveException due) { 421 // being able to catch a DeviceUnresponsiveException here implies that recovery was 422 // successful, and test execution should proceed to next module. 423 CLog.w( 424 "Ignored DeviceUnresponsiveException because recovery was " 425 + "successful, proceeding with next module. Stack trace:"); 426 CLog.w(due); 427 CLog.w("Proceeding to the next test."); 428 runListener.testRunFailed(due.getMessage()); 429 } catch (DeviceNotAvailableException dnae) { 430 // TODO: See if it's possible to report IReportNotExecuted 431 runListener.testRunFailed( 432 "Run in progress was not completed due to: " + dnae.getMessage()); 433 // Device Not Available Exception are rethrown. 434 throw dnae; 435 } finally { 436 mRetryAttemptForwarder.incrementAttempt(); 437 } 438 } 439 440 /** Get the merged TestRunResults from each {@link IRemoteTest} run. */ getFinalTestRunResults()441 public final List<TestRunResult> getFinalTestRunResults() { 442 // TODO: Once we are ready to report break-down of results and option will override this. 443 MergeStrategy strategy = MergeStrategy.ONE_TESTCASE_PASS_IS_PASS; 444 switch (mRetryStrategy) { 445 case ITERATIONS: 446 strategy = MergeStrategy.ANY_FAIL_IS_FAIL; 447 break; 448 case RERUN_UNTIL_FAILURE: 449 strategy = MergeStrategy.ANY_FAIL_IS_FAIL; 450 break; 451 case RETRY_ANY_FAILURE: 452 strategy = MergeStrategy.ANY_PASS_IS_PASS; 453 break; 454 case RETRY_TEST_CASE_FAILURE: 455 strategy = MergeStrategy.ONE_TESTCASE_PASS_IS_PASS; 456 break; 457 case RETRY_TEST_RUN_FAILURE: 458 strategy = MergeStrategy.ONE_TESTRUN_PASS_IS_PASS; 459 break; 460 } 461 462 mMainGranularRunListener.setMergeStrategy(strategy); 463 return mMainGranularRunListener.getMergedTestRunResults(); 464 } 465 466 @VisibleForTesting getTestRunResultCollected()467 Map<String, List<TestRunResult>> getTestRunResultCollected() { 468 Map<String, List<TestRunResult>> runResultMap = new LinkedHashMap<>(); 469 for (String runName : mMainGranularRunListener.getTestRunNames()) { 470 runResultMap.put(runName, mMainGranularRunListener.getTestRunAttempts(runName)); 471 } 472 return runResultMap; 473 } 474 475 @VisibleForTesting cloneCollectors(List<IMetricCollector> originalCollectors)476 List<IMetricCollector> cloneCollectors(List<IMetricCollector> originalCollectors) { 477 return CollectorHelper.cloneCollectors(originalCollectors); 478 } 479 480 /** Check if any testRunResult has ever failed. This check is used for bug report only. */ hasFailed()481 public boolean hasFailed() { 482 return mMainGranularRunListener.hasFailed(); 483 } 484 485 /** 486 * Calculate the number of testcases in the {@link IRemoteTest}. This value distincts the same 487 * testcases that are rescheduled multiple times. 488 */ getExpectedTestsCount()489 public final int getExpectedTestsCount() { 490 return mMainGranularRunListener.getExpectedTests(); 491 } 492 493 /** Returns the elapsed time in retry attempts. */ getRetryTime()494 public final long getRetryTime() { 495 return mRetryTime; 496 } 497 498 /** Returns the number of tests we managed to change status from failed to pass. */ getRetrySuccess()499 public final long getRetrySuccess() { 500 return mSuccessRetried; 501 } 502 503 /** Returns the number of tests we couldn't change status from failed to pass. */ getRetryFailed()504 public final long getRetryFailed() { 505 return mFailedRetried; 506 } 507 508 /** Returns the listener containing all the results. */ getResultListener()509 public ModuleListener getResultListener() { 510 return mMainGranularRunListener; 511 } 512 513 /** Returns the attempts that turned into success. */ getAttemptSuccessStats()514 public Map<String, Integer> getAttemptSuccessStats() { 515 return mAttemptSuccess; 516 } 517 518 /** Forwarder that also handles passing the current attempt we are at. */ 519 private class RetryLogSaverResultForwarder extends LogSaverResultForwarder { 520 521 private int mAttemptNumber = 0; 522 RetryLogSaverResultForwarder( ILogSaver logSaver, List<ITestInvocationListener> listeners)523 public RetryLogSaverResultForwarder( 524 ILogSaver logSaver, List<ITestInvocationListener> listeners) { 525 super(logSaver, listeners); 526 } 527 528 @Override testRunStarted(String runName, int testCount)529 public void testRunStarted(String runName, int testCount) { 530 super.testRunStarted(runName, testCount, mAttemptNumber); 531 } 532 533 @Override testRunStarted(String runName, int testCount, int attemptNumber)534 public void testRunStarted(String runName, int testCount, int attemptNumber) { 535 if (attemptNumber != mAttemptNumber) { 536 CLog.w( 537 "Test reported an attempt %s, while the suite is at attempt %s", 538 attemptNumber, mAttemptNumber); 539 } 540 // We enforce our attempt number 541 super.testRunStarted(runName, testCount, mAttemptNumber); 542 } 543 544 /** Increment the attempt number. */ incrementAttempt()545 public void incrementAttempt() { 546 mAttemptNumber++; 547 } 548 } 549 550 @Override setCollectTestsOnly(boolean shouldCollectTest)551 public void setCollectTestsOnly(boolean shouldCollectTest) { 552 mCollectTestsOnly = shouldCollectTest; 553 } 554 } 555