1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype.suite; 17 18 import com.android.ddmlib.Log.LogLevel; 19 import com.android.tradefed.build.IBuildInfo; 20 import com.android.tradefed.config.ConfigurationDescriptor; 21 import com.android.tradefed.config.IConfiguration; 22 import com.android.tradefed.device.DeviceNotAvailableException; 23 import com.android.tradefed.device.ITestDevice; 24 import com.android.tradefed.device.ITestDevice.RecoveryMode; 25 import com.android.tradefed.device.StubDevice; 26 import com.android.tradefed.device.metric.IMetricCollector; 27 import com.android.tradefed.device.metric.LogcatOnFailureCollector; 28 import com.android.tradefed.device.metric.ScreenshotOnFailureCollector; 29 import com.android.tradefed.invoker.IInvocationContext; 30 import com.android.tradefed.invoker.InvocationContext; 31 import com.android.tradefed.invoker.shard.token.TokenProperty; 32 import com.android.tradefed.log.ILogRegistry.EventType; 33 import com.android.tradefed.log.ITestLogger; 34 import com.android.tradefed.log.LogRegistry; 35 import com.android.tradefed.log.LogUtil.CLog; 36 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 37 import com.android.tradefed.result.ILogSaver; 38 import com.android.tradefed.result.ILogSaverListener; 39 import com.android.tradefed.result.ITestInvocationListener; 40 import com.android.tradefed.result.ITestLoggerReceiver; 41 import com.android.tradefed.result.LogFile; 42 import com.android.tradefed.result.ResultForwarder; 43 import com.android.tradefed.result.TestDescription; 44 import com.android.tradefed.result.TestResult; 45 import com.android.tradefed.result.TestRunResult; 46 import com.android.tradefed.suite.checker.ISystemStatusCheckerReceiver; 47 import com.android.tradefed.targetprep.BuildError; 48 import com.android.tradefed.targetprep.ITargetCleaner; 49 import com.android.tradefed.targetprep.ITargetPreparer; 50 import com.android.tradefed.targetprep.TargetSetupError; 51 import com.android.tradefed.targetprep.multi.IMultiTargetPreparer; 52 import com.android.tradefed.testtype.IBuildReceiver; 53 import com.android.tradefed.testtype.IDeviceTest; 54 import com.android.tradefed.testtype.IInvocationContextReceiver; 55 import com.android.tradefed.testtype.IMultiDeviceTest; 56 import com.android.tradefed.testtype.IRemoteTest; 57 import com.android.tradefed.testtype.IRuntimeHintProvider; 58 import com.android.tradefed.testtype.ITestCollector; 59 import com.android.tradefed.testtype.suite.ITestSuite.RetryStrategy; 60 import com.android.tradefed.testtype.suite.module.BaseModuleController; 61 import com.android.tradefed.testtype.suite.module.IModuleController.RunStrategy; 62 import com.android.tradefed.util.StreamUtil; 63 import com.android.tradefed.util.proto.TfMetricProtoUtil; 64 65 import com.google.api.client.repackaged.com.google.common.base.Joiner; 66 import com.google.common.annotations.VisibleForTesting; 67 68 import java.io.PrintWriter; 69 import java.io.StringWriter; 70 import java.util.ArrayList; 71 import java.util.Collection; 72 import java.util.Collections; 73 import java.util.HashMap; 74 import java.util.HashSet; 75 import java.util.LinkedHashMap; 76 import java.util.List; 77 import java.util.ListIterator; 78 import java.util.Map; 79 import java.util.Map.Entry; 80 import java.util.Set; 81 82 /** 83 * Container for the test run configuration. This class is an helper to prepare and run the tests. 84 */ 85 public class ModuleDefinition implements Comparable<ModuleDefinition>, ITestCollector { 86 87 /** key names used for saving module info into {@link IInvocationContext} */ 88 /** 89 * Module name is the base name associated with the module, usually coming from the Xml TF 90 * config file the module was loaded from. 91 */ 92 public static final String MODULE_NAME = "module-name"; 93 public static final String MODULE_ABI = "module-abi"; 94 /** 95 * Module ID the name that will be used to identify uniquely the module during testRunStart. It 96 * will usually be a combination of MODULE_ABI + MODULE_NAME. 97 */ 98 public static final String MODULE_ID = "module-id"; 99 100 public static final String MODULE_CONTROLLER = "module_controller"; 101 102 public static final String PREPARATION_TIME = "PREP_TIME"; 103 public static final String TEAR_DOWN_TIME = "TEARDOWN_TIME"; 104 public static final String TEST_TIME = "TEST_TIME"; 105 public static final String RETRY_TIME = "MODULE_RETRY_TIME"; 106 public static final String RETRY_SUCCESS_COUNT = "MODULE_RETRY_SUCCESS"; 107 public static final String RETRY_FAIL_COUNT = "MODULE_RETRY_FAILED"; 108 109 private static final String FLAKE_DATE_PREFIX = "FLAKE_DATA:"; 110 111 private final IInvocationContext mModuleInvocationContext; 112 private final IConfiguration mModuleConfiguration; 113 private ILogSaver mLogSaver; 114 115 private final String mId; 116 private Collection<IRemoteTest> mTests = null; 117 private Map<String, List<ITargetPreparer>> mPreparersPerDevice = null; 118 119 private List<IMultiTargetPreparer> mMultiPreparers = new ArrayList<>(); 120 private IBuildInfo mBuild; 121 private ITestDevice mDevice; 122 private Map<ITestDevice, IBuildInfo> mDeviceInfos; 123 private List<IMetricCollector> mRunMetricCollectors = new ArrayList<>(); 124 private boolean mCollectTestsOnly = false; 125 126 private List<TestRunResult> mTestsResults = new ArrayList<>(); 127 private List<ModuleListener> mRunListenersResults = new ArrayList<>(); 128 private int mExpectedTests = 0; 129 private boolean mIsFailedModule = false; 130 131 // Tracking of preparers performance 132 private long mElapsedPreparation = 0l; 133 private long mElapsedTearDown = 0l; 134 135 private long mStartTestTime = 0l; 136 137 // Tracking of retry performance 138 private long mRetryTime = 0L; 139 /** The number of test cases that passed after a failed attempt */ 140 private long mSuccessRetried = 0L; 141 /** The number of test cases that remained failed after all retry attempts */ 142 private long mFailedRetried = 0L; 143 144 private RetryStrategy mRetryStrategy = RetryStrategy.RETRY_TEST_CASE_FAILURE; 145 private boolean mMergeAttempts = true; 146 private boolean mRebootAtLastRetry = false; 147 148 // Token during sharding 149 private Set<TokenProperty> mRequiredTokens = new HashSet<>(); 150 151 /** 152 * Constructor 153 * 154 * @param name unique name of the test configuration. 155 * @param tests list of {@link IRemoteTest} that needs to run. 156 * @param preparersPerDevice list of {@link ITargetPreparer} to be used to setup the device. 157 * @param moduleConfig the {@link IConfiguration} of the underlying module config. 158 */ ModuleDefinition( String name, Collection<IRemoteTest> tests, Map<String, List<ITargetPreparer>> preparersPerDevice, List<IMultiTargetPreparer> multiPreparers, IConfiguration moduleConfig)159 public ModuleDefinition( 160 String name, 161 Collection<IRemoteTest> tests, 162 Map<String, List<ITargetPreparer>> preparersPerDevice, 163 List<IMultiTargetPreparer> multiPreparers, 164 IConfiguration moduleConfig) { 165 mId = name; 166 mTests = tests; 167 mModuleConfiguration = moduleConfig; 168 ConfigurationDescriptor configDescriptor = moduleConfig.getConfigurationDescription(); 169 mModuleInvocationContext = new InvocationContext(); 170 mModuleInvocationContext.setConfigurationDescriptor(configDescriptor.clone()); 171 172 // If available in the suite, add the abi name 173 if (configDescriptor.getAbi() != null) { 174 mModuleInvocationContext.addInvocationAttribute( 175 MODULE_ABI, configDescriptor.getAbi().getName()); 176 } 177 if (configDescriptor.getModuleName() != null) { 178 mModuleInvocationContext.addInvocationAttribute( 179 MODULE_NAME, configDescriptor.getModuleName()); 180 } 181 // If there is no specific abi, module-id should be module-name 182 mModuleInvocationContext.addInvocationAttribute(MODULE_ID, mId); 183 184 mMultiPreparers.addAll(multiPreparers); 185 mPreparersPerDevice = preparersPerDevice; 186 187 // Get the tokens of the module 188 List<String> tokens = configDescriptor.getMetaData(ITestSuite.TOKEN_KEY); 189 if (tokens != null) { 190 for (String token : tokens) { 191 mRequiredTokens.add(TokenProperty.valueOf(token.toUpperCase())); 192 } 193 } 194 } 195 196 /** 197 * Returns the next {@link IRemoteTest} from the list of tests. The list of tests of a module 198 * may be shared with another one in case of sharding. 199 */ poll()200 IRemoteTest poll() { 201 synchronized (mTests) { 202 if (mTests.isEmpty()) { 203 return null; 204 } 205 IRemoteTest test = mTests.iterator().next(); 206 mTests.remove(test); 207 return test; 208 } 209 } 210 211 /** 212 * Add some {@link IRemoteTest} to be executed as part of the module. Used when merging two 213 * modules. 214 */ addTests(List<IRemoteTest> test)215 void addTests(List<IRemoteTest> test) { 216 synchronized (mTests) { 217 mTests.addAll(test); 218 } 219 } 220 221 /** Returns the current number of {@link IRemoteTest} waiting to be executed. */ numTests()222 public int numTests() { 223 synchronized (mTests) { 224 return mTests.size(); 225 } 226 } 227 228 /** 229 * Return True if the Module still has {@link IRemoteTest} to run in its pool. False otherwise. 230 */ hasTests()231 protected boolean hasTests() { 232 synchronized (mTests) { 233 return mTests.isEmpty(); 234 } 235 } 236 237 /** Return the unique module name. */ getId()238 public String getId() { 239 return mId; 240 } 241 242 /** 243 * {@inheritDoc} 244 */ 245 @Override compareTo(ModuleDefinition moduleDef)246 public int compareTo(ModuleDefinition moduleDef) { 247 return getId().compareTo(moduleDef.getId()); 248 } 249 250 /** 251 * Inject the {@link IBuildInfo} to be used during the tests. 252 */ setBuild(IBuildInfo build)253 public void setBuild(IBuildInfo build) { 254 mBuild = build; 255 } 256 257 /** 258 * Inject the {@link ITestDevice} to be used during the tests. 259 */ setDevice(ITestDevice device)260 public void setDevice(ITestDevice device) { 261 mDevice = device; 262 } 263 264 /** 265 * Inject the {@link Map} of {@link ITestDevice} and {@link IBuildInfo} for the configuration. 266 */ setDeviceInfos(Map<ITestDevice, IBuildInfo> deviceInfos)267 public void setDeviceInfos(Map<ITestDevice, IBuildInfo> deviceInfos) { 268 mDeviceInfos = deviceInfos; 269 } 270 271 /** Inject the List of {@link IMetricCollector} to be used by the module. */ setMetricCollectors(List<IMetricCollector> collectors)272 public void setMetricCollectors(List<IMetricCollector> collectors) { 273 if (collectors == null) { 274 return; 275 } 276 mRunMetricCollectors.addAll(collectors); 277 } 278 279 /** Pass the invocation log saver to the module so it can use it if necessary. */ setLogSaver(ILogSaver logSaver)280 public void setLogSaver(ILogSaver logSaver) { 281 mLogSaver = logSaver; 282 } 283 284 /** 285 * Run all the {@link IRemoteTest} contained in the module and use all the preparers before and 286 * after to setup and clean the device. 287 * 288 * @param listener the {@link ITestInvocationListener} where to report results. 289 * @throws DeviceNotAvailableException in case of device going offline. 290 */ run(ITestInvocationListener listener)291 public final void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 292 run(listener, null, null); 293 } 294 295 /** 296 * Run all the {@link IRemoteTest} contained in the module and use all the preparers before and 297 * after to setup and clean the device. 298 * 299 * @param listener the {@link ITestInvocationListener} where to report results. 300 * @param moduleLevelListeners The list of listeners at the module level. 301 * @param failureListener a particular listener to collect logs on testFail. Can be null. 302 * @throws DeviceNotAvailableException in case of device going offline. 303 */ run( ITestInvocationListener listener, List<ITestInvocationListener> moduleLevelListeners, TestFailureListener failureListener)304 public final void run( 305 ITestInvocationListener listener, 306 List<ITestInvocationListener> moduleLevelListeners, 307 TestFailureListener failureListener) 308 throws DeviceNotAvailableException { 309 run(listener, moduleLevelListeners, failureListener, 1); 310 } 311 312 /** 313 * Run all the {@link IRemoteTest} contained in the module and use all the preparers before and 314 * after to setup and clean the device. 315 * 316 * @param listener the {@link ITestInvocationListener} where to report results. 317 * @param moduleLevelListeners The list of listeners at the module level. 318 * @param failureListener a particular listener to collect logs on testFail. Can be null. 319 * @param maxRunLimit the max number of runs for each testcase. 320 * @throws DeviceNotAvailableException in case of device going offline. 321 */ run( ITestInvocationListener listener, List<ITestInvocationListener> moduleLevelListeners, TestFailureListener failureListener, int maxRunLimit)322 public final void run( 323 ITestInvocationListener listener, 324 List<ITestInvocationListener> moduleLevelListeners, 325 TestFailureListener failureListener, 326 int maxRunLimit) 327 throws DeviceNotAvailableException { 328 // Load extra configuration for the module from module_controller 329 // TODO: make module_controller a full TF object 330 boolean skipTestCases = false; 331 RunStrategy rs = applyConfigurationControl(failureListener); 332 if (RunStrategy.FULL_MODULE_BYPASS.equals(rs)) { 333 CLog.d("module_controller applied and module %s should not run.", getId()); 334 return; 335 } else if (RunStrategy.SKIP_MODULE_TESTCASES.equals(rs)) { 336 CLog.d("All tests cases for %s will be marked skipped.", getId()); 337 skipTestCases = true; 338 } 339 340 CLog.d("Running module %s", getId()); 341 // Exception generated during setUp or run of the tests 342 Throwable preparationException = null; 343 DeviceNotAvailableException runException = null; 344 // Setup 345 long prepStartTime = getCurrentTime(); 346 347 for (int i = 0; i < mModuleInvocationContext.getDeviceConfigNames().size(); i++) { 348 String deviceName = mModuleInvocationContext.getDeviceConfigNames().get(i); 349 ITestDevice device = mModuleInvocationContext.getDevice(deviceName); 350 if (i >= mPreparersPerDevice.size()) { 351 CLog.d( 352 "Main configuration has more devices than the module configuration. '%s' " 353 + "will not run any preparation.", 354 deviceName); 355 continue; 356 } 357 List<ITargetPreparer> preparers = mPreparersPerDevice.get(deviceName); 358 if (preparers == null) { 359 CLog.w( 360 "Module configuration devices mismatch the main configuration " 361 + "(Missing device '%s'), resolving preparers by index.", 362 deviceName); 363 String key = new ArrayList<>(mPreparersPerDevice.keySet()).get(i); 364 preparers = mPreparersPerDevice.get(key); 365 } 366 for (ITargetPreparer preparer : preparers) { 367 preparationException = 368 runPreparerSetup( 369 device, 370 mModuleInvocationContext.getBuildInfo(deviceName), 371 preparer, 372 listener); 373 if (preparationException != null) { 374 mIsFailedModule = true; 375 CLog.e("Some preparation step failed. failing the module %s", getId()); 376 break; 377 } 378 } 379 if (preparationException != null) { 380 // If one device errored out, we skip the remaining devices. 381 break; 382 } 383 } 384 385 // Skip multi-preparation if preparation already failed. 386 if (preparationException == null) { 387 for (IMultiTargetPreparer multiPreparer : mMultiPreparers) { 388 preparationException = runMultiPreparerSetup(multiPreparer, listener); 389 if (preparationException != null) { 390 mIsFailedModule = true; 391 CLog.e("Some preparation step failed. failing the module %s", getId()); 392 break; 393 } 394 } 395 } 396 mElapsedPreparation = getCurrentTime() - prepStartTime; 397 // Run the tests 398 try { 399 if (preparationException != null) { 400 List<ITestInvocationListener> allListeners = new ArrayList<>(); 401 allListeners.add(listener); 402 if (moduleLevelListeners != null) { 403 allListeners.addAll(moduleLevelListeners); 404 } 405 // Report the early module failures to the moduleListeners too in order for them 406 // to know about it. 407 ITestInvocationListener forwarder = new ResultForwarder(allListeners); 408 // For reporting purpose we create a failure placeholder with the error stack 409 // similar to InitializationError of JUnit. 410 forwarder.testRunStarted(getId(), 1, 0, System.currentTimeMillis()); 411 StringWriter sw = new StringWriter(); 412 preparationException.printStackTrace(new PrintWriter(sw)); 413 forwarder.testRunFailed(sw.toString()); 414 HashMap<String, Metric> metricsProto = new HashMap<>(); 415 metricsProto.put( 416 TEST_TIME, TfMetricProtoUtil.createSingleValue(0L, "milliseconds")); 417 forwarder.testRunEnded(0, metricsProto); 418 // If it was a not available exception rethrow it to signal the new device state. 419 if (preparationException instanceof DeviceNotAvailableException) { 420 throw (DeviceNotAvailableException) preparationException; 421 } 422 return; 423 } 424 mStartTestTime = getCurrentTime(); 425 while (true) { 426 IRemoteTest test = poll(); 427 if (test == null) { 428 return; 429 } 430 431 if (test instanceof IBuildReceiver) { 432 ((IBuildReceiver) test).setBuild(mBuild); 433 } 434 if (test instanceof IDeviceTest) { 435 ((IDeviceTest) test).setDevice(mDevice); 436 } 437 if (test instanceof IMultiDeviceTest) { 438 ((IMultiDeviceTest) test).setDeviceInfos(mDeviceInfos); 439 } 440 if (test instanceof IInvocationContextReceiver) { 441 ((IInvocationContextReceiver) test) 442 .setInvocationContext(mModuleInvocationContext); 443 } 444 if (test instanceof ISystemStatusCheckerReceiver) { 445 // We do not pass down Status checker because they are already running at the 446 // top level suite. 447 ((ISystemStatusCheckerReceiver) test).setSystemStatusChecker(new ArrayList<>()); 448 } 449 if (test instanceof ITestCollector) { 450 if (skipTestCases) { 451 mCollectTestsOnly = true; 452 } 453 ((ITestCollector) test).setCollectTestsOnly(mCollectTestsOnly); 454 } 455 456 GranularRetriableTestWrapper retriableTest = 457 prepareGranularRetriableWrapper( 458 test, 459 listener, 460 failureListener, 461 moduleLevelListeners, 462 skipTestCases, 463 maxRunLimit); 464 retriableTest.setCollectTestsOnly(mCollectTestsOnly); 465 try { 466 retriableTest.run(listener); 467 } catch (DeviceNotAvailableException dnae) { 468 runException = dnae; 469 // We do special logging of some information in Context of the module for easier 470 // debugging. 471 CLog.e( 472 "Module %s threw a DeviceNotAvailableException on device %s during " 473 + "test %s", 474 getId(), mDevice.getSerialNumber(), test.getClass()); 475 CLog.e(dnae); 476 // log an events 477 logDeviceEvent( 478 EventType.MODULE_DEVICE_NOT_AVAILABLE, 479 mDevice.getSerialNumber(), 480 dnae, 481 getId()); 482 throw dnae; 483 } finally { 484 if (mMergeAttempts) { 485 // A single module can generate several test runs 486 mTestsResults.addAll(retriableTest.getFinalTestRunResults()); 487 } else { 488 // Keep track of each listener for attempts 489 mRunListenersResults.add(retriableTest.getResultListener()); 490 } 491 492 mExpectedTests += retriableTest.getExpectedTestsCount(); 493 // Get information about retry 494 mRetryTime += retriableTest.getRetryTime(); 495 mSuccessRetried += retriableTest.getRetrySuccess(); 496 mFailedRetried += retriableTest.getRetryFailed(); 497 498 addAttemptStatsToBuild(mBuild, retriableTest.getAttemptSuccessStats()); 499 } 500 // After the run, if the test failed (even after retry the final result passed) has 501 // failed, capture a bugreport. 502 if (retriableTest.hasFailed()) { 503 captureBugreport(listener, getId()); 504 } 505 } 506 } finally { 507 long cleanStartTime = getCurrentTime(); 508 RuntimeException tearDownException = null; 509 try { 510 Throwable exception = (runException != null) ? runException : preparationException; 511 // Tear down 512 runTearDown(exception); 513 } catch (DeviceNotAvailableException dnae) { 514 CLog.e( 515 "Module %s failed during tearDown with: %s", 516 getId(), StreamUtil.getStackTrace(dnae)); 517 throw dnae; 518 } catch (RuntimeException e) { 519 CLog.e("Exception while running tearDown:"); 520 CLog.e(e); 521 tearDownException = e; 522 } finally { 523 if (failureListener != null) { 524 failureListener.join(); 525 } 526 mElapsedTearDown = getCurrentTime() - cleanStartTime; 527 // finalize results 528 if (preparationException == null) { 529 if (mMergeAttempts) { 530 reportFinalResults( 531 listener, mExpectedTests, mTestsResults, null, tearDownException); 532 } else { 533 // Push the attempts one by one 534 for (int i = 0; i < maxRunLimit; i++) { 535 // Get all the results for the attempt 536 List<TestRunResult> runResultList = new ArrayList<TestRunResult>(); 537 int expectedCount = 0; 538 for (ModuleListener attemptListener : mRunListenersResults) { 539 for (String runName : attemptListener.getTestRunNames()) { 540 TestRunResult run = 541 attemptListener.getTestRunAtAttempt(runName, i); 542 if (run != null) { 543 runResultList.add(run); 544 expectedCount += run.getExpectedTestCount(); 545 } 546 } 547 } 548 549 if (!runResultList.isEmpty()) { 550 reportFinalResults( 551 listener, 552 expectedCount, 553 runResultList, 554 i, 555 tearDownException); 556 } else { 557 CLog.d("No results to be forwarded for attempt %s.", i); 558 } 559 } 560 } 561 } 562 } 563 } 564 } 565 566 /** 567 * Create a wrapper class for the {@link IRemoteTest} which has built-in logic to schedule 568 * multiple test runs for the same module, and have the ability to run testcases at a more 569 * granular level (a subset of testcases in the module). 570 * 571 * @param test the {@link IRemoteTest} that is being wrapped. 572 * @param failureListener a particular listener to collect logs on testFail. Can be null. 573 * @param skipTestCases A run strategy when SKIP_MODULE_TESTCASES is defined. 574 * @param maxRunLimit a rate-limiter on testcases retrying times. 575 */ 576 @VisibleForTesting prepareGranularRetriableWrapper( IRemoteTest test, ITestInvocationListener listener, TestFailureListener failureListener, List<ITestInvocationListener> moduleLevelListeners, boolean skipTestCases, int maxRunLimit)577 GranularRetriableTestWrapper prepareGranularRetriableWrapper( 578 IRemoteTest test, 579 ITestInvocationListener listener, 580 TestFailureListener failureListener, 581 List<ITestInvocationListener> moduleLevelListeners, 582 boolean skipTestCases, 583 int maxRunLimit) { 584 GranularRetriableTestWrapper retriableTest = 585 new GranularRetriableTestWrapper( 586 test, listener, failureListener, moduleLevelListeners, maxRunLimit); 587 retriableTest.setModuleId(getId()); 588 retriableTest.setMarkTestsSkipped(skipTestCases); 589 retriableTest.setMetricCollectors(mRunMetricCollectors); 590 retriableTest.setModuleConfig(mModuleConfiguration); 591 retriableTest.setInvocationContext(mModuleInvocationContext); 592 retriableTest.setLogSaver(mLogSaver); 593 retriableTest.setRetryStrategy(mRetryStrategy); 594 retriableTest.setRebootAtLastRetry(mRebootAtLastRetry); 595 return retriableTest; 596 } 597 captureBugreport(ITestLogger listener, String moduleId)598 private void captureBugreport(ITestLogger listener, String moduleId) { 599 for (ITestDevice device : mModuleInvocationContext.getDevices()) { 600 if (device.getIDevice() instanceof StubDevice) { 601 continue; 602 } 603 device.logBugreport( 604 String.format( 605 "module-%s-failure-%s-bugreport", moduleId, device.getSerialNumber()), 606 listener); 607 } 608 } 609 610 /** Helper to log the device events. */ logDeviceEvent(EventType event, String serial, Throwable t, String moduleId)611 private void logDeviceEvent(EventType event, String serial, Throwable t, String moduleId) { 612 Map<String, String> args = new HashMap<>(); 613 args.put("serial", serial); 614 args.put("trace", StreamUtil.getStackTrace(t)); 615 args.put("module-id", moduleId); 616 LogRegistry.getLogRegistry().logEvent(LogLevel.DEBUG, event, args); 617 } 618 619 /** Finalize results to report them all and count if there are missing tests. */ reportFinalResults( ITestInvocationListener listener, int totalExpectedTests, List<TestRunResult> listResults, Integer attempt, RuntimeException tearDownException)620 private void reportFinalResults( 621 ITestInvocationListener listener, 622 int totalExpectedTests, 623 List<TestRunResult> listResults, 624 Integer attempt, 625 RuntimeException tearDownException) { 626 long elapsedTime = 0l; 627 HashMap<String, Metric> metricsProto = new HashMap<>(); 628 if (attempt != null) { 629 listener.testRunStarted(getId(), totalExpectedTests, attempt, mStartTestTime); 630 } else { 631 listener.testRunStarted(getId(), totalExpectedTests, 0, mStartTestTime); 632 } 633 int numResults = 0; 634 Map<String, LogFile> aggLogFiles = new LinkedHashMap<>(); 635 List<String> runFailureMessages = new ArrayList<>(); 636 for (TestRunResult runResult : listResults) { 637 numResults += runResult.getTestResults().size(); 638 forwardTestResults(runResult.getTestResults(), listener); 639 if (runResult.isRunFailure()) { 640 runFailureMessages.add(runResult.getRunFailureMessage()); 641 mIsFailedModule = true; 642 } 643 elapsedTime += runResult.getElapsedTime(); 644 // put metrics from the tests 645 metricsProto.putAll(runResult.getRunProtoMetrics()); 646 aggLogFiles.putAll(runResult.getRunLoggedFiles()); 647 } 648 // put metrics from the preparation 649 metricsProto.put( 650 PREPARATION_TIME, 651 TfMetricProtoUtil.createSingleValue(mElapsedPreparation, "milliseconds")); 652 metricsProto.put( 653 TEAR_DOWN_TIME, 654 TfMetricProtoUtil.createSingleValue(mElapsedTearDown, "milliseconds")); 655 metricsProto.put( 656 TEST_TIME, TfMetricProtoUtil.createSingleValue(elapsedTime, "milliseconds")); 657 // Report all the retry informations 658 if (mRetryTime > 0L) { 659 metricsProto.put( 660 RETRY_TIME, TfMetricProtoUtil.createSingleValue(mRetryTime, "milliseconds")); 661 metricsProto.put( 662 RETRY_SUCCESS_COUNT, TfMetricProtoUtil.createSingleValue(mSuccessRetried, "")); 663 metricsProto.put( 664 RETRY_FAIL_COUNT, TfMetricProtoUtil.createSingleValue(mFailedRetried, "")); 665 } 666 667 if (totalExpectedTests != numResults) { 668 String error = 669 String.format( 670 "Module %s only ran %d out of %d expected tests.", 671 getId(), numResults, totalExpectedTests); 672 runFailureMessages.add(error); 673 CLog.e(error); 674 mIsFailedModule = true; 675 } 676 677 if (tearDownException != null) { 678 runFailureMessages.add(tearDownException.getMessage()); 679 } 680 // If there is any errors report them all at once 681 if (!runFailureMessages.isEmpty()) { 682 listener.testRunFailed(Joiner.on(TestRunResult.ERROR_DIVIDER).join(runFailureMessages)); 683 } 684 685 // Provide a strong association of the run to its logs. 686 for (Entry<String, LogFile> logFile : aggLogFiles.entrySet()) { 687 if (listener instanceof ILogSaverListener) { 688 ((ILogSaverListener) listener).logAssociation(logFile.getKey(), logFile.getValue()); 689 } 690 } 691 listener.testRunEnded(getCurrentTime() - mStartTestTime, metricsProto); 692 } 693 forwardTestResults( Map<TestDescription, TestResult> testResults, ITestInvocationListener listener)694 private void forwardTestResults( 695 Map<TestDescription, TestResult> testResults, ITestInvocationListener listener) { 696 for (Map.Entry<TestDescription, TestResult> testEntry : testResults.entrySet()) { 697 listener.testStarted(testEntry.getKey(), testEntry.getValue().getStartTime()); 698 switch (testEntry.getValue().getStatus()) { 699 case FAILURE: 700 listener.testFailed(testEntry.getKey(), testEntry.getValue().getStackTrace()); 701 break; 702 case ASSUMPTION_FAILURE: 703 listener.testAssumptionFailure( 704 testEntry.getKey(), testEntry.getValue().getStackTrace()); 705 break; 706 case IGNORED: 707 listener.testIgnored(testEntry.getKey()); 708 break; 709 case INCOMPLETE: 710 listener.testFailed( 711 testEntry.getKey(), "Test did not complete due to exception."); 712 break; 713 default: 714 break; 715 } 716 // Provide a strong association of the test to its logs. 717 for (Entry<String, LogFile> logFile : 718 testEntry.getValue().getLoggedFiles().entrySet()) { 719 if (listener instanceof ILogSaverListener) { 720 ((ILogSaverListener) listener) 721 .logAssociation(logFile.getKey(), logFile.getValue()); 722 } 723 } 724 listener.testEnded( 725 testEntry.getKey(), 726 testEntry.getValue().getEndTime(), 727 testEntry.getValue().getProtoMetrics()); 728 } 729 } 730 731 /** Run all the prepare steps. */ runPreparerSetup( ITestDevice device, IBuildInfo build, ITargetPreparer preparer, ITestLogger logger)732 private Throwable runPreparerSetup( 733 ITestDevice device, IBuildInfo build, ITargetPreparer preparer, ITestLogger logger) { 734 if (preparer.isDisabled()) { 735 // If disabled skip completely. 736 return null; 737 } 738 CLog.d("Running setup preparer: %s", preparer.getClass().getSimpleName()); 739 try { 740 // set the logger in case they need it. 741 if (preparer instanceof ITestLoggerReceiver) { 742 ((ITestLoggerReceiver) preparer).setTestLogger(logger); 743 } 744 if (preparer instanceof IInvocationContextReceiver) { 745 ((IInvocationContextReceiver) preparer) 746 .setInvocationContext(mModuleInvocationContext); 747 } 748 preparer.setUp(device, build); 749 return null; 750 } catch (BuildError 751 | TargetSetupError 752 | DeviceNotAvailableException 753 | RuntimeException 754 | AssertionError e) { 755 // We catch all the TargetPreparer possible exception + RuntimeException to avoid 756 // specific issues + AssertionError since it's widely used in tests and doesn't notify 757 // something very wrong with the harness. 758 CLog.e("Unexpected Exception from preparer: %s", preparer.getClass().getName()); 759 CLog.e(e); 760 return e; 761 } 762 } 763 764 /** Run all multi target preparer step. */ runMultiPreparerSetup(IMultiTargetPreparer preparer, ITestLogger logger)765 private Throwable runMultiPreparerSetup(IMultiTargetPreparer preparer, ITestLogger logger) { 766 if (preparer.isDisabled()) { 767 // If disabled skip completely. 768 return null; 769 } 770 CLog.d("Running setup multi preparer: %s", preparer.getClass().getSimpleName()); 771 try { 772 // set the logger in case they need it. 773 if (preparer instanceof ITestLoggerReceiver) { 774 ((ITestLoggerReceiver) preparer).setTestLogger(logger); 775 } 776 if (preparer instanceof IInvocationContextReceiver) { 777 ((IInvocationContextReceiver) preparer) 778 .setInvocationContext(mModuleInvocationContext); 779 } 780 preparer.setUp(mModuleInvocationContext); 781 return null; 782 } catch (BuildError 783 | TargetSetupError 784 | DeviceNotAvailableException 785 | RuntimeException 786 | AssertionError e) { 787 // We catch all the MultiTargetPreparer possible exception + RuntimeException to avoid 788 // specific issues + AssertionError since it's widely used in tests and doesn't notify 789 // something very wrong with the harness. 790 CLog.e("Unexpected Exception from preparer: %s", preparer.getClass().getName()); 791 CLog.e(e); 792 return e; 793 } 794 } 795 796 /** Run all the tear down steps from preparers. */ runTearDown(Throwable exception)797 private void runTearDown(Throwable exception) throws DeviceNotAvailableException { 798 // Tear down 799 List<IMultiTargetPreparer> cleanerList = new ArrayList<>(mMultiPreparers); 800 Collections.reverse(cleanerList); 801 for (IMultiTargetPreparer multiCleaner : cleanerList) { 802 if (multiCleaner.isDisabled() || multiCleaner.isTearDownDisabled()) { 803 // If disabled skip completely. 804 continue; 805 } 806 CLog.d("Running teardown multi cleaner: %s", multiCleaner.getClass().getSimpleName()); 807 multiCleaner.tearDown(mModuleInvocationContext, exception); 808 } 809 810 for (int i = 0; i < mModuleInvocationContext.getDeviceConfigNames().size(); i++) { 811 String deviceName = mModuleInvocationContext.getDeviceConfigNames().get(i); 812 ITestDevice device = mModuleInvocationContext.getDevice(deviceName); 813 if (i >= mPreparersPerDevice.size()) { 814 CLog.d( 815 "Main configuration has more devices than the module configuration. '%s' " 816 + "will not run any tear down.", 817 deviceName); 818 continue; 819 } 820 List<ITargetPreparer> preparers = mPreparersPerDevice.get(deviceName); 821 if (preparers == null) { 822 CLog.w( 823 "Module configuration devices mismatch the main configuration " 824 + "(Missing device '%s'), resolving preparers by index.", 825 deviceName); 826 String key = new ArrayList<>(mPreparersPerDevice.keySet()).get(i); 827 preparers = mPreparersPerDevice.get(key); 828 } 829 ListIterator<ITargetPreparer> itr = preparers.listIterator(preparers.size()); 830 while (itr.hasPrevious()) { 831 ITargetPreparer preparer = itr.previous(); 832 if (preparer instanceof ITargetCleaner) { 833 ITargetCleaner cleaner = (ITargetCleaner) preparer; 834 // do not call the cleaner if it was disabled 835 if (cleaner.isDisabled() || cleaner.isTearDownDisabled()) { 836 CLog.d("%s has been disabled. skipping.", cleaner); 837 continue; 838 } 839 840 RecoveryMode origMode = null; 841 try { 842 // If an exception was generated in setup with a DNAE do not attempt any 843 // recovery again in case we hit the device not available again. 844 if (exception != null && exception instanceof DeviceNotAvailableException) { 845 origMode = device.getRecoveryMode(); 846 device.setRecoveryMode(RecoveryMode.NONE); 847 } 848 cleaner.tearDown( 849 device, 850 mModuleInvocationContext.getBuildInfo(deviceName), 851 exception); 852 } finally { 853 if (origMode != null) { 854 device.setRecoveryMode(origMode); 855 } 856 } 857 } 858 } 859 } 860 } 861 862 /** Returns the current time. */ getCurrentTime()863 private long getCurrentTime() { 864 return System.currentTimeMillis(); 865 } 866 867 @Override setCollectTestsOnly(boolean collectTestsOnly)868 public void setCollectTestsOnly(boolean collectTestsOnly) { 869 mCollectTestsOnly = collectTestsOnly; 870 } 871 872 /** Sets the {@link RetryStrategy} to be used when retrying. */ setRetryStrategy(RetryStrategy retryStrategy, boolean mergeAttempts)873 public final void setRetryStrategy(RetryStrategy retryStrategy, boolean mergeAttempts) { 874 mRetryStrategy = retryStrategy; 875 mMergeAttempts = mergeAttempts; 876 } 877 878 /** Sets the flag to reboot devices at the last intra-module retry. */ setRebootAtLastRetry(boolean rebootAtLastRetry)879 public final void setRebootAtLastRetry(boolean rebootAtLastRetry) { 880 mRebootAtLastRetry = rebootAtLastRetry; 881 } 882 883 /** Returns a list of tests that ran in this module. */ getTestsResults()884 List<TestRunResult> getTestsResults() { 885 return mTestsResults; 886 } 887 888 /** Returns the number of tests that was expected to be run */ getNumExpectedTests()889 int getNumExpectedTests() { 890 return mExpectedTests; 891 } 892 893 /** Returns True if a testRunFailure has been called on the module * */ hasModuleFailed()894 public boolean hasModuleFailed() { 895 return mIsFailedModule; 896 } 897 getRequiredTokens()898 public Set<TokenProperty> getRequiredTokens() { 899 return mRequiredTokens; 900 } 901 902 /** {@inheritDoc} */ 903 @Override toString()904 public String toString() { 905 return getId(); 906 } 907 908 /** Returns the approximate time to run all the tests in the module. */ getRuntimeHint()909 public long getRuntimeHint() { 910 long hint = 0l; 911 for (IRemoteTest test : mTests) { 912 if (test instanceof IRuntimeHintProvider) { 913 hint += ((IRuntimeHintProvider) test).getRuntimeHint(); 914 } else { 915 hint += 60000; 916 } 917 } 918 return hint; 919 } 920 921 /** Returns the list of {@link IRemoteTest} defined for this module. */ 922 @VisibleForTesting getTests()923 List<IRemoteTest> getTests() { 924 return new ArrayList<>(mTests); 925 } 926 927 /** Returns the list of {@link ITargetPreparer} associated with the given device name */ 928 @VisibleForTesting getTargetPreparerForDevice(String deviceName)929 List<ITargetPreparer> getTargetPreparerForDevice(String deviceName) { 930 return mPreparersPerDevice.get(deviceName); 931 } 932 933 /** Returns the {@link IInvocationContext} associated with the module. */ getModuleInvocationContext()934 public IInvocationContext getModuleInvocationContext() { 935 return mModuleInvocationContext; 936 } 937 938 /** Report completely not executed modules. */ reportNotExecuted(ITestInvocationListener listener, String message)939 public final void reportNotExecuted(ITestInvocationListener listener, String message) { 940 listener.testModuleStarted(getModuleInvocationContext()); 941 listener.testRunStarted(getId(), 0, 0, System.currentTimeMillis()); 942 listener.testRunFailed(message); 943 listener.testRunEnded(0, new HashMap<String, Metric>()); 944 listener.testModuleEnded(); 945 } 946 947 /** 948 * Allow to load a module_controller object to tune how should a particular module run. 949 * 950 * @param failureListener The {@link TestFailureListener} taking actions on tests failures. 951 * @return The strategy to use to run the tests. 952 */ applyConfigurationControl(TestFailureListener failureListener)953 private RunStrategy applyConfigurationControl(TestFailureListener failureListener) { 954 Object ctrlObject = mModuleConfiguration.getConfigurationObject(MODULE_CONTROLLER); 955 if (ctrlObject != null && ctrlObject instanceof BaseModuleController) { 956 BaseModuleController controller = (BaseModuleController) ctrlObject; 957 // module_controller can also control the log collection for the one module 958 if (failureListener != null) { 959 failureListener.applyModuleConfiguration(controller.shouldCaptureBugreport()); 960 } 961 if (!controller.shouldCaptureLogcat()) { 962 mRunMetricCollectors.removeIf(c -> (c instanceof LogcatOnFailureCollector)); 963 } 964 if (!controller.shouldCaptureScreenshot()) { 965 mRunMetricCollectors.removeIf(c -> (c instanceof ScreenshotOnFailureCollector)); 966 } 967 return controller.shouldRunModule(mModuleInvocationContext); 968 } 969 return RunStrategy.RUN; 970 } 971 addAttemptStatsToBuild(IBuildInfo build, Map<String, Integer> attemptStats)972 private void addAttemptStatsToBuild(IBuildInfo build, Map<String, Integer> attemptStats) { 973 for (Entry<String, Integer> entry : attemptStats.entrySet()) { 974 String key = String.format("%s%s:%s", FLAKE_DATE_PREFIX, getId(), entry.getKey()); 975 build.addBuildAttribute(key, Integer.toString(entry.getValue())); 976 } 977 } 978 } 979