1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype.suite; 17 18 import com.android.tradefed.build.BuildRetrievalError; 19 import com.android.tradefed.build.IBuildInfo; 20 import com.android.tradefed.config.Configuration; 21 import com.android.tradefed.config.ConfigurationDescriptor; 22 import com.android.tradefed.config.ConfigurationException; 23 import com.android.tradefed.config.DeviceConfigurationHolder; 24 import com.android.tradefed.config.DynamicRemoteFileResolver; 25 import com.android.tradefed.config.IConfiguration; 26 import com.android.tradefed.config.IConfigurationReceiver; 27 import com.android.tradefed.config.IDeviceConfiguration; 28 import com.android.tradefed.dependencies.ExternalDependency; 29 import com.android.tradefed.dependencies.IExternalDependency; 30 import com.android.tradefed.device.DeviceNotAvailableException; 31 import com.android.tradefed.device.ITestDevice; 32 import com.android.tradefed.device.ITestDevice.RecoveryMode; 33 import com.android.tradefed.device.StubDevice; 34 import com.android.tradefed.device.connection.AdbTcpConnection; 35 import com.android.tradefed.device.metric.BugreportzOnTestCaseFailureCollector; 36 import com.android.tradefed.device.metric.IMetricCollector; 37 import com.android.tradefed.device.metric.LogcatOnFailureCollector; 38 import com.android.tradefed.device.metric.ScreenshotOnFailureCollector; 39 import com.android.tradefed.error.HarnessRuntimeException; 40 import com.android.tradefed.error.IHarnessException; 41 import com.android.tradefed.invoker.IInvocationContext; 42 import com.android.tradefed.invoker.InvocationContext; 43 import com.android.tradefed.invoker.TestInformation; 44 import com.android.tradefed.invoker.logger.CurrentInvocation; 45 import com.android.tradefed.invoker.logger.InvocationMetricLogger; 46 import com.android.tradefed.invoker.logger.InvocationMetricLogger.InvocationMetricKey; 47 import com.android.tradefed.invoker.logger.TfObjectTracker; 48 import com.android.tradefed.invoker.shard.token.TokenProperty; 49 import com.android.tradefed.invoker.tracing.CloseableTraceScope; 50 import com.android.tradefed.log.ILogRegistry.EventType; 51 import com.android.tradefed.log.ITestLogger; 52 import com.android.tradefed.log.Log.LogLevel; 53 import com.android.tradefed.log.LogRegistry; 54 import com.android.tradefed.log.LogUtil.CLog; 55 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 56 import com.android.tradefed.result.FailureDescription; 57 import com.android.tradefed.result.ILogSaver; 58 import com.android.tradefed.result.ILogSaverListener; 59 import com.android.tradefed.result.ITestInvocationListener; 60 import com.android.tradefed.result.ITestLoggerReceiver; 61 import com.android.tradefed.result.LogFile; 62 import com.android.tradefed.result.LogSaverResultForwarder; 63 import com.android.tradefed.result.MultiFailureDescription; 64 import com.android.tradefed.result.TestDescription; 65 import com.android.tradefed.result.TestResult; 66 import com.android.tradefed.result.TestRunResult; 67 import com.android.tradefed.result.error.DeviceErrorIdentifier; 68 import com.android.tradefed.result.error.ErrorIdentifier; 69 import com.android.tradefed.result.error.InfraErrorIdentifier; 70 import com.android.tradefed.result.error.TestErrorIdentifier; 71 import com.android.tradefed.result.proto.TestRecordProto.FailureStatus; 72 import com.android.tradefed.retry.IRetryDecision; 73 import com.android.tradefed.retry.RetryPreparationDecision; 74 import com.android.tradefed.retry.RetryStatistics; 75 import com.android.tradefed.retry.RetryStrategy; 76 import com.android.tradefed.suite.checker.ISystemStatusCheckerReceiver; 77 import com.android.tradefed.targetprep.BuildError; 78 import com.android.tradefed.targetprep.ITargetPreparer; 79 import com.android.tradefed.targetprep.TargetSetupError; 80 import com.android.tradefed.targetprep.multi.IMultiTargetPreparer; 81 import com.android.tradefed.testtype.IBuildReceiver; 82 import com.android.tradefed.testtype.IDeviceTest; 83 import com.android.tradefed.testtype.IInvocationContextReceiver; 84 import com.android.tradefed.testtype.IRemoteTest; 85 import com.android.tradefed.testtype.IRuntimeHintProvider; 86 import com.android.tradefed.testtype.ITestCollector; 87 import com.android.tradefed.testtype.ITestFileFilterReceiver; 88 import com.android.tradefed.testtype.ITestFilterReceiver; 89 import com.android.tradefed.testtype.suite.module.BaseModuleController; 90 import com.android.tradefed.testtype.suite.module.IModuleController.RunStrategy; 91 import com.android.tradefed.util.FileUtil; 92 import com.android.tradefed.util.MultiMap; 93 import com.android.tradefed.util.StreamUtil; 94 import com.android.tradefed.util.SystemUtil; 95 import com.android.tradefed.util.proto.TfMetricProtoUtil; 96 97 import com.google.api.client.util.Joiner; 98 import com.google.common.annotations.VisibleForTesting; 99 100 import java.io.File; 101 import java.io.IOException; 102 import java.util.ArrayList; 103 import java.util.Arrays; 104 import java.util.Collection; 105 import java.util.Collections; 106 import java.util.HashMap; 107 import java.util.HashSet; 108 import java.util.Iterator; 109 import java.util.LinkedHashSet; 110 import java.util.List; 111 import java.util.ListIterator; 112 import java.util.Map; 113 import java.util.Map.Entry; 114 import java.util.Set; 115 import java.util.stream.Collectors; 116 117 /** 118 * Container for the test run configuration. This class is an helper to prepare and run the tests. 119 */ 120 public class ModuleDefinition implements Comparable<ModuleDefinition>, ITestCollector { 121 122 /** key names used for saving module info into {@link IInvocationContext} */ 123 /** 124 * Module name is the base name associated with the module, usually coming from the Xml TF 125 * config file the module was loaded from. 126 */ 127 public static final String MODULE_NAME = "module-name"; 128 public static final String MODULE_ABI = "module-abi"; 129 public static final String MODULE_PARAMETERIZATION = "module-param"; 130 public static final String MODULE_EXTERNAL_DEPENDENCIES = "module-external-dependencies"; 131 /** 132 * Module ID the name that will be used to identify uniquely the module during testRunStart. It 133 * will usually be a combination of MODULE_ABI + MODULE_NAME. 134 */ 135 public static final String MODULE_ID = "module-id"; 136 /** This property is set to true if the module was running on a freshly prepared device. */ 137 public static final String MODULE_ISOLATED = "module-isolated"; 138 /** This property is set to true if the test module results were cached. */ 139 public static final String MODULE_CACHED = "module-cached"; 140 141 /** This property is set to true if the test module was skipped */ 142 public static final String MODULE_SKIPPED = "module-skipped"; 143 144 /** This property is set to true if only module level events are reported. */ 145 public static final String SPARSE_MODULE = "sparse-module"; 146 147 public static final String MODULE_CONTROLLER = "module_controller"; 148 149 public static final String PREPARATION_TIME = "PREP_TIME"; 150 public static final String TEAR_DOWN_TIME = "TEARDOWN_TIME"; 151 public static final String TEST_TIME = "TEST_TIME"; 152 public static final String MODULE_TEST_COUNT = "MODULE_TEST_COUNT"; 153 public static final String RETRY_TIME = "MODULE_RETRY_TIME"; 154 public static final String ISOLATION_COST = "ISOLATION_COST"; 155 public static final String RETRY_SUCCESS_COUNT = "MODULE_RETRY_SUCCESS"; 156 public static final String RETRY_FAIL_COUNT = "MODULE_RETRY_FAILED"; 157 public static final String MODULE_INVOCATION_ATTRIBUTE_FLAG_OVERRIDES_KEY = 158 "module-flag-overrides"; 159 160 private final IInvocationContext mModuleInvocationContext; 161 private final IConfiguration mModuleConfiguration; 162 private IConfiguration mInternalTestConfiguration; 163 private IConfiguration mInternalTargetPreparerConfiguration; 164 private ILogSaver mLogSaver; 165 private TestInformation mModuleInfo; 166 private ITestInvocationListener mInvocationListener; 167 168 private final String mId; 169 private Collection<IRemoteTest> mTests = null; 170 private Integer mIntraModuleShardCount = null; 171 private Integer mIntraModuleShardIndex = null; 172 173 private Map<String, List<ITargetPreparer>> mPreparersPerDevice = null; 174 private Map<String, List<ITargetPreparer>> mSuitePreparersPerDevice = null; 175 176 private List<IMultiTargetPreparer> mMultiPreparers = new ArrayList<>(); 177 private IBuildInfo mBuild; 178 private ITestDevice mDevice; 179 private List<IMetricCollector> mRunMetricCollectors = new ArrayList<>(); 180 private boolean mCollectTestsOnly = false; 181 182 private List<TestRunResult> mTestsResults = new ArrayList<>(); 183 private List<ModuleListener> mRunListenersResults = new ArrayList<>(); 184 private int mExpectedTests = 0; 185 private boolean mIsFailedModule = false; 186 private boolean mRetriedModulePreparationSuccess = false; 187 188 // Tracking of preparers performance 189 private long mElapsedPreparation = 0L; 190 private long mElapsedTearDown = 0L; 191 192 private long mStartTestTime = 0L; 193 private boolean mReportModuleStart = true; 194 private boolean mReportModuleEnd = true; 195 private boolean mFinalResultsReported = false; 196 197 // Tracking of retry performance 198 private List<RetryStatistics> mRetryStats = new ArrayList<>(); 199 private boolean mDisableAutoRetryTimeReporting = false; 200 201 private boolean mMergeAttempts = true; 202 private IRetryDecision mRetryDecision; 203 204 // Token during sharding 205 private Set<TokenProperty> mRequiredTokens = new HashSet<>(); 206 207 private boolean mEnableDynamicDownload = false; 208 private GranularRetriableTestWrapper mCurrentTestWrapper = null; 209 private int mMaxRetry = 1; 210 // TODO(b/226453043): Eventually we need to remove this. 211 private int mTargetPreparerRetryCount = 0; 212 213 private Set<TestDescription> mPassThroughFilters = new LinkedHashSet<>(); 214 215 private boolean mRecoverVirtualDevice = false; 216 217 private boolean mUseModuleResultsForwarder = false; 218 219 @VisibleForTesting ModuleDefinition()220 public ModuleDefinition() { 221 mModuleInvocationContext = null; 222 mModuleConfiguration = null; 223 mId = ""; 224 } 225 226 /** 227 * Constructor 228 * 229 * @param name unique name of the test configuration. 230 * @param tests list of {@link IRemoteTest} that needs to run. 231 * @param preparersPerDevice list of {@link ITargetPreparer} to be used to setup the device. 232 * @param moduleConfig the {@link IConfiguration} of the underlying module config. 233 */ ModuleDefinition( String name, Collection<IRemoteTest> tests, Map<String, List<ITargetPreparer>> preparersPerDevice, List<IMultiTargetPreparer> multiPreparers, IConfiguration moduleConfig)234 public ModuleDefinition( 235 String name, 236 Collection<IRemoteTest> tests, 237 Map<String, List<ITargetPreparer>> preparersPerDevice, 238 List<IMultiTargetPreparer> multiPreparers, 239 IConfiguration moduleConfig) { 240 this(name, tests, preparersPerDevice, null, multiPreparers, moduleConfig); 241 } 242 243 /** 244 * Constructor 245 * 246 * @param name unique name of the test configuration. 247 * @param tests list of {@link IRemoteTest} that needs to run. 248 * @param preparersPerDevice list of {@link ITargetPreparer} to be used to setup the device. 249 * @param moduleConfig the {@link IConfiguration} of the underlying module config. 250 */ ModuleDefinition( String name, Collection<IRemoteTest> tests, Map<String, List<ITargetPreparer>> preparersPerDevice, Map<String, List<ITargetPreparer>> suitePreparersPerDevice, List<IMultiTargetPreparer> multiPreparers, IConfiguration moduleConfig)251 public ModuleDefinition( 252 String name, 253 Collection<IRemoteTest> tests, 254 Map<String, List<ITargetPreparer>> preparersPerDevice, 255 Map<String, List<ITargetPreparer>> suitePreparersPerDevice, 256 List<IMultiTargetPreparer> multiPreparers, 257 IConfiguration moduleConfig) { 258 mId = name; 259 mTests = tests; 260 mModuleConfiguration = moduleConfig; 261 ConfigurationDescriptor configDescriptor = moduleConfig.getConfigurationDescription(); 262 mModuleInvocationContext = new InvocationContext(); 263 mModuleInvocationContext.setConfigurationDescriptor(configDescriptor.clone()); 264 // Copy the command options invocation attributes to the invocation context 265 mModuleInvocationContext.addInvocationAttributes( 266 moduleConfig.getCommandOptions().getInvocationData()); 267 268 // If available in the suite, add the abi name 269 if (configDescriptor.getAbi() != null) { 270 mModuleInvocationContext.addInvocationAttribute( 271 MODULE_ABI, configDescriptor.getAbi().getName()); 272 } 273 if (configDescriptor.getModuleName() != null) { 274 mModuleInvocationContext.addInvocationAttribute( 275 MODULE_NAME, configDescriptor.getModuleName()); 276 } 277 String parameterization = 278 configDescriptor 279 .getAllMetaData() 280 .getUniqueMap() 281 .get(ConfigurationDescriptor.ACTIVE_PARAMETER_KEY); 282 if (parameterization != null) { 283 mModuleInvocationContext.addInvocationAttribute( 284 MODULE_PARAMETERIZATION, parameterization); 285 } 286 // If there is no specific abi, module-id should be module-name 287 mModuleInvocationContext.addInvocationAttribute(MODULE_ID, mId); 288 289 // Add External Dependencies of this module to the module context 290 Set<ExternalDependency> externalDependencies = new LinkedHashSet<>(); 291 for (IDeviceConfiguration deviceConfig : moduleConfig.getDeviceConfig()) { 292 for (Object obj : deviceConfig.getAllObjects()) { 293 if (obj instanceof IExternalDependency) { 294 externalDependencies.addAll(((IExternalDependency) obj).getDependencies()); 295 } 296 } 297 } 298 if (!externalDependencies.isEmpty()) { 299 final List<String> dependencyClassNames = 300 externalDependencies.stream() 301 .map(dependency -> dependency.getClass().getName()) 302 .collect(Collectors.toList()); 303 mModuleInvocationContext.addInvocationAttribute( 304 MODULE_EXTERNAL_DEPENDENCIES, String.join(", ", dependencyClassNames)); 305 } 306 307 mMultiPreparers.addAll(multiPreparers); 308 mPreparersPerDevice = preparersPerDevice; 309 mSuitePreparersPerDevice = suitePreparersPerDevice; 310 311 // Get the tokens of the module 312 List<String> tokens = configDescriptor.getMetaData(ITestSuite.TOKEN_KEY); 313 if (tokens != null) { 314 for (String token : tokens) { 315 mRequiredTokens.add(TokenProperty.valueOf(token.toUpperCase())); 316 } 317 } 318 } 319 setIntraModuleInformation(int shardCount, int shardIndex)320 public void setIntraModuleInformation(int shardCount, int shardIndex) { 321 mIntraModuleShardCount = shardCount; 322 mIntraModuleShardIndex = shardIndex; 323 } 324 getIntraModuleShardCount()325 public Integer getIntraModuleShardCount() { 326 return mIntraModuleShardCount; 327 } 328 getIntraModuleShardIndex()329 public Integer getIntraModuleShardIndex() { 330 return mIntraModuleShardIndex; 331 } 332 333 /** Returns the number of devices expected to run this test. */ neededDevices()334 public int neededDevices() { 335 return mModuleConfiguration.getDeviceConfig().size(); 336 } 337 338 /** 339 * Returns the next {@link IRemoteTest} from the list of tests. The list of tests of a module 340 * may be shared with another one in case of sharding. 341 */ poll()342 IRemoteTest poll() { 343 synchronized (mTests) { 344 if (mTests.isEmpty()) { 345 return null; 346 } 347 IRemoteTest test = mTests.iterator().next(); 348 mTests.remove(test); 349 return test; 350 } 351 } 352 353 /** 354 * Add some {@link IRemoteTest} to be executed as part of the module. Used when merging two 355 * modules. 356 */ addTests(List<IRemoteTest> test)357 void addTests(List<IRemoteTest> test) { 358 synchronized (mTests) { 359 mTests.addAll(test); 360 } 361 } 362 363 /** Returns the current number of {@link IRemoteTest} waiting to be executed. */ numTests()364 public int numTests() { 365 synchronized (mTests) { 366 return mTests.size(); 367 } 368 } 369 370 /** 371 * Return True if the Module still has {@link IRemoteTest} to run in its pool. False otherwise. 372 */ hasTests()373 protected boolean hasTests() { 374 synchronized (mTests) { 375 return !mTests.isEmpty(); 376 } 377 } 378 379 /** Return the unique module name. */ getId()380 public String getId() { 381 return mId; 382 } 383 384 /** 385 * {@inheritDoc} 386 */ 387 @Override compareTo(ModuleDefinition moduleDef)388 public int compareTo(ModuleDefinition moduleDef) { 389 return getId().compareTo(moduleDef.getId()); 390 } 391 392 /** 393 * Inject the {@link IBuildInfo} to be used during the tests. 394 */ setBuild(IBuildInfo build)395 public void setBuild(IBuildInfo build) { 396 mBuild = build; 397 } 398 399 /** 400 * Inject the {@link ITestDevice} to be used during the tests. 401 */ setDevice(ITestDevice device)402 public void setDevice(ITestDevice device) { 403 mDevice = device; 404 } 405 406 /** Inject the List of {@link IMetricCollector} to be used by the module. */ setMetricCollectors(List<IMetricCollector> collectors)407 public void setMetricCollectors(List<IMetricCollector> collectors) { 408 if (collectors == null) { 409 return; 410 } 411 mRunMetricCollectors.addAll(collectors); 412 } 413 414 /** Pass the invocation log saver to the module so it can use it if necessary. */ setLogSaver(ILogSaver logSaver)415 public void setLogSaver(ILogSaver logSaver) { 416 mLogSaver = logSaver; 417 } 418 419 /** 420 * Run all the {@link IRemoteTest} contained in the module and use all the preparers before and 421 * after to setup and clean the device. 422 * 423 * @param listener the {@link ITestInvocationListener} where to report results. 424 * @throws DeviceNotAvailableException in case of device going offline. 425 */ run(TestInformation moduleInfo, ITestInvocationListener listener)426 public final void run(TestInformation moduleInfo, ITestInvocationListener listener) 427 throws DeviceNotAvailableException { 428 run(moduleInfo, listener, 1); 429 } 430 431 /** 432 * Run all the {@link IRemoteTest} contained in the module and use all the preparers before and 433 * after to setup and clean the device. 434 * 435 * @param moduleInfo the {@link TestInformation} for the module. 436 * @param listener the {@link ITestInvocationListener} where to report results. 437 * @param maxRunLimit the max number of runs for each testcase. 438 * @throws DeviceNotAvailableException in case of device going offline. 439 */ run( TestInformation moduleInfo, ITestInvocationListener listener, int maxRunLimit)440 public final void run( 441 TestInformation moduleInfo, 442 ITestInvocationListener listener, 443 int maxRunLimit) 444 throws DeviceNotAvailableException { 445 mMaxRetry = maxRunLimit; 446 mModuleInfo = moduleInfo; 447 // until RetryLogSaverResultForwarder is created, wrap a temp log saver if needed. 448 if (!(listener instanceof LogSaverResultForwarder)) { 449 mInvocationListener = 450 new LogSaverResultForwarder( 451 mLogSaver, Arrays.asList(listener), mModuleConfiguration, false); 452 } else { 453 mInvocationListener = listener; 454 } 455 456 // Load extra configuration for the module from module_controller 457 // TODO: make module_controller a full TF object 458 boolean skipTestCases = false; 459 RunStrategy rs = applyConfigurationControl(); 460 if (RunStrategy.FULL_MODULE_BYPASS.equals(rs)) { 461 CLog.d("module_controller applied and module %s should not run.", getId()); 462 return; 463 } else if (RunStrategy.SKIP_MODULE_TESTCASES.equals(rs)) { 464 CLog.d("All tests cases for %s will be marked skipped.", getId()); 465 skipTestCases = true; 466 } 467 468 CLog.logAndDisplay(LogLevel.DEBUG, "Running module %s", getId()); 469 // set the module context so it's available widely during the module run period. 470 CurrentInvocation.setModuleContext(mModuleInvocationContext); 471 // Exception generated during setUp or run of the tests 472 Throwable preparationException; 473 DeviceNotAvailableException runException = null; 474 long start = System.currentTimeMillis(); 475 // Resolve dynamic files except for the IRemoteTest ones 476 try (CloseableTraceScope ignored = new CloseableTraceScope("download_files")) { 477 preparationException = 478 invokeRemoteDynamic(moduleInfo.getDevice(), mModuleConfiguration); 479 480 if (preparationException == null) { 481 mInternalTargetPreparerConfiguration = 482 new Configuration("tmp-download", "tmp-download"); 483 mInternalTargetPreparerConfiguration 484 .getCommandOptions() 485 .getDynamicDownloadArgs() 486 .putAll(mModuleConfiguration.getCommandOptions().getDynamicDownloadArgs()); 487 for (String device : mPreparersPerDevice.keySet()) { 488 mInternalTargetPreparerConfiguration.setDeviceConfig( 489 new DeviceConfigurationHolder(device)); 490 for (ITargetPreparer preparer : mPreparersPerDevice.get(device)) { 491 try { 492 mInternalTargetPreparerConfiguration 493 .getDeviceConfigByName(device) 494 .addSpecificConfig(preparer); 495 } catch (ConfigurationException e) { 496 // unset the module context since module run is ending. 497 CurrentInvocation.setModuleContext(null); 498 // Shouldn't happen; 499 throw new RuntimeException(e); 500 } 501 } 502 } 503 mInternalTargetPreparerConfiguration.setMultiTargetPreparers(mMultiPreparers); 504 preparationException = 505 invokeRemoteDynamic( 506 moduleInfo.getDevice(), mInternalTargetPreparerConfiguration); 507 } 508 } 509 // Setup 510 if (preparationException == null) { 511 try (CloseableTraceScope ignored = new CloseableTraceScope("module_preparation")) { 512 preparationException = runPreparation(false); 513 } 514 } 515 516 while (preparationException != null) { 517 RetryPreparationDecision retryDecision = 518 mRetryDecision.shouldRetryPreparation( 519 this, 520 mTargetPreparerRetryCount, 521 maxRunLimit); 522 boolean shouldFailRun = retryDecision.shouldFailRun(); 523 reportSetupFailure( 524 preparationException, 525 mInvocationListener, 526 mTargetPreparerRetryCount, 527 shouldFailRun); 528 if (shouldFailRun) { 529 // unset the module context since module run is ending. 530 CurrentInvocation.setModuleContext(null); 531 return; 532 } 533 mTargetPreparerRetryCount++; 534 if (!retryDecision.shouldRetry()) { 535 CLog.i("Retry target preparers for module: %s successfully at the %s retry", 536 this.getId(), mTargetPreparerRetryCount); 537 // This flag is set true for any target preparation error, set it false when the 538 // retrying succeed. 539 mIsFailedModule = false; 540 mRetriedModulePreparationSuccess = true; 541 InvocationMetricLogger.addInvocationMetrics( 542 InvocationMetricKey.DEVICE_RESET_MODULES_FOR_TARGET_PREPARER, this.getId()); 543 } 544 preparationException = retryDecision.getPreviousException(); 545 } 546 547 // Total count of retried (preparation + test) shouldn't exceed maxRunLimit and mMaxRetry. 548 maxRunLimit = maxRunLimit - mTargetPreparerRetryCount; 549 mMaxRetry = mMaxRetry - mTargetPreparerRetryCount; 550 InvocationMetricLogger.addInvocationPairMetrics( 551 InvocationMetricKey.MODULE_SETUP_PAIR, start, System.currentTimeMillis()); 552 553 // Run the tests 554 try { 555 mStartTestTime = getCurrentTime(); 556 int perModuleRetryQuota = mMaxRetry; 557 while (true) { 558 IRemoteTest test = poll(); 559 if (test == null) { 560 return; 561 } 562 TfObjectTracker.countWithParents(test.getClass()); 563 if (test instanceof IBuildReceiver) { 564 ((IBuildReceiver) test).setBuild(mBuild); 565 } 566 if (test instanceof IDeviceTest) { 567 ((IDeviceTest) test).setDevice(mDevice); 568 } 569 if (test instanceof IInvocationContextReceiver) { 570 ((IInvocationContextReceiver) test) 571 .setInvocationContext(mModuleInvocationContext); 572 } 573 mInternalTestConfiguration = new Configuration("tmp-download", "tmp-download"); 574 mInternalTestConfiguration 575 .getCommandOptions() 576 .getDynamicDownloadArgs() 577 .putAll(mModuleConfiguration.getCommandOptions().getDynamicDownloadArgs()); 578 // We do it before the official set, otherwise the IConfiguration will not be the 579 // right one. 580 mInternalTestConfiguration.setTest(test); 581 if (test instanceof IConfigurationReceiver) { 582 ((IConfigurationReceiver) test).setConfiguration(mModuleConfiguration); 583 } 584 if (mDevice instanceof IConfigurationReceiver) { 585 ((IConfigurationReceiver) mDevice).setConfiguration(mModuleConfiguration); 586 } 587 if (test instanceof ISystemStatusCheckerReceiver) { 588 // We do not pass down Status checker because they are already running at the 589 // top level suite. 590 ((ISystemStatusCheckerReceiver) test).setSystemStatusChecker(new ArrayList<>()); 591 } 592 if (test instanceof ITestCollector) { 593 if (skipTestCases) { 594 mCollectTestsOnly = true; 595 } 596 ((ITestCollector) test).setCollectTestsOnly(mCollectTestsOnly); 597 } 598 if (!mPassThroughFilters.isEmpty()) { 599 applyFilterToTest(test, mPassThroughFilters); 600 } 601 mCurrentTestWrapper = 602 prepareGranularRetriableWrapper( 603 test, 604 listener, 605 skipTestCases, 606 perModuleRetryQuota); 607 mCurrentTestWrapper.setCollectTestsOnly(mCollectTestsOnly); 608 // Resolve the dynamic options for that one test. 609 preparationException = 610 invokeRemoteDynamic(moduleInfo.getDevice(), mInternalTestConfiguration); 611 if (preparationException != null) { 612 reportSetupFailure( 613 preparationException, 614 mInvocationListener, 615 mTargetPreparerRetryCount, 616 true); 617 return; 618 } 619 try (CloseableTraceScope ignored = new CloseableTraceScope("module_test")) { 620 mCurrentTestWrapper.run(moduleInfo, listener); 621 } catch (DeviceNotAvailableException dnae) { 622 runException = dnae; 623 // We do special logging of some information in Context of the module for easier 624 // debugging. 625 CLog.e( 626 "Module %s threw a DeviceNotAvailableException on device %s during " 627 + "test %s", 628 getId(), mDevice.getSerialNumber(), test.getClass()); 629 CLog.e(dnae); 630 // log an events 631 logDeviceEvent( 632 EventType.MODULE_DEVICE_NOT_AVAILABLE, 633 mDevice.getSerialNumber(), 634 dnae, 635 getId()); 636 throw dnae; 637 } finally { 638 mInternalTestConfiguration.cleanConfigurationData(); 639 mInternalTestConfiguration = null; 640 if (mMergeAttempts) { 641 // A single module can generate several test runs 642 mTestsResults.addAll(mCurrentTestWrapper.getFinalTestRunResults()); 643 } else { 644 // Keep track of each listener for attempts 645 mRunListenersResults.add(mCurrentTestWrapper.getResultListener()); 646 } 647 if (mModuleConfiguration 648 .getConfigurationDescription() 649 .isNotIRemoteTestShardable()) { 650 mPassThroughFilters.addAll(mCurrentTestWrapper.getPassedTests()); 651 } 652 // Limit escalating retries across all sub-IRemoteTests 653 if (mRetryDecision != null 654 && RetryStrategy.RETRY_ANY_FAILURE.equals( 655 mRetryDecision.getRetryStrategy())) { 656 perModuleRetryQuota -= mCurrentTestWrapper.getRetryCount(); 657 } 658 659 mExpectedTests += mCurrentTestWrapper.getExpectedTestsCount(); 660 // Get information about retry 661 if (mRetryDecision != null) { 662 RetryStatistics res = mRetryDecision.getRetryStatistics(); 663 if (res != null) { 664 addRetryTime(res.mRetryTime); 665 mRetryStats.add(res); 666 } 667 } 668 } 669 // After the run, if the test failed (even after retry the final result passed) has 670 // failed, capture a bugreport. 671 if (mCurrentTestWrapper.getResultListener().hasLastAttemptFailed()) { 672 captureBugreport( 673 mInvocationListener, 674 getId(), 675 mCurrentTestWrapper 676 .getResultListener() 677 .getCurrentRunResults() 678 .getRunFailureDescription()); 679 } 680 } 681 } finally { 682 mPassThroughFilters.clear(); 683 // Clean target preparers dynamic files. 684 if (mInternalTargetPreparerConfiguration != null) { 685 mInternalTargetPreparerConfiguration.cleanConfigurationData(); 686 mInternalTargetPreparerConfiguration = null; 687 } 688 long cleanStartTime = getCurrentTime(); 689 RuntimeException tearDownException = null; 690 try (CloseableTraceScope ignored = new CloseableTraceScope("module_teardown")) { 691 Throwable exception = (runException != null) ? runException : preparationException; 692 try { 693 // Tear down 694 runTearDown(moduleInfo, exception); 695 } catch (DeviceNotAvailableException dnae) { 696 if (runException == null) { 697 // Ignore the exception and attempt recovery. 698 CLog.e( 699 "Module %s failed during tearDown with: %s", 700 getId(), StreamUtil.getStackTrace(dnae)); 701 recoverDevice(moduleInfo, dnae); 702 } else { 703 throw dnae; 704 } 705 } 706 // If still available, verify that device didn't crash 707 if (runException == null) { 708 checkEndModuleDevice(moduleInfo); 709 } 710 } catch (DeviceNotAvailableException dnae) { 711 CLog.e( 712 "Module %s failed during tearDown with: %s", 713 getId(), StreamUtil.getStackTrace(dnae)); 714 throw dnae; 715 } catch (RuntimeException e) { 716 CLog.e("Exception while running tearDown:"); 717 CLog.e(e); 718 tearDownException = e; 719 } finally { 720 InvocationMetricLogger 721 .addInvocationPairMetrics(InvocationMetricKey.MODULE_TEARDOWN_PAIR, 722 cleanStartTime, getCurrentTime()); 723 mElapsedTearDown = getCurrentTime() - cleanStartTime; 724 // finalize results 725 if (preparationException == null) { 726 mModuleConfiguration.cleanConfigurationData(); 727 if (mMergeAttempts) { 728 reportFinalResults( 729 mInvocationListener, 730 mExpectedTests, 731 mTestsResults, 732 null, 733 tearDownException); 734 mTestsResults.clear(); 735 mExpectedTests = 0; 736 } else { 737 boolean reported = false; 738 // Push the attempts one by one 739 for (int i = 0; i < maxRunLimit; i++) { 740 // Get all the results for the attempt 741 List<TestRunResult> runResultList = new ArrayList<TestRunResult>(); 742 int expectedCount = 0; 743 for (ModuleListener attemptListener : mRunListenersResults) { 744 for (String runName : attemptListener.getTestRunNames()) { 745 TestRunResult run = 746 attemptListener.getTestRunAtAttempt(runName, i); 747 if (run != null) { 748 runResultList.add(run); 749 expectedCount += run.getExpectedTestCount(); 750 } 751 } 752 } 753 754 if (!runResultList.isEmpty() || ( 755 !reported && mRetriedModulePreparationSuccess)) { 756 if (runResultList.isEmpty()) { 757 reported = true; 758 CLog.i("Module preparation retry pass but no test cases were " + 759 "executed. Keep reporting the result to notify it " + 760 "failed in the 1st run but passed after retrying."); 761 } 762 reportFinalResults( 763 mInvocationListener, 764 expectedCount, 765 runResultList, 766 i, 767 tearDownException); 768 } else { 769 CLog.d("No results to be forwarded for attempt %s.", i); 770 } 771 } 772 } 773 } 774 // unset the module context since module run is ending. 775 CurrentInvocation.setModuleContext(null); 776 } 777 } 778 } 779 780 /** 781 * Create a wrapper class for the {@link IRemoteTest} which has built-in logic to schedule 782 * multiple test runs for the same module, and have the ability to run testcases at a more 783 * granular level (a subset of testcases in the module). 784 * 785 * @param test the {@link IRemoteTest} that is being wrapped. 786 * @param failureListener a particular listener to collect logs on testFail. Can be null. 787 * @param skipTestCases A run strategy when SKIP_MODULE_TESTCASES is defined. 788 * @param maxRunLimit a rate-limiter on testcases retrying times. 789 */ 790 @VisibleForTesting prepareGranularRetriableWrapper( IRemoteTest test, ITestInvocationListener listener, boolean skipTestCases, int maxRunLimit)791 GranularRetriableTestWrapper prepareGranularRetriableWrapper( 792 IRemoteTest test, 793 ITestInvocationListener listener, 794 boolean skipTestCases, 795 int maxRunLimit) { 796 GranularRetriableTestWrapper retriableTest = 797 new GranularRetriableTestWrapper( 798 test, this, listener, maxRunLimit, mUseModuleResultsForwarder); 799 retriableTest.setModuleId(getId()); 800 retriableTest.setMarkTestsSkipped(skipTestCases); 801 retriableTest.setMetricCollectors(mRunMetricCollectors); 802 retriableTest.setModuleConfig(mModuleConfiguration); 803 retriableTest.setInvocationContext(mModuleInvocationContext); 804 retriableTest.setLogSaver(mLogSaver); 805 retriableTest.setRetryDecision(mRetryDecision); 806 return retriableTest; 807 } 808 captureBugreport( ITestLogger listener, String moduleId, FailureDescription failure)809 private void captureBugreport( 810 ITestLogger listener, String moduleId, FailureDescription failure) { 811 FailureStatus status = failure.getFailureStatus(); 812 if (!FailureStatus.LOST_SYSTEM_UNDER_TEST.equals(status) 813 && !FailureStatus.SYSTEM_UNDER_TEST_CRASHED.equals(status)) { 814 return; 815 } 816 for (ITestDevice device : mModuleInvocationContext.getDevices()) { 817 if (device.getIDevice() instanceof StubDevice) { 818 continue; 819 } 820 device.logBugreport( 821 String.format( 822 "module-%s-failure-%s-bugreport", moduleId, device.getSerialNumber()), 823 listener); 824 } 825 } 826 827 /** Helper to log the device events. */ logDeviceEvent(EventType event, String serial, Throwable t, String moduleId)828 private void logDeviceEvent(EventType event, String serial, Throwable t, String moduleId) { 829 Map<String, String> args = new HashMap<>(); 830 args.put("serial", serial); 831 args.put("trace", StreamUtil.getStackTrace(t)); 832 args.put("module-id", moduleId); 833 LogRegistry.getLogRegistry().logEvent(LogLevel.DEBUG, event, args); 834 } 835 836 /** Finalize results to report them all and count if there are missing tests. */ reportFinalResults( ITestInvocationListener listener, int totalExpectedTests, List<TestRunResult> listResults, Integer attempt, RuntimeException tearDownException)837 private void reportFinalResults( 838 ITestInvocationListener listener, 839 int totalExpectedTests, 840 List<TestRunResult> listResults, 841 Integer attempt, 842 RuntimeException tearDownException) { 843 long elapsedTime = 0L; 844 HashMap<String, Metric> metricsProto = new HashMap<>(); 845 if (attempt != null) { 846 long startTime = 847 listResults.isEmpty() ? mStartTestTime : listResults.get(0).getStartTime(); 848 listener.testRunStarted( 849 getId(), totalExpectedTests, attempt + mTargetPreparerRetryCount, startTime); 850 } else { 851 listener.testRunStarted( 852 getId(), totalExpectedTests, mTargetPreparerRetryCount, mStartTestTime); 853 } 854 int numResults = 0; 855 MultiMap<String, LogFile> aggLogFiles = new MultiMap<>(); 856 List<FailureDescription> runFailureMessages = new ArrayList<>(); 857 for (TestRunResult runResult : listResults) { 858 numResults += runResult.getTestResults().size(); 859 forwardTestResults(runResult.getTestResults(), listener); 860 if (runResult.isRunFailure()) { 861 runFailureMessages.add(runResult.getRunFailureDescription()); 862 } 863 elapsedTime += runResult.getElapsedTime(); 864 // put metrics from the tests 865 metricsProto.putAll(runResult.getRunProtoMetrics()); 866 aggLogFiles.putAll(runResult.getRunLoggedFiles()); 867 } 868 // put metrics from the preparation 869 metricsProto.put( 870 PREPARATION_TIME, 871 TfMetricProtoUtil.createSingleValue(mElapsedPreparation, "milliseconds")); 872 metricsProto.put( 873 TEAR_DOWN_TIME, 874 TfMetricProtoUtil.createSingleValue(mElapsedTearDown, "milliseconds")); 875 metricsProto.put( 876 TEST_TIME, TfMetricProtoUtil.createSingleValue(elapsedTime, "milliseconds")); 877 metricsProto.put(MODULE_TEST_COUNT, TfMetricProtoUtil.createSingleValue(numResults, "int")); 878 // Report all the retry informations 879 if (!mRetryStats.isEmpty()) { 880 if (attempt != null) { 881 long cost = RetryStatistics.isolationCostPerAttempt(attempt, mRetryStats); 882 if (cost != 0L) { 883 metricsProto.put( 884 ISOLATION_COST, 885 TfMetricProtoUtil.createSingleValue(cost, "milliseconds")); 886 } 887 } else { 888 RetryStatistics agg = RetryStatistics.aggregateStatistics(mRetryStats); 889 metricsProto.put( 890 RETRY_TIME, 891 TfMetricProtoUtil.createSingleValue(agg.mRetryTime, "milliseconds")); 892 metricsProto.put( 893 RETRY_SUCCESS_COUNT, 894 TfMetricProtoUtil.createSingleValue(agg.mRetrySuccess, "")); 895 metricsProto.put( 896 RETRY_FAIL_COUNT, 897 TfMetricProtoUtil.createSingleValue(agg.mRetryFailure, "")); 898 } 899 } 900 901 // Only report the mismatch if there were no error during the run. 902 if (runFailureMessages.isEmpty() && totalExpectedTests != numResults) { 903 String error = 904 String.format( 905 "Module %s only ran %d out of %d expected tests.", 906 getId(), numResults, totalExpectedTests); 907 FailureDescription mismatch = 908 FailureDescription.create(error) 909 .setFailureStatus(FailureStatus.TEST_FAILURE) 910 .setErrorIdentifier(InfraErrorIdentifier.EXPECTED_TESTS_MISMATCH); 911 runFailureMessages.add(mismatch); 912 CLog.e(error); 913 } 914 915 if (tearDownException != null) { 916 FailureDescription failure = 917 CurrentInvocation.createFailure( 918 StreamUtil.getStackTrace(tearDownException), null) 919 .setCause(tearDownException); 920 runFailureMessages.add(failure); 921 } 922 // If there is any errors report them all at once 923 if (!runFailureMessages.isEmpty()) { 924 if (runFailureMessages.size() == 1) { 925 listener.testRunFailed(runFailureMessages.get(0)); 926 } else { 927 listener.testRunFailed(new MultiFailureDescription(runFailureMessages)); 928 } 929 mIsFailedModule = true; 930 } 931 932 // Provide a strong association of the run to its logs. 933 for (String key : aggLogFiles.keySet()) { 934 for (LogFile logFile : aggLogFiles.get(key)) { 935 if (listener instanceof ILogSaverListener) { 936 ((ILogSaverListener) listener).logAssociation(key, logFile); 937 } 938 } 939 } 940 // Allow each attempt to have its own start/end time 941 if (attempt != null) { 942 listener.testRunEnded(elapsedTime, metricsProto); 943 } else { 944 listener.testRunEnded(getCurrentTime() - mStartTestTime, metricsProto); 945 } 946 mFinalResultsReported = true; 947 } 948 forwardTestResults( Map<TestDescription, TestResult> testResults, ITestInvocationListener listener)949 private void forwardTestResults( 950 Map<TestDescription, TestResult> testResults, ITestInvocationListener listener) { 951 Iterator<Map.Entry<TestDescription, TestResult>> iterator = 952 testResults.entrySet().iterator(); 953 while (iterator.hasNext()) { 954 Map.Entry<TestDescription, TestResult> testEntry = iterator.next(); 955 listener.testStarted(testEntry.getKey(), testEntry.getValue().getStartTime()); 956 switch (testEntry.getValue().getResultStatus()) { 957 case FAILURE: 958 listener.testFailed(testEntry.getKey(), testEntry.getValue().getFailure()); 959 break; 960 case ASSUMPTION_FAILURE: 961 listener.testAssumptionFailure( 962 testEntry.getKey(), testEntry.getValue().getFailure()); 963 break; 964 case IGNORED: 965 listener.testIgnored(testEntry.getKey()); 966 break; 967 case SKIPPED: 968 listener.testSkipped(testEntry.getKey(), testEntry.getValue().getSkipReason()); 969 break; 970 case INCOMPLETE: 971 listener.testFailed( 972 testEntry.getKey(), 973 FailureDescription.create( 974 "Test did not complete due to exception.", 975 FailureStatus.TEST_FAILURE)); 976 break; 977 default: 978 break; 979 } 980 // Provide a strong association of the test to its logs. 981 for (Entry<String, LogFile> logFile : 982 testEntry.getValue().getLoggedFiles().entrySet()) { 983 if (listener instanceof ILogSaverListener) { 984 ((ILogSaverListener) listener) 985 .logAssociation(logFile.getKey(), logFile.getValue()); 986 } 987 } 988 listener.testEnded( 989 testEntry.getKey(), 990 testEntry.getValue().getEndTime(), 991 testEntry.getValue().getProtoMetrics()); 992 // Remove the test result from the map to release memory. 993 iterator.remove(); 994 } 995 } 996 997 /** 998 * Run preparers of the test, including suite level preparers if specified. 999 * 1000 * @param includeSuitePreparers Set to {@code true} to also run suite level preparers. 1001 * @return {@link Throwable} of any exception raised when running preparers. 1002 */ runPreparation(boolean includeSuitePreparers)1003 public Throwable runPreparation(boolean includeSuitePreparers) { 1004 Throwable preparationException = null; 1005 long prepStartTime = getCurrentTime(); 1006 if (includeSuitePreparers) { 1007 // Run suite level preparers. 1008 preparationException = runTargetPreparation(mSuitePreparersPerDevice); 1009 } 1010 1011 if (preparationException == null) { 1012 preparationException = runTargetPreparation(mPreparersPerDevice); 1013 } 1014 // Skip multi-preparation if preparation already failed. 1015 if (preparationException == null) { 1016 for (IMultiTargetPreparer multiPreparer : mMultiPreparers) { 1017 preparationException = runMultiPreparerSetup(multiPreparer); 1018 if (preparationException != null) { 1019 mIsFailedModule = true; 1020 CLog.e("Some preparation step failed. failing the module %s", getId()); 1021 break; 1022 } 1023 } 1024 } 1025 mElapsedPreparation = getCurrentTime() - prepStartTime; 1026 return preparationException; 1027 } 1028 1029 /** Run all the prepare steps. */ runPreparerSetup( ITargetPreparer preparer, int deviceIndex)1030 private Throwable runPreparerSetup( 1031 ITargetPreparer preparer, 1032 int deviceIndex) { 1033 if (preparer.isDisabled()) { 1034 // If disabled skip completely. 1035 return null; 1036 } 1037 TfObjectTracker.countWithParents(preparer.getClass()); 1038 CLog.d("Running setup preparer: %s", preparer.getClass().getSimpleName()); 1039 try (CloseableTraceScope ignored = 1040 new CloseableTraceScope(preparer.getClass().getName())) { 1041 if (preparer instanceof IConfigurationReceiver) { 1042 ((IConfigurationReceiver) preparer).setConfiguration(mModuleConfiguration); 1043 } 1044 // set the logger in case they need it. 1045 if (preparer instanceof ITestLoggerReceiver) { 1046 ((ITestLoggerReceiver) preparer).setTestLogger(mInvocationListener); 1047 } 1048 if (preparer instanceof IInvocationContextReceiver) { 1049 ((IInvocationContextReceiver) preparer) 1050 .setInvocationContext(mModuleInvocationContext); 1051 } 1052 mModuleInfo.setActiveDeviceIndex(deviceIndex); 1053 preparer.setUp(mModuleInfo); 1054 return null; 1055 } catch (BuildError 1056 | TargetSetupError 1057 | DeviceNotAvailableException 1058 | RuntimeException 1059 | AssertionError 1060 | LinkageError e) { 1061 // We catch all the TargetPreparer possible exception + RuntimeException to avoid 1062 // specific issues + AssertionError since it's widely used in tests and doesn't notify 1063 // something very wrong with the harness. 1064 CLog.e("Unexpected Exception from preparer: %s", preparer.getClass().getName()); 1065 CLog.e(e); 1066 return e; 1067 } finally { 1068 mModuleInfo.setActiveDeviceIndex(0); 1069 } 1070 } 1071 1072 /** Run all multi target preparer step. */ runMultiPreparerSetup(IMultiTargetPreparer preparer)1073 private Throwable runMultiPreparerSetup(IMultiTargetPreparer preparer) { 1074 if (preparer.isDisabled()) { 1075 // If disabled skip completely. 1076 return null; 1077 } 1078 TfObjectTracker.countWithParents(preparer.getClass()); 1079 CLog.d("Running setup multi preparer: %s", preparer.getClass().getSimpleName()); 1080 try (CloseableTraceScope ignored = 1081 new CloseableTraceScope(preparer.getClass().getName())) { 1082 if (preparer instanceof IConfigurationReceiver) { 1083 ((IConfigurationReceiver) preparer).setConfiguration(mModuleConfiguration); 1084 } 1085 // set the logger in case they need it. 1086 if (preparer instanceof ITestLoggerReceiver) { 1087 ((ITestLoggerReceiver) preparer).setTestLogger(mInvocationListener); 1088 } 1089 if (preparer instanceof IInvocationContextReceiver) { 1090 ((IInvocationContextReceiver) preparer) 1091 .setInvocationContext(mModuleInvocationContext); 1092 } 1093 preparer.setUp(mModuleInfo); 1094 return null; 1095 } catch (BuildError 1096 | TargetSetupError 1097 | DeviceNotAvailableException 1098 | RuntimeException 1099 | AssertionError 1100 | LinkageError e) { 1101 // We catch all the MultiTargetPreparer possible exception + RuntimeException to avoid 1102 // specific issues + AssertionError since it's widely used in tests and doesn't notify 1103 // something very wrong with the harness. 1104 CLog.e("Unexpected Exception from preparer: %s", preparer.getClass().getName()); 1105 CLog.e(e); 1106 return e; 1107 } 1108 } 1109 1110 /** Run all the tear down steps from preparers. */ runTearDown(TestInformation moduleInfo, Throwable exception)1111 private void runTearDown(TestInformation moduleInfo, Throwable exception) 1112 throws DeviceNotAvailableException { 1113 // Tear down 1114 List<IMultiTargetPreparer> cleanerList = new ArrayList<>(mMultiPreparers); 1115 Collections.reverse(cleanerList); 1116 for (IMultiTargetPreparer multiCleaner : cleanerList) { 1117 if (multiCleaner.isDisabled() || multiCleaner.isTearDownDisabled()) { 1118 // If disabled skip completely. 1119 continue; 1120 } 1121 CLog.d("Running teardown multi cleaner: %s", multiCleaner.getClass().getSimpleName()); 1122 multiCleaner.tearDown(moduleInfo, exception); 1123 } 1124 1125 for (int i = 0; i < mModuleInvocationContext.getDeviceConfigNames().size(); i++) { 1126 String deviceName = mModuleInvocationContext.getDeviceConfigNames().get(i); 1127 ITestDevice device = mModuleInvocationContext.getDevice(deviceName); 1128 if (i >= mPreparersPerDevice.size()) { 1129 CLog.d( 1130 "Main configuration has more devices than the module configuration. '%s' " 1131 + "will not run any tear down.", 1132 deviceName); 1133 continue; 1134 } 1135 List<ITargetPreparer> preparers = mPreparersPerDevice.get(deviceName); 1136 if (preparers == null) { 1137 CLog.w( 1138 "Module configuration devices mismatch the main configuration " 1139 + "(Missing device '%s'), resolving preparers by index.", 1140 deviceName); 1141 String key = new ArrayList<>(mPreparersPerDevice.keySet()).get(i); 1142 preparers = mPreparersPerDevice.get(key); 1143 } 1144 ListIterator<ITargetPreparer> itr = preparers.listIterator(preparers.size()); 1145 while (itr.hasPrevious()) { 1146 ITargetPreparer preparer = itr.previous(); 1147 // do not call the cleaner if it was disabled 1148 if (preparer.isDisabled() || preparer.isTearDownDisabled()) { 1149 CLog.d("%s has been disabled. skipping.", preparer); 1150 continue; 1151 } 1152 1153 RecoveryMode origMode = null; 1154 try (CloseableTraceScope ignored = 1155 new CloseableTraceScope(preparer.getClass().getName())) { 1156 // If an exception was generated in setup with a DNAE do not attempt any 1157 // recovery again in case we hit the device not available again. 1158 if (exception != null && exception instanceof DeviceNotAvailableException) { 1159 origMode = device.getRecoveryMode(); 1160 device.setRecoveryMode(RecoveryMode.NONE); 1161 } 1162 moduleInfo.setActiveDeviceIndex(i); 1163 preparer.tearDown(moduleInfo, exception); 1164 } finally { 1165 moduleInfo.setActiveDeviceIndex(0); 1166 if (origMode != null) { 1167 device.setRecoveryMode(origMode); 1168 } 1169 } 1170 } 1171 } 1172 } 1173 1174 /** Verify that the device did not crash after the module. */ checkEndModuleDevice(TestInformation testInfo)1175 private void checkEndModuleDevice(TestInformation testInfo) throws DeviceNotAvailableException { 1176 if (SystemUtil.isLocalMode()) { 1177 CLog.d("Skipping check for device availability after end of module for local run."); 1178 return; 1179 } 1180 try (CloseableTraceScope check = new CloseableTraceScope("checkEndModuleDevice")) { 1181 for (ITestDevice device : testInfo.getDevices()) { 1182 if (device.getIDevice() instanceof StubDevice) { 1183 continue; 1184 } 1185 // Check device is still online 1186 try { 1187 device.waitForDeviceAvailable(); 1188 } catch (DeviceNotAvailableException e) { 1189 // Wrap exception for better message 1190 String error_msg = 1191 String.format("Device went offline after running module '%s'", mId); 1192 // TODO: If module is the last one, it won't need to do device recovery. 1193 if (!mRecoverVirtualDevice) { 1194 throw new DeviceNotAvailableException( 1195 error_msg, 1196 e, 1197 e.getSerial(), 1198 DeviceErrorIdentifier.DEVICE_UNAVAILABLE); 1199 } 1200 CLog.d(error_msg); 1201 String snapshotId = null; 1202 if (device.getConnection() instanceof AdbTcpConnection) { 1203 snapshotId = 1204 ((AdbTcpConnection) device.getConnection()) 1205 .getSuiteSnapshots() 1206 .get(device); 1207 } 1208 device.getConnection().recoverVirtualDevice(device, snapshotId, e); 1209 } 1210 } 1211 } 1212 } 1213 recoverDevice(TestInformation testInfo, DeviceNotAvailableException e)1214 private void recoverDevice(TestInformation testInfo, DeviceNotAvailableException e) 1215 throws DeviceNotAvailableException { 1216 if (SystemUtil.isLocalMode()) { 1217 CLog.d("Skipping device recovery for local run."); 1218 throw e; 1219 } 1220 if (!mRecoverVirtualDevice) { 1221 CLog.d("Skipping device recovery for as option recover-device-by-cvd is not enabled."); 1222 throw e; 1223 } 1224 try (CloseableTraceScope check = new CloseableTraceScope("recover_device")) { 1225 for (ITestDevice device : testInfo.getDevices()) { 1226 if (device.getIDevice() instanceof StubDevice) { 1227 continue; 1228 } 1229 String snapshotId = null; 1230 if (device.getConnection() instanceof AdbTcpConnection) { 1231 snapshotId = 1232 ((AdbTcpConnection) device.getConnection()) 1233 .getSuiteSnapshots() 1234 .get(device); 1235 } 1236 device.getConnection().recoverVirtualDevice(device, snapshotId, e); 1237 } 1238 } 1239 } 1240 1241 /** Returns the current time. */ getCurrentTime()1242 private long getCurrentTime() { 1243 return System.currentTimeMillis(); 1244 } 1245 1246 @Override setCollectTestsOnly(boolean collectTestsOnly)1247 public void setCollectTestsOnly(boolean collectTestsOnly) { 1248 mCollectTestsOnly = collectTestsOnly; 1249 } 1250 1251 /** Sets should recover virtual device. */ setRecoverVirtualDevice(boolean recoverVirtualDevice)1252 public void setRecoverVirtualDevice(boolean recoverVirtualDevice) { 1253 mRecoverVirtualDevice = recoverVirtualDevice; 1254 } 1255 1256 /** Returns if we should recover virtual device. */ shouldRecoverVirtualDevice()1257 public boolean shouldRecoverVirtualDevice() { 1258 return mRecoverVirtualDevice; 1259 } 1260 1261 /** Sets whether or not we should merge results. */ setMergeAttemps(boolean mergeAttempts)1262 public final void setMergeAttemps(boolean mergeAttempts) { 1263 mMergeAttempts = mergeAttempts; 1264 } 1265 1266 /** Sets the {@link IRetryDecision} to be used for intra-module retry. */ setRetryDecision(IRetryDecision decision)1267 public final void setRetryDecision(IRetryDecision decision) { 1268 mRetryDecision = decision; 1269 // Carry the retry decision to the module configuration 1270 mModuleConfiguration.setRetryDecision(decision); 1271 } 1272 1273 /** Returns True if a testRunFailure has been called on the module * */ hasModuleFailed()1274 public boolean hasModuleFailed() { 1275 return mIsFailedModule; 1276 } 1277 getRequiredTokens(TestInformation testInfo)1278 public Set<TokenProperty> getRequiredTokens(TestInformation testInfo) { 1279 // If there are no controllers just return directly 1280 List<?> ctrlObjectList = mModuleConfiguration.getConfigurationObjectList(MODULE_CONTROLLER); 1281 if (ctrlObjectList == null) { 1282 return mRequiredTokens; 1283 } 1284 // Clone the module context to get its metadata and then provide the device information 1285 // the same as ITestSuite would do during execution to run only the controllers 1286 InvocationContext clonedContext = 1287 InvocationContext.fromProto(mModuleInvocationContext.toProto()); 1288 for (String deviceName : testInfo.getContext().getDeviceConfigNames()) { 1289 clonedContext.addAllocatedDevice( 1290 deviceName, testInfo.getContext().getDevice(deviceName)); 1291 clonedContext.addDeviceBuildInfo( 1292 deviceName, testInfo.getContext().getBuildInfo(deviceName)); 1293 } 1294 try { 1295 if (!RunStrategy.RUN.equals(shouldRunWithController(clonedContext))) { 1296 // Bypass token since the module isn't expected to run 1297 return null; 1298 } 1299 } catch (RuntimeException | DeviceNotAvailableException e) { 1300 CLog.e(e); 1301 } 1302 return mRequiredTokens; 1303 } 1304 1305 /** {@inheritDoc} */ 1306 @Override toString()1307 public String toString() { 1308 return getId(); 1309 } 1310 1311 /** Returns the approximate time to run all the tests in the module. */ getRuntimeHint()1312 public long getRuntimeHint() { 1313 long hint = 0L; 1314 for (IRemoteTest test : mTests) { 1315 if (test instanceof IRuntimeHintProvider) { 1316 hint += ((IRuntimeHintProvider) test).getRuntimeHint(); 1317 } else { 1318 hint += 60000; 1319 } 1320 } 1321 return hint; 1322 } 1323 1324 /** Returns the list of {@link IRemoteTest} defined for this module. */ 1325 @VisibleForTesting getTests()1326 List<IRemoteTest> getTests() { 1327 return new ArrayList<>(mTests); 1328 } 1329 1330 /** Returns the list of {@link ITargetPreparer} associated with the given device name */ 1331 @VisibleForTesting getTargetPreparerForDevice(String deviceName)1332 List<ITargetPreparer> getTargetPreparerForDevice(String deviceName) { 1333 return mPreparersPerDevice.get(deviceName); 1334 } 1335 1336 /** 1337 * Returns the list of suite level {@link ITargetPreparer} associated with the given device name 1338 */ 1339 @VisibleForTesting getSuitePreparerForDevice(String deviceName)1340 List<ITargetPreparer> getSuitePreparerForDevice(String deviceName) { 1341 return mSuitePreparersPerDevice.get(deviceName); 1342 } 1343 1344 /** 1345 * When running unit tests for ModuleDefinition we don't want to unnecessarily report some auto 1346 * retry times. 1347 */ 1348 @VisibleForTesting disableAutoRetryReportingTime()1349 void disableAutoRetryReportingTime() { 1350 mDisableAutoRetryTimeReporting = true; 1351 } 1352 1353 /** Returns the {@link IInvocationContext} associated with the module. */ getModuleInvocationContext()1354 public IInvocationContext getModuleInvocationContext() { 1355 return mModuleInvocationContext; 1356 } 1357 getModuleConfiguration()1358 public IConfiguration getModuleConfiguration() { 1359 return mModuleConfiguration; 1360 } 1361 setReportModuleStart(boolean shouldReportModuleStart)1362 public void setReportModuleStart(boolean shouldReportModuleStart) { 1363 mReportModuleStart = shouldReportModuleStart; 1364 } 1365 setReportModuleEnd(boolean shouldReportModuleEnd)1366 public void setReportModuleEnd(boolean shouldReportModuleEnd) { 1367 mReportModuleEnd = shouldReportModuleEnd; 1368 } 1369 1370 /** Report completely not executed modules. */ reportNotExecuted(ITestInvocationListener listener, String message)1371 public final void reportNotExecuted(ITestInvocationListener listener, String message) { 1372 if (mReportModuleStart) { 1373 listener.testModuleStarted(getModuleInvocationContext()); 1374 } 1375 if (mCurrentTestWrapper != null) { 1376 // do not report results if already reported once 1377 if (!mFinalResultsReported) { 1378 mRunListenersResults.add(mCurrentTestWrapper.getResultListener()); 1379 HarnessRuntimeException interruptedException = 1380 new HarnessRuntimeException( 1381 message, TestErrorIdentifier.MODULE_DID_NOT_EXECUTE); 1382 for (int i = 0; i < mMaxRetry; i++) { 1383 // Get all the results for the attempt 1384 List<TestRunResult> runResultList = new ArrayList<TestRunResult>(); 1385 int expectedCount = 0; 1386 for (ModuleListener attemptListener : mRunListenersResults) { 1387 for (String runName : attemptListener.getTestRunNames()) { 1388 TestRunResult run = attemptListener.getTestRunAtAttempt(runName, i); 1389 if (run != null) { 1390 runResultList.add(run); 1391 expectedCount += run.getExpectedTestCount(); 1392 } 1393 } 1394 } 1395 1396 if (!runResultList.isEmpty()) { 1397 reportFinalResults( 1398 listener, expectedCount, runResultList, i, interruptedException); 1399 } else { 1400 CLog.d("No results to be forwarded for attempt %s.", i); 1401 } 1402 } 1403 } 1404 } else { 1405 listener.testRunStarted( 1406 getId(), 0, mTargetPreparerRetryCount, System.currentTimeMillis()); 1407 FailureDescription description = 1408 FailureDescription.create(message) 1409 .setFailureStatus(FailureStatus.NOT_EXECUTED) 1410 .setErrorIdentifier(TestErrorIdentifier.MODULE_DID_NOT_EXECUTE); 1411 listener.testRunFailed(description); 1412 listener.testRunEnded(0, new HashMap<String, Metric>()); 1413 } 1414 if (mReportModuleEnd) { 1415 listener.testModuleEnded(); 1416 } 1417 } 1418 1419 /** Whether or not to enable dynamic download at module level. */ setEnableDynamicDownload(boolean enableDynamicDownload)1420 public void setEnableDynamicDownload(boolean enableDynamicDownload) { 1421 mEnableDynamicDownload = enableDynamicDownload; 1422 } 1423 1424 /** Copy a few of the suite level configuration */ transferSuiteLevelOptions(IConfiguration mSuiteConfiguration)1425 public void transferSuiteLevelOptions(IConfiguration mSuiteConfiguration) { 1426 mModuleConfiguration 1427 .getCommandOptions() 1428 .getDynamicDownloadArgs() 1429 .putAll(mSuiteConfiguration.getCommandOptions().getDynamicDownloadArgs()); 1430 mModuleConfiguration 1431 .getCommandOptions() 1432 .setReportTestCaseCount( 1433 mSuiteConfiguration.getCommandOptions().reportTestCaseCount()); 1434 } 1435 1436 /** 1437 * Allow to load module_controller objects to tune how should a particular module run. They will 1438 * be applied in order of appearance in the XML. 1439 * 1440 * @param failureListener The {@link TestFailureListener} taking actions on tests failures. 1441 * @return The strategy to use to run the tests. 1442 */ applyConfigurationControl()1443 private RunStrategy applyConfigurationControl() throws DeviceNotAvailableException { 1444 List<?> ctrlObjectList = mModuleConfiguration.getConfigurationObjectList(MODULE_CONTROLLER); 1445 if (ctrlObjectList == null) { 1446 return RunStrategy.RUN; 1447 } 1448 for (Object ctrlObject : ctrlObjectList) { 1449 if (ctrlObject instanceof BaseModuleController) { 1450 BaseModuleController controller = (BaseModuleController) ctrlObject; 1451 // Track usage of the controller 1452 TfObjectTracker.countWithParents(controller.getClass()); 1453 if (!controller.shouldCaptureLogcat()) { 1454 mRunMetricCollectors.removeIf(c -> (c instanceof LogcatOnFailureCollector)); 1455 } 1456 if (!controller.shouldCaptureScreenshot()) { 1457 mRunMetricCollectors.removeIf(c -> (c instanceof ScreenshotOnFailureCollector)); 1458 } 1459 if (!controller.shouldCaptureBugreport()) { 1460 mRunMetricCollectors.removeIf( 1461 c -> (c instanceof BugreportzOnTestCaseFailureCollector)); 1462 } 1463 } 1464 } 1465 return shouldRunWithController(mModuleInvocationContext); 1466 } 1467 shouldRunWithController(IInvocationContext context)1468 private RunStrategy shouldRunWithController(IInvocationContext context) 1469 throws DeviceNotAvailableException { 1470 List<?> ctrlObjectList = mModuleConfiguration.getConfigurationObjectList(MODULE_CONTROLLER); 1471 if (ctrlObjectList == null) { 1472 return RunStrategy.RUN; 1473 } 1474 // We keep the most stringent strategy across controllers. 1475 RunStrategy current = RunStrategy.RUN; 1476 for (Object ctrlObject : ctrlObjectList) { 1477 if (ctrlObject instanceof BaseModuleController) { 1478 BaseModuleController controller = (BaseModuleController) ctrlObject; 1479 RunStrategy strategy = controller.shouldRunModule(context); 1480 if (RunStrategy.FULL_MODULE_BYPASS.equals(strategy)) { 1481 current = strategy; 1482 } else if (RunStrategy.SKIP_MODULE_TESTCASES.equals(strategy) 1483 && RunStrategy.RUN.equals(current)) { 1484 current = strategy; 1485 } 1486 } 1487 } 1488 return current; 1489 } 1490 addRetryTime(long retryTimeMs)1491 private void addRetryTime(long retryTimeMs) { 1492 if (retryTimeMs <= 0 || mDisableAutoRetryTimeReporting) { 1493 return; 1494 } 1495 InvocationMetricLogger.addInvocationMetrics( 1496 InvocationMetricKey.AUTO_RETRY_TIME, retryTimeMs); 1497 } 1498 runTargetPreparation(Map<String, List<ITargetPreparer>> preparersPerDevice)1499 private Throwable runTargetPreparation(Map<String, List<ITargetPreparer>> preparersPerDevice) { 1500 Throwable preparationException = null; 1501 for (int i = 0; i < mModuleInvocationContext.getDeviceConfigNames().size(); i++) { 1502 String deviceName = mModuleInvocationContext.getDeviceConfigNames().get(i); 1503 if (i >= preparersPerDevice.size()) { 1504 CLog.d( 1505 "Main configuration has more devices than the module configuration. '%s' " 1506 + "will not run any preparation.", 1507 deviceName); 1508 continue; 1509 } 1510 List<ITargetPreparer> preparers = preparersPerDevice.get(deviceName); 1511 if (preparers == null) { 1512 CLog.w( 1513 "Module configuration devices mismatch the main configuration " 1514 + "(Missing device '%s'), resolving preparers by index.", 1515 deviceName); 1516 String key = new ArrayList<>(preparersPerDevice.keySet()).get(i); 1517 preparers = preparersPerDevice.get(key); 1518 } 1519 for (ITargetPreparer preparer : preparers) { 1520 preparationException = runPreparerSetup(preparer, i); 1521 if (preparationException != null) { 1522 mIsFailedModule = true; 1523 CLog.e("Some preparation step failed. failing the module %s", getId()); 1524 // If one device errored out, we skip the remaining devices. 1525 return preparationException; 1526 } 1527 } 1528 } 1529 return null; 1530 } 1531 1532 /** 1533 * Handle calling the {@link IConfiguration#resolveDynamicOptions(DynamicRemoteFileResolver)}. 1534 */ invokeRemoteDynamic(ITestDevice device, IConfiguration moduleConfiguration)1535 private Exception invokeRemoteDynamic(ITestDevice device, IConfiguration moduleConfiguration) { 1536 if (!mEnableDynamicDownload) { 1537 return null; 1538 } 1539 // TODO: Add elapsed time tracking 1540 try { 1541 CLog.d("Attempting to resolve dynamic files from %s", getId()); 1542 DynamicRemoteFileResolver resolver = new DynamicRemoteFileResolver(); 1543 resolver.setDevice(device); 1544 resolver.addExtraArgs(moduleConfiguration.getCommandOptions().getDynamicDownloadArgs()); 1545 moduleConfiguration.resolveDynamicOptions(resolver); 1546 return null; 1547 } catch (RuntimeException | ConfigurationException | BuildRetrievalError e) { 1548 mIsFailedModule = true; 1549 return e; 1550 } 1551 } 1552 1553 /** Report a setup exception as a run failure and notify all the listeners. */ reportSetupFailure( Throwable setupException, ITestInvocationListener invocListener, int attemptNumber, boolean shouldFail)1554 private void reportSetupFailure( 1555 Throwable setupException, 1556 ITestInvocationListener invocListener, 1557 int attemptNumber, 1558 boolean shouldFail) 1559 throws DeviceNotAvailableException { 1560 // For reporting purpose we create a failure placeholder with the error stack 1561 // similar to InitializationError of JUnit. 1562 invocListener.testRunStarted(getId(), 1, attemptNumber, System.currentTimeMillis()); 1563 FailureDescription failureDescription = 1564 CurrentInvocation.createFailure(StreamUtil.getStackTrace(setupException), null); 1565 if (setupException instanceof IHarnessException 1566 && ((IHarnessException) setupException).getErrorId() != null) { 1567 ErrorIdentifier id = ((IHarnessException) setupException).getErrorId(); 1568 failureDescription.setErrorIdentifier(id); 1569 failureDescription.setFailureStatus(id.status()); 1570 failureDescription.setOrigin(((IHarnessException) setupException).getOrigin()); 1571 } else if (setupException instanceof RuntimeException) { 1572 // TODO: switch to customer_issue 1573 failureDescription.setFailureStatus(FailureStatus.UNSET); 1574 failureDescription.setErrorIdentifier( 1575 InfraErrorIdentifier.MODULE_SETUP_RUNTIME_EXCEPTION); 1576 } else { 1577 failureDescription.setFailureStatus(FailureStatus.UNSET); 1578 } 1579 failureDescription.setCause(setupException); 1580 invocListener.testRunFailed(failureDescription); 1581 HashMap<String, Metric> metricsProto = new HashMap<>(); 1582 metricsProto.put(TEST_TIME, TfMetricProtoUtil.createSingleValue(0L, "milliseconds")); 1583 invocListener.testRunEnded(0, metricsProto); 1584 // If it was a not available exception rethrow it to signal the new device state. 1585 if (setupException instanceof DeviceNotAvailableException) { 1586 if (!shouldFail) { 1587 CLog.i("Do not report the exception as module error, returning..."); 1588 return; 1589 } 1590 throw (DeviceNotAvailableException) setupException; 1591 } 1592 } 1593 applyFilterToTest(IRemoteTest test, Set<TestDescription> filters)1594 private void applyFilterToTest(IRemoteTest test, Set<TestDescription> filters) { 1595 Set<String> filterNames = 1596 filters.stream().map(f -> f.toString()).collect(Collectors.toSet()); 1597 if (test instanceof ITestFileFilterReceiver) { 1598 File excludeFilterFile = ((ITestFileFilterReceiver) test).getExcludeTestFile(); 1599 if (excludeFilterFile == null) { 1600 try { 1601 excludeFilterFile = FileUtil.createTempFile("exclude-filter", ".txt"); 1602 } catch (IOException e) { 1603 throw new HarnessRuntimeException( 1604 e.getMessage(), e, InfraErrorIdentifier.FAIL_TO_CREATE_FILE); 1605 } 1606 ((ITestFileFilterReceiver) test).setExcludeTestFile(excludeFilterFile); 1607 } 1608 try { 1609 FileUtil.writeToFile(Joiner.on('\n').join(filterNames), excludeFilterFile, true); 1610 } catch (IOException e) { 1611 CLog.e(e); 1612 } 1613 } else if (test instanceof ITestFilterReceiver) { 1614 ((ITestFilterReceiver) test).addAllExcludeFilters(filterNames); 1615 } 1616 } 1617 setUseModuleResultsForwarder(boolean useModuleResultsForwarder)1618 public void setUseModuleResultsForwarder(boolean useModuleResultsForwarder) { 1619 mUseModuleResultsForwarder = useModuleResultsForwarder; 1620 } 1621 } 1622