1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.testtype.suite; 18 19 import com.android.tradefed.config.ConfigurationDescriptor; 20 import com.android.tradefed.config.IConfiguration; 21 import com.android.tradefed.config.IConfigurationReceiver; 22 import com.android.tradefed.device.DeviceNotAvailableException; 23 import com.android.tradefed.device.DeviceUnresponsiveException; 24 import com.android.tradefed.device.metric.CollectorHelper; 25 import com.android.tradefed.device.metric.CountTestCasesCollector; 26 import com.android.tradefed.device.metric.IMetricCollector; 27 import com.android.tradefed.device.metric.IMetricCollectorReceiver; 28 import com.android.tradefed.error.IHarnessException; 29 import com.android.tradefed.invoker.IInvocationContext; 30 import com.android.tradefed.invoker.TestInformation; 31 import com.android.tradefed.invoker.logger.CurrentInvocation; 32 import com.android.tradefed.invoker.logger.CurrentInvocation.IsolationGrade; 33 import com.android.tradefed.invoker.tracing.CloseableTraceScope; 34 import com.android.tradefed.log.LogUtil.CLog; 35 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 36 import com.android.tradefed.result.FailureDescription; 37 import com.android.tradefed.result.ILogSaver; 38 import com.android.tradefed.result.ITestInvocationListener; 39 import com.android.tradefed.result.ModuleResultsAndMetricsForwarder; 40 import com.android.tradefed.result.ResultAndLogForwarder; 41 import com.android.tradefed.result.TestDescription; 42 import com.android.tradefed.result.TestRunResult; 43 import com.android.tradefed.result.TestStatus; 44 import com.android.tradefed.result.error.ErrorIdentifier; 45 import com.android.tradefed.retry.IRetryDecision; 46 import com.android.tradefed.retry.MergeStrategy; 47 import com.android.tradefed.retry.RetryLogSaverResultForwarder; 48 import com.android.tradefed.retry.RetryStatistics; 49 import com.android.tradefed.testtype.IRemoteTest; 50 import com.android.tradefed.testtype.ITestCollector; 51 import com.android.tradefed.testtype.ITestFilterReceiver; 52 import com.android.tradefed.util.StreamUtil; 53 54 import com.google.common.annotations.VisibleForTesting; 55 56 import java.time.Duration; 57 import java.util.ArrayList; 58 import java.util.Arrays; 59 import java.util.HashMap; 60 import java.util.LinkedHashMap; 61 import java.util.LinkedHashSet; 62 import java.util.List; 63 import java.util.Map; 64 import java.util.Set; 65 66 /** 67 * A wrapper class works on the {@link IRemoteTest} to granulate the IRemoteTest in testcase level. 68 * An IRemoteTest can contain multiple testcases. Previously, these testcases are treated as a 69 * whole: When IRemoteTest runs, all testcases will run. Some IRemoteTest (The ones that implements 70 * ITestFilterReceiver) can accept an allowlist of testcases and only run those testcases. This 71 * class takes advantage of the existing feature and provides a more flexible way to run test suite. 72 * 73 * <ul> 74 * <li>Single testcase can be retried multiple times (within the same IRemoteTest run) to reduce 75 * the non-test-error failure rates. 76 * <li>The retried testcases are dynamically collected from previous run failures. 77 * </ul> 78 * 79 * <p>Note: 80 * 81 * <ul> 82 * <li>The prerequisite to run a subset of test cases is that the test type should implement the 83 * interface {@link ITestFilterReceiver}. 84 * <li>X is customized max retry number. 85 * </ul> 86 */ 87 public class GranularRetriableTestWrapper implements IRemoteTest, ITestCollector { 88 89 private IRetryDecision mRetryDecision; 90 private IRemoteTest mTest; 91 private ModuleDefinition mModule; 92 private List<IMetricCollector> mRunMetricCollectors; 93 private IInvocationContext mModuleInvocationContext; 94 private IConfiguration mModuleConfiguration; 95 private ModuleListener mMainGranularRunListener; 96 private RetryLogSaverResultForwarder mRetryAttemptForwarder; 97 private ITestInvocationListener mRemoteTestTimeOutEnforcer; 98 private ModuleResultsAndMetricsForwarder listenerWithModuleMetricsForwarder; 99 private ILogSaver mLogSaver; 100 private String mModuleId; 101 private int mMaxRunLimit; 102 103 private boolean mCollectTestsOnly = false; 104 105 // Tracking of the metrics 106 private RetryStatistics mRetryStats = null; 107 private int mCountRetryUsed = 0; 108 109 private boolean mUseModuleResultsForwarder = false; 110 GranularRetriableTestWrapper( IRemoteTest test, ITestInvocationListener mainListener, int maxRunLimit)111 public GranularRetriableTestWrapper( 112 IRemoteTest test, 113 ITestInvocationListener mainListener, 114 int maxRunLimit) { 115 this(test, null, mainListener, maxRunLimit); 116 } 117 GranularRetriableTestWrapper( IRemoteTest test, ModuleDefinition module, ITestInvocationListener mainListener, int maxRunLimit)118 public GranularRetriableTestWrapper( 119 IRemoteTest test, 120 ModuleDefinition module, 121 ITestInvocationListener mainListener, 122 int maxRunLimit) { 123 this(test, module, mainListener, maxRunLimit, false); 124 } 125 GranularRetriableTestWrapper( IRemoteTest test, ModuleDefinition module, ITestInvocationListener mainListener, int maxRunLimit, boolean useModuleResultsForwarder)126 public GranularRetriableTestWrapper( 127 IRemoteTest test, 128 ModuleDefinition module, 129 ITestInvocationListener mainListener, 130 int maxRunLimit, 131 boolean useModuleResultsForwarder) { 132 mTest = test; 133 mModule = module; 134 mUseModuleResultsForwarder = useModuleResultsForwarder; 135 IInvocationContext context = null; 136 if (module != null) { 137 context = module.getModuleInvocationContext(); 138 } 139 initializeGranularRunListener(mainListener, context); 140 mMaxRunLimit = maxRunLimit; 141 } 142 143 /** Sets the {@link IRetryDecision} to be used. */ setRetryDecision(IRetryDecision decision)144 public void setRetryDecision(IRetryDecision decision) { 145 mRetryDecision = decision; 146 } 147 148 /** 149 * Set the {@link ModuleDefinition} name as a {@link GranularRetriableTestWrapper} attribute. 150 * 151 * @param moduleId the name of the moduleDefinition. 152 */ setModuleId(String moduleId)153 public void setModuleId(String moduleId) { 154 mModuleId = moduleId; 155 } 156 157 /** 158 * Set the {@link ModuleDefinition} RunStrategy as a {@link GranularRetriableTestWrapper} 159 * attribute. 160 * 161 * @param skipTestCases whether the testcases should be skipped. 162 */ setMarkTestsSkipped(boolean skipTestCases)163 public void setMarkTestsSkipped(boolean skipTestCases) { 164 mMainGranularRunListener.setMarkTestsSkipped(skipTestCases); 165 } 166 167 /** 168 * Set the {@link ModuleDefinition}'s runMetricCollector as a {@link 169 * GranularRetriableTestWrapper} attribute. 170 * 171 * @param runMetricCollectors A list of MetricCollector for the module. 172 */ setMetricCollectors(List<IMetricCollector> runMetricCollectors)173 public void setMetricCollectors(List<IMetricCollector> runMetricCollectors) { 174 mRunMetricCollectors = runMetricCollectors; 175 } 176 177 /** 178 * Set the {@link ModuleDefinition}'s ModuleConfig as a {@link GranularRetriableTestWrapper} 179 * attribute. 180 * 181 * @param moduleConfiguration Provide the module metrics. 182 */ setModuleConfig(IConfiguration moduleConfiguration)183 public void setModuleConfig(IConfiguration moduleConfiguration) { 184 mModuleConfiguration = moduleConfiguration; 185 } 186 187 /** 188 * Set the {@link IInvocationContext} as a {@link GranularRetriableTestWrapper} attribute. 189 * 190 * @param moduleInvocationContext The wrapper uses the InvocationContext to initialize the 191 * MetricCollector when necessary. 192 */ setInvocationContext(IInvocationContext moduleInvocationContext)193 public void setInvocationContext(IInvocationContext moduleInvocationContext) { 194 mModuleInvocationContext = moduleInvocationContext; 195 } 196 197 /** 198 * Set the Module's {@link ILogSaver} as a {@link GranularRetriableTestWrapper} attribute. 199 * 200 * @param logSaver The listeners for each test run should save the logs. 201 */ setLogSaver(ILogSaver logSaver)202 public void setLogSaver(ILogSaver logSaver) { 203 mLogSaver = logSaver; 204 } 205 206 /** 207 * Initialize granular run listener with {@link RemoteTestTimeOutEnforcer} if timeout is set. 208 * And set the test-mapping sources in granular run listener. 209 * 210 * @param listener The listener for each test run should be wrapped. 211 * @param moduleContext the invocation context of the module 212 */ initializeGranularRunListener( ITestInvocationListener listener, IInvocationContext moduleContext)213 private void initializeGranularRunListener( 214 ITestInvocationListener listener, IInvocationContext moduleContext) { 215 ModuleResultsAndMetricsForwarder mListenerWithModuleMetricsForwarder = null; 216 if (mUseModuleResultsForwarder) { 217 mListenerWithModuleMetricsForwarder = new ModuleResultsAndMetricsForwarder(listener); 218 mListenerWithModuleMetricsForwarder.setModuleId(mModuleId); 219 listener = mListenerWithModuleMetricsForwarder; 220 } 221 mMainGranularRunListener = new ModuleListener(listener, moduleContext); 222 mMainGranularRunListener.setUseModuleResultsForwarder(mUseModuleResultsForwarder); 223 if (mModule != null) { 224 ConfigurationDescriptor configDesc = 225 mModule.getModuleInvocationContext().getConfigurationDescriptor(); 226 if (configDesc.getMetaData( 227 RemoteTestTimeOutEnforcer.REMOTE_TEST_TIMEOUT_OPTION) != null) { 228 Duration duration = Duration.parse( 229 configDesc.getMetaData( 230 RemoteTestTimeOutEnforcer.REMOTE_TEST_TIMEOUT_OPTION).get(0)); 231 mRemoteTestTimeOutEnforcer = new RemoteTestTimeOutEnforcer( 232 mMainGranularRunListener, mModule, mTest, duration); 233 } 234 List<String> testMappingSources = 235 configDesc.getMetaData(Integer.toString(mTest.hashCode())); 236 if (testMappingSources != null) { 237 mMainGranularRunListener.setTestMappingSources(testMappingSources); 238 if (mListenerWithModuleMetricsForwarder != null) { 239 mListenerWithModuleMetricsForwarder.setTestMappingSources(testMappingSources); 240 } 241 } 242 } 243 } 244 245 /** 246 * Initialize a new {@link ModuleListener} for each test run. 247 * 248 * @return a {@link ITestInvocationListener} listener which contains the new {@link 249 * ModuleListener}, the main {@link ITestInvocationListener} and main {@link 250 * TestFailureListener}, and wrapped by RunMetricsCollector and Module MetricCollector (if 251 * not initialized). 252 */ initializeListeners()253 private ITestInvocationListener initializeListeners() throws DeviceNotAvailableException { 254 List<ITestInvocationListener> currentTestListener = new ArrayList<>(); 255 currentTestListener.add(mMainGranularRunListener); 256 257 if (mRemoteTestTimeOutEnforcer != null) { 258 currentTestListener.add(mRemoteTestTimeOutEnforcer); 259 } 260 261 mRetryAttemptForwarder = 262 new RetryLogSaverResultForwarder( 263 mLogSaver, currentTestListener, mModuleConfiguration); 264 ITestInvocationListener runListener = mRetryAttemptForwarder; 265 266 // The module collectors itself are added: this list will be very limited. 267 // We clone them since the configuration object is shared across shards. 268 for (IMetricCollector collector : 269 CollectorHelper.cloneCollectors(mModuleConfiguration.getMetricCollectors())) { 270 if (collector.isDisabled()) { 271 CLog.d("%s has been disabled. Skipping.", collector); 272 } else { 273 try (CloseableTraceScope ignored = 274 new CloseableTraceScope( 275 "init_attempt_" + collector.getClass().getSimpleName())) { 276 if (collector instanceof IConfigurationReceiver) { 277 ((IConfigurationReceiver) collector).setConfiguration(mModuleConfiguration); 278 } 279 runListener = collector.init(mModuleInvocationContext, runListener); 280 } 281 } 282 } 283 284 return runListener; 285 } 286 287 /** 288 * Schedule a series of {@link IRemoteTest#run(TestInformation, ITestInvocationListener)}. 289 * 290 * @param listener The ResultForwarder listener which contains a new moduleListener for each 291 * run. 292 */ 293 @Override run(TestInformation testInfo, ITestInvocationListener listener)294 public void run(TestInformation testInfo, ITestInvocationListener listener) 295 throws DeviceNotAvailableException { 296 mMainGranularRunListener.setCollectTestsOnly(mCollectTestsOnly); 297 ITestInvocationListener allListeners = initializeListeners(); 298 // First do the regular run, not retried. 299 DeviceNotAvailableException dnae = intraModuleRun(testInfo, allListeners, 0); 300 301 if (mMaxRunLimit <= 1) { 302 // TODO: If module is the last one and there is no retry quota, it won't need to do 303 // device recovery. 304 if (dnae == null || !mModule.shouldRecoverVirtualDevice()) { 305 if (dnae != null) { 306 throw dnae; 307 } 308 return; 309 } 310 } 311 312 if (mRetryDecision == null) { 313 CLog.e("RetryDecision is null. Something is misconfigured this shouldn't happen"); 314 return; 315 } 316 317 // Bail out early if there is no need to retry at all. 318 if (!mRetryDecision.shouldRetry( 319 mTest, mModule, 0, mMainGranularRunListener.getTestRunForAttempts(0), dnae)) { 320 return; 321 } 322 323 // Avoid rechecking the shouldRetry below the first time as it could retrigger reboot. 324 boolean firstCheck = true; 325 326 // Deal with retried attempted 327 long startTime = System.currentTimeMillis(); 328 try { 329 CLog.d("Starting intra-module retry."); 330 for (int attemptNumber = 1; attemptNumber < mMaxRunLimit; attemptNumber++) { 331 if (!firstCheck) { 332 boolean retry = 333 mRetryDecision.shouldRetry( 334 mTest, 335 mModule, 336 attemptNumber - 1, 337 mMainGranularRunListener.getTestRunForAttempts( 338 attemptNumber - 1), 339 dnae); 340 if (!retry) { 341 return; 342 } 343 } 344 firstCheck = false; 345 mCountRetryUsed++; 346 CLog.d("Intra-module retry attempt number %s", attemptNumber); 347 // Run the tests again 348 dnae = intraModuleRun(testInfo, allListeners, attemptNumber); 349 } 350 // Feed the last attempt if we reached here. 351 mRetryDecision.addLastAttempt( 352 mMainGranularRunListener.getTestRunForAttempts(mMaxRunLimit - 1)); 353 } finally { 354 mRetryStats = mRetryDecision.getRetryStatistics(); 355 // Track how long we spend in retry 356 mRetryStats.mRetryTime = System.currentTimeMillis() - startTime; 357 } 358 } 359 360 /** 361 * The workflow for each individual {@link IRemoteTest} run. 362 * 363 * @return DeviceNotAvailableException while DNAE happened, null otherwise. 364 */ intraModuleRun( TestInformation testInfo, ITestInvocationListener runListener, int attempt)365 private final DeviceNotAvailableException intraModuleRun( 366 TestInformation testInfo, ITestInvocationListener runListener, int attempt) { 367 DeviceNotAvailableException exception = null; 368 mMainGranularRunListener.setAttemptIsolation(CurrentInvocation.runCurrentIsolation()); 369 if (listenerWithModuleMetricsForwarder != null) { 370 listenerWithModuleMetricsForwarder.setAttemptIsolation( 371 CurrentInvocation.runCurrentIsolation()); 372 } 373 StartEndCollector startEndCollector = new StartEndCollector(runListener); 374 runListener = startEndCollector; 375 try (CloseableTraceScope ignored = 376 new CloseableTraceScope( 377 "attempt " + attempt + " " + mTest.getClass().getCanonicalName())) { 378 List<IMetricCollector> clonedCollectors = cloneCollectors(mRunMetricCollectors); 379 if (mTest instanceof IMetricCollectorReceiver) { 380 ((IMetricCollectorReceiver) mTest).setMetricCollectors(clonedCollectors); 381 // If test can receive collectors then let it handle how to set them up 382 mTest.run(testInfo, runListener); 383 } else { 384 if (mModuleConfiguration.getCommandOptions().reportTestCaseCount()) { 385 CountTestCasesCollector counter = new CountTestCasesCollector(mTest); 386 clonedCollectors.add(counter); 387 } 388 // Module only init the collectors here to avoid triggering the collectors when 389 // replaying the cached events at the end. This ensures metrics are capture at 390 // the proper time in the invocation. 391 for (IMetricCollector collector : clonedCollectors) { 392 if (collector.isDisabled()) { 393 CLog.d("%s has been disabled. Skipping.", collector); 394 } else { 395 try (CloseableTraceScope ignoreCollector = 396 new CloseableTraceScope( 397 "init_run_" + collector.getClass().getSimpleName())) { 398 if (collector instanceof IConfigurationReceiver) { 399 ((IConfigurationReceiver) collector) 400 .setConfiguration(mModuleConfiguration); 401 } 402 runListener = collector.init(mModuleInvocationContext, runListener); 403 } 404 } 405 } 406 mTest.run(testInfo, runListener); 407 } 408 } catch (RuntimeException | AssertionError re) { 409 CLog.e("Module '%s' - test '%s' threw exception:", mModuleId, mTest.getClass()); 410 CLog.e(re); 411 CLog.e("Proceeding to the next test."); 412 if (!startEndCollector.mRunStartReported) { 413 CLog.e("Event mismatch ! the test runner didn't report any testRunStart."); 414 runListener.testRunStarted(mModule.getId(), 0); 415 } 416 runListener.testRunFailed(createFromException(re)); 417 if (!startEndCollector.mRunEndedReported) { 418 CLog.e("Event mismatch ! the test runner didn't report any testRunEnded."); 419 runListener.testRunEnded(0L, new HashMap<String, Metric>()); 420 } 421 } catch (DeviceUnresponsiveException due) { 422 // being able to catch a DeviceUnresponsiveException here implies that recovery was 423 // successful, and test execution should proceed to next module. 424 CLog.w( 425 "Ignored DeviceUnresponsiveException because recovery was " 426 + "successful, proceeding with next module. Stack trace:"); 427 CLog.w(due); 428 CLog.w("Proceeding to the next test."); 429 // If it already was marked as failure do not remark it. 430 if (!mMainGranularRunListener.hasLastAttemptFailed()) { 431 runListener.testRunFailed(createFromException(due)); 432 } 433 } catch (DeviceNotAvailableException dnae) { 434 // TODO: See if it's possible to report IReportNotExecuted 435 CLog.e("Run in progress was not completed due to:"); 436 CLog.e(dnae); 437 // If it already was marked as failure do not remark it. 438 if (!mMainGranularRunListener.hasLastAttemptFailed()) { 439 runListener.testRunFailed(createFromException(dnae)); 440 } 441 exception = dnae; 442 } finally { 443 mRetryAttemptForwarder.incrementAttempt(); 444 // After one run, do not consider follow up isolated without action. 445 CurrentInvocation.setRunIsolation(IsolationGrade.NOT_ISOLATED); 446 } 447 return exception; 448 } 449 450 /** Get the merged TestRunResults from each {@link IRemoteTest} run. */ getFinalTestRunResults()451 public final List<TestRunResult> getFinalTestRunResults() { 452 MergeStrategy strategy = MergeStrategy.getMergeStrategy(mRetryDecision.getRetryStrategy()); 453 mMainGranularRunListener.setMergeStrategy(strategy); 454 return mMainGranularRunListener.getMergedTestRunResults(); 455 } 456 457 @VisibleForTesting getTestRunResultCollected()458 Map<String, List<TestRunResult>> getTestRunResultCollected() { 459 Map<String, List<TestRunResult>> runResultMap = new LinkedHashMap<>(); 460 for (String runName : mMainGranularRunListener.getTestRunNames()) { 461 runResultMap.put(runName, mMainGranularRunListener.getTestRunAttempts(runName)); 462 } 463 return runResultMap; 464 } 465 466 @VisibleForTesting cloneCollectors(List<IMetricCollector> originalCollectors)467 List<IMetricCollector> cloneCollectors(List<IMetricCollector> originalCollectors) { 468 return CollectorHelper.cloneCollectors(originalCollectors); 469 } 470 471 /** 472 * Calculate the number of testcases in the {@link IRemoteTest}. This value distincts the same 473 * testcases that are rescheduled multiple times. 474 */ getExpectedTestsCount()475 public final int getExpectedTestsCount() { 476 return mMainGranularRunListener.getExpectedTests(); 477 } 478 getPassedTests()479 public final Set<TestDescription> getPassedTests() { 480 Set<TestDescription> nonFailedTests = new LinkedHashSet<>(); 481 for (TestRunResult runResult : mMainGranularRunListener.getMergedTestRunResults()) { 482 nonFailedTests.addAll( 483 runResult.getTestsInState( 484 Arrays.asList( 485 TestStatus.PASSED, 486 TestStatus.IGNORED, 487 TestStatus.ASSUMPTION_FAILURE))); 488 } 489 return nonFailedTests; 490 } 491 492 /** Returns the listener containing all the results. */ getResultListener()493 public ModuleListener getResultListener() { 494 return mMainGranularRunListener; 495 } 496 getRetryCount()497 public int getRetryCount() { 498 return mCountRetryUsed; 499 } 500 501 @Override setCollectTestsOnly(boolean shouldCollectTest)502 public void setCollectTestsOnly(boolean shouldCollectTest) { 503 mCollectTestsOnly = shouldCollectTest; 504 } 505 createFromException(Throwable exception)506 private FailureDescription createFromException(Throwable exception) { 507 String message = 508 (exception.getMessage() == null) 509 ? String.format( 510 "No error message reported for: %s", 511 StreamUtil.getStackTrace(exception)) 512 : exception.getMessage(); 513 FailureDescription failure = 514 CurrentInvocation.createFailure(message, null).setCause(exception); 515 if (exception instanceof IHarnessException) { 516 ErrorIdentifier id = ((IHarnessException) exception).getErrorId(); 517 failure.setErrorIdentifier(id); 518 if (id != null) { 519 failure.setFailureStatus(id.status()); 520 } 521 failure.setOrigin(((IHarnessException) exception).getOrigin()); 522 } 523 return failure; 524 } 525 526 /** Class helper to catch missing run start and end. */ 527 public class StartEndCollector extends ResultAndLogForwarder { 528 529 public boolean mRunStartReported = false; 530 public boolean mRunEndedReported = false; 531 StartEndCollector(ITestInvocationListener listener)532 StartEndCollector(ITestInvocationListener listener) { 533 super(listener); 534 } 535 536 @Override testRunStarted(String runName, int testCount)537 public void testRunStarted(String runName, int testCount) { 538 super.testRunStarted(runName, testCount); 539 mRunStartReported = true; 540 } 541 542 @Override testRunStarted(String runName, int testCount, int attemptNumber)543 public void testRunStarted(String runName, int testCount, int attemptNumber) { 544 super.testRunStarted(runName, testCount, attemptNumber); 545 mRunStartReported = true; 546 } 547 548 @Override testRunStarted( String runName, int testCount, int attemptNumber, long startTime)549 public void testRunStarted( 550 String runName, int testCount, int attemptNumber, long startTime) { 551 super.testRunStarted(runName, testCount, attemptNumber, startTime); 552 mRunStartReported = true; 553 } 554 555 @Override testRunEnded(long elapsedTime, HashMap<String, Metric> runMetrics)556 public void testRunEnded(long elapsedTime, HashMap<String, Metric> runMetrics) { 557 super.testRunEnded(elapsedTime, runMetrics); 558 mRunEndedReported = true; 559 } 560 561 @Override testRunEnded(long elapsedTimeMillis, Map<String, String> runMetrics)562 public void testRunEnded(long elapsedTimeMillis, Map<String, String> runMetrics) { 563 super.testRunEnded(elapsedTimeMillis, runMetrics); 564 mRunEndedReported = true; 565 } 566 } 567 } 568