• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2016 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.testtype.suite;
17 
18 import com.android.tradefed.build.BuildRetrievalError;
19 import com.android.tradefed.build.IBuildInfo;
20 import com.android.tradefed.config.Configuration;
21 import com.android.tradefed.config.ConfigurationDescriptor;
22 import com.android.tradefed.config.ConfigurationException;
23 import com.android.tradefed.config.DeviceConfigurationHolder;
24 import com.android.tradefed.config.DynamicRemoteFileResolver;
25 import com.android.tradefed.config.IConfiguration;
26 import com.android.tradefed.config.IConfigurationReceiver;
27 import com.android.tradefed.config.IDeviceConfiguration;
28 import com.android.tradefed.dependencies.ExternalDependency;
29 import com.android.tradefed.dependencies.IExternalDependency;
30 import com.android.tradefed.device.DeviceNotAvailableException;
31 import com.android.tradefed.device.ITestDevice;
32 import com.android.tradefed.device.ITestDevice.RecoveryMode;
33 import com.android.tradefed.device.StubDevice;
34 import com.android.tradefed.device.connection.AdbTcpConnection;
35 import com.android.tradefed.device.metric.BugreportzOnTestCaseFailureCollector;
36 import com.android.tradefed.device.metric.IMetricCollector;
37 import com.android.tradefed.device.metric.LogcatOnFailureCollector;
38 import com.android.tradefed.device.metric.ScreenshotOnFailureCollector;
39 import com.android.tradefed.error.HarnessRuntimeException;
40 import com.android.tradefed.error.IHarnessException;
41 import com.android.tradefed.invoker.IInvocationContext;
42 import com.android.tradefed.invoker.InvocationContext;
43 import com.android.tradefed.invoker.TestInformation;
44 import com.android.tradefed.invoker.logger.CurrentInvocation;
45 import com.android.tradefed.invoker.logger.InvocationMetricLogger;
46 import com.android.tradefed.invoker.logger.InvocationMetricLogger.InvocationMetricKey;
47 import com.android.tradefed.invoker.logger.TfObjectTracker;
48 import com.android.tradefed.invoker.shard.token.TokenProperty;
49 import com.android.tradefed.invoker.tracing.CloseableTraceScope;
50 import com.android.tradefed.log.ILogRegistry.EventType;
51 import com.android.tradefed.log.ITestLogger;
52 import com.android.tradefed.log.Log.LogLevel;
53 import com.android.tradefed.log.LogRegistry;
54 import com.android.tradefed.log.LogUtil.CLog;
55 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
56 import com.android.tradefed.result.FailureDescription;
57 import com.android.tradefed.result.ILogSaver;
58 import com.android.tradefed.result.ILogSaverListener;
59 import com.android.tradefed.result.ITestInvocationListener;
60 import com.android.tradefed.result.ITestLoggerReceiver;
61 import com.android.tradefed.result.LogFile;
62 import com.android.tradefed.result.LogSaverResultForwarder;
63 import com.android.tradefed.result.MultiFailureDescription;
64 import com.android.tradefed.result.TestDescription;
65 import com.android.tradefed.result.TestResult;
66 import com.android.tradefed.result.TestRunResult;
67 import com.android.tradefed.result.error.DeviceErrorIdentifier;
68 import com.android.tradefed.result.error.ErrorIdentifier;
69 import com.android.tradefed.result.error.InfraErrorIdentifier;
70 import com.android.tradefed.result.error.TestErrorIdentifier;
71 import com.android.tradefed.result.proto.TestRecordProto.FailureStatus;
72 import com.android.tradefed.retry.IRetryDecision;
73 import com.android.tradefed.retry.RetryPreparationDecision;
74 import com.android.tradefed.retry.RetryStatistics;
75 import com.android.tradefed.retry.RetryStrategy;
76 import com.android.tradefed.suite.checker.ISystemStatusCheckerReceiver;
77 import com.android.tradefed.targetprep.BuildError;
78 import com.android.tradefed.targetprep.ITargetPreparer;
79 import com.android.tradefed.targetprep.TargetSetupError;
80 import com.android.tradefed.targetprep.multi.IMultiTargetPreparer;
81 import com.android.tradefed.testtype.IBuildReceiver;
82 import com.android.tradefed.testtype.IDeviceTest;
83 import com.android.tradefed.testtype.IInvocationContextReceiver;
84 import com.android.tradefed.testtype.IRemoteTest;
85 import com.android.tradefed.testtype.IRuntimeHintProvider;
86 import com.android.tradefed.testtype.ITestCollector;
87 import com.android.tradefed.testtype.ITestFileFilterReceiver;
88 import com.android.tradefed.testtype.ITestFilterReceiver;
89 import com.android.tradefed.testtype.suite.module.BaseModuleController;
90 import com.android.tradefed.testtype.suite.module.IModuleController.RunStrategy;
91 import com.android.tradefed.util.FileUtil;
92 import com.android.tradefed.util.MultiMap;
93 import com.android.tradefed.util.StreamUtil;
94 import com.android.tradefed.util.SystemUtil;
95 import com.android.tradefed.util.proto.TfMetricProtoUtil;
96 
97 import com.google.api.client.util.Joiner;
98 import com.google.common.annotations.VisibleForTesting;
99 
100 import java.io.File;
101 import java.io.IOException;
102 import java.util.ArrayList;
103 import java.util.Arrays;
104 import java.util.Collection;
105 import java.util.Collections;
106 import java.util.HashMap;
107 import java.util.HashSet;
108 import java.util.Iterator;
109 import java.util.LinkedHashSet;
110 import java.util.List;
111 import java.util.ListIterator;
112 import java.util.Map;
113 import java.util.Map.Entry;
114 import java.util.Set;
115 import java.util.stream.Collectors;
116 
117 /**
118  * Container for the test run configuration. This class is an helper to prepare and run the tests.
119  */
120 public class ModuleDefinition implements Comparable<ModuleDefinition>, ITestCollector {
121 
122     /** key names used for saving module info into {@link IInvocationContext} */
123     /**
124      * Module name is the base name associated with the module, usually coming from the Xml TF
125      * config file the module was loaded from.
126      */
127     public static final String MODULE_NAME = "module-name";
128     public static final String MODULE_ABI = "module-abi";
129     public static final String MODULE_PARAMETERIZATION = "module-param";
130     public static final String MODULE_EXTERNAL_DEPENDENCIES = "module-external-dependencies";
131     /**
132      * Module ID the name that will be used to identify uniquely the module during testRunStart. It
133      * will usually be a combination of MODULE_ABI + MODULE_NAME.
134      */
135     public static final String MODULE_ID = "module-id";
136     /** This property is set to true if the module was running on a freshly prepared device. */
137     public static final String MODULE_ISOLATED = "module-isolated";
138     /** This property is set to true if the test module results were cached. */
139     public static final String MODULE_CACHED = "module-cached";
140 
141     /** This property is set to true if the test module was skipped */
142     public static final String MODULE_SKIPPED = "module-skipped";
143 
144     /** This property is set to true if only module level events are reported. */
145     public static final String SPARSE_MODULE = "sparse-module";
146 
147     public static final String MODULE_CONTROLLER = "module_controller";
148 
149     public static final String PREPARATION_TIME = "PREP_TIME";
150     public static final String TEAR_DOWN_TIME = "TEARDOWN_TIME";
151     public static final String TEST_TIME = "TEST_TIME";
152     public static final String MODULE_TEST_COUNT = "MODULE_TEST_COUNT";
153     public static final String RETRY_TIME = "MODULE_RETRY_TIME";
154     public static final String ISOLATION_COST = "ISOLATION_COST";
155     public static final String RETRY_SUCCESS_COUNT = "MODULE_RETRY_SUCCESS";
156     public static final String RETRY_FAIL_COUNT = "MODULE_RETRY_FAILED";
157     public static final String MODULE_INVOCATION_ATTRIBUTE_FLAG_OVERRIDES_KEY =
158             "module-flag-overrides";
159 
160     private final IInvocationContext mModuleInvocationContext;
161     private final IConfiguration mModuleConfiguration;
162     private IConfiguration mInternalTestConfiguration;
163     private IConfiguration mInternalTargetPreparerConfiguration;
164     private ILogSaver mLogSaver;
165     private TestInformation mModuleInfo;
166     private ITestInvocationListener mInvocationListener;
167 
168     private final String mId;
169     private Collection<IRemoteTest> mTests = null;
170     private Integer mIntraModuleShardCount = null;
171     private Integer mIntraModuleShardIndex = null;
172 
173     private Map<String, List<ITargetPreparer>> mPreparersPerDevice = null;
174     private Map<String, List<ITargetPreparer>> mSuitePreparersPerDevice = null;
175 
176     private List<IMultiTargetPreparer> mMultiPreparers = new ArrayList<>();
177     private IBuildInfo mBuild;
178     private ITestDevice mDevice;
179     private List<IMetricCollector> mRunMetricCollectors = new ArrayList<>();
180     private boolean mCollectTestsOnly = false;
181 
182     private List<TestRunResult> mTestsResults = new ArrayList<>();
183     private List<ModuleListener> mRunListenersResults = new ArrayList<>();
184     private int mExpectedTests = 0;
185     private boolean mIsFailedModule = false;
186     private boolean mRetriedModulePreparationSuccess = false;
187 
188     // Tracking of preparers performance
189     private long mElapsedPreparation = 0L;
190     private long mElapsedTearDown = 0L;
191 
192     private long mStartTestTime = 0L;
193     private boolean mReportModuleStart = true;
194     private boolean mReportModuleEnd = true;
195     private boolean mFinalResultsReported = false;
196 
197     // Tracking of retry performance
198     private List<RetryStatistics> mRetryStats = new ArrayList<>();
199     private boolean mDisableAutoRetryTimeReporting = false;
200 
201     private boolean mMergeAttempts = true;
202     private IRetryDecision mRetryDecision;
203 
204     // Token during sharding
205     private Set<TokenProperty> mRequiredTokens = new HashSet<>();
206 
207     private boolean mEnableDynamicDownload = false;
208     private GranularRetriableTestWrapper mCurrentTestWrapper = null;
209     private int mMaxRetry = 1;
210     // TODO(b/226453043): Eventually we need to remove this.
211     private int mTargetPreparerRetryCount = 0;
212 
213     private Set<TestDescription> mPassThroughFilters = new LinkedHashSet<>();
214 
215     private boolean mRecoverVirtualDevice = false;
216 
217     private boolean mUseModuleResultsForwarder = false;
218 
219     @VisibleForTesting
ModuleDefinition()220     public ModuleDefinition() {
221         mModuleInvocationContext = null;
222         mModuleConfiguration = null;
223         mId = "";
224     }
225 
226     /**
227      * Constructor
228      *
229      * @param name unique name of the test configuration.
230      * @param tests list of {@link IRemoteTest} that needs to run.
231      * @param preparersPerDevice list of {@link ITargetPreparer} to be used to setup the device.
232      * @param moduleConfig the {@link IConfiguration} of the underlying module config.
233      */
ModuleDefinition( String name, Collection<IRemoteTest> tests, Map<String, List<ITargetPreparer>> preparersPerDevice, List<IMultiTargetPreparer> multiPreparers, IConfiguration moduleConfig)234     public ModuleDefinition(
235             String name,
236             Collection<IRemoteTest> tests,
237             Map<String, List<ITargetPreparer>> preparersPerDevice,
238             List<IMultiTargetPreparer> multiPreparers,
239             IConfiguration moduleConfig) {
240         this(name, tests, preparersPerDevice, null, multiPreparers, moduleConfig);
241     }
242 
243     /**
244      * Constructor
245      *
246      * @param name unique name of the test configuration.
247      * @param tests list of {@link IRemoteTest} that needs to run.
248      * @param preparersPerDevice list of {@link ITargetPreparer} to be used to setup the device.
249      * @param moduleConfig the {@link IConfiguration} of the underlying module config.
250      */
ModuleDefinition( String name, Collection<IRemoteTest> tests, Map<String, List<ITargetPreparer>> preparersPerDevice, Map<String, List<ITargetPreparer>> suitePreparersPerDevice, List<IMultiTargetPreparer> multiPreparers, IConfiguration moduleConfig)251     public ModuleDefinition(
252             String name,
253             Collection<IRemoteTest> tests,
254             Map<String, List<ITargetPreparer>> preparersPerDevice,
255             Map<String, List<ITargetPreparer>> suitePreparersPerDevice,
256             List<IMultiTargetPreparer> multiPreparers,
257             IConfiguration moduleConfig) {
258         mId = name;
259         mTests = tests;
260         mModuleConfiguration = moduleConfig;
261         ConfigurationDescriptor configDescriptor = moduleConfig.getConfigurationDescription();
262         mModuleInvocationContext = new InvocationContext();
263         mModuleInvocationContext.setConfigurationDescriptor(configDescriptor.clone());
264         // Copy the command options invocation attributes to the invocation context
265         mModuleInvocationContext.addInvocationAttributes(
266                 moduleConfig.getCommandOptions().getInvocationData());
267 
268         // If available in the suite, add the abi name
269         if (configDescriptor.getAbi() != null) {
270             mModuleInvocationContext.addInvocationAttribute(
271                     MODULE_ABI, configDescriptor.getAbi().getName());
272         }
273         if (configDescriptor.getModuleName() != null) {
274             mModuleInvocationContext.addInvocationAttribute(
275                     MODULE_NAME, configDescriptor.getModuleName());
276         }
277         String parameterization =
278                 configDescriptor
279                         .getAllMetaData()
280                         .getUniqueMap()
281                         .get(ConfigurationDescriptor.ACTIVE_PARAMETER_KEY);
282         if (parameterization != null) {
283             mModuleInvocationContext.addInvocationAttribute(
284                     MODULE_PARAMETERIZATION, parameterization);
285         }
286         // If there is no specific abi, module-id should be module-name
287         mModuleInvocationContext.addInvocationAttribute(MODULE_ID, mId);
288 
289         // Add External Dependencies of this module to the module context
290         Set<ExternalDependency> externalDependencies = new LinkedHashSet<>();
291         for (IDeviceConfiguration deviceConfig : moduleConfig.getDeviceConfig()) {
292             for (Object obj : deviceConfig.getAllObjects()) {
293                 if (obj instanceof IExternalDependency) {
294                     externalDependencies.addAll(((IExternalDependency) obj).getDependencies());
295                 }
296             }
297         }
298         if (!externalDependencies.isEmpty()) {
299             final List<String> dependencyClassNames =
300                     externalDependencies.stream()
301                             .map(dependency -> dependency.getClass().getName())
302                             .collect(Collectors.toList());
303             mModuleInvocationContext.addInvocationAttribute(
304                     MODULE_EXTERNAL_DEPENDENCIES, String.join(", ", dependencyClassNames));
305         }
306 
307         mMultiPreparers.addAll(multiPreparers);
308         mPreparersPerDevice = preparersPerDevice;
309         mSuitePreparersPerDevice = suitePreparersPerDevice;
310 
311         // Get the tokens of the module
312         List<String> tokens = configDescriptor.getMetaData(ITestSuite.TOKEN_KEY);
313         if (tokens != null) {
314             for (String token : tokens) {
315                 mRequiredTokens.add(TokenProperty.valueOf(token.toUpperCase()));
316             }
317         }
318     }
319 
setIntraModuleInformation(int shardCount, int shardIndex)320     public void setIntraModuleInformation(int shardCount, int shardIndex) {
321         mIntraModuleShardCount = shardCount;
322         mIntraModuleShardIndex = shardIndex;
323     }
324 
getIntraModuleShardCount()325     public Integer getIntraModuleShardCount() {
326         return mIntraModuleShardCount;
327     }
328 
getIntraModuleShardIndex()329     public Integer getIntraModuleShardIndex() {
330         return mIntraModuleShardIndex;
331     }
332 
333     /** Returns the number of devices expected to run this test. */
neededDevices()334     public int neededDevices() {
335         return mModuleConfiguration.getDeviceConfig().size();
336     }
337 
338     /**
339      * Returns the next {@link IRemoteTest} from the list of tests. The list of tests of a module
340      * may be shared with another one in case of sharding.
341      */
poll()342     IRemoteTest poll() {
343         synchronized (mTests) {
344             if (mTests.isEmpty()) {
345                 return null;
346             }
347             IRemoteTest test = mTests.iterator().next();
348             mTests.remove(test);
349             return test;
350         }
351     }
352 
353     /**
354      * Add some {@link IRemoteTest} to be executed as part of the module. Used when merging two
355      * modules.
356      */
addTests(List<IRemoteTest> test)357     void addTests(List<IRemoteTest> test) {
358         synchronized (mTests) {
359             mTests.addAll(test);
360         }
361     }
362 
363     /** Returns the current number of {@link IRemoteTest} waiting to be executed. */
numTests()364     public int numTests() {
365         synchronized (mTests) {
366             return mTests.size();
367         }
368     }
369 
370     /**
371      * Return True if the Module still has {@link IRemoteTest} to run in its pool. False otherwise.
372      */
hasTests()373     protected boolean hasTests() {
374         synchronized (mTests) {
375             return !mTests.isEmpty();
376         }
377     }
378 
379     /** Return the unique module name. */
getId()380     public String getId() {
381         return mId;
382     }
383 
384     /**
385      * {@inheritDoc}
386      */
387     @Override
compareTo(ModuleDefinition moduleDef)388     public int compareTo(ModuleDefinition moduleDef) {
389         return getId().compareTo(moduleDef.getId());
390     }
391 
392     /**
393      * Inject the {@link IBuildInfo} to be used during the tests.
394      */
setBuild(IBuildInfo build)395     public void setBuild(IBuildInfo build) {
396         mBuild = build;
397     }
398 
399     /**
400      * Inject the {@link ITestDevice} to be used during the tests.
401      */
setDevice(ITestDevice device)402     public void setDevice(ITestDevice device) {
403         mDevice = device;
404     }
405 
406     /** Inject the List of {@link IMetricCollector} to be used by the module. */
setMetricCollectors(List<IMetricCollector> collectors)407     public void setMetricCollectors(List<IMetricCollector> collectors) {
408         if (collectors == null) {
409             return;
410         }
411         mRunMetricCollectors.addAll(collectors);
412     }
413 
414     /** Pass the invocation log saver to the module so it can use it if necessary. */
setLogSaver(ILogSaver logSaver)415     public void setLogSaver(ILogSaver logSaver) {
416         mLogSaver = logSaver;
417     }
418 
419     /**
420      * Run all the {@link IRemoteTest} contained in the module and use all the preparers before and
421      * after to setup and clean the device.
422      *
423      * @param listener the {@link ITestInvocationListener} where to report results.
424      * @throws DeviceNotAvailableException in case of device going offline.
425      */
run(TestInformation moduleInfo, ITestInvocationListener listener)426     public final void run(TestInformation moduleInfo, ITestInvocationListener listener)
427             throws DeviceNotAvailableException {
428         run(moduleInfo, listener, 1);
429     }
430 
431     /**
432      * Run all the {@link IRemoteTest} contained in the module and use all the preparers before and
433      * after to setup and clean the device.
434      *
435      * @param moduleInfo the {@link TestInformation} for the module.
436      * @param listener the {@link ITestInvocationListener} where to report results.
437      * @param maxRunLimit the max number of runs for each testcase.
438      * @throws DeviceNotAvailableException in case of device going offline.
439      */
run( TestInformation moduleInfo, ITestInvocationListener listener, int maxRunLimit)440     public final void run(
441             TestInformation moduleInfo,
442             ITestInvocationListener listener,
443             int maxRunLimit)
444             throws DeviceNotAvailableException {
445         mMaxRetry = maxRunLimit;
446         mModuleInfo = moduleInfo;
447         // until RetryLogSaverResultForwarder is created, wrap a temp log saver if needed.
448         if (!(listener instanceof LogSaverResultForwarder)) {
449             mInvocationListener =
450                     new LogSaverResultForwarder(
451                             mLogSaver, Arrays.asList(listener), mModuleConfiguration, false);
452         } else {
453             mInvocationListener = listener;
454         }
455 
456         // Load extra configuration for the module from module_controller
457         // TODO: make module_controller a full TF object
458         boolean skipTestCases = false;
459         RunStrategy rs = applyConfigurationControl();
460         if (RunStrategy.FULL_MODULE_BYPASS.equals(rs)) {
461             CLog.d("module_controller applied and module %s should not run.", getId());
462             return;
463         } else if (RunStrategy.SKIP_MODULE_TESTCASES.equals(rs)) {
464             CLog.d("All tests cases for %s will be marked skipped.", getId());
465             skipTestCases = true;
466         }
467 
468         CLog.logAndDisplay(LogLevel.DEBUG, "Running module %s", getId());
469         // set the module context so it's available widely during the module run period.
470         CurrentInvocation.setModuleContext(mModuleInvocationContext);
471         // Exception generated during setUp or run of the tests
472         Throwable preparationException;
473         DeviceNotAvailableException runException = null;
474         long start = System.currentTimeMillis();
475         // Resolve dynamic files except for the IRemoteTest ones
476         try (CloseableTraceScope ignored = new CloseableTraceScope("download_files")) {
477             preparationException =
478                     invokeRemoteDynamic(moduleInfo.getDevice(), mModuleConfiguration);
479 
480             if (preparationException == null) {
481                 mInternalTargetPreparerConfiguration =
482                         new Configuration("tmp-download", "tmp-download");
483                 mInternalTargetPreparerConfiguration
484                         .getCommandOptions()
485                         .getDynamicDownloadArgs()
486                         .putAll(mModuleConfiguration.getCommandOptions().getDynamicDownloadArgs());
487                 for (String device : mPreparersPerDevice.keySet()) {
488                     mInternalTargetPreparerConfiguration.setDeviceConfig(
489                             new DeviceConfigurationHolder(device));
490                     for (ITargetPreparer preparer : mPreparersPerDevice.get(device)) {
491                         try {
492                             mInternalTargetPreparerConfiguration
493                                     .getDeviceConfigByName(device)
494                                     .addSpecificConfig(preparer);
495                         } catch (ConfigurationException e) {
496                             // unset the module context since module run is ending.
497                             CurrentInvocation.setModuleContext(null);
498                             // Shouldn't happen;
499                             throw new RuntimeException(e);
500                         }
501                     }
502                 }
503                 mInternalTargetPreparerConfiguration.setMultiTargetPreparers(mMultiPreparers);
504                 preparationException =
505                         invokeRemoteDynamic(
506                                 moduleInfo.getDevice(), mInternalTargetPreparerConfiguration);
507             }
508         }
509         // Setup
510         if (preparationException == null) {
511             try (CloseableTraceScope ignored = new CloseableTraceScope("module_preparation")) {
512                 preparationException = runPreparation(false);
513             }
514         }
515 
516         while (preparationException != null) {
517             RetryPreparationDecision retryDecision =
518                     mRetryDecision.shouldRetryPreparation(
519                             this,
520                             mTargetPreparerRetryCount,
521                             maxRunLimit);
522             boolean shouldFailRun = retryDecision.shouldFailRun();
523             reportSetupFailure(
524                     preparationException,
525                     mInvocationListener,
526                     mTargetPreparerRetryCount,
527                     shouldFailRun);
528             if (shouldFailRun) {
529                 // unset the module context since module run is ending.
530                 CurrentInvocation.setModuleContext(null);
531                 return;
532             }
533             mTargetPreparerRetryCount++;
534             if (!retryDecision.shouldRetry()) {
535                 CLog.i("Retry target preparers for module: %s successfully at the %s retry",
536                         this.getId(), mTargetPreparerRetryCount);
537                 // This flag is set true for any target preparation error, set it false when the
538                 // retrying succeed.
539                 mIsFailedModule = false;
540                 mRetriedModulePreparationSuccess = true;
541                 InvocationMetricLogger.addInvocationMetrics(
542                         InvocationMetricKey.DEVICE_RESET_MODULES_FOR_TARGET_PREPARER, this.getId());
543             }
544             preparationException = retryDecision.getPreviousException();
545         }
546 
547         // Total count of retried (preparation + test) shouldn't exceed maxRunLimit and mMaxRetry.
548         maxRunLimit = maxRunLimit - mTargetPreparerRetryCount;
549         mMaxRetry = mMaxRetry - mTargetPreparerRetryCount;
550         InvocationMetricLogger.addInvocationPairMetrics(
551                 InvocationMetricKey.MODULE_SETUP_PAIR, start, System.currentTimeMillis());
552 
553         // Run the tests
554         try {
555             mStartTestTime = getCurrentTime();
556             int perModuleRetryQuota = mMaxRetry;
557             while (true) {
558                 IRemoteTest test = poll();
559                 if (test == null) {
560                     return;
561                 }
562                 TfObjectTracker.countWithParents(test.getClass());
563                 if (test instanceof IBuildReceiver) {
564                     ((IBuildReceiver) test).setBuild(mBuild);
565                 }
566                 if (test instanceof IDeviceTest) {
567                     ((IDeviceTest) test).setDevice(mDevice);
568                 }
569                 if (test instanceof IInvocationContextReceiver) {
570                     ((IInvocationContextReceiver) test)
571                             .setInvocationContext(mModuleInvocationContext);
572                 }
573                 mInternalTestConfiguration = new Configuration("tmp-download", "tmp-download");
574                 mInternalTestConfiguration
575                         .getCommandOptions()
576                         .getDynamicDownloadArgs()
577                         .putAll(mModuleConfiguration.getCommandOptions().getDynamicDownloadArgs());
578                 // We do it before the official set, otherwise the IConfiguration will not be the
579                 // right one.
580                 mInternalTestConfiguration.setTest(test);
581                 if (test instanceof IConfigurationReceiver) {
582                     ((IConfigurationReceiver) test).setConfiguration(mModuleConfiguration);
583                 }
584                 if (mDevice instanceof IConfigurationReceiver) {
585                     ((IConfigurationReceiver) mDevice).setConfiguration(mModuleConfiguration);
586                 }
587                 if (test instanceof ISystemStatusCheckerReceiver) {
588                     // We do not pass down Status checker because they are already running at the
589                     // top level suite.
590                     ((ISystemStatusCheckerReceiver) test).setSystemStatusChecker(new ArrayList<>());
591                 }
592                 if (test instanceof ITestCollector) {
593                     if (skipTestCases) {
594                         mCollectTestsOnly = true;
595                     }
596                     ((ITestCollector) test).setCollectTestsOnly(mCollectTestsOnly);
597                 }
598                 if (!mPassThroughFilters.isEmpty()) {
599                     applyFilterToTest(test, mPassThroughFilters);
600                 }
601                 mCurrentTestWrapper =
602                         prepareGranularRetriableWrapper(
603                                 test,
604                                 listener,
605                                 skipTestCases,
606                                 perModuleRetryQuota);
607                 mCurrentTestWrapper.setCollectTestsOnly(mCollectTestsOnly);
608                 // Resolve the dynamic options for that one test.
609                 preparationException =
610                         invokeRemoteDynamic(moduleInfo.getDevice(), mInternalTestConfiguration);
611                 if (preparationException != null) {
612                     reportSetupFailure(
613                             preparationException,
614                             mInvocationListener,
615                             mTargetPreparerRetryCount,
616                             true);
617                     return;
618                 }
619                 try (CloseableTraceScope ignored = new CloseableTraceScope("module_test")) {
620                     mCurrentTestWrapper.run(moduleInfo, listener);
621                 } catch (DeviceNotAvailableException dnae) {
622                     runException = dnae;
623                     // We do special logging of some information in Context of the module for easier
624                     // debugging.
625                     CLog.e(
626                             "Module %s threw a DeviceNotAvailableException on device %s during "
627                                     + "test %s",
628                             getId(), mDevice.getSerialNumber(), test.getClass());
629                     CLog.e(dnae);
630                     // log an events
631                     logDeviceEvent(
632                             EventType.MODULE_DEVICE_NOT_AVAILABLE,
633                             mDevice.getSerialNumber(),
634                             dnae,
635                             getId());
636                     throw dnae;
637                 } finally {
638                     mInternalTestConfiguration.cleanConfigurationData();
639                     mInternalTestConfiguration = null;
640                     if (mMergeAttempts) {
641                         // A single module can generate several test runs
642                         mTestsResults.addAll(mCurrentTestWrapper.getFinalTestRunResults());
643                     } else {
644                         // Keep track of each listener for attempts
645                         mRunListenersResults.add(mCurrentTestWrapper.getResultListener());
646                     }
647                     if (mModuleConfiguration
648                             .getConfigurationDescription()
649                             .isNotIRemoteTestShardable()) {
650                         mPassThroughFilters.addAll(mCurrentTestWrapper.getPassedTests());
651                     }
652                     // Limit escalating retries across all sub-IRemoteTests
653                     if (mRetryDecision != null
654                             && RetryStrategy.RETRY_ANY_FAILURE.equals(
655                                     mRetryDecision.getRetryStrategy())) {
656                         perModuleRetryQuota -= mCurrentTestWrapper.getRetryCount();
657                     }
658 
659                     mExpectedTests += mCurrentTestWrapper.getExpectedTestsCount();
660                     // Get information about retry
661                     if (mRetryDecision != null) {
662                         RetryStatistics res = mRetryDecision.getRetryStatistics();
663                         if (res != null) {
664                             addRetryTime(res.mRetryTime);
665                             mRetryStats.add(res);
666                         }
667                     }
668                 }
669                 // After the run, if the test failed (even after retry the final result passed) has
670                 // failed, capture a bugreport.
671                 if (mCurrentTestWrapper.getResultListener().hasLastAttemptFailed()) {
672                     captureBugreport(
673                             mInvocationListener,
674                             getId(),
675                             mCurrentTestWrapper
676                                     .getResultListener()
677                                     .getCurrentRunResults()
678                                     .getRunFailureDescription());
679                 }
680             }
681         } finally {
682             mPassThroughFilters.clear();
683             // Clean target preparers dynamic files.
684             if (mInternalTargetPreparerConfiguration != null) {
685                 mInternalTargetPreparerConfiguration.cleanConfigurationData();
686                 mInternalTargetPreparerConfiguration = null;
687             }
688             long cleanStartTime = getCurrentTime();
689             RuntimeException tearDownException = null;
690             try (CloseableTraceScope ignored = new CloseableTraceScope("module_teardown")) {
691                 Throwable exception = (runException != null) ? runException : preparationException;
692                 try {
693                     // Tear down
694                     runTearDown(moduleInfo, exception);
695                 } catch (DeviceNotAvailableException dnae) {
696                     if (runException == null) {
697                         // Ignore the exception and attempt recovery.
698                         CLog.e(
699                                 "Module %s failed during tearDown with: %s",
700                                 getId(), StreamUtil.getStackTrace(dnae));
701                         recoverDevice(moduleInfo, dnae);
702                     } else {
703                         throw dnae;
704                     }
705                 }
706                 // If still available, verify that device didn't crash
707                 if (runException == null) {
708                     checkEndModuleDevice(moduleInfo);
709                 }
710             } catch (DeviceNotAvailableException dnae) {
711                 CLog.e(
712                         "Module %s failed during tearDown with: %s",
713                         getId(), StreamUtil.getStackTrace(dnae));
714                 throw dnae;
715             } catch (RuntimeException e) {
716                 CLog.e("Exception while running tearDown:");
717                 CLog.e(e);
718                 tearDownException = e;
719             } finally {
720                 InvocationMetricLogger
721                         .addInvocationPairMetrics(InvocationMetricKey.MODULE_TEARDOWN_PAIR,
722                                 cleanStartTime, getCurrentTime());
723                 mElapsedTearDown = getCurrentTime() - cleanStartTime;
724                 // finalize results
725                 if (preparationException == null) {
726                     mModuleConfiguration.cleanConfigurationData();
727                     if (mMergeAttempts) {
728                         reportFinalResults(
729                                 mInvocationListener,
730                                 mExpectedTests,
731                                 mTestsResults,
732                                 null,
733                                 tearDownException);
734                         mTestsResults.clear();
735                         mExpectedTests = 0;
736                     } else {
737                         boolean reported = false;
738                         // Push the attempts one by one
739                         for (int i = 0; i < maxRunLimit; i++) {
740                             // Get all the results for the attempt
741                             List<TestRunResult> runResultList = new ArrayList<TestRunResult>();
742                             int expectedCount = 0;
743                             for (ModuleListener attemptListener : mRunListenersResults) {
744                                 for (String runName : attemptListener.getTestRunNames()) {
745                                     TestRunResult run =
746                                             attemptListener.getTestRunAtAttempt(runName, i);
747                                     if (run != null) {
748                                         runResultList.add(run);
749                                         expectedCount += run.getExpectedTestCount();
750                                     }
751                                 }
752                             }
753 
754                             if (!runResultList.isEmpty() || (
755                                 !reported && mRetriedModulePreparationSuccess)) {
756                                 if (runResultList.isEmpty()) {
757                                     reported = true;
758                                     CLog.i("Module preparation retry pass but no test cases were " +
759                                             "executed. Keep reporting the result to notify it " +
760                                             "failed in the 1st run but passed after retrying.");
761                                 }
762                                 reportFinalResults(
763                                         mInvocationListener,
764                                         expectedCount,
765                                         runResultList,
766                                         i,
767                                         tearDownException);
768                             } else {
769                                 CLog.d("No results to be forwarded for attempt %s.", i);
770                             }
771                         }
772                     }
773                 }
774                 // unset the module context since module run is ending.
775                 CurrentInvocation.setModuleContext(null);
776             }
777         }
778     }
779 
780     /**
781      * Create a wrapper class for the {@link IRemoteTest} which has built-in logic to schedule
782      * multiple test runs for the same module, and have the ability to run testcases at a more
783      * granular level (a subset of testcases in the module).
784      *
785      * @param test the {@link IRemoteTest} that is being wrapped.
786      * @param failureListener a particular listener to collect logs on testFail. Can be null.
787      * @param skipTestCases A run strategy when SKIP_MODULE_TESTCASES is defined.
788      * @param maxRunLimit a rate-limiter on testcases retrying times.
789      */
790     @VisibleForTesting
prepareGranularRetriableWrapper( IRemoteTest test, ITestInvocationListener listener, boolean skipTestCases, int maxRunLimit)791     GranularRetriableTestWrapper prepareGranularRetriableWrapper(
792             IRemoteTest test,
793             ITestInvocationListener listener,
794             boolean skipTestCases,
795             int maxRunLimit) {
796         GranularRetriableTestWrapper retriableTest =
797                 new GranularRetriableTestWrapper(
798                         test, this, listener, maxRunLimit, mUseModuleResultsForwarder);
799         retriableTest.setModuleId(getId());
800         retriableTest.setMarkTestsSkipped(skipTestCases);
801         retriableTest.setMetricCollectors(mRunMetricCollectors);
802         retriableTest.setModuleConfig(mModuleConfiguration);
803         retriableTest.setInvocationContext(mModuleInvocationContext);
804         retriableTest.setLogSaver(mLogSaver);
805         retriableTest.setRetryDecision(mRetryDecision);
806         return retriableTest;
807     }
808 
captureBugreport( ITestLogger listener, String moduleId, FailureDescription failure)809     private void captureBugreport(
810             ITestLogger listener, String moduleId, FailureDescription failure) {
811         FailureStatus status = failure.getFailureStatus();
812         if (!FailureStatus.LOST_SYSTEM_UNDER_TEST.equals(status)
813                 && !FailureStatus.SYSTEM_UNDER_TEST_CRASHED.equals(status)) {
814             return;
815         }
816         for (ITestDevice device : mModuleInvocationContext.getDevices()) {
817             if (device.getIDevice() instanceof StubDevice) {
818                 continue;
819             }
820             device.logBugreport(
821                     String.format(
822                             "module-%s-failure-%s-bugreport", moduleId, device.getSerialNumber()),
823                     listener);
824         }
825     }
826 
827     /** Helper to log the device events. */
logDeviceEvent(EventType event, String serial, Throwable t, String moduleId)828     private void logDeviceEvent(EventType event, String serial, Throwable t, String moduleId) {
829         Map<String, String> args = new HashMap<>();
830         args.put("serial", serial);
831         args.put("trace", StreamUtil.getStackTrace(t));
832         args.put("module-id", moduleId);
833         LogRegistry.getLogRegistry().logEvent(LogLevel.DEBUG, event, args);
834     }
835 
836     /** Finalize results to report them all and count if there are missing tests. */
reportFinalResults( ITestInvocationListener listener, int totalExpectedTests, List<TestRunResult> listResults, Integer attempt, RuntimeException tearDownException)837     private void reportFinalResults(
838             ITestInvocationListener listener,
839             int totalExpectedTests,
840             List<TestRunResult> listResults,
841             Integer attempt,
842             RuntimeException tearDownException) {
843         long elapsedTime = 0L;
844         HashMap<String, Metric> metricsProto = new HashMap<>();
845         if (attempt != null) {
846             long startTime =
847                     listResults.isEmpty() ? mStartTestTime : listResults.get(0).getStartTime();
848             listener.testRunStarted(
849                     getId(), totalExpectedTests, attempt + mTargetPreparerRetryCount, startTime);
850         } else {
851             listener.testRunStarted(
852                     getId(), totalExpectedTests, mTargetPreparerRetryCount, mStartTestTime);
853         }
854         int numResults = 0;
855         MultiMap<String, LogFile> aggLogFiles = new MultiMap<>();
856         List<FailureDescription> runFailureMessages = new ArrayList<>();
857         for (TestRunResult runResult : listResults) {
858             numResults += runResult.getTestResults().size();
859             forwardTestResults(runResult.getTestResults(), listener);
860             if (runResult.isRunFailure()) {
861                 runFailureMessages.add(runResult.getRunFailureDescription());
862             }
863             elapsedTime += runResult.getElapsedTime();
864             // put metrics from the tests
865             metricsProto.putAll(runResult.getRunProtoMetrics());
866             aggLogFiles.putAll(runResult.getRunLoggedFiles());
867         }
868         // put metrics from the preparation
869         metricsProto.put(
870                 PREPARATION_TIME,
871                 TfMetricProtoUtil.createSingleValue(mElapsedPreparation, "milliseconds"));
872         metricsProto.put(
873                 TEAR_DOWN_TIME,
874                 TfMetricProtoUtil.createSingleValue(mElapsedTearDown, "milliseconds"));
875         metricsProto.put(
876                 TEST_TIME, TfMetricProtoUtil.createSingleValue(elapsedTime, "milliseconds"));
877         metricsProto.put(MODULE_TEST_COUNT, TfMetricProtoUtil.createSingleValue(numResults, "int"));
878         // Report all the retry informations
879         if (!mRetryStats.isEmpty()) {
880             if (attempt != null) {
881                 long cost = RetryStatistics.isolationCostPerAttempt(attempt, mRetryStats);
882                 if (cost != 0L) {
883                     metricsProto.put(
884                             ISOLATION_COST,
885                             TfMetricProtoUtil.createSingleValue(cost, "milliseconds"));
886                 }
887             } else {
888                 RetryStatistics agg = RetryStatistics.aggregateStatistics(mRetryStats);
889                 metricsProto.put(
890                         RETRY_TIME,
891                         TfMetricProtoUtil.createSingleValue(agg.mRetryTime, "milliseconds"));
892                 metricsProto.put(
893                         RETRY_SUCCESS_COUNT,
894                         TfMetricProtoUtil.createSingleValue(agg.mRetrySuccess, ""));
895                 metricsProto.put(
896                         RETRY_FAIL_COUNT,
897                         TfMetricProtoUtil.createSingleValue(agg.mRetryFailure, ""));
898             }
899         }
900 
901         // Only report the mismatch if there were no error during the run.
902         if (runFailureMessages.isEmpty() && totalExpectedTests != numResults) {
903             String error =
904                     String.format(
905                             "Module %s only ran %d out of %d expected tests.",
906                             getId(), numResults, totalExpectedTests);
907             FailureDescription mismatch =
908                     FailureDescription.create(error)
909                             .setFailureStatus(FailureStatus.TEST_FAILURE)
910                             .setErrorIdentifier(InfraErrorIdentifier.EXPECTED_TESTS_MISMATCH);
911             runFailureMessages.add(mismatch);
912             CLog.e(error);
913         }
914 
915         if (tearDownException != null) {
916             FailureDescription failure =
917                     CurrentInvocation.createFailure(
918                                     StreamUtil.getStackTrace(tearDownException), null)
919                             .setCause(tearDownException);
920             runFailureMessages.add(failure);
921         }
922         // If there is any errors report them all at once
923         if (!runFailureMessages.isEmpty()) {
924             if (runFailureMessages.size() == 1) {
925                 listener.testRunFailed(runFailureMessages.get(0));
926             } else {
927                 listener.testRunFailed(new MultiFailureDescription(runFailureMessages));
928             }
929             mIsFailedModule = true;
930         }
931 
932         // Provide a strong association of the run to its logs.
933         for (String key : aggLogFiles.keySet()) {
934             for (LogFile logFile : aggLogFiles.get(key)) {
935                 if (listener instanceof ILogSaverListener) {
936                     ((ILogSaverListener) listener).logAssociation(key, logFile);
937                 }
938             }
939         }
940         // Allow each attempt to have its own start/end time
941         if (attempt != null) {
942             listener.testRunEnded(elapsedTime, metricsProto);
943         } else {
944             listener.testRunEnded(getCurrentTime() - mStartTestTime, metricsProto);
945         }
946         mFinalResultsReported = true;
947     }
948 
forwardTestResults( Map<TestDescription, TestResult> testResults, ITestInvocationListener listener)949     private void forwardTestResults(
950             Map<TestDescription, TestResult> testResults, ITestInvocationListener listener) {
951         Iterator<Map.Entry<TestDescription, TestResult>> iterator =
952                 testResults.entrySet().iterator();
953         while (iterator.hasNext()) {
954             Map.Entry<TestDescription, TestResult> testEntry = iterator.next();
955             listener.testStarted(testEntry.getKey(), testEntry.getValue().getStartTime());
956             switch (testEntry.getValue().getResultStatus()) {
957                 case FAILURE:
958                     listener.testFailed(testEntry.getKey(), testEntry.getValue().getFailure());
959                     break;
960                 case ASSUMPTION_FAILURE:
961                     listener.testAssumptionFailure(
962                             testEntry.getKey(), testEntry.getValue().getFailure());
963                     break;
964                 case IGNORED:
965                     listener.testIgnored(testEntry.getKey());
966                     break;
967                 case SKIPPED:
968                     listener.testSkipped(testEntry.getKey(), testEntry.getValue().getSkipReason());
969                     break;
970                 case INCOMPLETE:
971                     listener.testFailed(
972                             testEntry.getKey(),
973                             FailureDescription.create(
974                                     "Test did not complete due to exception.",
975                                     FailureStatus.TEST_FAILURE));
976                     break;
977                 default:
978                     break;
979             }
980             // Provide a strong association of the test to its logs.
981             for (Entry<String, LogFile> logFile :
982                     testEntry.getValue().getLoggedFiles().entrySet()) {
983                 if (listener instanceof ILogSaverListener) {
984                     ((ILogSaverListener) listener)
985                             .logAssociation(logFile.getKey(), logFile.getValue());
986                 }
987             }
988             listener.testEnded(
989                     testEntry.getKey(),
990                     testEntry.getValue().getEndTime(),
991                     testEntry.getValue().getProtoMetrics());
992             // Remove the test result from the map to release memory.
993             iterator.remove();
994         }
995     }
996 
997     /**
998      * Run preparers of the test, including suite level preparers if specified.
999      *
1000      * @param includeSuitePreparers Set to {@code true} to also run suite level preparers.
1001      * @return {@link Throwable} of any exception raised when running preparers.
1002      */
runPreparation(boolean includeSuitePreparers)1003     public Throwable runPreparation(boolean includeSuitePreparers) {
1004         Throwable preparationException = null;
1005         long prepStartTime = getCurrentTime();
1006         if (includeSuitePreparers) {
1007             // Run suite level preparers.
1008             preparationException = runTargetPreparation(mSuitePreparersPerDevice);
1009         }
1010 
1011         if (preparationException == null) {
1012             preparationException = runTargetPreparation(mPreparersPerDevice);
1013         }
1014         // Skip multi-preparation if preparation already failed.
1015         if (preparationException == null) {
1016             for (IMultiTargetPreparer multiPreparer : mMultiPreparers) {
1017                 preparationException = runMultiPreparerSetup(multiPreparer);
1018                 if (preparationException != null) {
1019                     mIsFailedModule = true;
1020                     CLog.e("Some preparation step failed. failing the module %s", getId());
1021                     break;
1022                 }
1023             }
1024         }
1025         mElapsedPreparation = getCurrentTime() - prepStartTime;
1026         return preparationException;
1027     }
1028 
1029     /** Run all the prepare steps. */
runPreparerSetup( ITargetPreparer preparer, int deviceIndex)1030     private Throwable runPreparerSetup(
1031             ITargetPreparer preparer,
1032             int deviceIndex) {
1033         if (preparer.isDisabled()) {
1034             // If disabled skip completely.
1035             return null;
1036         }
1037         TfObjectTracker.countWithParents(preparer.getClass());
1038         CLog.d("Running setup preparer: %s", preparer.getClass().getSimpleName());
1039         try (CloseableTraceScope ignored =
1040                 new CloseableTraceScope(preparer.getClass().getName())) {
1041             if (preparer instanceof IConfigurationReceiver) {
1042                 ((IConfigurationReceiver) preparer).setConfiguration(mModuleConfiguration);
1043             }
1044             // set the logger in case they need it.
1045             if (preparer instanceof ITestLoggerReceiver) {
1046                 ((ITestLoggerReceiver) preparer).setTestLogger(mInvocationListener);
1047             }
1048             if (preparer instanceof IInvocationContextReceiver) {
1049                 ((IInvocationContextReceiver) preparer)
1050                         .setInvocationContext(mModuleInvocationContext);
1051             }
1052             mModuleInfo.setActiveDeviceIndex(deviceIndex);
1053             preparer.setUp(mModuleInfo);
1054             return null;
1055         } catch (BuildError
1056                 | TargetSetupError
1057                 | DeviceNotAvailableException
1058                 | RuntimeException
1059                 | AssertionError
1060                 | LinkageError e) {
1061             // We catch all the TargetPreparer possible exception + RuntimeException to avoid
1062             // specific issues + AssertionError since it's widely used in tests and doesn't notify
1063             // something very wrong with the harness.
1064             CLog.e("Unexpected Exception from preparer: %s", preparer.getClass().getName());
1065             CLog.e(e);
1066             return e;
1067         } finally {
1068             mModuleInfo.setActiveDeviceIndex(0);
1069         }
1070     }
1071 
1072     /** Run all multi target preparer step. */
runMultiPreparerSetup(IMultiTargetPreparer preparer)1073     private Throwable runMultiPreparerSetup(IMultiTargetPreparer preparer) {
1074         if (preparer.isDisabled()) {
1075             // If disabled skip completely.
1076             return null;
1077         }
1078         TfObjectTracker.countWithParents(preparer.getClass());
1079         CLog.d("Running setup multi preparer: %s", preparer.getClass().getSimpleName());
1080         try (CloseableTraceScope ignored =
1081                 new CloseableTraceScope(preparer.getClass().getName())) {
1082             if (preparer instanceof IConfigurationReceiver) {
1083                 ((IConfigurationReceiver) preparer).setConfiguration(mModuleConfiguration);
1084             }
1085             // set the logger in case they need it.
1086             if (preparer instanceof ITestLoggerReceiver) {
1087                 ((ITestLoggerReceiver) preparer).setTestLogger(mInvocationListener);
1088             }
1089             if (preparer instanceof IInvocationContextReceiver) {
1090                 ((IInvocationContextReceiver) preparer)
1091                         .setInvocationContext(mModuleInvocationContext);
1092             }
1093             preparer.setUp(mModuleInfo);
1094             return null;
1095         } catch (BuildError
1096                 | TargetSetupError
1097                 | DeviceNotAvailableException
1098                 | RuntimeException
1099                 | AssertionError
1100                 | LinkageError e) {
1101             // We catch all the MultiTargetPreparer possible exception + RuntimeException to avoid
1102             // specific issues + AssertionError since it's widely used in tests and doesn't notify
1103             // something very wrong with the harness.
1104             CLog.e("Unexpected Exception from preparer: %s", preparer.getClass().getName());
1105             CLog.e(e);
1106             return e;
1107         }
1108     }
1109 
1110     /** Run all the tear down steps from preparers. */
runTearDown(TestInformation moduleInfo, Throwable exception)1111     private void runTearDown(TestInformation moduleInfo, Throwable exception)
1112             throws DeviceNotAvailableException {
1113         // Tear down
1114         List<IMultiTargetPreparer> cleanerList = new ArrayList<>(mMultiPreparers);
1115         Collections.reverse(cleanerList);
1116         for (IMultiTargetPreparer multiCleaner : cleanerList) {
1117             if (multiCleaner.isDisabled() || multiCleaner.isTearDownDisabled()) {
1118                 // If disabled skip completely.
1119                 continue;
1120             }
1121             CLog.d("Running teardown multi cleaner: %s", multiCleaner.getClass().getSimpleName());
1122             multiCleaner.tearDown(moduleInfo, exception);
1123         }
1124 
1125         for (int i = 0; i < mModuleInvocationContext.getDeviceConfigNames().size(); i++) {
1126             String deviceName = mModuleInvocationContext.getDeviceConfigNames().get(i);
1127             ITestDevice device = mModuleInvocationContext.getDevice(deviceName);
1128             if (i >= mPreparersPerDevice.size()) {
1129                 CLog.d(
1130                         "Main configuration has more devices than the module configuration. '%s' "
1131                                 + "will not run any tear down.",
1132                         deviceName);
1133                 continue;
1134             }
1135             List<ITargetPreparer> preparers = mPreparersPerDevice.get(deviceName);
1136             if (preparers == null) {
1137                 CLog.w(
1138                         "Module configuration devices mismatch the main configuration "
1139                                 + "(Missing device '%s'), resolving preparers by index.",
1140                         deviceName);
1141                 String key = new ArrayList<>(mPreparersPerDevice.keySet()).get(i);
1142                 preparers = mPreparersPerDevice.get(key);
1143             }
1144             ListIterator<ITargetPreparer> itr = preparers.listIterator(preparers.size());
1145             while (itr.hasPrevious()) {
1146                 ITargetPreparer preparer = itr.previous();
1147                 // do not call the cleaner if it was disabled
1148                 if (preparer.isDisabled() || preparer.isTearDownDisabled()) {
1149                     CLog.d("%s has been disabled. skipping.", preparer);
1150                     continue;
1151                 }
1152 
1153                 RecoveryMode origMode = null;
1154                 try (CloseableTraceScope ignored =
1155                         new CloseableTraceScope(preparer.getClass().getName())) {
1156                     // If an exception was generated in setup with a DNAE do not attempt any
1157                     // recovery again in case we hit the device not available again.
1158                     if (exception != null && exception instanceof DeviceNotAvailableException) {
1159                         origMode = device.getRecoveryMode();
1160                         device.setRecoveryMode(RecoveryMode.NONE);
1161                     }
1162                     moduleInfo.setActiveDeviceIndex(i);
1163                     preparer.tearDown(moduleInfo, exception);
1164                 } finally {
1165                     moduleInfo.setActiveDeviceIndex(0);
1166                     if (origMode != null) {
1167                         device.setRecoveryMode(origMode);
1168                     }
1169                 }
1170             }
1171         }
1172     }
1173 
1174     /** Verify that the device did not crash after the module. */
checkEndModuleDevice(TestInformation testInfo)1175     private void checkEndModuleDevice(TestInformation testInfo) throws DeviceNotAvailableException {
1176         if (SystemUtil.isLocalMode()) {
1177             CLog.d("Skipping check for device availability after end of module for local run.");
1178             return;
1179         }
1180         try (CloseableTraceScope check = new CloseableTraceScope("checkEndModuleDevice")) {
1181             for (ITestDevice device : testInfo.getDevices()) {
1182                 if (device.getIDevice() instanceof StubDevice) {
1183                     continue;
1184                 }
1185                 // Check device is still online
1186                 try {
1187                     device.waitForDeviceAvailable();
1188                 } catch (DeviceNotAvailableException e) {
1189                     // Wrap exception for better message
1190                     String error_msg =
1191                             String.format("Device went offline after running module '%s'", mId);
1192                     // TODO: If module is the last one, it won't need to do device recovery.
1193                     if (!mRecoverVirtualDevice) {
1194                         throw new DeviceNotAvailableException(
1195                                 error_msg,
1196                                 e,
1197                                 e.getSerial(),
1198                                 DeviceErrorIdentifier.DEVICE_UNAVAILABLE);
1199                     }
1200                     CLog.d(error_msg);
1201                     String snapshotId = null;
1202                     if (device.getConnection() instanceof AdbTcpConnection) {
1203                         snapshotId =
1204                                 ((AdbTcpConnection) device.getConnection())
1205                                         .getSuiteSnapshots()
1206                                         .get(device);
1207                     }
1208                     device.getConnection().recoverVirtualDevice(device, snapshotId, e);
1209                 }
1210             }
1211         }
1212     }
1213 
recoverDevice(TestInformation testInfo, DeviceNotAvailableException e)1214     private void recoverDevice(TestInformation testInfo, DeviceNotAvailableException e)
1215             throws DeviceNotAvailableException {
1216         if (SystemUtil.isLocalMode()) {
1217             CLog.d("Skipping device recovery for local run.");
1218             throw e;
1219         }
1220         if (!mRecoverVirtualDevice) {
1221             CLog.d("Skipping device recovery for as option recover-device-by-cvd is not enabled.");
1222             throw e;
1223         }
1224         try (CloseableTraceScope check = new CloseableTraceScope("recover_device")) {
1225             for (ITestDevice device : testInfo.getDevices()) {
1226                 if (device.getIDevice() instanceof StubDevice) {
1227                     continue;
1228                 }
1229                 String snapshotId = null;
1230                 if (device.getConnection() instanceof AdbTcpConnection) {
1231                     snapshotId =
1232                             ((AdbTcpConnection) device.getConnection())
1233                                     .getSuiteSnapshots()
1234                                     .get(device);
1235                 }
1236                 device.getConnection().recoverVirtualDevice(device, snapshotId, e);
1237             }
1238         }
1239     }
1240 
1241     /** Returns the current time. */
getCurrentTime()1242     private long getCurrentTime() {
1243         return System.currentTimeMillis();
1244     }
1245 
1246     @Override
setCollectTestsOnly(boolean collectTestsOnly)1247     public void setCollectTestsOnly(boolean collectTestsOnly) {
1248         mCollectTestsOnly = collectTestsOnly;
1249     }
1250 
1251     /** Sets should recover virtual device. */
setRecoverVirtualDevice(boolean recoverVirtualDevice)1252     public void setRecoverVirtualDevice(boolean recoverVirtualDevice) {
1253         mRecoverVirtualDevice = recoverVirtualDevice;
1254     }
1255 
1256     /** Returns if we should recover virtual device. */
shouldRecoverVirtualDevice()1257     public boolean shouldRecoverVirtualDevice() {
1258         return mRecoverVirtualDevice;
1259     }
1260 
1261     /** Sets whether or not we should merge results. */
setMergeAttemps(boolean mergeAttempts)1262     public final void setMergeAttemps(boolean mergeAttempts) {
1263         mMergeAttempts = mergeAttempts;
1264     }
1265 
1266     /** Sets the {@link IRetryDecision} to be used for intra-module retry. */
setRetryDecision(IRetryDecision decision)1267     public final void setRetryDecision(IRetryDecision decision) {
1268         mRetryDecision = decision;
1269         // Carry the retry decision to the module configuration
1270         mModuleConfiguration.setRetryDecision(decision);
1271     }
1272 
1273     /** Returns True if a testRunFailure has been called on the module * */
hasModuleFailed()1274     public boolean hasModuleFailed() {
1275         return mIsFailedModule;
1276     }
1277 
getRequiredTokens(TestInformation testInfo)1278     public Set<TokenProperty> getRequiredTokens(TestInformation testInfo) {
1279         // If there are no controllers just return directly
1280         List<?> ctrlObjectList = mModuleConfiguration.getConfigurationObjectList(MODULE_CONTROLLER);
1281         if (ctrlObjectList == null) {
1282             return mRequiredTokens;
1283         }
1284         // Clone the module context to get its metadata and then provide the device information
1285         // the same as ITestSuite would do during execution to run only the controllers
1286         InvocationContext clonedContext =
1287                 InvocationContext.fromProto(mModuleInvocationContext.toProto());
1288         for (String deviceName : testInfo.getContext().getDeviceConfigNames()) {
1289             clonedContext.addAllocatedDevice(
1290                     deviceName, testInfo.getContext().getDevice(deviceName));
1291             clonedContext.addDeviceBuildInfo(
1292                     deviceName, testInfo.getContext().getBuildInfo(deviceName));
1293         }
1294         try {
1295             if (!RunStrategy.RUN.equals(shouldRunWithController(clonedContext))) {
1296                 // Bypass token since the module isn't expected to run
1297                 return null;
1298             }
1299         } catch (RuntimeException | DeviceNotAvailableException e) {
1300             CLog.e(e);
1301         }
1302         return mRequiredTokens;
1303     }
1304 
1305     /** {@inheritDoc} */
1306     @Override
toString()1307     public String toString() {
1308         return getId();
1309     }
1310 
1311     /** Returns the approximate time to run all the tests in the module. */
getRuntimeHint()1312     public long getRuntimeHint() {
1313         long hint = 0L;
1314         for (IRemoteTest test : mTests) {
1315             if (test instanceof IRuntimeHintProvider) {
1316                 hint += ((IRuntimeHintProvider) test).getRuntimeHint();
1317             } else {
1318                 hint += 60000;
1319             }
1320         }
1321         return hint;
1322     }
1323 
1324     /** Returns the list of {@link IRemoteTest} defined for this module. */
1325     @VisibleForTesting
getTests()1326     List<IRemoteTest> getTests() {
1327         return new ArrayList<>(mTests);
1328     }
1329 
1330     /** Returns the list of {@link ITargetPreparer} associated with the given device name */
1331     @VisibleForTesting
getTargetPreparerForDevice(String deviceName)1332     List<ITargetPreparer> getTargetPreparerForDevice(String deviceName) {
1333         return mPreparersPerDevice.get(deviceName);
1334     }
1335 
1336     /**
1337      * Returns the list of suite level {@link ITargetPreparer} associated with the given device name
1338      */
1339     @VisibleForTesting
getSuitePreparerForDevice(String deviceName)1340     List<ITargetPreparer> getSuitePreparerForDevice(String deviceName) {
1341         return mSuitePreparersPerDevice.get(deviceName);
1342     }
1343 
1344     /**
1345      * When running unit tests for ModuleDefinition we don't want to unnecessarily report some auto
1346      * retry times.
1347      */
1348     @VisibleForTesting
disableAutoRetryReportingTime()1349     void disableAutoRetryReportingTime() {
1350         mDisableAutoRetryTimeReporting = true;
1351     }
1352 
1353     /** Returns the {@link IInvocationContext} associated with the module. */
getModuleInvocationContext()1354     public IInvocationContext getModuleInvocationContext() {
1355         return mModuleInvocationContext;
1356     }
1357 
getModuleConfiguration()1358     public IConfiguration getModuleConfiguration() {
1359         return mModuleConfiguration;
1360     }
1361 
setReportModuleStart(boolean shouldReportModuleStart)1362     public void setReportModuleStart(boolean shouldReportModuleStart) {
1363         mReportModuleStart = shouldReportModuleStart;
1364     }
1365 
setReportModuleEnd(boolean shouldReportModuleEnd)1366     public void setReportModuleEnd(boolean shouldReportModuleEnd) {
1367         mReportModuleEnd = shouldReportModuleEnd;
1368     }
1369 
1370     /** Report completely not executed modules. */
reportNotExecuted(ITestInvocationListener listener, String message)1371     public final void reportNotExecuted(ITestInvocationListener listener, String message) {
1372         if (mReportModuleStart) {
1373             listener.testModuleStarted(getModuleInvocationContext());
1374         }
1375         if (mCurrentTestWrapper != null)  {
1376             // do not report results if already reported once
1377             if (!mFinalResultsReported) {
1378                 mRunListenersResults.add(mCurrentTestWrapper.getResultListener());
1379                 HarnessRuntimeException interruptedException =
1380                         new HarnessRuntimeException(
1381                                 message, TestErrorIdentifier.MODULE_DID_NOT_EXECUTE);
1382                 for (int i = 0; i < mMaxRetry; i++) {
1383                     // Get all the results for the attempt
1384                     List<TestRunResult> runResultList = new ArrayList<TestRunResult>();
1385                     int expectedCount = 0;
1386                     for (ModuleListener attemptListener : mRunListenersResults) {
1387                         for (String runName : attemptListener.getTestRunNames()) {
1388                             TestRunResult run = attemptListener.getTestRunAtAttempt(runName, i);
1389                             if (run != null) {
1390                                 runResultList.add(run);
1391                                 expectedCount += run.getExpectedTestCount();
1392                             }
1393                         }
1394                     }
1395 
1396                     if (!runResultList.isEmpty()) {
1397                         reportFinalResults(
1398                                 listener, expectedCount, runResultList, i, interruptedException);
1399                     } else {
1400                         CLog.d("No results to be forwarded for attempt %s.", i);
1401                     }
1402                 }
1403             }
1404         } else {
1405             listener.testRunStarted(
1406                     getId(), 0, mTargetPreparerRetryCount, System.currentTimeMillis());
1407             FailureDescription description =
1408                     FailureDescription.create(message)
1409                             .setFailureStatus(FailureStatus.NOT_EXECUTED)
1410                             .setErrorIdentifier(TestErrorIdentifier.MODULE_DID_NOT_EXECUTE);
1411             listener.testRunFailed(description);
1412             listener.testRunEnded(0, new HashMap<String, Metric>());
1413         }
1414         if (mReportModuleEnd) {
1415             listener.testModuleEnded();
1416         }
1417     }
1418 
1419     /** Whether or not to enable dynamic download at module level. */
setEnableDynamicDownload(boolean enableDynamicDownload)1420     public void setEnableDynamicDownload(boolean enableDynamicDownload) {
1421         mEnableDynamicDownload = enableDynamicDownload;
1422     }
1423 
1424     /** Copy a few of the suite level configuration */
transferSuiteLevelOptions(IConfiguration mSuiteConfiguration)1425     public void transferSuiteLevelOptions(IConfiguration mSuiteConfiguration) {
1426         mModuleConfiguration
1427                 .getCommandOptions()
1428                 .getDynamicDownloadArgs()
1429                 .putAll(mSuiteConfiguration.getCommandOptions().getDynamicDownloadArgs());
1430         mModuleConfiguration
1431                 .getCommandOptions()
1432                 .setReportTestCaseCount(
1433                         mSuiteConfiguration.getCommandOptions().reportTestCaseCount());
1434     }
1435 
1436     /**
1437      * Allow to load module_controller objects to tune how should a particular module run. They will
1438      * be applied in order of appearance in the XML.
1439      *
1440      * @param failureListener The {@link TestFailureListener} taking actions on tests failures.
1441      * @return The strategy to use to run the tests.
1442      */
applyConfigurationControl()1443     private RunStrategy applyConfigurationControl() throws DeviceNotAvailableException {
1444         List<?> ctrlObjectList = mModuleConfiguration.getConfigurationObjectList(MODULE_CONTROLLER);
1445         if (ctrlObjectList == null) {
1446             return RunStrategy.RUN;
1447         }
1448         for (Object ctrlObject : ctrlObjectList) {
1449             if (ctrlObject instanceof BaseModuleController) {
1450                 BaseModuleController controller = (BaseModuleController) ctrlObject;
1451                 // Track usage of the controller
1452                 TfObjectTracker.countWithParents(controller.getClass());
1453                 if (!controller.shouldCaptureLogcat()) {
1454                     mRunMetricCollectors.removeIf(c -> (c instanceof LogcatOnFailureCollector));
1455                 }
1456                 if (!controller.shouldCaptureScreenshot()) {
1457                     mRunMetricCollectors.removeIf(c -> (c instanceof ScreenshotOnFailureCollector));
1458                 }
1459                 if (!controller.shouldCaptureBugreport()) {
1460                     mRunMetricCollectors.removeIf(
1461                             c -> (c instanceof BugreportzOnTestCaseFailureCollector));
1462                 }
1463             }
1464         }
1465         return shouldRunWithController(mModuleInvocationContext);
1466     }
1467 
shouldRunWithController(IInvocationContext context)1468     private RunStrategy shouldRunWithController(IInvocationContext context)
1469             throws DeviceNotAvailableException {
1470         List<?> ctrlObjectList = mModuleConfiguration.getConfigurationObjectList(MODULE_CONTROLLER);
1471         if (ctrlObjectList == null) {
1472             return RunStrategy.RUN;
1473         }
1474         // We keep the most stringent strategy across controllers.
1475         RunStrategy current = RunStrategy.RUN;
1476         for (Object ctrlObject : ctrlObjectList) {
1477             if (ctrlObject instanceof BaseModuleController) {
1478                 BaseModuleController controller = (BaseModuleController) ctrlObject;
1479                 RunStrategy strategy = controller.shouldRunModule(context);
1480                 if (RunStrategy.FULL_MODULE_BYPASS.equals(strategy)) {
1481                     current = strategy;
1482                 } else if (RunStrategy.SKIP_MODULE_TESTCASES.equals(strategy)
1483                         && RunStrategy.RUN.equals(current)) {
1484                     current = strategy;
1485                 }
1486             }
1487         }
1488         return current;
1489     }
1490 
addRetryTime(long retryTimeMs)1491     private void addRetryTime(long retryTimeMs) {
1492         if (retryTimeMs <= 0 || mDisableAutoRetryTimeReporting) {
1493             return;
1494         }
1495         InvocationMetricLogger.addInvocationMetrics(
1496                 InvocationMetricKey.AUTO_RETRY_TIME, retryTimeMs);
1497     }
1498 
runTargetPreparation(Map<String, List<ITargetPreparer>> preparersPerDevice)1499     private Throwable runTargetPreparation(Map<String, List<ITargetPreparer>> preparersPerDevice) {
1500         Throwable preparationException = null;
1501         for (int i = 0; i < mModuleInvocationContext.getDeviceConfigNames().size(); i++) {
1502             String deviceName = mModuleInvocationContext.getDeviceConfigNames().get(i);
1503             if (i >= preparersPerDevice.size()) {
1504                 CLog.d(
1505                         "Main configuration has more devices than the module configuration. '%s' "
1506                                 + "will not run any preparation.",
1507                         deviceName);
1508                 continue;
1509             }
1510             List<ITargetPreparer> preparers = preparersPerDevice.get(deviceName);
1511             if (preparers == null) {
1512                 CLog.w(
1513                         "Module configuration devices mismatch the main configuration "
1514                                 + "(Missing device '%s'), resolving preparers by index.",
1515                         deviceName);
1516                 String key = new ArrayList<>(preparersPerDevice.keySet()).get(i);
1517                 preparers = preparersPerDevice.get(key);
1518             }
1519             for (ITargetPreparer preparer : preparers) {
1520                 preparationException = runPreparerSetup(preparer, i);
1521                 if (preparationException != null) {
1522                     mIsFailedModule = true;
1523                     CLog.e("Some preparation step failed. failing the module %s", getId());
1524                     // If one device errored out, we skip the remaining devices.
1525                     return preparationException;
1526                 }
1527             }
1528         }
1529         return null;
1530     }
1531 
1532     /**
1533      * Handle calling the {@link IConfiguration#resolveDynamicOptions(DynamicRemoteFileResolver)}.
1534      */
invokeRemoteDynamic(ITestDevice device, IConfiguration moduleConfiguration)1535     private Exception invokeRemoteDynamic(ITestDevice device, IConfiguration moduleConfiguration) {
1536         if (!mEnableDynamicDownload) {
1537             return null;
1538         }
1539         // TODO: Add elapsed time tracking
1540         try {
1541             CLog.d("Attempting to resolve dynamic files from %s", getId());
1542             DynamicRemoteFileResolver resolver = new DynamicRemoteFileResolver();
1543             resolver.setDevice(device);
1544             resolver.addExtraArgs(moduleConfiguration.getCommandOptions().getDynamicDownloadArgs());
1545             moduleConfiguration.resolveDynamicOptions(resolver);
1546             return null;
1547         } catch (RuntimeException | ConfigurationException | BuildRetrievalError e) {
1548             mIsFailedModule = true;
1549             return e;
1550         }
1551     }
1552 
1553     /** Report a setup exception as a run failure and notify all the listeners. */
reportSetupFailure( Throwable setupException, ITestInvocationListener invocListener, int attemptNumber, boolean shouldFail)1554     private void reportSetupFailure(
1555             Throwable setupException,
1556             ITestInvocationListener invocListener,
1557             int attemptNumber,
1558             boolean shouldFail)
1559         throws DeviceNotAvailableException {
1560         // For reporting purpose we create a failure placeholder with the error stack
1561         // similar to InitializationError of JUnit.
1562         invocListener.testRunStarted(getId(), 1, attemptNumber, System.currentTimeMillis());
1563         FailureDescription failureDescription =
1564                 CurrentInvocation.createFailure(StreamUtil.getStackTrace(setupException), null);
1565         if (setupException instanceof IHarnessException
1566                 && ((IHarnessException) setupException).getErrorId() != null) {
1567             ErrorIdentifier id = ((IHarnessException) setupException).getErrorId();
1568             failureDescription.setErrorIdentifier(id);
1569             failureDescription.setFailureStatus(id.status());
1570             failureDescription.setOrigin(((IHarnessException) setupException).getOrigin());
1571         } else if (setupException instanceof RuntimeException) {
1572             // TODO: switch to customer_issue
1573             failureDescription.setFailureStatus(FailureStatus.UNSET);
1574             failureDescription.setErrorIdentifier(
1575                     InfraErrorIdentifier.MODULE_SETUP_RUNTIME_EXCEPTION);
1576         } else {
1577             failureDescription.setFailureStatus(FailureStatus.UNSET);
1578         }
1579         failureDescription.setCause(setupException);
1580         invocListener.testRunFailed(failureDescription);
1581         HashMap<String, Metric> metricsProto = new HashMap<>();
1582         metricsProto.put(TEST_TIME, TfMetricProtoUtil.createSingleValue(0L, "milliseconds"));
1583         invocListener.testRunEnded(0, metricsProto);
1584         // If it was a not available exception rethrow it to signal the new device state.
1585         if (setupException instanceof DeviceNotAvailableException) {
1586             if (!shouldFail) {
1587               CLog.i("Do not report the exception as module error, returning...");
1588               return;
1589             }
1590             throw (DeviceNotAvailableException) setupException;
1591         }
1592     }
1593 
applyFilterToTest(IRemoteTest test, Set<TestDescription> filters)1594     private void applyFilterToTest(IRemoteTest test, Set<TestDescription> filters) {
1595         Set<String> filterNames =
1596                 filters.stream().map(f -> f.toString()).collect(Collectors.toSet());
1597         if (test instanceof ITestFileFilterReceiver) {
1598             File excludeFilterFile = ((ITestFileFilterReceiver) test).getExcludeTestFile();
1599             if (excludeFilterFile == null) {
1600                 try {
1601                     excludeFilterFile = FileUtil.createTempFile("exclude-filter", ".txt");
1602                 } catch (IOException e) {
1603                     throw new HarnessRuntimeException(
1604                             e.getMessage(), e, InfraErrorIdentifier.FAIL_TO_CREATE_FILE);
1605                 }
1606                 ((ITestFileFilterReceiver) test).setExcludeTestFile(excludeFilterFile);
1607             }
1608             try {
1609                 FileUtil.writeToFile(Joiner.on('\n').join(filterNames), excludeFilterFile, true);
1610             } catch (IOException e) {
1611                 CLog.e(e);
1612             }
1613         } else if (test instanceof ITestFilterReceiver) {
1614             ((ITestFilterReceiver) test).addAllExcludeFilters(filterNames);
1615         }
1616     }
1617 
setUseModuleResultsForwarder(boolean useModuleResultsForwarder)1618     public void setUseModuleResultsForwarder(boolean useModuleResultsForwarder) {
1619         mUseModuleResultsForwarder = useModuleResultsForwarder;
1620     }
1621 }
1622