• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2015 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.compatibility.common.tradefed.testtype;
18 
19 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
20 import com.android.compatibility.common.tradefed.result.InvocationFailureHandler;
21 import com.android.compatibility.common.tradefed.result.SubPlanHelper;
22 import com.android.compatibility.common.tradefed.targetprep.NetworkConnectivityChecker;
23 import com.android.compatibility.common.tradefed.testtype.suite.CompatibilityTestSuite;
24 import com.android.compatibility.common.tradefed.util.RetryFilterHelper;
25 import com.android.compatibility.common.tradefed.util.RetryType;
26 import com.android.compatibility.common.tradefed.util.UniqueModuleCountUtil;
27 import com.android.compatibility.common.util.IInvocationResult;
28 import com.android.compatibility.common.util.ResultHandler;
29 import com.android.compatibility.common.util.TestFilter;
30 import com.android.ddmlib.Log.LogLevel;
31 import com.android.tradefed.build.IBuildInfo;
32 import com.android.tradefed.config.ConfigurationException;
33 import com.android.tradefed.config.Option;
34 import com.android.tradefed.config.Option.Importance;
35 import com.android.tradefed.config.OptionClass;
36 import com.android.tradefed.config.OptionCopier;
37 import com.android.tradefed.device.DeviceNotAvailableException;
38 import com.android.tradefed.device.DeviceUnresponsiveException;
39 import com.android.tradefed.device.ITestDevice;
40 import com.android.tradefed.invoker.IInvocationContext;
41 import com.android.tradefed.invoker.InvocationContext;
42 import com.android.tradefed.log.ITestLogger;
43 import com.android.tradefed.log.LogUtil.CLog;
44 import com.android.tradefed.result.ITestInvocationListener;
45 import com.android.tradefed.result.InputStreamSource;
46 import com.android.tradefed.result.LogDataType;
47 import com.android.tradefed.suite.checker.ISystemStatusChecker;
48 import com.android.tradefed.suite.checker.ISystemStatusCheckerReceiver;
49 import com.android.tradefed.suite.checker.StatusCheckerResult;
50 import com.android.tradefed.suite.checker.StatusCheckerResult.CheckStatus;
51 import com.android.tradefed.testtype.Abi;
52 import com.android.tradefed.testtype.IAbi;
53 import com.android.tradefed.testtype.IBuildReceiver;
54 import com.android.tradefed.testtype.IDeviceTest;
55 import com.android.tradefed.testtype.IInvocationContextReceiver;
56 import com.android.tradefed.testtype.IRemoteTest;
57 import com.android.tradefed.testtype.IShardableTest;
58 import com.android.tradefed.testtype.ITestCollector;
59 import com.android.tradefed.testtype.suite.TestSuiteInfo;
60 import com.android.tradefed.util.AbiFormatter;
61 import com.android.tradefed.util.AbiUtils;
62 import com.android.tradefed.util.ArrayUtil;
63 import com.android.tradefed.util.MultiMap;
64 import com.android.tradefed.util.StreamUtil;
65 import com.android.tradefed.util.TimeUtil;
66 
67 import com.google.common.annotations.VisibleForTesting;
68 
69 import java.io.ByteArrayOutputStream;
70 import java.io.FileNotFoundException;
71 import java.io.PrintWriter;
72 import java.util.ArrayList;
73 import java.util.Arrays;
74 import java.util.Collection;
75 import java.util.HashSet;
76 import java.util.LinkedHashSet;
77 import java.util.LinkedList;
78 import java.util.List;
79 import java.util.Set;
80 import java.util.concurrent.CountDownLatch;
81 import java.util.concurrent.TimeUnit;
82 
83 /**
84  * A Test for running Compatibility Suites.
85  * @deprecated use {@link CompatibilityTestSuite} instead.
86  */
87 @Deprecated
88 @OptionClass(alias = "compatibility")
89 public class CompatibilityTest implements IDeviceTest, IShardableTest, IBuildReceiver,
90         ISystemStatusCheckerReceiver, ITestCollector,
91         IInvocationContextReceiver {
92 
93     public static final String INCLUDE_FILTER_OPTION = "include-filter";
94     public static final String EXCLUDE_FILTER_OPTION = "exclude-filter";
95     public static final String SUBPLAN_OPTION = "subplan";
96     public static final String MODULE_OPTION = "module";
97     public static final String TEST_OPTION = "test";
98     public static final String PRECONDITION_ARG_OPTION = "precondition-arg";
99     public static final String MODULE_ARG_OPTION = "module-arg";
100     public static final String TEST_ARG_OPTION = "test-arg";
101     public static final char TEST_OPTION_SHORT_NAME = 't';
102     public static final String RETRY_OPTION = "retry";
103     public static final String RETRY_TYPE_OPTION = "retry-type";
104     public static final String ABI_OPTION = "abi";
105     public static final String SHARD_OPTION = "shards";
106     public static final String SKIP_DEVICE_INFO_OPTION = "skip-device-info";
107     public static final String SKIP_PRECONDITIONS_OPTION = "skip-preconditions";
108     public static final String SKIP_HOST_ARCH_CHECK = "skip-host-arch-check";
109     public static final String PRIMARY_ABI_RUN = "primary-abi-only";
110     public static final String DEVICE_TOKEN_OPTION = "device-token";
111     public static final String LOGCAT_ON_FAILURE_SIZE_OPTION = "logcat-on-failure-size";
112 
113     // Constants for checking invocation or preconditions preparation failure
114     private static final int NUM_PREP_ATTEMPTS = 10;
115     private static final int MINUTES_PER_PREP_ATTEMPT = 2;
116 
117     @Option(name = SUBPLAN_OPTION,
118             description = "the subplan to run",
119             importance = Importance.IF_UNSET)
120     private String mSubPlan;
121 
122     @Option(name = INCLUDE_FILTER_OPTION,
123             description = "the include module filters to apply.",
124             importance = Importance.ALWAYS)
125     private Set<String> mIncludeFilters = new HashSet<>();
126 
127     @Option(name = EXCLUDE_FILTER_OPTION,
128             description = "the exclude module filters to apply.",
129             importance = Importance.ALWAYS)
130     private Set<String> mExcludeFilters = new HashSet<>();
131 
132     @Option(name = MODULE_OPTION,
133             shortName = 'm',
134             description = "the test module to run.",
135             importance = Importance.IF_UNSET)
136     private String mModuleName = null;
137 
138     @Option(name = TEST_OPTION,
139             shortName = TEST_OPTION_SHORT_NAME,
140             description = "the test run.",
141             importance = Importance.IF_UNSET)
142     private String mTestName = null;
143 
144     @Option(name = PRECONDITION_ARG_OPTION,
145             description = "the arguments to pass to a precondition. The expected format is"
146                     + "\"<arg-name>:<arg-value>\"",
147             importance = Importance.ALWAYS)
148     private List<String> mPreconditionArgs = new ArrayList<>();
149 
150     @Option(name = MODULE_ARG_OPTION,
151             description = "the arguments to pass to a module. The expected format is"
152                     + "\"<module-name>:<arg-name>:[<arg-key>:=]<arg-value>\"",
153             importance = Importance.ALWAYS)
154     private List<String> mModuleArgs = new ArrayList<>();
155 
156     @Option(name = TEST_ARG_OPTION,
157             description = "the arguments to pass to a test. The expected format is"
158                     + "\"<test-class>:<arg-name>:[<arg-key>:=]<arg-value>\"",
159             importance = Importance.ALWAYS)
160     private List<String> mTestArgs = new ArrayList<>();
161 
162     @Option(name = RETRY_OPTION,
163             shortName = 'r',
164             description = "retry a previous session's failed and not executed tests.",
165             importance = Importance.IF_UNSET)
166     private Integer mRetrySessionId = null;
167 
168     @Option(name = RETRY_TYPE_OPTION,
169             description = "used with " + RETRY_OPTION + ", retry tests of a certain status. "
170             + "Possible values include \"failed\", \"not_executed\", and \"custom\".",
171             importance = Importance.IF_UNSET)
172     private RetryType mRetryType = null;
173 
174     @Option(name = ABI_OPTION,
175             shortName = 'a',
176             description = "the abi to test.",
177             importance = Importance.IF_UNSET)
178     private String mAbiName = null;
179 
180     @Option(name = SHARD_OPTION,
181             description = "split the modules up to run on multiple devices concurrently. "
182                     + "Deprecated, use --shard-count instead.")
183     @Deprecated
184     private int mShards = 1;
185 
186     @Option(name = SKIP_DEVICE_INFO_OPTION,
187             shortName = 'd',
188             description = "Whether device info collection should be skipped")
189     private boolean mSkipDeviceInfo = false;
190 
191     @Option(name = SKIP_HOST_ARCH_CHECK,
192             description = "Whether host architecture check should be skipped")
193     private boolean mSkipHostArchCheck = false;
194 
195     @Option(name = SKIP_PRECONDITIONS_OPTION,
196             shortName = 'o',
197             description = "Whether preconditions should be skipped")
198     private boolean mSkipPreconditions = false;
199 
200     @Option(name = PRIMARY_ABI_RUN,
201             description = "Whether to run tests with only the device primary abi. "
202                     + "This override the --abi option.")
203     private boolean mPrimaryAbiRun = false;
204 
205     @Option(name = DEVICE_TOKEN_OPTION,
206             description = "Holds the devices' tokens, used when scheduling tests that have"
207                     + "prerequisites such as requiring a SIM card. Format is <serial>:<token>",
208             importance = Importance.ALWAYS)
209     private List<String> mDeviceTokens = new ArrayList<>();
210 
211     @Option(name = "bugreport-on-failure",
212             description = "Take a bugreport on every test failure. " +
213                     "Warning: can potentially use a lot of disk space.")
214     private boolean mBugReportOnFailure = false;
215 
216     @Option(name = "logcat-on-failure",
217             description = "Take a logcat snapshot on every test failure.")
218     private boolean mLogcatOnFailure = false;
219 
220     @Option(name = LOGCAT_ON_FAILURE_SIZE_OPTION,
221             description = "The max number of logcat data in bytes to capture when "
222             + "--logcat-on-failure is on. Should be an amount that can comfortably fit in memory.")
223     private int mMaxLogcatBytes = 500 * 1024; // 500K
224 
225     @Option(name = "screenshot-on-failure",
226             description = "Take a screenshot on every test failure.")
227     private boolean mScreenshotOnFailure = false;
228 
229     @Option(name = "reboot-before-test",
230             description = "Reboot the device before the test suite starts.")
231     private boolean mRebootBeforeTest = false;
232 
233     @Option(name = "reboot-on-failure",
234             description = "Reboot the device after every test failure.")
235     private boolean mRebootOnFailure = false;
236 
237     @Option(name = "reboot-per-module",
238             description = "Reboot the device before every module run.")
239     private boolean mRebootPerModule = false;
240 
241     @Option(name = "skip-connectivity-check",
242             description = "Don't verify device connectivity between module execution.")
243     private boolean mSkipConnectivityCheck = false;
244 
245     @Option(name = "preparer-whitelist",
246             description = "Only run specific preparers."
247             + "Specify zero or more ITargetPreparers as canonical class names. "
248             + "e.g. \"com.android.compatibility.common.tradefed.targetprep.ApkInstaller\" "
249             + "If not specified, all configured preparers are run.")
250     private Set<String> mPreparerWhitelist = new HashSet<>();
251 
252     @Option(name = "skip-all-system-status-check",
253             description = "Whether all system status check between modules should be skipped")
254     private boolean mSkipAllSystemStatusCheck = false;
255 
256     @Option(name = "skip-system-status-check",
257             description = "Disable specific system status checkers."
258             + "Specify zero or more SystemStatusChecker as canonical class names. e.g. "
259             + "\"com.android.compatibility.common.tradefed.targetprep.NetworkConnectivityChecker\" "
260             + "If not specified, all configured or whitelisted system status checkers are run.")
261     private Set<String> mSystemStatusCheckBlacklist = new HashSet<>();
262 
263     @Option(name = "system-status-check-whitelist",
264             description = "Only run specific system status checkers."
265             + "Specify zero or more SystemStatusChecker as canonical class names. e.g. "
266             + "\"com.android.compatibility.common.tradefed.targetprep.NetworkConnectivityChecker\" "
267             + "If not specified, all configured system status checkers are run.")
268     private Set<String> mSystemStatusCheckWhitelist = new HashSet<>();
269 
270     private List<ISystemStatusChecker> mListCheckers = new ArrayList<>();
271 
272     @Option(name = "collect-tests-only",
273             description = "Only invoke the suite to collect list of applicable test cases. All "
274                     + "test run callbacks will be triggered, but test execution will not be "
275                     + "actually carried out.")
276     private Boolean mCollectTestsOnly = null;
277 
278     @Option(name = "module-metadata-include-filter",
279             description = "Include modules for execution based on matching of metadata fields: "
280                     + "for any of the specified filter name and value, if a module has a metadata "
281                     + "field with the same name and value, it will be included. When both module "
282                     + "inclusion and exclusion rules are applied, inclusion rules will be "
283                     + "evaluated first. Using this together with test filter inclusion rules may "
284                     + "result in no tests to execute if the rules don't overlap.")
285     private MultiMap<String, String> mModuleMetadataIncludeFilter = new MultiMap<>();
286 
287     @Option(name = "module-metadata-exclude-filter",
288             description = "Exclude modules for execution based on matching of metadata fields: "
289                     + "for any of the specified filter name and value, if a module has a metadata "
290                     + "field with the same name and value, it will be excluded. When both module "
291                     + "inclusion and exclusion rules are applied, inclusion rules will be "
292                     + "evaluated first.")
293     private MultiMap<String, String> mModuleMetadataExcludeFilter = new MultiMap<>();
294 
295     private int mTotalShards;
296     private Integer mShardIndex = null;
297     private IModuleRepo mModuleRepo;
298     private ITestDevice mDevice;
299     private CompatibilityBuildHelper mBuildHelper;
300 
301     // variables used for local sharding scenario
302     private static CountDownLatch sPreparedLatch;
303     private boolean mIsLocalSharding = false;
304     private boolean mIsSharded = false;
305 
306     private IInvocationContext mInvocationContext;
307 
308     /**
309      * Create a new {@link CompatibilityTest} that will run the default list of
310      * modules.
311      */
CompatibilityTest()312     public CompatibilityTest() {
313         this(1 /* totalShards */, new ModuleRepo(), 0);
314     }
315 
316     /**
317      * Create a new {@link CompatibilityTest} that will run a sublist of
318      * modules.
319      */
CompatibilityTest(int totalShards, IModuleRepo moduleRepo, Integer shardIndex)320     public CompatibilityTest(int totalShards, IModuleRepo moduleRepo, Integer shardIndex) {
321         if (totalShards < 1) {
322             throw new IllegalArgumentException(
323                     "Must be at least 1 shard. Given:" + totalShards);
324         }
325         mTotalShards = totalShards;
326         mModuleRepo = moduleRepo;
327         mShardIndex = shardIndex;
328     }
329 
330     /**
331      * {@inheritDoc}
332      */
333     @Override
getDevice()334     public ITestDevice getDevice() {
335         return mDevice;
336     }
337 
338     /**
339      * {@inheritDoc}
340      */
341     @Override
setDevice(ITestDevice device)342     public void setDevice(ITestDevice device) {
343         mDevice = device;
344     }
345 
346     /**
347      * {@inheritDoc}
348      */
349     @Override
setBuild(IBuildInfo buildInfo)350     public void setBuild(IBuildInfo buildInfo) {
351         mBuildHelper = new CompatibilityBuildHelper(buildInfo);
352     }
353 
354     /**
355      * {@inheritDoc}
356      */
357     @Override
run(ITestInvocationListener listener)358     public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
359         try {
360             List<ISystemStatusChecker> checkers = new ArrayList<>();
361             // Get system status checkers
362             if (mSkipAllSystemStatusCheck) {
363                 CLog.d("Skipping system status checkers");
364             } else {
365                 checkSystemStatusBlackAndWhiteList();
366                 for (ISystemStatusChecker checker : mListCheckers) {
367                     if(shouldIncludeSystemStatusChecker(checker)) {
368                         checkers.add(checker);
369                     }
370                 }
371             }
372 
373             LinkedList<IModuleDef> modules = initializeModuleRepo();
374 
375             mExcludeFilters.clear();
376             mIncludeFilters.clear();
377             // Update BuildInfo in each shard to store the original command-line arguments from
378             // the session to be retried. These arguments will be serialized in the report later.
379             if (mRetrySessionId != null) {
380                 loadRetryCommandLineArgs(mRetrySessionId);
381             }
382 
383             listener = new FailureListener(listener, getDevice(), mBugReportOnFailure,
384                     mLogcatOnFailure, mScreenshotOnFailure, mRebootOnFailure, mMaxLogcatBytes);
385             int moduleCount = modules.size();
386             if (moduleCount == 0) {
387                 CLog.logAndDisplay(LogLevel.INFO, "No module to run on %s.",
388                         mDevice.getSerialNumber());
389                 // Make sure we unlock other shards.
390                 if (sPreparedLatch != null) {
391                     sPreparedLatch.countDown();
392                 }
393                 return;
394             } else {
395                 int uniqueModuleCount = UniqueModuleCountUtil.countUniqueModules(modules);
396                 CLog.logAndDisplay(LogLevel.INFO, "Starting %d test sub-module%s on %s",
397                         uniqueModuleCount, (uniqueModuleCount > 1) ? "s" : "",
398                                 mDevice.getSerialNumber());
399             }
400 
401             if (mRebootBeforeTest) {
402                 CLog.d("Rebooting device before test starts as requested.");
403                 mDevice.reboot();
404             }
405 
406             if (mSkipConnectivityCheck) {
407                 String clazz = NetworkConnectivityChecker.class.getCanonicalName();
408                 CLog.logAndDisplay(LogLevel.INFO, "\"--skip-connectivity-check\" is deprecated, "
409                         + "please use \"--skip-system-status-check %s\" instead", clazz);
410                 mSystemStatusCheckBlacklist.add(clazz);
411             }
412 
413             // Set values and run preconditions
414             boolean isPrepared = true; // whether the device has been successfully prepared
415             for (int i = 0; i < moduleCount; i++) {
416                 IModuleDef module = modules.get(i);
417                 module.setBuild(mBuildHelper.getBuildInfo());
418                 module.setDevice(mDevice);
419                 module.setPreparerWhitelist(mPreparerWhitelist);
420                 // don't set a value if unspecified
421                 if (mCollectTestsOnly != null) {
422                     module.setCollectTestsOnly(mCollectTestsOnly);
423                 }
424                 isPrepared &= (module.prepare(mSkipPreconditions, mPreconditionArgs));
425             }
426             if (!isPrepared) {
427                 throw new RuntimeException(String.format("Failed preconditions on %s",
428                         mDevice.getSerialNumber()));
429             }
430             if (mIsLocalSharding) {
431                 try {
432                     sPreparedLatch.countDown();
433                     int attempt = 1;
434                     while(!sPreparedLatch.await(MINUTES_PER_PREP_ATTEMPT, TimeUnit.MINUTES)) {
435                         if (attempt > NUM_PREP_ATTEMPTS ||
436                                 InvocationFailureHandler.hasFailed(mBuildHelper)) {
437                             CLog.logAndDisplay(LogLevel.ERROR,
438                                     "Incorrect preparation detected, exiting test run from %s",
439                                     mDevice.getSerialNumber());
440                             return;
441                         }
442                         CLog.logAndDisplay(LogLevel.WARN, "waiting on preconditions");
443                         attempt++;
444                     }
445                 } catch (InterruptedException e) {
446                     throw new RuntimeException(e);
447                 }
448             }
449             // Module Repo is not useful anymore
450             mModuleRepo.tearDown();
451             mModuleRepo = null;
452             // Run the tests
453             while (!modules.isEmpty()) {
454                 // Make sure we remove the modules from the reference list when we are done with
455                 // them.
456                 IModuleDef module = modules.poll();
457                 long start = System.currentTimeMillis();
458 
459                 if (mRebootPerModule) {
460                     if ("user".equals(mDevice.getProperty("ro.build.type"))) {
461                         CLog.e("reboot-per-module should only be used during development, "
462                             + "this is a\" user\" build device");
463                     } else {
464                         CLog.logAndDisplay(LogLevel.INFO, "Rebooting device before starting next "
465                             + "module");
466                         mDevice.reboot();
467                     }
468                 }
469 
470                 // execute pre module execution checker
471                 if (checkers != null && !checkers.isEmpty()) {
472                     runPreModuleCheck(module.getName(), checkers, mDevice, listener);
473                 }
474                 IInvocationContext moduleContext = new InvocationContext();
475                 moduleContext.setConfigurationDescriptor(module.getConfigurationDescriptor());
476                 moduleContext.addInvocationAttribute(IModuleDef.MODULE_NAME, module.getName());
477                 moduleContext.addInvocationAttribute(IModuleDef.MODULE_ABI,
478                         module.getAbi().getName());
479                 // This format is not always true but for the deprecated runner this is best effort.
480                 moduleContext.addInvocationAttribute(
481                         IModuleDef.MODULE_ID,
482                         String.format("%s %s", module.getAbi().getName(), module.getName()));
483                 mInvocationContext.setModuleInvocationContext(moduleContext);
484                 // Populate the module context with devices and builds
485                 for (String deviceName : mInvocationContext.getDeviceConfigNames()) {
486                     moduleContext.addAllocatedDevice(
487                             deviceName, mInvocationContext.getDevice(deviceName));
488                     moduleContext.addDeviceBuildInfo(
489                             deviceName, mInvocationContext.getBuildInfo(deviceName));
490                 }
491                 module.setInvocationContext(moduleContext);
492                 try {
493                     listener.testModuleStarted(moduleContext);
494                     module.run(listener);
495                 } catch (DeviceUnresponsiveException due) {
496                     // being able to catch a DeviceUnresponsiveException here implies that recovery
497                     // was successful, and test execution should proceed to next module
498                     ByteArrayOutputStream stack = new ByteArrayOutputStream();
499                     due.printStackTrace(new PrintWriter(stack, true));
500                     StreamUtil.close(stack);
501                     CLog.w("Ignored DeviceUnresponsiveException because recovery was successful, "
502                             + "proceeding with next module. Stack trace: %s",
503                             stack.toString());
504                     CLog.w("This may be due to incorrect timeout setting on module %s",
505                             module.getName());
506                 } finally {
507                     // clear out module invocation context since we are now done with module
508                     // execution
509                     mInvocationContext.setModuleInvocationContext(null);
510                     listener.testModuleEnded();
511                 }
512                 long duration = System.currentTimeMillis() - start;
513                 long expected = module.getRuntimeHint();
514                 long delta = Math.abs(duration - expected);
515                 // Show warning if delta is more than 10% of expected
516                 if (expected > 0 && ((float)delta / (float)expected) > 0.1f) {
517                     CLog.logAndDisplay(LogLevel.WARN,
518                             "Inaccurate runtime hint for %s, expected %s was %s",
519                             module.getId(),
520                             TimeUtil.formatElapsedTime(expected),
521                             TimeUtil.formatElapsedTime(duration));
522                 }
523                 if (checkers != null && !checkers.isEmpty()) {
524                     runPostModuleCheck(module.getName(), checkers, mDevice, listener);
525                 }
526                 module = null;
527             }
528         } catch (FileNotFoundException fnfe) {
529             throw new RuntimeException("Failed to initialize modules", fnfe);
530         }
531     }
532 
533     /**
534      * Initialize module repo.
535      *
536      * @return A list of module definition
537      * @throws DeviceNotAvailableException
538      * @throws FileNotFoundException
539      */
initializeModuleRepo()540     protected LinkedList<IModuleDef> initializeModuleRepo()
541             throws DeviceNotAvailableException, FileNotFoundException {
542         // FIXME: Each shard will do a full initialization which is not optimal. Need a way
543         // to be more specific on what to initialize.
544         synchronized (mModuleRepo) {
545             if (!mModuleRepo.isInitialized()) {
546                 setupFilters();
547                 // Initialize the repository, {@link CompatibilityBuildHelper#getTestsDir} can
548                 // throw a {@link FileNotFoundException}
549                 mModuleRepo.initialize(mTotalShards, mShardIndex, mBuildHelper.getTestsDir(),
550                         getAbis(), mDeviceTokens, mTestArgs, mModuleArgs, mIncludeFilters,
551                         mExcludeFilters, mModuleMetadataIncludeFilter, mModuleMetadataExcludeFilter,
552                         mBuildHelper.getBuildInfo());
553 
554                 // Add the entire list of modules to the CompatibilityBuildHelper for reporting
555                 mBuildHelper.setModuleIds(mModuleRepo.getModuleIds());
556 
557                 int count = UniqueModuleCountUtil.countUniqueModules(mModuleRepo.getTokenModules())
558                         + UniqueModuleCountUtil.countUniqueModules(
559                                   mModuleRepo.getNonTokenModules());
560                 CLog.logAndDisplay(LogLevel.INFO, "========================================");
561                 CLog.logAndDisplay(LogLevel.INFO, "Starting a run with %s unique modules.", count);
562                 CLog.logAndDisplay(LogLevel.INFO, "========================================");
563             } else {
564                 CLog.d("ModuleRepo already initialized.");
565             }
566             // Get the tests to run in this shard
567             return mModuleRepo.getModules(getDevice().getSerialNumber(), mShardIndex);
568         }
569     }
570 
571     /**
572      * Gets the set of ABIs supported by both Compatibility and the device under test
573      *
574      * @return The set of ABIs to run the tests on
575      * @throws DeviceNotAvailableException
576      */
getAbis()577     Set<IAbi> getAbis() throws DeviceNotAvailableException {
578         Set<IAbi> abis = new LinkedHashSet<>();
579         Set<String> archAbis = getAbisForBuildTargetArch();
580         if (mPrimaryAbiRun) {
581             if (mAbiName == null) {
582                 // Get the primary from the device and make it the --abi to run.
583                 mAbiName = mDevice.getProperty("ro.product.cpu.abi").trim();
584             } else {
585                 CLog.d("Option --%s supersedes the option --%s, using abi: %s", ABI_OPTION,
586                         PRIMARY_ABI_RUN, mAbiName);
587             }
588         }
589         if (mAbiName != null) {
590             // A particular abi was requested, it still need to be supported by the build.
591             if ((!mSkipHostArchCheck && !archAbis.contains(mAbiName)) ||
592                     !AbiUtils.isAbiSupportedByCompatibility(mAbiName)) {
593                 throw new IllegalArgumentException(String.format("Your CTS hasn't been built with "
594                         + "abi '%s' support, this CTS currently supports '%s'.",
595                         mAbiName, archAbis));
596             } else {
597                 abis.add(new Abi(mAbiName, AbiUtils.getBitness(mAbiName)));
598                 return abis;
599             }
600         } else {
601             // Run on all abi in common between the device and CTS.
602             List<String> deviceAbis = Arrays.asList(AbiFormatter.getSupportedAbis(mDevice, ""));
603             for (String abi : deviceAbis) {
604                 if ((mSkipHostArchCheck || archAbis.contains(abi)) &&
605                         AbiUtils.isAbiSupportedByCompatibility(abi)) {
606                     abis.add(new Abi(abi, AbiUtils.getBitness(abi)));
607                 } else {
608                     CLog.d("abi '%s' is supported by device but not by this CTS build (%s), tests "
609                             + "will not run against it.", abi, archAbis);
610                 }
611             }
612             if (abis.isEmpty()) {
613                 throw new IllegalArgumentException(String.format("None of the abi supported by this"
614                        + " CTS build ('%s') are supported by the device ('%s').",
615                        archAbis, deviceAbis));
616             }
617             return abis;
618         }
619     }
620 
621     /**
622      * Return the abis supported by the Host build target architecture.
623      * Exposed for testing.
624      */
getAbisForBuildTargetArch()625     protected Set<String> getAbisForBuildTargetArch() {
626         return AbiUtils.getAbisForArch(TestSuiteInfo.getInstance().getTargetArchs().get(0));
627     }
628 
629     /**
630      * Check that the system status checker specified by option are valid.
631      */
checkSystemStatusBlackAndWhiteList()632     protected void checkSystemStatusBlackAndWhiteList() {
633         for (String checker : mSystemStatusCheckWhitelist) {
634             try {
635                 Class.forName(checker);
636             } catch (ClassNotFoundException e) {
637                 ConfigurationException ex = new ConfigurationException(
638                         String.format("--system-status-check-whitelist must contains valid class, "
639                                 + "%s was not found", checker), e);
640                 throw new RuntimeException(ex);
641             }
642         }
643         for (String checker : mSystemStatusCheckBlacklist) {
644             try {
645                 Class.forName(checker);
646             } catch (ClassNotFoundException e) {
647                 ConfigurationException ex = new ConfigurationException(
648                         String.format("--skip-system-status-check must contains valid class, "
649                                 + "%s was not found", checker), e);
650                 throw new RuntimeException(ex);
651             }
652         }
653     }
654 
655     /**
656      * Resolve the inclusion and exclusion logic of system status checkers
657      *
658      * @param s the {@link ISystemStatusChecker} to perform filtering logic on
659      * @return True if the {@link ISystemStatusChecker} should be included, false otherwise.
660      */
shouldIncludeSystemStatusChecker(ISystemStatusChecker s)661     private boolean shouldIncludeSystemStatusChecker(ISystemStatusChecker s) {
662         String clazz = s.getClass().getCanonicalName();
663         boolean shouldInclude = mSystemStatusCheckWhitelist.isEmpty()
664                 || mSystemStatusCheckWhitelist.contains(clazz);
665         boolean shouldExclude = !mSystemStatusCheckBlacklist.isEmpty()
666                 && mSystemStatusCheckBlacklist.contains(clazz);
667         return shouldInclude && !shouldExclude;
668     }
669 
670     @VisibleForTesting
runPreModuleCheck(String moduleName, List<ISystemStatusChecker> checkers, ITestDevice device, ITestLogger logger)671     void runPreModuleCheck(String moduleName, List<ISystemStatusChecker> checkers,
672             ITestDevice device, ITestLogger logger) throws DeviceNotAvailableException {
673         CLog.i("Running system status checker before module execution: %s", moduleName);
674         List<String> failures = new ArrayList<>();
675         for (ISystemStatusChecker checker : checkers) {
676             StatusCheckerResult result = checker.preExecutionCheck(device);
677             if (!CheckStatus.SUCCESS.equals(result.getStatus())) {
678                 failures.add(checker.getClass().getCanonicalName());
679                 CLog.w("System status checker [%s] failed", checker.getClass().getCanonicalName());
680             }
681         }
682         if (!failures.isEmpty()) {
683             CLog.w("There are failed system status checkers: %s capturing a bugreport",
684                     failures.toString());
685             try (InputStreamSource bugSource = device.getBugreport()) {
686                 logger.testLog(String.format("bugreport-checker-pre-module-%s", moduleName),
687                         LogDataType.BUGREPORT, bugSource);
688             }
689         }
690     }
691 
692     @VisibleForTesting
runPostModuleCheck(String moduleName, List<ISystemStatusChecker> checkers, ITestDevice device, ITestLogger logger)693     void runPostModuleCheck(String moduleName, List<ISystemStatusChecker> checkers,
694             ITestDevice device, ITestLogger logger) throws DeviceNotAvailableException {
695         CLog.i("Running system status checker after module execution: %s", moduleName);
696         List<String> failures = new ArrayList<>();
697         for (ISystemStatusChecker checker : checkers) {
698             StatusCheckerResult result = checker.postExecutionCheck(device);
699             if (!CheckStatus.SUCCESS.equals(result.getStatus())) {
700                 failures.add(checker.getClass().getCanonicalName());
701                 CLog.w("System status checker [%s] failed", checker.getClass().getCanonicalName());
702             }
703         }
704         if (!failures.isEmpty()) {
705             CLog.w("There are failed system status checkers: %s capturing a bugreport",
706                     failures.toString());
707             try (InputStreamSource bugSource = device.getBugreport()) {
708                 logger.testLog(String.format("bugreport-checker-post-module-%s", moduleName),
709                         LogDataType.BUGREPORT, bugSource);
710             }
711         }
712     }
713 
714     /**
715      * Sets the retry command-line args to be stored in the BuildInfo and serialized into the
716      * report upon completion of the invocation.
717      */
loadRetryCommandLineArgs(Integer sessionId)718     void loadRetryCommandLineArgs(Integer sessionId) {
719         IInvocationResult result = null;
720         try {
721             result = ResultHandler.findResult(mBuildHelper.getResultsDir(), sessionId);
722         } catch (FileNotFoundException e) {
723             // We should never reach this point, because this method should only be called
724             // after setupFilters(), so result exists if we've gotten this far
725             throw new RuntimeException(e);
726         }
727         if (result == null) {
728             // Again, this should never happen
729             throw new IllegalArgumentException(String.format(
730                     "Could not find session with id %d", sessionId));
731         }
732         String retryCommandLineArgs = result.getCommandLineArgs();
733         if (retryCommandLineArgs != null) {
734             mBuildHelper.setRetryCommandLineArgs(retryCommandLineArgs);
735         }
736     }
737 
738     /**
739      * Sets the include/exclude filters up based on if a module name was given or whether this is a
740      * retry run.
741      */
setupFilters()742     void setupFilters() throws DeviceNotAvailableException {
743         if (mRetrySessionId != null) {
744             // Load the invocation result
745             RetryFilterHelper helper = createRetryFilterHelper(mRetrySessionId);
746             helper.validateBuildFingerprint(mDevice);
747             helper.setCommandLineOptionsFor(this);
748             helper.populateRetryFilters();
749             mIncludeFilters = helper.getIncludeFilters();
750             mExcludeFilters = helper.getExcludeFilters();
751             helper.tearDown();
752         } else {
753             if (mSubPlan != null) {
754                 ISubPlan subPlan = SubPlanHelper.getSubPlanByName(mBuildHelper, mSubPlan);
755                 mIncludeFilters.addAll(subPlan.getIncludeFilters());
756                 mExcludeFilters.addAll(subPlan.getExcludeFilters());
757             }
758             if (mModuleName != null) {
759                 try {
760                     List<String> modules = ModuleRepo.getModuleNamesMatching(
761                             mBuildHelper.getTestsDir(), mModuleName);
762                     if (modules.size() == 0) {
763                         throw new IllegalArgumentException(
764                                 String.format("No modules found matching %s", mModuleName));
765                     } else if (modules.size() > 1) {
766                         throw new IllegalArgumentException(String.format("Multiple modules found"
767                                 + " matching %s:\n%s\nWhich one did you mean?\n",
768                                 mModuleName, ArrayUtil.join("\n", modules)));
769                     } else {
770                         String module = modules.get(0);
771                         cleanFilters(mIncludeFilters, module);
772                         cleanFilters(mExcludeFilters, module);
773                         mIncludeFilters.add(
774                                 new TestFilter(mAbiName, module, mTestName).toString());
775                     }
776                 } catch (FileNotFoundException e) {
777                     throw new RuntimeException(e);
778                 }
779             } else if (mTestName != null) {
780                 throw new IllegalArgumentException(
781                         "Test name given without module name. Add --module <module-name>");
782             }
783         }
784     }
785 
786     /* Creates a new {@link RetryFilterHelper} from attributes of this object. */
createRetryFilterHelper(Integer retrySessionId)787     protected RetryFilterHelper createRetryFilterHelper(Integer retrySessionId) {
788         return new RetryFilterHelper(mBuildHelper, retrySessionId,
789                 mSubPlan, mIncludeFilters, mExcludeFilters, mAbiName, mModuleName, mTestName,
790                 mRetryType);
791     }
792 
793     /* Helper method designed to remove filters in a list not applicable to the given module */
cleanFilters(Set<String> filters, String module)794     private static void cleanFilters(Set<String> filters, String module) {
795         Set<String> cleanedFilters = new HashSet<String>();
796         for (String filter : filters) {
797             if (module.equals(TestFilter.createFrom(filter).getName())) {
798                 cleanedFilters.add(filter); // Module name matches, filter passes
799             }
800         }
801         filters.clear();
802         filters.addAll(cleanedFilters);
803     }
804 
805     /**
806      * {@inheritDoc}
807      */
808     @Override
split()809     public Collection<IRemoteTest> split() {
810         if (mShards <= 1) {
811             return null;
812         }
813         mIsLocalSharding = true;
814         List<IRemoteTest> shardQueue = new LinkedList<>();
815         for (int i = 0; i < mShards; i++) {
816             CompatibilityTest test = (CompatibilityTest) getTestShard(mShards, i);
817             test.mIsLocalSharding = true;
818             shardQueue.add(test);
819         }
820         sPreparedLatch = new CountDownLatch(shardQueue.size());
821         return shardQueue;
822     }
823 
824     /**
825      * {@inheritDoc}
826      */
827     @Override
split(int shardCount)828     public Collection<IRemoteTest> split(int shardCount) {
829         if (shardCount <= 1 || mIsSharded) {
830             return null;
831         }
832         mIsSharded = true;
833         List<IRemoteTest> shardQueue = new LinkedList<>();
834         for (int i = 0; i < shardCount; i++) {
835             CompatibilityTest test = (CompatibilityTest) getTestShard(shardCount, i);
836             shardQueue.add(test);
837             test.mIsSharded = true;
838         }
839         return shardQueue;
840     }
841 
getTestShard(int shardCount, int shardIndex)842     private IRemoteTest getTestShard(int shardCount, int shardIndex) {
843         CompatibilityTest test = new CompatibilityTest(shardCount, mModuleRepo, shardIndex);
844         OptionCopier.copyOptionsNoThrow(this, test);
845         // Set the shard count because the copy option on the previous line
846         // copies over the mShard value
847         test.mShards = 0;
848         return test;
849     }
850 
851     /**
852      * {@inheritDoc}
853      */
854     @Override
setSystemStatusChecker(List<ISystemStatusChecker> systemCheckers)855     public void setSystemStatusChecker(List<ISystemStatusChecker> systemCheckers) {
856         mListCheckers = systemCheckers;
857     }
858 
859     @Override
setCollectTestsOnly(boolean collectTestsOnly)860     public void setCollectTestsOnly(boolean collectTestsOnly) {
861         mCollectTestsOnly = collectTestsOnly;
862     }
863 
864     /**
865      * Sets include-filters for the compatibility test
866      */
setIncludeFilter(Set<String> includeFilters)867     public void setIncludeFilter(Set<String> includeFilters) {
868         mIncludeFilters.addAll(includeFilters);
869     }
870 
871     /**
872      * Sets exclude-filters for the compatibility test
873      */
setExcludeFilter(Set<String> excludeFilters)874     public void setExcludeFilter(Set<String> excludeFilters) {
875         mExcludeFilters.addAll(excludeFilters);
876     }
877 
878     @Override
setInvocationContext(IInvocationContext invocationContext)879     public void setInvocationContext(IInvocationContext invocationContext) {
880         mInvocationContext = invocationContext;
881     }
882 
883     /**
884      * @return the mSubPlan
885      */
getSubPlan()886     protected String getSubPlan() {
887         return mSubPlan;
888     }
889 
890     /**
891      * @return the mIncludeFilters
892      */
getIncludeFilters()893     protected Set<String> getIncludeFilters() {
894         return mIncludeFilters;
895     }
896 
897     /**
898      * @return the mExcludeFilters
899      */
getExcludeFilters()900     protected Set<String> getExcludeFilters() {
901         return mExcludeFilters;
902     }
903 
904     /**
905      * @return the mModuleName
906      */
getModuleName()907     protected String getModuleName() {
908         return mModuleName;
909     }
910 
911     /**
912      * @return the mTestName
913      */
getTestName()914     protected String getTestName() {
915         return mTestName;
916     }
917 
918     /**
919      * @return the mModuleArgs
920      */
getModuleArgs()921     protected List<String> getModuleArgs() {
922         return mModuleArgs;
923     }
924 
925     /**
926      * @return the mTestArgs
927      */
getTestArgs()928     protected List<String> getTestArgs() {
929         return mTestArgs;
930     }
931 
932     /**
933      * @return the mRetryType
934      */
getRetryType()935     protected RetryType getRetryType() {
936         return mRetryType;
937     }
938 
939     /**
940      * @return the mAbiName
941      */
getAbiName()942     protected String getAbiName() {
943         return mAbiName;
944     }
945 
946     /**
947      * @return the mDeviceTokens
948      */
getDeviceTokens()949     protected List<String> getDeviceTokens() {
950         return mDeviceTokens;
951     }
952 
953     /**
954      * @return the mModuleMetadataIncludeFilter
955      */
getModuleMetadataIncludeFilter()956     protected MultiMap<String, String> getModuleMetadataIncludeFilter() {
957         return mModuleMetadataIncludeFilter;
958     }
959 
960     /**
961      * @return the mModuleMetadataExcludeFilter
962      */
getModuleMetadataExcludeFilter()963     protected MultiMap<String, String> getModuleMetadataExcludeFilter() {
964         return mModuleMetadataExcludeFilter;
965     }
966 
967     /**
968      * @return the mTotalShards
969      */
getTotalShards()970     protected int getTotalShards() {
971         return mTotalShards;
972     }
973 
974     /**
975      * @return the mShardIndex
976      */
getShardIndex()977     protected Integer getShardIndex() {
978         return mShardIndex;
979     }
980 
981     /**
982      * @return the mBuildHelper
983      */
getBuildHelper()984     protected CompatibilityBuildHelper getBuildHelper() {
985         return mBuildHelper;
986     }
987 
988     /**
989      * @return the mInvocationContext
990      */
getInvocationContext()991     protected IInvocationContext getInvocationContext() {
992         return mInvocationContext;
993     }
994 
995     /**
996      * @return the mModuleRepo
997      */
getModuleRepo()998     protected IModuleRepo getModuleRepo() {
999         return mModuleRepo;
1000     }
1001 }
1002