1 /* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.compatibility.common.tradefed.testtype; 18 19 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 20 import com.android.compatibility.common.tradefed.result.InvocationFailureHandler; 21 import com.android.compatibility.common.tradefed.result.SubPlanHelper; 22 import com.android.compatibility.common.tradefed.targetprep.NetworkConnectivityChecker; 23 import com.android.compatibility.common.tradefed.testtype.suite.CompatibilityTestSuite; 24 import com.android.compatibility.common.tradefed.util.RetryFilterHelper; 25 import com.android.compatibility.common.tradefed.util.RetryType; 26 import com.android.compatibility.common.tradefed.util.UniqueModuleCountUtil; 27 import com.android.compatibility.common.util.IInvocationResult; 28 import com.android.compatibility.common.util.ResultHandler; 29 import com.android.compatibility.common.util.TestFilter; 30 import com.android.ddmlib.Log.LogLevel; 31 import com.android.tradefed.build.IBuildInfo; 32 import com.android.tradefed.config.ConfigurationException; 33 import com.android.tradefed.config.Option; 34 import com.android.tradefed.config.Option.Importance; 35 import com.android.tradefed.config.OptionClass; 36 import com.android.tradefed.config.OptionCopier; 37 import com.android.tradefed.device.DeviceNotAvailableException; 38 import com.android.tradefed.device.DeviceUnresponsiveException; 39 import com.android.tradefed.device.ITestDevice; 40 import com.android.tradefed.invoker.IInvocationContext; 41 import com.android.tradefed.invoker.InvocationContext; 42 import com.android.tradefed.log.ITestLogger; 43 import com.android.tradefed.log.LogUtil.CLog; 44 import com.android.tradefed.result.ITestInvocationListener; 45 import com.android.tradefed.result.InputStreamSource; 46 import com.android.tradefed.result.LogDataType; 47 import com.android.tradefed.suite.checker.ISystemStatusChecker; 48 import com.android.tradefed.suite.checker.ISystemStatusCheckerReceiver; 49 import com.android.tradefed.suite.checker.StatusCheckerResult; 50 import com.android.tradefed.suite.checker.StatusCheckerResult.CheckStatus; 51 import com.android.tradefed.testtype.Abi; 52 import com.android.tradefed.testtype.IAbi; 53 import com.android.tradefed.testtype.IBuildReceiver; 54 import com.android.tradefed.testtype.IDeviceTest; 55 import com.android.tradefed.testtype.IInvocationContextReceiver; 56 import com.android.tradefed.testtype.IRemoteTest; 57 import com.android.tradefed.testtype.IShardableTest; 58 import com.android.tradefed.testtype.ITestCollector; 59 import com.android.tradefed.testtype.suite.TestSuiteInfo; 60 import com.android.tradefed.util.AbiFormatter; 61 import com.android.tradefed.util.AbiUtils; 62 import com.android.tradefed.util.ArrayUtil; 63 import com.android.tradefed.util.MultiMap; 64 import com.android.tradefed.util.StreamUtil; 65 import com.android.tradefed.util.TimeUtil; 66 67 import com.google.common.annotations.VisibleForTesting; 68 69 import java.io.ByteArrayOutputStream; 70 import java.io.FileNotFoundException; 71 import java.io.PrintWriter; 72 import java.util.ArrayList; 73 import java.util.Arrays; 74 import java.util.Collection; 75 import java.util.HashSet; 76 import java.util.LinkedHashSet; 77 import java.util.LinkedList; 78 import java.util.List; 79 import java.util.Set; 80 import java.util.concurrent.CountDownLatch; 81 import java.util.concurrent.TimeUnit; 82 83 /** 84 * A Test for running Compatibility Suites. 85 * @deprecated use {@link CompatibilityTestSuite} instead. 86 */ 87 @Deprecated 88 @OptionClass(alias = "compatibility") 89 public class CompatibilityTest implements IDeviceTest, IShardableTest, IBuildReceiver, 90 ISystemStatusCheckerReceiver, ITestCollector, 91 IInvocationContextReceiver { 92 93 public static final String INCLUDE_FILTER_OPTION = "include-filter"; 94 public static final String EXCLUDE_FILTER_OPTION = "exclude-filter"; 95 public static final String SUBPLAN_OPTION = "subplan"; 96 public static final String MODULE_OPTION = "module"; 97 public static final String TEST_OPTION = "test"; 98 public static final String PRECONDITION_ARG_OPTION = "precondition-arg"; 99 public static final String MODULE_ARG_OPTION = "module-arg"; 100 public static final String TEST_ARG_OPTION = "test-arg"; 101 public static final char TEST_OPTION_SHORT_NAME = 't'; 102 public static final String RETRY_OPTION = "retry"; 103 public static final String RETRY_TYPE_OPTION = "retry-type"; 104 public static final String ABI_OPTION = "abi"; 105 public static final String SHARD_OPTION = "shards"; 106 public static final String SKIP_DEVICE_INFO_OPTION = "skip-device-info"; 107 public static final String SKIP_PRECONDITIONS_OPTION = "skip-preconditions"; 108 public static final String SKIP_HOST_ARCH_CHECK = "skip-host-arch-check"; 109 public static final String PRIMARY_ABI_RUN = "primary-abi-only"; 110 public static final String DEVICE_TOKEN_OPTION = "device-token"; 111 public static final String LOGCAT_ON_FAILURE_SIZE_OPTION = "logcat-on-failure-size"; 112 113 // Constants for checking invocation or preconditions preparation failure 114 private static final int NUM_PREP_ATTEMPTS = 10; 115 private static final int MINUTES_PER_PREP_ATTEMPT = 2; 116 117 @Option(name = SUBPLAN_OPTION, 118 description = "the subplan to run", 119 importance = Importance.IF_UNSET) 120 private String mSubPlan; 121 122 @Option(name = INCLUDE_FILTER_OPTION, 123 description = "the include module filters to apply.", 124 importance = Importance.ALWAYS) 125 private Set<String> mIncludeFilters = new HashSet<>(); 126 127 @Option(name = EXCLUDE_FILTER_OPTION, 128 description = "the exclude module filters to apply.", 129 importance = Importance.ALWAYS) 130 private Set<String> mExcludeFilters = new HashSet<>(); 131 132 @Option(name = MODULE_OPTION, 133 shortName = 'm', 134 description = "the test module to run.", 135 importance = Importance.IF_UNSET) 136 private String mModuleName = null; 137 138 @Option(name = TEST_OPTION, 139 shortName = TEST_OPTION_SHORT_NAME, 140 description = "the test run.", 141 importance = Importance.IF_UNSET) 142 private String mTestName = null; 143 144 @Option(name = PRECONDITION_ARG_OPTION, 145 description = "the arguments to pass to a precondition. The expected format is" 146 + "\"<arg-name>:<arg-value>\"", 147 importance = Importance.ALWAYS) 148 private List<String> mPreconditionArgs = new ArrayList<>(); 149 150 @Option(name = MODULE_ARG_OPTION, 151 description = "the arguments to pass to a module. The expected format is" 152 + "\"<module-name>:<arg-name>:[<arg-key>:=]<arg-value>\"", 153 importance = Importance.ALWAYS) 154 private List<String> mModuleArgs = new ArrayList<>(); 155 156 @Option(name = TEST_ARG_OPTION, 157 description = "the arguments to pass to a test. The expected format is" 158 + "\"<test-class>:<arg-name>:[<arg-key>:=]<arg-value>\"", 159 importance = Importance.ALWAYS) 160 private List<String> mTestArgs = new ArrayList<>(); 161 162 @Option(name = RETRY_OPTION, 163 shortName = 'r', 164 description = "retry a previous session's failed and not executed tests.", 165 importance = Importance.IF_UNSET) 166 private Integer mRetrySessionId = null; 167 168 @Option(name = RETRY_TYPE_OPTION, 169 description = "used with " + RETRY_OPTION + ", retry tests of a certain status. " 170 + "Possible values include \"failed\", \"not_executed\", and \"custom\".", 171 importance = Importance.IF_UNSET) 172 private RetryType mRetryType = null; 173 174 @Option(name = ABI_OPTION, 175 shortName = 'a', 176 description = "the abi to test.", 177 importance = Importance.IF_UNSET) 178 private String mAbiName = null; 179 180 @Option(name = SHARD_OPTION, 181 description = "split the modules up to run on multiple devices concurrently. " 182 + "Deprecated, use --shard-count instead.") 183 @Deprecated 184 private int mShards = 1; 185 186 @Option(name = SKIP_DEVICE_INFO_OPTION, 187 shortName = 'd', 188 description = "Whether device info collection should be skipped") 189 private boolean mSkipDeviceInfo = false; 190 191 @Option(name = SKIP_HOST_ARCH_CHECK, 192 description = "Whether host architecture check should be skipped") 193 private boolean mSkipHostArchCheck = false; 194 195 @Option(name = SKIP_PRECONDITIONS_OPTION, 196 shortName = 'o', 197 description = "Whether preconditions should be skipped") 198 private boolean mSkipPreconditions = false; 199 200 @Option(name = PRIMARY_ABI_RUN, 201 description = "Whether to run tests with only the device primary abi. " 202 + "This override the --abi option.") 203 private boolean mPrimaryAbiRun = false; 204 205 @Option(name = DEVICE_TOKEN_OPTION, 206 description = "Holds the devices' tokens, used when scheduling tests that have" 207 + "prerequisites such as requiring a SIM card. Format is <serial>:<token>", 208 importance = Importance.ALWAYS) 209 private List<String> mDeviceTokens = new ArrayList<>(); 210 211 @Option(name = "bugreport-on-failure", 212 description = "Take a bugreport on every test failure. " + 213 "Warning: can potentially use a lot of disk space.") 214 private boolean mBugReportOnFailure = false; 215 216 @Option(name = "logcat-on-failure", 217 description = "Take a logcat snapshot on every test failure.") 218 private boolean mLogcatOnFailure = false; 219 220 @Option(name = LOGCAT_ON_FAILURE_SIZE_OPTION, 221 description = "The max number of logcat data in bytes to capture when " 222 + "--logcat-on-failure is on. Should be an amount that can comfortably fit in memory.") 223 private int mMaxLogcatBytes = 500 * 1024; // 500K 224 225 @Option(name = "screenshot-on-failure", 226 description = "Take a screenshot on every test failure.") 227 private boolean mScreenshotOnFailure = false; 228 229 @Option(name = "reboot-before-test", 230 description = "Reboot the device before the test suite starts.") 231 private boolean mRebootBeforeTest = false; 232 233 @Option(name = "reboot-on-failure", 234 description = "Reboot the device after every test failure.") 235 private boolean mRebootOnFailure = false; 236 237 @Option(name = "reboot-per-module", 238 description = "Reboot the device before every module run.") 239 private boolean mRebootPerModule = false; 240 241 @Option(name = "skip-connectivity-check", 242 description = "Don't verify device connectivity between module execution.") 243 private boolean mSkipConnectivityCheck = false; 244 245 @Option(name = "preparer-whitelist", 246 description = "Only run specific preparers." 247 + "Specify zero or more ITargetPreparers as canonical class names. " 248 + "e.g. \"com.android.compatibility.common.tradefed.targetprep.ApkInstaller\" " 249 + "If not specified, all configured preparers are run.") 250 private Set<String> mPreparerWhitelist = new HashSet<>(); 251 252 @Option(name = "skip-all-system-status-check", 253 description = "Whether all system status check between modules should be skipped") 254 private boolean mSkipAllSystemStatusCheck = false; 255 256 @Option(name = "skip-system-status-check", 257 description = "Disable specific system status checkers." 258 + "Specify zero or more SystemStatusChecker as canonical class names. e.g. " 259 + "\"com.android.compatibility.common.tradefed.targetprep.NetworkConnectivityChecker\" " 260 + "If not specified, all configured or whitelisted system status checkers are run.") 261 private Set<String> mSystemStatusCheckBlacklist = new HashSet<>(); 262 263 @Option(name = "system-status-check-whitelist", 264 description = "Only run specific system status checkers." 265 + "Specify zero or more SystemStatusChecker as canonical class names. e.g. " 266 + "\"com.android.compatibility.common.tradefed.targetprep.NetworkConnectivityChecker\" " 267 + "If not specified, all configured system status checkers are run.") 268 private Set<String> mSystemStatusCheckWhitelist = new HashSet<>(); 269 270 private List<ISystemStatusChecker> mListCheckers = new ArrayList<>(); 271 272 @Option(name = "collect-tests-only", 273 description = "Only invoke the suite to collect list of applicable test cases. All " 274 + "test run callbacks will be triggered, but test execution will not be " 275 + "actually carried out.") 276 private Boolean mCollectTestsOnly = null; 277 278 @Option(name = "module-metadata-include-filter", 279 description = "Include modules for execution based on matching of metadata fields: " 280 + "for any of the specified filter name and value, if a module has a metadata " 281 + "field with the same name and value, it will be included. When both module " 282 + "inclusion and exclusion rules are applied, inclusion rules will be " 283 + "evaluated first. Using this together with test filter inclusion rules may " 284 + "result in no tests to execute if the rules don't overlap.") 285 private MultiMap<String, String> mModuleMetadataIncludeFilter = new MultiMap<>(); 286 287 @Option(name = "module-metadata-exclude-filter", 288 description = "Exclude modules for execution based on matching of metadata fields: " 289 + "for any of the specified filter name and value, if a module has a metadata " 290 + "field with the same name and value, it will be excluded. When both module " 291 + "inclusion and exclusion rules are applied, inclusion rules will be " 292 + "evaluated first.") 293 private MultiMap<String, String> mModuleMetadataExcludeFilter = new MultiMap<>(); 294 295 private int mTotalShards; 296 private Integer mShardIndex = null; 297 private IModuleRepo mModuleRepo; 298 private ITestDevice mDevice; 299 private CompatibilityBuildHelper mBuildHelper; 300 301 // variables used for local sharding scenario 302 private static CountDownLatch sPreparedLatch; 303 private boolean mIsLocalSharding = false; 304 private boolean mIsSharded = false; 305 306 private IInvocationContext mInvocationContext; 307 308 /** 309 * Create a new {@link CompatibilityTest} that will run the default list of 310 * modules. 311 */ CompatibilityTest()312 public CompatibilityTest() { 313 this(1 /* totalShards */, new ModuleRepo(), 0); 314 } 315 316 /** 317 * Create a new {@link CompatibilityTest} that will run a sublist of 318 * modules. 319 */ CompatibilityTest(int totalShards, IModuleRepo moduleRepo, Integer shardIndex)320 public CompatibilityTest(int totalShards, IModuleRepo moduleRepo, Integer shardIndex) { 321 if (totalShards < 1) { 322 throw new IllegalArgumentException( 323 "Must be at least 1 shard. Given:" + totalShards); 324 } 325 mTotalShards = totalShards; 326 mModuleRepo = moduleRepo; 327 mShardIndex = shardIndex; 328 } 329 330 /** 331 * {@inheritDoc} 332 */ 333 @Override getDevice()334 public ITestDevice getDevice() { 335 return mDevice; 336 } 337 338 /** 339 * {@inheritDoc} 340 */ 341 @Override setDevice(ITestDevice device)342 public void setDevice(ITestDevice device) { 343 mDevice = device; 344 } 345 346 /** 347 * {@inheritDoc} 348 */ 349 @Override setBuild(IBuildInfo buildInfo)350 public void setBuild(IBuildInfo buildInfo) { 351 mBuildHelper = new CompatibilityBuildHelper(buildInfo); 352 } 353 354 /** 355 * {@inheritDoc} 356 */ 357 @Override run(ITestInvocationListener listener)358 public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 359 try { 360 List<ISystemStatusChecker> checkers = new ArrayList<>(); 361 // Get system status checkers 362 if (mSkipAllSystemStatusCheck) { 363 CLog.d("Skipping system status checkers"); 364 } else { 365 checkSystemStatusBlackAndWhiteList(); 366 for (ISystemStatusChecker checker : mListCheckers) { 367 if(shouldIncludeSystemStatusChecker(checker)) { 368 checkers.add(checker); 369 } 370 } 371 } 372 373 LinkedList<IModuleDef> modules = initializeModuleRepo(); 374 375 mExcludeFilters.clear(); 376 mIncludeFilters.clear(); 377 // Update BuildInfo in each shard to store the original command-line arguments from 378 // the session to be retried. These arguments will be serialized in the report later. 379 if (mRetrySessionId != null) { 380 loadRetryCommandLineArgs(mRetrySessionId); 381 } 382 383 listener = new FailureListener(listener, getDevice(), mBugReportOnFailure, 384 mLogcatOnFailure, mScreenshotOnFailure, mRebootOnFailure, mMaxLogcatBytes); 385 int moduleCount = modules.size(); 386 if (moduleCount == 0) { 387 CLog.logAndDisplay(LogLevel.INFO, "No module to run on %s.", 388 mDevice.getSerialNumber()); 389 // Make sure we unlock other shards. 390 if (sPreparedLatch != null) { 391 sPreparedLatch.countDown(); 392 } 393 return; 394 } else { 395 int uniqueModuleCount = UniqueModuleCountUtil.countUniqueModules(modules); 396 CLog.logAndDisplay(LogLevel.INFO, "Starting %d test sub-module%s on %s", 397 uniqueModuleCount, (uniqueModuleCount > 1) ? "s" : "", 398 mDevice.getSerialNumber()); 399 } 400 401 if (mRebootBeforeTest) { 402 CLog.d("Rebooting device before test starts as requested."); 403 mDevice.reboot(); 404 } 405 406 if (mSkipConnectivityCheck) { 407 String clazz = NetworkConnectivityChecker.class.getCanonicalName(); 408 CLog.logAndDisplay(LogLevel.INFO, "\"--skip-connectivity-check\" is deprecated, " 409 + "please use \"--skip-system-status-check %s\" instead", clazz); 410 mSystemStatusCheckBlacklist.add(clazz); 411 } 412 413 // Set values and run preconditions 414 boolean isPrepared = true; // whether the device has been successfully prepared 415 for (int i = 0; i < moduleCount; i++) { 416 IModuleDef module = modules.get(i); 417 module.setBuild(mBuildHelper.getBuildInfo()); 418 module.setDevice(mDevice); 419 module.setPreparerWhitelist(mPreparerWhitelist); 420 // don't set a value if unspecified 421 if (mCollectTestsOnly != null) { 422 module.setCollectTestsOnly(mCollectTestsOnly); 423 } 424 isPrepared &= (module.prepare(mSkipPreconditions, mPreconditionArgs)); 425 } 426 if (!isPrepared) { 427 throw new RuntimeException(String.format("Failed preconditions on %s", 428 mDevice.getSerialNumber())); 429 } 430 if (mIsLocalSharding) { 431 try { 432 sPreparedLatch.countDown(); 433 int attempt = 1; 434 while(!sPreparedLatch.await(MINUTES_PER_PREP_ATTEMPT, TimeUnit.MINUTES)) { 435 if (attempt > NUM_PREP_ATTEMPTS || 436 InvocationFailureHandler.hasFailed(mBuildHelper)) { 437 CLog.logAndDisplay(LogLevel.ERROR, 438 "Incorrect preparation detected, exiting test run from %s", 439 mDevice.getSerialNumber()); 440 return; 441 } 442 CLog.logAndDisplay(LogLevel.WARN, "waiting on preconditions"); 443 attempt++; 444 } 445 } catch (InterruptedException e) { 446 throw new RuntimeException(e); 447 } 448 } 449 // Module Repo is not useful anymore 450 mModuleRepo.tearDown(); 451 mModuleRepo = null; 452 // Run the tests 453 while (!modules.isEmpty()) { 454 // Make sure we remove the modules from the reference list when we are done with 455 // them. 456 IModuleDef module = modules.poll(); 457 long start = System.currentTimeMillis(); 458 459 if (mRebootPerModule) { 460 if ("user".equals(mDevice.getProperty("ro.build.type"))) { 461 CLog.e("reboot-per-module should only be used during development, " 462 + "this is a\" user\" build device"); 463 } else { 464 CLog.logAndDisplay(LogLevel.INFO, "Rebooting device before starting next " 465 + "module"); 466 mDevice.reboot(); 467 } 468 } 469 470 // execute pre module execution checker 471 if (checkers != null && !checkers.isEmpty()) { 472 runPreModuleCheck(module.getName(), checkers, mDevice, listener); 473 } 474 IInvocationContext moduleContext = new InvocationContext(); 475 moduleContext.setConfigurationDescriptor(module.getConfigurationDescriptor()); 476 moduleContext.addInvocationAttribute(IModuleDef.MODULE_NAME, module.getName()); 477 moduleContext.addInvocationAttribute(IModuleDef.MODULE_ABI, 478 module.getAbi().getName()); 479 // This format is not always true but for the deprecated runner this is best effort. 480 moduleContext.addInvocationAttribute( 481 IModuleDef.MODULE_ID, 482 String.format("%s %s", module.getAbi().getName(), module.getName())); 483 mInvocationContext.setModuleInvocationContext(moduleContext); 484 // Populate the module context with devices and builds 485 for (String deviceName : mInvocationContext.getDeviceConfigNames()) { 486 moduleContext.addAllocatedDevice( 487 deviceName, mInvocationContext.getDevice(deviceName)); 488 moduleContext.addDeviceBuildInfo( 489 deviceName, mInvocationContext.getBuildInfo(deviceName)); 490 } 491 module.setInvocationContext(moduleContext); 492 try { 493 listener.testModuleStarted(moduleContext); 494 module.run(listener); 495 } catch (DeviceUnresponsiveException due) { 496 // being able to catch a DeviceUnresponsiveException here implies that recovery 497 // was successful, and test execution should proceed to next module 498 ByteArrayOutputStream stack = new ByteArrayOutputStream(); 499 due.printStackTrace(new PrintWriter(stack, true)); 500 StreamUtil.close(stack); 501 CLog.w("Ignored DeviceUnresponsiveException because recovery was successful, " 502 + "proceeding with next module. Stack trace: %s", 503 stack.toString()); 504 CLog.w("This may be due to incorrect timeout setting on module %s", 505 module.getName()); 506 } finally { 507 // clear out module invocation context since we are now done with module 508 // execution 509 mInvocationContext.setModuleInvocationContext(null); 510 listener.testModuleEnded(); 511 } 512 long duration = System.currentTimeMillis() - start; 513 long expected = module.getRuntimeHint(); 514 long delta = Math.abs(duration - expected); 515 // Show warning if delta is more than 10% of expected 516 if (expected > 0 && ((float)delta / (float)expected) > 0.1f) { 517 CLog.logAndDisplay(LogLevel.WARN, 518 "Inaccurate runtime hint for %s, expected %s was %s", 519 module.getId(), 520 TimeUtil.formatElapsedTime(expected), 521 TimeUtil.formatElapsedTime(duration)); 522 } 523 if (checkers != null && !checkers.isEmpty()) { 524 runPostModuleCheck(module.getName(), checkers, mDevice, listener); 525 } 526 module = null; 527 } 528 } catch (FileNotFoundException fnfe) { 529 throw new RuntimeException("Failed to initialize modules", fnfe); 530 } 531 } 532 533 /** 534 * Initialize module repo. 535 * 536 * @return A list of module definition 537 * @throws DeviceNotAvailableException 538 * @throws FileNotFoundException 539 */ initializeModuleRepo()540 protected LinkedList<IModuleDef> initializeModuleRepo() 541 throws DeviceNotAvailableException, FileNotFoundException { 542 // FIXME: Each shard will do a full initialization which is not optimal. Need a way 543 // to be more specific on what to initialize. 544 synchronized (mModuleRepo) { 545 if (!mModuleRepo.isInitialized()) { 546 setupFilters(); 547 // Initialize the repository, {@link CompatibilityBuildHelper#getTestsDir} can 548 // throw a {@link FileNotFoundException} 549 mModuleRepo.initialize(mTotalShards, mShardIndex, mBuildHelper.getTestsDir(), 550 getAbis(), mDeviceTokens, mTestArgs, mModuleArgs, mIncludeFilters, 551 mExcludeFilters, mModuleMetadataIncludeFilter, mModuleMetadataExcludeFilter, 552 mBuildHelper.getBuildInfo()); 553 554 // Add the entire list of modules to the CompatibilityBuildHelper for reporting 555 mBuildHelper.setModuleIds(mModuleRepo.getModuleIds()); 556 557 int count = UniqueModuleCountUtil.countUniqueModules(mModuleRepo.getTokenModules()) 558 + UniqueModuleCountUtil.countUniqueModules( 559 mModuleRepo.getNonTokenModules()); 560 CLog.logAndDisplay(LogLevel.INFO, "========================================"); 561 CLog.logAndDisplay(LogLevel.INFO, "Starting a run with %s unique modules.", count); 562 CLog.logAndDisplay(LogLevel.INFO, "========================================"); 563 } else { 564 CLog.d("ModuleRepo already initialized."); 565 } 566 // Get the tests to run in this shard 567 return mModuleRepo.getModules(getDevice().getSerialNumber(), mShardIndex); 568 } 569 } 570 571 /** 572 * Gets the set of ABIs supported by both Compatibility and the device under test 573 * 574 * @return The set of ABIs to run the tests on 575 * @throws DeviceNotAvailableException 576 */ getAbis()577 Set<IAbi> getAbis() throws DeviceNotAvailableException { 578 Set<IAbi> abis = new LinkedHashSet<>(); 579 Set<String> archAbis = getAbisForBuildTargetArch(); 580 if (mPrimaryAbiRun) { 581 if (mAbiName == null) { 582 // Get the primary from the device and make it the --abi to run. 583 mAbiName = mDevice.getProperty("ro.product.cpu.abi").trim(); 584 } else { 585 CLog.d("Option --%s supersedes the option --%s, using abi: %s", ABI_OPTION, 586 PRIMARY_ABI_RUN, mAbiName); 587 } 588 } 589 if (mAbiName != null) { 590 // A particular abi was requested, it still need to be supported by the build. 591 if ((!mSkipHostArchCheck && !archAbis.contains(mAbiName)) || 592 !AbiUtils.isAbiSupportedByCompatibility(mAbiName)) { 593 throw new IllegalArgumentException(String.format("Your CTS hasn't been built with " 594 + "abi '%s' support, this CTS currently supports '%s'.", 595 mAbiName, archAbis)); 596 } else { 597 abis.add(new Abi(mAbiName, AbiUtils.getBitness(mAbiName))); 598 return abis; 599 } 600 } else { 601 // Run on all abi in common between the device and CTS. 602 List<String> deviceAbis = Arrays.asList(AbiFormatter.getSupportedAbis(mDevice, "")); 603 for (String abi : deviceAbis) { 604 if ((mSkipHostArchCheck || archAbis.contains(abi)) && 605 AbiUtils.isAbiSupportedByCompatibility(abi)) { 606 abis.add(new Abi(abi, AbiUtils.getBitness(abi))); 607 } else { 608 CLog.d("abi '%s' is supported by device but not by this CTS build (%s), tests " 609 + "will not run against it.", abi, archAbis); 610 } 611 } 612 if (abis.isEmpty()) { 613 throw new IllegalArgumentException(String.format("None of the abi supported by this" 614 + " CTS build ('%s') are supported by the device ('%s').", 615 archAbis, deviceAbis)); 616 } 617 return abis; 618 } 619 } 620 621 /** 622 * Return the abis supported by the Host build target architecture. 623 * Exposed for testing. 624 */ getAbisForBuildTargetArch()625 protected Set<String> getAbisForBuildTargetArch() { 626 return AbiUtils.getAbisForArch(TestSuiteInfo.getInstance().getTargetArchs().get(0)); 627 } 628 629 /** 630 * Check that the system status checker specified by option are valid. 631 */ checkSystemStatusBlackAndWhiteList()632 protected void checkSystemStatusBlackAndWhiteList() { 633 for (String checker : mSystemStatusCheckWhitelist) { 634 try { 635 Class.forName(checker); 636 } catch (ClassNotFoundException e) { 637 ConfigurationException ex = new ConfigurationException( 638 String.format("--system-status-check-whitelist must contains valid class, " 639 + "%s was not found", checker), e); 640 throw new RuntimeException(ex); 641 } 642 } 643 for (String checker : mSystemStatusCheckBlacklist) { 644 try { 645 Class.forName(checker); 646 } catch (ClassNotFoundException e) { 647 ConfigurationException ex = new ConfigurationException( 648 String.format("--skip-system-status-check must contains valid class, " 649 + "%s was not found", checker), e); 650 throw new RuntimeException(ex); 651 } 652 } 653 } 654 655 /** 656 * Resolve the inclusion and exclusion logic of system status checkers 657 * 658 * @param s the {@link ISystemStatusChecker} to perform filtering logic on 659 * @return True if the {@link ISystemStatusChecker} should be included, false otherwise. 660 */ shouldIncludeSystemStatusChecker(ISystemStatusChecker s)661 private boolean shouldIncludeSystemStatusChecker(ISystemStatusChecker s) { 662 String clazz = s.getClass().getCanonicalName(); 663 boolean shouldInclude = mSystemStatusCheckWhitelist.isEmpty() 664 || mSystemStatusCheckWhitelist.contains(clazz); 665 boolean shouldExclude = !mSystemStatusCheckBlacklist.isEmpty() 666 && mSystemStatusCheckBlacklist.contains(clazz); 667 return shouldInclude && !shouldExclude; 668 } 669 670 @VisibleForTesting runPreModuleCheck(String moduleName, List<ISystemStatusChecker> checkers, ITestDevice device, ITestLogger logger)671 void runPreModuleCheck(String moduleName, List<ISystemStatusChecker> checkers, 672 ITestDevice device, ITestLogger logger) throws DeviceNotAvailableException { 673 CLog.i("Running system status checker before module execution: %s", moduleName); 674 List<String> failures = new ArrayList<>(); 675 for (ISystemStatusChecker checker : checkers) { 676 StatusCheckerResult result = checker.preExecutionCheck(device); 677 if (!CheckStatus.SUCCESS.equals(result.getStatus())) { 678 failures.add(checker.getClass().getCanonicalName()); 679 CLog.w("System status checker [%s] failed", checker.getClass().getCanonicalName()); 680 } 681 } 682 if (!failures.isEmpty()) { 683 CLog.w("There are failed system status checkers: %s capturing a bugreport", 684 failures.toString()); 685 try (InputStreamSource bugSource = device.getBugreport()) { 686 logger.testLog(String.format("bugreport-checker-pre-module-%s", moduleName), 687 LogDataType.BUGREPORT, bugSource); 688 } 689 } 690 } 691 692 @VisibleForTesting runPostModuleCheck(String moduleName, List<ISystemStatusChecker> checkers, ITestDevice device, ITestLogger logger)693 void runPostModuleCheck(String moduleName, List<ISystemStatusChecker> checkers, 694 ITestDevice device, ITestLogger logger) throws DeviceNotAvailableException { 695 CLog.i("Running system status checker after module execution: %s", moduleName); 696 List<String> failures = new ArrayList<>(); 697 for (ISystemStatusChecker checker : checkers) { 698 StatusCheckerResult result = checker.postExecutionCheck(device); 699 if (!CheckStatus.SUCCESS.equals(result.getStatus())) { 700 failures.add(checker.getClass().getCanonicalName()); 701 CLog.w("System status checker [%s] failed", checker.getClass().getCanonicalName()); 702 } 703 } 704 if (!failures.isEmpty()) { 705 CLog.w("There are failed system status checkers: %s capturing a bugreport", 706 failures.toString()); 707 try (InputStreamSource bugSource = device.getBugreport()) { 708 logger.testLog(String.format("bugreport-checker-post-module-%s", moduleName), 709 LogDataType.BUGREPORT, bugSource); 710 } 711 } 712 } 713 714 /** 715 * Sets the retry command-line args to be stored in the BuildInfo and serialized into the 716 * report upon completion of the invocation. 717 */ loadRetryCommandLineArgs(Integer sessionId)718 void loadRetryCommandLineArgs(Integer sessionId) { 719 IInvocationResult result = null; 720 try { 721 result = ResultHandler.findResult(mBuildHelper.getResultsDir(), sessionId); 722 } catch (FileNotFoundException e) { 723 // We should never reach this point, because this method should only be called 724 // after setupFilters(), so result exists if we've gotten this far 725 throw new RuntimeException(e); 726 } 727 if (result == null) { 728 // Again, this should never happen 729 throw new IllegalArgumentException(String.format( 730 "Could not find session with id %d", sessionId)); 731 } 732 String retryCommandLineArgs = result.getCommandLineArgs(); 733 if (retryCommandLineArgs != null) { 734 mBuildHelper.setRetryCommandLineArgs(retryCommandLineArgs); 735 } 736 } 737 738 /** 739 * Sets the include/exclude filters up based on if a module name was given or whether this is a 740 * retry run. 741 */ setupFilters()742 void setupFilters() throws DeviceNotAvailableException { 743 if (mRetrySessionId != null) { 744 // Load the invocation result 745 RetryFilterHelper helper = createRetryFilterHelper(mRetrySessionId); 746 helper.validateBuildFingerprint(mDevice); 747 helper.setCommandLineOptionsFor(this); 748 helper.populateRetryFilters(); 749 mIncludeFilters = helper.getIncludeFilters(); 750 mExcludeFilters = helper.getExcludeFilters(); 751 helper.tearDown(); 752 } else { 753 if (mSubPlan != null) { 754 ISubPlan subPlan = SubPlanHelper.getSubPlanByName(mBuildHelper, mSubPlan); 755 mIncludeFilters.addAll(subPlan.getIncludeFilters()); 756 mExcludeFilters.addAll(subPlan.getExcludeFilters()); 757 } 758 if (mModuleName != null) { 759 try { 760 List<String> modules = ModuleRepo.getModuleNamesMatching( 761 mBuildHelper.getTestsDir(), mModuleName); 762 if (modules.size() == 0) { 763 throw new IllegalArgumentException( 764 String.format("No modules found matching %s", mModuleName)); 765 } else if (modules.size() > 1) { 766 throw new IllegalArgumentException(String.format("Multiple modules found" 767 + " matching %s:\n%s\nWhich one did you mean?\n", 768 mModuleName, ArrayUtil.join("\n", modules))); 769 } else { 770 String module = modules.get(0); 771 cleanFilters(mIncludeFilters, module); 772 cleanFilters(mExcludeFilters, module); 773 mIncludeFilters.add( 774 new TestFilter(mAbiName, module, mTestName).toString()); 775 } 776 } catch (FileNotFoundException e) { 777 throw new RuntimeException(e); 778 } 779 } else if (mTestName != null) { 780 throw new IllegalArgumentException( 781 "Test name given without module name. Add --module <module-name>"); 782 } 783 } 784 } 785 786 /* Creates a new {@link RetryFilterHelper} from attributes of this object. */ createRetryFilterHelper(Integer retrySessionId)787 protected RetryFilterHelper createRetryFilterHelper(Integer retrySessionId) { 788 return new RetryFilterHelper(mBuildHelper, retrySessionId, 789 mSubPlan, mIncludeFilters, mExcludeFilters, mAbiName, mModuleName, mTestName, 790 mRetryType); 791 } 792 793 /* Helper method designed to remove filters in a list not applicable to the given module */ cleanFilters(Set<String> filters, String module)794 private static void cleanFilters(Set<String> filters, String module) { 795 Set<String> cleanedFilters = new HashSet<String>(); 796 for (String filter : filters) { 797 if (module.equals(TestFilter.createFrom(filter).getName())) { 798 cleanedFilters.add(filter); // Module name matches, filter passes 799 } 800 } 801 filters.clear(); 802 filters.addAll(cleanedFilters); 803 } 804 805 /** 806 * {@inheritDoc} 807 */ 808 @Override split()809 public Collection<IRemoteTest> split() { 810 if (mShards <= 1) { 811 return null; 812 } 813 mIsLocalSharding = true; 814 List<IRemoteTest> shardQueue = new LinkedList<>(); 815 for (int i = 0; i < mShards; i++) { 816 CompatibilityTest test = (CompatibilityTest) getTestShard(mShards, i); 817 test.mIsLocalSharding = true; 818 shardQueue.add(test); 819 } 820 sPreparedLatch = new CountDownLatch(shardQueue.size()); 821 return shardQueue; 822 } 823 824 /** 825 * {@inheritDoc} 826 */ 827 @Override split(int shardCount)828 public Collection<IRemoteTest> split(int shardCount) { 829 if (shardCount <= 1 || mIsSharded) { 830 return null; 831 } 832 mIsSharded = true; 833 List<IRemoteTest> shardQueue = new LinkedList<>(); 834 for (int i = 0; i < shardCount; i++) { 835 CompatibilityTest test = (CompatibilityTest) getTestShard(shardCount, i); 836 shardQueue.add(test); 837 test.mIsSharded = true; 838 } 839 return shardQueue; 840 } 841 getTestShard(int shardCount, int shardIndex)842 private IRemoteTest getTestShard(int shardCount, int shardIndex) { 843 CompatibilityTest test = new CompatibilityTest(shardCount, mModuleRepo, shardIndex); 844 OptionCopier.copyOptionsNoThrow(this, test); 845 // Set the shard count because the copy option on the previous line 846 // copies over the mShard value 847 test.mShards = 0; 848 return test; 849 } 850 851 /** 852 * {@inheritDoc} 853 */ 854 @Override setSystemStatusChecker(List<ISystemStatusChecker> systemCheckers)855 public void setSystemStatusChecker(List<ISystemStatusChecker> systemCheckers) { 856 mListCheckers = systemCheckers; 857 } 858 859 @Override setCollectTestsOnly(boolean collectTestsOnly)860 public void setCollectTestsOnly(boolean collectTestsOnly) { 861 mCollectTestsOnly = collectTestsOnly; 862 } 863 864 /** 865 * Sets include-filters for the compatibility test 866 */ setIncludeFilter(Set<String> includeFilters)867 public void setIncludeFilter(Set<String> includeFilters) { 868 mIncludeFilters.addAll(includeFilters); 869 } 870 871 /** 872 * Sets exclude-filters for the compatibility test 873 */ setExcludeFilter(Set<String> excludeFilters)874 public void setExcludeFilter(Set<String> excludeFilters) { 875 mExcludeFilters.addAll(excludeFilters); 876 } 877 878 @Override setInvocationContext(IInvocationContext invocationContext)879 public void setInvocationContext(IInvocationContext invocationContext) { 880 mInvocationContext = invocationContext; 881 } 882 883 /** 884 * @return the mSubPlan 885 */ getSubPlan()886 protected String getSubPlan() { 887 return mSubPlan; 888 } 889 890 /** 891 * @return the mIncludeFilters 892 */ getIncludeFilters()893 protected Set<String> getIncludeFilters() { 894 return mIncludeFilters; 895 } 896 897 /** 898 * @return the mExcludeFilters 899 */ getExcludeFilters()900 protected Set<String> getExcludeFilters() { 901 return mExcludeFilters; 902 } 903 904 /** 905 * @return the mModuleName 906 */ getModuleName()907 protected String getModuleName() { 908 return mModuleName; 909 } 910 911 /** 912 * @return the mTestName 913 */ getTestName()914 protected String getTestName() { 915 return mTestName; 916 } 917 918 /** 919 * @return the mModuleArgs 920 */ getModuleArgs()921 protected List<String> getModuleArgs() { 922 return mModuleArgs; 923 } 924 925 /** 926 * @return the mTestArgs 927 */ getTestArgs()928 protected List<String> getTestArgs() { 929 return mTestArgs; 930 } 931 932 /** 933 * @return the mRetryType 934 */ getRetryType()935 protected RetryType getRetryType() { 936 return mRetryType; 937 } 938 939 /** 940 * @return the mAbiName 941 */ getAbiName()942 protected String getAbiName() { 943 return mAbiName; 944 } 945 946 /** 947 * @return the mDeviceTokens 948 */ getDeviceTokens()949 protected List<String> getDeviceTokens() { 950 return mDeviceTokens; 951 } 952 953 /** 954 * @return the mModuleMetadataIncludeFilter 955 */ getModuleMetadataIncludeFilter()956 protected MultiMap<String, String> getModuleMetadataIncludeFilter() { 957 return mModuleMetadataIncludeFilter; 958 } 959 960 /** 961 * @return the mModuleMetadataExcludeFilter 962 */ getModuleMetadataExcludeFilter()963 protected MultiMap<String, String> getModuleMetadataExcludeFilter() { 964 return mModuleMetadataExcludeFilter; 965 } 966 967 /** 968 * @return the mTotalShards 969 */ getTotalShards()970 protected int getTotalShards() { 971 return mTotalShards; 972 } 973 974 /** 975 * @return the mShardIndex 976 */ getShardIndex()977 protected Integer getShardIndex() { 978 return mShardIndex; 979 } 980 981 /** 982 * @return the mBuildHelper 983 */ getBuildHelper()984 protected CompatibilityBuildHelper getBuildHelper() { 985 return mBuildHelper; 986 } 987 988 /** 989 * @return the mInvocationContext 990 */ getInvocationContext()991 protected IInvocationContext getInvocationContext() { 992 return mInvocationContext; 993 } 994 995 /** 996 * @return the mModuleRepo 997 */ getModuleRepo()998 protected IModuleRepo getModuleRepo() { 999 return mModuleRepo; 1000 } 1001 } 1002