1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype.suite.retry; 17 18 import com.android.annotations.VisibleForTesting; 19 import com.android.tradefed.config.ConfigurationException; 20 import com.android.tradefed.config.ConfigurationFactory; 21 import com.android.tradefed.config.IConfiguration; 22 import com.android.tradefed.config.IConfigurationFactory; 23 import com.android.tradefed.config.IConfigurationReceiver; 24 import com.android.tradefed.config.Option; 25 import com.android.tradefed.config.Option.Importance; 26 import com.android.tradefed.device.DeviceNotAvailableException; 27 import com.android.tradefed.invoker.IRescheduler; 28 import com.android.tradefed.log.FileLogger; 29 import com.android.tradefed.log.ILeveledLogOutput; 30 import com.android.tradefed.log.LogUtil.CLog; 31 import com.android.tradefed.result.CollectingTestListener; 32 import com.android.tradefed.result.ITestInvocationListener; 33 import com.android.tradefed.result.TestDescription; 34 import com.android.tradefed.result.TestResult; 35 import com.android.tradefed.result.TestRunResult; 36 import com.android.tradefed.result.TextResultReporter; 37 import com.android.tradefed.result.proto.TestRecordProto.TestRecord; 38 import com.android.tradefed.testtype.IRemoteTest; 39 import com.android.tradefed.testtype.suite.BaseTestSuite; 40 import com.android.tradefed.testtype.suite.SuiteTestFilter; 41 import com.android.tradefed.util.AbiUtils; 42 import com.android.tradefed.util.QuotationAwareTokenizer; 43 import com.android.tradefed.util.TestRecordInterpreter; 44 45 import com.google.inject.Inject; 46 47 import java.util.ArrayList; 48 import java.util.HashSet; 49 import java.util.LinkedHashMap; 50 import java.util.LinkedHashSet; 51 import java.util.List; 52 import java.util.Map; 53 import java.util.Map.Entry; 54 import java.util.Set; 55 56 /** 57 * A special runner that allows to reschedule a previous run tests that failed or where not 58 * executed. 59 * 60 * <p>TODO: Ensure a configuration should not have several of that runner. Consider having this 61 * configuration built-in TF. 62 */ 63 public final class RetryRescheduler implements IRemoteTest, IConfigurationReceiver { 64 65 /** The types of the tests that can be retried. */ 66 public enum RetryType { 67 FAILED, 68 NOT_EXECUTED, 69 } 70 71 @Option( 72 name = "retry-type", 73 description = 74 "used to retry tests of a certain status. Possible values include \"failed\" " 75 + "and \"not_executed\".") 76 private RetryType mRetryType = null; 77 78 /** 79 * It's possible to add extra exclusion from the rerun. But these tests will not change their 80 * state. 81 */ 82 @Option( 83 name = BaseTestSuite.EXCLUDE_FILTER_OPTION, 84 description = "the exclude module filters to apply.", 85 importance = Importance.ALWAYS 86 ) 87 private Set<String> mExcludeFilters = new HashSet<>(); 88 89 public static final String PREVIOUS_LOADER_NAME = "previous_loader"; 90 91 private IConfiguration mConfiguration; 92 private IRescheduler mRescheduler; 93 94 private IConfigurationFactory mFactory; 95 96 private IConfiguration mRescheduledConfiguration; 97 98 @Override run(ITestInvocationListener listener)99 public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 100 // Get the re-loader for previous results 101 Object loader = mConfiguration.getConfigurationObject(PREVIOUS_LOADER_NAME); 102 if (loader == null) { 103 throw new RuntimeException( 104 String.format( 105 "An <object> of type %s was expected in the retry.", 106 PREVIOUS_LOADER_NAME)); 107 } 108 if (!(loader instanceof ITestSuiteResultLoader)) { 109 throw new RuntimeException( 110 String.format( 111 "%s should be implementing %s", 112 loader.getClass().getCanonicalName(), 113 ITestSuiteResultLoader.class.getCanonicalName())); 114 } 115 116 ITestSuiteResultLoader previousLoader = (ITestSuiteResultLoader) loader; 117 // First init the reloader. 118 previousLoader.init(); 119 // Then get the command line of the previous run 120 String commandLine = previousLoader.getCommandLine(); 121 IConfiguration originalConfig; 122 try { 123 originalConfig = 124 getFactory() 125 .createConfigurationFromArgs( 126 QuotationAwareTokenizer.tokenizeLine(commandLine)); 127 // Transfer the sharding options from the original command. 128 originalConfig 129 .getCommandOptions() 130 .setShardCount(mConfiguration.getCommandOptions().getShardCount()); 131 originalConfig 132 .getCommandOptions() 133 .setShardIndex(mConfiguration.getCommandOptions().getShardIndex()); 134 // TODO: Use serial from parent config 135 List<String> serials = mConfiguration.getDeviceRequirements().getSerials(); 136 originalConfig.getDeviceRequirements().setSerial(serials.toArray(new String[0])); 137 138 // Transfer log level from retry to subconfig 139 ILeveledLogOutput originalLogger = originalConfig.getLogOutput(); 140 originalLogger.setLogLevel(mConfiguration.getLogOutput().getLogLevel()); 141 if (originalLogger instanceof FileLogger) { 142 ((FileLogger) originalLogger) 143 .setLogLevelDisplay(mConfiguration.getLogOutput().getLogLevel()); 144 } 145 146 handleExtraResultReporter(originalConfig, mConfiguration); 147 } catch (ConfigurationException e) { 148 throw new RuntimeException(e); 149 } 150 // Get previous results 151 TestRecord previousRecord = previousLoader.loadPreviousRecord(); 152 CollectingTestListener collectedTests = 153 TestRecordInterpreter.interpreteRecord(previousRecord); 154 155 previousLoader.cleanUp(); 156 previousRecord = null; 157 158 // Appropriately update the configuration 159 IRemoteTest test = originalConfig.getTests().get(0); 160 if (!(test instanceof BaseTestSuite)) { 161 throw new RuntimeException( 162 "RetryScheduler only works for BaseTestSuite implementations"); 163 } 164 BaseTestSuite suite = (BaseTestSuite) test; 165 ResultsPlayer replayer = new ResultsPlayer(); 166 updateRunner(suite, collectedTests, replayer); 167 collectedTests = null; 168 updateConfiguration(originalConfig, replayer); 169 // Do the customization of the configuration for specialized use cases. 170 customizeConfig(previousLoader, originalConfig); 171 172 mRescheduledConfiguration = originalConfig; 173 174 if (mRescheduler != null) { 175 // At the end, reschedule if requested 176 boolean res = mRescheduler.scheduleConfig(originalConfig); 177 if (!res) { 178 CLog.e("Something went wrong, failed to kick off the retry run."); 179 } 180 } 181 } 182 183 @Inject setRescheduler(IRescheduler rescheduler)184 public void setRescheduler(IRescheduler rescheduler) { 185 mRescheduler = rescheduler; 186 } 187 188 @Override setConfiguration(IConfiguration configuration)189 public void setConfiguration(IConfiguration configuration) { 190 mConfiguration = configuration; 191 } 192 getFactory()193 private IConfigurationFactory getFactory() { 194 if (mFactory != null) { 195 return mFactory; 196 } 197 return ConfigurationFactory.getInstance(); 198 } 199 200 @VisibleForTesting setConfigurationFactory(IConfigurationFactory factory)201 void setConfigurationFactory(IConfigurationFactory factory) { 202 mFactory = factory; 203 } 204 205 /** Returns the {@link IConfiguration} that should be retried. */ getRetryConfiguration()206 public final IConfiguration getRetryConfiguration() { 207 return mRescheduledConfiguration; 208 } 209 210 /** 211 * Update the configuration to be ready for re-run. 212 * 213 * @param suite The {@link BaseTestSuite} that will be re-run. 214 * @param results The results of the previous run. 215 * @param replayer The {@link ResultsPlayer} that will replay the non-retried use cases. 216 */ updateRunner( BaseTestSuite suite, CollectingTestListener results, ResultsPlayer replayer)217 private void updateRunner( 218 BaseTestSuite suite, CollectingTestListener results, ResultsPlayer replayer) { 219 List<RetryType> types = new ArrayList<>(); 220 if (mRetryType == null) { 221 types.add(RetryType.FAILED); 222 types.add(RetryType.NOT_EXECUTED); 223 } else { 224 types.add(mRetryType); 225 } 226 227 // Expand the exclude-filter in case no abi is specified. 228 Set<String> extendedExcludeRetryFilters = new HashSet<>(); 229 for (String excludeFilter : mExcludeFilters) { 230 SuiteTestFilter suiteFilter = SuiteTestFilter.createFrom(excludeFilter); 231 // Keep the current exclude-filter 232 extendedExcludeRetryFilters.add(excludeFilter); 233 if (suiteFilter.getAbi() == null) { 234 // If no abi is specified, exclude them all. 235 Set<String> abis = AbiUtils.getAbisSupportedByCompatibility(); 236 for (String abi : abis) { 237 SuiteTestFilter namingFilter = 238 new SuiteTestFilter(abi, suiteFilter.getName(), suiteFilter.getTest()); 239 extendedExcludeRetryFilters.add(namingFilter.toString()); 240 } 241 } 242 } 243 244 // Prepare exclusion filters 245 for (TestRunResult moduleResult : results.getMergedTestRunResults()) { 246 // If the module is explicitly excluded from retries, preserve the original results. 247 if (!extendedExcludeRetryFilters.contains(moduleResult.getName()) 248 && RetryResultHelper.shouldRunModule(moduleResult, types)) { 249 if (types.contains(RetryType.NOT_EXECUTED)) { 250 // Clear the run failure since we are attempting to rerun all non-executed 251 moduleResult.resetRunFailure(); 252 } 253 254 Map<TestDescription, TestResult> parameterizedMethods = new LinkedHashMap<>(); 255 256 for (Entry<TestDescription, TestResult> result : 257 moduleResult.getTestResults().entrySet()) { 258 // Put aside all parameterized methods 259 if (isParameterized(result.getKey())) { 260 parameterizedMethods.put(result.getKey(), result.getValue()); 261 continue; 262 } 263 if (!RetryResultHelper.shouldRunTest(result.getValue(), types)) { 264 addExcludeToConfig(suite, moduleResult, result.getKey().toString()); 265 replayer.addToReplay( 266 results.getModuleContextForRunResult(moduleResult.getName()), 267 moduleResult, 268 result); 269 } 270 } 271 272 // Handle parameterized methods 273 for (Entry<String, Map<TestDescription, TestResult>> subMap : 274 sortMethodToClass(parameterizedMethods).entrySet()) { 275 boolean shouldNotrerunAnything = 276 subMap.getValue() 277 .entrySet() 278 .stream() 279 .noneMatch( 280 (v) -> 281 RetryResultHelper.shouldRunTest( 282 v.getValue(), types) 283 == true); 284 // If None of the base method need to be rerun exclude it 285 if (shouldNotrerunAnything) { 286 // Exclude the base method 287 addExcludeToConfig(suite, moduleResult, subMap.getKey()); 288 // Replay all test cases 289 for (Entry<TestDescription, TestResult> result : 290 subMap.getValue().entrySet()) { 291 replayer.addToReplay( 292 results.getModuleContextForRunResult(moduleResult.getName()), 293 moduleResult, 294 result); 295 } 296 } 297 } 298 } else { 299 // Exclude the module completely - it will keep its current status 300 addExcludeToConfig(suite, moduleResult, null); 301 replayer.addToReplay( 302 results.getModuleContextForRunResult(moduleResult.getName()), 303 moduleResult, 304 null); 305 } 306 } 307 } 308 309 /** Update the configuration to put the replayer before all the actual real tests. */ updateConfiguration(IConfiguration config, ResultsPlayer replayer)310 private void updateConfiguration(IConfiguration config, ResultsPlayer replayer) { 311 List<IRemoteTest> tests = config.getTests(); 312 List<IRemoteTest> newList = new ArrayList<>(); 313 // Add the replayer first to replay all the tests cases first. 314 newList.add(replayer); 315 newList.addAll(tests); 316 config.setTests(newList); 317 } 318 319 /** Allow the specialized loader to customize the config before re-running it. */ customizeConfig(ITestSuiteResultLoader loader, IConfiguration originalConfig)320 private void customizeConfig(ITestSuiteResultLoader loader, IConfiguration originalConfig) { 321 loader.customizeConfiguration(originalConfig); 322 } 323 324 /** Add the filter to the suite. */ addExcludeToConfig( BaseTestSuite suite, TestRunResult moduleResult, String testDescription)325 private void addExcludeToConfig( 326 BaseTestSuite suite, TestRunResult moduleResult, String testDescription) { 327 String filter = moduleResult.getName(); 328 if (testDescription != null) { 329 filter = String.format("%s %s", filter, testDescription); 330 } 331 SuiteTestFilter testFilter = SuiteTestFilter.createFrom(filter); 332 Set<String> excludeFilter = new LinkedHashSet<>(); 333 excludeFilter.add(testFilter.toString()); 334 suite.setExcludeFilter(excludeFilter); 335 } 336 337 /** Returns True if a test case is a parameterized one. */ isParameterized(TestDescription description)338 private boolean isParameterized(TestDescription description) { 339 return !description.getTestName().equals(description.getTestNameWithoutParams()); 340 } 341 sortMethodToClass( Map<TestDescription, TestResult> paramMethods)342 private Map<String, Map<TestDescription, TestResult>> sortMethodToClass( 343 Map<TestDescription, TestResult> paramMethods) { 344 Map<String, Map<TestDescription, TestResult>> returnMap = new LinkedHashMap<>(); 345 for (Entry<TestDescription, TestResult> entry : paramMethods.entrySet()) { 346 String noParamName = 347 String.format( 348 "%s#%s", 349 entry.getKey().getClassName(), 350 entry.getKey().getTestNameWithoutParams()); 351 Map<TestDescription, TestResult> forClass = returnMap.get(noParamName); 352 if (forClass == null) { 353 forClass = new LinkedHashMap<>(); 354 returnMap.put(noParamName, forClass); 355 } 356 forClass.put(entry.getKey(), entry.getValue()); 357 } 358 return returnMap; 359 } 360 361 /** 362 * Fetch additional result_reporter from the retry configuration and add them to the original 363 * command. This is the only allowed modification of the original command: add more result 364 * end-points. 365 */ handleExtraResultReporter( IConfiguration originalConfig, IConfiguration retryConfig)366 private void handleExtraResultReporter( 367 IConfiguration originalConfig, IConfiguration retryConfig) { 368 // Since we always have 1 default reporter, avoid carrying it for no reason. Only carry 369 // reporters if some actual ones were specified. 370 if (retryConfig.getTestInvocationListeners().size() == 1 371 && (mConfiguration.getTestInvocationListeners().get(0) 372 instanceof TextResultReporter)) { 373 return; 374 } 375 List<ITestInvocationListener> listeners = originalConfig.getTestInvocationListeners(); 376 listeners.addAll(retryConfig.getTestInvocationListeners()); 377 originalConfig.setTestInvocationListeners(listeners); 378 } 379 } 380