1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.invoker.shard; 17 18 import com.android.annotations.VisibleForTesting; 19 import com.android.ddmlib.Log.LogLevel; 20 import com.android.tradefed.build.IBuildInfo; 21 import com.android.tradefed.config.Configuration; 22 import com.android.tradefed.config.ConfigurationException; 23 import com.android.tradefed.config.IConfiguration; 24 import com.android.tradefed.config.IConfigurationReceiver; 25 import com.android.tradefed.device.DeviceNotAvailableException; 26 import com.android.tradefed.device.DeviceUnresponsiveException; 27 import com.android.tradefed.device.ITestDevice; 28 import com.android.tradefed.device.cloud.NestedRemoteDevice; 29 import com.android.tradefed.device.metric.IMetricCollector; 30 import com.android.tradefed.device.metric.IMetricCollectorReceiver; 31 import com.android.tradefed.invoker.IInvocationContext; 32 import com.android.tradefed.invoker.shard.token.ITokenProvider; 33 import com.android.tradefed.invoker.shard.token.ITokenRequest; 34 import com.android.tradefed.invoker.shard.token.TokenProperty; 35 import com.android.tradefed.invoker.shard.token.TokenProviderHelper; 36 import com.android.tradefed.log.ILogRegistry; 37 import com.android.tradefed.log.ILogRegistry.EventType; 38 import com.android.tradefed.log.ITestLogger; 39 import com.android.tradefed.log.LogRegistry; 40 import com.android.tradefed.log.LogUtil.CLog; 41 import com.android.tradefed.result.ITestInvocationListener; 42 import com.android.tradefed.suite.checker.ISystemStatusChecker; 43 import com.android.tradefed.suite.checker.ISystemStatusCheckerReceiver; 44 import com.android.tradefed.testtype.IBuildReceiver; 45 import com.android.tradefed.testtype.IDeviceTest; 46 import com.android.tradefed.testtype.IInvocationContextReceiver; 47 import com.android.tradefed.testtype.IMultiDeviceTest; 48 import com.android.tradefed.testtype.IRemoteTest; 49 import com.android.tradefed.testtype.IReportNotExecuted; 50 import com.android.tradefed.util.StreamUtil; 51 import com.android.tradefed.util.TimeUtil; 52 53 import com.google.common.collect.Sets; 54 55 import java.util.ArrayList; 56 import java.util.Collection; 57 import java.util.HashMap; 58 import java.util.Iterator; 59 import java.util.List; 60 import java.util.Map; 61 import java.util.Set; 62 import java.util.concurrent.CountDownLatch; 63 64 /** 65 * Tests wrapper that allow to execute all the tests of a pool of tests. Tests can be shared by 66 * another {@link TestsPoolPoller} so synchronization is required. 67 * 68 * <p>TODO: Add handling for token module/tests. 69 */ 70 public final class TestsPoolPoller 71 implements IRemoteTest, 72 IConfigurationReceiver, 73 IDeviceTest, 74 IBuildReceiver, 75 IMultiDeviceTest, 76 IInvocationContextReceiver, 77 ISystemStatusCheckerReceiver, 78 IMetricCollectorReceiver { 79 80 private static final long WAIT_RECOVERY_TIME = 15 * 60 * 1000; 81 82 private Collection<IRemoteTest> mGenericPool; 83 private Collection<ITokenRequest> mTokenPool; 84 private CountDownLatch mTracker; 85 private Set<ITokenRequest> mRejectedToken; 86 87 private ITestDevice mDevice; 88 private IBuildInfo mBuildInfo; 89 private IInvocationContext mContext; 90 private Map<ITestDevice, IBuildInfo> mDeviceInfos; 91 private IConfiguration mConfig; 92 private List<ISystemStatusChecker> mSystemStatusCheckers; 93 private List<IMetricCollector> mCollectors; 94 95 private ILogRegistry mRegistry = null; 96 97 /** 98 * Ctor where the pool of {@link IRemoteTest} is provided. 99 * 100 * @param tests {@link IRemoteTest}s pool of all tests. 101 * @param tracker a {@link CountDownLatch} shared to get the number of running poller. 102 */ TestsPoolPoller(Collection<IRemoteTest> tests, CountDownLatch tracker)103 public TestsPoolPoller(Collection<IRemoteTest> tests, CountDownLatch tracker) { 104 mGenericPool = tests; 105 mTracker = tracker; 106 } 107 TestsPoolPoller( Collection<IRemoteTest> tests, Collection<ITokenRequest> tokenTests, CountDownLatch tracker)108 public TestsPoolPoller( 109 Collection<IRemoteTest> tests, 110 Collection<ITokenRequest> tokenTests, 111 CountDownLatch tracker) { 112 this(tests, tracker); 113 mTokenPool = tokenTests; 114 mRejectedToken = Sets.newConcurrentHashSet(); 115 } 116 117 /** Returns the first {@link IRemoteTest} from the pool or null if none remaining. */ poll()118 IRemoteTest poll() { 119 return poll(false); 120 } 121 122 /** Returns the first {@link IRemoteTest} from the pool or null if none remaining. */ poll(boolean reportNotExecuted)123 private IRemoteTest poll(boolean reportNotExecuted) { 124 if (mTokenPool != null) { 125 synchronized (mTokenPool) { 126 if (!mTokenPool.isEmpty()) { 127 Iterator<ITokenRequest> itr = mTokenPool.iterator(); 128 while (itr.hasNext()) { 129 ITokenRequest test = itr.next(); 130 if (reportNotExecuted) { 131 // Return to report not executed tests, regardless of if they can 132 // actually execute or not. 133 mRejectedToken.remove(test); 134 mTokenPool.remove(test); 135 return test; 136 } 137 if (mRejectedToken.contains(test)) { 138 // If the poller already rejected the tests once, do not re-evaluate. 139 continue; 140 } 141 Set<TokenProperty> tokens = test.getRequiredTokens(); 142 if (tokens == null || tokens.isEmpty() || isSupported(tokens)) { 143 // No Token can run anywhere, or supported can run 144 mTokenPool.remove(test); 145 mRejectedToken.remove(test); 146 return test; 147 } 148 149 // Track as rejected 150 mRejectedToken.add(test); 151 } 152 } 153 } 154 } 155 synchronized (mGenericPool) { 156 if (mGenericPool.isEmpty()) { 157 return null; 158 } 159 IRemoteTest test = mGenericPool.iterator().next(); 160 mGenericPool.remove(test); 161 return test; 162 } 163 } 164 pollRejectedTokenModule()165 private ITokenRequest pollRejectedTokenModule() { 166 if (mTokenPool == null) { 167 return null; 168 } 169 synchronized (mTokenPool) { 170 if (mRejectedToken.isEmpty()) { 171 return null; 172 } 173 ITokenRequest test = mRejectedToken.iterator().next(); 174 mRejectedToken.remove(test); 175 mTokenPool.remove(test); 176 return test; 177 } 178 } 179 180 /** {@inheritDoc} */ 181 @Override run(ITestInvocationListener listener)182 public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 183 try { 184 ITestInvocationListener listenerWithCollectors = listener; 185 for (IMetricCollector collector : mCollectors) { 186 listenerWithCollectors = collector.init(mContext, listenerWithCollectors); 187 } 188 while (true) { 189 IRemoteTest test = poll(); 190 if (test == null) { 191 return; 192 } 193 if (test instanceof IBuildReceiver) { 194 ((IBuildReceiver) test).setBuild(mBuildInfo); 195 } 196 if (test instanceof IConfigurationReceiver) { 197 ((IConfigurationReceiver) test).setConfiguration(mConfig); 198 } 199 if (test instanceof IDeviceTest) { 200 ((IDeviceTest) test).setDevice(mDevice); 201 } 202 if (test instanceof IInvocationContextReceiver) { 203 ((IInvocationContextReceiver) test).setInvocationContext(mContext); 204 } 205 if (test instanceof IMultiDeviceTest) { 206 ((IMultiDeviceTest) test).setDeviceInfos(mDeviceInfos); 207 } 208 if (test instanceof ISystemStatusCheckerReceiver) { 209 ((ISystemStatusCheckerReceiver) test) 210 .setSystemStatusChecker(mSystemStatusCheckers); 211 } 212 IConfiguration validationConfig = new Configuration("validation", "validation"); 213 try { 214 // At this point only the <test> object needs to be validated for options, this 215 // ensures that the object is fine before running it. 216 validationConfig.setTest(test); 217 validationConfig.validateOptions(true); 218 // Run the test itself and prevent random exception from stopping the poller. 219 if (test instanceof IMetricCollectorReceiver) { 220 ((IMetricCollectorReceiver) test).setMetricCollectors(mCollectors); 221 // If test can receive collectors then let it handle the how to set them up 222 test.run(listener); 223 } else { 224 test.run(listenerWithCollectors); 225 } 226 } catch (RuntimeException e) { 227 CLog.e( 228 "Caught an Exception in a test: %s. Proceeding to next test.", 229 test.getClass()); 230 CLog.e(e); 231 } catch (DeviceUnresponsiveException due) { 232 // being able to catch a DeviceUnresponsiveException here implies that recovery 233 // was successful, and test execution should proceed to next test. 234 CLog.w( 235 "Ignored DeviceUnresponsiveException because recovery was " 236 + "successful, proceeding with next test. Stack trace:"); 237 CLog.w(due); 238 CLog.w("Proceeding to the next test."); 239 } catch (DeviceNotAvailableException dnae) { 240 HandleDeviceNotAvailable(listener, dnae, test); 241 } catch (ConfigurationException e) { 242 CLog.w( 243 "Failed to validate the @options of test: %s. Proceeding to next test.", 244 test.getClass()); 245 CLog.w(e); 246 } finally { 247 validationConfig.cleanDynamicOptionFiles(); 248 } 249 } 250 } finally { 251 mTracker.countDown(); 252 if (mTracker.getCount() == 0) { 253 // If the last poller is also disconnected we want to know about the tests that 254 // did not execute. 255 reportNotExecuted(listener); 256 } 257 } 258 } 259 260 /** 261 * Helper to wait for the device to maybe come back online, in that case we reboot it to refresh 262 * the state and proceed with execution. 263 */ HandleDeviceNotAvailable( ITestLogger logger, DeviceNotAvailableException originalException, IRemoteTest test)264 void HandleDeviceNotAvailable( 265 ITestLogger logger, DeviceNotAvailableException originalException, IRemoteTest test) 266 throws DeviceNotAvailableException { 267 try { 268 if (mDevice instanceof NestedRemoteDevice) { 269 // If it's not the last device, reset it. 270 if (((NestedRemoteDevice) mDevice) 271 .resetVirtualDevice(logger, mBuildInfo, /* Collect the logs */ true)) { 272 CLog.d("Successful virtual device reset."); 273 return; 274 } 275 // Original exception will be thrown below 276 CLog.e("Virtual device %s reset failed.", mDevice.getSerialNumber()); 277 } else if (mTracker.getCount() > 1) { 278 CLog.d( 279 "Wait %s for device to maybe come back online.", 280 TimeUtil.formatElapsedTime(WAIT_RECOVERY_TIME)); 281 mDevice.waitForDeviceAvailable(WAIT_RECOVERY_TIME); 282 mDevice.reboot(); 283 CLog.d("TestPoller was recovered after %s went offline", mDevice.getSerialNumber()); 284 return; 285 } 286 } catch (DeviceNotAvailableException e) { 287 // ignore this exception 288 } 289 // We catch and rethrow in order to log that the poller associated with the device 290 // that went offline is terminating. 291 CLog.e( 292 "Test %s threw DeviceNotAvailableException. Test poller associated with " 293 + "device %s is terminating.", 294 test.getClass(), mDevice.getSerialNumber()); 295 // Log an event to track more easily the failure 296 logDeviceEvent( 297 EventType.SHARD_POLLER_EARLY_TERMINATION, 298 mDevice.getSerialNumber(), 299 originalException); 300 throw originalException; 301 } 302 303 /** Go through the remaining IRemoteTest and report them as not executed. */ reportNotExecuted(ITestInvocationListener listener)304 private void reportNotExecuted(ITestInvocationListener listener) { 305 // Report non-executed token test first 306 ITokenRequest tokenTest = pollRejectedTokenModule(); 307 while (tokenTest != null) { 308 if (tokenTest instanceof IReportNotExecuted) { 309 String message = 310 String.format( 311 "Test did not run. No token '%s' matching it on any device.", 312 tokenTest.getRequiredTokens()); 313 ((IReportNotExecuted) tokenTest).reportNotExecuted(listener, message); 314 } else { 315 CLog.e( 316 "Could not report not executed tests from %s.", 317 tokenTest.getClass().getCanonicalName()); 318 } 319 tokenTest = pollRejectedTokenModule(); 320 } 321 // Report all remaining test 322 IRemoteTest test = poll(true); 323 while (test != null) { 324 if (test instanceof IReportNotExecuted) { 325 ((IReportNotExecuted) test).reportNotExecuted(listener); 326 } else { 327 CLog.e( 328 "Could not report not executed tests from %s.", 329 test.getClass().getCanonicalName()); 330 } 331 test = poll(true); 332 } 333 } 334 335 /** Helper to log the device events. */ logDeviceEvent(EventType event, String serial, Throwable t)336 private void logDeviceEvent(EventType event, String serial, Throwable t) { 337 Map<String, String> args = new HashMap<>(); 338 args.put("serial", serial); 339 args.put("trace", StreamUtil.getStackTrace(t)); 340 getLogRegistry().logEvent(LogLevel.DEBUG, event, args); 341 } 342 getLogRegistry()343 private ILogRegistry getLogRegistry() { 344 if (mRegistry != null) { 345 return mRegistry; 346 } 347 return LogRegistry.getLogRegistry(); 348 } 349 isSupported(Set<TokenProperty> requiredTokens)350 private boolean isSupported(Set<TokenProperty> requiredTokens) { 351 for (TokenProperty prop : requiredTokens) { 352 ITokenProvider provider = TokenProviderHelper.getTokenProvider(prop); 353 if (provider == null) { 354 CLog.e("No provider for token %s", prop); 355 return false; 356 } 357 if (!provider.hasToken(mDevice, prop)) { 358 return false; 359 } 360 } 361 return true; 362 } 363 364 @VisibleForTesting setLogRegistry(ILogRegistry registry)365 public void setLogRegistry(ILogRegistry registry) { 366 mRegistry = registry; 367 } 368 369 @Override setBuild(IBuildInfo buildInfo)370 public void setBuild(IBuildInfo buildInfo) { 371 mBuildInfo = buildInfo; 372 } 373 374 @Override setDevice(ITestDevice device)375 public void setDevice(ITestDevice device) { 376 mDevice = device; 377 } 378 379 @Override getDevice()380 public ITestDevice getDevice() { 381 return mDevice; 382 } 383 384 @Override setInvocationContext(IInvocationContext invocationContext)385 public void setInvocationContext(IInvocationContext invocationContext) { 386 mContext = invocationContext; 387 } 388 389 @Override setDeviceInfos(Map<ITestDevice, IBuildInfo> deviceInfos)390 public void setDeviceInfos(Map<ITestDevice, IBuildInfo> deviceInfos) { 391 mDeviceInfos = deviceInfos; 392 } 393 394 @Override setConfiguration(IConfiguration configuration)395 public void setConfiguration(IConfiguration configuration) { 396 mConfig = configuration; 397 } 398 399 @Override setSystemStatusChecker(List<ISystemStatusChecker> systemCheckers)400 public void setSystemStatusChecker(List<ISystemStatusChecker> systemCheckers) { 401 mSystemStatusCheckers = systemCheckers; 402 } 403 404 @Override setMetricCollectors(List<IMetricCollector> collectors)405 public void setMetricCollectors(List<IMetricCollector> collectors) { 406 mCollectors = collectors; 407 } 408 409 /** Get a copy of the pool of token tests. For testing only. */ 410 @VisibleForTesting getTokenPool()411 List<ITokenRequest> getTokenPool() { 412 if (mTokenPool == null) { 413 return null; 414 } 415 synchronized (mTokenPool) { 416 return new ArrayList<>(mTokenPool); 417 } 418 } 419 } 420