1 /* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.ota.tests; 17 18 import com.android.ddmlib.Log; 19 import com.android.tradefed.build.IBuildInfo; 20 import com.android.tradefed.config.ConfigurationException; 21 import com.android.tradefed.config.IConfiguration; 22 import com.android.tradefed.config.IConfigurationReceiver; 23 import com.android.tradefed.config.Option; 24 import com.android.tradefed.config.Option.Importance; 25 import com.android.tradefed.config.OptionClass; 26 import com.android.tradefed.device.DeviceNotAvailableException; 27 import com.android.tradefed.device.ITestDevice; 28 import com.android.tradefed.log.LogUtil.CLog; 29 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 30 import com.android.tradefed.result.FileInputStreamSource; 31 import com.android.tradefed.result.ITestInvocationListener; 32 import com.android.tradefed.result.InputStreamSource; 33 import com.android.tradefed.result.LogDataType; 34 import com.android.tradefed.targetprep.BuildError; 35 import com.android.tradefed.targetprep.ITargetPreparer; 36 import com.android.tradefed.targetprep.TargetSetupError; 37 import com.android.tradefed.testtype.IBuildReceiver; 38 import com.android.tradefed.testtype.IDeviceTest; 39 import com.android.tradefed.testtype.IRemoteTest; 40 import com.android.tradefed.testtype.IResumableTest; 41 import com.android.tradefed.testtype.IShardableTest; 42 import com.android.tradefed.util.FileUtil; 43 import com.android.tradefed.util.IRunUtil; 44 import com.android.tradefed.util.RunUtil; 45 import com.android.tradefed.util.StreamUtil; 46 import com.android.tradefed.util.proto.TfMetricProtoUtil; 47 48 import org.junit.Assert; 49 50 import java.io.File; 51 import java.io.IOException; 52 import java.util.ArrayList; 53 import java.util.Collection; 54 import java.util.HashMap; 55 56 /** 57 * A test that will flash a build on a device, wait for the device to be OTA-ed to another build, 58 * and then repeat N times. 59 * 60 * <p>Note: this test assumes that the {@link ITargetPreparer}s included in this test's {@link 61 * IConfiguration} will flash the device back to a baseline build, and prepare the device to receive 62 * the OTA to a new build. 63 */ 64 @OptionClass(alias = "ota-stability") 65 public class OtaStabilityTest 66 implements IDeviceTest, 67 IBuildReceiver, 68 IConfigurationReceiver, 69 IShardableTest, 70 IResumableTest { 71 72 private static final String LOG_TAG = "OtaStabilityTest"; 73 private IBuildInfo mDeviceBuild; 74 private IConfiguration mConfiguration; 75 private ITestDevice mDevice; 76 77 @Option( 78 name = "run-name", 79 description = "The name of the ota stability test run. Used to report metrics.") 80 private String mRunName = "ota-stability"; 81 82 @Option( 83 name = "iterations", 84 description = "Number of ota stability 'flash + wait for ota' iterations to run.") 85 private int mIterations = 20; 86 87 @Option( 88 name = "wait-recovery-time", 89 description = "Number of minutes to wait for device to begin installing ota.") 90 private int mWaitRecoveryTime = 15; 91 92 @Option( 93 name = "wait-install-time", 94 description = 95 "Number of minutes to wait for device to be online after beginning ota installation.") 96 private int mWaitInstallTime = 10; 97 98 @Option( 99 name = "shards", 100 description = 101 "Optional number of shards to split test into. " 102 + "Iterations will be split evenly among shards.", 103 importance = Importance.IF_UNSET) 104 private Integer mShards = null; 105 106 @Option( 107 name = "resume", 108 description = 109 "Resume the ota test run if an device setup error " 110 + "stopped the previous test run.") 111 private boolean mResumeMode = false; 112 113 /** controls if this test should be resumed. Only used if mResumeMode is enabled */ 114 private boolean mResumable = true; 115 116 /** {@inheritDoc} */ 117 @Override setConfiguration(IConfiguration configuration)118 public void setConfiguration(IConfiguration configuration) { 119 mConfiguration = configuration; 120 } 121 122 /** {@inheritDoc} */ 123 @Override setBuild(IBuildInfo buildInfo)124 public void setBuild(IBuildInfo buildInfo) { 125 mDeviceBuild = buildInfo; 126 } 127 128 /** {@inheritDoc} */ 129 @Override setDevice(ITestDevice device)130 public void setDevice(ITestDevice device) { 131 mDevice = device; 132 } 133 134 /** {@inheritDoc} */ 135 @Override getDevice()136 public ITestDevice getDevice() { 137 return mDevice; 138 } 139 140 /** Set the wait recovery time */ setWaitRecoveryTime(int waitRecoveryTime)141 void setWaitRecoveryTime(int waitRecoveryTime) { 142 mWaitRecoveryTime = waitRecoveryTime; 143 } 144 145 /** Set the wait install time */ setWaitInstallTime(int waitInstallTime)146 void setWaitInstallTime(int waitInstallTime) { 147 mWaitInstallTime = waitInstallTime; 148 } 149 150 /** Set the run name */ setRunName(String runName)151 void setRunName(String runName) { 152 mRunName = runName; 153 } 154 155 /** 156 * Return the number of iterations. 157 * 158 * <p>Exposed for unit testing 159 */ getIterations()160 public int getIterations() { 161 return mIterations; 162 } 163 164 /** Set the iterations */ setIterations(int iterations)165 void setIterations(int iterations) { 166 mIterations = iterations; 167 } 168 169 /** Set the number of shards */ setShards(int shards)170 void setShards(int shards) { 171 mShards = shards; 172 } 173 174 /** {@inheritDoc} */ 175 @Override split()176 public Collection<IRemoteTest> split() { 177 if (mShards == null || mShards <= 1) { 178 return null; 179 } 180 Collection<IRemoteTest> shards = new ArrayList<>(mShards); 181 int remainingIterations = mIterations; 182 for (int i = mShards; i > 0; i--) { 183 OtaStabilityTest testShard = new OtaStabilityTest(); 184 // device and configuration will be set by test invoker 185 testShard.setRunName(mRunName); 186 testShard.setWaitInstallTime(mWaitInstallTime); 187 testShard.setWaitRecoveryTime(mWaitRecoveryTime); 188 // attempt to divide iterations evenly among shards with no remainder 189 int iterationsForShard = Math.round(remainingIterations / i); 190 if (iterationsForShard > 0) { 191 testShard.setIterations(iterationsForShard); 192 remainingIterations -= iterationsForShard; 193 shards.add(testShard); 194 } 195 } 196 return shards; 197 } 198 199 /** {@inheritDoc} */ 200 @Override run(ITestInvocationListener listener)201 public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 202 // started run, turn to off 203 mResumable = false; 204 checkFields(); 205 206 long startTime = System.currentTimeMillis(); 207 listener.testRunStarted(mRunName, 0); 208 int actualIterations = 0; 209 try { 210 waitForOta(listener); 211 for (actualIterations = 1; actualIterations < mIterations; actualIterations++) { 212 flashDevice(); 213 String buildId = waitForOta(listener); 214 Log.i( 215 LOG_TAG, 216 String.format( 217 "Device %s successfully OTA-ed to build %s. Iteration: %d", 218 mDevice.getSerialNumber(), buildId, actualIterations)); 219 } 220 } catch (AssertionError error) { 221 Log.e(LOG_TAG, error); 222 } catch (TargetSetupError e) { 223 CLog.i("Encountered TargetSetupError, marking this test as resumable"); 224 mResumable = true; 225 CLog.e(e); 226 // throw up an exception so this test can be resumed 227 Assert.fail(e.toString()); 228 } catch (BuildError e) { 229 Log.e(LOG_TAG, e); 230 } catch (ConfigurationException e) { 231 Log.e(LOG_TAG, e); 232 } finally { 233 HashMap<String, Metric> metrics = new HashMap<>(); 234 metrics.put( 235 "iterations", 236 TfMetricProtoUtil.stringToMetric(Integer.toString(actualIterations))); 237 long endTime = System.currentTimeMillis() - startTime; 238 listener.testRunEnded(endTime, metrics); 239 } 240 } 241 242 /** 243 * Flash the device back to baseline build. 244 * 245 * <p>Currently does this by re-running {@link ITargetPreparer#setUp(ITestDevice, IBuildInfo)} 246 * 247 * @throws DeviceNotAvailableException 248 * @throws BuildError 249 * @throws TargetSetupError 250 * @throws ConfigurationException 251 */ flashDevice()252 private void flashDevice() 253 throws TargetSetupError, BuildError, DeviceNotAvailableException, 254 ConfigurationException { 255 // assume the target preparers will flash the device back to device build 256 for (ITargetPreparer preparer : mConfiguration.getTargetPreparers()) { 257 preparer.setUp(mDevice, mDeviceBuild); 258 } 259 } 260 261 /** 262 * Get the {@link IRunUtil} instance to use. 263 * 264 * <p>Exposed so unit tests can mock. 265 */ getRunUtil()266 IRunUtil getRunUtil() { 267 return RunUtil.getDefault(); 268 } 269 270 /** 271 * Blocks and waits for OTA package to be installed. 272 * 273 * @param listener the {@link ITestInvocationListener} 274 * @return the build id the device ota-ed to 275 * @throws DeviceNotAvailableException 276 * @throws AssertionError 277 */ waitForOta(ITestInvocationListener listener)278 private String waitForOta(ITestInvocationListener listener) 279 throws DeviceNotAvailableException, AssertionError { 280 String currentBuildId = mDevice.getBuildId(); 281 Assert.assertEquals( 282 String.format( 283 "device %s does not have expected build id on boot.", 284 mDevice.getSerialNumber()), 285 currentBuildId, 286 mDeviceBuild.getBuildId()); 287 // give some time for device to settle 288 getRunUtil().sleep(5 * 1000); 289 // force a checkin so device downloads OTA immediately 290 mDevice.executeShellCommand( 291 "am broadcast -a android.server.checkin.CHECKIN com.google.android.gms"); 292 Assert.assertTrue( 293 String.format( 294 "Device %s did not enter recovery after %d min.", 295 mDevice.getSerialNumber(), mWaitRecoveryTime), 296 mDevice.waitForDeviceInRecovery(mWaitRecoveryTime * 60 * 1000)); 297 try { 298 mDevice.waitForDeviceOnline(mWaitInstallTime * 60 * 1000); 299 } catch (DeviceNotAvailableException e) { 300 Log.e( 301 LOG_TAG, 302 String.format( 303 "Device %s did not come back online after leaving recovery", 304 mDevice.getSerialNumber())); 305 sendRecoveryLog(listener); 306 throw e; 307 } 308 try { 309 mDevice.waitForDeviceAvailable(); 310 } catch (DeviceNotAvailableException e) { 311 Log.e( 312 LOG_TAG, 313 String.format( 314 "Device %s did not boot up successfully after leaving recovery/installing OTA", 315 mDevice.getSerialNumber())); 316 throw e; 317 } 318 currentBuildId = mDevice.getBuildId(); 319 // TODO: should exact expected build id be checked? 320 Assert.assertNotEquals( 321 String.format( 322 "Device %s build id did not change after leaving recovery", 323 mDevice.getSerialNumber()), 324 currentBuildId, 325 mDeviceBuild.getBuildId()); 326 return currentBuildId; 327 } 328 sendRecoveryLog(ITestInvocationListener listener)329 private void sendRecoveryLog(ITestInvocationListener listener) 330 throws DeviceNotAvailableException { 331 File destFile = null; 332 InputStreamSource destSource = null; 333 try { 334 // get recovery log 335 destFile = FileUtil.createTempFile("recovery", "log"); 336 boolean gotFile = mDevice.pullFile("/tmp/recovery.log", destFile); 337 if (gotFile) { 338 destSource = new FileInputStreamSource(destFile); 339 listener.testLog("recovery_log", LogDataType.TEXT, destSource); 340 } 341 } catch (IOException e) { 342 Log.e( 343 LOG_TAG, 344 String.format( 345 "Failed to get recovery log from device %s", 346 mDevice.getSerialNumber())); 347 Log.e(LOG_TAG, e); 348 } finally { 349 FileUtil.deleteFile(destFile); 350 StreamUtil.cancel(destSource); 351 } 352 } 353 checkFields()354 private void checkFields() { 355 if (mDevice == null) { 356 throw new IllegalArgumentException("missing device"); 357 } 358 if (mConfiguration == null) { 359 throw new IllegalArgumentException("missing configuration"); 360 } 361 if (mDeviceBuild == null) { 362 throw new IllegalArgumentException("missing build info"); 363 } 364 } 365 366 /** {@inheritDoc} */ 367 @Override isResumable()368 public boolean isResumable() { 369 return mResumeMode && mResumable; 370 } 371 } 372