1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.testtype; 18 19 import com.android.tradefed.config.ConfigurationException; 20 import com.android.tradefed.config.OptionCopier; 21 import com.android.tradefed.device.DeviceNotAvailableException; 22 import com.android.tradefed.invoker.TestInformation; 23 import com.android.tradefed.log.LogUtil.CLog; 24 import com.android.tradefed.result.CollectingTestListener; 25 import com.android.tradefed.result.FilteredResultForwarder; 26 import com.android.tradefed.result.ITestInvocationListener; 27 import com.android.tradefed.result.TestDescription; 28 import com.android.tradefed.util.FileUtil; 29 30 import com.google.common.annotations.VisibleForTesting; 31 32 import java.io.BufferedWriter; 33 import java.io.File; 34 import java.io.FileWriter; 35 import java.io.IOException; 36 import java.util.Collection; 37 import java.util.LinkedHashMap; 38 import java.util.Map; 39 40 /** 41 * Runs a set of instrumentation tests by specifying a list of line separated test classes and 42 * methods in a file pushed to device (expected format: com.android.foo.FooClassName#testMethodName) 43 * <p> 44 * Note: Requires a runner that supports test execution from a file. Will default to serial tests 45 * execution via rerun if any issues with file creation are encountered 46 * or if all tests in the created file fail to successfully finish execution. 47 */ 48 class InstrumentationFileTest implements IRemoteTest { 49 50 // on device test folder location where the test file should be saved 51 private static final String ON_DEVICE_TEST_DIR_LOCATION = "/data/local/tmp/"; 52 /** Key that matches the -e package option for instrumentation. */ 53 private static final String PACKAGE_ARG_KEY = "package"; 54 55 private InstrumentationTest mInstrumentationTest = null; 56 57 /** the set of tests to run */ 58 private final Collection<TestDescription> mTests; 59 60 private String mFilePathOnDevice = null; 61 62 private int mAttemps; 63 64 private int mMaxAttemps; 65 66 /** 67 * Creates a {@link InstrumentationFileTest}. 68 * 69 * @param instrumentationTest {@link InstrumentationTest} used to configure this class 70 * @param testsToRun a {@link Collection} of tests to run. Note this {@link Collection} will be 71 * used as is (ie a reference to the testsToRun object will be kept). 72 */ InstrumentationFileTest( InstrumentationTest instrumentationTest, Collection<TestDescription> testsToRun, int maxAttempts)73 InstrumentationFileTest( 74 InstrumentationTest instrumentationTest, 75 Collection<TestDescription> testsToRun, 76 int maxAttempts) 77 throws ConfigurationException { 78 // reuse the InstrumentationTest class to perform actual test run 79 mInstrumentationTest = createInstrumentationTest(); 80 // copy all options from the original InstrumentationTest 81 OptionCopier.copyOptions(instrumentationTest, mInstrumentationTest); 82 mInstrumentationTest.setConfiguration(instrumentationTest.getConfiguration()); 83 mInstrumentationTest.setDevice(instrumentationTest.getDevice()); 84 mInstrumentationTest.setForceAbi(instrumentationTest.getForceAbi()); 85 mInstrumentationTest.setReRunUsingTestFile(true); 86 // no need to rerun when executing tests one by one 87 mInstrumentationTest.setRerunMode(false); 88 mInstrumentationTest.setIsRerun(true); 89 // keep local copy of tests to be run 90 mTests = testsToRun; 91 mAttemps = 0; 92 mMaxAttemps = maxAttempts; 93 } 94 95 /** {@inheritDoc} */ 96 @Override run(TestInformation testInfo, final ITestInvocationListener listener)97 public void run(TestInformation testInfo, final ITestInvocationListener listener) 98 throws DeviceNotAvailableException { 99 if (mInstrumentationTest.getDevice() == null) { 100 throw new IllegalArgumentException("Device has not been set"); 101 } 102 // reuse InstrumentationTest class to perform actual test run 103 writeTestsToFileAndRun(mTests, testInfo, listener); 104 } 105 106 /** 107 * Creates a file based on the {@link Collection} of tests to run. Upon successful file creation 108 * will push the file onto the test device and attempt to run them via {@link 109 * InstrumentationTest}. If something goes wrong, will default to serial test execution. 110 * 111 * @param tests a {@link Collection} of tests to run 112 * @param listener the test result listener 113 * @throws DeviceNotAvailableException 114 */ writeTestsToFileAndRun( Collection<TestDescription> tests, TestInformation testInfo, final ITestInvocationListener listener)115 private void writeTestsToFileAndRun( 116 Collection<TestDescription> tests, 117 TestInformation testInfo, 118 final ITestInvocationListener listener) 119 throws DeviceNotAvailableException { 120 mAttemps += 1; 121 if (mMaxAttemps > 0 && mAttemps <= mMaxAttemps) { 122 CLog.d("Try to run tests from file for the %d/%d attempts", 123 mAttemps, mMaxAttemps); 124 } else if (mMaxAttemps > 0) { 125 CLog.d("Running tests from file exceeded max attempts. Ignore the rest tests"); 126 return; 127 } 128 File testFile = null; 129 try { 130 // create and populate test file 131 testFile = FileUtil.createTempFile( 132 "tf_testFile_" + InstrumentationFileTest.class.getCanonicalName(), ".txt"); 133 try (BufferedWriter bw = new BufferedWriter(new FileWriter(testFile))) { 134 // Remove parameterized tests to only re-run their base method. 135 Collection<TestDescription> uniqueMethods = createRerunSet(tests); 136 137 for (TestDescription testToRun : uniqueMethods) { 138 // We use getTestNameNoParams to avoid attempting re-running individual 139 // parameterized tests. Instead ask the base method to re-run them all. 140 bw.write( 141 String.format( 142 "%s#%s", 143 testToRun.getClassName(), 144 testToRun.getTestNameWithoutParams())); 145 bw.newLine(); 146 } 147 CLog.d("Test file %s was successfully created", testFile.getAbsolutePath()); 148 } 149 // push test file to the device and run 150 mFilePathOnDevice = ON_DEVICE_TEST_DIR_LOCATION + testFile.getName(); 151 if (pushFileToTestDevice(testFile, mFilePathOnDevice)) { 152 // Unset package name if any just in case to avoid conflict with classname. 153 // Since at that point we explicitly request the class to rerun there is no need to 154 // keep any of the original package options. 155 mInstrumentationTest.setTestPackageName(null); 156 mInstrumentationTest.removeFromInstrumentationArg(PACKAGE_ARG_KEY); 157 mInstrumentationTest.setTestFilePathOnDevice(mFilePathOnDevice); 158 CLog.d("Test file %s was successfully pushed to %s on device", 159 testFile.getAbsolutePath(), mFilePathOnDevice); 160 runTests(mInstrumentationTest, testInfo, listener); 161 } else { 162 CLog.e("Failed to push file to device, ignore the rest of tests"); 163 } 164 } catch (IOException e) { 165 CLog.e("Failed to push file to device, ignore the rest of tests"); 166 CLog.e(e); 167 } finally { 168 // clean up test file, if it was created 169 FileUtil.deleteFile(testFile); 170 } 171 } 172 173 /** 174 * Run all tests from file. Attempt to re-run not finished tests. If all tests in file fail to 175 * run default to executing them serially. 176 */ runTests( InstrumentationTest runner, TestInformation testInfo, ITestInvocationListener listener)177 private void runTests( 178 InstrumentationTest runner, TestInformation testInfo, ITestInvocationListener listener) 179 throws DeviceNotAvailableException { 180 CollectingTestListener testTracker = new CollectingTestListener(); 181 try { 182 runner.run(testInfo, new FilteredResultForwarder(mTests, listener, testTracker)); 183 } finally { 184 deleteTestFileFromDevice(mFilePathOnDevice); 185 Collection<TestDescription> completedTests = 186 InstrumentationTest.excludeNonExecuted(testTracker.getCurrentRunResults()); 187 if (mTests.removeAll(completedTests) && !mTests.isEmpty()) { 188 // re-run remaining tests from file 189 writeTestsToFileAndRun(mTests, testInfo, listener); 190 } else if (!mTests.isEmpty()) { 191 CLog.e("all remaining tests failed to run from file, will be ignored"); 192 } 193 } 194 } 195 196 /** 197 * Returns a new collection of {@link TestDescription} where only one instance of each 198 * parameterized method is in the list. 199 */ createRerunSet(Collection<TestDescription> tests)200 private Collection<TestDescription> createRerunSet(Collection<TestDescription> tests) { 201 Map<String, TestDescription> uniqueMethods = new LinkedHashMap<>(); 202 for (TestDescription test : tests) { 203 uniqueMethods.put(test.getTestNameWithoutParams(), test); 204 } 205 return uniqueMethods.values(); 206 } 207 208 /** 209 * Util method to push file to a device. Exposed for unit testing. 210 * 211 * @return if file was pushed to the device successfully 212 * @throws DeviceNotAvailableException 213 */ pushFileToTestDevice(File file, String destinationPath)214 boolean pushFileToTestDevice(File file, String destinationPath) 215 throws DeviceNotAvailableException { 216 return mInstrumentationTest.getDevice().pushFile(file, destinationPath); 217 } 218 219 /** 220 * Delete file from the device if it exists 221 */ deleteTestFileFromDevice(String pathToFile)222 void deleteTestFileFromDevice(String pathToFile) throws DeviceNotAvailableException { 223 if (mInstrumentationTest.getDevice().doesFileExist(pathToFile)) { 224 mInstrumentationTest.getDevice() 225 .executeShellCommand(String.format("rm %s", pathToFile)); 226 CLog.d("Removed test file from device: %s", pathToFile); 227 } 228 } 229 230 /** @return the {@link InstrumentationTest} to use. Exposed for unit testing. */ 231 @VisibleForTesting createInstrumentationTest()232 InstrumentationTest createInstrumentationTest() { 233 return new InstrumentationTest(); 234 } 235 } 236