1 /* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.testtype; 18 19 import com.android.tradefed.config.ConfigurationException; 20 import com.android.tradefed.config.OptionCopier; 21 import com.android.tradefed.device.DeviceNotAvailableException; 22 import com.android.tradefed.log.LogUtil.CLog; 23 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 24 import com.android.tradefed.result.CollectingTestListener; 25 import com.android.tradefed.result.FilteredResultForwarder; 26 import com.android.tradefed.result.ITestInvocationListener; 27 import com.android.tradefed.result.RetryResultForwarder; 28 import com.android.tradefed.result.TestDescription; 29 import com.android.tradefed.result.TestRunResult; 30 31 import java.util.Arrays; 32 import java.util.Collection; 33 import java.util.HashMap; 34 35 /** 36 * A Test that runs a set of instrumentation tests by running one adb command for per test. 37 */ 38 class InstrumentationSerialTest implements IRemoteTest { 39 40 /** number of attempts to make if test fails to run */ 41 static final int FAILED_RUN_TEST_ATTEMPTS = 2; 42 43 /** the set of tests to run */ 44 private final Collection<TestDescription> mTests; 45 46 private final InstrumentationTest mInstrumentationTest; 47 48 /** 49 * Creates a {@link InstrumentationSerialTest}. 50 * 51 * @param instrumentationTest {@link InstrumentationTest} used to configure this class 52 * @param testsToRun a {@link Collection} of tests to run. Note this {@link Collection} will be 53 * used as is (ie a reference to the testsToRun object will be kept). 54 */ InstrumentationSerialTest( InstrumentationTest instrumentationTest, Collection<TestDescription> testsToRun)55 InstrumentationSerialTest( 56 InstrumentationTest instrumentationTest, Collection<TestDescription> testsToRun) 57 throws ConfigurationException { 58 // reuse the InstrumentationTest class to perform actual test run 59 mInstrumentationTest = createInstrumentationTest(instrumentationTest); 60 // keep local copy of tests to be run 61 mTests = testsToRun; 62 } 63 64 /** 65 * Create and initialize new instance of {@link InstrumentationTest}. Exposed for unit testing. 66 * 67 * @param instrumentationTest {@link InstrumentationTest} used to configure this class 68 * @return the newly created {@link InstrumentationTest} 69 */ createInstrumentationTest(InstrumentationTest instrumentationTest)70 InstrumentationTest createInstrumentationTest(InstrumentationTest instrumentationTest) 71 throws ConfigurationException { 72 InstrumentationTest runner = new InstrumentationTest(); 73 OptionCopier.copyOptions(instrumentationTest, runner); 74 runner.setDevice(instrumentationTest.getDevice()); 75 runner.setForceAbi(instrumentationTest.getForceAbi()); 76 // ensure testFile is not used. 77 runner.setReRunUsingTestFile(false); 78 // no need to rerun when executing tests one by one 79 runner.setRerunMode(false); 80 runner.setIsRerun(true); 81 return runner; 82 } 83 84 /** 85 * {@inheritDoc} 86 */ 87 @Override run(final ITestInvocationListener listener)88 public void run(final ITestInvocationListener listener) throws DeviceNotAvailableException { 89 if (mInstrumentationTest.getDevice() == null) { 90 throw new IllegalArgumentException("Device has not been set"); 91 } 92 // reuse the InstrumentationTest class to perform actual test run 93 try { 94 for (TestDescription testToRun : mTests) { 95 InstrumentationTest runner = createInstrumentationTest(mInstrumentationTest); 96 runner.setClassName(testToRun.getClassName()); 97 // We use getTestNameNoParams to avoid attempting re-running individual 98 // parameterized tests. Instead ask the base method to re-run them all. 99 runner.setMethodName(testToRun.getTestNameWithoutParams()); 100 // Unset package name if any just in case to avoid conflict with classname. 101 runner.setTestPackageName(null); 102 runTest(runner, listener, testToRun); 103 } 104 } catch (ConfigurationException e) { 105 CLog.e("Failed to create new InstrumentationTest: %s", e.getMessage()); 106 } 107 } 108 runTest( InstrumentationTest runner, ITestInvocationListener listener, TestDescription testToRun)109 private void runTest( 110 InstrumentationTest runner, ITestInvocationListener listener, TestDescription testToRun) 111 throws DeviceNotAvailableException { 112 // use a listener filter, to track if the test failed to run 113 CollectingTestListener trackingListener = 114 new CollectingTestListener() { 115 @Override 116 public void testRunStarted(String name, int numTests, int attemptNumber) { 117 // Make the tracker unaware of attempts to track the current retry attempt 118 super.testRunStarted(name, 0, 0); 119 } 120 }; 121 for (int i=1; i <= FAILED_RUN_TEST_ATTEMPTS; i++) { 122 runner.run( 123 new RetryResultForwarder( 124 i, 125 trackingListener, 126 new FilteredResultForwarder(Arrays.asList(testToRun), listener)) { 127 // Avoid any test count to avoid recounting the tests. 128 @Override 129 public void testRunStarted(String runName, int testCount) { 130 super.testRunStarted(runName, 0); 131 } 132 133 @Override 134 public void testRunStarted( 135 String runName, int testCount, int attemptNumber) { 136 super.testRunStarted(runName, 0, attemptNumber); 137 } 138 }); 139 if (trackingListener.getCurrentRunResults().getTestResults().containsKey(testToRun)) { 140 return; 141 } 142 CLog.w( 143 "Expected test %s did not run on attempt %d of %d", 144 testToRun, i, FAILED_RUN_TEST_ATTEMPTS); 145 } 146 markTestAsFailed(testToRun, trackingListener.getCurrentRunResults(), listener); 147 } 148 markTestAsFailed( TestDescription test, TestRunResult testRun, ITestInvocationListener listener)149 private void markTestAsFailed( 150 TestDescription test, TestRunResult testRun, ITestInvocationListener listener) { 151 // Set test count at 0 to avoid re-counting the number of tests. 152 listener.testRunStarted(testRun.getName(), 0); 153 listener.testStarted(test); 154 155 String message = 156 testRun.isRunFailure() 157 ? testRun.getRunFailureMessage() 158 : "The test was not initialized by the test runner."; 159 listener.testFailed( 160 test, String.format("Test failed to run. Test run failed due to : %s", message)); 161 listener.testEnded(test, new HashMap<String, Metric>()); 162 listener.testRunEnded(0, new HashMap<String, Metric>()); 163 } 164 } 165