1 /* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.cts.core.runner; 18 19 import android.app.Activity; 20 import android.app.Instrumentation; 21 import android.os.Bundle; 22 import android.os.Debug; 23 import android.support.test.internal.runner.listener.InstrumentationResultPrinter; 24 import android.support.test.internal.runner.listener.InstrumentationRunListener; 25 import android.support.test.internal.util.AndroidRunnerParams; 26 import android.util.Log; 27 import com.android.cts.core.runner.support.ExtendedAndroidRunnerBuilder; 28 import com.google.common.base.Splitter; 29 import java.io.BufferedReader; 30 import java.io.FileReader; 31 import java.io.IOException; 32 import java.util.ArrayList; 33 import java.util.Arrays; 34 import java.util.Collection; 35 import java.util.Collections; 36 import java.util.LinkedHashSet; 37 import java.util.List; 38 import java.util.Set; 39 import java.util.HashSet; 40 import javax.annotation.Nullable; 41 import org.junit.runner.Computer; 42 import org.junit.runner.JUnitCore; 43 import org.junit.runner.Request; 44 import org.junit.runner.Runner; 45 import org.junit.runner.manipulation.Filter; 46 import org.junit.runner.manipulation.Filterable; 47 import org.junit.runner.manipulation.NoTestsRemainException; 48 import org.junit.runner.notification.RunListener; 49 import org.junit.runners.model.InitializationError; 50 import org.junit.runners.model.RunnerBuilder; 51 import vogar.ExpectationStore; 52 import vogar.ModeId; 53 54 import static com.android.cts.core.runner.AndroidJUnitRunnerConstants.ARGUMENT_COUNT; 55 import static com.android.cts.core.runner.AndroidJUnitRunnerConstants.ARGUMENT_DEBUG; 56 import static com.android.cts.core.runner.AndroidJUnitRunnerConstants.ARGUMENT_LOG_ONLY; 57 import static com.android.cts.core.runner.AndroidJUnitRunnerConstants.ARGUMENT_NOT_TEST_CLASS; 58 import static com.android.cts.core.runner.AndroidJUnitRunnerConstants.ARGUMENT_NOT_TEST_FILE; 59 import static com.android.cts.core.runner.AndroidJUnitRunnerConstants.ARGUMENT_NOT_TEST_PACKAGE; 60 import static com.android.cts.core.runner.AndroidJUnitRunnerConstants.ARGUMENT_TEST_CLASS; 61 import static com.android.cts.core.runner.AndroidJUnitRunnerConstants.ARGUMENT_TEST_FILE; 62 import static com.android.cts.core.runner.AndroidJUnitRunnerConstants.ARGUMENT_TEST_PACKAGE; 63 import static com.android.cts.core.runner.AndroidJUnitRunnerConstants.ARGUMENT_TIMEOUT; 64 65 /** 66 * A drop-in replacement for AndroidJUnitTestRunner, which understands the same arguments, and has 67 * similar functionality, but can filter by expectations and allows a custom runner-builder to be 68 * provided. 69 */ 70 public class CoreTestRunner extends Instrumentation { 71 72 public static final String TAG = "LibcoreTestRunner"; 73 74 private static final java.lang.String ARGUMENT_ROOT_CLASSES = "core-root-classes"; 75 76 private static final String ARGUMENT_EXPECTATIONS = "core-expectations"; 77 78 private static final String ARGUMENT_CORE_LISTENER = "core-listener"; 79 80 private static final Splitter CLASS_LIST_SPLITTER = Splitter.on(',').trimResults(); 81 82 /** The args for the runner. */ 83 private Bundle args; 84 85 /** Only count the number of tests, and not run them. */ 86 private boolean testCountOnly; 87 88 /** Only log the number of tests, and not run them. */ 89 private boolean logOnly; 90 91 /** The amount of time in millis to wait for a single test to complete. */ 92 private long testTimeout; 93 94 /** 95 * The container for any test expectations. 96 */ 97 @Nullable 98 private ExpectationStore expectationStore; 99 100 /** 101 * The list of tests to run. 102 */ 103 private TestList testList; 104 105 /** 106 * The list of {@link RunListener} classes to create. 107 */ 108 private List<Class<? extends RunListener>> listenerClasses; 109 110 @Override onCreate(final Bundle args)111 public void onCreate(final Bundle args) { 112 super.onCreate(args); 113 this.args = args; 114 115 boolean debug = "true".equalsIgnoreCase(args.getString(ARGUMENT_DEBUG)); 116 if (debug) { 117 Log.i(TAG, "Waiting for debugger to connect..."); 118 Debug.waitForDebugger(); 119 Log.i(TAG, "Debugger connected."); 120 } 121 122 // Log the message only after getting a value from the args so that the args are 123 // unparceled. 124 Log.d(TAG, "In OnCreate: " + args); 125 126 this.logOnly = "true".equalsIgnoreCase(args.getString(ARGUMENT_LOG_ONLY)); 127 this.testCountOnly = args.getBoolean(ARGUMENT_COUNT); 128 this.testTimeout = parseUnsignedLong(args.getString(ARGUMENT_TIMEOUT), ARGUMENT_TIMEOUT); 129 130 try { 131 // Get the set of resource names containing the expectations. 132 Set<String> expectationResources = new LinkedHashSet<>( 133 getExpectationResourcePaths(args)); 134 expectationStore = ExpectationStore.parseResources( 135 getClass(), expectationResources, ModeId.DEVICE); 136 } catch (IOException e) { 137 Log.e(TAG, "Could not initialize ExpectationStore: ", e); 138 } 139 140 // The test can be run specifying a list of tests to run, or as cts-tradefed does it, 141 // by passing a fileName with a test to run on each line. 142 Set<String> testNameSet = new HashSet<>(); 143 String arg; 144 if ((arg = args.getString(ARGUMENT_TEST_FILE)) != null) { 145 // The tests are specified in a file. 146 try { 147 testNameSet.addAll(readTestsFromFile(arg)); 148 } catch (IOException err) { 149 finish(Activity.RESULT_CANCELED, new Bundle()); 150 return; 151 } 152 } else if ((arg = args.getString(ARGUMENT_TEST_CLASS)) != null) { 153 // The tests are specified in a String passed in the bundle. 154 String[] tests = arg.split(","); 155 testNameSet.addAll(Arrays.asList(tests)); 156 } 157 158 // Tests may be excluded from the run by passing a list of tests not to run, 159 // or by passing a fileName with a test not to run on each line. 160 Set<String> notTestNameSet = new HashSet<>(); 161 if ((arg = args.getString(ARGUMENT_NOT_TEST_FILE)) != null) { 162 // The tests are specified in a file. 163 try { 164 notTestNameSet.addAll(readTestsFromFile(arg)); 165 } catch (IOException err) { 166 finish(Activity.RESULT_CANCELED, new Bundle()); 167 return; 168 } 169 } else if ((arg = args.getString(ARGUMENT_NOT_TEST_CLASS)) != null) { 170 // The classes are specified in a String passed in the bundle 171 String[] tests = arg.split(","); 172 notTestNameSet.addAll(Arrays.asList(tests)); 173 } 174 175 Set<String> packageNameSet = new HashSet<>(); 176 if ((arg = args.getString(ARGUMENT_TEST_PACKAGE)) != null) { 177 // The packages are specified in a String passed in the bundle 178 String[] packages = arg.split(","); 179 packageNameSet.addAll(Arrays.asList(packages)); 180 } 181 182 Set<String> notPackageNameSet = new HashSet<>(); 183 if ((arg = args.getString(ARGUMENT_NOT_TEST_PACKAGE)) != null) { 184 // The packages are specified in a String passed in the bundle 185 String[] packages = arg.split(","); 186 notPackageNameSet.addAll(Arrays.asList(packages)); 187 } 188 189 List<String> roots = getRootClassNames(args); 190 if (roots == null) { 191 // Find all test classes 192 Collection<Class<?>> classes = TestClassFinder.getClasses( 193 Collections.singletonList(getContext().getPackageCodePath()), 194 getClass().getClassLoader()); 195 testList = new TestList(classes); 196 } else { 197 testList = TestList.rootList(roots); 198 } 199 200 testList.addIncludeTestPackages(packageNameSet); 201 testList.addExcludeTestPackages(notPackageNameSet); 202 testList.addIncludeTests(testNameSet); 203 testList.addExcludeTests(notTestNameSet); 204 205 listenerClasses = new ArrayList<>(); 206 String listenerArg = args.getString(ARGUMENT_CORE_LISTENER); 207 if (listenerArg != null) { 208 List<String> listenerClassNames = CLASS_LIST_SPLITTER.splitToList(listenerArg); 209 for (String listenerClassName : listenerClassNames) { 210 try { 211 Class<? extends RunListener> listenerClass = Class.forName(listenerClassName) 212 .asSubclass(RunListener.class); 213 listenerClasses.add(listenerClass); 214 } catch (ClassNotFoundException e) { 215 Log.e(TAG, "Could not load listener class: " + listenerClassName, e); 216 } 217 } 218 } 219 220 start(); 221 } 222 getExpectationResourcePaths(Bundle args)223 protected List<String> getExpectationResourcePaths(Bundle args) { 224 return CLASS_LIST_SPLITTER.splitToList(args.getString(ARGUMENT_EXPECTATIONS)); 225 } 226 getRootClassNames(Bundle args)227 protected List<String> getRootClassNames(Bundle args) { 228 String rootClasses = args.getString(ARGUMENT_ROOT_CLASSES); 229 List<String> roots; 230 if (rootClasses == null) { 231 roots = null; 232 } else { 233 roots = CLASS_LIST_SPLITTER.splitToList(rootClasses); 234 } 235 return roots; 236 } 237 238 @Override onStart()239 public void onStart() { 240 if (logOnly || testCountOnly) { 241 Log.d(TAG, "Counting/logging tests only"); 242 } else { 243 Log.d(TAG, "Running tests"); 244 } 245 246 AndroidRunnerParams runnerParams = new AndroidRunnerParams(this, args, 247 logOnly || testCountOnly, testTimeout, false /*ignoreSuiteMethods*/); 248 249 JUnitCore core = new JUnitCore(); 250 251 Request request; 252 try { 253 RunnerBuilder runnerBuilder = new ExtendedAndroidRunnerBuilder(runnerParams); 254 Class[] classes = testList.getClassesToRun(); 255 for (Class cls : classes) { 256 Log.d(TAG, "Found class to run: " + cls.getName()); 257 } 258 Runner suite = new Computer().getSuite(runnerBuilder, classes); 259 260 if (suite instanceof Filterable) { 261 Filterable filterable = (Filterable) suite; 262 263 // Filter out all the tests that are expected to fail. 264 Filter filter = new TestFilter(testList, expectationStore); 265 266 try { 267 filterable.filter(filter); 268 } catch (NoTestsRemainException e) { 269 // Sometimes filtering will remove all tests but we do not care about that. 270 } 271 } 272 273 request = Request.runner(suite); 274 275 } catch (InitializationError e) { 276 throw new RuntimeException("Could not create a suite", e); 277 } 278 279 InstrumentationResultPrinter instrumentationResultPrinter = 280 new InstrumentationResultPrinter(); 281 instrumentationResultPrinter.setInstrumentation(this); 282 core.addListener(instrumentationResultPrinter); 283 284 for (Class<? extends RunListener> listenerClass : listenerClasses) { 285 try { 286 RunListener runListener = listenerClass.newInstance(); 287 if (runListener instanceof InstrumentationRunListener) { 288 ((InstrumentationRunListener) runListener).setInstrumentation(this); 289 } 290 core.addListener(runListener); 291 } catch (InstantiationException | IllegalAccessException e) { 292 Log.e(TAG, "Could not create instance of listener: " + listenerClass, e); 293 } 294 } 295 296 Bundle results = new Bundle(); 297 try { 298 core.run(request); 299 } catch (RuntimeException e) { 300 final String msg = "Fatal exception when running tests"; 301 Log.e(TAG, msg, e); 302 // report the exception to instrumentation out 303 results.putString(Instrumentation.REPORT_KEY_STREAMRESULT, 304 msg + "\n" + Log.getStackTraceString(e)); 305 } 306 307 Log.d(TAG, "Finished"); 308 finish(Activity.RESULT_OK, results); 309 } 310 311 /** 312 * Read tests from a specified file. 313 * 314 * @return class names of tests. If there was an error reading the file, null is returned. 315 */ readTestsFromFile(String fileName)316 private static List<String> readTestsFromFile(String fileName) throws IOException { 317 try (BufferedReader br = new BufferedReader(new FileReader(fileName))) { 318 List<String> tests = new ArrayList<>(); 319 String line; 320 while ((line = br.readLine()) != null) { 321 tests.add(line); 322 } 323 return tests; 324 } catch (IOException err) { 325 Log.e(TAG, "There was an error reading the test class list: " + err.getMessage()); 326 throw err; 327 } 328 } 329 330 /** 331 * Parse long from given value - except either Long or String. 332 * 333 * @return the value, -1 if not found 334 * @throws NumberFormatException if value is negative or not a number 335 */ parseUnsignedLong(Object value, String name)336 private static long parseUnsignedLong(Object value, String name) { 337 if (value != null) { 338 long longValue = Long.parseLong(value.toString()); 339 if (longValue < 0) { 340 throw new NumberFormatException(name + " can not be negative"); 341 } 342 return longValue; 343 } 344 return -1; 345 } 346 347 } 348