1 /* 2 * Copyright (C) 2007 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.test; 18 19 import static android.test.suitebuilder.TestPredicates.REJECT_PERFORMANCE; 20 21 import com.android.internal.util.Predicate; 22 import com.android.internal.util.Predicates; 23 24 import android.app.Activity; 25 import android.app.Instrumentation; 26 import android.os.Bundle; 27 import android.os.Debug; 28 import android.os.Looper; 29 import android.os.Parcelable; 30 import android.os.PerformanceCollector; 31 import android.os.PerformanceCollector.PerformanceResultsWriter; 32 import android.test.suitebuilder.TestMethod; 33 import android.test.suitebuilder.TestPredicates; 34 import android.test.suitebuilder.TestSuiteBuilder; 35 import android.test.suitebuilder.annotation.HasAnnotation; 36 import android.test.suitebuilder.annotation.LargeTest; 37 import android.util.Log; 38 39 import java.io.ByteArrayOutputStream; 40 import java.io.File; 41 import java.io.PrintStream; 42 import java.lang.annotation.Annotation; 43 import java.lang.reflect.InvocationTargetException; 44 import java.lang.reflect.Method; 45 import java.util.ArrayList; 46 import java.util.List; 47 48 import junit.framework.AssertionFailedError; 49 import junit.framework.Test; 50 import junit.framework.TestCase; 51 import junit.framework.TestListener; 52 import junit.framework.TestResult; 53 import junit.framework.TestSuite; 54 import junit.runner.BaseTestRunner; 55 import junit.textui.ResultPrinter; 56 57 /** 58 * An {@link Instrumentation} that runs various types of {@link junit.framework.TestCase}s against 59 * an Android package (application). Typical usage: 60 * <ol> 61 * <li>Write {@link junit.framework.TestCase}s that perform unit, functional, or performance tests 62 * against the classes in your package. Typically these are subclassed from: 63 * <ul><li>{@link android.test.ActivityInstrumentationTestCase2}</li> 64 * <li>{@link android.test.ActivityUnitTestCase}</li> 65 * <li>{@link android.test.AndroidTestCase}</li> 66 * <li>{@link android.test.ApplicationTestCase}</li> 67 * <li>{@link android.test.InstrumentationTestCase}</li> 68 * <li>{@link android.test.ProviderTestCase}</li> 69 * <li>{@link android.test.ServiceTestCase}</li> 70 * <li>{@link android.test.SingleLaunchActivityTestCase}</li></ul> 71 * <li>In an appropriate AndroidManifest.xml, define the this instrumentation with 72 * the appropriate android:targetPackage set. 73 * <li>Run the instrumentation using "adb shell am instrument -w", 74 * with no optional arguments, to run all tests (except performance tests). 75 * <li>Run the instrumentation using "adb shell am instrument -w", 76 * with the argument '-e func true' to run all functional tests. These are tests that derive from 77 * {@link android.test.InstrumentationTestCase}. 78 * <li>Run the instrumentation using "adb shell am instrument -w", 79 * with the argument '-e unit true' to run all unit tests. These are tests that <i>do not</i>derive 80 * from {@link android.test.InstrumentationTestCase} (and are not performance tests). 81 * <li>Run the instrumentation using "adb shell am instrument -w", 82 * with the argument '-e class' set to run an individual {@link junit.framework.TestCase}. 83 * </ol> 84 * <p/> 85 * <b>Running all tests:</b> adb shell am instrument -w 86 * com.android.foo/android.test.InstrumentationTestRunner 87 * <p/> 88 * <b>Running all small tests:</b> adb shell am instrument -w 89 * -e size small 90 * com.android.foo/android.test.InstrumentationTestRunner 91 * <p/> 92 * <b>Running all medium tests:</b> adb shell am instrument -w 93 * -e size medium 94 * com.android.foo/android.test.InstrumentationTestRunner 95 * <p/> 96 * <b>Running all large tests:</b> adb shell am instrument -w 97 * -e size large 98 * com.android.foo/android.test.InstrumentationTestRunner 99 * <p/> 100 * <b>Filter test run to tests with given annotation:</b> adb shell am instrument -w 101 * -e annotation com.android.foo.MyAnnotation 102 * com.android.foo/android.test.InstrumentationTestRunner 103 * <p/> 104 * If used with other options, the resulting test run will contain the union of the two options. 105 * e.g. "-e size large -e annotation com.android.foo.MyAnnotation" will run only tests with both 106 * the {@link LargeTest} and "com.android.foo.MyAnnotation" annotations. 107 * <p/> 108 * <b>Filter test run to tests <i>without</i> given annotation:</b> adb shell am instrument -w 109 * -e notAnnotation com.android.foo.MyAnnotation 110 * com.android.foo/android.test.InstrumentationTestRunner 111 * <p/> 112 * <b>Running a single testcase:</b> adb shell am instrument -w 113 * -e class com.android.foo.FooTest 114 * com.android.foo/android.test.InstrumentationTestRunner 115 * <p/> 116 * <b>Running a single test:</b> adb shell am instrument -w 117 * -e class com.android.foo.FooTest#testFoo 118 * com.android.foo/android.test.InstrumentationTestRunner 119 * <p/> 120 * <b>Running multiple tests:</b> adb shell am instrument -w 121 * -e class com.android.foo.FooTest,com.android.foo.TooTest 122 * com.android.foo/android.test.InstrumentationTestRunner 123 * <p/> 124 * <b>Running all tests in a java package:</b> adb shell am instrument -w 125 * -e package com.android.foo.subpkg 126 * com.android.foo/android.test.InstrumentationTestRunner 127 * <p/> 128 * <b>Including performance tests:</b> adb shell am instrument -w 129 * -e perf true 130 * com.android.foo/android.test.InstrumentationTestRunner 131 * <p/> 132 * <b>To debug your tests, set a break point in your code and pass:</b> 133 * -e debug true 134 * <p/> 135 * <b>To run in 'log only' mode</b> 136 * -e log true 137 * This option will load and iterate through all test classes and methods, but will bypass actual 138 * test execution. Useful for quickly obtaining info on the tests to be executed by an 139 * instrumentation command. 140 * <p/> 141 * <b>To generate EMMA code coverage:</b> 142 * -e coverage true 143 * Note: this requires an emma instrumented build. By default, the code coverage results file 144 * will be saved in a /data/<app>/coverage.ec file, unless overridden by coverageFile flag (see 145 * below) 146 * <p/> 147 * <b> To specify EMMA code coverage results file path:</b> 148 * -e coverageFile /sdcard/myFile.ec 149 * <br/> 150 * in addition to the other arguments. 151 */ 152 153 /* (not JavaDoc) 154 * Although not necessary in most case, another way to use this class is to extend it and have the 155 * derived class return the desired test suite from the {@link #getTestSuite()} method. The test 156 * suite returned from this method will be used if no target class is defined in the meta-data or 157 * command line argument parameters. If a derived class is used it needs to be added as an 158 * instrumentation to the AndroidManifest.xml and the command to run it would look like: 159 * <p/> 160 * adb shell am instrument -w com.android.foo/<i>com.android.FooInstrumentationTestRunner</i> 161 * <p/> 162 * Where <i>com.android.FooInstrumentationTestRunner</i> is the derived class. 163 * 164 * This model is used by many existing app tests, but can probably be deprecated. 165 */ 166 public class InstrumentationTestRunner extends Instrumentation implements TestSuiteProvider { 167 168 /** @hide */ 169 public static final String ARGUMENT_TEST_CLASS = "class"; 170 /** @hide */ 171 public static final String ARGUMENT_TEST_PACKAGE = "package"; 172 /** @hide */ 173 public static final String ARGUMENT_TEST_SIZE_PREDICATE = "size"; 174 /** @hide */ 175 public static final String ARGUMENT_INCLUDE_PERF = "perf"; 176 /** @hide */ 177 public static final String ARGUMENT_DELAY_MSEC = "delay_msec"; 178 179 private static final String SMALL_SUITE = "small"; 180 private static final String MEDIUM_SUITE = "medium"; 181 private static final String LARGE_SUITE = "large"; 182 183 private static final String ARGUMENT_LOG_ONLY = "log"; 184 /** @hide */ 185 static final String ARGUMENT_ANNOTATION = "annotation"; 186 /** @hide */ 187 static final String ARGUMENT_NOT_ANNOTATION = "notAnnotation"; 188 189 /** 190 * This constant defines the maximum allowed runtime (in ms) for a test included in the "small" 191 * suite. It is used to make an educated guess at what suite an unlabeled test belongs. 192 */ 193 private static final float SMALL_SUITE_MAX_RUNTIME = 100; 194 195 /** 196 * This constant defines the maximum allowed runtime (in ms) for a test included in the 197 * "medium" suite. It is used to make an educated guess at what suite an unlabeled test belongs. 198 */ 199 private static final float MEDIUM_SUITE_MAX_RUNTIME = 1000; 200 201 /** 202 * The following keys are used in the status bundle to provide structured reports to 203 * an IInstrumentationWatcher. 204 */ 205 206 /** 207 * This value, if stored with key {@link android.app.Instrumentation#REPORT_KEY_IDENTIFIER}, 208 * identifies InstrumentationTestRunner as the source of the report. This is sent with all 209 * status messages. 210 */ 211 public static final String REPORT_VALUE_ID = "InstrumentationTestRunner"; 212 /** 213 * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 214 * identifies the total number of tests that are being run. This is sent with all status 215 * messages. 216 */ 217 public static final String REPORT_KEY_NUM_TOTAL = "numtests"; 218 /** 219 * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 220 * identifies the sequence number of the current test. This is sent with any status message 221 * describing a specific test being started or completed. 222 */ 223 public static final String REPORT_KEY_NUM_CURRENT = "current"; 224 /** 225 * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 226 * identifies the name of the current test class. This is sent with any status message 227 * describing a specific test being started or completed. 228 */ 229 public static final String REPORT_KEY_NAME_CLASS = "class"; 230 /** 231 * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 232 * identifies the name of the current test. This is sent with any status message 233 * describing a specific test being started or completed. 234 */ 235 public static final String REPORT_KEY_NAME_TEST = "test"; 236 /** 237 * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 238 * reports the run time in seconds of the current test. 239 */ 240 private static final String REPORT_KEY_RUN_TIME = "runtime"; 241 /** 242 * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 243 * reports the guessed suite assignment for the current test. 244 */ 245 private static final String REPORT_KEY_SUITE_ASSIGNMENT = "suiteassignment"; 246 /** 247 * If included in the status or final bundle sent to an IInstrumentationWatcher, this key 248 * identifies the path to the generated code coverage file. 249 */ 250 private static final String REPORT_KEY_COVERAGE_PATH = "coverageFilePath"; 251 252 /** 253 * The test is starting. 254 */ 255 public static final int REPORT_VALUE_RESULT_START = 1; 256 /** 257 * The test completed successfully. 258 */ 259 public static final int REPORT_VALUE_RESULT_OK = 0; 260 /** 261 * The test completed with an error. 262 */ 263 public static final int REPORT_VALUE_RESULT_ERROR = -1; 264 /** 265 * The test completed with a failure. 266 */ 267 public static final int REPORT_VALUE_RESULT_FAILURE = -2; 268 /** 269 * If included in the status bundle sent to an IInstrumentationWatcher, this key 270 * identifies a stack trace describing an error or failure. This is sent with any status 271 * message describing a specific test being completed. 272 */ 273 public static final String REPORT_KEY_STACK = "stack"; 274 275 // Default file name for code coverage 276 private static final String DEFAULT_COVERAGE_FILE_NAME = "coverage.ec"; 277 278 private static final String LOG_TAG = "InstrumentationTestRunner"; 279 280 private final Bundle mResults = new Bundle(); 281 private AndroidTestRunner mTestRunner; 282 private boolean mDebug; 283 private boolean mJustCount; 284 private boolean mSuiteAssignmentMode; 285 private int mTestCount; 286 private String mPackageOfTests; 287 private boolean mCoverage; 288 private String mCoverageFilePath; 289 private int mDelayMsec; 290 291 @Override onCreate(Bundle arguments)292 public void onCreate(Bundle arguments) { 293 super.onCreate(arguments); 294 295 // Apk paths used to search for test classes when using TestSuiteBuilders. 296 String[] apkPaths = 297 {getTargetContext().getPackageCodePath(), getContext().getPackageCodePath()}; 298 ClassPathPackageInfoSource.setApkPaths(apkPaths); 299 300 Predicate<TestMethod> testSizePredicate = null; 301 Predicate<TestMethod> testAnnotationPredicate = null; 302 Predicate<TestMethod> testNotAnnotationPredicate = null; 303 boolean includePerformance = false; 304 String testClassesArg = null; 305 boolean logOnly = false; 306 307 if (arguments != null) { 308 // Test class name passed as an argument should override any meta-data declaration. 309 testClassesArg = arguments.getString(ARGUMENT_TEST_CLASS); 310 mDebug = getBooleanArgument(arguments, "debug"); 311 mJustCount = getBooleanArgument(arguments, "count"); 312 mSuiteAssignmentMode = getBooleanArgument(arguments, "suiteAssignment"); 313 mPackageOfTests = arguments.getString(ARGUMENT_TEST_PACKAGE); 314 testSizePredicate = getSizePredicateFromArg( 315 arguments.getString(ARGUMENT_TEST_SIZE_PREDICATE)); 316 testAnnotationPredicate = getAnnotationPredicate( 317 arguments.getString(ARGUMENT_ANNOTATION)); 318 testNotAnnotationPredicate = getNotAnnotationPredicate( 319 arguments.getString(ARGUMENT_NOT_ANNOTATION)); 320 321 includePerformance = getBooleanArgument(arguments, ARGUMENT_INCLUDE_PERF); 322 logOnly = getBooleanArgument(arguments, ARGUMENT_LOG_ONLY); 323 mCoverage = getBooleanArgument(arguments, "coverage"); 324 mCoverageFilePath = arguments.getString("coverageFile"); 325 326 try { 327 Object delay = arguments.get(ARGUMENT_DELAY_MSEC); // Accept either string or int 328 if (delay != null) mDelayMsec = Integer.parseInt(delay.toString()); 329 } catch (NumberFormatException e) { 330 Log.e(LOG_TAG, "Invalid delay_msec parameter", e); 331 } 332 } 333 334 TestSuiteBuilder testSuiteBuilder = new TestSuiteBuilder(getClass().getName(), 335 getTargetContext().getClassLoader()); 336 337 if (testSizePredicate != null) { 338 testSuiteBuilder.addRequirements(testSizePredicate); 339 } 340 if (testAnnotationPredicate != null) { 341 testSuiteBuilder.addRequirements(testAnnotationPredicate); 342 } 343 if (testNotAnnotationPredicate != null) { 344 testSuiteBuilder.addRequirements(testNotAnnotationPredicate); 345 } 346 if (!includePerformance) { 347 testSuiteBuilder.addRequirements(REJECT_PERFORMANCE); 348 } 349 350 if (testClassesArg == null) { 351 if (mPackageOfTests != null) { 352 testSuiteBuilder.includePackages(mPackageOfTests); 353 } else { 354 TestSuite testSuite = getTestSuite(); 355 if (testSuite != null) { 356 testSuiteBuilder.addTestSuite(testSuite); 357 } else { 358 // no package or class bundle arguments were supplied, and no test suite 359 // provided so add all tests in application 360 testSuiteBuilder.includePackages(""); 361 } 362 } 363 } else { 364 parseTestClasses(testClassesArg, testSuiteBuilder); 365 } 366 367 testSuiteBuilder.addRequirements(getBuilderRequirements()); 368 369 mTestRunner = getAndroidTestRunner(); 370 mTestRunner.setContext(getTargetContext()); 371 mTestRunner.setInstrumentation(this); 372 mTestRunner.setSkipExecution(logOnly); 373 mTestRunner.setTest(testSuiteBuilder.build()); 374 mTestCount = mTestRunner.getTestCases().size(); 375 if (mSuiteAssignmentMode) { 376 mTestRunner.addTestListener(new SuiteAssignmentPrinter()); 377 } else { 378 WatcherResultPrinter resultPrinter = new WatcherResultPrinter(mTestCount); 379 mTestRunner.addTestListener(new TestPrinter("TestRunner", false)); 380 mTestRunner.addTestListener(resultPrinter); 381 mTestRunner.setPerformanceResultsWriter(resultPrinter); 382 } 383 start(); 384 } 385 getBuilderRequirements()386 List<Predicate<TestMethod>> getBuilderRequirements() { 387 return new ArrayList<Predicate<TestMethod>>(); 388 } 389 390 /** 391 * Parses and loads the specified set of test classes 392 * 393 * @param testClassArg - comma-separated list of test classes and methods 394 * @param testSuiteBuilder - builder to add tests to 395 */ parseTestClasses(String testClassArg, TestSuiteBuilder testSuiteBuilder)396 private void parseTestClasses(String testClassArg, TestSuiteBuilder testSuiteBuilder) { 397 String[] testClasses = testClassArg.split(","); 398 for (String testClass : testClasses) { 399 parseTestClass(testClass, testSuiteBuilder); 400 } 401 } 402 403 /** 404 * Parse and load the given test class and, optionally, method 405 * 406 * @param testClassName - full package name of test class and optionally method to add. 407 * Expected format: com.android.TestClass#testMethod 408 * @param testSuiteBuilder - builder to add tests to 409 */ parseTestClass(String testClassName, TestSuiteBuilder testSuiteBuilder)410 private void parseTestClass(String testClassName, TestSuiteBuilder testSuiteBuilder) { 411 int methodSeparatorIndex = testClassName.indexOf('#'); 412 String testMethodName = null; 413 414 if (methodSeparatorIndex > 0) { 415 testMethodName = testClassName.substring(methodSeparatorIndex + 1); 416 testClassName = testClassName.substring(0, methodSeparatorIndex); 417 } 418 testSuiteBuilder.addTestClassByName(testClassName, testMethodName, getTargetContext()); 419 } 420 getAndroidTestRunner()421 protected AndroidTestRunner getAndroidTestRunner() { 422 return new AndroidTestRunner(); 423 } 424 getBooleanArgument(Bundle arguments, String tag)425 private boolean getBooleanArgument(Bundle arguments, String tag) { 426 String tagString = arguments.getString(tag); 427 return tagString != null && Boolean.parseBoolean(tagString); 428 } 429 430 /* 431 * Returns the size predicate object, corresponding to the "size" argument value. 432 */ getSizePredicateFromArg(String sizeArg)433 private Predicate<TestMethod> getSizePredicateFromArg(String sizeArg) { 434 435 if (SMALL_SUITE.equals(sizeArg)) { 436 return TestPredicates.SELECT_SMALL; 437 } else if (MEDIUM_SUITE.equals(sizeArg)) { 438 return TestPredicates.SELECT_MEDIUM; 439 } else if (LARGE_SUITE.equals(sizeArg)) { 440 return TestPredicates.SELECT_LARGE; 441 } else { 442 return null; 443 } 444 } 445 446 /** 447 * Returns the test predicate object, corresponding to the annotation class value provided via 448 * the {@link ARGUMENT_ANNOTATION} argument. 449 * 450 * @return the predicate or <code>null</code> 451 */ getAnnotationPredicate(String annotationClassName)452 private Predicate<TestMethod> getAnnotationPredicate(String annotationClassName) { 453 Class<? extends Annotation> annotationClass = getAnnotationClass(annotationClassName); 454 if (annotationClass != null) { 455 return new HasAnnotation(annotationClass); 456 } 457 return null; 458 } 459 460 /** 461 * Returns the negative test predicate object, corresponding to the annotation class value 462 * provided via the {@link ARGUMENT_NOT_ANNOTATION} argument. 463 * 464 * @return the predicate or <code>null</code> 465 */ getNotAnnotationPredicate(String annotationClassName)466 private Predicate<TestMethod> getNotAnnotationPredicate(String annotationClassName) { 467 Class<? extends Annotation> annotationClass = getAnnotationClass(annotationClassName); 468 if (annotationClass != null) { 469 return Predicates.not(new HasAnnotation(annotationClass)); 470 } 471 return null; 472 } 473 474 /** 475 * Helper method to return the annotation class with specified name 476 * 477 * @param annotationClassName the fully qualified name of the class 478 * @return the annotation class or <code>null</code> 479 */ getAnnotationClass(String annotationClassName)480 private Class<? extends Annotation> getAnnotationClass(String annotationClassName) { 481 if (annotationClassName == null) { 482 return null; 483 } 484 try { 485 Class<?> annotationClass = Class.forName(annotationClassName); 486 if (annotationClass.isAnnotation()) { 487 return (Class<? extends Annotation>)annotationClass; 488 } else { 489 Log.e(LOG_TAG, String.format("Provided annotation value %s is not an Annotation", 490 annotationClassName)); 491 } 492 } catch (ClassNotFoundException e) { 493 Log.e(LOG_TAG, String.format("Could not find class for specified annotation %s", 494 annotationClassName)); 495 } 496 return null; 497 } 498 499 /** 500 * Initialize the current thread as a looper. 501 * <p/> 502 * Exposed for unit testing. 503 */ prepareLooper()504 void prepareLooper() { 505 Looper.prepare(); 506 } 507 508 @Override onStart()509 public void onStart() { 510 prepareLooper(); 511 512 if (mJustCount) { 513 mResults.putString(Instrumentation.REPORT_KEY_IDENTIFIER, REPORT_VALUE_ID); 514 mResults.putInt(REPORT_KEY_NUM_TOTAL, mTestCount); 515 finish(Activity.RESULT_OK, mResults); 516 } else { 517 if (mDebug) { 518 Debug.waitForDebugger(); 519 } 520 521 ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); 522 PrintStream writer = new PrintStream(byteArrayOutputStream); 523 try { 524 StringResultPrinter resultPrinter = new StringResultPrinter(writer); 525 526 mTestRunner.addTestListener(resultPrinter); 527 528 long startTime = System.currentTimeMillis(); 529 mTestRunner.runTest(); 530 long runTime = System.currentTimeMillis() - startTime; 531 532 resultPrinter.print(mTestRunner.getTestResult(), runTime); 533 } catch (Throwable t) { 534 // catch all exceptions so a more verbose error message can be outputted 535 writer.println(String.format("Test run aborted due to unexpected exception: %s", 536 t.getMessage())); 537 t.printStackTrace(writer); 538 } finally { 539 mResults.putString(Instrumentation.REPORT_KEY_STREAMRESULT, 540 String.format("\nTest results for %s=%s", 541 mTestRunner.getTestClassName(), 542 byteArrayOutputStream.toString())); 543 544 if (mCoverage) { 545 generateCoverageReport(); 546 } 547 writer.close(); 548 549 finish(Activity.RESULT_OK, mResults); 550 } 551 } 552 } 553 getTestSuite()554 public TestSuite getTestSuite() { 555 return getAllTests(); 556 } 557 558 /** 559 * Override this to define all of the tests to run in your package. 560 */ getAllTests()561 public TestSuite getAllTests() { 562 return null; 563 } 564 565 /** 566 * Override this to provide access to the class loader of your package. 567 */ getLoader()568 public ClassLoader getLoader() { 569 return null; 570 } 571 generateCoverageReport()572 private void generateCoverageReport() { 573 // use reflection to call emma dump coverage method, to avoid 574 // always statically compiling against emma jar 575 String coverageFilePath = getCoverageFilePath(); 576 java.io.File coverageFile = new java.io.File(coverageFilePath); 577 try { 578 Class<?> emmaRTClass = Class.forName("com.vladium.emma.rt.RT"); 579 Method dumpCoverageMethod = emmaRTClass.getMethod("dumpCoverageData", 580 coverageFile.getClass(), boolean.class, boolean.class); 581 582 dumpCoverageMethod.invoke(null, coverageFile, false, false); 583 // output path to generated coverage file so it can be parsed by a test harness if 584 // needed 585 mResults.putString(REPORT_KEY_COVERAGE_PATH, coverageFilePath); 586 // also output a more user friendly msg 587 final String currentStream = mResults.getString( 588 Instrumentation.REPORT_KEY_STREAMRESULT); 589 mResults.putString(Instrumentation.REPORT_KEY_STREAMRESULT, 590 String.format("%s\nGenerated code coverage data to %s", currentStream, 591 coverageFilePath)); 592 } catch (ClassNotFoundException e) { 593 reportEmmaError("Is emma jar on classpath?", e); 594 } catch (SecurityException e) { 595 reportEmmaError(e); 596 } catch (NoSuchMethodException e) { 597 reportEmmaError(e); 598 } catch (IllegalArgumentException e) { 599 reportEmmaError(e); 600 } catch (IllegalAccessException e) { 601 reportEmmaError(e); 602 } catch (InvocationTargetException e) { 603 reportEmmaError(e); 604 } 605 } 606 getCoverageFilePath()607 private String getCoverageFilePath() { 608 if (mCoverageFilePath == null) { 609 return getTargetContext().getFilesDir().getAbsolutePath() + File.separator + 610 DEFAULT_COVERAGE_FILE_NAME; 611 } else { 612 return mCoverageFilePath; 613 } 614 } 615 reportEmmaError(Exception e)616 private void reportEmmaError(Exception e) { 617 reportEmmaError("", e); 618 } 619 reportEmmaError(String hint, Exception e)620 private void reportEmmaError(String hint, Exception e) { 621 String msg = "Failed to generate emma coverage. " + hint; 622 Log.e(LOG_TAG, msg, e); 623 mResults.putString(Instrumentation.REPORT_KEY_STREAMRESULT, "\nError: " + msg); 624 } 625 626 // TODO kill this, use status() and prettyprint model for better output 627 private class StringResultPrinter extends ResultPrinter { 628 StringResultPrinter(PrintStream writer)629 public StringResultPrinter(PrintStream writer) { 630 super(writer); 631 } 632 print(TestResult result, long runTime)633 synchronized void print(TestResult result, long runTime) { 634 printHeader(runTime); 635 printFooter(result); 636 } 637 } 638 639 /** 640 * This class sends status reports back to the IInstrumentationWatcher about 641 * which suite each test belongs. 642 */ 643 private class SuiteAssignmentPrinter implements TestListener { 644 645 private Bundle mTestResult; 646 private long mStartTime; 647 private long mEndTime; 648 private boolean mTimingValid; 649 SuiteAssignmentPrinter()650 public SuiteAssignmentPrinter() { 651 } 652 653 /** 654 * send a status for the start of a each test, so long tests can be seen as "running" 655 */ startTest(Test test)656 public void startTest(Test test) { 657 mTimingValid = true; 658 mStartTime = System.currentTimeMillis(); 659 } 660 661 /** 662 * @see junit.framework.TestListener#addError(Test, Throwable) 663 */ addError(Test test, Throwable t)664 public void addError(Test test, Throwable t) { 665 mTimingValid = false; 666 } 667 668 /** 669 * @see junit.framework.TestListener#addFailure(Test, AssertionFailedError) 670 */ addFailure(Test test, AssertionFailedError t)671 public void addFailure(Test test, AssertionFailedError t) { 672 mTimingValid = false; 673 } 674 675 /** 676 * @see junit.framework.TestListener#endTest(Test) 677 */ endTest(Test test)678 public void endTest(Test test) { 679 float runTime; 680 String assignmentSuite; 681 mEndTime = System.currentTimeMillis(); 682 mTestResult = new Bundle(); 683 684 if (!mTimingValid || mStartTime < 0) { 685 assignmentSuite = "NA"; 686 runTime = -1; 687 } else { 688 runTime = mEndTime - mStartTime; 689 if (runTime < SMALL_SUITE_MAX_RUNTIME 690 && !InstrumentationTestCase.class.isAssignableFrom(test.getClass())) { 691 assignmentSuite = SMALL_SUITE; 692 } else if (runTime < MEDIUM_SUITE_MAX_RUNTIME) { 693 assignmentSuite = MEDIUM_SUITE; 694 } else { 695 assignmentSuite = LARGE_SUITE; 696 } 697 } 698 // Clear mStartTime so that we can verify that it gets set next time. 699 mStartTime = -1; 700 701 mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, 702 test.getClass().getName() + "#" + ((TestCase) test).getName() 703 + "\nin " + assignmentSuite + " suite\nrunTime: " 704 + String.valueOf(runTime) + "\n"); 705 mTestResult.putFloat(REPORT_KEY_RUN_TIME, runTime); 706 mTestResult.putString(REPORT_KEY_SUITE_ASSIGNMENT, assignmentSuite); 707 708 sendStatus(0, mTestResult); 709 } 710 } 711 712 /** 713 * This class sends status reports back to the IInstrumentationWatcher 714 */ 715 private class WatcherResultPrinter implements TestListener, PerformanceResultsWriter { 716 private final Bundle mResultTemplate; 717 Bundle mTestResult; 718 int mTestNum = 0; 719 int mTestResultCode = 0; 720 String mTestClass = null; 721 PerformanceCollector mPerfCollector = new PerformanceCollector(); 722 boolean mIsTimedTest = false; 723 boolean mIncludeDetailedStats = false; 724 WatcherResultPrinter(int numTests)725 public WatcherResultPrinter(int numTests) { 726 mResultTemplate = new Bundle(); 727 mResultTemplate.putString(Instrumentation.REPORT_KEY_IDENTIFIER, REPORT_VALUE_ID); 728 mResultTemplate.putInt(REPORT_KEY_NUM_TOTAL, numTests); 729 } 730 731 /** 732 * send a status for the start of a each test, so long tests can be seen 733 * as "running" 734 */ startTest(Test test)735 public void startTest(Test test) { 736 String testClass = test.getClass().getName(); 737 String testName = ((TestCase)test).getName(); 738 mTestResult = new Bundle(mResultTemplate); 739 mTestResult.putString(REPORT_KEY_NAME_CLASS, testClass); 740 mTestResult.putString(REPORT_KEY_NAME_TEST, testName); 741 mTestResult.putInt(REPORT_KEY_NUM_CURRENT, ++mTestNum); 742 // pretty printing 743 if (testClass != null && !testClass.equals(mTestClass)) { 744 mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, 745 String.format("\n%s:", testClass)); 746 mTestClass = testClass; 747 } else { 748 mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, ""); 749 } 750 751 // The delay_msec parameter is normally used to provide buffers of idle time 752 // for power measurement purposes. To make sure there is a delay before and after 753 // every test in a suite, we delay *after* every test (see endTest below) and also 754 // delay *before* the first test. So, delay test1 delay test2 delay. 755 756 try { 757 if (mTestNum == 1) Thread.sleep(mDelayMsec); 758 } catch (InterruptedException e) { 759 throw new IllegalStateException(e); 760 } 761 762 sendStatus(REPORT_VALUE_RESULT_START, mTestResult); 763 mTestResultCode = 0; 764 765 mIsTimedTest = false; 766 mIncludeDetailedStats = false; 767 try { 768 // Look for TimedTest annotation on both test class and test method 769 if (test.getClass().getMethod(testName).isAnnotationPresent(TimedTest.class)) { 770 mIsTimedTest = true; 771 mIncludeDetailedStats = test.getClass().getMethod(testName).getAnnotation( 772 TimedTest.class).includeDetailedStats(); 773 } else if (test.getClass().isAnnotationPresent(TimedTest.class)) { 774 mIsTimedTest = true; 775 mIncludeDetailedStats = test.getClass().getAnnotation( 776 TimedTest.class).includeDetailedStats(); 777 } 778 } catch (SecurityException e) { 779 // ignore - the test with given name cannot be accessed. Will be handled during 780 // test execution 781 } catch (NoSuchMethodException e) { 782 // ignore- the test with given name does not exist. Will be handled during test 783 // execution 784 } 785 786 if (mIsTimedTest && mIncludeDetailedStats) { 787 mPerfCollector.beginSnapshot(""); 788 } else if (mIsTimedTest) { 789 mPerfCollector.startTiming(""); 790 } 791 } 792 793 /** 794 * @see junit.framework.TestListener#addError(Test, Throwable) 795 */ addError(Test test, Throwable t)796 public void addError(Test test, Throwable t) { 797 mTestResult.putString(REPORT_KEY_STACK, BaseTestRunner.getFilteredTrace(t)); 798 mTestResultCode = REPORT_VALUE_RESULT_ERROR; 799 // pretty printing 800 mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, 801 String.format("\nError in %s:\n%s", 802 ((TestCase)test).getName(), BaseTestRunner.getFilteredTrace(t))); 803 } 804 805 /** 806 * @see junit.framework.TestListener#addFailure(Test, AssertionFailedError) 807 */ addFailure(Test test, AssertionFailedError t)808 public void addFailure(Test test, AssertionFailedError t) { 809 mTestResult.putString(REPORT_KEY_STACK, BaseTestRunner.getFilteredTrace(t)); 810 mTestResultCode = REPORT_VALUE_RESULT_FAILURE; 811 // pretty printing 812 mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, 813 String.format("\nFailure in %s:\n%s", 814 ((TestCase)test).getName(), BaseTestRunner.getFilteredTrace(t))); 815 } 816 817 /** 818 * @see junit.framework.TestListener#endTest(Test) 819 */ endTest(Test test)820 public void endTest(Test test) { 821 if (mIsTimedTest && mIncludeDetailedStats) { 822 mTestResult.putAll(mPerfCollector.endSnapshot()); 823 } else if (mIsTimedTest) { 824 writeStopTiming(mPerfCollector.stopTiming("")); 825 } 826 827 if (mTestResultCode == 0) { 828 mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, "."); 829 } 830 sendStatus(mTestResultCode, mTestResult); 831 832 try { // Sleep after every test, if specified 833 Thread.sleep(mDelayMsec); 834 } catch (InterruptedException e) { 835 throw new IllegalStateException(e); 836 } 837 } 838 writeBeginSnapshot(String label)839 public void writeBeginSnapshot(String label) { 840 // Do nothing 841 } 842 writeEndSnapshot(Bundle results)843 public void writeEndSnapshot(Bundle results) { 844 // Copy all snapshot data fields into mResults, which is outputted 845 // via Instrumentation.finish 846 mResults.putAll(results); 847 } 848 writeStartTiming(String label)849 public void writeStartTiming(String label) { 850 // Do nothing 851 } 852 writeStopTiming(Bundle results)853 public void writeStopTiming(Bundle results) { 854 // Copy results into mTestResult by flattening list of iterations, 855 // which is outputted via WatcherResultPrinter.endTest 856 int i = 0; 857 for (Parcelable p : 858 results.getParcelableArrayList(PerformanceCollector.METRIC_KEY_ITERATIONS)) { 859 Bundle iteration = (Bundle)p; 860 String index = "iteration" + i + "."; 861 mTestResult.putString(index + PerformanceCollector.METRIC_KEY_LABEL, 862 iteration.getString(PerformanceCollector.METRIC_KEY_LABEL)); 863 mTestResult.putLong(index + PerformanceCollector.METRIC_KEY_CPU_TIME, 864 iteration.getLong(PerformanceCollector.METRIC_KEY_CPU_TIME)); 865 mTestResult.putLong(index + PerformanceCollector.METRIC_KEY_EXECUTION_TIME, 866 iteration.getLong(PerformanceCollector.METRIC_KEY_EXECUTION_TIME)); 867 i++; 868 } 869 } 870 writeMeasurement(String label, long value)871 public void writeMeasurement(String label, long value) { 872 mTestResult.putLong(label, value); 873 } 874 writeMeasurement(String label, float value)875 public void writeMeasurement(String label, float value) { 876 mTestResult.putFloat(label, value); 877 } 878 writeMeasurement(String label, String value)879 public void writeMeasurement(String label, String value) { 880 mTestResult.putString(label, value); 881 } 882 883 // TODO report the end of the cycle 884 } 885 } 886