1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.testtype; 18 19 import static org.junit.Assert.assertEquals; 20 import static org.junit.Assert.assertNotNull; 21 import static org.junit.Assert.assertTrue; 22 import static org.junit.Assert.fail; 23 24 import com.android.ddmlib.IDevice; 25 import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner; 26 import com.android.ddmlib.testrunner.ITestRunListener; 27 import com.android.tradefed.config.ConfigurationException; 28 import com.android.tradefed.device.DeviceNotAvailableException; 29 import com.android.tradefed.device.ITestDevice; 30 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 31 import com.android.tradefed.result.ITestInvocationListener; 32 import com.android.tradefed.result.ITestLifeCycleReceiver; 33 import com.android.tradefed.result.TestDescription; 34 import com.android.tradefed.result.ddmlib.TestRunToTestInvocationForwarder; 35 36 import org.easymock.EasyMock; 37 import org.easymock.IAnswer; 38 import org.junit.Before; 39 import org.junit.Test; 40 import org.junit.runner.RunWith; 41 import org.junit.runners.JUnit4; 42 import org.mockito.Mockito; 43 44 import java.io.BufferedReader; 45 import java.io.File; 46 import java.io.FileReader; 47 import java.io.IOException; 48 import java.util.ArrayList; 49 import java.util.Collection; 50 import java.util.Collections; 51 import java.util.HashMap; 52 53 /** Unit tests for {@link InstrumentationFileTest}. */ 54 @RunWith(JUnit4.class) 55 public class InstrumentationFileTestTest { 56 57 private static final String TEST_PACKAGE_VALUE = "com.foo"; 58 59 /** The {@link InstrumentationFileTest} under test, with all dependencies mocked out */ 60 private InstrumentationFileTest mInstrumentationFileTest; 61 62 private ITestDevice mMockTestDevice; 63 private ITestInvocationListener mMockListener; 64 private InstrumentationTest mMockITest; 65 66 private File mTestFile; 67 68 @Before setUp()69 public void setUp() throws Exception { 70 mTestFile = null; 71 72 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 73 mMockTestDevice = EasyMock.createMock(ITestDevice.class); 74 mMockListener = EasyMock.createMock(ITestInvocationListener.class); 75 76 EasyMock.expect(mMockTestDevice.getIDevice()).andStubReturn(mockIDevice); 77 EasyMock.expect(mMockTestDevice.getSerialNumber()).andStubReturn("serial"); 78 79 // mock out InstrumentationTest that will be used to create InstrumentationFileTest 80 mMockITest = new InstrumentationTest() { 81 @Override 82 protected String queryRunnerName() { 83 return "runner"; 84 } 85 }; 86 mMockITest.setDevice(mMockTestDevice); 87 mMockITest.setPackageName(TEST_PACKAGE_VALUE); 88 mMockITest = Mockito.spy(mMockITest); 89 } 90 91 /** Test normal run scenario with a single test. */ 92 @Test testRun_singleSuccessfulTest()93 public void testRun_singleSuccessfulTest() 94 throws DeviceNotAvailableException, ConfigurationException { 95 final Collection<TestDescription> testsList = new ArrayList<>(1); 96 final TestDescription test = new TestDescription("ClassFoo", "methodBar"); 97 testsList.add(test); 98 99 // verify the mock listener is passed through to the runner 100 RunTestAnswer runTestResponse = 101 new RunTestAnswer() { 102 @Override 103 public Boolean answer( 104 IRemoteAndroidTestRunner runner, ITestRunListener listener) { 105 listener.testRunStarted(TEST_PACKAGE_VALUE, 1); 106 listener.testStarted(TestDescription.convertToIdentifier(test)); 107 listener.testEnded( 108 TestDescription.convertToIdentifier(test), Collections.emptyMap()); 109 listener.testRunEnded(0, Collections.emptyMap()); 110 return true; 111 } 112 }; 113 setRunTestExpectations(runTestResponse); 114 mInstrumentationFileTest = new InstrumentationFileTest(mMockITest, testsList, true, -1) { 115 @Override 116 InstrumentationTest createInstrumentationTest() { 117 return mMockITest; 118 } 119 @Override 120 boolean pushFileToTestDevice(File file, String destinationPath) 121 throws DeviceNotAvailableException { 122 // simulate successful push and store created file 123 mTestFile = file; 124 // verify that the content of the testFile contains all expected tests 125 verifyTestFile(testsList); 126 return true; 127 } 128 @Override 129 void deleteTestFileFromDevice(String pathToFile) throws DeviceNotAvailableException { 130 //ignore 131 } 132 }; 133 134 // mock successful test run lifecycle 135 mMockListener.testRunStarted(TEST_PACKAGE_VALUE, 1); 136 mMockListener.testStarted(EasyMock.eq(test), EasyMock.anyLong()); 137 mMockListener.testEnded( 138 EasyMock.eq(test), EasyMock.anyLong(), EasyMock.eq(new HashMap<String, Metric>())); 139 mMockListener.testRunEnded(EasyMock.anyLong(), EasyMock.eq(new HashMap<String, Metric>())); 140 141 EasyMock.replay(mMockListener, mMockTestDevice); 142 mInstrumentationFileTest.run(mMockListener); 143 assertEquals(mMockTestDevice, mMockITest.getDevice()); 144 // Ensure that we unset the package name 145 Mockito.verify(mMockITest).setTestPackageName(null); 146 Mockito.verify(mMockITest).removeFromInstrumentationArg("package"); 147 } 148 149 /** 150 * Test re-run scenario when 1 out of 3 tests fails to complete but is successful after re-run 151 */ 152 @Test testRun_reRunOneFailedToCompleteTest()153 public void testRun_reRunOneFailedToCompleteTest() 154 throws DeviceNotAvailableException, ConfigurationException { 155 final Collection<TestDescription> testsList = new ArrayList<>(1); 156 final TestDescription test1 = new TestDescription("ClassFoo1", "methodBar1"); 157 final TestDescription test2 = new TestDescription("ClassFoo2", "methodBar2"); 158 final TestDescription test3 = new TestDescription("ClassFoo3", "methodBar3"); 159 testsList.add(test1); 160 testsList.add(test2); 161 testsList.add(test3); 162 163 // verify the test1 is completed and test2 was started but never finished 164 RunTestAnswer firstRunAnswer = 165 new RunTestAnswer() { 166 @Override 167 public Boolean answer( 168 IRemoteAndroidTestRunner runner, ITestRunListener listener) { 169 // first test started and ended successfully 170 listener.testRunStarted(TEST_PACKAGE_VALUE, 2); 171 listener.testStarted(TestDescription.convertToIdentifier(test1)); 172 listener.testEnded( 173 TestDescription.convertToIdentifier(test1), Collections.emptyMap()); 174 listener.testRunEnded(1, Collections.emptyMap()); 175 // second test started but never finished 176 listener.testStarted(TestDescription.convertToIdentifier(test2)); 177 return true; 178 } 179 }; 180 setRunTestExpectations(firstRunAnswer); 181 182 // now expect second run to rerun remaining test3 and test2 183 RunTestAnswer secondRunAnswer = 184 new RunTestAnswer() { 185 @Override 186 public Boolean answer( 187 IRemoteAndroidTestRunner runner, ITestRunListener listener) { 188 // third test started and ended successfully 189 listener.testRunStarted(TEST_PACKAGE_VALUE, 2); 190 listener.testStarted(TestDescription.convertToIdentifier(test3)); 191 listener.testEnded( 192 TestDescription.convertToIdentifier(test3), Collections.emptyMap()); 193 listener.testRunEnded(1, Collections.emptyMap()); 194 // second test is rerun but completed successfully this time 195 listener.testStarted(TestDescription.convertToIdentifier(test2)); 196 listener.testEnded( 197 TestDescription.convertToIdentifier(test2), Collections.emptyMap()); 198 listener.testRunEnded(1, Collections.emptyMap()); 199 return true; 200 } 201 }; 202 setRunTestExpectations(secondRunAnswer); 203 mInstrumentationFileTest = new InstrumentationFileTest(mMockITest, testsList, true, -1) { 204 @Override 205 InstrumentationTest createInstrumentationTest() { 206 return mMockITest; 207 } 208 @Override 209 boolean pushFileToTestDevice(File file, String destinationPath) 210 throws DeviceNotAvailableException { 211 // simulate successful push and store created file 212 mTestFile = file; 213 // verify that the content of the testFile contains all expected tests 214 verifyTestFile(testsList); 215 return true; 216 } 217 @Override 218 void deleteTestFileFromDevice(String pathToFile) throws DeviceNotAvailableException { 219 //ignore 220 } 221 }; 222 223 // First run: 224 mMockListener.testRunStarted(TEST_PACKAGE_VALUE, 2); 225 // expect test1 to start and finish successfully 226 mMockListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong()); 227 mMockListener.testEnded( 228 EasyMock.eq(test1), EasyMock.anyLong(), EasyMock.eq(new HashMap<String, Metric>())); 229 mMockListener.testRunEnded(EasyMock.anyLong(), EasyMock.eq(new HashMap<String, Metric>())); 230 // expect test2 to start but never finish 231 mMockListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong()); 232 // Second run: 233 mMockListener.testRunStarted(TEST_PACKAGE_VALUE, 2); 234 // expect test3 to start and finish successfully 235 mMockListener.testStarted(EasyMock.eq(test3), EasyMock.anyLong()); 236 mMockListener.testEnded( 237 EasyMock.eq(test3), EasyMock.anyLong(), EasyMock.eq(new HashMap<String, Metric>())); 238 mMockListener.testRunEnded(EasyMock.anyLong(), EasyMock.eq(new HashMap<String, Metric>())); 239 // expect to rerun test2 successfully 240 mMockListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong()); 241 mMockListener.testEnded( 242 EasyMock.eq(test2), EasyMock.anyLong(), EasyMock.eq(new HashMap<String, Metric>())); 243 mMockListener.testRunEnded(EasyMock.anyLong(), EasyMock.eq(new HashMap<String, Metric>())); 244 245 EasyMock.replay(mMockListener, mMockTestDevice); 246 mInstrumentationFileTest.run(mMockListener); 247 assertEquals(mMockTestDevice, mMockITest.getDevice()); 248 } 249 250 /** Test re-run scenario when 2 remaining tests fail to complete and need to be run serially */ 251 @Test testRun_serialReRunOfTwoFailedToCompleteTests()252 public void testRun_serialReRunOfTwoFailedToCompleteTests() 253 throws DeviceNotAvailableException, ConfigurationException { 254 final Collection<TestDescription> testsList = new ArrayList<>(1); 255 final TestDescription test1 = new TestDescription("ClassFoo1", "methodBar1"); 256 final TestDescription test2 = new TestDescription("ClassFoo2", "methodBar2"); 257 testsList.add(test1); 258 testsList.add(test2); 259 260 // verify the test1 and test2 started but never completed 261 RunTestAnswer firstRunAnswer = 262 new RunTestAnswer() { 263 @Override 264 public Boolean answer( 265 IRemoteAndroidTestRunner runner, ITestRunListener listener) { 266 // first and second tests started but never completed 267 listener.testRunStarted(TEST_PACKAGE_VALUE, 2); 268 listener.testStarted(TestDescription.convertToIdentifier(test1)); 269 listener.testStarted(TestDescription.convertToIdentifier(test2)); 270 // verify that the content of the testFile contains all expected tests 271 verifyTestFile(testsList); 272 return true; 273 } 274 }; 275 setRunTestExpectations(firstRunAnswer); 276 277 // verify successful serial execution of test1 278 RunTestAnswer firstSerialRunAnswer = 279 new RunTestAnswer() { 280 @Override 281 public Boolean answer( 282 IRemoteAndroidTestRunner runner, ITestRunListener listener) { 283 // first test started and ended successfully in serial mode 284 listener.testRunStarted(TEST_PACKAGE_VALUE, 1); 285 listener.testStarted(TestDescription.convertToIdentifier(test1)); 286 listener.testEnded( 287 TestDescription.convertToIdentifier(test1), Collections.emptyMap()); 288 listener.testRunEnded(1, Collections.emptyMap()); 289 return true; 290 } 291 }; 292 setRunTestExpectations(firstSerialRunAnswer); 293 294 // verify successful serial execution of test2 295 RunTestAnswer secdondSerialRunAnswer = 296 new RunTestAnswer() { 297 @Override 298 public Boolean answer( 299 IRemoteAndroidTestRunner runner, ITestRunListener listener) { 300 // Second test started and ended successfully in serial mode 301 listener.testRunStarted(TEST_PACKAGE_VALUE, 1); 302 listener.testStarted(TestDescription.convertToIdentifier(test2)); 303 listener.testEnded( 304 TestDescription.convertToIdentifier(test2), Collections.emptyMap()); 305 listener.testRunEnded(1, Collections.emptyMap()); 306 return true; 307 } 308 }; 309 setRunTestExpectations(secdondSerialRunAnswer); 310 311 mInstrumentationFileTest = new InstrumentationFileTest(mMockITest, testsList, true, -1) { 312 @Override 313 InstrumentationTest createInstrumentationTest() { 314 return mMockITest; 315 } 316 @Override 317 boolean pushFileToTestDevice(File file, String destinationPath) 318 throws DeviceNotAvailableException { 319 // simulate successful push and store created file 320 mTestFile = file; 321 return true; 322 } 323 @Override 324 void deleteTestFileFromDevice(String pathToFile) throws DeviceNotAvailableException { 325 //ignore 326 } 327 }; 328 329 // First run: 330 mMockListener.testRunStarted(TEST_PACKAGE_VALUE, 2); 331 // expect test1 and test 2 to start but never finish 332 mMockListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong()); 333 mMockListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong()); 334 335 // re-run test1 and test2 serially 336 // first serial re-run: 337 mMockListener.testRunStarted(TEST_PACKAGE_VALUE, 1); 338 // expect test1 to start and finish successfully 339 mMockListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong()); 340 mMockListener.testEnded( 341 EasyMock.eq(test1), EasyMock.anyLong(), EasyMock.eq(new HashMap<String, Metric>())); 342 mMockListener.testRunEnded(EasyMock.anyLong(), EasyMock.eq(new HashMap<String, Metric>())); 343 // first serial re-run: 344 mMockListener.testRunStarted(TEST_PACKAGE_VALUE, 0, 1); 345 // expect test2 to start and finish successfully 346 mMockListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong()); 347 mMockListener.testEnded( 348 EasyMock.eq(test2), EasyMock.anyLong(), EasyMock.eq(new HashMap<String, Metric>())); 349 mMockListener.testRunEnded(EasyMock.anyLong(), EasyMock.eq(new HashMap<String, Metric>())); 350 351 EasyMock.replay(mMockListener, mMockTestDevice); 352 mInstrumentationFileTest.run(mMockListener); 353 assertEquals(mMockTestDevice, mMockITest.getDevice()); 354 // test file is expected to be null since we defaulted to serial test execution 355 assertEquals(null, mMockITest.getTestFilePathOnDevice()); 356 } 357 358 /** Test no serial re-run tests fail to complete. */ 359 @Test testRun_noSerialReRun()360 public void testRun_noSerialReRun() throws DeviceNotAvailableException, ConfigurationException { 361 final Collection<TestDescription> testsList = new ArrayList<>(1); 362 final TestDescription test1 = new TestDescription("ClassFoo1", "methodBar1"); 363 final TestDescription test2 = new TestDescription("ClassFoo2", "methodBar2"); 364 testsList.add(test1); 365 testsList.add(test2); 366 367 // verify the test1 and test2 started but never completed 368 RunTestAnswer firstRunAnswer = 369 new RunTestAnswer() { 370 @Override 371 public Boolean answer( 372 IRemoteAndroidTestRunner runner, ITestRunListener listener) { 373 // first and second tests started but never completed 374 listener.testRunStarted(TEST_PACKAGE_VALUE, 2); 375 listener.testStarted(TestDescription.convertToIdentifier(test1)); 376 listener.testStarted(TestDescription.convertToIdentifier(test2)); 377 // verify that the content of the testFile contains all expected tests 378 verifyTestFile(testsList); 379 return true; 380 } 381 }; 382 setRunTestExpectations(firstRunAnswer); 383 384 mInstrumentationFileTest = new InstrumentationFileTest(mMockITest, testsList, false, -1) { 385 @Override 386 InstrumentationTest createInstrumentationTest() { 387 return mMockITest; 388 } 389 @Override 390 boolean pushFileToTestDevice(File file, String destinationPath) 391 throws DeviceNotAvailableException { 392 // simulate successful push and store created file 393 mTestFile = file; 394 return true; 395 } 396 @Override 397 void deleteTestFileFromDevice(String pathToFile) throws DeviceNotAvailableException { 398 //ignore 399 } 400 }; 401 402 // First run: 403 mMockListener.testRunStarted(TEST_PACKAGE_VALUE, 2); 404 // expect test1 and test 2 to start but never finish 405 mMockListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong()); 406 mMockListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong()); 407 408 EasyMock.replay(mMockListener, mMockTestDevice); 409 mInstrumentationFileTest.run(mMockListener); 410 assertEquals(mMockTestDevice, mMockITest.getDevice()); 411 } 412 413 /** Test attempting times exceed max attempts. */ 414 @Test testRun_exceedMaxAttempts()415 public void testRun_exceedMaxAttempts() 416 throws DeviceNotAvailableException, ConfigurationException { 417 final ArrayList<TestDescription> testsList = new ArrayList<>(1); 418 final TestDescription test1 = new TestDescription("ClassFoo1", "methodBar1"); 419 final TestDescription test2 = new TestDescription("ClassFoo2", "methodBar2"); 420 final TestDescription test3 = new TestDescription("ClassFoo3", "methodBar3"); 421 final TestDescription test4 = new TestDescription("ClassFoo4", "methodBar4"); 422 final TestDescription test5 = new TestDescription("ClassFoo5", "methodBar5"); 423 final TestDescription test6 = new TestDescription("ClassFoo6", "methodBar6"); 424 425 testsList.add(test1); 426 testsList.add(test2); 427 testsList.add(test3); 428 testsList.add(test4); 429 testsList.add(test5); 430 testsList.add(test6); 431 432 final ArrayList<TestDescription> expectedTestsList = new ArrayList<>(testsList); 433 434 // test1 fininshed, test2 started but not finished. 435 RunTestAnswer firstRunAnswer = 436 new RunTestAnswer() { 437 @Override 438 public Boolean answer( 439 IRemoteAndroidTestRunner runner, ITestRunListener listener) { 440 listener.testRunStarted(TEST_PACKAGE_VALUE, 6); 441 // first test started and ended successfully 442 listener.testStarted(TestDescription.convertToIdentifier(test1)); 443 listener.testEnded( 444 TestDescription.convertToIdentifier(test1), Collections.emptyMap()); 445 listener.testRunEnded(1, Collections.emptyMap()); 446 // second test started but never finished 447 listener.testStarted(TestDescription.convertToIdentifier(test2)); 448 // verify that the content of the testFile contains all expected tests 449 verifyTestFile(expectedTestsList); 450 return true; 451 } 452 }; 453 setRunTestExpectations(firstRunAnswer); 454 455 // test2 finished, test3 started but not finished. 456 RunTestAnswer secondRunAnswer = 457 new RunTestAnswer() { 458 @Override 459 public Boolean answer( 460 IRemoteAndroidTestRunner runner, ITestRunListener listener) { 461 // test2 started and ended successfully 462 listener.testRunStarted(TEST_PACKAGE_VALUE, 5); 463 listener.testStarted(TestDescription.convertToIdentifier(test2)); 464 listener.testEnded( 465 TestDescription.convertToIdentifier(test2), Collections.emptyMap()); 466 listener.testRunEnded(1, Collections.emptyMap()); 467 // test3 started but never finished 468 listener.testStarted(TestDescription.convertToIdentifier(test3)); 469 // verify that the content of the testFile contains all expected tests 470 verifyTestFile(expectedTestsList.subList(1, expectedTestsList.size())); 471 return true; 472 } 473 }; 474 setRunTestExpectations(secondRunAnswer); 475 476 // test3 finished, test4 started but not finished. 477 RunTestAnswer thirdRunAnswer = 478 new RunTestAnswer() { 479 @Override 480 public Boolean answer( 481 IRemoteAndroidTestRunner runner, ITestRunListener listener) { 482 // test3 started and ended successfully 483 listener.testRunStarted(TEST_PACKAGE_VALUE, 4); 484 listener.testStarted(TestDescription.convertToIdentifier(test3)); 485 listener.testEnded( 486 TestDescription.convertToIdentifier(test3), Collections.emptyMap()); 487 listener.testRunEnded(1, Collections.emptyMap()); 488 // test4 started but never finished 489 listener.testStarted(TestDescription.convertToIdentifier(test4)); 490 // verify that the content of the testFile contains all expected tests 491 verifyTestFile(expectedTestsList.subList(2, expectedTestsList.size())); 492 return true; 493 } 494 }; 495 setRunTestExpectations(thirdRunAnswer); 496 497 mInstrumentationFileTest = new InstrumentationFileTest(mMockITest, testsList, false, 3) { 498 @Override 499 InstrumentationTest createInstrumentationTest() { 500 return mMockITest; 501 } 502 @Override 503 boolean pushFileToTestDevice(File file, String destinationPath) 504 throws DeviceNotAvailableException { 505 // simulate successful push and store created file 506 mTestFile = file; 507 return true; 508 } 509 @Override 510 void deleteTestFileFromDevice(String pathToFile) throws DeviceNotAvailableException { 511 //ignore 512 } 513 }; 514 515 // First run: 516 mMockListener.testRunStarted(TEST_PACKAGE_VALUE, 6); 517 mMockListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong()); 518 mMockListener.testEnded( 519 EasyMock.eq(test1), EasyMock.anyLong(), EasyMock.eq(new HashMap<String, Metric>())); 520 mMockListener.testRunEnded(EasyMock.anyLong(), EasyMock.eq(new HashMap<String, Metric>())); 521 mMockListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong()); 522 523 // Second run: 524 mMockListener.testRunStarted(TEST_PACKAGE_VALUE, 5); 525 mMockListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong()); 526 mMockListener.testEnded( 527 EasyMock.eq(test2), EasyMock.anyLong(), EasyMock.eq(new HashMap<String, Metric>())); 528 mMockListener.testRunEnded(EasyMock.anyLong(), EasyMock.eq(new HashMap<String, Metric>())); 529 mMockListener.testStarted(EasyMock.eq(test3), EasyMock.anyLong()); 530 531 // Third run: 532 mMockListener.testRunStarted(TEST_PACKAGE_VALUE, 4); 533 mMockListener.testStarted(EasyMock.eq(test3), EasyMock.anyLong()); 534 mMockListener.testEnded( 535 EasyMock.eq(test3), EasyMock.anyLong(), EasyMock.eq(new HashMap<String, Metric>())); 536 mMockListener.testRunEnded(EasyMock.anyLong(), EasyMock.eq(new HashMap<String, Metric>())); 537 mMockListener.testStarted(EasyMock.eq(test4), EasyMock.anyLong()); 538 539 // MAX_ATTEMPTS is 3, so there will be no forth run. 540 541 EasyMock.replay(mMockListener, mMockTestDevice); 542 mInstrumentationFileTest.run(mMockListener); 543 assertEquals(mMockTestDevice, mMockITest.getDevice()); 544 } 545 546 /** Test re-run a test instrumentation when some methods are parameterized. */ 547 @Test testRun_parameterized()548 public void testRun_parameterized() throws DeviceNotAvailableException, ConfigurationException { 549 final Collection<TestDescription> testsList = new ArrayList<>(); 550 final TestDescription test = new TestDescription("ClassFoo", "methodBar"); 551 final TestDescription test1 = new TestDescription("ClassFoo", "paramMethod[0]"); 552 final TestDescription test2 = new TestDescription("ClassFoo", "paramMethod[1]"); 553 testsList.add(test); 554 testsList.add(test1); 555 testsList.add(test2); 556 557 // verify the mock listener is passed through to the runner, the first test pass 558 RunTestAnswer runTestResponse = 559 new RunTestAnswer() { 560 @Override 561 public Boolean answer( 562 IRemoteAndroidTestRunner runner, ITestRunListener listener) { 563 listener.testRunStarted(TEST_PACKAGE_VALUE, 3); 564 listener.testStarted(TestDescription.convertToIdentifier(test)); 565 listener.testEnded( 566 TestDescription.convertToIdentifier(test), Collections.emptyMap()); 567 listener.testStarted(TestDescription.convertToIdentifier(test1)); 568 listener.testRunEnded(0, Collections.emptyMap()); 569 return true; 570 } 571 }; 572 setRunTestExpectations(runTestResponse); 573 574 RunTestAnswer secondRunAnswer = 575 new RunTestAnswer() { 576 @Override 577 public Boolean answer( 578 IRemoteAndroidTestRunner runner, ITestRunListener listener) { 579 // test2 started and ended successfully 580 listener.testRunStarted(TEST_PACKAGE_VALUE, 2); 581 listener.testStarted(TestDescription.convertToIdentifier(test1)); 582 listener.testEnded( 583 TestDescription.convertToIdentifier(test1), Collections.emptyMap()); 584 listener.testStarted(TestDescription.convertToIdentifier(test2)); 585 listener.testEnded( 586 TestDescription.convertToIdentifier(test2), Collections.emptyMap()); 587 listener.testRunEnded(1, Collections.emptyMap()); 588 return true; 589 } 590 }; 591 setRunTestExpectations(secondRunAnswer); 592 593 mInstrumentationFileTest = 594 new InstrumentationFileTest(mMockITest, testsList, true, -1) { 595 @Override 596 InstrumentationTest createInstrumentationTest() { 597 return mMockITest; 598 } 599 600 @Override 601 boolean pushFileToTestDevice(File file, String destinationPath) 602 throws DeviceNotAvailableException { 603 // simulate successful push and store created file 604 mTestFile = file; 605 // verify that the content of the testFile contains all expected tests 606 Collection<TestDescription> updatedList = new ArrayList<>(); 607 updatedList.add(test); 608 updatedList.add(new TestDescription("ClassFoo", "paramMethod")); 609 verifyTestFile(updatedList); 610 return true; 611 } 612 613 @Override 614 void deleteTestFileFromDevice(String pathToFile) 615 throws DeviceNotAvailableException { 616 //ignore 617 } 618 }; 619 620 // mock successful test run lifecycle for the first test 621 mMockListener.testRunStarted(TEST_PACKAGE_VALUE, 3); 622 mMockListener.testStarted(EasyMock.eq(test), EasyMock.anyLong()); 623 mMockListener.testEnded( 624 EasyMock.eq(test), EasyMock.anyLong(), EasyMock.eq(new HashMap<String, Metric>())); 625 mMockListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong()); 626 mMockListener.testRunEnded(EasyMock.anyLong(), EasyMock.eq(new HashMap<String, Metric>())); 627 628 // Second run: 629 mMockListener.testRunStarted(TEST_PACKAGE_VALUE, 2); 630 mMockListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong()); 631 mMockListener.testEnded( 632 EasyMock.eq(test1), EasyMock.anyLong(), EasyMock.eq(new HashMap<String, Metric>())); 633 mMockListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong()); 634 mMockListener.testEnded( 635 EasyMock.eq(test2), EasyMock.anyLong(), EasyMock.eq(new HashMap<String, Metric>())); 636 mMockListener.testRunEnded(EasyMock.anyLong(), EasyMock.eq(new HashMap<String, Metric>())); 637 638 EasyMock.replay(mMockListener, mMockTestDevice); 639 mInstrumentationFileTest.run(mMockListener); 640 assertEquals(mMockTestDevice, mMockITest.getDevice()); 641 } 642 643 /** 644 * Helper class that verifies tetFile's content match the expected list of test to be run 645 * 646 * @param testsList list of test to be executed 647 */ verifyTestFile(Collection<TestDescription> testsList)648 private void verifyTestFile(Collection<TestDescription> testsList) { 649 // fail if the file was never created 650 assertNotNull(mTestFile); 651 652 try (BufferedReader br = new BufferedReader(new FileReader(mTestFile))) { 653 String line; 654 while ((line = br.readLine()) != null) { 655 String[] str = line.split("#"); 656 TestDescription test = new TestDescription(str[0], str[1]); 657 assertTrue(String.format( 658 "Test with class name: %s and method name: %s does not exists", 659 test.getClassName(), test.getTestName()), testsList.contains(test)); 660 } 661 } catch (IOException e) { 662 // fail if the file is corrupt in any way 663 fail("failed reading test file"); 664 } 665 } 666 667 /** 668 * Helper class for providing an EasyMock {@link IAnswer} to a 669 * {@link ITestDevice#runInstrumentationTests(IRemoteAndroidTestRunner, Collection)} call. 670 */ 671 private static abstract class RunTestAnswer implements IAnswer<Boolean> { 672 @Override answer()673 public Boolean answer() throws Throwable { 674 Object[] args = EasyMock.getCurrentArguments(); 675 return answer( 676 (IRemoteAndroidTestRunner) args[0], 677 new TestRunToTestInvocationForwarder((ITestLifeCycleReceiver) args[1])); 678 } 679 answer(IRemoteAndroidTestRunner runner, ITestRunListener listener)680 public abstract Boolean answer(IRemoteAndroidTestRunner runner, 681 ITestRunListener listener) throws DeviceNotAvailableException; 682 } 683 setRunTestExpectations(RunTestAnswer runTestResponse)684 private void setRunTestExpectations(RunTestAnswer runTestResponse) 685 throws DeviceNotAvailableException { 686 687 EasyMock.expect( 688 mMockTestDevice.runInstrumentationTests( 689 (IRemoteAndroidTestRunner) EasyMock.anyObject(), 690 (ITestLifeCycleReceiver) EasyMock.anyObject())) 691 .andAnswer(runTestResponse); 692 } 693 } 694