1 /* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.result; 17 18 import static com.google.common.truth.Truth.assertThat; 19 20 import static org.junit.Assert.assertEquals; 21 import static org.junit.Assert.assertFalse; 22 import static org.junit.Assert.assertTrue; 23 24 import com.android.ddmlib.testrunner.TestResult.TestStatus; 25 import com.android.tradefed.build.BuildInfo; 26 import com.android.tradefed.invoker.IInvocationContext; 27 import com.android.tradefed.invoker.InvocationContext; 28 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 29 import com.android.tradefed.util.proto.TfMetricProtoUtil; 30 31 import org.junit.Before; 32 import org.junit.Test; 33 import org.junit.runner.RunWith; 34 import org.junit.runners.JUnit4; 35 36 import java.util.HashMap; 37 import java.util.Iterator; 38 import java.util.List; 39 import java.util.Map; 40 41 /** Unit tests for {@link CollectingTestListener}. */ 42 @RunWith(JUnit4.class) 43 public final class CollectingTestListenerTest { 44 45 private static final String METRIC_VALUE = "value"; 46 private static final String TEST_KEY = "key"; 47 private static final String METRIC_VALUE2 = "value2"; 48 private static final String RUN_KEY = "key2"; 49 50 private CollectingTestListener mCollectingTestListener; 51 52 @Before setUp()53 public void setUp() throws Exception { 54 mCollectingTestListener = new CollectingTestListener(); 55 IInvocationContext context = new InvocationContext(); 56 context.addDeviceBuildInfo("fakeDevice", new BuildInfo()); 57 mCollectingTestListener.invocationStarted(context); 58 } 59 60 /** Test the listener under a single normal test run. */ 61 @Test testSingleRun()62 public void testSingleRun() { 63 final TestDescription test = injectTestRun("run", "testFoo", METRIC_VALUE, 0); 64 TestRunResult runResult = mCollectingTestListener.getCurrentRunResults(); 65 66 assertThat(runResult.isRunComplete()).isTrue(); 67 assertThat(runResult.isRunFailure()).isFalse(); 68 assertThat(mCollectingTestListener.getNumTotalTests()).isEqualTo(1); 69 assertThat(runResult.getTestResults().get(test).getStatus()).isEqualTo(TestStatus.PASSED); 70 assertThat(runResult.getTestResults().get(test).getStartTime()).isGreaterThan(0L); 71 assertThat(runResult.getTestResults().get(test).getEndTime()) 72 .isGreaterThan(runResult.getTestResults().get(test).getStartTime() - 1); 73 } 74 75 /** Test the listener where test run has failed. */ 76 @Test testRunFailed()77 public void testRunFailed() { 78 mCollectingTestListener.testRunStarted("foo", 1); 79 mCollectingTestListener.testRunFailed("error"); 80 mCollectingTestListener.testRunEnded(0, new HashMap<String, Metric>()); 81 TestRunResult runResult = mCollectingTestListener.getCurrentRunResults(); 82 assertThat(runResult.isRunComplete()).isTrue(); 83 assertThat(runResult.isRunFailure()).isTrue(); 84 assertThat(runResult.getRunFailureMessage()).isEqualTo("error"); 85 } 86 87 /** Test the listener where test run has failed. */ 88 @Test testRunFailed_counting()89 public void testRunFailed_counting() { 90 mCollectingTestListener.testRunStarted("foo1", 1); 91 mCollectingTestListener.testRunFailed("error1"); 92 mCollectingTestListener.testRunEnded(0, new HashMap<String, Metric>()); 93 mCollectingTestListener.testRunStarted("foo2", 1); 94 mCollectingTestListener.testRunEnded(0, new HashMap<String, Metric>()); 95 mCollectingTestListener.testRunStarted("foo3", 1); 96 mCollectingTestListener.testRunFailed("error3"); 97 mCollectingTestListener.testRunEnded(0, new HashMap<String, Metric>()); 98 assertThat(mCollectingTestListener.getNumAllFailedTestRuns()).isEqualTo(2); 99 } 100 101 /** Test the listener when invocation is composed of two test runs. */ 102 @Test testTwoRuns()103 public void testTwoRuns() { 104 final TestDescription test1 = injectTestRun("run1", "testFoo1", METRIC_VALUE, 0); 105 final TestDescription test2 = injectTestRun("run2", "testFoo2", METRIC_VALUE2, 0); 106 assertThat(mCollectingTestListener.getNumTotalTests()).isEqualTo(2); 107 assertThat(mCollectingTestListener.getNumTestsInState(TestStatus.PASSED)).isEqualTo(2); 108 assertThat(mCollectingTestListener.getRunResults()).hasSize(2); 109 assertThat(mCollectingTestListener.getMergedTestRunResults()).hasSize(2); 110 Iterator<TestRunResult> runIter = 111 mCollectingTestListener.getMergedTestRunResults().iterator(); 112 final TestRunResult runResult1 = runIter.next(); 113 final TestRunResult runResult2 = runIter.next(); 114 115 assertThat(runResult1.getName()).isEqualTo("run1"); 116 assertThat(runResult2.getName()).isEqualTo("run2"); 117 assertThat(runResult1.getTestResults().get(test1).getStatus()).isEqualTo(TestStatus.PASSED); 118 assertThat(runResult2.getTestResults().get(test2).getStatus()).isEqualTo(TestStatus.PASSED); 119 assertThat(runResult1.getRunMetrics().get(RUN_KEY)).isEqualTo(METRIC_VALUE); 120 assertThat(runResult1.getTestResults().get(test1).getMetrics().get(TEST_KEY)) 121 .isEqualTo(METRIC_VALUE); 122 assertThat(runResult2.getTestResults().get(test2).getMetrics().get(TEST_KEY)) 123 .isEqualTo(METRIC_VALUE2); 124 } 125 126 /** Test the listener when invocation is composed of a re-executed test run. */ 127 @Test testReRun()128 public void testReRun() { 129 final TestDescription test1 = injectTestRun("run", "testFoo1", METRIC_VALUE, 0); 130 final TestDescription test2 = injectTestRun("run", "testFoo2", METRIC_VALUE2, 0); 131 assertThat(mCollectingTestListener.getNumTotalTests()).isEqualTo(2); 132 assertThat(mCollectingTestListener.getNumTestsInState(TestStatus.PASSED)).isEqualTo(2); 133 assertThat(mCollectingTestListener.getMergedTestRunResults()).hasSize(1); 134 TestRunResult runResult = mCollectingTestListener.getCurrentRunResults(); 135 assertThat(runResult.getNumTestsInState(TestStatus.PASSED)).isEqualTo(2); 136 assertThat(runResult.getCompletedTests()).contains(test1); 137 assertThat(runResult.getCompletedTests()).contains(test2); 138 } 139 140 /** 141 * Test the listener when invocation is composed of a re-executed test run, containing the same 142 * tests 143 */ 144 @Test testReRun_overlap()145 public void testReRun_overlap() { 146 injectTestRun("run", "testFoo1", METRIC_VALUE, 0); 147 injectTestRun("run", "testFoo1", METRIC_VALUE2, 0, true); 148 assertThat(mCollectingTestListener.getNumTotalTests()).isEqualTo(1); 149 assertThat(mCollectingTestListener.getNumTestsInState(TestStatus.PASSED)).isEqualTo(0); 150 assertThat(mCollectingTestListener.getNumTestsInState(TestStatus.FAILURE)).isEqualTo(1); 151 assertThat(mCollectingTestListener.getMergedTestRunResults()).hasSize(1); 152 TestRunResult runResult = mCollectingTestListener.getCurrentRunResults(); 153 assertThat(runResult.getNumTestsInState(TestStatus.PASSED)).isEqualTo(0); 154 assertThat(runResult.getNumTestsInState(TestStatus.FAILURE)).isEqualTo(1); 155 assertThat(runResult.getNumTests()).isEqualTo(1); 156 } 157 158 @Test testRun_attempts()159 public void testRun_attempts() { 160 injectTestRun("run", "testFoo1", METRIC_VALUE, 0); 161 injectTestRun("run", "testFoo1", METRIC_VALUE, 1); 162 injectTestRun("run", "testFoo2", METRIC_VALUE, 2); 163 injectTestRun("run", "testFoo2", METRIC_VALUE, 0); 164 injectTestRun("run", "testFoo2", METRIC_VALUE, 1); 165 assertThat(mCollectingTestListener.getTestRunNames()).containsExactly("run"); 166 assertThat(mCollectingTestListener.getTestRunAttemptCount("run")).isEqualTo(3); 167 final TestRunResult run1 = mCollectingTestListener.getTestRunAtAttempt("run", 0); 168 final TestRunResult run2 = mCollectingTestListener.getTestRunAtAttempt("run", 1); 169 final TestRunResult run3 = mCollectingTestListener.getTestRunAtAttempt("run", 2); 170 assertThat(mCollectingTestListener.getTestRunAttempts("run")) 171 .containsExactly(run1, run2, run3); 172 assertThat(run1.getNumTests()).isEqualTo(2); 173 assertThat(run2.getNumTests()).isEqualTo(2); 174 assertThat(run3.getNumTests()).isEqualTo(1); 175 assertThat(mCollectingTestListener.getNumTotalTests()).isEqualTo(2); 176 // Check results which do not exist 177 assertThat(mCollectingTestListener.getTestRunAttemptCount("notrun")).isEqualTo(0); 178 assertThat(mCollectingTestListener.getTestRunAttempts("notrun")).isNull(); 179 assertThat(mCollectingTestListener.getTestRunAtAttempt("notrun", 0)).isNull(); 180 assertThat(mCollectingTestListener.getTestRunAtAttempt("run", 3)).isNull(); 181 } 182 183 @Test testRun_invalidAttempts()184 public void testRun_invalidAttempts() { 185 injectTestRun("run", "testFoo1", METRIC_VALUE, 0); 186 injectTestRun("run", "testFoo1", METRIC_VALUE, 2); 187 List<TestRunResult> results = mCollectingTestListener.getTestRunAttempts("run"); 188 assertEquals(3, results.size()); 189 assertFalse(results.get(0).isRunFailure()); 190 assertTrue(results.get(1).isRunFailure()); // Missing run is failed 191 assertFalse(results.get(2).isRunFailure()); 192 } 193 194 /** Test run with incomplete tests */ 195 @Test 196 @SuppressWarnings("unchecked") testSingleRun_incomplete()197 public void testSingleRun_incomplete() { 198 mCollectingTestListener.testRunStarted("run", 1); 199 mCollectingTestListener.testStarted(new TestDescription("FooTest", "incomplete")); 200 mCollectingTestListener.testRunEnded(0, new HashMap<String, Metric>()); 201 assertThat(mCollectingTestListener.getNumTestsInState(TestStatus.INCOMPLETE)).isEqualTo(1); 202 } 203 204 /** Test aggregating of metrics with long values */ 205 @Test testRunEnded_aggregateLongMetrics()206 public void testRunEnded_aggregateLongMetrics() { 207 mCollectingTestListener.setIsAggregrateMetrics(true); 208 injectTestRun("run", "testFoo1", "1", 0); 209 injectTestRun("run", "testFoo1", "1", 0); 210 assertThat(mCollectingTestListener.getCurrentRunResults().getRunMetrics().get(RUN_KEY)) 211 .isEqualTo("2"); 212 } 213 214 /** Test aggregating of metrics with double values */ 215 @Test testRunEnded_aggregateDoubleMetrics()216 public void testRunEnded_aggregateDoubleMetrics() { 217 mCollectingTestListener.setIsAggregrateMetrics(true); 218 injectTestRun("run", "testFoo1", "1.1", 0); 219 injectTestRun("run", "testFoo1", "1.1", 0); 220 assertThat(mCollectingTestListener.getCurrentRunResults().getRunMetrics().get(RUN_KEY)) 221 .isEqualTo("2.2"); 222 } 223 224 /** Test aggregating of metrics with different data types */ 225 @Test testRunEnded_aggregateMixedMetrics()226 public void testRunEnded_aggregateMixedMetrics() { 227 mCollectingTestListener.setIsAggregrateMetrics(true); 228 injectTestRun("run", "testFoo1", "1", 0); 229 injectTestRun("run", "testFoo1", "1.1", 0); 230 mCollectingTestListener.invocationEnded(0); 231 assertThat(mCollectingTestListener.getCurrentRunResults().getRunMetrics().get(RUN_KEY)) 232 .isEqualTo("2.1"); 233 } 234 235 /** Test aggregating of metrics when new metric isn't a number */ 236 @Test testRunEnded_aggregateNewStringMetrics()237 public void testRunEnded_aggregateNewStringMetrics() { 238 mCollectingTestListener.setIsAggregrateMetrics(true); 239 injectTestRun("run", "testFoo1", "1", 0); 240 injectTestRun("run", "testFoo1", "bar", 0); 241 mCollectingTestListener.invocationEnded(0); 242 assertThat(mCollectingTestListener.getCurrentRunResults().getRunMetrics().get(RUN_KEY)) 243 .isEqualTo("bar"); 244 } 245 246 /** Test aggregating of metrics when existing metric isn't a number */ 247 @Test testRunEnded_aggregateExistingStringMetrics()248 public void testRunEnded_aggregateExistingStringMetrics() { 249 mCollectingTestListener.setIsAggregrateMetrics(true); 250 injectTestRun("run", "testFoo1", "bar", 0); 251 injectTestRun("run", "testFoo1", "1", 0); 252 mCollectingTestListener.invocationEnded(0); 253 assertThat(mCollectingTestListener.getCurrentRunResults().getRunMetrics().get(RUN_KEY)) 254 .isEqualTo("1"); 255 } 256 257 @Test testGetNumTestsInState()258 public void testGetNumTestsInState() { 259 injectTestRun("run", "testFoo1", "1", 0); 260 injectTestRun("run", "testFoo2", "1", 0); 261 int testsPassed = mCollectingTestListener.getNumTestsInState(TestStatus.PASSED); 262 assertThat(testsPassed).isEqualTo(2); 263 injectTestRun("run", "testFoo3", "1", 0); 264 testsPassed = mCollectingTestListener.getNumTestsInState(TestStatus.PASSED); 265 assertThat(testsPassed).isEqualTo(3); 266 } 267 268 @Test testGetNumTotalTests()269 public void testGetNumTotalTests() { 270 injectTestRun("run", "testFoo1", "1", 0); 271 injectTestRun("run", "testFoo2", "1", 0); 272 int total = mCollectingTestListener.getNumTotalTests(); 273 assertThat(total).isEqualTo(2); 274 injectTestRun("run", "testFoo3", "1", 0, true); 275 total = mCollectingTestListener.getNumTotalTests(); 276 assertThat(total).isEqualTo(3); 277 } 278 279 @Test testEarlyFailure()280 public void testEarlyFailure() { 281 CollectingTestListener listener = new CollectingTestListener(); 282 listener.testRunFailed("early failure!"); 283 List<TestRunResult> results = listener.getMergedTestRunResults(); 284 assertThat(results.size()).isEqualTo(1); 285 TestRunResult res = results.get(0); 286 // We are in an odd state, but we should not loose the failure. 287 assertThat(res.isRunFailure()).isTrue(); 288 assertThat(res.getName()).isEqualTo("not started"); 289 } 290 291 /** Test the listener under a single normal test run that gets sharded */ 292 @Test testSingleRun_multi()293 public void testSingleRun_multi() { 294 mCollectingTestListener.testRunStarted("run1", 1); 295 final TestDescription test = new TestDescription("FooTest", "testName1"); 296 mCollectingTestListener.testStarted(test); 297 mCollectingTestListener.testEnded(test, new HashMap<String, Metric>()); 298 mCollectingTestListener.testRunEnded(0, new HashMap<String, Metric>()); 299 300 mCollectingTestListener.testRunStarted("run1", 3); 301 final TestDescription test2 = new TestDescription("FooTest", "testName2"); 302 mCollectingTestListener.testStarted(test2); 303 mCollectingTestListener.testEnded(test2, new HashMap<String, Metric>()); 304 mCollectingTestListener.testRunFailed("missing tests"); 305 mCollectingTestListener.testRunEnded(0, new HashMap<String, Metric>()); 306 307 TestRunResult runResult = mCollectingTestListener.getCurrentRunResults(); 308 309 assertThat(runResult.isRunComplete()).isTrue(); 310 assertThat(runResult.isRunFailure()).isTrue(); 311 assertThat(mCollectingTestListener.getNumTotalTests()).isEqualTo(2); 312 assertThat(mCollectingTestListener.getExpectedTests()).isEqualTo(4); 313 } 314 315 /** Test the listener under a single normal test run that gets sharded */ 316 @Test testSingleRun_multi_failureRunFirst()317 public void testSingleRun_multi_failureRunFirst() { 318 mCollectingTestListener.testRunStarted("run1", 3); 319 final TestDescription test2 = new TestDescription("FooTest", "testName2"); 320 mCollectingTestListener.testStarted(test2); 321 mCollectingTestListener.testEnded(test2, new HashMap<String, Metric>()); 322 mCollectingTestListener.testRunFailed("missing tests"); 323 mCollectingTestListener.testRunEnded(0, new HashMap<String, Metric>()); 324 325 mCollectingTestListener.testRunStarted("run1", 1); 326 final TestDescription test = new TestDescription("FooTest", "testName1"); 327 mCollectingTestListener.testStarted(test); 328 mCollectingTestListener.testEnded(test, new HashMap<String, Metric>()); 329 mCollectingTestListener.testRunEnded(0, new HashMap<String, Metric>()); 330 331 TestRunResult runResult = mCollectingTestListener.getCurrentRunResults(); 332 333 assertThat(runResult.isRunComplete()).isTrue(); 334 assertThat(runResult.isRunFailure()).isTrue(); 335 assertThat(mCollectingTestListener.getNumTotalTests()).isEqualTo(2); 336 assertThat(mCollectingTestListener.getExpectedTests()).isEqualTo(4); 337 } 338 339 @Test testSingleRun_multi_failureRunFirst_attempts()340 public void testSingleRun_multi_failureRunFirst_attempts() { 341 mCollectingTestListener.testRunStarted("run1", 3); 342 final TestDescription test2 = new TestDescription("FooTest", "testName2"); 343 mCollectingTestListener.testStarted(test2); 344 mCollectingTestListener.testEnded(test2, new HashMap<String, Metric>()); 345 mCollectingTestListener.testRunFailed("missing tests"); 346 mCollectingTestListener.testRunEnded(0, new HashMap<String, Metric>()); 347 348 mCollectingTestListener.testRunStarted("run1", 1, 1); 349 final TestDescription test = new TestDescription("FooTest", "testName1"); 350 mCollectingTestListener.testStarted(test); 351 mCollectingTestListener.testEnded(test, new HashMap<String, Metric>()); 352 mCollectingTestListener.testRunEnded(0, new HashMap<String, Metric>()); 353 354 TestRunResult runResult = mCollectingTestListener.getMergedTestRunResults().get(0); 355 356 assertThat(runResult.isRunComplete()).isTrue(); 357 assertThat(runResult.isRunFailure()).isTrue(); 358 assertThat(mCollectingTestListener.getNumTotalTests()).isEqualTo(2); 359 // Accross attempt we do not increment the total expected test 360 assertThat(mCollectingTestListener.getExpectedTests()).isEqualTo(3); 361 } 362 363 /** 364 * Injects a single test run with 1 passed test into the {@link CollectingTestListener} under 365 * test 366 * 367 * @return the {@link TestDescription} of added test 368 */ injectTestRun( String runName, String testName, String metricValue, int attempt)369 private TestDescription injectTestRun( 370 String runName, String testName, String metricValue, int attempt) { 371 return injectTestRun(runName, testName, metricValue, attempt, false); 372 } 373 374 /** 375 * Injects a single test run with 1 test into the {@link CollectingTestListener} under test. 376 * 377 * @return the {@link TestDescription} of added test 378 */ injectTestRun( String runName, String testName, String metricValue, int attempt, boolean failtest)379 private TestDescription injectTestRun( 380 String runName, String testName, String metricValue, int attempt, boolean failtest) { 381 Map<String, String> runMetrics = new HashMap<String, String>(1); 382 runMetrics.put(RUN_KEY, metricValue); 383 Map<String, String> testMetrics = new HashMap<String, String>(1); 384 testMetrics.put(TEST_KEY, metricValue); 385 386 mCollectingTestListener.testRunStarted(runName, 1, attempt); 387 final TestDescription test = new TestDescription("FooTest", testName); 388 mCollectingTestListener.testStarted(test); 389 if (failtest) { 390 mCollectingTestListener.testFailed(test, "trace"); 391 } 392 mCollectingTestListener.testEnded(test, TfMetricProtoUtil.upgradeConvert(testMetrics)); 393 mCollectingTestListener.testRunEnded(0, TfMetricProtoUtil.upgradeConvert(runMetrics)); 394 return test; 395 } 396 } 397