1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.util; 17 18 import static org.junit.Assert.assertEquals; 19 import static org.junit.Assert.assertFalse; 20 import static org.junit.Assert.assertTrue; 21 import static org.junit.Assert.fail; 22 23 import com.android.tradefed.build.BuildInfo; 24 import com.android.tradefed.build.IBuildInfo; 25 import com.android.tradefed.invoker.IInvocationContext; 26 import com.android.tradefed.invoker.InvocationContext; 27 import com.android.tradefed.log.LogUtil.CLog; 28 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 29 import com.android.tradefed.result.ILogSaverListener; 30 import com.android.tradefed.result.ITestInvocationListener; 31 import com.android.tradefed.result.LogDataType; 32 import com.android.tradefed.result.LogFile; 33 import com.android.tradefed.result.TestDescription; 34 35 import org.easymock.Capture; 36 import org.easymock.EasyMock; 37 import org.junit.Test; 38 import org.junit.runner.RunWith; 39 import org.junit.runners.JUnit4; 40 41 import java.io.BufferedReader; 42 import java.io.File; 43 import java.io.IOException; 44 import java.io.InputStream; 45 import java.io.InputStreamReader; 46 import java.io.PrintWriter; 47 import java.net.Socket; 48 import java.util.HashMap; 49 import java.util.Map; 50 import java.util.Vector; 51 52 /** Unit Tests for {@link SubprocessTestResultsParser} */ 53 @RunWith(JUnit4.class) 54 public class SubprocessTestResultsParserTest { 55 56 private static final String TEST_TYPE_DIR = "testdata"; 57 private static final String SUBPROC_OUTPUT_FILE_1 = "subprocess1.txt"; 58 private static final String SUBPROC_OUTPUT_FILE_2 = "subprocess2.txt"; 59 60 /** 61 * Helper to read a file from the res/testdata directory and return its contents as a String[] 62 * 63 * @param filename the name of the file (without the extension) in the res/testdata directory 64 * @return a String[] of the 65 */ readInFile(String filename)66 private String[] readInFile(String filename) { 67 Vector<String> fileContents = new Vector<String>(); 68 try { 69 InputStream gtestResultStream1 = getClass().getResourceAsStream(File.separator + 70 TEST_TYPE_DIR + File.separator + filename); 71 BufferedReader reader = new BufferedReader(new InputStreamReader(gtestResultStream1)); 72 String line = null; 73 while ((line = reader.readLine()) != null) { 74 fileContents.add(line); 75 } 76 } 77 catch (NullPointerException e) { 78 CLog.e("Gest output file does not exist: " + filename); 79 } 80 catch (IOException e) { 81 CLog.e("Unable to read contents of gtest output file: " + filename); 82 } 83 return fileContents.toArray(new String[fileContents.size()]); 84 } 85 86 /** Tests the parser for cases of test failed, ignored, assumption failure */ 87 @Test testParse_randomEvents()88 public void testParse_randomEvents() throws Exception { 89 String[] contents = readInFile(SUBPROC_OUTPUT_FILE_1); 90 ITestInvocationListener mockRunListener = 91 EasyMock.createMock(ITestInvocationListener.class); 92 mockRunListener.testRunStarted("arm64-v8a CtsGestureTestCases", 4); 93 mockRunListener.testStarted((TestDescription) EasyMock.anyObject(), EasyMock.anyLong()); 94 EasyMock.expectLastCall().times(4); 95 mockRunListener.testEnded( 96 (TestDescription) EasyMock.anyObject(), 97 EasyMock.anyLong(), 98 (HashMap<String, Metric>) EasyMock.anyObject()); 99 EasyMock.expectLastCall().times(4); 100 mockRunListener.testRunEnded( 101 EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 102 EasyMock.expectLastCall().times(1); 103 mockRunListener.testIgnored((TestDescription) EasyMock.anyObject()); 104 EasyMock.expectLastCall(); 105 mockRunListener.testFailed( 106 (TestDescription) EasyMock.anyObject(), (String) EasyMock.anyObject()); 107 EasyMock.expectLastCall(); 108 mockRunListener.testAssumptionFailure( 109 (TestDescription) EasyMock.anyObject(), (String) EasyMock.anyObject()); 110 EasyMock.expectLastCall(); 111 EasyMock.replay(mockRunListener); 112 File tmp = FileUtil.createTempFile("sub", "unit"); 113 SubprocessTestResultsParser resultParser = null; 114 try { 115 resultParser = 116 new SubprocessTestResultsParser(mockRunListener, new InvocationContext()); 117 resultParser.processNewLines(contents); 118 EasyMock.verify(mockRunListener); 119 } finally { 120 StreamUtil.close(resultParser); 121 FileUtil.deleteFile(tmp); 122 } 123 } 124 125 /** Tests the parser for cases of test starting without closing. */ 126 @Test testParse_invalidEventOrder()127 public void testParse_invalidEventOrder() throws Exception { 128 String[] contents = readInFile(SUBPROC_OUTPUT_FILE_2); 129 ITestInvocationListener mockRunListener = 130 EasyMock.createMock(ITestInvocationListener.class); 131 mockRunListener.testRunStarted("arm64-v8a CtsGestureTestCases", 4); 132 mockRunListener.testStarted((TestDescription) EasyMock.anyObject(), EasyMock.anyLong()); 133 EasyMock.expectLastCall().times(4); 134 mockRunListener.testEnded( 135 (TestDescription) EasyMock.anyObject(), 136 EasyMock.anyLong(), 137 (HashMap<String, Metric>) EasyMock.anyObject()); 138 EasyMock.expectLastCall().times(3); 139 mockRunListener.testRunFailed((String)EasyMock.anyObject()); 140 EasyMock.expectLastCall().times(1); 141 mockRunListener.testRunEnded( 142 EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 143 EasyMock.expectLastCall().times(1); 144 mockRunListener.testIgnored((TestDescription) EasyMock.anyObject()); 145 EasyMock.expectLastCall(); 146 mockRunListener.testAssumptionFailure( 147 (TestDescription) EasyMock.anyObject(), (String) EasyMock.anyObject()); 148 EasyMock.expectLastCall(); 149 EasyMock.replay(mockRunListener); 150 File tmp = FileUtil.createTempFile("sub", "unit"); 151 SubprocessTestResultsParser resultParser = null; 152 try { 153 resultParser = 154 new SubprocessTestResultsParser(mockRunListener, new InvocationContext()); 155 resultParser.processNewLines(contents); 156 EasyMock.verify(mockRunListener); 157 } finally { 158 StreamUtil.close(resultParser); 159 FileUtil.deleteFile(tmp); 160 } 161 } 162 163 /** Tests the parser for cases of test starting without closing. */ 164 @Test testParse_testNotStarted()165 public void testParse_testNotStarted() throws Exception { 166 ITestInvocationListener mockRunListener = 167 EasyMock.createMock(ITestInvocationListener.class); 168 mockRunListener.testRunStarted("arm64-v8a CtsGestureTestCases", 4); 169 mockRunListener.testEnded( 170 (TestDescription) EasyMock.anyObject(), 171 EasyMock.anyLong(), 172 (HashMap<String, Metric>) EasyMock.anyObject()); 173 EasyMock.expectLastCall().times(1); 174 EasyMock.replay(mockRunListener); 175 File tmp = FileUtil.createTempFile("sub", "unit"); 176 SubprocessTestResultsParser resultParser = null; 177 try { 178 resultParser = 179 new SubprocessTestResultsParser(mockRunListener, new InvocationContext()); 180 String startRun = 181 "TEST_RUN_STARTED {\"testCount\":4,\"runName\":\"arm64-v8a " 182 + "CtsGestureTestCases\"}\n"; 183 FileUtil.writeToFile(startRun, tmp, true); 184 String testEnded = 185 "03-22 14:04:02 E/SubprocessResultsReporter: TEST_ENDED " 186 + "{\"end_time\":1489160958359,\"className\":\"android.gesture.cts." 187 + "GestureLibraryTest\",\"testName\":\"testGetGestures\",\"extra\":\"" 188 + "data\"}\n"; 189 FileUtil.writeToFile(testEnded, tmp, true); 190 resultParser.parseFile(tmp); 191 EasyMock.verify(mockRunListener); 192 } finally { 193 StreamUtil.close(resultParser); 194 FileUtil.deleteFile(tmp); 195 } 196 } 197 198 /** Tests the parser for a cases when there is no start/end time stamp. */ 199 @Test testParse_noTimeStamp()200 public void testParse_noTimeStamp() throws Exception { 201 ITestInvocationListener mockRunListener = 202 EasyMock.createMock(ITestInvocationListener.class); 203 mockRunListener.testRunStarted("arm64-v8a CtsGestureTestCases", 4); 204 mockRunListener.testStarted(EasyMock.anyObject()); 205 mockRunListener.testEnded( 206 (TestDescription) EasyMock.anyObject(), 207 (HashMap<String, Metric>) EasyMock.anyObject()); 208 EasyMock.expectLastCall().times(1); 209 EasyMock.replay(mockRunListener); 210 File tmp = FileUtil.createTempFile("sub", "unit"); 211 SubprocessTestResultsParser resultParser = null; 212 try { 213 resultParser = 214 new SubprocessTestResultsParser(mockRunListener, new InvocationContext()); 215 String startRun = "TEST_RUN_STARTED {\"testCount\":4,\"runName\":\"arm64-v8a " 216 + "CtsGestureTestCases\"}\n"; 217 FileUtil.writeToFile(startRun, tmp, true); 218 String testStarted = 219 "03-22 14:04:02 E/SubprocessResultsReporter: TEST_STARTED " 220 + "{\"className\":\"android.gesture.cts." 221 + "GestureLibraryTest\",\"testName\":\"testGetGestures\"}\n"; 222 FileUtil.writeToFile(testStarted, tmp, true); 223 String testEnded = 224 "03-22 14:04:02 E/SubprocessResultsReporter: TEST_ENDED " 225 + "{\"className\":\"android.gesture.cts." 226 + "GestureLibraryTest\",\"testName\":\"testGetGestures\",\"extra\":\"" 227 + "data\"}\n"; 228 FileUtil.writeToFile(testEnded, tmp, true); 229 resultParser.parseFile(tmp); 230 EasyMock.verify(mockRunListener); 231 } finally { 232 StreamUtil.close(resultParser); 233 FileUtil.deleteFile(tmp); 234 } 235 } 236 237 /** Test injecting an invocation failure and verify the callback is called. */ 238 @Test testParse_invocationFailed()239 public void testParse_invocationFailed() throws Exception { 240 ITestInvocationListener mockRunListener = 241 EasyMock.createMock(ITestInvocationListener.class); 242 Capture<Throwable> cap = new Capture<Throwable>(); 243 mockRunListener.invocationFailed((EasyMock.capture(cap))); 244 EasyMock.replay(mockRunListener); 245 File tmp = FileUtil.createTempFile("sub", "unit"); 246 SubprocessTestResultsParser resultParser = null; 247 try { 248 resultParser = 249 new SubprocessTestResultsParser(mockRunListener, new InvocationContext()); 250 String cause = "com.android.tradefed.targetprep." 251 + "TargetSetupError: Not all target preparation steps completed\n\tat " 252 + "com.android.compatibility.common.tradefed.targetprep." 253 + "ApkInstrumentationPreparer.run(ApkInstrumentationPreparer.java:88)\n"; 254 String startRun = "03-23 11:50:12 E/SubprocessResultsReporter: " 255 + "INVOCATION_FAILED {\"cause\":\"com.android.tradefed.targetprep." 256 + "TargetSetupError: Not all target preparation steps completed\\n\\tat " 257 + "com.android.compatibility.common.tradefed.targetprep." 258 + "ApkInstrumentationPreparer.run(ApkInstrumentationPreparer.java:88)\\n\"}\n"; 259 FileUtil.writeToFile(startRun, tmp, true); 260 resultParser.parseFile(tmp); 261 EasyMock.verify(mockRunListener); 262 String expected = cap.getValue().getMessage(); 263 assertEquals(cause, expected); 264 } finally { 265 StreamUtil.close(resultParser); 266 FileUtil.deleteFile(tmp); 267 } 268 } 269 270 /** Report results when received from socket. */ 271 @Test testParser_receiveFromSocket()272 public void testParser_receiveFromSocket() throws Exception { 273 ITestInvocationListener mockRunListener = 274 EasyMock.createMock(ITestInvocationListener.class); 275 mockRunListener.testRunStarted("arm64-v8a CtsGestureTestCases", 4); 276 mockRunListener.testEnded( 277 (TestDescription) EasyMock.anyObject(), 278 EasyMock.anyLong(), 279 (HashMap<String, Metric>) EasyMock.anyObject()); 280 EasyMock.expectLastCall().times(1); 281 EasyMock.replay(mockRunListener); 282 SubprocessTestResultsParser resultParser = null; 283 Socket socket = null; 284 try { 285 resultParser = 286 new SubprocessTestResultsParser(mockRunListener, true, new InvocationContext()); 287 socket = new Socket("localhost", resultParser.getSocketServerPort()); 288 if (!socket.isConnected()) { 289 fail("socket did not connect"); 290 } 291 PrintWriter out = new PrintWriter(socket.getOutputStream(), true); 292 String startRun = "TEST_RUN_STARTED {\"testCount\":4,\"runName\":\"arm64-v8a " 293 + "CtsGestureTestCases\"}\n"; 294 out.print(startRun); 295 out.flush(); 296 String testEnded = 297 "03-22 14:04:02 E/SubprocessResultsReporter: TEST_ENDED " 298 + "{\"end_time\":1489160958359,\"className\":\"android.gesture.cts." 299 + "GestureLibraryTest\",\"testName\":\"testGetGestures\",\"extra\":\"" 300 + "data\"}\n"; 301 out.print(testEnded); 302 out.flush(); 303 StreamUtil.close(socket); 304 assertTrue(resultParser.joinReceiver(500)); 305 EasyMock.verify(mockRunListener); 306 } finally { 307 StreamUtil.close(resultParser); 308 StreamUtil.close(socket); 309 } 310 } 311 312 /** When the receiver thread fails to join then an exception is thrown. */ 313 @Test testParser_failToJoin()314 public void testParser_failToJoin() throws Exception { 315 ITestInvocationListener mockRunListener = 316 EasyMock.createMock(ITestInvocationListener.class); 317 EasyMock.replay(mockRunListener); 318 SubprocessTestResultsParser resultParser = null; 319 try { 320 resultParser = 321 new SubprocessTestResultsParser(mockRunListener, true, new InvocationContext()); 322 assertFalse(resultParser.joinReceiver(50)); 323 EasyMock.verify(mockRunListener); 324 } finally { 325 StreamUtil.close(resultParser); 326 } 327 } 328 329 /** Tests the parser receiving event on updating test tag. */ 330 @Test testParse_testTag()331 public void testParse_testTag() throws Exception { 332 final String subTestTag = "test_tag_in_subprocess"; 333 InvocationContext context = new InvocationContext(); 334 context.setTestTag("stub"); 335 336 ITestInvocationListener mockRunListener = 337 EasyMock.createMock(ITestInvocationListener.class); 338 EasyMock.replay(mockRunListener); 339 File tmp = FileUtil.createTempFile("sub", "unit"); 340 SubprocessTestResultsParser resultParser = null; 341 try { 342 resultParser = new SubprocessTestResultsParser(mockRunListener, false, context); 343 String testTagEvent = 344 String.format( 345 "INVOCATION_STARTED {\"testTag\": \"%s\",\"start_time\":250}", 346 subTestTag); 347 FileUtil.writeToFile(testTagEvent, tmp, true); 348 resultParser.parseFile(tmp); 349 EasyMock.verify(mockRunListener); 350 assertEquals(subTestTag, context.getTestTag()); 351 assertEquals(250l, resultParser.getStartTime().longValue()); 352 } finally { 353 StreamUtil.close(resultParser); 354 FileUtil.deleteFile(tmp); 355 } 356 } 357 358 /** Tests the parser smoothly handling case where there is no build info. */ 359 @Test testParse_testInvocationEndedWithoutBuildInfo()360 public void testParse_testInvocationEndedWithoutBuildInfo() throws Exception { 361 InvocationContext context = new InvocationContext(); 362 context.setTestTag("stub"); 363 364 ITestInvocationListener mockRunListener = 365 EasyMock.createMock(ITestInvocationListener.class); 366 EasyMock.replay(mockRunListener); 367 File tmp = FileUtil.createTempFile("sub", "unit"); 368 SubprocessTestResultsParser resultParser = null; 369 try { 370 resultParser = new SubprocessTestResultsParser(mockRunListener, false, context); 371 String event = "INVOCATION_ENDED {\"foo\": \"bar\"}"; 372 FileUtil.writeToFile(event, tmp, true); 373 resultParser.parseFile(tmp); 374 EasyMock.verify(mockRunListener); 375 } finally { 376 StreamUtil.close(resultParser); 377 FileUtil.deleteFile(tmp); 378 } 379 } 380 381 /** Tests the parser propagating up build attributes. */ 382 @Test testParse_testInvocationEnded()383 public void testParse_testInvocationEnded() throws Exception { 384 InvocationContext context = new InvocationContext(); 385 IBuildInfo info = new BuildInfo(); 386 context.setTestTag("stub"); 387 context.addDeviceBuildInfo("device1", info); 388 info.addBuildAttribute("baz", "qux"); 389 390 ITestInvocationListener mockRunListener = 391 EasyMock.createMock(ITestInvocationListener.class); 392 EasyMock.replay(mockRunListener); 393 File tmp = FileUtil.createTempFile("sub", "unit"); 394 SubprocessTestResultsParser resultParser = null; 395 try { 396 resultParser = new SubprocessTestResultsParser(mockRunListener, false, context); 397 String event = "INVOCATION_ENDED {\"foo\": \"bar\", \"baz\": \"wrong\"}"; 398 FileUtil.writeToFile(event, tmp, true); 399 resultParser.parseFile(tmp); 400 Map<String, String> attributes = info.getBuildAttributes(); 401 // foo=bar is propagated up 402 assertEquals("bar", attributes.get("foo")); 403 // baz=qux is not overwritten 404 assertEquals("qux", attributes.get("baz")); 405 EasyMock.verify(mockRunListener); 406 } finally { 407 StreamUtil.close(resultParser); 408 FileUtil.deleteFile(tmp); 409 } 410 } 411 412 /** Tests the parser should not overwrite the test tag in parent process if it's already set. */ 413 @Test testParse_testTagNotOverwrite()414 public void testParse_testTagNotOverwrite() throws Exception { 415 final String subTestTag = "test_tag_in_subprocess"; 416 final String parentTestTag = "test_tag_in_parent_process"; 417 InvocationContext context = new InvocationContext(); 418 context.setTestTag(parentTestTag); 419 420 ITestInvocationListener mockRunListener = 421 EasyMock.createMock(ITestInvocationListener.class); 422 EasyMock.replay(mockRunListener); 423 File tmp = FileUtil.createTempFile("sub", "unit"); 424 SubprocessTestResultsParser resultParser = null; 425 try { 426 resultParser = new SubprocessTestResultsParser(mockRunListener, false, context); 427 String testTagEvent = String.format("TEST_TAG %s", subTestTag); 428 FileUtil.writeToFile(testTagEvent, tmp, true); 429 resultParser.parseFile(tmp); 430 EasyMock.verify(mockRunListener); 431 assertEquals(parentTestTag, context.getTestTag()); 432 } finally { 433 StreamUtil.close(resultParser); 434 FileUtil.deleteFile(tmp); 435 } 436 } 437 438 /** Test that module start and end is properly parsed when reported. */ 439 @Test testParse_moduleStarted_end()440 public void testParse_moduleStarted_end() throws Exception { 441 ITestInvocationListener mockRunListener = 442 EasyMock.createMock(ITestInvocationListener.class); 443 mockRunListener.testModuleStarted(EasyMock.anyObject()); 444 mockRunListener.testModuleEnded(); 445 EasyMock.replay(mockRunListener); 446 IInvocationContext fakeModuleContext = new InvocationContext(); 447 File tmp = FileUtil.createTempFile("sub", "unit"); 448 SubprocessTestResultsParser resultParser = null; 449 File serializedModule = null; 450 try { 451 serializedModule = SerializationUtil.serialize(fakeModuleContext); 452 resultParser = 453 new SubprocessTestResultsParser(mockRunListener, new InvocationContext()); 454 String moduleStart = 455 String.format( 456 "TEST_MODULE_STARTED {\"moduleContextFileName\":\"%s\"}\n", 457 serializedModule.getAbsolutePath()); 458 FileUtil.writeToFile(moduleStart, tmp, true); 459 String moduleEnd = "TEST_MODULE_ENDED {}\n"; 460 FileUtil.writeToFile(moduleEnd, tmp, true); 461 462 resultParser.parseFile(tmp); 463 EasyMock.verify(mockRunListener); 464 } finally { 465 StreamUtil.close(resultParser); 466 FileUtil.deleteFile(tmp); 467 FileUtil.deleteFile(serializedModule); 468 } 469 } 470 471 /** Test that logAssociation event is properly passed and parsed. */ 472 @Test testParse_logAssociation()473 public void testParse_logAssociation() throws Exception { 474 ILogSaverListener mockRunListener = EasyMock.createMock(ILogSaverListener.class); 475 Capture<LogFile> capture = new Capture<>(); 476 mockRunListener.logAssociation(EasyMock.eq("dataname"), EasyMock.capture(capture)); 477 EasyMock.replay(mockRunListener); 478 LogFile logFile = new LogFile("path", "url", LogDataType.TEXT); 479 File serializedLogFile = null; 480 File tmp = FileUtil.createTempFile("sub", "unit"); 481 SubprocessTestResultsParser resultParser = null; 482 try { 483 serializedLogFile = SerializationUtil.serialize(logFile); 484 resultParser = 485 new SubprocessTestResultsParser(mockRunListener, new InvocationContext()); 486 String logAssocation = 487 String.format( 488 "LOG_ASSOCIATION {\"loggedFile\":\"%s\",\"dataName\":\"dataname\"}\n", 489 serializedLogFile.getAbsolutePath()); 490 FileUtil.writeToFile(logAssocation, tmp, true); 491 resultParser.parseFile(tmp); 492 EasyMock.verify(mockRunListener); 493 } finally { 494 StreamUtil.close(resultParser); 495 FileUtil.deleteFile(serializedLogFile); 496 FileUtil.deleteFile(tmp); 497 } 498 LogFile received = capture.getValue(); 499 assertEquals(logFile.getPath(), received.getPath()); 500 assertEquals(logFile.getUrl(), received.getUrl()); 501 assertEquals(logFile.getType(), received.getType()); 502 } 503 } 504