1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.result.suite; 17 18 import static org.junit.Assert.assertEquals; 19 import static org.junit.Assert.assertFalse; 20 import static org.junit.Assert.assertNotNull; 21 import static org.junit.Assert.assertTrue; 22 23 import com.android.tradefed.build.BuildInfo; 24 import com.android.tradefed.invoker.IInvocationContext; 25 import com.android.tradefed.invoker.InvocationContext; 26 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 27 import com.android.tradefed.result.FailureDescription; 28 import com.android.tradefed.result.LogDataType; 29 import com.android.tradefed.result.LogFile; 30 import com.android.tradefed.result.TestDescription; 31 import com.android.tradefed.result.TestResult; 32 import com.android.tradefed.result.TestRunResult; 33 import com.android.tradefed.result.TestStatus; 34 import com.android.tradefed.result.error.TestErrorIdentifier; 35 import com.android.tradefed.testtype.Abi; 36 import com.android.tradefed.testtype.IAbi; 37 import com.android.tradefed.util.FileUtil; 38 import com.android.tradefed.util.proto.TfMetricProtoUtil; 39 40 import org.junit.After; 41 import org.junit.Before; 42 import org.junit.Test; 43 import org.junit.runner.RunWith; 44 import org.junit.runners.JUnit4; 45 import org.w3c.dom.Element; 46 import org.w3c.dom.NodeList; 47 import org.xml.sax.InputSource; 48 49 import java.io.File; 50 import java.io.StringReader; 51 import java.util.ArrayList; 52 import java.util.Arrays; 53 import java.util.Collection; 54 import java.util.HashMap; 55 import java.util.List; 56 import java.util.Map; 57 import java.util.Map.Entry; 58 59 import javax.xml.xpath.XPath; 60 import javax.xml.xpath.XPathConstants; 61 import javax.xml.xpath.XPathExpressionException; 62 import javax.xml.xpath.XPathFactory; 63 64 /** Unit tests for {@link XmlSuiteResultFormatter}. */ 65 @RunWith(JUnit4.class) 66 public class XmlSuiteResultFormatterTest { 67 private XmlSuiteResultFormatter mFormatter; 68 private SuiteResultHolder mResultHolder; 69 private IInvocationContext mContext; 70 private File mResultDir; 71 72 @Before setUp()73 public void setUp() throws Exception { 74 mFormatter = new XmlSuiteResultFormatter(); 75 mResultHolder = new SuiteResultHolder(); 76 mContext = new InvocationContext(); 77 mResultDir = FileUtil.createTempDir("result-dir"); 78 } 79 80 @After tearDown()81 public void tearDown() throws Exception { 82 FileUtil.recursiveDelete(mResultDir); 83 } 84 85 /** Check that the basic overall structure is good an contains all the information. */ 86 @Test testBasicFormat()87 public void testBasicFormat() throws Exception { 88 mResultHolder.context = mContext; 89 // Ensure attributes are escaped if needed 90 mContext.addInvocationAttribute("test-type-count:JarHostTest", "5"); 91 92 Collection<TestRunResult> runResults = new ArrayList<>(); 93 runResults.add(createFakeResult("module1", 2, 0, 0, 0)); 94 runResults.add(createFakeResult("module2", 1, 0, 0, 0)); 95 mResultHolder.runResults = runResults; 96 97 Map<String, IAbi> modulesAbi = new HashMap<>(); 98 modulesAbi.put("module1", new Abi("armeabi-v7a", "32")); 99 modulesAbi.put("module2", new Abi("armeabi-v7a", "32")); 100 mResultHolder.modulesAbi = modulesAbi; 101 102 mResultHolder.completeModules = 2; 103 mResultHolder.totalModules = 2; 104 mResultHolder.passedTests = 2; 105 mResultHolder.failedTests = 0; 106 mResultHolder.startTime = 0L; 107 mResultHolder.endTime = 10L; 108 File res = mFormatter.writeResults(mResultHolder, mResultDir); 109 String content = FileUtil.readStringFromFile(res); 110 assertXmlContainsNode(content, "Result"); 111 // Verify that RunHistory tag should not exist if run history is empty. 112 assertXmlNotContainNode(content, "Result/RunHistory"); 113 // Verify that the summary has been populated 114 assertXmlContainsNode(content, "Result/Summary"); 115 assertXmlContainsAttribute(content, "Result/Summary", "pass", "2"); 116 assertXmlContainsAttribute(content, "Result/Summary", "failed", "0"); 117 assertXmlContainsAttribute(content, "Result/Summary", "modules_done", "2"); 118 assertXmlContainsAttribute(content, "Result/Summary", "modules_total", "2"); 119 // Verify that each module results are available 120 assertXmlContainsNode(content, "Result/Module"); 121 assertXmlContainsAttribute(content, "Result/Module", "name", "module1"); 122 assertXmlContainsAttribute(content, "Result/Module", "abi", "armeabi-v7a"); 123 assertXmlContainsAttribute(content, "Result/Module", "runtime", "10"); 124 assertXmlContainsAttribute(content, "Result/Module", "done", "true"); 125 assertXmlContainsAttribute(content, "Result/Module", "pass", "2"); 126 // Verify the test cases that passed are present 127 assertXmlContainsNode(content, "Result/Module/TestCase"); 128 assertXmlContainsAttribute(content, "Result/Module/TestCase", "name", "com.class.module1"); 129 assertXmlContainsAttribute( 130 content, "Result/Module/TestCase/Test", "name", "module1.method0"); 131 assertXmlContainsAttribute( 132 content, "Result/Module/TestCase/Test", "name", "module1.method1"); 133 134 assertXmlContainsAttribute(content, "Result/Module", "name", "module2"); 135 assertXmlContainsAttribute(content, "Result/Module", "pass", "1"); 136 assertXmlContainsAttribute(content, "Result/Module/TestCase", "name", "com.class.module2"); 137 assertXmlContainsAttribute( 138 content, "Result/Module/TestCase/Test", "name", "module2.method0"); 139 } 140 141 /** Check that the test failures are properly reported. */ 142 @Test testFailuresReporting()143 public void testFailuresReporting() throws Exception { 144 mResultHolder.context = mContext; 145 146 Collection<TestRunResult> runResults = new ArrayList<>(); 147 runResults.add(createFakeResult("module1", 2, 1, 0, 0)); 148 mResultHolder.runResults = runResults; 149 150 Map<String, IAbi> modulesAbi = new HashMap<>(); 151 modulesAbi.put("module1", new Abi("armeabi-v7a", "32")); 152 mResultHolder.modulesAbi = modulesAbi; 153 154 mResultHolder.completeModules = 2; 155 mResultHolder.totalModules = 1; 156 mResultHolder.passedTests = 2; 157 mResultHolder.failedTests = 1; 158 mResultHolder.startTime = 0L; 159 mResultHolder.endTime = 10L; 160 File res = mFormatter.writeResults(mResultHolder, mResultDir); 161 String content = FileUtil.readStringFromFile(res); 162 163 assertXmlContainsNode(content, "Result/Module"); 164 assertXmlContainsAttribute(content, "Result/Module/TestCase", "name", "com.class.module1"); 165 assertXmlContainsAttribute( 166 content, "Result/Module/TestCase/Test", "name", "module1.method0"); 167 assertXmlContainsAttribute( 168 content, "Result/Module/TestCase/Test", "name", "module1.method1"); 169 // Check that failures are showing in the xml for the test cases with error identifiers 170 assertXmlContainsAttribute( 171 content, "Result/Module/TestCase/Test", "name", "module1.failed0"); 172 assertXmlContainsAttribute(content, "Result/Module/TestCase/Test", "result", "fail"); 173 assertXmlContainsAttribute( 174 content, "Result/Module/TestCase/Test/Failure", "message", "module1 failed."); 175 assertXmlContainsAttribute( 176 content, 177 "Result/Module/TestCase/Test/Failure", 178 "error_name", 179 TestErrorIdentifier.TEST_ABORTED.name()); 180 assertXmlContainsAttribute( 181 content, 182 "Result/Module/TestCase/Test/Failure", 183 "error_code", 184 Long.toString(TestErrorIdentifier.TEST_ABORTED.code())); 185 assertXmlContainsValue( 186 content, 187 "Result/Module/TestCase/Test/Failure/StackTrace", 188 mFormatter.sanitizeXmlContent("module1 failed.\nstack\nstack\0")); 189 // Test that we can read back the informations 190 SuiteResultHolder holder = mFormatter.parseResults(mResultDir, false); 191 assertEquals(holder.completeModules, mResultHolder.completeModules); 192 assertEquals(holder.totalModules, mResultHolder.totalModules); 193 assertEquals(holder.passedTests, mResultHolder.passedTests); 194 assertEquals(holder.failedTests, mResultHolder.failedTests); 195 assertEquals(holder.startTime, mResultHolder.startTime); 196 assertEquals(holder.endTime, mResultHolder.endTime); 197 assertEquals( 198 holder.modulesAbi.get("armeabi-v7a module1"), 199 mResultHolder.modulesAbi.get("module1")); 200 assertEquals(holder.runResults.size(), mResultHolder.runResults.size()); 201 } 202 203 @Test testFailuresReporting_notDone()204 public void testFailuresReporting_notDone() throws Exception { 205 mResultHolder.context = mContext; 206 207 List<TestRunResult> runResults = new ArrayList<>(); 208 runResults.add(createFakeResult("module1", 2, 1, 0, 0)); 209 FailureDescription failureDescription = 210 FailureDescription.create("Failed module") 211 .setErrorIdentifier(TestErrorIdentifier.TEST_ABORTED); 212 runResults.get(0).testRunFailed(failureDescription); 213 mResultHolder.runResults = runResults; 214 215 Map<String, IAbi> modulesAbi = new HashMap<>(); 216 modulesAbi.put("module1", new Abi("armeabi-v7a", "32")); 217 mResultHolder.modulesAbi = modulesAbi; 218 219 mResultHolder.completeModules = 2; 220 mResultHolder.totalModules = 1; 221 mResultHolder.passedTests = 2; 222 mResultHolder.failedTests = 1; 223 mResultHolder.startTime = 0L; 224 mResultHolder.endTime = 10L; 225 File res = mFormatter.writeResults(mResultHolder, mResultDir); 226 String content = FileUtil.readStringFromFile(res); 227 228 assertXmlContainsNode(content, "Result/Module"); 229 assertXmlContainsAttribute(content, "Result/Module", "done", "false"); 230 assertXmlContainsNode(content, "Result/Module/Reason"); 231 // Check that Reason contains error identifiers 232 assertXmlContainsAttribute( 233 content, 234 "Result/Module/Reason", 235 "error_name", 236 TestErrorIdentifier.TEST_ABORTED.name()); 237 assertXmlContainsAttribute( 238 content, 239 "Result/Module/Reason", 240 "error_code", 241 Long.toString(TestErrorIdentifier.TEST_ABORTED.code())); 242 assertXmlContainsAttribute(content, "Result/Module/TestCase", "name", "com.class.module1"); 243 assertXmlContainsAttribute( 244 content, "Result/Module/TestCase/Test", "name", "module1.method0"); 245 assertXmlContainsAttribute( 246 content, "Result/Module/TestCase/Test", "name", "module1.method1"); 247 // Check that failures are showing in the xml for the test cases 248 assertXmlContainsAttribute( 249 content, "Result/Module/TestCase/Test", "name", "module1.failed0"); 250 assertXmlContainsAttribute(content, "Result/Module/TestCase/Test", "result", "fail"); 251 assertXmlContainsAttribute( 252 content, "Result/Module/TestCase/Test/Failure", "message", "module1 failed."); 253 assertXmlContainsValue( 254 content, 255 "Result/Module/TestCase/Test/Failure/StackTrace", 256 mFormatter.sanitizeXmlContent("module1 failed.\nstack\nstack\0")); 257 // Test that we can read back the informations 258 SuiteResultHolder holder = mFormatter.parseResults(mResultDir, false); 259 assertEquals(holder.completeModules, mResultHolder.completeModules); 260 assertEquals(holder.totalModules, mResultHolder.totalModules); 261 assertEquals(holder.passedTests, mResultHolder.passedTests); 262 assertEquals(holder.failedTests, mResultHolder.failedTests); 263 assertEquals(holder.startTime, mResultHolder.startTime); 264 assertEquals(holder.endTime, mResultHolder.endTime); 265 assertEquals( 266 holder.modulesAbi.get("armeabi-v7a module1"), 267 mResultHolder.modulesAbi.get("module1")); 268 assertEquals(1, holder.runResults.size()); 269 TestRunResult result = new ArrayList<>(holder.runResults).get(0); 270 assertFalse(result.isRunComplete()); 271 } 272 273 @Test testFailuresReporting_largeStackTrace()274 public void testFailuresReporting_largeStackTrace() throws Exception { 275 mResultHolder.context = mContext; 276 277 List<TestRunResult> runResults = new ArrayList<>(); 278 runResults.add(createFakeResult("module1", 2, 1, 0, 0, 1024 * 1024, false, false)); 279 mResultHolder.runResults = runResults; 280 281 Map<String, IAbi> modulesAbi = new HashMap<>(); 282 modulesAbi.put("module1", new Abi("armeabi-v7a", "32")); 283 mResultHolder.modulesAbi = modulesAbi; 284 285 mResultHolder.completeModules = 2; 286 mResultHolder.totalModules = 1; 287 mResultHolder.passedTests = 2; 288 mResultHolder.failedTests = 1; 289 mResultHolder.startTime = 0L; 290 mResultHolder.endTime = 10L; 291 File res = mFormatter.writeResults(mResultHolder, mResultDir); 292 String content = FileUtil.readStringFromFile(res); 293 294 assertXmlContainsNode(content, "Result/Module"); 295 assertXmlContainsAttribute(content, "Result/Module/TestCase", "name", "com.class.module1"); 296 assertXmlContainsAttribute( 297 content, "Result/Module/TestCase/Test", "name", "module1.method0"); 298 assertXmlContainsAttribute( 299 content, "Result/Module/TestCase/Test", "name", "module1.method1"); 300 // Check that failures are showing in the xml for the test cases with error identifiers 301 assertXmlContainsAttribute( 302 content, "Result/Module/TestCase/Test", "name", "module1.failed0"); 303 assertXmlContainsAttribute(content, "Result/Module/TestCase/Test", "result", "fail"); 304 assertXmlContainsAttribute( 305 content, "Result/Module/TestCase/Test/Failure", "message", "module1 failed."); 306 assertXmlContainsAttribute( 307 content, 308 "Result/Module/TestCase/Test/Failure", 309 "error_name", 310 TestErrorIdentifier.TEST_ABORTED.name()); 311 assertXmlContainsAttribute( 312 content, 313 "Result/Module/TestCase/Test/Failure", 314 "error_code", 315 Long.toString(TestErrorIdentifier.TEST_ABORTED.code())); 316 assertXmlContainsValue( 317 content, 318 "Result/Module/TestCase/Test/Failure/StackTrace", 319 mFormatter.sanitizeXmlContent("module1 failed." + "\nstack".repeat(174760) + "\n")); 320 // Test that we can read back the informations 321 SuiteResultHolder holder = mFormatter.parseResults(mResultDir, false); 322 assertEquals(holder.completeModules, mResultHolder.completeModules); 323 assertEquals(holder.totalModules, mResultHolder.totalModules); 324 assertEquals(holder.passedTests, mResultHolder.passedTests); 325 assertEquals(holder.failedTests, mResultHolder.failedTests); 326 assertEquals(holder.startTime, mResultHolder.startTime); 327 assertEquals(holder.endTime, mResultHolder.endTime); 328 assertEquals( 329 holder.modulesAbi.get("armeabi-v7a module1"), 330 mResultHolder.modulesAbi.get("module1")); 331 assertEquals(holder.runResults.size(), mResultHolder.runResults.size()); 332 } 333 334 /** Test that assumption failures and ignored tests are correctly reported in the xml. */ 335 @Test testAssumptionFailures_Ignore_Reporting()336 public void testAssumptionFailures_Ignore_Reporting() throws Exception { 337 mResultHolder.context = mContext; 338 339 Collection<TestRunResult> runResults = new ArrayList<>(); 340 runResults.add(createFakeResult("module1", 2, 0, 1, 1)); 341 mResultHolder.runResults = runResults; 342 343 Map<String, IAbi> modulesAbi = new HashMap<>(); 344 modulesAbi.put("module1", new Abi("armeabi-v7a", "32")); 345 mResultHolder.modulesAbi = modulesAbi; 346 347 mResultHolder.completeModules = 1; 348 mResultHolder.totalModules = 1; 349 mResultHolder.passedTests = 2; 350 mResultHolder.failedTests = 0L; 351 mResultHolder.startTime = 0L; 352 mResultHolder.endTime = 10L; 353 File res = mFormatter.writeResults(mResultHolder, mResultDir); 354 String content = FileUtil.readStringFromFile(res); 355 356 assertXmlContainsNode(content, "Result/Module"); 357 assertXmlContainsAttribute(content, "Result/Module/TestCase", "name", "com.class.module1"); 358 assertXmlContainsAttribute( 359 content, "Result/Module/TestCase/Test", "name", "module1.method0"); 360 assertXmlContainsAttribute( 361 content, "Result/Module/TestCase/Test", "name", "module1.method1"); 362 // Check that failures are showing in the xml for the test cases 363 assertXmlContainsAttribute( 364 content, "Result/Module/TestCase/Test", "name", "module1.assumpFail0"); 365 assertXmlContainsAttribute( 366 content, "Result/Module/TestCase/Test", "result", "ASSUMPTION_FAILURE"); 367 assertXmlContainsAttribute( 368 content, "Result/Module/TestCase/Test/Failure", "message", "module1 failed."); 369 assertXmlContainsValue( 370 content, 371 "Result/Module/TestCase/Test/Failure/StackTrace", 372 "module1 failed.\nstack\nstack"); 373 // Test that we can read back the informations 374 SuiteResultHolder holder = mFormatter.parseResults(mResultDir, false); 375 assertEquals(holder.completeModules, mResultHolder.completeModules); 376 assertEquals(holder.totalModules, mResultHolder.totalModules); 377 assertEquals(holder.passedTests, mResultHolder.passedTests); 378 assertEquals(holder.failedTests, mResultHolder.failedTests); 379 assertEquals(holder.startTime, mResultHolder.startTime); 380 assertEquals(holder.endTime, mResultHolder.endTime); 381 assertEquals( 382 holder.modulesAbi.get("armeabi-v7a module1"), 383 mResultHolder.modulesAbi.get("module1")); 384 assertEquals(holder.runResults.size(), mResultHolder.runResults.size()); 385 386 // Test that the results are loadable with expected values 387 SuiteResultHolder reloaded = mFormatter.parseResults(mResultDir, false); 388 assertEquals(1, reloaded.runResults.size()); 389 TestRunResult result = reloaded.runResults.iterator().next(); 390 assertEquals(0, result.getNumTestsInState(TestStatus.FAILURE)); 391 assertEquals(1, result.getNumTestsInState(TestStatus.ASSUMPTION_FAILURE)); 392 assertEquals(1, result.getNumTestsInState(TestStatus.IGNORED)); 393 } 394 395 /** Check that the logs for each test case are reported. */ 396 @Test testLogReporting()397 public void testLogReporting() throws Exception { 398 mResultHolder.context = mContext; 399 400 Collection<TestRunResult> runResults = new ArrayList<>(); 401 runResults.add(createResultWithLog("armeabi-v7a module1", 1, LogDataType.LOGCAT)); 402 runResults.add(createResultWithLog("module2", 1, LogDataType.BUGREPORT)); 403 runResults.add(createResultWithLog("module3", 1, LogDataType.PNG)); 404 mResultHolder.runResults = runResults; 405 406 Map<String, IAbi> modulesAbi = new HashMap<>(); 407 modulesAbi.put("armeabi-v7a module1", new Abi("armeabi-v7a", "32")); 408 mResultHolder.modulesAbi = modulesAbi; 409 410 mResultHolder.completeModules = 2; 411 mResultHolder.totalModules = 2; 412 mResultHolder.passedTests = 2; 413 mResultHolder.failedTests = 0; 414 mResultHolder.startTime = 0L; 415 mResultHolder.endTime = 10L; 416 File res = mFormatter.writeResults(mResultHolder, mResultDir); 417 String content = FileUtil.readStringFromFile(res); 418 // One logcat and one bugreport are found in the report 419 assertXmlContainsValue( 420 content, "Result/Module/TestCase/Test/Logcat", "http:url/armeabi-v7a module1"); 421 assertXmlContainsValue( 422 content, "Result/Module/TestCase/Test/BugReport", "http:url/module2"); 423 assertXmlContainsValue( 424 content, "Result/Module/TestCase/Test/Screenshot", "http:url/module3"); 425 426 // Test that we can read back the informations for log files 427 SuiteResultHolder holder = mFormatter.parseResults(mResultDir, false); 428 assertEquals( 429 holder.modulesAbi.get("armeabi-v7a module1"), 430 mResultHolder.modulesAbi.get("armeabi-v7a module1")); 431 assertEquals(holder.runResults.size(), mResultHolder.runResults.size()); 432 for (TestRunResult result : holder.runResults) { 433 TestDescription description = 434 new TestDescription( 435 "com.class." + result.getName(), result.getName() + ".method0"); 436 // Check that we reloaded the logged files. 437 assertTrue( 438 result.getTestResults() 439 .get(description) 440 .getLoggedFiles() 441 .get(result.getName() + "log0") 442 != null); 443 } 444 } 445 446 /** Check that the metrics with test cases are properly reported. */ 447 @Test testMetricReporting()448 public void testMetricReporting() throws Exception { 449 mResultHolder.context = mContext; 450 451 Collection<TestRunResult> runResults = new ArrayList<>(); 452 runResults.add(createFakeResult("module1", 2, 1, 0, 0, true, false)); 453 mResultHolder.runResults = runResults; 454 455 Map<String, IAbi> modulesAbi = new HashMap<>(); 456 modulesAbi.put("module1", new Abi("armeabi-v7a", "32")); 457 mResultHolder.modulesAbi = modulesAbi; 458 459 mResultHolder.completeModules = 1; 460 mResultHolder.totalModules = 1; 461 mResultHolder.passedTests = 2; 462 mResultHolder.failedTests = 1; 463 mResultHolder.startTime = 0L; 464 mResultHolder.endTime = 10L; 465 File res = mFormatter.writeResults(mResultHolder, mResultDir); 466 String content = FileUtil.readStringFromFile(res); 467 468 assertXmlContainsNode(content, "Result/Module"); 469 assertXmlContainsAttribute(content, "Result/Module/TestCase", "name", "com.class.module1"); 470 assertXmlContainsAttribute( 471 content, "Result/Module/TestCase/Test", "name", "module1.method0"); 472 assertXmlContainsAttribute( 473 content, "Result/Module/TestCase/Test", "name", "module1.method1"); 474 // Check that failures are showing in the xml for the test cases 475 assertXmlContainsAttribute( 476 content, "Result/Module/TestCase/Test", "name", "module1.failed0"); 477 assertXmlContainsAttribute(content, "Result/Module/TestCase/Test", "result", "fail"); 478 assertXmlContainsAttribute( 479 content, "Result/Module/TestCase/Test/Failure", "message", "module1 failed."); 480 // Test that we can read back the informations 481 SuiteResultHolder holder = mFormatter.parseResults(mResultDir, false); 482 assertEquals(holder.completeModules, mResultHolder.completeModules); 483 assertEquals(holder.totalModules, mResultHolder.totalModules); 484 assertEquals(holder.passedTests, mResultHolder.passedTests); 485 assertEquals(holder.failedTests, mResultHolder.failedTests); 486 assertEquals(holder.startTime, mResultHolder.startTime); 487 assertEquals(holder.endTime, mResultHolder.endTime); 488 assertEquals( 489 holder.modulesAbi.get("armeabi-v7a module1"), 490 mResultHolder.modulesAbi.get("module1")); 491 assertEquals(1, holder.runResults.size()); 492 TestRunResult result = holder.runResults.iterator().next(); 493 // 2 passed tests and 1 failed with metrics 494 assertEquals(3, result.getTestResults().size()); 495 assertEquals(1, result.getNumTestsInState(TestStatus.FAILURE)); 496 for (Entry<TestDescription, TestResult> entry : result.getTestResults().entrySet()) { 497 if (TestStatus.FAILURE.equals(entry.getValue().getResultStatus())) { 498 assertEquals("value00", entry.getValue().getMetrics().get("metric00")); 499 assertEquals("value10", entry.getValue().getMetrics().get("metric10")); 500 } 501 } 502 } 503 504 /** Test that the device format is properly done. */ 505 @Test testDeviceSerials()506 public void testDeviceSerials() throws Exception { 507 mResultHolder.context = mContext; 508 mResultHolder.context.addSerialsFromShard(0, Arrays.asList("serial1", "serial2")); 509 mResultHolder.context.addSerialsFromShard(1, Arrays.asList("serial3", "serial4")); 510 511 Collection<TestRunResult> runResults = new ArrayList<>(); 512 runResults.add(createFakeResult("module1", 2, 0, 0, 0)); 513 runResults.add(createFakeResult("module2", 1, 0, 0, 0)); 514 mResultHolder.runResults = runResults; 515 516 Map<String, IAbi> modulesAbi = new HashMap<>(); 517 modulesAbi.put("module1", new Abi("armeabi-v7a", "32")); 518 modulesAbi.put("module2", new Abi("armeabi-v7a", "32")); 519 mResultHolder.modulesAbi = modulesAbi; 520 521 mResultHolder.completeModules = 2; 522 mResultHolder.totalModules = 2; 523 mResultHolder.passedTests = 2; 524 mResultHolder.failedTests = 0; 525 mResultHolder.startTime = 0L; 526 mResultHolder.endTime = 10L; 527 File res = mFormatter.writeResults(mResultHolder, mResultDir); 528 String content = FileUtil.readStringFromFile(res); 529 assertXmlContainsNode(content, "Result"); 530 assertXmlContainsAttribute(content, "Result", "devices", "serial1,serial2,serial3,serial4"); 531 } 532 533 /** Test writing then loading a shallow representation of the results. */ 534 @Test testBasicFormat_shallow()535 public void testBasicFormat_shallow() throws Exception { 536 mResultHolder.context = mContext; 537 538 Collection<TestRunResult> runResults = new ArrayList<>(); 539 runResults.add(createFakeResult("module1", 2, 1, 0, 0, true, false)); 540 mResultHolder.runResults = runResults; 541 542 Map<String, IAbi> modulesAbi = new HashMap<>(); 543 modulesAbi.put("module1", new Abi("armeabi-v7a", "32")); 544 mResultHolder.modulesAbi = modulesAbi; 545 546 mResultHolder.completeModules = 1; 547 mResultHolder.totalModules = 1; 548 mResultHolder.passedTests = 2; 549 mResultHolder.failedTests = 1; 550 mResultHolder.startTime = 0L; 551 mResultHolder.endTime = 10L; 552 File res = mFormatter.writeResults(mResultHolder, mResultDir); 553 String content = FileUtil.readStringFromFile(res); 554 555 assertXmlContainsNode(content, "Result/Module"); 556 assertXmlContainsAttribute(content, "Result/Module/TestCase", "name", "com.class.module1"); 557 assertXmlContainsAttribute( 558 content, "Result/Module/TestCase/Test", "name", "module1.method0"); 559 assertXmlContainsAttribute( 560 content, "Result/Module/TestCase/Test", "name", "module1.method1"); 561 // Check that failures are showing in the xml for the test cases 562 assertXmlContainsAttribute( 563 content, "Result/Module/TestCase/Test", "name", "module1.failed0"); 564 assertXmlContainsAttribute(content, "Result/Module/TestCase/Test", "result", "fail"); 565 assertXmlContainsAttribute( 566 content, "Result/Module/TestCase/Test/Failure", "message", "module1 failed."); 567 assertXmlContainsValue( 568 content, 569 "Result/Module/TestCase/Test/Failure/StackTrace", 570 mFormatter.sanitizeXmlContent("module1 failed.\nstack\nstack\0")); 571 // Test that we can read back the informations 572 SuiteResultHolder holder = mFormatter.parseResults(mResultDir, true); 573 assertEquals(holder.completeModules, mResultHolder.completeModules); 574 assertEquals(holder.totalModules, mResultHolder.totalModules); 575 assertEquals(holder.passedTests, mResultHolder.passedTests); 576 assertEquals(holder.failedTests, mResultHolder.failedTests); 577 assertEquals(holder.startTime, mResultHolder.startTime); 578 assertEquals(holder.endTime, mResultHolder.endTime); 579 580 // Shallow loading doesn't load complex run informations. 581 assertTrue(holder.runResults == null); 582 assertTrue(holder.modulesAbi == null); 583 } 584 585 @Test testMetricReporting_badKey()586 public void testMetricReporting_badKey() throws Exception { 587 mResultHolder.context = mContext; 588 589 Collection<TestRunResult> runResults = new ArrayList<>(); 590 runResults.add(createFakeResult("module1", 2, 1, 0, 0, true, true)); 591 mResultHolder.runResults = runResults; 592 593 Map<String, IAbi> modulesAbi = new HashMap<>(); 594 modulesAbi.put("module1", new Abi("armeabi-v7a", "32")); 595 mResultHolder.modulesAbi = modulesAbi; 596 597 mResultHolder.completeModules = 1; 598 mResultHolder.totalModules = 1; 599 mResultHolder.passedTests = 2; 600 mResultHolder.failedTests = 1; 601 mResultHolder.startTime = 0L; 602 mResultHolder.endTime = 10L; 603 File res = mFormatter.writeResults(mResultHolder, mResultDir); 604 String content = FileUtil.readStringFromFile(res); 605 606 assertXmlContainsNode(content, "Result/Module"); 607 assertXmlContainsAttribute(content, "Result/Module/TestCase", "name", "com.class.module1"); 608 assertXmlContainsAttribute( 609 content, "Result/Module/TestCase/Test", "name", "module1.method0"); 610 assertXmlContainsAttribute( 611 content, "Result/Module/TestCase/Test", "name", "module1.method1"); 612 // Check that failures are showing in the xml for the test cases 613 assertXmlContainsAttribute( 614 content, "Result/Module/TestCase/Test", "name", "module1.failed0"); 615 assertXmlContainsAttribute(content, "Result/Module/TestCase/Test", "result", "fail"); 616 assertXmlContainsAttribute( 617 content, "Result/Module/TestCase/Test/Failure", "message", "module1 failed."); 618 // Test that we can read back the informations 619 SuiteResultHolder holder = mFormatter.parseResults(mResultDir, false); 620 assertEquals(holder.completeModules, mResultHolder.completeModules); 621 assertEquals(holder.totalModules, mResultHolder.totalModules); 622 assertEquals(holder.passedTests, mResultHolder.passedTests); 623 assertEquals(holder.failedTests, mResultHolder.failedTests); 624 assertEquals(holder.startTime, mResultHolder.startTime); 625 assertEquals(holder.endTime, mResultHolder.endTime); 626 assertEquals( 627 holder.modulesAbi.get("armeabi-v7a module1"), 628 mResultHolder.modulesAbi.get("module1")); 629 assertEquals(1, holder.runResults.size()); 630 TestRunResult result = holder.runResults.iterator().next(); 631 // 2 passed tests and 1 failed with metrics 632 assertEquals(3, result.getTestResults().size()); 633 assertEquals(1, result.getNumTestsInState(TestStatus.FAILURE)); 634 for (Entry<TestDescription, TestResult> entry : result.getTestResults().entrySet()) { 635 if (TestStatus.FAILURE.equals(entry.getValue().getResultStatus())) { 636 assertEquals("value00", entry.getValue().getMetrics().get("metric00")); 637 assertEquals("value10", entry.getValue().getMetrics().get("metric10")); 638 } 639 } 640 } 641 642 /** Check that run history is properly reported. */ 643 @Test testRunHistoryReporting()644 public void testRunHistoryReporting() throws Exception { 645 final String RUN_HISTORY = 646 "[{\"startTime\":10000000000000,\"endTime\":10000000100000,\"passedTests\":10," 647 + "\"failedTests\":5,\"commandLineArgs\":\"cts\"," 648 + "\"hostName\":\"user.android.com\"}," 649 + "{\"startTime\":10000000200000,\"endTime\":10000000300000," 650 + "\"passedTests\":3,\"failedTests\":2," 651 + "\"commandLineArgs\":\"cts\"," 652 + "\"hostName\":\"user.android.com\"}]"; 653 mResultHolder.context = mContext; 654 mResultHolder.context.addInvocationAttribute("run_history", RUN_HISTORY); 655 656 Collection<TestRunResult> runResults = new ArrayList<>(); 657 runResults.add(createFakeResult("module1", 2, 1, 0, 0)); 658 mResultHolder.runResults = runResults; 659 660 Map<String, IAbi> modulesAbi = new HashMap<>(); 661 modulesAbi.put("module1", new Abi("armeabi-v7a", "32")); 662 mResultHolder.modulesAbi = modulesAbi; 663 664 mResultHolder.completeModules = 1; 665 mResultHolder.totalModules = 1; 666 mResultHolder.passedTests = 1; 667 mResultHolder.failedTests = 0; 668 mResultHolder.startTime = 0L; 669 mResultHolder.endTime = 10L; 670 File res = mFormatter.writeResults(mResultHolder, mResultDir); 671 String content = FileUtil.readStringFromFile(res); 672 673 assertXmlContainsAttribute(content, "Result/Build", "run_history", RUN_HISTORY); 674 assertXmlContainsNode(content, "Result/RunHistory"); 675 assertXmlContainsAttribute(content, "Result/RunHistory/Run", "start", "10000000000000"); 676 assertXmlContainsAttribute(content, "Result/RunHistory/Run", "end", "10000000100000"); 677 assertXmlContainsAttribute(content, "Result/RunHistory/Run", "pass", "10"); 678 assertXmlContainsAttribute(content, "Result/RunHistory/Run", "failed", "5"); 679 assertXmlContainsAttribute(content, "Result/RunHistory/Run", "command_line_args", "cts"); 680 assertXmlContainsAttribute( 681 content, "Result/RunHistory/Run", "host_name", "user.android.com"); 682 assertXmlContainsAttribute(content, "Result/RunHistory/Run", "start", "10000000200000"); 683 assertXmlContainsAttribute(content, "Result/RunHistory/Run", "end", "10000000300000"); 684 assertXmlContainsAttribute(content, "Result/RunHistory/Run", "pass", "3"); 685 assertXmlContainsAttribute(content, "Result/RunHistory/Run", "failed", "2"); 686 assertXmlContainsAttribute(content, "Result/RunHistory/Run", "command_line_args", "cts"); 687 assertXmlContainsAttribute( 688 content, "Result/RunHistory/Run", "host_name", "user.android.com"); 689 // Test that we can read back the information. 690 SuiteResultHolder holder = mFormatter.parseResults(mResultDir, false); 691 assertEquals(RUN_HISTORY, holder.context.getAttributes().getUniqueMap().get("run_history")); 692 } 693 694 /** Ensure the order is sorted according to module name and abi. */ 695 @Test testSortModules()696 public void testSortModules() { 697 List<TestRunResult> originalList = new ArrayList<>(); 698 originalList.add(createFakeResult("armeabi-v7a module1", 1, 0, 0, 0)); 699 originalList.add(createFakeResult("arm64-v8a module3", 1, 0, 0, 0)); 700 originalList.add(createFakeResult("armeabi-v7a module2", 1, 0, 0, 0)); 701 originalList.add(createFakeResult("arm64-v8a module1", 1, 0, 0, 0)); 702 originalList.add(createFakeResult("armeabi-v7a module4", 1, 0, 0, 0)); 703 originalList.add(createFakeResult("arm64-v8a module2", 1, 0, 0, 0)); 704 Map<String, IAbi> moduleAbis = new HashMap<>(); 705 moduleAbis.put("armeabi-v7a module1", new Abi("armeabi-v7a", "32")); 706 moduleAbis.put("arm64-v8a module1", new Abi("arm64-v8a", "64")); 707 moduleAbis.put("armeabi-v7a module2", new Abi("armeabi-v7a", "32")); 708 moduleAbis.put("arm64-v8a module2", new Abi("arm64-v8a", "64")); 709 moduleAbis.put("arm64-v8a module3", new Abi("arm64-v8a", "64")); 710 moduleAbis.put("armeabi-v7a module4", new Abi("armeabi-v7a", "32")); 711 712 List<TestRunResult> sortedResult = mFormatter.sortModules(originalList, moduleAbis); 713 assertEquals(6, sortedResult.size()); 714 assertEquals("arm64-v8a module1", sortedResult.get(0).getName()); 715 assertEquals("armeabi-v7a module1", sortedResult.get(1).getName()); 716 assertEquals("arm64-v8a module2", sortedResult.get(2).getName()); 717 assertEquals("armeabi-v7a module2", sortedResult.get(3).getName()); 718 assertEquals("arm64-v8a module3", sortedResult.get(4).getName()); 719 assertEquals("armeabi-v7a module4", sortedResult.get(5).getName()); 720 } 721 722 /** Check that the build info contains all information. */ 723 @Test testBasicFormat_buildInfo()724 public void testBasicFormat_buildInfo() throws Exception { 725 mResultHolder.context = mContext; 726 mContext.addInvocationAttribute("invocation-attr", "attr"); 727 BuildInfo buildInfo = new BuildInfo(); 728 buildInfo.addBuildAttribute("device_kernel_info", "kernel info"); 729 buildInfo.addBuildAttribute("system_img_info", "system img info"); 730 buildInfo.addBuildAttribute("vendor_img_info", "vendor img info"); 731 mContext.addDeviceBuildInfo("device", buildInfo); 732 733 Collection<TestRunResult> runResults = new ArrayList<>(); 734 runResults.add(createFakeResult("module1", 2, 0, 0, 0)); 735 runResults.add(createFakeResult("module2", 1, 0, 0, 0)); 736 mResultHolder.runResults = runResults; 737 738 Map<String, IAbi> modulesAbi = new HashMap<>(); 739 modulesAbi.put("module1", new Abi("armeabi-v7a", "32")); 740 modulesAbi.put("module2", new Abi("armeabi-v7a", "32")); 741 mResultHolder.modulesAbi = modulesAbi; 742 743 mResultHolder.completeModules = 2; 744 mResultHolder.totalModules = 2; 745 mResultHolder.passedTests = 2; 746 mResultHolder.failedTests = 0; 747 mResultHolder.startTime = 0L; 748 mResultHolder.endTime = 10L; 749 File res = mFormatter.writeResults(mResultHolder, mResultDir); 750 String content = FileUtil.readStringFromFile(res); 751 752 assertXmlContainsNode(content, "Result/Build"); 753 assertXmlContainsAttribute(content, "Result/Build", "invocation-attr", "attr"); 754 assertXmlContainsAttribute(content, "Result/Build", "device_kernel_info", "kernel info"); 755 assertXmlContainsAttribute(content, "Result/Build", "system_img_info", "system img info"); 756 assertXmlContainsAttribute(content, "Result/Build", "vendor_img_info", "vendor img info"); 757 } 758 createResultWithLog(String runName, int count, LogDataType type)759 private TestRunResult createResultWithLog(String runName, int count, LogDataType type) { 760 TestRunResult fakeRes = new TestRunResult(); 761 fakeRes.testRunStarted(runName, count); 762 for (int i = 0; i < count; i++) { 763 TestDescription description = 764 new TestDescription("com.class." + runName, runName + ".method" + i); 765 fakeRes.testStarted(description); 766 fakeRes.testLogSaved( 767 runName + "log" + i, new LogFile("path", "http:url/" + runName, type)); 768 fakeRes.testEnded(description, new HashMap<String, Metric>()); 769 } 770 fakeRes.testRunEnded(10L, new HashMap<String, String>()); 771 return fakeRes; 772 } 773 createFakeResult( String runName, int passed, int failed, int assumptionFailures, int testIgnored)774 private TestRunResult createFakeResult( 775 String runName, int passed, int failed, int assumptionFailures, int testIgnored) { 776 return createFakeResult( 777 runName, passed, failed, assumptionFailures, testIgnored, false, false); 778 } 779 createFakeResult( String runName, int passed, int failed, int assumptionFailures, int testIgnored, boolean withMetrics, boolean withBadKey)780 private TestRunResult createFakeResult( 781 String runName, 782 int passed, 783 int failed, 784 int assumptionFailures, 785 int testIgnored, 786 boolean withMetrics, 787 boolean withBadKey) { 788 return createFakeResult( 789 runName, 790 passed, 791 failed, 792 assumptionFailures, 793 testIgnored, 794 2, 795 withMetrics, 796 withBadKey); 797 } 798 createFakeResult( String runName, int passed, int failed, int assumptionFailures, int testIgnored, int stackDepth, boolean withMetrics, boolean withBadKey)799 private TestRunResult createFakeResult( 800 String runName, 801 int passed, 802 int failed, 803 int assumptionFailures, 804 int testIgnored, 805 int stackDepth, 806 boolean withMetrics, 807 boolean withBadKey) { 808 TestRunResult fakeRes = new TestRunResult(); 809 fakeRes.testRunStarted(runName, passed + failed); 810 for (int i = 0; i < passed; i++) { 811 TestDescription description = 812 new TestDescription("com.class." + runName, runName + ".method" + i); 813 fakeRes.testStarted(description); 814 fakeRes.testEnded(description, new HashMap<String, Metric>()); 815 } 816 for (int i = 0; i < failed; i++) { 817 TestDescription description = 818 new TestDescription("com.class." + runName, runName + ".failed" + i); 819 fakeRes.testStarted(description); 820 // Include a null character \0 that is not XML supported 821 FailureDescription failureDescription = 822 FailureDescription.create( 823 runName + " failed." + "\nstack".repeat(stackDepth) + "\0") 824 .setErrorIdentifier(TestErrorIdentifier.TEST_ABORTED); 825 fakeRes.testFailed(description, failureDescription); 826 HashMap<String, Metric> metrics = new HashMap<String, Metric>(); 827 if (withMetrics) { 828 metrics.put("metric0" + i, TfMetricProtoUtil.stringToMetric("value0" + i)); 829 metrics.put("metric1" + i, TfMetricProtoUtil.stringToMetric("value1" + i)); 830 } 831 if (withBadKey) { 832 metrics.put("%_capacity" + i, TfMetricProtoUtil.stringToMetric("0.00")); 833 metrics.put("&_capacity" + i, TfMetricProtoUtil.stringToMetric("0.00")); 834 } 835 fakeRes.testEnded(description, metrics); 836 } 837 for (int i = 0; i < assumptionFailures; i++) { 838 TestDescription description = 839 new TestDescription("com.class." + runName, runName + ".assumpFail" + i); 840 fakeRes.testStarted(description); 841 fakeRes.testAssumptionFailure( 842 description, runName + " failed." + "\nstack".repeat(stackDepth)); 843 fakeRes.testEnded(description, new HashMap<String, Metric>()); 844 } 845 for (int i = 0; i < testIgnored; i++) { 846 TestDescription description = 847 new TestDescription("com.class." + runName, runName + ".ignored" + i); 848 fakeRes.testStarted(description); 849 fakeRes.testIgnored(description); 850 fakeRes.testEnded(description, new HashMap<String, Metric>()); 851 } 852 fakeRes.testRunEnded(10L, new HashMap<String, Metric>()); 853 return fakeRes; 854 } 855 856 /** Return all XML nodes that match the given xPathExpression. */ getXmlNodes(String xml, String xPathExpression)857 private NodeList getXmlNodes(String xml, String xPathExpression) 858 throws XPathExpressionException { 859 860 InputSource inputSource = new InputSource(new StringReader(xml)); 861 XPath xpath = XPathFactory.newInstance().newXPath(); 862 return (NodeList) xpath.evaluate(xPathExpression, inputSource, XPathConstants.NODESET); 863 } 864 865 /** Assert that the XML contains a node matching the given xPathExpression. */ assertXmlContainsNode(String xml, String xPathExpression)866 private NodeList assertXmlContainsNode(String xml, String xPathExpression) 867 throws XPathExpressionException { 868 NodeList nodes = getXmlNodes(xml, xPathExpression); 869 assertNotNull( 870 String.format("XML '%s' returned null for xpath '%s'.", xml, xPathExpression), 871 nodes); 872 assertTrue( 873 String.format( 874 "XML '%s' should have returned at least 1 node for xpath '%s', " 875 + "but returned %s nodes instead.", 876 xml, xPathExpression, nodes.getLength()), 877 nodes.getLength() >= 1); 878 return nodes; 879 } 880 881 /** Assert that the XML does not contain a node matching the given xPathExpression. */ assertXmlNotContainNode(String xml, String xPathExpression)882 private void assertXmlNotContainNode(String xml, String xPathExpression) 883 throws XPathExpressionException { 884 NodeList nodes = getXmlNodes(xml, xPathExpression); 885 assertNotNull( 886 String.format("XML '%s' returned null for xpath '%s'.", xml, xPathExpression), 887 nodes); 888 assertEquals( 889 String.format( 890 "XML '%s' should have returned at least 1 node for xpath '%s', " 891 + "but returned %s nodes instead.", 892 xml, xPathExpression, nodes.getLength()), 893 0, 894 nodes.getLength()); 895 } 896 897 /** 898 * Assert that the XML contains a node matching the given xPathExpression and that the node has 899 * a given value. 900 */ assertXmlContainsValue(String xml, String xPathExpression, String value)901 private void assertXmlContainsValue(String xml, String xPathExpression, String value) 902 throws XPathExpressionException { 903 NodeList nodes = assertXmlContainsNode(xml, xPathExpression); 904 boolean found = false; 905 906 for (int i = 0; i < nodes.getLength(); i++) { 907 Element element = (Element) nodes.item(i); 908 if (element.getTextContent().equals(value)) { 909 found = true; 910 break; 911 } 912 } 913 914 assertTrue( 915 String.format( 916 "xPath '%s' should contain value '%s' but does not. XML: '%s'", 917 xPathExpression, value, xml), 918 found); 919 } 920 921 /** 922 * Assert that the XML contains a node matching the given xPathExpression and that the node has 923 * a given value. 924 */ assertXmlContainsAttribute( String xml, String xPathExpression, String attributeName, String attributeValue)925 private void assertXmlContainsAttribute( 926 String xml, String xPathExpression, String attributeName, String attributeValue) 927 throws XPathExpressionException { 928 NodeList nodes = assertXmlContainsNode(xml, xPathExpression); 929 boolean found = false; 930 931 for (int i = 0; i < nodes.getLength(); i++) { 932 Element element = (Element) nodes.item(i); 933 String value = element.getAttribute(attributeName); 934 if (attributeValue.equals(value)) { 935 found = true; 936 break; 937 } 938 } 939 940 assertTrue( 941 String.format( 942 "xPath '%s' should contain attribute '%s' but does not. XML: '%s'", 943 xPathExpression, attributeName, xml), 944 found); 945 } 946 } 947