1 /* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License 15 */ 16 package com.android.compatibility.common.util; 17 18 import com.android.tradefed.util.FileUtil; 19 20 import junit.framework.TestCase; 21 22 import java.io.File; 23 import java.io.FileWriter; 24 import java.net.InetAddress; 25 import java.net.UnknownHostException; 26 import java.util.Arrays; 27 import java.util.List; 28 import java.util.Map; 29 import java.util.Set; 30 31 /** 32 * Unit tests for {@link ResultHandler} 33 */ 34 public class ResultHandlerTest extends TestCase { 35 36 private static final String SUITE_NAME = "CTS"; 37 private static final String SUITE_VERSION = "5.0"; 38 private static final String SUITE_PLAN = "cts"; 39 private static final String SUITE_BUILD = "12345"; 40 private static final String REPORT_VERSION = "5.0"; 41 private static final String OS_NAME = System.getProperty("os.name"); 42 private static final String OS_VERSION = System.getProperty("os.version"); 43 private static final String OS_ARCH = System.getProperty("os.arch"); 44 private static final String JAVA_VENDOR = System.getProperty("java.vendor"); 45 private static final String JAVA_VERSION = System.getProperty("java.version"); 46 private static final String NAME_A = "ModuleA"; 47 private static final String NAME_B = "ModuleB"; 48 private static final String ABI = "mips64"; 49 private static final String ID_A = AbiUtils.createId(ABI, NAME_A); 50 private static final String ID_B = AbiUtils.createId(ABI, NAME_B); 51 52 private static final String BUILD_ID = "build_id"; 53 private static final String BUILD_PRODUCT = "build_product"; 54 private static final String EXAMPLE_BUILD_ID = "XYZ"; 55 private static final String EXAMPLE_BUILD_PRODUCT = "wolverine"; 56 57 private static final String DEVICE_A = "device123"; 58 private static final String DEVICE_B = "device456"; 59 private static final String DEVICES = "device456,device123"; 60 private static final String CLASS_A = "android.test.Foor"; 61 private static final String CLASS_B = "android.test.Bar"; 62 private static final String METHOD_1 = "testBlah1"; 63 private static final String METHOD_2 = "testBlah2"; 64 private static final String METHOD_3 = "testBlah3"; 65 private static final String METHOD_4 = "testBlah4"; 66 private static final String SUMMARY_SOURCE = String.format("%s#%s:20", CLASS_B, METHOD_4); 67 private static final String DETAILS_SOURCE = String.format("%s#%s:18", CLASS_B, METHOD_4); 68 private static final String SUMMARY_MESSAGE = "Headline"; 69 private static final double SUMMARY_VALUE = 9001; 70 private static final String DETAILS_MESSAGE = "Deats"; 71 private static final double DETAILS_VALUE_1 = 14; 72 private static final double DETAILS_VALUE_2 = 18; 73 private static final double DETAILS_VALUE_3 = 17; 74 private static final String MESSAGE = "Something small is not alright"; 75 private static final String STACK_TRACE = "Something small is not alright\n " + 76 "at four.big.insects.Marley.sing(Marley.java:10)"; 77 private static final long START_MS = 1431586801000L; 78 private static final long END_MS = 1431673199000L; 79 private static final String START_DISPLAY = "Fri Aug 20 15:13:03 PDT 2010"; 80 private static final String END_DISPLAY = "Fri Aug 20 15:13:04 PDT 2010"; 81 82 private static final String REFERENCE_URL="http://android.com"; 83 private static final String LOG_URL ="file:///path/to/logs"; 84 private static final String COMMAND_LINE_ARGS = "cts -m CtsMyModuleTestCases"; 85 private static final String JOIN = "%s%s"; 86 private static final String XML_BASE = 87 "<?xml version='1.0' encoding='UTF-8' standalone='no' ?>" + 88 "<?xml-stylesheet type=\"text/xsl\" href=\"compatibility_result.xsl\"?>\n" + 89 "<Result start=\"%d\" end=\"%d\" start_display=\"%s\"" + 90 "end_display=\"%s\" suite_name=\"%s\" suite_version=\"%s\" " + 91 "suite_plan=\"%s\" suite_build_number=\"%s\" report_version=\"%s\" " + 92 "devices=\"%s\" host_name=\"%s\"" + 93 "os_name=\"%s\" os_version=\"%s\" os_arch=\"%s\" java_vendor=\"%s\"" + 94 "java_version=\"%s\" reference_url=\"%s\" log_url=\"%s\"" + 95 "command_line_args=\"%s\">\n" + 96 "%s%s%s" + 97 "</Result>"; 98 private static final String XML_BUILD_INFO = 99 " <Build build_fingerprint=\"%s\" " + BUILD_ID + "=\"%s\" " + 100 BUILD_PRODUCT + "=\"%s\" />\n"; 101 private static final String XML_SUMMARY = 102 " <Summary pass=\"%d\" failed=\"%d\" not_executed=\"%d\" " + 103 "modules_done=\"1\" modules_total=\"1\" />\n"; 104 private static final String XML_MODULE = 105 " <Module name=\"%s\" abi=\"%s\" device=\"%s\">\n" + 106 "%s" + 107 " </Module>\n"; 108 private static final String XML_CASE = 109 " <TestCase name=\"%s\">\n" + 110 "%s" + 111 " </TestCase>\n"; 112 private static final String XML_TEST_PASS = 113 " <Test result=\"pass\" name=\"%s\"/>\n"; 114 private static final String XML_TEST_NOT_EXECUTED = 115 " <Test result=\"not_executed\" name=\"%s\"/>\n"; 116 private static final String XML_TEST_FAIL = 117 " <Test result=\"fail\" name=\"%s\">\n" + 118 " <Failure message=\"%s\">\n" + 119 " <StackTrace>%s</StackTrace>\n" + 120 " </Failure>\n" + 121 " </Test>\n"; 122 private static final String XML_TEST_RESULT = 123 " <Test result=\"pass\" name=\"%s\">\n" + 124 " <Summary>\n" + 125 " <Metric source=\"%s\" message=\"%s\" score_type=\"%s\" score_unit=\"%s\">\n" + 126 " <Value>%s</Value>\n" + 127 " </Metric>\n" + 128 " </Summary>\n" + 129 " </Test>\n"; 130 private File resultsDir = null; 131 private File resultDir = null; 132 133 @Override setUp()134 public void setUp() throws Exception { 135 resultsDir = FileUtil.createTempDir("results"); 136 resultDir = FileUtil.createTempDir("12345", resultsDir); 137 } 138 139 @Override tearDown()140 public void tearDown() throws Exception { 141 if (resultsDir != null) { 142 FileUtil.recursiveDelete(resultsDir); 143 } 144 } 145 testSerialization()146 public void testSerialization() throws Exception { 147 IInvocationResult result = new InvocationResult(); 148 result.setStartTime(START_MS); 149 result.setTestPlan(SUITE_PLAN); 150 result.addDeviceSerial(DEVICE_A); 151 result.addDeviceSerial(DEVICE_B); 152 result.addInvocationInfo(BUILD_ID, EXAMPLE_BUILD_ID); 153 result.addInvocationInfo(BUILD_PRODUCT, EXAMPLE_BUILD_PRODUCT); 154 IModuleResult moduleA = result.getOrCreateModule(ID_A); 155 ICaseResult moduleACase = moduleA.getOrCreateResult(CLASS_A); 156 ITestResult moduleATest1 = moduleACase.getOrCreateResult(METHOD_1); 157 moduleATest1.setResultStatus(TestStatus.PASS); 158 ITestResult moduleATest2 = moduleACase.getOrCreateResult(METHOD_2); 159 moduleATest2.setResultStatus(TestStatus.NOT_EXECUTED); 160 161 IModuleResult moduleB = result.getOrCreateModule(ID_B); 162 ICaseResult moduleBCase = moduleB.getOrCreateResult(CLASS_B); 163 ITestResult moduleBTest3 = moduleBCase.getOrCreateResult(METHOD_3); 164 moduleBTest3.setResultStatus(TestStatus.FAIL); 165 moduleBTest3.setMessage(MESSAGE); 166 moduleBTest3.setStackTrace(STACK_TRACE); 167 ITestResult moduleBTest4 = moduleBCase.getOrCreateResult(METHOD_4); 168 moduleBTest4.setResultStatus(TestStatus.PASS); 169 ReportLog report = new ReportLog(); 170 ReportLog.Metric summary = new ReportLog.Metric(SUMMARY_SOURCE, SUMMARY_MESSAGE, 171 SUMMARY_VALUE, ResultType.HIGHER_BETTER, ResultUnit.SCORE); 172 report.setSummary(summary); 173 moduleBTest4.setReportLog(report); 174 175 // Serialize to file 176 ResultHandler.writeResults(SUITE_NAME, SUITE_VERSION, SUITE_PLAN, SUITE_BUILD, 177 result, resultDir, START_MS, END_MS, REFERENCE_URL, LOG_URL, 178 COMMAND_LINE_ARGS); 179 180 // Parse the results and assert correctness 181 checkResult(ResultHandler.getResults(resultsDir), resultDir); 182 } 183 testParsing()184 public void testParsing() throws Exception { 185 File resultsDir = null; 186 FileWriter writer = null; 187 try { 188 resultsDir = FileUtil.createTempDir("results"); 189 File resultDir = FileUtil.createTempDir("12345", resultsDir); 190 // Create the result file 191 File resultFile = new File(resultDir, ResultHandler.TEST_RESULT_FILE_NAME); 192 writer = new FileWriter(resultFile); 193 String buildInfo = String.format(XML_BUILD_INFO, DEVICE_A, 194 EXAMPLE_BUILD_ID, EXAMPLE_BUILD_PRODUCT); 195 String summary = String.format(XML_SUMMARY, 2, 1, 1); 196 String moduleATest1 = String.format(XML_TEST_PASS, METHOD_1); 197 String moduleATest2 = String.format(XML_TEST_NOT_EXECUTED, METHOD_2); 198 String moduleATests = String.format(JOIN, moduleATest1, moduleATest2); 199 String moduleACases = String.format(XML_CASE, CLASS_A, moduleATests); 200 String moduleA = String.format(XML_MODULE, NAME_A, ABI, DEVICE_A, moduleACases); 201 String moduleBTest3 = String.format(XML_TEST_FAIL, METHOD_3, MESSAGE, STACK_TRACE); 202 String moduleBTest4 = String.format(XML_TEST_RESULT, METHOD_4, 203 SUMMARY_SOURCE, SUMMARY_MESSAGE, ResultType.HIGHER_BETTER.toReportString(), 204 ResultUnit.SCORE.toReportString(), Double.toString(SUMMARY_VALUE), 205 DETAILS_SOURCE, DETAILS_MESSAGE, ResultType.LOWER_BETTER.toReportString(), 206 ResultUnit.MS.toReportString(), Double.toString(DETAILS_VALUE_1), 207 Double.toString(DETAILS_VALUE_2), Double.toString(DETAILS_VALUE_3)); 208 String moduleBTests = String.format(JOIN, moduleBTest3, moduleBTest4); 209 String moduleBCases = String.format(XML_CASE, CLASS_B, moduleBTests); 210 String moduleB = String.format(XML_MODULE, NAME_B, ABI, DEVICE_B, moduleBCases); 211 String modules = String.format(JOIN, moduleA, moduleB); 212 String hostName = ""; 213 try { 214 hostName = InetAddress.getLocalHost().getHostName(); 215 } catch (UnknownHostException ignored) {} 216 String output = String.format(XML_BASE, START_MS, END_MS, START_DISPLAY, END_DISPLAY, 217 SUITE_NAME, SUITE_VERSION, SUITE_PLAN, SUITE_BUILD, REPORT_VERSION, DEVICES, 218 hostName, OS_NAME, OS_VERSION, OS_ARCH, JAVA_VENDOR, 219 JAVA_VERSION, REFERENCE_URL, LOG_URL, COMMAND_LINE_ARGS, 220 buildInfo, summary, modules); 221 writer.write(output); 222 writer.flush(); 223 224 // Parse the results and assert correctness 225 checkResult(ResultHandler.getResults(resultsDir), resultDir); 226 } finally { 227 if (writer != null) { 228 writer.close(); 229 } 230 } 231 } 232 checkResult(List<IInvocationResult> results, File resultDir)233 private void checkResult(List<IInvocationResult> results, File resultDir) throws Exception { 234 assertEquals("Expected 1 result", 1, results.size()); 235 IInvocationResult result = results.get(0); 236 assertEquals("Expected 2 passes", 2, result.countResults(TestStatus.PASS)); 237 assertEquals("Expected 1 failure", 1, result.countResults(TestStatus.FAIL)); 238 assertEquals("Expected 1 not executed", 1, result.countResults(TestStatus.NOT_EXECUTED)); 239 240 Map<String, String> buildInfo = result.getInvocationInfo(); 241 assertEquals("Incorrect Build ID", EXAMPLE_BUILD_ID, buildInfo.get(BUILD_ID)); 242 assertEquals("Incorrect Build Product", 243 EXAMPLE_BUILD_PRODUCT, buildInfo.get(BUILD_PRODUCT)); 244 245 Set<String> serials = result.getDeviceSerials(); 246 assertTrue("Missing device", serials.contains(DEVICE_A)); 247 assertTrue("Missing device", serials.contains(DEVICE_B)); 248 assertEquals("Expected 2 devices", 2, serials.size()); 249 assertTrue("Incorrect devices", serials.contains(DEVICE_A) && serials.contains(DEVICE_B)); 250 assertEquals("Incorrect start time", START_MS, result.getStartTime()); 251 assertEquals("Incorrect test plan", SUITE_PLAN, result.getTestPlan()); 252 253 List<IModuleResult> modules = result.getModules(); 254 assertEquals("Expected 2 modules", 2, modules.size()); 255 256 IModuleResult moduleA = modules.get(0); 257 assertEquals("Expected 1 pass", 1, moduleA.countResults(TestStatus.PASS)); 258 assertEquals("Expected 0 failures", 0, moduleA.countResults(TestStatus.FAIL)); 259 assertEquals("Expected 1 not executed", 1, moduleA.countResults(TestStatus.NOT_EXECUTED)); 260 assertEquals("Incorrect ABI", ABI, moduleA.getAbi()); 261 assertEquals("Incorrect name", NAME_A, moduleA.getName()); 262 assertEquals("Incorrect ID", ID_A, moduleA.getId()); 263 List<ICaseResult> moduleACases = moduleA.getResults(); 264 assertEquals("Expected 1 test case", 1, moduleACases.size()); 265 ICaseResult moduleACase = moduleACases.get(0); 266 assertEquals("Incorrect name", CLASS_A, moduleACase.getName()); 267 List<ITestResult> moduleAResults = moduleACase.getResults(); 268 assertEquals("Expected 2 results", 2, moduleAResults.size()); 269 ITestResult moduleATest1 = moduleAResults.get(0); 270 assertEquals("Incorrect name", METHOD_1, moduleATest1.getName()); 271 assertEquals("Incorrect result", TestStatus.PASS, moduleATest1.getResultStatus()); 272 assertNull("Unexpected bugreport", moduleATest1.getBugReport()); 273 assertNull("Unexpected log", moduleATest1.getLog()); 274 assertNull("Unexpected screenshot", moduleATest1.getScreenshot()); 275 assertNull("Unexpected message", moduleATest1.getMessage()); 276 assertNull("Unexpected stack trace", moduleATest1.getStackTrace()); 277 assertNull("Unexpected report", moduleATest1.getReportLog()); 278 ITestResult moduleATest2 = moduleAResults.get(1); 279 assertEquals("Incorrect name", METHOD_2, moduleATest2.getName()); 280 assertEquals("Incorrect result", TestStatus.NOT_EXECUTED, moduleATest2.getResultStatus()); 281 assertNull("Unexpected bugreport", moduleATest2.getBugReport()); 282 assertNull("Unexpected log", moduleATest2.getLog()); 283 assertNull("Unexpected screenshot", moduleATest2.getScreenshot()); 284 assertNull("Unexpected message", moduleATest2.getMessage()); 285 assertNull("Unexpected stack trace", moduleATest2.getStackTrace()); 286 assertNull("Unexpected report", moduleATest2.getReportLog()); 287 288 IModuleResult moduleB = modules.get(1); 289 assertEquals("Expected 1 pass", 1, moduleB.countResults(TestStatus.PASS)); 290 assertEquals("Expected 1 failure", 1, moduleB.countResults(TestStatus.FAIL)); 291 assertEquals("Expected 0 not executed", 0, moduleB.countResults(TestStatus.NOT_EXECUTED)); 292 assertEquals("Incorrect ABI", ABI, moduleB.getAbi()); 293 assertEquals("Incorrect name", NAME_B, moduleB.getName()); 294 assertEquals("Incorrect ID", ID_B, moduleB.getId()); 295 List<ICaseResult> moduleBCases = moduleB.getResults(); 296 assertEquals("Expected 1 test case", 1, moduleBCases.size()); 297 ICaseResult moduleBCase = moduleBCases.get(0); 298 assertEquals("Incorrect name", CLASS_B, moduleBCase.getName()); 299 List<ITestResult> moduleBResults = moduleBCase.getResults(); 300 assertEquals("Expected 2 results", 2, moduleBResults.size()); 301 ITestResult moduleBTest3 = moduleBResults.get(0); 302 assertEquals("Incorrect name", METHOD_3, moduleBTest3.getName()); 303 assertEquals("Incorrect result", TestStatus.FAIL, moduleBTest3.getResultStatus()); 304 assertNull("Unexpected bugreport", moduleBTest3.getBugReport()); 305 assertNull("Unexpected log", moduleBTest3.getLog()); 306 assertNull("Unexpected screenshot", moduleBTest3.getScreenshot()); 307 assertEquals("Incorrect message", MESSAGE, moduleBTest3.getMessage()); 308 assertEquals("Incorrect stack trace", STACK_TRACE, moduleBTest3.getStackTrace()); 309 assertNull("Unexpected report", moduleBTest3.getReportLog()); 310 ITestResult moduleBTest4 = moduleBResults.get(1); 311 assertEquals("Incorrect name", METHOD_4, moduleBTest4.getName()); 312 assertEquals("Incorrect result", TestStatus.PASS, moduleBTest4.getResultStatus()); 313 assertNull("Unexpected bugreport", moduleBTest4.getBugReport()); 314 assertNull("Unexpected log", moduleBTest4.getLog()); 315 assertNull("Unexpected screenshot", moduleBTest4.getScreenshot()); 316 assertNull("Unexpected message", moduleBTest4.getMessage()); 317 assertNull("Unexpected stack trace", moduleBTest4.getStackTrace()); 318 ReportLog report = moduleBTest4.getReportLog(); 319 assertNotNull("Expected report", report); 320 ReportLog.Metric summary = report.getSummary(); 321 assertNotNull("Expected report summary", summary); 322 assertEquals("Incorrect source", SUMMARY_SOURCE, summary.getSource()); 323 assertEquals("Incorrect message", SUMMARY_MESSAGE, summary.getMessage()); 324 assertEquals("Incorrect type", ResultType.HIGHER_BETTER, summary.getType()); 325 assertEquals("Incorrect unit", ResultUnit.SCORE, summary.getUnit()); 326 assertTrue("Incorrect values", Arrays.equals(new double[] { SUMMARY_VALUE }, 327 summary.getValues())); 328 } 329 } 330