1 /* 2 * Copyright (C) 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.icu.tradefed.testtype; 17 18 import com.android.tradefed.log.LogUtil.CLog; 19 import com.android.tradefed.result.ITestInvocationListener; 20 import com.android.tradefed.result.TestDescription; 21 import com.android.tradefed.util.CommandResult; 22 import org.w3c.dom.Document; 23 import org.w3c.dom.Element; 24 import org.w3c.dom.NodeList; 25 import org.xml.sax.SAXException; 26 import org.xml.sax.helpers.DefaultHandler; 27 28 import java.io.File; 29 import java.io.IOException; 30 import java.util.ArrayList; 31 import java.util.Collection; 32 import java.util.Collections; 33 import java.util.HashMap; 34 import java.util.Map; 35 36 import javax.xml.parsers.DocumentBuilder; 37 import javax.xml.parsers.DocumentBuilderFactory; 38 import javax.xml.parsers.ParserConfigurationException; 39 40 /** Parses the XUnit results of ICU4C tests and informs a ITestInvocationListener of the results. */ 41 public class ICU4CXmlResultParser { 42 43 private static final String TEST_CASE_TAG = "testcase"; 44 45 private final String mTestRunName; 46 private final String mModuleName; 47 private int mNumTestsRun = 0; 48 private int mNumTestsExpected = 0; 49 private long mTotalRunTime = 0; 50 private final Collection<ITestInvocationListener> mTestListeners; 51 52 /** 53 * Creates the ICU4CXmlResultParser. 54 * 55 * @param moduleName module name 56 * @param testRunName the test run name to provide to {@link 57 * ITestInvocationListener#testRunStarted(String, int)} 58 * @param listeners informed of test results as the tests are executing 59 */ ICU4CXmlResultParser(String moduleName, String testRunName, Collection<ITestInvocationListener> listeners)60 public ICU4CXmlResultParser(String moduleName, String testRunName, 61 Collection<ITestInvocationListener> listeners) { 62 mModuleName = moduleName; 63 mTestRunName = testRunName; 64 mTestListeners = new ArrayList<>(listeners); 65 } 66 67 /** 68 * Creates the ICU4CXmlResultParser for a single listener. 69 * 70 * @param moduleName module name 71 * @param testRunName the test run name to provide to {@link 72 * ITestInvocationListener#testRunStarted(String, int)} 73 * @param listener informed of test results as the tests are executing 74 */ ICU4CXmlResultParser(String moduleName, String testRunName, ITestInvocationListener listener)75 public ICU4CXmlResultParser(String moduleName, String testRunName, 76 ITestInvocationListener listener) { 77 mModuleName = moduleName; 78 mTestRunName = testRunName; 79 mTestListeners = new ArrayList<>(); 80 if (listener != null) { 81 mTestListeners.add(listener); 82 } 83 } 84 85 /** 86 * Parse the xml results 87 * 88 * @param f {@link File} containing the outputed xml 89 * @param output The output collected from the execution run to complete the logs if necessary 90 */ parseResult(File f, CommandResult commandResult)91 public void parseResult(File f, CommandResult commandResult) { 92 DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); 93 Document result = null; 94 try { 95 DocumentBuilder db = dbf.newDocumentBuilder(); 96 db.setErrorHandler(new DefaultHandler()); 97 result = db.parse(f); 98 } catch (SAXException | IOException | ParserConfigurationException e) { 99 reportTestRunStarted(); 100 for (ITestInvocationListener listener : mTestListeners) { 101 String errorMessage = 102 String.format( 103 "Failed to get an xml output from tests," + " it probably crashed"); 104 errorMessage += "\nstdout:\n" + commandResult.getStdout(); 105 errorMessage += "\nstderr:\n" + commandResult.getStderr(); 106 CLog.e(errorMessage); 107 listener.testRunFailed(errorMessage); 108 listener.testRunEnded(mTotalRunTime, Collections.emptyMap()); 109 } 110 return; 111 } 112 Element rootNode = result.getDocumentElement(); 113 114 getTestSuitesInfo(rootNode); 115 reportTestRunStarted(); 116 117 // The ICU4C test runner doesn't put out the same format as GTest. 118 // There's no root <testsuites> node. The root node is <testsuite> and 119 // its children are <testcase>. 120 NodeList testcasesList = rootNode.getElementsByTagName(TEST_CASE_TAG); 121 // Iterate other the test cases in the test suite. 122 if (testcasesList != null && testcasesList.getLength() > 0) { 123 for (int i = 0; i < testcasesList.getLength(); i++) { 124 processTestResult((Element) testcasesList.item(i)); 125 } 126 } 127 128 if (mNumTestsExpected > mNumTestsRun) { 129 for (ITestInvocationListener listener : mTestListeners) { 130 listener.testRunFailed( 131 String.format( 132 "Test run incomplete. Expected %d tests, received %d", 133 mNumTestsExpected, mNumTestsRun)); 134 } 135 } 136 for (ITestInvocationListener listener : mTestListeners) { 137 listener.testRunEnded(mTotalRunTime, Collections.emptyMap()); 138 } 139 } 140 getTestSuitesInfo(Element rootNode)141 private void getTestSuitesInfo(Element rootNode) { 142 mNumTestsExpected = rootNode.getElementsByTagName(TEST_CASE_TAG).getLength(); 143 // TODO(danalbert): Teach the ICU4C test runner to output this. 144 mTotalRunTime = 0L; 145 } 146 147 /** 148 * Reports the start of a test run, and the total test count, if it has not been previously 149 * reported. 150 */ reportTestRunStarted()151 private void reportTestRunStarted() { 152 for (ITestInvocationListener listener : mTestListeners) { 153 listener.testRunStarted(mTestRunName, mNumTestsExpected); 154 } 155 } 156 157 /** 158 * Processes and informs listener when we encounter a tag indicating that a test has started. 159 * 160 * @param testcase Raw log output of the form classname.testname, with an optional time (x ms) 161 */ processTestResult(Element testcase)162 private void processTestResult(Element testcase) { 163 String classname = testcase.getAttribute("classname"); 164 String testname = testcase.getAttribute("name"); 165 String runtime = testcase.getAttribute("time"); 166 167 // Remove extra leading '/' character 168 if (classname.startsWith("/")) { 169 classname = classname.substring(1); 170 } 171 172 // For test reporting on Android, prefix module name to the class name 173 // and replace '/' with '.' 174 classname = mModuleName + '.' + classname.replace('/', '.'); 175 176 // TODO: Fix the duplicate test name in the testId 177 // Currently, testId is like spoof#spoof or spoof/testBug8654#testBug8654 178 // in order to avoid empty test name in the case of spoof#spoof. We should remove 179 // spoof#spoof from the test report because it's not a test case. 180 TestDescription testId = new TestDescription(classname, testname); 181 mNumTestsRun++; 182 for (ITestInvocationListener listener : mTestListeners) { 183 listener.testStarted(testId); 184 } 185 186 // If there is a failure tag report failure 187 if (testcase.getElementsByTagName("failure").getLength() != 0) { 188 String trace = 189 ((Element) testcase.getElementsByTagName("failure").item(0)) 190 .getAttribute("message"); 191 if (!trace.contains("Failed")) { 192 // For some reason, the alternative ICU4C format doesn't specify Failed in the 193 // trace and error doesn't show properly in reporter, so adding it here. 194 trace += "\nFailed"; 195 } 196 for (ITestInvocationListener listener : mTestListeners) { 197 listener.testFailed(testId, trace); 198 } 199 } 200 201 Map<String, String> map = new HashMap<>(); 202 map.put("runtime", runtime); 203 for (ITestInvocationListener listener : mTestListeners) { 204 listener.testEnded(testId, map); 205 } 206 } 207 } 208