1 /* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.cts.tradefed.result; 18 19 import com.android.cts.tradefed.build.CtsBuildHelper; 20 import com.android.cts.tradefed.device.DeviceInfoCollector; 21 import com.android.cts.tradefed.testtype.CtsTest; 22 import com.android.ddmlib.Log; 23 import com.android.ddmlib.Log.LogLevel; 24 import com.android.ddmlib.testrunner.TestIdentifier; 25 import com.android.tradefed.build.IBuildInfo; 26 import com.android.tradefed.build.IFolderBuildInfo; 27 import com.android.tradefed.config.Option; 28 import com.android.tradefed.log.LogUtil.CLog; 29 import com.android.tradefed.result.ILogFileSaver; 30 import com.android.tradefed.result.ITestInvocationListener; 31 import com.android.tradefed.result.InputStreamSource; 32 import com.android.tradefed.result.LogDataType; 33 import com.android.tradefed.result.LogFileSaver; 34 import com.android.tradefed.result.TestSummary; 35 import com.android.tradefed.util.FileUtil; 36 import com.android.tradefed.util.StreamUtil; 37 38 import org.kxml2.io.KXmlSerializer; 39 40 import java.io.File; 41 import java.io.FileNotFoundException; 42 import java.io.FileOutputStream; 43 import java.io.IOException; 44 import java.io.InputStream; 45 import java.io.OutputStream; 46 import java.util.Map; 47 48 /** 49 * Writes results to an XML files in the CTS format. 50 * <p/> 51 * Collects all test info in memory, then dumps to file when invocation is complete. 52 * <p/> 53 * Outputs xml in format governed by the cts_result.xsd 54 */ 55 public class CtsXmlResultReporter implements ITestInvocationListener { 56 private static final String LOG_TAG = "CtsXmlResultReporter"; 57 58 static final String TEST_RESULT_FILE_NAME = "testResult.xml"; 59 private static final String CTS_RESULT_FILE_VERSION = "1.13"; 60 private static final String[] CTS_RESULT_RESOURCES = {"cts_result.xsl", "cts_result.css", 61 "logo.gif", "newrule-green.png"}; 62 63 /** the XML namespace */ 64 static final String ns = null; 65 66 static final String RESULT_TAG = "TestResult"; 67 static final String PLAN_ATTR = "testPlan"; 68 static final String STARTTIME_ATTR = "starttime"; 69 70 private static final String REPORT_DIR_NAME = "output-file-path"; 71 @Option(name=REPORT_DIR_NAME, description="root file system path to directory to store xml " + 72 "test results and associated logs. If not specified, results will be stored at " + 73 "<cts root>/repository/results") 74 protected File mReportDir = null; 75 76 // listen in on the plan option provided to CtsTest 77 @Option(name = CtsTest.PLAN_OPTION, description = "the test plan to run.") 78 private String mPlanName = "NA"; 79 80 // listen in on the continue-session option provided to CtsTest 81 @Option(name = CtsTest.CONTINUE_OPTION, description = "the test result session to continue.") 82 private Integer mContinueSessionId = null; 83 84 @Option(name = "quiet-output", description = "Mute display of test results.") 85 private boolean mQuietOutput = false; 86 87 @Option(name = "result-server", description = "Server to publish test results.") 88 private String mResultServer; 89 90 protected IBuildInfo mBuildInfo; 91 private String mStartTime; 92 private String mDeviceSerial; 93 private TestResults mResults = new TestResults(); 94 private TestPackageResult mCurrentPkgResult = null; 95 private boolean mIsDeviceInfoRun = false; 96 97 private File mLogDir; 98 setReportDir(File reportDir)99 public void setReportDir(File reportDir) { 100 mReportDir = reportDir; 101 } 102 103 /** 104 * {@inheritDoc} 105 */ 106 @Override invocationStarted(IBuildInfo buildInfo)107 public void invocationStarted(IBuildInfo buildInfo) { 108 mBuildInfo = buildInfo; 109 if (!(buildInfo instanceof IFolderBuildInfo)) { 110 throw new IllegalArgumentException("build info is not a IFolderBuildInfo"); 111 } 112 IFolderBuildInfo ctsBuild = (IFolderBuildInfo)buildInfo; 113 mDeviceSerial = buildInfo.getDeviceSerial() == null ? "unknown_device" : 114 buildInfo.getDeviceSerial(); 115 if (mContinueSessionId != null) { 116 CLog.d("Continuing session %d", mContinueSessionId); 117 // reuse existing directory 118 TestResultRepo resultRepo = new TestResultRepo(getBuildHelper(ctsBuild).getResultsDir()); 119 mResults = resultRepo.getResult(mContinueSessionId); 120 if (mResults == null) { 121 throw new IllegalArgumentException(String.format("Could not find session %d", 122 mContinueSessionId)); 123 } 124 mPlanName = resultRepo.getSummaries().get(mContinueSessionId).getTestPlan(); 125 mStartTime = resultRepo.getSummaries().get(mContinueSessionId).getStartTime(); 126 mReportDir = resultRepo.getReportDir(mContinueSessionId); 127 } else { 128 if (mReportDir == null) { 129 mReportDir = getBuildHelper(ctsBuild).getResultsDir(); 130 } 131 // create a unique directory for saving results, using old cts host convention 132 // TODO: in future, consider using LogFileSaver to create build-specific directories 133 mReportDir = new File(mReportDir, TimeUtil.getResultTimestamp()); 134 mReportDir.mkdirs(); 135 mStartTime = getTimestamp(); 136 logResult("Created result dir %s", mReportDir.getName()); 137 } 138 // TODO: allow customization of log dir 139 // create a unique directory for saving logs, with same name as result dir 140 File rootLogDir = getBuildHelper(ctsBuild).getLogsDir(); 141 mLogDir = new File(rootLogDir, mReportDir.getName()); 142 mLogDir.mkdirs(); 143 } 144 145 /** 146 * Helper method to retrieve the {@link CtsBuildHelper}. 147 * @param ctsBuild 148 */ getBuildHelper(IFolderBuildInfo ctsBuild)149 CtsBuildHelper getBuildHelper(IFolderBuildInfo ctsBuild) { 150 CtsBuildHelper buildHelper = new CtsBuildHelper(ctsBuild.getRootDir()); 151 try { 152 buildHelper.validateStructure(); 153 } catch (FileNotFoundException e) { 154 // just log an error - it might be expected if we failed to retrieve a build 155 CLog.e("Invalid CTS build %s", ctsBuild.getRootDir()); 156 } 157 return buildHelper; 158 } 159 160 /** 161 * {@inheritDoc} 162 */ 163 @Override testLog(String dataName, LogDataType dataType, InputStreamSource dataStream)164 public void testLog(String dataName, LogDataType dataType, InputStreamSource dataStream) { 165 try { 166 File logFile = getLogFileSaver().saveAndZipLogData(dataName, dataType, 167 dataStream.createInputStream()); 168 logResult(String.format("Saved log %s", logFile.getName())); 169 } catch (IOException e) { 170 CLog.e("Failed to write log for %s", dataName); 171 } 172 } 173 174 /** 175 * Return the {@link ILogFileSaver} to use. 176 * <p/> 177 * Exposed for unit testing. 178 */ getLogFileSaver()179 ILogFileSaver getLogFileSaver() { 180 return new LogFileSaver(mLogDir); 181 } 182 183 /** 184 * {@inheritDoc} 185 */ 186 @Override testRunStarted(String name, int numTests)187 public void testRunStarted(String name, int numTests) { 188 if (mCurrentPkgResult != null && !name.equals(mCurrentPkgResult.getAppPackageName())) { 189 // display results from previous run 190 logCompleteRun(mCurrentPkgResult); 191 } 192 mIsDeviceInfoRun = name.equals(DeviceInfoCollector.APP_PACKAGE_NAME); 193 if (mIsDeviceInfoRun) { 194 logResult("Collecting device info"); 195 } else { 196 if (mCurrentPkgResult == null || !name.equals(mCurrentPkgResult.getAppPackageName())) { 197 logResult("-----------------------------------------"); 198 logResult("Test package %s started", name); 199 logResult("-----------------------------------------"); 200 } 201 mCurrentPkgResult = mResults.getOrCreatePackage(name); 202 } 203 204 } 205 206 /** 207 * {@inheritDoc} 208 */ 209 @Override testStarted(TestIdentifier test)210 public void testStarted(TestIdentifier test) { 211 mCurrentPkgResult.insertTest(test); 212 } 213 214 /** 215 * {@inheritDoc} 216 */ 217 @Override testFailed(TestFailure status, TestIdentifier test, String trace)218 public void testFailed(TestFailure status, TestIdentifier test, String trace) { 219 mCurrentPkgResult.reportTestFailure(test, CtsTestStatus.FAIL, trace); 220 } 221 222 /** 223 * {@inheritDoc} 224 */ 225 @Override testEnded(TestIdentifier test, Map<String, String> testMetrics)226 public void testEnded(TestIdentifier test, Map<String, String> testMetrics) { 227 mCurrentPkgResult.reportTestEnded(test); 228 Test result = mCurrentPkgResult.findTest(test); 229 String stack = result.getStackTrace() == null ? "" : "\n" + result.getStackTrace(); 230 logResult("%s#%s %s %s", test.getClassName(), test.getTestName(), result.getResult(), 231 stack); 232 } 233 234 /** 235 * {@inheritDoc} 236 */ 237 @Override testRunEnded(long elapsedTime, Map<String, String> runMetrics)238 public void testRunEnded(long elapsedTime, Map<String, String> runMetrics) { 239 if (mIsDeviceInfoRun) { 240 mResults.populateDeviceInfoMetrics(runMetrics); 241 } else { 242 mCurrentPkgResult.populateMetrics(runMetrics); 243 } 244 } 245 246 /** 247 * {@inheritDoc} 248 */ 249 @Override invocationEnded(long elapsedTime)250 public void invocationEnded(long elapsedTime) { 251 // display the results of the last completed run 252 if (mCurrentPkgResult != null) { 253 logCompleteRun(mCurrentPkgResult); 254 } 255 if (mReportDir == null || mStartTime == null) { 256 // invocationStarted must have failed, abort 257 CLog.w("Unable to create XML report"); 258 return; 259 } 260 261 File reportFile = getResultFile(mReportDir); 262 createXmlResult(reportFile, mStartTime, elapsedTime); 263 copyFormattingFiles(mReportDir); 264 zipResults(mReportDir); 265 266 try { 267 ResultReporter reporter = new ResultReporter(mResultServer, reportFile); 268 reporter.reportResult(); 269 } catch (IOException e) { 270 CLog.e(e); 271 } 272 } 273 logResult(String format, Object... args)274 private void logResult(String format, Object... args) { 275 if (mQuietOutput) { 276 CLog.i(format, args); 277 } else { 278 Log.logAndDisplay(LogLevel.INFO, mDeviceSerial, String.format(format, args)); 279 } 280 } 281 logCompleteRun(TestPackageResult pkgResult)282 private void logCompleteRun(TestPackageResult pkgResult) { 283 if (pkgResult.getAppPackageName().equals(DeviceInfoCollector.APP_PACKAGE_NAME)) { 284 logResult("Device info collection complete"); 285 return; 286 } 287 logResult("%s package complete: Passed %d, Failed %d, Not Executed %d", 288 pkgResult.getAppPackageName(), pkgResult.countTests(CtsTestStatus.PASS), 289 pkgResult.countTests(CtsTestStatus.FAIL), 290 pkgResult.countTests(CtsTestStatus.NOT_EXECUTED)); 291 } 292 293 /** 294 * Creates a report file and populates it with the report data from the completed tests. 295 */ createXmlResult(File reportFile, String startTimestamp, long elapsedTime)296 private void createXmlResult(File reportFile, String startTimestamp, long elapsedTime) { 297 String endTime = getTimestamp(); 298 OutputStream stream = null; 299 try { 300 stream = createOutputResultStream(reportFile); 301 KXmlSerializer serializer = new KXmlSerializer(); 302 serializer.setOutput(stream, "UTF-8"); 303 serializer.startDocument("UTF-8", false); 304 serializer.setFeature( 305 "http://xmlpull.org/v1/doc/features.html#indent-output", true); 306 serializer.processingInstruction("xml-stylesheet type=\"text/xsl\" " + 307 "href=\"cts_result.xsl\""); 308 serializeResultsDoc(serializer, startTimestamp, endTime); 309 serializer.endDocument(); 310 String msg = String.format("XML test result file generated at %s. Passed %d, " + 311 "Failed %d, Not Executed %d", mReportDir.getName(), 312 mResults.countTests(CtsTestStatus.PASS), 313 mResults.countTests(CtsTestStatus.FAIL), 314 mResults.countTests(CtsTestStatus.NOT_EXECUTED)); 315 logResult(msg); 316 logResult("Time: %s", TimeUtil.formatElapsedTime(elapsedTime)); 317 } catch (IOException e) { 318 Log.e(LOG_TAG, "Failed to generate report data"); 319 } finally { 320 StreamUtil.closeStream(stream); 321 } 322 } 323 324 /** 325 * Output the results XML. 326 * 327 * @param serializer the {@link KXmlSerializer} to use 328 * @param startTime the user-friendly starting time of the test invocation 329 * @param endTime the user-friendly ending time of the test invocation 330 * @throws IOException 331 */ serializeResultsDoc(KXmlSerializer serializer, String startTime, String endTime)332 private void serializeResultsDoc(KXmlSerializer serializer, String startTime, String endTime) 333 throws IOException { 334 serializer.startTag(ns, RESULT_TAG); 335 serializer.attribute(ns, PLAN_ATTR, mPlanName); 336 serializer.attribute(ns, STARTTIME_ATTR, startTime); 337 serializer.attribute(ns, "endtime", endTime); 338 serializer.attribute(ns, "version", CTS_RESULT_FILE_VERSION); 339 340 mResults.serialize(serializer); 341 // TODO: not sure why, but the serializer doesn't like this statement 342 //serializer.endTag(ns, RESULT_TAG); 343 } 344 getResultFile(File reportDir)345 private File getResultFile(File reportDir) { 346 return new File(reportDir, TEST_RESULT_FILE_NAME); 347 } 348 349 /** 350 * Creates the output stream to use for test results. Exposed for mocking. 351 */ createOutputResultStream(File reportFile)352 OutputStream createOutputResultStream(File reportFile) throws IOException { 353 logResult("Created xml report file at file://%s", reportFile.getAbsolutePath()); 354 return new FileOutputStream(reportFile); 355 } 356 357 /** 358 * Copy the xml formatting files stored in this jar to the results directory 359 * 360 * @param resultsDir 361 */ copyFormattingFiles(File resultsDir)362 private void copyFormattingFiles(File resultsDir) { 363 for (String resultFileName : CTS_RESULT_RESOURCES) { 364 InputStream configStream = getClass().getResourceAsStream(String.format("/report/%s", 365 resultFileName)); 366 if (configStream != null) { 367 File resultFile = new File(resultsDir, resultFileName); 368 try { 369 FileUtil.writeToFile(configStream, resultFile); 370 } catch (IOException e) { 371 Log.w(LOG_TAG, String.format("Failed to write %s to file", resultFileName)); 372 } 373 } else { 374 Log.w(LOG_TAG, String.format("Failed to load %s from jar", resultFileName)); 375 } 376 } 377 } 378 379 /** 380 * Zip the contents of the given results directory. 381 * 382 * @param resultsDir 383 */ zipResults(File resultsDir)384 private void zipResults(File resultsDir) { 385 try { 386 // create a file in parent directory, with same name as resultsDir 387 File zipResultFile = new File(resultsDir.getParent(), String.format("%s.zip", 388 resultsDir.getName())); 389 FileUtil.createZip(resultsDir, zipResultFile); 390 } catch (IOException e) { 391 Log.w(LOG_TAG, String.format("Failed to create zip for %s", resultsDir.getName())); 392 } 393 } 394 395 /** 396 * Get a String version of the current time. 397 * <p/> 398 * Exposed so unit tests can mock. 399 */ getTimestamp()400 String getTimestamp() { 401 return TimeUtil.getTimestamp(); 402 } 403 404 /** 405 * {@inheritDoc} 406 */ 407 @Override testRunFailed(String errorMessage)408 public void testRunFailed(String errorMessage) { 409 // ignore 410 } 411 412 /** 413 * {@inheritDoc} 414 */ 415 @Override testRunStopped(long elapsedTime)416 public void testRunStopped(long elapsedTime) { 417 // ignore 418 } 419 420 /** 421 * {@inheritDoc} 422 */ 423 @Override invocationFailed(Throwable cause)424 public void invocationFailed(Throwable cause) { 425 // ignore 426 } 427 428 /** 429 * {@inheritDoc} 430 */ 431 @Override getSummary()432 public TestSummary getSummary() { 433 return null; 434 } 435 } 436