1 /* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.result; 17 18 import com.android.tradefed.config.Option; 19 import com.android.tradefed.config.OptionClass; 20 import com.android.tradefed.invoker.IInvocationContext; 21 22 import java.io.PrintStream; 23 import java.text.SimpleDateFormat; 24 import java.util.ArrayList; 25 import java.util.Collections; 26 import java.util.Date; 27 import java.util.LinkedHashMap; 28 import java.util.LinkedHashSet; 29 import java.util.List; 30 import java.util.Map; 31 import java.util.Map.Entry; 32 import java.util.Set; 33 34 /** 35 * Result reporter to print the test results to the console. 36 * 37 * <p>Prints each test run, each test case, and test metrics, test logs, and test file locations. 38 * 39 * <p> 40 */ 41 @OptionClass(alias = "console-result-reporter") 42 public class ConsoleResultReporter extends TestResultListener 43 implements ILogSaverListener, ITestInvocationListener { 44 45 private static final SimpleDateFormat sTimeStampFormat = new SimpleDateFormat("HH:mm:ss"); 46 47 @Option( 48 name = "suppress-passed-tests", 49 description = 50 "For functional tests, ommit summary for " 51 + "passing tests, only print failed and ignored ones") 52 private boolean mSuppressPassedTest = false; 53 54 @Option( 55 name = "display-failure-summary", 56 description = "Display all the failures at the very end for easier visualization.") 57 private boolean mDisplayFailureSummary = true; 58 59 @Option( 60 name = "display-invocation-attributes", 61 description = 62 "Display all the invocation attributes at the very end for easier" 63 + " visualization.") 64 private boolean mDisplayInvocationAttributes = false; 65 66 private final PrintStream mStream; 67 private Set<LogFile> mLoggedFiles = new LinkedHashSet<>(); 68 private Map<TestDescription, TestResult> mFailures = new LinkedHashMap<>(); 69 private String mTestTag; 70 private String mRunInProgress; 71 private CountingTestResultListener mResultCountListener = new CountingTestResultListener(); 72 private IInvocationContext mContext; 73 ConsoleResultReporter()74 public ConsoleResultReporter() { 75 this(System.out); 76 } 77 ConsoleResultReporter(PrintStream outputStream)78 ConsoleResultReporter(PrintStream outputStream) { 79 mStream = outputStream; 80 } 81 82 @Override invocationStarted(IInvocationContext context)83 public void invocationStarted(IInvocationContext context) { 84 mTestTag = context.getTestTag(); 85 mContext = context; 86 } 87 88 @Override testResult(TestDescription test, TestResult result)89 public void testResult(TestDescription test, TestResult result) { 90 mResultCountListener.testResult(test, result); 91 if (mSuppressPassedTest && TestStatus.PASSED.equals(result.getResultStatus())) { 92 return; 93 } 94 if (mDisplayFailureSummary && TestStatus.FAILURE.equals(result.getResultStatus())) { 95 mFailures.put(test, result); 96 } 97 print(getTestSummary(mTestTag, test, result)); 98 } 99 100 @Override testRunStarted(String runName, int testCount)101 public void testRunStarted(String runName, int testCount) { 102 super.testRunStarted(runName, testCount); 103 mRunInProgress = runName; 104 } 105 106 @Override testRunFailed(String errorMessage)107 public void testRunFailed(String errorMessage) { 108 print(String.format("%s: run failed: %s\n", mRunInProgress, errorMessage)); 109 } 110 111 @Override testRunFailed(FailureDescription failure)112 public void testRunFailed(FailureDescription failure) { 113 print(String.format("%s: run failed: %s\n", mRunInProgress, failure)); 114 } 115 116 @Override testRunEnded(long elapsedTimeMillis, Map<String, String> metrics)117 public void testRunEnded(long elapsedTimeMillis, Map<String, String> metrics) { 118 super.testRunEnded(elapsedTimeMillis, metrics); 119 if (metrics != null && !metrics.isEmpty()) { 120 String tag = mTestTag != null ? mTestTag : "unknown"; 121 String runName = mRunInProgress != null ? mRunInProgress : "unknown"; 122 StringBuilder sb = new StringBuilder(tag); 123 sb.append(": "); 124 sb.append(runName); 125 sb.append(": "); 126 List<String> metricKeys = new ArrayList<String>(metrics.keySet()); 127 Collections.sort(metricKeys); 128 for (String metricKey : metricKeys) { 129 sb.append(String.format("%s=%s\n", metricKey, metrics.get(metricKey))); 130 } 131 print(sb.toString()); 132 } 133 mRunInProgress = null; 134 } 135 136 /** {@inheritDoc} */ 137 @Override invocationEnded(long elapsedTime)138 public void invocationEnded(long elapsedTime) { 139 int[] results = mResultCountListener.getResultCounts(); 140 StringBuilder sb = new StringBuilder(); 141 sb.append("========== Result Summary =========="); 142 sb.append(String.format("\nResults summary for test-tag '%s': ", mTestTag)); 143 sb.append(mResultCountListener.getTotalTests()); 144 sb.append(" Tests ["); 145 sb.append(results[TestStatus.PASSED.ordinal()]); 146 sb.append(" Passed"); 147 if (results[TestStatus.FAILURE.ordinal()] > 0) { 148 sb.append(" "); 149 sb.append(results[TestStatus.FAILURE.ordinal()]); 150 sb.append(" Failed"); 151 } 152 if (results[TestStatus.IGNORED.ordinal()] > 0) { 153 sb.append(" "); 154 sb.append(results[TestStatus.IGNORED.ordinal()]); 155 sb.append(" Ignored"); 156 } 157 if (results[TestStatus.SKIPPED.ordinal()] > 0) { 158 sb.append(" "); 159 sb.append(results[TestStatus.SKIPPED.ordinal()]); 160 sb.append(" Skipped"); 161 } 162 if (results[TestStatus.ASSUMPTION_FAILURE.ordinal()] > 0) { 163 sb.append(" "); 164 sb.append(results[TestStatus.ASSUMPTION_FAILURE.ordinal()]); 165 sb.append(" Assumption failures"); 166 } 167 if (results[TestStatus.INCOMPLETE.ordinal()] > 0) { 168 sb.append(" "); 169 sb.append(results[TestStatus.INCOMPLETE.ordinal()]); 170 sb.append(" Incomplete"); 171 } 172 sb.append("] \r\n"); 173 print(sb.toString()); 174 if (mDisplayInvocationAttributes && !mContext.getAttributes().isEmpty()) { 175 StringBuilder metricPrint = new StringBuilder(); 176 metricPrint.append(" Metrics:\n"); 177 for (String key : mContext.getAttributes().keySet()) { 178 metricPrint.append( 179 " " + key + "=" + mContext.getAttributes().get(key).toString() + "\n"); 180 } 181 print(metricPrint.toString()); 182 } 183 if (mDisplayFailureSummary) { 184 for (Entry<TestDescription, TestResult> entry : mFailures.entrySet()) { 185 print(getTestSummary(mTestTag, entry.getKey(), entry.getValue())); 186 } 187 } 188 // Print the logs 189 for (LogFile logFile : mLoggedFiles) { 190 printLog(logFile); 191 } 192 } 193 194 /** {@inheritDoc} */ 195 @Override logAssociation(String dataName, LogFile logFile)196 public void logAssociation(String dataName, LogFile logFile) { 197 mLoggedFiles.add(logFile); 198 } 199 200 /** {@inheritDoc} */ 201 @Override testLogSaved( String dataName, LogDataType dataType, InputStreamSource dataStream, LogFile logFile)202 public void testLogSaved( 203 String dataName, LogDataType dataType, InputStreamSource dataStream, LogFile logFile) { 204 mLoggedFiles.add(logFile); 205 } 206 printLog(LogFile logFile)207 private void printLog(LogFile logFile) { 208 if (mSuppressPassedTest && !mResultCountListener.hasFailedTests()) { 209 // all tests passed, skip logging 210 return; 211 } 212 String logDesc = logFile.getUrl() == null ? logFile.getPath() : logFile.getUrl(); 213 print("Log: " + logDesc + "\r\n"); 214 } 215 216 /** Get the test summary as string including test metrics. */ getTestSummary(String testTag, TestDescription testId, TestResult testResult)217 static String getTestSummary(String testTag, TestDescription testId, TestResult testResult) { 218 StringBuilder sb = new StringBuilder(); 219 sb.append( 220 String.format( 221 "%s: %s: %s (%dms)\n", 222 testTag, 223 testId.toString(), 224 testResult.getStatus(), 225 testResult.getEndTime() - testResult.getStartTime())); 226 String stack = testResult.getStackTrace(); 227 if (stack != null && !stack.isEmpty()) { 228 sb.append(" stack=\n"); 229 String lines[] = stack.split("\\r?\\n"); 230 for (String line : lines) { 231 sb.append(String.format(" %s\n", line)); 232 } 233 } 234 Map<String, String> metrics = testResult.getMetrics(); 235 if (metrics != null && !metrics.isEmpty()) { 236 List<String> metricKeys = new ArrayList<String>(metrics.keySet()); 237 Collections.sort(metricKeys); 238 for (String metricKey : metricKeys) { 239 sb.append(String.format(" %s: %s\n", metricKey, metrics.get(metricKey))); 240 } 241 } 242 243 return sb.toString(); 244 } 245 print(String msg)246 private void print(String msg) { 247 mStream.print(sTimeStampFormat.format(new Date()) + " " + msg); 248 } 249 } 250