1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.result; 17 18 import com.android.ddmlib.testrunner.TestResult.TestStatus; 19 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 20 21 import com.google.common.base.Joiner; 22 23 import java.util.ArrayList; 24 import java.util.Arrays; 25 import java.util.HashMap; 26 import java.util.LinkedHashMap; 27 import java.util.List; 28 import java.util.Map; 29 30 /** Container for a result of a single test. */ 31 public class TestResult { 32 33 private TestStatus mStatus; 34 private String mStackTrace; 35 private Map<String, String> mMetrics; 36 private HashMap<String, Metric> mProtoMetrics; 37 private Map<String, LogFile> mLoggedFiles; 38 // the start and end time of the test, measured via {@link System#currentTimeMillis()} 39 private long mStartTime = 0; 40 private long mEndTime = 0; 41 TestResult()42 public TestResult() { 43 mStatus = TestStatus.INCOMPLETE; 44 mStartTime = System.currentTimeMillis(); 45 mLoggedFiles = new LinkedHashMap<String, LogFile>(); 46 mMetrics = new HashMap<>(); 47 mProtoMetrics = new HashMap<>(); 48 } 49 50 /** Get the {@link TestStatus} result of the test. */ getStatus()51 public TestStatus getStatus() { 52 return mStatus; 53 } 54 55 /** 56 * Get the associated {@link String} stack trace. Should be <code>null</code> if {@link 57 * #getStatus()} is {@link TestStatus#PASSED}. 58 */ getStackTrace()59 public String getStackTrace() { 60 return mStackTrace; 61 } 62 63 /** Get the associated test metrics. */ getMetrics()64 public Map<String, String> getMetrics() { 65 return mMetrics; 66 } 67 68 /** Get the associated test metrics in proto format. */ getProtoMetrics()69 public HashMap<String, Metric> getProtoMetrics() { 70 return mProtoMetrics; 71 } 72 73 /** Set the test metrics, overriding any previous values. */ setMetrics(Map<String, String> metrics)74 public void setMetrics(Map<String, String> metrics) { 75 mMetrics = metrics; 76 } 77 78 /** Set the test proto metrics format, overriding any previous values. */ setProtoMetrics(HashMap<String, Metric> metrics)79 public void setProtoMetrics(HashMap<String, Metric> metrics) { 80 mProtoMetrics = metrics; 81 } 82 83 /** Add a logged file tracking associated with that test case */ addLoggedFile(String dataName, LogFile loggedFile)84 public void addLoggedFile(String dataName, LogFile loggedFile) { 85 mLoggedFiles.put(dataName, loggedFile); 86 } 87 88 /** Returns a copy of the map containing all the logged file associated with that test case. */ getLoggedFiles()89 public Map<String, LogFile> getLoggedFiles() { 90 return new LinkedHashMap<>(mLoggedFiles); 91 } 92 93 /** 94 * Return the {@link System#currentTimeMillis()} time that the {@link 95 * ITestInvocationListener#testStarted(TestDescription)} event was received. 96 */ getStartTime()97 public long getStartTime() { 98 return mStartTime; 99 } 100 101 /** 102 * Allows to set the time when the test was started, to be used with {@link 103 * ITestInvocationListener#testStarted(TestDescription, long)}. 104 */ setStartTime(long startTime)105 public void setStartTime(long startTime) { 106 mStartTime = startTime; 107 } 108 109 /** 110 * Return the {@link System#currentTimeMillis()} time that the {@link 111 * ITestInvocationListener#testEnded(TestDescription, Map)} event was received. 112 */ getEndTime()113 public long getEndTime() { 114 return mEndTime; 115 } 116 117 /** Set the {@link TestStatus}. */ setStatus(TestStatus status)118 public TestResult setStatus(TestStatus status) { 119 mStatus = status; 120 return this; 121 } 122 123 /** Set the stack trace. */ setStackTrace(String trace)124 public void setStackTrace(String trace) { 125 mStackTrace = trace; 126 } 127 128 /** Sets the end time */ setEndTime(long currentTimeMillis)129 public void setEndTime(long currentTimeMillis) { 130 mEndTime = currentTimeMillis; 131 } 132 133 @Override hashCode()134 public int hashCode() { 135 return Arrays.hashCode(new Object[] {mMetrics, mStackTrace, mStatus}); 136 } 137 138 @Override equals(Object obj)139 public boolean equals(Object obj) { 140 if (this == obj) { 141 return true; 142 } 143 if (obj == null) { 144 return false; 145 } 146 if (getClass() != obj.getClass()) { 147 return false; 148 } 149 TestResult other = (TestResult) obj; 150 return equal(mMetrics, other.mMetrics) 151 && equal(mStackTrace, other.mStackTrace) 152 && equal(mStatus, other.mStatus); 153 } 154 equal(Object a, Object b)155 private static boolean equal(Object a, Object b) { 156 return a == b || (a != null && a.equals(b)); 157 } 158 159 /** 160 * Merge the attempts for a same test case based on the merging strategy. 161 * 162 * @param results List of {@link TestResult} that will be merged 163 * @param strategy the {@link MergeStrategy} to be used to determine the merging outcome. 164 * @return the merged {@link TestResult} or null if there is nothing to merge. 165 */ merge(List<TestResult> results, MergeStrategy strategy)166 public static TestResult merge(List<TestResult> results, MergeStrategy strategy) { 167 if (results.isEmpty()) { 168 return null; 169 } 170 if (MergeStrategy.NO_MERGE.equals(strategy)) { 171 throw new IllegalArgumentException( 172 "TestResult#merge cannot be called with NO_MERGE strategy."); 173 } 174 TestResult mergedResult = new TestResult(); 175 176 long earliestStartTime = Long.MAX_VALUE; 177 long latestEndTime = Long.MIN_VALUE; 178 179 List<String> errorMsg = new ArrayList<>(); 180 int pass = 0; 181 int fail = 0; 182 int assumption_failure = 0; 183 int ignored = 0; 184 int incomplete = 0; 185 186 for (TestResult attempt : results) { 187 mergedResult.mProtoMetrics.putAll(attempt.getProtoMetrics()); 188 mergedResult.mMetrics.putAll(attempt.getMetrics()); 189 mergedResult.mLoggedFiles.putAll(attempt.getLoggedFiles()); 190 earliestStartTime = Math.min(attempt.getStartTime(), earliestStartTime); 191 latestEndTime = Math.max(attempt.getEndTime(), latestEndTime); 192 switch (attempt.getStatus()) { 193 case PASSED: 194 pass++; 195 break; 196 case FAILURE: 197 fail++; 198 if (attempt.getStackTrace() != null) { 199 errorMsg.add(attempt.getStackTrace()); 200 } 201 break; 202 case INCOMPLETE: 203 incomplete++; 204 errorMsg.add("incomplete test case result."); 205 break; 206 case ASSUMPTION_FAILURE: 207 assumption_failure++; 208 if (attempt.getStackTrace() != null) { 209 errorMsg.add(attempt.getStackTrace()); 210 } 211 break; 212 case IGNORED: 213 ignored++; 214 break; 215 } 216 } 217 218 switch (strategy) { 219 case ANY_PASS_IS_PASS: 220 case ONE_TESTCASE_PASS_IS_PASS: 221 // We prioritize passing the test due to the merging strategy. 222 if (pass > 0) { 223 mergedResult.setStatus(TestStatus.PASSED); 224 } else if (fail == 0) { 225 if (ignored > 0) { 226 mergedResult.setStatus(TestStatus.IGNORED); 227 } else if (assumption_failure > 0) { 228 mergedResult.setStatus(TestStatus.ASSUMPTION_FAILURE); 229 } else if (incomplete > 0) { 230 mergedResult.setStatus(TestStatus.INCOMPLETE); 231 } 232 } else { 233 mergedResult.setStatus(TestStatus.FAILURE); 234 } 235 break; 236 default: 237 // We keep a sane default of one failure is a failure that should be reported. 238 if (fail > 0) { 239 mergedResult.setStatus(TestStatus.FAILURE); 240 } else { 241 if (ignored > 0) { 242 mergedResult.setStatus(TestStatus.IGNORED); 243 } else if (assumption_failure > 0) { 244 mergedResult.setStatus(TestStatus.ASSUMPTION_FAILURE); 245 } else if (incomplete > 0) { 246 mergedResult.setStatus(TestStatus.INCOMPLETE); 247 } else { 248 mergedResult.setStatus(TestStatus.PASSED); 249 } 250 } 251 break; 252 } 253 if (errorMsg.isEmpty()) { 254 mergedResult.mStackTrace = null; 255 } else { 256 mergedResult.mStackTrace = Joiner.on("\n\n").join(errorMsg); 257 } 258 mergedResult.setStartTime(earliestStartTime); 259 mergedResult.setEndTime(latestEndTime); 260 return mergedResult; 261 } 262 } 263