1#!/usr/bin/env python3.4 2# 3# Copyright 2016 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17 18"""This module is where all the record definitions and record containers live. 19""" 20 21import json 22import pprint 23 24from acts.signals import TestSignal 25from acts.utils import epoch_to_human_time 26from acts.utils import get_current_epoch_time 27 28class TestResultEnums(object): 29 """Enums used for TestResultRecord class. 30 31 Includes the tokens to mark test result with, and the string names for each 32 field in TestResultRecord. 33 """ 34 35 RECORD_NAME = "Test Name" 36 RECORD_CLASS = "Test Class" 37 RECORD_BEGIN_TIME = "Begin Time" 38 RECORD_END_TIME = "End Time" 39 RECORD_RESULT = "Result" 40 RECORD_UID = "UID" 41 RECORD_EXTRAS = "Extras" 42 RECORD_EXTRA_ERRORS = "Extra Errors" 43 RECORD_DETAILS = "Details" 44 TEST_RESULT_PASS = "PASS" 45 TEST_RESULT_FAIL = "FAIL" 46 TEST_RESULT_SKIP = "SKIP" 47 TEST_RESULT_UNKNOWN = "UNKNOWN" 48 49class TestResultRecord(object): 50 """A record that holds the information of a test case execution. 51 52 Attributes: 53 test_name: A string representing the name of the test case. 54 begin_time: Epoch timestamp of when the test case started. 55 end_time: Epoch timestamp of when the test case ended. 56 self.uid: Unique identifier of a test case. 57 self.result: Test result, PASS/FAIL/SKIP. 58 self.extras: User defined extra information of the test result. 59 self.details: A string explaining the details of the test case. 60 """ 61 62 def __init__(self, t_name, t_class=None): 63 self.test_name = t_name 64 self.test_class = t_class 65 self.begin_time = None 66 self.end_time = None 67 self.uid = None 68 self.result = None 69 self.extras = None 70 self.details = None 71 self.extra_errors = {} 72 73 def test_begin(self): 74 """Call this when the test case it records begins execution. 75 76 Sets the begin_time of this record. 77 """ 78 self.begin_time = get_current_epoch_time() 79 80 def _test_end(self, result, e): 81 """Class internal function to signal the end of a test case execution. 82 83 Args: 84 result: One of the TEST_RESULT enums in TestResultEnums. 85 e: A test termination signal (usually an exception object). It can 86 be any exception instance or of any subclass of 87 base_test._TestSignal. 88 """ 89 self.end_time = get_current_epoch_time() 90 self.result = result 91 if isinstance(e, TestSignal): 92 self.details = e.details 93 self.extras = e.extras 94 elif e: 95 self.details = str(e) 96 97 def test_pass(self, e=None): 98 """To mark the test as passed in this record. 99 100 Args: 101 e: An instance of acts.signals.TestPass. 102 """ 103 self._test_end(TestResultEnums.TEST_RESULT_PASS, e) 104 105 def test_fail(self, e=None): 106 """To mark the test as failed in this record. 107 108 Only test_fail does instance check because we want "assert xxx" to also 109 fail the test same way assert_true does. 110 111 Args: 112 e: An exception object. It can be an instance of AssertionError or 113 acts.base_test.TestFailure. 114 """ 115 self._test_end(TestResultEnums.TEST_RESULT_FAIL, e) 116 117 def test_skip(self, e=None): 118 """To mark the test as skipped in this record. 119 120 Args: 121 e: An instance of acts.signals.TestSkip. 122 """ 123 self._test_end(TestResultEnums.TEST_RESULT_SKIP, e) 124 125 def test_unknown(self, e=None): 126 """To mark the test as unknown in this record. 127 128 Args: 129 e: An exception object. 130 """ 131 self._test_end(TestResultEnums.TEST_RESULT_UNKNOWN, e) 132 133 def add_error(self, tag, e): 134 """Add extra error happened during a test mark the test result as 135 UNKNOWN. 136 137 If an error is added the test record, the record's result is equivalent 138 to the case where an uncaught exception happened. 139 140 Args: 141 tag: A string describing where this error came from, e.g. 'on_pass'. 142 e: An exception object. 143 """ 144 self.result = TestResultEnums.TEST_RESULT_UNKNOWN 145 self.extra_errors[tag] = str(e) 146 147 def __str__(self): 148 d = self.to_dict() 149 l = ["%s = %s" % (k, v) for k, v in d.items()] 150 s = ', '.join(l) 151 return s 152 153 def __repr__(self): 154 """This returns a short string representation of the test record.""" 155 t = epoch_to_human_time(self.begin_time) 156 return "%s %s %s" % (t, self.test_name, self.result) 157 158 def to_dict(self): 159 """Gets a dictionary representating the content of this class. 160 161 Returns: 162 A dictionary representating the content of this class. 163 """ 164 d = {} 165 d[TestResultEnums.RECORD_NAME] = self.test_name 166 d[TestResultEnums.RECORD_CLASS] = self.test_class 167 d[TestResultEnums.RECORD_BEGIN_TIME] = self.begin_time 168 d[TestResultEnums.RECORD_END_TIME] = self.end_time 169 d[TestResultEnums.RECORD_RESULT] = self.result 170 d[TestResultEnums.RECORD_UID] = self.uid 171 d[TestResultEnums.RECORD_EXTRAS] = self.extras 172 d[TestResultEnums.RECORD_DETAILS] = self.details 173 d[TestResultEnums.RECORD_EXTRA_ERRORS] = self.extra_errors 174 return d 175 176 def json_str(self): 177 """Converts this test record to a string in json format. 178 179 Format of the json string is: 180 { 181 'Test Name': <test name>, 182 'Begin Time': <epoch timestamp>, 183 'Details': <details>, 184 ... 185 } 186 187 Returns: 188 A json-format string representing the test record. 189 """ 190 return json.dumps(self.to_dict()) 191 192class TestResult(object): 193 """A class that contains metrics of a test run. 194 195 This class is essentially a container of TestResultRecord objects. 196 197 Attributes: 198 self.requested: A list of strings, each is the name of a test requested 199 by user. 200 self.failed: A list of records for tests failed. 201 self.executed: A list of records for tests that were actually executed. 202 self.passed: A list of records for tests passed. 203 self.skipped: A list of records for tests skipped. 204 self.unknown: A list of records for tests with unknown result token. 205 """ 206 207 def __init__(self): 208 self.requested = [] 209 self.failed = [] 210 self.executed = [] 211 self.passed = [] 212 self.skipped = [] 213 self.unknown = [] 214 215 def __add__(self, r): 216 """Overrides '+' operator for TestResult class. 217 218 The add operator merges two TestResult objects by concatenating all of 219 their lists together. 220 221 Args: 222 r: another instance of TestResult to be added 223 224 Returns: 225 A TestResult instance that's the sum of two TestResult instances. 226 """ 227 assert isinstance(r, TestResult) 228 sum_result = TestResult() 229 for name in sum_result.__dict__: 230 l_value = list(getattr(self, name)) 231 r_value = list(getattr(r, name)) 232 setattr(sum_result, name, l_value + r_value) 233 return sum_result 234 235 def add_record(self, record): 236 """Adds a test record to test result. 237 238 A record is considered executed once it's added to the test result. 239 240 Args: 241 record: A test record object to add. 242 """ 243 self.executed.append(record) 244 if record.result == TestResultEnums.TEST_RESULT_FAIL: 245 self.failed.append(record) 246 elif record.result == TestResultEnums.TEST_RESULT_SKIP: 247 self.skipped.append(record) 248 elif record.result == TestResultEnums.TEST_RESULT_PASS: 249 self.passed.append(record) 250 else: 251 self.unknown.append(record) 252 253 def fail_class(self, class_name, e): 254 """Add a record to indicate a test class setup has failed and no test 255 in the class was executed. 256 257 Args: 258 class_name: A string that is the name of the failed test class. 259 e: An exception object. 260 """ 261 record = TestResultRecord("", class_name) 262 record.test_begin() 263 if isinstance(e, TestSignal): 264 new_e = type(e)("setup_class failed for %s: %s" % ( 265 class_name, e.details), e.extras) 266 else: 267 new_e = type(e)("setup_class failed for %s: %s" % ( 268 class_name, str(e))) 269 record.test_fail(new_e) 270 self.executed.append(record) 271 self.failed.append(record) 272 273 def json_str(self): 274 """Converts this test result to a string in json format. 275 276 Format of the json string is: 277 { 278 "Results": [ 279 {<executed test record 1>}, 280 {<executed test record 2>}, 281 ... 282 ], 283 "Summary": <summary dict> 284 } 285 286 Returns: 287 A json-format string representing the test results. 288 """ 289 d = {} 290 executed = [record.to_dict() for record in self.executed] 291 d["Results"] = executed 292 d["Summary"] = self.summary_dict() 293 json_str = json.dumps(d, indent=4, sort_keys=True) 294 return json_str 295 296 def summary_str(self): 297 """Gets a string that summarizes the stats of this test result. 298 299 The summary rovides the counts of how many test cases fall into each 300 category, like "Passed", "Failed" etc. 301 302 Format of the string is: 303 Requested <int>, Executed <int>, ... 304 305 Returns: 306 A summary string of this test result. 307 """ 308 l = ["%s %d" % (k, v) for k, v in self.summary_dict().items()] 309 # Sort the list so the order is the same every time. 310 msg = ", ".join(sorted(l)) 311 return msg 312 313 def summary_dict(self): 314 """Gets a dictionary that summarizes the stats of this test result. 315 316 The summary rovides the counts of how many test cases fall into each 317 category, like "Passed", "Failed" etc. 318 319 Returns: 320 A dictionary with the stats of this test result. 321 """ 322 d = {} 323 d["Requested"] = len(self.requested) 324 d["Executed"] = len(self.executed) 325 d["Passed"] = len(self.passed) 326 d["Failed"] = len(self.failed) 327 d["Skipped"] = len(self.skipped) 328 d["Unknown"] = len(self.unknown) 329 return d 330