1# Copyright (C) 2010 Google Inc. All rights reserved. 2# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged 3# 4# Redistribution and use in source and binary forms, with or without 5# modification, are permitted provided that the following conditions are 6# met: 7# 8# * Redistributions of source code must retain the above copyright 9# notice, this list of conditions and the following disclaimer. 10# * Redistributions in binary form must reproduce the above 11# copyright notice, this list of conditions and the following disclaimer 12# in the documentation and/or other materials provided with the 13# distribution. 14# * Neither the name of Google Inc. nor the names of its 15# contributors may be used to endorse or promote products derived from 16# this software without specific prior written permission. 17# 18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30import logging 31import signal 32import time 33 34from webkitpy.layout_tests.models import test_expectations 35from webkitpy.layout_tests.models import test_failures 36 37 38_log = logging.getLogger(__name__) 39 40OK_EXIT_STATUS = 0 41 42# This matches what the shell does on POSIX. 43INTERRUPTED_EXIT_STATUS = signal.SIGINT + 128 44 45# POSIX limits status codes to 0-255. Normally run-webkit-tests returns the number 46# of tests that failed. These indicate exceptional conditions triggered by the 47# script itself, so we count backwards from 255 (aka -1) to enumerate them. 48# 49# FIXME: crbug.com/357866. We really shouldn't return the number of failures 50# in the exit code at all. 51EARLY_EXIT_STATUS = 251 52SYS_DEPS_EXIT_STATUS = 252 53NO_TESTS_EXIT_STATUS = 253 54NO_DEVICES_EXIT_STATUS = 254 55UNEXPECTED_ERROR_EXIT_STATUS = 255 56 57ERROR_CODES = ( 58 INTERRUPTED_EXIT_STATUS, 59 EARLY_EXIT_STATUS, 60 SYS_DEPS_EXIT_STATUS, 61 NO_TESTS_EXIT_STATUS, 62 NO_DEVICES_EXIT_STATUS, 63 UNEXPECTED_ERROR_EXIT_STATUS, 64) 65 66# In order to avoid colliding with the above codes, we put a ceiling on 67# the value returned by num_regressions 68MAX_FAILURES_EXIT_STATUS = 101 69 70class TestRunException(Exception): 71 def __init__(self, code, msg): 72 self.code = code 73 self.msg = msg 74 75 76class TestRunResults(object): 77 def __init__(self, expectations, num_tests): 78 self.total = num_tests 79 self.remaining = self.total 80 self.expectations = expectations 81 self.expected = 0 82 self.expected_failures = 0 83 self.unexpected = 0 84 self.unexpected_failures = 0 85 self.unexpected_crashes = 0 86 self.unexpected_timeouts = 0 87 self.tests_by_expectation = {} 88 self.tests_by_timeline = {} 89 self.results_by_name = {} # Map of test name to the last result for the test. 90 self.all_results = [] # All results from a run, including every iteration of every test. 91 self.unexpected_results_by_name = {} 92 self.failures_by_name = {} 93 self.total_failures = 0 94 self.expected_skips = 0 95 for expectation in test_expectations.TestExpectations.EXPECTATIONS.values(): 96 self.tests_by_expectation[expectation] = set() 97 for timeline in test_expectations.TestExpectations.TIMELINES.values(): 98 self.tests_by_timeline[timeline] = expectations.get_tests_with_timeline(timeline) 99 self.slow_tests = set() 100 self.interrupted = False 101 self.keyboard_interrupted = False 102 self.run_time = 0 # The wall clock time spent running the tests (layout_test_runner.run()). 103 104 def add(self, test_result, expected, test_is_slow): 105 result_type_for_stats = test_result.type 106 if test_expectations.WONTFIX in self.expectations.model().get_expectations(test_result.test_name): 107 result_type_for_stats = test_expectations.WONTFIX 108 self.tests_by_expectation[result_type_for_stats].add(test_result.test_name) 109 110 self.results_by_name[test_result.test_name] = test_result 111 if test_result.type != test_expectations.SKIP: 112 self.all_results.append(test_result) 113 self.remaining -= 1 114 if len(test_result.failures): 115 self.total_failures += 1 116 self.failures_by_name[test_result.test_name] = test_result.failures 117 if expected: 118 self.expected += 1 119 if test_result.type == test_expectations.SKIP: 120 self.expected_skips += 1 121 elif test_result.type != test_expectations.PASS: 122 self.expected_failures += 1 123 else: 124 self.unexpected_results_by_name[test_result.test_name] = test_result 125 self.unexpected += 1 126 if len(test_result.failures): 127 self.unexpected_failures += 1 128 if test_result.type == test_expectations.CRASH: 129 self.unexpected_crashes += 1 130 elif test_result.type == test_expectations.TIMEOUT: 131 self.unexpected_timeouts += 1 132 if test_is_slow: 133 self.slow_tests.add(test_result.test_name) 134 135 136class RunDetails(object): 137 def __init__(self, exit_code, summarized_full_results=None, summarized_failing_results=None, initial_results=None, retry_results=None, enabled_pixel_tests_in_retry=False): 138 self.exit_code = exit_code 139 self.summarized_full_results = summarized_full_results 140 self.summarized_failing_results = summarized_failing_results 141 self.initial_results = initial_results 142 self.retry_results = retry_results 143 self.enabled_pixel_tests_in_retry = enabled_pixel_tests_in_retry 144 145 146def _interpret_test_failures(failures): 147 test_dict = {} 148 failure_types = [type(failure) for failure in failures] 149 # FIXME: get rid of all this is_* values once there is a 1:1 map between 150 # TestFailure type and test_expectations.EXPECTATION. 151 if test_failures.FailureMissingAudio in failure_types: 152 test_dict['is_missing_audio'] = True 153 154 if test_failures.FailureMissingResult in failure_types: 155 test_dict['is_missing_text'] = True 156 157 if test_failures.FailureMissingImage in failure_types or test_failures.FailureMissingImageHash in failure_types: 158 test_dict['is_missing_image'] = True 159 160 if test_failures.FailureTestHarnessAssertion in failure_types: 161 test_dict['is_testharness_test'] = True 162 163 return test_dict 164 165 166def summarize_results(port_obj, expectations, initial_results, retry_results, enabled_pixel_tests_in_retry, only_include_failing=False): 167 """Returns a dictionary containing a summary of the test runs, with the following fields: 168 'version': a version indicator 169 'fixable': The number of fixable tests (NOW - PASS) 170 'skipped': The number of skipped tests (NOW & SKIPPED) 171 'num_regressions': The number of non-flaky failures 172 'num_flaky': The number of flaky failures 173 'num_passes': The number of unexpected passes 174 'tests': a dict of tests -> {'expected': '...', 'actual': '...'} 175 """ 176 results = {} 177 results['version'] = 3 178 179 tbe = initial_results.tests_by_expectation 180 tbt = initial_results.tests_by_timeline 181 results['fixable'] = len(tbt[test_expectations.NOW] - tbe[test_expectations.PASS]) 182 # FIXME: Remove this. It is redundant with results['num_failures_by_type']. 183 results['skipped'] = len(tbt[test_expectations.NOW] & tbe[test_expectations.SKIP]) 184 185 num_passes = 0 186 num_flaky = 0 187 num_regressions = 0 188 keywords = {} 189 for expecation_string, expectation_enum in test_expectations.TestExpectations.EXPECTATIONS.iteritems(): 190 keywords[expectation_enum] = expecation_string.upper() 191 192 num_failures_by_type = {} 193 for expectation in initial_results.tests_by_expectation: 194 tests = initial_results.tests_by_expectation[expectation] 195 if expectation != test_expectations.WONTFIX: 196 tests &= tbt[test_expectations.NOW] 197 num_failures_by_type[keywords[expectation]] = len(tests) 198 # The number of failures by type. 199 results['num_failures_by_type'] = num_failures_by_type 200 201 tests = {} 202 203 for test_name, result in initial_results.results_by_name.iteritems(): 204 expected = expectations.get_expectations_string(test_name) 205 result_type = result.type 206 actual = [keywords[result_type]] 207 208 if only_include_failing and result.type == test_expectations.SKIP: 209 continue 210 211 if result_type == test_expectations.PASS: 212 num_passes += 1 213 if not result.has_stderr and only_include_failing: 214 continue 215 elif result_type != test_expectations.SKIP and test_name in initial_results.unexpected_results_by_name: 216 if retry_results: 217 if test_name not in retry_results.unexpected_results_by_name: 218 # The test failed unexpectedly at first, but ran as expected the second time -> flaky. 219 actual.extend(expectations.get_expectations_string(test_name).split(" ")) 220 num_flaky += 1 221 else: 222 retry_result_type = retry_results.unexpected_results_by_name[test_name].type 223 if retry_result_type == test_expectations.PASS: 224 # The test failed unexpectedly at first, then passed unexpectedly -> unexpected pass. 225 num_passes += 1 226 if not result.has_stderr and only_include_failing: 227 continue 228 else: 229 # The test failed unexpectedly both times -> regression. 230 num_regressions += 1 231 if not keywords[retry_result_type] in actual: 232 actual.append(keywords[retry_result_type]) 233 else: 234 # The test failed unexpectedly, but we didn't do any retries -> regression. 235 num_regressions += 1 236 237 test_dict = {} 238 239 rounded_run_time = round(result.test_run_time, 1) 240 if rounded_run_time: 241 test_dict['time'] = rounded_run_time 242 243 if result.has_stderr: 244 test_dict['has_stderr'] = True 245 246 bugs = expectations.model().get_expectation_line(test_name).bugs 247 if bugs: 248 test_dict['bugs'] = bugs 249 250 if result.reftest_type: 251 test_dict.update(reftest_type=list(result.reftest_type)) 252 253 test_dict['expected'] = expected 254 test_dict['actual'] = " ".join(actual) 255 256 def is_expected(actual_result): 257 return expectations.matches_an_expected_result(test_name, result_type, 258 port_obj.get_option('pixel_tests') or result.reftest_type, 259 port_obj.get_option('enable_sanitizer')) 260 261 # To avoid bloating the output results json too much, only add an entry for whether the failure is unexpected. 262 if not all(is_expected(actual_result) for actual_result in actual): 263 test_dict['is_unexpected'] = True 264 265 test_dict.update(_interpret_test_failures(result.failures)) 266 267 if retry_results: 268 retry_result = retry_results.unexpected_results_by_name.get(test_name) 269 if retry_result: 270 test_dict.update(_interpret_test_failures(retry_result.failures)) 271 272 if (result.has_repaint_overlay): 273 test_dict['has_repaint_overlay'] = True 274 275 # Store test hierarchically by directory. e.g. 276 # foo/bar/baz.html: test_dict 277 # foo/bar/baz1.html: test_dict 278 # 279 # becomes 280 # foo: { 281 # bar: { 282 # baz.html: test_dict, 283 # baz1.html: test_dict 284 # } 285 # } 286 parts = test_name.split('/') 287 current_map = tests 288 for i, part in enumerate(parts): 289 if i == (len(parts) - 1): 290 current_map[part] = test_dict 291 break 292 if part not in current_map: 293 current_map[part] = {} 294 current_map = current_map[part] 295 296 results['tests'] = tests 297 # FIXME: Remove this. It is redundant with results['num_failures_by_type']. 298 results['num_passes'] = num_passes 299 results['num_flaky'] = num_flaky 300 # FIXME: Remove this. It is redundant with results['num_failures_by_type']. 301 results['num_regressions'] = num_regressions 302 results['interrupted'] = initial_results.interrupted # Does results.html have enough information to compute this itself? (by checking total number of results vs. total number of tests?) 303 results['layout_tests_dir'] = port_obj.layout_tests_dir() 304 results['has_wdiff'] = port_obj.wdiff_available() 305 results['has_pretty_patch'] = port_obj.pretty_patch_available() 306 results['pixel_tests_enabled'] = port_obj.get_option('pixel_tests') 307 results['seconds_since_epoch'] = int(time.time()) 308 results['build_number'] = port_obj.get_option('build_number') 309 results['builder_name'] = port_obj.get_option('builder_name') 310 311 # Don't do this by default since it takes >100ms. 312 # It's only used for uploading data to the flakiness dashboard. 313 results['chromium_revision'] = '' 314 results['blink_revision'] = '' 315 if port_obj.get_option('builder_name'): 316 for (name, path) in port_obj.repository_paths(): 317 scm = port_obj.host.scm_for_path(path) 318 if scm: 319 rev = scm.svn_revision(path) 320 if rev: 321 results[name.lower() + '_revision'] = rev 322 else: 323 _log.warn('Failed to determine svn revision for %s, ' 324 'leaving "%s_revision" key blank in full_results.json.' 325 % (path, name)) 326 327 return results 328