1# Copyright (C) 2010 Google Inc. All rights reserved. 2# 3# Redistribution and use in source and binary forms, with or without 4# modification, are permitted provided that the following conditions are 5# met: 6# 7# * Redistributions of source code must retain the above copyright 8# notice, this list of conditions and the following disclaimer. 9# * Redistributions in binary form must reproduce the above 10# copyright notice, this list of conditions and the following disclaimer 11# in the documentation and/or other materials provided with the 12# distribution. 13# * Neither the name of Google Inc. nor the names of its 14# contributors may be used to endorse or promote products derived from 15# this software without specific prior written permission. 16# 17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29import logging 30 31from webkitpy.layout_tests.layout_package import json_results_generator 32from webkitpy.layout_tests.layout_package import test_expectations 33from webkitpy.layout_tests.layout_package import test_failures 34import webkitpy.thirdparty.simplejson as simplejson 35 36 37class JSONLayoutResultsGenerator(json_results_generator.JSONResultsGeneratorBase): 38 """A JSON results generator for layout tests.""" 39 40 LAYOUT_TESTS_PATH = "LayoutTests" 41 42 # Additional JSON fields. 43 WONTFIX = "wontfixCounts" 44 45 FAILURE_TO_CHAR = {test_expectations.PASS: json_results_generator.JSONResultsGeneratorBase.PASS_RESULT, 46 test_expectations.SKIP: json_results_generator.JSONResultsGeneratorBase.SKIP_RESULT, 47 test_expectations.FAIL: "Y", 48 test_expectations.CRASH: "C", 49 test_expectations.TIMEOUT: "T", 50 test_expectations.IMAGE: "I", 51 test_expectations.TEXT: "F", 52 test_expectations.MISSING: "O", 53 test_expectations.AUDIO: "A", 54 test_expectations.IMAGE_PLUS_TEXT: "Z"} 55 56 def __init__(self, port, builder_name, build_name, build_number, 57 results_file_base_path, builder_base_url, 58 test_timings, expectations, result_summary, all_tests, 59 test_results_server=None, test_type="", master_name=""): 60 """Modifies the results.json file. Grabs it off the archive directory 61 if it is not found locally. 62 63 Args: 64 result_summary: ResultsSummary object storing the summary of the test 65 results. 66 """ 67 super(JSONLayoutResultsGenerator, self).__init__( 68 port, builder_name, build_name, build_number, results_file_base_path, 69 builder_base_url, {}, port.test_repository_paths(), 70 test_results_server, test_type, master_name) 71 72 self._expectations = expectations 73 74 # We want relative paths to LayoutTest root for JSON output. 75 path_to_name = self._get_path_relative_to_layout_test_root 76 self._result_summary = result_summary 77 self._failures = dict( 78 (path_to_name(test), test_failures.determine_result_type(failures)) 79 for (test, failures) in result_summary.failures.iteritems()) 80 self._all_tests = [path_to_name(test) for test in all_tests] 81 self._test_timings = dict( 82 (path_to_name(test_tuple.filename), test_tuple.test_run_time) 83 for test_tuple in test_timings) 84 85 self.generate_json_output() 86 87 def _get_path_relative_to_layout_test_root(self, test): 88 """Returns the path of the test relative to the layout test root. 89 For example, for: 90 src/third_party/WebKit/LayoutTests/fast/forms/foo.html 91 We would return 92 fast/forms/foo.html 93 """ 94 index = test.find(self.LAYOUT_TESTS_PATH) 95 if index is not -1: 96 index += len(self.LAYOUT_TESTS_PATH) 97 98 if index is -1: 99 # Already a relative path. 100 relativePath = test 101 else: 102 relativePath = test[index + 1:] 103 104 # Make sure all paths are unix-style. 105 return relativePath.replace('\\', '/') 106 107 # override 108 def _get_test_timing(self, test_name): 109 if test_name in self._test_timings: 110 # Floor for now to get time in seconds. 111 return int(self._test_timings[test_name]) 112 return 0 113 114 # override 115 def _get_failed_test_names(self): 116 return set(self._failures.keys()) 117 118 # override 119 def _get_modifier_char(self, test_name): 120 if test_name not in self._all_tests: 121 return self.NO_DATA_RESULT 122 123 if test_name in self._failures: 124 return self.FAILURE_TO_CHAR[self._failures[test_name]] 125 126 return self.PASS_RESULT 127 128 # override 129 def _get_result_char(self, test_name): 130 return self._get_modifier_char(test_name) 131 132 # override 133 def _convert_json_to_current_version(self, results_json): 134 archive_version = None 135 if self.VERSION_KEY in results_json: 136 archive_version = results_json[self.VERSION_KEY] 137 138 super(JSONLayoutResultsGenerator, 139 self)._convert_json_to_current_version(results_json) 140 141 # version 2->3 142 if archive_version == 2: 143 for results_for_builder in results_json.itervalues(): 144 try: 145 test_results = results_for_builder[self.TESTS] 146 except: 147 continue 148 149 for test in test_results: 150 # Make sure all paths are relative 151 test_path = self._get_path_relative_to_layout_test_root(test) 152 if test_path != test: 153 test_results[test_path] = test_results[test] 154 del test_results[test] 155 156 # override 157 def _insert_failure_summaries(self, results_for_builder): 158 summary = self._result_summary 159 160 self._insert_item_into_raw_list(results_for_builder, 161 len((set(summary.failures.keys()) | 162 summary.tests_by_expectation[test_expectations.SKIP]) & 163 summary.tests_by_timeline[test_expectations.NOW]), 164 self.FIXABLE_COUNT) 165 self._insert_item_into_raw_list(results_for_builder, 166 self._get_failure_summary_entry(test_expectations.NOW), 167 self.FIXABLE) 168 self._insert_item_into_raw_list(results_for_builder, 169 len(self._expectations.get_tests_with_timeline( 170 test_expectations.NOW)), self.ALL_FIXABLE_COUNT) 171 self._insert_item_into_raw_list(results_for_builder, 172 self._get_failure_summary_entry(test_expectations.WONTFIX), 173 self.WONTFIX) 174 175 # override 176 def _normalize_results_json(self, test, test_name, tests): 177 super(JSONLayoutResultsGenerator, self)._normalize_results_json( 178 test, test_name, tests) 179 180 # Remove tests that don't exist anymore. 181 full_path = self._fs.join(self._port.layout_tests_dir(), test_name) 182 full_path = self._fs.normpath(full_path) 183 if not self._fs.exists(full_path): 184 del tests[test_name] 185 186 def _get_failure_summary_entry(self, timeline): 187 """Creates a summary object to insert into the JSON. 188 189 Args: 190 summary ResultSummary object with test results 191 timeline current test_expectations timeline to build entry for 192 (e.g., test_expectations.NOW, etc.) 193 """ 194 entry = {} 195 summary = self._result_summary 196 timeline_tests = summary.tests_by_timeline[timeline] 197 entry[self.SKIP_RESULT] = len( 198 summary.tests_by_expectation[test_expectations.SKIP] & 199 timeline_tests) 200 entry[self.PASS_RESULT] = len( 201 summary.tests_by_expectation[test_expectations.PASS] & 202 timeline_tests) 203 for failure_type in summary.tests_by_expectation.keys(): 204 if failure_type not in self.FAILURE_TO_CHAR: 205 continue 206 count = len(summary.tests_by_expectation[failure_type] & 207 timeline_tests) 208 entry[self.FAILURE_TO_CHAR[failure_type]] = count 209 return entry 210