1# Copyright (C) 2010 Google Inc. All rights reserved. 2# 3# Redistribution and use in source and binary forms, with or without 4# modification, are permitted provided that the following conditions are 5# met: 6# 7# * Redistributions of source code must retain the above copyright 8# notice, this list of conditions and the following disclaimer. 9# * Redistributions in binary form must reproduce the above 10# copyright notice, this list of conditions and the following disclaimer 11# in the documentation and/or other materials provided with the 12# distribution. 13# * Neither the name of Google Inc. nor the names of its 14# contributors may be used to endorse or promote products derived from 15# this software without specific prior written permission. 16# 17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29import webkitpy.thirdparty.unittest2 as unittest 30import json 31import optparse 32import random 33 34from webkitpy.common.host_mock import MockHost 35from webkitpy.layout_tests.layout_package import json_results_generator 36from webkitpy.layout_tests.models import test_expectations 37from webkitpy.layout_tests.port import test 38from webkitpy.thirdparty.mock import Mock 39 40 41class JSONGeneratorTest(unittest.TestCase): 42 def setUp(self): 43 self.builder_name = 'DUMMY_BUILDER_NAME' 44 self.build_name = 'DUMMY_BUILD_NAME' 45 self.build_number = 'DUMMY_BUILDER_NUMBER' 46 47 # For archived results. 48 self._json = None 49 self._num_runs = 0 50 self._tests_set = set([]) 51 self._test_timings = {} 52 self._failed_count_map = {} 53 54 self._PASS_count = 0 55 self._DISABLED_count = 0 56 self._FLAKY_count = 0 57 self._FAILS_count = 0 58 self._fixable_count = 0 59 60 def test_strip_json_wrapper(self): 61 json = "['contents']" 62 self.assertEqual(json_results_generator.strip_json_wrapper(json_results_generator._JSON_PREFIX + json + json_results_generator._JSON_SUFFIX), json) 63 self.assertEqual(json_results_generator.strip_json_wrapper(json), json) 64 65 def _test_json_generation(self, passed_tests_list, failed_tests_list): 66 tests_set = set(passed_tests_list) | set(failed_tests_list) 67 68 DISABLED_tests = set([t for t in tests_set 69 if t.startswith('DISABLED_')]) 70 FLAKY_tests = set([t for t in tests_set 71 if t.startswith('FLAKY_')]) 72 FAILS_tests = set([t for t in tests_set 73 if t.startswith('FAILS_')]) 74 PASS_tests = tests_set - (DISABLED_tests | FLAKY_tests | FAILS_tests) 75 76 failed_tests = set(failed_tests_list) - DISABLED_tests 77 failed_count_map = dict([(t, 1) for t in failed_tests]) 78 79 test_timings = {} 80 i = 0 81 for test in tests_set: 82 test_timings[test] = float(self._num_runs * 100 + i) 83 i += 1 84 85 test_results_map = dict() 86 for test in tests_set: 87 test_results_map[test] = json_results_generator.TestResult(test, 88 failed=(test in failed_tests), 89 elapsed_time=test_timings[test]) 90 91 host = MockHost() 92 port = Mock() 93 port._filesystem = host.filesystem 94 generator = json_results_generator.JSONResultsGeneratorBase(port, 95 self.builder_name, self.build_name, self.build_number, 96 '', 97 None, # don't fetch past json results archive 98 test_results_map) 99 100 failed_count_map = dict([(t, 1) for t in failed_tests]) 101 102 # Test incremental json results 103 incremental_json = generator.get_json() 104 self._verify_json_results( 105 tests_set, 106 test_timings, 107 failed_count_map, 108 len(PASS_tests), 109 len(DISABLED_tests), 110 len(FLAKY_tests), 111 len(DISABLED_tests | failed_tests), 112 incremental_json, 113 1) 114 115 # We don't verify the results here, but at least we make sure the code runs without errors. 116 generator.generate_json_output() 117 generator.generate_times_ms_file() 118 119 def _verify_json_results(self, tests_set, test_timings, failed_count_map, 120 PASS_count, DISABLED_count, FLAKY_count, 121 fixable_count, 122 json, num_runs): 123 # Aliasing to a short name for better access to its constants. 124 JRG = json_results_generator.JSONResultsGeneratorBase 125 126 self.assertIn(JRG.VERSION_KEY, json) 127 self.assertIn(self.builder_name, json) 128 129 buildinfo = json[self.builder_name] 130 self.assertIn(JRG.FIXABLE, buildinfo) 131 self.assertIn(JRG.TESTS, buildinfo) 132 self.assertEqual(len(buildinfo[JRG.BUILD_NUMBERS]), num_runs) 133 self.assertEqual(buildinfo[JRG.BUILD_NUMBERS][0], self.build_number) 134 135 if tests_set or DISABLED_count: 136 fixable = {} 137 for fixable_items in buildinfo[JRG.FIXABLE]: 138 for (type, count) in fixable_items.iteritems(): 139 if type in fixable: 140 fixable[type] = fixable[type] + count 141 else: 142 fixable[type] = count 143 144 if PASS_count: 145 self.assertEqual(fixable[JRG.PASS_RESULT], PASS_count) 146 else: 147 self.assertTrue(JRG.PASS_RESULT not in fixable or 148 fixable[JRG.PASS_RESULT] == 0) 149 if DISABLED_count: 150 self.assertEqual(fixable[JRG.SKIP_RESULT], DISABLED_count) 151 else: 152 self.assertTrue(JRG.SKIP_RESULT not in fixable or 153 fixable[JRG.SKIP_RESULT] == 0) 154 if FLAKY_count: 155 self.assertEqual(fixable[JRG.FLAKY_RESULT], FLAKY_count) 156 else: 157 self.assertTrue(JRG.FLAKY_RESULT not in fixable or 158 fixable[JRG.FLAKY_RESULT] == 0) 159 160 if failed_count_map: 161 tests = buildinfo[JRG.TESTS] 162 for test_name in failed_count_map.iterkeys(): 163 test = self._find_test_in_trie(test_name, tests) 164 165 failed = 0 166 for result in test[JRG.RESULTS]: 167 if result[1] == JRG.FAIL_RESULT: 168 failed += result[0] 169 self.assertEqual(failed_count_map[test_name], failed) 170 171 timing_count = 0 172 for timings in test[JRG.TIMES]: 173 if timings[1] == test_timings[test_name]: 174 timing_count = timings[0] 175 self.assertEqual(1, timing_count) 176 177 if fixable_count: 178 self.assertEqual(sum(buildinfo[JRG.FIXABLE_COUNT]), fixable_count) 179 180 def _find_test_in_trie(self, path, trie): 181 nodes = path.split("/") 182 sub_trie = trie 183 for node in nodes: 184 self.assertIn(node, sub_trie) 185 sub_trie = sub_trie[node] 186 return sub_trie 187 188 def test_json_generation(self): 189 self._test_json_generation([], []) 190 self._test_json_generation(['A1', 'B1'], []) 191 self._test_json_generation([], ['FAILS_A2', 'FAILS_B2']) 192 self._test_json_generation(['DISABLED_A3', 'DISABLED_B3'], []) 193 self._test_json_generation(['A4'], ['B4', 'FAILS_C4']) 194 self._test_json_generation(['DISABLED_C5', 'DISABLED_D5'], ['A5', 'B5']) 195 self._test_json_generation( 196 ['A6', 'B6', 'FAILS_C6', 'DISABLED_E6', 'DISABLED_F6'], 197 ['FAILS_D6']) 198 199 # Generate JSON with the same test sets. (Both incremental results and 200 # archived results must be updated appropriately.) 201 self._test_json_generation( 202 ['A', 'FLAKY_B', 'DISABLED_C'], 203 ['FAILS_D', 'FLAKY_E']) 204 self._test_json_generation( 205 ['A', 'DISABLED_C', 'FLAKY_E'], 206 ['FLAKY_B', 'FAILS_D']) 207 self._test_json_generation( 208 ['FLAKY_B', 'DISABLED_C', 'FAILS_D'], 209 ['A', 'FLAKY_E']) 210 211 def test_hierarchical_json_generation(self): 212 # FIXME: Re-work tests to be more comprehensible and comprehensive. 213 self._test_json_generation(['foo/A'], ['foo/B', 'bar/C']) 214 215 def test_test_timings_trie(self): 216 test_port = test.TestPort(MockHost()) 217 individual_test_timings = [] 218 individual_test_timings.append(json_results_generator.TestResult('foo/bar/baz.html', elapsed_time=1.2)) 219 individual_test_timings.append(json_results_generator.TestResult('bar.html', elapsed_time=0.0001)) 220 trie = json_results_generator.test_timings_trie(test_port, individual_test_timings) 221 222 expected_trie = { 223 'bar.html': 0, 224 'foo': { 225 'bar': { 226 'baz.html': 1200, 227 } 228 } 229 } 230 231 self.assertEqual(json.dumps(trie), json.dumps(expected_trie)) 232