1# Copyright (C) 2010 Google Inc. All rights reserved. 2# 3# Redistribution and use in source and binary forms, with or without 4# modification, are permitted provided that the following conditions are 5# met: 6# 7# * Redistributions of source code must retain the above copyright 8# notice, this list of conditions and the following disclaimer. 9# * Redistributions in binary form must reproduce the above 10# copyright notice, this list of conditions and the following disclaimer 11# in the documentation and/or other materials provided with the 12# distribution. 13# * Neither the name of Google Inc. nor the names of its 14# contributors may be used to endorse or promote products derived from 15# this software without specific prior written permission. 16# 17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29import logging 30import subprocess 31import sys 32import time 33import urllib2 34import xml.dom.minidom 35 36from webkitpy.layout_tests.layout_package import test_results_uploader 37 38import webkitpy.thirdparty.simplejson as simplejson 39 40# A JSON results generator for generic tests. 41# FIXME: move this code out of the layout_package directory. 42 43_log = logging.getLogger("webkitpy.layout_tests.layout_package.json_results_generator") 44 45_JSON_PREFIX = "ADD_RESULTS(" 46_JSON_SUFFIX = ");" 47 48 49def strip_json_wrapper(json_content): 50 return json_content[len(_JSON_PREFIX):len(json_content) - len(_JSON_SUFFIX)] 51 52 53def load_json(filesystem, file_path): 54 content = filesystem.read_text_file(file_path) 55 content = strip_json_wrapper(content) 56 return simplejson.loads(content) 57 58 59def write_json(filesystem, json_object, file_path): 60 # Specify separators in order to get compact encoding. 61 json_data = simplejson.dumps(json_object, separators=(',', ':')) 62 json_string = _JSON_PREFIX + json_data + _JSON_SUFFIX 63 filesystem.write_text_file(file_path, json_string) 64 65# FIXME: We already have a TestResult class in test_results.py 66class TestResult(object): 67 """A simple class that represents a single test result.""" 68 69 # Test modifier constants. 70 (NONE, FAILS, FLAKY, DISABLED) = range(4) 71 72 def __init__(self, name, failed=False, elapsed_time=0): 73 self.name = name 74 self.failed = failed 75 self.time = elapsed_time 76 77 test_name = name 78 try: 79 test_name = name.split('.')[1] 80 except IndexError: 81 _log.warn("Invalid test name: %s.", name) 82 pass 83 84 if test_name.startswith('FAILS_'): 85 self.modifier = self.FAILS 86 elif test_name.startswith('FLAKY_'): 87 self.modifier = self.FLAKY 88 elif test_name.startswith('DISABLED_'): 89 self.modifier = self.DISABLED 90 else: 91 self.modifier = self.NONE 92 93 def fixable(self): 94 return self.failed or self.modifier == self.DISABLED 95 96 97class JSONResultsGeneratorBase(object): 98 """A JSON results generator for generic tests.""" 99 100 MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG = 750 101 # Min time (seconds) that will be added to the JSON. 102 MIN_TIME = 1 103 104 # Note that in non-chromium tests those chars are used to indicate 105 # test modifiers (FAILS, FLAKY, etc) but not actual test results. 106 PASS_RESULT = "P" 107 SKIP_RESULT = "X" 108 FAIL_RESULT = "F" 109 FLAKY_RESULT = "L" 110 NO_DATA_RESULT = "N" 111 112 MODIFIER_TO_CHAR = {TestResult.NONE: PASS_RESULT, 113 TestResult.DISABLED: SKIP_RESULT, 114 TestResult.FAILS: FAIL_RESULT, 115 TestResult.FLAKY: FLAKY_RESULT} 116 117 VERSION = 3 118 VERSION_KEY = "version" 119 RESULTS = "results" 120 TIMES = "times" 121 BUILD_NUMBERS = "buildNumbers" 122 TIME = "secondsSinceEpoch" 123 TESTS = "tests" 124 125 FIXABLE_COUNT = "fixableCount" 126 FIXABLE = "fixableCounts" 127 ALL_FIXABLE_COUNT = "allFixableCount" 128 129 RESULTS_FILENAME = "results.json" 130 FULL_RESULTS_FILENAME = "full_results.json" 131 INCREMENTAL_RESULTS_FILENAME = "incremental_results.json" 132 133 URL_FOR_TEST_LIST_JSON = \ 134 "http://%s/testfile?builder=%s&name=%s&testlistjson=1&testtype=%s" 135 136 # FIXME: Remove generate_incremental_results once the reference to it in 137 # http://src.chromium.org/viewvc/chrome/trunk/tools/build/scripts/slave/gtest_slave_utils.py 138 # has been removed. 139 def __init__(self, port, builder_name, build_name, build_number, 140 results_file_base_path, builder_base_url, 141 test_results_map, svn_repositories=None, 142 test_results_server=None, 143 test_type="", 144 master_name="", 145 generate_incremental_results=None): 146 """Modifies the results.json file. Grabs it off the archive directory 147 if it is not found locally. 148 149 Args 150 port: port-specific wrapper 151 builder_name: the builder name (e.g. Webkit). 152 build_name: the build name (e.g. webkit-rel). 153 build_number: the build number. 154 results_file_base_path: Absolute path to the directory containing the 155 results json file. 156 builder_base_url: the URL where we have the archived test results. 157 If this is None no archived results will be retrieved. 158 test_results_map: A dictionary that maps test_name to TestResult. 159 svn_repositories: A (json_field_name, svn_path) pair for SVN 160 repositories that tests rely on. The SVN revision will be 161 included in the JSON with the given json_field_name. 162 test_results_server: server that hosts test results json. 163 test_type: test type string (e.g. 'layout-tests'). 164 master_name: the name of the buildbot master. 165 """ 166 self._port = port 167 self._fs = port._filesystem 168 self._builder_name = builder_name 169 self._build_name = build_name 170 self._build_number = build_number 171 self._builder_base_url = builder_base_url 172 self._results_directory = results_file_base_path 173 174 self._test_results_map = test_results_map 175 self._test_results = test_results_map.values() 176 177 self._svn_repositories = svn_repositories 178 if not self._svn_repositories: 179 self._svn_repositories = {} 180 181 self._test_results_server = test_results_server 182 self._test_type = test_type 183 self._master_name = master_name 184 185 self._archived_results = None 186 187 def generate_json_output(self): 188 json = self.get_json() 189 if json: 190 file_path = self._fs.join(self._results_directory, self.INCREMENTAL_RESULTS_FILENAME) 191 write_json(self._fs, json, file_path) 192 193 def generate_full_results_file(self): 194 # Use the same structure as the compacted version of TestRunner.summarize_results. 195 # For now we only include the times as this is only used for treemaps and 196 # expected/actual don't make sense for gtests. 197 results = {} 198 results['version'] = 1 199 200 tests = {} 201 202 for test in self._test_results_map: 203 time_seconds = self._test_results_map[test].time 204 tests[test] = {} 205 tests[test]['time_ms'] = int(1000 * time_seconds) 206 207 results['tests'] = tests 208 file_path = self._fs.join(self._results_directory, self.FULL_RESULTS_FILENAME) 209 write_json(self._fs, results, file_path) 210 211 def get_json(self): 212 """Gets the results for the results.json file.""" 213 results_json = {} 214 215 if not results_json: 216 results_json, error = self._get_archived_json_results() 217 if error: 218 # If there was an error don't write a results.json 219 # file at all as it would lose all the information on the 220 # bot. 221 _log.error("Archive directory is inaccessible. Not " 222 "modifying or clobbering the results.json " 223 "file: " + str(error)) 224 return None 225 226 builder_name = self._builder_name 227 if results_json and builder_name not in results_json: 228 _log.debug("Builder name (%s) is not in the results.json file." 229 % builder_name) 230 231 self._convert_json_to_current_version(results_json) 232 233 if builder_name not in results_json: 234 results_json[builder_name] = ( 235 self._create_results_for_builder_json()) 236 237 results_for_builder = results_json[builder_name] 238 239 self._insert_generic_metadata(results_for_builder) 240 241 self._insert_failure_summaries(results_for_builder) 242 243 # Update the all failing tests with result type and time. 244 tests = results_for_builder[self.TESTS] 245 all_failing_tests = self._get_failed_test_names() 246 all_failing_tests.update(tests.iterkeys()) 247 for test in all_failing_tests: 248 self._insert_test_time_and_result(test, tests) 249 250 return results_json 251 252 def set_archived_results(self, archived_results): 253 self._archived_results = archived_results 254 255 def upload_json_files(self, json_files): 256 """Uploads the given json_files to the test_results_server (if the 257 test_results_server is given).""" 258 if not self._test_results_server: 259 return 260 261 if not self._master_name: 262 _log.error("--test-results-server was set, but --master-name was not. Not uploading JSON files.") 263 return 264 265 _log.info("Uploading JSON files for builder: %s", self._builder_name) 266 attrs = [("builder", self._builder_name), 267 ("testtype", self._test_type), 268 ("master", self._master_name)] 269 270 files = [(file, self._fs.join(self._results_directory, file)) 271 for file in json_files] 272 273 uploader = test_results_uploader.TestResultsUploader( 274 self._test_results_server) 275 try: 276 # Set uploading timeout in case appengine server is having problem. 277 # 120 seconds are more than enough to upload test results. 278 uploader.upload(attrs, files, 120) 279 except Exception, err: 280 _log.error("Upload failed: %s" % err) 281 return 282 283 _log.info("JSON files uploaded.") 284 285 def _get_test_timing(self, test_name): 286 """Returns test timing data (elapsed time) in second 287 for the given test_name.""" 288 if test_name in self._test_results_map: 289 # Floor for now to get time in seconds. 290 return int(self._test_results_map[test_name].time) 291 return 0 292 293 def _get_failed_test_names(self): 294 """Returns a set of failed test names.""" 295 return set([r.name for r in self._test_results if r.failed]) 296 297 def _get_modifier_char(self, test_name): 298 """Returns a single char (e.g. SKIP_RESULT, FAIL_RESULT, 299 PASS_RESULT, NO_DATA_RESULT, etc) that indicates the test modifier 300 for the given test_name. 301 """ 302 if test_name not in self._test_results_map: 303 return self.__class__.NO_DATA_RESULT 304 305 test_result = self._test_results_map[test_name] 306 if test_result.modifier in self.MODIFIER_TO_CHAR.keys(): 307 return self.MODIFIER_TO_CHAR[test_result.modifier] 308 309 return self.__class__.PASS_RESULT 310 311 def _get_result_char(self, test_name): 312 """Returns a single char (e.g. SKIP_RESULT, FAIL_RESULT, 313 PASS_RESULT, NO_DATA_RESULT, etc) that indicates the test result 314 for the given test_name. 315 """ 316 if test_name not in self._test_results_map: 317 return self.__class__.NO_DATA_RESULT 318 319 test_result = self._test_results_map[test_name] 320 if test_result.modifier == TestResult.DISABLED: 321 return self.__class__.SKIP_RESULT 322 323 if test_result.failed: 324 return self.__class__.FAIL_RESULT 325 326 return self.__class__.PASS_RESULT 327 328 # FIXME: Callers should use scm.py instead. 329 # FIXME: Identify and fix the run-time errors that were observed on Windows 330 # chromium buildbot when we had updated this code to use scm.py once before. 331 def _get_svn_revision(self, in_directory): 332 """Returns the svn revision for the given directory. 333 334 Args: 335 in_directory: The directory where svn is to be run. 336 """ 337 if self._fs.exists(self._fs.join(in_directory, '.svn')): 338 # Note: Not thread safe: http://bugs.python.org/issue2320 339 output = subprocess.Popen(["svn", "info", "--xml"], 340 cwd=in_directory, 341 shell=(sys.platform == 'win32'), 342 stdout=subprocess.PIPE).communicate()[0] 343 try: 344 dom = xml.dom.minidom.parseString(output) 345 return dom.getElementsByTagName('entry')[0].getAttribute( 346 'revision') 347 except xml.parsers.expat.ExpatError: 348 return "" 349 return "" 350 351 def _get_archived_json_results(self): 352 """Download JSON file that only contains test 353 name list from test-results server. This is for generating incremental 354 JSON so the file generated has info for tests that failed before but 355 pass or are skipped from current run. 356 357 Returns (archived_results, error) tuple where error is None if results 358 were successfully read. 359 """ 360 results_json = {} 361 old_results = None 362 error = None 363 364 if not self._test_results_server: 365 return {}, None 366 367 results_file_url = (self.URL_FOR_TEST_LIST_JSON % 368 (urllib2.quote(self._test_results_server), 369 urllib2.quote(self._builder_name), 370 self.RESULTS_FILENAME, 371 urllib2.quote(self._test_type))) 372 373 try: 374 results_file = urllib2.urlopen(results_file_url) 375 info = results_file.info() 376 old_results = results_file.read() 377 except urllib2.HTTPError, http_error: 378 # A non-4xx status code means the bot is hosed for some reason 379 # and we can't grab the results.json file off of it. 380 if (http_error.code < 400 and http_error.code >= 500): 381 error = http_error 382 except urllib2.URLError, url_error: 383 error = url_error 384 385 if old_results: 386 # Strip the prefix and suffix so we can get the actual JSON object. 387 old_results = strip_json_wrapper(old_results) 388 389 try: 390 results_json = simplejson.loads(old_results) 391 except: 392 _log.debug("results.json was not valid JSON. Clobbering.") 393 # The JSON file is not valid JSON. Just clobber the results. 394 results_json = {} 395 else: 396 _log.debug('Old JSON results do not exist. Starting fresh.') 397 results_json = {} 398 399 return results_json, error 400 401 def _insert_failure_summaries(self, results_for_builder): 402 """Inserts aggregate pass/failure statistics into the JSON. 403 This method reads self._test_results and generates 404 FIXABLE, FIXABLE_COUNT and ALL_FIXABLE_COUNT entries. 405 406 Args: 407 results_for_builder: Dictionary containing the test results for a 408 single builder. 409 """ 410 # Insert the number of tests that failed or skipped. 411 fixable_count = len([r for r in self._test_results if r.fixable()]) 412 self._insert_item_into_raw_list(results_for_builder, 413 fixable_count, self.FIXABLE_COUNT) 414 415 # Create a test modifiers (FAILS, FLAKY etc) summary dictionary. 416 entry = {} 417 for test_name in self._test_results_map.iterkeys(): 418 result_char = self._get_modifier_char(test_name) 419 entry[result_char] = entry.get(result_char, 0) + 1 420 421 # Insert the pass/skip/failure summary dictionary. 422 self._insert_item_into_raw_list(results_for_builder, entry, 423 self.FIXABLE) 424 425 # Insert the number of all the tests that are supposed to pass. 426 all_test_count = len(self._test_results) 427 self._insert_item_into_raw_list(results_for_builder, 428 all_test_count, self.ALL_FIXABLE_COUNT) 429 430 def _insert_item_into_raw_list(self, results_for_builder, item, key): 431 """Inserts the item into the list with the given key in the results for 432 this builder. Creates the list if no such list exists. 433 434 Args: 435 results_for_builder: Dictionary containing the test results for a 436 single builder. 437 item: Number or string to insert into the list. 438 key: Key in results_for_builder for the list to insert into. 439 """ 440 if key in results_for_builder: 441 raw_list = results_for_builder[key] 442 else: 443 raw_list = [] 444 445 raw_list.insert(0, item) 446 raw_list = raw_list[:self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG] 447 results_for_builder[key] = raw_list 448 449 def _insert_item_run_length_encoded(self, item, encoded_results): 450 """Inserts the item into the run-length encoded results. 451 452 Args: 453 item: String or number to insert. 454 encoded_results: run-length encoded results. An array of arrays, e.g. 455 [[3,'A'],[1,'Q']] encodes AAAQ. 456 """ 457 if len(encoded_results) and item == encoded_results[0][1]: 458 num_results = encoded_results[0][0] 459 if num_results <= self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG: 460 encoded_results[0][0] = num_results + 1 461 else: 462 # Use a list instead of a class for the run-length encoding since 463 # we want the serialized form to be concise. 464 encoded_results.insert(0, [1, item]) 465 466 def _insert_generic_metadata(self, results_for_builder): 467 """ Inserts generic metadata (such as version number, current time etc) 468 into the JSON. 469 470 Args: 471 results_for_builder: Dictionary containing the test results for 472 a single builder. 473 """ 474 self._insert_item_into_raw_list(results_for_builder, 475 self._build_number, self.BUILD_NUMBERS) 476 477 # Include SVN revisions for the given repositories. 478 for (name, path) in self._svn_repositories: 479 self._insert_item_into_raw_list(results_for_builder, 480 self._get_svn_revision(path), 481 name + 'Revision') 482 483 self._insert_item_into_raw_list(results_for_builder, 484 int(time.time()), 485 self.TIME) 486 487 def _insert_test_time_and_result(self, test_name, tests): 488 """ Insert a test item with its results to the given tests dictionary. 489 490 Args: 491 tests: Dictionary containing test result entries. 492 """ 493 494 result = self._get_result_char(test_name) 495 time = self._get_test_timing(test_name) 496 497 if test_name not in tests: 498 tests[test_name] = self._create_results_and_times_json() 499 500 thisTest = tests[test_name] 501 if self.RESULTS in thisTest: 502 self._insert_item_run_length_encoded(result, thisTest[self.RESULTS]) 503 else: 504 thisTest[self.RESULTS] = [[1, result]] 505 506 if self.TIMES in thisTest: 507 self._insert_item_run_length_encoded(time, thisTest[self.TIMES]) 508 else: 509 thisTest[self.TIMES] = [[1, time]] 510 511 def _convert_json_to_current_version(self, results_json): 512 """If the JSON does not match the current version, converts it to the 513 current version and adds in the new version number. 514 """ 515 if (self.VERSION_KEY in results_json and 516 results_json[self.VERSION_KEY] == self.VERSION): 517 return 518 519 results_json[self.VERSION_KEY] = self.VERSION 520 521 def _create_results_and_times_json(self): 522 results_and_times = {} 523 results_and_times[self.RESULTS] = [] 524 results_and_times[self.TIMES] = [] 525 return results_and_times 526 527 def _create_results_for_builder_json(self): 528 results_for_builder = {} 529 results_for_builder[self.TESTS] = {} 530 return results_for_builder 531 532 def _remove_items_over_max_number_of_builds(self, encoded_list): 533 """Removes items from the run-length encoded list after the final 534 item that exceeds the max number of builds to track. 535 536 Args: 537 encoded_results: run-length encoded results. An array of arrays, e.g. 538 [[3,'A'],[1,'Q']] encodes AAAQ. 539 """ 540 num_builds = 0 541 index = 0 542 for result in encoded_list: 543 num_builds = num_builds + result[0] 544 index = index + 1 545 if num_builds > self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG: 546 return encoded_list[:index] 547 return encoded_list 548 549 def _normalize_results_json(self, test, test_name, tests): 550 """ Prune tests where all runs pass or tests that no longer exist and 551 truncate all results to maxNumberOfBuilds. 552 553 Args: 554 test: ResultsAndTimes object for this test. 555 test_name: Name of the test. 556 tests: The JSON object with all the test results for this builder. 557 """ 558 test[self.RESULTS] = self._remove_items_over_max_number_of_builds( 559 test[self.RESULTS]) 560 test[self.TIMES] = self._remove_items_over_max_number_of_builds( 561 test[self.TIMES]) 562 563 is_all_pass = self._is_results_all_of_type(test[self.RESULTS], 564 self.PASS_RESULT) 565 is_all_no_data = self._is_results_all_of_type(test[self.RESULTS], 566 self.NO_DATA_RESULT) 567 max_time = max([time[1] for time in test[self.TIMES]]) 568 569 # Remove all passes/no-data from the results to reduce noise and 570 # filesize. If a test passes every run, but takes > MIN_TIME to run, 571 # don't throw away the data. 572 if is_all_no_data or (is_all_pass and max_time <= self.MIN_TIME): 573 del tests[test_name] 574 575 def _is_results_all_of_type(self, results, type): 576 """Returns whether all the results are of the given type 577 (e.g. all passes).""" 578 return len(results) == 1 and results[0][1] == type 579 580 581# Left here not to break anything. 582class JSONResultsGenerator(JSONResultsGeneratorBase): 583 pass 584