1# Copyright (C) 2010 Google Inc. All rights reserved. 2# 3# Redistribution and use in source and binary forms, with or without 4# modification, are permitted provided that the following conditions are 5# met: 6# 7# * Redistributions of source code must retain the above copyright 8# notice, this list of conditions and the following disclaimer. 9# * Redistributions in binary form must reproduce the above 10# copyright notice, this list of conditions and the following disclaimer 11# in the documentation and/or other materials provided with the 12# distribution. 13# * Neither the name of Google Inc. nor the names of its 14# contributors may be used to endorse or promote products derived from 15# this software without specific prior written permission. 16# 17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29import json 30import logging 31import re 32import sys 33import traceback 34 35from testfile import TestFile 36 37JSON_RESULTS_FILE = "results.json" 38JSON_RESULTS_FILE_SMALL = "results-small.json" 39JSON_RESULTS_PREFIX = "ADD_RESULTS(" 40JSON_RESULTS_SUFFIX = ");" 41 42JSON_RESULTS_MIN_TIME = 3 43JSON_RESULTS_HIERARCHICAL_VERSION = 4 44JSON_RESULTS_MAX_BUILDS = 500 45JSON_RESULTS_MAX_BUILDS_SMALL = 100 46 47ACTUAL_KEY = "actual" 48BUG_KEY = "bugs" 49BUILD_NUMBERS_KEY = "buildNumbers" 50BUILDER_NAME_KEY = "builder_name" 51EXPECTED_KEY = "expected" 52FAILURE_MAP_KEY = "failure_map" 53FAILURES_BY_TYPE_KEY = "num_failures_by_type" 54FIXABLE_COUNTS_KEY = "fixableCounts" 55RESULTS_KEY = "results" 56TESTS_KEY = "tests" 57TIME_KEY = "time" 58TIMES_KEY = "times" 59VERSIONS_KEY = "version" 60 61AUDIO = "A" 62CRASH = "C" 63FAIL = "Q" 64# This is only output by gtests. 65FLAKY = "L" 66IMAGE = "I" 67IMAGE_PLUS_TEXT = "Z" 68MISSING = "O" 69NO_DATA = "N" 70NOTRUN = "Y" 71PASS = "P" 72SKIP = "X" 73TEXT = "F" 74TIMEOUT = "T" 75LEAK = "K" 76 77AUDIO_STRING = "AUDIO" 78CRASH_STRING = "CRASH" 79IMAGE_PLUS_TEXT_STRING = "IMAGE+TEXT" 80IMAGE_STRING = "IMAGE" 81FAIL_STRING = "FAIL" 82FLAKY_STRING = "FLAKY" 83MISSING_STRING = "MISSING" 84NO_DATA_STRING = "NO DATA" 85NOTRUN_STRING = "NOTRUN" 86PASS_STRING = "PASS" 87SKIP_STRING = "SKIP" 88TEXT_STRING = "TEXT" 89TIMEOUT_STRING = "TIMEOUT" 90LEAK_STRING = "LEAK" 91 92FAILURE_TO_CHAR = { 93 AUDIO_STRING: AUDIO, 94 CRASH_STRING: CRASH, 95 IMAGE_PLUS_TEXT_STRING: IMAGE_PLUS_TEXT, 96 IMAGE_STRING: IMAGE, 97 FLAKY_STRING: FLAKY, 98 FAIL_STRING: FAIL, 99 MISSING_STRING: MISSING, 100 NO_DATA_STRING: NO_DATA, 101 NOTRUN_STRING: NOTRUN, 102 PASS_STRING: PASS, 103 SKIP_STRING: SKIP, 104 TEXT_STRING: TEXT, 105 TIMEOUT_STRING: TIMEOUT, 106 LEAK_STRING: LEAK, 107} 108 109# FIXME: Use dict comprehensions once we update the server to python 2.7. 110CHAR_TO_FAILURE = dict((value, key) for key, value in FAILURE_TO_CHAR.items()) 111 112def _is_directory(subtree): 113 return RESULTS_KEY not in subtree 114 115 116class JsonResults(object): 117 @classmethod 118 def _strip_prefix_suffix(cls, data): 119 if data.startswith(JSON_RESULTS_PREFIX) and data.endswith(JSON_RESULTS_SUFFIX): 120 return data[len(JSON_RESULTS_PREFIX):len(data) - len(JSON_RESULTS_SUFFIX)] 121 return data 122 123 @classmethod 124 def _generate_file_data(cls, jsonObject, sort_keys=False): 125 return json.dumps(jsonObject, separators=(',', ':'), sort_keys=sort_keys) 126 127 @classmethod 128 def _load_json(cls, file_data): 129 json_results_str = cls._strip_prefix_suffix(file_data) 130 if not json_results_str: 131 logging.warning("No json results data.") 132 return None 133 134 try: 135 return json.loads(json_results_str) 136 except: 137 logging.debug(json_results_str) 138 logging.error("Failed to load json results: %s", traceback.print_exception(*sys.exc_info())) 139 return None 140 141 @classmethod 142 def _merge_json(cls, aggregated_json, incremental_json, num_runs): 143 # We have to delete expected entries because the incremental json may not have any 144 # entry for every test in the aggregated json. But, the incremental json will have 145 # all the correct expected entries for that run. 146 cls._delete_expected_entries(aggregated_json[TESTS_KEY]) 147 cls._merge_non_test_data(aggregated_json, incremental_json, num_runs) 148 incremental_tests = incremental_json[TESTS_KEY] 149 if incremental_tests: 150 aggregated_tests = aggregated_json[TESTS_KEY] 151 cls._merge_tests(aggregated_tests, incremental_tests, num_runs) 152 153 @classmethod 154 def _delete_expected_entries(cls, aggregated_json): 155 for key in aggregated_json: 156 item = aggregated_json[key] 157 if _is_directory(item): 158 cls._delete_expected_entries(item) 159 else: 160 if EXPECTED_KEY in item: 161 del item[EXPECTED_KEY] 162 if BUG_KEY in item: 163 del item[BUG_KEY] 164 165 @classmethod 166 def _merge_non_test_data(cls, aggregated_json, incremental_json, num_runs): 167 incremental_builds = incremental_json[BUILD_NUMBERS_KEY] 168 aggregated_builds = aggregated_json[BUILD_NUMBERS_KEY] 169 aggregated_build_number = int(aggregated_builds[0]) 170 171 # FIXME: It's no longer possible to have multiple runs worth of data in the incremental_json, 172 # So we can get rid of this for-loop and the associated index. 173 for index in reversed(range(len(incremental_builds))): 174 build_number = int(incremental_builds[index]) 175 logging.debug("Merging build %s, incremental json index: %d.", build_number, index) 176 177 # Merge this build into aggreagated results. 178 cls._merge_one_build(aggregated_json, incremental_json, index, num_runs) 179 180 @classmethod 181 def _merge_one_build(cls, aggregated_json, incremental_json, incremental_index, num_runs): 182 for key in incremental_json.keys(): 183 # Merge json results except "tests" properties (results, times etc). 184 # "tests" properties will be handled separately. 185 if key == TESTS_KEY or key == FAILURE_MAP_KEY: 186 continue 187 188 if key in aggregated_json: 189 if key == FAILURES_BY_TYPE_KEY: 190 cls._merge_one_build(aggregated_json[key], incremental_json[key], incremental_index, num_runs=num_runs) 191 else: 192 aggregated_json[key].insert(0, incremental_json[key][incremental_index]) 193 aggregated_json[key] = aggregated_json[key][:num_runs] 194 else: 195 aggregated_json[key] = incremental_json[key] 196 197 @classmethod 198 def _merge_tests(cls, aggregated_json, incremental_json, num_runs): 199 # FIXME: Some data got corrupted and has results/times at the directory level. 200 # Once the data is fixe, this should assert that the directory level does not have 201 # results or times and just return "RESULTS_KEY not in subtree". 202 if RESULTS_KEY in aggregated_json: 203 del aggregated_json[RESULTS_KEY] 204 if TIMES_KEY in aggregated_json: 205 del aggregated_json[TIMES_KEY] 206 207 all_tests = set(aggregated_json.iterkeys()) 208 if incremental_json: 209 all_tests |= set(incremental_json.iterkeys()) 210 211 for test_name in all_tests: 212 if test_name not in aggregated_json: 213 aggregated_json[test_name] = incremental_json[test_name] 214 continue 215 216 incremental_sub_result = incremental_json[test_name] if incremental_json and test_name in incremental_json else None 217 if _is_directory(aggregated_json[test_name]): 218 cls._merge_tests(aggregated_json[test_name], incremental_sub_result, num_runs) 219 continue 220 221 aggregated_test = aggregated_json[test_name] 222 223 if incremental_sub_result: 224 results = incremental_sub_result[RESULTS_KEY] 225 times = incremental_sub_result[TIMES_KEY] 226 if EXPECTED_KEY in incremental_sub_result and incremental_sub_result[EXPECTED_KEY] != PASS_STRING: 227 aggregated_test[EXPECTED_KEY] = incremental_sub_result[EXPECTED_KEY] 228 if BUG_KEY in incremental_sub_result: 229 aggregated_test[BUG_KEY] = incremental_sub_result[BUG_KEY] 230 else: 231 results = [[1, NO_DATA]] 232 times = [[1, 0]] 233 234 cls._insert_item_run_length_encoded(results, aggregated_test[RESULTS_KEY], num_runs) 235 cls._insert_item_run_length_encoded(times, aggregated_test[TIMES_KEY], num_runs) 236 237 @classmethod 238 def _insert_item_run_length_encoded(cls, incremental_item, aggregated_item, num_runs): 239 for item in incremental_item: 240 if len(aggregated_item) and item[1] == aggregated_item[0][1]: 241 aggregated_item[0][0] = min(aggregated_item[0][0] + item[0], num_runs) 242 else: 243 aggregated_item.insert(0, item) 244 245 @classmethod 246 def _normalize_results(cls, aggregated_json, num_runs, run_time_pruning_threshold): 247 names_to_delete = [] 248 for test_name in aggregated_json: 249 if _is_directory(aggregated_json[test_name]): 250 cls._normalize_results(aggregated_json[test_name], num_runs, run_time_pruning_threshold) 251 # If normalizing deletes all the children of this directory, also delete the directory. 252 if not aggregated_json[test_name]: 253 names_to_delete.append(test_name) 254 else: 255 leaf = aggregated_json[test_name] 256 leaf[RESULTS_KEY] = cls._remove_items_over_max_number_of_builds(leaf[RESULTS_KEY], num_runs) 257 leaf[TIMES_KEY] = cls._remove_items_over_max_number_of_builds(leaf[TIMES_KEY], num_runs) 258 if cls._should_delete_leaf(leaf, run_time_pruning_threshold): 259 names_to_delete.append(test_name) 260 261 for test_name in names_to_delete: 262 del aggregated_json[test_name] 263 264 @classmethod 265 def _should_delete_leaf(cls, leaf, run_time_pruning_threshold): 266 if leaf.get(EXPECTED_KEY, PASS_STRING) != PASS_STRING: 267 return False 268 269 if BUG_KEY in leaf: 270 return False 271 272 deletable_types = set((PASS, NO_DATA, NOTRUN)) 273 for result in leaf[RESULTS_KEY]: 274 if result[1] not in deletable_types: 275 return False 276 277 for time in leaf[TIMES_KEY]: 278 if time[1] >= run_time_pruning_threshold: 279 return False 280 281 return True 282 283 @classmethod 284 def _remove_items_over_max_number_of_builds(cls, encoded_list, num_runs): 285 num_builds = 0 286 index = 0 287 for result in encoded_list: 288 num_builds = num_builds + result[0] 289 index = index + 1 290 if num_builds >= num_runs: 291 return encoded_list[:index] 292 293 return encoded_list 294 295 @classmethod 296 def _convert_gtest_json_to_aggregate_results_format(cls, json): 297 # FIXME: Change gtests over to uploading the full results format like layout-tests 298 # so we don't have to do this normalizing. 299 # http://crbug.com/247192. 300 301 if FAILURES_BY_TYPE_KEY in json: 302 # This is already in the right format. 303 return 304 305 failures_by_type = {} 306 for fixableCount in json[FIXABLE_COUNTS_KEY]: 307 for failure_type, count in fixableCount.items(): 308 failure_string = CHAR_TO_FAILURE[failure_type] 309 if failure_string not in failures_by_type: 310 failures_by_type[failure_string] = [] 311 failures_by_type[failure_string].append(count) 312 json[FAILURES_BY_TYPE_KEY] = failures_by_type 313 314 @classmethod 315 def _check_json(cls, builder, json): 316 version = json[VERSIONS_KEY] 317 if version > JSON_RESULTS_HIERARCHICAL_VERSION: 318 return "Results JSON version '%s' is not supported." % version 319 320 if not builder in json: 321 return "Builder '%s' is not in json results." % builder 322 323 results_for_builder = json[builder] 324 if not BUILD_NUMBERS_KEY in results_for_builder: 325 return "Missing build number in json results." 326 327 cls._convert_gtest_json_to_aggregate_results_format(json[builder]) 328 329 # FIXME: Remove this once all the bots have cycled with this code. 330 # The failure map was moved from the top-level to being below the builder 331 # like everything else. 332 if FAILURE_MAP_KEY in json: 333 del json[FAILURE_MAP_KEY] 334 335 # FIXME: Remove this code once the gtests switch over to uploading the full_results.json format. 336 # Once the bots have cycled with this code, we can move this loop into _convert_gtest_json_to_aggregate_results_format. 337 KEYS_TO_DELETE = ["fixableCount", "fixableCounts", "allFixableCount"] 338 for key in KEYS_TO_DELETE: 339 if key in json[builder]: 340 del json[builder][key] 341 342 return "" 343 344 @classmethod 345 def _populate_tests_from_full_results(cls, full_results, new_results): 346 if EXPECTED_KEY in full_results: 347 expected = full_results[EXPECTED_KEY] 348 if expected != PASS_STRING and expected != NOTRUN_STRING: 349 new_results[EXPECTED_KEY] = expected 350 time = int(round(full_results[TIME_KEY])) if TIME_KEY in full_results else 0 351 new_results[TIMES_KEY] = [[1, time]] 352 353 actual_failures = full_results[ACTUAL_KEY] 354 # Treat unexpected skips like NOTRUNs to avoid exploding the results JSON files 355 # when a bot exits early (e.g. due to too many crashes/timeouts). 356 if expected != SKIP_STRING and actual_failures == SKIP_STRING: 357 expected = first_actual_failure = NOTRUN_STRING 358 elif expected == NOTRUN_STRING: 359 first_actual_failure = expected 360 else: 361 # FIXME: Include the retry result as well and find a nice way to display it in the flakiness dashboard. 362 first_actual_failure = actual_failures.split(' ')[0] 363 new_results[RESULTS_KEY] = [[1, FAILURE_TO_CHAR[first_actual_failure]]] 364 365 if BUG_KEY in full_results: 366 new_results[BUG_KEY] = full_results[BUG_KEY] 367 return 368 369 for key in full_results: 370 new_results[key] = {} 371 cls._populate_tests_from_full_results(full_results[key], new_results[key]) 372 373 @classmethod 374 def _convert_full_results_format_to_aggregate(cls, full_results_format): 375 num_total_tests = 0 376 num_failing_tests = 0 377 failures_by_type = full_results_format[FAILURES_BY_TYPE_KEY] 378 379 tests = {} 380 cls._populate_tests_from_full_results(full_results_format[TESTS_KEY], tests) 381 382 aggregate_results_format = { 383 VERSIONS_KEY: JSON_RESULTS_HIERARCHICAL_VERSION, 384 full_results_format[BUILDER_NAME_KEY]: { 385 # FIXME: Use dict comprehensions once we update the server to python 2.7. 386 FAILURES_BY_TYPE_KEY: dict((key, [value]) for key, value in failures_by_type.items()), 387 TESTS_KEY: tests, 388 # FIXME: Have all the consumers of this switch over to the full_results_format keys 389 # so we don't have to do this silly conversion. Or switch the full_results_format keys 390 # to be camel-case. 391 BUILD_NUMBERS_KEY: [full_results_format['build_number']], 392 'chromeRevision': [full_results_format['chromium_revision']], 393 'blinkRevision': [full_results_format['blink_revision']], 394 'secondsSinceEpoch': [full_results_format['seconds_since_epoch']], 395 } 396 } 397 return aggregate_results_format 398 399 @classmethod 400 def _get_incremental_json(cls, builder, incremental_string, is_full_results_format): 401 if not incremental_string: 402 return "No incremental JSON data to merge.", 403 403 404 logging.info("Loading incremental json.") 405 incremental_json = cls._load_json(incremental_string) 406 if not incremental_json: 407 return "Incremental JSON data is not valid JSON.", 403 408 409 if is_full_results_format: 410 logging.info("Converting full results format to aggregate.") 411 incremental_json = cls._convert_full_results_format_to_aggregate(incremental_json) 412 413 logging.info("Checking incremental json.") 414 check_json_error_string = cls._check_json(builder, incremental_json) 415 if check_json_error_string: 416 return check_json_error_string, 403 417 return incremental_json, 200 418 419 @classmethod 420 def _get_aggregated_json(cls, builder, aggregated_string): 421 logging.info("Loading existing aggregated json.") 422 aggregated_json = cls._load_json(aggregated_string) 423 if not aggregated_json: 424 return None, 200 425 426 logging.info("Checking existing aggregated json.") 427 check_json_error_string = cls._check_json(builder, aggregated_json) 428 if check_json_error_string: 429 return check_json_error_string, 500 430 431 return aggregated_json, 200 432 433 @classmethod 434 def merge(cls, builder, aggregated_string, incremental_json, num_runs, sort_keys=False): 435 aggregated_json, status_code = cls._get_aggregated_json(builder, aggregated_string) 436 if not aggregated_json: 437 aggregated_json = incremental_json 438 elif status_code != 200: 439 return aggregated_json, status_code 440 else: 441 if aggregated_json[builder][BUILD_NUMBERS_KEY][0] == incremental_json[builder][BUILD_NUMBERS_KEY][0]: 442 status_string = "Incremental JSON's build number %s is the latest build number in the aggregated JSON." % str(aggregated_json[builder][BUILD_NUMBERS_KEY][0]) 443 return status_string, 409 444 445 logging.info("Merging json results.") 446 try: 447 cls._merge_json(aggregated_json[builder], incremental_json[builder], num_runs) 448 except: 449 return "Failed to merge json results: %s", traceback.print_exception(*sys.exc_info()), 500 450 451 aggregated_json[VERSIONS_KEY] = JSON_RESULTS_HIERARCHICAL_VERSION 452 aggregated_json[builder][FAILURE_MAP_KEY] = CHAR_TO_FAILURE 453 454 is_debug_builder = re.search(r"(Debug|Dbg)", builder, re.I) 455 run_time_pruning_threshold = 3 * JSON_RESULTS_MIN_TIME if is_debug_builder else JSON_RESULTS_MIN_TIME 456 cls._normalize_results(aggregated_json[builder][TESTS_KEY], num_runs, run_time_pruning_threshold) 457 return cls._generate_file_data(aggregated_json, sort_keys), 200 458 459 @classmethod 460 def _get_file(cls, master, builder, test_type, filename): 461 files = TestFile.get_files(master, builder, test_type, filename) 462 if files: 463 return files[0] 464 465 file = TestFile() 466 file.master = master 467 file.builder = builder 468 file.test_type = test_type 469 file.name = filename 470 file.data = "" 471 return file 472 473 @classmethod 474 def update(cls, master, builder, test_type, incremental_string, is_full_results_format): 475 logging.info("Updating %s and %s." % (JSON_RESULTS_FILE_SMALL, JSON_RESULTS_FILE)) 476 small_file = cls._get_file(master, builder, test_type, JSON_RESULTS_FILE_SMALL) 477 large_file = cls._get_file(master, builder, test_type, JSON_RESULTS_FILE) 478 return cls.update_files(builder, incremental_string, small_file, large_file, is_full_results_format) 479 480 @classmethod 481 def update_files(cls, builder, incremental_string, small_file, large_file, is_full_results_format): 482 incremental_json, status_code = cls._get_incremental_json(builder, incremental_string, is_full_results_format) 483 if status_code != 200: 484 return incremental_json, status_code 485 486 status_string, status_code = cls.update_file(builder, small_file, incremental_json, JSON_RESULTS_MAX_BUILDS_SMALL) 487 if status_code != 200: 488 return status_string, status_code 489 490 return cls.update_file(builder, large_file, incremental_json, JSON_RESULTS_MAX_BUILDS) 491 492 @classmethod 493 def update_file(cls, builder, file, incremental_json, num_runs): 494 new_results, status_code = cls.merge(builder, file.data, incremental_json, num_runs) 495 if status_code != 200: 496 return new_results, status_code 497 return TestFile.save_file(file, new_results) 498 499 @classmethod 500 def _delete_results_and_times(cls, tests): 501 for key in tests.keys(): 502 if key in (RESULTS_KEY, TIMES_KEY): 503 del tests[key] 504 else: 505 cls._delete_results_and_times(tests[key]) 506 507 @classmethod 508 def get_test_list(cls, builder, json_file_data): 509 logging.debug("Loading test results json...") 510 json = cls._load_json(json_file_data) 511 if not json: 512 return None 513 514 logging.debug("Checking test results json...") 515 516 check_json_error_string = cls._check_json(builder, json) 517 if check_json_error_string: 518 return None 519 520 test_list_json = {} 521 tests = json[builder][TESTS_KEY] 522 cls._delete_results_and_times(tests) 523 test_list_json[builder] = {TESTS_KEY: tests} 524 return cls._generate_file_data(test_list_json) 525