1# Copyright (C) 2010 Google Inc. All rights reserved. 2# 3# Redistribution and use in source and binary forms, with or without 4# modification, are permitted provided that the following conditions are 5# met: 6# 7# * Redistributions of source code must retain the above copyright 8# notice, this list of conditions and the following disclaimer. 9# * Redistributions in binary form must reproduce the above 10# copyright notice, this list of conditions and the following disclaimer 11# in the documentation and/or other materials provided with the 12# distribution. 13# * Neither the name of Google Inc. nor the names of its 14# contributors may be used to endorse or promote products derived from 15# this software without specific prior written permission. 16# 17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29"""A helper class for reading in and dealing with tests expectations 30for layout tests. 31""" 32 33import logging 34import re 35 36from webkitpy.layout_tests.models.test_configuration import TestConfigurationConverter 37 38_log = logging.getLogger(__name__) 39 40 41# Test expectation and specifier constants. 42# 43# FIXME: range() starts with 0 which makes if expectation checks harder 44# as PASS is 0. 45(PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, TIMEOUT, CRASH, SKIP, WONTFIX, 46 SLOW, REBASELINE, NEEDS_REBASELINE, NEEDS_MANUAL_REBASELINE, MISSING, FLAKY, NOW, NONE) = range(18) 47 48# FIXME: Perhas these two routines should be part of the Port instead? 49BASELINE_SUFFIX_LIST = ('png', 'wav', 'txt') 50 51WEBKIT_BUG_PREFIX = 'webkit.org/b/' 52CHROMIUM_BUG_PREFIX = 'crbug.com/' 53V8_BUG_PREFIX = 'code.google.com/p/v8/issues/detail?id=' 54NAMED_BUG_PREFIX = 'Bug(' 55 56MISSING_KEYWORD = 'Missing' 57NEEDS_REBASELINE_KEYWORD = 'NeedsRebaseline' 58NEEDS_MANUAL_REBASELINE_KEYWORD = 'NeedsManualRebaseline' 59 60class ParseError(Exception): 61 def __init__(self, warnings): 62 super(ParseError, self).__init__() 63 self.warnings = warnings 64 65 def __str__(self): 66 return '\n'.join(map(str, self.warnings)) 67 68 def __repr__(self): 69 return 'ParseError(warnings=%s)' % self.warnings 70 71 72class TestExpectationParser(object): 73 """Provides parsing facilities for lines in the test_expectation.txt file.""" 74 75 # FIXME: Rename these to *_KEYWORD as in MISSING_KEYWORD above, but make the case studdly-caps to match the actual file contents. 76 REBASELINE_MODIFIER = 'rebaseline' 77 NEEDS_REBASELINE_MODIFIER = 'needsrebaseline' 78 NEEDS_MANUAL_REBASELINE_MODIFIER = 'needsmanualrebaseline' 79 PASS_EXPECTATION = 'pass' 80 SKIP_MODIFIER = 'skip' 81 SLOW_MODIFIER = 'slow' 82 WONTFIX_MODIFIER = 'wontfix' 83 84 TIMEOUT_EXPECTATION = 'timeout' 85 86 MISSING_BUG_WARNING = 'Test lacks BUG specifier.' 87 88 def __init__(self, port, full_test_list, is_lint_mode): 89 self._port = port 90 self._test_configuration_converter = TestConfigurationConverter(set(port.all_test_configurations()), port.configuration_specifier_macros()) 91 self._full_test_list = full_test_list 92 self._is_lint_mode = is_lint_mode 93 94 def parse(self, filename, expectations_string): 95 expectation_lines = [] 96 line_number = 0 97 for line in expectations_string.split("\n"): 98 line_number += 1 99 test_expectation = self._tokenize_line(filename, line, line_number) 100 self._parse_line(test_expectation) 101 expectation_lines.append(test_expectation) 102 return expectation_lines 103 104 def _create_expectation_line(self, test_name, expectations, file_name): 105 expectation_line = TestExpectationLine() 106 expectation_line.original_string = test_name 107 expectation_line.name = test_name 108 expectation_line.filename = file_name 109 expectation_line.expectations = expectations 110 return expectation_line 111 112 def expectation_line_for_test(self, test_name, expectations): 113 expectation_line = self._create_expectation_line(test_name, expectations, '<Bot TestExpectations>') 114 self._parse_line(expectation_line) 115 return expectation_line 116 117 118 def expectation_for_skipped_test(self, test_name): 119 if not self._port.test_exists(test_name): 120 _log.warning('The following test %s from the Skipped list doesn\'t exist' % test_name) 121 expectation_line = self._create_expectation_line(test_name, [TestExpectationParser.PASS_EXPECTATION], '<Skipped file>') 122 expectation_line.expectations = [TestExpectationParser.SKIP_MODIFIER, TestExpectationParser.WONTFIX_MODIFIER] 123 expectation_line.is_skipped_outside_expectations_file = True 124 self._parse_line(expectation_line) 125 return expectation_line 126 127 def _parse_line(self, expectation_line): 128 if not expectation_line.name: 129 return 130 131 if not self._check_test_exists(expectation_line): 132 return 133 134 expectation_line.is_file = self._port.test_isfile(expectation_line.name) 135 if expectation_line.is_file: 136 expectation_line.path = expectation_line.name 137 else: 138 expectation_line.path = self._port.normalize_test_name(expectation_line.name) 139 140 self._collect_matching_tests(expectation_line) 141 142 self._parse_specifiers(expectation_line) 143 self._parse_expectations(expectation_line) 144 145 def _parse_specifiers(self, expectation_line): 146 if self._is_lint_mode: 147 self._lint_line(expectation_line) 148 149 parsed_specifiers = set([specifier.lower() for specifier in expectation_line.specifiers]) 150 expectation_line.matching_configurations = self._test_configuration_converter.to_config_set(parsed_specifiers, expectation_line.warnings) 151 152 def _lint_line(self, expectation_line): 153 expectations = [expectation.lower() for expectation in expectation_line.expectations] 154 if not expectation_line.bugs and self.WONTFIX_MODIFIER not in expectations: 155 expectation_line.warnings.append(self.MISSING_BUG_WARNING) 156 if self.REBASELINE_MODIFIER in expectations: 157 expectation_line.warnings.append('REBASELINE should only be used for running rebaseline.py. Cannot be checked in.') 158 159 def _parse_expectations(self, expectation_line): 160 result = set() 161 for part in expectation_line.expectations: 162 expectation = TestExpectations.expectation_from_string(part) 163 if expectation is None: # Careful, PASS is currently 0. 164 expectation_line.warnings.append('Unsupported expectation: %s' % part) 165 continue 166 result.add(expectation) 167 expectation_line.parsed_expectations = result 168 169 def _check_test_exists(self, expectation_line): 170 # WebKit's way of skipping tests is to add a -disabled suffix. 171 # So we should consider the path existing if the path or the 172 # -disabled version exists. 173 if not self._port.test_exists(expectation_line.name) and not self._port.test_exists(expectation_line.name + '-disabled'): 174 # Log a warning here since you hit this case any 175 # time you update TestExpectations without syncing 176 # the LayoutTests directory 177 expectation_line.warnings.append('Path does not exist.') 178 return False 179 return True 180 181 def _collect_matching_tests(self, expectation_line): 182 """Convert the test specification to an absolute, normalized 183 path and make sure directories end with the OS path separator.""" 184 # FIXME: full_test_list can quickly contain a big amount of 185 # elements. We should consider at some point to use a more 186 # efficient structure instead of a list. Maybe a dictionary of 187 # lists to represent the tree of tests, leaves being test 188 # files and nodes being categories. 189 190 if not self._full_test_list: 191 expectation_line.matching_tests = [expectation_line.path] 192 return 193 194 if not expectation_line.is_file: 195 # this is a test category, return all the tests of the category. 196 expectation_line.matching_tests = [test for test in self._full_test_list if test.startswith(expectation_line.path)] 197 return 198 199 # this is a test file, do a quick check if it's in the 200 # full test suite. 201 if expectation_line.path in self._full_test_list: 202 expectation_line.matching_tests.append(expectation_line.path) 203 204 # FIXME: Update the original specifiers and remove this once the old syntax is gone. 205 _configuration_tokens_list = [ 206 'Mac', 'SnowLeopard', 'Lion', 'Retina', 'MountainLion', 'Mavericks', 207 'Win', 'XP', 'Win7', 208 'Linux', 209 'Android', 210 'Release', 211 'Debug', 212 ] 213 214 _configuration_tokens = dict((token, token.upper()) for token in _configuration_tokens_list) 215 _inverted_configuration_tokens = dict((value, name) for name, value in _configuration_tokens.iteritems()) 216 217 # FIXME: Update the original specifiers list and remove this once the old syntax is gone. 218 _expectation_tokens = { 219 'Crash': 'CRASH', 220 'Failure': 'FAIL', 221 'ImageOnlyFailure': 'IMAGE', 222 MISSING_KEYWORD: 'MISSING', 223 'Pass': 'PASS', 224 'Rebaseline': 'REBASELINE', 225 NEEDS_REBASELINE_KEYWORD: 'NEEDSREBASELINE', 226 NEEDS_MANUAL_REBASELINE_KEYWORD: 'NEEDSMANUALREBASELINE', 227 'Skip': 'SKIP', 228 'Slow': 'SLOW', 229 'Timeout': 'TIMEOUT', 230 'WontFix': 'WONTFIX', 231 } 232 233 _inverted_expectation_tokens = dict([(value, name) for name, value in _expectation_tokens.iteritems()] + 234 [('TEXT', 'Failure'), ('IMAGE+TEXT', 'Failure'), ('AUDIO', 'Failure')]) 235 236 # FIXME: Seems like these should be classmethods on TestExpectationLine instead of TestExpectationParser. 237 @classmethod 238 def _tokenize_line(cls, filename, expectation_string, line_number): 239 """Tokenizes a line from TestExpectations and returns an unparsed TestExpectationLine instance using the old format. 240 241 The new format for a test expectation line is: 242 243 [[bugs] [ "[" <configuration specifiers> "]" <name> [ "[" <expectations> "]" ["#" <comment>] 244 245 Any errant whitespace is not preserved. 246 247 """ 248 expectation_line = TestExpectationLine() 249 expectation_line.original_string = expectation_string 250 expectation_line.filename = filename 251 expectation_line.line_numbers = str(line_number) 252 253 comment_index = expectation_string.find("#") 254 if comment_index == -1: 255 comment_index = len(expectation_string) 256 else: 257 expectation_line.comment = expectation_string[comment_index + 1:] 258 259 remaining_string = re.sub(r"\s+", " ", expectation_string[:comment_index].strip()) 260 if len(remaining_string) == 0: 261 return expectation_line 262 263 # special-case parsing this so that we fail immediately instead of treating this as a test name 264 if remaining_string.startswith('//'): 265 expectation_line.warnings = ['use "#" instead of "//" for comments'] 266 return expectation_line 267 268 bugs = [] 269 specifiers = [] 270 name = None 271 expectations = [] 272 warnings = [] 273 has_unrecognized_expectation = False 274 275 tokens = remaining_string.split() 276 state = 'start' 277 for token in tokens: 278 if (token.startswith(WEBKIT_BUG_PREFIX) or 279 token.startswith(CHROMIUM_BUG_PREFIX) or 280 token.startswith(V8_BUG_PREFIX) or 281 token.startswith(NAMED_BUG_PREFIX)): 282 if state != 'start': 283 warnings.append('"%s" is not at the start of the line.' % token) 284 break 285 if token.startswith(WEBKIT_BUG_PREFIX): 286 bugs.append(token) 287 elif token.startswith(CHROMIUM_BUG_PREFIX): 288 bugs.append(token) 289 elif token.startswith(V8_BUG_PREFIX): 290 bugs.append(token) 291 else: 292 match = re.match('Bug\((\w+)\)$', token) 293 if not match: 294 warnings.append('unrecognized bug identifier "%s"' % token) 295 break 296 else: 297 bugs.append(token) 298 elif token == '[': 299 if state == 'start': 300 state = 'configuration' 301 elif state == 'name_found': 302 state = 'expectations' 303 else: 304 warnings.append('unexpected "["') 305 break 306 elif token == ']': 307 if state == 'configuration': 308 state = 'name' 309 elif state == 'expectations': 310 state = 'done' 311 else: 312 warnings.append('unexpected "]"') 313 break 314 elif token in ('//', ':', '='): 315 warnings.append('"%s" is not legal in the new TestExpectations syntax.' % token) 316 break 317 elif state == 'configuration': 318 specifiers.append(cls._configuration_tokens.get(token, token)) 319 elif state == 'expectations': 320 if token not in cls._expectation_tokens: 321 has_unrecognized_expectation = True 322 warnings.append('Unrecognized expectation "%s"' % token) 323 else: 324 expectations.append(cls._expectation_tokens.get(token, token)) 325 elif state == 'name_found': 326 warnings.append('expecting "[", "#", or end of line instead of "%s"' % token) 327 break 328 else: 329 name = token 330 state = 'name_found' 331 332 if not warnings: 333 if not name: 334 warnings.append('Did not find a test name.') 335 elif state not in ('name_found', 'done'): 336 warnings.append('Missing a "]"') 337 338 if 'WONTFIX' in expectations and 'SKIP' not in expectations: 339 expectations.append('SKIP') 340 341 if ('SKIP' in expectations or 'WONTFIX' in expectations) and len(set(expectations) - set(['SKIP', 'WONTFIX'])): 342 warnings.append('A test marked Skip or WontFix must not have other expectations.') 343 344 if not expectations and not has_unrecognized_expectation: 345 warnings.append('Missing expectations.') 346 347 expectation_line.bugs = bugs 348 expectation_line.specifiers = specifiers 349 expectation_line.expectations = expectations 350 expectation_line.name = name 351 expectation_line.warnings = warnings 352 return expectation_line 353 354 @classmethod 355 def _split_space_separated(cls, space_separated_string): 356 """Splits a space-separated string into an array.""" 357 return [part.strip() for part in space_separated_string.strip().split(' ')] 358 359 360class TestExpectationLine(object): 361 """Represents a line in test expectations file.""" 362 363 def __init__(self): 364 """Initializes a blank-line equivalent of an expectation.""" 365 self.original_string = None 366 self.filename = None # this is the path to the expectations file for this line 367 self.line_numbers = "0" 368 self.name = None # this is the path in the line itself 369 self.path = None # this is the normpath of self.name 370 self.bugs = [] 371 self.specifiers = [] 372 self.parsed_specifiers = [] 373 self.matching_configurations = set() 374 self.expectations = [] 375 self.parsed_expectations = set() 376 self.comment = None 377 self.matching_tests = [] 378 self.warnings = [] 379 self.is_skipped_outside_expectations_file = False 380 381 def __eq__(self, other): 382 return (self.original_string == other.original_string 383 and self.filename == other.filename 384 and self.line_numbers == other.line_numbers 385 and self.name == other.name 386 and self.path == other.path 387 and self.bugs == other.bugs 388 and self.specifiers == other.specifiers 389 and self.parsed_specifiers == other.parsed_specifiers 390 and self.matching_configurations == other.matching_configurations 391 and self.expectations == other.expectations 392 and self.parsed_expectations == other.parsed_expectations 393 and self.comment == other.comment 394 and self.matching_tests == other.matching_tests 395 and self.warnings == other.warnings 396 and self.is_skipped_outside_expectations_file == other.is_skipped_outside_expectations_file) 397 398 def is_invalid(self): 399 return self.warnings and self.warnings != [TestExpectationParser.MISSING_BUG_WARNING] 400 401 def is_flaky(self): 402 return len(self.parsed_expectations) > 1 403 404 def is_whitespace_or_comment(self): 405 return bool(re.match("^\s*$", self.original_string.split('#')[0])) 406 407 @staticmethod 408 def create_passing_expectation(test): 409 expectation_line = TestExpectationLine() 410 expectation_line.name = test 411 expectation_line.path = test 412 expectation_line.parsed_expectations = set([PASS]) 413 expectation_line.expectations = set(['PASS']) 414 expectation_line.matching_tests = [test] 415 return expectation_line 416 417 @staticmethod 418 def merge_expectation_lines(line1, line2, model_all_expectations): 419 """Merges the expectations of line2 into line1 and returns a fresh object.""" 420 if line1 is None: 421 return line2 422 if line2 is None: 423 return line1 424 if model_all_expectations and line1.filename != line2.filename: 425 return line2 426 427 # Don't merge original_string or comment. 428 result = TestExpectationLine() 429 # We only care about filenames when we're linting, in which case the filenames are the same. 430 # Not clear that there's anything better to do when not linting and the filenames are different. 431 if model_all_expectations: 432 result.filename = line2.filename 433 result.line_numbers = line1.line_numbers + "," + line2.line_numbers 434 result.name = line1.name 435 result.path = line1.path 436 result.parsed_expectations = set(line1.parsed_expectations) | set(line2.parsed_expectations) 437 result.expectations = list(set(line1.expectations) | set(line2.expectations)) 438 result.bugs = list(set(line1.bugs) | set(line2.bugs)) 439 result.specifiers = list(set(line1.specifiers) | set(line2.specifiers)) 440 result.parsed_specifiers = list(set(line1.parsed_specifiers) | set(line2.parsed_specifiers)) 441 result.matching_configurations = set(line1.matching_configurations) | set(line2.matching_configurations) 442 result.matching_tests = list(list(set(line1.matching_tests) | set(line2.matching_tests))) 443 result.warnings = list(set(line1.warnings) | set(line2.warnings)) 444 result.is_skipped_outside_expectations_file = line1.is_skipped_outside_expectations_file or line2.is_skipped_outside_expectations_file 445 return result 446 447 def to_string(self, test_configuration_converter, include_specifiers=True, include_expectations=True, include_comment=True): 448 parsed_expectation_to_string = dict([[parsed_expectation, expectation_string] for expectation_string, parsed_expectation in TestExpectations.EXPECTATIONS.items()]) 449 450 if self.is_invalid(): 451 return self.original_string or '' 452 453 if self.name is None: 454 return '' if self.comment is None else "#%s" % self.comment 455 456 if test_configuration_converter and self.bugs: 457 specifiers_list = test_configuration_converter.to_specifiers_list(self.matching_configurations) 458 result = [] 459 for specifiers in specifiers_list: 460 # FIXME: this is silly that we join the specifiers and then immediately split them. 461 specifiers = self._serialize_parsed_specifiers(test_configuration_converter, specifiers).split() 462 expectations = self._serialize_parsed_expectations(parsed_expectation_to_string).split() 463 result.append(self._format_line(self.bugs, specifiers, self.name, expectations, self.comment)) 464 return "\n".join(result) if result else None 465 466 return self._format_line(self.bugs, self.specifiers, self.name, self.expectations, self.comment, 467 include_specifiers, include_expectations, include_comment) 468 469 def to_csv(self): 470 # Note that this doesn't include the comments. 471 return '%s,%s,%s,%s' % (self.name, ' '.join(self.bugs), ' '.join(self.specifiers), ' '.join(self.expectations)) 472 473 def _serialize_parsed_expectations(self, parsed_expectation_to_string): 474 result = [] 475 for index in TestExpectations.EXPECTATIONS.values(): 476 if index in self.parsed_expectations: 477 result.append(parsed_expectation_to_string[index]) 478 return ' '.join(result) 479 480 def _serialize_parsed_specifiers(self, test_configuration_converter, specifiers): 481 result = [] 482 result.extend(sorted(self.parsed_specifiers)) 483 result.extend(test_configuration_converter.specifier_sorter().sort_specifiers(specifiers)) 484 return ' '.join(result) 485 486 @staticmethod 487 def _filter_redundant_expectations(expectations): 488 if set(expectations) == set(['Pass', 'Skip']): 489 return ['Skip'] 490 if set(expectations) == set(['Pass', 'Slow']): 491 return ['Slow'] 492 return expectations 493 494 @staticmethod 495 def _format_line(bugs, specifiers, name, expectations, comment, include_specifiers=True, include_expectations=True, include_comment=True): 496 new_specifiers = [] 497 new_expectations = [] 498 for specifier in specifiers: 499 # FIXME: Make this all work with the mixed-cased specifiers (e.g. WontFix, Slow, etc). 500 specifier = specifier.upper() 501 new_specifiers.append(TestExpectationParser._inverted_configuration_tokens.get(specifier, specifier)) 502 503 for expectation in expectations: 504 expectation = expectation.upper() 505 new_expectations.append(TestExpectationParser._inverted_expectation_tokens.get(expectation, expectation)) 506 507 result = '' 508 if include_specifiers and (bugs or new_specifiers): 509 if bugs: 510 result += ' '.join(bugs) + ' ' 511 if new_specifiers: 512 result += '[ %s ] ' % ' '.join(new_specifiers) 513 result += name 514 if include_expectations and new_expectations: 515 new_expectations = TestExpectationLine._filter_redundant_expectations(new_expectations) 516 result += ' [ %s ]' % ' '.join(sorted(set(new_expectations))) 517 if include_comment and comment is not None: 518 result += " #%s" % comment 519 return result 520 521 522# FIXME: Refactor API to be a proper CRUD. 523class TestExpectationsModel(object): 524 """Represents relational store of all expectations and provides CRUD semantics to manage it.""" 525 526 def __init__(self, shorten_filename=None): 527 # Maps a test to its list of expectations. 528 self._test_to_expectations = {} 529 530 # Maps a test to list of its specifiers (string values) 531 self._test_to_specifiers = {} 532 533 # Maps a test to a TestExpectationLine instance. 534 self._test_to_expectation_line = {} 535 536 self._expectation_to_tests = self._dict_of_sets(TestExpectations.EXPECTATIONS) 537 self._timeline_to_tests = self._dict_of_sets(TestExpectations.TIMELINES) 538 self._result_type_to_tests = self._dict_of_sets(TestExpectations.RESULT_TYPES) 539 540 self._shorten_filename = shorten_filename or (lambda x: x) 541 542 def _merge_test_map(self, self_map, other_map): 543 for test in other_map: 544 new_expectations = set(other_map[test]) 545 if test in self_map: 546 new_expectations |= set(self_map[test]) 547 self_map[test] = list(new_expectations) if isinstance(other_map[test], list) else new_expectations 548 549 def _merge_dict_of_sets(self, self_dict, other_dict): 550 for key in other_dict: 551 self_dict[key] |= other_dict[key] 552 553 def merge_model(self, other): 554 self._merge_test_map(self._test_to_expectations, other._test_to_expectations) 555 556 for test, line in other._test_to_expectation_line.items(): 557 if test in self._test_to_expectation_line: 558 line = TestExpectationLine.merge_expectation_lines(self._test_to_expectation_line[test], line, model_all_expectations=False) 559 self._test_to_expectation_line[test] = line 560 561 self._merge_dict_of_sets(self._expectation_to_tests, other._expectation_to_tests) 562 self._merge_dict_of_sets(self._timeline_to_tests, other._timeline_to_tests) 563 self._merge_dict_of_sets(self._result_type_to_tests, other._result_type_to_tests) 564 565 def _dict_of_sets(self, strings_to_constants): 566 """Takes a dict of strings->constants and returns a dict mapping 567 each constant to an empty set.""" 568 d = {} 569 for c in strings_to_constants.values(): 570 d[c] = set() 571 return d 572 573 def get_test_set(self, expectation, include_skips=True): 574 tests = self._expectation_to_tests[expectation] 575 if not include_skips: 576 tests = tests - self.get_test_set(SKIP) 577 return tests 578 579 def get_test_set_for_keyword(self, keyword): 580 expectation_enum = TestExpectations.EXPECTATIONS.get(keyword.lower(), None) 581 if expectation_enum is not None: 582 return self._expectation_to_tests[expectation_enum] 583 584 matching_tests = set() 585 for test, specifiers in self._test_to_specifiers.iteritems(): 586 if keyword.lower() in specifiers: 587 matching_tests.add(test) 588 return matching_tests 589 590 def get_tests_with_result_type(self, result_type): 591 return self._result_type_to_tests[result_type] 592 593 def get_tests_with_timeline(self, timeline): 594 return self._timeline_to_tests[timeline] 595 596 def has_test(self, test): 597 return test in self._test_to_expectation_line 598 599 def get_expectation_line(self, test): 600 return self._test_to_expectation_line.get(test) 601 602 def get_expectations(self, test): 603 return self._test_to_expectations[test] 604 605 def get_expectations_string(self, test): 606 """Returns the expectatons for the given test as an uppercase string. 607 If there are no expectations for the test, then "PASS" is returned.""" 608 if self.get_expectation_line(test).is_skipped_outside_expectations_file: 609 return 'NOTRUN' 610 611 expectations = self.get_expectations(test) 612 retval = [] 613 614 # FIXME: WontFix should cause the test to get skipped without artificially adding SKIP to the expectations list. 615 if WONTFIX in expectations and SKIP in expectations: 616 expectations.remove(SKIP) 617 618 for expectation in expectations: 619 retval.append(self.expectation_to_string(expectation)) 620 621 return " ".join(retval) 622 623 def expectation_to_string(self, expectation): 624 """Return the uppercased string equivalent of a given expectation.""" 625 for item in TestExpectations.EXPECTATIONS.items(): 626 if item[1] == expectation: 627 return item[0].upper() 628 raise ValueError(expectation) 629 630 def remove_expectation_line(self, test): 631 if not self.has_test(test): 632 return 633 self._clear_expectations_for_test(test) 634 del self._test_to_expectation_line[test] 635 636 def add_expectation_line(self, expectation_line, 637 model_all_expectations=False): 638 """Returns a list of warnings encountered while matching specifiers.""" 639 640 if expectation_line.is_invalid(): 641 return 642 643 for test in expectation_line.matching_tests: 644 if self._already_seen_better_match(test, expectation_line): 645 continue 646 647 if model_all_expectations: 648 expectation_line = TestExpectationLine.merge_expectation_lines(self.get_expectation_line(test), expectation_line, model_all_expectations) 649 650 self._clear_expectations_for_test(test) 651 self._test_to_expectation_line[test] = expectation_line 652 self._add_test(test, expectation_line) 653 654 def _add_test(self, test, expectation_line): 655 """Sets the expected state for a given test. 656 657 This routine assumes the test has not been added before. If it has, 658 use _clear_expectations_for_test() to reset the state prior to 659 calling this.""" 660 self._test_to_expectations[test] = expectation_line.parsed_expectations 661 for expectation in expectation_line.parsed_expectations: 662 self._expectation_to_tests[expectation].add(test) 663 664 self._test_to_specifiers[test] = expectation_line.specifiers 665 666 if WONTFIX in expectation_line.parsed_expectations: 667 self._timeline_to_tests[WONTFIX].add(test) 668 else: 669 self._timeline_to_tests[NOW].add(test) 670 671 if SKIP in expectation_line.parsed_expectations: 672 self._result_type_to_tests[SKIP].add(test) 673 elif expectation_line.parsed_expectations == set([PASS]): 674 self._result_type_to_tests[PASS].add(test) 675 elif expectation_line.is_flaky(): 676 self._result_type_to_tests[FLAKY].add(test) 677 else: 678 # FIXME: What is this? 679 self._result_type_to_tests[FAIL].add(test) 680 681 def _clear_expectations_for_test(self, test): 682 """Remove prexisting expectations for this test. 683 This happens if we are seeing a more precise path 684 than a previous listing. 685 """ 686 if self.has_test(test): 687 self._test_to_expectations.pop(test, '') 688 self._remove_from_sets(test, self._expectation_to_tests) 689 self._remove_from_sets(test, self._timeline_to_tests) 690 self._remove_from_sets(test, self._result_type_to_tests) 691 692 def _remove_from_sets(self, test, dict_of_sets_of_tests): 693 """Removes the given test from the sets in the dictionary. 694 695 Args: 696 test: test to look for 697 dict: dict of sets of files""" 698 for set_of_tests in dict_of_sets_of_tests.itervalues(): 699 if test in set_of_tests: 700 set_of_tests.remove(test) 701 702 def _already_seen_better_match(self, test, expectation_line): 703 """Returns whether we've seen a better match already in the file. 704 705 Returns True if we've already seen a expectation_line.name that matches more of the test 706 than this path does 707 """ 708 # FIXME: See comment below about matching test configs and specificity. 709 if not self.has_test(test): 710 # We've never seen this test before. 711 return False 712 713 prev_expectation_line = self._test_to_expectation_line[test] 714 715 if prev_expectation_line.filename != expectation_line.filename: 716 # We've moved on to a new expectation file, which overrides older ones. 717 return False 718 719 if len(prev_expectation_line.path) > len(expectation_line.path): 720 # The previous path matched more of the test. 721 return True 722 723 if len(prev_expectation_line.path) < len(expectation_line.path): 724 # This path matches more of the test. 725 return False 726 727 # At this point we know we have seen a previous exact match on this 728 # base path, so we need to check the two sets of specifiers. 729 730 # FIXME: This code was originally designed to allow lines that matched 731 # more specifiers to override lines that matched fewer specifiers. 732 # However, we currently view these as errors. 733 # 734 # To use the "more specifiers wins" policy, change the errors for overrides 735 # to be warnings and return False". 736 737 if prev_expectation_line.matching_configurations == expectation_line.matching_configurations: 738 expectation_line.warnings.append('Duplicate or ambiguous entry lines %s:%s and %s:%s.' % ( 739 self._shorten_filename(prev_expectation_line.filename), prev_expectation_line.line_numbers, 740 self._shorten_filename(expectation_line.filename), expectation_line.line_numbers)) 741 return True 742 743 if prev_expectation_line.matching_configurations >= expectation_line.matching_configurations: 744 expectation_line.warnings.append('More specific entry for %s on line %s:%s overrides line %s:%s.' % (expectation_line.name, 745 self._shorten_filename(prev_expectation_line.filename), prev_expectation_line.line_numbers, 746 self._shorten_filename(expectation_line.filename), expectation_line.line_numbers)) 747 # FIXME: return False if we want more specific to win. 748 return True 749 750 if prev_expectation_line.matching_configurations <= expectation_line.matching_configurations: 751 expectation_line.warnings.append('More specific entry for %s on line %s:%s overrides line %s:%s.' % (expectation_line.name, 752 self._shorten_filename(expectation_line.filename), expectation_line.line_numbers, 753 self._shorten_filename(prev_expectation_line.filename), prev_expectation_line.line_numbers)) 754 return True 755 756 if prev_expectation_line.matching_configurations & expectation_line.matching_configurations: 757 expectation_line.warnings.append('Entries for %s on lines %s:%s and %s:%s match overlapping sets of configurations.' % (expectation_line.name, 758 self._shorten_filename(prev_expectation_line.filename), prev_expectation_line.line_numbers, 759 self._shorten_filename(expectation_line.filename), expectation_line.line_numbers)) 760 return True 761 762 # Configuration sets are disjoint, then. 763 return False 764 765 766class TestExpectations(object): 767 """Test expectations consist of lines with specifications of what 768 to expect from layout test cases. The test cases can be directories 769 in which case the expectations apply to all test cases in that 770 directory and any subdirectory. The format is along the lines of: 771 772 LayoutTests/fast/js/fixme.js [ Failure ] 773 LayoutTests/fast/js/flaky.js [ Failure Pass ] 774 LayoutTests/fast/js/crash.js [ Crash Failure Pass Timeout ] 775 ... 776 777 To add specifiers: 778 LayoutTests/fast/js/no-good.js 779 [ Debug ] LayoutTests/fast/js/no-good.js [ Pass Timeout ] 780 [ Debug ] LayoutTests/fast/js/no-good.js [ Pass Skip Timeout ] 781 [ Linux Debug ] LayoutTests/fast/js/no-good.js [ Pass Skip Timeout ] 782 [ Linux Win ] LayoutTests/fast/js/no-good.js [ Pass Skip Timeout ] 783 784 Skip: Doesn't run the test. 785 Slow: The test takes a long time to run, but does not timeout indefinitely. 786 WontFix: For tests that we never intend to pass on a given platform (treated like Skip). 787 788 Notes: 789 -A test cannot be both SLOW and TIMEOUT 790 -A test can be included twice, but not via the same path. 791 -If a test is included twice, then the more precise path wins. 792 -CRASH tests cannot be WONTFIX 793 """ 794 795 # FIXME: Update to new syntax once the old format is no longer supported. 796 EXPECTATIONS = {'pass': PASS, 797 'audio': AUDIO, 798 'fail': FAIL, 799 'image': IMAGE, 800 'image+text': IMAGE_PLUS_TEXT, 801 'text': TEXT, 802 'timeout': TIMEOUT, 803 'crash': CRASH, 804 'missing': MISSING, 805 TestExpectationParser.SKIP_MODIFIER: SKIP, 806 TestExpectationParser.NEEDS_REBASELINE_MODIFIER: NEEDS_REBASELINE, 807 TestExpectationParser.NEEDS_MANUAL_REBASELINE_MODIFIER: NEEDS_MANUAL_REBASELINE, 808 TestExpectationParser.WONTFIX_MODIFIER: WONTFIX, 809 TestExpectationParser.SLOW_MODIFIER: SLOW, 810 TestExpectationParser.REBASELINE_MODIFIER: REBASELINE, 811 } 812 813 EXPECTATIONS_TO_STRING = dict((k, v) for (v, k) in EXPECTATIONS.iteritems()) 814 815 # (aggregated by category, pass/fail/skip, type) 816 EXPECTATION_DESCRIPTIONS = {SKIP: 'skipped', 817 PASS: 'passes', 818 FAIL: 'failures', 819 IMAGE: 'image-only failures', 820 TEXT: 'text-only failures', 821 IMAGE_PLUS_TEXT: 'image and text failures', 822 AUDIO: 'audio failures', 823 CRASH: 'crashes', 824 TIMEOUT: 'timeouts', 825 MISSING: 'missing results'} 826 827 NON_TEST_OUTCOME_EXPECTATIONS = (REBASELINE, SKIP, SLOW, WONTFIX) 828 829 BUILD_TYPES = ('debug', 'release') 830 831 TIMELINES = {TestExpectationParser.WONTFIX_MODIFIER: WONTFIX, 832 'now': NOW} 833 834 RESULT_TYPES = {'skip': SKIP, 835 'pass': PASS, 836 'fail': FAIL, 837 'flaky': FLAKY} 838 839 @classmethod 840 def expectation_from_string(cls, string): 841 assert(' ' not in string) # This only handles one expectation at a time. 842 return cls.EXPECTATIONS.get(string.lower()) 843 844 @staticmethod 845 def result_was_expected(result, expected_results, test_needs_rebaselining): 846 """Returns whether we got a result we were expecting. 847 Args: 848 result: actual result of a test execution 849 expected_results: set of results listed in test_expectations 850 test_needs_rebaselining: whether test was marked as REBASELINE""" 851 if not (set(expected_results) - (set(TestExpectations.NON_TEST_OUTCOME_EXPECTATIONS))): 852 expected_results = set([PASS]) 853 854 if result in expected_results: 855 return True 856 if result in (PASS, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, MISSING) and (NEEDS_REBASELINE in expected_results or NEEDS_MANUAL_REBASELINE in expected_results): 857 return True 858 if result in (TEXT, IMAGE_PLUS_TEXT, AUDIO) and (FAIL in expected_results): 859 return True 860 if result == MISSING and test_needs_rebaselining: 861 return True 862 if result == SKIP: 863 return True 864 return False 865 866 @staticmethod 867 def remove_pixel_failures(expected_results): 868 """Returns a copy of the expected results for a test, except that we 869 drop any pixel failures and return the remaining expectations. For example, 870 if we're not running pixel tests, then tests expected to fail as IMAGE 871 will PASS.""" 872 expected_results = expected_results.copy() 873 if IMAGE in expected_results: 874 expected_results.remove(IMAGE) 875 expected_results.add(PASS) 876 return expected_results 877 878 @staticmethod 879 def has_pixel_failures(actual_results): 880 return IMAGE in actual_results or FAIL in actual_results 881 882 @staticmethod 883 def suffixes_for_expectations(expectations): 884 suffixes = set() 885 if IMAGE in expectations: 886 suffixes.add('png') 887 if FAIL in expectations: 888 suffixes.add('txt') 889 suffixes.add('png') 890 suffixes.add('wav') 891 return set(suffixes) 892 893 @staticmethod 894 def suffixes_for_actual_expectations_string(expectations): 895 suffixes = set() 896 if 'TEXT' in expectations: 897 suffixes.add('txt') 898 if 'IMAGE' in expectations: 899 suffixes.add('png') 900 if 'AUDIO' in expectations: 901 suffixes.add('wav') 902 if 'MISSING' in expectations: 903 suffixes.add('txt') 904 suffixes.add('png') 905 suffixes.add('wav') 906 return suffixes 907 908 # FIXME: This constructor does too much work. We should move the actual parsing of 909 # the expectations into separate routines so that linting and handling overrides 910 # can be controlled separately, and the constructor can be more of a no-op. 911 def __init__(self, port, tests=None, include_overrides=True, expectations_dict=None, model_all_expectations=False, is_lint_mode=False): 912 self._full_test_list = tests 913 self._test_config = port.test_configuration() 914 self._is_lint_mode = is_lint_mode 915 self._model_all_expectations = self._is_lint_mode or model_all_expectations 916 self._model = TestExpectationsModel(self._shorten_filename) 917 self._parser = TestExpectationParser(port, tests, self._is_lint_mode) 918 self._port = port 919 self._skipped_tests_warnings = [] 920 self._expectations = [] 921 922 if not expectations_dict: 923 expectations_dict = port.expectations_dict() 924 925 # Always parse the generic expectations (the generic file is required 926 # to be the first one in the expectations_dict, which must be an OrderedDict). 927 generic_path, generic_exps = expectations_dict.items()[0] 928 expectations = self._parser.parse(generic_path, generic_exps) 929 self._add_expectations(expectations, self._model) 930 self._expectations += expectations 931 932 # Now add the overrides if so requested. 933 if include_overrides: 934 for path, contents in expectations_dict.items()[1:]: 935 expectations = self._parser.parse(path, contents) 936 model = TestExpectationsModel(self._shorten_filename) 937 self._add_expectations(expectations, model) 938 self._expectations += expectations 939 self._model.merge_model(model) 940 941 # FIXME: move ignore_tests into port.skipped_layout_tests() 942 self.add_extra_skipped_tests(port.skipped_layout_tests(tests).union(set(port.get_option('ignore_tests', [])))) 943 self.add_expectations_from_bot() 944 945 self._has_warnings = False 946 self._report_warnings() 947 self._process_tests_without_expectations() 948 949 # TODO(ojan): Allow for removing skipped tests when getting the list of 950 # tests to run, but not when getting metrics. 951 def model(self): 952 return self._model 953 954 def get_needs_rebaseline_failures(self): 955 return self._model.get_test_set(NEEDS_REBASELINE) 956 957 def get_rebaselining_failures(self): 958 return self._model.get_test_set(REBASELINE) 959 960 # FIXME: Change the callsites to use TestExpectationsModel and remove. 961 def get_expectations(self, test): 962 return self._model.get_expectations(test) 963 964 # FIXME: Change the callsites to use TestExpectationsModel and remove. 965 def get_tests_with_result_type(self, result_type): 966 return self._model.get_tests_with_result_type(result_type) 967 968 # FIXME: Change the callsites to use TestExpectationsModel and remove. 969 def get_test_set(self, expectation, include_skips=True): 970 return self._model.get_test_set(expectation, include_skips) 971 972 # FIXME: Change the callsites to use TestExpectationsModel and remove. 973 def get_tests_with_timeline(self, timeline): 974 return self._model.get_tests_with_timeline(timeline) 975 976 def get_expectations_string(self, test): 977 return self._model.get_expectations_string(test) 978 979 def expectation_to_string(self, expectation): 980 return self._model.expectation_to_string(expectation) 981 982 def matches_an_expected_result(self, test, result, pixel_tests_are_enabled): 983 expected_results = self._model.get_expectations(test) 984 if not pixel_tests_are_enabled: 985 expected_results = self.remove_pixel_failures(expected_results) 986 return self.result_was_expected(result, expected_results, self.is_rebaselining(test)) 987 988 def is_rebaselining(self, test): 989 return REBASELINE in self._model.get_expectations(test) 990 991 def _shorten_filename(self, filename): 992 if filename.startswith(self._port.path_from_webkit_base()): 993 return self._port.host.filesystem.relpath(filename, self._port.path_from_webkit_base()) 994 return filename 995 996 def _report_warnings(self): 997 warnings = [] 998 for expectation in self._expectations: 999 for warning in expectation.warnings: 1000 warnings.append('%s:%s %s %s' % (self._shorten_filename(expectation.filename), expectation.line_numbers, 1001 warning, expectation.name if expectation.expectations else expectation.original_string)) 1002 1003 if warnings: 1004 self._has_warnings = True 1005 if self._is_lint_mode: 1006 raise ParseError(warnings) 1007 _log.warning('--lint-test-files warnings:') 1008 for warning in warnings: 1009 _log.warning(warning) 1010 _log.warning('') 1011 1012 def _process_tests_without_expectations(self): 1013 if self._full_test_list: 1014 for test in self._full_test_list: 1015 if not self._model.has_test(test): 1016 self._model.add_expectation_line(TestExpectationLine.create_passing_expectation(test)) 1017 1018 def has_warnings(self): 1019 return self._has_warnings 1020 1021 def remove_configuration_from_test(self, test, test_configuration): 1022 expectations_to_remove = [] 1023 modified_expectations = [] 1024 1025 for expectation in self._expectations: 1026 if expectation.name != test or not expectation.parsed_expectations: 1027 continue 1028 if test_configuration not in expectation.matching_configurations: 1029 continue 1030 1031 expectation.matching_configurations.remove(test_configuration) 1032 if expectation.matching_configurations: 1033 modified_expectations.append(expectation) 1034 else: 1035 expectations_to_remove.append(expectation) 1036 1037 for expectation in expectations_to_remove: 1038 index = self._expectations.index(expectation) 1039 self._expectations.remove(expectation) 1040 1041 if index == len(self._expectations) or self._expectations[index].is_whitespace_or_comment(): 1042 while index and self._expectations[index - 1].is_whitespace_or_comment(): 1043 index = index - 1 1044 self._expectations.pop(index) 1045 1046 return self.list_to_string(self._expectations, self._parser._test_configuration_converter, modified_expectations) 1047 1048 def _add_expectations(self, expectation_list, model): 1049 for expectation_line in expectation_list: 1050 if not expectation_line.expectations: 1051 continue 1052 1053 if self._model_all_expectations or self._test_config in expectation_line.matching_configurations: 1054 model.add_expectation_line(expectation_line, model_all_expectations=self._model_all_expectations) 1055 1056 def add_extra_skipped_tests(self, tests_to_skip): 1057 if not tests_to_skip: 1058 return 1059 for test in self._expectations: 1060 if test.name and test.name in tests_to_skip: 1061 test.warnings.append('%s:%s %s is also in a Skipped file.' % (test.filename, test.line_numbers, test.name)) 1062 1063 model = TestExpectationsModel(self._shorten_filename) 1064 for test_name in tests_to_skip: 1065 expectation_line = self._parser.expectation_for_skipped_test(test_name) 1066 model.add_expectation_line(expectation_line) 1067 self._model.merge_model(model) 1068 1069 def add_expectations_from_bot(self): 1070 # FIXME: With mode 'very-flaky' and 'maybe-flaky', this will show the expectations entry in the flakiness 1071 # dashboard rows for each test to be whatever the bot thinks they should be. Is this a good thing? 1072 bot_expectations = self._port.bot_expectations() 1073 model = TestExpectationsModel(self._shorten_filename) 1074 for test_name in bot_expectations: 1075 expectation_line = self._parser.expectation_line_for_test(test_name, bot_expectations[test_name]) 1076 1077 # Unexpected results are merged into existing expectations. 1078 merge = self._port.get_option('ignore_flaky_tests') == 'unexpected' 1079 model.add_expectation_line(expectation_line) 1080 self._model.merge_model(model) 1081 1082 def add_expectation_line(self, expectation_line): 1083 self._model.add_expectation_line(expectation_line) 1084 self._expectations += [expectation_line] 1085 1086 def remove_expectation_line(self, test): 1087 if not self._model.has_test(test): 1088 return 1089 self._expectations.remove(self._model.get_expectation_line(test)) 1090 self._model.remove_expectation_line(test) 1091 1092 @staticmethod 1093 def list_to_string(expectation_lines, test_configuration_converter=None, reconstitute_only_these=None): 1094 def serialize(expectation_line): 1095 # If reconstitute_only_these is an empty list, we want to return original_string. 1096 # So we need to compare reconstitute_only_these to None, not just check if it's falsey. 1097 if reconstitute_only_these is None or expectation_line in reconstitute_only_these: 1098 return expectation_line.to_string(test_configuration_converter) 1099 return expectation_line.original_string 1100 1101 def nones_out(expectation_line): 1102 return expectation_line is not None 1103 1104 return "\n".join(filter(nones_out, map(serialize, expectation_lines))) 1105