1#!/usr/bin/env vpython3 2# Copyright 2020 The Chromium Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6import datetime 7import os 8import tempfile 9import unittest 10from unittest import mock 11 12from pyfakefs import fake_filesystem_unittest 13 14from unexpected_passes_common import data_types 15from unexpected_passes_common import expectations 16from unexpected_passes_common import unittest_utils as uu 17 18# Protected access is allowed for unittests. 19# pylint: disable=protected-access 20 21FAKE_EXPECTATION_FILE_CONTENTS = """\ 22# tags: [ win linux ] 23# results: [ Failure RetryOnFailure Skip Pass ] 24crbug.com/1234 [ win ] foo/test [ Failure ] 25crbug.com/5678 crbug.com/6789 [ win ] foo/another/test [ RetryOnFailure ] 26 27[ linux ] foo/test [ Failure ] 28 29crbug.com/2345 [ linux ] bar/* [ RetryOnFailure ] 30crbug.com/3456 [ linux ] some/bad/test [ Skip ] 31crbug.com/4567 [ linux ] some/good/test [ Pass ] 32""" 33 34SECONDARY_FAKE_EXPECTATION_FILE_CONTENTS = """\ 35# tags: [ mac ] 36# results: [ Failure ] 37 38crbug.com/4567 [ mac ] foo/test [ Failure ] 39""" 40 41FAKE_EXPECTATION_FILE_CONTENTS_WITH_TYPO = """\ 42# tags: [ win linux ] 43# results: [ Failure RetryOnFailure Skip ] 44crbug.com/1234 [ wine ] foo/test [ Failure ] 45 46[ linux ] foo/test [ Failure ] 47 48crbug.com/2345 [ linux ] bar/* [ RetryOnFailure ] 49crbug.com/3456 [ linux ] some/bad/test [ Skip ] 50""" 51 52FAKE_EXPECTATION_FILE_CONTENTS_WITH_COMPLEX_TAGS = """\ 53# tags: [ win win10 54# linux 55# mac ] 56# tags: [ nvidia nvidia-0x1111 57# intel intel-0x2222 58# amd amd-0x3333] 59# tags: [ release debug ] 60# results: [ Failure RetryOnFailure ] 61 62crbug.com/1234 [ win ] foo/test [ Failure ] 63crbug.com/2345 [ linux ] foo/test [ RetryOnFailure ] 64""" 65 66 67class CreateTestExpectationMapUnittest(unittest.TestCase): 68 def setUp(self) -> None: 69 self.instance = expectations.Expectations() 70 71 self._expectation_content = {} 72 self._content_patcher = mock.patch.object( 73 self.instance, '_GetNonRecentExpectationContent') 74 self._content_mock = self._content_patcher.start() 75 self.addCleanup(self._content_patcher.stop) 76 77 def SideEffect(filepath, _): 78 return self._expectation_content[filepath] 79 80 self._content_mock.side_effect = SideEffect 81 82 def testExclusiveOr(self) -> None: 83 """Tests that only one input can be specified.""" 84 with self.assertRaises(AssertionError): 85 self.instance.CreateTestExpectationMap(None, None, 86 datetime.timedelta(days=0)) 87 with self.assertRaises(AssertionError): 88 self.instance.CreateTestExpectationMap('foo', ['bar'], 89 datetime.timedelta(days=0)) 90 91 def testExpectationFile(self) -> None: 92 """Tests reading expectations from an expectation file.""" 93 filename = '/tmp/foo' 94 self._expectation_content[filename] = FAKE_EXPECTATION_FILE_CONTENTS 95 expectation_map = self.instance.CreateTestExpectationMap( 96 filename, None, datetime.timedelta(days=0)) 97 # Skip expectations should be omitted, but everything else should be 98 # present. 99 # yapf: disable 100 expected_expectation_map = { 101 filename: { 102 data_types.Expectation( 103 'foo/test', ['win'], ['Failure'], 'crbug.com/1234'): {}, 104 data_types.Expectation( 105 'foo/another/test', ['win'], ['RetryOnFailure'], 106 'crbug.com/5678 crbug.com/6789'): {}, 107 data_types.Expectation('foo/test', ['linux'], ['Failure']): {}, 108 data_types.Expectation( 109 'bar/*', ['linux'], ['RetryOnFailure'], 'crbug.com/2345'): {}, 110 }, 111 } 112 # yapf: enable 113 self.assertEqual(expectation_map, expected_expectation_map) 114 self.assertIsInstance(expectation_map, data_types.TestExpectationMap) 115 116 def testMultipleExpectationFiles(self) -> None: 117 """Tests reading expectations from multiple files.""" 118 filename1 = '/tmp/foo' 119 filename2 = '/tmp/bar' 120 expectation_files = [filename1, filename2] 121 self._expectation_content[filename1] = FAKE_EXPECTATION_FILE_CONTENTS 122 self._expectation_content[ 123 filename2] = SECONDARY_FAKE_EXPECTATION_FILE_CONTENTS 124 125 expectation_map = self.instance.CreateTestExpectationMap( 126 expectation_files, None, datetime.timedelta(days=0)) 127 # yapf: disable 128 expected_expectation_map = { 129 expectation_files[0]: { 130 data_types.Expectation( 131 'foo/test', ['win'], ['Failure'], 'crbug.com/1234'): {}, 132 data_types.Expectation( 133 'foo/another/test', ['win'], ['RetryOnFailure'], 134 'crbug.com/5678 crbug.com/6789'): {}, 135 data_types.Expectation('foo/test', ['linux'], ['Failure']): {}, 136 data_types.Expectation( 137 'bar/*', ['linux'], ['RetryOnFailure'], 'crbug.com/2345'): {}, 138 }, 139 expectation_files[1]: { 140 data_types.Expectation( 141 'foo/test', ['mac'], ['Failure'], 'crbug.com/4567'): {}, 142 } 143 } 144 # yapf: enable 145 self.assertEqual(expectation_map, expected_expectation_map) 146 self.assertIsInstance(expectation_map, data_types.TestExpectationMap) 147 148 def testIndividualTests(self) -> None: 149 """Tests reading expectations from a list of tests.""" 150 expectation_map = self.instance.CreateTestExpectationMap( 151 None, ['foo/test', 'bar/*'], datetime.timedelta(days=0)) 152 expected_expectation_map = { 153 '': { 154 data_types.Expectation('foo/test', [], ['RetryOnFailure']): {}, 155 data_types.Expectation('bar/*', [], ['RetryOnFailure']): {}, 156 }, 157 } 158 self.assertEqual(expectation_map, expected_expectation_map) 159 self.assertIsInstance(expectation_map, data_types.TestExpectationMap) 160 161 162class GetNonRecentExpectationContentUnittest(unittest.TestCase): 163 def setUp(self) -> None: 164 self.instance = uu.CreateGenericExpectations() 165 self._output_patcher = mock.patch( 166 'unexpected_passes_common.expectations.subprocess.check_output') 167 self._output_mock = self._output_patcher.start() 168 self.addCleanup(self._output_patcher.stop) 169 170 def testBasic(self) -> None: 171 """Tests that only expectations that are old enough are kept.""" 172 today_date = datetime.date.today() 173 yesterday_date = today_date - datetime.timedelta(days=1) 174 older_date = today_date - datetime.timedelta(days=2) 175 today_str = today_date.isoformat() 176 yesterday_str = yesterday_date.isoformat() 177 older_str = older_date.isoformat() 178 # pylint: disable=line-too-long 179 blame_output = """\ 1805f03bc04975c04 (Some R. Author {today_date} 00:00:00 +0000 1)# tags: [ tag1 ] 18198637cd80f8c15 (Some R. Author {yesterday_date} 00:00:00 +0000 2)# tags: [ tag2 ] 1823fcadac9d861d0 (Some R. Author {older_date} 00:00:00 +0000 3)# results: [ Failure ] 1835f03bc04975c04 (Some R. Author {today_date} 00:00:00 +0000 4) 1845f03bc04975c04 (Some R. Author {today_date} 00:00:00 +0000 5)crbug.com/1234 [ tag1 ] testname [ Failure ] 18598637cd80f8c15 (Some R. Author {yesterday_date} 00:00:00 +0000 6)[ tag2 ] testname [ Failure ] # Comment 1863fcadac9d861d0 (Some R. Author {older_date} 00:00:00 +0000 7)[ tag1 ] othertest [ Failure ] 1875f03bc04975c04 (Some R. Author {today_date} 00:00:00 +0000 8)crbug.com/2345 testname [ Failure ] 1883fcadac9d861d0 (Some R. Author {older_date} 00:00:00 +0000 9)crbug.com/3456 othertest [ Failure ]""" 189 # pylint: enable=line-too-long 190 blame_output = blame_output.format(today_date=today_str, 191 yesterday_date=yesterday_str, 192 older_date=older_str) 193 self._output_mock.return_value = blame_output.encode('utf-8') 194 195 expected_content = """\ 196# tags: [ tag1 ] 197# tags: [ tag2 ] 198# results: [ Failure ] 199 200[ tag1 ] othertest [ Failure ] 201crbug.com/3456 othertest [ Failure ]""" 202 self.assertEqual( 203 self.instance._GetNonRecentExpectationContent( 204 '', datetime.timedelta(days=1)), expected_content) 205 206 def testNegativeGracePeriod(self) -> None: 207 """Tests that setting a negative grace period disables filtering.""" 208 today_date = datetime.date.today() 209 yesterday_date = today_date - datetime.timedelta(days=1) 210 older_date = today_date - datetime.timedelta(days=2) 211 today_str = today_date.isoformat() 212 yesterday_str = yesterday_date.isoformat() 213 older_str = older_date.isoformat() 214 # pylint: disable=line-too-long 215 blame_output = """\ 2165f03bc04975c04 (Some R. Author {today_date} 00:00:00 +0000 1)# tags: [ tag1 ] 21798637cd80f8c15 (Some R. Author {yesterday_date} 00:00:00 +0000 2)# tags: [ tag2 ] 2183fcadac9d861d0 (Some R. Author {older_date} 00:00:00 +0000 3)# results: [ Failure ] 2195f03bc04975c04 (Some R. Author {today_date} 00:00:00 +0000 4) 2205f03bc04975c04 (Some R. Author {today_date} 00:00:00 +0000 5)crbug.com/1234 [ tag1 ] testname [ Failure ] 22198637cd80f8c15 (Some R. Author {yesterday_date} 00:00:00 +0000 6)[ tag2 ] testname [ Failure ] # Comment 2223fcadac9d861d0 (Some R. Author {older_date} 00:00:00 +0000 7)[ tag1 ] othertest [ Failure ]""" 223 # pylint: enable=line-too-long 224 blame_output = blame_output.format(today_date=today_str, 225 yesterday_date=yesterday_str, 226 older_date=older_str) 227 self._output_mock.return_value = blame_output.encode('utf-8') 228 229 expected_content = """\ 230# tags: [ tag1 ] 231# tags: [ tag2 ] 232# results: [ Failure ] 233 234crbug.com/1234 [ tag1 ] testname [ Failure ] 235[ tag2 ] testname [ Failure ] # Comment 236[ tag1 ] othertest [ Failure ]""" 237 self.assertEqual( 238 self.instance._GetNonRecentExpectationContent( 239 '', datetime.timedelta(days=-1)), expected_content) 240 241 242class RemoveExpectationsFromFileUnittest(fake_filesystem_unittest.TestCase): 243 def setUp(self) -> None: 244 self.instance = uu.CreateGenericExpectations() 245 self.header = self.instance._GetExpectationFileTagHeader(None) 246 self.setUpPyfakefs() 247 with tempfile.NamedTemporaryFile(delete=False) as f: 248 self.filename = f.name 249 250 def testExpectationRemoval(self) -> None: 251 """Tests that expectations are properly removed from a file.""" 252 contents = self.header + """ 253 254# This is a test comment 255crbug.com/1234 [ win ] foo/test [ Failure ] 256crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 257 258# Another comment 259[ linux ] bar/test [ RetryOnFailure ] 260[ win ] bar/test [ RetryOnFailure ] 261""" 262 263 stale_expectations = [ 264 data_types.Expectation('foo/test', ['win'], ['Failure'], 265 'crbug.com/1234'), 266 data_types.Expectation('bar/test', ['linux'], ['RetryOnFailure']) 267 ] 268 269 expected_contents = self.header + """ 270 271# This is a test comment 272crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 273 274# Another comment 275[ win ] bar/test [ RetryOnFailure ] 276""" 277 278 with open(self.filename, 'w') as f: 279 f.write(contents) 280 281 removed_urls = self.instance.RemoveExpectationsFromFile( 282 stale_expectations, self.filename, expectations.RemovalType.STALE) 283 self.assertEqual(removed_urls, set(['crbug.com/1234'])) 284 with open(self.filename) as f: 285 self.assertEqual(f.read(), expected_contents) 286 287 def testRemovalWithMultipleBugs(self) -> None: 288 """Tests removal of expectations with multiple associated bugs.""" 289 contents = self.header + """ 290 291# This is a test comment 292crbug.com/1234 crbug.com/3456 crbug.com/4567 [ win ] foo/test [ Failure ] 293crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 294 295# Another comment 296[ linux ] bar/test [ RetryOnFailure ] 297[ win ] bar/test [ RetryOnFailure ] 298""" 299 300 stale_expectations = [ 301 data_types.Expectation('foo/test', ['win'], ['Failure'], 302 'crbug.com/1234 crbug.com/3456 crbug.com/4567'), 303 ] 304 expected_contents = self.header + """ 305 306# This is a test comment 307crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 308 309# Another comment 310[ linux ] bar/test [ RetryOnFailure ] 311[ win ] bar/test [ RetryOnFailure ] 312""" 313 with open(self.filename, 'w') as f: 314 f.write(contents) 315 316 removed_urls = self.instance.RemoveExpectationsFromFile( 317 stale_expectations, self.filename, expectations.RemovalType.STALE) 318 self.assertEqual( 319 removed_urls, 320 set(['crbug.com/1234', 'crbug.com/3456', 'crbug.com/4567'])) 321 with open(self.filename) as f: 322 self.assertEqual(f.read(), expected_contents) 323 324 def testGeneralBlockComments(self) -> None: 325 """Tests that expectations in a disable block comment are not removed.""" 326 contents = self.header + """ 327crbug.com/1234 [ win ] foo/test [ Failure ] 328# finder:disable-general 329crbug.com/2345 [ win ] foo/test [ Failure ] 330crbug.com/3456 [ win ] foo/test [ Failure ] 331# finder:enable-general 332crbug.com/4567 [ win ] foo/test [ Failure ] 333""" 334 stale_expectations = [ 335 data_types.Expectation('foo/test', ['win'], ['Failure'], 336 'crbug.com/1234'), 337 data_types.Expectation('foo/test', ['win'], ['Failure'], 338 'crbug.com/2345'), 339 data_types.Expectation('foo/test', ['win'], ['Failure'], 340 'crbug.com/3456'), 341 data_types.Expectation('foo/test', ['win'], ['Failure'], 342 'crbug.com/4567'), 343 ] 344 expected_contents = self.header + """ 345# finder:disable-general 346crbug.com/2345 [ win ] foo/test [ Failure ] 347crbug.com/3456 [ win ] foo/test [ Failure ] 348# finder:enable-general 349""" 350 for removal_type in (expectations.RemovalType.STALE, 351 expectations.RemovalType.UNUSED): 352 with open(self.filename, 'w') as f: 353 f.write(contents) 354 removed_urls = self.instance.RemoveExpectationsFromFile( 355 stale_expectations, self.filename, removal_type) 356 self.assertEqual(removed_urls, set(['crbug.com/1234', 'crbug.com/4567'])) 357 with open(self.filename) as f: 358 self.assertEqual(f.read(), expected_contents) 359 360 def testStaleBlockComments(self) -> None: 361 """Tests that stale expectations in a stale disable block are not removed""" 362 contents = self.header + """ 363crbug.com/1234 [ win ] not_stale [ Failure ] 364crbug.com/1234 [ win ] before_block [ Failure ] 365# finder:disable-stale 366crbug.com/2345 [ win ] in_block [ Failure ] 367# finder:enable-stale 368crbug.com/3456 [ win ] after_block [ Failure ] 369""" 370 stale_expectations = [ 371 data_types.Expectation('before_block', ['win'], 'Failure', 372 'crbug.com/1234'), 373 data_types.Expectation('in_block', ['win'], 'Failure', 374 'crbug.com/2345'), 375 data_types.Expectation('after_block', ['win'], 'Failure', 376 'crbug.com/3456'), 377 ] 378 expected_contents = self.header + """ 379crbug.com/1234 [ win ] not_stale [ Failure ] 380# finder:disable-stale 381crbug.com/2345 [ win ] in_block [ Failure ] 382# finder:enable-stale 383""" 384 with open(self.filename, 'w') as f: 385 f.write(contents) 386 removed_urls = self.instance.RemoveExpectationsFromFile( 387 stale_expectations, self.filename, expectations.RemovalType.STALE) 388 self.assertEqual(removed_urls, set(['crbug.com/1234', 'crbug.com/3456'])) 389 with open(self.filename) as f: 390 self.assertEqual(f.read(), expected_contents) 391 392 def testUnusedBlockComments(self) -> None: 393 """Tests that stale expectations in unused disable blocks are not removed""" 394 contents = self.header + """ 395crbug.com/1234 [ win ] not_unused [ Failure ] 396crbug.com/1234 [ win ] before_block [ Failure ] 397# finder:disable-unused 398crbug.com/2345 [ win ] in_block [ Failure ] 399# finder:enable-unused 400crbug.com/3456 [ win ] after_block [ Failure ] 401""" 402 unused_expectations = [ 403 data_types.Expectation('before_block', ['win'], 'Failure', 404 'crbug.com/1234'), 405 data_types.Expectation('in_block', ['win'], 'Failure', 406 'crbug.com/2345'), 407 data_types.Expectation('after_block', ['win'], 'Failure', 408 'crbug.com/3456'), 409 ] 410 expected_contents = self.header + """ 411crbug.com/1234 [ win ] not_unused [ Failure ] 412# finder:disable-unused 413crbug.com/2345 [ win ] in_block [ Failure ] 414# finder:enable-unused 415""" 416 with open(self.filename, 'w') as f: 417 f.write(contents) 418 removed_urls = self.instance.RemoveExpectationsFromFile( 419 unused_expectations, self.filename, expectations.RemovalType.UNUSED) 420 self.assertEqual(removed_urls, set(['crbug.com/1234', 'crbug.com/3456'])) 421 with open(self.filename) as f: 422 self.assertEqual(f.read(), expected_contents) 423 424 def testMismatchedBlockComments(self) -> None: 425 """Tests that block comments for the wrong removal type do nothing.""" 426 contents = self.header + """ 427crbug.com/1234 [ win ] do_not_remove [ Failure ] 428# finder:disable-stale 429crbug.com/2345 [ win ] disabled_stale [ Failure ] 430# finder:enable-stale 431# finder:disable-unused 432crbug.com/3456 [ win ] disabled_unused [ Failure ] 433# finder:enable-unused 434crbug.com/4567 [ win ] also_do_not_remove [ Failure ] 435""" 436 expectations_to_remove = [ 437 data_types.Expectation('disabled_stale', ['win'], 'Failure', 438 'crbug.com/2345'), 439 data_types.Expectation('disabled_unused', ['win'], 'Failure', 440 'crbug.com/3456'), 441 ] 442 443 expected_contents = self.header + """ 444crbug.com/1234 [ win ] do_not_remove [ Failure ] 445# finder:disable-stale 446crbug.com/2345 [ win ] disabled_stale [ Failure ] 447# finder:enable-stale 448crbug.com/4567 [ win ] also_do_not_remove [ Failure ] 449""" 450 with open(self.filename, 'w') as f: 451 f.write(contents) 452 removed_urls = self.instance.RemoveExpectationsFromFile( 453 expectations_to_remove, self.filename, expectations.RemovalType.STALE) 454 self.assertEqual(removed_urls, set(['crbug.com/3456'])) 455 with open(self.filename) as f: 456 self.assertEqual(f.read(), expected_contents) 457 458 expected_contents = self.header + """ 459crbug.com/1234 [ win ] do_not_remove [ Failure ] 460# finder:disable-unused 461crbug.com/3456 [ win ] disabled_unused [ Failure ] 462# finder:enable-unused 463crbug.com/4567 [ win ] also_do_not_remove [ Failure ] 464""" 465 with open(self.filename, 'w') as f: 466 f.write(contents) 467 removed_urls = self.instance.RemoveExpectationsFromFile( 468 expectations_to_remove, self.filename, expectations.RemovalType.UNUSED) 469 self.assertEqual(removed_urls, set(['crbug.com/2345'])) 470 with open(self.filename) as f: 471 self.assertEqual(f.read(), expected_contents) 472 473 def testInlineGeneralComments(self) -> None: 474 """Tests that expectations with inline disable comments are not removed.""" 475 contents = self.header + """ 476crbug.com/1234 [ win ] foo/test [ Failure ] 477crbug.com/2345 [ win ] foo/test [ Failure ] # finder:disable-general 478crbug.com/3456 [ win ] foo/test [ Failure ] 479""" 480 stale_expectations = [ 481 data_types.Expectation('foo/test', ['win'], ['Failure'], 482 'crbug.com/1234'), 483 data_types.Expectation('foo/test', ['win'], ['Failure'], 484 'crbug.com/2345'), 485 data_types.Expectation('foo/test', ['win'], ['Failure'], 486 'crbug.com/3456'), 487 ] 488 expected_contents = self.header + """ 489crbug.com/2345 [ win ] foo/test [ Failure ] # finder:disable-general 490""" 491 for removal_type in (expectations.RemovalType.STALE, 492 expectations.RemovalType.UNUSED): 493 with open(self.filename, 'w') as f: 494 f.write(contents) 495 removed_urls = self.instance.RemoveExpectationsFromFile( 496 stale_expectations, self.filename, removal_type) 497 self.assertEqual(removed_urls, set(['crbug.com/1234', 'crbug.com/3456'])) 498 with open(self.filename) as f: 499 self.assertEqual(f.read(), expected_contents) 500 501 def testInlineStaleComments(self) -> None: 502 """Tests that expectations with inline stale disable comments not removed""" 503 contents = self.header + """ 504crbug.com/1234 [ win ] not_disabled [ Failure ] 505crbug.com/2345 [ win ] stale_disabled [ Failure ] # finder:disable-stale 506crbug.com/3456 [ win ] unused_disabled [ Failure ] # finder:disable-unused 507""" 508 stale_expectations = [ 509 data_types.Expectation('not_disabled', ['win'], 'Failure', 510 'crbug.com/1234'), 511 data_types.Expectation('stale_disabled', ['win'], 'Failure', 512 'crbug.com/2345'), 513 data_types.Expectation('unused_disabled', ['win'], 'Failure', 514 'crbug.com/3456') 515 ] 516 expected_contents = self.header + """ 517crbug.com/2345 [ win ] stale_disabled [ Failure ] # finder:disable-stale 518""" 519 with open(self.filename, 'w') as f: 520 f.write(contents) 521 removed_urls = self.instance.RemoveExpectationsFromFile( 522 stale_expectations, self.filename, expectations.RemovalType.STALE) 523 self.assertEqual(removed_urls, set(['crbug.com/1234', 'crbug.com/3456'])) 524 with open(self.filename) as f: 525 self.assertEqual(f.read(), expected_contents) 526 527 def testInlineUnusedComments(self) -> None: 528 """Tests that expectations with inline unused comments not removed""" 529 contents = self.header + """ 530crbug.com/1234 [ win ] not_disabled [ Failure ] 531crbug.com/2345 [ win ] stale_disabled [ Failure ] # finder:disable-stale 532crbug.com/3456 [ win ] unused_disabled [ Failure ] # finder:disable-unused 533""" 534 stale_expectations = [ 535 data_types.Expectation('not_disabled', ['win'], 'Failure', 536 'crbug.com/1234'), 537 data_types.Expectation('stale_disabled', ['win'], 'Failure', 538 'crbug.com/2345'), 539 data_types.Expectation('unused_disabled', ['win'], 'Failure', 540 'crbug.com/3456') 541 ] 542 expected_contents = self.header + """ 543crbug.com/3456 [ win ] unused_disabled [ Failure ] # finder:disable-unused 544""" 545 with open(self.filename, 'w') as f: 546 f.write(contents) 547 removed_urls = self.instance.RemoveExpectationsFromFile( 548 stale_expectations, self.filename, expectations.RemovalType.UNUSED) 549 self.assertEqual(removed_urls, set(['crbug.com/1234', 'crbug.com/2345'])) 550 with open(self.filename) as f: 551 self.assertEqual(f.read(), expected_contents) 552 553 def testGetDisableReasonFromComment(self): 554 """Tests that the disable reason can be pulled from a line.""" 555 self.assertEqual( 556 expectations._GetDisableReasonFromComment( 557 '# finder:disable-general foo'), 'foo') 558 self.assertEqual( 559 expectations._GetDisableReasonFromComment( 560 'crbug.com/1234 [ win ] bar/test [ Failure ] ' 561 '# finder:disable-general foo'), 'foo') 562 563 def testGroupBlockAllRemovable(self): 564 """Tests that a group with all members removable is removed.""" 565 contents = self.header + """ 566 567# This is a test comment 568crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 569 570# Another comment 571 572# finder:group-start some group name 573[ linux ] bar/test [ RetryOnFailure ] 574crbug.com/1234 [ win ] foo/test [ Failure ] 575# finder:group-end 576[ win ] bar/test [ RetryOnFailure ] 577""" 578 579 stale_expectations = [ 580 data_types.Expectation('foo/test', ['win'], ['Failure'], 581 'crbug.com/1234'), 582 data_types.Expectation('bar/test', ['linux'], ['RetryOnFailure']), 583 ] 584 585 expected_contents = self.header + """ 586 587# This is a test comment 588crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 589 590# Another comment 591 592[ win ] bar/test [ RetryOnFailure ] 593""" 594 595 with open(self.filename, 'w') as f: 596 f.write(contents) 597 598 removed_urls = self.instance.RemoveExpectationsFromFile( 599 stale_expectations, self.filename, expectations.RemovalType.STALE) 600 self.assertEqual(removed_urls, set(['crbug.com/1234'])) 601 with open(self.filename) as f: 602 self.assertEqual(f.read(), expected_contents) 603 604 def testLargeGroupBlockAllRemovable(self): 605 """Tests that a large group with all members removable is removed.""" 606 # This test exists because we've had issues that passed tests with 607 # relatively small groups, but failed on larger ones. 608 contents = self.header + """ 609 610# This is a test comment 611crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 612 613# Another comment 614 615# finder:group-start some group name 616[ linux ] a [ RetryOnFailure ] 617[ linux ] b [ RetryOnFailure ] 618[ linux ] c [ RetryOnFailure ] 619[ linux ] d [ RetryOnFailure ] 620[ linux ] e [ RetryOnFailure ] 621[ linux ] f [ RetryOnFailure ] 622[ linux ] g [ RetryOnFailure ] 623[ linux ] h [ RetryOnFailure ] 624[ linux ] i [ RetryOnFailure ] 625[ linux ] j [ RetryOnFailure ] 626[ linux ] k [ RetryOnFailure ] 627# finder:group-end 628[ win ] bar/test [ RetryOnFailure ] 629""" 630 631 stale_expectations = [ 632 data_types.Expectation('a', ['linux'], ['RetryOnFailure']), 633 data_types.Expectation('b', ['linux'], ['RetryOnFailure']), 634 data_types.Expectation('c', ['linux'], ['RetryOnFailure']), 635 data_types.Expectation('d', ['linux'], ['RetryOnFailure']), 636 data_types.Expectation('e', ['linux'], ['RetryOnFailure']), 637 data_types.Expectation('f', ['linux'], ['RetryOnFailure']), 638 data_types.Expectation('g', ['linux'], ['RetryOnFailure']), 639 data_types.Expectation('h', ['linux'], ['RetryOnFailure']), 640 data_types.Expectation('i', ['linux'], ['RetryOnFailure']), 641 data_types.Expectation('j', ['linux'], ['RetryOnFailure']), 642 data_types.Expectation('k', ['linux'], ['RetryOnFailure']), 643 ] 644 645 expected_contents = self.header + """ 646 647# This is a test comment 648crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 649 650# Another comment 651 652[ win ] bar/test [ RetryOnFailure ] 653""" 654 655 with open(self.filename, 'w') as f: 656 f.write(contents) 657 658 removed_urls = self.instance.RemoveExpectationsFromFile( 659 stale_expectations, self.filename, expectations.RemovalType.STALE) 660 self.assertEqual(removed_urls, set([])) 661 with open(self.filename) as f: 662 self.assertEqual(f.read(), expected_contents) 663 664 def testNestedGroupAndNarrowingAllRemovable(self): 665 """Tests that a disable block within a group can be properly removed.""" 666 contents = self.header + """ 667crbug.com/2345 [ win ] baz/test [ Failure ] 668 669# Description 670# finder:group-start name 671# finder:disable-narrowing 672crbug.com/1234 [ win ] foo/test [ Failure ] 673crbug.com/1234 [ win ] bar/test [ Failure ] 674# finder:enable-narrowing 675# finder:group-end 676 677crbug.com/3456 [ linux ] foo/test [ Failure ] 678""" 679 680 stale_expectations = [ 681 data_types.Expectation('foo/test', ['win'], ['Failure'], 682 'crbug.com/1234'), 683 data_types.Expectation('bar/test', ['win'], ['Failure'], 684 'crbug.com/1234'), 685 ] 686 687 expected_contents = self.header + """ 688crbug.com/2345 [ win ] baz/test [ Failure ] 689 690 691crbug.com/3456 [ linux ] foo/test [ Failure ] 692""" 693 694 with open(self.filename, 'w') as f: 695 f.write(contents) 696 697 removed_urls = self.instance.RemoveExpectationsFromFile( 698 stale_expectations, self.filename, expectations.RemovalType.STALE) 699 self.assertEqual(removed_urls, set(['crbug.com/1234'])) 700 with open(self.filename) as f: 701 self.assertEqual(f.read(), expected_contents) 702 703 def testGroupBlockNotAllRemovable(self): 704 """Tests that a group with not all members removable is not removed.""" 705 contents = self.header + """ 706 707# This is a test comment 708crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 709 710# Another comment 711# finder:group-start some group name 712[ linux ] bar/test [ RetryOnFailure ] 713crbug.com/1234 [ win ] foo/test [ Failure ] 714# finder:group-end 715[ win ] bar/test [ RetryOnFailure ] 716""" 717 718 stale_expectations = [ 719 data_types.Expectation('bar/test', ['linux'], ['RetryOnFailure']) 720 ] 721 722 expected_contents = contents 723 724 with open(self.filename, 'w') as f: 725 f.write(contents) 726 727 removed_urls = self.instance.RemoveExpectationsFromFile( 728 stale_expectations, self.filename, expectations.RemovalType.STALE) 729 self.assertEqual(removed_urls, set()) 730 with open(self.filename) as f: 731 self.assertEqual(f.read(), expected_contents) 732 733 def testGroupSplitAllRemovable(self): 734 """Tests that a split group with all members removable is removed.""" 735 contents = self.header + """ 736 737# This is a test comment 738crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 739 740# Another comment 741 742# finder:group-start some group name 743[ linux ] bar/test [ RetryOnFailure ] 744# finder:group-end 745 746# finder:group-start some group name 747crbug.com/1234 [ win ] foo/test [ Failure ] 748# finder:group-end 749[ win ] bar/test [ RetryOnFailure ] 750""" 751 752 stale_expectations = [ 753 data_types.Expectation('foo/test', ['win'], ['Failure'], 754 'crbug.com/1234'), 755 data_types.Expectation('bar/test', ['linux'], ['RetryOnFailure']) 756 ] 757 758 expected_contents = self.header + """ 759 760# This is a test comment 761crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 762 763# Another comment 764 765 766[ win ] bar/test [ RetryOnFailure ] 767""" 768 769 with open(self.filename, 'w') as f: 770 f.write(contents) 771 772 removed_urls = self.instance.RemoveExpectationsFromFile( 773 stale_expectations, self.filename, expectations.RemovalType.STALE) 774 self.assertEqual(removed_urls, set(['crbug.com/1234'])) 775 with open(self.filename) as f: 776 self.assertEqual(f.read(), expected_contents) 777 778 def testGroupSplitNotAllRemovable(self): 779 """Tests that a split group without all members removable is not removed.""" 780 contents = self.header + """ 781 782# This is a test comment 783crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 784 785# Another comment 786# finder:group-start some group name 787[ linux ] bar/test [ RetryOnFailure ] 788# finder:group-end 789 790# finder:group-start some group name 791crbug.com/1234 [ win ] foo/test [ Failure ] 792# finder:group-end 793[ win ] bar/test [ RetryOnFailure ] 794""" 795 796 stale_expectations = [ 797 data_types.Expectation('bar/test', ['linux'], ['RetryOnFailure']) 798 ] 799 800 expected_contents = contents 801 802 with open(self.filename, 'w') as f: 803 f.write(contents) 804 805 removed_urls = self.instance.RemoveExpectationsFromFile( 806 stale_expectations, self.filename, expectations.RemovalType.STALE) 807 self.assertEqual(removed_urls, set()) 808 with open(self.filename) as f: 809 self.assertEqual(f.read(), expected_contents) 810 811 def testGroupMultipleGroupsAllRemovable(self): 812 """Tests that multiple groups with all members removable are removed.""" 813 contents = self.header + """ 814 815# This is a test comment 816crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 817 818# Another comment 819 820# finder:group-start some group name 821[ linux ] bar/test [ RetryOnFailure ] 822# finder:group-end 823 824# finder:group-start another group name 825crbug.com/1234 [ win ] foo/test [ Failure ] 826# finder:group-end 827[ win ] bar/test [ RetryOnFailure ] 828""" 829 830 stale_expectations = [ 831 data_types.Expectation('foo/test', ['win'], ['Failure'], 832 'crbug.com/1234'), 833 data_types.Expectation('bar/test', ['linux'], ['RetryOnFailure']) 834 ] 835 836 expected_contents = self.header + """ 837 838# This is a test comment 839crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 840 841# Another comment 842 843 844[ win ] bar/test [ RetryOnFailure ] 845""" 846 847 with open(self.filename, 'w') as f: 848 f.write(contents) 849 850 removed_urls = self.instance.RemoveExpectationsFromFile( 851 stale_expectations, self.filename, expectations.RemovalType.STALE) 852 self.assertEqual(removed_urls, set(['crbug.com/1234'])) 853 with open(self.filename) as f: 854 self.assertEqual(f.read(), expected_contents) 855 856 def testGroupMultipleGroupsSomeRemovable(self): 857 """Tests that multiple groups are handled separately.""" 858 contents = self.header + """ 859 860# This is a test comment 861crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 862 863# Another comment 864 865# finder:group-start some group name 866[ linux ] bar/test [ RetryOnFailure ] 867# finder:group-end 868 869# finder:group-start another group name 870crbug.com/1234 [ win ] foo/test [ Failure ] 871crbug.com/1234 [ linux ] foo/test [ Failure ] 872# finder:group-end 873[ win ] bar/test [ RetryOnFailure ] 874""" 875 876 stale_expectations = [ 877 data_types.Expectation('foo/test', ['win'], ['Failure'], 878 'crbug.com/1234'), 879 data_types.Expectation('bar/test', ['linux'], ['RetryOnFailure']) 880 ] 881 882 expected_contents = self.header + """ 883 884# This is a test comment 885crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 886 887# Another comment 888 889 890# finder:group-start another group name 891crbug.com/1234 [ win ] foo/test [ Failure ] 892crbug.com/1234 [ linux ] foo/test [ Failure ] 893# finder:group-end 894[ win ] bar/test [ RetryOnFailure ] 895""" 896 897 with open(self.filename, 'w') as f: 898 f.write(contents) 899 900 removed_urls = self.instance.RemoveExpectationsFromFile( 901 stale_expectations, self.filename, expectations.RemovalType.STALE) 902 self.assertEqual(removed_urls, set()) 903 with open(self.filename) as f: 904 self.assertEqual(f.read(), expected_contents) 905 906 def testNestedGroupStart(self): 907 """Tests that nested groups are disallowed.""" 908 contents = self.header + """ 909 910# This is a test comment 911crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 912 913# Another comment 914# finder:group-start some group name 915[ linux ] bar/test [ RetryOnFailure ] 916# finder:group-start another group name 917# finder:group-end 918# finder:group-end 919[ win ] bar/test [ RetryOnFailure ] 920""" 921 922 with open(self.filename, 'w') as f: 923 f.write(contents) 924 925 with self.assertRaisesRegex(RuntimeError, 926 'that is inside another group block'): 927 self.instance.RemoveExpectationsFromFile([], self.filename, 928 expectations.RemovalType.STALE) 929 930 def testOrphanedGroupEnd(self): 931 """Tests that orphaned group ends are disallowed.""" 932 contents = self.header + """ 933 934# This is a test comment 935crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 936 937# Another comment 938# finder:group-end 939[ linux ] bar/test [ RetryOnFailure ] 940[ win ] bar/test [ RetryOnFailure ] 941""" 942 943 with open(self.filename, 'w') as f: 944 f.write(contents) 945 946 with self.assertRaisesRegex(RuntimeError, 'without a group start comment'): 947 self.instance.RemoveExpectationsFromFile([], self.filename, 948 expectations.RemovalType.STALE) 949 950 def testNoGroupName(self): 951 """Tests that unnamed groups are disallowed.""" 952 contents = self.header + """ 953 954# This is a test comment 955crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 956 957# Another comment 958# finder:group-start 959# finder:group-end 960[ linux ] bar/test [ RetryOnFailure ] 961[ win ] bar/test [ RetryOnFailure ] 962""" 963 964 with open(self.filename, 'w') as f: 965 f.write(contents) 966 967 with self.assertRaisesRegex(RuntimeError, 'did not have a group name'): 968 self.instance.RemoveExpectationsFromFile([], self.filename, 969 expectations.RemovalType.STALE) 970 971 def testRemoveCommentBlockSimpleTrailingWhitespace(self): 972 """Tests stale comment removal in a simple case with trailing whitespace.""" 973 contents = self.header + """ 974# Comment line 1 975# Comment line 2 976crbug.com/1234 [ linux ] foo/test [ Failure ] 977 978crbug.com/2345 [ win ] bar/test [ Failure ] 979""" 980 981 stale_expectations = [ 982 data_types.Expectation('foo/test', ['linux'], ['Failure'], 983 'crbug.com/1234'), 984 ] 985 986 expected_contents = self.header + """ 987 988crbug.com/2345 [ win ] bar/test [ Failure ] 989""" 990 991 with open(self.filename, 'w') as f: 992 f.write(contents) 993 994 removed_urls = self.instance.RemoveExpectationsFromFile( 995 stale_expectations, self.filename, expectations.RemovalType.STALE) 996 self.assertEqual(removed_urls, {'crbug.com/1234'}) 997 with open(self.filename) as f: 998 self.assertEqual(f.read(), expected_contents) 999 1000 def testRemoveCommentBlockSimpleTrailingComment(self): 1001 """Tests stale comment removal in a simple case with trailing comment.""" 1002 contents = self.header + """ 1003# Comment line 1 1004# Comment line 2 1005crbug.com/1234 [ linux ] foo/test [ Failure ] 1006# Comment line 3 1007crbug.com/2345 [ win ] bar/test [ Failure ] 1008""" 1009 1010 stale_expectations = [ 1011 data_types.Expectation('foo/test', ['linux'], ['Failure'], 1012 'crbug.com/1234'), 1013 ] 1014 1015 expected_contents = self.header + """ 1016# Comment line 3 1017crbug.com/2345 [ win ] bar/test [ Failure ] 1018""" 1019 1020 with open(self.filename, 'w') as f: 1021 f.write(contents) 1022 1023 removed_urls = self.instance.RemoveExpectationsFromFile( 1024 stale_expectations, self.filename, expectations.RemovalType.STALE) 1025 self.assertEqual(removed_urls, {'crbug.com/1234'}) 1026 with open(self.filename) as f: 1027 self.assertEqual(f.read(), expected_contents) 1028 1029 def testRemoveCommentBlockSimpleEndOfFile(self): 1030 """Tests stale comment removal in a simple case at file end.""" 1031 contents = self.header + """ 1032crbug.com/2345 [ win ] bar/test [ Failure ] 1033 1034# Comment line 1 1035# Comment line 2 1036crbug.com/1234 [ linux ] foo/test [ Failure ]""" 1037 1038 stale_expectations = [ 1039 data_types.Expectation('foo/test', ['linux'], ['Failure'], 1040 'crbug.com/1234'), 1041 ] 1042 1043 expected_contents = self.header + """ 1044crbug.com/2345 [ win ] bar/test [ Failure ] 1045 1046""" 1047 1048 with open(self.filename, 'w') as f: 1049 f.write(contents) 1050 1051 removed_urls = self.instance.RemoveExpectationsFromFile( 1052 stale_expectations, self.filename, expectations.RemovalType.STALE) 1053 self.assertEqual(removed_urls, {'crbug.com/1234'}) 1054 with open(self.filename) as f: 1055 self.assertEqual(f.read(), expected_contents) 1056 1057 def testRemoveCommentBlockWithAnnotations(self): 1058 """Tests stale comment removal with annotations on both ends.""" 1059 contents = self.header + """ 1060# Comment line 1 1061# Comment line 2 1062# finder:disable-unused 1063crbug.com/1234 [ linux ] foo/test [ Failure ] 1064# finder:enable-unused 1065# Comment line 3 1066crbug.com/2345 [ win ] bar/test [ Failure ] 1067""" 1068 1069 stale_expectations = [ 1070 data_types.Expectation('foo/test', ['linux'], ['Failure'], 1071 'crbug.com/1234'), 1072 ] 1073 1074 expected_contents = self.header + """ 1075# Comment line 3 1076crbug.com/2345 [ win ] bar/test [ Failure ] 1077""" 1078 1079 with open(self.filename, 'w') as f: 1080 f.write(contents) 1081 1082 removed_urls = self.instance.RemoveExpectationsFromFile( 1083 stale_expectations, self.filename, expectations.RemovalType.STALE) 1084 self.assertEqual(removed_urls, {'crbug.com/1234'}) 1085 with open(self.filename) as f: 1086 self.assertEqual(f.read(), expected_contents) 1087 1088 def testRemoveCommentBlockWithMissingTrailingAnnotation(self): 1089 """Tests stale comment removal with a missing trailing annotation.""" 1090 contents = self.header + """ 1091# Comment line 1 1092# Comment line 2 1093# finder:disable-unused 1094crbug.com/1234 [ linux ] foo/test [ Failure ] 1095 1096crbug.com/1234 [ win ] foo/test [ Failure ] 1097# finder:enable-unused 1098 1099# Comment line 3 1100crbug.com/2345 [ win ] bar/test [ Failure ] 1101""" 1102 1103 stale_expectations = [ 1104 data_types.Expectation('foo/test', ['linux'], ['Failure'], 1105 'crbug.com/1234'), 1106 ] 1107 1108 expected_contents = self.header + """ 1109# Comment line 1 1110# Comment line 2 1111# finder:disable-unused 1112 1113crbug.com/1234 [ win ] foo/test [ Failure ] 1114# finder:enable-unused 1115 1116# Comment line 3 1117crbug.com/2345 [ win ] bar/test [ Failure ] 1118""" 1119 1120 with open(self.filename, 'w') as f: 1121 f.write(contents) 1122 1123 removed_urls = self.instance.RemoveExpectationsFromFile( 1124 stale_expectations, self.filename, expectations.RemovalType.STALE) 1125 self.assertEqual(removed_urls, {'crbug.com/1234'}) 1126 with open(self.filename) as f: 1127 self.assertEqual(f.read(), expected_contents) 1128 1129 def testRemoveCommentBlockWithMissingStartAnnotation(self): 1130 """Tests stale comment removal with a missing start annotation.""" 1131 contents = self.header + """ 1132# finder:disable-unused 1133crbug.com/1234 [ win ] foo/test [ Failure ] 1134# Comment line 1 1135# Comment line 2 1136crbug.com/1234 [ linux ] foo/test [ Failure ] 1137# finder:enable-unused 1138# Comment line 3 1139crbug.com/2345 [ win ] bar/test [ Failure ] 1140""" 1141 1142 stale_expectations = [ 1143 data_types.Expectation('foo/test', ['linux'], ['Failure'], 1144 'crbug.com/1234'), 1145 ] 1146 1147 expected_contents = self.header + """ 1148# finder:disable-unused 1149crbug.com/1234 [ win ] foo/test [ Failure ] 1150# finder:enable-unused 1151# Comment line 3 1152crbug.com/2345 [ win ] bar/test [ Failure ] 1153""" 1154 1155 with open(self.filename, 'w') as f: 1156 f.write(contents) 1157 1158 removed_urls = self.instance.RemoveExpectationsFromFile( 1159 stale_expectations, self.filename, expectations.RemovalType.STALE) 1160 self.assertEqual(removed_urls, {'crbug.com/1234'}) 1161 with open(self.filename) as f: 1162 self.assertEqual(f.read(), expected_contents) 1163 1164 def testRemoveCommentBlockMultipleExpectations(self): 1165 """Tests stale comment removal with multiple expectations in a block.""" 1166 contents = self.header + """ 1167# Comment line 1 1168# Comment line 2 1169# finder:disable-unused 1170crbug.com/1234 [ linux ] foo/test [ Failure ] 1171crbug.com/3456 [ mac ] foo/test [ Failure ] 1172# finder:enable-unused 1173 1174# Comment line 3 1175crbug.com/2345 [ win ] bar/test [ Failure ] 1176""" 1177 1178 stale_expectations = [ 1179 data_types.Expectation('foo/test', ['linux'], ['Failure'], 1180 'crbug.com/1234'), 1181 data_types.Expectation('foo/test', ['mac'], ['Failure'], 1182 'crbug.com/3456'), 1183 ] 1184 1185 expected_contents = self.header + """ 1186 1187# Comment line 3 1188crbug.com/2345 [ win ] bar/test [ Failure ] 1189""" 1190 1191 with open(self.filename, 'w') as f: 1192 f.write(contents) 1193 1194 removed_urls = self.instance.RemoveExpectationsFromFile( 1195 stale_expectations, self.filename, expectations.RemovalType.STALE) 1196 self.assertEqual(removed_urls, {'crbug.com/1234', 'crbug.com/3456'}) 1197 with open(self.filename) as f: 1198 self.assertEqual(f.read(), expected_contents) 1199 1200 def testRemoveCommentBlockMultipleBlocks(self): 1201 """Tests stale comment removal with expectations in multiple blocks.""" 1202 contents = self.header + """ 1203# Comment line 1 1204# Comment line 2 1205# finder:disable-unused 1206crbug.com/1234 [ linux ] foo/test [ Failure ] 1207# finder:enable-unused 1208 1209# Comment line 4 1210# finder:disable-unused 1211crbug.com/3456 [ mac ] foo/test [ Failure ] 1212# finder:enable-unused 1213# Comment line 3 1214crbug.com/2345 [ win ] bar/test [ Failure ] 1215""" 1216 1217 stale_expectations = [ 1218 data_types.Expectation('foo/test', ['linux'], ['Failure'], 1219 'crbug.com/1234'), 1220 data_types.Expectation('foo/test', ['mac'], ['Failure'], 1221 'crbug.com/3456'), 1222 ] 1223 1224 expected_contents = self.header + """ 1225 1226# Comment line 3 1227crbug.com/2345 [ win ] bar/test [ Failure ] 1228""" 1229 1230 with open(self.filename, 'w') as f: 1231 f.write(contents) 1232 1233 removed_urls = self.instance.RemoveExpectationsFromFile( 1234 stale_expectations, self.filename, expectations.RemovalType.STALE) 1235 self.assertEqual(removed_urls, {'crbug.com/1234', 'crbug.com/3456'}) 1236 with open(self.filename) as f: 1237 self.assertEqual(f.read(), expected_contents) 1238 1239 def testRemoveStaleAnnotationBlocks(self): 1240 """Tests removal of annotation blocks not associated with removals.""" 1241 contents = self.header + """ 1242# finder:disable-general 1243# finder:enable-general 1244 1245# finder:disable-stale 1246 1247# finder:enable-stale 1248 1249# finder:disable-unused 1250# comment 1251# finder:enable-unused 1252 1253# finder:disable-narrowing description 1254# comment 1255# finder:enable-narrowing 1256 1257# finder:group-start name 1258# finder:group-end 1259""" 1260 1261 stale_expectations = [] 1262 1263 expected_contents = self.header + """ 1264 1265 1266 1267 1268""" 1269 1270 with open(self.filename, 'w') as f: 1271 f.write(contents) 1272 1273 removed_urls = self.instance.RemoveExpectationsFromFile( 1274 stale_expectations, self.filename, expectations.RemovalType.STALE) 1275 self.assertEqual(removed_urls, set()) 1276 with open(self.filename) as f: 1277 self.assertEqual(f.read(), expected_contents) 1278 1279 def testGroupNameExtraction(self): 1280 """Tests that group names are properly extracted.""" 1281 group_name = expectations._GetGroupNameFromCommentLine( 1282 '# finder:group-start group name') 1283 self.assertEqual(group_name, 'group name') 1284 1285 1286class GetDisableAnnotatedExpectationsFromFileUnittest(unittest.TestCase): 1287 def setUp(self) -> None: 1288 self.instance = uu.CreateGenericExpectations() 1289 1290 def testNestedBlockComments(self) -> None: 1291 """Tests that nested disable block comments throw exceptions.""" 1292 contents = """ 1293# finder:disable-general 1294# finder:disable-general 1295crbug.com/1234 [ win ] foo/test [ Failure ] 1296# finder:enable-general 1297# finder:enable-general 1298""" 1299 with self.assertRaises(RuntimeError): 1300 self.instance._GetDisableAnnotatedExpectationsFromFile( 1301 'expectation_file', contents) 1302 1303 contents = """ 1304# finder:disable-general 1305# finder:disable-stale 1306crbug.com/1234 [ win ] foo/test [ Failure ] 1307# finder:enable-stale 1308# finder:enable-general 1309""" 1310 with self.assertRaises(RuntimeError): 1311 self.instance._GetDisableAnnotatedExpectationsFromFile( 1312 'expectation_file', contents) 1313 1314 contents = """ 1315# finder:enable-general 1316crbug.com/1234 [ win ] foo/test [ Failure ] 1317""" 1318 with self.assertRaises(RuntimeError): 1319 self.instance._GetDisableAnnotatedExpectationsFromFile( 1320 'expectation_file', contents) 1321 1322 def testBlockComments(self) -> None: 1323 """Tests that disable block comments are properly parsed.""" 1324 contents = """ 1325# finder:disable-general general-reason 1326crbug.com/1234 [ win ] foo/test [ Failure ] 1327# finder:enable-general 1328 1329# finder:disable-stale 1330crbug.com/1234 [ mac ] foo/test [ Failure ] 1331# finder:enable-stale 1332 1333# finder:disable-unused unused reason 1334crbug.com/1234 [ linux ] foo/test [ Failure ] 1335# finder:enable-unused 1336 1337# finder:disable-narrowing 1338crbug.com/1234 [ win ] bar/test [ Failure ] 1339# finder:enable-narrowing 1340 1341crbug.com/1234 [ mac ] bar/test [ Failure ] 1342""" 1343 annotated_expectations = ( 1344 self.instance._GetDisableAnnotatedExpectationsFromFile( 1345 'expectation_file', contents)) 1346 self.assertEqual(len(annotated_expectations), 4) 1347 self.assertEqual( 1348 annotated_expectations[data_types.Expectation('foo/test', ['win'], 1349 'Failure', 1350 'crbug.com/1234')], 1351 ('-general', 'general-reason')) 1352 self.assertEqual( 1353 annotated_expectations[data_types.Expectation('foo/test', ['mac'], 1354 'Failure', 1355 'crbug.com/1234')], 1356 ('-stale', '')) 1357 self.assertEqual( 1358 annotated_expectations[data_types.Expectation('foo/test', ['linux'], 1359 'Failure', 1360 'crbug.com/1234')], 1361 ('-unused', 'unused reason')) 1362 self.assertEqual( 1363 annotated_expectations[data_types.Expectation('bar/test', ['win'], 1364 'Failure', 1365 'crbug.com/1234')], 1366 ('-narrowing', '')) 1367 1368 def testInlineComments(self) -> None: 1369 """Tests that inline disable comments are properly parsed.""" 1370 # pylint: disable=line-too-long 1371 contents = """ 1372crbug.com/1234 [ win ] foo/test [ Failure ] # finder:disable-general general-reason 1373 1374crbug.com/1234 [ mac ] foo/test [ Failure ] # finder:disable-stale 1375 1376crbug.com/1234 [ linux ] foo/test [ Failure ] # finder:disable-unused unused reason 1377 1378crbug.com/1234 [ win ] bar/test [ Failure ] # finder:disable-narrowing 1379 1380crbug.com/1234 [ mac ] bar/test [ Failure ] 1381""" 1382 # pylint: enable=line-too-long 1383 annotated_expectations = ( 1384 self.instance._GetDisableAnnotatedExpectationsFromFile( 1385 'expectation_file', contents)) 1386 self.assertEqual(len(annotated_expectations), 4) 1387 self.assertEqual( 1388 annotated_expectations[data_types.Expectation('foo/test', ['win'], 1389 'Failure', 1390 'crbug.com/1234')], 1391 ('-general', 'general-reason')) 1392 self.assertEqual( 1393 annotated_expectations[data_types.Expectation('foo/test', ['mac'], 1394 'Failure', 1395 'crbug.com/1234')], 1396 ('-stale', '')) 1397 self.assertEqual( 1398 annotated_expectations[data_types.Expectation('foo/test', ['linux'], 1399 'Failure', 1400 'crbug.com/1234')], 1401 ('-unused', 'unused reason')) 1402 self.assertEqual( 1403 annotated_expectations[data_types.Expectation('bar/test', ['win'], 1404 'Failure', 1405 'crbug.com/1234')], 1406 ('-narrowing', '')) 1407 1408 1409class GetExpectationLineUnittest(unittest.TestCase): 1410 def setUp(self) -> None: 1411 self.instance = uu.CreateGenericExpectations() 1412 1413 def testNoMatchingExpectation(self) -> None: 1414 """Tests that the case of no matching expectation is handled.""" 1415 expectation = data_types.Expectation('foo', ['win'], 'Failure') 1416 line, line_number = self.instance._GetExpectationLine( 1417 expectation, FAKE_EXPECTATION_FILE_CONTENTS, 'expectation_file') 1418 self.assertIsNone(line) 1419 self.assertIsNone(line_number) 1420 1421 def testMatchingExpectation(self) -> None: 1422 """Tests that matching expectations are found.""" 1423 expectation = data_types.Expectation('foo/test', ['win'], 'Failure', 1424 'crbug.com/1234') 1425 line, line_number = self.instance._GetExpectationLine( 1426 expectation, FAKE_EXPECTATION_FILE_CONTENTS, 'expectation_file') 1427 self.assertEqual(line, 'crbug.com/1234 [ win ] foo/test [ Failure ]') 1428 self.assertEqual(line_number, 3) 1429 1430 1431class FilterToMostSpecificTypTagsUnittest(fake_filesystem_unittest.TestCase): 1432 def setUp(self) -> None: 1433 self._expectations = uu.CreateGenericExpectations() 1434 self.setUpPyfakefs() 1435 with tempfile.NamedTemporaryFile(delete=False, mode='w') as f: 1436 self.filename = f.name 1437 1438 def testBasic(self) -> None: 1439 """Tests that only the most specific tags are kept.""" 1440 expectation_file_contents = """\ 1441# tags: [ win win10 1442# linux 1443# mac ] 1444# tags: [ nvidia nvidia-0x1111 1445# intel intel-0x2222 1446# amd amd-0x3333] 1447# tags: [ release debug ] 1448""" 1449 with open(self.filename, 'w') as outfile: 1450 outfile.write(expectation_file_contents) 1451 tags = frozenset(['win', 'nvidia', 'nvidia-0x1111', 'release']) 1452 filtered_tags = self._expectations._FilterToMostSpecificTypTags( 1453 tags, self.filename) 1454 self.assertEqual(filtered_tags, set(['win', 'nvidia-0x1111', 'release'])) 1455 1456 def testSingleTags(self) -> None: 1457 """Tests that functionality works with single tags.""" 1458 expectation_file_contents = """\ 1459# tags: [ tag1_most_specific ] 1460# tags: [ tag2_most_specific ]""" 1461 with open(self.filename, 'w') as outfile: 1462 outfile.write(expectation_file_contents) 1463 1464 tags = frozenset(['tag1_most_specific', 'tag2_most_specific']) 1465 filtered_tags = self._expectations._FilterToMostSpecificTypTags( 1466 tags, self.filename) 1467 self.assertEqual(filtered_tags, tags) 1468 1469 def testUnusedTags(self) -> None: 1470 """Tests that functionality works as expected with extra/unused tags.""" 1471 expectation_file_contents = """\ 1472# tags: [ tag1_least_specific tag1_middle_specific tag1_most_specific ] 1473# tags: [ tag2_least_specific tag2_middle_specific tag2_most_specific ] 1474# tags: [ some_unused_tag ]""" 1475 with open(self.filename, 'w') as outfile: 1476 outfile.write(expectation_file_contents) 1477 1478 tags = frozenset([ 1479 'tag1_least_specific', 'tag1_most_specific', 'tag2_middle_specific', 1480 'tag2_least_specific' 1481 ]) 1482 filtered_tags = self._expectations._FilterToMostSpecificTypTags( 1483 tags, self.filename) 1484 self.assertEqual(filtered_tags, 1485 set(['tag1_most_specific', 'tag2_middle_specific'])) 1486 1487 def testMissingTags(self) -> None: 1488 """Tests that a file not having all tags is an error.""" 1489 expectation_file_contents = """\ 1490# tags: [ tag1_least_specific tag1_middle_specific ] 1491# tags: [ tag2_least_specific tag2_middle_specific tag2_most_specific ]""" 1492 with open(self.filename, 'w') as outfile: 1493 outfile.write(expectation_file_contents) 1494 1495 tags = frozenset([ 1496 'tag1_least_specific', 'tag1_most_specific', 'tag2_middle_specific', 1497 'tag2_least_specific' 1498 ]) 1499 with self.assertRaisesRegex(RuntimeError, r'.*tag1_most_specific.*'): 1500 self._expectations._FilterToMostSpecificTypTags(tags, self.filename) 1501 1502 def testTagsLowerCased(self) -> None: 1503 """Tests that found tags are lower cased to match internal tags.""" 1504 expectation_file_contents = """\ 1505# tags: [ Win Win10 1506# Linux 1507# Mac ] 1508# tags: [ nvidia nvidia-0x1111 1509# intel intel-0x2222 1510# amd amd-0x3333] 1511# tags: [ release debug ] 1512""" 1513 with open(self.filename, 'w') as outfile: 1514 outfile.write(expectation_file_contents) 1515 tags = frozenset(['win', 'win10', 'nvidia', 'release']) 1516 filtered_tags = self._expectations._FilterToMostSpecificTypTags( 1517 tags, self.filename) 1518 self.assertEqual(filtered_tags, set(['win10', 'nvidia', 'release'])) 1519 1520 1521class NarrowSemiStaleExpectationScopeUnittest(fake_filesystem_unittest.TestCase 1522 ): 1523 def setUp(self) -> None: 1524 self.setUpPyfakefs() 1525 self.instance = uu.CreateGenericExpectations() 1526 1527 with tempfile.NamedTemporaryFile(delete=False, mode='w') as f: 1528 f.write(FAKE_EXPECTATION_FILE_CONTENTS_WITH_COMPLEX_TAGS) 1529 self.filename = f.name 1530 1531 def testEmptyExpectationMap(self) -> None: 1532 """Tests that scope narrowing with an empty map is a no-op.""" 1533 urls = self.instance.NarrowSemiStaleExpectationScope( 1534 data_types.TestExpectationMap({})) 1535 self.assertEqual(urls, set()) 1536 with open(self.filename) as infile: 1537 self.assertEqual(infile.read(), 1538 FAKE_EXPECTATION_FILE_CONTENTS_WITH_COMPLEX_TAGS) 1539 1540 def testWildcard(self) -> None: 1541 """Regression test to ensure that wildcards are modified correctly.""" 1542 file_contents = """\ 1543# tags: [ win ] 1544# tags: [ amd intel ] 1545# results: [ Failure ] 1546 1547crbug.com/1234 [ win ] foo/bar* [ Failure ] 1548""" 1549 with open(self.filename, 'w') as f: 1550 f.write(file_contents) 1551 1552 amd_stats = data_types.BuildStats() 1553 amd_stats.AddPassedBuild(frozenset(['win', 'amd'])) 1554 intel_stats = data_types.BuildStats() 1555 intel_stats.AddFailedBuild('1', frozenset(['win', 'intel'])) 1556 # yapf: disable 1557 test_expectation_map = data_types.TestExpectationMap({ 1558 self.filename: 1559 data_types.ExpectationBuilderMap({ 1560 data_types.Expectation( 1561 'foo/bar*', ['win'], 'Failure', 'crbug.com/1234'): 1562 data_types.BuilderStepMap({ 1563 'win_builder': 1564 data_types.StepBuildStatsMap({ 1565 'amd': amd_stats, 1566 'intel': intel_stats, 1567 }), 1568 }), 1569 }), 1570 }) 1571 # yap: enable 1572 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 1573 expected_contents = """\ 1574# tags: [ win ] 1575# tags: [ amd intel ] 1576# results: [ Failure ] 1577 1578crbug.com/1234 [ intel win ] foo/bar* [ Failure ] 1579""" 1580 with open(self.filename) as infile: 1581 self.assertEqual(infile.read(), expected_contents) 1582 self.assertEqual(urls, {'crbug.com/1234'}) 1583 1584 def testMultipleSteps(self) -> None: 1585 """Tests that scope narrowing works across multiple steps.""" 1586 amd_stats = data_types.BuildStats() 1587 amd_stats.AddPassedBuild(frozenset(['win', 'amd'])) 1588 intel_stats = data_types.BuildStats() 1589 intel_stats.AddFailedBuild('1', frozenset(['win', 'intel'])) 1590 # yapf: disable 1591 test_expectation_map = data_types.TestExpectationMap({ 1592 self.filename: 1593 data_types.ExpectationBuilderMap({ 1594 data_types.Expectation( 1595 'foo/test', ['win'], 'Failure', 'crbug.com/1234'): 1596 data_types.BuilderStepMap({ 1597 'win_builder': 1598 data_types.StepBuildStatsMap({ 1599 'amd': amd_stats, 1600 'intel': intel_stats, 1601 }), 1602 }), 1603 }), 1604 }) 1605 # yapf: enable 1606 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 1607 expected_contents = """\ 1608# tags: [ win win10 1609# linux 1610# mac ] 1611# tags: [ nvidia nvidia-0x1111 1612# intel intel-0x2222 1613# amd amd-0x3333] 1614# tags: [ release debug ] 1615# results: [ Failure RetryOnFailure ] 1616 1617crbug.com/1234 [ intel win ] foo/test [ Failure ] 1618crbug.com/2345 [ linux ] foo/test [ RetryOnFailure ] 1619""" 1620 with open(self.filename) as infile: 1621 self.assertEqual(infile.read(), expected_contents) 1622 self.assertEqual(urls, set(['crbug.com/1234'])) 1623 1624 def testMultipleBuilders(self) -> None: 1625 """Tests that scope narrowing works across multiple builders.""" 1626 amd_stats = data_types.BuildStats() 1627 amd_stats.AddPassedBuild(frozenset(['win', 'amd'])) 1628 intel_stats = data_types.BuildStats() 1629 intel_stats.AddFailedBuild('1', frozenset(['win', 'intel'])) 1630 # yapf: disable 1631 test_expectation_map = data_types.TestExpectationMap({ 1632 self.filename: 1633 data_types.ExpectationBuilderMap({ 1634 data_types.Expectation( 1635 'foo/test', ['win'], 'Failure', 'crbug.com/1234'): 1636 data_types.BuilderStepMap({ 1637 'win_amd_builder': 1638 data_types.StepBuildStatsMap({ 1639 'amd': amd_stats, 1640 }), 1641 'win_intel_builder': 1642 data_types.StepBuildStatsMap({ 1643 'intel': intel_stats, 1644 }), 1645 }), 1646 }), 1647 }) 1648 # yapf: enable 1649 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 1650 expected_contents = """\ 1651# tags: [ win win10 1652# linux 1653# mac ] 1654# tags: [ nvidia nvidia-0x1111 1655# intel intel-0x2222 1656# amd amd-0x3333] 1657# tags: [ release debug ] 1658# results: [ Failure RetryOnFailure ] 1659 1660crbug.com/1234 [ intel win ] foo/test [ Failure ] 1661crbug.com/2345 [ linux ] foo/test [ RetryOnFailure ] 1662""" 1663 with open(self.filename) as infile: 1664 self.assertEqual(infile.read(), expected_contents) 1665 self.assertEqual(urls, set(['crbug.com/1234'])) 1666 1667 def testMultipleExpectations(self) -> None: 1668 """Tests that scope narrowing works across multiple expectations.""" 1669 amd_stats = data_types.BuildStats() 1670 amd_stats.AddPassedBuild(frozenset(['win', 'amd'])) 1671 failed_amd_stats = data_types.BuildStats() 1672 failed_amd_stats.AddFailedBuild('1', frozenset(['win', 'amd'])) 1673 multi_amd_stats = data_types.BuildStats() 1674 multi_amd_stats.AddFailedBuild('1', frozenset(['win', 'amd', 'debug'])) 1675 multi_amd_stats.AddFailedBuild('1', frozenset(['win', 'amd', 'release'])) 1676 intel_stats = data_types.BuildStats() 1677 intel_stats.AddFailedBuild('1', frozenset(['win', 'intel'])) 1678 debug_stats = data_types.BuildStats() 1679 debug_stats.AddFailedBuild('1', frozenset(['linux', 'debug'])) 1680 release_stats = data_types.BuildStats() 1681 release_stats.AddPassedBuild(frozenset(['linux', 'release'])) 1682 # yapf: disable 1683 test_expectation_map = data_types.TestExpectationMap({ 1684 self.filename: 1685 data_types.ExpectationBuilderMap({ 1686 data_types.Expectation( 1687 'foo/test', ['win'], 'Failure', 'crbug.com/1234'): 1688 data_types.BuilderStepMap({ 1689 'win_builder': 1690 data_types.StepBuildStatsMap({ 1691 'amd': amd_stats, 1692 'intel': intel_stats, 1693 }), 1694 }), 1695 # These two expectations are here to ensure that our continue logic 1696 # works as expected when we hit cases we can't handle, i.e. that 1697 # later expectations are still handled properly. 1698 data_types.Expectation('bar/test', ['win'], 'Failure', ''): 1699 data_types.BuilderStepMap({ 1700 'win_builder': 1701 data_types.StepBuildStatsMap({ 1702 'win1': amd_stats, 1703 'win2': failed_amd_stats, 1704 }), 1705 }), 1706 data_types.Expectation('baz/test', ['win'], 'Failure', ''): 1707 data_types.BuilderStepMap({ 1708 'win_builder': 1709 data_types.StepBuildStatsMap({ 1710 'win1': amd_stats, 1711 'win2': multi_amd_stats, 1712 }), 1713 }), 1714 data_types.Expectation( 1715 'foo/test', ['linux'], 'RetryOnFailure', 'crbug.com/2345'): 1716 data_types.BuilderStepMap({ 1717 'linux_builder': 1718 data_types.StepBuildStatsMap({ 1719 'debug': debug_stats, 1720 'release': release_stats, 1721 }), 1722 }), 1723 }), 1724 }) 1725 # yapf: enable 1726 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 1727 expected_contents = """\ 1728# tags: [ win win10 1729# linux 1730# mac ] 1731# tags: [ nvidia nvidia-0x1111 1732# intel intel-0x2222 1733# amd amd-0x3333] 1734# tags: [ release debug ] 1735# results: [ Failure RetryOnFailure ] 1736 1737crbug.com/1234 [ intel win ] foo/test [ Failure ] 1738crbug.com/2345 [ debug linux ] foo/test [ RetryOnFailure ] 1739""" 1740 with open(self.filename) as infile: 1741 self.assertEqual(infile.read(), expected_contents) 1742 self.assertEqual(urls, set(['crbug.com/1234', 'crbug.com/2345'])) 1743 1744 def testMultipleOutputLines(self) -> None: 1745 """Tests that scope narrowing works with multiple output lines.""" 1746 amd_stats = data_types.BuildStats() 1747 amd_stats.AddPassedBuild(frozenset(['win', 'amd'])) 1748 intel_stats = data_types.BuildStats() 1749 intel_stats.AddFailedBuild('1', frozenset(['win', 'intel'])) 1750 nvidia_stats = data_types.BuildStats() 1751 nvidia_stats.AddFailedBuild('1', frozenset(['win', 'nvidia'])) 1752 # yapf: disable 1753 test_expectation_map = data_types.TestExpectationMap({ 1754 self.filename: 1755 data_types.ExpectationBuilderMap({ 1756 data_types.Expectation( 1757 'foo/test', ['win'], 'Failure', 'crbug.com/1234'): 1758 data_types.BuilderStepMap({ 1759 'win_amd_builder': 1760 data_types.StepBuildStatsMap({ 1761 'amd': amd_stats, 1762 }), 1763 'win_intel_builder': 1764 data_types.StepBuildStatsMap({ 1765 'intel': intel_stats, 1766 }), 1767 'win_nvidia_builder': 1768 data_types.StepBuildStatsMap({ 1769 'nvidia': nvidia_stats, 1770 }) 1771 }), 1772 }), 1773 }) 1774 # yapf: enable 1775 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 1776 expected_contents = """\ 1777# tags: [ win win10 1778# linux 1779# mac ] 1780# tags: [ nvidia nvidia-0x1111 1781# intel intel-0x2222 1782# amd amd-0x3333] 1783# tags: [ release debug ] 1784# results: [ Failure RetryOnFailure ] 1785 1786crbug.com/1234 [ intel win ] foo/test [ Failure ] 1787crbug.com/1234 [ nvidia win ] foo/test [ Failure ] 1788crbug.com/2345 [ linux ] foo/test [ RetryOnFailure ] 1789""" 1790 with open(self.filename) as infile: 1791 self.assertEqual(infile.read(), expected_contents) 1792 self.assertEqual(urls, set(['crbug.com/1234'])) 1793 1794 def testMultipleTagSets(self) -> None: 1795 """Tests that multiple tag sets result in a scope narrowing no-op.""" 1796 amd_stats = data_types.BuildStats() 1797 amd_stats.AddPassedBuild(frozenset(['win', 'amd', 'release'])) 1798 amd_stats.AddPassedBuild(frozenset(['win', 'amd', 'debug'])) 1799 intel_stats = data_types.BuildStats() 1800 intel_stats.AddFailedBuild('1', frozenset(['win', 'intel'])) 1801 # yapf: disable 1802 test_expectation_map = data_types.TestExpectationMap({ 1803 self.filename: 1804 data_types.ExpectationBuilderMap({ 1805 data_types.Expectation( 1806 'foo/test', ['win'], 'Failure', 'crbug.com/1234'): 1807 data_types.BuilderStepMap({ 1808 'win_builder': 1809 data_types.StepBuildStatsMap({ 1810 'amd': amd_stats, 1811 'intel': intel_stats, 1812 }), 1813 }), 1814 }), 1815 }) 1816 # yapf: enable 1817 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 1818 with open(self.filename) as infile: 1819 self.assertEqual(infile.read(), 1820 FAKE_EXPECTATION_FILE_CONTENTS_WITH_COMPLEX_TAGS) 1821 self.assertEqual(urls, set()) 1822 1823 def testAmbiguousTags(self): 1824 """Tests that ambiguous tag sets result in a scope narrowing no-op.""" 1825 amd_stats = data_types.BuildStats() 1826 amd_stats.AddPassedBuild(frozenset(['win', 'amd'])) 1827 bad_amd_stats = data_types.BuildStats() 1828 bad_amd_stats.AddFailedBuild('1', frozenset(['win', 'amd'])) 1829 intel_stats = data_types.BuildStats() 1830 intel_stats.AddFailedBuild('1', frozenset(['win', 'intel'])) 1831 # yapf: disable 1832 test_expectation_map = data_types.TestExpectationMap({ 1833 self.filename: 1834 data_types.ExpectationBuilderMap({ 1835 data_types.Expectation( 1836 'foo/test', ['win'], 'Failure', 'crbug.com/1234'): 1837 data_types.BuilderStepMap({ 1838 'win_builder': 1839 data_types.StepBuildStatsMap({ 1840 'amd': amd_stats, 1841 'intel': intel_stats, 1842 'bad_amd': bad_amd_stats, 1843 }), 1844 }), 1845 }), 1846 }) 1847 # yapf: enable 1848 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 1849 with open(self.filename) as infile: 1850 self.assertEqual(infile.read(), 1851 FAKE_EXPECTATION_FILE_CONTENTS_WITH_COMPLEX_TAGS) 1852 self.assertEqual(urls, set()) 1853 1854 def testRemoveCommonTags(self) -> None: 1855 """Tests that scope narrowing removes common/redundant tags.""" 1856 amd_stats = data_types.BuildStats() 1857 amd_stats.AddPassedBuild(frozenset(['win', 'amd', 'desktop'])) 1858 intel_stats = data_types.BuildStats() 1859 intel_stats.AddFailedBuild('1', frozenset(['win', 'intel', 'desktop'])) 1860 # yapf: disable 1861 test_expectation_map = data_types.TestExpectationMap({ 1862 self.filename: 1863 data_types.ExpectationBuilderMap({ 1864 data_types.Expectation( 1865 'foo/test', ['win'], 'Failure', 'crbug.com/1234'): 1866 data_types.BuilderStepMap({ 1867 'win_builder': 1868 data_types.StepBuildStatsMap({ 1869 'amd': amd_stats, 1870 'intel': intel_stats, 1871 }), 1872 }), 1873 }), 1874 }) 1875 # yapf: enable 1876 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 1877 expected_contents = """\ 1878# tags: [ win win10 1879# linux 1880# mac ] 1881# tags: [ nvidia nvidia-0x1111 1882# intel intel-0x2222 1883# amd amd-0x3333] 1884# tags: [ release debug ] 1885# results: [ Failure RetryOnFailure ] 1886 1887crbug.com/1234 [ intel win ] foo/test [ Failure ] 1888crbug.com/2345 [ linux ] foo/test [ RetryOnFailure ] 1889""" 1890 with open(self.filename) as infile: 1891 self.assertEqual(infile.read(), expected_contents) 1892 self.assertEqual(urls, {'crbug.com/1234'}) 1893 1894 def testConsolidateKnownOverlappingTags(self) -> None: 1895 """Tests that scope narrowing consolidates known overlapping tags.""" 1896 1897 # This specific example emulates a dual GPU machine where we remove the 1898 # integrated GPU tag. 1899 def SideEffect(tags): 1900 return tags - {'intel'} 1901 1902 amd_stats = data_types.BuildStats() 1903 amd_stats.AddPassedBuild(frozenset(['win', 'amd'])) 1904 nvidia_dgpu_intel_igpu_stats = data_types.BuildStats() 1905 nvidia_dgpu_intel_igpu_stats.AddFailedBuild( 1906 '1', frozenset(['win', 'nvidia', 'intel'])) 1907 # yapf: disable 1908 test_expectation_map = data_types.TestExpectationMap({ 1909 self.filename: 1910 data_types.ExpectationBuilderMap({ 1911 data_types.Expectation( 1912 'foo/test', ['win'], 'Failure', 'crbug.com/1234'): 1913 data_types.BuilderStepMap({ 1914 'win_builder': 1915 data_types.StepBuildStatsMap({ 1916 'amd': amd_stats, 1917 'dual_gpu': nvidia_dgpu_intel_igpu_stats, 1918 }), 1919 }), 1920 }), 1921 }) 1922 # yapf: enable 1923 with mock.patch.object(self.instance, 1924 '_ConsolidateKnownOverlappingTags', 1925 side_effect=SideEffect): 1926 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 1927 expected_contents = """\ 1928# tags: [ win win10 1929# linux 1930# mac ] 1931# tags: [ nvidia nvidia-0x1111 1932# intel intel-0x2222 1933# amd amd-0x3333] 1934# tags: [ release debug ] 1935# results: [ Failure RetryOnFailure ] 1936 1937crbug.com/1234 [ nvidia win ] foo/test [ Failure ] 1938crbug.com/2345 [ linux ] foo/test [ RetryOnFailure ] 1939""" 1940 with open(self.filename) as infile: 1941 self.assertEqual(infile.read(), expected_contents) 1942 self.assertEqual(urls, set(['crbug.com/1234'])) 1943 1944 def testFilterToSpecificTags(self) -> None: 1945 """Tests that scope narrowing filters to the most specific tags.""" 1946 amd_stats = data_types.BuildStats() 1947 amd_stats.AddPassedBuild(frozenset(['win', 'amd', 'amd-0x1111'])) 1948 intel_stats = data_types.BuildStats() 1949 intel_stats.AddFailedBuild('1', frozenset(['win', 'intel', 'intel-0x2222'])) 1950 # yapf: disable 1951 test_expectation_map = data_types.TestExpectationMap({ 1952 self.filename: 1953 data_types.ExpectationBuilderMap({ 1954 data_types.Expectation( 1955 'foo/test', ['win'], 'Failure', 'crbug.com/1234'): 1956 data_types.BuilderStepMap({ 1957 'win_builder': 1958 data_types.StepBuildStatsMap({ 1959 'amd': amd_stats, 1960 'intel': intel_stats, 1961 }), 1962 }), 1963 }), 1964 }) 1965 # yapf: enable 1966 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 1967 expected_contents = """\ 1968# tags: [ win win10 1969# linux 1970# mac ] 1971# tags: [ nvidia nvidia-0x1111 1972# intel intel-0x2222 1973# amd amd-0x3333] 1974# tags: [ release debug ] 1975# results: [ Failure RetryOnFailure ] 1976 1977crbug.com/1234 [ intel-0x2222 win ] foo/test [ Failure ] 1978crbug.com/2345 [ linux ] foo/test [ RetryOnFailure ] 1979""" 1980 with open(self.filename) as infile: 1981 self.assertEqual(infile.read(), expected_contents) 1982 self.assertEqual(urls, set(['crbug.com/1234'])) 1983 1984 def testSupersetsRemoved(self) -> None: 1985 """Tests that superset tags (i.e. conflicts) are omitted.""" 1986 # These stats are set up so that the raw new tag sets are: 1987 # [{win, amd}, {win, amd, debug}] since the passed Intel build also has 1988 # "release". Thus, if we aren't correctly filtering out supersets, we'll 1989 # end up with [ amd win ] and [ amd debug win ] in the expectation file 1990 # instead of just [ amd win ]. 1991 amd_release_stats = data_types.BuildStats() 1992 amd_release_stats.AddFailedBuild('1', frozenset(['win', 'amd', 'release'])) 1993 amd_debug_stats = data_types.BuildStats() 1994 amd_debug_stats.AddFailedBuild('1', frozenset(['win', 'amd', 'debug'])) 1995 intel_stats = data_types.BuildStats() 1996 intel_stats.AddPassedBuild(frozenset(['win', 'intel', 'release'])) 1997 # yapf: disable 1998 test_expectation_map = data_types.TestExpectationMap({ 1999 self.filename: 2000 data_types.ExpectationBuilderMap({ 2001 data_types.Expectation( 2002 'foo/test', ['win'], 'Failure', 'crbug.com/1234'): 2003 data_types.BuilderStepMap({ 2004 'win_builder': 2005 data_types.StepBuildStatsMap({ 2006 'amd_release': amd_release_stats, 2007 'amd_debug': amd_debug_stats, 2008 'intel': intel_stats, 2009 }), 2010 }), 2011 }), 2012 }) 2013 # yapf: enable 2014 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 2015 expected_contents = """\ 2016# tags: [ win win10 2017# linux 2018# mac ] 2019# tags: [ nvidia nvidia-0x1111 2020# intel intel-0x2222 2021# amd amd-0x3333] 2022# tags: [ release debug ] 2023# results: [ Failure RetryOnFailure ] 2024 2025crbug.com/1234 [ amd win ] foo/test [ Failure ] 2026crbug.com/2345 [ linux ] foo/test [ RetryOnFailure ] 2027""" 2028 with open(self.filename) as infile: 2029 self.assertEqual(infile.read(), expected_contents) 2030 self.assertEqual(urls, set(['crbug.com/1234'])) 2031 2032 def testNoPassingOverlap(self): 2033 """Tests that scope narrowing works with no overlap between passing tags.""" 2034 # There is no commonality between [ amd debug ] and [ intel release ], so 2035 # the resulting expectation we generate should just be the tags from the 2036 # failed build. 2037 amd_stats = data_types.BuildStats() 2038 amd_stats.AddPassedBuild(frozenset(['win', 'amd', 'debug'])) 2039 intel_stats_debug = data_types.BuildStats() 2040 intel_stats_debug.AddFailedBuild('1', frozenset(['win', 'intel', 'debug'])) 2041 intel_stats_release = data_types.BuildStats() 2042 intel_stats_release.AddPassedBuild(frozenset(['win', 'intel', 'release'])) 2043 # yapf: disable 2044 test_expectation_map = data_types.TestExpectationMap({ 2045 self.filename: 2046 data_types.ExpectationBuilderMap({ 2047 data_types.Expectation( 2048 'foo/test', ['win'], 'Failure', 'crbug.com/1234'): 2049 data_types.BuilderStepMap({ 2050 'win_builder': 2051 data_types.StepBuildStatsMap({ 2052 'amd': amd_stats, 2053 'intel_debug': intel_stats_debug, 2054 'intel_release': intel_stats_release, 2055 }), 2056 }), 2057 }), 2058 }) 2059 # yapf: enable 2060 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 2061 expected_contents = """\ 2062# tags: [ win win10 2063# linux 2064# mac ] 2065# tags: [ nvidia nvidia-0x1111 2066# intel intel-0x2222 2067# amd amd-0x3333] 2068# tags: [ release debug ] 2069# results: [ Failure RetryOnFailure ] 2070 2071crbug.com/1234 [ debug intel win ] foo/test [ Failure ] 2072crbug.com/2345 [ linux ] foo/test [ RetryOnFailure ] 2073""" 2074 with open(self.filename) as infile: 2075 self.assertEqual(infile.read(), expected_contents) 2076 self.assertEqual(urls, set(['crbug.com/1234'])) 2077 2078 def testMultipleOverlap(self): 2079 """Tests that scope narrowing works with multiple potential overlaps.""" 2080 # [ win intel debug ], [ win intel release ], and [ win amd debug ] each 2081 # have 2/3 tags overlapping with each other, so we expect one pair to be 2082 # simplified and the other to remain the same. 2083 intel_debug_stats = data_types.BuildStats() 2084 intel_debug_stats.AddFailedBuild('1', frozenset(['win', 'intel', 'debug'])) 2085 intel_release_stats = data_types.BuildStats() 2086 intel_release_stats.AddFailedBuild('1', 2087 frozenset(['win', 'intel', 'release'])) 2088 amd_debug_stats = data_types.BuildStats() 2089 amd_debug_stats.AddFailedBuild('1', frozenset(['win', 'amd', 'debug'])) 2090 amd_release_stats = data_types.BuildStats() 2091 amd_release_stats.AddPassedBuild(frozenset(['win', 'amd', 'release'])) 2092 # yapf: disable 2093 test_expectation_map = data_types.TestExpectationMap({ 2094 self.filename: 2095 data_types.ExpectationBuilderMap({ 2096 data_types.Expectation( 2097 'foo/test', ['win'], 'Failure', 'crbug.com/1234'): 2098 data_types.BuilderStepMap({ 2099 'win_builder': 2100 data_types.StepBuildStatsMap({ 2101 'amd_debug': amd_debug_stats, 2102 'amd_release': amd_release_stats, 2103 'intel_debug': intel_debug_stats, 2104 'intel_release': intel_release_stats, 2105 }), 2106 }), 2107 }), 2108 }) 2109 # yapf: enable 2110 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 2111 # Python sets are not stable between different processes due to random hash 2112 # seeds that are on by default. Since there are two valid ways to simplify 2113 # the tags we provided, this means that the test is flaky if we only check 2114 # for one due to the non-deterministic order the tags are processed, so 2115 # instead, accept either valid output. 2116 # 2117 # Random hash seeds can be disabled by setting PYTHONHASHSEED, but that 2118 # requires that we either ensure that this test is always run with that set 2119 # (difficult/error-prone), or we manually set the seed and recreate the 2120 # process (hacky). Simply accepting either valid value instead of trying to 2121 # force a certain order seems like the better approach. 2122 expected_contents1 = """\ 2123# tags: [ win win10 2124# linux 2125# mac ] 2126# tags: [ nvidia nvidia-0x1111 2127# intel intel-0x2222 2128# amd amd-0x3333] 2129# tags: [ release debug ] 2130# results: [ Failure RetryOnFailure ] 2131 2132crbug.com/1234 [ amd debug win ] foo/test [ Failure ] 2133crbug.com/1234 [ intel win ] foo/test [ Failure ] 2134crbug.com/2345 [ linux ] foo/test [ RetryOnFailure ] 2135""" 2136 expected_contents2 = """\ 2137# tags: [ win win10 2138# linux 2139# mac ] 2140# tags: [ nvidia nvidia-0x1111 2141# intel intel-0x2222 2142# amd amd-0x3333] 2143# tags: [ release debug ] 2144# results: [ Failure RetryOnFailure ] 2145 2146crbug.com/1234 [ debug win ] foo/test [ Failure ] 2147crbug.com/1234 [ intel release win ] foo/test [ Failure ] 2148crbug.com/2345 [ linux ] foo/test [ RetryOnFailure ] 2149""" 2150 with open(self.filename) as infile: 2151 self.assertIn(infile.read(), (expected_contents1, expected_contents2)) 2152 self.assertEqual(urls, set(['crbug.com/1234'])) 2153 2154 def testMultipleOverlapRepeatedIntersection(self): 2155 """Edge case where intersection checks need to be repeated to work.""" 2156 original_contents = """\ 2157# tags: [ mac 2158# win ] 2159# tags: [ amd amd-0x3333 2160# intel intel-0x2222 intel-0x4444 2161# nvidia nvidia-0x1111 ] 2162# results: [ Failure ] 2163 2164crbug.com/1234 foo/test [ Failure ] 2165""" 2166 with open(self.filename, 'w') as outfile: 2167 outfile.write(original_contents) 2168 amd_stats = data_types.BuildStats() 2169 amd_stats.AddFailedBuild('1', frozenset(['mac', 'amd', 'amd-0x3333'])) 2170 intel_stats_1 = data_types.BuildStats() 2171 intel_stats_1.AddFailedBuild('1', 2172 frozenset(['mac', 'intel', 'intel-0x2222'])) 2173 intel_stats_2 = data_types.BuildStats() 2174 intel_stats_2.AddFailedBuild('1', 2175 frozenset(['mac', 'intel', 'intel-0x4444'])) 2176 nvidia_stats = data_types.BuildStats() 2177 nvidia_stats.AddPassedBuild(frozenset(['win', 'nvidia', 'nvidia-0x1111'])) 2178 2179 # yapf: disable 2180 test_expectation_map = data_types.TestExpectationMap({ 2181 self.filename: 2182 data_types.ExpectationBuilderMap({ 2183 data_types.Expectation( 2184 'foo/test', [], 'Failure', 'crbug.com/1234'): 2185 data_types.BuilderStepMap({ 2186 'mixed_builder': 2187 data_types.StepBuildStatsMap({ 2188 'amd': amd_stats, 2189 'intel_1': intel_stats_1, 2190 'intel_2': intel_stats_2, 2191 'nvidia': nvidia_stats, 2192 }), 2193 }), 2194 }), 2195 }) 2196 # yapf: enable 2197 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 2198 expected_contents = """\ 2199# tags: [ mac 2200# win ] 2201# tags: [ amd amd-0x3333 2202# intel intel-0x2222 intel-0x4444 2203# nvidia nvidia-0x1111 ] 2204# results: [ Failure ] 2205 2206crbug.com/1234 [ mac ] foo/test [ Failure ] 2207""" 2208 with open(self.filename) as infile: 2209 self.assertEqual(infile.read(), expected_contents) 2210 self.assertEqual(urls, set(['crbug.com/1234'])) 2211 2212 def testBlockDisableAnnotation(self) -> None: 2213 """Tests that narrowing is skipped if block annotations are present.""" 2214 original_contents = """\ 2215# tags: [ mac ] 2216# tags: [ amd intel ] 2217# results: [ Failure ] 2218 2219crbug.com/1234 [ mac ] foo/test [ Failure ] 2220# finder:disable-narrowing 2221crbug.com/2345 [ mac ] bar/test [ Failure ] 2222# finder:enable-narrowing 2223""" 2224 with open(self.filename, 'w') as outfile: 2225 outfile.write(original_contents) 2226 2227 amd_stats = data_types.BuildStats() 2228 amd_stats.AddPassedBuild(frozenset(['mac', 'amd'])) 2229 intel_stats = data_types.BuildStats() 2230 intel_stats.AddFailedBuild('1', frozenset(['mac', 'intel'])) 2231 2232 # yapf: disable 2233 test_expectation_map = data_types.TestExpectationMap({ 2234 self.filename: 2235 data_types.ExpectationBuilderMap({ 2236 data_types.Expectation( 2237 'foo/test', ['mac'], 'Failure', 'crbug.com/1234'): 2238 data_types.BuilderStepMap({ 2239 'mac_builder': 2240 data_types.StepBuildStatsMap({ 2241 'amd': amd_stats, 2242 'intel': intel_stats, 2243 }), 2244 }), 2245 data_types.Expectation( 2246 'bar/test', ['mac'], 'Failure', 'crbug.com/2345'): 2247 data_types.BuilderStepMap({ 2248 'mac_builder': 2249 data_types.StepBuildStatsMap({ 2250 'amd': amd_stats, 2251 'intel': intel_stats, 2252 }), 2253 }), 2254 }), 2255 }) 2256 # yapf: enable 2257 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 2258 expected_contents = """\ 2259# tags: [ mac ] 2260# tags: [ amd intel ] 2261# results: [ Failure ] 2262 2263crbug.com/1234 [ intel mac ] foo/test [ Failure ] 2264# finder:disable-narrowing 2265crbug.com/2345 [ mac ] bar/test [ Failure ] 2266# finder:enable-narrowing 2267""" 2268 with open(self.filename) as infile: 2269 self.assertEqual(infile.read(), expected_contents) 2270 self.assertEqual(urls, set(['crbug.com/1234'])) 2271 2272 def testNoOverlapsInNarrowedExpectations(self): 2273 """Tests that scope narrowing does not produce overlapping tag sets.""" 2274 original_contents = """\ 2275# tags: [ Linux 2276# Mac Mac10.15 Mac11 Mac11-arm64 Mac12 Mac12-arm64 2277# Win Win10.20h2 Win11 ] 2278# tags: [ Release Debug ] 2279# results: [ Failure ] 2280 2281crbug.com/874695 foo/test [ Failure ] 2282""" 2283 with open(self.filename, 'w') as outfile: 2284 outfile.write(original_contents) 2285 2286 linux_debug_stats = data_types.BuildStats() 2287 linux_debug_stats.AddPassedBuild(frozenset(['debug', 'linux'])) 2288 linux_release_stats = data_types.BuildStats() 2289 linux_release_stats.AddFailedBuild('1', frozenset(['linux', 'release'])) 2290 mac10_release_stats = data_types.BuildStats() 2291 mac10_release_stats.AddFailedBuild( 2292 '1', frozenset(['mac', 'mac10.15', 'release'])) 2293 mac11_arm_release_stats = data_types.BuildStats() 2294 mac11_arm_release_stats.AddFailedBuild( 2295 '1', frozenset(['mac', 'mac11-arm64', 'release'])) 2296 mac11_release_stats = data_types.BuildStats() 2297 mac11_release_stats.AddFailedBuild('1', 2298 frozenset(['mac', 'mac11', 'release'])) 2299 mac12_arm_release_stats = data_types.BuildStats() 2300 mac12_arm_release_stats.AddFailedBuild( 2301 '1', frozenset(['mac', 'mac12-arm64', 'release'])) 2302 mac12_debug_stats = data_types.BuildStats() 2303 mac12_debug_stats.AddFailedBuild('1', frozenset(['debug', 'mac', 'mac12'])) 2304 mac12_release_stats = data_types.BuildStats() 2305 mac12_release_stats.AddFailedBuild('1', 2306 frozenset(['mac', 'mac12', 'release'])) 2307 win10_release_stats = data_types.BuildStats() 2308 win10_release_stats.AddFailedBuild( 2309 '1', frozenset(['release', 'win', 'win10.20h2'])) 2310 win11_release_stats = data_types.BuildStats() 2311 win11_release_stats.AddFailedBuild('1', 2312 frozenset(['release', 'win', 'win11'])) 2313 # yapf: disable 2314 test_expectation_map = data_types.TestExpectationMap({ 2315 self.filename: 2316 data_types.ExpectationBuilderMap({ 2317 data_types.Expectation( 2318 'foo/test', 2319 [], 'Failure', 'crbug.com/874695'): 2320 data_types.BuilderStepMap({ 2321 'Linux Tests (dbg)(1)': 2322 data_types.StepBuildStatsMap({ 2323 'blink_web_tests': linux_debug_stats, 2324 }), 2325 'mac11-arm64-rel-tests': 2326 data_types.StepBuildStatsMap({ 2327 'blink_web_tests': mac11_arm_release_stats, 2328 }), 2329 'Mac11 Tests': 2330 data_types.StepBuildStatsMap({ 2331 'blink_web_tests': mac11_release_stats, 2332 }), 2333 'mac12-arm64-rel-tests': 2334 data_types.StepBuildStatsMap({ 2335 'blink_web_tests': mac12_arm_release_stats, 2336 }), 2337 'Mac12 Tests (dbg)': 2338 data_types.StepBuildStatsMap({ 2339 'blink_web_tests': mac12_debug_stats, 2340 }), 2341 'Mac12 Tests': 2342 data_types.StepBuildStatsMap({ 2343 'blink_web_tests': mac12_release_stats, 2344 }), 2345 'Linux Tests': 2346 data_types.StepBuildStatsMap({ 2347 'blink_web_tests': linux_release_stats, 2348 }), 2349 'WebKit Win10': 2350 data_types.StepBuildStatsMap({ 2351 'blink_web_tests': win10_release_stats, 2352 }), 2353 'Win11 Tests x64': 2354 data_types.StepBuildStatsMap({ 2355 'blink_web_tests': win11_release_stats, 2356 }), 2357 }), 2358 }), 2359 }) 2360 # yapf: enable 2361 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 2362 # Python sets are not stable between different processes due to random hash 2363 # seeds that are on by default. Since there are two valid ways to simplify 2364 # the tags we provided, this means that the test is flaky if we only check 2365 # for one due to the non-deterministic order the tags are processed, so 2366 # instead, accept either valid output. 2367 # 2368 # Random hash seeds can be disabled by setting PYTHONHASHSEED, but that 2369 # requires that we either ensure that this test is always run with that set 2370 # (difficult/error-prone), or we manually set the seed and recreate the 2371 # process (hacky). Simply accepting either valid value instead of trying to 2372 # force a certain order seems like the better approach. 2373 expected_contents1 = """\ 2374# tags: [ Linux 2375# Mac Mac10.15 Mac11 Mac11-arm64 Mac12 Mac12-arm64 2376# Win Win10.20h2 Win11 ] 2377# tags: [ Release Debug ] 2378# results: [ Failure ] 2379 2380crbug.com/874695 [ debug mac12 ] foo/test [ Failure ] 2381crbug.com/874695 [ release ] foo/test [ Failure ] 2382""" 2383 expected_contents2 = """\ 2384# tags: [ Linux 2385# Mac Mac10.15 Mac11 Mac11-arm64 Mac12 Mac12-arm64 2386# Win Win10.20h2 Win11 ] 2387# tags: [ Release Debug ] 2388# results: [ Failure ] 2389 2390crbug.com/874695 [ linux release ] foo/test [ Failure ] 2391crbug.com/874695 [ mac ] foo/test [ Failure ] 2392crbug.com/874695 [ release win ] foo/test [ Failure ] 2393""" 2394 with open(self.filename) as infile: 2395 self.assertIn(infile.read(), (expected_contents1, expected_contents2)) 2396 self.assertEqual(urls, set(['crbug.com/874695'])) 2397 2398 def testInlineDisableAnnotation(self) -> None: 2399 """Tests that narrowing is skipped if inline annotations are present.""" 2400 original_contents = """\ 2401# tags: [ mac ] 2402# tags: [ amd intel ] 2403# results: [ Failure ] 2404 2405crbug.com/1234 [ mac ] foo/test [ Failure ] 2406crbug.com/2345 [ mac ] bar/test [ Failure ] # finder:disable-narrowing 2407""" 2408 with open(self.filename, 'w') as outfile: 2409 outfile.write(original_contents) 2410 2411 amd_stats = data_types.BuildStats() 2412 amd_stats.AddPassedBuild(frozenset(['mac', 'amd'])) 2413 intel_stats = data_types.BuildStats() 2414 intel_stats.AddFailedBuild('1', frozenset(['mac', 'intel'])) 2415 2416 # yapf: disable 2417 test_expectation_map = data_types.TestExpectationMap({ 2418 self.filename: 2419 data_types.ExpectationBuilderMap({ 2420 data_types.Expectation( 2421 'foo/test', ['mac'], 'Failure', 'crbug.com/1234'): 2422 data_types.BuilderStepMap({ 2423 'mac_builder': 2424 data_types.StepBuildStatsMap({ 2425 'amd': amd_stats, 2426 'intel': intel_stats, 2427 }), 2428 }), 2429 data_types.Expectation( 2430 'bar/test', ['mac'], 'Failure', 'crbug.com/2345'): 2431 data_types.BuilderStepMap({ 2432 'mac_builder': 2433 data_types.StepBuildStatsMap({ 2434 'amd': amd_stats, 2435 'intel': intel_stats, 2436 }), 2437 }), 2438 }), 2439 }) 2440 # yapf: enable 2441 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 2442 expected_contents = """\ 2443# tags: [ mac ] 2444# tags: [ amd intel ] 2445# results: [ Failure ] 2446 2447crbug.com/1234 [ intel mac ] foo/test [ Failure ] 2448crbug.com/2345 [ mac ] bar/test [ Failure ] # finder:disable-narrowing 2449""" 2450 with open(self.filename) as infile: 2451 self.assertEqual(infile.read(), expected_contents) 2452 self.assertEqual(urls, set(['crbug.com/1234'])) 2453 2454 2455class FindOrphanedBugsUnittest(fake_filesystem_unittest.TestCase): 2456 def CreateFile(self, *args, **kwargs) -> None: 2457 # TODO(crbug.com/40160566): Remove this and just use fs.create_file() when 2458 # Catapult is updated to a newer version of pyfakefs that is compatible with 2459 # Chromium's version. 2460 if hasattr(self.fs, 'create_file'): 2461 self.fs.create_file(*args, **kwargs) 2462 else: 2463 self.fs.CreateFile(*args, **kwargs) 2464 2465 def setUp(self) -> None: 2466 expectations_dir = os.path.join(os.path.dirname(__file__), 'expectations') 2467 self.setUpPyfakefs() 2468 self.instance = expectations.Expectations() 2469 self.filepath_patcher = mock.patch.object( 2470 self.instance, 2471 'GetExpectationFilepaths', 2472 return_value=[os.path.join(expectations_dir, 'real_expectations.txt')]) 2473 self.filepath_mock = self.filepath_patcher.start() 2474 self.addCleanup(self.filepath_patcher.stop) 2475 2476 real_contents = 'crbug.com/1\ncrbug.com/2' 2477 skipped_contents = 'crbug.com/4' 2478 self.CreateFile(os.path.join(expectations_dir, 'real_expectations.txt'), 2479 contents=real_contents) 2480 self.CreateFile(os.path.join(expectations_dir, 'fake.txt'), 2481 contents=skipped_contents) 2482 2483 def testNoOrphanedBugs(self) -> None: 2484 bugs = ['crbug.com/1', 'crbug.com/2'] 2485 self.assertEqual(self.instance.FindOrphanedBugs(bugs), set()) 2486 2487 def testOrphanedBugs(self) -> None: 2488 bugs = ['crbug.com/1', 'crbug.com/3', 'crbug.com/4'] 2489 self.assertEqual(self.instance.FindOrphanedBugs(bugs), 2490 set(['crbug.com/3', 'crbug.com/4'])) 2491 2492 2493if __name__ == '__main__': 2494 unittest.main(verbosity=2) 2495