1#!/usr/bin/env vpython3 2# Copyright 2020 The Chromium Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6from __future__ import print_function 7 8import json 9import os 10import sys 11from typing import Any, Dict, Set, Tuple 12import unittest 13 14if sys.version_info[0] == 2: 15 import mock 16else: 17 import unittest.mock as mock 18 19from pyfakefs import fake_filesystem_unittest 20 21from unexpected_passes_common import builders 22from unexpected_passes_common import constants 23from unexpected_passes_common import data_types 24from unexpected_passes_common import multiprocessing_utils 25from unexpected_passes_common import unittest_utils 26 27 28class FakeFilesystemTestCaseWithFileCreation(fake_filesystem_unittest.TestCase): 29 def CreateFile(self, *args, **kwargs): 30 # TODO(crbug.com/1156806): Remove this and just use fs.create_file() when 31 # Catapult is updated to a newer version of pyfakefs that is compatible with 32 # Chromium's version. 33 if hasattr(self.fs, 'create_file'): 34 self.fs.create_file(*args, **kwargs) 35 else: 36 self.fs.CreateFile(*args, **kwargs) 37 38 39class GetCiBuildersUnittest(FakeFilesystemTestCaseWithFileCreation): 40 def setUp(self) -> None: 41 self._builders_instance = unittest_utils.GenericBuilders( 42 suite='webgl_conformance') 43 self._isolate_patcher = mock.patch.object( 44 self._builders_instance, 45 'GetIsolateNames', 46 return_value={'telemetry_gpu_integration_test'}) 47 self._isolate_mock = self._isolate_patcher.start() 48 self.addCleanup(self._isolate_patcher.stop) 49 50 def testJsonContentLoaded(self) -> None: 51 """Tests that the correct JSON data is loaded in.""" 52 self.setUpPyfakefs() 53 gpu_json = { 54 'AAAAA1 AUTOGENERATED FILE DO NOT EDIT': {}, 55 'Android Release (Nexus 5X)': { 56 'isolated_scripts': [{ 57 'args': [ 58 'webgl_conformance', 59 ], 60 'isolate_name': 61 'telemetry_gpu_integration_test', 62 }], 63 }, 64 'GPU Linux Builder': {}, 65 } 66 gpu_fyi_json = { 67 'AAAAA1 AUTOGENERATED FILE DO NOT EDIT': {}, 68 'ANGLE GPU Android Release (Nexus 5X)': { 69 'isolated_scripts': [{ 70 'args': [ 71 'webgl_conformance', 72 ], 73 'isolate_name': 74 'telemetry_gpu_integration_test', 75 }], 76 }, 77 'GPU FYI Linux Builder': {}, 78 } 79 # Should be ignored. 80 tryserver_json = { 81 'AAAAA1 AUTOGENERATED FILE DO NOT EDIT': {}, 82 'Trybot': { 83 'isolated_scripts': [{ 84 'args': [ 85 'webgl_conformance', 86 ], 87 'isolate_name': 88 'telemetry_gpu_integration_test', 89 }], 90 }, 91 } 92 # Also should be ignored. 93 not_buildbot_json = { 94 'Not buildbot': { 95 'isolated_scripts': [{ 96 'args': [ 97 'webgl_conformance', 98 ], 99 'isolate_name': 100 'telemetry_gpu_integration_test', 101 }], 102 }, 103 } 104 105 self.CreateFile(os.path.join(builders.TESTING_BUILDBOT_DIR, 106 'chromium.gpu.json'), 107 contents=json.dumps(gpu_json)) 108 self.CreateFile(os.path.join(builders.TESTING_BUILDBOT_DIR, 109 'chromium.gpu.fyi.json'), 110 contents=json.dumps(gpu_fyi_json)) 111 self.CreateFile(os.path.join(builders.TESTING_BUILDBOT_DIR, 112 'tryserver.gpu.json'), 113 contents=json.dumps(tryserver_json)) 114 self.CreateFile(os.path.join(builders.TESTING_BUILDBOT_DIR, 115 'not_buildbot.json'), 116 contents=json.dumps(not_buildbot_json)) 117 118 gpu_builders = self._builders_instance.GetCiBuilders() 119 self.assertEqual( 120 gpu_builders, 121 set([ 122 data_types.BuilderEntry('Android Release (Nexus 5X)', 123 constants.BuilderTypes.CI, False), 124 data_types.BuilderEntry('ANGLE GPU Android Release (Nexus 5X)', 125 constants.BuilderTypes.CI, False), 126 data_types.BuilderEntry('GPU Linux Builder', 127 constants.BuilderTypes.CI, False), 128 data_types.BuilderEntry('GPU FYI Linux Builder', 129 constants.BuilderTypes.CI, False), 130 ])) 131 132 def testPublicInternalBuilders(self) -> None: 133 """Tests that public internal builders are treated as internal.""" 134 self.setUpPyfakefs() 135 gpu_json = { 136 'AAAAA1 AUTOGENERATED FILE DO NOT EDIT': {}, 137 'Android Release (Nexus 5X)': { 138 'isolated_scripts': [{ 139 'args': [ 140 'webgl_conformance', 141 ], 142 'isolate_name': 143 'telemetry_gpu_integration_test', 144 }], 145 }, 146 'GPU Linux Builder': {}, 147 } 148 gpu_internal_json = { 149 'AAAAA1 AUTOGENERATED FILE DO NOT EDIT': {}, 150 'Android Chrome Release (Nexus 5X)': { 151 'isolated_scripts': [{ 152 'args': [ 153 'webgl_conformance', 154 ], 155 'isolate_name': 156 'telemetry_gpu_integration_test', 157 }], 158 }, 159 'GPU Chrome Linux Builder': {}, 160 } 161 internal_json = { 162 'AAAAA1 AUTOGENERATED FILE DO NOT EDIT': {}, 163 'Android Internal Release (Nexus 5X)': { 164 'isolated_scripts': [{ 165 'args': [ 166 'webgl_conformance', 167 ], 168 'isolate_name': 169 'telemetry_gpu_integration_test', 170 }], 171 }, 172 'GPU Internal Linux Builder': {}, 173 } 174 175 self.CreateFile(os.path.join(builders.TESTING_BUILDBOT_DIR, 176 'chromium.gpu.json'), 177 contents=json.dumps(gpu_json)) 178 self.CreateFile(os.path.join(builders.TESTING_BUILDBOT_DIR, 179 'chrome.gpu.fyi.json'), 180 contents=json.dumps(gpu_internal_json)) 181 self.CreateFile(os.path.join(builders.INTERNAL_TESTING_BUILDBOT_DIR, 182 'internal.json'), 183 contents=json.dumps(internal_json)) 184 185 gpu_builders = self._builders_instance.GetCiBuilders() 186 self.assertEqual( 187 gpu_builders, 188 set([ 189 data_types.BuilderEntry('Android Release (Nexus 5X)', 190 constants.BuilderTypes.CI, False), 191 data_types.BuilderEntry('GPU Linux Builder', 192 constants.BuilderTypes.CI, False), 193 ])) 194 195 internal_instance = unittest_utils.GenericBuilders( 196 suite='webgl_conformance', include_internal_builders=True) 197 with mock.patch.object(internal_instance, 198 'GetIsolateNames', 199 return_value={'telemetry_gpu_integration_test'}): 200 gpu_builders = internal_instance.GetCiBuilders() 201 self.assertEqual( 202 gpu_builders, 203 set([ 204 data_types.BuilderEntry('Android Release (Nexus 5X)', 205 constants.BuilderTypes.CI, False), 206 data_types.BuilderEntry('Android Chrome Release (Nexus 5X)', 207 constants.BuilderTypes.CI, True), 208 data_types.BuilderEntry('Android Internal Release (Nexus 5X)', 209 constants.BuilderTypes.CI, True), 210 data_types.BuilderEntry('GPU Linux Builder', 211 constants.BuilderTypes.CI, False), 212 data_types.BuilderEntry('GPU Chrome Linux Builder', 213 constants.BuilderTypes.CI, True), 214 data_types.BuilderEntry('GPU Internal Linux Builder', 215 constants.BuilderTypes.CI, True), 216 ])) 217 218 def testFilterBySuite(self) -> None: 219 """Tests that only builders that run the given suite are returned.""" 220 221 def SideEffect(tm: Dict[str, Any]) -> bool: 222 tests = tm.get('isolated_scripts', []) 223 for t in tests: 224 if t.get('isolate_name') == 'foo_integration_test': 225 if 'webgl_conformance' in t.get('args', []): 226 return True 227 return False 228 229 self.setUpPyfakefs() 230 gpu_json = { 231 'AAAAA1 AUTOGENERATED FILE DO NOT EDIT': {}, 232 'Android Tester': { 233 'isolated_scripts': [ 234 { 235 'args': [ 236 'webgl_conformance', 237 ], 238 'isolate_name': 'not_telemetry', 239 }, 240 ], 241 }, 242 'Linux Tester': { 243 'isolated_scripts': [ 244 { 245 'args': [ 246 'not_a_suite', 247 ], 248 'isolate_name': 'foo_integration_test', 249 }, 250 ], 251 }, 252 'Windows Tester': { 253 'isolated_scripts': [ 254 { 255 'args': [ 256 'webgl_conformance', 257 ], 258 'isolate_name': 'foo_integration_test', 259 }, 260 ], 261 }, 262 } 263 264 self.CreateFile(os.path.join(builders.TESTING_BUILDBOT_DIR, 265 'chromium.json'), 266 contents=json.dumps(gpu_json)) 267 268 with mock.patch.object(self._builders_instance, 269 '_BuilderRunsTestOfInterest', 270 side_effect=SideEffect): 271 gpu_builders = self._builders_instance.GetCiBuilders() 272 self.assertEqual( 273 gpu_builders, 274 set([ 275 data_types.BuilderEntry('Windows Tester', constants.BuilderTypes.CI, 276 False) 277 ])) 278 279 def testRealContentCanBeLoaded(self) -> None: 280 """Tests that *something* from the real JSON files can be loaded.""" 281 # This directory is not available on swarming, so if it doesn't exist, just 282 # skip the test. 283 if not os.path.exists(builders.TESTING_BUILDBOT_DIR): 284 return 285 self.assertNotEqual(len(self._builders_instance.GetCiBuilders()), 0) 286 287 288class GetMirroredBuildersForCiBuilderUnittest(unittest.TestCase): 289 def setUp(self) -> None: 290 self._builders_instance = builders.Builders('suite', False) 291 self._bb_patcher = mock.patch.object(self._builders_instance, 292 '_GetBuildbucketOutputForCiBuilder') 293 self._bb_mock = self._bb_patcher.start() 294 self.addCleanup(self._bb_patcher.stop) 295 self._fake_ci_patcher = mock.patch.object(self._builders_instance, 296 'GetFakeCiBuilders', 297 return_value={}) 298 self._fake_ci_mock = self._fake_ci_patcher.start() 299 self.addCleanup(self._fake_ci_patcher.stop) 300 self._non_chromium_patcher = mock.patch.object( 301 self._builders_instance, 302 'GetNonChromiumBuilders', 303 return_value={'foo_non_chromium'}) 304 self._non_chromium_mock = self._non_chromium_patcher.start() 305 self.addCleanup(self._non_chromium_patcher.stop) 306 307 def testFakeCiBuilder(self) -> None: 308 """Tests that a fake CI builder gets properly mapped.""" 309 self._fake_ci_mock.return_value = { 310 data_types.BuilderEntry('foo_ci', constants.BuilderTypes.CI, False): 311 {data_types.BuilderEntry('foo_try', constants.BuilderTypes.TRY, False)} 312 } 313 try_builder, found_mirror = ( 314 self._builders_instance._GetMirroredBuildersForCiBuilder( 315 data_types.BuilderEntry('foo_ci', constants.BuilderTypes.CI, 316 False))) 317 self.assertTrue(found_mirror) 318 self.assertEqual( 319 try_builder, 320 set([ 321 data_types.BuilderEntry('foo_try', constants.BuilderTypes.TRY, 322 False) 323 ])) 324 self._bb_mock.assert_not_called() 325 326 def testNoBuildbucketOutput(self) -> None: 327 """Tests that a failure to get Buildbucket output is surfaced.""" 328 self._bb_mock.return_value = '' 329 builder_entry = data_types.BuilderEntry('nonexistent', 330 constants.BuilderTypes.CI, False) 331 try_builder, found_mirror = ( 332 self._builders_instance._GetMirroredBuildersForCiBuilder(builder_entry)) 333 self.assertFalse(found_mirror) 334 self.assertEqual(try_builder, set([builder_entry])) 335 336 def testBuildbucketOutput(self): 337 """Tests that Buildbucket output is parsed correctly.""" 338 self._bb_mock.return_value = json.dumps({ 339 'output': { 340 'properties': { 341 'mirrored_builders': [ 342 'try:foo_try', 343 'try:bar_try', 344 ] 345 } 346 } 347 }) 348 try_builders, found_mirror = ( 349 self._builders_instance._GetMirroredBuildersForCiBuilder( 350 data_types.BuilderEntry('foo_ci', constants.BuilderTypes.CI, 351 False))) 352 self.assertTrue(found_mirror) 353 self.assertEqual( 354 try_builders, 355 set([ 356 data_types.BuilderEntry('foo_try', constants.BuilderTypes.TRY, 357 False), 358 data_types.BuilderEntry('bar_try', constants.BuilderTypes.TRY, 359 False) 360 ])) 361 362 def testBuildbucketOutputInternal(self) -> None: 363 """Tests that internal Buildbucket output is parsed correctly.""" 364 self._bb_mock.return_value = json.dumps({ 365 'output': { 366 'properties': { 367 'mirrored_builders': [ 368 'try:foo_try', 369 'try:bar_try', 370 ] 371 } 372 } 373 }) 374 try_builders, found_mirror = ( 375 self._builders_instance._GetMirroredBuildersForCiBuilder( 376 data_types.BuilderEntry('foo_ci', constants.BuilderTypes.CI, True))) 377 self.assertTrue(found_mirror) 378 self.assertEqual( 379 try_builders, 380 set([ 381 data_types.BuilderEntry('foo_try', constants.BuilderTypes.TRY, 382 True), 383 data_types.BuilderEntry('bar_try', constants.BuilderTypes.TRY, True) 384 ])) 385 386 387class GetTryBuildersUnittest(FakeFilesystemTestCaseWithFileCreation): 388 def setUp(self) -> None: 389 self._builders_instance = builders.Builders('suite', False) 390 self._get_patcher = mock.patch.object(self._builders_instance, 391 '_GetMirroredBuildersForCiBuilder') 392 self._get_mock = self._get_patcher.start() 393 self.addCleanup(self._get_patcher.stop) 394 self._runs_test_patcher = mock.patch.object(self._builders_instance, 395 '_BuilderRunsTestOfInterest') 396 self._runs_test_mock = self._runs_test_patcher.start() 397 self.addCleanup(self._runs_test_patcher.stop) 398 self._pool_patcher = mock.patch.object(multiprocessing_utils, 399 'GetProcessPool') 400 self._pool_mock = self._pool_patcher.start() 401 self._pool_mock.return_value = unittest_utils.FakePool() 402 self.addCleanup(self._pool_patcher.stop) 403 404 self.setUpPyfakefs() 405 # Make sure the directory exists. 406 self.CreateFile( 407 os.path.join(builders.TESTING_BUILDBOT_DIR, 'placeholder.txt')) 408 409 def testMirrorNoOutputCausesFailure(self) -> None: 410 """Tests that a failure to get Buildbot output raises an exception.""" 411 builder = data_types.BuilderEntry('foo_ci', constants.BuilderTypes.CI, 412 False) 413 self._get_mock.return_value = (set([builder]), False) 414 self._runs_test_mock.return_value = True 415 with self.assertRaises(RuntimeError): 416 self._builders_instance.GetTryBuilders([builder]) 417 418 def testMirrorOutputReturned(self) -> None: 419 """Tests that parsed, mirrored builders get returned on success.""" 420 421 def SideEffect(ci_builder: data_types.BuilderEntry 422 ) -> Tuple[Set[data_types.BuilderEntry], bool]: 423 b = [ 424 data_types.BuilderEntry(ci_builder.name.replace('ci', 'try'), 425 constants.BuilderTypes.TRY, False), 426 data_types.BuilderEntry(ci_builder.name.replace('ci', 'try2'), 427 constants.BuilderTypes.TRY, False), 428 ] 429 return set(b), True 430 431 self._get_mock.side_effect = SideEffect 432 self._runs_test_mock.return_value = False 433 mirrored_builders = self._builders_instance.GetTryBuilders([ 434 data_types.BuilderEntry('foo_ci', constants.BuilderTypes.CI, False), 435 data_types.BuilderEntry('bar_ci', constants.BuilderTypes.CI, False), 436 ]) 437 self.assertEqual( 438 mirrored_builders, 439 set([ 440 data_types.BuilderEntry('foo_try', constants.BuilderTypes.TRY, 441 False), 442 data_types.BuilderEntry('foo_try2', constants.BuilderTypes.TRY, 443 False), 444 data_types.BuilderEntry('bar_try', constants.BuilderTypes.TRY, 445 False), 446 data_types.BuilderEntry('bar_try2', constants.BuilderTypes.TRY, 447 False), 448 ])) 449 450 def testDedicatedJsonContentLoaded(self) -> None: 451 """Tests that tryserver JSON content is loaded.""" 452 453 def SideEffect(test_spec: Dict[str, Any]) -> bool: 454 # Treat non-empty test specs as valid. 455 return bool(test_spec) 456 457 self._runs_test_mock.side_effect = SideEffect 458 # Should be ignored. 459 gpu_json = { 460 'AAAAA1 AUTOGENERATED FILE DO NOT EDIT': {}, 461 'Android Release (Nexus 5X)': { 462 'isolated_scripts': [{ 463 'args': [ 464 'webgl_conformance', 465 ], 466 'isolate_name': 467 'telemetry_gpu_integration_test', 468 }], 469 }, 470 'GPU Linux Builder': {}, 471 } 472 # Should be ignored. 473 gpu_fyi_json = { 474 'AAAAA1 AUTOGENERATED FILE DO NOT EDIT': {}, 475 'ANGLE GPU Android Release (Nexus 5X)': { 476 'isolated_scripts': [{ 477 'args': [ 478 'webgl_conformance', 479 ], 480 'isolate_name': 481 'telemetry_gpu_integration_test', 482 }], 483 }, 484 'GPU FYI Linux Builder': {}, 485 } 486 tryserver_json = { 487 'AAAAA1 AUTOGENERATED FILE DO NOT EDIT': {}, 488 'Trybot': { 489 'isolated_scripts': [{ 490 'args': [ 491 'webgl_conformance', 492 ], 493 'isolate_name': 494 'telemetry_gpu_integration_test', 495 }], 496 }, 497 'Trybot Empty': {}, 498 } 499 # Also should be ignored. 500 not_buildbot_json = { 501 'Not buildbot': { 502 'isolated_scripts': [{ 503 'args': [ 504 'webgl_conformance', 505 ], 506 'isolate_name': 507 'telemetry_gpu_integration_test', 508 }], 509 }, 510 } 511 512 self.CreateFile(os.path.join(builders.TESTING_BUILDBOT_DIR, 513 'chromium.gpu.json'), 514 contents=json.dumps(gpu_json)) 515 self.CreateFile(os.path.join(builders.TESTING_BUILDBOT_DIR, 516 'chromium.gpu.fyi.json'), 517 contents=json.dumps(gpu_fyi_json)) 518 self.CreateFile(os.path.join(builders.TESTING_BUILDBOT_DIR, 519 'tryserver.gpu.json'), 520 contents=json.dumps(tryserver_json)) 521 self.CreateFile(os.path.join(builders.TESTING_BUILDBOT_DIR, 522 'not_buildbot.json'), 523 contents=json.dumps(not_buildbot_json)) 524 525 gpu_builders = self._builders_instance.GetTryBuilders({}) 526 self.assertEqual( 527 gpu_builders, 528 set([ 529 data_types.BuilderEntry('Trybot', constants.BuilderTypes.TRY, 530 False), 531 ])) 532 533 def testDedicatedFilterBySuite(self) -> None: 534 """Tests that only builders that run the given suite are returned.""" 535 536 def SideEffect(tm: Dict[str, Any]) -> bool: 537 tests = tm.get('isolated_scripts', []) 538 for t in tests: 539 if t.get('isolate_name') == 'foo_integration_test': 540 if 'webgl_conformance' in t.get('args', []): 541 return True 542 return False 543 544 self._runs_test_mock.side_effect = SideEffect 545 gpu_json = { 546 'AAAAA1 AUTOGENERATED FILE DO NOT EDIT': {}, 547 'Android Tester': { 548 'isolated_scripts': [ 549 { 550 'args': [ 551 'webgl_conformance', 552 ], 553 'isolate_name': 'not_telemetry', 554 }, 555 ], 556 }, 557 'Linux Tester': { 558 'isolated_scripts': [ 559 { 560 'args': [ 561 'not_a_suite', 562 ], 563 'isolate_name': 'foo_integration_test', 564 }, 565 ], 566 }, 567 'Windows Tester': { 568 'isolated_scripts': [ 569 { 570 'args': [ 571 'webgl_conformance', 572 ], 573 'isolate_name': 'foo_integration_test', 574 }, 575 ], 576 }, 577 } 578 579 self.CreateFile(os.path.join(builders.TESTING_BUILDBOT_DIR, 580 'tryserver.chromium.json'), 581 contents=json.dumps(gpu_json)) 582 583 gpu_builders = self._builders_instance.GetTryBuilders({}) 584 self.assertEqual( 585 gpu_builders, 586 set([ 587 data_types.BuilderEntry('Windows Tester', 588 constants.BuilderTypes.TRY, False) 589 ])) 590 591 def testDedicatedAndMirroredCombined(self) -> None: 592 """Tests that both dedicated and mirrored trybots are returned.""" 593 594 def SideEffect(_: Any) -> Tuple[Set[data_types.BuilderEntry], bool]: 595 return set({ 596 data_types.BuilderEntry('mirrored_trybot', constants.BuilderTypes.TRY, 597 False) 598 }), True 599 600 self._get_mock.side_effect = SideEffect 601 self._runs_test_mock.return_value = True 602 tryserver_json = { 603 'AAAAA1 AUTOGENERATED FILE DO NOT EDIT': {}, 604 'Trybot': { 605 'isolated_scripts': [{ 606 'args': [ 607 'webgl_conformance', 608 ], 609 'isolate_name': 610 'telemetry_gpu_integration_test', 611 }], 612 }, 613 } 614 615 self.CreateFile(os.path.join(builders.TESTING_BUILDBOT_DIR, 616 'tryserver.chromium.json'), 617 contents=json.dumps(tryserver_json)) 618 619 try_builders = self._builders_instance.GetTryBuilders({ 620 data_types.BuilderEntry('ci_builder', constants.BuilderTypes.CI, False) 621 }) 622 self.assertEqual( 623 try_builders, { 624 data_types.BuilderEntry('mirrored_trybot', 625 constants.BuilderTypes.TRY, False), 626 data_types.BuilderEntry('Trybot', constants.BuilderTypes.TRY, False) 627 }) 628 629 630if __name__ == '__main__': 631 unittest.main(verbosity=2) 632