1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# Copyright 2019 The ChromiumOS Authors 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7# pylint: disable=protected-access 8 9"""Tests for LLVM bisection.""" 10 11 12import json 13import os 14import subprocess 15import unittest 16import unittest.mock as mock 17 18import chroot 19import get_llvm_hash 20import git_llvm_rev 21import llvm_bisection 22import modify_a_tryjob 23import test_helpers 24 25 26class LLVMBisectionTest(unittest.TestCase): 27 """Unittests for LLVM bisection.""" 28 29 def testGetRemainingRangePassed(self): 30 start = 100 31 end = 150 32 33 test_tryjobs = [ 34 { 35 "rev": 110, 36 "status": "good", 37 "link": "https://some_tryjob_1_url.com", 38 }, 39 { 40 "rev": 120, 41 "status": "good", 42 "link": "https://some_tryjob_2_url.com", 43 }, 44 { 45 "rev": 130, 46 "status": "pending", 47 "link": "https://some_tryjob_3_url.com", 48 }, 49 { 50 "rev": 135, 51 "status": "skip", 52 "link": "https://some_tryjob_4_url.com", 53 }, 54 { 55 "rev": 140, 56 "status": "bad", 57 "link": "https://some_tryjob_5_url.com", 58 }, 59 ] 60 61 # Tuple consists of the new good revision, the new bad revision, a set of 62 # 'pending' revisions, and a set of 'skip' revisions. 63 expected_revisions_tuple = 120, 140, {130}, {135} 64 65 self.assertEqual( 66 llvm_bisection.GetRemainingRange(start, end, test_tryjobs), 67 expected_revisions_tuple, 68 ) 69 70 def testGetRemainingRangeFailedWithMissingStatus(self): 71 start = 100 72 end = 150 73 74 test_tryjobs = [ 75 { 76 "rev": 105, 77 "status": "good", 78 "link": "https://some_tryjob_1_url.com", 79 }, 80 { 81 "rev": 120, 82 "status": None, 83 "link": "https://some_tryjob_2_url.com", 84 }, 85 { 86 "rev": 140, 87 "status": "bad", 88 "link": "https://some_tryjob_3_url.com", 89 }, 90 ] 91 92 with self.assertRaises(ValueError) as err: 93 llvm_bisection.GetRemainingRange(start, end, test_tryjobs) 94 95 error_message = ( 96 '"status" is missing or has no value, please ' 97 "go to %s and update it" % test_tryjobs[1]["link"] 98 ) 99 self.assertEqual(str(err.exception), error_message) 100 101 def testGetRemainingRangeFailedWithInvalidRange(self): 102 start = 100 103 end = 150 104 105 test_tryjobs = [ 106 { 107 "rev": 110, 108 "status": "bad", 109 "link": "https://some_tryjob_1_url.com", 110 }, 111 { 112 "rev": 125, 113 "status": "skip", 114 "link": "https://some_tryjob_2_url.com", 115 }, 116 { 117 "rev": 140, 118 "status": "good", 119 "link": "https://some_tryjob_3_url.com", 120 }, 121 ] 122 123 with self.assertRaises(AssertionError) as err: 124 llvm_bisection.GetRemainingRange(start, end, test_tryjobs) 125 126 expected_error_message = ( 127 "Bisection is broken because %d (good) is >= " 128 "%d (bad)" % (test_tryjobs[2]["rev"], test_tryjobs[0]["rev"]) 129 ) 130 131 self.assertEqual(str(err.exception), expected_error_message) 132 133 @mock.patch.object(get_llvm_hash, "GetGitHashFrom") 134 def testGetCommitsBetweenPassed(self, mock_get_git_hash): 135 start = git_llvm_rev.base_llvm_revision 136 end = start + 10 137 test_pending_revisions = {start + 7} 138 test_skip_revisions = { 139 start + 1, 140 start + 2, 141 start + 4, 142 start + 8, 143 start + 9, 144 } 145 parallel = 3 146 abs_path_to_src = "/abs/path/to/src" 147 148 revs = ["a123testhash3", "a123testhash5"] 149 mock_get_git_hash.side_effect = revs 150 151 git_hashes = [ 152 git_llvm_rev.base_llvm_revision + 3, 153 git_llvm_rev.base_llvm_revision + 5, 154 ] 155 156 self.assertEqual( 157 llvm_bisection.GetCommitsBetween( 158 start, 159 end, 160 parallel, 161 abs_path_to_src, 162 test_pending_revisions, 163 test_skip_revisions, 164 ), 165 (git_hashes, revs), 166 ) 167 168 def testLoadStatusFilePassedWithExistingFile(self): 169 start = 100 170 end = 150 171 172 test_bisect_state = {"start": start, "end": end, "jobs": []} 173 174 # Simulate that the status file exists. 175 with test_helpers.CreateTemporaryJsonFile() as temp_json_file: 176 with open(temp_json_file, "w") as f: 177 test_helpers.WritePrettyJsonFile(test_bisect_state, f) 178 179 self.assertEqual( 180 llvm_bisection.LoadStatusFile(temp_json_file, start, end), 181 test_bisect_state, 182 ) 183 184 def testLoadStatusFilePassedWithoutExistingFile(self): 185 start = 200 186 end = 250 187 188 expected_bisect_state = {"start": start, "end": end, "jobs": []} 189 190 last_tested = "/abs/path/to/file_that_does_not_exist.json" 191 192 self.assertEqual( 193 llvm_bisection.LoadStatusFile(last_tested, start, end), 194 expected_bisect_state, 195 ) 196 197 @mock.patch.object(modify_a_tryjob, "AddTryjob") 198 def testBisectPassed(self, mock_add_tryjob): 199 200 git_hash_list = ["a123testhash1", "a123testhash2", "a123testhash3"] 201 revisions_list = [102, 104, 106] 202 203 # Simulate behavior of `AddTryjob()` when successfully launched a tryjob for 204 # the updated packages. 205 @test_helpers.CallCountsToMockFunctions 206 def MockAddTryjob( 207 call_count, 208 _packages, 209 _git_hash, 210 _revision, 211 _chroot_path, 212 _patch_file, 213 _extra_cls, 214 _options, 215 _builder, 216 _verbose, 217 _svn_revision, 218 ): 219 220 if call_count < 2: 221 return {"rev": revisions_list[call_count], "status": "pending"} 222 223 # Simulate an exception happened along the way when updating the 224 # packages' `LLVM_NEXT_HASH`. 225 if call_count == 2: 226 raise ValueError("Unable to launch tryjob") 227 228 assert False, "Called `AddTryjob()` more than expected." 229 230 # Use the test function to simulate `AddTryjob()`. 231 mock_add_tryjob.side_effect = MockAddTryjob 232 233 start = 100 234 end = 110 235 236 bisection_contents = {"start": start, "end": end, "jobs": []} 237 238 args_output = test_helpers.ArgsOutputTest() 239 240 packages = ["sys-devel/llvm"] 241 patch_file = "/abs/path/to/PATCHES.json" 242 243 # Create a temporary .JSON file to simulate a status file for bisection. 244 with test_helpers.CreateTemporaryJsonFile() as temp_json_file: 245 with open(temp_json_file, "w") as f: 246 test_helpers.WritePrettyJsonFile(bisection_contents, f) 247 248 # Verify that the status file is updated when an exception happened when 249 # attempting to launch a revision (i.e. progress is not lost). 250 with self.assertRaises(ValueError) as err: 251 llvm_bisection.Bisect( 252 revisions_list, 253 git_hash_list, 254 bisection_contents, 255 temp_json_file, 256 packages, 257 args_output.chroot_path, 258 patch_file, 259 args_output.extra_change_lists, 260 args_output.options, 261 args_output.builders, 262 args_output.verbose, 263 ) 264 265 expected_bisection_contents = { 266 "start": start, 267 "end": end, 268 "jobs": [ 269 {"rev": revisions_list[0], "status": "pending"}, 270 {"rev": revisions_list[1], "status": "pending"}, 271 ], 272 } 273 274 # Verify that the launched tryjobs were added to the status file when 275 # an exception happened. 276 with open(temp_json_file) as f: 277 json_contents = json.load(f) 278 279 self.assertEqual(json_contents, expected_bisection_contents) 280 281 self.assertEqual(str(err.exception), "Unable to launch tryjob") 282 283 self.assertEqual(mock_add_tryjob.call_count, 3) 284 285 @mock.patch.object(subprocess, "check_output", return_value=None) 286 @mock.patch.object( 287 get_llvm_hash.LLVMHash, "GetLLVMHash", return_value="a123testhash4" 288 ) 289 @mock.patch.object(llvm_bisection, "GetCommitsBetween") 290 @mock.patch.object(llvm_bisection, "GetRemainingRange") 291 @mock.patch.object(llvm_bisection, "LoadStatusFile") 292 @mock.patch.object(chroot, "VerifyOutsideChroot", return_value=True) 293 def testMainPassed( 294 self, 295 mock_outside_chroot, 296 mock_load_status_file, 297 mock_get_range, 298 mock_get_revision_and_hash_list, 299 _mock_get_bad_llvm_hash, 300 mock_abandon_cl, 301 ): 302 303 start = 500 304 end = 502 305 cl = 1 306 307 bisect_state = { 308 "start": start, 309 "end": end, 310 "jobs": [{"rev": 501, "status": "bad", "cl": cl}], 311 } 312 313 skip_revisions = {501} 314 pending_revisions = {} 315 316 mock_load_status_file.return_value = bisect_state 317 318 mock_get_range.return_value = ( 319 start, 320 end, 321 pending_revisions, 322 skip_revisions, 323 ) 324 325 mock_get_revision_and_hash_list.return_value = [], [] 326 327 args_output = test_helpers.ArgsOutputTest() 328 args_output.start_rev = start 329 args_output.end_rev = end 330 args_output.parallel = 3 331 args_output.src_path = None 332 args_output.chroot_path = "somepath" 333 args_output.cleanup = True 334 335 self.assertEqual( 336 llvm_bisection.main(args_output), 337 llvm_bisection.BisectionExitStatus.BISECTION_COMPLETE.value, 338 ) 339 340 mock_outside_chroot.assert_called_once() 341 342 mock_load_status_file.assert_called_once() 343 344 mock_get_range.assert_called_once() 345 346 mock_get_revision_and_hash_list.assert_called_once() 347 348 mock_abandon_cl.assert_called_once() 349 self.assertEqual( 350 mock_abandon_cl.call_args, 351 mock.call( 352 [ 353 os.path.join( 354 args_output.chroot_path, "chromite/bin/gerrit" 355 ), 356 "abandon", 357 str(cl), 358 ], 359 stderr=subprocess.STDOUT, 360 encoding="utf-8", 361 ), 362 ) 363 364 @mock.patch.object(llvm_bisection, "LoadStatusFile") 365 @mock.patch.object(chroot, "VerifyOutsideChroot", return_value=True) 366 def testMainFailedWithInvalidRange( 367 self, mock_outside_chroot, mock_load_status_file 368 ): 369 370 start = 500 371 end = 502 372 373 bisect_state = { 374 "start": start - 1, 375 "end": end, 376 } 377 378 mock_load_status_file.return_value = bisect_state 379 380 args_output = test_helpers.ArgsOutputTest() 381 args_output.start_rev = start 382 args_output.end_rev = end 383 args_output.parallel = 3 384 args_output.src_path = None 385 386 with self.assertRaises(ValueError) as err: 387 llvm_bisection.main(args_output) 388 389 error_message = ( 390 f"The start {start} or the end {end} version provided is " 391 f'different than "start" {bisect_state["start"]} or "end" ' 392 f'{bisect_state["end"]} in the .JSON file' 393 ) 394 395 self.assertEqual(str(err.exception), error_message) 396 397 mock_outside_chroot.assert_called_once() 398 399 mock_load_status_file.assert_called_once() 400 401 @mock.patch.object(llvm_bisection, "GetCommitsBetween") 402 @mock.patch.object(llvm_bisection, "GetRemainingRange") 403 @mock.patch.object(llvm_bisection, "LoadStatusFile") 404 @mock.patch.object(chroot, "VerifyOutsideChroot", return_value=True) 405 def testMainFailedWithPendingBuilds( 406 self, 407 mock_outside_chroot, 408 mock_load_status_file, 409 mock_get_range, 410 mock_get_revision_and_hash_list, 411 ): 412 413 start = 500 414 end = 502 415 rev = 501 416 417 bisect_state = { 418 "start": start, 419 "end": end, 420 "jobs": [{"rev": rev, "status": "pending"}], 421 } 422 423 skip_revisions = {} 424 pending_revisions = {rev} 425 426 mock_load_status_file.return_value = bisect_state 427 428 mock_get_range.return_value = ( 429 start, 430 end, 431 pending_revisions, 432 skip_revisions, 433 ) 434 435 mock_get_revision_and_hash_list.return_value = [], [] 436 437 args_output = test_helpers.ArgsOutputTest() 438 args_output.start_rev = start 439 args_output.end_rev = end 440 args_output.parallel = 3 441 args_output.src_path = None 442 443 with self.assertRaises(ValueError) as err: 444 llvm_bisection.main(args_output) 445 446 error_message = ( 447 f"No revisions between start {start} and end {end} to " 448 "create tryjobs\nThe following tryjobs are pending:\n" 449 f"{rev}\n" 450 ) 451 452 self.assertEqual(str(err.exception), error_message) 453 454 mock_outside_chroot.assert_called_once() 455 456 mock_load_status_file.assert_called_once() 457 458 mock_get_range.assert_called_once() 459 460 mock_get_revision_and_hash_list.assert_called_once() 461 462 @mock.patch.object(llvm_bisection, "GetCommitsBetween") 463 @mock.patch.object(llvm_bisection, "GetRemainingRange") 464 @mock.patch.object(llvm_bisection, "LoadStatusFile") 465 @mock.patch.object(chroot, "VerifyOutsideChroot", return_value=True) 466 def testMainFailedWithDuplicateBuilds( 467 self, 468 mock_outside_chroot, 469 mock_load_status_file, 470 mock_get_range, 471 mock_get_revision_and_hash_list, 472 ): 473 474 start = 500 475 end = 502 476 rev = 501 477 git_hash = "a123testhash1" 478 479 bisect_state = { 480 "start": start, 481 "end": end, 482 "jobs": [{"rev": rev, "status": "pending"}], 483 } 484 485 skip_revisions = {} 486 pending_revisions = {rev} 487 488 mock_load_status_file.return_value = bisect_state 489 490 mock_get_range.return_value = ( 491 start, 492 end, 493 pending_revisions, 494 skip_revisions, 495 ) 496 497 mock_get_revision_and_hash_list.return_value = [rev], [git_hash] 498 499 args_output = test_helpers.ArgsOutputTest() 500 args_output.start_rev = start 501 args_output.end_rev = end 502 args_output.parallel = 3 503 args_output.src_path = None 504 505 with self.assertRaises(ValueError) as err: 506 llvm_bisection.main(args_output) 507 508 error_message = 'Revision %d exists already in "jobs"' % rev 509 self.assertEqual(str(err.exception), error_message) 510 511 mock_outside_chroot.assert_called_once() 512 513 mock_load_status_file.assert_called_once() 514 515 mock_get_range.assert_called_once() 516 517 mock_get_revision_and_hash_list.assert_called_once() 518 519 @mock.patch.object(subprocess, "check_output", return_value=None) 520 @mock.patch.object( 521 get_llvm_hash.LLVMHash, "GetLLVMHash", return_value="a123testhash4" 522 ) 523 @mock.patch.object(llvm_bisection, "GetCommitsBetween") 524 @mock.patch.object(llvm_bisection, "GetRemainingRange") 525 @mock.patch.object(llvm_bisection, "LoadStatusFile") 526 @mock.patch.object(chroot, "VerifyOutsideChroot", return_value=True) 527 def testMainFailedToAbandonCL( 528 self, 529 mock_outside_chroot, 530 mock_load_status_file, 531 mock_get_range, 532 mock_get_revision_and_hash_list, 533 _mock_get_bad_llvm_hash, 534 mock_abandon_cl, 535 ): 536 537 start = 500 538 end = 502 539 540 bisect_state = { 541 "start": start, 542 "end": end, 543 "jobs": [{"rev": 501, "status": "bad", "cl": 0}], 544 } 545 546 skip_revisions = {501} 547 pending_revisions = {} 548 549 mock_load_status_file.return_value = bisect_state 550 551 mock_get_range.return_value = ( 552 start, 553 end, 554 pending_revisions, 555 skip_revisions, 556 ) 557 558 mock_get_revision_and_hash_list.return_value = ([], []) 559 560 error_message = "Error message." 561 mock_abandon_cl.side_effect = subprocess.CalledProcessError( 562 returncode=1, cmd=[], output=error_message 563 ) 564 565 args_output = test_helpers.ArgsOutputTest() 566 args_output.start_rev = start 567 args_output.end_rev = end 568 args_output.parallel = 3 569 args_output.src_path = None 570 args_output.cleanup = True 571 572 with self.assertRaises(subprocess.CalledProcessError) as err: 573 llvm_bisection.main(args_output) 574 575 self.assertEqual(err.exception.output, error_message) 576 577 mock_outside_chroot.assert_called_once() 578 579 mock_load_status_file.assert_called_once() 580 581 mock_get_range.assert_called_once() 582 583 584if __name__ == "__main__": 585 unittest.main() 586