1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# Copyright 2019 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7# pylint: disable=protected-access 8 9"""Tests for LLVM bisection.""" 10 11from __future__ import print_function 12 13import json 14import os 15import subprocess 16import unittest 17import unittest.mock as mock 18 19import chroot 20import get_llvm_hash 21import git_llvm_rev 22import llvm_bisection 23import modify_a_tryjob 24import test_helpers 25 26 27class LLVMBisectionTest(unittest.TestCase): 28 """Unittests for LLVM bisection.""" 29 30 def testGetRemainingRangePassed(self): 31 start = 100 32 end = 150 33 34 test_tryjobs = [{ 35 'rev': 110, 36 'status': 'good', 37 'link': 'https://some_tryjob_1_url.com' 38 }, { 39 'rev': 120, 40 'status': 'good', 41 'link': 'https://some_tryjob_2_url.com' 42 }, { 43 'rev': 130, 44 'status': 'pending', 45 'link': 'https://some_tryjob_3_url.com' 46 }, { 47 'rev': 135, 48 'status': 'skip', 49 'link': 'https://some_tryjob_4_url.com' 50 }, { 51 'rev': 140, 52 'status': 'bad', 53 'link': 'https://some_tryjob_5_url.com' 54 }] 55 56 # Tuple consists of the new good revision, the new bad revision, a set of 57 # 'pending' revisions, and a set of 'skip' revisions. 58 expected_revisions_tuple = 120, 140, {130}, {135} 59 60 self.assertEqual( 61 llvm_bisection.GetRemainingRange(start, end, test_tryjobs), 62 expected_revisions_tuple) 63 64 def testGetRemainingRangeFailedWithMissingStatus(self): 65 start = 100 66 end = 150 67 68 test_tryjobs = [{ 69 'rev': 105, 70 'status': 'good', 71 'link': 'https://some_tryjob_1_url.com' 72 }, { 73 'rev': 120, 74 'status': None, 75 'link': 'https://some_tryjob_2_url.com' 76 }, { 77 'rev': 140, 78 'status': 'bad', 79 'link': 'https://some_tryjob_3_url.com' 80 }] 81 82 with self.assertRaises(ValueError) as err: 83 llvm_bisection.GetRemainingRange(start, end, test_tryjobs) 84 85 error_message = ('"status" is missing or has no value, please ' 86 'go to %s and update it' % test_tryjobs[1]['link']) 87 self.assertEqual(str(err.exception), error_message) 88 89 def testGetRemainingRangeFailedWithInvalidRange(self): 90 start = 100 91 end = 150 92 93 test_tryjobs = [{ 94 'rev': 110, 95 'status': 'bad', 96 'link': 'https://some_tryjob_1_url.com' 97 }, { 98 'rev': 125, 99 'status': 'skip', 100 'link': 'https://some_tryjob_2_url.com' 101 }, { 102 'rev': 140, 103 'status': 'good', 104 'link': 'https://some_tryjob_3_url.com' 105 }] 106 107 with self.assertRaises(AssertionError) as err: 108 llvm_bisection.GetRemainingRange(start, end, test_tryjobs) 109 110 expected_error_message = ('Bisection is broken because %d (good) is >= ' 111 '%d (bad)' % 112 (test_tryjobs[2]['rev'], test_tryjobs[0]['rev'])) 113 114 self.assertEqual(str(err.exception), expected_error_message) 115 116 @mock.patch.object(get_llvm_hash, 'GetGitHashFrom') 117 def testGetCommitsBetweenPassed(self, mock_get_git_hash): 118 start = git_llvm_rev.base_llvm_revision 119 end = start + 10 120 test_pending_revisions = {start + 7} 121 test_skip_revisions = { 122 start + 1, start + 2, start + 4, start + 8, start + 9 123 } 124 parallel = 3 125 abs_path_to_src = '/abs/path/to/src' 126 127 revs = ['a123testhash3', 'a123testhash5'] 128 mock_get_git_hash.side_effect = revs 129 130 git_hashes = [ 131 git_llvm_rev.base_llvm_revision + 3, git_llvm_rev.base_llvm_revision + 5 132 ] 133 134 self.assertEqual( 135 llvm_bisection.GetCommitsBetween(start, end, parallel, abs_path_to_src, 136 test_pending_revisions, 137 test_skip_revisions), 138 (git_hashes, revs)) 139 140 def testLoadStatusFilePassedWithExistingFile(self): 141 start = 100 142 end = 150 143 144 test_bisect_state = {'start': start, 'end': end, 'jobs': []} 145 146 # Simulate that the status file exists. 147 with test_helpers.CreateTemporaryJsonFile() as temp_json_file: 148 with open(temp_json_file, 'w') as f: 149 test_helpers.WritePrettyJsonFile(test_bisect_state, f) 150 151 self.assertEqual( 152 llvm_bisection.LoadStatusFile(temp_json_file, start, end), 153 test_bisect_state) 154 155 def testLoadStatusFilePassedWithoutExistingFile(self): 156 start = 200 157 end = 250 158 159 expected_bisect_state = {'start': start, 'end': end, 'jobs': []} 160 161 last_tested = '/abs/path/to/file_that_does_not_exist.json' 162 163 self.assertEqual( 164 llvm_bisection.LoadStatusFile(last_tested, start, end), 165 expected_bisect_state) 166 167 @mock.patch.object(modify_a_tryjob, 'AddTryjob') 168 def testBisectPassed(self, mock_add_tryjob): 169 170 git_hash_list = ['a123testhash1', 'a123testhash2', 'a123testhash3'] 171 revisions_list = [102, 104, 106] 172 173 # Simulate behavior of `AddTryjob()` when successfully launched a tryjob for 174 # the updated packages. 175 @test_helpers.CallCountsToMockFunctions 176 def MockAddTryjob(call_count, _packages, _git_hash, _revision, _chroot_path, 177 _patch_file, _extra_cls, _options, _builder, _verbose, 178 _svn_revision): 179 180 if call_count < 2: 181 return {'rev': revisions_list[call_count], 'status': 'pending'} 182 183 # Simulate an exception happened along the way when updating the 184 # packages' `LLVM_NEXT_HASH`. 185 if call_count == 2: 186 raise ValueError('Unable to launch tryjob') 187 188 assert False, 'Called `AddTryjob()` more than expected.' 189 190 # Use the test function to simulate `AddTryjob()`. 191 mock_add_tryjob.side_effect = MockAddTryjob 192 193 start = 100 194 end = 110 195 196 bisection_contents = {'start': start, 'end': end, 'jobs': []} 197 198 args_output = test_helpers.ArgsOutputTest() 199 200 packages = ['sys-devel/llvm'] 201 patch_file = '/abs/path/to/PATCHES.json' 202 203 # Create a temporary .JSON file to simulate a status file for bisection. 204 with test_helpers.CreateTemporaryJsonFile() as temp_json_file: 205 with open(temp_json_file, 'w') as f: 206 test_helpers.WritePrettyJsonFile(bisection_contents, f) 207 208 # Verify that the status file is updated when an exception happened when 209 # attempting to launch a revision (i.e. progress is not lost). 210 with self.assertRaises(ValueError) as err: 211 llvm_bisection.Bisect(revisions_list, git_hash_list, bisection_contents, 212 temp_json_file, packages, args_output.chroot_path, 213 patch_file, args_output.extra_change_lists, 214 args_output.options, args_output.builders, 215 args_output.verbose) 216 217 expected_bisection_contents = { 218 'start': 219 start, 220 'end': 221 end, 222 'jobs': [{ 223 'rev': revisions_list[0], 224 'status': 'pending' 225 }, { 226 'rev': revisions_list[1], 227 'status': 'pending' 228 }] 229 } 230 231 # Verify that the launched tryjobs were added to the status file when 232 # an exception happened. 233 with open(temp_json_file) as f: 234 json_contents = json.load(f) 235 236 self.assertEqual(json_contents, expected_bisection_contents) 237 238 self.assertEqual(str(err.exception), 'Unable to launch tryjob') 239 240 self.assertEqual(mock_add_tryjob.call_count, 3) 241 242 @mock.patch.object(subprocess, 'check_output', return_value=None) 243 @mock.patch.object( 244 get_llvm_hash.LLVMHash, 'GetLLVMHash', return_value='a123testhash4') 245 @mock.patch.object(llvm_bisection, 'GetCommitsBetween') 246 @mock.patch.object(llvm_bisection, 'GetRemainingRange') 247 @mock.patch.object(llvm_bisection, 'LoadStatusFile') 248 @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True) 249 def testMainPassed(self, mock_outside_chroot, mock_load_status_file, 250 mock_get_range, mock_get_revision_and_hash_list, 251 _mock_get_bad_llvm_hash, mock_abandon_cl): 252 253 start = 500 254 end = 502 255 cl = 1 256 257 bisect_state = { 258 'start': start, 259 'end': end, 260 'jobs': [{ 261 'rev': 501, 262 'status': 'bad', 263 'cl': cl 264 }] 265 } 266 267 skip_revisions = {501} 268 pending_revisions = {} 269 270 mock_load_status_file.return_value = bisect_state 271 272 mock_get_range.return_value = (start, end, pending_revisions, 273 skip_revisions) 274 275 mock_get_revision_and_hash_list.return_value = [], [] 276 277 args_output = test_helpers.ArgsOutputTest() 278 args_output.start_rev = start 279 args_output.end_rev = end 280 args_output.parallel = 3 281 args_output.src_path = None 282 args_output.chroot_path = 'somepath' 283 args_output.cleanup = True 284 285 self.assertEqual( 286 llvm_bisection.main(args_output), 287 llvm_bisection.BisectionExitStatus.BISECTION_COMPLETE.value) 288 289 mock_outside_chroot.assert_called_once() 290 291 mock_load_status_file.assert_called_once() 292 293 mock_get_range.assert_called_once() 294 295 mock_get_revision_and_hash_list.assert_called_once() 296 297 mock_abandon_cl.assert_called_once() 298 self.assertEqual( 299 mock_abandon_cl.call_args, 300 mock.call( 301 [ 302 os.path.join(args_output.chroot_path, 'chromite/bin/gerrit'), 303 'abandon', 304 str(cl), 305 ], 306 stderr=subprocess.STDOUT, 307 encoding='utf-8', 308 )) 309 310 @mock.patch.object(llvm_bisection, 'LoadStatusFile') 311 @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True) 312 def testMainFailedWithInvalidRange(self, mock_outside_chroot, 313 mock_load_status_file): 314 315 start = 500 316 end = 502 317 318 bisect_state = { 319 'start': start - 1, 320 'end': end, 321 } 322 323 mock_load_status_file.return_value = bisect_state 324 325 args_output = test_helpers.ArgsOutputTest() 326 args_output.start_rev = start 327 args_output.end_rev = end 328 args_output.parallel = 3 329 args_output.src_path = None 330 331 with self.assertRaises(ValueError) as err: 332 llvm_bisection.main(args_output) 333 334 error_message = (f'The start {start} or the end {end} version provided is ' 335 f'different than "start" {bisect_state["start"]} or "end" ' 336 f'{bisect_state["end"]} in the .JSON file') 337 338 self.assertEqual(str(err.exception), error_message) 339 340 mock_outside_chroot.assert_called_once() 341 342 mock_load_status_file.assert_called_once() 343 344 @mock.patch.object(llvm_bisection, 'GetCommitsBetween') 345 @mock.patch.object(llvm_bisection, 'GetRemainingRange') 346 @mock.patch.object(llvm_bisection, 'LoadStatusFile') 347 @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True) 348 def testMainFailedWithPendingBuilds(self, mock_outside_chroot, 349 mock_load_status_file, mock_get_range, 350 mock_get_revision_and_hash_list): 351 352 start = 500 353 end = 502 354 rev = 501 355 356 bisect_state = { 357 'start': start, 358 'end': end, 359 'jobs': [{ 360 'rev': rev, 361 'status': 'pending' 362 }] 363 } 364 365 skip_revisions = {} 366 pending_revisions = {rev} 367 368 mock_load_status_file.return_value = bisect_state 369 370 mock_get_range.return_value = (start, end, pending_revisions, 371 skip_revisions) 372 373 mock_get_revision_and_hash_list.return_value = [], [] 374 375 args_output = test_helpers.ArgsOutputTest() 376 args_output.start_rev = start 377 args_output.end_rev = end 378 args_output.parallel = 3 379 args_output.src_path = None 380 381 with self.assertRaises(ValueError) as err: 382 llvm_bisection.main(args_output) 383 384 error_message = (f'No revisions between start {start} and end {end} to ' 385 'create tryjobs\nThe following tryjobs are pending:\n' 386 f'{rev}\n') 387 388 self.assertEqual(str(err.exception), error_message) 389 390 mock_outside_chroot.assert_called_once() 391 392 mock_load_status_file.assert_called_once() 393 394 mock_get_range.assert_called_once() 395 396 mock_get_revision_and_hash_list.assert_called_once() 397 398 @mock.patch.object(llvm_bisection, 'GetCommitsBetween') 399 @mock.patch.object(llvm_bisection, 'GetRemainingRange') 400 @mock.patch.object(llvm_bisection, 'LoadStatusFile') 401 @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True) 402 def testMainFailedWithDuplicateBuilds(self, mock_outside_chroot, 403 mock_load_status_file, mock_get_range, 404 mock_get_revision_and_hash_list): 405 406 start = 500 407 end = 502 408 rev = 501 409 git_hash = 'a123testhash1' 410 411 bisect_state = { 412 'start': start, 413 'end': end, 414 'jobs': [{ 415 'rev': rev, 416 'status': 'pending' 417 }] 418 } 419 420 skip_revisions = {} 421 pending_revisions = {rev} 422 423 mock_load_status_file.return_value = bisect_state 424 425 mock_get_range.return_value = (start, end, pending_revisions, 426 skip_revisions) 427 428 mock_get_revision_and_hash_list.return_value = [rev], [git_hash] 429 430 args_output = test_helpers.ArgsOutputTest() 431 args_output.start_rev = start 432 args_output.end_rev = end 433 args_output.parallel = 3 434 args_output.src_path = None 435 436 with self.assertRaises(ValueError) as err: 437 llvm_bisection.main(args_output) 438 439 error_message = ('Revision %d exists already in "jobs"' % rev) 440 self.assertEqual(str(err.exception), error_message) 441 442 mock_outside_chroot.assert_called_once() 443 444 mock_load_status_file.assert_called_once() 445 446 mock_get_range.assert_called_once() 447 448 mock_get_revision_and_hash_list.assert_called_once() 449 450 @mock.patch.object(subprocess, 'check_output', return_value=None) 451 @mock.patch.object( 452 get_llvm_hash.LLVMHash, 'GetLLVMHash', return_value='a123testhash4') 453 @mock.patch.object(llvm_bisection, 'GetCommitsBetween') 454 @mock.patch.object(llvm_bisection, 'GetRemainingRange') 455 @mock.patch.object(llvm_bisection, 'LoadStatusFile') 456 @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True) 457 def testMainFailedToAbandonCL(self, mock_outside_chroot, 458 mock_load_status_file, mock_get_range, 459 mock_get_revision_and_hash_list, 460 _mock_get_bad_llvm_hash, mock_abandon_cl): 461 462 start = 500 463 end = 502 464 465 bisect_state = { 466 'start': start, 467 'end': end, 468 'jobs': [{ 469 'rev': 501, 470 'status': 'bad', 471 'cl': 0 472 }] 473 } 474 475 skip_revisions = {501} 476 pending_revisions = {} 477 478 mock_load_status_file.return_value = bisect_state 479 480 mock_get_range.return_value = (start, end, pending_revisions, 481 skip_revisions) 482 483 mock_get_revision_and_hash_list.return_value = ([], []) 484 485 error_message = 'Error message.' 486 mock_abandon_cl.side_effect = subprocess.CalledProcessError( 487 returncode=1, cmd=[], output=error_message) 488 489 args_output = test_helpers.ArgsOutputTest() 490 args_output.start_rev = start 491 args_output.end_rev = end 492 args_output.parallel = 3 493 args_output.src_path = None 494 args_output.cleanup = True 495 496 with self.assertRaises(subprocess.CalledProcessError) as err: 497 llvm_bisection.main(args_output) 498 499 self.assertEqual(err.exception.output, error_message) 500 501 mock_outside_chroot.assert_called_once() 502 503 mock_load_status_file.assert_called_once() 504 505 mock_get_range.assert_called_once() 506 507 508if __name__ == '__main__': 509 unittest.main() 510