1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# Copyright 2019 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""Tests when updating a tryjob's status.""" 8 9from __future__ import print_function 10 11import json 12import os 13import subprocess 14import unittest 15import unittest.mock as mock 16 17from test_helpers import CreateTemporaryJsonFile 18from test_helpers import WritePrettyJsonFile 19from update_tryjob_status import TryjobStatus 20from update_tryjob_status import CustomScriptStatus 21import update_tryjob_status 22 23 24class UpdateTryjobStatusTest(unittest.TestCase): 25 """Unittests for updating a tryjob's 'status'.""" 26 27 def testFoundTryjobIndex(self): 28 test_tryjobs = [{ 29 'rev': 123, 30 'url': 'https://some_url_to_CL.com', 31 'cl': 'https://some_link_to_tryjob.com', 32 'status': 'good', 33 'buildbucket_id': 91835 34 }, 35 { 36 'rev': 1000, 37 'url': 'https://some_url_to_CL.com', 38 'cl': 'https://some_link_to_tryjob.com', 39 'status': 'pending', 40 'buildbucket_id': 10931 41 }] 42 43 expected_index = 0 44 45 revision_to_find = 123 46 47 self.assertEqual( 48 update_tryjob_status.FindTryjobIndex(revision_to_find, test_tryjobs), 49 expected_index) 50 51 def testNotFindTryjobIndex(self): 52 test_tryjobs = [{ 53 'rev': 500, 54 'url': 'https://some_url_to_CL.com', 55 'cl': 'https://some_link_to_tryjob.com', 56 'status': 'bad', 57 'buildbucket_id': 390 58 }, 59 { 60 'rev': 10, 61 'url': 'https://some_url_to_CL.com', 62 'cl': 'https://some_link_to_tryjob.com', 63 'status': 'skip', 64 'buildbucket_id': 10 65 }] 66 67 revision_to_find = 250 68 69 self.assertIsNone( 70 update_tryjob_status.FindTryjobIndex(revision_to_find, test_tryjobs)) 71 72 # Simulate the behavior of `ChrootRunCommand()` when executing a command 73 # inside the chroot. 74 @mock.patch.object(update_tryjob_status, 'ChrootRunCommand') 75 def testGetStatusFromCrosBuildResult(self, mock_chroot_command): 76 tryjob_contents = { 77 '192': { 78 'status': 'good', 79 'CleanUpChroot': 'pass', 80 'artifacts_url': None 81 } 82 } 83 84 # Use the test function to simulate 'ChrootRunCommand()' behavior. 85 mock_chroot_command.return_value = json.dumps(tryjob_contents) 86 87 buildbucket_id = 192 88 89 chroot_path = '/some/path/to/chroot' 90 91 self.assertEqual( 92 update_tryjob_status.GetStatusFromCrosBuildResult( 93 chroot_path, buildbucket_id), 'good') 94 95 expected_cmd = [ 96 'cros', 'buildresult', '--buildbucket-id', 97 str(buildbucket_id), '--report', 'json' 98 ] 99 100 mock_chroot_command.assert_called_once_with(chroot_path, expected_cmd) 101 102 # Simulate the behavior of `GetStatusFromCrosBuildResult()` when `cros 103 # buildresult` returned a string that is not in the mapping. 104 @mock.patch.object( 105 update_tryjob_status, 106 'GetStatusFromCrosBuildResult', 107 return_value='querying') 108 def testInvalidCrosBuildResultValue(self, mock_cros_buildresult): 109 chroot_path = '/some/path/to/chroot' 110 buildbucket_id = 50 111 112 # Verify the exception is raised when the return value of `cros buildresult` 113 # is not in the `builder_status_mapping`. 114 with self.assertRaises(ValueError) as err: 115 update_tryjob_status.GetAutoResult(chroot_path, buildbucket_id) 116 117 self.assertEqual( 118 str(err.exception), 119 '"cros buildresult" return value is invalid: querying') 120 121 mock_cros_buildresult.assert_called_once_with(chroot_path, buildbucket_id) 122 123 # Simulate the behavior of `GetStatusFromCrosBuildResult()` when `cros 124 # buildresult` returned a string that is in the mapping. 125 @mock.patch.object( 126 update_tryjob_status, 127 'GetStatusFromCrosBuildResult', 128 return_value=update_tryjob_status.BuilderStatus.PASS.value) 129 def testValidCrosBuildResultValue(self, mock_cros_buildresult): 130 chroot_path = '/some/path/to/chroot' 131 buildbucket_id = 100 132 133 self.assertEqual( 134 update_tryjob_status.GetAutoResult(chroot_path, buildbucket_id), 135 TryjobStatus.GOOD.value) 136 137 mock_cros_buildresult.assert_called_once_with(chroot_path, buildbucket_id) 138 139 @mock.patch.object(subprocess, 'Popen') 140 # Simulate the behavior of `os.rename()` when successfully renamed a file. 141 @mock.patch.object(os, 'rename', return_value=None) 142 # Simulate the behavior of `os.path.basename()` when successfully retrieved 143 # the basename of the temp .JSON file. 144 @mock.patch.object(os.path, 'basename', return_value='tmpFile.json') 145 def testInvalidExitCodeByCustomScript(self, mock_basename, mock_rename_file, 146 mock_exec_custom_script): 147 148 error_message_by_custom_script = 'Failed to parse .JSON file' 149 150 # Simulate the behavior of 'subprocess.Popen()' when executing the custom 151 # script. 152 # 153 # `Popen.communicate()` returns a tuple of `stdout` and `stderr`. 154 mock_exec_custom_script.return_value.communicate.return_value = ( 155 None, error_message_by_custom_script) 156 157 # Exit code of 1 is not in the mapping, so an exception will be raised. 158 custom_script_exit_code = 1 159 160 mock_exec_custom_script.return_value.returncode = custom_script_exit_code 161 162 tryjob_contents = { 163 'status': 'good', 164 'rev': 1234, 165 'url': 'https://some_url_to_CL.com', 166 'link': 'https://some_url_to_tryjob.com' 167 } 168 169 custom_script_path = '/abs/path/to/script.py' 170 status_file_path = '/abs/path/to/status_file.json' 171 172 name_json_file = os.path.join( 173 os.path.dirname(status_file_path), 'tmpFile.json') 174 175 expected_error_message = ( 176 'Custom script %s exit code %d did not match ' 177 'any of the expected exit codes: %s for "good", ' 178 '%d for "bad", or %d for "skip".\nPlease check ' 179 '%s for information about the tryjob: %s' % 180 (custom_script_path, custom_script_exit_code, 181 CustomScriptStatus.GOOD.value, CustomScriptStatus.BAD.value, 182 CustomScriptStatus.SKIP.value, name_json_file, 183 error_message_by_custom_script)) 184 185 # Verify the exception is raised when the exit code by the custom script 186 # does not match any of the exit codes in the mapping of 187 # `custom_script_exit_value_mapping`. 188 with self.assertRaises(ValueError) as err: 189 update_tryjob_status.GetCustomScriptResult( 190 custom_script_path, status_file_path, tryjob_contents) 191 192 self.assertEqual(str(err.exception), expected_error_message) 193 194 mock_exec_custom_script.assert_called_once() 195 196 mock_rename_file.assert_called_once() 197 198 mock_basename.assert_called_once() 199 200 @mock.patch.object(subprocess, 'Popen') 201 # Simulate the behavior of `os.rename()` when successfully renamed a file. 202 @mock.patch.object(os, 'rename', return_value=None) 203 # Simulate the behavior of `os.path.basename()` when successfully retrieved 204 # the basename of the temp .JSON file. 205 @mock.patch.object(os.path, 'basename', return_value='tmpFile.json') 206 def testValidExitCodeByCustomScript(self, mock_basename, mock_rename_file, 207 mock_exec_custom_script): 208 209 # Simulate the behavior of 'subprocess.Popen()' when executing the custom 210 # script. 211 # 212 # `Popen.communicate()` returns a tuple of `stdout` and `stderr`. 213 mock_exec_custom_script.return_value.communicate.return_value = (None, None) 214 215 mock_exec_custom_script.return_value.returncode = \ 216 CustomScriptStatus.GOOD.value 217 218 tryjob_contents = { 219 'status': 'good', 220 'rev': 1234, 221 'url': 'https://some_url_to_CL.com', 222 'link': 'https://some_url_to_tryjob.com' 223 } 224 225 custom_script_path = '/abs/path/to/script.py' 226 status_file_path = '/abs/path/to/status_file.json' 227 228 self.assertEqual( 229 update_tryjob_status.GetCustomScriptResult( 230 custom_script_path, status_file_path, tryjob_contents), 231 TryjobStatus.GOOD.value) 232 233 mock_exec_custom_script.assert_called_once() 234 235 mock_rename_file.assert_not_called() 236 237 mock_basename.assert_not_called() 238 239 def testNoTryjobsInStatusFileWhenUpdatingTryjobStatus(self): 240 bisect_test_contents = {'start': 369410, 'end': 369420, 'jobs': []} 241 242 # Create a temporary .JSON file to simulate a .JSON file that has bisection 243 # contents. 244 with CreateTemporaryJsonFile() as temp_json_file: 245 with open(temp_json_file, 'w') as f: 246 WritePrettyJsonFile(bisect_test_contents, f) 247 248 revision_to_update = 369412 249 250 chroot_path = '/abs/path/to/chroot' 251 252 custom_script = None 253 254 # Verify the exception is raised when the `status_file` does not have any 255 # `jobs` (empty). 256 with self.assertRaises(SystemExit) as err: 257 update_tryjob_status.UpdateTryjobStatus( 258 revision_to_update, TryjobStatus.GOOD, temp_json_file, chroot_path, 259 custom_script) 260 261 self.assertEqual(str(err.exception), 'No tryjobs in %s' % temp_json_file) 262 263 # Simulate the behavior of `FindTryjobIndex()` when the tryjob does not exist 264 # in the status file. 265 @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=None) 266 def testNotFindTryjobIndexWhenUpdatingTryjobStatus(self, 267 mock_find_tryjob_index): 268 269 bisect_test_contents = { 270 'start': 369410, 271 'end': 369420, 272 'jobs': [{ 273 'rev': 369411, 274 'status': 'pending' 275 }] 276 } 277 278 # Create a temporary .JSON file to simulate a .JSON file that has bisection 279 # contents. 280 with CreateTemporaryJsonFile() as temp_json_file: 281 with open(temp_json_file, 'w') as f: 282 WritePrettyJsonFile(bisect_test_contents, f) 283 284 revision_to_update = 369416 285 286 chroot_path = '/abs/path/to/chroot' 287 288 custom_script = None 289 290 # Verify the exception is raised when the `status_file` does not have any 291 # `jobs` (empty). 292 with self.assertRaises(ValueError) as err: 293 update_tryjob_status.UpdateTryjobStatus( 294 revision_to_update, TryjobStatus.SKIP, temp_json_file, chroot_path, 295 custom_script) 296 297 self.assertEqual( 298 str(err.exception), 'Unable to find tryjob for %d in %s' % 299 (revision_to_update, temp_json_file)) 300 301 mock_find_tryjob_index.assert_called_once() 302 303 # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the 304 # status file. 305 @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0) 306 def testSuccessfullyUpdatedTryjobStatusToGood(self, mock_find_tryjob_index): 307 bisect_test_contents = { 308 'start': 369410, 309 'end': 369420, 310 'jobs': [{ 311 'rev': 369411, 312 'status': 'pending' 313 }] 314 } 315 316 # Create a temporary .JSON file to simulate a .JSON file that has bisection 317 # contents. 318 with CreateTemporaryJsonFile() as temp_json_file: 319 with open(temp_json_file, 'w') as f: 320 WritePrettyJsonFile(bisect_test_contents, f) 321 322 revision_to_update = 369411 323 324 # Index of the tryjob that is going to have its 'status' value updated. 325 tryjob_index = 0 326 327 chroot_path = '/abs/path/to/chroot' 328 329 custom_script = None 330 331 update_tryjob_status.UpdateTryjobStatus(revision_to_update, 332 TryjobStatus.GOOD, temp_json_file, 333 chroot_path, custom_script) 334 335 # Verify that the tryjob's 'status' has been updated in the status file. 336 with open(temp_json_file) as status_file: 337 bisect_contents = json.load(status_file) 338 339 self.assertEqual(bisect_contents['jobs'][tryjob_index]['status'], 340 TryjobStatus.GOOD.value) 341 342 mock_find_tryjob_index.assert_called_once() 343 344 # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the 345 # status file. 346 @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0) 347 def testSuccessfullyUpdatedTryjobStatusToBad(self, mock_find_tryjob_index): 348 bisect_test_contents = { 349 'start': 369410, 350 'end': 369420, 351 'jobs': [{ 352 'rev': 369411, 353 'status': 'pending' 354 }] 355 } 356 357 # Create a temporary .JSON file to simulate a .JSON file that has bisection 358 # contents. 359 with CreateTemporaryJsonFile() as temp_json_file: 360 with open(temp_json_file, 'w') as f: 361 WritePrettyJsonFile(bisect_test_contents, f) 362 363 revision_to_update = 369411 364 365 # Index of the tryjob that is going to have its 'status' value updated. 366 tryjob_index = 0 367 368 chroot_path = '/abs/path/to/chroot' 369 370 custom_script = None 371 372 update_tryjob_status.UpdateTryjobStatus(revision_to_update, 373 TryjobStatus.BAD, temp_json_file, 374 chroot_path, custom_script) 375 376 # Verify that the tryjob's 'status' has been updated in the status file. 377 with open(temp_json_file) as status_file: 378 bisect_contents = json.load(status_file) 379 380 self.assertEqual(bisect_contents['jobs'][tryjob_index]['status'], 381 TryjobStatus.BAD.value) 382 383 mock_find_tryjob_index.assert_called_once() 384 385 # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the 386 # status file. 387 @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0) 388 def testSuccessfullyUpdatedTryjobStatusToPending(self, 389 mock_find_tryjob_index): 390 bisect_test_contents = { 391 'start': 369410, 392 'end': 369420, 393 'jobs': [{ 394 'rev': 369411, 395 'status': 'skip' 396 }] 397 } 398 399 # Create a temporary .JSON file to simulate a .JSON file that has bisection 400 # contents. 401 with CreateTemporaryJsonFile() as temp_json_file: 402 with open(temp_json_file, 'w') as f: 403 WritePrettyJsonFile(bisect_test_contents, f) 404 405 revision_to_update = 369411 406 407 # Index of the tryjob that is going to have its 'status' value updated. 408 tryjob_index = 0 409 410 chroot_path = '/abs/path/to/chroot' 411 412 custom_script = None 413 414 update_tryjob_status.UpdateTryjobStatus( 415 revision_to_update, update_tryjob_status.TryjobStatus.SKIP, 416 temp_json_file, chroot_path, custom_script) 417 418 # Verify that the tryjob's 'status' has been updated in the status file. 419 with open(temp_json_file) as status_file: 420 bisect_contents = json.load(status_file) 421 422 self.assertEqual(bisect_contents['jobs'][tryjob_index]['status'], 423 update_tryjob_status.TryjobStatus.SKIP.value) 424 425 mock_find_tryjob_index.assert_called_once() 426 427 # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the 428 # status file. 429 @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0) 430 def testSuccessfullyUpdatedTryjobStatusToSkip(self, mock_find_tryjob_index): 431 bisect_test_contents = { 432 'start': 369410, 433 'end': 369420, 434 'jobs': [{ 435 'rev': 369411, 436 'status': 'pending', 437 }] 438 } 439 440 # Create a temporary .JSON file to simulate a .JSON file that has bisection 441 # contents. 442 with CreateTemporaryJsonFile() as temp_json_file: 443 with open(temp_json_file, 'w') as f: 444 WritePrettyJsonFile(bisect_test_contents, f) 445 446 revision_to_update = 369411 447 448 # Index of the tryjob that is going to have its 'status' value updated. 449 tryjob_index = 0 450 451 chroot_path = '/abs/path/to/chroot' 452 453 custom_script = None 454 455 update_tryjob_status.UpdateTryjobStatus( 456 revision_to_update, update_tryjob_status.TryjobStatus.PENDING, 457 temp_json_file, chroot_path, custom_script) 458 459 # Verify that the tryjob's 'status' has been updated in the status file. 460 with open(temp_json_file) as status_file: 461 bisect_contents = json.load(status_file) 462 463 self.assertEqual(bisect_contents['jobs'][tryjob_index]['status'], 464 update_tryjob_status.TryjobStatus.PENDING.value) 465 466 mock_find_tryjob_index.assert_called_once() 467 468 # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the 469 # status file. 470 @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0) 471 # Simulate the behavior of `GetAutoResult()` when `cros buildresult` returns 472 # a value that is in the mapping. 473 @mock.patch.object( 474 update_tryjob_status, 475 'GetAutoResult', 476 return_value=TryjobStatus.GOOD.value) 477 def testSuccessfullyUpdatedTryjobStatusToAuto(self, mock_get_auto_result, 478 mock_find_tryjob_index): 479 bisect_test_contents = { 480 'start': 369410, 481 'end': 369420, 482 'jobs': [{ 483 'rev': 369411, 484 'status': 'pending', 485 'buildbucket_id': 1200 486 }] 487 } 488 489 # Create a temporary .JSON file to simulate a .JSON file that has bisection 490 # contents. 491 with CreateTemporaryJsonFile() as temp_json_file: 492 with open(temp_json_file, 'w') as f: 493 WritePrettyJsonFile(bisect_test_contents, f) 494 495 revision_to_update = 369411 496 497 # Index of the tryjob that is going to have its 'status' value updated. 498 tryjob_index = 0 499 500 path_to_chroot = '/abs/path/to/chroot' 501 502 custom_script = None 503 504 update_tryjob_status.UpdateTryjobStatus( 505 revision_to_update, update_tryjob_status.TryjobStatus.AUTO, 506 temp_json_file, path_to_chroot, custom_script) 507 508 # Verify that the tryjob's 'status' has been updated in the status file. 509 with open(temp_json_file) as status_file: 510 bisect_contents = json.load(status_file) 511 512 self.assertEqual(bisect_contents['jobs'][tryjob_index]['status'], 513 update_tryjob_status.TryjobStatus.GOOD.value) 514 515 mock_get_auto_result.assert_called_once_with( 516 path_to_chroot, 517 bisect_test_contents['jobs'][tryjob_index]['buildbucket_id']) 518 519 mock_find_tryjob_index.assert_called_once() 520 521 # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the 522 # status file. 523 @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0) 524 # Simulate the behavior of `GetCustomScriptResult()` when the custom script 525 # exit code is in the mapping. 526 @mock.patch.object( 527 update_tryjob_status, 528 'GetCustomScriptResult', 529 return_value=TryjobStatus.SKIP.value) 530 def testSuccessfullyUpdatedTryjobStatusToAuto( 531 self, mock_get_custom_script_result, mock_find_tryjob_index): 532 bisect_test_contents = { 533 'start': 369410, 534 'end': 369420, 535 'jobs': [{ 536 'rev': 369411, 537 'status': 'pending', 538 'buildbucket_id': 1200 539 }] 540 } 541 542 # Create a temporary .JSON file to simulate a .JSON file that has bisection 543 # contents. 544 with CreateTemporaryJsonFile() as temp_json_file: 545 with open(temp_json_file, 'w') as f: 546 WritePrettyJsonFile(bisect_test_contents, f) 547 548 revision_to_update = 369411 549 550 # Index of the tryjob that is going to have its 'status' value updated. 551 tryjob_index = 0 552 553 path_to_chroot = '/abs/path/to/chroot' 554 555 custom_script_path = '/abs/path/to/custom_script.py' 556 557 update_tryjob_status.UpdateTryjobStatus( 558 revision_to_update, update_tryjob_status.TryjobStatus.CUSTOM_SCRIPT, 559 temp_json_file, path_to_chroot, custom_script_path) 560 561 # Verify that the tryjob's 'status' has been updated in the status file. 562 with open(temp_json_file) as status_file: 563 bisect_contents = json.load(status_file) 564 565 self.assertEqual(bisect_contents['jobs'][tryjob_index]['status'], 566 update_tryjob_status.TryjobStatus.SKIP.value) 567 568 mock_get_custom_script_result.assert_called_once() 569 570 mock_find_tryjob_index.assert_called_once() 571 572 # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the 573 # status file. 574 @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0) 575 def testSetStatusDoesNotExistWhenUpdatingTryjobStatus(self, 576 mock_find_tryjob_index): 577 578 bisect_test_contents = { 579 'start': 369410, 580 'end': 369420, 581 'jobs': [{ 582 'rev': 369411, 583 'status': 'pending', 584 'buildbucket_id': 1200 585 }] 586 } 587 588 # Create a temporary .JSON file to simulate a .JSON file that has bisection 589 # contents. 590 with CreateTemporaryJsonFile() as temp_json_file: 591 with open(temp_json_file, 'w') as f: 592 WritePrettyJsonFile(bisect_test_contents, f) 593 594 revision_to_update = 369411 595 596 path_to_chroot = '/abs/path/to/chroot' 597 598 nonexistent_update_status = 'revert_status' 599 600 custom_script = None 601 602 # Verify the exception is raised when the `set_status` command line 603 # argument does not exist in the mapping. 604 with self.assertRaises(ValueError) as err: 605 update_tryjob_status.UpdateTryjobStatus( 606 revision_to_update, nonexistent_update_status, temp_json_file, 607 path_to_chroot, custom_script) 608 609 self.assertEqual( 610 str(err.exception), 611 'Invalid "set_status" option provided: revert_status') 612 613 mock_find_tryjob_index.assert_called_once() 614 615 616if __name__ == '__main__': 617 unittest.main() 618