1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# Copyright 2019 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""Tests when updating a tryjob's status.""" 8 9from __future__ import print_function 10 11import json 12import os 13import subprocess 14import unittest 15import unittest.mock as mock 16 17from test_helpers import CreateTemporaryJsonFile 18from test_helpers import WritePrettyJsonFile 19from update_tryjob_status import TryjobStatus 20from update_tryjob_status import CustomScriptStatus 21import update_tryjob_status 22 23 24class UpdateTryjobStatusTest(unittest.TestCase): 25 """Unittests for updating a tryjob's 'status'.""" 26 27 def testFoundTryjobIndex(self): 28 test_tryjobs = [{ 29 'rev': 123, 30 'url': 'https://some_url_to_CL.com', 31 'cl': 'https://some_link_to_tryjob.com', 32 'status': 'good', 33 'buildbucket_id': 91835 34 }, { 35 'rev': 1000, 36 'url': 'https://some_url_to_CL.com', 37 'cl': 'https://some_link_to_tryjob.com', 38 'status': 'pending', 39 'buildbucket_id': 10931 40 }] 41 42 expected_index = 0 43 44 revision_to_find = 123 45 46 self.assertEqual( 47 update_tryjob_status.FindTryjobIndex(revision_to_find, test_tryjobs), 48 expected_index) 49 50 def testNotFindTryjobIndex(self): 51 test_tryjobs = [{ 52 'rev': 500, 53 'url': 'https://some_url_to_CL.com', 54 'cl': 'https://some_link_to_tryjob.com', 55 'status': 'bad', 56 'buildbucket_id': 390 57 }, { 58 'rev': 10, 59 'url': 'https://some_url_to_CL.com', 60 'cl': 'https://some_link_to_tryjob.com', 61 'status': 'skip', 62 'buildbucket_id': 10 63 }] 64 65 revision_to_find = 250 66 67 self.assertIsNone( 68 update_tryjob_status.FindTryjobIndex(revision_to_find, test_tryjobs)) 69 70 @mock.patch.object(subprocess, 'Popen') 71 # Simulate the behavior of `os.rename()` when successfully renamed a file. 72 @mock.patch.object(os, 'rename', return_value=None) 73 # Simulate the behavior of `os.path.basename()` when successfully retrieved 74 # the basename of the temp .JSON file. 75 @mock.patch.object(os.path, 'basename', return_value='tmpFile.json') 76 def testInvalidExitCodeByCustomScript(self, mock_basename, mock_rename_file, 77 mock_exec_custom_script): 78 79 error_message_by_custom_script = 'Failed to parse .JSON file' 80 81 # Simulate the behavior of 'subprocess.Popen()' when executing the custom 82 # script. 83 # 84 # `Popen.communicate()` returns a tuple of `stdout` and `stderr`. 85 mock_exec_custom_script.return_value.communicate.return_value = ( 86 None, error_message_by_custom_script) 87 88 # Exit code of 1 is not in the mapping, so an exception will be raised. 89 custom_script_exit_code = 1 90 91 mock_exec_custom_script.return_value.returncode = custom_script_exit_code 92 93 tryjob_contents = { 94 'status': 'good', 95 'rev': 1234, 96 'url': 'https://some_url_to_CL.com', 97 'link': 'https://some_url_to_tryjob.com' 98 } 99 100 custom_script_path = '/abs/path/to/script.py' 101 status_file_path = '/abs/path/to/status_file.json' 102 103 name_json_file = os.path.join( 104 os.path.dirname(status_file_path), 'tmpFile.json') 105 106 expected_error_message = ( 107 'Custom script %s exit code %d did not match ' 108 'any of the expected exit codes: %s for "good", ' 109 '%d for "bad", or %d for "skip".\nPlease check ' 110 '%s for information about the tryjob: %s' % 111 (custom_script_path, custom_script_exit_code, 112 CustomScriptStatus.GOOD.value, CustomScriptStatus.BAD.value, 113 CustomScriptStatus.SKIP.value, name_json_file, 114 error_message_by_custom_script)) 115 116 # Verify the exception is raised when the exit code by the custom script 117 # does not match any of the exit codes in the mapping of 118 # `custom_script_exit_value_mapping`. 119 with self.assertRaises(ValueError) as err: 120 update_tryjob_status.GetCustomScriptResult(custom_script_path, 121 status_file_path, 122 tryjob_contents) 123 124 self.assertEqual(str(err.exception), expected_error_message) 125 126 mock_exec_custom_script.assert_called_once() 127 128 mock_rename_file.assert_called_once() 129 130 mock_basename.assert_called_once() 131 132 @mock.patch.object(subprocess, 'Popen') 133 # Simulate the behavior of `os.rename()` when successfully renamed a file. 134 @mock.patch.object(os, 'rename', return_value=None) 135 # Simulate the behavior of `os.path.basename()` when successfully retrieved 136 # the basename of the temp .JSON file. 137 @mock.patch.object(os.path, 'basename', return_value='tmpFile.json') 138 def testValidExitCodeByCustomScript(self, mock_basename, mock_rename_file, 139 mock_exec_custom_script): 140 141 # Simulate the behavior of 'subprocess.Popen()' when executing the custom 142 # script. 143 # 144 # `Popen.communicate()` returns a tuple of `stdout` and `stderr`. 145 mock_exec_custom_script.return_value.communicate.return_value = (None, None) 146 147 mock_exec_custom_script.return_value.returncode = ( 148 CustomScriptStatus.GOOD.value) 149 150 tryjob_contents = { 151 'status': 'good', 152 'rev': 1234, 153 'url': 'https://some_url_to_CL.com', 154 'link': 'https://some_url_to_tryjob.com' 155 } 156 157 custom_script_path = '/abs/path/to/script.py' 158 status_file_path = '/abs/path/to/status_file.json' 159 160 self.assertEqual( 161 update_tryjob_status.GetCustomScriptResult(custom_script_path, 162 status_file_path, 163 tryjob_contents), 164 TryjobStatus.GOOD.value) 165 166 mock_exec_custom_script.assert_called_once() 167 168 mock_rename_file.assert_not_called() 169 170 mock_basename.assert_not_called() 171 172 def testNoTryjobsInStatusFileWhenUpdatingTryjobStatus(self): 173 bisect_test_contents = {'start': 369410, 'end': 369420, 'jobs': []} 174 175 # Create a temporary .JSON file to simulate a .JSON file that has bisection 176 # contents. 177 with CreateTemporaryJsonFile() as temp_json_file: 178 with open(temp_json_file, 'w') as f: 179 WritePrettyJsonFile(bisect_test_contents, f) 180 181 revision_to_update = 369412 182 183 custom_script = None 184 185 # Verify the exception is raised when the `status_file` does not have any 186 # `jobs` (empty). 187 with self.assertRaises(SystemExit) as err: 188 update_tryjob_status.UpdateTryjobStatus(revision_to_update, 189 TryjobStatus.GOOD, 190 temp_json_file, custom_script) 191 192 self.assertEqual(str(err.exception), 'No tryjobs in %s' % temp_json_file) 193 194 # Simulate the behavior of `FindTryjobIndex()` when the tryjob does not exist 195 # in the status file. 196 @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=None) 197 def testNotFindTryjobIndexWhenUpdatingTryjobStatus(self, 198 mock_find_tryjob_index): 199 200 bisect_test_contents = { 201 'start': 369410, 202 'end': 369420, 203 'jobs': [{ 204 'rev': 369411, 205 'status': 'pending' 206 }] 207 } 208 209 # Create a temporary .JSON file to simulate a .JSON file that has bisection 210 # contents. 211 with CreateTemporaryJsonFile() as temp_json_file: 212 with open(temp_json_file, 'w') as f: 213 WritePrettyJsonFile(bisect_test_contents, f) 214 215 revision_to_update = 369416 216 217 custom_script = None 218 219 # Verify the exception is raised when the `status_file` does not have any 220 # `jobs` (empty). 221 with self.assertRaises(ValueError) as err: 222 update_tryjob_status.UpdateTryjobStatus(revision_to_update, 223 TryjobStatus.SKIP, 224 temp_json_file, custom_script) 225 226 self.assertEqual( 227 str(err.exception), 'Unable to find tryjob for %d in %s' % 228 (revision_to_update, temp_json_file)) 229 230 mock_find_tryjob_index.assert_called_once() 231 232 # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the 233 # status file. 234 @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0) 235 def testSuccessfullyUpdatedTryjobStatusToGood(self, mock_find_tryjob_index): 236 bisect_test_contents = { 237 'start': 369410, 238 'end': 369420, 239 'jobs': [{ 240 'rev': 369411, 241 'status': 'pending' 242 }] 243 } 244 245 # Create a temporary .JSON file to simulate a .JSON file that has bisection 246 # contents. 247 with CreateTemporaryJsonFile() as temp_json_file: 248 with open(temp_json_file, 'w') as f: 249 WritePrettyJsonFile(bisect_test_contents, f) 250 251 revision_to_update = 369411 252 253 # Index of the tryjob that is going to have its 'status' value updated. 254 tryjob_index = 0 255 256 custom_script = None 257 258 update_tryjob_status.UpdateTryjobStatus(revision_to_update, 259 TryjobStatus.GOOD, temp_json_file, 260 custom_script) 261 262 # Verify that the tryjob's 'status' has been updated in the status file. 263 with open(temp_json_file) as status_file: 264 bisect_contents = json.load(status_file) 265 266 self.assertEqual(bisect_contents['jobs'][tryjob_index]['status'], 267 TryjobStatus.GOOD.value) 268 269 mock_find_tryjob_index.assert_called_once() 270 271 # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the 272 # status file. 273 @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0) 274 def testSuccessfullyUpdatedTryjobStatusToBad(self, mock_find_tryjob_index): 275 bisect_test_contents = { 276 'start': 369410, 277 'end': 369420, 278 'jobs': [{ 279 'rev': 369411, 280 'status': 'pending' 281 }] 282 } 283 284 # Create a temporary .JSON file to simulate a .JSON file that has bisection 285 # contents. 286 with CreateTemporaryJsonFile() as temp_json_file: 287 with open(temp_json_file, 'w') as f: 288 WritePrettyJsonFile(bisect_test_contents, f) 289 290 revision_to_update = 369411 291 292 # Index of the tryjob that is going to have its 'status' value updated. 293 tryjob_index = 0 294 295 custom_script = None 296 297 update_tryjob_status.UpdateTryjobStatus(revision_to_update, 298 TryjobStatus.BAD, temp_json_file, 299 custom_script) 300 301 # Verify that the tryjob's 'status' has been updated in the status file. 302 with open(temp_json_file) as status_file: 303 bisect_contents = json.load(status_file) 304 305 self.assertEqual(bisect_contents['jobs'][tryjob_index]['status'], 306 TryjobStatus.BAD.value) 307 308 mock_find_tryjob_index.assert_called_once() 309 310 # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the 311 # status file. 312 @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0) 313 def testSuccessfullyUpdatedTryjobStatusToPending(self, 314 mock_find_tryjob_index): 315 bisect_test_contents = { 316 'start': 369410, 317 'end': 369420, 318 'jobs': [{ 319 'rev': 369411, 320 'status': 'skip' 321 }] 322 } 323 324 # Create a temporary .JSON file to simulate a .JSON file that has bisection 325 # contents. 326 with CreateTemporaryJsonFile() as temp_json_file: 327 with open(temp_json_file, 'w') as f: 328 WritePrettyJsonFile(bisect_test_contents, f) 329 330 revision_to_update = 369411 331 332 # Index of the tryjob that is going to have its 'status' value updated. 333 tryjob_index = 0 334 335 custom_script = None 336 337 update_tryjob_status.UpdateTryjobStatus( 338 revision_to_update, update_tryjob_status.TryjobStatus.SKIP, 339 temp_json_file, custom_script) 340 341 # Verify that the tryjob's 'status' has been updated in the status file. 342 with open(temp_json_file) as status_file: 343 bisect_contents = json.load(status_file) 344 345 self.assertEqual(bisect_contents['jobs'][tryjob_index]['status'], 346 update_tryjob_status.TryjobStatus.SKIP.value) 347 348 mock_find_tryjob_index.assert_called_once() 349 350 # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the 351 # status file. 352 @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0) 353 def testSuccessfullyUpdatedTryjobStatusToSkip(self, mock_find_tryjob_index): 354 bisect_test_contents = { 355 'start': 369410, 356 'end': 369420, 357 'jobs': [{ 358 'rev': 369411, 359 'status': 'pending', 360 }] 361 } 362 363 # Create a temporary .JSON file to simulate a .JSON file that has bisection 364 # contents. 365 with CreateTemporaryJsonFile() as temp_json_file: 366 with open(temp_json_file, 'w') as f: 367 WritePrettyJsonFile(bisect_test_contents, f) 368 369 revision_to_update = 369411 370 371 # Index of the tryjob that is going to have its 'status' value updated. 372 tryjob_index = 0 373 374 custom_script = None 375 376 update_tryjob_status.UpdateTryjobStatus( 377 revision_to_update, update_tryjob_status.TryjobStatus.PENDING, 378 temp_json_file, custom_script) 379 380 # Verify that the tryjob's 'status' has been updated in the status file. 381 with open(temp_json_file) as status_file: 382 bisect_contents = json.load(status_file) 383 384 self.assertEqual(bisect_contents['jobs'][tryjob_index]['status'], 385 update_tryjob_status.TryjobStatus.PENDING.value) 386 387 mock_find_tryjob_index.assert_called_once() 388 389 @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0) 390 @mock.patch.object( 391 update_tryjob_status, 392 'GetCustomScriptResult', 393 return_value=TryjobStatus.SKIP.value) 394 def testUpdatedTryjobStatusToAutoPassedWithCustomScript( 395 self, mock_get_custom_script_result, mock_find_tryjob_index): 396 bisect_test_contents = { 397 'start': 369410, 398 'end': 369420, 399 'jobs': [{ 400 'rev': 369411, 401 'status': 'pending', 402 'buildbucket_id': 1200 403 }] 404 } 405 406 # Create a temporary .JSON file to simulate a .JSON file that has bisection 407 # contents. 408 with CreateTemporaryJsonFile() as temp_json_file: 409 with open(temp_json_file, 'w') as f: 410 WritePrettyJsonFile(bisect_test_contents, f) 411 412 revision_to_update = 369411 413 414 # Index of the tryjob that is going to have its 'status' value updated. 415 tryjob_index = 0 416 417 custom_script_path = '/abs/path/to/custom_script.py' 418 419 update_tryjob_status.UpdateTryjobStatus( 420 revision_to_update, update_tryjob_status.TryjobStatus.CUSTOM_SCRIPT, 421 temp_json_file, custom_script_path) 422 423 # Verify that the tryjob's 'status' has been updated in the status file. 424 with open(temp_json_file) as status_file: 425 bisect_contents = json.load(status_file) 426 427 self.assertEqual(bisect_contents['jobs'][tryjob_index]['status'], 428 update_tryjob_status.TryjobStatus.SKIP.value) 429 430 mock_get_custom_script_result.assert_called_once() 431 432 mock_find_tryjob_index.assert_called_once() 433 434 # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the 435 # status file. 436 @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0) 437 def testSetStatusDoesNotExistWhenUpdatingTryjobStatus(self, 438 mock_find_tryjob_index): 439 440 bisect_test_contents = { 441 'start': 369410, 442 'end': 369420, 443 'jobs': [{ 444 'rev': 369411, 445 'status': 'pending', 446 'buildbucket_id': 1200 447 }] 448 } 449 450 # Create a temporary .JSON file to simulate a .JSON file that has bisection 451 # contents. 452 with CreateTemporaryJsonFile() as temp_json_file: 453 with open(temp_json_file, 'w') as f: 454 WritePrettyJsonFile(bisect_test_contents, f) 455 456 revision_to_update = 369411 457 458 nonexistent_update_status = 'revert_status' 459 460 custom_script = None 461 462 # Verify the exception is raised when the `set_status` command line 463 # argument does not exist in the mapping. 464 with self.assertRaises(ValueError) as err: 465 update_tryjob_status.UpdateTryjobStatus(revision_to_update, 466 nonexistent_update_status, 467 temp_json_file, custom_script) 468 469 self.assertEqual( 470 str(err.exception), 471 'Invalid "set_status" option provided: revert_status') 472 473 mock_find_tryjob_index.assert_called_once() 474 475 476if __name__ == '__main__': 477 unittest.main() 478