1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# Copyright 2019 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""Tests for auto bisection of LLVM.""" 8 9from __future__ import print_function 10 11import os 12import subprocess 13import time 14import traceback 15import unittest 16import unittest.mock as mock 17 18import auto_llvm_bisection 19import chroot 20import llvm_bisection 21import test_helpers 22 23 24class AutoLLVMBisectionTest(unittest.TestCase): 25 """Unittests for auto bisection of LLVM.""" 26 27 # Simulate the behavior of `VerifyOutsideChroot()` when successfully invoking 28 # the script outside of the chroot. 29 @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True) 30 # Simulate behavior of `time.sleep()` when waiting for errors to settle caused 31 # by `llvm_bisection.main()` (e.g. network issue, etc.). 32 @mock.patch.object(time, 'sleep') 33 # Simulate behavior of `traceback.print_exc()` when an exception happened in 34 # `llvm_bisection.main()`. 35 @mock.patch.object(traceback, 'print_exc') 36 # Simulate behavior of `llvm_bisection.main()` when failed to launch tryjobs 37 # (exception happened along the way, etc.). 38 @mock.patch.object(llvm_bisection, 'main') 39 # Simulate behavior of `os.path.isfile()` when starting a new bisection. 40 @mock.patch.object(os.path, 'isfile', return_value=False) 41 # Simulate behavior of `GetPathToUpdateAllTryjobsWithAutoScript()` when 42 # returning the absolute path to that script that updates all 'pending' 43 # tryjobs to the result of `cros buildresult`. 44 @mock.patch.object( 45 auto_llvm_bisection, 46 'GetPathToUpdateAllTryjobsWithAutoScript', 47 return_value='/abs/path/to/update_tryjob.py') 48 # Simulate `llvm_bisection.GetCommandLineArgs()` when parsing the command line 49 # arguments required by the bisection script. 50 @mock.patch.object( 51 llvm_bisection, 52 'GetCommandLineArgs', 53 return_value=test_helpers.ArgsOutputTest()) 54 def testFailedToStartBisection( 55 self, mock_get_args, mock_get_auto_script, mock_is_file, 56 mock_llvm_bisection, mock_traceback, mock_sleep, mock_outside_chroot): 57 58 def MockLLVMBisectionRaisesException(_args_output): 59 raise ValueError('Failed to launch more tryjobs.') 60 61 # Use the test function to simulate the behavior of an exception happening 62 # when launching more tryjobs. 63 mock_llvm_bisection.side_effect = MockLLVMBisectionRaisesException 64 65 # Verify the exception is raised when the number of attempts to launched 66 # more tryjobs is exceeded, so unable to continue 67 # bisection. 68 with self.assertRaises(SystemExit) as err: 69 auto_llvm_bisection.main() 70 71 self.assertEqual(err.exception.code, 1) 72 73 mock_outside_chroot.assert_called_once() 74 mock_get_args.assert_called_once() 75 mock_get_auto_script.assert_called_once() 76 self.assertEqual(mock_is_file.call_count, 2) 77 self.assertEqual(mock_llvm_bisection.call_count, 3) 78 self.assertEqual(mock_traceback.call_count, 3) 79 self.assertEqual(mock_sleep.call_count, 2) 80 81 # Simulate the behavior of `subprocess.call()` when successfully updated all 82 # tryjobs whose 'status' value is 'pending'. 83 @mock.patch.object(subprocess, 'call', return_value=0) 84 # Simulate the behavior of `VerifyOutsideChroot()` when successfully invoking 85 # the script outside of the chroot. 86 @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True) 87 # Simulate behavior of `time.sleep()` when waiting for errors to settle caused 88 # by `llvm_bisection.main()` (e.g. network issue, etc.). 89 @mock.patch.object(time, 'sleep') 90 # Simulate behavior of `traceback.print_exc()` when an exception happened in 91 # `llvm_bisection.main()`. 92 @mock.patch.object(traceback, 'print_exc') 93 # Simulate behavior of `llvm_bisection.main()` when failed to launch tryjobs 94 # (exception happened along the way, etc.). 95 @mock.patch.object(llvm_bisection, 'main') 96 # Simulate behavior of `os.path.isfile()` when starting a new bisection. 97 @mock.patch.object(os.path, 'isfile') 98 # Simulate behavior of `GetPathToUpdateAllTryjobsWithAutoScript()` when 99 # returning the absolute path to that script that updates all 'pending' 100 # tryjobs to the result of `cros buildresult`. 101 @mock.patch.object( 102 auto_llvm_bisection, 103 'GetPathToUpdateAllTryjobsWithAutoScript', 104 return_value='/abs/path/to/update_tryjob.py') 105 # Simulate `llvm_bisection.GetCommandLineArgs()` when parsing the command line 106 # arguments required by the bisection script. 107 @mock.patch.object( 108 llvm_bisection, 109 'GetCommandLineArgs', 110 return_value=test_helpers.ArgsOutputTest()) 111 def testSuccessfullyBisectedLLVMRevision( 112 self, mock_get_args, mock_get_auto_script, mock_is_file, 113 mock_llvm_bisection, mock_traceback, mock_sleep, mock_outside_chroot, 114 mock_update_tryjobs): 115 116 # Simulate the behavior of `os.path.isfile()` when checking whether the 117 # status file provided exists. 118 @test_helpers.CallCountsToMockFunctions 119 def MockStatusFileCheck(call_count, _last_tested): 120 # Simulate that the status file does not exist, so the LLVM bisection 121 # script would create the status file and launch tryjobs. 122 if call_count < 2: 123 return False 124 125 # Simulate when the status file exists and `subprocess.call()` executes 126 # the script that updates all the 'pending' tryjobs to the result of `cros 127 # buildresult`. 128 if call_count == 2: 129 return True 130 131 assert False, 'os.path.isfile() called more times than expected.' 132 133 # Simulate behavior of `llvm_bisection.main()` when successfully bisected 134 # between the good and bad LLVM revision. 135 @test_helpers.CallCountsToMockFunctions 136 def MockLLVMBisectionReturnValue(call_count, _args_output): 137 # Simulate that successfully launched more tryjobs. 138 if call_count == 0: 139 return 0 140 141 # Simulate that failed to launch more tryjobs. 142 if call_count == 1: 143 raise ValueError('Failed to launch more tryjobs.') 144 145 # Simulate that the bad revision has been found. 146 if call_count == 2: 147 return llvm_bisection.BisectionExitStatus.BISECTION_COMPLETE.value 148 149 assert False, 'Called `llvm_bisection.main()` more than expected.' 150 151 # Use the test function to simulate the behavior of `llvm_bisection.main()`. 152 mock_llvm_bisection.side_effect = MockLLVMBisectionReturnValue 153 154 # Use the test function to simulate the behavior of `os.path.isfile()`. 155 mock_is_file.side_effect = MockStatusFileCheck 156 157 # Verify the excpetion is raised when successfully found the bad revision. 158 # Uses `sys.exit(0)` to indicate success. 159 with self.assertRaises(SystemExit) as err: 160 auto_llvm_bisection.main() 161 162 self.assertEqual(err.exception.code, 0) 163 164 mock_outside_chroot.assert_called_once() 165 mock_get_args.assert_called_once() 166 mock_get_auto_script.assert_called_once() 167 self.assertEqual(mock_is_file.call_count, 3) 168 self.assertEqual(mock_llvm_bisection.call_count, 3) 169 mock_traceback.assert_called_once() 170 mock_sleep.assert_called_once() 171 mock_update_tryjobs.assert_called_once() 172 173 # Simulate behavior of `subprocess.call()` when failed to update tryjobs to 174 # `cros buildresult` (script failed). 175 @mock.patch.object(subprocess, 'call', return_value=1) 176 # Simulate behavior of `time.time()` when determining the time passed when 177 # updating tryjobs whose 'status' is 'pending'. 178 @mock.patch.object(time, 'time') 179 # Simulate the behavior of `VerifyOutsideChroot()` when successfully invoking 180 # the script outside of the chroot. 181 @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True) 182 # Simulate behavior of `time.sleep()` when waiting for errors to settle caused 183 # by `llvm_bisection.main()` (e.g. network issue, etc.). 184 @mock.patch.object(time, 'sleep') 185 # Simulate behavior of `traceback.print_exc()` when resuming bisection. 186 @mock.patch.object(os.path, 'isfile', return_value=True) 187 # Simulate behavior of `GetPathToUpdateAllTryjobsWithAutoScript()` when 188 # returning the absolute path to that script that updates all 'pending' 189 # tryjobs to the result of `cros buildresult`. 190 @mock.patch.object( 191 auto_llvm_bisection, 192 'GetPathToUpdateAllTryjobsWithAutoScript', 193 return_value='/abs/path/to/update_tryjob.py') 194 # Simulate `llvm_bisection.GetCommandLineArgs()` when parsing the command line 195 # arguments required by the bisection script. 196 @mock.patch.object( 197 llvm_bisection, 198 'GetCommandLineArgs', 199 return_value=test_helpers.ArgsOutputTest()) 200 def testFailedToUpdatePendingTryJobs( 201 self, mock_get_args, mock_get_auto_script, mock_is_file, mock_sleep, 202 mock_outside_chroot, mock_time, mock_update_tryjobs): 203 204 # Simulate behavior of `time.time()` for time passed. 205 @test_helpers.CallCountsToMockFunctions 206 def MockTimePassed(call_count): 207 if call_count < 3: 208 return call_count 209 210 assert False, 'Called `time.time()` more than expected.' 211 212 # Use the test function to simulate the behavior of `time.time()`. 213 mock_time.side_effect = MockTimePassed 214 215 # Reduce the polling limit for the test case to terminate faster. 216 auto_llvm_bisection.POLLING_LIMIT_SECS = 1 217 218 # Verify the exception is raised when unable to update tryjobs whose 219 # 'status' value is 'pending'. 220 with self.assertRaises(SystemExit) as err: 221 auto_llvm_bisection.main() 222 223 self.assertEqual(err.exception.code, 1) 224 225 mock_outside_chroot.assert_called_once() 226 mock_get_args.assert_called_once() 227 mock_get_auto_script.assert_called_once() 228 self.assertEqual(mock_is_file.call_count, 2) 229 mock_sleep.assert_called_once() 230 self.assertEqual(mock_time.call_count, 3) 231 self.assertEqual(mock_update_tryjobs.call_count, 2) 232 233 234if __name__ == '__main__': 235 unittest.main() 236