1 2# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Module of benchmark runs.""" 7from __future__ import print_function 8 9import datetime 10import threading 11import time 12import traceback 13 14from cros_utils import command_executer 15from cros_utils import timeline 16 17from suite_runner import SuiteRunner 18from results_cache import MockResult 19from results_cache import MockResultsCache 20from results_cache import Result 21from results_cache import ResultsCache 22 23STATUS_FAILED = 'FAILED' 24STATUS_SUCCEEDED = 'SUCCEEDED' 25STATUS_IMAGING = 'IMAGING' 26STATUS_RUNNING = 'RUNNING' 27STATUS_WAITING = 'WAITING' 28STATUS_PENDING = 'PENDING' 29 30 31class BenchmarkRun(threading.Thread): 32 """The benchmarkrun class.""" 33 def __init__(self, name, benchmark, label, iteration, cache_conditions, 34 machine_manager, logger_to_use, log_level, share_cache): 35 threading.Thread.__init__(self) 36 self.name = name 37 self._logger = logger_to_use 38 self.log_level = log_level 39 self.benchmark = benchmark 40 self.iteration = iteration 41 self.label = label 42 self.result = None 43 self.terminated = False 44 self.retval = None 45 self.run_completed = False 46 self.machine_manager = machine_manager 47 self.suite_runner = SuiteRunner(self._logger, self.log_level) 48 self.machine = None 49 self.cache_conditions = cache_conditions 50 self.runs_complete = 0 51 self.cache_hit = False 52 self.failure_reason = '' 53 self.test_args = benchmark.test_args 54 self.cache = None 55 self.profiler_args = self.GetExtraAutotestArgs() 56 self._ce = command_executer.GetCommandExecuter(self._logger, 57 log_level=self.log_level) 58 self.timeline = timeline.Timeline() 59 self.timeline.Record(STATUS_PENDING) 60 self.share_cache = share_cache 61 self.cache_has_been_read = False 62 63 # This is used by schedv2. 64 self.owner_thread = None 65 66 def ReadCache(self): 67 # Just use the first machine for running the cached version, 68 # without locking it. 69 self.cache = ResultsCache() 70 self.cache.Init(self.label.chromeos_image, self.label.chromeos_root, 71 self.benchmark.test_name, self.iteration, self.test_args, 72 self.profiler_args, self.machine_manager, self.machine, 73 self.label.board, self.cache_conditions, self._logger, 74 self.log_level, self.label, self.share_cache, 75 self.benchmark.suite, self.benchmark.show_all_results, 76 self.benchmark.run_local) 77 78 self.result = self.cache.ReadResult() 79 self.cache_hit = (self.result is not None) 80 self.cache_has_been_read = True 81 82 def run(self): 83 try: 84 if not self.cache_has_been_read: 85 self.ReadCache() 86 87 if self.result: 88 self._logger.LogOutput('%s: Cache hit.' % self.name) 89 self._logger.LogOutput(self.result.out, print_to_console=False) 90 self._logger.LogError(self.result.err, print_to_console=False) 91 92 elif self.label.cache_only: 93 self._logger.LogOutput('%s: No cache hit.' % self.name) 94 output = '%s: No Cache hit.' % self.name 95 retval = 1 96 err = 'No cache hit.' 97 self.result = Result.CreateFromRun( 98 self._logger, self.log_level, self.label, self.machine, output, err, 99 retval, self.benchmark.test_name, 100 self.benchmark.suite) 101 102 else: 103 self._logger.LogOutput('%s: No cache hit.' % self.name) 104 self.timeline.Record(STATUS_WAITING) 105 # Try to acquire a machine now. 106 self.machine = self.AcquireMachine() 107 self.cache.machine = self.machine 108 self.result = self.RunTest(self.machine) 109 110 self.cache.remote = self.machine.name 111 self.label.chrome_version = self.machine_manager.GetChromeVersion( 112 self.machine) 113 self.cache.StoreResult(self.result) 114 115 if not self.label.chrome_version: 116 if self.machine: 117 self.label.chrome_version = self.machine_manager.GetChromeVersion( 118 self.machine) 119 elif self.result.chrome_version: 120 self.label.chrome_version = self.result.chrome_version 121 122 if self.terminated: 123 return 124 125 if not self.result.retval: 126 self.timeline.Record(STATUS_SUCCEEDED) 127 else: 128 if self.timeline.GetLastEvent() != STATUS_FAILED: 129 self.failure_reason = 'Return value of test suite was non-zero.' 130 self.timeline.Record(STATUS_FAILED) 131 132 except Exception, e: 133 self._logger.LogError("Benchmark run: '%s' failed: %s" % (self.name, e)) 134 traceback.print_exc() 135 if self.timeline.GetLastEvent() != STATUS_FAILED: 136 self.timeline.Record(STATUS_FAILED) 137 self.failure_reason = str(e) 138 finally: 139 if self.owner_thread is not None: 140 # In schedv2 mode, we do not lock machine locally. So noop here. 141 pass 142 elif self.machine: 143 if not self.machine.IsReachable(): 144 self._logger.LogOutput('Machine %s is not reachable, removing it.' % 145 self.machine.name) 146 self.machine_manager.RemoveMachine(self.machine.name) 147 self._logger.LogOutput('Releasing machine: %s' % self.machine.name) 148 self.machine_manager.ReleaseMachine(self.machine) 149 self._logger.LogOutput('Released machine: %s' % self.machine.name) 150 151 def Terminate(self): 152 self.terminated = True 153 self.suite_runner.Terminate() 154 if self.timeline.GetLastEvent() != STATUS_FAILED: 155 self.timeline.Record(STATUS_FAILED) 156 self.failure_reason = 'Thread terminated.' 157 158 def AcquireMachine(self): 159 if self.owner_thread is not None: 160 # No need to lock machine locally, DutWorker, which is a thread, is 161 # responsible for running br. 162 return self.owner_thread.dut() 163 while True: 164 machine = None 165 if self.terminated: 166 raise RuntimeError('Thread terminated while trying to acquire machine.') 167 168 machine = self.machine_manager.AcquireMachine(self.label) 169 170 if machine: 171 self._logger.LogOutput('%s: Machine %s acquired at %s' % 172 (self.name, machine.name, 173 datetime.datetime.now())) 174 break 175 time.sleep(10) 176 return machine 177 178 def GetExtraAutotestArgs(self): 179 if self.benchmark.perf_args and self.benchmark.suite == 'telemetry': 180 self._logger.LogError('Telemetry does not support profiler.') 181 self.benchmark.perf_args = '' 182 183 if self.benchmark.perf_args and self.benchmark.suite == 'test_that': 184 self._logger.LogError('test_that does not support profiler.') 185 self.benchmark.perf_args = '' 186 187 if self.benchmark.perf_args: 188 perf_args_list = self.benchmark.perf_args.split(' ') 189 perf_args_list = [perf_args_list[0]] + ['-a'] + perf_args_list[1:] 190 perf_args = ' '.join(perf_args_list) 191 if not perf_args_list[0] in ['record', 'stat']: 192 raise SyntaxError('perf_args must start with either record or stat') 193 extra_test_args = ['--profiler=custom_perf', 194 ("--profiler_args='perf_options=\"%s\"'" % perf_args)] 195 return ' '.join(extra_test_args) 196 else: 197 return '' 198 199 def RunTest(self, machine): 200 self.timeline.Record(STATUS_IMAGING) 201 if self.owner_thread is not None: 202 # In schedv2 mode, do not even call ImageMachine. Machine image is 203 # guarenteed. 204 pass 205 else: 206 self.machine_manager.ImageMachine(machine, self.label) 207 self.timeline.Record(STATUS_RUNNING) 208 retval, out, err = self.suite_runner.Run(machine.name, self.label, 209 self.benchmark, self.test_args, 210 self.profiler_args) 211 self.run_completed = True 212 return Result.CreateFromRun(self._logger, self.log_level, self.label, 213 self.machine, out, err, retval, 214 self.benchmark.test_name, self.benchmark.suite) 215 216 def SetCacheConditions(self, cache_conditions): 217 self.cache_conditions = cache_conditions 218 219 def logger(self): 220 """Return the logger, only used by unittest. 221 222 Returns: 223 self._logger 224 """ 225 226 return self._logger 227 228 def __str__(self): 229 """For better debugging.""" 230 231 return 'BenchmarkRun[name="{}"]'.format(self.name) 232 233 234class MockBenchmarkRun(BenchmarkRun): 235 """Inherited from BenchmarkRun.""" 236 237 def ReadCache(self): 238 # Just use the first machine for running the cached version, 239 # without locking it. 240 self.cache = MockResultsCache() 241 self.cache.Init(self.label.chromeos_image, self.label.chromeos_root, 242 self.benchmark.test_name, self.iteration, self.test_args, 243 self.profiler_args, self.machine_manager, self.machine, 244 self.label.board, self.cache_conditions, self._logger, 245 self.log_level, self.label, self.share_cache, 246 self.benchmark.suite, self.benchmark.show_all_results, 247 self.benchmark.run_local) 248 249 self.result = self.cache.ReadResult() 250 self.cache_hit = (self.result is not None) 251 252 def RunTest(self, machine): 253 """Remove Result.CreateFromRun for testing.""" 254 self.timeline.Record(STATUS_IMAGING) 255 self.machine_manager.ImageMachine(machine, self.label) 256 self.timeline.Record(STATUS_RUNNING) 257 [retval, out, err] = self.suite_runner.Run(machine.name, self.label, 258 self.benchmark, self.test_args, 259 self.profiler_args) 260 self.run_completed = True 261 rr = MockResult('logger', self.label, self.log_level, machine) 262 rr.out = out 263 rr.err = err 264 rr.retval = retval 265 return rr 266