1#!/usr/bin/python2 2# 3# Copyright (c) 2014 The Chromium OS Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7import datetime as datetime_base 8import logging 9from datetime import datetime 10 11import common 12 13from autotest_lib.client.common_lib import global_config 14from autotest_lib.server import utils 15from autotest_lib.server.cros.dynamic_suite import reporting_utils 16 17CONFIG = global_config.global_config 18 19 20class DUTsNotAvailableError(utils.TestLabException): 21 """Raised when a DUT label combination is not available in the lab.""" 22 23 24class NotEnoughDutsError(utils.TestLabException): 25 """Rasied when the lab doesn't have the minimum number of duts.""" 26 27 def __init__(self, labels, num_available, num_required, hosts): 28 """Initialize instance. 29 30 Please pass arguments by keyword. 31 32 @param labels: Labels required, including board an pool labels. 33 @param num_available: Number of available hosts. 34 @param num_required: Number of hosts required. 35 @param hosts: Sequence of Host instances for given board and pool. 36 """ 37 self.labels = labels 38 self.num_available = num_available 39 self.num_required = num_required 40 self.hosts = hosts 41 self.bug_id = None 42 self.suite_name = None 43 self.build = None 44 45 46 def __repr__(self): 47 return ( 48 '<{cls} at 0x{id:x} with' 49 ' labels={this.labels!r},' 50 ' num_available={this.num_available!r},' 51 ' num_required={this.num_required!r},' 52 ' bug_id={this.bug_id!r},' 53 ' suite_name={this.suite_name!r},' 54 ' build={this.build!r}>' 55 .format(cls=type(self).__name__, id=id(self), this=self) 56 ) 57 58 59 def __str__(self): 60 msg_parts = [ 61 'Not enough DUTs for requirements: {this.labels};' 62 ' required: {this.num_required}, found: {this.num_available}' 63 ] 64 format_dict = {'this': self} 65 if self.bug_id is not None: 66 msg_parts.append('bug: {bug_url}') 67 format_dict['bug_url'] = reporting_utils.link_crbug(self.bug_id) 68 if self.suite_name is not None: 69 msg_parts.append('suite: {this.suite_name}') 70 if self.build is not None: 71 msg_parts.append('build: {this.build}') 72 return ', '.join(msg_parts).format(**format_dict) 73 74 75class SimpleTimer(object): 76 """A simple timer used to periodically check if a deadline has passed.""" 77 78 def _reset(self): 79 """Reset the deadline.""" 80 if not self.interval_hours or self.interval_hours < 0: 81 logging.error('Bad interval %s', self.interval_hours) 82 self.deadline = None 83 return 84 self.deadline = datetime.now() + datetime_base.timedelta( 85 hours=self.interval_hours) 86 87 88 def __init__(self, interval_hours=0.5): 89 """Initialize a simple periodic deadline timer. 90 91 @param interval_hours: Interval of the deadline. 92 """ 93 self.interval_hours = interval_hours 94 self._reset() 95 96 97 def poll(self): 98 """Poll the timer to see if we've hit the deadline. 99 100 This method resets the deadline if it has passed. If the deadline 101 hasn't been set, or the current time is less than the deadline, the 102 method returns False. 103 104 @return: True if the deadline has passed, False otherwise. 105 """ 106 if not self.deadline or datetime.now() < self.deadline: 107 return False 108 self._reset() 109 return True 110 111 112class JobTimer(object): 113 """Utility class capable of measuring job timeouts. 114 """ 115 116 # Format used in datetime - string conversion. 117 time_format = '%m-%d-%Y [%H:%M:%S]' 118 119 def __init__(self, job_created_time, timeout_mins): 120 """JobTimer constructor. 121 122 @param job_created_time: float representing the time a job was 123 created. Eg: time.time() 124 @param timeout_mins: float representing the timeout in minutes. 125 """ 126 self.job_created_time = datetime.fromtimestamp(job_created_time) 127 self.timeout_hours = datetime_base.timedelta(hours=timeout_mins/60.0) 128 self.debug_output_timer = SimpleTimer(interval_hours=0.5) 129 self.past_halftime = False 130 131 132 @classmethod 133 def format_time(cls, datetime_obj): 134 """Get the string formatted version of the datetime object. 135 136 @param datetime_obj: A datetime.datetime object. 137 Eg: datetime.datetime.now() 138 139 @return: A formatted string containing the date/time of the 140 input datetime. 141 """ 142 return datetime_obj.strftime(cls.time_format) 143 144 145 def elapsed_time(self): 146 """Get the time elapsed since this job was created. 147 148 @return: A timedelta object representing the elapsed time. 149 """ 150 return datetime.now() - self.job_created_time 151 152 153 def is_suite_timeout(self): 154 """Check if the suite timed out. 155 156 @return: True if more than timeout_hours has elapsed since the suite job 157 was created. 158 """ 159 if self.elapsed_time() >= self.timeout_hours: 160 logging.info('Suite timed out. Started on %s, timed out on %s', 161 self.format_time(self.job_created_time), 162 self.format_time(datetime.now())) 163 return True 164 return False 165 166 167 def first_past_halftime(self): 168 """Check if we just crossed half time. 169 170 This method will only return True once, the first time it is called 171 after a job's elapsed time is past half its timeout. 172 173 @return True: If this is the first call of the method after halftime. 174 """ 175 if (not self.past_halftime and 176 self.elapsed_time() > self.timeout_hours/2): 177 self.past_halftime = True 178 return True 179 return False 180 181 182class RPCHelper(object): 183 """A class to help diagnose a suite run through the rpc interface. 184 """ 185 186 def __init__(self, rpc_interface): 187 """Constructor for rpc helper class. 188 189 @param rpc_interface: An rpc object, eg: A RetryingAFE instance. 190 """ 191 self.rpc_interface = rpc_interface 192 193 194 def check_dut_availability(self, labels, minimum_duts=0, 195 skip_duts_check=False): 196 """Check if DUT availability for a given board and pool is less than 197 minimum. 198 199 @param labels: DUT label dependencies, including board and pool 200 labels. 201 @param minimum_duts: Minimum Number of available machines required to 202 run the suite. Default is set to 0, which means do 203 not force the check of available machines before 204 running the suite. 205 @param skip_duts_check: If True, skip minimum available DUTs check. 206 @raise: NotEnoughDutsError if DUT availability is lower than minimum. 207 @raise: DUTsNotAvailableError if no host found for requested 208 board/pool. 209 """ 210 if minimum_duts == 0: 211 return 212 213 hosts = self.rpc_interface.get_hosts( 214 invalid=False, multiple_labels=labels) 215 if not hosts: 216 raise DUTsNotAvailableError( 217 'No hosts found for labels %r. The test lab ' 218 'currently does not cover test for those DUTs.' % 219 (labels,)) 220 221 if skip_duts_check: 222 # Bypass minimum avilable DUTs check 223 logging.debug('skip_duts_check is on, do not enforce minimum ' 224 'DUTs check.') 225 return 226 227 if len(hosts) < minimum_duts: 228 logging.debug('The total number of DUTs for %r is %d, ' 229 'which is less than %d, the required minimum ' 230 'number of available DUTS', labels, len(hosts), 231 minimum_duts) 232 233 available_hosts = 0 234 for host in hosts: 235 if host.is_available(): 236 available_hosts += 1 237 logging.debug('%d of %d DUTs are available for %r.', 238 available_hosts, len(hosts), labels) 239 if available_hosts < minimum_duts: 240 raise NotEnoughDutsError( 241 labels=labels, 242 num_available=available_hosts, 243 num_required=minimum_duts, 244 hosts=hosts) 245 246 247 def diagnose_job(self, job_id, instance_server): 248 """Diagnose a suite job. 249 250 Logs information about the jobs that are still to run in the suite. 251 252 @param job_id: The id of the suite job to get information about. 253 No meaningful information gets logged if the id is for a sub-job. 254 @param instance_server: The instance server. 255 Eg: cautotest, cautotest-cq, localhost. 256 """ 257 incomplete_jobs = self.rpc_interface.get_jobs( 258 parent_job_id=job_id, summary=True, 259 hostqueueentry__complete=False) 260 if incomplete_jobs: 261 logging.info('\n%s printing summary of incomplete jobs (%s):\n', 262 JobTimer.format_time(datetime.now()), 263 len(incomplete_jobs)) 264 for job in incomplete_jobs: 265 logging.info('%s: %s', job.testname[job.testname.rfind('/')+1:], 266 reporting_utils.link_job(job.id, instance_server)) 267 else: 268 logging.info('All jobs in suite have already completed.') 269