• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python2
2#
3# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7import datetime as datetime_base
8import logging
9from datetime import datetime
10
11import common
12
13from autotest_lib.client.common_lib import global_config
14from autotest_lib.server import utils
15from autotest_lib.server.cros.dynamic_suite import reporting_utils
16
17CONFIG = global_config.global_config
18
19
20class DUTsNotAvailableError(utils.TestLabException):
21    """Raised when a DUT label combination is not available in the lab."""
22
23
24class NotEnoughDutsError(utils.TestLabException):
25    """Rasied when the lab doesn't have the minimum number of duts."""
26
27    def __init__(self, labels, num_available, num_required, hosts):
28        """Initialize instance.
29
30        Please pass arguments by keyword.
31
32        @param labels: Labels required, including board an pool labels.
33        @param num_available: Number of available hosts.
34        @param num_required: Number of hosts required.
35        @param hosts: Sequence of Host instances for given board and pool.
36        """
37        self.labels = labels
38        self.num_available = num_available
39        self.num_required = num_required
40        self.hosts = hosts
41        self.bug_id = None
42        self.suite_name = None
43        self.build = None
44
45
46    def __repr__(self):
47        return (
48            '<{cls} at 0x{id:x} with'
49            ' labels={this.labels!r},'
50            ' num_available={this.num_available!r},'
51            ' num_required={this.num_required!r},'
52            ' bug_id={this.bug_id!r},'
53            ' suite_name={this.suite_name!r},'
54            ' build={this.build!r}>'
55            .format(cls=type(self).__name__, id=id(self), this=self)
56        )
57
58
59    def __str__(self):
60        msg_parts = [
61            'Not enough DUTs for requirements: {this.labels};'
62            ' required: {this.num_required}, found: {this.num_available}'
63        ]
64        format_dict = {'this': self}
65        if self.bug_id is not None:
66            msg_parts.append('bug: {bug_url}')
67            format_dict['bug_url'] = reporting_utils.link_crbug(self.bug_id)
68        if self.suite_name is not None:
69            msg_parts.append('suite: {this.suite_name}')
70        if self.build is not None:
71            msg_parts.append('build: {this.build}')
72        return ', '.join(msg_parts).format(**format_dict)
73
74
75class SimpleTimer(object):
76    """A simple timer used to periodically check if a deadline has passed."""
77
78    def _reset(self):
79        """Reset the deadline."""
80        if not self.interval_hours or self.interval_hours < 0:
81            logging.error('Bad interval %s', self.interval_hours)
82            self.deadline = None
83            return
84        self.deadline = datetime.now() + datetime_base.timedelta(
85                hours=self.interval_hours)
86
87
88    def __init__(self, interval_hours=0.5):
89        """Initialize a simple periodic deadline timer.
90
91        @param interval_hours: Interval of the deadline.
92        """
93        self.interval_hours = interval_hours
94        self._reset()
95
96
97    def poll(self):
98        """Poll the timer to see if we've hit the deadline.
99
100        This method resets the deadline if it has passed. If the deadline
101        hasn't been set, or the current time is less than the deadline, the
102        method returns False.
103
104        @return: True if the deadline has passed, False otherwise.
105        """
106        if not self.deadline or datetime.now() < self.deadline:
107            return False
108        self._reset()
109        return True
110
111
112class JobTimer(object):
113    """Utility class capable of measuring job timeouts.
114    """
115
116    # Format used in datetime - string conversion.
117    time_format = '%m-%d-%Y [%H:%M:%S]'
118
119    def __init__(self, job_created_time, timeout_mins):
120        """JobTimer constructor.
121
122        @param job_created_time: float representing the time a job was
123            created. Eg: time.time()
124        @param timeout_mins: float representing the timeout in minutes.
125        """
126        self.job_created_time = datetime.fromtimestamp(job_created_time)
127        self.timeout_hours = datetime_base.timedelta(hours=timeout_mins/60.0)
128        self.debug_output_timer = SimpleTimer(interval_hours=0.5)
129        self.past_halftime = False
130
131
132    @classmethod
133    def format_time(cls, datetime_obj):
134        """Get the string formatted version of the datetime object.
135
136        @param datetime_obj: A datetime.datetime object.
137            Eg: datetime.datetime.now()
138
139        @return: A formatted string containing the date/time of the
140            input datetime.
141        """
142        return datetime_obj.strftime(cls.time_format)
143
144
145    def elapsed_time(self):
146        """Get the time elapsed since this job was created.
147
148        @return: A timedelta object representing the elapsed time.
149        """
150        return datetime.now() - self.job_created_time
151
152
153    def is_suite_timeout(self):
154        """Check if the suite timed out.
155
156        @return: True if more than timeout_hours has elapsed since the suite job
157            was created.
158        """
159        if self.elapsed_time() >= self.timeout_hours:
160            logging.info('Suite timed out. Started on %s, timed out on %s',
161                         self.format_time(self.job_created_time),
162                         self.format_time(datetime.now()))
163            return True
164        return False
165
166
167    def first_past_halftime(self):
168        """Check if we just crossed half time.
169
170        This method will only return True once, the first time it is called
171        after a job's elapsed time is past half its timeout.
172
173        @return True: If this is the first call of the method after halftime.
174        """
175        if (not self.past_halftime and
176            self.elapsed_time() > self.timeout_hours/2):
177            self.past_halftime = True
178            return True
179        return False
180
181
182class RPCHelper(object):
183    """A class to help diagnose a suite run through the rpc interface.
184    """
185
186    def __init__(self, rpc_interface):
187        """Constructor for rpc helper class.
188
189        @param rpc_interface: An rpc object, eg: A RetryingAFE instance.
190        """
191        self.rpc_interface = rpc_interface
192
193
194    def check_dut_availability(self, labels, minimum_duts=0,
195                               skip_duts_check=False):
196        """Check if DUT availability for a given board and pool is less than
197        minimum.
198
199        @param labels: DUT label dependencies, including board and pool
200                       labels.
201        @param minimum_duts: Minimum Number of available machines required to
202                             run the suite. Default is set to 0, which means do
203                             not force the check of available machines before
204                             running the suite.
205        @param skip_duts_check: If True, skip minimum available DUTs check.
206        @raise: NotEnoughDutsError if DUT availability is lower than minimum.
207        @raise: DUTsNotAvailableError if no host found for requested
208                board/pool.
209        """
210        if minimum_duts == 0:
211            return
212
213        hosts = self.rpc_interface.get_hosts(
214                invalid=False, multiple_labels=labels)
215        if not hosts:
216            raise DUTsNotAvailableError(
217                    'No hosts found for labels %r. The test lab '
218                    'currently does not cover test for those DUTs.' %
219                    (labels,))
220
221        if skip_duts_check:
222            # Bypass minimum avilable DUTs check
223            logging.debug('skip_duts_check is on, do not enforce minimum '
224                          'DUTs check.')
225            return
226
227        if len(hosts) < minimum_duts:
228            logging.debug('The total number of DUTs for %r is %d, '
229                          'which is less than %d, the required minimum '
230                          'number of available DUTS', labels, len(hosts),
231                          minimum_duts)
232
233        available_hosts = 0
234        for host in hosts:
235            if host.is_available():
236                available_hosts += 1
237        logging.debug('%d of %d DUTs are available for %r.',
238                      available_hosts, len(hosts), labels)
239        if available_hosts < minimum_duts:
240            raise NotEnoughDutsError(
241                labels=labels,
242                num_available=available_hosts,
243                num_required=minimum_duts,
244                hosts=hosts)
245
246
247    def diagnose_job(self, job_id, instance_server):
248        """Diagnose a suite job.
249
250        Logs information about the jobs that are still to run in the suite.
251
252        @param job_id: The id of the suite job to get information about.
253            No meaningful information gets logged if the id is for a sub-job.
254        @param instance_server: The instance server.
255            Eg: cautotest, cautotest-cq, localhost.
256        """
257        incomplete_jobs = self.rpc_interface.get_jobs(
258                parent_job_id=job_id, summary=True,
259                hostqueueentry__complete=False)
260        if incomplete_jobs:
261            logging.info('\n%s printing summary of incomplete jobs (%s):\n',
262                         JobTimer.format_time(datetime.now()),
263                         len(incomplete_jobs))
264            for job in incomplete_jobs:
265                logging.info('%s: %s', job.testname[job.testname.rfind('/')+1:],
266                             reporting_utils.link_job(job.id, instance_server))
267        else:
268            logging.info('All jobs in suite have already completed.')
269