#!/usr/bin/env python # Copyright (c) 2014 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. # This script is used to compare the performance of duts when running the same # test/special task. For example: # # python compare_dut_perf.py -l 240 --board stumpy # # compares the test runtime of all stumpy for the last 10 days. Sample output: # ============================================================================== # Test hardware_MemoryTotalSize # ============================================================================== # chromeos2-row2-rack8-host8 : min= 479, max= 479, mean= 479, med= 479, cnt= 1 # chromeos2-row2-rack8-host12 : min= 440, max= 440, mean= 440, med= 440, cnt= 1 # chromeos2-row2-rack8-host11 : min= 504, max= 504, mean= 504, med= 504, cnt= 1 # # At the end of each row, it also lists the last 5 jobs running in the dut. import argparse import datetime import multiprocessing.pool import pprint import time from itertools import groupby import common import numpy from autotest_lib.frontend import setup_django_environment from autotest_lib.frontend.afe import models from autotest_lib.frontend.afe import rpc_utils from autotest_lib.frontend.tko import models as tko_models from autotest_lib.server.cros.dynamic_suite import frontend_wrappers def get_matched_duts(hostnames=None, board=None, pool=None, other_labels=None): """Get duts with matching board and pool labels from given autotest instance @param hostnames: A list of hostnames. @param board: board of DUT, set to None if board doesn't need to match. Default is None. @param pool: pool of DUT, set to None if pool doesn't need to match. Default is None. @param other_labels: Other labels to filter duts. @return: A list of duts that match the specified board and pool. """ if hostnames: hosts = models.Host.objects.filter(hostname__in=hostnames) else: multiple_labels = () if pool: multiple_labels += ('pool:%s' % pool,) if board: multiple_labels += ('board:%s' % board,) if other_labels: for label in other_labels: multiple_labels += (label,) hosts = rpc_utils.get_host_query(multiple_labels, exclude_only_if_needed_labels=False, exclude_atomic_group_hosts=False, valid_only=True, filter_data={}) return [host_obj.get_object_dict() for host_obj in hosts] def get_job_runtime(input): """Get all test jobs and special tasks' runtime for a given host during a give time period. @param input: input arguments, including: start_time: Start time of the search interval. end_time: End time of the search interval. host_id: id of the dut. hostname: Name of the dut. @return: A list of records, e.g., [{'job_name':'dummy_Pass', 'time_used': 3, 'id': 12313, 'hostname': '1.2.3.4'}, {'task_name':'Cleanup', 'time_used': 30, 'id': 5687, 'hostname': '1.2.3.4'}] """ start_time = input['start_time'] end_time = input['end_time'] host_id = input['host_id'] hostname = input['hostname'] records = [] special_tasks = models.SpecialTask.objects.filter( host_id=host_id, time_started__gte=start_time, time_started__lte=end_time, time_started__isnull=False, time_finished__isnull=False).values('task', 'id', 'time_started', 'time_finished') for task in special_tasks: time_used = task['time_finished'] - task['time_started'] records.append({'name': task['task'], 'id': task['id'], 'time_used': time_used.total_seconds(), 'hostname': hostname}) hqes = models.HostQueueEntry.objects.filter( host_id=host_id, started_on__gte=start_time, started_on__lte=end_time, started_on__isnull=False, finished_on__isnull=False) for hqe in hqes: time_used = (hqe.finished_on - hqe.started_on).total_seconds() records.append({'name': hqe.job.name.split('/')[-1], 'id': hqe.job.id, 'time_used': time_used, 'hostname': hostname}) return records def get_job_stats(jobs): """Get the stats of a list of jobs. @param jobs: A list of jobs. @return: Stats of the jobs' runtime, including: t_min: minimum runtime. t_max: maximum runtime. t_average: average runtime. t_median: median runtime. """ runtimes = [job['time_used'] for job in jobs] t_min = min(runtimes) t_max = max(runtimes) t_mean = numpy.mean(runtimes) t_median = numpy.median(runtimes) return t_min, t_max, t_mean, t_median, len(runtimes) def process_results(results): """Compare the results. @param results: A list of a list of job/task information. """ # Merge list of all results. all_results = [] for result in results: all_results.extend(result) all_results = sorted(all_results, key=lambda r: r['name']) for name,jobs_for_test in groupby(all_results, lambda r: r['name']): print '='*80 print 'Test %s' % name print '='*80 for hostname,jobs_for_dut in groupby(jobs_for_test, lambda j: j['hostname']): jobs = list(jobs_for_dut) t_min, t_max, t_mean, t_median, count = get_job_stats(jobs) ids = [str(job['id']) for job in jobs] print ('%-28s: min= %-3.0f max= %-3.0f mean= %-3.0f med= %-3.0f ' 'cnt= %-3s IDs: %s' % (hostname, t_min, t_max, t_mean, t_median, count, ','.join(sorted(ids)[-5:]))) def main(): """main script. """ t_now = time.time() t_now_minus_one_day = t_now - 3600 * 24 parser = argparse.ArgumentParser() parser.add_argument('-l', type=float, dest='last', help='last hours to search results across', default=24) parser.add_argument('--board', type=str, dest='board', help='restrict query by board', default=None) parser.add_argument('--pool', type=str, dest='pool', help='restrict query by pool', default=None) parser.add_argument('--hosts', nargs='+', dest='hosts', help='Enter space deliminated hostnames', default=[]) parser.add_argument('--start', type=str, dest='start', help=('Enter start time as: yyyy-mm-dd hh-mm-ss,' 'defualts to 24h ago.')) parser.add_argument('--end', type=str, dest='end', help=('Enter end time in as: yyyy-mm-dd hh-mm-ss,' 'defualts to current time.')) options = parser.parse_args() if not options.start or not options.end: end_time = datetime.datetime.now() start_time = end_time - datetime.timedelta(seconds=3600 * options.last) else: start_time = time_utils.time_string_to_datetime(options.start) end_time = time_utils.time_string_to_datetime(options.end) hosts = get_matched_duts(hostnames=options.hosts, board=options.board, pool=options.pool) if not hosts: raise Exception('No host found to search for history.') print 'Found %d duts.' % len(hosts) print 'Start time: %s' % start_time print 'End time: %s' % end_time args = [] for host in hosts: args.append({'start_time': start_time, 'end_time': end_time, 'host_id': host['id'], 'hostname': host['hostname']}) get_job_runtime(args[0]) # Parallizing this process. pool = multiprocessing.pool.ThreadPool() results = pool.imap_unordered(get_job_runtime, args) process_results(results) if __name__ == '__main__': main()