#!/usr/bin/python import optparse import sys import sqlite3 import scipy.stats import numpy import adbutil from devices import DEVICES DB_PATH="/data/data/com.android.benchmark/databases/BenchmarkResults" OUT_PATH = "db/" QUERY_BAD_FRAME = ("select run_id, name, total_duration from ui_results " "where total_duration >=12 order by run_id, name") QUERY_PERCENT_JANK = ("select run_id, name, sum(jank_frame) as jank_count, count (*) as total " "from ui_results group by run_id, name") class IterationResult: def __init__(self): self.durations = [] self.jank_count = 0 self.total_count = 0 def get_scoremap(dbpath): db = sqlite3.connect(dbpath) rows = db.execute(QUERY_BAD_FRAME) scoremap = {} for row in rows: run_id = row[0] name = row[1] total_duration = row[2] if not run_id in scoremap: scoremap[run_id] = {} if not name in scoremap[run_id]: scoremap[run_id][name] = IterationResult() scoremap[run_id][name].durations.append(float(total_duration)) for row in db.execute(QUERY_PERCENT_JANK): run_id = row[0] name = row[1] jank_count = row[2] total_count = row[3] if run_id in scoremap.keys() and name in scoremap[run_id].keys(): scoremap[run_id][name].jank_count = long(jank_count) scoremap[run_id][name].total_count = long(total_count) db.close() return scoremap def score_device(name, serial, pull = False, verbose = False): dbpath = OUT_PATH + name + ".db" if pull: adbutil.root(serial) adbutil.pull(serial, DB_PATH, dbpath) scoremap = None try: scoremap = get_scoremap(dbpath) except sqlite3.DatabaseError: print "Database corrupt, fetching..." adbutil.root(serial) adbutil.pull(serial, DB_PATH, dbpath) scoremap = get_scoremap(dbpath) per_test_score = {} per_test_sample_count = {} global_overall = {} for run_id in iter(scoremap): overall = [] if len(scoremap[run_id]) < 1: if verbose: print "Skipping short run %s" % run_id continue print "Run: %s" % run_id for test in iter(scoremap[run_id]): if verbose: print "\t%s" % test scores = [] sample_count = 0 res = scoremap[run_id][test] stddev = numpy.std(res.durations) mean = numpy.mean(res.durations) sample_count = len(res.durations) pj = 100 * res.jank_count / float(res.total_count) score = stddev * mean *pj if score == 0: score = 1 scores.append(score) if verbose: print "\tScore = %f x %f x %f = %f (%d samples)" % (stddev, mean, pj, score, len(res.durations)) geo_run = scipy.stats.gmean(scores) if test not in per_test_score: per_test_score[test] = [] if test not in per_test_sample_count: per_test_sample_count[test] = [] per_test_score[test].append(geo_run) per_test_sample_count[test].append(int(sample_count)) overall.append(geo_run) if not verbose: print "\t%s:\t%0.2f (%0.2f avg. sample count)" % (test, geo_run, sample_count) else: print "\tOverall:\t%0.2f (%0.2f avg. sample count)" % (geo_run, sample_count) print "" global_overall[run_id] = scipy.stats.gmean(overall) print "Run Overall: %f" % global_overall[run_id] print "" print "" print "Variability (CV) - %s:" % name for test in per_test_score: print "\t%s:\t%0.2f%% (%0.2f avg sample count)" % (test, 100 * scipy.stats.variation(per_test_score[test]), numpy.mean(per_test_sample_count[test])) print "\tOverall: %0.2f%%" % (100 * scipy.stats.variation([x for x in global_overall.values()])) print "" def parse_options(argv): usage = 'Usage: %prog [options]' desc = 'Example: %prog' parser = optparse.OptionParser(usage=usage, description=desc) parser.add_option("-p", dest='pull', action="store_true") parser.add_option("-d", dest='device', action="store") parser.add_option("-v", dest='verbose', action="store_true") options, categories = parser.parse_args(argv[1:]) return options def main(): options = parse_options(sys.argv) if options.device != None: score_device(options.device, DEVICES[options.device], options.pull, options.verbose) else: for name, serial in DEVICES.iteritems(): print "======== %s =========" % name score_device(name, serial, options.pull, options.verbose) if __name__ == "__main__": main()