1#!/usr/bin/env python 2# SPDX-License-Identifier: Apache-2.0 3# 4# Copyright (C) 2017, ARM Limited, Google, and contributors. 5# 6# Licensed under the Apache License, Version 2.0 (the "License"); you may 7# not use this file except in compliance with the License. 8# You may obtain a copy of the License at 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17# 18# 19# Analysis of framestats deltas between 2+ runs 20# This analysis outputs potential regressions when comparing 21# a baseline folder output of microbenchmarks like UiBench with 22# a test folder with similar stats. The base and test folders 23# can contain the output of one or more runs of run_uibench.py or 24# similar script. 25 26# For example: 27# python framestats_analysis.py --baseline_dir=/usr/local/google/home/andresoportus/b/master2/external/lisa/results/UiBench_walleye_em_baseline_b/ --results_dir=/usr/local/google/home/andresoportus/b/master2/external/lisa/results/UiBench_walleye_em_first/ --threshold_ave 0.322 28# No handlers could be found for logger "EnergyModel" 29# Potential regression in: 30# avg-frame-time-50 avg-frame-time-90 avg-frame-time-95 avg-frame-time-99 31# UiBenchJankTests#testTrivialAnimation 32# all base [ 5. 5. 5. 5. 5. 5. 5.] [ 5. 5. 5. 5. 5. 5. 5.] [ 5. 5. 5. 5. 5. 5. 5.] [ 5. 5.2 5.1 5.2 5.9 5.2 5.2] 33# all test [ 5. 5. 5. 5. 5. 5.] [ 5. 5. 5. 5. 5. 5.] [ 5. 5. 5. 5. 5. 5.] [ 5.4 6. 5.3 5.8 6. 5.1] 34# ave base [ 5.] [ 5.] [ 5.] [ 5.25714286] 35# ave test [ 5.] [ 5.] [ 5.] [ 5.6] 36# avg delta [ 0.] [ 0.] [ 0.] [ 0.34285714] 37 38import os 39import sys 40import argparse 41import numpy as np 42import pandas as pd 43from collections import OrderedDict 44from android.workloads.uibench import UiBench 45 46averages = ['avg-frame-time-50', 'avg-frame-time-90', 47 'avg-frame-time-95', 'avg-frame-time-99'] 48stats_file = 'framestats.txt' 49 50""" 51Parses a directory to find stats 52 53:param stats_dir: path to stats dir 54:type stats_dir: str 55 56:param stats_file: name of stats file 57:type stats_file: str 58""" 59def parse_stats_dir(stats_dir, stats_file): 60 df = {} 61 run_name = os.path.basename(os.path.normpath(stats_dir)) 62 for root, dirs, files in os.walk(stats_dir): 63 if stats_file in files: 64 test_name = os.path.basename(os.path.normpath(root)) 65 if test_name not in df: 66 df[test_name] = pd.DataFrame() 67 df[test_name] = df[test_name].append(UiBench.get_results(root)) 68 return df 69 70""" 71Prints a header line 72 73:param df: pandas dataframe to extract columns names from 74:type df: pd.DataFrame 75""" 76def print_header(df): 77 sys.stdout.write('{:16}'.format('')) 78 for c in df.columns: 79 sys.stdout.write(' {:32}'.format(str(c))) 80 sys.stdout.write('\n') 81 82""" 83Prints a pandas DataFrame as a row 84 85:param df: pandas dataframe to extract values from 86:type df: pd.DataFrame 87""" 88def print_as_row(name, df): 89 sys.stdout.write('{:16}'.format(name)) 90 for c in df.columns: 91 sys.stdout.write(' {:32}'.format(str(df[c].values))) 92 sys.stdout.write('\n') 93 94def main(): 95 header_printed = False 96 pd.set_option('precision', 2) 97 base = parse_stats_dir(args.baseline_dir, stats_file) 98 test = parse_stats_dir(args.results_dir, stats_file) 99 for name in base: 100 try: 101 # DataFrame created from the Series output of mean() needs to 102 # be transposed to make averages be columns again 103 ave_base = pd.DataFrame(base[name][averages].mean()).T 104 ave_test = pd.DataFrame(test[name][averages].mean()).T 105 if args.verbose or \ 106 ((ave_test - ave_base) > args.threshold_ave).iloc[0].any(): 107 if not header_printed: 108 if not args.verbose: 109 print "Potential regression in:" 110 print_header(base[name][averages]) 111 header_printed = True 112 print name 113 print_as_row('all base', base[name][averages]) 114 print_as_row('all test', test[name][averages]) 115 print_as_row('ave base', ave_base) 116 print_as_row('ave test', ave_test) 117 print_as_row('avg delta', ave_test - ave_base) 118 except KeyError: 119 sys.stdout.write('\n') 120 if not header_printed: 121 print "No regression found" 122 123if __name__ == "__main__": 124 parser = argparse.ArgumentParser( 125 description="Framestats analysis") 126 127 parser.add_argument("--results_dir", "-d", type=str, 128 default=os.path.join(os.environ["LISA_HOME"], 129 "results/UiBench_default"), 130 help="The test directory to read from. (default \ 131 LISA_HOME/restuls/UiBench_deafult)") 132 parser.add_argument("--baseline_dir", "-b", type=str, required=True, 133 help="The directory that provides baseline test to \ 134 compare against") 135 parser.add_argument("--threshold_ave", "-t", type=float, default=0.99, 136 help="Amount to filter noise in average values") 137 parser.add_argument("--verbose", "-v", action='store_true', 138 help="Verbose output") 139 args = parser.parse_args() 140 main() 141