1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# Copyright (c) 2021-2022 Huawei Device Co., Ltd. 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16import argparse 17import os 18import json 19import subprocess 20 21SRC_PATH = os.path.realpath(os.path.dirname(__file__)) 22 23parser = argparse.ArgumentParser("Compare benchmark results") 24parser.add_argument("--old", metavar="JSON_FILE_PATH", required=True, 25 help="Base or reference benchmark result") 26parser.add_argument("--new", metavar="JSON_FILE_PATH", required=True, 27 help="Benchmark result to be compared with the reference") 28parser.add_argument("--failed", metavar="JSON_FILE_PATH", required=False, 29 help="File to log error messages from c2p") 30 31 32def exists(name, d): 33 if name in d: 34 return True 35 return False 36 37 38args = parser.parse_args() 39 40if __name__ == '__main__': 41 if not os.path.exists(args.old): 42 print("Input file (%s) does not exists." % args.old) 43 exit(1) 44 45 if not os.path.exists(args.new): 46 print("Input file (%s) does not exists." % args.new) 47 exit(1) 48 49 with open(args.old, 'r') as old_fp, open(args.new, 'r') as new_fp: 50 old_res, new_res = json.load(old_fp), json.load(new_fp) 51 52 result = {} 53 failed_log_new = {} 54 failed_log_old = {} 55 flog = {} 56 optimized_files = 0 # number of files that have been optimized 57 not_optimized_files = 0 # files that are not optimized 58 de_optimized_files = 0 # code_item section is bigger than baseline 59 empty_files = 0 # files with no code_item section 60 old_size = 0 61 new_size = 0 62 sum_old = 0 63 sum_new = 0 64 failed_old = 0 65 failed_new = 0 66 sum_optimized_old = 0 67 sum_optimized_new = 0 68 69 for filename in old_res.keys(): 70 if not exists("error", old_res[filename]): 71 if not exists("error", new_res[filename]): 72 if exists("code_item section", new_res[filename]): 73 old_size = old_res[filename]["code_item section"] 74 new_size = new_res[filename]["code_item section"] 75 sum_old += old_size 76 sum_new += new_size 77 diff = old_size - new_size 78 result[filename] = {"old_size": old_size, 79 "new_size": new_size, "diff": diff} 80 if diff > 0: 81 optimized_files += 1 82 sum_optimized_old += old_size 83 sum_optimized_new += new_size 84 elif diff < 0: 85 de_optimized_files += 1 86 else: 87 not_optimized_files += 1 88 else: 89 empty_files += 1 90 else: 91 failed_new += 1 92 failed_log_new[filename] = new_res[filename] 93 else: 94 failed_old += 1 95 failed_log_old[filename] = old_res[filename] 96 97 print("Classes that have been optimized:\n Code_item section size:\n|Old: |New: |Diff:|Per: |File:") 98 for r in result: 99 if result[r]["diff"] > 0: 100 print("|{:5d}|{:5d}|{:5d}|{:5.2f}%| {:s}".format( 101 result[r]["old_size"], 102 result[r]["new_size"], 103 result[r]["diff"], 104 100 * (1 - float(result[r]["new_size"]) / result[r]["old_size"]) if result[r]["old_size"] != 0 else 0, 105 r)) 106 107 print("""\nSummary:\n=============\ 108 \n Total code_item section size of baseline files: {:d} bytes\ 109 \n Total code_item section size of compared files: {:d} bytes\ 110 \n Difference: {:d} bytes [{:3.2f}%]\ 111 \n Number of optimized files: {:d}\ 112 \n Number of not optimized files : {:d}\ 113 \n Files with no code item section: {:d}\ 114 \n Files that are bigger than baseline: {:d}\ 115 \n Failed tests on baseline: {:d}\ 116 \n Failed tests compared to baseline: {:d}\ 117 \n=============""" 118 .format(sum_old, sum_new, sum_old - sum_new, 119 100 * (1 - float(sum_new) / sum_old) if sum_old != 0 else 0, 120 optimized_files, not_optimized_files, empty_files, de_optimized_files, failed_old, failed_new)) 121 122 print("""\nStatistics on optimized files:\n============= \ 123 \n Total code_item section size of baseline files: {:d} bytes\ 124 \n Total code_item section size of compared files: {:d} bytes\ 125 \n Difference: {:d} bytes [{:3.2f}%]\ 126 \n=============""" 127 .format(sum_optimized_old, sum_optimized_new, 128 sum_optimized_old - sum_optimized_new, 129 100 * (1 - float(sum_optimized_new) / sum_optimized_old) if sum_optimized_old != 0 else 0)) 130 131 if args.failed: 132 with open(args.failed, 'w') as fp: 133 flog = {"Errors on baseline tests": failed_log_old, 134 "Errors on compared tests": failed_log_new} 135 136 json.dump(flog, fp, indent=4) 137 fp.write("\n") 138