1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# Copyright (c) 2021-2022 Huawei Device Co., Ltd. 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16import argparse 17import os 18import json 19import subprocess 20import tempfile 21 22SRC_PATH = os.path.realpath(os.path.dirname(__file__)) 23testdir = os.path.join(SRC_PATH, "suite") 24bindir = os.path.join(SRC_PATH, "..", "..", "..", "build", "bin") 25 26parser = argparse.ArgumentParser("Run bytecode benchmarks") 27parser.add_argument("--testdir", default=testdir, 28 help="Directory with tests (*.class or *.pa). Default: './%s'" % os.path.relpath(testdir)) 29parser.add_argument("--bindir", default=bindir, 30 help="Directory with compiled binaries (eg.: c2p). Default: './%s'" % os.path.relpath(bindir)) 31parser.add_argument("--input-type", choices=["class", "pa"], default="class", 32 help="Type of the test input. Default: '%(default)s'"), 33parser.add_argument("--compiler-options", metavar="LIST", 34 help="Comma separated list of compiler options for C2P (see '%s --help' for details" 35 % (os.path.relpath(os.path.join(bindir, "c2p")))), 36parser.add_argument("--compiler-options-file", metavar="FILE", 37 help="Input file containing compiler options for C2P (see '%s --help' for details" 38 % (os.path.relpath(os.path.join(bindir, "c2p")))), 39parser.add_argument("--json", metavar="FILE", 40 help="JSON dump file name"), 41parser.add_argument("--verbose", "-v", action='store_true', 42 help="Enable verbose messages") 43 44args = parser.parse_args() 45 46 47def log(msg): 48 if args.verbose: 49 print(msg) 50 51 52def parse_c2p_output(name, stdout, stderr, returncode, d): 53 d[name] = {} # {'name': name: {'code_item section': 11}, {'total': 123}, ..} 54 result = {} # { 'name': name } 55 # TODO: Handle segmentation fault correctly 56 if returncode != 0: 57 d[name] = {"error": stderr.decode('ascii')} 58 else: 59 for stdout_line in stdout.decode('ascii').split("\n"): 60 if not stdout_line: 61 continue 62 key, value = stdout_line.split(":")[:2] 63 if value: 64 result[key] = int(value.strip()) 65 d[name][key] = int(value.strip()) 66 return result 67 68 69def print_sizes_dict(sizes): 70 for d in sorted(sizes): 71 print("%28s: %d" % (d, sizes[d])) 72 73 74def calc_statistics(sizes): 75 total = len(sizes) 76 if total == 0: 77 return None 78 keys = sizes[0].keys() 79 avgs = dict.fromkeys(keys, 0) 80 mins = dict.fromkeys(keys, 99999999) 81 maxs = dict.fromkeys(keys, 0) 82 for d in sizes: 83 for k in keys: 84 if k not in d: 85 continue 86 avgs[k] += d[k] 87 88 if d[k] < mins[k]: 89 mins[k] = d[k] 90 if d[k] > maxs[k]: 91 maxs[k] = d[k] 92 93 for d in avgs: 94 avgs[d] = round(avgs[d] / total, 1) 95 96 print("\nAverage sizes (in bytes):") 97 print_sizes_dict(avgs) 98 print("\nMinimum sizes (in bytes):") 99 print_sizes_dict(mins) 100 print("\nMaximum sizes (in bytes):") 101 print_sizes_dict(maxs) 102 103 return {"Average sizes (in bytes)": avgs, 104 "Minimum sizes (in bytes)": mins, 105 "Maximum sizes (in bytes)": maxs} 106 107 108def run_c2p(test_dir, bin_dir, c2p_opts): 109 sizes = [] 110 result = {} 111 tests_passed = 0 112 tests_failed = 0 113 114 c2p = os.path.join(bin_dir, "c2p") 115 if not os.path.exists(c2p): 116 print("c2p executable does not exists (%s)." % os.path.relpath(c2p)) 117 exit(2) 118 fp = tempfile.NamedTemporaryFile() 119 for dirpath, dirnames, filenames in os.walk(test_dir): 120 for name in filenames: 121 if name.endswith(".class"): 122 test_path = os.path.join(dirpath, name) 123 proc = subprocess.Popen([c2p, "--size-stat"] + c2p_opts + [ 124 test_path, fp.name], stdout=subprocess.PIPE, stderr=subprocess.PIPE) 125 stdout, stderr = proc.communicate() 126 sizes.append(parse_c2p_output(test_path, stdout, 127 stderr, proc.returncode, result)) 128 if proc.returncode == 0: 129 tests_passed += 1 130 else: 131 tests_failed += 1 132 log("Could not process the class file (%s)." % test_path) 133 pass 134 fp.close() 135 return sizes, tests_passed, tests_failed, result 136 137 138############################################ 139 140if __name__ == '__main__': 141 if not os.path.exists(args.testdir): 142 print("Test directory (%s) does not exists." % args.testdir) 143 exit(1) 144 145 num_of_tests = 0 146 passed_tests = 0 147 failed_tests = 0 148 149 if args.input_type == "class": 150 c2p_options = [] 151 if args.compiler_options_file: 152 with open(args.compiler_options_file) as conf_fp: 153 lines = conf_fp.readlines() 154 for line in lines: 155 c2p_options.append(line.strip()) 156 157 if args.compiler_options: 158 c2p_options += args.compiler_options.split(",") 159 160 c2p_sizes, passed_tests, failed_tests, c2p_res = run_c2p( 161 args.testdir, args.bindir, c2p_options) 162 num_of_tests = passed_tests + failed_tests 163 stats = calc_statistics(c2p_sizes) 164 if args.json: 165 with open(args.json, 'w') as json_file: 166 json.dump(c2p_res, json_file, indent=4, sort_keys=True) 167 json_file.write("\n") 168 169 else: 170 print("Non class input types have not been implemented.") 171 exit(2) 172 173 print("Summary:\n========\n Tests : %d\n Passed: %d\n Failed: %d\n" 174 % (num_of_tests, passed_tests, failed_tests)) 175