1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3 4# 5# Copyright (c) 2022-2024 Huawei Device Co., Ltd. 6# Licensed under the Apache License, Version 2.0 (the "License"); 7# you may not use this file except in compliance with the License. 8# You may obtain a copy of the License at 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17# 18 19from pathlib import Path 20import os 21import subprocess 22import argparse 23 24SRC_PATH = os.path.realpath(os.path.dirname(__file__)) 25 26 27class RunStats: 28 def __init__(self): 29 self.failed_exec = [] 30 self.failed_aot_compile = [] 31 self.failed_compile = [] 32 self.passed = [] 33 self.time = {} 34 35 def record_failure_compile(self, path: Path): 36 self.failed_compile.append(path) 37 38 def record_failure_aot_compile(self, path: Path): 39 self.failed_aot_compile.append(path) 40 41 def record_failure_exec(self, path: Path): 42 self.failed_exec.append(path) 43 44 def record_success(self, path: Path): 45 self.passed.append(path) 46 47 def record_time(self, path: Path, time: float): 48 self.time[path] = time 49 50 51class Logger: 52 def __init__(self, args): 53 self.log_level = args.log_level 54 55 def info(self, message): 56 if self.log_level == "info" or self.log_level == "debug": 57 print(message) 58 59 def debug(self, message): 60 if self.log_level == "debug": 61 print(message) 62 63 def silence(self, message): 64 print(message) 65 66# ======================================================= 67# Main runner class 68 69 70class EtsBenchmarksRunner: 71 def __init__(self, args, logger, ark_opts, aot_opts): 72 self.logger = logger 73 self.mode = args.mode 74 self.interpreter_type = args.interpreter_type 75 self.is_device = args.target == "device" 76 self.host_build_dir = args.host_builddir if self.is_device else os.path.join( 77 args.bindir, "..") 78 self.host_output_dir = os.path.join( 79 self.host_build_dir, "plugins", "ets", "tests", "micro-benchmarks") 80 self.device_output_dir = os.path.join( 81 args.bindir, "..", "micro-benchmarks") if self.is_device else None 82 self.current_bench_name = None 83 self.source_dir = os.path.join( 84 SRC_PATH, "..", "..", "micro-benchmarks") 85 self.wrapper_asm_filepath = os.path.join( 86 self.source_dir, "wrapper", "test_wrapper.pa") 87 if self.is_device: 88 self.stdlib_path = os.path.join(args.libdir, "etsstdlib.abc") 89 else: 90 self.stdlib_path = os.path.join( 91 args.bindir, "..", "plugins", "ets", "etsstdlib.abc") 92 self.ark_asm = os.path.join(args.bindir, "ark_asm") 93 self.ark_aot = os.path.join(args.bindir, "ark_aot") 94 self.ark = os.path.join(args.bindir, "ark") 95 self.prefix = [ 96 "hdc", "shell", f"LD_LIBRARY_PATH={args.libdir}"] if self.is_device else [] 97 self.ark_opts = ark_opts 98 self.aot_opts = aot_opts 99 100 def dump_output_to_file(self, pipe, file_ext): 101 dumpfile = os.fdopen(os.open(os.path.join(self.host_output_dir, 102 self.current_bench_name, f"test.{file_ext}"), 103 os.O_RDWR | os.O_CREAT, 0o755), "w") 104 dumpfile.write(pipe.decode('ascii')) 105 dumpfile.close() 106 107 def dump_stdout(self, pipe, tp): 108 self.dump_output_to_file(pipe, f"{tp}_out") 109 110 def dump_stderr(self, pipe, tp): 111 self.dump_output_to_file(pipe, f"{tp}_err") 112 113 def compile_test(self, asm_filepath, bin_filepath): 114 cmd = self.prefix + [self.ark_asm, 115 str(asm_filepath), str(bin_filepath)] 116 proc = subprocess.Popen( 117 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 118 stdout, stderr = proc.communicate(timeout=5400) 119 self.dump_stdout(stdout, "asm") 120 if proc.returncode == 0: 121 return True 122 self.logger.debug(f"{self.current_bench_name} FAIL (compile)") 123 self.logger.debug("How to reproduce: " + " ".join(cmd) + "\n") 124 self.dump_stderr(stderr, "asm") 125 return False 126 127 def compile_aot_test(self, bin_filepath, aot_filepath): 128 cmd = self.prefix + [ 129 self.ark_aot, "--paoc-mode=aot", f"--boot-panda-files={self.stdlib_path}", 130 "--load-runtimes=ets", "--compiler-ignore-failures=false" 131 ] + self.aot_opts + [ 132 "--paoc-panda-files", bin_filepath, 133 "--paoc-output", aot_filepath 134 ] 135 proc = subprocess.Popen( 136 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 137 stdout, stderr = proc.communicate(timeout=5400) 138 self.dump_stdout(stdout, "aot") 139 if proc.returncode == 0: 140 return True 141 self.logger.debug(f"{self.current_bench_name} FAIL (aot)") 142 self.logger.debug("How to reproduce: " + " ".join(cmd) + "\n") 143 self.dump_stderr(stderr, "aot") 144 return False 145 146 def run_test(self, bin_filepath): 147 additional_opts = [] 148 if self.mode == "aot": 149 additional_opts += ["--aot-file", 150 bin_filepath.replace(".abc", ".an")] 151 # NOTE(ipetrov, #14164): return limit standard allocation after fix in taskmanager 152 cmd = self.prefix + [self.ark, f"--boot-panda-files={self.stdlib_path}", "--load-runtimes=ets", 153 "--compiler-ignore-failures=false"] \ 154 + self.ark_opts + additional_opts + \ 155 [bin_filepath, "_GLOBAL::main"] 156 proc = subprocess.Popen( 157 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 158 stdout, stderr = proc.communicate(timeout=5400) 159 self.dump_stdout(stdout, "ark") 160 if proc.returncode == 0: 161 self.logger.debug(f"{self.current_bench_name} PASS") 162 return True, float(stdout.decode('ascii').split("\n")[0]) 163 self.logger.debug(f"{self.current_bench_name} FAIL (execute)") 164 self.logger.debug("How to reproduce: " + " ".join(cmd) + "\n") 165 self.dump_stderr(stderr, "ark") 166 return False, 0 167 168 def generate_benchmark(self, filename): 169 base_asm_file_path = os.path.join(self.source_dir, filename) 170 current_output_dir = os.path.join( 171 self.host_output_dir, self.current_bench_name) 172 tmp_asm_file_path = os.path.join(current_output_dir, "test.pa") 173 subprocess.run(["mkdir", "-p", current_output_dir]) 174 fd_read = os.open(self.wrapper_asm_filepath, os.O_RDONLY, 0o755) 175 fd_read_two = os.open(base_asm_file_path, os.O_RDONLY, 0o755) 176 file_to_read = os.fdopen(fd_read, "r") 177 file_to_read_two = os.fdopen(fd_read_two, "r") 178 os.fdopen(os.open(tmp_asm_file_path, os.O_WRONLY | os.O_CREAT, 0o755), "w").write(file_to_read.read() + 179 file_to_read_two.read()) 180 181 if self.is_device: 182 device_current_output_dir = os.path.join( 183 self.device_output_dir, self.current_bench_name) 184 subprocess.run(["hdc", "shell", f"mkdir -p {device_current_output_dir}"]) 185 subprocess.run( 186 ["hdc", "file", "send", tmp_asm_file_path, f"{device_current_output_dir}/"]) 187 tmp_asm_file_path = os.path.join( 188 device_current_output_dir, "test.pa") 189 return tmp_asm_file_path 190 191 def run_separate_bench(self, filename, stats): 192 bench_name = Path(filename).stem 193 self.current_bench_name = f"{bench_name}_{self.mode}_{self.interpreter_type}" 194 195 tmp_asm_file_path = self.generate_benchmark(filename) 196 197 bin_file_path = tmp_asm_file_path.replace(".pa", ".abc") 198 res = self.compile_test(tmp_asm_file_path, bin_file_path) 199 if not res: 200 stats.record_time(bench_name, 0) 201 stats.record_failure_compile(bench_name) 202 return 203 if self.mode == "aot": 204 aot_file_path = tmp_asm_file_path.replace(".pa", ".an") 205 res = self.compile_aot_test(bin_file_path, aot_file_path) 206 if not res: 207 stats.record_time(bench_name, 0) 208 stats.record_failure_aot_compile(bench_name) 209 return 210 res, exec_time = self.run_test(bin_file_path) 211 stats.record_time(bench_name, exec_time) 212 if res: 213 stats.record_success(bench_name) 214 else: 215 stats.record_failure_exec(bench_name) 216 217 def run(self, args): 218 stats = RunStats() 219 220 if self.is_device: 221 subprocess.run(["hdc", "shell", f"mkdir -p {self.device_output_dir}"]) 222 else: 223 subprocess.run(["mkdir", "-p", self.host_output_dir]) 224 225 if args.test_name is not None: 226 self.run_separate_bench(args.test_name + ".pa", stats) 227 return stats 228 229 if args.test_list is not None: 230 with open(args.test_list, "r") as testlist_file: 231 testlist = [line.strip() for line in testlist_file] 232 for test_name in testlist: 233 self.run_separate_bench(test_name + ".pa", stats) 234 return stats 235 236 for dirpath, dirnames, filepathes in os.walk(self.source_dir): 237 for filepath in filepathes: 238 if Path(filepath).suffix != ".pa": 239 continue 240 self.run_separate_bench(filepath, stats) 241 break 242 return stats 243 244# ======================================================= 245# Helpers 246 247 248def dump_time_stats(logger, stats): 249 max_width = max([len(x) for x in stats.time.keys()]) 250 keys = list(stats.time.keys()) 251 keys.sort() 252 for name in keys: 253 if name in stats.passed: 254 logger.info(name.split(".")[0].ljust( 255 max_width + 5) + str(round(stats.time[name], 3))) 256 else: 257 logger.info(f"{name.split('.')[0].ljust(max_width + 5)}None") 258 259 260def dump_pass_rate(logger, skiplist_name, passed_tests, failed_tests): 261 if os.path.isfile(skiplist_name): 262 with open(skiplist_name, "r") as skiplist: 263 skipset = set([line.strip() for line in skiplist]) 264 else: 265 skipset = set() 266 new_failed = set(failed_tests) - skipset 267 new_passed = skipset.intersection(set(passed_tests)) - set(failed_tests) 268 if len(new_passed) != 0: 269 logger.info("\nNew passed:") 270 for name in list(new_passed): 271 logger.info(name) 272 if len(new_failed) != 0: 273 logger.info("\nNew failures:") 274 for name in list(new_failed): 275 logger.info(name) 276 return False 277 return True 278 279 280def parse_results(logger, stats, mode, int_type): 281 all_failed_tests = stats.failed_exec + \ 282 stats.failed_aot_compile + stats.failed_compile 283 if len(all_failed_tests) == 0: 284 return True 285 return dump_pass_rate(logger, f"skiplist_{mode}_{int_type}.txt", stats.passed, all_failed_tests) 286 287 288# ======================================================= 289# Entry 290 291parser = argparse.ArgumentParser(description='Run ETS micro-benchmarks.') 292parser.add_argument("--bindir", required=True, 293 help="Directory with compiled binaries (eg.: ark_asm, ark_aot, ark)") 294parser.add_argument("--sourcedir", 295 help="Directory with source asm files (for arm64)") 296parser.add_argument("--libdir", 297 help="Directory with etsstdlib (for arm64)") 298parser.add_argument("--host-builddir", 299 help="Host build directory (for arm64)") 300parser.add_argument("--mode", choices=["int", "jit", "aot"], default="int", 301 help="Running mode. Default: '%(default)s'") 302parser.add_argument("--interpreter-type", choices=["cpp", "irtoc", "llvm"], default="cpp", 303 help="Type of interpreter. Default: '%(default)s'") 304parser.add_argument("--target", choices=["host", "device"], default="host", 305 help="Launch target. Default: '%(default)s'") 306parser.add_argument("--aot-options", metavar="LIST", 307 help="Comma separated list of aot options for ARK_AOT") 308parser.add_argument("--runtime-options", metavar="LIST", 309 help="Comma separated list of runtime options for ARK") 310parser.add_argument("--test-name", 311 help="Run a specific benchmark.") 312parser.add_argument("--test-list", 313 help="List with benchmarks to be launched.") 314parser.add_argument("--log-level", choices=["silence", "info", "debug"], default="info", 315 help="Log level. Default: '%(default)s'") 316 317 318def main(): 319 args = parser.parse_args() 320 if args.test_list and not os.path.isfile(args.test_list): 321 logger.silence(f"ERROR: wrong path to testlist '{args.test_list}' ") 322 exit(1) 323 324 if args.target == "host" and not Path(args.bindir).is_dir(): 325 logger.silence(f"ERROR: Path '{args.bindir}' must be a directory") 326 exit(1) 327 328 # ======================================================= 329 # Prepare ARK and AOT options 330 331 ark_opts = [] 332 aot_opts = [] 333 if args.runtime_options: 334 ark_opts = ("--" + args.runtime_options.replace(" ", 335 "").replace(",", " --")).split(" ") 336 if args.aot_options: 337 aot_opts = ("--" + args.aot_options.replace(" ", 338 "").replace(",", " --")).split(" ") 339 if args.test_list and not os.path.isfile(args.test_list): 340 print_silence(f"ERROR: wrong path to testlist '{args.test_list}' ") 341 exit(1) 342 343 found_int_type_opt = False 344 found_jit_opt = False 345 for elem in ark_opts: 346 if elem.find("compiler-enable-jit=true") != -1: 347 args.mode = "jit" 348 found_jit_opt = True 349 if elem.find("interpreter-type") == -1: 350 continue 351 352 found_int_type_opt = True 353 if elem.find("interpreter-type=cpp") != -1: 354 args.interpreter_type = "cpp" 355 if elem.find("interpreter-type=irtoc") != -1: 356 args.interpreter_type = "irtoc" 357 if elem.find("interpreter-type=llvm") != -1: 358 args.interpreter_type = "llvm" 359 360 if not found_jit_opt: 361 ark_opts += ["--compiler-enable-jit=" + 362 ("true" if args.mode == "jit" else "false")] 363 if not found_int_type_opt: 364 ark_opts += [f"--interpreter-type={args.interpreter_type}"] 365 366 # ======================================================= 367 # Run benchmarks 368 369 logger = Logger(args) 370 runner = EtsBenchmarksRunner(args, logger, ark_opts, aot_opts) 371 372 stats = runner.run(args) 373 374 # ======================================================= 375 # Prepare and output results 376 377 if not stats: 378 logger.info("\nFAIL!") 379 exit(1) 380 381 all_tests_amount = len(stats.failed_exec) + len(stats.failed_aot_compile) + \ 382 len(stats.failed_compile) + len(stats.passed) 383 logger.info("\n=====") 384 if args.mode == "aot": 385 logger.info(f"TESTS {all_tests_amount} | " 386 f"FAILED (compile) {len(stats.failed_compile)} | " 387 f"FAILED (aot compile) {len(stats.failed_aot_compile)} | " 388 f"FAILED (exec) {len(stats.failed_exec)}\n") 389 else: 390 logger.info(f"TESTS {all_tests_amount} | " 391 f"FAILED (compile) {len(stats.failed_compile)} | " 392 f"FAILED (exec) {len(stats.failed_exec)}\n") 393 dump_time_stats(logger, stats) 394 395 if not parse_results(logger, stats, args.mode, args.interpreter_type): 396 logger.info("\nFAIL!") 397 exit(1) 398 else: 399 logger.info("\nSUCCESS!") 400 401 402if __name__ == "__main__": 403 main() 404