1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3 4# Copyright (c) 2024-2025 Huawei Device Co., Ltd. 5# Licensed under the Apache License, Version 2.0 (the 'License'); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an 'AS IS' BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17 18import argparse 19import sys 20import logging 21import re 22from typing import Any, List, Dict, Set 23from pathlib import Path 24from vmb.helpers import StringEnum, split_params, read_list_file, die 25from vmb.tool import ToolMode, OptFlags 26 27LOG_LEVELS = ('fatal', 'pass', 'error', 'warn', 'info', 'debug', 'trace') 28log = logging.getLogger('vmb') 29 30 31class Command(StringEnum): 32 """VMB commands enum.""" 33 34 GEN = 'gen' 35 RUN = 'run' 36 REPORT = 'report' 37 ALL = 'all' 38 VERSION = 'version' 39 LIST = 'list' 40 41 42def comma_separated_list(arg_val: str) -> Set[str]: 43 if not arg_val: 44 return set() 45 return set(split_params(arg_val)) 46 47 48def add_measurement_opts(parser: argparse.ArgumentParser) -> None: 49 """Add options wich should be processed in @Benchmarks too.""" 50 parser.add_argument('-wi', '--warmup-iters', default=None, type=int, 51 help='Number of warmup iterations') 52 parser.add_argument('-mi', '--measure-iters', default=None, type=int, 53 help='Number of measurement iterations') 54 parser.add_argument('-it', '--iter-time', default=None, type=int, 55 help='Iteration time, sec') 56 parser.add_argument('-wt', '--warmup-time', default=None, type=int, 57 help='Warmup iteration time, sec') 58 parser.add_argument('-fi', '--fast-iters', default=None, type=int, 59 help='Number of fast iterations') 60 parser.add_argument('-gc', '--sys-gc-pause', default=None, type=int, 61 help='If <val> >= 0 invoke GC twice ' 62 'and wait <val> ms before iteration') 63 parser.add_argument("-aot-co", "--aot-compiler-options", default=[], 64 type=str, action="append", 65 help="Sets ahead-of-time compiler options") 66 parser.add_argument("-aot-lib-co", "--aot-lib-compiler-options", default=[], 67 type=str, action="append", 68 help="Sets ahead-of-time compiler options for libraries") 69 parser.add_argument("-c", "--concurrency-level", 70 default=None, type=str, 71 help="Concurrency level (DEPRECATED)") 72 parser.add_argument("-compiler-inlning", "--compiler-inlining", 73 default=None, type=str, 74 help="enable compiler inlining") 75 76 77def add_gen_opts(parser: argparse.ArgumentParser, command: Command) -> None: 78 parser.add_argument('-l', '--langs', 79 type=comma_separated_list, 80 default=set(), 81 required=(command == Command.GEN), 82 help='Comma-separated list of lang plugins') 83 parser.add_argument('-o', '--outdir', default='generated', type=str, 84 help='Dir for generated benches') 85 parser.add_argument('-t', '--tests', 86 default=set(), 87 type=comma_separated_list, 88 help='Filter by name (comma-separated list)') 89 parser.add_argument('-L', '--src-langs', 90 default=set(), 91 type=comma_separated_list, 92 help='Override src file extentions ' 93 '(comma-separated list)') 94 95 96def add_run_opts(parser: argparse.ArgumentParser) -> None: 97 parser.add_argument('-p', '--platform', type=str, required=True, 98 help='Platform plugin name') 99 parser.add_argument('-m', '--mode', type=str, 100 default='default', choices=ToolMode.getall(), help='Run VM mode') 101 parser.add_argument('-n', '--name', type=str, 102 default='', help='Description of run') 103 parser.add_argument('--timeout', default=None, type=float, help='Timeout (seconds)') 104 parser.add_argument('--device', type=str, default='', help='Device ID (serial)') 105 parser.add_argument('--device-host', type=str, default='', 106 help='device server in form server:port in case you use remote device') 107 parser.add_argument('--device-dir', type=str, default='/data/local/tmp/vmb', 108 help='Base dir on device (%(default)s)') 109 parser.add_argument('--hooks', type=str, default='', 110 help='Comma-separated list of hook plugins') 111 parser.add_argument('-A', '--aot-skip-libs', action='store_true', 112 help='Skip AOT compilation of stdlib') 113 parser.add_argument('-g', '--enable-gc-logs', action='store_true', 114 help='Runs benchmark with GC logs enabled') 115 parser.add_argument('--dry-run', action='store_true', 116 help='Generate and compile, no execution') 117 parser.add_argument('--report-json', default='', type=str, metavar='FILE_NAME', 118 help='Save json report as FILE_NAME') 119 parser.add_argument('--report-json-compact', action='store_true', 120 help='Json file without indentation') 121 parser.add_argument('--report-csv', default='', type=str, 122 metavar='FILE_NAME', help='Save csv report as FILE_NAME') 123 parser.add_argument('--exclude-list', default='', type=str, 124 metavar='EXCLUDE_LIST', help='Path to exclude list') 125 parser.add_argument('--fail-logs', default='', type=str, 126 metavar='FAIL_LOGS_DIR', help='Save failure messages to folder') 127 parser.add_argument('--cpumask', default='', type=str, 128 help='Use cores mask in hex or bin format. ' 129 'E.g., 0x38 or 0b111000 = high cores') 130 parser.add_argument('--aot-stats', action='store_true', 131 help='Collect aot compilation data') 132 parser.add_argument('--jit-stats', action='store_true', 133 help='Collect jit compilation data') 134 parser.add_argument('--aot-whitelist', default='', type=str, 135 metavar='FILE_NAME', help='Get methods names from FILE_NAME') 136 parser.add_argument('--tests-per-batch', default=25, type=int, 137 help='Test count per one batch run (%(default)s)') 138 139 140def add_report_opts(parser: argparse.ArgumentParser) -> None: 141 """Add options specific to vmb report.""" 142 parser.add_argument('--full', action='store_true', 143 help='Produce full report') 144 parser.add_argument('--json', default='', type=str, 145 help='Save out as JSON') 146 parser.add_argument('--aot-stats-json', default='', type=str, 147 help='File path to save aot stats comparison') 148 parser.add_argument('--aot-passes-json', default='', type=str, 149 help='File path to save aot passes comparison') 150 parser.add_argument("--gc-stats-json", default='', type=str, 151 help='File path to save GC stats comparison') 152 parser.add_argument('--compare', action='store_true', 153 help='Compare 2 reports') 154 parser.add_argument('--flaky-list', default='', type=str, 155 help='Exclude list file') 156 parser.add_argument('--tolerance', default=0.5, type=float, 157 help='Percentage of tolerance in comparison') 158 159 160def add_filter_opts(parser: argparse.ArgumentParser) -> None: 161 parser.add_argument('-T', '--tags', 162 default=set(), 163 type=comma_separated_list, 164 help='Filter by tag (comma-separated list)') 165 parser.add_argument('-ST', '--skip-tags', 166 default=set(), 167 type=comma_separated_list, 168 help='Skip if tagged (comma-separated list)') 169 170 171class _ArgumentParser(argparse.ArgumentParser): 172 173 def __init__(self, command: Command) -> None: 174 super().__init__( 175 prog=f'vmb {command.value}', 176 formatter_class=argparse.RawDescriptionHelpFormatter, 177 epilog=self.__epilog() 178 ) 179 # Options common for all commands 180 self.add_argument('paths', nargs='*', help='Dirs or files') 181 self.add_argument('-e', '--extra-plugins', default=None, 182 help='Path to extra plugins') 183 self.add_argument('-v', '--log-level', default='info', 184 choices=LOG_LEVELS, 185 help='Log level (default: %(default)s)') 186 self.add_argument('--abort-on-fail', action='store_true', 187 help='Abort run on first error') 188 self.add_argument('--no-color', action='store_true', 189 help='Disable color logging') 190 # Generator-specific options 191 if command in (Command.GEN, Command.ALL): 192 add_gen_opts(self, command) 193 add_measurement_opts(self) 194 add_filter_opts(self) 195 # Runner-specific options 196 if command in (Command.RUN, Command.ALL): 197 add_run_opts(self) 198 if command in (Command.REPORT,): 199 add_report_opts(self) 200 add_filter_opts(self) 201 202 @staticmethod 203 def __epilog() -> str: 204 return '' 205 206 207class Args(argparse.Namespace): 208 """Args parser for VMB.""" 209 210 def __init__(self) -> None: 211 super().__init__() 212 self.command = None 213 self.custom_opts: Dict[str, List[str]] = {} 214 if len(sys.argv) < 2 \ 215 or sys.argv[1] == 'help' \ 216 or sys.argv[1] not in Command.getall(): 217 print('Usage: vmb COMMAND [options] [paths]') 218 print(f' COMMAND {{{",".join(Command.getall())}}}') 219 for c in Command: 220 print('=' * 80) 221 try: 222 Args.print_help(c) 223 except SystemExit: 224 continue 225 print('=' * 80) 226 sys.exit(1) 227 self.command = Command(sys.argv[1]) 228 args = sys.argv[2:] 229 _, unknown = _ArgumentParser(self.command).parse_known_args( 230 args, namespace=self) 231 self.process_custom_opts(unknown) 232 # provide some defaults 233 self.paths: List[Path] = [Path(p) for p in self.paths] if self.paths \ 234 else [Path.cwd()] 235 self.src_langs = {f'.{x.lstrip(".")}' 236 for x in self.get('src_langs', []) if x} 237 # pylint: disable-next=access-member-before-definition 238 if self.extra_plugins: # type: ignore 239 # pylint: disable-next=access-member-before-definition 240 self.extra_plugins = Path(self.extra_plugins) # type: ignore 241 excl = self.get('exclude_list', None) 242 self.exclude_list = read_list_file(excl) if excl else [] 243 self.dry_run = self.get('dry_run', False) 244 self.fail_logs = self.get('fail_logs', '') 245 if self.fail_logs: 246 Path(self.fail_logs).mkdir(exist_ok=True) 247 248 def __repr__(self) -> str: 249 return '\n'.join(super().__repr__().split(',')) 250 251 @staticmethod 252 def print_help(cmd: Command) -> None: 253 """Print usage for VMB command.""" 254 _ArgumentParser(cmd).parse_args(['--help']) 255 256 def process_custom_opts(self, custom: List[str]) -> None: 257 re_custom = re.compile(r'^--(?P<tool>\w+)-custom-option=(?P<opt>.+)$') 258 for opt in custom: 259 m = re.search(re_custom, opt) 260 if m: 261 tool = m.group('tool') 262 custom_opt = m.group('opt') 263 if not self.custom_opts.get(tool): 264 self.custom_opts[tool] = [custom_opt] 265 else: 266 self.custom_opts[tool].append(custom_opt) 267 else: 268 die(not m, f'Unknown option: {opt}') 269 270 def get(self, arg: str, default=None) -> Any: 271 return vars(self).get(arg, default) 272 273 def get_opts_flags(self) -> OptFlags: 274 flags = OptFlags.NONE 275 mode = ToolMode(self.get('mode')) 276 if ToolMode.AOT == mode: 277 flags |= OptFlags.AOT 278 elif ToolMode.AOTPGO == mode: 279 flags |= OptFlags.AOTPGO 280 elif ToolMode.LLVMAOT == mode: 281 flags |= OptFlags.AOT | OptFlags.LLVMAOT 282 elif ToolMode.INT == mode: 283 flags |= OptFlags.INT 284 elif ToolMode.INT_CPP == mode: 285 flags |= OptFlags.INT | OptFlags.INT_CPP 286 elif ToolMode.INT_IRTOC == mode: 287 flags |= OptFlags.INT | OptFlags.INT_IRTOC 288 elif ToolMode.INT_LLVM == mode: 289 flags |= OptFlags.INT | OptFlags.INT_LLVM 290 elif ToolMode.JIT == mode: 291 flags |= OptFlags.JIT 292 if self.get('dry_run', False): 293 flags |= OptFlags.DRY_RUN 294 if self.get('aot_skip_libs', False): 295 flags |= OptFlags.AOT_SKIP_LIBS 296 if self.get('enable_gc_logs', False): 297 flags |= OptFlags.GC_STATS 298 if self.get('aot_stats', False): 299 flags |= OptFlags.AOT_STATS 300 if self.get('jit_stats', False): 301 flags |= OptFlags.JIT_STATS 302 # backward compatibility 303 if 'false' == self.get('compiler_inlining', ''): 304 flags |= OptFlags.DISABLE_INLINING 305 return flags 306 307 def get_shared_path(self) -> str: 308 path = self.get('outdir', '') 309 if path: 310 return path 311 return self.get('path', ['.'])[0] 312