1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3 4# Copyright (c) 2024 Huawei Device Co., Ltd. 5# Licensed under the Apache License, Version 2.0 (the 'License'); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an 'AS IS' BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17 18import argparse 19import sys 20import logging 21import re 22from typing import Any, List, Dict, Set 23from pathlib import Path 24from vmb.helpers import StringEnum, split_params, read_list_file, die 25from vmb.tool import ToolMode, OptFlags 26 27LOG_LEVELS = ('fatal', 'pass', 'error', 'warn', 'info', 'debug', 'trace') 28log = logging.getLogger('vmb') 29 30 31class Command(StringEnum): 32 """VMB commands enum.""" 33 34 GEN = 'gen' 35 RUN = 'run' 36 REPORT = 'report' 37 ALL = 'all' 38 VERSION = 'version' 39 LIST = 'list' 40 41 42def comma_separated_list(arg_val: str) -> Set[str]: 43 if not arg_val: 44 return set() 45 return set(split_params(arg_val)) 46 47 48def add_measurement_opts(parser: argparse.ArgumentParser) -> None: 49 """Add options wich should be processed in @Benchmarks too.""" 50 parser.add_argument('-wi', '--warmup-iters', default=None, type=int, 51 help='Number of warmup iterations') 52 parser.add_argument('-mi', '--measure-iters', default=None, type=int, 53 help='Number of measurement iterations') 54 parser.add_argument('-it', '--iter-time', default=None, type=int, 55 help='Iteration time, sec') 56 parser.add_argument('-wt', '--warmup-time', default=None, type=int, 57 help='Warmup iteration time, sec') 58 parser.add_argument('-fi', '--fast-iters', default=None, type=int, 59 help='Number of fast iterations') 60 parser.add_argument('-gc', '--sys-gc-pause', default=None, type=int, 61 help='If <val> >= 0 invoke GC twice ' 62 'and wait <val> ms before iteration') 63 parser.add_argument("-aot-co", "--aot-compiler-options", default=[], 64 type=str, action="append", help="aot-compiler options") 65 parser.add_argument("-c", "--concurrency-level", 66 default=None, type=str, 67 help="Concurrency level (DEPRECATED)") 68 parser.add_argument("-compiler-inlning", "--compiler-inlining", 69 default=None, type=str, 70 help="enable compiler inlining") 71 72 73def add_gen_opts(parser: argparse.ArgumentParser, command: Command) -> None: 74 parser.add_argument('-l', '--langs', 75 type=comma_separated_list, 76 default=set(), 77 required=(command == Command.GEN), 78 help='Comma-separated list of lang plugins') 79 parser.add_argument('-o', '--outdir', default='generated', type=str, 80 help='Dir for generated benches') 81 parser.add_argument('-T', '--tags', 82 default=set(), 83 type=comma_separated_list, 84 help='Filter by tag (comma-separated list)') 85 parser.add_argument('-ST', '--skip-tags', 86 default=set(), 87 type=comma_separated_list, 88 help='Skip if tagged (comma-separated list)') 89 parser.add_argument('-t', '--tests', 90 default=set(), 91 type=comma_separated_list, 92 help='Filter by name (comma-separated list)') 93 parser.add_argument('-L', '--src-langs', 94 default=set(), 95 type=comma_separated_list, 96 help='Override src file extentions ' 97 '(comma-separated list)') 98 99 100def add_run_opts(parser: argparse.ArgumentParser) -> None: 101 parser.add_argument('-p', '--platform', type=str, 102 required=True, 103 help='Platform plugin name') 104 parser.add_argument('-m', '--mode', type=str, 105 default='default', 106 choices=ToolMode.getall(), 107 help='Run VM mode') 108 parser.add_argument('-n', '--name', type=str, 109 default='', help='Description of run') 110 parser.add_argument('--timeout', default=None, type=float, 111 help='Timeout (seconds)') 112 parser.add_argument('--device', type=str, 113 default='', help='Device ID (serial)') 114 parser.add_argument('--device-dir', type=str, 115 default='/data/local/tmp/vmb', 116 help='Base dir on device (%(default)s)') 117 parser.add_argument('--hooks', type=str, default='', 118 help='Comma-separated list of hook plugins') 119 parser.add_argument('-A', '--aot-skip-libs', action='store_true', 120 help='Skip AOT compilation of stdlib') 121 parser.add_argument('-g', '--enable-gc-logs', action='store_true', 122 help='Runs benchmark with GC logs enabled') 123 parser.add_argument('--dry-run', action='store_true', 124 help='Generate and compile, no execution') 125 parser.add_argument('--report-json', default='', type=str, 126 metavar='FILE_NAME', 127 help='Save json report as FILE_NAME') 128 parser.add_argument('--report-json-compact', action='store_true', 129 help='Json file without indentation') 130 parser.add_argument('--report-csv', default='', type=str, 131 metavar='FILE_NAME', 132 help='Save csv report as FILE_NAME') 133 parser.add_argument('--exclude-list', default='', type=str, 134 metavar='EXCLUDE_LIST', 135 help='Path to exclude list') 136 parser.add_argument('--fail-logs', default='', type=str, 137 metavar='FAIL_LOGS_DIR', 138 help='Save failure messages to folder') 139 parser.add_argument('--cpumask', default='', type=str, 140 help='Use cores mask. F.e. 11110000 = low cores') 141 parser.add_argument('--aot-stats', action='store_true', 142 help='Collect aot compilation data') 143 parser.add_argument('--jit-stats', action='store_true', 144 help='Collect jit compilation data') 145 parser.add_argument('--aot-whitelist', default='', type=str, 146 metavar='FILE_NAME', 147 help='Get methods names from FILE_NAME') 148 149 150def add_report_opts(parser: argparse.ArgumentParser) -> None: 151 """Add options specific to vmb report.""" 152 parser.add_argument('--full', action='store_true', 153 help='Produce full report') 154 parser.add_argument('--json', default='', type=str, 155 help='Save out as JSON') 156 parser.add_argument('--aot-stats-json', default='', type=str, 157 help='File path to save aot stats comparison') 158 parser.add_argument('--aot-passes-json', default='', type=str, 159 help='File path to save aot passes comparison') 160 parser.add_argument("--gc-stats-json", default='', type=str, 161 help='File path to save GC stats comparison') 162 parser.add_argument('--compare', action='store_true', 163 help='Compare 2 reports') 164 parser.add_argument('--flaky-list', default='', type=str, 165 help='Exclude list file') 166 parser.add_argument('--tolerance', default=0.5, type=float, 167 help='Percentage of tolerance in comparison') 168 169 170class _ArgumentParser(argparse.ArgumentParser): 171 172 def __init__(self, command: Command) -> None: 173 super().__init__( 174 prog=f'vmb {command.value}', 175 formatter_class=argparse.RawDescriptionHelpFormatter, 176 epilog=self.__epilog() 177 ) 178 # Options common for all commands 179 self.add_argument('paths', nargs='*', help='Dirs or files') 180 self.add_argument('-e', '--extra-plugins', default=None, 181 help='Path to extra plugins') 182 self.add_argument('-v', '--log-level', default='info', 183 choices=LOG_LEVELS, 184 help='Log level (default: %(default)s)') 185 self.add_argument('--abort-on-fail', action='store_true', 186 help='Abort run on first error') 187 self.add_argument('--no-color', action='store_true', 188 help='Disable color logging') 189 # Generator-specific options 190 if command in (Command.GEN, Command.ALL): 191 add_gen_opts(self, command) 192 add_measurement_opts(self) 193 # Runner-specific options 194 if command in (Command.RUN, Command.ALL): 195 add_run_opts(self) 196 if command in (Command.REPORT,): 197 add_report_opts(self) 198 199 @staticmethod 200 def __epilog() -> str: 201 return '' 202 203 204class Args(argparse.Namespace): 205 """Args parser for VMB.""" 206 207 def __init__(self) -> None: 208 super().__init__() 209 self.command = None 210 self.custom_opts: Dict[str, List[str]] = {} 211 if len(sys.argv) < 2 \ 212 or sys.argv[1] == 'help' \ 213 or sys.argv[1] not in Command.getall(): 214 print('Usage: vmb COMMAND [options] [paths]') 215 print(f' COMMAND {{{",".join(Command.getall())}}}') 216 for c in Command: 217 print('=' * 80) 218 try: 219 Args.print_help(c) 220 except SystemExit: 221 continue 222 print('=' * 80) 223 sys.exit(1) 224 self.command = Command(sys.argv[1]) 225 args = sys.argv[2:] 226 _, unknown = _ArgumentParser(self.command).parse_known_args( 227 args, namespace=self) 228 self.process_custom_opts(unknown) 229 # provide some defaults 230 self.paths: List[Path] = [Path(p) for p in self.paths] if self.paths \ 231 else [Path.cwd()] 232 self.src_langs = {f'.{x.lstrip(".")}' 233 for x in self.get('src_langs', []) if x} 234 # pylint: disable-next=access-member-before-definition 235 if self.extra_plugins: # type: ignore 236 # pylint: disable-next=access-member-before-definition 237 self.extra_plugins = Path(self.extra_plugins) # type: ignore 238 excl = self.get('exclude_list', None) 239 self.exclude_list = read_list_file(excl) if excl else [] 240 self.dry_run = self.get('dry_run', False) 241 self.fail_logs = self.get('fail_logs', '') 242 if self.fail_logs: 243 Path(self.fail_logs).mkdir(exist_ok=True) 244 245 def __repr__(self) -> str: 246 return '\n'.join(super().__repr__().split(',')) 247 248 @staticmethod 249 def print_help(cmd: Command) -> None: 250 """Print usage for VMB command.""" 251 _ArgumentParser(cmd).parse_args(['--help']) 252 253 def process_custom_opts(self, custom: List[str]) -> None: 254 re_custom = re.compile(r'^--(?P<tool>\w+)-custom-option=(?P<opt>.+)$') 255 for opt in custom: 256 m = re.search(re_custom, opt) 257 if m: 258 tool = m.group('tool') 259 custom_opt = m.group('opt') 260 if not self.custom_opts.get(tool): 261 self.custom_opts[tool] = [custom_opt] 262 else: 263 self.custom_opts[tool].append(custom_opt) 264 else: 265 die(not m, f'Unknown option: {opt}') 266 267 def get(self, arg: str, default=None) -> Any: 268 return vars(self).get(arg, default) 269 270 def get_opts_flags(self) -> OptFlags: 271 flags = OptFlags.NONE 272 mode = ToolMode(self.get('mode')) 273 if ToolMode.AOT == mode: 274 flags |= OptFlags.AOT 275 elif ToolMode.INT == mode: 276 flags |= OptFlags.INT 277 elif ToolMode.JIT == mode: 278 flags |= OptFlags.JIT 279 if self.get('dry_run', False): 280 flags |= OptFlags.DRY_RUN 281 if self.get('aot_skip_libs', False): 282 flags |= OptFlags.AOT_SKIP_LIBS 283 if self.get('enable_gc_logs', False): 284 flags |= OptFlags.GC_STATS 285 if self.get('aot_stats', False): 286 flags |= OptFlags.AOT_STATS 287 if self.get('jit_stats', False): 288 flags |= OptFlags.JIT_STATS 289 # backward compatibility 290 if 'false' == self.get('compiler_inlining', ''): 291 flags |= OptFlags.DISABLE_INLINING 292 return flags 293 294 def get_custom_opts(self, name: str) -> str: 295 opts = self.custom_opts.get(name, []) 296 return ' '.join(opts) 297 298 def get_shared_path(self) -> str: 299 path = self.get('outdir', '') 300 if path: 301 return path 302 return self.get('path', ['.'])[0] 303