1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3 4""" 5Copyright (c) 2021 Huawei Device Co., Ltd. 6Licensed under the Apache License, Version 2.0 (the "License"); 7you may not use this file except in compliance with the License. 8You may obtain a copy of the License at 9 10 http://www.apache.org/licenses/LICENSE-2.0 11 12Unless required by applicable law or agreed to in writing, software 13distributed under the License is distributed on an "AS IS" BASIS, 14WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15See the License for the specific language governing permissions and 16limitations under the License. 17 18Description: Use ark to execute test 262 test suite 19""" 20 21import argparse 22import datetime 23import collections 24import json 25import os 26import sys 27import subprocess 28from multiprocessing import Pool 29import platform 30from utils import * 31from config import * 32 33 34def parse_args(): 35 parser = argparse.ArgumentParser() 36 parser.add_argument('--dir', metavar='DIR', 37 help='Directory to test ') 38 parser.add_argument('--file', metavar='FILE', 39 help='File to test') 40 parser.add_argument('--mode', 41 nargs='?', choices=[1, 2, 3], type=int, 42 help='selection information as: ' + 43 '1: only default \n ' + 44 '2: only strict mode \n' + 45 '3: both default and strict mode\n') 46 parser.add_argument('--es51', action='store_true', 47 help='Run test262 ES5.1 version') 48 parser.add_argument('--es2021', default=False, const='all', 49 nargs='?', choices=['all', 'only'], 50 help='Run test262 - ES2021. ' + 51 'all: Contains all use cases for es5_tests and es2015_tests and es2021_tests and intl_tests' + 52 'only: Only include use cases for ES2021') 53 parser.add_argument('--intl', default=False, const='intl', 54 nargs='?', choices=['intl'], 55 help='Run test262 - Intltest. ' + 56 'intl: Only include use cases for intlcsae') 57 parser.add_argument('--es2015', default=False, const='es2015', 58 nargs='?', choices=['es2015'], 59 help='Run test262 - es2015. ' + 60 'es2015: Only include use cases for es2015') 61 parser.add_argument('--ci-build', action='store_true', 62 help='Run test262 ES2015 filter cases for build version') 63 parser.add_argument('--esnext', action='store_true', 64 help='Run test262 - ES.next.') 65 parser.add_argument('--engine', metavar='FILE', 66 help='Other engine binarys to run tests(as:d8,qjs...)') 67 parser.add_argument('--babel', action='store_true', 68 help='Whether to use Babel conversion') 69 parser.add_argument('--timeout', default=DEFAULT_TIMEOUT, type=int, 70 help='Set a custom test timeout in milliseconds !!!\n') 71 parser.add_argument('--threads', default=DEFAULT_THREADS, type=int, 72 help="Run this many tests in parallel.") 73 parser.add_argument('--hostArgs', 74 help="command-line arguments to pass to eshost host\n") 75 parser.add_argument('--ark-tool', 76 help="ark's binary tool") 77 parser.add_argument('--ark-aot', action='store_true', 78 help="Run test262 with aot") 79 parser.add_argument('--ark-aot-tool', 80 help="ark's aot tool") 81 parser.add_argument("--libs-dir", 82 help="The path collection of dependent so has been divided by':'") 83 parser.add_argument('--ark-frontend', 84 nargs='?', choices=ARK_FRONTEND_LIST, type=str, 85 help="Choose one of them") 86 parser.add_argument('--ark-frontend-binary', 87 help="ark frontend conversion binary tool") 88 parser.add_argument('--ark-arch', 89 default=DEFAULT_ARK_ARCH, 90 nargs='?', choices=ARK_ARCH_LIST, type=str, 91 help="Choose one of them") 92 parser.add_argument('--ark-arch-root', 93 default=DEFAULT_ARK_ARCH, 94 help="the root path for qemu-aarch64 or qemu-arm") 95 parser.add_argument('--opt-level', 96 default=DEFAULT_OPT_LEVEL, 97 help="the opt level for es2abc") 98 parser.add_argument('--es2abc-thread-count', 99 default=DEFAULT_ES2ABC_THREAD_COUNT, 100 help="the thread count for es2abc") 101 parser.add_argument('--merge-abc-binary', 102 help="frontend merge abc binary tool") 103 parser.add_argument('--merge-abc-mode', 104 help="run test for merge abc mode") 105 return parser.parse_args() 106 107 108def run_check(runnable, env=None): 109 report_command('Test command:', runnable, env=env) 110 111 if env is not None: 112 full_env = dict(os.environ) 113 full_env.update(env) 114 env = full_env 115 116 proc = subprocess.Popen(runnable, env=env) 117 proc.wait() 118 return proc.returncode 119 120 121def excuting_npm_install(args): 122 ark_frontend = DEFAULT_ARK_FRONTEND 123 if args.ark_frontend: 124 ark_frontend = args.ark_frontend 125 126 if ark_frontend != ARK_FRONTEND_LIST[0]: 127 return 128 129 ark_frontend_binary = os.path.join(ARK_FRONTEND_BINARY_LIST[0]) 130 if args.ark_frontend_binary: 131 ark_frontend_binary = os.path.join(args.ark_frontend_binary) 132 133 ts2abc_build_dir = os.path.join(os.path.dirname( 134 os.path.realpath(ark_frontend_binary)), "..") 135 136 if not os.path.exists(os.path.join(ts2abc_build_dir, "package.json")) and \ 137 not os.path.exists(os.path.join(ts2abc_build_dir, "..", "package.json")): 138 return 139 140 if os.path.exists(os.path.join(ts2abc_build_dir, "..", "package.json")) and \ 141 not os.path.exists(os.path.join(ts2abc_build_dir, "package.json")): 142 ts2abc_build_dir = os.path.join(ts2abc_build_dir, "..") 143 # copy deps/ohos-typescript 144 deps_dir = os.path.join(ts2abc_build_dir, "deps") 145 mkdir(deps_dir) 146 if platform.system() == "Windows" : 147 proc = subprocess.Popen("cp %s %s" % (OHOS_TYPESCRIPT_TGZ_PATH, os.path.join(deps_dir, OHOS_TYPESCRIPT))) 148 proc.wait() 149 else : 150 subprocess.getstatusoutput("cp %s %s" % (OHOS_TYPESCRIPT_TGZ_PATH, os.path.join(deps_dir, OHOS_TYPESCRIPT))) 151 152 npm_install(ts2abc_build_dir) 153 154 155def init(args): 156 remove_dir(BASE_OUT_DIR) 157 remove_dir(TEST_ES5_DIR) 158 remove_dir(TEST_ES2015_DIR) 159 remove_dir(TEST_INTL_DIR) 160 remove_dir(TEST_ES2021_DIR) 161 remove_dir(TEST_CI_DIR) 162 get_all_skip_tests(args) 163 excuting_npm_install(args) 164 165 166def get_all_skip_tests(args): 167 # !!! plz correct the condition when changing the default frontend 168 if args.ark_frontend and args.ark_frontend == ARK_FRONTEND_LIST[1]: 169 SKIP_LIST_FILES.append(ES2ABC_SKIP_LIST_FILE) 170 else: 171 SKIP_LIST_FILES.append(TS2ABC_SKIP_LIST_FILE) 172 173 for file in SKIP_LIST_FILES: 174 with open(file) as jsonfile: 175 json_data = json.load(jsonfile) 176 for key in json_data: 177 ALL_SKIP_TESTS.extend(key["files"]) 178 179 180def collect_files(path): 181 if os.path.isfile(path): 182 yield path 183 return 184 185 if not os.path.isdir(path): 186 raise ValueError(f'Not found: "{path}"') 187 188 for root, _, file_names in os.walk(path): 189 for file_name in file_names: 190 if file_name.startswith('.') or not file_name.endswith(".js"): 191 continue 192 193 yield os.path.join(root, file_name) 194 195 196def mkdstdir(file, src_dir, dist_dir): 197 idx = file.rfind(src_dir) 198 if idx == -1: 199 raise SystemExit(f'{file} can not found in {src_dir}') 200 201 fpath, fname = os.path.split(file[idx:]) 202 fpath = fpath.replace(src_dir, dist_dir) 203 mkdir(fpath) 204 205 206class TestPrepare(): 207 def __init__(self, args): 208 self.args = args 209 self.out_dir = BASE_OUT_DIR 210 211 212 def prepare_test262_code(self): 213 if not os.path.isdir(os.path.join(DATA_DIR, '.git')): 214 git_clone(TEST262_GIT_URL, DATA_DIR) 215 git_checkout(TEST262_GIT_HASH, DATA_DIR) 216 217 if not os.path.isdir(os.path.join(ESHOST_DIR, '.git')): 218 git_clone(ESHOST_GIT_URL, ESHOST_DIR) 219 git_checkout(ESHOST_GIT_HASH, ESHOST_DIR) 220 git_apply('../eshost.patch', ESHOST_DIR) 221 222 npm_install(ESHOST_DIR) 223 224 if not os.path.isdir(os.path.join(HARNESS_DIR, '.git')): 225 git_clone(HARNESS_GIT_URL, HARNESS_DIR) 226 git_checkout(HARNESS_GIT_HASH, HARNESS_DIR) 227 git_apply('../harness.patch', HARNESS_DIR) 228 229 npm_install(HARNESS_DIR) 230 231 def prepare_clean_data(self): 232 git_clean(DATA_DIR) 233 git_checkout(TEST262_GIT_HASH, DATA_DIR) 234 235 def patching_the_plugin(self): 236 remove_file(os.path.join(ESHOST_DIR, "lib/agents/panda.js")) 237 remove_file(os.path.join(ESHOST_DIR, "runtimes/panda.js")) 238 239 git_clean(ESHOST_DIR) 240 git_apply("../eshost.patch", ESHOST_DIR) 241 git_clean(HARNESS_DIR) 242 git_apply("../harness.patch", HARNESS_DIR) 243 244 def prepare_args_es51_es2021(self): 245 if self.args.dir: 246 if TEST_ES5_DIR in self.args.dir: 247 self.args.es51 = True 248 elif TEST_ES2015_DIR in self.args.dir: 249 self.args.es2015 = "es2015" 250 elif TEST_INTL_DIR in self.args.dir: 251 self.args.intl = "intl" 252 elif TEST_ES2021_DIR in self.args.dir: 253 self.args.es2021 = "all" 254 255 if self.args.file: 256 if TEST_ES5_DIR in self.args.file: 257 self.args.es51 = True 258 elif TEST_ES2015_DIR in self.args.file: 259 self.args.es2015 = "es2015" 260 elif TEST_INTL_DIR in self.args.file: 261 self.args.intl = "intl" 262 elif TEST_ES2021_DIR in self.args.file: 263 self.args.es2021 = "all" 264 265 def prepare_out_dir(self): 266 if self.args.es51: 267 self.out_dir = os.path.join(BASE_OUT_DIR, "test_es51") 268 elif self.args.es2015: 269 self.out_dir = os.path.join(BASE_OUT_DIR, "test_es2015") 270 elif self.args.intl: 271 self.out_dir = os.path.join(BASE_OUT_DIR, "test_intl") 272 elif self.args.es2021: 273 self.out_dir = os.path.join(BASE_OUT_DIR, "test_es2021") 274 elif self.args.ci_build: 275 self.out_dir = os.path.join(BASE_OUT_DIR, "test_CI") 276 else: 277 self.out_dir = os.path.join(BASE_OUT_DIR, "test") 278 279 def prepare_args_testdir(self): 280 if self.args.dir: 281 return 282 283 if self.args.es51: 284 self.args.dir = TEST_ES5_DIR 285 elif self.args.es2015: 286 self.args.dir = TEST_ES2015_DIR 287 elif self.args.intl: 288 self.args.dir = TEST_INTL_DIR 289 elif self.args.es2021: 290 self.args.dir = TEST_ES2021_DIR 291 elif self.args.ci_build: 292 self.args.dir = TEST_CI_DIR 293 else: 294 self.args.dir = os.path.join(DATA_DIR, "test") 295 296 def copyfile(self, file, all_skips): 297 dstdir = os.path.join(DATA_DIR, "test") 298 file = file.strip() 299 file = file.strip('\n') 300 file = file.replace("\\", "/") 301 if file in all_skips: 302 return 303 304 srcdir = os.path.join(DATA_DIR, "test", file) 305 if self.args.es51: 306 dstdir = os.path.join(TEST_ES5_DIR, file) 307 elif self.args.es2015: 308 dstdir = os.path.join(TEST_ES2015_DIR, file) 309 elif self.args.intl: 310 dstdir = os.path.join(TEST_INTL_DIR, file) 311 elif self.args.es2021: 312 dstdir = os.path.join(TEST_ES2021_DIR, file) 313 elif self.args.ci_build: 314 dstdir = os.path.join(TEST_CI_DIR, file) 315 if platform.system() == "Windows" : 316 proc = subprocess.Popen("cp %s %s" % (srcdir, dstdir)) 317 proc.wait() 318 else : 319 subprocess.getstatusoutput("cp %s %s" % (srcdir, dstdir)) 320 321 322 def collect_tests(self): 323 files = [] 324 origin_dir = os.path.join(DATA_DIR, "test/") 325 file_names = collect_files(origin_dir) 326 esid = "" 327 if self.args.es51: 328 esid = "es5id" 329 elif self.args.es2021: 330 esid = "es6id" 331 332 for file_name in file_names: 333 with open(file_name, 'r', encoding='utf-8') as file: 334 content = file.read() 335 if esid in content: 336 files.append(file_name.split(origin_dir)[1]) 337 return files 338 339 def get_tests_from_file(self, file): 340 with open(file) as fopen: 341 files = fopen.readlines() 342 return files 343 344 def prepare_es2021_tests(self): 345 files = [] 346 files = self.collect_tests() 347 files.extend(self.get_tests_from_file(ES2021_LIST_FILE)) 348 if self.args.es2021 == "all": 349 files.extend(self.get_tests_from_file(ES5_LIST_FILE)) 350 files.extend(self.get_tests_from_file(INTL_LIST_FILE)) 351 files.extend(self.get_tests_from_file(ES2015_LIST_FILE)) 352 return files 353 354 def prepare_intl_tests(self): 355 files = [] 356 files = self.collect_tests() 357 if self.args.intl: 358 files = self.get_tests_from_file(INTL_LIST_FILE) 359 return files 360 361 def prepare_es2015_tests(self): 362 files = [] 363 files = self.collect_tests() 364 if self.args.es2015: 365 files = self.get_tests_from_file(ES2015_LIST_FILE) 366 return files 367 368 def prepare_test_suit(self): 369 files = [] 370 test_dir = "" 371 if self.args.es51: 372 test_dir = TEST_ES5_DIR 373 files = self.get_tests_from_file(ES5_LIST_FILE) 374 elif self.args.es2015: 375 test_dir = TEST_ES2015_DIR 376 files = self.prepare_es2015_tests() 377 elif self.args.intl: 378 test_dir = TEST_INTL_DIR 379 files = self.prepare_intl_tests() 380 elif self.args.es2021: 381 test_dir = TEST_ES2021_DIR 382 files = self.prepare_es2021_tests() 383 elif self.args.ci_build: 384 test_dir = TEST_CI_DIR 385 files = self.get_tests_from_file(CI_LIST_FILE) 386 387 for file in files: 388 path = os.path.split(file)[0] 389 path = os.path.join(test_dir, path) 390 mkdir(path) 391 392 pool = Pool(DEFAULT_THREADS) 393 for it in files: 394 pool.apply(self.copyfile, (it, ALL_SKIP_TESTS)) 395 pool.close() 396 pool.join() 397 398 def prepare_test262_test(self): 399 src_dir = TEST_FULL_DIR 400 if self.args.es51: 401 self.prepare_test_suit() 402 src_dir = TEST_ES5_DIR 403 elif self.args.es2015: 404 self.prepare_test_suit() 405 src_dir = TEST_ES2015_DIR 406 elif self.args.intl: 407 self.prepare_test_suit() 408 src_dir = TEST_INTL_DIR 409 elif self.args.es2021: 410 self.prepare_test_suit() 411 src_dir = TEST_ES2021_DIR 412 elif self.args.ci_build: 413 self.prepare_test_suit() 414 src_dir = TEST_CI_DIR 415 elif self.args.esnext: 416 git_checkout(ESNEXT_GIT_HASH, DATA_DIR) 417 else: 418 git_checkout(TEST262_GIT_HASH, DATA_DIR) 419 420 if self.args.file: 421 mkdstdir(self.args.file, src_dir, self.out_dir) 422 return 423 424 files = collect_files(self.args.dir) 425 for file in files: 426 mkdstdir(file, src_dir, self.out_dir) 427 428 def run(self): 429 self.prepare_test262_code() 430 self.prepare_clean_data() 431 self.patching_the_plugin() 432 self.prepare_args_es51_es2021() 433 self.prepare_out_dir() 434 self.prepare_args_testdir() 435 self.prepare_test262_test() 436 437 438def run_test262_prepare(args): 439 init(args) 440 441 test_prepare = TestPrepare(args) 442 test_prepare.run() 443 444 445def modetype_to_string(mode): 446 if mode == 1: 447 return "only default" 448 if mode == 2: 449 return "only strict mode" 450 return "both default and strict mode" 451 452 453def run_test262_mode(args): 454 if args.mode: 455 return modetype_to_string(args.mode) 456 return modetype_to_string(DEFAULT_MODE) 457 458 459def get_execute_arg(args): 460 execute_args = "" 461 462 if args.file: 463 execute_args = args.file 464 else: 465 execute_args = os.path.join(args.dir, "**", "*.js") 466 return execute_args 467 468 469def get_host_path_type(args): 470 host_path = DEFAULT_HOST_PATH 471 host_type = DEFAULT_HOST_TYPE 472 if args.engine: 473 host_path = args.engine 474 host_type = os.path.split(args.engine.strip())[1] 475 return host_path, host_type 476 477 478def get_timeout(args, threads): 479 timeout = DEFAULT_TIMEOUT * threads 480 if args.timeout: 481 timeout = args.timeout 482 return timeout 483 484 485def get_threads(args): 486 threads = DEFAULT_THREADS 487 if args.threads: 488 threads = args.threads 489 return threads 490 491 492def get_host_args(args, host_type): 493 host_args = "" 494 ark_tool = DEFAULT_ARK_TOOL 495 ark_aot_tool = DEFAULT_ARK_AOT_TOOL 496 libs_dir = DEFAULT_LIBS_DIR 497 ark_frontend = DEFAULT_ARK_FRONTEND 498 ark_frontend_binary = DEFAULT_ARK_FRONTEND_BINARY 499 ark_arch = DEFAULT_ARK_ARCH 500 opt_level = DEFAULT_OPT_LEVEL 501 es2abc_thread_count = DEFAULT_ES2ABC_THREAD_COUNT 502 merge_abc_binary = DEFAULT_MERGE_ABC_BINARY 503 merge_abc_mode = DEFAULT_MERGE_ABC_MODE 504 505 if args.hostArgs: 506 host_args = args.hostArgs 507 508 if args.ark_tool: 509 ark_tool = args.ark_tool 510 511 if args.ark_aot_tool: 512 ark_aot_tool = args.ark_aot_tool 513 514 if args.libs_dir: 515 libs_dir = args.libs_dir 516 517 if args.ark_frontend: 518 ark_frontend = args.ark_frontend 519 520 if args.ark_frontend_binary: 521 ark_frontend_binary = args.ark_frontend_binary 522 523 if args.opt_level: 524 opt_level = args.opt_level 525 526 if args.es2abc_thread_count: 527 es2abc_thread_count = args.es2abc_thread_count 528 529 if args.merge_abc_binary: 530 merge_abc_binary = args.merge_abc_binary 531 532 if args.merge_abc_mode: 533 merge_abc_mode = args.merge_abc_mode 534 535 if host_type == DEFAULT_HOST_TYPE: 536 host_args = f"-B test262/run_sunspider.py " 537 host_args += f"--ark-tool={ark_tool} " 538 if args.ark_aot: 539 host_args += f"--ark-aot " 540 host_args += f"--ark-aot-tool={ark_aot_tool} " 541 host_args += f"--libs-dir={libs_dir} " 542 host_args += f"--ark-frontend={ark_frontend} " 543 host_args += f"--ark-frontend-binary={ark_frontend_binary} " 544 host_args += f"--opt-level={opt_level} " 545 host_args += f"--es2abc-thread-count={es2abc_thread_count} " 546 host_args += f"--merge-abc-binary={merge_abc_binary} " 547 host_args += f"--merge-abc-mode={merge_abc_mode} " 548 549 if args.ark_arch != ark_arch: 550 host_args += f"--ark-arch={args.ark_arch} " 551 host_args += f"--ark-arch-root={args.ark_arch_root} " 552 553 return host_args 554 555 556def run_test262_test(args): 557 execute_args = get_execute_arg(args) 558 host_path, host_type = get_host_path_type(args) 559 host_args = get_host_args(args, host_type) 560 threads = get_threads(args) 561 timeout = get_timeout(args, threads) 562 563 test_cmd = ["node", TEST262_RUNNER_SCRIPT] 564 test_cmd.append(f"--hostType={host_type}") 565 test_cmd.append(f"--hostPath={host_path}") 566 if host_args != "": 567 test_cmd.append(f"--hostArgs='{host_args}'") 568 test_cmd.append(f"--threads={threads}") 569 test_cmd.append(f"--mode={run_test262_mode(args)}") 570 test_cmd.append(f"--timeout={timeout}") 571 if platform.system() == "Windows" : 572 global BASE_OUT_DIR 573 global DATA_DIR 574 BASE_OUT_DIR = BASE_OUT_DIR.replace("/","\\") 575 DATA_DIR = DATA_DIR.replace("/","\\") 576 execute_args = execute_args.replace("/","\\") 577 test_cmd.append(f"--tempDir={BASE_OUT_DIR}") 578 test_cmd.append(f"--test262Dir={DATA_DIR}") 579 580 if args.babel: 581 test_cmd.append("--preprocessor='test262/babel-preprocessor.js'") 582 test_cmd.append(DEFAULT_OTHER_ARGS) 583 test_cmd.append(execute_args) 584 585 run_check(test_cmd) 586 587 588Check = collections.namedtuple('Check', ['enabled', 'runner', 'arg']) 589 590 591def main(args): 592 print("\nWait a moment..........\n") 593 starttime = datetime.datetime.now() 594 run_test262_prepare(args) 595 check = Check(True, run_test262_test, args) 596 ret = check.runner(check.arg) 597 if ret: 598 sys.exit(ret) 599 endtime = datetime.datetime.now() 600 print(f"used time is: {str(endtime - starttime)}") 601 602 603if __name__ == "__main__": 604 sys.exit(main(parse_args())) 605