1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# 4# Copyright (c) 2021-2022 Huawei Device Co., Ltd. 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17from glob import glob 18from os import path 19from enum import Enum 20import argparse 21import fnmatch 22import multiprocessing 23import os 24import re 25import shutil 26import subprocess 27import sys 28import test262util 29 30 31def is_directory(parser, arg): 32 if not path.isdir(arg): 33 parser.error("The directory '%s' does not exist" % arg) 34 35 return path.abspath(arg) 36 37 38def is_file(parser, arg): 39 if not path.isfile(arg): 40 parser.error("The file '%s' does not exist" % arg) 41 42 return path.abspath(arg) 43 44def prepare_tsc_testcases(test_root): 45 third_party_tsc = path.join(test_root, "TypeScript") 46 ohos_third_party_tsc = path.join(test_root, "../../../../third_party/typescript") 47 48 if not path.isdir(third_party_tsc): 49 if (path.isdir(ohos_third_party_tsc)): 50 return path.abspath(ohos_third_party_tsc) 51 subprocess.run( 52 f"git clone https://gitee.com/openharmony/third_party_typescript.git {third_party_tsc}", 53 shell=True, 54 stdout=subprocess.DEVNULL, 55 ) 56 else: 57 subprocess.run( 58 f"cd {third_party_tsc} && git clean -f > /dev/null 2>&1", 59 shell=True, 60 stdout=subprocess.DEVNULL, 61 ) 62 return third_party_tsc 63 64def check_timeout(value): 65 ivalue = int(value) 66 if ivalue <= 0: 67 raise argparse.ArgumentTypeError( 68 "%s is an invalid timeout value" % value) 69 return ivalue 70 71 72def get_args(): 73 parser = argparse.ArgumentParser(description="Regression test runner") 74 parser.add_argument( 75 'build_dir', type=lambda arg: is_directory(parser, arg), 76 help='panda build directory') 77 parser.add_argument( 78 '--test262', '-t', action='store_true', dest='test262', default=False, 79 help='run test262 tests') 80 parser.add_argument( 81 '--error', action='store_true', dest='error', default=False, 82 help='capture stderr') 83 parser.add_argument( 84 '--regression', '-r', action='store_true', dest='regression', 85 default=False, help='run regression tests') 86 parser.add_argument( 87 '--compiler', '-c', action='store_true', dest='compiler', 88 default=False, help='run compiler tests') 89 parser.add_argument( 90 '--tsc', action='store_true', dest='tsc', 91 default=False, help='run tsc tests') 92 parser.add_argument( 93 '--type-extractor', action='store_true', dest='type_extractor', 94 default=False, help='run type extractor tests') 95 parser.add_argument( 96 '--no-progress', action='store_false', dest='progress', default=True, 97 help='don\'t show progress bar') 98 parser.add_argument( 99 '--no-skip', action='store_false', dest='skip', default=True, 100 help='don\'t use skiplists') 101 parser.add_argument( 102 '--update', action='store_true', dest='update', default=False, 103 help='update skiplist') 104 parser.add_argument( 105 '--no-run-gc-in-place', action='store_true', dest='no_gip', default=False, 106 help='enable --run-gc-in-place mode') 107 parser.add_argument( 108 '--filter', '-f', action='store', dest='filter', 109 default="*", help='test filter regexp') 110 parser.add_argument( 111 '--es2panda-timeout', type=check_timeout, 112 dest='es2panda_timeout', default=60, help='es2panda translator timeout') 113 parser.add_argument( 114 '--paoc-timeout', type=check_timeout, 115 dest='paoc_timeout', default=600, help='paoc compiler timeout') 116 parser.add_argument( 117 '--timeout', type=check_timeout, 118 dest='timeout', default=10, help='JS runtime timeout') 119 parser.add_argument( 120 '--gc-type', dest='gc_type', default="g1-gc", help='Type of garbage collector') 121 parser.add_argument( 122 '--aot', action='store_true', dest='aot', default=False, 123 help='use AOT compilation') 124 parser.add_argument( 125 '--no-bco', action='store_false', dest='bco', default=True, 126 help='disable bytecodeopt') 127 parser.add_argument( 128 '--jit', action='store_true', dest='jit', default=False, 129 help='use JIT in interpreter') 130 parser.add_argument( 131 '--arm64-compiler-skip', action='store_true', dest='arm64_compiler_skip', default=False, 132 help='use skiplist for tests failing on aarch64 in AOT or JIT mode') 133 parser.add_argument( 134 '--arm64-qemu', action='store_true', dest='arm64_qemu', default=False, 135 help='launch all binaries in qemu aarch64') 136 parser.add_argument( 137 '--arm32-qemu', action='store_true', dest='arm32_qemu', default=False, 138 help='launch all binaries in qemu arm') 139 parser.add_argument( 140 '--test-list', dest='test_list', default=None, type=lambda arg: is_file(parser, arg), 141 help='run tests listed in file') 142 parser.add_argument( 143 '--aot-args', action='append', dest='aot_args', default=[], 144 help='Additional arguments that will passed to ark_aot') 145 parser.add_argument( 146 '--verbose', '-v', action='store_true', dest='verbose', default=False, 147 help='Enable verbose output') 148 parser.add_argument( 149 '--js-runtime', dest='js_runtime_path', default=None, type=lambda arg: is_directory(parser, arg), 150 help='the path of js vm runtime') 151 parser.add_argument( 152 '--LD_LIBRARY_PATH', dest='ld_library_path', default=None, help='LD_LIBRARY_PATH') 153 parser.add_argument( 154 '--tsc-path', dest='tsc_path', default=None, type=lambda arg: is_directory(parser, arg), 155 help='the path of tsc') 156 parser.add_argument('--hotfix', dest='hotfix', action='store_true', default=False, 157 help='run hotfix tests') 158 parser.add_argument('--hotreload', dest='hotreload', action='store_true', default=False, 159 help='run hotreload tests') 160 parser.add_argument('--coldfix', dest='coldfix', action='store_true', default=False, 161 help='run coldfix tests') 162 parser.add_argument('--base64', dest='base64', action='store_true', default=False, 163 help='run base64 tests') 164 parser.add_argument('--bytecode', dest='bytecode', action='store_true', default=False, 165 help='run bytecode tests') 166 parser.add_argument('--debugger', dest='debugger', action='store_true', default=False, 167 help='run debugger tests') 168 169 return parser.parse_args() 170 171 172class Test: 173 def __init__(self, test_path, flags): 174 self.path = test_path 175 self.flags = flags 176 self.output = None 177 self.error = None 178 self.passed = None 179 self.skipped = None 180 self.reproduce = "" 181 182 def log_cmd(self, cmd): 183 self.reproduce += "\n" + ' '.join(cmd) 184 185 def get_path_to_expected(self): 186 if self.path.find(".d.ts") == -1: 187 return "%s-expected.txt" % (path.splitext(self.path)[0]) 188 return "%s-expected.txt" % (self.path[:self.path.find(".d.ts")]) 189 190 def run(self, runner): 191 test_abc_name = ("%s.abc" % (path.splitext(self.path)[0])).replace("/", "_") 192 test_abc_path = path.join(runner.build_dir, test_abc_name) 193 cmd = runner.cmd_prefix + [runner.es2panda] 194 cmd.extend(self.flags) 195 cmd.extend(["--output=" + test_abc_path]) 196 cmd.append(self.path) 197 198 self.log_cmd(cmd) 199 process = subprocess.Popen( 200 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 201 out, err = process.communicate() 202 self.output = out.decode("utf-8", errors="ignore") + err.decode("utf-8", errors="ignore") 203 204 expected_path = self.get_path_to_expected() 205 try: 206 with open(expected_path, 'r') as fp: 207 expected = fp.read() 208 self.passed = expected == self.output and process.returncode in [ 209 0, 1] 210 except Exception: 211 self.passed = False 212 213 if not self.passed: 214 self.error = err.decode("utf-8", errors="ignore") 215 216 if os.path.exists(test_abc_path): 217 os.remove(test_abc_path) 218 219 return self 220 221 222class Test262Test(Test): 223 def __init__(self, test_path, flags, test_id, with_optimizer): 224 Test.__init__(self, test_path, flags) 225 self.test_id = test_id 226 self.fail_kind = None 227 self.with_optimizer = with_optimizer 228 229 class FailKind(Enum): 230 ES2PANDA_FAIL = 1 231 RUNTIME_FAIL = 2 232 AOT_FAIL = 3 233 ES2PANDA_TIMEOUT = 4 234 RUNTIME_TIMEOUT = 5 235 AOT_TIMEOUT = 6 236 237 def run(self, runner): 238 with open(self.path, 'r') as fp: 239 header = runner.util.get_header(fp.read()) 240 desc = runner.util.parse_descriptor(header) 241 242 test_abc = path.join(runner.tmp_dir, "%s.abc" % self.test_id) 243 test_an = path.join(runner.tmp_dir, "%s.an" % self.test_id) 244 245 directory = path.dirname(test_abc) 246 os.makedirs(directory, exist_ok=True) 247 248 cmd = runner.cmd_prefix + [runner.es2panda] 249 if self.with_optimizer: 250 cmd.append('--opt-level=2') 251 cmd.extend(['--thread=0', '--output=%s' % (test_abc)]) 252 253 if 'module' in desc['flags']: 254 cmd.append("--module") 255 256 if 'noStrict' in desc['flags']: 257 self.skipped = True 258 return self 259 260 cmd.append(self.path) 261 262 self.log_cmd(cmd) 263 264 if runner.args.verbose: 265 print('Run es2panda: %s' % ' '.join(cmd), file=sys.stderr) 266 267 process = subprocess.Popen( 268 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=runner.cmd_env) 269 270 try: 271 output_res, err = process.communicate(runner.args.es2panda_timeout) 272 except subprocess.TimeoutExpired: 273 process.kill() 274 self.passed = False 275 self.fail_kind = self.FailKind.ES2PANDA_TIMEOUT 276 self.error = self.fail_kind.name 277 return self 278 279 out = output_res.decode("utf-8", errors="ignore") 280 err = err.decode("utf-8", errors="ignore") 281 self.passed, need_exec = runner.util.validate_parse_result( 282 process.returncode, err, desc, out) 283 284 if not self.passed: 285 self.fail_kind = self.FailKind.ES2PANDA_FAIL 286 self.error = "out:{}\nerr:{}\ncode:{}".format( 287 out, err, process.returncode) 288 print(self.error) 289 return self 290 291 if not need_exec: 292 self.passed = True 293 return self 294 295 if runner.args.aot: 296 cmd = runner.cmd_prefix + [runner.arkaot] 297 cmd.extend(runner.aot_args) 298 cmd.extend(['--paoc-panda-files', test_abc]) 299 cmd.extend(['--paoc-output', test_an]) 300 301 if os.path.isfile(test_an): 302 os.remove(test_an) 303 304 self.log_cmd(cmd) 305 306 if runner.args.verbose: 307 print('Run ark_aot: %s' % ' '.join(cmd), file=sys.stderr) 308 309 process = subprocess.Popen( 310 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=runner.cmd_env) 311 312 try: 313 out, err = process.communicate(runner.args.paoc_timeout) 314 except subprocess.TimeoutExpired: 315 process.kill() 316 self.passed = False 317 self.fail_kind = self.FailKind.AOT_TIMEOUT 318 self.error = self.fail_kind.name 319 return self 320 321 if process.returncode != 0: 322 self.passed = False 323 self.fail_kind = self.FailKind.AOT_FAIL 324 self.error = err.decode("utf-8", errors="ignore") 325 return self 326 327 cmd = runner.cmd_prefix + [runner.runtime] 328 329 if runner.args.verbose: 330 print('Run aot for arm64: %s' % ' '.join(cmd), file=sys.stderr) 331 332 cmd.extend(runner.runtime_args) 333 334 if runner.args.aot: 335 cmd.extend(['--aot-files', test_an]) 336 337 if runner.args.jit: 338 cmd.extend(['--compiler-enable-jit=true', '--compiler-hotness-threshold=0']) 339 else: 340 cmd.extend(['--compiler-enable-jit=false']) 341 342 cmd.extend([test_abc, "_GLOBAL::func_main_0"]) 343 344 self.log_cmd(cmd) 345 346 if runner.args.verbose: 347 print('Run ark: %s' % ' '.join(cmd), file=sys.stderr) 348 349 process = subprocess.Popen( 350 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=runner.cmd_env) 351 352 try: 353 out, err = process.communicate(timeout=runner.args.timeout) 354 except subprocess.TimeoutExpired: 355 process.kill() 356 self.passed = False 357 self.fail_kind = self.FailKind.RUNTIME_TIMEOUT 358 self.error = self.fail_kind.name 359 return self 360 361 out = out.decode("utf-8", errors="ignore") 362 err = err.decode("utf-8", errors="ignore") 363 self.passed = runner.util.validate_runtime_result( 364 process.returncode, err, desc, out) 365 366 if not self.passed: 367 self.fail_kind = self.FailKind.RUNTIME_FAIL 368 self.error = "out:{}\nerr:{}\ncode:{}".format( 369 out, err, process.returncode) 370 print(self.error) 371 372 return self 373 374 375class TSCTest(Test): 376 def __init__(self, test_path, flags): 377 Test.__init__(self, test_path, flags) 378 self.options = self.parse_options() 379 380 def parse_options(self): 381 test_options = {} 382 383 with open(self.path, "r", encoding="latin1") as f: 384 lines = f.read() 385 options = re.findall(r"//\s?@\w+:.*\n", lines) 386 387 for option in options: 388 separated = option.split(":") 389 opt = re.findall(r"\w+", separated[0])[0].lower() 390 value = separated[1].strip().lower() 391 392 if opt == "filename": 393 if opt in options: 394 test_options[opt].append(value) 395 else: 396 test_options[opt] = [value] 397 398 elif opt == "lib" or opt == "module": 399 test_options[opt] = [each.strip() 400 for each in value.split(",")] 401 elif value == "true" or value == "false": 402 test_options[opt] = value.lower() == "true" 403 else: 404 test_options[opt] = value 405 406 # TODO: Possibility of error: all exports will be catched, even the commented ones 407 if 'module' not in test_options and re.search(r"export ", lines): 408 test_options['module'] = [] 409 410 return test_options 411 412 def run(self, runner): 413 cmd = runner.cmd_prefix + [runner.es2panda, '--parse-only'] 414 cmd.extend(self.flags) 415 if "module" in self.options: 416 cmd.append('--module') 417 cmd.append(self.path) 418 419 self.log_cmd(cmd) 420 process = subprocess.Popen( 421 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 422 out, err = process.communicate() 423 self.output = out.decode("utf-8", errors="ignore") 424 425 self.passed = True if process.returncode == 0 else False 426 427 if not self.passed: 428 self.error = err.decode("utf-8", errors="ignore") 429 430 return self 431 432 433class Runner: 434 def __init__(self, args, name): 435 self.test_root = path.dirname(path.abspath(__file__)) 436 self.args = args 437 self.name = name 438 self.tests = [] 439 self.failed = 0 440 self.passed = 0 441 self.es2panda = path.join(args.build_dir, 'es2abc') 442 self.build_dir = args.build_dir 443 self.cmd_prefix = [] 444 self.ark_js_vm = "" 445 self.ark_aot_compiler = "" 446 self.ld_library_path = "" 447 448 if args.js_runtime_path: 449 self.ark_js_vm = path.join(args.js_runtime_path, 'ark_js_vm') 450 self.ark_aot_compiler = path.join(args.js_runtime_path, 'ark_aot_compiler') 451 452 if args.ld_library_path: 453 self.ld_library_path = args.ld_library_path 454 455 if args.arm64_qemu: 456 self.cmd_prefix = ["qemu-aarch64", "-L", "/usr/aarch64-linux-gnu/"] 457 458 if args.arm32_qemu: 459 self.cmd_prefix = ["qemu-arm", "-L", "/usr/arm-linux-gnueabi"] 460 461 if not path.isfile(self.es2panda): 462 raise Exception("Cannot find es2panda binary: %s" % self.es2panda) 463 464 def add_directory(self, directory, extension, flags): 465 pass 466 467 def test_path(self, src): 468 pass 469 470 def run_test(self, test): 471 return test.run(self) 472 473 def run(self): 474 pool = multiprocessing.Pool() 475 result_iter = pool.imap_unordered( 476 self.run_test, self.tests, chunksize=32) 477 pool.close() 478 479 if self.args.progress: 480 from tqdm import tqdm 481 result_iter = tqdm(result_iter, total=len(self.tests)) 482 483 results = [] 484 for res in result_iter: 485 results.append(res) 486 487 self.tests = results 488 pool.join() 489 490 def deal_error(self, test): 491 path_str = test.path 492 err_col = {} 493 if test.error: 494 err_str = test.error.split('[')[0] if "patchfix" not in test.path else " patchfix throw error failed" 495 err_col = {"path" : [path_str], "status": ["fail"], "error" : [test.error], "type" : [err_str]} 496 else: 497 err_col = {"path" : [path_str], "status": ["fail"], "error" : ["Segmentation fault"], 498 "type" : ["Segmentation fault"]} 499 return err_col 500 501 def summarize(self): 502 print("") 503 fail_list = [] 504 success_list = [] 505 506 for test in self.tests: 507 assert(test.passed is not None) 508 if not test.passed: 509 fail_list.append(test) 510 else: 511 success_list.append(test) 512 513 if len(fail_list): 514 if self.args.error: 515 import pandas as pd 516 test_list = pd.DataFrame(columns=["path", "status", "error", "type"]) 517 for test in success_list: 518 suc_col = {"path" : [test.path], "status": ["success"], "error" : ["success"], "type" : ["success"]} 519 if self.args.error: 520 test_list = pd.concat([test_list, pd.DataFrame(suc_col)]) 521 print("Failed tests:") 522 for test in fail_list: 523 print(self.test_path(test.path)) 524 525 if self.args.error: 526 print("steps:", test.reproduce) 527 print("error:") 528 print(test.error) 529 print("\n") 530 err_col = self.deal_error(test) 531 test_list = pd.concat([test_list, pd.DataFrame(err_col)]) 532 533 if self.args.error: 534 test_list.to_csv('test_statistics.csv', index=False) 535 test_list["type"].value_counts().to_csv('type_statistics.csv', index_label="error") 536 print("Type statistics:\n", test_list["type"].value_counts()) 537 print("") 538 539 print("Summary(%s):" % self.name) 540 print("\033[37mTotal: %5d" % (len(self.tests))) 541 print("\033[92mPassed: %5d" % (len(self.tests) - len(fail_list))) 542 print("\033[91mFailed: %5d" % (len(fail_list))) 543 print("\033[0m") 544 545 return len(fail_list) 546 547 548class RegressionRunner(Runner): 549 def __init__(self, args): 550 Runner.__init__(self, args, "Regression") 551 552 def add_directory(self, directory, extension, flags, func=Test): 553 glob_expression = path.join( 554 self.test_root, directory, "*.%s" % (extension)) 555 files = glob(glob_expression) 556 files = fnmatch.filter(files, self.test_root + '**' + self.args.filter) 557 558 self.tests += list(map(lambda f: func(f, flags), files)) 559 560 def test_path(self, src): 561 return src 562 563 564class Test262Runner(Runner): 565 def __init__(self, args): 566 Runner.__init__(self, args, "Test262 ark"), 567 568 self.cmd_env = os.environ.copy() 569 for san in ["ASAN_OPTIONS", "TSAN_OPTIONS", "MSAN_OPTIONS", "LSAN_OPTIONS"]: 570 # we don't want to interpret asan failures as SyntaxErrors 571 self.cmd_env[san] = ":exitcode=255" 572 573 self.update = args.update 574 self.enable_skiplists = False if self.update else args.skip 575 self.normal_skiplist_file = "test262skiplist.txt" 576 self.long_flaky_skiplist_files = ["test262skiplist-long.txt", "test262skiplist-flaky.txt"] 577 self.normal_skiplist = set([]) 578 self.runtime = path.join(args.build_dir, 'bin', 'ark') 579 if not path.isfile(self.runtime): 580 raise Exception("Cannot find runtime binary: %s" % self.runtime) 581 582 self.runtime_args = [ 583 '--boot-panda-files=%s/pandastdlib/arkstdlib.abc' 584 % args.build_dir, 585 '--load-runtimes=ecmascript', 586 '--gc-type=%s' % args.gc_type, 587 ] 588 589 if not args.no_gip: 590 self.runtime_args += ['--run-gc-in-place'] 591 592 if args.aot: 593 self.arkaot = path.join(args.build_dir, 'bin', 'ark_aot') 594 if not path.isfile(self.arkaot): 595 raise Exception("Cannot find aot binary: %s" % self.arkaot) 596 597 self.aot_args = [ 598 '--boot-panda-files=%s/pandastdlib/arkstdlib.abc' 599 % args.build_dir, 600 '--load-runtimes=ecmascript', 601 '--gc-type=%s' % args.gc_type, 602 ] 603 604 if not args.no_gip: 605 self.aot_args += ['--run-gc-in-place'] 606 607 self.aot_args += args.aot_args 608 else: 609 self.aot_args = [] 610 611 self.skiplist_name_list = self.long_flaky_skiplist_files if self.update else [] 612 self.skiplist_bco_name = "" 613 614 if self.enable_skiplists: 615 self.skiplist_name_list.append(self.normal_skiplist_file) 616 self.skiplist_name_list.extend(self.long_flaky_skiplist_files) 617 618 if args.bco: 619 self.skiplist_bco_name = "test262skiplist-bco.txt" 620 if args.arm64_compiler_skip: 621 self.skiplist_name_list.append("test262skiplist-compiler-arm64.txt") 622 623 self.tmp_dir = path.join(path.sep, 'tmp', 'panda', 'test262') 624 os.makedirs(self.tmp_dir, exist_ok=True) 625 626 self.util = test262util.Test262Util() 627 self.test262_dir = self.util.generate( 628 '281eb10b2844929a7c0ac04527f5b42ce56509fd', 629 args.build_dir, 630 path.join(self.test_root, "test262harness.js"), 631 args.progress) 632 633 self.add_directory(self.test262_dir, "js", args.test_list, []) 634 635 def add_directory(self, directory, extension, test_list_path, flags): 636 glob_expression = path.join(directory, "**/*.%s" % (extension)) 637 files = glob(glob_expression, recursive=True) 638 files = fnmatch.filter(files, path.join(directory, self.args.filter)) 639 640 def load_list(p): 641 with open(p, 'r') as fp: 642 return set(map(lambda e: path.join(directory, e.strip()), fp)) 643 644 skiplist = set([]) 645 646 for sl in self.skiplist_name_list: 647 skiplist.update(load_list(path.join(self.test_root, sl))) 648 649 if self.update: 650 self.normal_skiplist.update(load_list(path.join(self.test_root, self.normal_skiplist_file))) 651 652 skiplist_bco = set([]) 653 if self.skiplist_bco_name != "": 654 skiplist_bco = load_list(path.join(self.test_root, self.skiplist_bco_name)) 655 656 if test_list_path is not None: 657 test_list = load_list(path.abspath(test_list_path)) 658 files = filter(lambda f: f in test_list, files) 659 660 def get_test_id(file): 661 return path.relpath(path.splitext(file)[0], self.test262_dir) 662 663 self.tests = list(map(lambda test: Test262Test(test, flags, get_test_id(test), test not in skiplist_bco), 664 filter(lambda f: f not in skiplist, files))) 665 666 def test_path(self, src): 667 return path.relpath(src, self.test262_dir) 668 669 def run(self): 670 Runner.run(self) 671 self.update_skiplist() 672 673 def summarize(self): 674 print("") 675 676 fail_lists = {} 677 for kind in Test262Test.FailKind: 678 fail_lists[kind] = [] 679 680 num_failed = 0 681 num_skipped = 0 682 for test in self.tests: 683 if test.skipped: 684 num_skipped += 1 685 continue 686 687 assert(test.passed is not None) 688 if not test.passed: 689 fail_lists[test.fail_kind].append(test) 690 num_failed += 1 691 692 def summarize_list(name, tests_list): 693 if len(tests_list): 694 tests_list.sort(key=lambda test: test.path) 695 print("# " + name) 696 for test in tests_list: 697 print(self.test_path(test.path)) 698 if self.args.error: 699 print("steps:", test.reproduce) 700 print(test.error) 701 print("") 702 703 total_tests = len(self.tests) - num_skipped 704 705 if not self.update: 706 for kind in Test262Test.FailKind: 707 summarize_list(kind.name, fail_lists[kind]) 708 709 print("Summary(%s):" % self.name) 710 print("\033[37mTotal: %5d" % (total_tests)) 711 print("\033[92mPassed: %5d" % (total_tests - num_failed)) 712 print("\033[91mFailed: %5d" % (num_failed)) 713 print("\033[0m") 714 715 return num_failed 716 717 def update_skiplist(self): 718 if not self.update: 719 return 720 721 skiplist_es2panda = list({x.test_id + ".js" for x in self.tests 722 if not x.skipped and not x.passed and 723 x.fail_kind == Test262Test.FailKind.ES2PANDA_FAIL}) 724 skiplist_runtime = list({x.test_id + ".js" for x in self.tests 725 if not x.skipped and not x.passed and 726 x.fail_kind == Test262Test.FailKind.RUNTIME_FAIL}) 727 728 skiplist_es2panda.sort() 729 skiplist_runtime.sort() 730 731 new_skiplist = skiplist_es2panda + skiplist_runtime 732 733 new_pass = list(filter(lambda x: len(x) and not x.startswith('#') 734 and x not in new_skiplist, self.normal_skiplist)) 735 new_fail = list(filter(lambda x: x not in self.normal_skiplist, new_skiplist)) 736 new_pass.sort() 737 new_fail.sort() 738 739 if new_pass: 740 print("\033[92mRemoved from skiplist:") 741 print("\n".join(new_pass)) 742 print("\033[0m") 743 744 if new_fail: 745 print("\033[91mNew tests on skiplist:") 746 print("\n".join(new_fail)) 747 print("\033[0m") 748 749 fd = os.open(path.join(self.test_root, self.normal_skiplist_file), os.O_RDWR | os.O_CREAT | os.O_TRUNC) 750 file = os.fdopen(fd, "w+") 751 file.write("\n".join(["# ES2PANDA_FAIL"] + skiplist_es2panda + ["", "# RUNTIME_FAIL"] + skiplist_runtime)) 752 file.write("\n") 753 file.close() 754 755 756class TSCRunner(Runner): 757 def __init__(self, args): 758 Runner.__init__(self, args, "TSC") 759 760 if self.args.tsc_path: 761 self.tsc_path = self.args.tsc_path 762 else : 763 self.tsc_path = prepare_tsc_testcases(self.test_root) 764 765 self.add_directory("conformance", []) 766 self.add_directory("compiler", []) 767 768 def add_directory(self, directory, flags): 769 ts_suite_dir = path.join(self.tsc_path, 'tests/cases') 770 771 glob_expression = path.join( 772 ts_suite_dir, directory, "**/*.ts") 773 files = glob(glob_expression, recursive=True) 774 files = fnmatch.filter(files, ts_suite_dir + '**' + self.args.filter) 775 776 for f in files: 777 test_name = path.basename(f.split(".ts")[0]) 778 negative_references = path.join( 779 self.tsc_path, 'tests/baselines/reference') 780 is_negative = path.isfile(path.join(negative_references, 781 test_name + ".errors.txt")) 782 test = TSCTest(f, flags) 783 784 if 'target' in test.options: 785 targets = test.options['target'].replace(" ", "").split(',') 786 for target in targets: 787 if path.isfile(path.join(negative_references, 788 test_name + "(target=%s).errors.txt" % (target))): 789 is_negative = True 790 break 791 792 if is_negative or "filename" in test.options: 793 continue 794 795 with open(path.join(self.test_root, 'test_tsc_ignore_list.txt'), 'r') as failed_references: 796 if self.args.skip: 797 if path.relpath(f, self.tsc_path) in failed_references.read(): 798 continue 799 800 self.tests.append(test) 801 802 def test_path(self, src): 803 return src 804 805 806class CompilerRunner(Runner): 807 def __init__(self, args): 808 Runner.__init__(self, args, "Compiler") 809 810 def add_directory(self, directory, extension, flags): 811 if directory.endswith("projects"): 812 projects_path = path.join(self.test_root, directory) 813 for project in os.listdir(projects_path): 814 glob_expression = path.join(projects_path, project, "**/*.%s" % (extension)) 815 files = glob(glob_expression, recursive=True) 816 files = fnmatch.filter(files, self.test_root + '**' + self.args.filter) 817 self.tests.append(CompilerProjectTest(projects_path, project, files, flags)) 818 else: 819 glob_expression = path.join( 820 self.test_root, directory, "**/*.%s" % (extension)) 821 files = glob(glob_expression, recursive=True) 822 files = fnmatch.filter(files, self.test_root + '**' + self.args.filter) 823 self.tests += list(map(lambda f: CompilerTest(f, flags), files)) 824 825 def test_path(self, src): 826 return src 827 828 829class CompilerTest(Test): 830 def __init__(self, test_path, flags): 831 Test.__init__(self, test_path, flags) 832 833 def run(self, runner): 834 test_abc_name = ("%s.abc" % (path.splitext(self.path)[0])).replace("/", "_") 835 test_abc_path = path.join(runner.build_dir, test_abc_name) 836 es2abc_cmd = runner.cmd_prefix + [runner.es2panda] 837 es2abc_cmd.extend(self.flags) 838 es2abc_cmd.extend(["--output=" + test_abc_path]) 839 es2abc_cmd.append(self.path) 840 self.log_cmd(es2abc_cmd) 841 842 process = subprocess.Popen(es2abc_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 843 out, err = process.communicate() 844 if err: 845 self.passed = False 846 self.error = err.decode("utf-8", errors="ignore") 847 return self 848 849 ld_library_path = runner.ld_library_path 850 os.environ.setdefault("LD_LIBRARY_PATH", ld_library_path) 851 run_abc_cmd = [runner.ark_js_vm] 852 run_abc_cmd.extend([test_abc_path]) 853 self.log_cmd(run_abc_cmd) 854 855 process = subprocess.Popen(run_abc_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 856 out, err = process.communicate() 857 self.output = out.decode("utf-8", errors="ignore") + err.decode("utf-8", errors="ignore") 858 expected_path = self.get_path_to_expected() 859 try: 860 with open(expected_path, 'r') as fp: 861 expected = fp.read() 862 self.passed = expected == self.output and process.returncode in [0, 1] 863 except Exception: 864 self.passed = False 865 866 if not self.passed: 867 self.error = err.decode("utf-8", errors="ignore") 868 869 os.remove(test_abc_path) 870 871 return self 872 873 874class CompilerProjectTest(Test): 875 def __init__(self, projects_path, project, test_paths, flags): 876 Test.__init__(self, "", flags) 877 self.projects_path = projects_path 878 self.project = project 879 self.test_paths = test_paths 880 self.files_info_path = os.path.join(os.path.join(self.projects_path, self.project), 'filesInfo.txt') 881 882 def remove_project(self, runner): 883 project_path = runner.build_dir + "/" + self.project 884 if path.exists(project_path): 885 shutil.rmtree(project_path) 886 if path.exists(self.files_info_path): 887 os.remove(self.files_info_path) 888 889 def get_file_absolute_path_and_name(self, runner): 890 sub_path = self.path[len(self.projects_path):] 891 file_relative_path = path.split(sub_path)[0] 892 file_name = path.split(sub_path)[1] 893 file_absolute_path = runner.build_dir + "/" + file_relative_path 894 return [file_absolute_path, file_name] 895 896 def gen_single_abc(self, runner): 897 for test_path in self.test_paths: 898 self.path = test_path 899 [file_absolute_path, file_name] = self.get_file_absolute_path_and_name(runner) 900 if not path.exists(file_absolute_path): 901 os.makedirs(file_absolute_path) 902 903 test_abc_name = ("%s.abc" % (path.splitext(file_name)[0])) 904 test_abc_path = path.join(file_absolute_path, test_abc_name) 905 es2abc_cmd = runner.cmd_prefix + [runner.es2panda] 906 es2abc_cmd.extend(self.flags) 907 es2abc_cmd.extend(['%s%s' % ("--output=", test_abc_path)]) 908 es2abc_cmd.append(self.path) 909 self.log_cmd(es2abc_cmd) 910 911 process = subprocess.Popen(es2abc_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 912 out, err = process.communicate() 913 if err: 914 self.passed = False 915 self.error = err.decode("utf-8", errors="ignore") 916 self.remove_project(runner) 917 return self 918 919 def gen_files_info(self, runner): 920 fd = os.open(self.files_info_path, os.O_RDWR | os.O_CREAT | os.O_TRUNC) 921 f = os.fdopen(fd, "w") 922 for test_path in self.test_paths: 923 record_name = os.path.relpath(test_path, os.path.dirname(self.files_info_path)).split('.')[0] 924 module_kind = "esm" 925 file_info = ('%s;%s;%s;%s;%s' % (test_path, record_name, module_kind, test_path, record_name)) 926 f.writelines(file_info + '\n') 927 f.close() 928 929 def gen_merged_abc(self, runner): 930 for test_path in self.test_paths: 931 self.path = test_path 932 if (self.path.endswith("-exec.ts")): 933 exec_file_path = self.path 934 [file_absolute_path, file_name] = self.get_file_absolute_path_and_name(runner) 935 if not path.exists(file_absolute_path): 936 os.makedirs(file_absolute_path) 937 test_abc_name = ("%s.abc" % (path.splitext(file_name)[0])) 938 output_abc_name = path.join(file_absolute_path, test_abc_name) 939 es2abc_cmd = runner.cmd_prefix + [runner.es2panda] 940 es2abc_cmd.extend(self.flags) 941 es2abc_cmd.extend(['%s%s' % ("--output=", output_abc_name)]) 942 es2abc_cmd.append('@' + os.path.join(os.path.dirname(exec_file_path), "filesInfo.txt")) 943 self.log_cmd(es2abc_cmd) 944 process = subprocess.Popen(es2abc_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 945 out, err = process.communicate() 946 if err: 947 self.passed = False 948 self.error = err.decode("utf-8", errors="ignore") 949 self.remove_project(runner) 950 return self 951 952 def run(self, runner): 953 # Compile all ts source files in the project to abc files. 954 if ("--merge-abc" in self.flags): 955 self.gen_files_info(runner) 956 self.gen_merged_abc(runner) 957 else: 958 self.gen_single_abc(runner) 959 960 # Run test files that need to be executed in the project. 961 for test_path in self.test_paths: 962 self.path = test_path 963 if self.path.endswith("-exec.ts"): 964 [file_absolute_path, file_name] = self.get_file_absolute_path_and_name(runner) 965 966 entry_point_name = path.splitext(file_name)[0] 967 test_abc_name = ("%s.abc" % entry_point_name) 968 test_abc_path = path.join(file_absolute_path, test_abc_name) 969 970 ld_library_path = runner.ld_library_path 971 os.environ.setdefault("LD_LIBRARY_PATH", ld_library_path) 972 run_abc_cmd = [runner.ark_js_vm] 973 if ("--merge-abc" in self.flags): 974 run_abc_cmd.extend(["--entry-point", entry_point_name]) 975 run_abc_cmd.extend([test_abc_path]) 976 self.log_cmd(run_abc_cmd) 977 978 process = subprocess.Popen(run_abc_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 979 out, err = process.communicate() 980 self.output = out.decode("utf-8", errors="ignore") + err.decode("utf-8", errors="ignore") 981 expected_path = self.get_path_to_expected() 982 try: 983 with open(expected_path, 'r') as fp: 984 expected = fp.read() 985 self.passed = expected == self.output and process.returncode in [0, 1] 986 except Exception: 987 self.passed = False 988 989 if not self.passed: 990 self.error = err.decode("utf-8", errors="ignore") 991 self.remove_project(runner) 992 return self 993 994 self.passed = True 995 996 self.remove_project(runner) 997 return self 998 999 1000class TSDeclarationTest(Test): 1001 def get_path_to_expected(self): 1002 file_name = self.path[:self.path.find(".d.ts")] 1003 return "%s-expected.txt" % file_name 1004 1005 1006class TransformerRunner(Runner): 1007 def __init__(self, args): 1008 Runner.__init__(self, args, "Transformer") 1009 1010 def add_directory(self, directory, extension, flags): 1011 glob_expression = path.join( 1012 self.test_root, directory, "**/*.%s" % (extension)) 1013 files = glob(glob_expression, recursive=True) 1014 files = fnmatch.filter(files, self.test_root + '**' + self.args.filter) 1015 1016 self.tests += list(map(lambda f: TransformerTest(f, flags), files)) 1017 1018 def test_path(self, src): 1019 return src 1020 1021 1022class TransformerTest(Test): 1023 def __init__(self, test_path, flags): 1024 Test.__init__(self, test_path, flags) 1025 1026 def get_path_to_expected(self): 1027 return "%s-transformed-expected.txt" % (path.splitext(self.path)[0]) 1028 1029 def run(self, runner): 1030 cmd = runner.cmd_prefix + [runner.es2panda] 1031 cmd.extend(self.flags) 1032 cmd.append(self.path) 1033 1034 self.log_cmd(cmd) 1035 process = subprocess.Popen( 1036 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 1037 out, err = process.communicate() 1038 self.output = out.decode("utf-8", errors="ignore") + err.decode("utf-8", errors="ignore") 1039 1040 expected_path = self.get_path_to_expected() 1041 try: 1042 with open(expected_path, 'r') as fp: 1043 expected = fp.read() 1044 self.passed = expected == self.output and process.returncode in [0, 1] 1045 except Exception: 1046 self.passed = False 1047 1048 if not self.passed: 1049 self.error = err.decode("utf-8", errors="ignore") 1050 1051 return self 1052 1053 1054class PatchTest(Test): 1055 def __init__(self, test_path, mode_arg): 1056 Test.__init__(self, test_path, "") 1057 self.mode = mode_arg 1058 1059 def run(self, runner): 1060 symbol_table_file = 'base.map' 1061 origin_input_file = 'base.js' 1062 origin_output_abc = 'base.abc' 1063 modified_input_file = 'base_mod.js' 1064 modified_output_abc = 'patch.abc' 1065 1066 gen_base_cmd = runner.cmd_prefix + [runner.es2panda, '--module'] 1067 if 'record-name-with-dots' in os.path.basename(self.path): 1068 gen_base_cmd.extend(['--merge-abc', '--record-name=record.name.with.dots']) 1069 gen_base_cmd.extend(['--dump-symbol-table', os.path.join(self.path, symbol_table_file)]) 1070 gen_base_cmd.extend(['--output', os.path.join(self.path, origin_output_abc)]) 1071 gen_base_cmd.extend([os.path.join(self.path, origin_input_file)]) 1072 self.log_cmd(gen_base_cmd) 1073 1074 if self.mode == 'hotfix': 1075 mode_arg = ["--generate-patch"] 1076 elif self.mode == 'hotreload': 1077 mode_arg = ["--hot-reload"] 1078 elif self.mode == 'coldfix': 1079 mode_arg = ["--generate-patch", "--cold-fix"] 1080 1081 patch_test_cmd = runner.cmd_prefix + [runner.es2panda, '--module'] 1082 patch_test_cmd.extend(mode_arg) 1083 patch_test_cmd.extend(['--input-symbol-table', os.path.join(self.path, symbol_table_file)]) 1084 patch_test_cmd.extend(['--output', os.path.join(self.path, modified_output_abc)]) 1085 patch_test_cmd.extend([os.path.join(self.path, modified_input_file)]) 1086 if 'record-name-with-dots' in os.path.basename(self.path): 1087 patch_test_cmd.extend(['--merge-abc', '--record-name=record.name.with.dots']) 1088 dump_assembly_testname = [ 1089 'modify-anon-content-keep-origin-name', 1090 'modify-class-memeber-function', 1091 'exist-lexenv-3', 1092 'lexenv-reduce', 1093 'lexenv-increase'] 1094 for name in dump_assembly_testname: 1095 if name in os.path.basename(self.path): 1096 patch_test_cmd.extend(['--dump-assembly']) 1097 self.log_cmd(patch_test_cmd) 1098 process_base = subprocess.Popen(gen_base_cmd, stdout=subprocess.PIPE, 1099 stderr=subprocess.PIPE) 1100 stdout_base, stderr_base = process_base.communicate(timeout=runner.args.es2panda_timeout) 1101 if stderr_base: 1102 self.passed = False 1103 self.error = stderr_base.decode("utf-8", errors="ignore") 1104 self.output = stdout_base.decode("utf-8", errors="ignore") + stderr_base.decode("utf-8", errors="ignore") 1105 else: 1106 process_patch = subprocess.Popen(patch_test_cmd, stdout=subprocess.PIPE, 1107 stderr=subprocess.PIPE) 1108 stdout_patch, stderr_patch = process_patch.communicate(timeout=runner.args.es2panda_timeout) 1109 if stderr_patch: 1110 self.passed = False 1111 self.error = stderr_patch.decode("utf-8", errors="ignore") 1112 self.output = stdout_patch.decode("utf-8", errors="ignore") + stderr_patch.decode("utf-8", errors="ignore") 1113 1114 expected_path = os.path.join(self.path, 'expected.txt') 1115 try: 1116 with open(expected_path, 'r') as fp: 1117 # ignore license description lines and skip leading blank lines 1118 expected = (''.join((fp.readlines()[12:]))).lstrip() 1119 self.passed = expected == self.output 1120 except Exception: 1121 self.passed = False 1122 1123 if not self.passed: 1124 self.error = "expected output:" + os.linesep + expected + os.linesep + "actual output:" + os.linesep +\ 1125 self.output 1126 1127 return self 1128 1129 1130class PatchRunner(Runner): 1131 def __init__(self, args, name): 1132 Runner.__init__(self, args, name) 1133 self.preserve_files = args.error 1134 1135 def __del__(self): 1136 if not self.preserve_files: 1137 self.clear_directory() 1138 1139 def add_directory(self): 1140 self.tests_in_dirs = [] 1141 for item in self.test_directory: 1142 glob_expression = path.join(item, "*") 1143 self.tests_in_dirs += glob(glob_expression, recursive=False) 1144 1145 def clear_directory(self): 1146 for test in self.tests_in_dirs: 1147 files_in_dir = os.listdir(test) 1148 filtered_files = [file for file in files_in_dir if file.endswith(".map") or file.endswith(".abc")] 1149 for file in filtered_files: 1150 os.remove(os.path.join(test, file)) 1151 1152 def test_path(self, src): 1153 return os.path.basename(src) 1154 1155 1156class HotfixRunner(PatchRunner): 1157 def __init__(self, args): 1158 PatchRunner.__init__(self, args, "Hotfix") 1159 self.test_directory = [path.join(self.test_root, "hotfix", "hotfix-throwerror"), 1160 path.join(self.test_root, "hotfix", "hotfix-noerror")] 1161 self.add_directory() 1162 self.tests += list(map(lambda t: PatchTest(t, "hotfix"), self.tests_in_dirs)) 1163 1164 1165class HotreloadRunner(PatchRunner): 1166 def __init__(self, args): 1167 PatchRunner.__init__(self, args, "Hotreload") 1168 self.test_directory = [path.join(self.test_root, "hotreload", "hotreload-throwerror"), 1169 path.join(self.test_root, "hotreload", "hotreload-noerror")] 1170 self.add_directory() 1171 self.tests += list(map(lambda t: PatchTest(t, "hotreload"), self.tests_in_dirs)) 1172 1173 1174class ColdfixRunner(PatchRunner): 1175 def __init__(self, args): 1176 PatchRunner.__init__(self, args, "Coldfix") 1177 self.test_directory = [path.join(self.test_root, "coldfix", "coldfix-throwerror"), 1178 path.join(self.test_root, "coldfix", "coldfix-noerror")] 1179 self.add_directory() 1180 self.tests += list(map(lambda t: PatchTest(t, "coldfix"), self.tests_in_dirs)) 1181 1182 1183class DebuggerTest(Test): 1184 def __init__(self, test_path, mode): 1185 Test.__init__(self, test_path, "") 1186 self.mode = mode 1187 1188 def run(self, runner): 1189 cmd = runner.cmd_prefix + [runner.es2panda, "--module"] 1190 input_file_name = 'base.js' 1191 if self.mode == "debug-mode": 1192 cmd.extend(['--debug-info']) 1193 cmd.extend([os.path.join(self.path, input_file_name)]) 1194 cmd.extend(['--dump-assembly']) 1195 1196 1197 self.log_cmd(cmd) 1198 1199 process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 1200 stdout, stderr = process.communicate(timeout=runner.args.es2panda_timeout) 1201 if stderr: 1202 self.passed = False 1203 self.error = stderr.decode("utf-8", errors="ignore") 1204 return self 1205 1206 self.output = stdout.decode("utf-8", errors="ignore") 1207 1208 expected_path = os.path.join(self.path, 'expected.txt') 1209 try: 1210 with open(expected_path, 'r') as fp: 1211 expected = (''.join((fp.readlines()[12:]))).lstrip() 1212 self.passed = expected == self.output 1213 except Exception: 1214 self.passed = False 1215 1216 if not self.passed: 1217 self.error = "expected output:" + os.linesep + expected + os.linesep + "actual output:" + os.linesep +\ 1218 self.output 1219 1220 return self 1221 1222 1223class DebuggerRunner(Runner): 1224 def __init__(self, args): 1225 Runner.__init__(self, args, "debugger") 1226 self.test_directory = path.join(self.test_root, "debugger") 1227 self.add_test() 1228 1229 def add_test(self): 1230 self.tests = [] 1231 self.tests.append(DebuggerTest(os.path.join(self.test_directory, "debugger-in-debug"), "debug-mode")) 1232 self.tests.append(DebuggerTest(os.path.join(self.test_directory, "debugger-in-release"), "release-mode")) 1233 1234 1235class Base64Test(Test): 1236 def __init__(self, test_path, input_type): 1237 Test.__init__(self, test_path, "") 1238 self.input_type = input_type 1239 1240 def run(self, runner): 1241 cmd = runner.cmd_prefix + [runner.es2panda, "--base64Output"] 1242 if self.input_type == "file": 1243 input_file_name = 'input.js' 1244 cmd.extend(['--source-file', input_file_name]) 1245 cmd.extend([os.path.join(self.path, input_file_name)]) 1246 elif self.input_type == "string": 1247 input_file = os.path.join(self.path, "input.txt") 1248 try: 1249 with open(input_file, 'r') as fp: 1250 base64_input = (''.join((fp.readlines()[12:]))).lstrip() # ignore license description lines 1251 cmd.extend(["--base64Input", base64_input]) 1252 except Exception: 1253 self.passed = False 1254 else: 1255 self.error = "Unsupported base64 input type" 1256 self.passed = False 1257 return self 1258 1259 self.log_cmd(cmd) 1260 1261 process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 1262 stdout, stderr = process.communicate(timeout=runner.args.es2panda_timeout) 1263 if stderr: 1264 self.passed = False 1265 self.error = stderr.decode("utf-8", errors="ignore") 1266 return self 1267 1268 self.output = stdout.decode("utf-8", errors="ignore") 1269 1270 expected_path = os.path.join(self.path, 'expected.txt') 1271 try: 1272 with open(expected_path, 'r') as fp: 1273 expected = (''.join((fp.readlines()[12:]))).lstrip() 1274 self.passed = expected == self.output 1275 except Exception: 1276 self.passed = False 1277 1278 if not self.passed: 1279 self.error = "expected output:" + os.linesep + expected + os.linesep + "actual output:" + os.linesep +\ 1280 self.output 1281 1282 return self 1283 1284 1285class Base64Runner(Runner): 1286 def __init__(self, args): 1287 Runner.__init__(self, args, "Base64") 1288 self.test_directory = path.join(self.test_root, "base64") 1289 self.add_test() 1290 1291 def add_test(self): 1292 self.tests = [] 1293 self.tests.append(Base64Test(os.path.join(self.test_directory, "inputFile"), "file")) 1294 self.tests.append(Base64Test(os.path.join(self.test_directory, "inputString"), "string")) 1295 1296 def test_path(self, src): 1297 return os.path.basename(src) 1298 1299 1300class TypeExtractorRunner(Runner): 1301 def __init__(self, args): 1302 Runner.__init__(self, args, "TypeExtractor") 1303 1304 if self.args.tsc_path: 1305 self.tsc_path = self.args.tsc_path 1306 else : 1307 self.tsc_path = prepare_tsc_testcases(self.test_root) 1308 1309 self.add_tsc_directory("conformance", []) 1310 self.add_directory("testcases", []) 1311 self.add_directory("dts-testcases", [], True) 1312 self.add_directory("testcases_with_assert", []) 1313 self.add_directory("testcases_with_assert/projects", [], False, True) 1314 self.add_directory("testcases_with_running", []) 1315 1316 def add_tsc_directory(self, directory, flags): 1317 ts_suite_dir = path.join(self.tsc_path, 'tests/cases') 1318 1319 glob_expression = path.join( 1320 ts_suite_dir, directory, "**/*.ts") 1321 files = glob(glob_expression, recursive=True) 1322 files = fnmatch.filter(files, ts_suite_dir + '**' + self.args.filter) 1323 1324 with open(path.join(self.test_root, 'type_extractor/testlist.txt'), 'r') as passed_references: 1325 for f in files: 1326 if path.relpath(f, self.tsc_path) in passed_references.read(): 1327 test = TypeExtractorTest(f, flags) 1328 self.tests.append(test) 1329 1330 def add_directory(self, directory, flags, is_dts_test=False, is_project=False): 1331 ts_suite_dir = path.join(self.test_root, 'type_extractor') 1332 1333 if is_project: 1334 glob_expression = path.join(ts_suite_dir, directory, "**/*-main.ts") 1335 elif is_dts_test: 1336 glob_expression = path.join(ts_suite_dir, directory, "**/*.d.ts") 1337 else: 1338 glob_expression = path.join(ts_suite_dir, directory, "*.ts") 1339 files = glob(glob_expression, recursive=True) 1340 files = fnmatch.filter(files, ts_suite_dir + '**' + self.args.filter) 1341 for f in files: 1342 if directory.startswith("testcases_with_assert") or directory.startswith("testcases_with_running"): 1343 if (self.ld_library_path == "" or self.ark_aot_compiler == ""): 1344 break 1345 test = TypeExtractorWithAOTTest(f, flags, directory.startswith("testcases_with_running"), directory.endswith("projects")) 1346 self.tests.append(test) 1347 else: 1348 test = TypeExtractorTest(f, flags, is_dts_test) 1349 self.tests.append(test) 1350 1351 def test_path(self, src): 1352 return src 1353 1354 1355class TypeExtractorTest(Test): 1356 def __init__(self, test_path, flags, is_dts_test=False): 1357 Test.__init__(self, test_path, flags) 1358 self.is_dts_test = is_dts_test 1359 1360 def run(self, runner): 1361 test_abc_name = ("%s.abc" % (path.splitext(self.path)[0])).replace("/", "_") 1362 cmd = runner.cmd_prefix + [runner.es2panda, 1363 '--module', '--dump-literal-buffer', '--opt-level=2', '--type-extractor'] 1364 if self.is_dts_test: 1365 cmd.append("--type-dts-builtin") 1366 cmd.extend(self.flags) 1367 cmd.extend(["--output=" + test_abc_name]) 1368 cmd.append(self.path) 1369 1370 self.log_cmd(cmd) 1371 process = subprocess.Popen( 1372 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 1373 out, err = process.communicate() 1374 self.output = out.decode("utf-8", errors="ignore") + err.decode("utf-8", errors="ignore") 1375 output_str = self.output.split("======> literal array buffer <======")[1] 1376 1377 if os.path.isfile(test_abc_name): 1378 os.remove(test_abc_name) 1379 1380 file_name = path.splitext(self.path)[0] 1381 if self.is_dts_test: 1382 expected_path = "%s-expected.txt" % (path.splitext(file_name)[0]) 1383 else: 1384 expected_path = "%s-expected.txt" % (file_name) 1385 if not os.path.isfile(expected_path): 1386 expected_path = path.dirname(path.abspath(__file__)) + "/type_extractor/tsc_expect/%s" % \ 1387 expected_path.split("tests/cases/")[-1] 1388 1389 try: 1390 with open(expected_path, 'r') as fp: 1391 expected = fp.read() 1392 expected_str = expected.split("======> literal array buffer <======")[1] 1393 1394 self.passed = expected_str == output_str and process.returncode in [ 1395 0, 1] 1396 except Exception: 1397 self.passed = False 1398 1399 if not self.passed: 1400 self.error = err.decode("utf-8", errors="ignore") 1401 1402 return self 1403 1404 1405class TypeExtractorWithAOTTest(Test): 1406 def __init__(self, test_path, flags, with_running=False, is_project=False): 1407 Test.__init__(self, test_path, flags) 1408 self.with_running = with_running 1409 self.is_project = is_project 1410 1411 def run_js_vm(self, runner, file_name, test_abc_name): 1412 expected_path = "%s-expected.txt" % (file_name) 1413 run_aot_cmd = [runner.ark_js_vm] 1414 run_aot_cmd.extend(["--aot-file=%s" % file_name]) 1415 run_aot_cmd.extend(["--entry-point=%s" % path.basename(file_name)]) 1416 run_aot_cmd.extend([test_abc_name]) 1417 self.log_cmd(run_aot_cmd) 1418 1419 process = subprocess.Popen(run_aot_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 1420 out, err = process.communicate(timeout=runner.args.timeout) 1421 self.output = out.decode("utf-8", errors="ignore") + err.decode("utf-8", errors="ignore") 1422 try: 1423 with open(expected_path, 'r') as fp: 1424 expected = fp.read() 1425 self.passed = expected == self.output and process.returncode in [0, 1] 1426 except Exception: 1427 self.passed = False 1428 1429 if not self.passed: 1430 self.error = err.decode("utf-8", errors="ignore") 1431 1432 if os.path.isfile("%s.an" % (file_name)): 1433 os.remove("%s.an" % (file_name)) 1434 if os.path.isfile("%s.ai" % (file_name)): 1435 os.remove("%s.ai" % (file_name)) 1436 1437 def run(self, runner): 1438 file_name = path.splitext(self.path)[0] 1439 test_abc_name = ("%s.abc" % path.basename(file_name)) 1440 cmd = runner.cmd_prefix + [runner.es2panda, 1441 '--module', '--merge-abc', '--opt-level=2', '--type-extractor'] 1442 cmd.extend(self.flags) 1443 cmd.extend(["--output=" + test_abc_name]) 1444 if self.is_project: 1445 cmd.append("--extension=ts") 1446 cmd.append(path.dirname(self.path)) 1447 else: 1448 cmd.append(self.path) 1449 1450 self.log_cmd(cmd) 1451 process = subprocess.Popen( 1452 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 1453 out, err = process.communicate() 1454 if (err): 1455 self.passed = False 1456 self.error = err.decode("utf-8", errors="ignore") 1457 return self 1458 1459 ld_library_path = runner.ld_library_path 1460 os.environ.setdefault("LD_LIBRARY_PATH", ld_library_path) 1461 aot_abc_cmd = [runner.ark_aot_compiler] 1462 aot_abc_cmd.extend(["--aot-file=%s" % file_name]) 1463 if not self.with_running: 1464 aot_abc_cmd.extend(["--compiler-assert-types=true"]) 1465 aot_abc_cmd.extend(["--compiler-opt-type-lowering=false"]) 1466 aot_abc_cmd.extend([test_abc_name]) 1467 self.log_cmd(aot_abc_cmd) 1468 1469 process = subprocess.Popen(aot_abc_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 1470 out, err = process.communicate() 1471 if err: 1472 self.passed = False 1473 self.error = err.decode("utf-8", errors="ignore") 1474 else: 1475 if self.with_running: 1476 self.run_js_vm(runner, file_name, test_abc_name) 1477 else: 1478 self.passed = True 1479 1480 if os.path.isfile(test_abc_name): 1481 os.remove(test_abc_name) 1482 if os.path.isfile("%s.an" % (file_name)): 1483 os.remove("%s.an" % (file_name)) 1484 if os.path.isfile("%s.ai" % (file_name)): 1485 os.remove("%s.ai" % (file_name)) 1486 1487 return self 1488 1489class BytecodeRunner(Runner): 1490 def __init__(self, args): 1491 Runner.__init__(self, args, "Bytecode") 1492 1493 def add_directory(self, directory, extension, flags, func=Test): 1494 glob_expression = path.join( 1495 self.test_root, directory, "**/*.%s" % (extension)) 1496 files = glob(glob_expression, recursive=True) 1497 files = fnmatch.filter(files, self.test_root + '**' + self.args.filter) 1498 self.tests += list(map(lambda f: func(f, flags), files)) 1499 1500 def test_path(self, src): 1501 return src 1502 1503def main(): 1504 args = get_args() 1505 1506 runners = [] 1507 1508 if args.regression: 1509 runner = RegressionRunner(args) 1510 runner.add_directory("parser/concurrent", "js", ["--module", "--dump-ast"]) 1511 runner.add_directory("parser/js", "js", ["--parse-only", "--dump-ast"]) 1512 runner.add_directory("parser/script", "ts", ["--parse-only", "--dump-ast"]) 1513 runner.add_directory("parser/ts", "ts", 1514 ["--parse-only", "--module", "--dump-ast"]) 1515 runner.add_directory("parser/ts/type_checker", "ts", 1516 ["--parse-only", "--enable-type-check", "--module", "--dump-ast"]) 1517 runner.add_directory("parser/ts/cases/declaration", "d.ts", 1518 ["--parse-only", "--module", "--dump-ast"], TSDeclarationTest) 1519 runner.add_directory("parser/commonjs", "js", ["--commonjs", "--parse-only", "--dump-ast"]) 1520 runner.add_directory("parser/binder", "js", ["--dump-assembly"]) 1521 runner.add_directory("parser/js/emptySource", "js", ["--dump-assembly"]) 1522 runner.add_directory("parser/js/language/arguments-object", "js", ["--parse-only"]) 1523 runner.add_directory("parser/js/language/statements/for-statement", "js", ["--parse-only", "--dump-ast"]) 1524 runner.add_directory("parser/js/language/expressions/optional-chain", "js", ["--parse-only", "--dump-ast"]) 1525 runner.add_directory("parser/sendable_class", "ts", ["--dump-assembly", "--dump-literal-buffer", "--module"]) 1526 runner.add_directory("parser/unicode", "js", ["--parse-only"]) 1527 runner.add_directory("parser/ts/stack_overflow", "ts", ["--parse-only", "--dump-ast"]) 1528 1529 runners.append(runner) 1530 1531 transformer_runner = TransformerRunner(args) 1532 transformer_runner.add_directory("parser/ts/transformed_cases", "ts", 1533 ["--parse-only", "--module", "--dump-transformed-ast", 1534 "--check-transformed-ast-structure"]) 1535 1536 runners.append(transformer_runner) 1537 1538 if args.test262: 1539 runners.append(Test262Runner(args)) 1540 1541 if args.tsc: 1542 runners.append(TSCRunner(args)) 1543 1544 if args.compiler: 1545 runner = CompilerRunner(args) 1546 runner.add_directory("compiler/js", "js", []) 1547 runner.add_directory("compiler/ts/cases", "ts", []) 1548 runner.add_directory("compiler/ts/projects", "ts", ["--module"]) 1549 runner.add_directory("compiler/ts/projects", "ts", ["--module", "--merge-abc"]) 1550 runner.add_directory("compiler/dts", "d.ts", ["--module", "--opt-level=0"]) 1551 runner.add_directory("compiler/commonjs", "js", ["--commonjs"]) 1552 runner.add_directory("compiler/recordsource/with-on", "js", ["--record-source"]) 1553 runner.add_directory("compiler/recordsource/with-off", "js", []) 1554 runner.add_directory("compiler/interpreter/lexicalEnv", "js", []) 1555 1556 runners.append(runner) 1557 1558 if args.hotfix: 1559 runners.append(HotfixRunner(args)) 1560 1561 if args.hotreload: 1562 runners.append(HotreloadRunner(args)) 1563 1564 if args.coldfix: 1565 runners.append(ColdfixRunner(args)) 1566 1567 if args.debugger: 1568 runners.append(DebuggerRunner(args)) 1569 1570 if args.base64: 1571 runners.append(Base64Runner(args)) 1572 1573 if args.type_extractor: 1574 runners.append(TypeExtractorRunner(args)) 1575 1576 if args.bytecode: 1577 runner = BytecodeRunner(args) 1578 runner.add_directory("bytecode/commonjs", "js", ["--commonjs", "--dump-assembly"]) 1579 runner.add_directory("bytecode/js", "js", ["--dump-assembly"]) 1580 1581 runners.append(runner) 1582 1583 failed_tests = 0 1584 1585 for runner in runners: 1586 runner.run() 1587 failed_tests += runner.summarize() 1588 1589 # TODO: exit 1 when we have failed tests after all tests are fixed 1590 exit(0) 1591 1592 1593if __name__ == "__main__": 1594 main() 1595