1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# 4# Copyright (c) 2021-2022 Huawei Device Co., Ltd. 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17from glob import glob 18from os import path 19from enum import Enum 20import argparse 21import fnmatch 22import multiprocessing 23import os 24import re 25import shutil 26import subprocess 27import sys 28import test262util 29 30 31def is_directory(parser, arg): 32 if not path.isdir(arg): 33 parser.error("The directory '%s' does not exist" % arg) 34 35 return path.abspath(arg) 36 37 38def is_file(parser, arg): 39 if not path.isfile(arg): 40 parser.error("The file '%s' does not exist" % arg) 41 42 return path.abspath(arg) 43 44def prepare_tsc_testcases(test_root): 45 third_party_tsc = path.join(test_root, "TypeScript") 46 ohos_third_party_tsc = path.join(test_root, "../../../../third_party/typescript") 47 48 if not path.isdir(third_party_tsc): 49 if (path.isdir(ohos_third_party_tsc)): 50 return path.abspath(ohos_third_party_tsc) 51 subprocess.run( 52 f"git clone https://gitee.com/openharmony/third_party_typescript.git {third_party_tsc}", 53 shell=True, 54 stdout=subprocess.DEVNULL, 55 ) 56 else: 57 subprocess.run( 58 f"cd {third_party_tsc} && git clean -f > /dev/null 2>&1", 59 shell=True, 60 stdout=subprocess.DEVNULL, 61 ) 62 return third_party_tsc 63 64def check_timeout(value): 65 ivalue = int(value) 66 if ivalue <= 0: 67 raise argparse.ArgumentTypeError( 68 "%s is an invalid timeout value" % value) 69 return ivalue 70 71 72def get_args(): 73 parser = argparse.ArgumentParser(description="Regression test runner") 74 parser.add_argument( 75 'build_dir', type=lambda arg: is_directory(parser, arg), 76 help='panda build directory') 77 parser.add_argument( 78 '--test262', '-t', action='store_true', dest='test262', default=False, 79 help='run test262 tests') 80 parser.add_argument( 81 '--error', action='store_true', dest='error', default=False, 82 help='capture stderr') 83 parser.add_argument( 84 '--regression', '-r', action='store_true', dest='regression', 85 default=False, help='run regression tests') 86 parser.add_argument( 87 '--compiler', '-c', action='store_true', dest='compiler', 88 default=False, help='run compiler tests') 89 parser.add_argument( 90 '--tsc', action='store_true', dest='tsc', 91 default=False, help='run tsc tests') 92 parser.add_argument( 93 '--type-extractor', action='store_true', dest='type_extractor', 94 default=False, help='run type extractor tests') 95 parser.add_argument( 96 '--no-progress', action='store_false', dest='progress', default=True, 97 help='don\'t show progress bar') 98 parser.add_argument( 99 '--no-skip', action='store_false', dest='skip', default=True, 100 help='don\'t use skiplists') 101 parser.add_argument( 102 '--update', action='store_true', dest='update', default=False, 103 help='update skiplist') 104 parser.add_argument( 105 '--no-run-gc-in-place', action='store_true', dest='no_gip', default=False, 106 help='enable --run-gc-in-place mode') 107 parser.add_argument( 108 '--filter', '-f', action='store', dest='filter', 109 default="*", help='test filter regexp') 110 parser.add_argument( 111 '--es2panda-timeout', type=check_timeout, 112 dest='es2panda_timeout', default=60, help='es2panda translator timeout') 113 parser.add_argument( 114 '--paoc-timeout', type=check_timeout, 115 dest='paoc_timeout', default=600, help='paoc compiler timeout') 116 parser.add_argument( 117 '--timeout', type=check_timeout, 118 dest='timeout', default=10, help='JS runtime timeout') 119 parser.add_argument( 120 '--gc-type', dest='gc_type', default="g1-gc", help='Type of garbage collector') 121 parser.add_argument( 122 '--aot', action='store_true', dest='aot', default=False, 123 help='use AOT compilation') 124 parser.add_argument( 125 '--no-bco', action='store_false', dest='bco', default=True, 126 help='disable bytecodeopt') 127 parser.add_argument( 128 '--jit', action='store_true', dest='jit', default=False, 129 help='use JIT in interpreter') 130 parser.add_argument( 131 '--arm64-compiler-skip', action='store_true', dest='arm64_compiler_skip', default=False, 132 help='use skiplist for tests failing on aarch64 in AOT or JIT mode') 133 parser.add_argument( 134 '--arm64-qemu', action='store_true', dest='arm64_qemu', default=False, 135 help='launch all binaries in qemu aarch64') 136 parser.add_argument( 137 '--arm32-qemu', action='store_true', dest='arm32_qemu', default=False, 138 help='launch all binaries in qemu arm') 139 parser.add_argument( 140 '--test-list', dest='test_list', default=None, type=lambda arg: is_file(parser, arg), 141 help='run tests listed in file') 142 parser.add_argument( 143 '--aot-args', action='append', dest='aot_args', default=[], 144 help='Additional arguments that will passed to ark_aot') 145 parser.add_argument( 146 '--verbose', '-v', action='store_true', dest='verbose', default=False, 147 help='Enable verbose output') 148 parser.add_argument( 149 '--js-runtime', dest='js_runtime_path', default=None, type=lambda arg: is_directory(parser, arg), 150 help='the path of js vm runtime') 151 parser.add_argument( 152 '--LD_LIBRARY_PATH', dest='ld_library_path', default=None, help='LD_LIBRARY_PATH') 153 parser.add_argument( 154 '--tsc-path', dest='tsc_path', default=None, type=lambda arg: is_directory(parser, arg), 155 help='the path of tsc') 156 parser.add_argument('--hotfix', dest='hotfix', action='store_true', default=False, 157 help='run hotfix tests') 158 parser.add_argument('--hotreload', dest='hotreload', action='store_true', default=False, 159 help='run hotreload tests') 160 parser.add_argument('--coldfix', dest='coldfix', action='store_true', default=False, 161 help='run coldfix tests') 162 parser.add_argument('--base64', dest='base64', action='store_true', default=False, 163 help='run base64 tests') 164 parser.add_argument('--bytecode', dest='bytecode', action='store_true', default=False, 165 help='run bytecode tests') 166 167 return parser.parse_args() 168 169 170class Test: 171 def __init__(self, test_path, flags): 172 self.path = test_path 173 self.flags = flags 174 self.output = None 175 self.error = None 176 self.passed = None 177 self.skipped = None 178 self.reproduce = "" 179 180 def log_cmd(self, cmd): 181 self.reproduce += "\n" + ' '.join(cmd) 182 183 def get_path_to_expected(self): 184 if self.path.find(".d.ts") == -1: 185 return "%s-expected.txt" % (path.splitext(self.path)[0]) 186 return "%s-expected.txt" % (self.path[:self.path.find(".d.ts")]) 187 188 def run(self, runner): 189 test_abc_name = ("%s.abc" % (path.splitext(self.path)[0])).replace("/", "_") 190 test_abc_path = path.join(runner.build_dir, test_abc_name) 191 cmd = runner.cmd_prefix + [runner.es2panda] 192 cmd.extend(self.flags) 193 cmd.extend(["--output=" + test_abc_path]) 194 cmd.append(self.path) 195 196 self.log_cmd(cmd) 197 process = subprocess.Popen( 198 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 199 out, err = process.communicate() 200 self.output = out.decode("utf-8", errors="ignore") + err.decode("utf-8", errors="ignore") 201 202 expected_path = self.get_path_to_expected() 203 try: 204 with open(expected_path, 'r') as fp: 205 expected = fp.read() 206 self.passed = expected == self.output and process.returncode in [ 207 0, 1] 208 except Exception: 209 self.passed = False 210 211 if not self.passed: 212 self.error = err.decode("utf-8", errors="ignore") 213 214 if os.path.exists(test_abc_path): 215 os.remove(test_abc_path) 216 217 return self 218 219 220class Test262Test(Test): 221 def __init__(self, test_path, flags, test_id, with_optimizer): 222 Test.__init__(self, test_path, flags) 223 self.test_id = test_id 224 self.fail_kind = None 225 self.with_optimizer = with_optimizer 226 227 class FailKind(Enum): 228 ES2PANDA_FAIL = 1 229 RUNTIME_FAIL = 2 230 AOT_FAIL = 3 231 ES2PANDA_TIMEOUT = 4 232 RUNTIME_TIMEOUT = 5 233 AOT_TIMEOUT = 6 234 235 def run(self, runner): 236 with open(self.path, 'r') as fp: 237 header = runner.util.get_header(fp.read()) 238 desc = runner.util.parse_descriptor(header) 239 240 test_abc = path.join(runner.tmp_dir, "%s.abc" % self.test_id) 241 test_an = path.join(runner.tmp_dir, "%s.an" % self.test_id) 242 243 directory = path.dirname(test_abc) 244 os.makedirs(directory, exist_ok=True) 245 246 cmd = runner.cmd_prefix + [runner.es2panda] 247 if self.with_optimizer: 248 cmd.append('--opt-level=2') 249 cmd.extend(['--thread=0', '--output=%s' % (test_abc)]) 250 251 if 'module' in desc['flags']: 252 cmd.append("--module") 253 254 if 'noStrict' in desc['flags']: 255 self.skipped = True 256 return self 257 258 cmd.append(self.path) 259 260 self.log_cmd(cmd) 261 262 if runner.args.verbose: 263 print('Run es2panda: %s' % ' '.join(cmd), file=sys.stderr) 264 265 process = subprocess.Popen( 266 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=runner.cmd_env) 267 268 try: 269 output_res, err = process.communicate(runner.args.es2panda_timeout) 270 except subprocess.TimeoutExpired: 271 process.kill() 272 self.passed = False 273 self.fail_kind = self.FailKind.ES2PANDA_TIMEOUT 274 self.error = self.fail_kind.name 275 return self 276 277 out = output_res.decode("utf-8", errors="ignore") 278 err = err.decode("utf-8", errors="ignore") 279 self.passed, need_exec = runner.util.validate_parse_result( 280 process.returncode, err, desc, out) 281 282 if not self.passed: 283 self.fail_kind = self.FailKind.ES2PANDA_FAIL 284 self.error = "out:{}\nerr:{}\ncode:{}".format( 285 out, err, process.returncode) 286 print(self.error) 287 return self 288 289 if not need_exec: 290 self.passed = True 291 return self 292 293 if runner.args.aot: 294 cmd = runner.cmd_prefix + [runner.arkaot] 295 cmd.extend(runner.aot_args) 296 cmd.extend(['--paoc-panda-files', test_abc]) 297 cmd.extend(['--paoc-output', test_an]) 298 299 if os.path.isfile(test_an): 300 os.remove(test_an) 301 302 self.log_cmd(cmd) 303 304 if runner.args.verbose: 305 print('Run ark_aot: %s' % ' '.join(cmd), file=sys.stderr) 306 307 process = subprocess.Popen( 308 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=runner.cmd_env) 309 310 try: 311 out, err = process.communicate(runner.args.paoc_timeout) 312 except subprocess.TimeoutExpired: 313 process.kill() 314 self.passed = False 315 self.fail_kind = self.FailKind.AOT_TIMEOUT 316 self.error = self.fail_kind.name 317 return self 318 319 if process.returncode != 0: 320 self.passed = False 321 self.fail_kind = self.FailKind.AOT_FAIL 322 self.error = err.decode("utf-8", errors="ignore") 323 return self 324 325 cmd = runner.cmd_prefix + [runner.runtime] 326 327 if runner.args.verbose: 328 print('Run aot for arm64: %s' % ' '.join(cmd), file=sys.stderr) 329 330 cmd.extend(runner.runtime_args) 331 332 if runner.args.aot: 333 cmd.extend(['--aot-files', test_an]) 334 335 if runner.args.jit: 336 cmd.extend(['--compiler-enable-jit=true', '--compiler-hotness-threshold=0']) 337 else: 338 cmd.extend(['--compiler-enable-jit=false']) 339 340 cmd.extend([test_abc, "_GLOBAL::func_main_0"]) 341 342 self.log_cmd(cmd) 343 344 if runner.args.verbose: 345 print('Run ark: %s' % ' '.join(cmd), file=sys.stderr) 346 347 process = subprocess.Popen( 348 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=runner.cmd_env) 349 350 try: 351 out, err = process.communicate(timeout=runner.args.timeout) 352 except subprocess.TimeoutExpired: 353 process.kill() 354 self.passed = False 355 self.fail_kind = self.FailKind.RUNTIME_TIMEOUT 356 self.error = self.fail_kind.name 357 return self 358 359 out = out.decode("utf-8", errors="ignore") 360 err = err.decode("utf-8", errors="ignore") 361 self.passed = runner.util.validate_runtime_result( 362 process.returncode, err, desc, out) 363 364 if not self.passed: 365 self.fail_kind = self.FailKind.RUNTIME_FAIL 366 self.error = "out:{}\nerr:{}\ncode:{}".format( 367 out, err, process.returncode) 368 print(self.error) 369 370 return self 371 372 373class TSCTest(Test): 374 def __init__(self, test_path, flags): 375 Test.__init__(self, test_path, flags) 376 self.options = self.parse_options() 377 378 def parse_options(self): 379 test_options = {} 380 381 with open(self.path, "r", encoding="latin1") as f: 382 lines = f.read() 383 options = re.findall(r"//\s?@\w+:.*\n", lines) 384 385 for option in options: 386 separated = option.split(":") 387 opt = re.findall(r"\w+", separated[0])[0].lower() 388 value = separated[1].strip().lower() 389 390 if opt == "filename": 391 if opt in options: 392 test_options[opt].append(value) 393 else: 394 test_options[opt] = [value] 395 396 elif opt == "lib" or opt == "module": 397 test_options[opt] = [each.strip() 398 for each in value.split(",")] 399 elif value == "true" or value == "false": 400 test_options[opt] = value.lower() == "true" 401 else: 402 test_options[opt] = value 403 404 # TODO: Possibility of error: all exports will be catched, even the commented ones 405 if 'module' not in test_options and re.search(r"export ", lines): 406 test_options['module'] = [] 407 408 return test_options 409 410 def run(self, runner): 411 cmd = runner.cmd_prefix + [runner.es2panda, '--parse-only'] 412 cmd.extend(self.flags) 413 if "module" in self.options: 414 cmd.append('--module') 415 cmd.append(self.path) 416 417 self.log_cmd(cmd) 418 process = subprocess.Popen( 419 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 420 out, err = process.communicate() 421 self.output = out.decode("utf-8", errors="ignore") 422 423 self.passed = True if process.returncode == 0 else False 424 425 if not self.passed: 426 self.error = err.decode("utf-8", errors="ignore") 427 428 return self 429 430 431class Runner: 432 def __init__(self, args, name): 433 self.test_root = path.dirname(path.abspath(__file__)) 434 self.args = args 435 self.name = name 436 self.tests = [] 437 self.failed = 0 438 self.passed = 0 439 self.es2panda = path.join(args.build_dir, 'es2abc') 440 self.build_dir = args.build_dir 441 self.cmd_prefix = [] 442 self.ark_js_vm = "" 443 self.ark_aot_compiler = "" 444 self.ld_library_path = "" 445 446 if args.js_runtime_path: 447 self.ark_js_vm = path.join(args.js_runtime_path, 'ark_js_vm') 448 self.ark_aot_compiler = path.join(args.js_runtime_path, 'ark_aot_compiler') 449 450 if args.ld_library_path: 451 self.ld_library_path = args.ld_library_path 452 453 if args.arm64_qemu: 454 self.cmd_prefix = ["qemu-aarch64", "-L", "/usr/aarch64-linux-gnu/"] 455 456 if args.arm32_qemu: 457 self.cmd_prefix = ["qemu-arm", "-L", "/usr/arm-linux-gnueabi"] 458 459 if not path.isfile(self.es2panda): 460 raise Exception("Cannot find es2panda binary: %s" % self.es2panda) 461 462 def add_directory(self, directory, extension, flags): 463 pass 464 465 def test_path(self, src): 466 pass 467 468 def run_test(self, test): 469 return test.run(self) 470 471 def run(self): 472 pool = multiprocessing.Pool() 473 result_iter = pool.imap_unordered( 474 self.run_test, self.tests, chunksize=32) 475 pool.close() 476 477 if self.args.progress: 478 from tqdm import tqdm 479 result_iter = tqdm(result_iter, total=len(self.tests)) 480 481 results = [] 482 for res in result_iter: 483 results.append(res) 484 485 self.tests = results 486 pool.join() 487 488 def deal_error(self, test): 489 path_str = test.path 490 err_col = {} 491 if test.error: 492 err_str = test.error.split('[')[0] if "patchfix" not in test.path else " patchfix throw error failed" 493 err_col = {"path" : [path_str], "status": ["fail"], "error" : [test.error], "type" : [err_str]} 494 else: 495 err_col = {"path" : [path_str], "status": ["fail"], "error" : ["Segmentation fault"], 496 "type" : ["Segmentation fault"]} 497 return err_col 498 499 def summarize(self): 500 print("") 501 fail_list = [] 502 success_list = [] 503 504 for test in self.tests: 505 assert(test.passed is not None) 506 if not test.passed: 507 fail_list.append(test) 508 else: 509 success_list.append(test) 510 511 if len(fail_list): 512 if self.args.error: 513 import pandas as pd 514 test_list = pd.DataFrame(columns=["path", "status", "error", "type"]) 515 for test in success_list: 516 suc_col = {"path" : [test.path], "status": ["success"], "error" : ["success"], "type" : ["success"]} 517 if self.args.error: 518 test_list = pd.concat([test_list, pd.DataFrame(suc_col)]) 519 print("Failed tests:") 520 for test in fail_list: 521 print(self.test_path(test.path)) 522 523 if self.args.error: 524 print("steps:", test.reproduce) 525 print("error:") 526 print(test.error) 527 print("\n") 528 err_col = self.deal_error(test) 529 test_list = pd.concat([test_list, pd.DataFrame(err_col)]) 530 531 if self.args.error: 532 test_list.to_csv('test_statistics.csv', index=False) 533 test_list["type"].value_counts().to_csv('type_statistics.csv', index_label="error") 534 print("Type statistics:\n", test_list["type"].value_counts()) 535 print("") 536 537 print("Summary(%s):" % self.name) 538 print("\033[37mTotal: %5d" % (len(self.tests))) 539 print("\033[92mPassed: %5d" % (len(self.tests) - len(fail_list))) 540 print("\033[91mFailed: %5d" % (len(fail_list))) 541 print("\033[0m") 542 543 return len(fail_list) 544 545 546class RegressionRunner(Runner): 547 def __init__(self, args): 548 Runner.__init__(self, args, "Regression") 549 550 def add_directory(self, directory, extension, flags, func=Test): 551 glob_expression = path.join( 552 self.test_root, directory, "*.%s" % (extension)) 553 files = glob(glob_expression) 554 files = fnmatch.filter(files, self.test_root + '**' + self.args.filter) 555 556 self.tests += list(map(lambda f: func(f, flags), files)) 557 558 def test_path(self, src): 559 return src 560 561 562class Test262Runner(Runner): 563 def __init__(self, args): 564 Runner.__init__(self, args, "Test262 ark"), 565 566 self.cmd_env = os.environ.copy() 567 for san in ["ASAN_OPTIONS", "TSAN_OPTIONS", "MSAN_OPTIONS", "LSAN_OPTIONS"]: 568 # we don't want to interpret asan failures as SyntaxErrors 569 self.cmd_env[san] = ":exitcode=255" 570 571 self.update = args.update 572 self.enable_skiplists = False if self.update else args.skip 573 self.normal_skiplist_file = "test262skiplist.txt" 574 self.long_flaky_skiplist_files = ["test262skiplist-long.txt", "test262skiplist-flaky.txt"] 575 self.normal_skiplist = set([]) 576 self.runtime = path.join(args.build_dir, 'bin', 'ark') 577 if not path.isfile(self.runtime): 578 raise Exception("Cannot find runtime binary: %s" % self.runtime) 579 580 self.runtime_args = [ 581 '--boot-panda-files=%s/pandastdlib/arkstdlib.abc' 582 % args.build_dir, 583 '--load-runtimes=ecmascript', 584 '--gc-type=%s' % args.gc_type, 585 ] 586 587 if not args.no_gip: 588 self.runtime_args += ['--run-gc-in-place'] 589 590 if args.aot: 591 self.arkaot = path.join(args.build_dir, 'bin', 'ark_aot') 592 if not path.isfile(self.arkaot): 593 raise Exception("Cannot find aot binary: %s" % self.arkaot) 594 595 self.aot_args = [ 596 '--boot-panda-files=%s/pandastdlib/arkstdlib.abc' 597 % args.build_dir, 598 '--load-runtimes=ecmascript', 599 '--gc-type=%s' % args.gc_type, 600 ] 601 602 if not args.no_gip: 603 self.aot_args += ['--run-gc-in-place'] 604 605 self.aot_args += args.aot_args 606 else: 607 self.aot_args = [] 608 609 self.skiplist_name_list = self.long_flaky_skiplist_files if self.update else [] 610 self.skiplist_bco_name = "" 611 612 if self.enable_skiplists: 613 self.skiplist_name_list.append(self.normal_skiplist_file) 614 self.skiplist_name_list.extend(self.long_flaky_skiplist_files) 615 616 if args.bco: 617 self.skiplist_bco_name = "test262skiplist-bco.txt" 618 if args.arm64_compiler_skip: 619 self.skiplist_name_list.append("test262skiplist-compiler-arm64.txt") 620 621 self.tmp_dir = path.join(path.sep, 'tmp', 'panda', 'test262') 622 os.makedirs(self.tmp_dir, exist_ok=True) 623 624 self.util = test262util.Test262Util() 625 self.test262_dir = self.util.generate( 626 '281eb10b2844929a7c0ac04527f5b42ce56509fd', 627 args.build_dir, 628 path.join(self.test_root, "test262harness.js"), 629 args.progress) 630 631 self.add_directory(self.test262_dir, "js", args.test_list, []) 632 633 def add_directory(self, directory, extension, test_list_path, flags): 634 glob_expression = path.join(directory, "**/*.%s" % (extension)) 635 files = glob(glob_expression, recursive=True) 636 files = fnmatch.filter(files, path.join(directory, self.args.filter)) 637 638 def load_list(p): 639 with open(p, 'r') as fp: 640 return set(map(lambda e: path.join(directory, e.strip()), fp)) 641 642 skiplist = set([]) 643 644 for sl in self.skiplist_name_list: 645 skiplist.update(load_list(path.join(self.test_root, sl))) 646 647 if self.update: 648 self.normal_skiplist.update(load_list(path.join(self.test_root, self.normal_skiplist_file))) 649 650 skiplist_bco = set([]) 651 if self.skiplist_bco_name != "": 652 skiplist_bco = load_list(path.join(self.test_root, self.skiplist_bco_name)) 653 654 if test_list_path is not None: 655 test_list = load_list(path.abspath(test_list_path)) 656 files = filter(lambda f: f in test_list, files) 657 658 def get_test_id(file): 659 return path.relpath(path.splitext(file)[0], self.test262_dir) 660 661 self.tests = list(map(lambda test: Test262Test(test, flags, get_test_id(test), test not in skiplist_bco), 662 filter(lambda f: f not in skiplist, files))) 663 664 def test_path(self, src): 665 return path.relpath(src, self.test262_dir) 666 667 def run(self): 668 Runner.run(self) 669 self.update_skiplist() 670 671 def summarize(self): 672 print("") 673 674 fail_lists = {} 675 for kind in Test262Test.FailKind: 676 fail_lists[kind] = [] 677 678 num_failed = 0 679 num_skipped = 0 680 for test in self.tests: 681 if test.skipped: 682 num_skipped += 1 683 continue 684 685 assert(test.passed is not None) 686 if not test.passed: 687 fail_lists[test.fail_kind].append(test) 688 num_failed += 1 689 690 def summarize_list(name, tests_list): 691 if len(tests_list): 692 tests_list.sort(key=lambda test: test.path) 693 print("# " + name) 694 for test in tests_list: 695 print(self.test_path(test.path)) 696 if self.args.error: 697 print("steps:", test.reproduce) 698 print(test.error) 699 print("") 700 701 total_tests = len(self.tests) - num_skipped 702 703 if not self.update: 704 for kind in Test262Test.FailKind: 705 summarize_list(kind.name, fail_lists[kind]) 706 707 print("Summary(%s):" % self.name) 708 print("\033[37mTotal: %5d" % (total_tests)) 709 print("\033[92mPassed: %5d" % (total_tests - num_failed)) 710 print("\033[91mFailed: %5d" % (num_failed)) 711 print("\033[0m") 712 713 return num_failed 714 715 def update_skiplist(self): 716 if not self.update: 717 return 718 719 skiplist_es2panda = list({x.test_id + ".js" for x in self.tests 720 if not x.skipped and not x.passed and 721 x.fail_kind == Test262Test.FailKind.ES2PANDA_FAIL}) 722 skiplist_runtime = list({x.test_id + ".js" for x in self.tests 723 if not x.skipped and not x.passed and 724 x.fail_kind == Test262Test.FailKind.RUNTIME_FAIL}) 725 726 skiplist_es2panda.sort() 727 skiplist_runtime.sort() 728 729 new_skiplist = skiplist_es2panda + skiplist_runtime 730 731 new_pass = list(filter(lambda x: len(x) and not x.startswith('#') 732 and x not in new_skiplist, self.normal_skiplist)) 733 new_fail = list(filter(lambda x: x not in self.normal_skiplist, new_skiplist)) 734 new_pass.sort() 735 new_fail.sort() 736 737 if new_pass: 738 print("\033[92mRemoved from skiplist:") 739 print("\n".join(new_pass)) 740 print("\033[0m") 741 742 if new_fail: 743 print("\033[91mNew tests on skiplist:") 744 print("\n".join(new_fail)) 745 print("\033[0m") 746 747 fd = os.open(path.join(self.test_root, self.normal_skiplist_file), os.O_RDWR | os.O_CREAT | os.O_TRUNC) 748 file = os.fdopen(fd, "w+") 749 file.write("\n".join(["# ES2PANDA_FAIL"] + skiplist_es2panda + ["", "# RUNTIME_FAIL"] + skiplist_runtime)) 750 file.write("\n") 751 file.close() 752 753 754class TSCRunner(Runner): 755 def __init__(self, args): 756 Runner.__init__(self, args, "TSC") 757 758 if self.args.tsc_path: 759 self.tsc_path = self.args.tsc_path 760 else : 761 self.tsc_path = prepare_tsc_testcases(self.test_root) 762 763 self.add_directory("conformance", []) 764 self.add_directory("compiler", []) 765 766 def add_directory(self, directory, flags): 767 ts_suite_dir = path.join(self.tsc_path, 'tests/cases') 768 769 glob_expression = path.join( 770 ts_suite_dir, directory, "**/*.ts") 771 files = glob(glob_expression, recursive=True) 772 files = fnmatch.filter(files, ts_suite_dir + '**' + self.args.filter) 773 774 for f in files: 775 test_name = path.basename(f.split(".ts")[0]) 776 negative_references = path.join( 777 self.tsc_path, 'tests/baselines/reference') 778 is_negative = path.isfile(path.join(negative_references, 779 test_name + ".errors.txt")) 780 test = TSCTest(f, flags) 781 782 if 'target' in test.options: 783 targets = test.options['target'].replace(" ", "").split(',') 784 for target in targets: 785 if path.isfile(path.join(negative_references, 786 test_name + "(target=%s).errors.txt" % (target))): 787 is_negative = True 788 break 789 790 if is_negative or "filename" in test.options: 791 continue 792 793 with open(path.join(self.test_root, 'test_tsc_ignore_list.txt'), 'r') as failed_references: 794 if self.args.skip: 795 if path.relpath(f, self.tsc_path) in failed_references.read(): 796 continue 797 798 self.tests.append(test) 799 800 def test_path(self, src): 801 return src 802 803 804class CompilerRunner(Runner): 805 def __init__(self, args): 806 Runner.__init__(self, args, "Compiler") 807 808 def add_directory(self, directory, extension, flags): 809 if directory.endswith("projects"): 810 projects_path = path.join(self.test_root, directory) 811 for project in os.listdir(projects_path): 812 glob_expression = path.join(projects_path, project, "**/*.%s" % (extension)) 813 files = glob(glob_expression, recursive=True) 814 files = fnmatch.filter(files, self.test_root + '**' + self.args.filter) 815 self.tests.append(CompilerProjectTest(projects_path, project, files, flags)) 816 else: 817 glob_expression = path.join( 818 self.test_root, directory, "**/*.%s" % (extension)) 819 files = glob(glob_expression, recursive=True) 820 files = fnmatch.filter(files, self.test_root + '**' + self.args.filter) 821 self.tests += list(map(lambda f: CompilerTest(f, flags), files)) 822 823 def test_path(self, src): 824 return src 825 826 827class CompilerTest(Test): 828 def __init__(self, test_path, flags): 829 Test.__init__(self, test_path, flags) 830 831 def run(self, runner): 832 test_abc_name = ("%s.abc" % (path.splitext(self.path)[0])).replace("/", "_") 833 test_abc_path = path.join(runner.build_dir, test_abc_name) 834 es2abc_cmd = runner.cmd_prefix + [runner.es2panda] 835 es2abc_cmd.extend(self.flags) 836 es2abc_cmd.extend(["--output=" + test_abc_path]) 837 es2abc_cmd.append(self.path) 838 self.log_cmd(es2abc_cmd) 839 840 process = subprocess.Popen(es2abc_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 841 out, err = process.communicate() 842 if err: 843 self.passed = False 844 self.error = err.decode("utf-8", errors="ignore") 845 return self 846 847 ld_library_path = runner.ld_library_path 848 os.environ.setdefault("LD_LIBRARY_PATH", ld_library_path) 849 run_abc_cmd = [runner.ark_js_vm] 850 run_abc_cmd.extend([test_abc_path]) 851 self.log_cmd(run_abc_cmd) 852 853 process = subprocess.Popen(run_abc_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 854 out, err = process.communicate() 855 self.output = out.decode("utf-8", errors="ignore") + err.decode("utf-8", errors="ignore") 856 expected_path = self.get_path_to_expected() 857 try: 858 with open(expected_path, 'r') as fp: 859 expected = fp.read() 860 self.passed = expected == self.output and process.returncode in [0, 1] 861 except Exception: 862 self.passed = False 863 864 if not self.passed: 865 self.error = err.decode("utf-8", errors="ignore") 866 867 os.remove(test_abc_path) 868 869 return self 870 871 872class CompilerProjectTest(Test): 873 def __init__(self, projects_path, project, test_paths, flags): 874 Test.__init__(self, "", flags) 875 self.projects_path = projects_path 876 self.project = project 877 self.test_paths = test_paths 878 self.files_info_path = os.path.join(os.path.join(self.projects_path, self.project), 'filesInfo.txt') 879 880 def remove_project(self, runner): 881 project_path = runner.build_dir + "/" + self.project 882 if path.exists(project_path): 883 shutil.rmtree(project_path) 884 if path.exists(self.files_info_path): 885 os.remove(self.files_info_path) 886 887 def get_file_absolute_path_and_name(self, runner): 888 sub_path = self.path[len(self.projects_path):] 889 file_relative_path = path.split(sub_path)[0] 890 file_name = path.split(sub_path)[1] 891 file_absolute_path = runner.build_dir + "/" + file_relative_path 892 return [file_absolute_path, file_name] 893 894 def gen_single_abc(self, runner): 895 for test_path in self.test_paths: 896 self.path = test_path 897 [file_absolute_path, file_name] = self.get_file_absolute_path_and_name(runner) 898 if not path.exists(file_absolute_path): 899 os.makedirs(file_absolute_path) 900 901 test_abc_name = ("%s.abc" % (path.splitext(file_name)[0])) 902 test_abc_path = path.join(file_absolute_path, test_abc_name) 903 es2abc_cmd = runner.cmd_prefix + [runner.es2panda] 904 es2abc_cmd.extend(self.flags) 905 es2abc_cmd.extend(['%s%s' % ("--output=", test_abc_path)]) 906 es2abc_cmd.append(self.path) 907 self.log_cmd(es2abc_cmd) 908 909 process = subprocess.Popen(es2abc_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 910 out, err = process.communicate() 911 if err: 912 self.passed = False 913 self.error = err.decode("utf-8", errors="ignore") 914 self.remove_project(runner) 915 return self 916 917 def gen_files_info(self, runner): 918 fd = os.open(self.files_info_path, os.O_RDWR | os.O_CREAT | os.O_TRUNC) 919 f = os.fdopen(fd, "w") 920 for test_path in self.test_paths: 921 record_name = os.path.relpath(test_path, os.path.dirname(self.files_info_path)).split('.')[0] 922 module_kind = "esm" 923 file_info = ('%s;%s;%s;%s;%s' % (test_path, record_name, module_kind, test_path, record_name)) 924 f.writelines(file_info + '\n') 925 f.close() 926 927 def gen_merged_abc(self, runner): 928 for test_path in self.test_paths: 929 self.path = test_path 930 if (self.path.endswith("-exec.ts")): 931 exec_file_path = self.path 932 [file_absolute_path, file_name] = self.get_file_absolute_path_and_name(runner) 933 if not path.exists(file_absolute_path): 934 os.makedirs(file_absolute_path) 935 test_abc_name = ("%s.abc" % (path.splitext(file_name)[0])) 936 output_abc_name = path.join(file_absolute_path, test_abc_name) 937 es2abc_cmd = runner.cmd_prefix + [runner.es2panda] 938 es2abc_cmd.extend(self.flags) 939 es2abc_cmd.extend(['%s%s' % ("--output=", output_abc_name)]) 940 es2abc_cmd.append('@' + os.path.join(os.path.dirname(exec_file_path), "filesInfo.txt")) 941 self.log_cmd(es2abc_cmd) 942 process = subprocess.Popen(es2abc_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 943 out, err = process.communicate() 944 if err: 945 self.passed = False 946 self.error = err.decode("utf-8", errors="ignore") 947 self.remove_project(runner) 948 return self 949 950 def run(self, runner): 951 # Compile all ts source files in the project to abc files. 952 if ("--merge-abc" in self.flags): 953 self.gen_files_info(runner) 954 self.gen_merged_abc(runner) 955 else: 956 self.gen_single_abc(runner) 957 958 # Run test files that need to be executed in the project. 959 for test_path in self.test_paths: 960 self.path = test_path 961 if self.path.endswith("-exec.ts"): 962 [file_absolute_path, file_name] = self.get_file_absolute_path_and_name(runner) 963 964 entry_point_name = path.splitext(file_name)[0] 965 test_abc_name = ("%s.abc" % entry_point_name) 966 test_abc_path = path.join(file_absolute_path, test_abc_name) 967 968 ld_library_path = runner.ld_library_path 969 os.environ.setdefault("LD_LIBRARY_PATH", ld_library_path) 970 run_abc_cmd = [runner.ark_js_vm] 971 if ("--merge-abc" in self.flags): 972 run_abc_cmd.extend(["--entry-point", entry_point_name]) 973 run_abc_cmd.extend([test_abc_path]) 974 self.log_cmd(run_abc_cmd) 975 976 process = subprocess.Popen(run_abc_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 977 out, err = process.communicate() 978 self.output = out.decode("utf-8", errors="ignore") + err.decode("utf-8", errors="ignore") 979 expected_path = self.get_path_to_expected() 980 try: 981 with open(expected_path, 'r') as fp: 982 expected = fp.read() 983 self.passed = expected == self.output and process.returncode in [0, 1] 984 except Exception: 985 self.passed = False 986 987 if not self.passed: 988 self.error = err.decode("utf-8", errors="ignore") 989 self.remove_project(runner) 990 return self 991 992 self.passed = True 993 994 self.remove_project(runner) 995 return self 996 997 998class TSDeclarationTest(Test): 999 def get_path_to_expected(self): 1000 file_name = self.path[:self.path.find(".d.ts")] 1001 return "%s-expected.txt" % file_name 1002 1003 1004class TransformerRunner(Runner): 1005 def __init__(self, args): 1006 Runner.__init__(self, args, "Transformer") 1007 1008 def add_directory(self, directory, extension, flags): 1009 glob_expression = path.join( 1010 self.test_root, directory, "**/*.%s" % (extension)) 1011 files = glob(glob_expression, recursive=True) 1012 files = fnmatch.filter(files, self.test_root + '**' + self.args.filter) 1013 1014 self.tests += list(map(lambda f: TransformerTest(f, flags), files)) 1015 1016 def test_path(self, src): 1017 return src 1018 1019 1020class TransformerTest(Test): 1021 def __init__(self, test_path, flags): 1022 Test.__init__(self, test_path, flags) 1023 1024 def get_path_to_expected(self): 1025 return "%s-transformed-expected.txt" % (path.splitext(self.path)[0]) 1026 1027 def run(self, runner): 1028 cmd = runner.cmd_prefix + [runner.es2panda] 1029 cmd.extend(self.flags) 1030 cmd.append(self.path) 1031 1032 self.log_cmd(cmd) 1033 process = subprocess.Popen( 1034 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 1035 out, err = process.communicate() 1036 self.output = out.decode("utf-8", errors="ignore") + err.decode("utf-8", errors="ignore") 1037 1038 expected_path = self.get_path_to_expected() 1039 try: 1040 with open(expected_path, 'r') as fp: 1041 expected = fp.read() 1042 self.passed = expected == self.output and process.returncode in [0, 1] 1043 except Exception: 1044 self.passed = False 1045 1046 if not self.passed: 1047 self.error = err.decode("utf-8", errors="ignore") 1048 1049 return self 1050 1051 1052class PatchTest(Test): 1053 def __init__(self, test_path, mode_arg): 1054 Test.__init__(self, test_path, "") 1055 self.mode = mode_arg 1056 1057 def run(self, runner): 1058 symbol_table_file = 'base.map' 1059 origin_input_file = 'base.js' 1060 origin_output_abc = 'base.abc' 1061 modified_input_file = 'base_mod.js' 1062 modified_output_abc = 'patch.abc' 1063 1064 gen_base_cmd = runner.cmd_prefix + [runner.es2panda, '--module'] 1065 if 'record-name-with-dots' in os.path.basename(self.path): 1066 gen_base_cmd.extend(['--merge-abc', '--record-name=record.name.with.dots']) 1067 gen_base_cmd.extend(['--dump-symbol-table', os.path.join(self.path, symbol_table_file)]) 1068 gen_base_cmd.extend(['--output', os.path.join(self.path, origin_output_abc)]) 1069 gen_base_cmd.extend([os.path.join(self.path, origin_input_file)]) 1070 self.log_cmd(gen_base_cmd) 1071 1072 if self.mode == 'hotfix': 1073 mode_arg = ["--generate-patch"] 1074 elif self.mode == 'hotreload': 1075 mode_arg = ["--hot-reload"] 1076 elif self.mode == 'coldfix': 1077 mode_arg = ["--generate-patch", "--cold-fix"] 1078 1079 patch_test_cmd = runner.cmd_prefix + [runner.es2panda, '--module'] 1080 patch_test_cmd.extend(mode_arg) 1081 patch_test_cmd.extend(['--input-symbol-table', os.path.join(self.path, symbol_table_file)]) 1082 patch_test_cmd.extend(['--output', os.path.join(self.path, modified_output_abc)]) 1083 patch_test_cmd.extend([os.path.join(self.path, modified_input_file)]) 1084 if 'record-name-with-dots' in os.path.basename(self.path): 1085 patch_test_cmd.extend(['--merge-abc', '--record-name=record.name.with.dots']) 1086 if ('modify-anon-content-keep-origin-name' in os.path.basename(self.path) or 1087 'modify-class-memeber-function' in os.path.basename(self.path)): 1088 patch_test_cmd.extend(['--dump-assembly']) 1089 self.log_cmd(patch_test_cmd) 1090 process_base = subprocess.Popen(gen_base_cmd, stdout=subprocess.PIPE, 1091 stderr=subprocess.PIPE) 1092 stdout_base, stderr_base = process_base.communicate(timeout=runner.args.es2panda_timeout) 1093 if stderr_base: 1094 self.passed = False 1095 self.error = stderr_base.decode("utf-8", errors="ignore") 1096 self.output = stdout_base.decode("utf-8", errors="ignore") + stderr_base.decode("utf-8", errors="ignore") 1097 else: 1098 process_patch = subprocess.Popen(patch_test_cmd, stdout=subprocess.PIPE, 1099 stderr=subprocess.PIPE) 1100 stdout_patch, stderr_patch = process_patch.communicate(timeout=runner.args.es2panda_timeout) 1101 if stderr_patch: 1102 self.passed = False 1103 self.error = stderr_patch.decode("utf-8", errors="ignore") 1104 self.output = stdout_patch.decode("utf-8", errors="ignore") + stderr_patch.decode("utf-8", errors="ignore") 1105 1106 expected_path = os.path.join(self.path, 'expected.txt') 1107 try: 1108 with open(expected_path, 'r') as fp: 1109 # ignore license description lines and skip leading blank lines 1110 expected = (''.join((fp.readlines()[12:]))).lstrip() 1111 self.passed = expected == self.output 1112 except Exception: 1113 self.passed = False 1114 1115 if not self.passed: 1116 self.error = "expected output:" + os.linesep + expected + os.linesep + "actual output:" + os.linesep +\ 1117 self.output 1118 1119 return self 1120 1121 1122class PatchRunner(Runner): 1123 def __init__(self, args, name): 1124 Runner.__init__(self, args, name) 1125 self.preserve_files = args.error 1126 1127 def __del__(self): 1128 if not self.preserve_files: 1129 self.clear_directory() 1130 1131 def add_directory(self): 1132 self.tests_in_dirs = [] 1133 for item in self.test_directory: 1134 glob_expression = path.join(item, "*") 1135 self.tests_in_dirs += glob(glob_expression, recursive=False) 1136 1137 def clear_directory(self): 1138 for test in self.tests_in_dirs: 1139 files_in_dir = os.listdir(test) 1140 filtered_files = [file for file in files_in_dir if file.endswith(".map") or file.endswith(".abc")] 1141 for file in filtered_files: 1142 os.remove(os.path.join(test, file)) 1143 1144 def test_path(self, src): 1145 return os.path.basename(src) 1146 1147 1148class HotfixRunner(PatchRunner): 1149 def __init__(self, args): 1150 PatchRunner.__init__(self, args, "Hotfix") 1151 self.test_directory = [path.join(self.test_root, "hotfix", "hotfix-throwerror"), 1152 path.join(self.test_root, "hotfix", "hotfix-noerror")] 1153 self.add_directory() 1154 self.tests += list(map(lambda t: PatchTest(t, "hotfix"), self.tests_in_dirs)) 1155 1156 1157class HotreloadRunner(PatchRunner): 1158 def __init__(self, args): 1159 PatchRunner.__init__(self, args, "Hotreload") 1160 self.test_directory = [path.join(self.test_root, "hotreload", "hotreload-throwerror"), 1161 path.join(self.test_root, "hotreload", "hotreload-noerror")] 1162 self.add_directory() 1163 self.tests += list(map(lambda t: PatchTest(t, "hotreload"), self.tests_in_dirs)) 1164 1165 1166class ColdfixRunner(PatchRunner): 1167 def __init__(self, args): 1168 PatchRunner.__init__(self, args, "Coldfix") 1169 self.test_directory = [path.join(self.test_root, "coldfix", "coldfix-throwerror"), 1170 path.join(self.test_root, "coldfix", "coldfix-noerror")] 1171 self.add_directory() 1172 self.tests += list(map(lambda t: PatchTest(t, "coldfix"), self.tests_in_dirs)) 1173 1174 1175class Base64Test(Test): 1176 def __init__(self, test_path, input_type): 1177 Test.__init__(self, test_path, "") 1178 self.input_type = input_type 1179 1180 def run(self, runner): 1181 cmd = runner.cmd_prefix + [runner.es2panda, "--base64Output"] 1182 if self.input_type == "file": 1183 input_file_name = 'input.js' 1184 cmd.extend(['--source-file', input_file_name]) 1185 cmd.extend([os.path.join(self.path, input_file_name)]) 1186 elif self.input_type == "string": 1187 input_file = os.path.join(self.path, "input.txt") 1188 try: 1189 with open(input_file, 'r') as fp: 1190 base64_input = (''.join((fp.readlines()[12:]))).lstrip() # ignore license description lines 1191 cmd.extend(["--base64Input", base64_input]) 1192 except Exception: 1193 self.passed = False 1194 else: 1195 self.error = "Unsupported base64 input type" 1196 self.passed = False 1197 return self 1198 1199 self.log_cmd(cmd) 1200 1201 process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 1202 stdout, stderr = process.communicate(timeout=runner.args.es2panda_timeout) 1203 if stderr: 1204 self.passed = False 1205 self.error = stderr.decode("utf-8", errors="ignore") 1206 return self 1207 1208 self.output = stdout.decode("utf-8", errors="ignore") 1209 1210 expected_path = os.path.join(self.path, 'expected.txt') 1211 try: 1212 with open(expected_path, 'r') as fp: 1213 expected = (''.join((fp.readlines()[12:]))).lstrip() 1214 self.passed = expected == self.output 1215 except Exception: 1216 self.passed = False 1217 1218 if not self.passed: 1219 self.error = "expected output:" + os.linesep + expected + os.linesep + "actual output:" + os.linesep +\ 1220 self.output 1221 1222 return self 1223 1224 1225class Base64Runner(Runner): 1226 def __init__(self, args): 1227 Runner.__init__(self, args, "Base64") 1228 self.test_directory = path.join(self.test_root, "base64") 1229 self.add_test() 1230 1231 def add_test(self): 1232 self.tests = [] 1233 self.tests.append(Base64Test(os.path.join(self.test_directory, "inputFile"), "file")) 1234 self.tests.append(Base64Test(os.path.join(self.test_directory, "inputString"), "string")) 1235 1236 def test_path(self, src): 1237 return os.path.basename(src) 1238 1239 1240class TypeExtractorRunner(Runner): 1241 def __init__(self, args): 1242 Runner.__init__(self, args, "TypeExtractor") 1243 1244 if self.args.tsc_path: 1245 self.tsc_path = self.args.tsc_path 1246 else : 1247 self.tsc_path = prepare_tsc_testcases(self.test_root) 1248 1249 self.add_tsc_directory("conformance", []) 1250 self.add_directory("testcases", []) 1251 self.add_directory("dts-testcases", [], True) 1252 self.add_directory("testcases_with_assert", []) 1253 self.add_directory("testcases_with_assert/projects", [], False, True) 1254 self.add_directory("testcases_with_running", []) 1255 1256 def add_tsc_directory(self, directory, flags): 1257 ts_suite_dir = path.join(self.tsc_path, 'tests/cases') 1258 1259 glob_expression = path.join( 1260 ts_suite_dir, directory, "**/*.ts") 1261 files = glob(glob_expression, recursive=True) 1262 files = fnmatch.filter(files, ts_suite_dir + '**' + self.args.filter) 1263 1264 with open(path.join(self.test_root, 'type_extractor/testlist.txt'), 'r') as passed_references: 1265 for f in files: 1266 if path.relpath(f, self.tsc_path) in passed_references.read(): 1267 test = TypeExtractorTest(f, flags) 1268 self.tests.append(test) 1269 1270 def add_directory(self, directory, flags, is_dts_test=False, is_project=False): 1271 ts_suite_dir = path.join(self.test_root, 'type_extractor') 1272 1273 if is_project: 1274 glob_expression = path.join(ts_suite_dir, directory, "**/*-main.ts") 1275 elif is_dts_test: 1276 glob_expression = path.join(ts_suite_dir, directory, "**/*.d.ts") 1277 else: 1278 glob_expression = path.join(ts_suite_dir, directory, "*.ts") 1279 files = glob(glob_expression, recursive=True) 1280 files = fnmatch.filter(files, ts_suite_dir + '**' + self.args.filter) 1281 for f in files: 1282 if directory.startswith("testcases_with_assert") or directory.startswith("testcases_with_running"): 1283 if (self.ld_library_path == "" or self.ark_aot_compiler == ""): 1284 break 1285 test = TypeExtractorWithAOTTest(f, flags, directory.startswith("testcases_with_running"), directory.endswith("projects")) 1286 self.tests.append(test) 1287 else: 1288 test = TypeExtractorTest(f, flags, is_dts_test) 1289 self.tests.append(test) 1290 1291 def test_path(self, src): 1292 return src 1293 1294 1295class TypeExtractorTest(Test): 1296 def __init__(self, test_path, flags, is_dts_test=False): 1297 Test.__init__(self, test_path, flags) 1298 self.is_dts_test = is_dts_test 1299 1300 def run(self, runner): 1301 test_abc_name = ("%s.abc" % (path.splitext(self.path)[0])).replace("/", "_") 1302 cmd = runner.cmd_prefix + [runner.es2panda, 1303 '--module', '--dump-literal-buffer', '--opt-level=2', '--type-extractor'] 1304 if self.is_dts_test: 1305 cmd.append("--type-dts-builtin") 1306 cmd.extend(self.flags) 1307 cmd.extend(["--output=" + test_abc_name]) 1308 cmd.append(self.path) 1309 1310 self.log_cmd(cmd) 1311 process = subprocess.Popen( 1312 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 1313 out, err = process.communicate() 1314 self.output = out.decode("utf-8", errors="ignore") + err.decode("utf-8", errors="ignore") 1315 output_str = self.output.split("======> literal array buffer <======")[1] 1316 1317 if os.path.isfile(test_abc_name): 1318 os.remove(test_abc_name) 1319 1320 file_name = path.splitext(self.path)[0] 1321 if self.is_dts_test: 1322 expected_path = "%s-expected.txt" % (path.splitext(file_name)[0]) 1323 else: 1324 expected_path = "%s-expected.txt" % (file_name) 1325 if not os.path.isfile(expected_path): 1326 expected_path = path.dirname(path.abspath(__file__)) + "/type_extractor/tsc_expect/%s" % \ 1327 expected_path.split("tests/cases/")[-1] 1328 1329 try: 1330 with open(expected_path, 'r') as fp: 1331 expected = fp.read() 1332 expected_str = expected.split("======> literal array buffer <======")[1] 1333 1334 self.passed = expected_str == output_str and process.returncode in [ 1335 0, 1] 1336 except Exception: 1337 self.passed = False 1338 1339 if not self.passed: 1340 self.error = err.decode("utf-8", errors="ignore") 1341 1342 return self 1343 1344 1345class TypeExtractorWithAOTTest(Test): 1346 def __init__(self, test_path, flags, with_running=False, is_project=False): 1347 Test.__init__(self, test_path, flags) 1348 self.with_running = with_running 1349 self.is_project = is_project 1350 1351 def run_js_vm(self, runner, file_name, test_abc_name): 1352 expected_path = "%s-expected.txt" % (file_name) 1353 run_aot_cmd = [runner.ark_js_vm] 1354 run_aot_cmd.extend(["--aot-file=%s" % file_name]) 1355 run_aot_cmd.extend(["--entry-point=%s" % path.basename(file_name)]) 1356 run_aot_cmd.extend([test_abc_name]) 1357 self.log_cmd(run_aot_cmd) 1358 1359 process = subprocess.Popen(run_aot_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 1360 out, err = process.communicate(timeout=runner.args.timeout) 1361 self.output = out.decode("utf-8", errors="ignore") + err.decode("utf-8", errors="ignore") 1362 try: 1363 with open(expected_path, 'r') as fp: 1364 expected = fp.read() 1365 self.passed = expected == self.output and process.returncode in [0, 1] 1366 except Exception: 1367 self.passed = False 1368 1369 if not self.passed: 1370 self.error = err.decode("utf-8", errors="ignore") 1371 1372 if os.path.isfile("%s.an" % (file_name)): 1373 os.remove("%s.an" % (file_name)) 1374 if os.path.isfile("%s.ai" % (file_name)): 1375 os.remove("%s.ai" % (file_name)) 1376 1377 def run(self, runner): 1378 file_name = path.splitext(self.path)[0] 1379 test_abc_name = ("%s.abc" % path.basename(file_name)) 1380 cmd = runner.cmd_prefix + [runner.es2panda, 1381 '--module', '--merge-abc', '--opt-level=2', '--type-extractor'] 1382 cmd.extend(self.flags) 1383 cmd.extend(["--output=" + test_abc_name]) 1384 if self.is_project: 1385 cmd.append("--extension=ts") 1386 cmd.append(path.dirname(self.path)) 1387 else: 1388 cmd.append(self.path) 1389 1390 self.log_cmd(cmd) 1391 process = subprocess.Popen( 1392 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 1393 out, err = process.communicate() 1394 if (err): 1395 self.passed = False 1396 self.error = err.decode("utf-8", errors="ignore") 1397 return self 1398 1399 ld_library_path = runner.ld_library_path 1400 os.environ.setdefault("LD_LIBRARY_PATH", ld_library_path) 1401 aot_abc_cmd = [runner.ark_aot_compiler] 1402 if self.with_running: 1403 aot_abc_cmd.extend(["--aot-file=%s" % file_name]) 1404 else: 1405 aot_abc_cmd.extend(["--compiler-assert-types=true"]) 1406 aot_abc_cmd.extend(["--compiler-opt-type-lowering=false"]) 1407 aot_abc_cmd.extend([test_abc_name]) 1408 self.log_cmd(aot_abc_cmd) 1409 1410 process = subprocess.Popen(aot_abc_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 1411 out, err = process.communicate() 1412 if err: 1413 self.passed = False 1414 self.error = err.decode("utf-8", errors="ignore") 1415 else: 1416 if self.with_running: 1417 self.run_js_vm(runner, file_name, test_abc_name) 1418 else: 1419 self.passed = True 1420 1421 if os.path.isfile(test_abc_name): 1422 os.remove(test_abc_name) 1423 if os.path.isfile("aot_file.an"): 1424 os.remove("aot_file.an") 1425 if os.path.isfile(".ai"): 1426 os.remove(".ai") 1427 1428 return self 1429 1430class BytecodeRunner(Runner): 1431 def __init__(self, args): 1432 Runner.__init__(self, args, "Bytecode") 1433 1434 def add_directory(self, directory, extension, flags, func=Test): 1435 glob_expression = path.join( 1436 self.test_root, directory, "**/*.%s" % (extension)) 1437 files = glob(glob_expression, recursive=True) 1438 files = fnmatch.filter(files, self.test_root + '**' + self.args.filter) 1439 self.tests += list(map(lambda f: func(f, flags), files)) 1440 1441 def test_path(self, src): 1442 return src 1443 1444def main(): 1445 args = get_args() 1446 1447 runners = [] 1448 1449 if args.regression: 1450 runner = RegressionRunner(args) 1451 runner.add_directory("parser/concurrent", "js", ["--module", "--dump-ast"]) 1452 runner.add_directory("parser/js", "js", ["--parse-only", "--dump-ast"]) 1453 runner.add_directory("parser/script", "ts", ["--parse-only", "--dump-ast"]) 1454 runner.add_directory("parser/ts", "ts", 1455 ["--parse-only", "--module", "--dump-ast"]) 1456 runner.add_directory("parser/ts/type_checker", "ts", 1457 ["--parse-only", "--enable-type-check", "--module", "--dump-ast"]) 1458 runner.add_directory("parser/ts/cases/declaration", "d.ts", 1459 ["--parse-only", "--module", "--dump-ast"], TSDeclarationTest) 1460 runner.add_directory("parser/commonjs", "js", ["--commonjs", "--parse-only", "--dump-ast"]) 1461 runner.add_directory("parser/binder", "js", ["--dump-assembly"]) 1462 runner.add_directory("parser/js/emptySource", "js", ["--dump-assembly"]) 1463 runner.add_directory("parser/js/language/arguments-object", "js", ["--parse-only"]) 1464 1465 runners.append(runner) 1466 1467 transformer_runner = TransformerRunner(args) 1468 transformer_runner.add_directory("parser/ts/transformed_cases", "ts", 1469 ["--parse-only", "--module", "--dump-transformed-ast", 1470 "--check-transformed-ast-structure"]) 1471 1472 runners.append(transformer_runner) 1473 1474 if args.test262: 1475 runners.append(Test262Runner(args)) 1476 1477 if args.tsc: 1478 runners.append(TSCRunner(args)) 1479 1480 if args.compiler: 1481 runner = CompilerRunner(args) 1482 runner.add_directory("compiler/js", "js", []) 1483 runner.add_directory("compiler/ts/cases", "ts", []) 1484 runner.add_directory("compiler/ts/projects", "ts", ["--module"]) 1485 runner.add_directory("compiler/ts/projects", "ts", ["--module", "--merge-abc"]) 1486 runner.add_directory("compiler/dts", "d.ts", ["--module", "--opt-level=0"]) 1487 runner.add_directory("compiler/commonjs", "js", ["--commonjs"]) 1488 1489 runners.append(runner) 1490 1491 if args.hotfix: 1492 runners.append(HotfixRunner(args)) 1493 1494 if args.hotreload: 1495 runners.append(HotreloadRunner(args)) 1496 1497 if args.coldfix: 1498 runners.append(ColdfixRunner(args)) 1499 1500 if args.base64: 1501 runners.append(Base64Runner(args)) 1502 1503 if args.type_extractor: 1504 runners.append(TypeExtractorRunner(args)) 1505 1506 if args.bytecode: 1507 runner = BytecodeRunner(args) 1508 runner.add_directory("bytecode/commonjs", "js", ["--commonjs", "--dump-assembly"]) 1509 runner.add_directory("bytecode/js", "js", ["--dump-assembly"]) 1510 1511 runners.append(runner) 1512 1513 failed_tests = 0 1514 1515 for runner in runners: 1516 runner.run() 1517 failed_tests += runner.summarize() 1518 1519 # TODO: exit 1 when we have failed tests after all tests are fixed 1520 exit(0) 1521 1522 1523if __name__ == "__main__": 1524 main() 1525