1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# 4# Copyright (c) 2021-2022 Huawei Device Co., Ltd. 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17from glob import glob 18from os import path 19from enum import Enum 20import argparse 21import fnmatch 22import multiprocessing 23import os 24import re 25import subprocess 26import sys 27import test262util 28 29 30def is_directory(parser, arg): 31 if not path.isdir(arg): 32 parser.error("The directory '%s' does not exist" % arg) 33 34 return path.abspath(arg) 35 36 37def is_file(parser, arg): 38 if not path.isfile(arg): 39 parser.error("The file '%s' does not exist" % arg) 40 41 return path.abspath(arg) 42 43 44def check_timeout(value): 45 ivalue = int(value) 46 if ivalue <= 0: 47 raise argparse.ArgumentTypeError( 48 "%s is an invalid timeout value" % value) 49 return ivalue 50 51 52def get_args(): 53 parser = argparse.ArgumentParser(description="Regression test runner") 54 parser.add_argument( 55 'build_dir', type=lambda arg: is_directory(parser, arg), 56 help='panda build directory') 57 parser.add_argument( 58 '--test262', '-t', action='store_true', dest='test262', default=False, 59 help='run test262 tests') 60 parser.add_argument( 61 '--error', action='store_true', dest='error', default=False, 62 help='capture stderr') 63 parser.add_argument( 64 '--regression', '-r', action='store_true', dest='regression', 65 default=False, help='run regression tests') 66 parser.add_argument( 67 '--compiler', '-c', action='store_true', dest='compiler', 68 default=False, help='run compiler tests') 69 parser.add_argument( 70 '--tsc', action='store_true', dest='tsc', 71 default=False, help='run tsc tests') 72 parser.add_argument( 73 '--no-progress', action='store_false', dest='progress', default=True, 74 help='don\'t show progress bar') 75 parser.add_argument( 76 '--no-skip', action='store_false', dest='skip', default=True, 77 help='don\'t use skiplists') 78 parser.add_argument( 79 '--update', action='store_true', dest='update', default=False, 80 help='update skiplist') 81 parser.add_argument( 82 '--no-run-gc-in-place', action='store_true', dest='no_gip', default=False, 83 help='enable --run-gc-in-place mode') 84 parser.add_argument( 85 '--filter', '-f', action='store', dest='filter', 86 default="*", help='test filter regexp') 87 parser.add_argument( 88 '--es2panda-timeout', type=check_timeout, 89 dest='es2panda_timeout', default=60, help='es2panda translator timeout') 90 parser.add_argument( 91 '--paoc-timeout', type=check_timeout, 92 dest='paoc_timeout', default=600, help='paoc compiler timeout') 93 parser.add_argument( 94 '--timeout', type=check_timeout, 95 dest='timeout', default=10, help='JS runtime timeout') 96 parser.add_argument( 97 '--gc-type', dest='gc_type', default="g1-gc", help='Type of garbage collector') 98 parser.add_argument( 99 '--aot', action='store_true', dest='aot', default=False, 100 help='use AOT compilation') 101 parser.add_argument( 102 '--no-bco', action='store_false', dest='bco', default=True, 103 help='disable bytecodeopt') 104 parser.add_argument( 105 '--jit', action='store_true', dest='jit', default=False, 106 help='use JIT in interpreter') 107 parser.add_argument( 108 '--arm64-compiler-skip', action='store_true', dest='arm64_compiler_skip', default=False, 109 help='use skiplist for tests failing on aarch64 in AOT or JIT mode') 110 parser.add_argument( 111 '--arm64-qemu', action='store_true', dest='arm64_qemu', default=False, 112 help='launch all binaries in qemu aarch64') 113 parser.add_argument( 114 '--arm32-qemu', action='store_true', dest='arm32_qemu', default=False, 115 help='launch all binaries in qemu arm') 116 parser.add_argument( 117 '--test-list', dest='test_list', default=None, type=lambda arg: is_file(parser, arg), 118 help='run tests listed in file') 119 parser.add_argument( 120 '--aot-args', action='append', dest='aot_args', default=[], 121 help='Additional arguments that will passed to ark_aot') 122 parser.add_argument( 123 '--verbose', '-v', action='store_true', dest='verbose', default=False, 124 help='Enable verbose output') 125 parser.add_argument( 126 '--js-runtime', dest='js_runtime_path', default=None, type=lambda arg: is_directory(parser, arg), 127 help='the path of js vm runtime') 128 parser.add_argument( 129 '--LD_LIBRARY_PATH', dest='ld_library_path', default=None, help='LD_LIBRARY_PATH') 130 parser.add_argument( 131 '--tsc-path', dest='tsc_path', default=None, type=lambda arg: is_directory(parser, arg), 132 help='the path of tsc') 133 parser.add_argument('--hotfix', dest='hotfix', action='store_true', default=False, 134 help='run hotfix tests') 135 parser.add_argument('--hotreload', dest='hotreload', action='store_true', default=False, 136 help='run hotreload tests') 137 parser.add_argument('--base64', dest='base64', action='store_true', default=False, 138 help='run base64 tests') 139 140 return parser.parse_args() 141 142 143class Test: 144 def __init__(self, test_path, flags): 145 self.path = test_path 146 self.flags = flags 147 self.output = None 148 self.error = None 149 self.passed = None 150 self.skipped = None 151 self.reproduce = "" 152 153 def log_cmd(self, cmd): 154 self.reproduce += "\n" + ' '.join(cmd) 155 156 def run(self, runner): 157 cmd = runner.cmd_prefix + [runner.es2panda, "--dump-ast"] 158 cmd.extend(self.flags) 159 cmd.append(self.path) 160 161 self.log_cmd(cmd) 162 process = subprocess.Popen( 163 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 164 out, err = process.communicate() 165 self.output = out.decode("utf-8", errors="ignore") + err.decode("utf-8", errors="ignore") 166 167 expected_path = "%s-expected.txt" % (path.splitext(self.path)[0]) 168 try: 169 with open(expected_path, 'r') as fp: 170 expected = fp.read() 171 self.passed = expected == self.output and process.returncode in [ 172 0, 1] 173 except Exception: 174 self.passed = False 175 176 if not self.passed: 177 self.error = err.decode("utf-8", errors="ignore") 178 179 return self 180 181 182class Test262Test(Test): 183 def __init__(self, test_path, flags, test_id, with_optimizer): 184 Test.__init__(self, test_path, flags) 185 self.test_id = test_id 186 self.fail_kind = None 187 self.with_optimizer = with_optimizer 188 189 class FailKind(Enum): 190 ES2PANDA_FAIL = 1 191 RUNTIME_FAIL = 2 192 AOT_FAIL = 3 193 ES2PANDA_TIMEOUT = 4 194 RUNTIME_TIMEOUT = 5 195 AOT_TIMEOUT = 6 196 197 def run(self, runner): 198 with open(self.path, 'r') as fp: 199 header = runner.util.get_header(fp.read()) 200 desc = runner.util.parse_descriptor(header) 201 202 test_abc = path.join(runner.tmp_dir, "%s.abc" % self.test_id) 203 test_an = path.join(runner.tmp_dir, "%s.an" % self.test_id) 204 205 directory = path.dirname(test_abc) 206 os.makedirs(directory, exist_ok=True) 207 208 cmd = runner.cmd_prefix + [runner.es2panda] 209 if self.with_optimizer: 210 cmd.append('--opt-level=2') 211 cmd.extend(['--thread=0', '--output=%s' % (test_abc)]) 212 213 if 'module' in desc['flags']: 214 cmd.append("--module") 215 216 if 'noStrict' in desc['flags']: 217 self.skipped = True 218 return self 219 220 cmd.append(self.path) 221 222 self.log_cmd(cmd) 223 224 if runner.args.verbose: 225 print('Run es2panda: %s' % ' '.join(cmd), file=sys.stderr) 226 227 process = subprocess.Popen( 228 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=runner.cmd_env) 229 230 try: 231 output_res, err = process.communicate(runner.args.es2panda_timeout) 232 except subprocess.TimeoutExpired: 233 process.kill() 234 self.passed = False 235 self.fail_kind = self.FailKind.ES2PANDA_TIMEOUT 236 self.error = self.fail_kind.name 237 return self 238 239 out = output_res.decode("utf-8", errors="ignore") 240 err = err.decode("utf-8", errors="ignore") 241 self.passed, need_exec = runner.util.validate_parse_result( 242 process.returncode, err, desc, out) 243 244 if not self.passed: 245 self.fail_kind = self.FailKind.ES2PANDA_FAIL 246 self.error = "out:{}\nerr:{}\ncode:{}".format( 247 out, err, process.returncode) 248 print(self.error) 249 return self 250 251 if not need_exec: 252 self.passed = True 253 return self 254 255 if runner.args.aot: 256 cmd = runner.cmd_prefix + [runner.arkaot] 257 cmd.extend(runner.aot_args) 258 cmd.extend(['--paoc-panda-files', test_abc]) 259 cmd.extend(['--paoc-output', test_an]) 260 261 if os.path.isfile(test_an): 262 os.remove(test_an) 263 264 self.log_cmd(cmd) 265 266 if runner.args.verbose: 267 print('Run ark_aot: %s' % ' '.join(cmd), file=sys.stderr) 268 269 process = subprocess.Popen( 270 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=runner.cmd_env) 271 272 try: 273 out, err = process.communicate(runner.args.paoc_timeout) 274 except subprocess.TimeoutExpired: 275 process.kill() 276 self.passed = False 277 self.fail_kind = self.FailKind.AOT_TIMEOUT 278 self.error = self.fail_kind.name 279 return self 280 281 if process.returncode != 0: 282 self.passed = False 283 self.fail_kind = self.FailKind.AOT_FAIL 284 self.error = err.decode("utf-8", errors="ignore") 285 return self 286 287 cmd = runner.cmd_prefix + [runner.runtime] 288 289 if runner.args.verbose: 290 print('Run aot for arm64: %s' % ' '.join(cmd), file=sys.stderr) 291 292 cmd.extend(runner.runtime_args) 293 294 if runner.args.aot: 295 cmd.extend(['--aot-files', test_an]) 296 297 if runner.args.jit: 298 cmd.extend(['--compiler-enable-jit=true', '--compiler-hotness-threshold=0']) 299 else: 300 cmd.extend(['--compiler-enable-jit=false']) 301 302 cmd.extend([test_abc, "_GLOBAL::func_main_0"]) 303 304 self.log_cmd(cmd) 305 306 if runner.args.verbose: 307 print('Run ark: %s' % ' '.join(cmd), file=sys.stderr) 308 309 process = subprocess.Popen( 310 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=runner.cmd_env) 311 312 try: 313 out, err = process.communicate(timeout=runner.args.timeout) 314 except subprocess.TimeoutExpired: 315 process.kill() 316 self.passed = False 317 self.fail_kind = self.FailKind.RUNTIME_TIMEOUT 318 self.error = self.fail_kind.name 319 return self 320 321 out = out.decode("utf-8", errors="ignore") 322 err = err.decode("utf-8", errors="ignore") 323 self.passed = runner.util.validate_runtime_result( 324 process.returncode, err, desc, out) 325 326 if not self.passed: 327 self.fail_kind = self.FailKind.RUNTIME_FAIL 328 self.error = "out:{}\nerr:{}\ncode:{}".format( 329 out, err, process.returncode) 330 print(self.error) 331 332 return self 333 334 335class TSCTest(Test): 336 def __init__(self, test_path, flags): 337 Test.__init__(self, test_path, flags) 338 self.options = self.parse_options() 339 340 def parse_options(self): 341 test_options = {} 342 343 with open(self.path, "r", encoding="latin1") as f: 344 lines = f.read() 345 options = re.findall(r"//\s?@\w+:.*\n", lines) 346 347 for option in options: 348 separated = option.split(":") 349 opt = re.findall(r"\w+", separated[0])[0].lower() 350 value = separated[1].strip().lower() 351 352 if opt == "filename": 353 if opt in options: 354 test_options[opt].append(value) 355 else: 356 test_options[opt] = [value] 357 358 elif opt == "lib" or opt == "module": 359 test_options[opt] = [each.strip() 360 for each in value.split(",")] 361 elif value == "true" or value == "false": 362 test_options[opt] = value.lower() == "true" 363 else: 364 test_options[opt] = value 365 366 # TODO: Possibility of error: all exports will be catched, even the commented ones 367 if 'module' not in test_options and re.search(r"export ", lines): 368 test_options['module'] = [] 369 370 return test_options 371 372 def run(self, runner): 373 cmd = runner.cmd_prefix + [runner.es2panda, '--parse-only', '--extension=ts'] 374 cmd.extend(self.flags) 375 if "module" in self.options: 376 cmd.append('--module') 377 cmd.append(self.path) 378 379 self.log_cmd(cmd) 380 process = subprocess.Popen( 381 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 382 out, err = process.communicate() 383 self.output = out.decode("utf-8", errors="ignore") 384 385 self.passed = True if process.returncode == 0 else False 386 387 if not self.passed: 388 self.error = err.decode("utf-8", errors="ignore") 389 390 return self 391 392 393class Runner: 394 def __init__(self, args, name): 395 self.test_root = path.dirname(path.abspath(__file__)) 396 self.args = args 397 self.name = name 398 self.tests = [] 399 self.failed = 0 400 self.passed = 0 401 self.es2panda = path.join(args.build_dir, 'es2abc') 402 self.build_dir = args.build_dir 403 self.cmd_prefix = [] 404 self.ark_js_vm = "" 405 self.ld_library_path = "" 406 407 if args.js_runtime_path: 408 self.ark_js_vm = path.join(args.js_runtime_path, 'ark_js_vm') 409 410 if args.ld_library_path: 411 self.ld_library_path = args.ld_library_path 412 413 if args.arm64_qemu: 414 self.cmd_prefix = ["qemu-aarch64", "-L", "/usr/aarch64-linux-gnu/"] 415 416 if args.arm32_qemu: 417 self.cmd_prefix = ["qemu-arm", "-L", "/usr/arm-linux-gnueabi"] 418 419 if not path.isfile(self.es2panda): 420 raise Exception("Cannot find es2panda binary: %s" % self.es2panda) 421 422 def add_directory(self, directory, extension, flags): 423 pass 424 425 def test_path(self, src): 426 pass 427 428 def run_test(self, test): 429 return test.run(self) 430 431 def run(self): 432 pool = multiprocessing.Pool() 433 result_iter = pool.imap_unordered( 434 self.run_test, self.tests, chunksize=32) 435 pool.close() 436 437 if self.args.progress: 438 from tqdm import tqdm 439 result_iter = tqdm(result_iter, total=len(self.tests)) 440 441 results = [] 442 for res in result_iter: 443 results.append(res) 444 445 self.tests = results 446 pool.join() 447 448 def deal_error(self, test): 449 path_str = test.path 450 err_col = {} 451 if test.error: 452 err_str = test.error.split('[')[0] if "hotfix" not in test.path else " hotfix throw error failed" 453 err_col = {"path" : [path_str], "status": ["fail"], "error" : [test.error], "type" : [err_str]} 454 else: 455 err_col = {"path" : [path_str], "status": ["fail"], "error" : ["Segmentation fault"], 456 "type" : ["Segmentation fault"]} 457 return err_col 458 459 def summarize(self): 460 print("") 461 fail_list = [] 462 success_list = [] 463 464 for test in self.tests: 465 assert(test.passed is not None) 466 if not test.passed: 467 fail_list.append(test) 468 else: 469 success_list.append(test) 470 471 if len(fail_list): 472 if self.args.error: 473 import pandas as pd 474 test_list = pd.DataFrame(columns=["path", "status", "error", "type"]) 475 for test in success_list: 476 suc_col = {"path" : [test.path], "status": ["success"], "error" : ["success"], "type" : ["success"]} 477 if self.args.error: 478 test_list = pd.concat([test_list, pd.DataFrame(suc_col)]) 479 print("Failed tests:") 480 for test in fail_list: 481 print(self.test_path(test.path)) 482 483 if self.args.error: 484 print("steps:", test.reproduce) 485 print("error:") 486 print(test.error) 487 print("\n") 488 err_col = self.deal_error(test) 489 test_list = pd.concat([test_list, pd.DataFrame(err_col)]) 490 491 if self.args.error: 492 test_list.to_csv('test_statistics.csv', index=False) 493 test_list["type"].value_counts().to_csv('type_statistics.csv', index_label="error") 494 print("Type statistics:\n", test_list["type"].value_counts()) 495 print("") 496 497 print("Summary(%s):" % self.name) 498 print("\033[37mTotal: %5d" % (len(self.tests))) 499 print("\033[92mPassed: %5d" % (len(self.tests) - len(fail_list))) 500 print("\033[91mFailed: %5d" % (len(fail_list))) 501 print("\033[0m") 502 503 return len(fail_list) 504 505 506class RegressionRunner(Runner): 507 def __init__(self, args): 508 Runner.__init__(self, args, "Regresssion") 509 510 def add_directory(self, directory, extension, flags): 511 glob_expression = path.join( 512 self.test_root, directory, "*.%s" % (extension)) 513 files = glob(glob_expression) 514 files = fnmatch.filter(files, self.test_root + '**' + self.args.filter) 515 516 self.tests += list(map(lambda f: Test(f, flags), files)) 517 518 def test_path(self, src): 519 return src 520 521 522class Test262Runner(Runner): 523 def __init__(self, args): 524 Runner.__init__(self, args, "Test262 ark"), 525 526 self.cmd_env = os.environ.copy() 527 for san in ["ASAN_OPTIONS", "TSAN_OPTIONS", "MSAN_OPTIONS", "LSAN_OPTIONS"]: 528 # we don't want to interpret asan failures as SyntaxErrors 529 self.cmd_env[san] = ":exitcode=255" 530 531 self.update = args.update 532 self.enable_skiplists = False if self.update else args.skip 533 self.normal_skiplist_file = "test262skiplist.txt" 534 self.long_flaky_skiplist_files = ["test262skiplist-long.txt", "test262skiplist-flaky.txt"] 535 self.normal_skiplist = set([]) 536 self.runtime = path.join(args.build_dir, 'bin', 'ark') 537 if not path.isfile(self.runtime): 538 raise Exception("Cannot find runtime binary: %s" % self.runtime) 539 540 self.runtime_args = [ 541 '--boot-panda-files=%s/pandastdlib/arkstdlib.abc' 542 % args.build_dir, 543 '--load-runtimes=ecmascript', 544 '--gc-type=%s' % args.gc_type, 545 ] 546 547 if not args.no_gip: 548 self.runtime_args += ['--run-gc-in-place'] 549 550 if args.aot: 551 self.arkaot = path.join(args.build_dir, 'bin', 'ark_aot') 552 if not path.isfile(self.arkaot): 553 raise Exception("Cannot find aot binary: %s" % self.arkaot) 554 555 self.aot_args = [ 556 '--boot-panda-files=%s/pandastdlib/arkstdlib.abc' 557 % args.build_dir, 558 '--load-runtimes=ecmascript', 559 '--gc-type=%s' % args.gc_type, 560 ] 561 562 if not args.no_gip: 563 self.aot_args += ['--run-gc-in-place'] 564 565 self.aot_args += args.aot_args 566 else: 567 self.aot_args = [] 568 569 self.skiplist_name_list = self.long_flaky_skiplist_files if self.update else [] 570 self.skiplist_bco_name = "" 571 572 if self.enable_skiplists: 573 self.skiplist_name_list.append(self.normal_skiplist_file) 574 self.skiplist_name_list.extend(self.long_flaky_skiplist_files) 575 576 if args.bco: 577 self.skiplist_bco_name = "test262skiplist-bco.txt" 578 if args.arm64_compiler_skip: 579 self.skiplist_name_list.append("test262skiplist-compiler-arm64.txt") 580 581 self.tmp_dir = path.join(path.sep, 'tmp', 'panda', 'test262') 582 os.makedirs(self.tmp_dir, exist_ok=True) 583 584 self.util = test262util.Test262Util() 585 self.test262_dir = self.util.generate( 586 '281eb10b2844929a7c0ac04527f5b42ce56509fd', 587 args.build_dir, 588 path.join(self.test_root, "test262harness.js"), 589 args.progress) 590 591 self.add_directory(self.test262_dir, "js", args.test_list, []) 592 593 def add_directory(self, directory, extension, test_list_path, flags): 594 glob_expression = path.join(directory, "**/*.%s" % (extension)) 595 files = glob(glob_expression, recursive=True) 596 files = fnmatch.filter(files, path.join(directory, self.args.filter)) 597 598 def load_list(p): 599 with open(p, 'r') as fp: 600 return set(map(lambda e: path.join(directory, e.strip()), fp)) 601 602 skiplist = set([]) 603 604 for sl in self.skiplist_name_list: 605 skiplist.update(load_list(path.join(self.test_root, sl))) 606 607 if self.update: 608 self.normal_skiplist.update(load_list(path.join(self.test_root, self.normal_skiplist_file))) 609 610 skiplist_bco = set([]) 611 if self.skiplist_bco_name != "": 612 skiplist_bco = load_list(path.join(self.test_root, self.skiplist_bco_name)) 613 614 if test_list_path is not None: 615 test_list = load_list(path.abspath(test_list_path)) 616 files = filter(lambda f: f in test_list, files) 617 618 def get_test_id(file): 619 return path.relpath(path.splitext(file)[0], self.test262_dir) 620 621 self.tests = list(map(lambda test: Test262Test(test, flags, get_test_id(test), test not in skiplist_bco), 622 filter(lambda f: f not in skiplist, files))) 623 624 def test_path(self, src): 625 return path.relpath(src, self.test262_dir) 626 627 def run(self): 628 Runner.run(self) 629 self.update_skiplist() 630 631 def summarize(self): 632 print("") 633 634 fail_lists = {} 635 for kind in Test262Test.FailKind: 636 fail_lists[kind] = [] 637 638 num_failed = 0 639 num_skipped = 0 640 for test in self.tests: 641 if test.skipped: 642 num_skipped += 1 643 continue 644 645 assert(test.passed is not None) 646 if not test.passed: 647 fail_lists[test.fail_kind].append(test) 648 num_failed += 1 649 650 def summarize_list(name, tests_list): 651 if len(tests_list): 652 tests_list.sort(key=lambda test: test.path) 653 print("# " + name) 654 for test in tests_list: 655 print(self.test_path(test.path)) 656 if self.args.error: 657 print("steps:", test.reproduce) 658 print(test.error) 659 print("") 660 661 total_tests = len(self.tests) - num_skipped 662 663 if not self.update: 664 for kind in Test262Test.FailKind: 665 summarize_list(kind.name, fail_lists[kind]) 666 667 print("Summary(%s):" % self.name) 668 print("\033[37mTotal: %5d" % (total_tests)) 669 print("\033[92mPassed: %5d" % (total_tests - num_failed)) 670 print("\033[91mFailed: %5d" % (num_failed)) 671 print("\033[0m") 672 673 return num_failed 674 675 def update_skiplist(self): 676 if not self.update: 677 return 678 679 skiplist_es2panda = list({x.test_id + ".js" for x in self.tests 680 if not x.skipped and not x.passed and 681 x.fail_kind == Test262Test.FailKind.ES2PANDA_FAIL}) 682 skiplist_runtime = list({x.test_id + ".js" for x in self.tests 683 if not x.skipped and not x.passed and 684 x.fail_kind == Test262Test.FailKind.RUNTIME_FAIL}) 685 686 skiplist_es2panda.sort() 687 skiplist_runtime.sort() 688 689 new_skiplist = skiplist_es2panda + skiplist_runtime 690 691 new_pass = list(filter(lambda x: len(x) and not x.startswith('#') 692 and x not in new_skiplist, self.normal_skiplist)) 693 new_fail = list(filter(lambda x: x not in self.normal_skiplist, new_skiplist)) 694 new_pass.sort() 695 new_fail.sort() 696 697 if new_pass: 698 print("\033[92mRemoved from skiplist:") 699 print("\n".join(new_pass)) 700 print("\033[0m") 701 702 if new_fail: 703 print("\033[91mNew tests on skiplist:") 704 print("\n".join(new_fail)) 705 print("\033[0m") 706 707 fd = os.open(path.join(self.test_root, self.normal_skiplist_file), os.O_RDWR | os.O_CREAT | os.O_TRUNC) 708 file = os.fdopen(fd, "w+") 709 file.write("\n".join(["# ES2PANDA_FAIL"] + skiplist_es2panda + ["", "# RUNTIME_FAIL"] + skiplist_runtime)) 710 file.write("\n") 711 file.close() 712 713 714class TSCRunner(Runner): 715 def __init__(self, args): 716 Runner.__init__(self, args, "TSC") 717 718 if self.args.tsc_path: 719 self.tsc_path = self.args.tsc_path 720 else : 721 ts_dir = path.join(self.test_root, "TypeScript") 722 ts_branch = "v4.2.4" 723 724 if not path.isdir(ts_dir): 725 subprocess.run( 726 f"git clone https://github.com/microsoft/TypeScript.git \ 727 {ts_dir} && cd {ts_dir} \ 728 && git checkout {ts_branch} > /dev/null 2>&1", 729 shell=True, 730 stdout=subprocess.DEVNULL, 731 ) 732 else: 733 subprocess.run( 734 f"cd {ts_dir} && git clean -f > /dev/null 2>&1", 735 shell=True, 736 stdout=subprocess.DEVNULL, 737 ) 738 self.tsc_path = ts_dir 739 740 self.add_directory("conformance", []) 741 self.add_directory("compiler", []) 742 743 def add_directory(self, directory, flags): 744 ts_suite_dir = path.join(self.tsc_path, 'tests/cases') 745 746 glob_expression = path.join( 747 ts_suite_dir, directory, "**/*.ts") 748 files = glob(glob_expression, recursive=True) 749 files = fnmatch.filter(files, ts_suite_dir + '**' + self.args.filter) 750 751 for f in files: 752 test_name = path.basename(f.split(".ts")[0]) 753 negative_references = path.join( 754 self.tsc_path, 'tests/baselines/reference') 755 is_negative = path.isfile(path.join(negative_references, 756 test_name + ".errors.txt")) 757 test = TSCTest(f, flags) 758 759 if 'target' in test.options: 760 targets = test.options['target'].replace(" ", "").split(',') 761 for target in targets: 762 if path.isfile(path.join(negative_references, 763 test_name + "(target=%s).errors.txt" % (target))): 764 is_negative = True 765 break 766 767 if is_negative or "filename" in test.options: 768 continue 769 770 with open(path.join(self.test_root, 'test_tsc_ignore_list.txt'), 'r') as failed_references: 771 if self.args.skip: 772 if path.relpath(f, self.tsc_path) in failed_references.read(): 773 continue 774 775 self.tests.append(test) 776 777 def test_path(self, src): 778 return src 779 780 781class CompilerRunner(Runner): 782 def __init__(self, args): 783 Runner.__init__(self, args, "Compiler") 784 785 def add_directory(self, directory, extension, flags): 786 glob_expression = path.join( 787 self.test_root, directory, "**/*.%s" % (extension)) 788 files = glob(glob_expression, recursive=True) 789 files = fnmatch.filter(files, self.test_root + '**' + self.args.filter) 790 791 self.tests += list(map(lambda f: CompilerTest(f, flags), files)) 792 793 def test_path(self, src): 794 return src 795 796 797class CompilerTest(Test): 798 def __init__(self, test_path, flags): 799 Test.__init__(self, test_path, flags) 800 801 def run(self, runner): 802 test_abc_name = ("%s.abc" % (path.splitext(self.path)[0])).replace("/", "_") 803 test_abc_path = path.join(runner.build_dir, test_abc_name) 804 es2abc_cmd = runner.cmd_prefix + [runner.es2panda] 805 es2abc_cmd.extend(self.flags) 806 es2abc_cmd.extend(['%s%s' % ("--output=", test_abc_path)]) 807 es2abc_cmd.append(self.path) 808 self.log_cmd(es2abc_cmd) 809 810 process = subprocess.Popen(es2abc_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 811 out, err = process.communicate() 812 if err: 813 self.passed = False 814 self.error = err.decode("utf-8", errors="ignore") 815 return self 816 817 ld_library_path = runner.ld_library_path 818 os.environ.setdefault("LD_LIBRARY_PATH", ld_library_path) 819 run_abc_cmd = [runner.ark_js_vm] 820 run_abc_cmd.extend([test_abc_path]) 821 self.log_cmd(run_abc_cmd) 822 823 process = subprocess.Popen(run_abc_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 824 out, err = process.communicate() 825 self.output = out.decode("utf-8", errors="ignore") + err.decode("utf-8", errors="ignore") 826 expected_path = "%s-expected.txt" % (path.splitext(self.path)[0]) 827 try: 828 with open(expected_path, 'r') as fp: 829 expected = fp.read() 830 self.passed = expected == self.output and process.returncode in [0, 1] 831 except Exception: 832 self.passed = False 833 834 if not self.passed: 835 self.error = err.decode("utf-8", errors="ignore") 836 837 os.remove(test_abc_path) 838 839 return self 840 841 842class PatchTest(Test): 843 def __init__(self, test_path, mode_arg): 844 Test.__init__(self, test_path, "") 845 self.mode = mode_arg 846 847 def run(self, runner): 848 symbol_table_file = 'base.map' 849 origin_input_file = 'base.js' 850 origin_output_abc = 'base.abc' 851 modified_input_file = 'base_mod.js' 852 modified_output_abc = 'patch.abc' 853 854 gen_base_cmd = runner.cmd_prefix + [runner.es2panda, '--module'] 855 gen_base_cmd.extend(['--dump-symbol-table', os.path.join(self.path, symbol_table_file)]) 856 gen_base_cmd.extend(['--output', os.path.join(self.path, origin_output_abc)]) 857 gen_base_cmd.extend([os.path.join(self.path, origin_input_file)]) 858 self.log_cmd(gen_base_cmd) 859 860 if self.mode == 'hotfix': 861 mode_arg = "--generate-patch" 862 elif self.mode == 'hotreload': 863 mode_arg = "--hot-reload" 864 865 patch_test_cmd = runner.cmd_prefix + [runner.es2panda, '--module', mode_arg] 866 patch_test_cmd.extend(['--input-symbol-table', os.path.join(self.path, symbol_table_file)]) 867 patch_test_cmd.extend(['--output', os.path.join(self.path, modified_output_abc)]) 868 patch_test_cmd.extend([os.path.join(self.path, modified_input_file)]) 869 self.log_cmd(patch_test_cmd) 870 871 process_base = subprocess.Popen(gen_base_cmd, stdout=subprocess.PIPE, 872 stderr=subprocess.PIPE) 873 stdout_base, stderr_base = process_base.communicate(timeout=runner.args.es2panda_timeout) 874 if stderr_base: 875 self.passed = False 876 self.error = stderr_base.decode("utf-8", errors="ignore") 877 return self 878 879 process_patch = subprocess.Popen(patch_test_cmd, stdout=subprocess.PIPE, 880 stderr=subprocess.PIPE) 881 stdout_patch, stderr_patch = process_patch.communicate(timeout=runner.args.es2panda_timeout) 882 if stderr_patch: 883 self.passed = False 884 self.error = stderr_patch.decode("utf-8", errors="ignore") 885 886 self.output = stdout_patch.decode("utf-8", errors="ignore") + stderr_patch.decode("utf-8", errors="ignore") 887 expected_path = os.path.join(self.path, 'expected.txt') 888 try: 889 with open(expected_path, 'r') as fp: 890 expected = (''.join((fp.readlines()[12:]))).lstrip() # ignore license description lines and skip leading blank lines 891 self.passed = expected == self.output 892 except Exception: 893 self.passed = False 894 895 if not self.passed: 896 self.error = "expected output:" + os.linesep + expected + os.linesep + "actual output:" + os.linesep +\ 897 self.output 898 899 return self 900 901 902class PatchRunner(Runner): 903 def __init__(self, args, name): 904 Runner.__init__(self, args, name) 905 self.preserve_files = args.error 906 907 def __del__(self): 908 if not self.preserve_files: 909 self.clear_directory() 910 911 def add_directory(self): 912 glob_expression = path.join(self.test_directory, "*") 913 self.tests_in_dirs = glob(glob_expression, recursive=False) 914 915 def clear_directory(self): 916 for test in self.tests_in_dirs: 917 files_in_dir = os.listdir(test) 918 filtered_files = [file for file in files_in_dir if file.endswith(".map") or file.endswith(".abc")] 919 for file in filtered_files: 920 os.remove(os.path.join(test, file)) 921 922 def test_path(self, src): 923 return os.path.basename(src) 924 925 926class HotfixRunner(PatchRunner): 927 def __init__(self, args): 928 PatchRunner.__init__(self, args, "Hotfix") 929 self.test_directory = path.join(self.test_root, "hotfix", "hotfix-throwerror") 930 self.add_directory() 931 self.tests += list(map(lambda t: PatchTest(t, "hotfix"), self.tests_in_dirs)) 932 933 934class HotreloadRunner(PatchRunner): 935 def __init__(self, args): 936 PatchRunner.__init__(self, args, "Hotreload") 937 self.test_directory = path.join(self.test_root, "hotreload") 938 self.add_directory() 939 self.tests += list(map(lambda t: PatchTest(t, "hotreload"), self.tests_in_dirs)) 940 941 942class Base64Test(Test): 943 def __init__(self, test_path, input_type): 944 Test.__init__(self, test_path, "") 945 self.input_type = input_type 946 947 def run(self, runner): 948 cmd = runner.cmd_prefix + [runner.es2panda, "--base64Output"] 949 if self.input_type == "file": 950 input_file_name = 'input.js' 951 cmd.extend(['--source-file', input_file_name]) 952 cmd.extend([os.path.join(self.path, input_file_name)]) 953 elif self.input_type == "string": 954 input_file = os.path.join(self.path, "input.txt") 955 try: 956 with open(input_file, 'r') as fp: 957 base64_input = (''.join((fp.readlines()[12:]))).lstrip() # ignore license description lines 958 cmd.extend(["--base64Input", base64_input]) 959 except Exception: 960 self.passed = False 961 else: 962 self.error = "Unsupported base64 input type" 963 self.passed = False 964 return self 965 966 self.log_cmd(cmd) 967 968 process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 969 stdout, stderr = process.communicate(timeout=runner.args.es2panda_timeout) 970 if stderr: 971 self.passed = False 972 self.error = stderr.decode("utf-8", errors="ignore") 973 return self 974 975 self.output = stdout.decode("utf-8", errors="ignore") 976 977 expected_path = os.path.join(self.path, 'expected.txt') 978 try: 979 with open(expected_path, 'r') as fp: 980 expected = (''.join((fp.readlines()[12:]))).lstrip() 981 self.passed = expected == self.output 982 except Exception: 983 self.passed = False 984 985 if not self.passed: 986 self.error = "expected output:" + os.linesep + expected + os.linesep + "actual output:" + os.linesep +\ 987 self.output 988 989 return self 990 991 992class Base64Runner(Runner): 993 def __init__(self, args): 994 Runner.__init__(self, args, "Base64") 995 self.test_directory = path.join(self.test_root, "base64") 996 self.add_test() 997 998 def add_test(self): 999 self.tests = [] 1000 self.tests.append(Base64Test(os.path.join(self.test_directory, "inputFile"), "file")) 1001 self.tests.append(Base64Test(os.path.join(self.test_directory, "inputString"), "string")) 1002 1003 def test_path(self, src): 1004 return os.path.basename(src) 1005 1006 1007def main(): 1008 args = get_args() 1009 1010 runners = [] 1011 1012 if args.regression: 1013 runner = RegressionRunner(args) 1014 runner.add_directory("parser/concurrent", "js", ["--module"]) 1015 runner.add_directory("parser/js", "js", ["--parse-only"]) 1016 runner.add_directory("parser/ts", "ts", 1017 ["--parse-only", "--module", "--extension=ts"]) 1018 runner.add_directory("parser/ts/type_checker", "ts", 1019 ["--parse-only", "--enable-type-check", "--module", "--extension=ts"]) 1020 runner.add_directory("parser/commonjs", "js", ["--commonjs", "--parse-only", "--dump-ast"]) 1021 1022 runners.append(runner) 1023 1024 if args.test262: 1025 runners.append(Test262Runner(args)) 1026 1027 if args.tsc: 1028 runners.append(TSCRunner(args)) 1029 1030 if args.compiler: 1031 runner = CompilerRunner(args) 1032 runner.add_directory("compiler/js", "js", []) 1033 runner.add_directory("compiler/ts", "ts", ["--extension=ts"]) 1034 runner.add_directory("compiler/commonjs", "js", ["--commonjs"]) 1035 1036 runners.append(runner) 1037 1038 if args.hotfix: 1039 runners.append(HotfixRunner(args)) 1040 1041 if args.hotreload: 1042 runners.append(HotreloadRunner(args)) 1043 1044 if args.base64: 1045 runners.append(Base64Runner(args)) 1046 1047 failed_tests = 0 1048 1049 for runner in runners: 1050 runner.run() 1051 failed_tests += runner.summarize() 1052 1053 # TODO: exit 1 when we have failed tests after all tests are fixed 1054 exit(0) 1055 1056 1057if __name__ == "__main__": 1058 main() 1059