1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# 4# Copyright (c) 2021-2022 Huawei Device Co., Ltd. 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17from glob import glob 18from os import path 19from enum import Enum 20import argparse 21import fnmatch 22import multiprocessing 23import os 24import re 25import shutil 26import subprocess 27import sys 28 29 30def is_directory(parser, arg): 31 if not path.isdir(arg): 32 parser.error("The directory '%s' does not exist" % arg) 33 34 return path.abspath(arg) 35 36 37def is_file(parser, arg): 38 if not path.isfile(arg): 39 parser.error("The file '%s' does not exist" % arg) 40 41 return path.abspath(arg) 42 43def prepare_tsc_testcases(test_root): 44 third_party_tsc = path.join(test_root, "TypeScript") 45 ohos_third_party_tsc = path.join(test_root, "../../../../third_party/typescript") 46 47 if not path.isdir(third_party_tsc): 48 if (path.isdir(ohos_third_party_tsc)): 49 return path.abspath(ohos_third_party_tsc) 50 subprocess.run( 51 f"git clone https://gitee.com/openharmony/third_party_typescript.git {third_party_tsc}", 52 shell=True, 53 stdout=subprocess.DEVNULL, 54 ) 55 else: 56 subprocess.run( 57 f"cd {third_party_tsc} && git clean -f > /dev/null 2>&1", 58 shell=True, 59 stdout=subprocess.DEVNULL, 60 ) 61 return third_party_tsc 62 63def check_timeout(value): 64 ivalue = int(value) 65 if ivalue <= 0: 66 raise argparse.ArgumentTypeError( 67 "%s is an invalid timeout value" % value) 68 return ivalue 69 70 71def get_args(): 72 parser = argparse.ArgumentParser(description="Regression test runner") 73 parser.add_argument( 74 'build_dir', type=lambda arg: is_directory(parser, arg), 75 help='panda build directory') 76 parser.add_argument( 77 '--error', action='store_true', dest='error', default=False, 78 help='capture stderr') 79 parser.add_argument( 80 '--abc-to-asm', action='store_true', dest='abc_to_asm', 81 default=False, help='run abc2asm tests') 82 parser.add_argument( 83 '--regression', '-r', action='store_true', dest='regression', 84 default=False, help='run regression tests') 85 parser.add_argument( 86 '--compiler', '-c', action='store_true', dest='compiler', 87 default=False, help='run compiler tests') 88 parser.add_argument( 89 '--tsc', action='store_true', dest='tsc', 90 default=False, help='run tsc tests') 91 parser.add_argument( 92 '--no-progress', action='store_false', dest='progress', default=True, 93 help='don\'t show progress bar') 94 parser.add_argument( 95 '--no-skip', action='store_false', dest='skip', default=True, 96 help='don\'t use skiplists') 97 parser.add_argument( 98 '--update', action='store_true', dest='update', default=False, 99 help='update skiplist') 100 parser.add_argument( 101 '--no-run-gc-in-place', action='store_true', dest='no_gip', default=False, 102 help='enable --run-gc-in-place mode') 103 parser.add_argument( 104 '--filter', '-f', action='store', dest='filter', 105 default="*", help='test filter regexp') 106 parser.add_argument( 107 '--es2panda-timeout', type=check_timeout, 108 dest='es2panda_timeout', default=60, help='es2panda translator timeout') 109 parser.add_argument( 110 '--paoc-timeout', type=check_timeout, 111 dest='paoc_timeout', default=600, help='paoc compiler timeout') 112 parser.add_argument( 113 '--timeout', type=check_timeout, 114 dest='timeout', default=10, help='JS runtime timeout') 115 parser.add_argument( 116 '--gc-type', dest='gc_type', default="g1-gc", help='Type of garbage collector') 117 parser.add_argument( 118 '--aot', action='store_true', dest='aot', default=False, 119 help='use AOT compilation') 120 parser.add_argument( 121 '--no-bco', action='store_false', dest='bco', default=True, 122 help='disable bytecodeopt') 123 parser.add_argument( 124 '--jit', action='store_true', dest='jit', default=False, 125 help='use JIT in interpreter') 126 parser.add_argument( 127 '--arm64-compiler-skip', action='store_true', dest='arm64_compiler_skip', default=False, 128 help='use skiplist for tests failing on aarch64 in AOT or JIT mode') 129 parser.add_argument( 130 '--arm64-qemu', action='store_true', dest='arm64_qemu', default=False, 131 help='launch all binaries in qemu aarch64') 132 parser.add_argument( 133 '--arm32-qemu', action='store_true', dest='arm32_qemu', default=False, 134 help='launch all binaries in qemu arm') 135 parser.add_argument( 136 '--test-list', dest='test_list', default=None, type=lambda arg: is_file(parser, arg), 137 help='run tests listed in file') 138 parser.add_argument( 139 '--aot-args', action='append', dest='aot_args', default=[], 140 help='Additional arguments that will passed to ark_aot') 141 parser.add_argument( 142 '--verbose', '-v', action='store_true', dest='verbose', default=False, 143 help='Enable verbose output') 144 parser.add_argument( 145 '--js-runtime', dest='js_runtime_path', default=None, type=lambda arg: is_directory(parser, arg), 146 help='the path of js vm runtime') 147 parser.add_argument( 148 '--LD_LIBRARY_PATH', dest='ld_library_path', default=None, help='LD_LIBRARY_PATH') 149 parser.add_argument( 150 '--tsc-path', dest='tsc_path', default=None, type=lambda arg: is_directory(parser, arg), 151 help='the path of tsc') 152 parser.add_argument('--hotfix', dest='hotfix', action='store_true', default=False, 153 help='run hotfix tests') 154 parser.add_argument('--hotreload', dest='hotreload', action='store_true', default=False, 155 help='run hotreload tests') 156 parser.add_argument('--coldfix', dest='coldfix', action='store_true', default=False, 157 help='run coldfix tests') 158 parser.add_argument('--coldreload', dest='coldreload', action='store_true', default=False, 159 help='run coldreload tests') 160 parser.add_argument('--base64', dest='base64', action='store_true', default=False, 161 help='run base64 tests') 162 parser.add_argument('--bytecode', dest='bytecode', action='store_true', default=False, 163 help='run bytecode tests') 164 parser.add_argument('--debugger', dest='debugger', action='store_true', default=False, 165 help='run debugger tests') 166 parser.add_argument('--debug', dest='debug', action='store_true', default=False, 167 help='run debug tests') 168 parser.add_argument('--enable-arkguard', action='store_true', dest='enable_arkguard', default=False, 169 help='enable arkguard for compiler tests') 170 parser.add_argument('--aop-transform', dest='aop_transform', action='store_true', default=False, 171 help='run debug tests') 172 173 return parser.parse_args() 174 175 176def run_subprocess_with_beta3(test_obj, cmd): 177 has_target_api = False 178 has_version_12 = False 179 has_sub_version = False 180 is_es2abc_cmd = False 181 182 for param in cmd: 183 if "es2abc" in param: 184 is_es2abc_cmd = True 185 if "--target-api-sub-version" in param: 186 has_sub_version = True 187 if "--target-api-version" in param: 188 has_target_api = True 189 if "12" in param: 190 has_version_12 = True 191 if is_es2abc_cmd and (not has_target_api or (has_version_12 and not has_sub_version)): 192 cmd.append("--target-api-sub-version=beta3") 193 if test_obj: 194 test_obj.log_cmd(cmd) 195 return subprocess.Popen( 196 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 197 198 199class Test: 200 def __init__(self, test_path, flags): 201 self.path = test_path 202 self.flags = flags 203 self.output = None 204 self.error = None 205 self.passed = None 206 self.skipped = None 207 self.reproduce = "" 208 209 def log_cmd(self, cmd): 210 self.reproduce += "\n" + ' '.join(cmd) 211 212 def get_path_to_expected(self): 213 if self.path.find(".d.ts") == -1: 214 return "%s-expected.txt" % (path.splitext(self.path)[0]) 215 return "%s-expected.txt" % (self.path[:self.path.find(".d.ts")]) 216 217 def run(self, runner): 218 test_abc_name = ("%s.abc" % (path.splitext(self.path)[0])).replace("/", "_") 219 test_abc_path = path.join(runner.build_dir, test_abc_name) 220 cmd = runner.cmd_prefix + [runner.es2panda] 221 cmd.extend(self.flags) 222 cmd.extend(["--output=" + test_abc_path]) 223 cmd.append(self.path) 224 process = run_subprocess_with_beta3(self, cmd) 225 out, err = process.communicate() 226 self.output = out.decode("utf-8", errors="ignore") + err.decode("utf-8", errors="ignore") 227 228 expected_path = self.get_path_to_expected() 229 try: 230 with open(expected_path, 'r') as fp: 231 expected = fp.read() 232 self.passed = expected == self.output and process.returncode in [ 233 0, 1] 234 except Exception: 235 self.passed = False 236 237 if not self.passed: 238 self.error = err.decode("utf-8", errors="ignore") 239 240 if os.path.exists(test_abc_path): 241 os.remove(test_abc_path) 242 243 return self 244 245 246class TSCTest(Test): 247 def __init__(self, test_path, flags): 248 Test.__init__(self, test_path, flags) 249 self.options = self.parse_options() 250 251 def parse_options(self): 252 test_options = {} 253 254 with open(self.path, "r", encoding="latin1") as f: 255 lines = f.read() 256 options = re.findall(r"//\s?@\w+:.*\n", lines) 257 258 for option in options: 259 separated = option.split(":") 260 opt = re.findall(r"\w+", separated[0])[0].lower() 261 value = separated[1].strip().lower() 262 263 if opt == "filename": 264 if opt in options: 265 test_options[opt].append(value) 266 else: 267 test_options[opt] = [value] 268 269 elif opt == "lib" or opt == "module": 270 test_options[opt] = [each.strip() 271 for each in value.split(",")] 272 elif value == "true" or value == "false": 273 test_options[opt] = value.lower() == "true" 274 else: 275 test_options[opt] = value 276 277 # TODO: Possibility of error: all exports will be catched, even the commented ones 278 if 'module' not in test_options and re.search(r"export ", lines): 279 test_options['module'] = [] 280 281 return test_options 282 283 def run(self, runner): 284 cmd = runner.cmd_prefix + [runner.es2panda, '--parse-only'] 285 cmd.extend(self.flags) 286 if "module" in self.options: 287 cmd.append('--module') 288 cmd.append(self.path) 289 process = run_subprocess_with_beta3(self, cmd) 290 out, err = process.communicate() 291 self.output = out.decode("utf-8", errors="ignore") 292 293 self.passed = True if process.returncode == 0 else False 294 295 if not self.passed: 296 self.error = err.decode("utf-8", errors="ignore") 297 298 return self 299 300 301class TestAop: 302 def __init__(self, cmd, compare_str, compare_abc_str, remove_file): 303 self.cmd = cmd 304 self.compare_str = compare_str 305 self.compare_abc_str = compare_abc_str 306 self.remove_file = remove_file 307 self.path = '' 308 self.output = None 309 self.error = None 310 self.passed = None 311 self.skipped = None 312 self.reproduce = "" 313 314 def log_cmd(self, cmd): 315 self.reproduce += ''.join(["\n", ' '.join(cmd)]) 316 317 def run(self, runner): 318 cmd = self.cmd 319 process = run_subprocess_with_beta3(self, cmd) 320 out, err = process.communicate() 321 self.output = out.decode("utf-8", errors="ignore") + err.decode("utf-8", errors="ignore") 322 323 if self.compare_str == '': 324 self.passed = True 325 else : 326 self.passed = self.output.startswith(self.compare_str) and process.returncode in [0, 1] 327 if self.remove_file != '' and os.path.exists(self.remove_file): 328 os.remove(self.remove_file) 329 330 if not self.passed: 331 self.error = err.decode("utf-8", errors="ignore") 332 333 abc_path = path.join(os.getcwd(), 'test_aop.abc') 334 if os.path.exists(abc_path): 335 if self.compare_abc_str != '': 336 with open(abc_path, "r") as abc_file: 337 self.passed = self.passed and abc_file.read() == self.compare_abc_str 338 os.remove(abc_path) 339 340 return self 341 342 343class Runner: 344 def __init__(self, args, name): 345 self.test_root = path.dirname(path.abspath(__file__)) 346 self.args = args 347 self.name = name 348 self.tests = [] 349 self.failed = 0 350 self.passed = 0 351 self.es2panda = path.join(args.build_dir, 'es2abc') 352 self.build_dir = args.build_dir 353 self.cmd_prefix = [] 354 self.ark_js_vm = "" 355 self.ark_aot_compiler = "" 356 self.ld_library_path = "" 357 358 if args.js_runtime_path: 359 self.ark_js_vm = path.join(args.js_runtime_path, 'ark_js_vm') 360 self.ark_aot_compiler = path.join(args.js_runtime_path, 'ark_aot_compiler') 361 362 if args.ld_library_path: 363 self.ld_library_path = args.ld_library_path 364 365 if args.arm64_qemu: 366 self.cmd_prefix = ["qemu-aarch64", "-L", "/usr/aarch64-linux-gnu/"] 367 368 if args.arm32_qemu: 369 self.cmd_prefix = ["qemu-arm", "-L", "/usr/arm-linux-gnueabi"] 370 371 if not path.isfile(self.es2panda): 372 raise Exception("Cannot find es2panda binary: %s" % self.es2panda) 373 374 def add_directory(self, directory, extension, flags): 375 pass 376 377 def test_path(self, src): 378 pass 379 380 def run_test(self, test): 381 return test.run(self) 382 383 def run(self): 384 pool = multiprocessing.Pool() 385 result_iter = pool.imap_unordered( 386 self.run_test, self.tests, chunksize=32) 387 pool.close() 388 389 if self.args.progress: 390 from tqdm import tqdm 391 result_iter = tqdm(result_iter, total=len(self.tests)) 392 393 results = [] 394 for res in result_iter: 395 results.append(res) 396 397 self.tests = results 398 pool.join() 399 400 def deal_error(self, test): 401 path_str = test.path 402 err_col = {} 403 if test.error: 404 err_str = test.error.split('[')[0] if "patchfix" not in test.path else " patchfix throw error failed" 405 err_col = {"path" : [path_str], "status": ["fail"], "error" : [test.error], "type" : [err_str]} 406 else: 407 err_col = {"path" : [path_str], "status": ["fail"], "error" : ["Segmentation fault"], 408 "type" : ["Segmentation fault"]} 409 return err_col 410 411 def summarize(self): 412 print("") 413 fail_list = [] 414 success_list = [] 415 416 for test in self.tests: 417 assert(test.passed is not None) 418 if not test.passed: 419 fail_list.append(test) 420 else: 421 success_list.append(test) 422 423 if len(fail_list): 424 if self.args.error: 425 import pandas as pd 426 test_list = pd.DataFrame(columns=["path", "status", "error", "type"]) 427 for test in success_list: 428 suc_col = {"path" : [test.path], "status": ["success"], "error" : ["success"], "type" : ["success"]} 429 if self.args.error: 430 test_list = pd.concat([test_list, pd.DataFrame(suc_col)]) 431 print("Failed tests:") 432 for test in fail_list: 433 print(self.test_path(test.path)) 434 435 if self.args.error: 436 print("steps:", test.reproduce) 437 print("error:") 438 print(test.error) 439 print("\n") 440 err_col = self.deal_error(test) 441 test_list = pd.concat([test_list, pd.DataFrame(err_col)]) 442 443 if self.args.error: 444 test_list.to_csv('test_statistics.csv', index=False) 445 test_list["type"].value_counts().to_csv('type_statistics.csv', index_label="error") 446 print("Type statistics:\n", test_list["type"].value_counts()) 447 print("") 448 449 print("Summary(%s):" % self.name) 450 print("\033[37mTotal: %5d" % (len(self.tests))) 451 print("\033[92mPassed: %5d" % (len(self.tests) - len(fail_list))) 452 print("\033[91mFailed: %5d" % (len(fail_list))) 453 print("\033[0m") 454 455 return len(fail_list) 456 457 458class RegressionRunner(Runner): 459 def __init__(self, args): 460 Runner.__init__(self, args, "Regression") 461 462 def add_directory(self, directory, extension, flags, func=Test): 463 glob_expression = path.join( 464 self.test_root, directory, "*.%s" % (extension)) 465 files = glob(glob_expression) 466 files = fnmatch.filter(files, self.test_root + '**' + self.args.filter) 467 468 self.tests += list(map(lambda f: func(f, flags), files)) 469 470 def test_path(self, src): 471 return src 472 473 474class AbcToAsmRunner(Runner): 475 def __init__(self, args, is_debug): 476 Runner.__init__(self, args, "Abc2asm" if not is_debug else "Abc2asmDebug") 477 self.is_debug = is_debug 478 479 def add_directory(self, directory, extension, flags, func=Test): 480 glob_expression = path.join( 481 self.test_root, directory, "*.%s" % (extension)) 482 files = glob(glob_expression) 483 files = fnmatch.filter(files, self.test_root + '**' + self.args.filter) 484 485 self.tests += list(map(lambda f: AbcToAsmTest(f, flags, self.is_debug), files)) 486 487 def test_path(self, src): 488 return os.path.basename(src) 489 490 491class AbcToAsmTest(Test): 492 def __init__(self, test_path, flags, is_debug): 493 Test.__init__(self, test_path, flags) 494 self.is_debug = is_debug 495 496 def run(self, runner): 497 output_abc_file = ("%s.abc" % (path.splitext(self.path)[0])).replace("/", "_") 498 # source code compilation, generate an abc file 499 gen_abc_cmd = runner.cmd_prefix + [runner.es2panda] 500 if (self.is_debug): 501 gen_abc_cmd.extend(["--debug-info"]) 502 gen_abc_cmd.extend(["--module", "--dump-normalized-asm-program", "--output=" + output_abc_file]) 503 gen_abc_cmd.append(self.path) 504 process_gen_abc = run_subprocess_with_beta3(self, gen_abc_cmd) 505 gen_abc_out, gen_abc_err = process_gen_abc.communicate() 506 gen_abc_output = gen_abc_out.decode("utf-8", errors="ignore") 507 508 # If no abc file is generated, an error occurs during parser, but abc2asm function is normal. 509 if not os.path.exists(output_abc_file): 510 self.passed = True 511 return self 512 513 # abc file compilation 514 abc_to_asm_cmd = runner.cmd_prefix + [runner.es2panda] 515 if (self.is_debug): 516 abc_to_asm_cmd.extend(["--debug-info"]) 517 abc_to_asm_cmd.extend(["--module", "--dump-normalized-asm-program", "--enable-abc-input"]) 518 abc_to_asm_cmd.append(output_abc_file) 519 process_abc_to_asm = run_subprocess_with_beta3(self, abc_to_asm_cmd) 520 abc_to_asm_out, abc_to_asm_err = process_abc_to_asm.communicate() 521 abc_to_asm_output = abc_to_asm_out.decode("utf-8", errors="ignore") 522 523 self.passed = gen_abc_output == abc_to_asm_output and process_abc_to_asm.returncode in [0, 1] 524 if not self.passed: 525 self.error = "Comparison of dump results between source code compilation and abc file compilation failed." 526 if gen_abc_err: 527 self.error += "\n" + gen_abc_err.decode("utf-8", errors="ignore") 528 if abc_to_asm_err: 529 self.error += "\n" + abc_to_asm_err.decode("utf-8", errors="ignore") 530 531 os.remove(output_abc_file) 532 return self 533 534 535class TSCRunner(Runner): 536 def __init__(self, args): 537 Runner.__init__(self, args, "TSC") 538 539 if self.args.tsc_path: 540 self.tsc_path = self.args.tsc_path 541 else : 542 self.tsc_path = prepare_tsc_testcases(self.test_root) 543 544 self.add_directory("conformance", []) 545 self.add_directory("compiler", []) 546 547 def add_directory(self, directory, flags): 548 ts_suite_dir = path.join(self.tsc_path, 'tests/cases') 549 550 glob_expression = path.join( 551 ts_suite_dir, directory, "**/*.ts") 552 files = glob(glob_expression, recursive=True) 553 files = fnmatch.filter(files, ts_suite_dir + '**' + self.args.filter) 554 555 for f in files: 556 test_name = path.basename(f.split(".ts")[0]) 557 negative_references = path.join( 558 self.tsc_path, 'tests/baselines/reference') 559 is_negative = path.isfile(path.join(negative_references, 560 test_name + ".errors.txt")) 561 test = TSCTest(f, flags) 562 563 if 'target' in test.options: 564 targets = test.options['target'].replace(" ", "").split(',') 565 for target in targets: 566 if path.isfile(path.join(negative_references, 567 test_name + "(target=%s).errors.txt" % (target))): 568 is_negative = True 569 break 570 571 if is_negative or "filename" in test.options: 572 continue 573 574 with open(path.join(self.test_root, 'test_tsc_ignore_list.txt'), 'r') as failed_references: 575 if self.args.skip: 576 if path.relpath(f, self.tsc_path) in failed_references.read(): 577 continue 578 579 self.tests.append(test) 580 581 def test_path(self, src): 582 return src 583 584 585class CompilerRunner(Runner): 586 def __init__(self, args): 587 Runner.__init__(self, args, "Compiler") 588 589 def add_directory(self, directory, extension, flags): 590 if directory.endswith("projects"): 591 projects_path = path.join(self.test_root, directory) 592 for project in os.listdir(projects_path): 593 glob_expression = path.join(projects_path, project, "**/*.%s" % (extension)) 594 files = glob(glob_expression, recursive=True) 595 files = fnmatch.filter(files, self.test_root + '**' + self.args.filter) 596 self.tests.append(CompilerProjectTest(projects_path, project, files, flags)) 597 else: 598 glob_expression = path.join( 599 self.test_root, directory, "**/*.%s" % (extension)) 600 files = glob(glob_expression, recursive=True) 601 files = fnmatch.filter(files, self.test_root + '**' + self.args.filter) 602 self.tests += list(map(lambda f: CompilerTest(f, flags), files)) 603 604 def test_path(self, src): 605 return src 606 607 608class CompilerTest(Test): 609 def __init__(self, test_path, flags): 610 Test.__init__(self, test_path, flags) 611 612 def execute_arkguard(self, runner): 613 input_file_path = self.path 614 arkguard_root_dir = os.path.join(runner.test_root, "../../arkguard") 615 arkgurad_entry_path = os.path.join(arkguard_root_dir, "lib/cli/SecHarmony.js") 616 config_path = os.path.join(arkguard_root_dir, "test/compilerTestConfig.json") 617 arkguard_cmd = [ 618 'node', 619 '--no-warnings', 620 arkgurad_entry_path, 621 input_file_path, 622 '--config-path', 623 config_path, 624 '--inplace' 625 ] 626 self.log_cmd(arkguard_cmd) 627 process = subprocess.Popen(arkguard_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 628 out, err = process.communicate() 629 process.wait() 630 success = True 631 if err or process.returncode != 0: 632 success = False 633 self.passed = False 634 self.error = err.decode("utf-8", errors="ignore") 635 return success 636 637 def run(self, runner): 638 test_abc_name = ("%s.abc" % (path.splitext(self.path)[0])).replace("/", "_") 639 test_abc_path = path.join(runner.build_dir, test_abc_name) 640 es2abc_cmd = runner.cmd_prefix + [runner.es2panda] 641 es2abc_cmd.extend(self.flags) 642 es2abc_cmd.extend(["--output=" + test_abc_path]) 643 es2abc_cmd.append(self.path) 644 enable_arkguard = runner.args.enable_arkguard 645 if enable_arkguard: 646 success = self.execute_arkguard(runner) 647 if not success: 648 return self 649 650 process = run_subprocess_with_beta3(self, es2abc_cmd) 651 out, err = process.communicate() 652 if "--dump-assembly" in self.flags: 653 pa_expected_path = "".join([self.get_path_to_expected()[:self.get_path_to_expected().rfind(".txt")], 654 ".pa.txt"]) 655 self.output = out.decode("utf-8", errors="ignore") + err.decode("utf-8", errors="ignore") 656 try: 657 with open(pa_expected_path, 'r') as fp: 658 expected = fp.read() 659 self.passed = expected == self.output and process.returncode in [0, 1] 660 except Exception: 661 self.passed = False 662 if not self.passed: 663 self.error = err.decode("utf-8", errors="ignore") 664 if os.path.exists(test_abc_path): 665 os.remove(test_abc_path) 666 return self 667 if "--dump-debug-info" in self.flags: 668 self.output = out.decode("utf-8", errors="ignore") + err.decode("utf-8", errors="ignore") 669 try: 670 with open(self.get_path_to_expected(), 'r') as fp: 671 expected = fp.read() 672 self.passed = expected == self.output and process.returncode in [0, 1] 673 if os.path.exists(test_abc_path): 674 os.remove(test_abc_path) 675 return self 676 except Exception: 677 self.passed = False 678 if not self.passed: 679 self.error = err.decode("utf-8", errors="ignore") 680 if os.path.exists(test_abc_path): 681 os.remove(test_abc_path) 682 return self 683 if err: 684 self.passed = False 685 self.error = err.decode("utf-8", errors="ignore") 686 return self 687 688 ld_library_path = runner.ld_library_path 689 os.environ.setdefault("LD_LIBRARY_PATH", ld_library_path) 690 run_abc_cmd = [runner.ark_js_vm, '--enable-force-gc=false', test_abc_path] 691 self.log_cmd(run_abc_cmd) 692 693 process = subprocess.Popen(run_abc_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 694 out, err = process.communicate() 695 self.output = out.decode("utf-8", errors="ignore") + err.decode("utf-8", errors="ignore") 696 expected_path = self.get_path_to_expected() 697 try: 698 with open(expected_path, 'r') as fp: 699 expected = fp.read() 700 self.passed = expected == self.output and process.returncode in [0, 1] 701 except Exception: 702 self.passed = False 703 704 if not self.passed: 705 self.error = err.decode("utf-8", errors="ignore") 706 707 os.remove(test_abc_path) 708 709 return self 710 711 712class CompilerProjectTest(Test): 713 def __init__(self, projects_path, project, test_paths, flags): 714 Test.__init__(self, "", flags) 715 self.projects_path = projects_path 716 self.project = project 717 self.test_paths = test_paths 718 self.files_info_path = os.path.join(os.path.join(self.projects_path, self.project), 'filesInfo.txt') 719 # Skip execution if --dump-assembly exists in flags 720 self.requires_execution = "--dump-assembly" not in self.flags 721 self.file_record_mapping = None 722 self.generated_abc_inputs_path = os.path.join(os.path.join(self.projects_path, self.project), "abcinputs_gen") 723 self.abc_input_filenames = None 724 self.record_names_path = os.path.join(os.path.join(self.projects_path, self.project), 'recordnames.txt') 725 self.abc_inputs_path = os.path.join(os.path.join(self.projects_path, self.project), 'abcinputs') 726 727 def remove_project(self, runner): 728 project_path = runner.build_dir + "/" + self.project 729 if path.exists(project_path): 730 shutil.rmtree(project_path) 731 if path.exists(self.files_info_path): 732 os.remove(self.files_info_path) 733 if path.exists(self.generated_abc_inputs_path): 734 shutil.rmtree(self.generated_abc_inputs_path) 735 736 def get_file_absolute_path_and_name(self, runner): 737 sub_path = self.path[len(self.projects_path):] 738 file_relative_path = path.split(sub_path)[0] 739 file_name = path.split(sub_path)[1] 740 file_absolute_path = runner.build_dir + "/" + file_relative_path 741 return [file_absolute_path, file_name] 742 743 def gen_single_abc(self, runner): 744 for test_path in self.test_paths: 745 self.path = test_path 746 [file_absolute_path, file_name] = self.get_file_absolute_path_and_name(runner) 747 if not path.exists(file_absolute_path): 748 os.makedirs(file_absolute_path) 749 750 test_abc_name = ("%s.abc" % (path.splitext(file_name)[0])) 751 test_abc_path = path.join(file_absolute_path, test_abc_name) 752 es2abc_cmd = runner.cmd_prefix + [runner.es2panda] 753 es2abc_cmd.extend(self.flags) 754 es2abc_cmd.extend(['%s%s' % ("--output=", test_abc_path)]) 755 es2abc_cmd.append(self.path) 756 process = run_subprocess_with_beta3(self, es2abc_cmd) 757 out, err = process.communicate() 758 if err: 759 self.passed = False 760 self.error = err.decode("utf-8", errors="ignore") 761 self.remove_project(runner) 762 return self 763 764 def collect_record_mapping(self): 765 # Collect record mappings from recordnames.txt, file format: 766 # 'source_file_name:record_name\n' * n 767 if path.exists(self.record_names_path): 768 with open(self.record_names_path) as mapping_fp: 769 mapping_lines = mapping_fp.readlines() 770 self.file_record_mapping = {} 771 for mapping_line in mapping_lines: 772 cur_mapping = mapping_line[:-1].split(":") 773 self.file_record_mapping[cur_mapping[0]] = cur_mapping[1] 774 775 def get_record_name(self, test_path): 776 record_name = os.path.relpath(test_path, os.path.dirname(self.files_info_path)).split('.')[0] 777 if (self.file_record_mapping is not None and record_name in self.file_record_mapping): 778 record_name = self.file_record_mapping[record_name] 779 return record_name 780 781 def collect_abc_inputs(self, runner): 782 # Collect abc input information from the 'abcinputs' directory. Each txt file in the directory 783 # will generate a merged abc file with the same filename and serve as the final abc input. 784 # file format: 'source_file_name.ts\n' * n 785 if not path.exists(self.abc_inputs_path): 786 return 787 if not path.exists(self.generated_abc_inputs_path): 788 os.makedirs(self.generated_abc_inputs_path) 789 self.abc_input_filenames = {} 790 filenames = os.listdir(self.abc_inputs_path) 791 for filename in filenames: 792 if not filename.endswith('.txt'): 793 self.remove_project(runner) 794 raise Exception("Invalid abc input file: %s, only txt files are allowed in abcinputs directory: %s" 795 % (filename, self.abc_inputs_path)) 796 with open(path.join(self.abc_inputs_path, filename)) as abc_inputs_fp: 797 abc_inputs_lines = abc_inputs_fp.readlines() 798 for abc_input_line in abc_inputs_lines: 799 # filename is 'xxx.txt', remove '.txt' here 800 self.abc_input_filenames[abc_input_line[:-1]] = filename[:-len('.txt')] 801 802 def get_belonging_abc_input(self, test_path): 803 filename = os.path.relpath(test_path, os.path.dirname(self.files_info_path)) 804 if (self.abc_input_filenames is not None and filename in self.abc_input_filenames): 805 return self.abc_input_filenames[filename] 806 return None 807 808 def gen_abc_input_files_infos(self, runner, abc_files_infos, final_file_info_f): 809 for abc_files_info_name in abc_files_infos: 810 abc_files_info = abc_files_infos[abc_files_info_name] 811 if len(abc_files_info) != 0: 812 abc_input_path = path.join(self.generated_abc_inputs_path, abc_files_info_name) 813 abc_files_info_path = ("%s-filesInfo.txt" % (abc_input_path)) 814 abc_files_info_fd = os.open(abc_files_info_path, os.O_RDWR | os.O_CREAT | os.O_TRUNC) 815 abc_files_info_f = os.fdopen(abc_files_info_fd, 'w') 816 abc_files_info_f.writelines(abc_files_info) 817 final_file_info_f.writelines('%s-abcinput.abc;;;;%s;\n' % (abc_input_path, abc_files_info_name)) 818 819 def gen_files_info(self, runner): 820 # After collect_record_mapping, self.file_record_mapping stores {'source file name' : 'source file record name'} 821 self.collect_record_mapping() 822 # After collect_abc_inputs, self.abc_input_filenames stores {'source file name' : 'belonging abc input name'} 823 self.collect_abc_inputs(runner) 824 825 fd = os.open(self.files_info_path, os.O_RDWR | os.O_CREAT | os.O_TRUNC) 826 f = os.fdopen(fd, 'w') 827 abc_files_infos = {} 828 for test_path in self.test_paths: 829 record_name = self.get_record_name(test_path) 830 module_kind = 'esm' 831 if (os.path.basename(test_path).startswith("commonjs")): 832 module_kind = 'commonjs' 833 is_shared_module = 'false' 834 if (os.path.basename(test_path).startswith("sharedmodule")): 835 is_shared_module = 'true' 836 file_info = ('%s;%s;%s;%s;%s;%s\n' % (test_path, record_name, module_kind, 837 os.path.relpath(test_path, self.projects_path), record_name, 838 is_shared_module)) 839 belonging_abc_input = self.get_belonging_abc_input(test_path) 840 if belonging_abc_input is not None: 841 if not belonging_abc_input in abc_files_infos: 842 abc_files_infos[belonging_abc_input] = [] 843 abc_files_infos[belonging_abc_input].append(file_info) 844 else: 845 f.writelines(file_info) 846 self.gen_abc_input_files_infos(runner, abc_files_infos, f) 847 f.close() 848 849 def gen_es2abc_cmd(self, runner, input_file, output_file): 850 es2abc_cmd = runner.cmd_prefix + [runner.es2panda] 851 es2abc_cmd.extend(self.flags) 852 es2abc_cmd.extend(['%s%s' % ("--output=", output_file)]) 853 es2abc_cmd.append(input_file) 854 return es2abc_cmd 855 856 def gen_merged_abc_for_abc_input(self, runner, files_info_name): 857 self.passed = True 858 if not files_info_name.endswith(".txt"): 859 return 860 abc_input_files_info_path = path.join(self.generated_abc_inputs_path, files_info_name) 861 abc_input_merged_abc_path = path.join(self.generated_abc_inputs_path, 862 '%s-abcinput.abc' % (files_info_name[:-len('-filesInfo.txt')])) 863 864 abc_input_file_path = '@' + abc_input_files_info_path 865 if "unmerged_abc_input" in self.generated_abc_inputs_path: 866 self.flags.remove("--merge-abc") 867 with open(abc_input_files_info_path, 'r') as fp: 868 abc_input_file_path = fp.read().split(';')[0] 869 870 es2abc_cmd = self.gen_es2abc_cmd(runner, abc_input_file_path, abc_input_merged_abc_path) 871 process = run_subprocess_with_beta3(self, es2abc_cmd) 872 out, err = process.communicate() 873 if err: 874 self.passed = False 875 self.error = err.decode("utf-8", errors="ignore") 876 877 def gen_merged_abc(self, runner): 878 # Generate abc inputs 879 if (os.path.exists(self.generated_abc_inputs_path)): 880 files_info_names = os.listdir(self.generated_abc_inputs_path) 881 for filename in files_info_names: 882 self.gen_merged_abc_for_abc_input(runner, filename) 883 if (not self.passed): 884 self.remove_project(runner) 885 return self 886 # Generate the abc to be tested 887 for test_path in self.test_paths: 888 self.path = test_path 889 if (self.path.endswith("-exec.ts")) or (self.path.endswith("-exec.js")): 890 exec_file_path = self.path 891 [file_absolute_path, file_name] = self.get_file_absolute_path_and_name(runner) 892 if not path.exists(file_absolute_path): 893 os.makedirs(file_absolute_path) 894 test_abc_name = ("%s.abc" % (path.splitext(file_name)[0])) 895 output_abc_name = path.join(file_absolute_path, test_abc_name) 896 897 # reverse merge-abc flag 898 if "merge_abc_consistence_check" in self.path: 899 if "--merge-abc" in self.flags: 900 self.flags.remove("--merge-abc") 901 else: 902 self.flags.append("--merge-abc") 903 904 es2abc_cmd = self.gen_es2abc_cmd(runner, '@' + self.files_info_path, output_abc_name) 905 compile_context_info_path = path.join(path.join(self.projects_path, self.project), "compileContextInfo.json") 906 if path.exists(compile_context_info_path): 907 es2abc_cmd.append("%s%s" % ("--compile-context-info=", compile_context_info_path)) 908 process = run_subprocess_with_beta3(self, es2abc_cmd) 909 self.path = exec_file_path 910 out, err = process.communicate() 911 912 # restore merge-abc flag 913 if "merge_abc_consistence_check" in self.path and "--merge-abc" not in self.flags: 914 self.flags.append("--merge-abc") 915 916 # Check dump-assembly outputs when required 917 if "--dump-assembly" in self.flags: 918 pa_expected_path = "".join([self.get_path_to_expected()[:self.get_path_to_expected().rfind(".txt")], 919 ".pa.txt"]) 920 self.output = out.decode("utf-8", errors="ignore") + err.decode("utf-8", errors="ignore") 921 if "merge_abc_consistence_check" in self.path: 922 self.output = self.output.split('.')[0] 923 try: 924 with open(pa_expected_path, 'r') as fp: 925 expected = fp.read() 926 self.passed = expected == self.output and process.returncode in [0, 1] 927 except Exception: 928 self.passed = False 929 if not self.passed: 930 self.error = err.decode("utf-8", errors="ignore") 931 self.remove_project(runner) 932 return self 933 else: 934 return self 935 936 if err: 937 self.passed = False 938 self.error = err.decode("utf-8", errors="ignore") 939 self.remove_project(runner) 940 return self 941 942 def run(self, runner): 943 # Compile all ts source files in the project to abc files. 944 if ("--merge-abc" in self.flags): 945 self.gen_files_info(runner) 946 self.gen_merged_abc(runner) 947 else: 948 self.gen_single_abc(runner) 949 950 if (not self.requires_execution): 951 self.remove_project(runner) 952 return self 953 954 # Run test files that need to be executed in the project. 955 for test_path in self.test_paths: 956 self.path = test_path 957 if self.path.endswith("-exec.ts"): 958 [file_absolute_path, file_name] = self.get_file_absolute_path_and_name(runner) 959 960 entry_point_name = path.splitext(file_name)[0] 961 test_abc_name = ("%s.abc" % entry_point_name) 962 test_abc_path = path.join(file_absolute_path, test_abc_name) 963 964 ld_library_path = runner.ld_library_path 965 os.environ.setdefault("LD_LIBRARY_PATH", ld_library_path) 966 run_abc_cmd = [runner.ark_js_vm] 967 if ("--merge-abc" in self.flags): 968 run_abc_cmd.extend(["--entry-point", entry_point_name]) 969 run_abc_cmd.extend([test_abc_path]) 970 self.log_cmd(run_abc_cmd) 971 972 process = subprocess.Popen(run_abc_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 973 out, err = process.communicate() 974 self.output = out.decode("utf-8", errors="ignore") + err.decode("utf-8", errors="ignore") 975 expected_path = self.get_path_to_expected() 976 try: 977 with open(expected_path, 'r') as fp: 978 expected = fp.read() 979 self.passed = expected == self.output and process.returncode in [0, 1] 980 except Exception: 981 self.passed = False 982 983 if not self.passed: 984 self.error = err.decode("utf-8", errors="ignore") 985 self.remove_project(runner) 986 return self 987 988 self.passed = True 989 990 self.remove_project(runner) 991 return self 992 993 994class TSDeclarationTest(Test): 995 def get_path_to_expected(self): 996 file_name = self.path[:self.path.find(".d.ts")] 997 return "%s-expected.txt" % file_name 998 999 1000class BcVersionRunner(Runner): 1001 def __init__(self, args): 1002 Runner.__init__(self, args, "Target bc version") 1003 self.ts2abc = path.join(self.test_root, '..', 'scripts', 'ts2abc.js') 1004 1005 def add_cmd(self): 1006 api_sub_version_list = ["beta1", "beta2", "beta3"] 1007 for api_version in range(8, 14): 1008 cmd = self.cmd_prefix + [self.es2panda] 1009 cmd += ["--target-bc-version"] 1010 cmd += ["--target-api-version"] 1011 cmd += [str(api_version)] 1012 self.tests += [BcVersionTest(cmd, api_version)] 1013 node_cmd = ["node"] + [self.ts2abc] 1014 node_cmd += ["".join(["es2abc=", self.es2panda])] 1015 node_cmd += ["--target-api-version"] 1016 node_cmd += [str(api_version)] 1017 self.tests += [BcVersionTest(node_cmd, api_version)] 1018 1019 # Add tests for "--target-api-sub-version" option 1020 if api_version == 12: 1021 for api_sub_version in api_sub_version_list: 1022 new_cmd = cmd.copy() 1023 new_cmd += ["--target-api-sub-version", api_sub_version] 1024 self.tests += [BcVersionTest(new_cmd, str(api_version) + '_' + api_sub_version)] 1025 new_node_cmd = node_cmd.copy() 1026 new_node_cmd += ["--target-api-sub-version", api_sub_version] 1027 self.tests += [BcVersionTest(new_node_cmd, str(api_version) + '_' + api_sub_version)] 1028 1029 def run(self): 1030 for test in self.tests: 1031 test.run() 1032 1033 1034class BcVersionTest(Test): 1035 def __init__(self, cmd, api_version): 1036 Test.__init__(self, "", 0) 1037 self.cmd = cmd 1038 self.api_version = api_version 1039 self.bc_version_expect = { 1040 8: "12.0.6.0", 1041 9: "9.0.0.0", 1042 10: "9.0.0.0", 1043 11: "11.0.2.0", 1044 12: "12.0.2.0", 1045 "12_beta1": "12.0.2.0", 1046 "12_beta2": "12.0.2.0", 1047 "12_beta3": "12.0.6.0", 1048 13: "12.0.6.0" 1049 } 1050 self.es2abc_script_expect = { 1051 8: "0.0.0.2", 1052 9: "9.0.0.0", 1053 10: "9.0.0.0", 1054 11: "11.0.2.0", 1055 12: "12.0.2.0", 1056 "12_beta1": "12.0.2.0", 1057 "12_beta2": "12.0.2.0", 1058 "12_beta3": "12.0.6.0", 1059 13: "12.0.6.0" 1060 } 1061 1062 def run(self): 1063 process = subprocess.Popen(self.cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 1064 out, err = process.communicate() 1065 self.output = out.decode("utf-8", errors="ignore") + err.decode("utf-8", errors="ignore") 1066 if self.cmd[0] == "node": 1067 self.passed = self.es2abc_script_expect.get(self.api_version) == self.output and process.returncode in [0, 1] 1068 else: 1069 self.passed = self.bc_version_expect.get(self.api_version) == self.output and process.returncode in [0, 1] 1070 if not self.passed: 1071 self.error = err.decode("utf-8", errors="ignore") 1072 return self 1073 1074 1075class TransformerRunner(Runner): 1076 def __init__(self, args): 1077 Runner.__init__(self, args, "Transformer") 1078 1079 def add_directory(self, directory, extension, flags): 1080 glob_expression = path.join( 1081 self.test_root, directory, "**/*.%s" % (extension)) 1082 files = glob(glob_expression, recursive=True) 1083 files = fnmatch.filter(files, self.test_root + '**' + self.args.filter) 1084 1085 self.tests += list(map(lambda f: TransformerTest(f, flags), files)) 1086 1087 def test_path(self, src): 1088 return src 1089 1090 1091class TransformerInTargetApiVersion10Runner(Runner): 1092 def __init__(self, args): 1093 Runner.__init__(self, args, "TransformerInTargetApiVersion10") 1094 1095 def add_directory(self, directory, extension, flags): 1096 glob_expression = path.join( 1097 self.test_root, directory, "**/*.%s" % (extension)) 1098 files = glob(glob_expression, recursive=True) 1099 files = fnmatch.filter(files, self.test_root + '**' + self.args.filter) 1100 1101 self.tests += list(map(lambda f: TransformerTest(f, flags), files)) 1102 1103 def test_path(self, src): 1104 return src 1105 1106 1107class TransformerTest(Test): 1108 def __init__(self, test_path, flags): 1109 Test.__init__(self, test_path, flags) 1110 1111 def get_path_to_expected(self): 1112 return "%s-transformed-expected.txt" % (path.splitext(self.path)[0]) 1113 1114 def run(self, runner): 1115 cmd = runner.cmd_prefix + [runner.es2panda] 1116 cmd.extend(self.flags) 1117 cmd.append(self.path) 1118 process = run_subprocess_with_beta3(self, cmd) 1119 out, err = process.communicate() 1120 self.output = out.decode("utf-8", errors="ignore") + err.decode("utf-8", errors="ignore") 1121 1122 expected_path = self.get_path_to_expected() 1123 try: 1124 with open(expected_path, 'r') as fp: 1125 expected = fp.read() 1126 self.passed = expected == self.output and process.returncode in [0, 1] 1127 except Exception: 1128 self.passed = False 1129 1130 if not self.passed: 1131 self.error = err.decode("utf-8", errors="ignore") 1132 1133 return self 1134 1135 1136class PatchTest(Test): 1137 def __init__(self, test_path, mode_arg, target_version, preserve_files): 1138 Test.__init__(self, test_path, "") 1139 self.mode = mode_arg 1140 self.target_version = target_version 1141 self.preserve_files = preserve_files 1142 1143 def gen_cmd(self, runner): 1144 symbol_table_file = os.path.join(self.path, 'base.map') 1145 origin_input_file = 'base.js' 1146 origin_output_abc = os.path.join(self.path, 'base.abc') 1147 modified_input_file = 'base_mod.js' 1148 modified_output_abc = os.path.join(self.path, 'patch.abc') 1149 target_version_cmd = "" 1150 if self.target_version > 0: 1151 target_version_cmd = "--target-api-version=" + str(self.target_version) 1152 1153 gen_base_cmd = runner.cmd_prefix + [runner.es2panda, '--module', target_version_cmd] 1154 if 'record-name-with-dots' in os.path.basename(self.path): 1155 gen_base_cmd.extend(['--merge-abc', '--record-name=record.name.with.dots']) 1156 gen_base_cmd.extend(['--dump-symbol-table', symbol_table_file]) 1157 gen_base_cmd.extend(['--output', origin_output_abc]) 1158 gen_base_cmd.extend([os.path.join(self.path, origin_input_file)]) 1159 self.log_cmd(gen_base_cmd) 1160 1161 if self.mode == 'hotfix': 1162 mode_arg = ["--generate-patch"] 1163 elif self.mode == 'hotreload': 1164 mode_arg = ["--hot-reload"] 1165 elif self.mode == 'coldfix': 1166 mode_arg = ["--generate-patch", "--cold-fix"] 1167 elif self.mode == 'coldreload': 1168 mode_arg = ["--cold-reload"] 1169 1170 patch_test_cmd = runner.cmd_prefix + [runner.es2panda, '--module', target_version_cmd] 1171 patch_test_cmd.extend(mode_arg) 1172 patch_test_cmd.extend(['--input-symbol-table', symbol_table_file]) 1173 patch_test_cmd.extend(['--output', modified_output_abc]) 1174 patch_test_cmd.extend([os.path.join(self.path, modified_input_file)]) 1175 if 'record-name-with-dots' in os.path.basename(self.path): 1176 patch_test_cmd.extend(['--merge-abc', '--record-name=record.name.with.dots']) 1177 dump_assembly_testname = [ 1178 'modify-anon-content-keep-origin-name', 1179 'modify-class-memeber-function', 1180 'exist-lexenv-3', 1181 'lexenv-reduce', 1182 'lexenv-increase'] 1183 for name in dump_assembly_testname: 1184 if name in os.path.basename(self.path): 1185 patch_test_cmd.extend(['--dump-assembly']) 1186 self.log_cmd(patch_test_cmd) 1187 1188 return gen_base_cmd, patch_test_cmd, symbol_table_file, origin_output_abc, modified_output_abc 1189 1190 def run(self, runner): 1191 gen_base_cmd, patch_test_cmd, symbol_table_file, origin_output_abc, modified_output_abc = self.gen_cmd(runner) 1192 1193 process_base = run_subprocess_with_beta3(None, gen_base_cmd) 1194 stdout_base, stderr_base = process_base.communicate(timeout=runner.args.es2panda_timeout) 1195 if stderr_base: 1196 self.passed = False 1197 self.error = stderr_base.decode("utf-8", errors="ignore") 1198 self.output = stdout_base.decode("utf-8", errors="ignore") + stderr_base.decode("utf-8", errors="ignore") 1199 else: 1200 process_patch = run_subprocess_with_beta3(None, patch_test_cmd) 1201 process_patch = subprocess.Popen(patch_test_cmd, stdout=subprocess.PIPE, 1202 stderr=subprocess.PIPE) 1203 stdout_patch, stderr_patch = process_patch.communicate(timeout=runner.args.es2panda_timeout) 1204 if stderr_patch: 1205 self.passed = False 1206 self.error = stderr_patch.decode("utf-8", errors="ignore") 1207 self.output = stdout_patch.decode("utf-8", errors="ignore") + stderr_patch.decode("utf-8", errors="ignore") 1208 1209 expected_path = os.path.join(self.path, 'expected.txt') 1210 try: 1211 with open(expected_path, 'r') as fp: 1212 # ignore license description lines and skip leading blank lines 1213 expected = (''.join((fp.readlines()[12:]))).lstrip() 1214 self.passed = expected == self.output 1215 except Exception: 1216 self.passed = False 1217 1218 if not self.passed: 1219 self.error = "expected output:" + os.linesep + expected + os.linesep + "actual output:" + os.linesep +\ 1220 self.output 1221 if not self.preserve_files: 1222 os.remove(symbol_table_file) 1223 os.remove(origin_output_abc) 1224 if (os.path.exists(modified_output_abc)): 1225 os.remove(modified_output_abc) 1226 return self 1227 1228 1229class PatchRunner(Runner): 1230 def __init__(self, args, name): 1231 Runner.__init__(self, args, name) 1232 self.preserve_files = args.error 1233 self.tests_in_dirs = [] 1234 dirs = os.listdir(path.join(self.test_root, "patch")) 1235 for target_version_path in dirs: 1236 self.add_tests(target_version_path, name) 1237 1238 def add_tests(self, target_version_path, name): 1239 name_dir = os.path.join(self.test_root, "patch", target_version_path, name) 1240 if not os.path.exists(name_dir): 1241 return 1242 target_version = 0 1243 if target_version_path.isdigit(): 1244 target_version = int(target_version_path) 1245 for sub_path in os.listdir(name_dir): 1246 test_base_path = os.path.join(name_dir, sub_path) 1247 if name != "coldreload": 1248 for test_dir in os.listdir(test_base_path): 1249 test_path = os.path.join(test_base_path, test_dir) 1250 self.tests_in_dirs.append(test_path) 1251 self.tests.append(PatchTest(test_path, name, target_version, self.preserve_files)) 1252 else: 1253 self.tests_in_dirs.append(test_base_path) 1254 self.tests.append(PatchTest(test_base_path, name, target_version, self.preserve_files)) 1255 1256 def test_path(self, src): 1257 return os.path.basename(src) 1258 1259 1260class HotfixRunner(PatchRunner): 1261 def __init__(self, args): 1262 PatchRunner.__init__(self, args, "hotfix") 1263 1264 1265class HotreloadRunner(PatchRunner): 1266 def __init__(self, args): 1267 PatchRunner.__init__(self, args, "hotreload") 1268 1269 1270class ColdfixRunner(PatchRunner): 1271 def __init__(self, args): 1272 PatchRunner.__init__(self, args, "coldfix") 1273 1274 1275class ColdreloadRunner(PatchRunner): 1276 def __init__(self, args): 1277 PatchRunner.__init__(self, args, "coldreload") 1278 1279 1280class DebuggerTest(Test): 1281 def __init__(self, test_path, mode): 1282 Test.__init__(self, test_path, "") 1283 self.mode = mode 1284 1285 def run(self, runner): 1286 cmd = runner.cmd_prefix + [runner.es2panda, "--module"] 1287 input_file_name = 'base.js' 1288 if self.mode == "debug-mode": 1289 cmd.extend(['--debug-info']) 1290 cmd.extend([os.path.join(self.path, input_file_name)]) 1291 cmd.extend(['--dump-assembly']) 1292 process = run_subprocess_with_beta3(self, cmd) 1293 stdout, stderr = process.communicate(timeout=runner.args.es2panda_timeout) 1294 if stderr: 1295 self.passed = False 1296 self.error = stderr.decode("utf-8", errors="ignore") 1297 return self 1298 1299 self.output = stdout.decode("utf-8", errors="ignore") 1300 1301 expected_path = os.path.join(self.path, 'expected.txt') 1302 try: 1303 with open(expected_path, 'r') as fp: 1304 expected = (''.join((fp.readlines()[12:]))).lstrip() 1305 self.passed = expected == self.output 1306 except Exception: 1307 self.passed = False 1308 1309 if not self.passed: 1310 self.error = "expected output:" + os.linesep + expected + os.linesep + "actual output:" + os.linesep +\ 1311 self.output 1312 1313 if os.path.exists("base.abc"): 1314 os.remove("base.abc") 1315 1316 return self 1317 1318 1319class DebuggerRunner(Runner): 1320 def __init__(self, args): 1321 Runner.__init__(self, args, "debugger") 1322 self.test_directory = path.join(self.test_root, "debugger") 1323 self.add_test() 1324 1325 def add_test(self): 1326 self.tests = [] 1327 self.tests.append(DebuggerTest(os.path.join(self.test_directory, "debugger-in-debug"), "debug-mode")) 1328 self.tests.append(DebuggerTest(os.path.join(self.test_directory, "debugger-in-release"), "release-mode")) 1329 1330 1331class Base64Test(Test): 1332 def __init__(self, test_path, input_type): 1333 Test.__init__(self, test_path, "") 1334 self.input_type = input_type 1335 1336 def run(self, runner): 1337 cmd = runner.cmd_prefix + [runner.es2panda, "--base64Output"] 1338 if self.input_type == "file": 1339 input_file_name = 'input.js' 1340 cmd.extend(['--source-file', input_file_name]) 1341 cmd.extend([os.path.join(self.path, input_file_name)]) 1342 elif self.input_type == "string": 1343 input_file = os.path.join(self.path, "input.txt") 1344 try: 1345 with open(input_file, 'r') as fp: 1346 base64_input = (''.join((fp.readlines()[12:]))).lstrip() # ignore license description lines 1347 cmd.extend(["--base64Input", base64_input]) 1348 except Exception: 1349 self.passed = False 1350 elif self.input_type == "targetApiVersion": 1351 # base64 test for all available target api version. 1352 version = os.path.basename(self.path) 1353 cmd.extend(['--target-api-version', version]) 1354 if version == "12": 1355 cmd.append("--target-api-sub-version=beta3") 1356 input_file = os.path.join(self.path, "input.txt") 1357 try: 1358 with open(input_file, 'r') as fp: 1359 base64_input = (''.join((fp.readlines()[12:]))).lstrip() # ignore license description lines 1360 cmd.extend(["--base64Input", base64_input]) 1361 except Exception: 1362 self.passed = False 1363 else: 1364 self.error = "Unsupported base64 input type" 1365 self.passed = False 1366 return self 1367 1368 version = os.path.basename(self.path) 1369 if not self.input_type == "targetApiVersion": 1370 cmd.append("--target-api-sub-version=beta3") 1371 1372 self.log_cmd(cmd) 1373 1374 process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 1375 stdout, stderr = process.communicate(timeout=runner.args.es2panda_timeout) 1376 if stderr: 1377 self.passed = False 1378 self.error = stderr.decode("utf-8", errors="ignore") 1379 return self 1380 1381 self.output = stdout.decode("utf-8", errors="ignore") 1382 1383 expected_path = os.path.join(self.path, 'expected.txt') 1384 try: 1385 with open(expected_path, 'r') as fp: 1386 expected = (''.join((fp.readlines()[12:]))).lstrip() 1387 self.passed = expected == self.output 1388 except Exception: 1389 self.passed = False 1390 1391 if not self.passed: 1392 self.error = "expected output:" + os.linesep + expected + os.linesep + "actual output:" + os.linesep +\ 1393 self.output 1394 1395 return self 1396 1397 1398class Base64Runner(Runner): 1399 def __init__(self, args): 1400 Runner.__init__(self, args, "Base64") 1401 self.test_directory = path.join(self.test_root, "base64") 1402 self.add_test() 1403 1404 def add_test(self): 1405 self.tests = [] 1406 self.tests.append(Base64Test(os.path.join(self.test_directory, "inputFile"), "file")) 1407 self.tests.append(Base64Test(os.path.join(self.test_directory, "inputString"), "string")) 1408 # current target api version is 12, once a new version is addded, a new testcase should be added here. 1409 current_version = 12 1410 available_target_api_versions = [9, 10, 11, current_version] 1411 for version in available_target_api_versions: 1412 self.tests.append(Base64Test(os.path.join(self.test_directory, "availableTargetApiVersion", str(version)), 1413 "targetApiVersion")) 1414 1415 def test_path(self, src): 1416 return os.path.basename(src) 1417 1418 1419class BytecodeRunner(Runner): 1420 def __init__(self, args): 1421 Runner.__init__(self, args, "Bytecode") 1422 1423 def add_directory(self, directory, extension, flags, func=Test): 1424 glob_expression = path.join( 1425 self.test_root, directory, "**/*.%s" % (extension)) 1426 files = glob(glob_expression, recursive=True) 1427 files = fnmatch.filter(files, self.test_root + '**' + self.args.filter) 1428 self.tests += list(map(lambda f: func(f, flags), files)) 1429 1430 def test_path(self, src): 1431 return src 1432 1433 1434class CompilerTestInfo(object): 1435 def __init__(self, directory, extension, flags): 1436 self.directory = directory 1437 self.extension = extension 1438 self.flags = flags 1439 1440 def update_dir(self, prefiex_dir): 1441 self.directory = os.path.sep.join([prefiex_dir, self.directory]) 1442 1443 1444# Copy compiler directory to test/.local directory, and do inplace obfuscation. 1445def prepare_for_obfuscation(compiler_test_infos, test_root): 1446 tmp_dir_name = ".local" 1447 tmp_path = os.path.join(test_root, tmp_dir_name) 1448 if not os.path.exists(tmp_path): 1449 os.mkdir(tmp_path) 1450 1451 test_root_dirs = set() 1452 for info in compiler_test_infos: 1453 root_dir = info.directory.split("/")[0] 1454 test_root_dirs.add(root_dir) 1455 1456 for test_dir in test_root_dirs: 1457 src_dir = os.path.join(test_root, test_dir) 1458 target_dir = os.path.join(tmp_path, test_dir) 1459 if os.path.exists(target_dir): 1460 shutil.rmtree(target_dir) 1461 shutil.copytree(src_dir, target_dir) 1462 1463 for info in compiler_test_infos: 1464 info.update_dir(tmp_dir_name) 1465 1466 1467def add_directory_for_regression(runners, args): 1468 runner = RegressionRunner(args) 1469 runner.add_directory("parser/concurrent", "js", ["--module", "--dump-ast"]) 1470 runner.add_directory("parser/js", "js", ["--parse-only", "--dump-ast"]) 1471 runner.add_directory("parser/script", "ts", ["--parse-only", "--dump-ast"]) 1472 runner.add_directory("parser/ts", "ts", 1473 ["--parse-only", "--module", "--dump-ast"]) 1474 runner.add_directory("parser/ts/type_checker", "ts", 1475 ["--parse-only", "--enable-type-check", "--module", "--dump-ast"]) 1476 runner.add_directory("parser/ts/cases/declaration", "d.ts", 1477 ["--parse-only", "--module", "--dump-ast"], TSDeclarationTest) 1478 runner.add_directory("parser/commonjs", "js", ["--commonjs", "--parse-only", "--dump-ast"]) 1479 runner.add_directory("parser/binder", "js", ["--dump-assembly", "--dump-literal-buffer", "--module", "--target-api-sub-version=beta3"]) 1480 runner.add_directory("parser/binder", "ts", ["--dump-assembly", "--dump-literal-buffer", "--module", "--target-api-sub-version=beta3"]) 1481 runner.add_directory("parser/binder/noModule", "ts", ["--dump-assembly", "--dump-literal-buffer", "--target-api-sub-version=beta3"]) 1482 runner.add_directory("parser/binder/api12beta2", "js", ["--dump-assembly", "--target-api-version=12", "--target-api-sub-version=beta2"]) 1483 runner.add_directory("parser/js/emptySource", "js", ["--dump-assembly"]) 1484 runner.add_directory("parser/js/language/arguments-object", "js", ["--parse-only"]) 1485 runner.add_directory("parser/js/language/statements/for-statement", "js", ["--parse-only", "--dump-ast"]) 1486 runner.add_directory("parser/js/language/expressions/optional-chain", "js", ["--parse-only", "--dump-ast"]) 1487 runner.add_directory("parser/js/language/import/syntax", "js", 1488 ["--parse-only", "--module", "--target-api-sub-version=beta3"]) 1489 runner.add_directory("parser/js/language/import/syntax/beta2", "js", 1490 ["--parse-only", "--module", "--target-api-version=12", "--target-api-sub-version=beta2"]) 1491 runner.add_directory("parser/js/language/import", "ts", 1492 ["--dump-assembly", "--dump-literal-buffer", "--module", "--target-api-sub-version=beta3"]) 1493 runner.add_directory("parser/sendable_class", "ts", 1494 ["--dump-assembly", "--dump-literal-buffer", "--module", "--target-api-sub-version=beta3"]) 1495 runner.add_directory("parser/sendable_class/api12beta2", "ts", 1496 ["--dump-assembly", "--dump-literal-buffer", "--module", "--target-api-version=12", "--target-api-sub-version=beta2"]) 1497 runner.add_directory("parser/unicode", "js", ["--parse-only"]) 1498 runner.add_directory("parser/ts/stack_overflow", "ts", ["--parse-only", "--dump-ast"]) 1499 runner.add_directory("parser/js/module-record/module-record-field-name-option.js", "js", 1500 ["--module-record-field-name=abc", "--source-file=abc", "--module", "--dump-normalized-asm-program"]) 1501 1502 runners.append(runner) 1503 1504 transformer_runner = TransformerRunner(args) 1505 transformer_runner.add_directory("parser/ts/transformed_cases", "ts", 1506 ["--parse-only", "--module", "--dump-transformed-ast", 1507 "--check-transformed-ast-structure"]) 1508 1509 runners.append(transformer_runner) 1510 1511 bc_version_runner = BcVersionRunner(args) 1512 bc_version_runner.add_cmd() 1513 1514 runners.append(bc_version_runner) 1515 1516 transformer_api_version_10_runner = TransformerInTargetApiVersion10Runner(args) 1517 transformer_api_version_10_runner.add_directory("parser/ts/transformed_cases_api_version_10", "ts", 1518 ["--parse-only", "--module", "--target-api-version=10", 1519 "--dump-transformed-ast"]) 1520 1521 runners.append(transformer_api_version_10_runner) 1522 1523def add_directory_for_asm(runners, args, mode = ""): 1524 runner = AbcToAsmRunner(args, True if mode == "debug" else False) 1525 runner.add_directory("abc2asm/js", "js", []) 1526 runner.add_directory("abc2asm/ts", "ts", []) 1527 runner.add_directory("compiler/js", "js", []) 1528 runner.add_directory("compiler/ts/cases/compiler", "ts", []) 1529 runner.add_directory("compiler/ts/projects", "ts", ["--module"]) 1530 runner.add_directory("compiler/ts/projects", "ts", ["--module", "--merge-abc"]) 1531 runner.add_directory("compiler/dts", "d.ts", ["--module", "--opt-level=0"]) 1532 runner.add_directory("compiler/commonjs", "js", ["--commonjs"]) 1533 runner.add_directory("parser/concurrent", "js", ["--module"]) 1534 runner.add_directory("parser/js", "js", []) 1535 runner.add_directory("parser/script", "ts", []) 1536 runner.add_directory("parser/ts", "ts", ["--module"]) 1537 runner.add_directory("parser/ts/type_checker", "ts", ["--enable-type-check", "--module"]) 1538 runner.add_directory("parser/commonjs", "js", ["--commonjs"]) 1539 runner.add_directory("parser/binder", "js", ["--dump-assembly", "--dump-literal-buffer", "--module"]) 1540 runner.add_directory("parser/binder", "ts", ["--dump-assembly", "--dump-literal-buffer", "--module"]) 1541 runner.add_directory("parser/binder/noModule", "ts", ["--dump-assembly", "--dump-literal-buffer"]) 1542 runner.add_directory("parser/js/emptySource", "js", []) 1543 runner.add_directory("parser/js/language/arguments-object", "js", []) 1544 runner.add_directory("parser/js/language/statements/for-statement", "js", []) 1545 runner.add_directory("parser/js/language/expressions/optional-chain", "js", []) 1546 runner.add_directory("parser/sendable_class", "ts", ["--module"]) 1547 runner.add_directory("parser/unicode", "js", []) 1548 runner.add_directory("parser/ts/stack_overflow", "ts", []) 1549 1550 runners.append(runner) 1551 1552 1553def add_directory_for_compiler(runners, args): 1554 runner = CompilerRunner(args) 1555 compiler_test_infos = [] 1556 compiler_test_infos.append(CompilerTestInfo("compiler/js", "js", ["--module"])) 1557 compiler_test_infos.append(CompilerTestInfo("compiler/ts/cases", "ts", [])) 1558 compiler_test_infos.append(CompilerTestInfo("compiler/ts/projects", "ts", ["--module"])) 1559 compiler_test_infos.append(CompilerTestInfo("compiler/ts/projects", "ts", ["--module", "--merge-abc"])) 1560 compiler_test_infos.append(CompilerTestInfo("compiler/dts", "d.ts", ["--module", "--opt-level=0"])) 1561 compiler_test_infos.append(CompilerTestInfo("compiler/commonjs", "js", ["--commonjs"])) 1562 compiler_test_infos.append(CompilerTestInfo("compiler/interpreter/lexicalEnv", "js", [])) 1563 compiler_test_infos.append(CompilerTestInfo("compiler/sendable", "ts", ["--module", "--target-api-sub-version=beta3"])) 1564 compiler_test_infos.append(CompilerTestInfo("optimizer/js/branch-elimination", "js", 1565 ["--module", "--branch-elimination", "--dump-assembly"])) 1566 compiler_test_infos.append(CompilerTestInfo("optimizer/js/opt-try-catch-func", "js", 1567 ["--module", "--dump-assembly"])) 1568 compiler_test_infos.append(CompilerTestInfo("compiler/debugInfo/", "js", 1569 ["--debug-info", "--dump-debug-info", "--source-file", "debug-info.js"])) 1570 compiler_test_infos.append(CompilerTestInfo("compiler/js/module-record-field-name-option.js", "js", 1571 ["--module", "--module-record-field-name=abc"])) 1572 # Following directories of test cases are for dump-assembly comparison only, and is not executed. 1573 # Check CompilerProjectTest for more details. 1574 compiler_test_infos.append(CompilerTestInfo("optimizer/ts/branch-elimination/projects", "ts", 1575 ["--module", "--branch-elimination", "--merge-abc", "--dump-assembly", 1576 "--file-threads=8"])) 1577 compiler_test_infos.append(CompilerTestInfo("compiler/bytecodehar/projects", "ts", 1578 ["--merge-abc", "--dump-assembly", "--enable-abc-input", 1579 "--dump-deps-info", "--remove-redundant-file", 1580 "--dump-literal-buffer", "--dump-string", "--abc-class-threads=4"])) 1581 compiler_test_infos.append(CompilerTestInfo("compiler/bytecodehar/js/projects", "js", 1582 ["--merge-abc", "--dump-assembly", "--enable-abc-input", 1583 "--dump-deps-info", "--remove-redundant-file", 1584 "--dump-literal-buffer", "--dump-string", "--abc-class-threads=4"])) 1585 compiler_test_infos.append(CompilerTestInfo("compiler/bytecodehar/merge_abc_consistence_check/projects", "js", 1586 ["--merge-abc", "--dump-assembly", "--enable-abc-input", 1587 "--abc-class-threads=4"])) 1588 1589 compiler_test_infos.append(CompilerTestInfo("compiler/ts/shared_module/projects", "ts", 1590 ["--module", "--merge-abc", "--dump-assembly"])) 1591 1592 if args.enable_arkguard: 1593 prepare_for_obfuscation(compiler_test_infos, runner.test_root) 1594 1595 for info in compiler_test_infos: 1596 runner.add_directory(info.directory, info.extension, info.flags) 1597 1598 runners.append(runner) 1599 1600 1601def add_directory_for_bytecode(runners, args): 1602 runner = BytecodeRunner(args) 1603 runner.add_directory("bytecode/commonjs", "js", ["--commonjs", "--dump-assembly"]) 1604 runner.add_directory("bytecode/js", "js", ["--dump-assembly"]) 1605 runner.add_directory("bytecode/ts/cases", "ts", ["--dump-assembly"]) 1606 runner.add_directory("bytecode/ts/ic", "ts", ["--dump-assembly"]) 1607 runner.add_directory("bytecode/ts/api11", "ts", ["--dump-assembly", "--module", "--target-api-version=11"]) 1608 runner.add_directory("bytecode/ts/api12", "ts", ["--dump-assembly", "--module", "--target-api-version=12"]) 1609 runner.add_directory("bytecode/watch-expression", "js", ["--debugger-evaluate-expression", "--dump-assembly"]) 1610 1611 runners.append(runner) 1612 1613 1614def add_directory_for_debug(runners, args): 1615 runner = RegressionRunner(args) 1616 runner.add_directory("debug/parser", "js", ["--parse-only", "--dump-ast"]) 1617 1618 runners.append(runner) 1619 1620 1621def add_cmd_for_aop_transform(runners, args): 1622 runner = AopTransform(args) 1623 1624 aop_file_path = path.join(runner.test_root, "aop") 1625 lib_suffix = '.so' 1626 #cpp src, deal type, result compare str, abc compare str 1627 msg_list = [ 1628 ["correct_modify.cpp", "compile", "aop_transform_start", "new_abc_content"], 1629 ["correct_no_modify.cpp", "compile", "aop_transform_start", ""], 1630 ["exec_error.cpp", "compile", "Transform exec fail", ""], 1631 ["no_func_transform.cpp", "compile", "os::library_loader::ResolveSymbol get func Transform error", ""], 1632 ["error_format.cpp", "copy_lib", "os::library_loader::Load error", ""], 1633 ["".join(["no_exist", lib_suffix]), "dirct_use", "Failed to find file", ""], 1634 ["error_suffix.xxx", "direct_use", "aop transform file suffix support", ""] 1635 ] 1636 for msg in msg_list: 1637 cpp_file = path.join(aop_file_path, msg[0]) 1638 if msg[1] == 'compile': 1639 lib_file = cpp_file.replace('.cpp', lib_suffix) 1640 remove_file = lib_file 1641 runner.add_cmd(["g++", "--share", "-o", lib_file, cpp_file], "", "", "") 1642 elif msg[1] == 'copy_lib': 1643 lib_file = cpp_file.replace('.cpp', lib_suffix) 1644 remove_file = lib_file 1645 if not os.path.exists(lib_file): 1646 with open(cpp_file, "r") as source_file: 1647 fd = os.open(lib_file, os.O_RDWR | os.O_CREAT | os.O_TRUNC) 1648 target_file = os.fdopen(fd, 'w') 1649 target_file.write(source_file.read()) 1650 elif msg[1] == 'direct_use': 1651 lib_file = cpp_file 1652 remove_file = "" 1653 1654 js_file = path.join(aop_file_path, "test_aop.js") 1655 runner.add_cmd([runner.es2panda, "--merge-abc", "--transform-lib", lib_file, js_file], msg[2], msg[3], remove_file) 1656 1657 runners.append(runner) 1658 1659 1660class AopTransform(Runner): 1661 def __init__(self, args): 1662 Runner.__init__(self, args, "AopTransform") 1663 1664 def add_cmd(self, cmd, compare_str, compare_abc_str, remove_file, func=TestAop): 1665 self.tests += [func(cmd, compare_str, compare_abc_str, remove_file)] 1666 1667 def test_path(self, src): 1668 return src 1669 1670 1671def main(): 1672 args = get_args() 1673 1674 runners = [] 1675 1676 if args.regression: 1677 add_directory_for_regression(runners, args) 1678 1679 if args.abc_to_asm: 1680 add_directory_for_asm(runners, args) 1681 add_directory_for_asm(runners, args, "debug") 1682 1683 if args.tsc: 1684 runners.append(TSCRunner(args)) 1685 1686 if args.compiler: 1687 add_directory_for_compiler(runners, args) 1688 1689 if args.hotfix: 1690 runners.append(HotfixRunner(args)) 1691 1692 if args.hotreload: 1693 runners.append(HotreloadRunner(args)) 1694 1695 if args.coldfix: 1696 runners.append(ColdfixRunner(args)) 1697 1698 if args.coldreload: 1699 runners.append(ColdreloadRunner(args)) 1700 1701 if args.debugger: 1702 runners.append(DebuggerRunner(args)) 1703 1704 if args.base64: 1705 runners.append(Base64Runner(args)) 1706 1707 if args.bytecode: 1708 add_directory_for_bytecode(runners, args) 1709 1710 if args.aop_transform: 1711 add_cmd_for_aop_transform(runners, args) 1712 1713 if args.debug: 1714 add_directory_for_debug(runners, args) 1715 1716 failed_tests = 0 1717 1718 for runner in runners: 1719 runner.run() 1720 failed_tests += runner.summarize() 1721 1722 # TODO: exit 1 when we have failed tests after all tests are fixed 1723 exit(0) 1724 1725 1726if __name__ == "__main__": 1727 main() 1728