1#!/usr/bin/env python 2 3# Copyright JS Foundation and other contributors, http://js.foundation 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17from __future__ import print_function 18import argparse 19import os 20import subprocess 21import sys 22 23import util 24 25def get_arguments(): 26 execution_runtime = os.environ.get('RUNTIME') 27 parser = argparse.ArgumentParser() 28 parser.add_argument('-q', '--quiet', action='store_true', 29 help='Only print out failing tests') 30 parser.add_argument('--runtime', metavar='FILE', default=execution_runtime, 31 help='Execution runtime (e.g. qemu)') 32 parser.add_argument('--engine', metavar='FILE', 33 help='JerryScript binary to run tests with') 34 parser.add_argument('--test-list', metavar='FILE', 35 help='File contains test paths to run') 36 parser.add_argument('--skip-list', metavar='LIST', 37 help='Add a comma separated list of patterns of the excluded JS-tests') 38 parser.add_argument('--test-dir', metavar='DIR', 39 help='Directory contains tests to run') 40 parser.add_argument('--snapshot', action='store_true', 41 help='Snapshot test') 42 43 script_args = parser.parse_args() 44 if script_args.skip_list: 45 script_args.skip_list = script_args.skip_list.split(',') 46 else: 47 script_args.skip_list = [] 48 49 return script_args 50 51 52def get_tests(test_dir, test_list, skip_list): 53 tests = [] 54 if test_dir: 55 tests = [] 56 for root, _, files in os.walk(test_dir): 57 tests.extend([os.path.join(root, test_file) for test_file in files if test_file.endswith('.js')]) 58 59 if test_list: 60 dirname = os.path.dirname(test_list) 61 with open(test_list, "r") as test_list_fd: 62 for test in test_list_fd: 63 tests.append(os.path.normpath(os.path.join(dirname, test.rstrip()))) 64 65 tests.sort() 66 67 def filter_tests(test): 68 for skipped in skip_list: 69 if skipped in test: 70 return False 71 return True 72 73 return [test for test in tests if filter_tests(test)] 74 75 76def get_platform_cmd_prefix(): 77 if sys.platform == 'win32': 78 return ['cmd', '/S', '/C'] 79 return [] 80 81 82def execute_test_command(test_cmd): 83 kwargs = {} 84 if sys.version_info.major >= 3: 85 kwargs['encoding'] = 'unicode_escape' 86 kwargs['text'] = True 87 process = subprocess.Popen(test_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kwargs) 88 stdout = process.communicate()[0] 89 return (process.returncode, stdout) 90 91 92def main(args): 93 tests = get_tests(args.test_dir, args.test_list, args.skip_list) 94 total = len(tests) 95 if total == 0: 96 print("No test to execute.") 97 return 1 98 99 if sys.platform == 'win32': 100 original_timezone = util.get_timezone() 101 util.set_sighdl_to_reset_timezone(original_timezone) 102 util.set_timezone('UTC') 103 104 if args.snapshot: 105 passed = run_snapshot_tests(args, tests) 106 else: 107 passed = run_normal_tests(args, tests) 108 109 if sys.platform == 'win32': 110 util.set_timezone(original_timezone) 111 112 failed = total - passed 113 114 summary_list = [os.path.relpath(args.engine)] 115 if args.snapshot: 116 summary_list.append('--snapshot') 117 if args.test_dir: 118 summary_list.append(os.path.relpath(args.test_dir)) 119 if args.test_list: 120 summary_list.append(os.path.relpath(args.test_list)) 121 util.print_test_summary(' '.join(summary_list), total, passed, failed) 122 123 return bool(failed) 124 125 126def run_normal_tests(args, tests): 127 test_cmd = get_platform_cmd_prefix() 128 if args.runtime: 129 test_cmd.append(args.runtime) 130 test_cmd.extend([args.engine, '--call-on-exit', '__checkAsync']) 131 132 total = len(tests) 133 tested = 0 134 passed = 0 135 for test in tests: 136 tested += 1 137 test_path = os.path.relpath(test) 138 is_expected_to_fail = os.path.join(os.path.sep, 'fail', '') in test 139 (returncode, stdout) = execute_test_command(test_cmd + [test]) 140 141 if (returncode == 0 and not is_expected_to_fail) or (returncode == 1 and is_expected_to_fail): 142 passed += 1 143 if not args.quiet: 144 passed_string = 'PASS' + (' (XFAIL)' if is_expected_to_fail else '') 145 util.print_test_result(tested, total, True, passed_string, test_path) 146 else: 147 passed_string = 'FAIL%s (%d)' % (' (XPASS)' if returncode == 0 and is_expected_to_fail else '', returncode) 148 util.print_test_result(tested, total, False, passed_string, test_path) 149 print("================================================") 150 print(stdout) 151 print("================================================") 152 153 return passed 154 155 156def run_snapshot_tests(args, tests): 157 execute_snapshot_cmd = get_platform_cmd_prefix() 158 generate_snapshot_cmd = get_platform_cmd_prefix() 159 if args.runtime: 160 execute_snapshot_cmd.append(args.runtime) 161 generate_snapshot_cmd.append(args.runtime) 162 163 execute_snapshot_cmd.extend([args.engine, '--exec-snapshot', 'js.snapshot']) 164 execute_snapshot_cmd.extend(['--call-on-exit', '__checkAsync']) 165 166 # engine: jerry[.exe] -> snapshot generator: jerry-snapshot[.exe] 167 engine = os.path.splitext(args.engine) 168 generate_snapshot_cmd.append(engine[0] + '-snapshot' + engine[1]) 169 generate_snapshot_cmd.append('generate') 170 171 total = len(tests) 172 tested = 0 173 passed = 0 174 for test in tests: 175 tested += 1 176 test_path = os.path.relpath(test) 177 is_expected_to_fail = os.path.join(os.path.sep, 'fail', '') in test 178 (returncode, stdout) = execute_test_command(generate_snapshot_cmd + [test]) 179 180 if (returncode == 0) or (returncode == 1 and is_expected_to_fail): 181 if not args.quiet: 182 passed_string = 'PASS' + (' (XFAIL)' if returncode else '') 183 util.print_test_result(tested, total, True, passed_string, test_path, True) 184 else: 185 util.print_test_result(tested, total, False, 'FAIL (%d)' % (returncode), test_path, True) 186 print("================================================") 187 print(stdout) 188 print("================================================") 189 190 if returncode: 191 if is_expected_to_fail: 192 passed += 1 193 continue 194 195 (returncode, stdout) = execute_test_command(execute_snapshot_cmd) 196 os.remove('js.snapshot') 197 198 if (returncode == 0 and not is_expected_to_fail) or (returncode == 1 and is_expected_to_fail): 199 passed += 1 200 if not args.quiet: 201 passed_string = 'PASS' + (' (XFAIL)' if is_expected_to_fail else '') 202 util.print_test_result(tested, total, True, passed_string, test_path, False) 203 else: 204 passed_string = 'FAIL%s (%d)' % (' (XPASS)' if returncode == 0 and is_expected_to_fail else '', returncode) 205 util.print_test_result(tested, total, False, passed_string, test_path, False) 206 print("================================================") 207 print(stdout) 208 print("================================================") 209 210 return passed 211 212 213if __name__ == "__main__": 214 sys.exit(main(get_arguments())) 215