1#!/usr/bin/env python 2 3# Copyright JS Foundation and other contributors, http://js.foundation 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17from __future__ import print_function 18import argparse 19import os 20import subprocess 21import sys 22 23import util 24 25def get_arguments(): 26 execution_runtime = os.environ.get('RUNTIME') 27 parser = argparse.ArgumentParser() 28 parser.add_argument('-q', '--quiet', action='store_true', 29 help='Only print out failing tests') 30 parser.add_argument('--runtime', metavar='FILE', default=execution_runtime, 31 help='Execution runtime (e.g. qemu)') 32 parser.add_argument('--engine', metavar='FILE', 33 help='JerryScript binary to run tests with') 34 parser.add_argument('--test-list', metavar='FILE', 35 help='File contains test paths to run') 36 parser.add_argument('--skip-list', metavar='LIST', 37 help='Add a comma separated list of patterns of the excluded JS-tests') 38 parser.add_argument('--skip-file', metavar='LIST', 39 help='Add a comma separated list of patterns of the excluded JS-tests') 40 parser.add_argument('--test-dir', metavar='DIR', 41 help='Directory contains tests to run') 42 parser.add_argument('--snapshot', action='store_true', 43 help='Snapshot test') 44 45 script_args = parser.parse_args() 46 if script_args.skip_list: 47 script_args.skip_list = script_args.skip_list.split(',') 48 else: 49 script_args.skip_list = [] 50 if script_args.skip_file: 51 script_args.skip_file = script_args.skip_file.split(',') 52 else: 53 script_args.skip_file = [] 54 55 return script_args 56 57 58def get_tests(test_dir, test_list, skip_list, skip_file): 59 tests = [] 60 if test_dir: 61 tests = [] 62 for root, _, files in os.walk(test_dir): 63 tests.extend([os.path.join(root, test_file) for test_file in files if test_file.endswith('.js')]) 64 65 if test_list: 66 dirname = os.path.dirname(test_list) 67 with open(test_list, "r") as test_list_fd: 68 for test in test_list_fd: 69 tests.append(os.path.normpath(os.path.join(dirname, test.rstrip()))) 70 71 tests.sort() 72 73 def filter_tests(test): 74 for skipped in skip_list: 75 if skipped in test: 76 return False 77 return True 78 def filter_file_tests(test): 79 for skipped in skip_file: 80 if skipped in test: 81 return False 82 return True 83 84 return [test for test in tests if filter_tests(test) and filter_file_tests(test)] 85 86 87 88def get_platform_cmd_prefix(): 89 if sys.platform == 'win32': 90 return ['cmd', '/S', '/C'] 91 return [] 92 93 94def execute_test_command(test_cmd): 95 kwargs = {} 96 if sys.version_info.major >= 3: 97 kwargs['encoding'] = 'unicode_escape' 98 kwargs['text'] = True 99 process = subprocess.Popen(test_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kwargs) 100 stdout = process.communicate()[0] 101 return (process.returncode, stdout) 102 103 104def main(args): 105 tests = get_tests(args.test_dir, args.test_list, args.skip_list, args.skip_file) 106 total = len(tests) 107 if total == 0: 108 print("No test to execute.") 109 return 1 110 111 if sys.platform == 'win32': 112 original_timezone = util.get_timezone() 113 util.set_sighdl_to_reset_timezone(original_timezone) 114 util.set_timezone('UTC') 115 116 if args.snapshot: 117 passed = run_snapshot_tests(args, tests) 118 else: 119 passed = run_normal_tests(args, tests) 120 121 if sys.platform == 'win32': 122 util.set_timezone(original_timezone) 123 124 failed = total - passed 125 126 summary_list = [os.path.relpath(args.engine)] 127 if args.snapshot: 128 summary_list.append('--snapshot') 129 if args.test_dir: 130 summary_list.append(os.path.relpath(args.test_dir)) 131 if args.test_list: 132 summary_list.append(os.path.relpath(args.test_list)) 133 util.print_test_summary(' '.join(summary_list), total, passed, failed) 134 135 return bool(failed) 136 137 138def run_normal_tests(args, tests): 139 test_cmd = get_platform_cmd_prefix() 140 if args.runtime: 141 test_cmd.append(args.runtime) 142 test_cmd.extend([args.engine, '--call-on-exit', '__checkAsync']) 143 144 total = len(tests) 145 tested = 0 146 passed = 0 147 for test in tests: 148 tested += 1 149 test_path = os.path.relpath(test) 150 is_expected_to_fail = os.path.join(os.path.sep, 'fail', '') in test 151 (returncode, stdout) = execute_test_command(test_cmd + [test]) 152 153 if (returncode == 0 and not is_expected_to_fail) or (returncode == 1 and is_expected_to_fail): 154 passed += 1 155 if not args.quiet: 156 passed_string = 'PASS' + (' (XFAIL)' if is_expected_to_fail else '') 157 util.print_test_result(tested, total, True, passed_string, test_path) 158 else: 159 passed_string = 'FAIL%s (%d)' % (' (XPASS)' if returncode == 0 and is_expected_to_fail else '', returncode) 160 util.print_test_result(tested, total, False, passed_string, test_path) 161 print("================================================") 162 print(stdout) 163 print("================================================") 164 165 return passed 166 167 168def run_snapshot_tests(args, tests): 169 execute_snapshot_cmd = get_platform_cmd_prefix() 170 generate_snapshot_cmd = get_platform_cmd_prefix() 171 if args.runtime: 172 execute_snapshot_cmd.append(args.runtime) 173 generate_snapshot_cmd.append(args.runtime) 174 175 execute_snapshot_cmd.extend([args.engine, '--exec-snapshot', 'js.snapshot']) 176 execute_snapshot_cmd.extend(['--call-on-exit', '__checkAsync']) 177 178 # engine: jerry[.exe] -> snapshot generator: jerry-snapshot[.exe] 179 engine = os.path.splitext(args.engine) 180 generate_snapshot_cmd.append(engine[0] + '-snapshot' + engine[1]) 181 generate_snapshot_cmd.append('generate') 182 183 total = len(tests) 184 tested = 0 185 passed = 0 186 for test in tests: 187 tested += 1 188 test_path = os.path.relpath(test) 189 is_expected_to_fail = os.path.join(os.path.sep, 'fail', '') in test 190 (returncode, stdout) = execute_test_command(generate_snapshot_cmd + [test]) 191 192 if (returncode == 0) or (returncode == 1 and is_expected_to_fail): 193 if not args.quiet: 194 passed_string = 'PASS' + (' (XFAIL)' if returncode else '') 195 util.print_test_result(tested, total, True, passed_string, test_path, True) 196 else: 197 util.print_test_result(tested, total, False, 'FAIL (%d)' % (returncode), test_path, True) 198 print("================================================") 199 print(stdout) 200 print("================================================") 201 202 if returncode: 203 if is_expected_to_fail: 204 passed += 1 205 continue 206 207 (returncode, stdout) = execute_test_command(execute_snapshot_cmd) 208 os.remove('js.snapshot') 209 210 if (returncode == 0 and not is_expected_to_fail) or (returncode == 1 and is_expected_to_fail): 211 passed += 1 212 if not args.quiet: 213 passed_string = 'PASS' + (' (XFAIL)' if is_expected_to_fail else '') 214 util.print_test_result(tested, total, True, passed_string, test_path, False) 215 else: 216 passed_string = 'FAIL%s (%d)' % (' (XPASS)' if returncode == 0 and is_expected_to_fail else '', returncode) 217 util.print_test_result(tested, total, False, passed_string, test_path, False) 218 print("================================================") 219 print(stdout) 220 print("================================================") 221 222 return passed 223 224 225if __name__ == "__main__": 226 sys.exit(main(get_arguments())) 227