• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3
4"""
5Copyright (c) 2023 Huawei Device Co., Ltd.
6Licensed under the Apache License, Version 2.0 (the "License");
7you may not use this file except in compliance with the License.
8You may obtain a copy of the License at
9
10    http://www.apache.org/licenses/LICENSE-2.0
11
12Unless required by applicable law or agreed to in writing, software
13distributed under the License is distributed on an "AS IS" BASIS,
14WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15See the License for the specific language governing permissions and
16limitations under the License.
17
18Description: run regress test case
19"""
20import argparse
21import datetime
22import json
23import logging
24import os
25import platform
26import re
27import shutil
28import signal
29import stat
30import subprocess
31import sys
32
33from regress_test_config import RegressTestConfig
34
35
36def init_log_file(args):
37    logging.basicConfig(filename=args.out_log, format=RegressTestConfig.DEFAULT_LOG_FORMAT, level=logging.INFO)
38
39
40def parse_args():
41    parser = argparse.ArgumentParser()
42    parser.add_argument('--test-dir', metavar='DIR',
43                        help='Directory to test ')
44    parser.add_argument('--test-file', metavar='FILE',
45                        help='File to test')
46    parser.add_argument('--timeout', default=RegressTestConfig.DEFAULT_TIMEOUT, type=int,
47                        help='Set a custom test timeout in milliseconds !!!\n')
48    parser.add_argument('--ark-tool',
49                        help="ark's binary tool")
50    parser.add_argument('--ark-frontend-binary',
51                        help="ark frontend conversion binary tool")
52    parser.add_argument('--LD_LIBRARY_PATH',
53                        dest='ld_library_path', default=None, help='LD_LIBRARY_PATH')
54    parser.add_argument('--icu-path',
55                        dest='icu_path', help='icu-data-path')
56    parser.add_argument('--out-dir',
57                        default=None, help='target out dir')
58    return parser.parse_args()
59
60
61def check_args(args):
62    current_dir = os.getcwd()
63    if args.ark_frontend_binary is None:
64        print('ark_frontend_binary is required, please add this parameter')
65        return False
66    else:
67        current_frontend_binary = os.path.join(current_dir, args.ark_frontend_binary)
68        test_tool_frontend_binary = os.path.join(RegressTestConfig.TEST_TOOL_FILE_DIR, args.ark_frontend_binary)
69        if not os.path.exists(current_frontend_binary) and not os.path.exists(test_tool_frontend_binary):
70            print('entered ark_frontend_binary does not exist. please confirm')
71            return False
72        else:
73            args.ark_frontend_binary = current_frontend_binary if os.path.exists(
74                current_frontend_binary) else test_tool_frontend_binary
75
76    if args.ark_tool is None:
77        print('ark_tool is required, please add this parameter')
78        return False
79    else:
80        current_ark_tool = os.path.join(current_dir, args.ark_tool)
81        test_tool_ark_tool = os.path.join(RegressTestConfig.TEST_TOOL_FILE_DIR, args.ark_tool)
82        if not os.path.exists(current_ark_tool) and not os.path.exists(test_tool_ark_tool):
83            print('entered ark_tool does not exist. please confirm')
84            return False
85        else:
86            args.ark_tool = current_ark_tool if os.path.exists(current_ark_tool) else test_tool_ark_tool
87    if args.ld_library_path is None:
88        args.ld_library_path = RegressTestConfig.DEFAULT_LIBS_DIR
89    if args.icu_path is None:
90        args.icu_path = RegressTestConfig.ICU_PATH
91    if args.out_dir is None:
92        args.out_dir = RegressTestConfig.PROJECT_BASE_OUT_DIR
93    else:
94        args.out_dir = os.path.join(RegressTestConfig.CURRENT_PATH, args.out_dir)
95    if not args.out_dir.endswith("/"):
96        args.out_dir = f"{args.out_dir}/"
97    args.regress_out_dir = os.path.join(args.out_dir, "regresstest")
98    args.out_result = os.path.join(args.regress_out_dir, 'result.txt')
99    args.out_log = os.path.join(args.regress_out_dir, 'test.log')
100    args.test_case_out_dir = os.path.join(args.regress_out_dir, RegressTestConfig.REGRESS_GIT_REPO)
101    return True
102
103
104def get_skip_test_cases():
105    skip_tests_list = []
106    with os.fdopen(os.open(RegressTestConfig.SKIP_LIST_FILE, os.O_RDONLY, stat.S_IRUSR), "r") as file_object:
107        json_data = json.load(file_object)
108        for key in json_data:
109            skip_tests_list.extend(key["files"])
110    return skip_tests_list
111
112
113def remove_dir(path):
114    if os.path.exists(path):
115        shutil.rmtree(path)
116
117
118def git_clone(git_url, code_dir):
119    cmds = ['git', 'clone', git_url, code_dir]
120    result = True
121    with subprocess.Popen(cmds) as proc:
122        ret = proc.wait()
123        if ret:
124            print(f"\n error: Cloning '{git_url}' failed.")
125            result = False
126    return result
127
128
129def output(msg):
130    print(str(msg))
131    logging.info(str(msg))
132
133
134def out_put_std(ret_code, cmds, msg):
135    error_messages = {
136        0: msg,
137        -6: f'{cmds}:{msg}\nAborted (core dumped)',
138        -4: f'{cmds}:{msg}\nAborted (core dumped)',
139        -11: f'{cmds}:{msg}\nSegmentation fault (core dumped)'
140    }
141    error_message = error_messages.get(ret_code, f'{cmds}: Unknown Error: {str(ret_code)}')
142    if error_message != '':
143        output(str(error_message))
144
145
146def exec_command(cmd_args, timeout=RegressTestConfig.DEFAULT_TIMEOUT):
147    code_format = 'utf-8'
148    if platform.system() == "Windows":
149        code_format = 'gbk'
150    code = 0
151    msg = ""
152    try:
153        with subprocess.Popen(cmd_args, stderr=subprocess.PIPE, stdout=subprocess.PIPE, close_fds=True,
154                              start_new_session=True) as process:
155            cmd_string = " ".join(cmd_args)
156            output_res, errs = process.communicate(timeout=timeout)
157            ret_code = process.poll()
158            if errs.decode(code_format, 'ignore'):
159                msg += errs.decode(code_format, 'ignore')
160            if ret_code and ret_code != 1:
161                code = ret_code
162                msg += f"Command {cmd_string}:\n error: {errs.decode(code_format, 'ignore')}"
163            else:
164                msg += output_res.decode(code_format, 'ignore')
165    except subprocess.TimeoutExpired:
166        process.kill()
167        process.terminate()
168        os.kill(process.pid, signal.SIGTERM)
169        code = 1
170        msg += f"Timeout: '{cmd_string}' timed out after {timeout} seconds"
171    except Exception as error:
172        code = 1
173        msg += f"{cmd_string}: unknown error: {error}"
174    out_put_std(code, cmd_args, msg)
175
176
177class RegressTestPrepare:
178    def __init__(self, args, skip_test_cases):
179        self.args = args
180        self.out_dir = args.regress_out_dir
181        self.skil_test = skip_test_cases
182
183    def run(self):
184        self.get_test_case()
185        self.prepare_clean_data()
186        test_list = self.get_regress_test_files()
187        self.gen_test_tool()
188        self.gen_abc_files(test_list)
189
190    def get_test_case(self):
191        if not os.path.isdir(os.path.join(RegressTestConfig.REGRESS_TEST_CASE_DIR, '.git')):
192            git_clone(RegressTestConfig.REGRESS_GIT_URL, RegressTestConfig.REGRESS_TEST_CASE_DIR)
193            return self.git_checkout(RegressTestConfig.REGRESS_GIT_HASH, RegressTestConfig.REGRESS_TEST_CASE_DIR)
194        return True
195
196    @staticmethod
197    def change_extension(path):
198        base_path, ext = os.path.splitext(path)
199        if ext:
200            new_path = base_path + ".abc"
201        else:
202            new_path = path + ".abc"
203        return new_path
204
205    def gen_test_tool(self):
206        self.gen_abc_files([RegressTestConfig.REGRESS_TEST_TOOL_DIR])
207
208    def gen_abc_files(self, test_list):
209        for file_path in test_list:
210            input_file_path = file_path
211            start_index = input_file_path.find(RegressTestConfig.REGRESS_GIT_REPO)
212            if start_index != -1:
213                test_case_path = input_file_path[start_index + len(RegressTestConfig.REGRESS_GIT_REPO) + 1:]
214                file_extensions = ['.out', '.txt', '.abc']
215                pattern = r'({})$'.format('|'.join(re.escape(ext) for ext in file_extensions))
216                if re.search(pattern, test_case_path) or test_case_path in self.skil_test:
217                    continue
218                out_flle = input_file_path.replace('.js', '.out')
219                expect_file_exits = os.path.exists(out_flle)
220                src_dir = RegressTestConfig.REGRESS_TEST_CASE_DIR
221                out_dir = self.args.test_case_out_dir
222                self.mk_dst_dir(input_file_path, src_dir, out_dir)
223                output_test_case_path = self.change_extension(test_case_path)
224                output_file = os.path.join(self.args.test_case_out_dir, output_test_case_path)
225                command = [self.args.ark_frontend_binary]
226                command.extend([input_file_path, f'--output={output_file}'])
227                exec_command(command)
228                if expect_file_exits:
229                    out_file_path = os.path.join(self.args.test_case_out_dir, test_case_path.replace('.js', '.out'))
230                    shutil.copy(out_flle, out_file_path)
231
232    def mk_dst_dir(self, file, src_dir, dist_dir):
233        idx = file.rfind(src_dir)
234        fpath, _ = os.path.split(file[idx:])
235        fpath = fpath.replace(src_dir, dist_dir)
236        self.mk_dir(fpath)
237
238    @staticmethod
239    def mk_dir(path):
240        if not os.path.exists(path):
241            os.makedirs(path)
242
243    def get_regress_test_files(self):
244        result = []
245        if self.args.test_file is not None and len(self.args.test_file) > 0:
246            test_file_list = os.path.join(RegressTestConfig.REGRESS_TEST_CASE_DIR, self.args.test_file)
247            result.append(test_file_list)
248            return result
249        elif self.args.test_dir is not None and len(self.args.test_dir) > 0:
250            test_file_list = os.path.join(RegressTestConfig.REGRESS_TEST_CASE_DIR, self.args.test_dir)
251        else:
252            test_file_list = RegressTestConfig.REGRESS_TEST_CASE_DIR
253        for dir_path, path, filenames in os.walk(test_file_list):
254            if dir_path.find(".git") != -1:
255                continue
256            for filename in filenames:
257                result.append(os.path.join(dir_path, filename))
258        return result
259
260    def prepare_clean_data(self):
261        self.git_clean(RegressTestConfig.REGRESS_TEST_CASE_DIR)
262        self.git_pull(RegressTestConfig.REGRESS_TEST_CASE_DIR)
263        self.git_checkout(RegressTestConfig.REGRESS_GIT_HASH, RegressTestConfig.REGRESS_TEST_CASE_DIR)
264
265    @staticmethod
266    def git_checkout(checkout_options, check_out_dir=os.getcwd()):
267        cmds = ['git', 'checkout', checkout_options]
268        result = True
269        with subprocess.Popen(cmds, cwd=check_out_dir) as proc:
270            ret = proc.wait()
271            if ret:
272                print(f"\n error: git checkout '{checkout_options}' failed.")
273                result = False
274        return result
275
276    @staticmethod
277    def git_pull(check_out_dir=os.getcwd()):
278        cmds = ['git', 'pull', '--rebase']
279        with subprocess.Popen(cmds, cwd=check_out_dir) as proc:
280            proc.wait()
281
282    @staticmethod
283    def git_clean(clean_dir=os.getcwd()):
284        cmds = ['git', 'checkout', '--', '.']
285        with subprocess.Popen(cmds, cwd=clean_dir) as proc:
286            proc.wait()
287
288
289def run_regress_test_prepare(args):
290    remove_dir(args.regress_out_dir)
291    remove_dir(RegressTestConfig.REGRESS_TEST_CASE_DIR)
292    os.makedirs(args.regress_out_dir)
293    init_log_file(args)
294    skip_tests_case = get_skip_test_cases()
295    test_prepare = RegressTestPrepare(args, skip_tests_case)
296    test_prepare.run()
297
298
299def read_expect_file(expect_file, test_case_file):
300    with os.fdopen(os.open(expect_file, os.O_RDWR, stat.S_IRUSR), "r+") as file_object:
301        lines = file_object.readlines()
302        lines = [line for line in lines if not line.strip().startswith('#')]
303        expect_output = ''.join(lines)
304        if test_case_file.startswith("/"):
305            test_case_file = test_case_file.lstrip("/")
306        expect_file = test_case_file.replace('regresstest/', '')
307        test_file_path = os.path.join(RegressTestConfig.REGRESS_BASE_TEST_DIR, expect_file)
308        expect_output_str = expect_output.replace('*%(basename)s', test_file_path)
309    return expect_output_str
310
311
312def open_write_file(file_path, append):
313    if append:
314        args = os.O_RDWR | os.O_CREAT | os.O_APPEND
315    else:
316        args = os.O_RDWR | os.O_CREAT
317    file_descriptor = os.open(file_path, args, stat.S_IRUSR | stat.S_IWUSR)
318    file_object = os.fdopen(file_descriptor, "w+")
319    return file_object
320
321
322def open_result_excel(file_path):
323    file_descriptor = os.open(file_path, os.O_RDWR | os.O_CREAT | os.O_APPEND, stat.S_IRUSR | stat.S_IWUSR)
324    file_object = os.fdopen(file_descriptor, "w+")
325    return file_object
326
327
328def run_test_case_with_expect(command, test_case_file, expect_file, result_file, timeout):
329    ret_code = 0
330    expect_output_str = read_expect_file(expect_file, test_case_file)
331    with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as process:
332        try:
333            out, err = process.communicate(timeout=timeout)
334            out_str = out.decode('UTF-8').strip()
335            err_str = err.decode('UTF-8').strip()
336            ret_code = process.poll()
337            # ret_code equals 0 means execute ok, ret_code equals 255 means having uncaughtable error
338            if (ret_code == 0 and (out_str == expect_output_str.strip() or err_str == expect_output_str.strip())) or \
339                ret_code == 255:
340                write_result_file(f'PASS {test_case_file} \n', result_file)
341                out_put_std(ret_code, command, f'PASS: {test_case_file}')
342                return True
343            else:
344                msg = f'FAIL: {test_case_file} \nexpect: [{expect_output_str}]\nbut got: [{err_str}]'
345                out_put_std(ret_code, command, msg)
346                write_result_file(f'FAIL {test_case_file} \n', result_file)
347                return False
348        except subprocess.TimeoutExpired as exception:
349            process.kill()
350            process.terminate()
351            os.kill(process.pid, signal.SIGTERM)
352            out_put_std(ret_code, command, f'FAIL: {test_case_file},err:{exception}')
353            write_result_file(f'FAIL {test_case_file} \n', result_file)
354            return False
355
356
357def run_test_case_with_assert(command, test_case_file, result_file, timeout):
358    ret_code = 0
359    with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as process:
360        try:
361            out, err = process.communicate(timeout=timeout)
362            ret_code = process.poll()
363            err_str = err.decode('UTF-8')
364            # ret_code equals 0 means execute ok, ret_code equals 255 means having uncaughtable error
365            if (ret_code != 0 and ret_code != 255) or (err_str != '' and "[ecmascript] Stack overflow" not in err_str):
366                out_put_std(ret_code, command, f'FAIL: {test_case_file} \nerr: {str(err_str)}')
367                write_result_file(f'FAIL {test_case_file} \n', result_file)
368                return False
369            else:
370                out_put_std(ret_code, command, f'PASS: {test_case_file}')
371                write_result_file(f'PASS {test_case_file} \n', result_file)
372                return True
373        except subprocess.TimeoutExpired as exception:
374            process.kill()
375            process.terminate()
376            os.kill(process.pid, signal.SIGTERM)
377            out_put_std(ret_code, command, f'FAIL: {test_case_file},err:{exception}')
378            write_result_file(f'FAIL {test_case_file} \n', result_file)
379            return False
380
381
382def run_test_case_file(command, test_case_file, expect_file, result_file, timeout):
383    expect_file_exits = os.path.exists(expect_file)
384    if expect_file_exits:
385        return run_test_case_with_expect(command, test_case_file, expect_file, result_file, timeout)
386    else:
387        return run_test_case_with_assert(command, test_case_file, result_file, timeout)
388
389
390def run_test_case_dir(args, test_abc_files, force_gc_files, timeout=RegressTestConfig.DEFAULT_TIMEOUT):
391    pass_count = 0
392    fail_count = 0
393    result_file = open_write_file(args.out_result, False)
394    for index, file in enumerate(test_abc_files):
395        test_file = file
396        if test_file.endswith(RegressTestConfig.TEST_TOOL_FILE_NAME):
397            continue
398        test_case_file = test_file.replace('abc', 'js').replace(f'{args.out_dir}', '')
399        assert_file = os.path.join(args.test_case_out_dir, RegressTestConfig.TEST_TOOL_FILE_NAME)
400        test_case = f'{assert_file}:{file}'
401        ld_library_path = args.ld_library_path
402        os.environ["LD_LIBRARY_PATH"] = ld_library_path
403        unforced_gc = False
404        force_gc_file = test_case_file.replace('regresstest/ark-regress/', '')
405        if force_gc_file in force_gc_files:
406            unforced_gc = True
407        asm_arg1 = "--enable-force-gc=true"
408        if unforced_gc:
409            asm_arg1 = "--enable-force-gc=false"
410        icu_path = f"--icu-data-path={args.icu_path}"
411        command = [args.ark_tool, asm_arg1, icu_path, test_case]
412        expect_file = test_file.replace('.abc', '.out')
413        test_res = run_test_case_file(command, test_case_file, expect_file, result_file, timeout)
414        if test_res:
415            pass_count += 1
416        else:
417            fail_count += 1
418    result_file.close()
419    return pass_count, fail_count
420
421
422def run_regress_test_case(args):
423    test_cast_list = get_regress_test_files(args.test_case_out_dir, 'abc')
424    force_gc_files = get_regress_force_gc_files()
425    return run_test_case_dir(args, test_cast_list, force_gc_files)
426
427
428def get_regress_force_gc_files():
429    skip_force_gc_tests_list = []
430    with os.fdopen(os.open(RegressTestConfig.FORCE_GC_FILE_LIST, os.O_RDONLY, stat.S_IRUSR), "r") as file_object:
431        json_data = json.load(file_object)
432        for key in json_data:
433            skip_force_gc_tests_list.extend(key["files"])
434    return skip_force_gc_tests_list
435
436
437def get_regress_test_files(start_dir, suffix):
438    result = []
439    for dir_path, dir_names, filenames in os.walk(start_dir):
440        for filename in filenames:
441            if filename.endswith(suffix):
442                result.append(os.path.join(dir_path, filename))
443    return result
444
445
446def write_result_file(msg, result_file):
447    result_file.write(f'{str(msg)}\n')
448
449
450def main(args):
451    if not check_args(args):
452        return
453    print("\nStart conducting regress test cases........\n")
454    start_time = datetime.datetime.now()
455    run_regress_test_prepare(args)
456    pass_count, fail_count = run_regress_test_case(args)
457    end_time = datetime.datetime.now()
458    print_result(args, pass_count, fail_count, end_time, start_time)
459
460
461def print_result(args, pass_count, fail_count, end_time, start_time):
462    result_file = open_write_file(args.out_result, True)
463    msg = f'\npass count: {pass_count}'
464    write_result_file(msg, result_file)
465    print(msg)
466    msg = f'fail count: {fail_count}'
467    write_result_file(msg, result_file)
468    print(msg)
469    msg = f'total count: {fail_count + pass_count}'
470    write_result_file(msg, result_file)
471    print(msg)
472    msg = f'used time is: {str(end_time - start_time)}'
473    write_result_file(msg, result_file)
474    print(msg)
475    result_file.close()
476
477
478if __name__ == "__main__":
479    sys.exit(main(parse_args()))
480