• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3#
4# Copyright (c) 2024 Huawei Device Co., Ltd.
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17
18import argparse
19import json
20import multiprocessing
21import sys
22import os
23import subprocess as spp
24import tempfile
25from collections import OrderedDict
26from typing import List
27import logging
28
29NPROC = multiprocessing.cpu_count()
30TMP_DIR = tempfile.gettempdir()
31
32SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
33OPTIONS_LIST_PATH = os.path.join(SCRIPT_DIR, 'options_list.json')
34
35
36def get_args():
37    parser = argparse.ArgumentParser(description="Abckit stress test")
38    parser.add_argument('--build-dir',
39                        type=str,
40                        required=True,
41                        help=f'Path to build dir')
42    parser.add_argument('--update-fail-list',
43                        action='store_true',
44                        default=False,
45                        help=f'Update fail list')
46    parser.add_argument("--ark-path",
47                        type=str,
48                        default=None,
49                        help=f'ARK runtime path')
50    parser.add_argument("--repeats",
51                        type=int,
52                        default=None,
53                        help=f'VM test retry attempts')
54    parser.add_argument("--with-runtime",
55                        type=bool,
56                        default=False,
57                        help=f'Run stress tests and compare their outputs')
58    return parser.parse_args()
59
60
61def collect_from(rd, func) -> List[str]:
62    tmp: List[str] = []
63    for root, _, file_names in os.walk(rd):
64        for file_name in file_names:
65            tmp.append(os.path.join(root, file_name))
66    return filter(func, tmp)
67
68
69class ExecRes:
70
71    def __init__(self, returncode, stdout, stderr):
72        self.returncode = returncode
73        self.stdout = stdout
74        self.stderr = stderr
75
76
77def stress_exec(cmd, **kwargs):
78    logger = create_logger()
79    default_kwargs = {
80        'cwd': os.getcwd(),
81        'allow_error': False,
82        'timeout': 900,
83        'print_output': False,
84        'print_command': True,
85        'env': {},
86        'repeats': 1,
87    }
88    kwargs = {**default_kwargs, **kwargs}
89
90    if kwargs['print_command']:
91        logger.debug(
92            '$ %s> %s %s', kwargs.get('cwd'), " ".join(
93                list(
94                    map(lambda x: f'{x}={kwargs.get("env").get(x)}',
95                        list(kwargs.get('env'))))), " ".join(cmd))
96
97    ct = kwargs['timeout']
98    sub_env = {**os.environ.copy(), **kwargs['env']}
99    for _ in range(kwargs['repeats']):
100        return_code, stdout, stderr = __exec_impl(
101            cmd,
102            cwd=kwargs.get('cwd'),
103            timeout=ct,
104            print_output=kwargs.get('print_output'),
105            env=sub_env)
106        if return_code == 0:
107            break
108        ct = ct + 60
109
110    if return_code != 0 and not kwargs['allow_error']:
111        raise Exception(
112            f"Error: Non-zero return code\nstdout: {stdout}\nstderr: {stderr}")
113    return ExecRes(return_code, stdout, stderr)
114
115
116def __exec_impl(cmd,
117                cwd=os.getcwd(),
118                timeout=900,
119                print_output=False,
120                env=None):
121    logger = create_logger()
122    proc = spp.Popen(cmd,
123                     cwd=cwd,
124                     stdout=spp.PIPE,
125                     stderr=spp.STDOUT,
126                     encoding='ISO-8859-1',
127                     env=env)
128    while True and print_output:
129        line = proc.stdout.readline()
130        logger.debug(line.strip())
131        if not line:
132            break
133    try:
134        stdout, stderr = proc.communicate(timeout=timeout)
135        return_code = proc.wait()
136    except spp.TimeoutExpired:
137        stdout, stderr = "timeout", "timeout"
138        return_code = -1
139
140    return return_code, stdout, stderr
141
142
143def parse_stdout(error: str, stdout):
144    if stdout is not None and 'ASSERTION FAILED:' in stdout:
145        for line in stdout.split('\n'):
146            if 'ASSERTION FAILED:' in line:
147                error = line
148    if stdout is not None and 'ERROR: plugin returned non-zero' in stdout:
149        for line in stdout.split('\n'):
150            if 'failed!' in line:
151                error = line.split(' ')[-2] + " " + line.split(' ')[-1]
152    return error
153
154
155def get_fail_list(result):
156    fail_list = OrderedDict()
157    for src_path in result:
158        if result[src_path]['error'] != '0':
159            fail_list[src_path] = result[src_path]['error']
160    return fail_list
161
162
163def check_regression_errors(kfl_path: str, fail_list):
164    logger = create_logger()
165    with open(kfl_path, 'r') as f:
166        old_fail_list = json.load(f)
167    regression_errors = []
168    for fail in fail_list:
169        if fail not in old_fail_list:
170            regression_errors.append(f'REGRESSION ERROR: new fail "{fail}"')
171    if regression_errors:
172        logger.debug('\n'.join(regression_errors))
173        return False
174    return True
175
176
177def update_fail_list(kfl_path: str, fail_list):
178    with os.fdopen(
179            os.open(kfl_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o755),
180            'w') as f:
181        json.dump(fail_list, f, indent=2, sort_keys=True)
182
183
184def check_fail_list(kfl_path: str, fail_list):
185    logger = create_logger()
186
187    with open(kfl_path, 'r') as f:
188        old_fail_list = json.load(f)
189
190    errors = []
191    for old_fail in old_fail_list:
192        if old_fail not in fail_list:
193            errors.append(f'ERROR: no file in new fail list "{old_fail}"')
194    if errors:
195        logger.debug('\n'.join(errors))
196        logger.debug(
197            'Please update fail list, rerun this script with "--update-fail-list"'
198        )
199        return False
200    return True
201
202
203def create_logger():
204    logger = multiprocessing.get_logger()
205    logger.setLevel(logging.DEBUG)
206    formatter = logging.Formatter(
207        '[%(asctime)s| %(levelname)s| %(processName)s] %(message)s')
208    handler = logging.StreamHandler(sys.stdout)
209    handler.setFormatter(formatter)
210
211    # this bit will make sure you won't have
212    # duplicated messages in the output
213    if not len(logger.handlers):
214        logger.addHandler(handler)
215    return logger
216
217
218def run_abckit(build_dir, source, input_abc, output_abc):
219    abckit = os.path.join(build_dir, 'arkcompiler', 'runtime_core', 'abckit')
220    stress_plugin = os.path.join(build_dir, 'arkcompiler', 'runtime_core',
221                                 'libabckit_stress_plugin.so')
222    ld_library_path = []
223    ld_library_path_old = os.environ.get('LD_LIBRARY_PATH')
224    if ld_library_path_old:
225        ld_library_path.append(ld_library_path_old)
226    ld_library_path = [
227        os.path.join(build_dir, 'arkcompiler', 'runtime_core'),
228        os.path.join(build_dir, 'arkcompiler', 'ets_runtime'),
229        os.path.join(build_dir, 'arkcompiler', 'ets_frontend'),
230        os.path.join(build_dir, 'thirdparty', 'icu'),
231        os.path.join(build_dir, 'thirdparty', 'zlib')
232    ]
233    cmd = [
234        abckit, '--plugin-path', stress_plugin, '--input-file', input_abc,
235        '--output-file', output_abc
236    ]
237    options_list = OrderedDict()
238    with open(OPTIONS_LIST_PATH, 'r') as f:
239        options_list = json.load(f)
240    if source in options_list:
241        cmd += [options_list[source][0], options_list[source][1]]
242    result = stress_exec(cmd,
243                         allow_error=True,
244                         print_command=True,
245                         env={"LD_LIBRARY_PATH": ':'.join(ld_library_path)})
246    return result
247