1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3 4# Copyright (c) 2021-2024 Huawei Device Co., Ltd. 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17 18import fnmatch 19import logging 20import multiprocessing 21import re 22import argparse 23from unittest import TestCase 24from abc import abstractmethod, ABC 25from collections import Counter 26from datetime import datetime 27from glob import glob 28from os import path 29from pathlib import Path 30from typing import List, Set, Tuple, Optional 31 32from tqdm import tqdm 33 34from runner.chapters import Chapters 35from runner.enum_types.configuration_kind import ConfigurationKind 36from runner.options.cli_args_wrapper import CliArgsWrapper 37from runner.plugins.ets.test_ets import TestETS 38from runner.logger import Log 39from runner.options.config import Config 40from runner.test_base import Test 41from runner.utils import get_group_number 42 43CONST_COMMENT = ["#"] 44TEST_COMMENT_EXPR = re.compile(r"^\s*(?P<test>[^# ]+)?(\s*#\s*(?P<comment>.+))?", re.MULTILINE) 45 46 47def load_list(test_root: str, test_list_path: str) -> List[str]: 48 result: List[str] = [] 49 if not path.exists(test_list_path): 50 return result 51 52 with open(test_list_path, 'r', encoding="utf-8") as file: 53 for line in file: 54 test, _ = get_test_and_comment_from_line(line.strip(" \n")) 55 if test is not None: 56 result.append(path.normpath(path.join(test_root, test))) 57 58 return result 59 60 61def get_test_id(file: str, start_directory: Path) -> str: 62 relpath = Path(file).relative_to(start_directory) 63 return str(relpath) 64 65 66def get_test_and_comment_from_line(line: str) -> Tuple[Optional[str], Optional[str]]: 67 line_parts = TEST_COMMENT_EXPR.search(line) 68 if line_parts: 69 return line_parts["test"], line_parts["comment"] 70 return None, None 71 72 73def correct_path(root: Path, test_list: str) -> str: 74 return path.abspath(test_list) if path.exists(test_list) else path.normpath(path.join(root, test_list)) 75 76 77_LOGGER = logging.getLogger("runner.runner_base") 78 79# pylint: disable=invalid-name,global-statement 80worker_cli_wrapper_args: Optional[argparse.Namespace] = None 81 82 83def init_worker(shared_args: argparse.Namespace) -> None: 84 global worker_cli_wrapper_args 85 worker_cli_wrapper_args = shared_args 86 87 88def run_test(test: Test) -> Test: 89 CliArgsWrapper.args = worker_cli_wrapper_args 90 return test.run() 91 92 93class Runner(ABC): 94 def __init__(self, config: Config, name: str) -> None: 95 # TODO(vpukhov): adjust es2panda path 96 default_ets_arktsconfig = path.normpath(path.join( 97 config.general.build, 98 "tools", "es2panda", "generated", "arktsconfig.json" 99 )) 100 if not path.exists(default_ets_arktsconfig): 101 default_ets_arktsconfig = path.normpath(path.join( 102 config.general.build, 103 "gen", # for GN build 104 "tools", "es2panda", "generated", "arktsconfig.json" 105 )) 106 107 # Roots: 108 # directory where test files are located - it's either set explicitly to the absolute value 109 # or the current folder (where this python file is located!) parent 110 self.test_root = config.general.test_root 111 if self.test_root is not None: 112 Log.summary(_LOGGER, f"TEST_ROOT set to {self.test_root}") 113 # directory where list files (files with list of ignored, excluded, and other tests) are located 114 # it's either set explicitly to the absolute value or 115 # the current folder (where this python file is located!) parent 116 self.default_list_root = Path(config.general.static_core_root) / 'tests' / 'tests-u-runner' / 'test-lists' 117 self.list_root = config.general.list_root 118 if self.list_root is not None: 119 Log.summary(_LOGGER, f"LIST_ROOT set to {self.list_root}") 120 121 # runner init time 122 self.start_time = datetime.now() 123 # root directory containing bin folder with binary files 124 self.build_dir = config.general.build 125 self.arktsconfig = config.es2panda.arktsconfig \ 126 if config.es2panda.arktsconfig is not None \ 127 else default_ets_arktsconfig 128 129 self.config = config 130 self.name: str = name 131 132 # Lists: 133 # excluded test is a test what should not be loaded and should be tried to run 134 # excluded_list: either absolute path or path relative from list_root to the file with the list of such tests 135 self.excluded_lists: List[str] = [] 136 self.excluded_tests: Set[str] = set([]) 137 # ignored test is a test what should be loaded and executed, but its failure should be ignored 138 # ignored_list: either absolute path or path relative from list_root to the file with the list of such tests 139 # aka: kfl = known failures list 140 self.ignored_lists: List[str] = [] 141 self.ignored_tests: Set[str] = set([]) 142 # list of file names, each is a name of a test. Every test should be executed 143 # So, it can contain ignored tests, but cannot contain excluded tests 144 self.tests: Set[Test] = set([]) 145 # list of results of every executed test 146 self.results: List[Test] = [] 147 # name of file with a list of only tests what should be executed 148 # if it's specified other tests are not executed 149 self.explicit_list = correct_path(self.list_root, config.test_lists.explicit_list) \ 150 if config.test_lists.explicit_list is not None and self.list_root is not None \ 151 else None 152 # name of the single test file in form of a relative path from test_root what should be executed 153 # if it's specified other tests are not executed even if test_list is set 154 self.explicit_test = config.test_lists.explicit_file 155 156 # Counters: 157 # failed + ignored + passed + excluded_after = len of all executed tests 158 # failed + ignored + passed + excluded_after + excluded = len of full set of tests 159 self.failed = 0 160 self.ignored = 0 161 self.passed = 0 162 self.excluded = 0 163 # Test chosen to execute can detect itself as excluded one 164 self.excluded_after = 0 165 self.update_excluded = config.test_lists.update_excluded 166 167 self.conf_kind = Runner.detect_conf(config) 168 169 @staticmethod 170 # pylint: disable=too-many-return-statements 171 def detect_conf(config: Config) -> ConfigurationKind: 172 if config.ark_aot.enable: 173 is_aot_full = len([ 174 arg for arg in config.ark_aot.aot_args 175 if "--compiler-inline-full-intrinsics=true" in arg 176 ]) > 0 177 if is_aot_full: 178 return ConfigurationKind.AOT_FULL 179 return ConfigurationKind.AOT 180 181 if config.ark.jit.enable: 182 return ConfigurationKind.JIT 183 184 if config.ark.interpreter_type: 185 return ConfigurationKind.OTHER_INT 186 187 if config.quick.enable: 188 return ConfigurationKind.QUICK 189 190 return ConfigurationKind.INT 191 192 @staticmethod 193 def _search_duplicates(original: List[str], kind: str) -> None: 194 main_counter = Counter(original) 195 dupes = [test for test, frequency in main_counter.items() if frequency > 1] 196 if len(dupes) > 0: 197 Log.summary(_LOGGER, f"There are {len(dupes)} duplicates in {kind} lists.") 198 for test in dupes: 199 Log.short(_LOGGER, f"\t{test}") 200 elif len(original) > 0: 201 Log.summary(_LOGGER, f"No duplicates found in {kind} lists.") 202 203 @abstractmethod 204 def create_test(self, test_file: str, flags: List[str], is_ignored: bool) -> Test: 205 pass 206 207 @abstractmethod 208 def summarize(self) -> int: 209 pass 210 211 @abstractmethod 212 def create_coverage_html(self) -> None: 213 pass 214 215 def run(self) -> None: 216 Log.all(_LOGGER, "Start test running") 217 with multiprocessing.Pool(processes=self.config.general.processes, 218 initializer=init_worker, initargs=(CliArgsWrapper.args,)) as pool: 219 results = pool.imap_unordered(run_test, self.tests, chunksize=self.config.general.chunksize) 220 if self.config.general.show_progress: 221 results = tqdm(results, total=len(self.tests)) 222 self.results = list(results) 223 pool.close() 224 pool.join() 225 226 def load_tests_from_lists(self, lists: List[str]) -> List[str]: 227 tests = [] 228 for list_name in lists: 229 list_path = correct_path(self.list_root, list_name) 230 Log.summary(_LOGGER, f"Loading tests from the list {list_path}") 231 TestCase().assertTrue(self.test_root, "TEST_ROOT not set to correct value") 232 tests.extend(load_list(self.test_root, list_path)) 233 return tests 234 235 # Read excluded_lists and load list of excluded tests 236 def load_excluded_tests(self) -> None: 237 excluded_tests = self.load_tests_from_lists(self.excluded_lists) 238 self.excluded_tests.update(excluded_tests) 239 self.excluded = len(self.excluded_tests) 240 self._search_duplicates(excluded_tests, "excluded") 241 242 # Read ignored_lists and load list of ignored tests 243 def load_ignored_tests(self) -> None: 244 ignored_tests = self.load_tests_from_lists(self.ignored_lists) 245 self.ignored_tests.update(ignored_tests) 246 self._search_duplicates(ignored_tests, "ignored") 247 248 # Browse the directory, search for files with the specified extension 249 # and add them as tests 250 def add_directory(self, directory: str, extension: str, flags: List[str]) -> None: 251 Log.summary(_LOGGER, f"Loading tests from the directory {directory}") 252 test_files = [] 253 if self.explicit_test is not None: 254 test_files.extend([correct_path(self.test_root, self.explicit_test)]) 255 elif self.explicit_list is not None: 256 test_files.extend(self.load_tests_from_lists([self.explicit_list])) 257 else: 258 if not self.config.test_lists.skip_test_lists: 259 self.load_excluded_tests() 260 self.load_ignored_tests() 261 test_files.extend(self.__load_test_files(directory, extension)) 262 263 self._search_both_excluded_and_ignored_tests() 264 self._search_not_used_ignored(test_files) 265 266 all_tests = {self.create_test(test, flags, test in self.ignored_tests) for test in test_files} 267 not_tests = {t for t in all_tests if isinstance(t, TestETS) and not t.is_valid_test} 268 valid_tests = all_tests - not_tests 269 270 if self.config.test_lists.groups.quantity > 1: 271 groups = self.config.test_lists.groups.quantity 272 n_group = self.config.test_lists.groups.number 273 n_group = n_group if n_group <= groups else groups 274 valid_tests = { 275 test for test in valid_tests 276 if get_group_number(test.path, groups) == n_group 277 } 278 279 self.tests.update(valid_tests) 280 Log.all(_LOGGER, f"Loaded {len(self.tests)} tests") 281 282 def _search_both_excluded_and_ignored_tests(self) -> None: 283 already_excluded = [test for test in self.ignored_tests if test in self.excluded_tests] 284 if not already_excluded: 285 return 286 Log.summary(_LOGGER, f"Found {len(already_excluded)} tests present both in excluded and ignored test " 287 f"lists.") 288 for test in already_excluded: 289 Log.all(_LOGGER, f"\t{test}") 290 self.ignored_tests.remove(test) 291 292 def _search_not_used_ignored(self, found_tests: List[str]) -> None: 293 ignored_absent = [test for test in self.ignored_tests if test not in found_tests] 294 if ignored_absent: 295 Log.summary(_LOGGER, f"Found {len(ignored_absent)} tests in ignored lists but absent on the file system:") 296 for test in ignored_absent: 297 Log.summary(_LOGGER, f"\t{test}") 298 else: 299 Log.short(_LOGGER, "All ignored tests are found on the file system") 300 301 def __load_test_files(self, directory: str, extension: str) -> List[str]: 302 if self.config.test_lists.filter != "*" and self.config.test_lists.groups.chapters: 303 Log.exception_and_raise( 304 _LOGGER, 305 "Incorrect configuration: specify either filter or chapter options" 306 ) 307 test_files: List[str] = [] 308 excluded: List[str] = list(self.excluded_tests)[:] 309 glob_expression = path.join(directory, f"**/*.{extension}") 310 test_files.extend(fnmatch.filter( 311 glob(glob_expression, recursive=True), 312 path.normpath(path.join(directory, self.config.test_lists.filter)) 313 )) 314 if self.config.test_lists.groups.chapters: 315 test_files = self.__filter_by_chapters(directory, test_files, extension) 316 return [ 317 test for test in test_files 318 if self.update_excluded or test not in excluded 319 ] 320 321 def __filter_by_chapters(self, base_folder: str, files: List[str], extension: str) -> List[str]: 322 test_files: Set[str] = set() 323 chapters: Chapters = self.__parse_chapters() 324 for chapter in self.config.test_lists.groups.chapters: 325 test_files.update(chapters.filter_by_chapter(chapter, base_folder, files, extension)) 326 return list(test_files) 327 328 def __parse_chapters(self) -> Chapters: 329 chapters: Optional[Chapters] = None 330 if path.isfile(self.config.test_lists.groups.chapters_file): 331 chapters = Chapters(self.config.test_lists.groups.chapters_file) 332 else: 333 corrected_chapters_file = correct_path(self.list_root, self.config.test_lists.groups.chapters_file) 334 if path.isfile(corrected_chapters_file): 335 chapters = Chapters(corrected_chapters_file) 336 else: 337 Log.exception_and_raise( 338 _LOGGER, 339 f"Not found either '{self.config.test_lists.groups.chapters_file}' or " 340 f"'{corrected_chapters_file}'", FileNotFoundError) 341 return chapters 342