1# 2# Copyright (C) 2020 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15# 16 17import os 18import logging 19 20import ltp_configs 21import ltp_enums 22import test_case 23from configs import stable_tests 24from configs import disabled_tests 25from common import filter_utils 26from typing import Set, Optional, List, Callable 27 28ltp_test_template = ' <option name="test-command-line" key="%s" value="&env_setup_cmd; ;' \ 29 ' cd <p_bin_dir; ; %s" />' 30 31class LtpTestCases(object): 32 """Load a ltp vts testcase definition file and parse it into a generator. 33 34 Attributes: 35 _data_path: string, the vts data path on host side 36 _filter_func: function, a filter method that will emit exception if a test is filtered 37 _ltp_tests_filter: list of string, filter for tests that are stable and disabled 38 _ltp_binaries: list of string, All ltp binaries that generate in build time 39 _ltp_config_lines: list of string: the context of the generated config 40 """ 41 42 def __init__(self, android_build_top: str, filter_func: Callable): 43 self._android_build_top = android_build_top 44 self._filter_func = filter_func 45 self._ltp_tests_filter = filter_utils.Filter( 46 set(stable_tests.STABLE_TESTS.keys()), 47 disabled_tests.DISABLED_TESTS, 48 enable_regex=True) 49 self._ltp_tests_filter.ExpandBitness() 50 self._ltp_binaries = [] 51 self._ltp_config_lines = [] 52 53 def ValidateDefinition(self, line: str) -> Optional[List[str]]: 54 """Validate a tab delimited test case definition. 55 56 Will check whether the given line of definition has three parts 57 separated by tabs. 58 It will also trim leading and ending white spaces for each part 59 in returned tuple (if valid). 60 61 Returns: 62 A tuple in format (test suite, test name, test command) if 63 definition is valid. None otherwise. 64 """ 65 items = [ 66 item.strip() 67 for item in line.split(ltp_enums.Delimiters.TESTCASE_DEFINITION) 68 ] 69 if not len(items) == 3 or not items: 70 return None 71 else: 72 return items 73 74 def ReadConfigTemplateFile(self) -> str: 75 """Read the template of the config file and return the context. 76 77 Returns: 78 String. 79 """ 80 file_name = ltp_configs.LTP_CONFIG_TEMPLATE_FILE_NAME 81 file_path = os.path.join(self._android_build_top, ltp_configs.LTP_CONFIG_TEMPLATE_DIR, file_name) 82 with open(file_path, 'r') as f: 83 return f.read() 84 85 def GetKernelModuleControllerOption(self, arch: str, n_bit: int, is_low_mem: bool = False, is_hwasan: bool = False) -> str: 86 """Get the Option of KernelModuleController. 87 88 Args: 89 arch: String, arch 90 n_bit: int, bitness 91 is_low_mem: bool, whether to use low memory device configuration 92 is_hwasan: bool, whether to use hwasan configuration 93 94 Returns: 95 String. 96 """ 97 arch_template = ' <option name="arch" value="{}"/>\n' 98 is_low_mem_template = ' <option name="is-low-mem" value="{}"/>\n' 99 is_hwasan_template = ' <option name="is-hwasan" value="{}"/>' 100 option_lines = arch_template + is_low_mem_template + is_hwasan_template 101 if n_bit == '64': 102 n_bit_string = str(n_bit) if arch == 'arm' or arch == 'riscv' else ('_'+str(n_bit)) 103 else: 104 n_bit_string = '' 105 arch_name = arch + n_bit_string 106 is_low_mem = 'true' if is_low_mem else 'false' 107 is_hwasan = 'true' if is_hwasan else 'false' 108 option_lines = option_lines.format(arch_name, 109 str(is_low_mem).lower(), 110 str(is_hwasan).lower()) 111 return option_lines 112 113 def GetLtpBinaries(self): 114 """Check the binary exist in the command. 115 116 Args: 117 command: String, the test command 118 """ 119 gen_bp_path = os.path.join(self._android_build_top, ltp_configs.LTP_GEN_BINARY_BP) 120 for line in open(gen_bp_path, 'r'): 121 line = line.strip() 122 if not line or line.startswith('#'): 123 continue 124 if line.startswith("stem:") or line.startswith('filename:'): 125 ltp_binary = line.split('"')[1] 126 self._ltp_binaries.append(ltp_binary) 127 128 def IsLtpBinaryExist(self, commands: str) -> bool: 129 """Check the binary exist in the command. 130 131 Args: 132 command: String, the test command 133 134 Returns: 135 bool: True if the binary in the gen.bp 136 """ 137 all_commands = commands.split(';') 138 for cmd in all_commands: 139 cmd = cmd.strip() 140 binary_name = cmd.split(' ')[0] 141 if binary_name in self._ltp_binaries: 142 return True 143 logging.info("Ltp binary not exist in cmd of '%s'", commands) 144 return False 145 146 def GenConfig(self, 147 arch: str, 148 n_bit: int, 149 test_filter: filter_utils.Filter, 150 output_file: str, 151 run_staging: bool = False, 152 is_low_mem: bool = False, 153 is_hwasan: bool = False): 154 """Read the definition file and generate the test config. 155 156 Args: 157 arch: String, arch 158 n_bit: int, bitness 159 test_filter: Filter object, test name filter from base_test 160 output_file: String, the file path of the generating config 161 run_staging: bool, whether to use staging configuration 162 is_low_mem: bool, whether to use low memory device configuration 163 """ 164 self.GetLtpBinaries() 165 scenario_groups = (ltp_configs.TEST_SUITES_LOW_MEM 166 if is_low_mem else ltp_configs.TEST_SUITES) 167 logging.info('LTP scenario groups: %s', scenario_groups) 168 start_append_test_keyword = 'option name="per-binary-timeout"' 169 config_lines = self.ReadConfigTemplateFile() 170 module_controller_option = self.GetKernelModuleControllerOption(arch, n_bit, 171 is_low_mem, 172 is_hwasan) 173 mandatory_test_cases = [] 174 skippable_test_cases = [] 175 run_script = self.GenerateLtpRunScript(scenario_groups) 176 for line in run_script: 177 items = self.ValidateDefinition(line) 178 if not items: 179 continue 180 181 testsuite, testname, command = items 182 if is_low_mem and testsuite.endswith( 183 ltp_configs.LOW_MEMORY_SCENARIO_GROUP_SUFFIX): 184 testsuite = testsuite[:-len( 185 ltp_configs.LOW_MEMORY_SCENARIO_GROUP_SUFFIX)] 186 187 # Tests failed to build will have prefix "DISABLED_" 188 if testname.startswith("DISABLED_"): 189 logging.info("[Parser] Skipping test case {}-{}. Reason: " 190 "not built".format(testsuite, testname)) 191 continue 192 193 # Some test cases have hardcoded "/tmp" in the command 194 # we replace that with ltp_configs.TMPDIR 195 command = command.replace('/tmp', ltp_configs.TMPDIR) 196 197 testcase = test_case.TestCase( 198 testsuite=testsuite, testname=testname, command=command) 199 test_display_name = "{}_{}bit".format(str(testcase), n_bit) 200 201 # Check runner's base_test filtering method 202 try: 203 self._filter_func(test_display_name) 204 except: 205 logging.info("[Parser] Skipping test case %s. Reason: " 206 "filtered" % testcase.fullname) 207 testcase.is_filtered = True 208 testcase.note = "filtered" 209 210 logging.info('ltp_test_cases Load(): test_display_name = %s\n' 211 'cmd = %s', test_display_name, command) 212 213 # For skipping tests that are not designed or ready for Android, 214 # check for bit specific test in disabled list as well as non-bit specific 215 if ((self._ltp_tests_filter.IsInExcludeFilter(str(testcase)) or 216 self._ltp_tests_filter.IsInExcludeFilter(test_display_name)) and 217 not test_filter.IsInIncludeFilter(test_display_name)): 218 logging.info("[Parser] Skipping test case %s. Reason: " 219 "disabled" % testcase.fullname) 220 continue 221 222 # For separating staging tests from stable tests 223 if not self._ltp_tests_filter.IsInIncludeFilter(test_display_name): 224 if not run_staging and not test_filter.IsInIncludeFilter( 225 test_display_name): 226 # Skip staging tests in stable run 227 continue 228 else: 229 testcase.is_staging = True 230 testcase.note = "staging" 231 else: 232 if run_staging: 233 # Skip stable tests in staging run 234 continue 235 236 if not testcase.is_staging: 237 if stable_tests.STABLE_TESTS.get(test_display_name, False): 238 testcase.is_mandatory = True 239 240 if is_hwasan: 241 if test_display_name in disabled_tests.DISABLED_TESTS_HWASAN: 242 continue 243 244 if self.IsLtpBinaryExist(command): 245 logging.info("[Parser] Adding test case %s." % testcase.fullname) 246 # Some test cases contain semicolons in their commands, 247 # and we replace them with && 248 command = command.replace(';', '&&') 249 # Replace the original command with '/data/local/tmp/ltp' 250 # e.g. mm.mmapstress07 251 command = command.replace(ltp_configs.LTPDIR, '<p_dir;') 252 ltp_test_line = ltp_test_template % (test_display_name, command) 253 if testcase.is_mandatory: 254 mandatory_test_cases.append(ltp_test_line) 255 else: 256 skippable_test_cases.append(ltp_test_line) 257 nativetest_bit_path = '64' if n_bit == '64' else '' 258 config_lines = config_lines.format( 259 nativetest_bit_path=nativetest_bit_path, 260 module_controller_option=module_controller_option, 261 mandatory_test_cases='\n'.join(mandatory_test_cases), 262 skippable_test_cases='\n'.join(skippable_test_cases)) 263 with open(output_file, 'w') as f: 264 f.write(config_lines) 265 266 def ReadCommentedTxt(self, filepath: str) -> Optional[Set[str]]: 267 '''Read a lines of a file that are not commented by #. 268 269 Args: 270 filepath: string, path of file to read 271 272 Returns: 273 A set of string representing non-commented lines in given file 274 ''' 275 if not filepath: 276 logging.error('Invalid file path') 277 return None 278 279 with open(filepath, 'r') as f: 280 lines_gen = (line.strip() for line in f) 281 return set( 282 line for line in lines_gen 283 if line and not line.startswith('#')) 284 285 def GenerateLtpTestCases(self, testsuite: str, disabled_tests_list: List[str]) -> List[str]: 286 '''Generate test cases for each ltp test suite. 287 288 Args: 289 testsuite: string, test suite name 290 291 Returns: 292 A list of string 293 ''' 294 testsuite_script = os.path.join(self._android_build_top, 295 ltp_configs.LTP_RUNTEST_DIR, testsuite) 296 297 result = [] 298 for line in open(testsuite_script, 'r'): 299 line = line.strip() 300 if not line or line.startswith('#'): 301 continue 302 303 testname = line.split()[0] 304 testname_prefix = ('DISABLED_' 305 if testname in disabled_tests_list else '') 306 testname_modified = testname_prefix + testname 307 308 result.append("\t".join( 309 [testsuite, testname_modified, line[len(testname):].strip()])) 310 return result 311 312 def GenerateLtpRunScript(self, scenario_groups: List[str]) -> List[str]: 313 '''Given a scenario group generate test case script. 314 315 Args: 316 scenario_groups: list of string, name of test scenario groups to use 317 318 Returns: 319 A list of string 320 ''' 321 disabled_tests_path = os.path.join( 322 self._android_build_top, ltp_configs.LTP_DISABLED_BUILD_TESTS_CONFIG_PATH) 323 disabled_tests_list = self.ReadCommentedTxt(disabled_tests_path) 324 325 result = [] 326 for testsuite in scenario_groups: 327 result.extend( 328 self.GenerateLtpTestCases(testsuite, disabled_tests_list)) 329 return result 330