1# 2# Copyright (C) 2020 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15# 16 17import os 18import logging 19 20import ltp_configs 21import ltp_enums 22import test_case 23from configs import stable_tests 24from configs import disabled_tests 25from common import filter_utils 26 27ltp_test_template = ' <option name="test-command-line" key="%s" value="&env_setup_cmd; ;' \ 28 ' cd <p_bin_dir; ; %s" />' 29 30class LtpTestCases(object): 31 """Load a ltp vts testcase definition file and parse it into a generator. 32 33 Attributes: 34 _data_path: string, the vts data path on host side 35 _filter_func: function, a filter method that will emit exception if a test is filtered 36 _ltp_tests_filter: list of string, filter for tests that are stable and disabled 37 _ltp_binaries: list of string, All ltp binaries that generate in build time 38 _ltp_config_lines: list of string: the context of the generated config 39 """ 40 41 def __init__(self, android_build_top, filter_func): 42 self._android_build_top = android_build_top 43 self._filter_func = filter_func 44 self._ltp_tests_filter = filter_utils.Filter( 45 [ i[0] for i in stable_tests.STABLE_TESTS ], 46 disabled_tests.DISABLED_TESTS, 47 enable_regex=True) 48 self._ltp_tests_filter.ExpandBitness() 49 self._ltp_binaries = [] 50 self._ltp_config_lines = [] 51 52 def ValidateDefinition(self, line): 53 """Validate a tab delimited test case definition. 54 55 Will check whether the given line of definition has three parts 56 separated by tabs. 57 It will also trim leading and ending white spaces for each part 58 in returned tuple (if valid). 59 60 Returns: 61 A tuple in format (test suite, test name, test command) if 62 definition is valid. None otherwise. 63 """ 64 items = [ 65 item.strip() 66 for item in line.split(ltp_enums.Delimiters.TESTCASE_DEFINITION) 67 ] 68 if not len(items) == 3 or not items: 69 return None 70 else: 71 return items 72 73 def ReadConfigTemplateFile(self): 74 """Read the template of the config file and return the context. 75 76 Returns: 77 String. 78 """ 79 file_name = ltp_configs.LTP_CONFIG_TEMPLATE_FILE_NAME 80 file_path = os.path.join(self._android_build_top, ltp_configs.LTP_CONFIG_TEMPLATE_DIR, file_name) 81 with open(file_path, 'r') as f: 82 return f.read() 83 84 def GetKernelModuleControllerOption(self, arch, n_bit, is_low_mem=False, is_hwasan=False): 85 """Get the Option of KernelModuleController. 86 87 Args: 88 arch: String, arch 89 n_bit: int, bitness 90 is_low_mem: bool, whether to use low memory device configuration 91 is_hwasan: bool, whether to use hwasan configuration 92 93 Returns: 94 String. 95 """ 96 arch_template = ' <option name="arch" value="{}"/>\n' 97 is_low_mem_template = ' <option name="is-low-mem" value="{}"/>\n' 98 is_hwasan_template = ' <option name="is-hwasan" value="{}"/>' 99 option_lines = arch_template + is_low_mem_template + is_hwasan_template 100 if n_bit == '64': 101 n_bit_string = str(n_bit) if arch == 'arm' else ('_'+str(n_bit)) 102 else: 103 n_bit_string = '' 104 arch_name = arch + n_bit_string 105 is_low_mem = 'true' if is_low_mem else 'false' 106 is_hwasan = 'true' if is_hwasan else 'false' 107 option_lines = option_lines.format(arch_name, 108 str(is_low_mem).lower(), 109 str(is_hwasan).lower()) 110 return option_lines 111 112 def GetLtpBinaries(self): 113 """Check the binary exist in the command. 114 115 Args: 116 command: String, the test command 117 118 Returns: 119 bool: True if the binary in the gen.bp 120 """ 121 gen_bp_path = os.path.join(self._android_build_top, ltp_configs.LTP_GEN_BINARY_BP) 122 for line in open(gen_bp_path, 'r'): 123 line = line.strip() 124 if not line or line.startswith('#'): 125 continue 126 if line.startswith("stem:") or line.startswith('filename:'): 127 ltp_binary = line.split('"')[1] 128 self._ltp_binaries.append(ltp_binary) 129 130 def IsLtpBinaryExist(self, commands): 131 """Check the binary exist in the command. 132 133 Args: 134 command: String, the test command 135 136 Returns: 137 bool: True if the binary in the gen.bp 138 """ 139 all_commands = commands.split(';') 140 for cmd in all_commands: 141 cmd = cmd.strip() 142 binary_name = cmd.split(' ')[0] 143 if binary_name in self._ltp_binaries: 144 return True 145 logging.info("Ltp binary not exist in cmd of '%s'", commands) 146 return False 147 148 def GenConfig(self, 149 arch, 150 n_bit, 151 test_filter, 152 output_file, 153 run_staging=False, 154 is_low_mem=False, 155 is_hwasan=False): 156 """Read the definition file and generate the test config. 157 158 Args: 159 arch: String, arch 160 n_bit: int, bitness 161 test_filter: Filter object, test name filter from base_test 162 output_file: String, the file path of the generating config 163 run_staging: bool, whether to use staging configuration 164 is_low_mem: bool, whether to use low memory device configuration 165 """ 166 self.GetLtpBinaries() 167 scenario_groups = (ltp_configs.TEST_SUITES_LOW_MEM 168 if is_low_mem else ltp_configs.TEST_SUITES) 169 logging.info('LTP scenario groups: %s', scenario_groups) 170 start_append_test_keyword = 'option name="per-binary-timeout"' 171 config_lines = self.ReadConfigTemplateFile() 172 module_controller_option = self.GetKernelModuleControllerOption(arch, n_bit, 173 is_low_mem, 174 is_hwasan) 175 test_case_string = '' 176 run_scritp = self.GenerateLtpRunScript(scenario_groups) 177 for line in run_scritp: 178 items = self.ValidateDefinition(line) 179 if not items: 180 continue 181 182 testsuite, testname, command = items 183 if is_low_mem and testsuite.endswith( 184 ltp_configs.LOW_MEMORY_SCENARIO_GROUP_SUFFIX): 185 testsuite = testsuite[:-len( 186 ltp_configs.LOW_MEMORY_SCENARIO_GROUP_SUFFIX)] 187 188 # Tests failed to build will have prefix "DISABLED_" 189 if testname.startswith("DISABLED_"): 190 logging.info("[Parser] Skipping test case {}-{}. Reason: " 191 "not built".format(testsuite, testname)) 192 continue 193 194 # Some test cases have hardcoded "/tmp" in the command 195 # we replace that with ltp_configs.TMPDIR 196 command = command.replace('/tmp', ltp_configs.TMPDIR) 197 198 testcase = test_case.TestCase( 199 testsuite=testsuite, testname=testname, command=command) 200 test_display_name = "{}_{}bit".format(str(testcase), n_bit) 201 202 # Check runner's base_test filtering method 203 try: 204 self._filter_func(test_display_name) 205 except: 206 logging.info("[Parser] Skipping test case %s. Reason: " 207 "filtered" % testcase.fullname) 208 testcase.is_filtered = True 209 testcase.note = "filtered" 210 211 logging.info('ltp_test_cases Load(): test_display_name = %s\n' 212 'cmd = %s', test_display_name, command) 213 214 # For skipping tests that are not designed or ready for Android, 215 # check for bit specific test in disabled list as well as non-bit specific 216 if ((self._ltp_tests_filter.IsInExcludeFilter(str(testcase)) or 217 self._ltp_tests_filter.IsInExcludeFilter(test_display_name)) and 218 not test_filter.IsInIncludeFilter(test_display_name)): 219 logging.info("[Parser] Skipping test case %s. Reason: " 220 "disabled" % testcase.fullname) 221 continue 222 223 # For separating staging tests from stable tests 224 if not self._ltp_tests_filter.IsInIncludeFilter(test_display_name): 225 if not run_staging and not test_filter.IsInIncludeFilter( 226 test_display_name): 227 # Skip staging tests in stable run 228 continue 229 else: 230 testcase.is_staging = True 231 testcase.note = "staging" 232 else: 233 if run_staging: 234 # Skip stable tests in staging run 235 continue 236 237 if not testcase.is_staging: 238 for x in stable_tests.STABLE_TESTS: 239 if x[0] == test_display_name and x[1]: 240 testcase.is_mandatory = True 241 break 242 243 if is_hwasan: 244 if x[0] in disabled_tests.DISABLED_TESTS_HWASAN: 245 continue 246 247 if self.IsLtpBinaryExist(command): 248 logging.info("[Parser] Adding test case %s." % testcase.fullname) 249 # Some test cases contain semicolons in their commands, 250 # and we replace them with && 251 command = command.replace(';', '&&') 252 # Replace the original command with '/data/local/tmp/ltp' 253 # e.g. mm.mmapstress07 254 command = command.replace(ltp_configs.LTPDIR, '<p_dir;') 255 ltp_test_line = ltp_test_template % (test_display_name, command) 256 test_case_string += (ltp_test_line + '\n') 257 nativetest_bit_path = '64' if n_bit == '64' else '' 258 config_lines = config_lines.format(nativetest_bit_path, module_controller_option, 259 test_case_string) 260 with open(output_file, 'w') as f: 261 f.write(config_lines) 262 263 def ReadCommentedTxt(self, filepath): 264 '''Read a lines of a file that are not commented by #. 265 266 Args: 267 filepath: string, path of file to read 268 269 Returns: 270 A set of string representing non-commented lines in given file 271 ''' 272 if not filepath: 273 logging.error('Invalid file path') 274 return None 275 276 with open(filepath, 'r') as f: 277 lines_gen = (line.strip() for line in f) 278 return set( 279 line for line in lines_gen 280 if line and not line.startswith('#')) 281 282 def GenerateLtpTestCases(self, testsuite, disabled_tests_list): 283 '''Generate test cases for each ltp test suite. 284 285 Args: 286 testsuite: string, test suite name 287 288 Returns: 289 A list of string 290 ''' 291 testsuite_script = os.path.join(self._android_build_top, 292 ltp_configs.LTP_RUNTEST_DIR, testsuite) 293 294 result = [] 295 for line in open(testsuite_script, 'r'): 296 line = line.strip() 297 if not line or line.startswith('#'): 298 continue 299 300 testname = line.split()[0] 301 testname_prefix = ('DISABLED_' 302 if testname in disabled_tests_list else '') 303 testname_modified = testname_prefix + testname 304 305 result.append("\t".join( 306 [testsuite, testname_modified, line[len(testname):].strip()])) 307 return result 308 309 def GenerateLtpRunScript(self, scenario_groups): 310 '''Given a scenario group generate test case script. 311 312 Args: 313 scenario_groups: list of string, name of test scenario groups to use 314 315 Returns: 316 A list of string 317 ''' 318 disabled_tests_path = os.path.join( 319 self._android_build_top, ltp_configs.LTP_DISABLED_BUILD_TESTS_CONFIG_PATH) 320 disabled_tests_list = self.ReadCommentedTxt(disabled_tests_path) 321 322 result = [] 323 for testsuite in scenario_groups: 324 result.extend( 325 self.GenerateLtpTestCases(testsuite, disabled_tests_list)) 326 return result 327