1# 2# Copyright (C) 2017 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15# 16 17import copy 18import logging 19import re 20from sre_constants import error as regex_error 21import types 22 23from vts.runners.host import const 24from vts.utils.python.common import list_utils 25 26REGEX_PREFIX = 'r(' 27REGEX_SUFFIX = ')' 28REGEX_PREFIX_ESCAPE = '\\r(' 29NEGATIVE_PATTERN_PREFIX = '-' 30_INCLUDE_FILTER = '_include_filter' 31_EXCLUDE_FILTER = '_exclude_filter' 32DEFAULT_EXCLUDE_OVER_INCLUDE = False 33_MODULE_NAME_PATTERN = '{module}.{test}' 34 35 36def ExpandBitness(input_list): 37 '''Expand filter items with bitness suffix. 38 39 If a filter item contains bitness suffix, only test name with that tag 40 will be included in output. 41 Otherwise, 2 more item with 32bit and 64bit suffix will be added to the output list. 42 43 This method removes duplicated item while keeping item order before returning output. 44 45 Examples of input -> output are: 46 [a_32bit] -> [a_32bit] 47 [a] -> [a, a_32bit, a_64bit] 48 [a_32bit, a] -> [a_32bit, a, a_64bit] 49 50 Args: 51 input_list: list of string, the list to expand 52 53 Returns: 54 A list of string 55 ''' 56 result = [] 57 for item in input_list: 58 result.append(str(item)) 59 if (not item.endswith(const.SUFFIX_32BIT) and 60 not item.endswith(const.SUFFIX_64BIT)): 61 result.append("%s_%s" % (item, const.SUFFIX_32BIT)) 62 result.append("%s_%s" % (item, const.SUFFIX_64BIT)) 63 return list_utils.DeduplicateKeepOrder(result) 64 65 66def ExpandAppendix(input_list, appendix_list, filter_pattern): 67 '''Expand each item in input_list with appendix in the appendix_list 68 69 For each item in input_list, expand it to N items (N=size of appendix_list) 70 by attaching it with each appendix form appendix_list. 71 Note, for items end with bitness info (e.g 32bit/63bit/_32bit/_64bit), 72 attach the appendix before the bitness info. (This is to make sure the 73 bitness info is always at the end of each item since the system rely on this 74 assumption to check the bitness info) 75 There are two cases when an item will not be expanded: 1) it is a Regex 76 filter item and 2) it has the pattern described by filter_pattern. 77 78 Examples of input -> output are: 79 [a] [_default] -> [a_default] 80 [a, b] [_default] -> [a_default, b_default] 81 [a] [_default, _test] -> [a_default, a_test] 82 [a, b_32bit] [_default, _test] 83 -> [a_default, a_test, b_default_32bit, b_test_32bit] 84 85 Args: 86 input_list: list of string, the list to expand 87 appendix_list: list of string, the appendix to be append. 88 filter_pattern: string, a Regex pattern to filter out the items that 89 should not expand. 90 91 Returns: 92 A list of string with expanded result. 93 ''' 94 result = [] 95 for item in input_list: 96 if IsRegexFilter(item) or re.compile(filter_pattern).match(item): 97 result.append(item) 98 continue 99 pos = len(item) 100 if (item.endswith(const.SUFFIX_32BIT) or 101 item.endswith(const.SUFFIX_64BIT)): 102 pos = len(item) - len(const.SUFFIX_32BIT) 103 if item[pos - 1] == "_": 104 pos = pos - 1 105 for appendix in appendix_list: 106 result.append(item[:pos] + appendix + item[pos:]) 107 return result 108 109 110def SplitFilterList(input_list): 111 '''Split filter items into exact and regex lists. 112 113 To specify a regex filter, the syntax is: 114 'r(suite.test)' for regex matching of 'suite.test', where '.' means 115 one of any char. 116 See Filter class docstring for details. 117 118 Args: 119 input_list: list of string, the list to split 120 121 Returns: 122 A tuple of lists: two lists where the first one is exact matching 123 list and second one is regex list where the wrapping 124 syntax 'r(..)' is removed. 125 ''' 126 exact = [] 127 regex = [] 128 for item in input_list: 129 if IsRegexFilter(item): 130 regex_item = item[len(REGEX_PREFIX):-len(REGEX_SUFFIX)] 131 try: 132 re.compile(regex_item) 133 regex.append(regex_item) 134 except regex_error: 135 logging.error('Invalid regex %s, ignored. Please refer to ' 136 'python re syntax documentation.' % regex_item) 137 elif item.startswith(REGEX_PREFIX_ESCAPE) and item.endswith( 138 REGEX_SUFFIX): 139 exact.append(REGEX_PREFIX + item[len(REGEX_PREFIX_ESCAPE):]) 140 else: 141 exact.append(item) 142 143 return (exact, regex) 144 145 146def SplitNegativePattern(input_list): 147 '''Split negative items out from an input filter list. 148 149 Items starting with the negative sign will be moved to the second returning 150 list. 151 152 Args: 153 input_list: list of string, the list to split 154 155 Returns: 156 A tuple of lists: two lists where the first one is positive patterns 157 and second one is negative items whose negative sign 158 is removed. 159 ''' 160 positive = [] 161 negative = [] 162 for item in input_list: 163 if item.startswith(NEGATIVE_PATTERN_PREFIX): 164 negative.append(item[len(NEGATIVE_PATTERN_PREFIX):]) 165 else: 166 positive.append(item) 167 return (positive, negative) 168 169 170def InRegexList(item, regex_list): 171 '''Checks whether a given string matches an item in the given regex list. 172 173 Args: 174 item: string, given string 175 regex_list: regex list 176 177 Returns: 178 bool, True if there is a match; False otherwise. 179 ''' 180 for regex in regex_list: 181 p = re.compile(regex) 182 m = p.match(item) 183 if m and m.start() == 0 and m.end() == len(item): 184 return True 185 186 return False 187 188 189def IsRegexFilter(item): 190 '''Checks whether the given item is a regex filter. 191 192 Args: 193 item: string, given string 194 195 Returns: 196 bool: true if the given item is a regex filter. 197 ''' 198 return item.startswith(REGEX_PREFIX) and item.endswith(REGEX_SUFFIX) 199 200 201class Filter(object): 202 '''A class to hold test filter rules and filter test names. 203 204 Regex matching is supported. Regex syntax is python re package syntax. 205 To specify a regex filter, the syntax is: 206 'suite.test' for exact matching 207 'r(suite.test)' for regex matching of 'suite.test', where '.' means 208 one of any char. 209 '\r(suite.test)' for exact matching of name 'r(suite.test)', where 210 '\r' is a two char string ('\\r' in code). 211 Since test name is not expected to start with backslash, the exact 212 string matching of name '\r(suite.test)' is not supported here. 213 214 Negative pattern is supported. If a test name starts with the negative 215 sign in include_filter, the negative sign will be removed and item will 216 be moved from include_filter to exclude_filter. Negative sign should 217 be added before regex prefix, i.e., '-r(negative.pattern)' 218 219 Attributes: 220 enable_regex: bool, whether regex is enabled. 221 include_filter: list of string, input include filter 222 exclude_filter: list of string, input exclude filter 223 include_filter_exact: list of string, exact include filter 224 include_filter_regex: list of string, exact include filter 225 exclude_filter_exact: list of string, exact exclude filter 226 exclude_filter_regex: list of string, exact exclude filter 227 exclude_over_include: bool, False for include over exclude; 228 True for exclude over include. 229 enable_native_pattern: bool, whether to enable negative pattern 230 processing in include_filter 231 enable_module_name_prefix_matching: bool, whether to perform auto 232 module name prefix matching 233 module_name: string, test module name for auto module name prefix 234 matching 235 expand_bitness: bool, whether to append bitness to filter items. 236 Default is False. When set to True, bitness will 237 be added to test name for filtering process, but 238 the original filter list will not be changed. 239 ''' 240 include_filter_exact = [] 241 include_filter_regex = [] 242 exclude_filter_exact = [] 243 exclude_filter_regex = [] 244 245 def __init__(self, 246 include_filter=[], 247 exclude_filter=[], 248 enable_regex=True, 249 exclude_over_include=None, 250 enable_negative_pattern=True, 251 enable_module_name_prefix_matching=False, 252 module_name=None, 253 expand_bitness=False): 254 self.enable_regex = enable_regex 255 self.expand_bitness = expand_bitness 256 257 self.enable_negative_pattern = enable_negative_pattern 258 self.include_filter = include_filter 259 self.exclude_filter = exclude_filter 260 if exclude_over_include is None: 261 exclude_over_include = DEFAULT_EXCLUDE_OVER_INCLUDE 262 self.exclude_over_include = exclude_over_include 263 self.enable_module_name_prefix_matching = enable_module_name_prefix_matching 264 self.module_name = module_name 265 266 # @Deprecated. Use expand_bitness parameter in construction method instead. 267 # This method will be removed after all legacy usage has been cleaned up 268 def ExpandBitness(self): 269 '''Expand bitness from filter. 270 271 Items in the filter that doesn't contain bitness suffix will be expended 272 to 3 items, 2 of which ending with bitness. This method is safe if 273 called multiple times. Regex items will not be expanded 274 ''' 275 self.include_filter_exact = ExpandBitness(self.include_filter_exact) 276 self.exclude_filter_exact = ExpandBitness(self.exclude_filter_exact) 277 self.expand_bitness = True 278 279 def IsIncludeFilterEmpty(self): 280 '''Check whether actual include filter is specified. 281 282 Since the input include filter may contain negative patterns, 283 checking self.include_filter is not always correct. 284 285 This method checks include_filter_exact and include_filter_regex. 286 ''' 287 return not self.include_filter_exact and not self.include_filter_regex 288 289 def ExpandAppendix(self, appendix_list, filter_pattern): 290 '''Expand filter with appendix from appendix_list. 291 292 Reset both include_filter and exclude_filter by expanding the filters 293 with appendix in appendix_list. 294 295 Args: 296 appendix_list: list of string to be append to the filters. 297 filter_pattern: string, a Regex pattern to filter out the items that 298 should not be expanded. 299 ''' 300 self.include_filter = ExpandAppendix(self.include_filter, 301 appendix_list, filter_pattern) 302 self.exclude_filter = ExpandAppendix(self.exclude_filter, 303 appendix_list, filter_pattern) 304 305 def Filter(self, item): 306 '''Filter a given string using the internal filters. 307 308 Rule: 309 By default, include_filter overrides exclude_filter. This means: 310 If include_filter is empty, only exclude_filter is checked. 311 Otherwise, only include_filter is checked 312 If exclude_over_include is set to True, exclude filter will first 313 be checked. 314 315 Args: 316 item: string, the string for filter check 317 318 Returns: 319 bool. True if it passed the filter; False otherwise 320 ''' 321 if self.exclude_over_include: 322 if self.IsInExcludeFilter(item): 323 return False 324 325 if not self.IsIncludeFilterEmpty(): 326 return self.IsInIncludeFilter(item) 327 328 return True 329 else: 330 if not self.IsIncludeFilterEmpty(): 331 return self.IsInIncludeFilter(item) 332 333 return not self.IsInExcludeFilter(item) 334 335 def IsInIncludeFilter(self, item): 336 '''Check if item is in include filter. 337 338 If enable_module_name_prefix_matching is set to True, module name 339 added to item as prefix will also be check from the include filter. 340 341 Args: 342 item: string, item to check filter 343 344 Returns: 345 bool, True if in include filter. 346 ''' 347 return self._ModuleNamePrefixMatchingCheck(item, 348 self._IsInIncludeFilter) 349 350 def IsInExcludeFilter(self, item): 351 '''Check if item is in exclude filter. 352 353 If enable_module_name_prefix_matching is set to True, module name 354 added to item as prefix will also be check from the exclude filter. 355 356 Args: 357 item: string, item to check filter 358 359 Returns: 360 bool, True if in exclude filter. 361 ''' 362 return self._ModuleNamePrefixMatchingCheck(item, 363 self._IsInExcludeFilter) 364 365 def _ModuleNamePrefixMatchingCheck(self, item, check_function): 366 '''Check item from filter after appending module name as prefix. 367 368 This function will first check whether enable_module_name_prefix_matching 369 is True and module_name is not empty. Then, the check_function will 370 be applied to the item. If the result is False and 371 enable_module_name_prefix_matching is True, module name will be added 372 as the prefix to the item, in format of '<module_name>.<item>', and 373 call the check_function again with the new resulting name. 374 375 This is mainly used for retry command where test module name are 376 automatically added to test case name. 377 378 Args: 379 item: string, test name for checking. 380 check_function: function to check item in filters. 381 382 Return: 383 bool, True if item pass the filter from the given check_function. 384 ''' 385 res = check_function(item) 386 387 if (not res and self.enable_module_name_prefix_matching and 388 self.module_name): 389 res = check_function( 390 _MODULE_NAME_PATTERN.format( 391 module=self.module_name, test=item)) 392 393 return res 394 395 def _IsInIncludeFilter(self, item): 396 '''Internal function to check if item is in include filter. 397 398 Args: 399 item: string, item to check filter 400 401 Returns: 402 bool, True if in include filter. 403 ''' 404 return item in self.include_filter_exact or InRegexList( 405 item, self.include_filter_regex) 406 407 def _IsInExcludeFilter(self, item): 408 '''Internal function to check if item is in exclude filter. 409 410 Args: 411 item: string, item to check filter 412 413 Returns: 414 bool, True if in exclude filter. 415 ''' 416 return item in self.exclude_filter_exact or InRegexList( 417 item, self.exclude_filter_regex) 418 419 @property 420 def include_filter(self): 421 '''Getter method for include_filter. 422 423 Use this method to print include_filter only. 424 425 If the items needed to be added, use add_to_exclude_filter method. 426 427 E.g. 428 self.add_to_exclude_filter('pattern1') 429 430 If the filter needs to be modified without using add_to_exclude_filter, 431 call refresh_filter() after modification. Otherwise, the change will 432 not take effect. 433 434 E.g. 435 Get and modify filter: 436 filter = Filter() 437 filter.include_filter.append('pattern1') 438 Refresh the filter: 439 filter.refresh_filter() 440 ''' 441 return getattr(self, _INCLUDE_FILTER, []) 442 443 @include_filter.setter 444 def include_filter(self, include_filter): 445 '''Setter method for include_filter''' 446 setattr(self, _INCLUDE_FILTER, include_filter) 447 self.refresh_filter() 448 449 @property 450 def exclude_filter(self): 451 '''Getter method for exclude_filter. 452 453 Use this method to print exclude_filter only. 454 455 If the items needed to be added, use add_to_exclude_filter method. 456 457 E.g. 458 self.add_to_exclude_filter('pattern1') 459 460 If the filter needs to be modified without using add_to_exclude_filter, 461 call refresh_filter() after modification. Otherwise, the change will 462 not take effect. 463 464 E.g. 465 Get and modify filter: 466 filter = Filter() 467 filter.exclude_filter.append('pattern1') 468 Refresh the filter: 469 filter.refresh_filter() 470 ''' 471 return getattr(self, _EXCLUDE_FILTER, []) 472 473 @exclude_filter.setter 474 def exclude_filter(self, exclude_filter): 475 '''Setter method for exclude_filter''' 476 setattr(self, _EXCLUDE_FILTER, exclude_filter) 477 self.refresh_filter() 478 479 def add_to_include_filter(self, pattern, auto_refresh=True): 480 '''Add an item to include_filter. 481 482 Args: 483 pattern: string or list of string. Item(s) to add 484 auto_refresh: bool, whether to automatically call refresh_filter(). 485 Default is True. Use False only if a large number of 486 items are added one by one in a sequence call. 487 In that case, call refresh_filter() at the end of the 488 sequence. 489 ''' 490 if not isinstance(pattern, types.ListType): 491 pattern = [pattern] 492 493 self.include_filter.extend(pattern) 494 495 if auto_refresh: 496 self.refresh_filter() 497 498 def add_to_exclude_filter(self, pattern, auto_refresh=True): 499 '''Add an item to exclude_filter. 500 501 Args: 502 pattern: string or list of string. Item(s) to add 503 auto_refresh: bool, whether to automatically call refresh_filter(). 504 Default is True. Use False only if a large number of 505 items are added one by one in a sequence call. 506 In that case, call refresh_filter() at the end of the 507 sequence. 508 ''' 509 if not isinstance(pattern, types.ListType): 510 pattern = [pattern] 511 512 self.exclude_filter.extend(pattern) 513 514 if auto_refresh: 515 self.refresh_filter() 516 517 def refresh_filter(self): 518 '''Process the filter patterns. 519 520 This method splits filter into exact and regex patterns. 521 Bitness will also be appended if expand_bitness is True. 522 ''' 523 include_filter = copy.copy(self.include_filter) 524 exclude_filter = copy.copy(self.exclude_filter) 525 526 if self.enable_negative_pattern: 527 include_filter, include_filter_negative = SplitNegativePattern( 528 include_filter) 529 exclude_filter.extend(include_filter_negative) 530 531 if self.enable_regex: 532 self.include_filter_exact, self.include_filter_regex = SplitFilterList( 533 include_filter) 534 self.exclude_filter_exact, self.exclude_filter_regex = SplitFilterList( 535 exclude_filter) 536 else: 537 self.include_filter_exact = include_filter 538 self.exclude_filter_exact = exclude_filter 539 540 if self.expand_bitness: 541 self.include_filter_exact = ExpandBitness( 542 self.include_filter_exact) 543 self.exclude_filter_exact = ExpandBitness( 544 self.exclude_filter_exact) 545 546 def __str__(self): 547 return ('Filter:\nenable_regex: {enable_regex}\n' 548 'enable_negative_pattern: {enable_negative_pattern}\n' 549 'enable_module_name_prefix_matching: ' 550 '{enable_module_name_prefix_matching}\n' 551 'module_name: {module_name}\n' 552 'include_filter: {include_filter}\n' 553 'exclude_filter: {exclude_filter}\n' 554 'include_filter_exact: {include_filter_exact}\n' 555 'include_filter_regex: {include_filter_regex}\n' 556 'exclude_filter_exact: {exclude_filter_exact}\n' 557 'exclude_filter_regex: {exclude_filter_regex}\n' 558 'expand_bitness: {expand_bitness}'.format( 559 enable_regex=self.enable_regex, 560 enable_negative_pattern=self.enable_negative_pattern, 561 enable_module_name_prefix_matching= 562 self.enable_module_name_prefix_matching, 563 module_name=self.module_name, 564 include_filter=self.include_filter, 565 exclude_filter=self.exclude_filter, 566 include_filter_exact=self.include_filter_exact, 567 include_filter_regex=self.include_filter_regex, 568 exclude_filter_exact=self.exclude_filter_exact, 569 exclude_filter_regex=self.exclude_filter_regex, 570 expand_bitness=self.expand_bitness)) 571