• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2012 the V8 project authors. All rights reserved.
2# Redistribution and use in source and binary forms, with or without
3# modification, are permitted provided that the following conditions are
4# met:
5#
6#     * Redistributions of source code must retain the above copyright
7#       notice, this list of conditions and the following disclaimer.
8#     * Redistributions in binary form must reproduce the above
9#       copyright notice, this list of conditions and the following
10#       disclaimer in the documentation and/or other materials provided
11#       with the distribution.
12#     * Neither the name of Google Inc. nor the names of its
13#       contributors may be used to endorse or promote products derived
14#       from this software without specific prior written permission.
15#
16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
28
29import fnmatch
30import imp
31import os
32
33from . import command
34from . import statusfile
35from . import utils
36from ..objects.testcase import TestCase
37from .variants import ALL_VARIANTS, ALL_VARIANT_FLAGS
38
39
40STANDARD_VARIANT = set(["default"])
41
42
43class VariantsGenerator(object):
44  def __init__(self, variants):
45    self._all_variants = [v for v in variants if v in ALL_VARIANTS]
46    self._standard_variant = [v for v in variants if v in STANDARD_VARIANT]
47
48  def gen(self, test):
49    """Generator producing (variant, flags, procid suffix) tuples."""
50    flags_set = self._get_flags_set(test)
51    for n, variant in enumerate(self._get_variants(test)):
52      yield (variant, flags_set[variant][0], n)
53
54  def _get_flags_set(self, test):
55    return ALL_VARIANT_FLAGS
56
57  def _get_variants(self, test):
58    if test.only_standard_variant:
59      return self._standard_variant
60    return self._all_variants
61
62
63class TestCombiner(object):
64  def get_group_key(self, test):
65    """To indicate what tests can be combined with each other we define a group
66    key for each test. Tests with the same group key can be combined. Test
67    without a group key (None) is not combinable with any other test.
68    """
69    raise NotImplementedError()
70
71  def combine(self, name, tests):
72    """Returns test combined from `tests`. Since we identify tests by their
73    suite and name, `name` parameter should be unique within one suite.
74    """
75    return self._combined_test_class()(name, tests)
76
77  def _combined_test_class(self):
78    raise NotImplementedError()
79
80
81class TestSuite(object):
82  @staticmethod
83  def LoadTestSuite(root, test_config):
84    name = root.split(os.path.sep)[-1]
85    f = None
86    try:
87      (f, pathname, description) = imp.find_module("testcfg", [root])
88      module = imp.load_module(name + "_testcfg", f, pathname, description)
89      return module.GetSuite(name, root, test_config)
90    finally:
91      if f:
92        f.close()
93
94  def __init__(self, name, root, test_config):
95    self.name = name  # string
96    self.root = root  # string containing path
97    self.test_config = test_config
98    self.tests = None  # list of TestCase objects
99    self.statusfile = None
100    self.suppress_internals = False
101
102  def status_file(self):
103    return "%s/%s.status" % (self.root, self.name)
104
105  def do_suppress_internals(self):
106    """Specifies if this test suite should suppress asserts based on internals.
107
108    Internals are e.g. testing against the outcome of native runtime functions.
109    This is switched off on some fuzzers that violate these contracts.
110    """
111    self.suppress_internals = True
112
113  def ListTests(self):
114    raise NotImplementedError
115
116  def get_variants_gen(self, variants):
117    return self._variants_gen_class()(variants)
118
119  def _variants_gen_class(self):
120    return VariantsGenerator
121
122  def test_combiner_available(self):
123    return bool(self._test_combiner_class())
124
125  def get_test_combiner(self):
126    cls = self._test_combiner_class()
127    if cls:
128      return cls()
129    return None
130
131  def _test_combiner_class(self):
132    """Returns Combiner subclass. None if suite doesn't support combining
133    tests.
134    """
135    return None
136
137  def ReadStatusFile(self, variables):
138    self.statusfile = statusfile.StatusFile(self.status_file(), variables)
139
140  def ReadTestCases(self):
141    self.tests = self.ListTests()
142
143
144  def FilterTestCasesByStatus(self,
145                              slow_tests_mode=None,
146                              pass_fail_tests_mode=None):
147    """Filters tests by outcomes from status file.
148
149    Status file has to be loaded before using this function.
150
151    Args:
152      slow_tests_mode: What to do with slow tests.
153      pass_fail_tests_mode: What to do with pass or fail tests.
154
155    Mode options:
156      None (default) - don't skip
157      "skip" - skip if slow/pass_fail
158      "run" - skip if not slow/pass_fail
159    """
160    def _skip_slow(is_slow, mode):
161      return (
162        (mode == 'run' and not is_slow) or
163        (mode == 'skip' and is_slow))
164
165    def _skip_pass_fail(pass_fail, mode):
166      return (
167        (mode == 'run' and not pass_fail) or
168        (mode == 'skip' and pass_fail))
169
170    def _compliant(test):
171      if test.do_skip:
172        return False
173      if _skip_slow(test.is_slow, slow_tests_mode):
174        return False
175      if _skip_pass_fail(test.is_pass_or_fail, pass_fail_tests_mode):
176        return False
177      return True
178
179    self.tests = filter(_compliant, self.tests)
180
181  def FilterTestCasesByArgs(self, args):
182    """Filter test cases based on command-line arguments.
183
184    args can be a glob: asterisks in any position of the argument
185    represent zero or more characters. Without asterisks, only exact matches
186    will be used with the exeption of the test-suite name as argument.
187    """
188    filtered = []
189    globs = []
190    for a in args:
191      argpath = a.split('/')
192      if argpath[0] != self.name:
193        continue
194      if len(argpath) == 1 or (len(argpath) == 2 and argpath[1] == '*'):
195        return  # Don't filter, run all tests in this suite.
196      path = '/'.join(argpath[1:])
197      globs.append(path)
198
199    for t in self.tests:
200      for g in globs:
201        if fnmatch.fnmatch(t.path, g):
202          filtered.append(t)
203          break
204    self.tests = filtered
205
206  def _create_test(self, path, **kwargs):
207    if self.suppress_internals:
208      test_class = self._suppressed_test_class()
209    else:
210      test_class = self._test_class()
211    return test_class(self, path, self._path_to_name(path), self.test_config,
212                      **kwargs)
213
214  def _suppressed_test_class(self):
215    """Optional testcase that suppresses assertions. Used by fuzzers that are
216    only interested in dchecks or tsan and that might violate the assertions
217    through fuzzing.
218    """
219    return self._test_class()
220
221  def _test_class(self):
222    raise NotImplementedError
223
224  def _path_to_name(self, path):
225    if utils.IsWindows():
226      return path.replace("\\", "/")
227    return path
228