• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2012 the V8 project authors. All rights reserved.
2# Redistribution and use in source and binary forms, with or without
3# modification, are permitted provided that the following conditions are
4# met:
5#
6#     * Redistributions of source code must retain the above copyright
7#       notice, this list of conditions and the following disclaimer.
8#     * Redistributions in binary form must reproduce the above
9#       copyright notice, this list of conditions and the following
10#       disclaimer in the documentation and/or other materials provided
11#       with the distribution.
12#     * Neither the name of Google Inc. nor the names of its
13#       contributors may be used to endorse or promote products derived
14#       from this software without specific prior written permission.
15#
16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
28
29import imp
30import os
31
32from . import commands
33from . import statusfile
34from . import utils
35from ..objects import testcase
36from variants import ALL_VARIANTS, ALL_VARIANT_FLAGS, FAST_VARIANT_FLAGS
37
38
39FAST_VARIANTS = set(["default", "turbofan"])
40STANDARD_VARIANT = set(["default"])
41
42
43class VariantGenerator(object):
44  def __init__(self, suite, variants):
45    self.suite = suite
46    self.all_variants = ALL_VARIANTS & variants
47    self.fast_variants = FAST_VARIANTS & variants
48    self.standard_variant = STANDARD_VARIANT & variants
49
50  def FilterVariantsByTest(self, testcase):
51    result = self.all_variants
52    if testcase.outcomes:
53      if statusfile.OnlyStandardVariant(testcase.outcomes):
54        return self.standard_variant
55      if statusfile.OnlyFastVariants(testcase.outcomes):
56        result = self.fast_variants
57    return result
58
59  def GetFlagSets(self, testcase, variant):
60    if testcase.outcomes and statusfile.OnlyFastVariants(testcase.outcomes):
61      return FAST_VARIANT_FLAGS[variant]
62    else:
63      return ALL_VARIANT_FLAGS[variant]
64
65
66class TestSuite(object):
67
68  @staticmethod
69  def LoadTestSuite(root, global_init=True):
70    name = root.split(os.path.sep)[-1]
71    f = None
72    try:
73      (f, pathname, description) = imp.find_module("testcfg", [root])
74      module = imp.load_module("testcfg", f, pathname, description)
75      return module.GetSuite(name, root)
76    except ImportError:
77      # Use default if no testcfg is present.
78      return GoogleTestSuite(name, root)
79    finally:
80      if f:
81        f.close()
82
83  def __init__(self, name, root):
84    # Note: This might be called concurrently from different processes.
85    self.name = name  # string
86    self.root = root  # string containing path
87    self.tests = None  # list of TestCase objects
88    self.rules = None  # dictionary mapping test path to list of outcomes
89    self.wildcards = None  # dictionary mapping test paths to list of outcomes
90    self.total_duration = None  # float, assigned on demand
91
92  def shell(self):
93    return "d8"
94
95  def suffix(self):
96    return ".js"
97
98  def status_file(self):
99    return "%s/%s.status" % (self.root, self.name)
100
101  # Used in the status file and for stdout printing.
102  def CommonTestName(self, testcase):
103    if utils.IsWindows():
104      return testcase.path.replace("\\", "/")
105    else:
106      return testcase.path
107
108  def ListTests(self, context):
109    raise NotImplementedError
110
111  def _VariantGeneratorFactory(self):
112    """The variant generator class to be used."""
113    return VariantGenerator
114
115  def CreateVariantGenerator(self, variants):
116    """Return a generator for the testing variants of this suite.
117
118    Args:
119      variants: List of variant names to be run as specified by the test
120                runner.
121    Returns: An object of type VariantGenerator.
122    """
123    return self._VariantGeneratorFactory()(self, set(variants))
124
125  def PrepareSources(self):
126    """Called once before multiprocessing for doing file-system operations.
127
128    This should not access the network. For network access use the method
129    below.
130    """
131    pass
132
133  def DownloadData(self):
134    pass
135
136  def ReadStatusFile(self, variables):
137    with open(self.status_file()) as f:
138      self.rules, self.wildcards = (
139          statusfile.ReadStatusFile(f.read(), variables))
140
141  def ReadTestCases(self, context):
142    self.tests = self.ListTests(context)
143
144  @staticmethod
145  def _FilterSlow(slow, mode):
146    return (mode == "run" and not slow) or (mode == "skip" and slow)
147
148  @staticmethod
149  def _FilterPassFail(pass_fail, mode):
150    return (mode == "run" and not pass_fail) or (mode == "skip" and pass_fail)
151
152  def FilterTestCasesByStatus(self, warn_unused_rules,
153                              slow_tests="dontcare",
154                              pass_fail_tests="dontcare",
155                              variants=False):
156
157    # Use only variants-dependent rules and wildcards when filtering
158    # respective test cases and generic rules when filtering generic test
159    # cases.
160    if not variants:
161      rules = self.rules[""]
162      wildcards = self.wildcards[""]
163    else:
164      # We set rules and wildcards to a variant-specific version for each test
165      # below.
166      rules = {}
167      wildcards = {}
168
169    filtered = []
170
171    # Remember used rules as tuples of (rule, variant), where variant is "" for
172    # variant-independent rules.
173    used_rules = set()
174
175    for t in self.tests:
176      slow = False
177      pass_fail = False
178      testname = self.CommonTestName(t)
179      variant = t.variant or ""
180      if variants:
181        rules = self.rules[variant]
182        wildcards = self.wildcards[variant]
183      if testname in rules:
184        used_rules.add((testname, variant))
185        # Even for skipped tests, as the TestCase object stays around and
186        # PrintReport() uses it.
187        t.outcomes = t.outcomes | rules[testname]
188        if statusfile.DoSkip(t.outcomes):
189          continue  # Don't add skipped tests to |filtered|.
190        for outcome in t.outcomes:
191          if outcome.startswith('Flags: '):
192            t.flags += outcome[7:].split()
193        slow = statusfile.IsSlow(t.outcomes)
194        pass_fail = statusfile.IsPassOrFail(t.outcomes)
195      skip = False
196      for rule in wildcards:
197        assert rule[-1] == '*'
198        if testname.startswith(rule[:-1]):
199          used_rules.add((rule, variant))
200          t.outcomes = t.outcomes | wildcards[rule]
201          if statusfile.DoSkip(t.outcomes):
202            skip = True
203            break  # "for rule in wildcards"
204          slow = slow or statusfile.IsSlow(t.outcomes)
205          pass_fail = pass_fail or statusfile.IsPassOrFail(t.outcomes)
206      if (skip
207          or self._FilterSlow(slow, slow_tests)
208          or self._FilterPassFail(pass_fail, pass_fail_tests)):
209        continue  # "for t in self.tests"
210      filtered.append(t)
211    self.tests = filtered
212
213    if not warn_unused_rules:
214      return
215
216    if not variants:
217      for rule in self.rules[""]:
218        if (rule, "") not in used_rules:
219          print("Unused rule: %s -> %s (variant independent)" % (
220              rule, self.rules[""][rule]))
221      for rule in self.wildcards[""]:
222        if (rule, "") not in used_rules:
223          print("Unused rule: %s -> %s (variant independent)" % (
224              rule, self.wildcards[""][rule]))
225    else:
226      for variant in ALL_VARIANTS:
227        for rule in self.rules[variant]:
228          if (rule, variant) not in used_rules:
229            print("Unused rule: %s -> %s (variant: %s)" % (
230                rule, self.rules[variant][rule], variant))
231        for rule in self.wildcards[variant]:
232          if (rule, variant) not in used_rules:
233            print("Unused rule: %s -> %s (variant: %s)" % (
234                rule, self.wildcards[variant][rule], variant))
235
236
237  def FilterTestCasesByArgs(self, args):
238    """Filter test cases based on command-line arguments.
239
240    An argument with an asterisk in the end will match all test cases
241    that have the argument as a prefix. Without asterisk, only exact matches
242    will be used with the exeption of the test-suite name as argument.
243    """
244    filtered = []
245    globs = []
246    exact_matches = []
247    for a in args:
248      argpath = a.split('/')
249      if argpath[0] != self.name:
250        continue
251      if len(argpath) == 1 or (len(argpath) == 2 and argpath[1] == '*'):
252        return  # Don't filter, run all tests in this suite.
253      path = '/'.join(argpath[1:])
254      if path[-1] == '*':
255        path = path[:-1]
256        globs.append(path)
257      else:
258        exact_matches.append(path)
259    for t in self.tests:
260      for a in globs:
261        if t.path.startswith(a):
262          filtered.append(t)
263          break
264      for a in exact_matches:
265        if t.path == a:
266          filtered.append(t)
267          break
268    self.tests = filtered
269
270  def GetFlagsForTestCase(self, testcase, context):
271    raise NotImplementedError
272
273  def GetSourceForTest(self, testcase):
274    return "(no source available)"
275
276  def IsFailureOutput(self, testcase):
277    return testcase.output.exit_code != 0
278
279  def IsNegativeTest(self, testcase):
280    return False
281
282  def HasFailed(self, testcase):
283    execution_failed = self.IsFailureOutput(testcase)
284    if self.IsNegativeTest(testcase):
285      return not execution_failed
286    else:
287      return execution_failed
288
289  def GetOutcome(self, testcase):
290    if testcase.output.HasCrashed():
291      return statusfile.CRASH
292    elif testcase.output.HasTimedOut():
293      return statusfile.TIMEOUT
294    elif self.HasFailed(testcase):
295      return statusfile.FAIL
296    else:
297      return statusfile.PASS
298
299  def HasUnexpectedOutput(self, testcase):
300    outcome = self.GetOutcome(testcase)
301    return not outcome in (testcase.outcomes or [statusfile.PASS])
302
303  def StripOutputForTransmit(self, testcase):
304    if not self.HasUnexpectedOutput(testcase):
305      testcase.output.stdout = ""
306      testcase.output.stderr = ""
307
308  def CalculateTotalDuration(self):
309    self.total_duration = 0.0
310    for t in self.tests:
311      self.total_duration += t.duration
312    return self.total_duration
313
314
315class StandardVariantGenerator(VariantGenerator):
316  def FilterVariantsByTest(self, testcase):
317    return self.standard_variant
318
319
320class GoogleTestSuite(TestSuite):
321  def __init__(self, name, root):
322    super(GoogleTestSuite, self).__init__(name, root)
323
324  def ListTests(self, context):
325    shell = os.path.abspath(os.path.join(context.shell_dir, self.shell()))
326    if utils.IsWindows():
327      shell += ".exe"
328
329    output = None
330    for i in xrange(3): # Try 3 times in case of errors.
331      output = commands.Execute(context.command_prefix +
332                                [shell, "--gtest_list_tests"] +
333                                context.extra_flags)
334      if output.exit_code == 0:
335        break
336      print "Test executable failed to list the tests (try %d).\n\nStdout:" % i
337      print output.stdout
338      print "\nStderr:"
339      print output.stderr
340      print "\nExit code: %d" % output.exit_code
341    else:
342      raise Exception("Test executable failed to list the tests.")
343
344    tests = []
345    test_case = ''
346    for line in output.stdout.splitlines():
347      test_desc = line.strip().split()[0]
348      if test_desc.endswith('.'):
349        test_case = test_desc
350      elif test_case and test_desc:
351        test = testcase.TestCase(self, test_case + test_desc)
352        tests.append(test)
353    tests.sort(key=lambda t: t.path)
354    return tests
355
356  def GetFlagsForTestCase(self, testcase, context):
357    return (testcase.flags + ["--gtest_filter=" + testcase.path] +
358            ["--gtest_random_seed=%s" % context.random_seed] +
359            ["--gtest_print_time=0"] +
360            context.mode_flags)
361
362  def _VariantGeneratorFactory(self):
363    return StandardVariantGenerator
364
365  def shell(self):
366    return self.name
367