• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2012 the V8 project authors. All rights reserved.
2# Redistribution and use in source and binary forms, with or without
3# modification, are permitted provided that the following conditions are
4# met:
5#
6#     * Redistributions of source code must retain the above copyright
7#       notice, this list of conditions and the following disclaimer.
8#     * Redistributions in binary form must reproduce the above
9#       copyright notice, this list of conditions and the following
10#       disclaimer in the documentation and/or other materials provided
11#       with the distribution.
12#     * Neither the name of Google Inc. nor the names of its
13#       contributors may be used to endorse or promote products derived
14#       from this software without specific prior written permission.
15#
16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
28import os
29import re
30
31from variants import ALL_VARIANTS
32from utils import Freeze
33
34# Possible outcomes
35FAIL = "FAIL"
36PASS = "PASS"
37TIMEOUT = "TIMEOUT"
38CRASH = "CRASH"
39
40# Outcomes only for status file, need special handling
41FAIL_OK = "FAIL_OK"
42FAIL_SLOPPY = "FAIL_SLOPPY"
43
44# Modifiers
45SKIP = "SKIP"
46SLOW = "SLOW"
47NO_VARIANTS = "NO_VARIANTS"
48
49ALWAYS = "ALWAYS"
50
51KEYWORDS = {}
52for key in [SKIP, FAIL, PASS, CRASH, SLOW, FAIL_OK, NO_VARIANTS, FAIL_SLOPPY,
53            ALWAYS]:
54  KEYWORDS[key] = key
55
56# Support arches, modes to be written as keywords instead of strings.
57VARIABLES = {ALWAYS: True}
58for var in ["debug", "release", "big", "little", "android",
59            "android_arm", "android_arm64", "android_ia32", "android_x64",
60            "arm", "arm64", "ia32", "mips", "mipsel", "mips64", "mips64el",
61            "x64", "ppc", "ppc64", "s390", "s390x", "macos", "windows",
62            "linux", "aix", "r1", "r2", "r3", "r5", "r6"]:
63  VARIABLES[var] = var
64
65# Allow using variants as keywords.
66for var in ALL_VARIANTS:
67  VARIABLES[var] = var
68
69class StatusFile(object):
70  def __init__(self, path, variables):
71    """
72    _rules:        {variant: {test name: [rule]}}
73    _prefix_rules: {variant: {test name prefix: [rule]}}
74    """
75    with open(path) as f:
76      self._rules, self._prefix_rules = ReadStatusFile(f.read(), variables)
77
78  def get_outcomes(self, testname, variant=None):
79    """Merges variant dependent and independent rules."""
80    outcomes = frozenset()
81
82    for key in set([variant or '', '']):
83      rules = self._rules.get(key, {})
84      prefix_rules = self._prefix_rules.get(key, {})
85
86      if testname in rules:
87        outcomes |= rules[testname]
88
89      for prefix in prefix_rules:
90        if testname.startswith(prefix):
91          outcomes |= prefix_rules[prefix]
92
93    return outcomes
94
95  def warn_unused_rules(self, tests, check_variant_rules=False):
96    """Finds and prints unused rules in status file.
97
98    Rule X is unused when it doesn't apply to any tests, which can also mean
99    that all matching tests were skipped by another rule before evaluating X.
100
101    Args:
102      tests: list of pairs (testname, variant)
103      check_variant_rules: if set variant dependent rules are checked
104    """
105
106    if check_variant_rules:
107      variants = list(ALL_VARIANTS)
108    else:
109      variants = ['']
110    used_rules = set()
111
112    for testname, variant in tests:
113      variant = variant or ''
114
115      if testname in self._rules.get(variant, {}):
116        used_rules.add((testname, variant))
117        if SKIP in self._rules[variant][testname]:
118          continue
119
120      for prefix in self._prefix_rules.get(variant, {}):
121        if testname.startswith(prefix):
122          used_rules.add((prefix, variant))
123          if SKIP in self._prefix_rules[variant][prefix]:
124            break
125
126    for variant in variants:
127      for rule, value in (
128          list(self._rules.get(variant, {}).iteritems()) +
129          list(self._prefix_rules.get(variant, {}).iteritems())):
130        if (rule, variant) not in used_rules:
131          if variant == '':
132            variant_desc = 'variant independent'
133          else:
134            variant_desc = 'variant: %s' % variant
135          print 'Unused rule: %s -> %s (%s)' % (rule, value, variant_desc)
136
137
138def _JoinsPassAndFail(outcomes1, outcomes2):
139  """Indicates if we join PASS and FAIL from two different outcome sets and
140  the first doesn't already contain both.
141  """
142  return (
143      PASS in outcomes1 and
144      not (FAIL in outcomes1 or FAIL_OK in outcomes1) and
145      (FAIL in outcomes2 or FAIL_OK in outcomes2)
146  )
147
148VARIANT_EXPRESSION = object()
149
150def _EvalExpression(exp, variables):
151  """Evaluates expression and returns its result. In case of NameError caused by
152  undefined "variant" identifier returns VARIANT_EXPRESSION marker.
153  """
154
155  try:
156    return eval(exp, variables)
157  except NameError as e:
158    identifier = re.match("name '(.*)' is not defined", e.message).group(1)
159    assert identifier == "variant", "Unknown identifier: %s" % identifier
160    return VARIANT_EXPRESSION
161
162
163def _EvalVariantExpression(
164  condition, section, variables, variant, rules, prefix_rules):
165  variables_with_variant = dict(variables)
166  variables_with_variant["variant"] = variant
167  result = _EvalExpression(condition, variables_with_variant)
168  assert result != VARIANT_EXPRESSION
169  if result is True:
170    _ReadSection(
171        section,
172        variables_with_variant,
173        rules[variant],
174        prefix_rules[variant],
175    )
176  else:
177    assert result is False, "Make sure expressions evaluate to boolean values"
178
179
180def _ParseOutcomeList(rule, outcomes, variables, target_dict):
181  """Outcome list format: [condition, outcome, outcome, ...]"""
182
183  result = set([])
184  if type(outcomes) == str:
185    outcomes = [outcomes]
186  for item in outcomes:
187    if type(item) == str:
188      result.add(item)
189    elif type(item) == list:
190      condition = item[0]
191      exp = _EvalExpression(condition, variables)
192      assert exp != VARIANT_EXPRESSION, (
193        "Nested variant expressions are not supported")
194      if exp is False:
195        continue
196
197      # Ensure nobody uses an identifier by mistake, like "default",
198      # which would evaluate to true here otherwise.
199      assert exp is True, "Make sure expressions evaluate to boolean values"
200
201      for outcome in item[1:]:
202        assert type(outcome) == str
203        result.add(outcome)
204    else:
205      assert False
206  if len(result) == 0:
207    return
208  if rule in target_dict:
209    # A FAIL without PASS in one rule has always precedence over a single
210    # PASS (without FAIL) in another. Otherwise the default PASS expectation
211    # in a rule with a modifier (e.g. PASS, SLOW) would be joined to a FAIL
212    # from another rule (which intended to mark a test as FAIL and not as
213    # PASS and FAIL).
214    if _JoinsPassAndFail(target_dict[rule], result):
215      target_dict[rule] -= set([PASS])
216    if _JoinsPassAndFail(result, target_dict[rule]):
217      result -= set([PASS])
218    target_dict[rule] |= result
219  else:
220    target_dict[rule] = result
221
222
223def ReadContent(content):
224  return eval(content, KEYWORDS)
225
226
227def ReadStatusFile(content, variables):
228  """Status file format
229  Status file := [section]
230  section = [CONDITION, section_rules]
231  section_rules := {path: outcomes}
232  outcomes := outcome | [outcome, ...]
233  outcome := SINGLE_OUTCOME | [CONDITION, SINGLE_OUTCOME, SINGLE_OUTCOME, ...]
234  """
235
236  # Empty defaults for rules and prefix_rules. Variant-independent
237  # rules are mapped by "", others by the variant name.
238  rules = {variant: {} for variant in ALL_VARIANTS}
239  rules[""] = {}
240  prefix_rules = {variant: {} for variant in ALL_VARIANTS}
241  prefix_rules[""] = {}
242
243  variables.update(VARIABLES)
244  for conditional_section in ReadContent(content):
245    assert type(conditional_section) == list
246    assert len(conditional_section) == 2
247    condition, section = conditional_section
248    exp = _EvalExpression(condition, variables)
249
250    # The expression is variant-independent and evaluates to False.
251    if exp is False:
252      continue
253
254    # The expression is variant-independent and evaluates to True.
255    if exp is True:
256      _ReadSection(
257          section,
258          variables,
259          rules[''],
260          prefix_rules[''],
261      )
262      continue
263
264    # The expression is variant-dependent (contains "variant" keyword)
265    if exp == VARIANT_EXPRESSION:
266      # If the expression contains one or more "variant" keywords, we evaluate
267      # it for all possible variants and create rules for those that apply.
268      for variant in ALL_VARIANTS:
269        _EvalVariantExpression(
270            condition, section, variables, variant, rules, prefix_rules)
271      continue
272
273    assert False, "Make sure expressions evaluate to boolean values"
274
275  return Freeze(rules), Freeze(prefix_rules)
276
277
278def _ReadSection(section, variables, rules, prefix_rules):
279  assert type(section) == dict
280  for rule, outcome_list in section.iteritems():
281    assert type(rule) == str
282
283    if rule[-1] == '*':
284      _ParseOutcomeList(rule[:-1], outcome_list, variables, prefix_rules)
285    else:
286      _ParseOutcomeList(rule, outcome_list, variables, rules)
287
288JS_TEST_PATHS = {
289  'debugger': [[]],
290  'inspector': [[]],
291  'intl': [[]],
292  'message': [[]],
293  'mjsunit': [[]],
294  'mozilla': [['data']],
295  'test262': [['data', 'test'], ['local-tests', 'test']],
296  'webkit': [[]],
297}
298
299def PresubmitCheck(path):
300  with open(path) as f:
301    contents = ReadContent(f.read())
302  basename = os.path.basename(os.path.dirname(path))
303  root_prefix = basename + "/"
304  status = {"success": True}
305  def _assert(check, message):  # Like "assert", but doesn't throw.
306    if not check:
307      print("%s: Error: %s" % (path, message))
308      status["success"] = False
309  try:
310    for section in contents:
311      _assert(type(section) == list, "Section must be a list")
312      _assert(len(section) == 2, "Section list must have exactly 2 entries")
313      section = section[1]
314      _assert(type(section) == dict,
315              "Second entry of section must be a dictionary")
316      for rule in section:
317        _assert(type(rule) == str, "Rule key must be a string")
318        _assert(not rule.startswith(root_prefix),
319                "Suite name prefix must not be used in rule keys")
320        _assert(not rule.endswith('.js'),
321                ".js extension must not be used in rule keys.")
322        _assert('*' not in rule or (rule.count('*') == 1 and rule[-1] == '*'),
323                "Only the last character of a rule key can be a wildcard")
324        if basename in JS_TEST_PATHS  and '*' not in rule:
325          _assert(any(os.path.exists(os.path.join(os.path.dirname(path),
326                                                  *(paths + [rule + ".js"])))
327                      for paths in JS_TEST_PATHS[basename]),
328                  "missing file for %s test %s" % (basename, rule))
329    return status["success"]
330  except Exception as e:
331    print e
332    return False
333