• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2012 the V8 project authors. All rights reserved.
2# Redistribution and use in source and binary forms, with or without
3# modification, are permitted provided that the following conditions are
4# met:
5#
6#     * Redistributions of source code must retain the above copyright
7#       notice, this list of conditions and the following disclaimer.
8#     * Redistributions in binary form must reproduce the above
9#       copyright notice, this list of conditions and the following
10#       disclaimer in the documentation and/or other materials provided
11#       with the distribution.
12#     * Neither the name of Google Inc. nor the names of its
13#       contributors may be used to endorse or promote products derived
14#       from this software without specific prior written permission.
15#
16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
28import os
29import re
30
31from testrunner.local.variants import ALL_VARIANTS
32from testrunner.local.utils import Freeze
33
34# Possible outcomes
35FAIL = "FAIL"
36PASS = "PASS"
37TIMEOUT = "TIMEOUT"
38CRASH = "CRASH"
39
40# Outcomes only for status file, need special handling
41FAIL_OK = "FAIL_OK"
42FAIL_SLOPPY = "FAIL_SLOPPY"
43
44# Modifiers
45HEAVY = "HEAVY"
46SKIP = "SKIP"
47SLOW = "SLOW"
48NO_VARIANTS = "NO_VARIANTS"
49FAIL_PHASE_ONLY = "FAIL_PHASE_ONLY"
50
51ALWAYS = "ALWAYS"
52
53KEYWORDS = {}
54for key in [SKIP, FAIL, PASS, CRASH, HEAVY, SLOW, FAIL_OK, NO_VARIANTS,
55            FAIL_SLOPPY, ALWAYS, FAIL_PHASE_ONLY]:
56  KEYWORDS[key] = key
57
58# Support arches, modes to be written as keywords instead of strings.
59VARIABLES = {ALWAYS: True}
60for var in ["debug", "release", "big", "little", "android",
61            "arm", "arm64", "ia32", "mips", "mipsel", "mips64", "mips64el",
62            "x64", "ppc", "ppc64", "s390", "s390x", "macos", "windows",
63            "linux", "aix", "r1", "r2", "r3", "r5", "r6", "riscv64", "loong64"]:
64  VARIABLES[var] = var
65
66# Allow using variants as keywords.
67for var in ALL_VARIANTS:
68  VARIABLES[var] = var
69
70class StatusFile(object):
71  def __init__(self, path, variables):
72    """
73    _rules:        {variant: {test name: [rule]}}
74    _prefix_rules: {variant: {test name prefix: [rule]}}
75    """
76    self.variables = variables
77    with open(path) as f:
78      self._rules, self._prefix_rules = ReadStatusFile(f.read(), variables)
79
80  def get_outcomes(self, testname, variant=None):
81    """Merges variant dependent and independent rules."""
82    outcomes = frozenset()
83
84    for key in set([variant or '', '']):
85      rules = self._rules.get(key, {})
86      prefix_rules = self._prefix_rules.get(key, {})
87
88      if testname in rules:
89        outcomes |= rules[testname]
90
91      for prefix in prefix_rules:
92        if testname.startswith(prefix):
93          outcomes |= prefix_rules[prefix]
94
95    return outcomes
96
97  def warn_unused_rules(self, tests, check_variant_rules=False):
98    """Finds and prints unused rules in status file.
99
100    Rule X is unused when it doesn't apply to any tests, which can also mean
101    that all matching tests were skipped by another rule before evaluating X.
102
103    Args:
104      tests: list of pairs (testname, variant)
105      check_variant_rules: if set variant dependent rules are checked
106    """
107
108    if check_variant_rules:
109      variants = list(ALL_VARIANTS)
110    else:
111      variants = ['']
112    used_rules = set()
113
114    for testname, variant in tests:
115      variant = variant or ''
116
117      if testname in self._rules.get(variant, {}):
118        used_rules.add((testname, variant))
119        if SKIP in self._rules[variant][testname]:
120          continue
121
122      for prefix in self._prefix_rules.get(variant, {}):
123        if testname.startswith(prefix):
124          used_rules.add((prefix, variant))
125          if SKIP in self._prefix_rules[variant][prefix]:
126            break
127
128    for variant in variants:
129      for rule, value in (
130          list(self._rules.get(variant, {}).items()) +
131          list(self._prefix_rules.get(variant, {}).items())):
132        if (rule, variant) not in used_rules:
133          if variant == '':
134            variant_desc = 'variant independent'
135          else:
136            variant_desc = 'variant: %s' % variant
137          print('Unused rule: %s -> %s (%s)' % (rule, value, variant_desc))
138
139
140def _JoinsPassAndFail(outcomes1, outcomes2):
141  """Indicates if we join PASS and FAIL from two different outcome sets and
142  the first doesn't already contain both.
143  """
144  return (
145      PASS in outcomes1 and
146      not (FAIL in outcomes1 or FAIL_OK in outcomes1) and
147      (FAIL in outcomes2 or FAIL_OK in outcomes2)
148  )
149
150VARIANT_EXPRESSION = object()
151
152def _EvalExpression(exp, variables):
153  """Evaluates expression and returns its result. In case of NameError caused by
154  undefined "variant" identifier returns VARIANT_EXPRESSION marker.
155  """
156
157  try:
158    return eval(exp, variables)
159  except NameError as e:
160    identifier = re.match("name '(.*)' is not defined", str(e)).group(1)
161    assert identifier == "variant", "Unknown identifier: %s" % identifier
162    return VARIANT_EXPRESSION
163
164
165def _EvalVariantExpression(
166  condition, section, variables, variant, rules, prefix_rules):
167  variables_with_variant = dict(variables)
168  variables_with_variant["variant"] = variant
169  result = _EvalExpression(condition, variables_with_variant)
170  assert result != VARIANT_EXPRESSION
171  if result is True:
172    _ReadSection(
173        section,
174        variables_with_variant,
175        rules[variant],
176        prefix_rules[variant],
177    )
178  else:
179    assert result is False, "Make sure expressions evaluate to boolean values"
180
181
182def _ParseOutcomeList(rule, outcomes, variables, target_dict):
183  """Outcome list format: [condition, outcome, outcome, ...]"""
184
185  result = set([])
186  if type(outcomes) == str:
187    outcomes = [outcomes]
188  for item in outcomes:
189    if type(item) == str:
190      result.add(item)
191    elif type(item) == list:
192      condition = item[0]
193      exp = _EvalExpression(condition, variables)
194      assert exp != VARIANT_EXPRESSION, (
195        "Nested variant expressions are not supported")
196      if exp is False:
197        continue
198
199      # Ensure nobody uses an identifier by mistake, like "default",
200      # which would evaluate to true here otherwise.
201      assert exp is True, "Make sure expressions evaluate to boolean values"
202
203      for outcome in item[1:]:
204        assert type(outcome) == str
205        result.add(outcome)
206    else:
207      assert False
208  if len(result) == 0:
209    return
210  if rule in target_dict:
211    # A FAIL without PASS in one rule has always precedence over a single
212    # PASS (without FAIL) in another. Otherwise the default PASS expectation
213    # in a rule with a modifier (e.g. PASS, SLOW) would be joined to a FAIL
214    # from another rule (which intended to mark a test as FAIL and not as
215    # PASS and FAIL).
216    if _JoinsPassAndFail(target_dict[rule], result):
217      target_dict[rule] -= set([PASS])
218    if _JoinsPassAndFail(result, target_dict[rule]):
219      result -= set([PASS])
220    target_dict[rule] |= result
221  else:
222    target_dict[rule] = result
223
224
225def ReadContent(content):
226  return eval(content, KEYWORDS)
227
228
229def ReadStatusFile(content, variables):
230  """Status file format
231  Status file := [section]
232  section = [CONDITION, section_rules]
233  section_rules := {path: outcomes}
234  outcomes := outcome | [outcome, ...]
235  outcome := SINGLE_OUTCOME | [CONDITION, SINGLE_OUTCOME, SINGLE_OUTCOME, ...]
236  """
237
238  # Empty defaults for rules and prefix_rules. Variant-independent
239  # rules are mapped by "", others by the variant name.
240  rules = {variant: {} for variant in ALL_VARIANTS}
241  rules[""] = {}
242  prefix_rules = {variant: {} for variant in ALL_VARIANTS}
243  prefix_rules[""] = {}
244
245  variables.update(VARIABLES)
246  for conditional_section in ReadContent(content):
247    assert type(conditional_section) == list
248    assert len(conditional_section) == 2
249    condition, section = conditional_section
250    exp = _EvalExpression(condition, variables)
251
252    # The expression is variant-independent and evaluates to False.
253    if exp is False:
254      continue
255
256    # The expression is variant-independent and evaluates to True.
257    if exp is True:
258      _ReadSection(
259          section,
260          variables,
261          rules[''],
262          prefix_rules[''],
263      )
264      continue
265
266    # The expression is variant-dependent (contains "variant" keyword)
267    if exp == VARIANT_EXPRESSION:
268      # If the expression contains one or more "variant" keywords, we evaluate
269      # it for all possible variants and create rules for those that apply.
270      for variant in ALL_VARIANTS:
271        _EvalVariantExpression(
272            condition, section, variables, variant, rules, prefix_rules)
273      continue
274
275    assert False, "Make sure expressions evaluate to boolean values"
276
277  return Freeze(rules), Freeze(prefix_rules)
278
279
280def _ReadSection(section, variables, rules, prefix_rules):
281  assert type(section) == dict
282  for rule, outcome_list in list(section.items()):
283    assert type(rule) == str
284
285    if rule[-1] == '*':
286      _ParseOutcomeList(rule[:-1], outcome_list, variables, prefix_rules)
287    else:
288      _ParseOutcomeList(rule, outcome_list, variables, rules)
289
290JS_TEST_PATHS = {
291  'debugger': [[]],
292  'inspector': [[]],
293  'intl': [[]],
294  'message': [[]],
295  'mjsunit': [[]],
296  'mozilla': [['data']],
297  'test262': [['data', 'test'], ['local-tests', 'test']],
298  'webkit': [[]],
299}
300
301FILE_EXTENSIONS = [".js", ".mjs"]
302
303def PresubmitCheck(path):
304  with open(path) as f:
305    contents = ReadContent(f.read())
306  basename = os.path.basename(os.path.dirname(path))
307  root_prefix = basename + "/"
308  status = {"success": True}
309  def _assert(check, message):  # Like "assert", but doesn't throw.
310    if not check:
311      print("%s: Error: %s" % (path, message))
312      status["success"] = False
313  try:
314    for section in contents:
315      _assert(type(section) == list, "Section must be a list")
316      _assert(len(section) == 2, "Section list must have exactly 2 entries")
317      section = section[1]
318      _assert(type(section) == dict,
319              "Second entry of section must be a dictionary")
320      for rule in section:
321        _assert(type(rule) == str, "Rule key must be a string")
322        _assert(not rule.startswith(root_prefix),
323                "Suite name prefix must not be used in rule keys")
324        _assert(not rule.endswith('.js'),
325                ".js extension must not be used in rule keys.")
326        _assert('*' not in rule or (rule.count('*') == 1 and rule[-1] == '*'),
327                "Only the last character of a rule key can be a wildcard")
328        if basename in JS_TEST_PATHS  and '*' not in rule:
329          def _any_exist(paths):
330            return any(os.path.exists(os.path.join(os.path.dirname(path),
331                                      *(paths + [rule + ext])))
332                       for ext in FILE_EXTENSIONS)
333          _assert(any(_any_exist(paths)
334                      for paths in JS_TEST_PATHS[basename]),
335                  "missing file for %s test %s" % (basename, rule))
336    return status["success"]
337  except Exception as e:
338    print(e)
339    return False
340