• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright 2009 the Sputnik authors.  All rights reserved.
3# This code is governed by the BSD license found in the LICENSE file.
4
5# This is derived from sputnik.py, the Sputnik console test runner,
6# with elements from packager.py, which is separately
7# copyrighted. TODO: Refactor so there is less duplication between
8# test262.py and packager.py.
9
10
11import logging
12import optparse
13import os
14from os import path
15import platform
16import re
17import subprocess
18import sys
19import tempfile
20import time
21import xml.dom.minidom
22import datetime
23import shutil
24import json
25import stat
26import xml.etree.ElementTree as xmlj
27import unicodedata
28from collections import Counter
29
30
31from parseTestRecord import parseTestRecord, stripHeader
32
33from _packagerConfig import *
34
35class Test262Error(Exception):
36  def __init__(self, message):
37    self.message = message
38
39def ReportError(s):
40  raise Test262Error(s)
41
42
43
44if not os.path.exists(EXCLUDED_FILENAME):
45    print "Cannot generate (JSON) test262 tests without a file," + \
46        " %s, showing which tests have been disabled!" % EXCLUDED_FILENAME
47    sys.exit(1)
48EXCLUDE_LIST = xml.dom.minidom.parse(EXCLUDED_FILENAME)
49EXCLUDE_REASON = EXCLUDE_LIST.getElementsByTagName("reason")
50EXCLUDE_LIST = EXCLUDE_LIST.getElementsByTagName("test")
51EXCLUDE_LIST = [x.getAttribute("id") for x in EXCLUDE_LIST]
52
53
54def BuildOptions():
55  result = optparse.OptionParser()
56  result.add_option("--command", default=None, help="The command-line to run")
57  result.add_option("--tests", default=path.abspath('.'),
58                    help="Path to the tests")
59  result.add_option("--cat", default=False, action="store_true",
60                    help="Print packaged test code that would be run")
61  result.add_option("--summary", default=False, action="store_true",
62                    help="Print summary after running tests")
63  result.add_option("--full-summary", default=False, action="store_true",
64                    help="Print summary and test output after running tests")
65  result.add_option("--strict_only", default=False, action="store_true",
66                    help="Test only strict mode")
67  result.add_option("--non_strict_only", default=False, action="store_true",
68                    help="Test only non-strict mode")
69  result.add_option("--unmarked_default", default="both",
70                    help="default mode for tests of unspecified strictness")
71  result.add_option("--logname", help="Filename to save stdout to")
72  result.add_option("--junitname", help="Filename to save test results in JUnit XML format")
73  result.add_option("--loglevel", default="warning",
74                    help="sets log level to debug, info, warning, error, or critical")
75  result.add_option("--print-handle", default="print", help="Command to print from console")
76  result.add_option("--list-includes", default=False, action="store_true",
77                    help="List includes required by tests")
78  return result
79
80
81def ValidateOptions(options):
82  if not options.command:
83    ReportError("A --command must be specified.")
84  if not path.exists(options.tests):
85    ReportError("Couldn't find test path '%s'" % options.tests)
86
87
88placeHolderPattern = re.compile(r"\{\{(\w+)\}\}")
89
90
91def IsWindows():
92  p = platform.system()
93  return (p == 'Windows') or (p == 'Microsoft')
94
95
96class TempFile(object):
97
98  def __init__(self, suffix="", prefix="tmp", text=False):
99    self.suffix = suffix
100    self.prefix = prefix
101    self.text = text
102    self.fd = None
103    self.name = None
104    self.is_closed = False
105    self.Open()
106
107  def Open(self):
108    (self.fd, self.name) = tempfile.mkstemp(
109        suffix = self.suffix,
110        prefix = self.prefix,
111        text = self.text)
112
113  def Write(self, str):
114    os.write(self.fd, str)
115
116  def Read(self):
117    f = file(self.name)
118    result = f.read()
119    f.close()
120    return result
121
122  def Close(self):
123    if not self.is_closed:
124      self.is_closed = True
125      os.close(self.fd)
126
127  def Dispose(self):
128    try:
129      self.Close()
130      os.unlink(self.name)
131    except OSError, e:
132      logging.error("Error disposing temp file: %s", str(e))
133
134
135class TestResult(object):
136
137  def __init__(self, exit_code, stdout, stderr, case):
138    self.exit_code = exit_code
139    self.stdout = stdout
140    self.stderr = stderr
141    self.case = case
142
143  def ReportOutcome(self, long_format):
144    name = self.case.GetName()
145    mode = self.case.GetMode()
146    if self.HasUnexpectedOutcome():
147      if self.case.IsNegative():
148        print "=== %s was expected to fail in %s, but didn't ===" % (name, mode)
149        print "--- expected error: %s ---\n" % self.case.GetNegativeType()
150      else:
151        if long_format:
152          print "=== %s failed in %s ===" % (name, mode)
153        else:
154          print "%s in %s: " % (name, mode)
155      self.WriteOutput(sys.stdout)
156      if long_format:
157        print "==="
158    elif self.case.IsNegative():
159      print "%s failed in %s as expected" % (name, mode)
160    else:
161      print "%s passed in %s" % (name, mode)
162
163  def WriteOutput(self, target):
164    out = self.stdout.strip()
165    if len(out) > 0:
166       target.write("--- output --- \n %s" % out)
167    err = self.stderr.strip()
168    if len(err) > 0:
169       target.write("--- errors ---  \n %s" % err)
170
171  # This is a way to make the output from the "whitespace" tests into valid XML
172  def SafeFormat(self, msg):
173    try:
174      msg = msg.encode(encoding='ascii', errors='strict')
175      msg = msg.replace('\u000Bx', '?')
176      msg = msg.replace('\u000Cx', '?')
177    except:
178      return 'Output contained invalid characters'
179
180  def XmlAssemble(self, result):
181    test_name = self.case.GetName()
182    test_mode = self.case.GetMode()
183    testCaseElement = xmlj.Element("testcase")
184    testpath = self.TestPathManipulation(test_name)
185    testCaseElement.attrib["classname"] = "%s.%s" % (testpath[0] , testpath[1])
186    testCaseElement.attrib["name"] = "%s %s" % (testpath[2].replace('.','_') , test_mode)
187    if self.HasUnexpectedOutcome():
188      failureElement = xmlj.Element("failure")
189      out = self.stdout.strip().decode('utf-8')
190      err = self.stderr.strip().decode('utf-8')
191      if len(out) > 0:
192        failureElement.text = self.SafeFormat(out)
193      if len(err) > 0:
194        failureElement.text = self.SafeFormat(err)
195      testCaseElement.append(failureElement)
196    return testCaseElement
197
198  def TestPathManipulation(self, test_name):
199    testdirlist = test_name.split('/')
200    testcase = testdirlist.pop()
201    testclass = testdirlist.pop()
202    testclass = testclass.replace('.','_')
203    if len(testdirlist) >= 1:
204       testpackage = testdirlist.pop(0)
205    else:
206       testpackage = testclass
207    return(testpackage,testclass,testcase)
208
209  def HasFailed(self):
210    return self.exit_code != 0
211
212  def AsyncHasFailed(self):
213    return 'Test262:AsyncTestComplete' not in self.stdout
214
215  def HasUnexpectedOutcome(self):
216    if self.case.IsAsyncTest():
217       return self.AsyncHasFailed() or self.HasFailed()
218    elif self.case.IsNegative():
219       return not (self.HasFailed() and self.case.NegativeMatch(self.GetErrorOutput()))
220    else:
221       return self.HasFailed()
222
223  def GetErrorOutput(self):
224    if len(self.stderr) != 0:
225      return self.stderr
226    return self.stdout
227
228
229class TestCase(object):
230
231  def __init__(self, suite, name, full_path, strict_mode):
232    self.suite = suite
233    self.name = name
234    self.full_path = full_path
235    self.strict_mode = strict_mode
236    f = open(self.full_path)
237    self.contents = f.read()
238    f.close()
239    testRecord = parseTestRecord(self.contents, name)
240    self.test = testRecord["test"]
241    del testRecord["test"]
242    del testRecord["header"]
243    testRecord.pop("commentary", None)    # do not throw if missing
244    self.testRecord = testRecord;
245
246    self.validate()
247
248  def NegativeMatch(self, stderr):
249    neg = re.compile(self.GetNegativeType())
250    return re.search(neg, stderr)
251
252  def GetNegative(self):
253    if not self.IsNegative():
254        return None
255    return self.testRecord["negative"]
256
257  def GetNegativeType(self):
258    negative = self.GetNegative()
259    return negative and negative["type"]
260
261  def GetNegativePhase(self):
262    negative = self.GetNegative()
263    return negative and negative["phase"]
264
265  def GetName(self):
266    return path.join(*self.name)
267
268  def GetMode(self):
269    if self.strict_mode:
270      return "strict mode"
271    else:
272      return "non-strict mode"
273
274  def GetPath(self):
275    return self.name
276
277  def IsNegative(self):
278    return 'negative' in self.testRecord
279
280  def IsOnlyStrict(self):
281    return 'onlyStrict' in self.testRecord
282
283  def IsNoStrict(self):
284    return 'noStrict' in self.testRecord or self.IsRaw()
285
286  def IsRaw(self):
287    return 'raw' in self.testRecord
288
289  def IsAsyncTest(self):
290    return 'async' in self.testRecord
291
292  def GetIncludeList(self):
293    if self.testRecord.get('includes'):
294      return self.testRecord['includes']
295    return []
296
297  def GetAdditionalIncludes(self):
298    return '\n'.join([self.suite.GetInclude(include) for include in self.GetIncludeList()])
299
300  def GetSource(self):
301    if self.IsRaw():
302        return self.test
303
304    source = self.suite.GetInclude("sta.js") + \
305        self.suite.GetInclude("cth.js") + \
306        self.suite.GetInclude("assert.js")
307
308    if self.IsAsyncTest():
309      source = source + \
310               self.suite.GetInclude("timer.js") + \
311               self.suite.GetInclude("doneprintHandle.js").replace('print', self.suite.print_handle)
312
313    source = source + \
314        self.GetAdditionalIncludes() + \
315        self.test + '\n'
316
317    if self.GetNegativePhase() == "early":
318        source = ("throw 'Expected an early error, but code was executed.';\n" +
319            source)
320
321    if self.strict_mode:
322      source = '"use strict";\nvar strict_mode = true;\n' + source
323    else:
324      # add comment line so line numbers match in both strict and non-strict version
325      source =  '//"no strict";\nvar strict_mode = false;\n' + source
326
327    return source
328
329  def InstantiateTemplate(self, template, params):
330    def GetParameter(match):
331      key = match.group(1)
332      return params.get(key, match.group(0))
333    return placeHolderPattern.sub(GetParameter, template)
334
335  def Execute(self, command):
336    if IsWindows():
337      args = '%s' % command
338    else:
339      args = command.split(" ")
340    stdout = TempFile(prefix="test262-out-")
341    stderr = TempFile(prefix="test262-err-")
342    try:
343      logging.info("exec: %s", str(args))
344      process = subprocess.Popen(
345        args,
346        shell = IsWindows(),
347        stdout = stdout.fd,
348        stderr = stderr.fd
349      )
350      code = process.wait()
351      out = stdout.Read()
352      err = stderr.Read()
353    finally:
354      stdout.Dispose()
355      stderr.Dispose()
356    return (code, out, err)
357
358  def RunTestIn(self, command_template, tmp):
359    tmp.Write(self.GetSource())
360    tmp.Close()
361    command = self.InstantiateTemplate(command_template, {
362      'path': tmp.name
363    })
364    (code, out, err) = self.Execute(command)
365    return TestResult(code, out, err, self)
366
367  def Run(self, command_template):
368    tmp = TempFile(suffix=".js", prefix="test262-", text=True)
369    try:
370      result = self.RunTestIn(command_template, tmp)
371    finally:
372      tmp.Dispose()
373    return result
374
375  def Print(self):
376    print self.GetSource()
377
378  def validate(self):
379    flags = self.testRecord.get("flags")
380    phase = self.GetNegativePhase()
381
382    if phase not in [None, "early", "runtime"]:
383        raise TypeError("Invalid value for negative phase: " + phase)
384
385    if not flags:
386        return
387
388    if 'raw' in flags:
389        if 'noStrict' in flags:
390            raise TypeError("The `raw` flag implies the `noStrict` flag")
391        elif 'onlyStrict' in flags:
392            raise TypeError(
393                "The `raw` flag is incompatible with the `onlyStrict` flag")
394        elif len(self.GetIncludeList()) > 0:
395            raise TypeError(
396                "The `raw` flag is incompatible with the `includes` tag")
397
398class ProgressIndicator(object):
399
400  def __init__(self, count):
401    self.count = count
402    self.succeeded = 0
403    self.failed = 0
404    self.failed_tests = []
405
406  def HasRun(self, result):
407    result.ReportOutcome(True)
408    if result.HasUnexpectedOutcome():
409      self.failed += 1
410      self.failed_tests.append(result)
411    else:
412      self.succeeded += 1
413
414
415def MakePlural(n):
416  if (n == 1):
417    return (n, "")
418  else:
419    return (n, "s")
420
421def PercentFormat(partial, total):
422  return "%i test%s (%.1f%%)" % (MakePlural(partial) +
423                                 ((100.0 * partial)/total,))
424
425
426class TestSuite(object):
427
428  def __init__(self, root, strict_only, non_strict_only, unmarked_default, print_handle):
429    # TODO: derive from packagerConfig.py
430    self.test_root = path.join(root, 'test')
431    self.lib_root = path.join(root, 'harness')
432    self.strict_only = strict_only
433    self.non_strict_only = non_strict_only
434    self.unmarked_default = unmarked_default
435    self.print_handle = print_handle
436    self.include_cache = { }
437
438
439  def Validate(self):
440    if not path.exists(self.test_root):
441      ReportError("No test repository found")
442    if not path.exists(self.lib_root):
443      ReportError("No test library found")
444
445  def IsHidden(self, path):
446    return path.startswith('.') or path == 'CVS'
447
448  def IsTestCase(self, path):
449    return path.endswith('.js')
450
451  def ShouldRun(self, rel_path, tests):
452    if len(tests) == 0:
453      return True
454    for test in tests:
455      if test in rel_path:
456        return True
457    return False
458
459  def GetInclude(self, name):
460    if not name in self.include_cache:
461      static = path.join(self.lib_root, name)
462      if path.exists(static):
463        f = open(static)
464        contents = stripHeader(f.read())
465        contents = re.sub(r'\r\n', '\n', contents)
466        self.include_cache[name] = contents + "\n"
467        f.close()
468      else:
469        ReportError("Can't find: " + static)
470    return self.include_cache[name]
471
472  def EnumerateTests(self, tests):
473    logging.info("Listing tests in %s", self.test_root)
474    cases = []
475    for root, dirs, files in os.walk(self.test_root):
476      for f in [x for x in dirs if self.IsHidden(x)]:
477        dirs.remove(f)
478      dirs.sort()
479      for f in sorted(files):
480        if self.IsTestCase(f):
481          full_path = path.join(root, f)
482          if full_path.startswith(self.test_root):
483            rel_path = full_path[len(self.test_root)+1:]
484          else:
485            logging.warning("Unexpected path %s", full_path)
486            rel_path = full_path
487          if self.ShouldRun(rel_path, tests):
488            basename = path.basename(full_path)[:-3]
489            name = rel_path.split(path.sep)[:-1] + [basename]
490            if EXCLUDE_LIST.count(basename) >= 1:
491              print 'Excluded: ' + basename
492            else:
493              if not self.non_strict_only:
494                strict_case = TestCase(self, name, full_path, True)
495                if not strict_case.IsNoStrict():
496                  if strict_case.IsOnlyStrict() or \
497                        self.unmarked_default in ['both', 'strict']:
498                    cases.append(strict_case)
499              if not self.strict_only:
500                non_strict_case = TestCase(self, name, full_path, False)
501                if not non_strict_case.IsOnlyStrict():
502                  if non_strict_case.IsNoStrict() or \
503                        self.unmarked_default in ['both', 'non_strict']:
504                    cases.append(non_strict_case)
505    logging.info("Done listing tests")
506    return cases
507
508
509  def PrintSummary(self, progress, logfile):
510
511    def write(s):
512      if logfile:
513        self.logf.write(s + "\n")
514      print s
515
516    print
517    write("=== Summary ===");
518    count = progress.count
519    succeeded = progress.succeeded
520    failed = progress.failed
521    write(" - Ran %i test%s" % MakePlural(count))
522    if progress.failed == 0:
523      write(" - All tests succeeded")
524    else:
525      write(" - Passed " + PercentFormat(succeeded, count))
526      write(" - Failed " + PercentFormat(failed, count))
527      positive = [c for c in progress.failed_tests if not c.case.IsNegative()]
528      negative = [c for c in progress.failed_tests if c.case.IsNegative()]
529      if len(positive) > 0:
530        print
531        write("Failed Tests")
532        for result in positive:
533          write("  %s in %s" % (result.case.GetName(), result.case.GetMode()))
534      if len(negative) > 0:
535        print
536        write("Expected to fail but passed ---")
537        for result in negative:
538          write("  %s in %s" % (result.case.GetName(), result.case.GetMode()))
539
540  def PrintFailureOutput(self, progress, logfile):
541    for result in progress.failed_tests:
542      if logfile:
543        self.WriteLog(result)
544      print
545      result.ReportOutcome(False)
546
547  def Run(self, command_template, tests, print_summary, full_summary, logname, junitfile):
548    if not "{{path}}" in command_template:
549      command_template += " {{path}}"
550    cases = self.EnumerateTests(tests)
551    if len(cases) == 0:
552      ReportError("No tests to run")
553    progress = ProgressIndicator(len(cases))
554    if logname:
555      self.logf = open(logname, "w")
556    if junitfile:
557      self.outfile = open(junitfile, "w")
558      TestSuitesElement = xmlj.Element("testsuites")
559      TestSuiteElement = xmlj.Element("testsuite")
560      TestSuitesElement.append(TestSuiteElement)
561      TestSuiteElement.attrib["name "] = "test262"
562      for x in range(len(EXCLUDE_LIST)):
563        if self.ShouldRun (unicode(EXCLUDE_LIST[x].encode('utf-8','ignore')), tests):
564          SkipCaseElement = xmlj.Element("testcase")
565          SkipCaseElement.attrib["classname"] = unicode(EXCLUDE_LIST[x]).encode('utf-8','ignore')
566          SkipCaseElement.attrib["name"] = unicode(EXCLUDE_LIST[x]).encode('utf-8','ignore')
567          SkipElement = xmlj.Element("skipped")
568          SkipElement.attrib["message"] = unicode(EXCLUDE_REASON[x].firstChild.nodeValue)
569          SkipCaseElement.append(SkipElement)
570          TestSuiteElement.append(SkipCaseElement)
571
572    for case in cases:
573      result = case.Run(command_template)
574      if junitfile:
575        TestCaseElement = result.XmlAssemble(result)
576        TestSuiteElement.append(TestCaseElement)
577        if case == cases[len(cases)-1]:
578             xmlj.ElementTree(TestSuitesElement).write(junitfile, "UTF-8")
579      if logname:
580        self.WriteLog(result)
581      progress.HasRun(result)
582
583    if print_summary:
584      self.PrintSummary(progress, logname)
585      if full_summary:
586        self.PrintFailureOutput(progress, logname)
587      else:
588        print
589        print "Use --full-summary to see output from failed tests"
590    print
591    return progress.failed
592
593  def WriteLog(self, result):
594    name = result.case.GetName()
595    mode = result.case.GetMode()
596    if result.HasUnexpectedOutcome():
597      if result.case.IsNegative():
598          self.logf.write("=== %s was expected to fail in %s, but didn't === \n" % (name, mode))
599          self.logf.write("--- expected error: %s ---\n" % result.case.GetNegativeType())
600          result.WriteOutput(self.logf)
601      else:
602          self.logf.write("=== %s failed in %s === \n" % (name, mode))
603          result.WriteOutput(self.logf)
604      self.logf.write("===\n")
605    elif result.case.IsNegative():
606       self.logf.write("%s failed in %s as expected \n" % (name, mode))
607    else:
608       self.logf.write("%s passed in %s \n" % (name, mode))
609
610  def Print(self, tests):
611    cases = self.EnumerateTests(tests)
612    if len(cases) > 0:
613      cases[0].Print()
614
615  def ListIncludes(self, tests):
616    cases = self.EnumerateTests(tests)
617    includes_dict = Counter()
618    for case in cases:
619      includes = case.GetIncludeList()
620      includes_dict.update(includes)
621
622    print includes_dict
623
624
625def Main():
626  code = 0
627  parser = BuildOptions()
628  (options, args) = parser.parse_args()
629  ValidateOptions(options)
630  test_suite = TestSuite(options.tests,
631                         options.strict_only,
632                         options.non_strict_only,
633                         options.unmarked_default,
634			 options.print_handle)
635  test_suite.Validate()
636  if options.loglevel == 'debug':
637    logging.basicConfig(level=logging.DEBUG)
638  elif options.loglevel == 'info':
639    logging.basicConfig(level=logging.INFO)
640  elif options.loglevel == 'warning':
641    logging.basicConfig(level=logging.WARNING)
642  elif options.loglevel == 'error':
643    logging.basicConfig(level=logging.ERROR)
644  elif options.loglevel == 'critical':
645    logging.basicConfig(level=logging.CRITICAL)
646  if options.cat:
647    test_suite.Print(args)
648  elif options.list_includes:
649    test_suite.ListIncludes(args)
650  else:
651    code = test_suite.Run(options.command, args,
652                          options.summary or options.full_summary,
653                          options.full_summary,
654                          options.logname,
655                          options.junitname)
656  return code
657
658if __name__ == '__main__':
659  try:
660    code = Main()
661    sys.exit(code)
662  except Test262Error, e:
663    print "Error: %s" % e.message
664    sys.exit(1)
665