• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2#
3# Copyright 2012 the V8 project authors. All rights reserved.
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9#       notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11#       copyright notice, this list of conditions and the following
12#       disclaimer in the documentation and/or other materials provided
13#       with the distribution.
14#     * Neither the name of Google Inc. nor the names of its
15#       contributors may be used to endorse or promote products derived
16#       from this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30
31import imp
32import optparse
33import os
34from os.path import join, dirname, abspath, basename, isdir, exists
35import platform
36import re
37import signal
38import subprocess
39import sys
40import tempfile
41import time
42import threading
43import utils
44from Queue import Queue, Empty
45
46
47VERBOSE = False
48
49
50# ---------------------------------------------
51# --- P r o g r e s s   I n d i c a t o r s ---
52# ---------------------------------------------
53
54
55class ProgressIndicator(object):
56
57  def __init__(self, cases):
58    self.cases = cases
59    self.queue = Queue(len(cases))
60    for case in cases:
61      self.queue.put_nowait(case)
62    self.succeeded = 0
63    self.remaining = len(cases)
64    self.total = len(cases)
65    self.failed = [ ]
66    self.crashed = 0
67    self.terminate = False
68    self.lock = threading.Lock()
69
70  def PrintFailureHeader(self, test):
71    if test.IsNegative():
72      negative_marker = '[negative] '
73    else:
74      negative_marker = ''
75    print "=== %(label)s %(negative)s===" % {
76      'label': test.GetLabel(),
77      'negative': negative_marker
78    }
79    print "Path: %s" % "/".join(test.path)
80
81  def Run(self, tasks):
82    self.Starting()
83    threads = []
84    # Spawn N-1 threads and then use this thread as the last one.
85    # That way -j1 avoids threading altogether which is a nice fallback
86    # in case of threading problems.
87    for i in xrange(tasks - 1):
88      thread = threading.Thread(target=self.RunSingle, args=[])
89      threads.append(thread)
90      thread.start()
91    try:
92      self.RunSingle()
93      # Wait for the remaining threads
94      for thread in threads:
95        # Use a timeout so that signals (ctrl-c) will be processed.
96        thread.join(timeout=10000000)
97    except Exception, e:
98      # If there's an exception we schedule an interruption for any
99      # remaining threads.
100      self.terminate = True
101      # ...and then reraise the exception to bail out
102      raise
103    self.Done()
104    return not self.failed
105
106  def RunSingle(self):
107    while not self.terminate:
108      try:
109        test = self.queue.get_nowait()
110      except Empty:
111        return
112      case = test.case
113      self.lock.acquire()
114      self.AboutToRun(case)
115      self.lock.release()
116      try:
117        start = time.time()
118        output = case.Run()
119        case.duration = (time.time() - start)
120      except BreakNowException:
121        self.terminate = True
122      except IOError, e:
123        assert self.terminate
124        return
125      if self.terminate:
126        return
127      self.lock.acquire()
128      if output.UnexpectedOutput():
129        self.failed.append(output)
130        if output.HasCrashed():
131          self.crashed += 1
132      else:
133        self.succeeded += 1
134      self.remaining -= 1
135      self.HasRun(output)
136      self.lock.release()
137
138
139def EscapeCommand(command):
140  parts = []
141  for part in command:
142    if ' ' in part:
143      # Escape spaces.  We may need to escape more characters for this
144      # to work properly.
145      parts.append('"%s"' % part)
146    else:
147      parts.append(part)
148  return " ".join(parts)
149
150
151class SimpleProgressIndicator(ProgressIndicator):
152
153  def Starting(self):
154    print 'Running %i tests' % len(self.cases)
155
156  def Done(self):
157    print
158    for failed in self.failed:
159      self.PrintFailureHeader(failed.test)
160      if failed.output.stderr:
161        print "--- stderr ---"
162        print failed.output.stderr.strip()
163      if failed.output.stdout:
164        print "--- stdout ---"
165        print failed.output.stdout.strip()
166      print "Command: %s" % EscapeCommand(failed.command)
167      if failed.HasCrashed():
168        print "--- CRASHED ---"
169      if failed.HasTimedOut():
170        print "--- TIMEOUT ---"
171    if len(self.failed) == 0:
172      print "==="
173      print "=== All tests succeeded"
174      print "==="
175    else:
176      print
177      print "==="
178      print "=== %i tests failed" % len(self.failed)
179      if self.crashed > 0:
180        print "=== %i tests CRASHED" % self.crashed
181      print "==="
182
183
184class VerboseProgressIndicator(SimpleProgressIndicator):
185
186  def AboutToRun(self, case):
187    print 'Starting %s...' % case.GetLabel()
188    sys.stdout.flush()
189
190  def HasRun(self, output):
191    if output.UnexpectedOutput():
192      if output.HasCrashed():
193        outcome = 'CRASH'
194      else:
195        outcome = 'FAIL'
196    else:
197      outcome = 'pass'
198    print 'Done running %s: %s' % (output.test.GetLabel(), outcome)
199
200
201class DotsProgressIndicator(SimpleProgressIndicator):
202
203  def AboutToRun(self, case):
204    pass
205
206  def HasRun(self, output):
207    total = self.succeeded + len(self.failed)
208    if (total > 1) and (total % 50 == 1):
209      sys.stdout.write('\n')
210    if output.UnexpectedOutput():
211      if output.HasCrashed():
212        sys.stdout.write('C')
213        sys.stdout.flush()
214      elif output.HasTimedOut():
215        sys.stdout.write('T')
216        sys.stdout.flush()
217      else:
218        sys.stdout.write('F')
219        sys.stdout.flush()
220    else:
221      sys.stdout.write('.')
222      sys.stdout.flush()
223
224
225class CompactProgressIndicator(ProgressIndicator):
226
227  def __init__(self, cases, templates):
228    super(CompactProgressIndicator, self).__init__(cases)
229    self.templates = templates
230    self.last_status_length = 0
231    self.start_time = time.time()
232
233  def Starting(self):
234    pass
235
236  def Done(self):
237    self.PrintProgress('Done')
238
239  def AboutToRun(self, case):
240    self.PrintProgress(case.GetLabel())
241
242  def HasRun(self, output):
243    if output.UnexpectedOutput():
244      self.ClearLine(self.last_status_length)
245      self.PrintFailureHeader(output.test)
246      stdout = output.output.stdout.strip()
247      if len(stdout):
248        print self.templates['stdout'] % stdout
249      stderr = output.output.stderr.strip()
250      if len(stderr):
251        print self.templates['stderr'] % stderr
252      print "Command: %s" % EscapeCommand(output.command)
253      if output.HasCrashed():
254        print "--- CRASHED ---"
255      if output.HasTimedOut():
256        print "--- TIMEOUT ---"
257
258  def Truncate(self, str, length):
259    if length and (len(str) > (length - 3)):
260      return str[:(length-3)] + "..."
261    else:
262      return str
263
264  def PrintProgress(self, name):
265    self.ClearLine(self.last_status_length)
266    elapsed = time.time() - self.start_time
267    status = self.templates['status_line'] % {
268      'passed': self.succeeded,
269      'remaining': (((self.total - self.remaining) * 100) // self.total),
270      'failed': len(self.failed),
271      'test': name,
272      'mins': int(elapsed) / 60,
273      'secs': int(elapsed) % 60
274    }
275    status = self.Truncate(status, 78)
276    self.last_status_length = len(status)
277    print status,
278    sys.stdout.flush()
279
280
281class ColorProgressIndicator(CompactProgressIndicator):
282
283  def __init__(self, cases):
284    templates = {
285      'status_line': "[%(mins)02i:%(secs)02i|\033[34m%%%(remaining) 4d\033[0m|\033[32m+%(passed) 4d\033[0m|\033[31m-%(failed) 4d\033[0m]: %(test)s",
286      'stdout': "\033[1m%s\033[0m",
287      'stderr': "\033[31m%s\033[0m",
288    }
289    super(ColorProgressIndicator, self).__init__(cases, templates)
290
291  def ClearLine(self, last_line_length):
292    print "\033[1K\r",
293
294
295class MonochromeProgressIndicator(CompactProgressIndicator):
296
297  def __init__(self, cases):
298    templates = {
299      'status_line': "[%(mins)02i:%(secs)02i|%%%(remaining) 4d|+%(passed) 4d|-%(failed) 4d]: %(test)s",
300      'stdout': '%s',
301      'stderr': '%s',
302      'clear': lambda last_line_length: ("\r" + (" " * last_line_length) + "\r"),
303      'max_length': 78
304    }
305    super(MonochromeProgressIndicator, self).__init__(cases, templates)
306
307  def ClearLine(self, last_line_length):
308    print ("\r" + (" " * last_line_length) + "\r"),
309
310
311PROGRESS_INDICATORS = {
312  'verbose': VerboseProgressIndicator,
313  'dots': DotsProgressIndicator,
314  'color': ColorProgressIndicator,
315  'mono': MonochromeProgressIndicator
316}
317
318
319# -------------------------
320# --- F r a m e w o r k ---
321# -------------------------
322
323class BreakNowException(Exception):
324  def __init__(self, value):
325    self.value = value
326  def __str__(self):
327    return repr(self.value)
328
329
330class CommandOutput(object):
331
332  def __init__(self, exit_code, timed_out, stdout, stderr):
333    self.exit_code = exit_code
334    self.timed_out = timed_out
335    self.stdout = stdout
336    self.stderr = stderr
337    self.failed = None
338
339
340class TestCase(object):
341
342  def __init__(self, context, path, mode):
343    self.path = path
344    self.context = context
345    self.duration = None
346    self.mode = mode
347
348  def IsNegative(self):
349    return False
350
351  def TestsIsolates(self):
352    return False
353
354  def CompareTime(self, other):
355    return cmp(other.duration, self.duration)
356
357  def DidFail(self, output):
358    if output.failed is None:
359      output.failed = self.IsFailureOutput(output)
360    return output.failed
361
362  def IsFailureOutput(self, output):
363    return output.exit_code != 0
364
365  def GetSource(self):
366    return "(no source available)"
367
368  def RunCommand(self, command):
369    full_command = self.context.processor(command)
370    output = Execute(full_command,
371                     self.context,
372                     self.context.GetTimeout(self, self.mode))
373    self.Cleanup()
374    return TestOutput(self,
375                      full_command,
376                      output,
377                      self.context.store_unexpected_output)
378
379  def BeforeRun(self):
380    pass
381
382  def AfterRun(self, result):
383    pass
384
385  def GetCustomFlags(self, mode):
386    return None
387
388  def Run(self):
389    self.BeforeRun()
390    result = None
391    try:
392      result = self.RunCommand(self.GetCommand())
393    except:
394      self.terminate = True
395      raise BreakNowException("User pressed CTRL+C or IO went wrong")
396    finally:
397      self.AfterRun(result)
398    return result
399
400  def Cleanup(self):
401    return
402
403
404class TestOutput(object):
405
406  def __init__(self, test, command, output, store_unexpected_output):
407    self.test = test
408    self.command = command
409    self.output = output
410    self.store_unexpected_output = store_unexpected_output
411
412  def UnexpectedOutput(self):
413    if self.HasCrashed():
414      outcome = CRASH
415    elif self.HasTimedOut():
416      outcome = TIMEOUT
417    elif self.HasFailed():
418      outcome = FAIL
419    else:
420      outcome = PASS
421    return not outcome in self.test.outcomes
422
423  def HasPreciousOutput(self):
424    return self.UnexpectedOutput() and self.store_unexpected_output
425
426  def HasCrashed(self):
427    if utils.IsWindows():
428      return 0x80000000 & self.output.exit_code and not (0x3FFFFF00 & self.output.exit_code)
429    else:
430      # Timed out tests will have exit_code -signal.SIGTERM.
431      if self.output.timed_out:
432        return False
433      return self.output.exit_code < 0 and \
434             self.output.exit_code != -signal.SIGABRT
435
436  def HasTimedOut(self):
437    return self.output.timed_out
438
439  def HasFailed(self):
440    execution_failed = self.test.DidFail(self.output)
441    if self.test.IsNegative():
442      return not execution_failed
443    else:
444      return execution_failed
445
446
447def KillProcessWithID(pid):
448  if utils.IsWindows():
449    os.popen('taskkill /T /F /PID %d' % pid)
450  else:
451    os.kill(pid, signal.SIGTERM)
452
453
454MAX_SLEEP_TIME = 0.1
455INITIAL_SLEEP_TIME = 0.0001
456SLEEP_TIME_FACTOR = 1.25
457
458SEM_INVALID_VALUE = -1
459SEM_NOGPFAULTERRORBOX = 0x0002 # Microsoft Platform SDK WinBase.h
460
461def Win32SetErrorMode(mode):
462  prev_error_mode = SEM_INVALID_VALUE
463  try:
464    import ctypes
465    prev_error_mode = ctypes.windll.kernel32.SetErrorMode(mode)
466  except ImportError:
467    pass
468  return prev_error_mode
469
470def RunProcess(context, timeout, args, **rest):
471  if context.verbose: print "#", " ".join(args)
472  popen_args = args
473  prev_error_mode = SEM_INVALID_VALUE
474  if utils.IsWindows():
475    popen_args = subprocess.list2cmdline(args)
476    if context.suppress_dialogs:
477      # Try to change the error mode to avoid dialogs on fatal errors. Don't
478      # touch any existing error mode flags by merging the existing error mode.
479      # See http://blogs.msdn.com/oldnewthing/archive/2004/07/27/198410.aspx.
480      error_mode = SEM_NOGPFAULTERRORBOX
481      prev_error_mode = Win32SetErrorMode(error_mode)
482      Win32SetErrorMode(error_mode | prev_error_mode)
483  process = subprocess.Popen(
484    shell = utils.IsWindows(),
485    args = popen_args,
486    **rest
487  )
488  if utils.IsWindows() and context.suppress_dialogs and prev_error_mode != SEM_INVALID_VALUE:
489    Win32SetErrorMode(prev_error_mode)
490  # Compute the end time - if the process crosses this limit we
491  # consider it timed out.
492  if timeout is None: end_time = None
493  else: end_time = time.time() + timeout
494  timed_out = False
495  # Repeatedly check the exit code from the process in a
496  # loop and keep track of whether or not it times out.
497  exit_code = None
498  sleep_time = INITIAL_SLEEP_TIME
499  while exit_code is None:
500    if (not end_time is None) and (time.time() >= end_time):
501      # Kill the process and wait for it to exit.
502      KillProcessWithID(process.pid)
503      exit_code = process.wait()
504      timed_out = True
505    else:
506      exit_code = process.poll()
507      time.sleep(sleep_time)
508      sleep_time = sleep_time * SLEEP_TIME_FACTOR
509      if sleep_time > MAX_SLEEP_TIME:
510        sleep_time = MAX_SLEEP_TIME
511  return (process, exit_code, timed_out)
512
513
514def PrintError(str):
515  sys.stderr.write(str)
516  sys.stderr.write('\n')
517
518
519def CheckedUnlink(name):
520  # On Windows, when run with -jN in parallel processes,
521  # OS often fails to unlink the temp file. Not sure why.
522  # Need to retry.
523  # Idea from https://bugs.webkit.org/attachment.cgi?id=75982&action=prettypatch
524  retry_count = 0
525  while retry_count < 30:
526    try:
527      os.unlink(name)
528      return
529    except OSError, e:
530      retry_count += 1
531      time.sleep(retry_count * 0.1)
532  PrintError("os.unlink() " + str(e))
533
534def Execute(args, context, timeout=None):
535  (fd_out, outname) = tempfile.mkstemp()
536  (fd_err, errname) = tempfile.mkstemp()
537  (process, exit_code, timed_out) = RunProcess(
538    context,
539    timeout,
540    args = args,
541    stdout = fd_out,
542    stderr = fd_err,
543  )
544  os.close(fd_out)
545  os.close(fd_err)
546  output = file(outname).read()
547  errors = file(errname).read()
548  CheckedUnlink(outname)
549  CheckedUnlink(errname)
550  return CommandOutput(exit_code, timed_out, output, errors)
551
552
553def ExecuteNoCapture(args, context, timeout=None):
554  (process, exit_code, timed_out) = RunProcess(
555    context,
556    timeout,
557    args = args,
558  )
559  return CommandOutput(exit_code, False, "", "")
560
561
562def CarCdr(path):
563  if len(path) == 0:
564    return (None, [ ])
565  else:
566    return (path[0], path[1:])
567
568
569# Use this to run several variants of the tests, e.g.:
570# VARIANT_FLAGS = [[], ['--always_compact', '--noflush_code']]
571VARIANT_FLAGS = [[],
572                 ['--stress-opt', '--always-opt'],
573                 ['--nocrankshaft']]
574
575
576class TestConfiguration(object):
577
578  def __init__(self, context, root):
579    self.context = context
580    self.root = root
581
582  def Contains(self, path, file):
583    if len(path) > len(file):
584      return False
585    for i in xrange(len(path)):
586      if not path[i].match(file[i]):
587        return False
588    return True
589
590  def GetTestStatus(self, sections, defs):
591    pass
592
593  def VariantFlags(self):
594    return VARIANT_FLAGS
595
596
597
598
599class TestSuite(object):
600
601  def __init__(self, name):
602    self.name = name
603
604  def GetName(self):
605    return self.name
606
607
608class TestRepository(TestSuite):
609
610  def __init__(self, path):
611    normalized_path = abspath(path)
612    super(TestRepository, self).__init__(basename(normalized_path))
613    self.path = normalized_path
614    self.is_loaded = False
615    self.config = None
616
617  def GetConfiguration(self, context):
618    if self.is_loaded:
619      return self.config
620    self.is_loaded = True
621    file = None
622    try:
623      (file, pathname, description) = imp.find_module('testcfg', [ self.path ])
624      module = imp.load_module('testcfg', file, pathname, description)
625      self.config = module.GetConfiguration(context, self.path)
626    finally:
627      if file:
628        file.close()
629    return self.config
630
631  def GetBuildRequirements(self, path, context):
632    return self.GetConfiguration(context).GetBuildRequirements()
633
634  def DownloadData(self, context):
635    config = self.GetConfiguration(context)
636    if 'DownloadData' in dir(config):
637      config.DownloadData()
638
639  def AddTestsToList(self, result, current_path, path, context, mode):
640    config = self.GetConfiguration(context)
641    for v in config.VariantFlags():
642      tests = config.ListTests(current_path, path, mode, v)
643      for t in tests: t.variant_flags = v
644      result += tests
645
646  def GetTestStatus(self, context, sections, defs):
647    self.GetConfiguration(context).GetTestStatus(sections, defs)
648
649
650class LiteralTestSuite(TestSuite):
651
652  def __init__(self, tests):
653    super(LiteralTestSuite, self).__init__('root')
654    self.tests = tests
655
656  def GetBuildRequirements(self, path, context):
657    (name, rest) = CarCdr(path)
658    result = [ ]
659    for test in self.tests:
660      if not name or name.match(test.GetName()):
661        result += test.GetBuildRequirements(rest, context)
662    return result
663
664  def DownloadData(self, path, context):
665    (name, rest) = CarCdr(path)
666    for test in self.tests:
667      if not name or name.match(test.GetName()):
668        test.DownloadData(context)
669
670  def ListTests(self, current_path, path, context, mode, variant_flags):
671    (name, rest) = CarCdr(path)
672    result = [ ]
673    for test in self.tests:
674      test_name = test.GetName()
675      if not name or name.match(test_name):
676        full_path = current_path + [test_name]
677        test.AddTestsToList(result, full_path, path, context, mode)
678    return result
679
680  def GetTestStatus(self, context, sections, defs):
681    for test in self.tests:
682      test.GetTestStatus(context, sections, defs)
683
684
685SUFFIX = {
686    'debug'   : '_g',
687    'release' : '' }
688FLAGS = {
689    'debug'   : ['--nobreak-on-abort', '--enable-slow-asserts', '--debug-code', '--verify-heap'],
690    'release' : ['--nobreak-on-abort']}
691TIMEOUT_SCALEFACTOR = {
692    'debug'   : 4,
693    'release' : 1 }
694
695
696class Context(object):
697
698  def __init__(self, workspace, buildspace, verbose, vm, timeout, processor, suppress_dialogs, store_unexpected_output):
699    self.workspace = workspace
700    self.buildspace = buildspace
701    self.verbose = verbose
702    self.vm_root = vm
703    self.timeout = timeout
704    self.processor = processor
705    self.suppress_dialogs = suppress_dialogs
706    self.store_unexpected_output = store_unexpected_output
707
708  def GetVm(self, mode):
709    name = self.vm_root + SUFFIX[mode]
710    if utils.IsWindows() and not name.endswith('.exe'):
711      name = name + '.exe'
712    return name
713
714  def GetVmCommand(self, testcase, mode):
715    return [self.GetVm(mode)] + self.GetVmFlags(testcase, mode)
716
717  def GetVmFlags(self, testcase, mode):
718    flags = testcase.GetCustomFlags(mode)
719    if flags is None:
720      flags = FLAGS[mode]
721    return testcase.variant_flags + flags
722
723  def GetTimeout(self, testcase, mode):
724    result = self.timeout * TIMEOUT_SCALEFACTOR[mode]
725    if '--stress-opt' in self.GetVmFlags(testcase, mode):
726      return result * 4
727    else:
728      return result
729
730def RunTestCases(cases_to_run, progress, tasks):
731  progress = PROGRESS_INDICATORS[progress](cases_to_run)
732  result = 0
733  try:
734    result = progress.Run(tasks)
735  except Exception, e:
736    print "\n", e
737  return result
738
739
740def BuildRequirements(context, requirements, mode, scons_flags):
741  command_line = (['scons', '-Y', context.workspace, 'mode=' + ",".join(mode)]
742                  + requirements
743                  + scons_flags)
744  output = ExecuteNoCapture(command_line, context)
745  return output.exit_code == 0
746
747
748# -------------------------------------------
749# --- T e s t   C o n f i g u r a t i o n ---
750# -------------------------------------------
751
752
753SKIP = 'skip'
754FAIL = 'fail'
755PASS = 'pass'
756OKAY = 'okay'
757TIMEOUT = 'timeout'
758CRASH = 'crash'
759SLOW = 'slow'
760
761
762class Expression(object):
763  pass
764
765
766class Constant(Expression):
767
768  def __init__(self, value):
769    self.value = value
770
771  def Evaluate(self, env, defs):
772    return self.value
773
774
775class Variable(Expression):
776
777  def __init__(self, name):
778    self.name = name
779
780  def GetOutcomes(self, env, defs):
781    if self.name in env: return ListSet([env[self.name]])
782    else: return Nothing()
783
784  def Evaluate(self, env, defs):
785    return env[self.name]
786
787
788class Outcome(Expression):
789
790  def __init__(self, name):
791    self.name = name
792
793  def GetOutcomes(self, env, defs):
794    if self.name in defs:
795      return defs[self.name].GetOutcomes(env, defs)
796    else:
797      return ListSet([self.name])
798
799
800class Set(object):
801  pass
802
803
804class ListSet(Set):
805
806  def __init__(self, elms):
807    self.elms = elms
808
809  def __str__(self):
810    return "ListSet%s" % str(self.elms)
811
812  def Intersect(self, that):
813    if not isinstance(that, ListSet):
814      return that.Intersect(self)
815    return ListSet([ x for x in self.elms if x in that.elms ])
816
817  def Union(self, that):
818    if not isinstance(that, ListSet):
819      return that.Union(self)
820    return ListSet(self.elms + [ x for x in that.elms if x not in self.elms ])
821
822  def IsEmpty(self):
823    return len(self.elms) == 0
824
825
826class Everything(Set):
827
828  def Intersect(self, that):
829    return that
830
831  def Union(self, that):
832    return self
833
834  def IsEmpty(self):
835    return False
836
837
838class Nothing(Set):
839
840  def Intersect(self, that):
841    return self
842
843  def Union(self, that):
844    return that
845
846  def IsEmpty(self):
847    return True
848
849
850class Operation(Expression):
851
852  def __init__(self, left, op, right):
853    self.left = left
854    self.op = op
855    self.right = right
856
857  def Evaluate(self, env, defs):
858    if self.op == '||' or self.op == ',':
859      return self.left.Evaluate(env, defs) or self.right.Evaluate(env, defs)
860    elif self.op == 'if':
861      return False
862    elif self.op == '==':
863      inter = self.left.GetOutcomes(env, defs).Intersect(self.right.GetOutcomes(env, defs))
864      return not inter.IsEmpty()
865    elif self.op == '!=':
866      inter = self.left.GetOutcomes(env, defs).Intersect(self.right.GetOutcomes(env, defs))
867      return inter.IsEmpty()
868    else:
869      assert self.op == '&&'
870      return self.left.Evaluate(env, defs) and self.right.Evaluate(env, defs)
871
872  def GetOutcomes(self, env, defs):
873    if self.op == '||' or self.op == ',':
874      return self.left.GetOutcomes(env, defs).Union(self.right.GetOutcomes(env, defs))
875    elif self.op == 'if':
876      if self.right.Evaluate(env, defs): return self.left.GetOutcomes(env, defs)
877      else: return Nothing()
878    else:
879      assert self.op == '&&'
880      return self.left.GetOutcomes(env, defs).Intersect(self.right.GetOutcomes(env, defs))
881
882
883def IsAlpha(str):
884  for char in str:
885    if not (char.isalpha() or char.isdigit() or char == '_'):
886      return False
887  return True
888
889
890class Tokenizer(object):
891  """A simple string tokenizer that chops expressions into variables,
892  parens and operators"""
893
894  def __init__(self, expr):
895    self.index = 0
896    self.expr = expr
897    self.length = len(expr)
898    self.tokens = None
899
900  def Current(self, length = 1):
901    if not self.HasMore(length): return ""
902    return self.expr[self.index:self.index+length]
903
904  def HasMore(self, length = 1):
905    return self.index < self.length + (length - 1)
906
907  def Advance(self, count = 1):
908    self.index = self.index + count
909
910  def AddToken(self, token):
911    self.tokens.append(token)
912
913  def SkipSpaces(self):
914    while self.HasMore() and self.Current().isspace():
915      self.Advance()
916
917  def Tokenize(self):
918    self.tokens = [ ]
919    while self.HasMore():
920      self.SkipSpaces()
921      if not self.HasMore():
922        return None
923      if self.Current() == '(':
924        self.AddToken('(')
925        self.Advance()
926      elif self.Current() == ')':
927        self.AddToken(')')
928        self.Advance()
929      elif self.Current() == '$':
930        self.AddToken('$')
931        self.Advance()
932      elif self.Current() == ',':
933        self.AddToken(',')
934        self.Advance()
935      elif IsAlpha(self.Current()):
936        buf = ""
937        while self.HasMore() and IsAlpha(self.Current()):
938          buf += self.Current()
939          self.Advance()
940        self.AddToken(buf)
941      elif self.Current(2) == '&&':
942        self.AddToken('&&')
943        self.Advance(2)
944      elif self.Current(2) == '||':
945        self.AddToken('||')
946        self.Advance(2)
947      elif self.Current(2) == '==':
948        self.AddToken('==')
949        self.Advance(2)
950      elif self.Current(2) == '!=':
951        self.AddToken('!=')
952        self.Advance(2)
953      else:
954        return None
955    return self.tokens
956
957
958class Scanner(object):
959  """A simple scanner that can serve out tokens from a given list"""
960
961  def __init__(self, tokens):
962    self.tokens = tokens
963    self.length = len(tokens)
964    self.index = 0
965
966  def HasMore(self):
967    return self.index < self.length
968
969  def Current(self):
970    return self.tokens[self.index]
971
972  def Advance(self):
973    self.index = self.index + 1
974
975
976def ParseAtomicExpression(scan):
977  if scan.Current() == "true":
978    scan.Advance()
979    return Constant(True)
980  elif scan.Current() == "false":
981    scan.Advance()
982    return Constant(False)
983  elif IsAlpha(scan.Current()):
984    name = scan.Current()
985    scan.Advance()
986    return Outcome(name.lower())
987  elif scan.Current() == '$':
988    scan.Advance()
989    if not IsAlpha(scan.Current()):
990      return None
991    name = scan.Current()
992    scan.Advance()
993    return Variable(name.lower())
994  elif scan.Current() == '(':
995    scan.Advance()
996    result = ParseLogicalExpression(scan)
997    if (not result) or (scan.Current() != ')'):
998      return None
999    scan.Advance()
1000    return result
1001  else:
1002    return None
1003
1004
1005BINARIES = ['==', '!=']
1006def ParseOperatorExpression(scan):
1007  left = ParseAtomicExpression(scan)
1008  if not left: return None
1009  while scan.HasMore() and (scan.Current() in BINARIES):
1010    op = scan.Current()
1011    scan.Advance()
1012    right = ParseOperatorExpression(scan)
1013    if not right:
1014      return None
1015    left = Operation(left, op, right)
1016  return left
1017
1018
1019def ParseConditionalExpression(scan):
1020  left = ParseOperatorExpression(scan)
1021  if not left: return None
1022  while scan.HasMore() and (scan.Current() == 'if'):
1023    scan.Advance()
1024    right = ParseOperatorExpression(scan)
1025    if not right:
1026      return None
1027    left = Operation(left, 'if', right)
1028  return left
1029
1030
1031LOGICALS = ["&&", "||", ","]
1032def ParseLogicalExpression(scan):
1033  left = ParseConditionalExpression(scan)
1034  if not left: return None
1035  while scan.HasMore() and (scan.Current() in LOGICALS):
1036    op = scan.Current()
1037    scan.Advance()
1038    right = ParseConditionalExpression(scan)
1039    if not right:
1040      return None
1041    left = Operation(left, op, right)
1042  return left
1043
1044
1045def ParseCondition(expr):
1046  """Parses a logical expression into an Expression object"""
1047  tokens = Tokenizer(expr).Tokenize()
1048  if not tokens:
1049    print "Malformed expression: '%s'" % expr
1050    return None
1051  scan = Scanner(tokens)
1052  ast = ParseLogicalExpression(scan)
1053  if not ast:
1054    print "Malformed expression: '%s'" % expr
1055    return None
1056  if scan.HasMore():
1057    print "Malformed expression: '%s'" % expr
1058    return None
1059  return ast
1060
1061
1062class ClassifiedTest(object):
1063
1064  def __init__(self, case, outcomes):
1065    self.case = case
1066    self.outcomes = outcomes
1067
1068  def TestsIsolates(self):
1069    return self.case.TestsIsolates()
1070
1071
1072class Configuration(object):
1073  """The parsed contents of a configuration file"""
1074
1075  def __init__(self, sections, defs):
1076    self.sections = sections
1077    self.defs = defs
1078
1079  def ClassifyTests(self, cases, env):
1080    sections = [s for s in self.sections if s.condition.Evaluate(env, self.defs)]
1081    all_rules = reduce(list.__add__, [s.rules for s in sections], [])
1082    unused_rules = set(all_rules)
1083    result = [ ]
1084    all_outcomes = set([])
1085    for case in cases:
1086      matches = [ r for r in all_rules if r.Contains(case.path) ]
1087      outcomes = set([])
1088      for rule in matches:
1089        outcomes = outcomes.union(rule.GetOutcomes(env, self.defs))
1090        unused_rules.discard(rule)
1091      if not outcomes:
1092        outcomes = [PASS]
1093      case.outcomes = outcomes
1094      all_outcomes = all_outcomes.union(outcomes)
1095      result.append(ClassifiedTest(case, outcomes))
1096    return (result, list(unused_rules), all_outcomes)
1097
1098
1099class Section(object):
1100  """A section of the configuration file.  Sections are enabled or
1101  disabled prior to running the tests, based on their conditions"""
1102
1103  def __init__(self, condition):
1104    self.condition = condition
1105    self.rules = [ ]
1106
1107  def AddRule(self, rule):
1108    self.rules.append(rule)
1109
1110
1111class Rule(object):
1112  """A single rule that specifies the expected outcome for a single
1113  test."""
1114
1115  def __init__(self, raw_path, path, value):
1116    self.raw_path = raw_path
1117    self.path = path
1118    self.value = value
1119
1120  def GetOutcomes(self, env, defs):
1121    set = self.value.GetOutcomes(env, defs)
1122    assert isinstance(set, ListSet)
1123    return set.elms
1124
1125  def Contains(self, path):
1126    if len(self.path) > len(path):
1127      return False
1128    for i in xrange(len(self.path)):
1129      if not self.path[i].match(path[i]):
1130        return False
1131    return True
1132
1133
1134HEADER_PATTERN = re.compile(r'\[([^]]+)\]')
1135RULE_PATTERN = re.compile(r'\s*([^: ]*)\s*:(.*)')
1136DEF_PATTERN = re.compile(r'^def\s*(\w+)\s*=(.*)$')
1137PREFIX_PATTERN = re.compile(r'^\s*prefix\s+([\w\_\.\-\/]+)$')
1138
1139
1140def ReadConfigurationInto(path, sections, defs):
1141  current_section = Section(Constant(True))
1142  sections.append(current_section)
1143  prefix = []
1144  for line in utils.ReadLinesFrom(path):
1145    header_match = HEADER_PATTERN.match(line)
1146    if header_match:
1147      condition_str = header_match.group(1).strip()
1148      condition = ParseCondition(condition_str)
1149      new_section = Section(condition)
1150      sections.append(new_section)
1151      current_section = new_section
1152      continue
1153    rule_match = RULE_PATTERN.match(line)
1154    if rule_match:
1155      path = prefix + SplitPath(rule_match.group(1).strip())
1156      value_str = rule_match.group(2).strip()
1157      value = ParseCondition(value_str)
1158      if not value:
1159        return False
1160      current_section.AddRule(Rule(rule_match.group(1), path, value))
1161      continue
1162    def_match = DEF_PATTERN.match(line)
1163    if def_match:
1164      name = def_match.group(1).lower()
1165      value = ParseCondition(def_match.group(2).strip())
1166      if not value:
1167        return False
1168      defs[name] = value
1169      continue
1170    prefix_match = PREFIX_PATTERN.match(line)
1171    if prefix_match:
1172      prefix = SplitPath(prefix_match.group(1).strip())
1173      continue
1174    print "Malformed line: '%s'." % line
1175    return False
1176  return True
1177
1178
1179# ---------------
1180# --- M a i n ---
1181# ---------------
1182
1183
1184ARCH_GUESS = utils.GuessArchitecture()
1185TIMEOUT_DEFAULT = 60;
1186
1187
1188def BuildOptions():
1189  result = optparse.OptionParser()
1190  result.add_option("-m", "--mode", help="The test modes in which to run (comma-separated)",
1191      default='release')
1192  result.add_option("-v", "--verbose", help="Verbose output",
1193      default=False, action="store_true")
1194  result.add_option("-S", dest="scons_flags", help="Flag to pass through to scons",
1195      default=[], action="append")
1196  result.add_option("-p", "--progress",
1197      help="The style of progress indicator (verbose, dots, color, mono)",
1198      choices=PROGRESS_INDICATORS.keys(), default="mono")
1199  result.add_option("--no-build", help="Don't build requirements",
1200      default=False, action="store_true")
1201  result.add_option("--build-only", help="Only build requirements, don't run the tests",
1202      default=False, action="store_true")
1203  result.add_option("--build-system", help="Build system in use (scons or gyp)",
1204      default='scons')
1205  result.add_option("--report", help="Print a summary of the tests to be run",
1206      default=False, action="store_true")
1207  result.add_option("--download-data", help="Download missing test suite data",
1208      default=False, action="store_true")
1209  result.add_option("-s", "--suite", help="A test suite",
1210      default=[], action="append")
1211  result.add_option("-t", "--timeout", help="Timeout in seconds",
1212      default=-1, type="int")
1213  result.add_option("--arch", help='The architecture to run tests for',
1214      default='none')
1215  result.add_option("--snapshot", help="Run the tests with snapshot turned on",
1216      default=False, action="store_true")
1217  result.add_option("--simulator", help="Run tests with architecture simulator",
1218      default='none')
1219  result.add_option("--special-command", default=None)
1220  result.add_option("--valgrind", help="Run tests through valgrind",
1221      default=False, action="store_true")
1222  result.add_option("--cat", help="Print the source of the tests",
1223      default=False, action="store_true")
1224  result.add_option("--warn-unused", help="Report unused rules",
1225      default=False, action="store_true")
1226  result.add_option("-j", help="The number of parallel tasks to run",
1227      default=1, type="int")
1228  result.add_option("--time", help="Print timing information after running",
1229      default=False, action="store_true")
1230  result.add_option("--suppress-dialogs", help="Suppress Windows dialogs for crashing tests",
1231        dest="suppress_dialogs", default=True, action="store_true")
1232  result.add_option("--no-suppress-dialogs", help="Display Windows dialogs for crashing tests",
1233        dest="suppress_dialogs", action="store_false")
1234  result.add_option("--mips-arch-variant", help="mips architecture variant: mips32r1/mips32r2", default="mips32r2");
1235  result.add_option("--shell", help="Path to V8 shell", default="d8")
1236  result.add_option("--isolates", help="Whether to test isolates", default=False, action="store_true")
1237  result.add_option("--store-unexpected-output",
1238      help="Store the temporary JS files from tests that fails",
1239      dest="store_unexpected_output", default=True, action="store_true")
1240  result.add_option("--no-store-unexpected-output",
1241      help="Deletes the temporary JS files from tests that fails",
1242      dest="store_unexpected_output", action="store_false")
1243  result.add_option("--stress-only",
1244                    help="Only run tests with --always-opt --stress-opt",
1245                    default=False, action="store_true")
1246  result.add_option("--nostress",
1247                    help="Don't run crankshaft --always-opt --stress-op test",
1248                    default=False, action="store_true")
1249  result.add_option("--crankshaft",
1250                    help="Run with the --crankshaft flag",
1251                    default=False, action="store_true")
1252  result.add_option("--shard-count",
1253                    help="Split testsuites into this number of shards",
1254                    default=1, type="int")
1255  result.add_option("--shard-run",
1256                    help="Run this shard from the split up tests.",
1257                    default=1, type="int")
1258  result.add_option("--noprof", help="Disable profiling support",
1259                    default=False)
1260  return result
1261
1262
1263def ProcessOptions(options):
1264  global VERBOSE
1265  VERBOSE = options.verbose
1266  options.mode = options.mode.split(',')
1267  for mode in options.mode:
1268    if not mode in ['debug', 'release']:
1269      print "Unknown mode %s" % mode
1270      return False
1271  if options.simulator != 'none':
1272    # Simulator argument was set. Make sure arch and simulator agree.
1273    if options.simulator != options.arch:
1274      if options.arch == 'none':
1275        options.arch = options.simulator
1276      else:
1277        print "Architecture %s does not match sim %s" %(options.arch, options.simulator)
1278        return False
1279    # Ensure that the simulator argument is handed down to scons.
1280    options.scons_flags.append("simulator=" + options.simulator)
1281  else:
1282    # If options.arch is not set by the command line and no simulator setting
1283    # was found, set the arch to the guess.
1284    if options.arch == 'none':
1285      options.arch = ARCH_GUESS
1286    options.scons_flags.append("arch=" + options.arch)
1287  # Simulators are slow, therefore allow a longer default timeout.
1288  if options.timeout == -1:
1289    if options.arch == 'arm' or options.arch == 'mips':
1290      options.timeout = 2 * TIMEOUT_DEFAULT;
1291    else:
1292      options.timeout = TIMEOUT_DEFAULT;
1293  if options.snapshot:
1294    options.scons_flags.append("snapshot=on")
1295  global VARIANT_FLAGS
1296  if options.mips_arch_variant:
1297    options.scons_flags.append("mips_arch_variant=" + options.mips_arch_variant)
1298
1299  if options.stress_only:
1300    VARIANT_FLAGS = [['--stress-opt', '--always-opt']]
1301  if options.nostress:
1302    VARIANT_FLAGS = [[],['--nocrankshaft']]
1303  if options.crankshaft:
1304    if options.special_command:
1305      options.special_command += " --crankshaft"
1306    else:
1307      options.special_command = "@ --crankshaft"
1308  if options.shell.endswith("d8"):
1309    if options.special_command:
1310      options.special_command += " --test"
1311    else:
1312      options.special_command = "@ --test"
1313  if options.noprof:
1314    options.scons_flags.append("prof=off")
1315    options.scons_flags.append("profilingsupport=off")
1316  if options.build_system == 'gyp':
1317    if options.build_only:
1318      print "--build-only not supported for gyp, please build manually."
1319      options.build_only = False
1320  return True
1321
1322
1323def DoSkip(case):
1324  return (SKIP in case.outcomes) or (SLOW in case.outcomes)
1325
1326
1327REPORT_TEMPLATE = """\
1328Total: %(total)i tests
1329 * %(skipped)4d tests will be skipped
1330 * %(timeout)4d tests are expected to timeout sometimes
1331 * %(nocrash)4d tests are expected to be flaky but not crash
1332 * %(pass)4d tests are expected to pass
1333 * %(fail_ok)4d tests are expected to fail that we won't fix
1334 * %(fail)4d tests are expected to fail that we should fix\
1335"""
1336
1337def PrintReport(cases):
1338  def IsFlaky(o):
1339    return (PASS in o) and (FAIL in o) and (not CRASH in o) and (not OKAY in o)
1340  def IsFailOk(o):
1341    return (len(o) == 2) and (FAIL in o) and (OKAY in o)
1342  unskipped = [c for c in cases if not DoSkip(c)]
1343  print REPORT_TEMPLATE % {
1344    'total': len(cases),
1345    'skipped': len(cases) - len(unskipped),
1346    'timeout': len([t for t in unskipped if TIMEOUT in t.outcomes]),
1347    'nocrash': len([t for t in unskipped if IsFlaky(t.outcomes)]),
1348    'pass': len([t for t in unskipped if list(t.outcomes) == [PASS]]),
1349    'fail_ok': len([t for t in unskipped if IsFailOk(t.outcomes)]),
1350    'fail': len([t for t in unskipped if list(t.outcomes) == [FAIL]])
1351  }
1352
1353
1354class Pattern(object):
1355
1356  def __init__(self, pattern):
1357    self.pattern = pattern
1358    self.compiled = None
1359
1360  def match(self, str):
1361    if not self.compiled:
1362      pattern = "^" + self.pattern.replace('*', '.*') + "$"
1363      self.compiled = re.compile(pattern)
1364    return self.compiled.match(str)
1365
1366  def __str__(self):
1367    return self.pattern
1368
1369
1370def SplitPath(s):
1371  stripped = [ c.strip() for c in s.split('/') ]
1372  return [ Pattern(s) for s in stripped if len(s) > 0 ]
1373
1374
1375def GetSpecialCommandProcessor(value):
1376  if (not value) or (value.find('@') == -1):
1377    def ExpandCommand(args):
1378      return args
1379    return ExpandCommand
1380  else:
1381    pos = value.find('@')
1382    import urllib
1383    prefix = urllib.unquote(value[:pos]).split()
1384    suffix = urllib.unquote(value[pos+1:]).split()
1385    def ExpandCommand(args):
1386      return prefix + args + suffix
1387    return ExpandCommand
1388
1389
1390BUILT_IN_TESTS = ['mjsunit', 'cctest', 'message', 'preparser']
1391
1392
1393def GetSuites(test_root):
1394  def IsSuite(path):
1395    return isdir(path) and exists(join(path, 'testcfg.py'))
1396  return [ f for f in os.listdir(test_root) if IsSuite(join(test_root, f)) ]
1397
1398
1399def FormatTime(d):
1400  millis = round(d * 1000) % 1000
1401  return time.strftime("%M:%S.", time.gmtime(d)) + ("%03i" % millis)
1402
1403def ShardTests(tests, options):
1404  if options.shard_count < 2:
1405    return tests
1406  if options.shard_run < 1 or options.shard_run > options.shard_count:
1407    print "shard-run not a valid number, should be in [1:shard-count]"
1408    print "defaulting back to running all tests"
1409    return tests
1410  count = 0
1411  shard = []
1412  for test in tests:
1413    if count % options.shard_count == options.shard_run - 1:
1414      shard.append(test)
1415    count += 1
1416  return shard
1417
1418def Main():
1419  parser = BuildOptions()
1420  (options, args) = parser.parse_args()
1421  if not ProcessOptions(options):
1422    parser.print_help()
1423    return 1
1424
1425  workspace = abspath(join(dirname(sys.argv[0]), '..'))
1426  suites = GetSuites(join(workspace, 'test'))
1427  repositories = [TestRepository(join(workspace, 'test', name)) for name in suites]
1428  repositories += [TestRepository(a) for a in options.suite]
1429
1430  root = LiteralTestSuite(repositories)
1431  if len(args) == 0:
1432    paths = [SplitPath(t) for t in BUILT_IN_TESTS]
1433  else:
1434    paths = [ ]
1435    for arg in args:
1436      path = SplitPath(arg)
1437      paths.append(path)
1438
1439  # Check for --valgrind option. If enabled, we overwrite the special
1440  # command flag with a command that uses the run-valgrind.py script.
1441  if options.valgrind:
1442    run_valgrind = join(workspace, "tools", "run-valgrind.py")
1443    options.special_command = "python -u " + run_valgrind + " @"
1444
1445  if options.build_system == 'gyp':
1446    SUFFIX['debug'] = ''
1447
1448  shell = abspath(options.shell)
1449  buildspace = dirname(shell)
1450
1451  context = Context(workspace, buildspace, VERBOSE,
1452                    shell,
1453                    options.timeout,
1454                    GetSpecialCommandProcessor(options.special_command),
1455                    options.suppress_dialogs,
1456                    options.store_unexpected_output)
1457  # First build the required targets
1458  if not options.no_build:
1459    reqs = [ ]
1460    for path in paths:
1461      reqs += root.GetBuildRequirements(path, context)
1462    reqs = list(set(reqs))
1463    if len(reqs) > 0:
1464      if options.j != 1:
1465        options.scons_flags += ['-j', str(options.j)]
1466      if not BuildRequirements(context, reqs, options.mode, options.scons_flags):
1467        return 1
1468
1469  # Just return if we are only building the targets for running the tests.
1470  if options.build_only:
1471    return 0
1472
1473  # Get status for tests
1474  sections = [ ]
1475  defs = { }
1476  root.GetTestStatus(context, sections, defs)
1477  config = Configuration(sections, defs)
1478
1479  # Download missing test suite data if requested.
1480  if options.download_data:
1481    for path in paths:
1482      root.DownloadData(path, context)
1483
1484  # List the tests
1485  all_cases = [ ]
1486  all_unused = [ ]
1487  unclassified_tests = [ ]
1488  globally_unused_rules = None
1489  for path in paths:
1490    for mode in options.mode:
1491      env = {
1492        'mode': mode,
1493        'system': utils.GuessOS(),
1494        'arch': options.arch,
1495        'simulator': options.simulator,
1496        'crankshaft': options.crankshaft,
1497        'isolates': options.isolates
1498      }
1499      test_list = root.ListTests([], path, context, mode, [])
1500      unclassified_tests += test_list
1501      (cases, unused_rules, all_outcomes) = config.ClassifyTests(test_list, env)
1502      if globally_unused_rules is None:
1503        globally_unused_rules = set(unused_rules)
1504      else:
1505        globally_unused_rules = globally_unused_rules.intersection(unused_rules)
1506      all_cases += ShardTests(cases, options)
1507      all_unused.append(unused_rules)
1508
1509  if options.cat:
1510    visited = set()
1511    for test in unclassified_tests:
1512      key = tuple(test.path)
1513      if key in visited:
1514        continue
1515      visited.add(key)
1516      print "--- begin source: %s ---" % test.GetLabel()
1517      source = test.GetSource().strip()
1518      print source
1519      print "--- end source: %s ---" % test.GetLabel()
1520    return 0
1521
1522  if options.warn_unused:
1523    for rule in globally_unused_rules:
1524      print "Rule for '%s' was not used." % '/'.join([str(s) for s in rule.path])
1525
1526  if not options.isolates:
1527    all_cases = [c for c in all_cases if not c.TestsIsolates()]
1528
1529  if options.report:
1530    PrintReport(all_cases)
1531
1532  result = None
1533  cases_to_run = [ c for c in all_cases if not DoSkip(c) ]
1534  if len(cases_to_run) == 0:
1535    print "No tests to run."
1536    return 0
1537  else:
1538    try:
1539      start = time.time()
1540      if RunTestCases(cases_to_run, options.progress, options.j):
1541        result = 0
1542      else:
1543        result = 1
1544      duration = time.time() - start
1545    except KeyboardInterrupt:
1546      print "Interrupted"
1547      return 1
1548
1549  if options.time:
1550    # Write the times to stderr to make it easy to separate from the
1551    # test output.
1552    print
1553    sys.stderr.write("--- Total time: %s ---\n" % FormatTime(duration))
1554    timed_tests = [ t.case for t in cases_to_run if not t.case.duration is None ]
1555    timed_tests.sort(lambda a, b: a.CompareTime(b))
1556    index = 1
1557    for entry in timed_tests[:20]:
1558      t = FormatTime(entry.duration)
1559      sys.stderr.write("%4i (%s) %s\n" % (index, t, entry.GetLabel()))
1560      index += 1
1561
1562  return result
1563
1564
1565if __name__ == '__main__':
1566  sys.exit(Main())
1567