• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2#
3# Copyright 2008 the V8 project authors. All rights reserved.
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9#       notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11#       copyright notice, this list of conditions and the following
12#       disclaimer in the documentation and/or other materials provided
13#       with the distribution.
14#     * Neither the name of Google Inc. nor the names of its
15#       contributors may be used to endorse or promote products derived
16#       from this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30
31import imp
32import optparse
33import os
34from os.path import join, dirname, abspath, basename, isdir, exists
35import platform
36import re
37import signal
38import subprocess
39import sys
40import tempfile
41import time
42import threading
43import utils
44from Queue import Queue, Empty
45
46
47VERBOSE = False
48
49
50# ---------------------------------------------
51# --- P r o g r e s s   I n d i c a t o r s ---
52# ---------------------------------------------
53
54
55class ProgressIndicator(object):
56
57  def __init__(self, cases):
58    self.cases = cases
59    self.queue = Queue(len(cases))
60    for case in cases:
61      self.queue.put_nowait(case)
62    self.succeeded = 0
63    self.remaining = len(cases)
64    self.total = len(cases)
65    self.failed = [ ]
66    self.crashed = 0
67    self.terminate = False
68    self.lock = threading.Lock()
69
70  def PrintFailureHeader(self, test):
71    if test.IsNegative():
72      negative_marker = '[negative] '
73    else:
74      negative_marker = ''
75    print "=== %(label)s %(negative)s===" % {
76      'label': test.GetLabel(),
77      'negative': negative_marker
78    }
79    print "Path: %s" % "/".join(test.path)
80
81  def Run(self, tasks):
82    self.Starting()
83    threads = []
84    # Spawn N-1 threads and then use this thread as the last one.
85    # That way -j1 avoids threading altogether which is a nice fallback
86    # in case of threading problems.
87    for i in xrange(tasks - 1):
88      thread = threading.Thread(target=self.RunSingle, args=[])
89      threads.append(thread)
90      thread.start()
91    try:
92      self.RunSingle()
93      # Wait for the remaining threads
94      for thread in threads:
95        # Use a timeout so that signals (ctrl-c) will be processed.
96        thread.join(timeout=10000000)
97    except Exception, e:
98      # If there's an exception we schedule an interruption for any
99      # remaining threads.
100      self.terminate = True
101      # ...and then reraise the exception to bail out
102      raise
103    self.Done()
104    return not self.failed
105
106  def RunSingle(self):
107    while not self.terminate:
108      try:
109        test = self.queue.get_nowait()
110      except Empty:
111        return
112      case = test.case
113      self.lock.acquire()
114      self.AboutToRun(case)
115      self.lock.release()
116      try:
117        start = time.time()
118        output = case.Run()
119        case.duration = (time.time() - start)
120      except IOError, e:
121        assert self.terminate
122        return
123      if self.terminate:
124        return
125      self.lock.acquire()
126      if output.UnexpectedOutput():
127        self.failed.append(output)
128        if output.HasCrashed():
129          self.crashed += 1
130      else:
131        self.succeeded += 1
132      self.remaining -= 1
133      self.HasRun(output)
134      self.lock.release()
135
136
137def EscapeCommand(command):
138  parts = []
139  for part in command:
140    if ' ' in part:
141      # Escape spaces.  We may need to escape more characters for this
142      # to work properly.
143      parts.append('"%s"' % part)
144    else:
145      parts.append(part)
146  return " ".join(parts)
147
148
149class SimpleProgressIndicator(ProgressIndicator):
150
151  def Starting(self):
152    print 'Running %i tests' % len(self.cases)
153
154  def Done(self):
155    print
156    for failed in self.failed:
157      self.PrintFailureHeader(failed.test)
158      if failed.output.stderr:
159        print "--- stderr ---"
160        print failed.output.stderr.strip()
161      if failed.output.stdout:
162        print "--- stdout ---"
163        print failed.output.stdout.strip()
164      print "Command: %s" % EscapeCommand(failed.command)
165      if failed.HasCrashed():
166        print "--- CRASHED ---"
167      if failed.HasTimedOut():
168        print "--- TIMEOUT ---"
169    if len(self.failed) == 0:
170      print "==="
171      print "=== All tests succeeded"
172      print "==="
173    else:
174      print
175      print "==="
176      print "=== %i tests failed" % len(self.failed)
177      if self.crashed > 0:
178        print "=== %i tests CRASHED" % self.crashed
179      print "==="
180
181
182class VerboseProgressIndicator(SimpleProgressIndicator):
183
184  def AboutToRun(self, case):
185    print 'Starting %s...' % case.GetLabel()
186    sys.stdout.flush()
187
188  def HasRun(self, output):
189    if output.UnexpectedOutput():
190      if output.HasCrashed():
191        outcome = 'CRASH'
192      else:
193        outcome = 'FAIL'
194    else:
195      outcome = 'pass'
196    print 'Done running %s: %s' % (output.test.GetLabel(), outcome)
197
198
199class DotsProgressIndicator(SimpleProgressIndicator):
200
201  def AboutToRun(self, case):
202    pass
203
204  def HasRun(self, output):
205    total = self.succeeded + len(self.failed)
206    if (total > 1) and (total % 50 == 1):
207      sys.stdout.write('\n')
208    if output.UnexpectedOutput():
209      if output.HasCrashed():
210        sys.stdout.write('C')
211        sys.stdout.flush()
212      elif output.HasTimedOut():
213        sys.stdout.write('T')
214        sys.stdout.flush()
215      else:
216        sys.stdout.write('F')
217        sys.stdout.flush()
218    else:
219      sys.stdout.write('.')
220      sys.stdout.flush()
221
222
223class CompactProgressIndicator(ProgressIndicator):
224
225  def __init__(self, cases, templates):
226    super(CompactProgressIndicator, self).__init__(cases)
227    self.templates = templates
228    self.last_status_length = 0
229    self.start_time = time.time()
230
231  def Starting(self):
232    pass
233
234  def Done(self):
235    self.PrintProgress('Done')
236
237  def AboutToRun(self, case):
238    self.PrintProgress(case.GetLabel())
239
240  def HasRun(self, output):
241    if output.UnexpectedOutput():
242      self.ClearLine(self.last_status_length)
243      self.PrintFailureHeader(output.test)
244      stdout = output.output.stdout.strip()
245      if len(stdout):
246        print self.templates['stdout'] % stdout
247      stderr = output.output.stderr.strip()
248      if len(stderr):
249        print self.templates['stderr'] % stderr
250      print "Command: %s" % EscapeCommand(output.command)
251      if output.HasCrashed():
252        print "--- CRASHED ---"
253      if output.HasTimedOut():
254        print "--- TIMEOUT ---"
255
256  def Truncate(self, str, length):
257    if length and (len(str) > (length - 3)):
258      return str[:(length-3)] + "..."
259    else:
260      return str
261
262  def PrintProgress(self, name):
263    self.ClearLine(self.last_status_length)
264    elapsed = time.time() - self.start_time
265    status = self.templates['status_line'] % {
266      'passed': self.succeeded,
267      'remaining': (((self.total - self.remaining) * 100) // self.total),
268      'failed': len(self.failed),
269      'test': name,
270      'mins': int(elapsed) / 60,
271      'secs': int(elapsed) % 60
272    }
273    status = self.Truncate(status, 78)
274    self.last_status_length = len(status)
275    print status,
276    sys.stdout.flush()
277
278
279class ColorProgressIndicator(CompactProgressIndicator):
280
281  def __init__(self, cases):
282    templates = {
283      'status_line': "[%(mins)02i:%(secs)02i|\033[34m%%%(remaining) 4d\033[0m|\033[32m+%(passed) 4d\033[0m|\033[31m-%(failed) 4d\033[0m]: %(test)s",
284      'stdout': "\033[1m%s\033[0m",
285      'stderr': "\033[31m%s\033[0m",
286    }
287    super(ColorProgressIndicator, self).__init__(cases, templates)
288
289  def ClearLine(self, last_line_length):
290    print "\033[1K\r",
291
292
293class MonochromeProgressIndicator(CompactProgressIndicator):
294
295  def __init__(self, cases):
296    templates = {
297      'status_line': "[%(mins)02i:%(secs)02i|%%%(remaining) 4d|+%(passed) 4d|-%(failed) 4d]: %(test)s",
298      'stdout': '%s',
299      'stderr': '%s',
300      'clear': lambda last_line_length: ("\r" + (" " * last_line_length) + "\r"),
301      'max_length': 78
302    }
303    super(MonochromeProgressIndicator, self).__init__(cases, templates)
304
305  def ClearLine(self, last_line_length):
306    print ("\r" + (" " * last_line_length) + "\r"),
307
308
309PROGRESS_INDICATORS = {
310  'verbose': VerboseProgressIndicator,
311  'dots': DotsProgressIndicator,
312  'color': ColorProgressIndicator,
313  'mono': MonochromeProgressIndicator
314}
315
316
317# -------------------------
318# --- F r a m e w o r k ---
319# -------------------------
320
321
322class CommandOutput(object):
323
324  def __init__(self, exit_code, timed_out, stdout, stderr):
325    self.exit_code = exit_code
326    self.timed_out = timed_out
327    self.stdout = stdout
328    self.stderr = stderr
329    self.failed = None
330
331
332class TestCase(object):
333
334  def __init__(self, context, path):
335    self.path = path
336    self.context = context
337    self.duration = None
338
339  def IsNegative(self):
340    return False
341
342  def CompareTime(self, other):
343    return cmp(other.duration, self.duration)
344
345  def DidFail(self, output):
346    if output.failed is None:
347      output.failed = self.IsFailureOutput(output)
348    return output.failed
349
350  def IsFailureOutput(self, output):
351    return output.exit_code != 0
352
353  def GetSource(self):
354    return "(no source available)"
355
356  def RunCommand(self, command):
357    full_command = self.context.processor(command)
358    output = Execute(full_command, self.context, self.context.timeout)
359    self.Cleanup()
360    return TestOutput(self, full_command, output)
361
362  def BeforeRun(self):
363    pass
364
365  def AfterRun(self):
366    pass
367
368  def Run(self):
369    self.BeforeRun()
370    try:
371      result = self.RunCommand(self.GetCommand())
372    finally:
373      self.AfterRun()
374    return result
375
376  def Cleanup(self):
377    return
378
379
380class TestOutput(object):
381
382  def __init__(self, test, command, output):
383    self.test = test
384    self.command = command
385    self.output = output
386
387  def UnexpectedOutput(self):
388    if self.HasCrashed():
389      outcome = CRASH
390    elif self.HasTimedOut():
391      outcome = TIMEOUT
392    elif self.HasFailed():
393      outcome = FAIL
394    else:
395      outcome = PASS
396    return not outcome in self.test.outcomes
397
398  def HasCrashed(self):
399    if utils.IsWindows():
400      return 0x80000000 & self.output.exit_code and not (0x3FFFFF00 & self.output.exit_code)
401    else:
402      # Timed out tests will have exit_code -signal.SIGTERM.
403      if self.output.timed_out:
404        return False
405      return self.output.exit_code < 0 and \
406             self.output.exit_code != -signal.SIGABRT
407
408  def HasTimedOut(self):
409    return self.output.timed_out;
410
411  def HasFailed(self):
412    execution_failed = self.test.DidFail(self.output)
413    if self.test.IsNegative():
414      return not execution_failed
415    else:
416      return execution_failed
417
418
419def KillProcessWithID(pid):
420  if utils.IsWindows():
421    os.popen('taskkill /T /F /PID %d' % pid)
422  else:
423    os.kill(pid, signal.SIGTERM)
424
425
426MAX_SLEEP_TIME = 0.1
427INITIAL_SLEEP_TIME = 0.0001
428SLEEP_TIME_FACTOR = 1.25
429
430SEM_INVALID_VALUE = -1
431SEM_NOGPFAULTERRORBOX = 0x0002 # Microsoft Platform SDK WinBase.h
432
433def Win32SetErrorMode(mode):
434  prev_error_mode = SEM_INVALID_VALUE
435  try:
436    import ctypes
437    prev_error_mode = ctypes.windll.kernel32.SetErrorMode(mode);
438  except ImportError:
439    pass
440  return prev_error_mode
441
442def RunProcess(context, timeout, args, **rest):
443  if context.verbose: print "#", " ".join(args)
444  popen_args = args
445  prev_error_mode = SEM_INVALID_VALUE;
446  if utils.IsWindows():
447    popen_args = '"' + subprocess.list2cmdline(args) + '"'
448    if context.suppress_dialogs:
449      # Try to change the error mode to avoid dialogs on fatal errors. Don't
450      # touch any existing error mode flags by merging the existing error mode.
451      # See http://blogs.msdn.com/oldnewthing/archive/2004/07/27/198410.aspx.
452      error_mode = SEM_NOGPFAULTERRORBOX;
453      prev_error_mode = Win32SetErrorMode(error_mode);
454      Win32SetErrorMode(error_mode | prev_error_mode);
455  process = subprocess.Popen(
456    shell = utils.IsWindows(),
457    args = popen_args,
458    **rest
459  )
460  if utils.IsWindows() and context.suppress_dialogs and prev_error_mode != SEM_INVALID_VALUE:
461    Win32SetErrorMode(prev_error_mode)
462  # Compute the end time - if the process crosses this limit we
463  # consider it timed out.
464  if timeout is None: end_time = None
465  else: end_time = time.time() + timeout
466  timed_out = False
467  # Repeatedly check the exit code from the process in a
468  # loop and keep track of whether or not it times out.
469  exit_code = None
470  sleep_time = INITIAL_SLEEP_TIME
471  while exit_code is None:
472    if (not end_time is None) and (time.time() >= end_time):
473      # Kill the process and wait for it to exit.
474      KillProcessWithID(process.pid)
475      exit_code = process.wait()
476      timed_out = True
477    else:
478      exit_code = process.poll()
479      time.sleep(sleep_time)
480      sleep_time = sleep_time * SLEEP_TIME_FACTOR
481      if sleep_time > MAX_SLEEP_TIME:
482        sleep_time = MAX_SLEEP_TIME
483  return (process, exit_code, timed_out)
484
485
486def PrintError(str):
487  sys.stderr.write(str)
488  sys.stderr.write('\n')
489
490
491def CheckedUnlink(name):
492  try:
493    os.unlink(name)
494  except OSError, e:
495    PrintError("os.unlink() " + str(e))
496
497
498def Execute(args, context, timeout=None):
499  (fd_out, outname) = tempfile.mkstemp()
500  (fd_err, errname) = tempfile.mkstemp()
501  (process, exit_code, timed_out) = RunProcess(
502    context,
503    timeout,
504    args = args,
505    stdout = fd_out,
506    stderr = fd_err,
507  )
508  os.close(fd_out)
509  os.close(fd_err)
510  output = file(outname).read()
511  errors = file(errname).read()
512  CheckedUnlink(outname)
513  CheckedUnlink(errname)
514  return CommandOutput(exit_code, timed_out, output, errors)
515
516
517def ExecuteNoCapture(args, context, timeout=None):
518  (process, exit_code, timed_out) = RunProcess(
519    context,
520    timeout,
521    args = args,
522  )
523  return CommandOutput(exit_code, False, "", "")
524
525
526def CarCdr(path):
527  if len(path) == 0:
528    return (None, [ ])
529  else:
530    return (path[0], path[1:])
531
532
533class TestConfiguration(object):
534
535  def __init__(self, context, root):
536    self.context = context
537    self.root = root
538
539  def Contains(self, path, file):
540    if len(path) > len(file):
541      return False
542    for i in xrange(len(path)):
543      if not path[i].match(file[i]):
544        return False
545    return True
546
547  def GetTestStatus(self, sections, defs):
548    pass
549
550
551class TestSuite(object):
552
553  def __init__(self, name):
554    self.name = name
555
556  def GetName(self):
557    return self.name
558
559
560class TestRepository(TestSuite):
561
562  def __init__(self, path):
563    normalized_path = abspath(path)
564    super(TestRepository, self).__init__(basename(normalized_path))
565    self.path = normalized_path
566    self.is_loaded = False
567    self.config = None
568
569  def GetConfiguration(self, context):
570    if self.is_loaded:
571      return self.config
572    self.is_loaded = True
573    file = None
574    try:
575      (file, pathname, description) = imp.find_module('testcfg', [ self.path ])
576      module = imp.load_module('testcfg', file, pathname, description)
577      self.config = module.GetConfiguration(context, self.path)
578    finally:
579      if file:
580        file.close()
581    return self.config
582
583  def GetBuildRequirements(self, path, context):
584    return self.GetConfiguration(context).GetBuildRequirements()
585
586  def ListTests(self, current_path, path, context, mode):
587    return self.GetConfiguration(context).ListTests(current_path, path, mode)
588
589  def GetTestStatus(self, context, sections, defs):
590    self.GetConfiguration(context).GetTestStatus(sections, defs)
591
592
593class LiteralTestSuite(TestSuite):
594
595  def __init__(self, tests):
596    super(LiteralTestSuite, self).__init__('root')
597    self.tests = tests
598
599  def GetBuildRequirements(self, path, context):
600    (name, rest) = CarCdr(path)
601    result = [ ]
602    for test in self.tests:
603      if not name or name.match(test.GetName()):
604        result += test.GetBuildRequirements(rest, context)
605    return result
606
607  def ListTests(self, current_path, path, context, mode):
608    (name, rest) = CarCdr(path)
609    result = [ ]
610    for test in self.tests:
611      test_name = test.GetName()
612      if not name or name.match(test_name):
613        full_path = current_path + [test_name]
614        result += test.ListTests(full_path, path, context, mode)
615    return result
616
617  def GetTestStatus(self, context, sections, defs):
618    for test in self.tests:
619      test.GetTestStatus(context, sections, defs)
620
621
622SUFFIX = {'debug': '_g', 'release': ''}
623
624
625class Context(object):
626
627  def __init__(self, workspace, buildspace, verbose, vm, timeout, processor, suppress_dialogs):
628    self.workspace = workspace
629    self.buildspace = buildspace
630    self.verbose = verbose
631    self.vm_root = vm
632    self.timeout = timeout
633    self.processor = processor
634    self.suppress_dialogs = suppress_dialogs
635
636  def GetVm(self, mode):
637    name = self.vm_root + SUFFIX[mode]
638    if utils.IsWindows() and not name.endswith('.exe'):
639      name = name + '.exe'
640    return name
641
642def RunTestCases(cases_to_run, progress, tasks):
643  progress = PROGRESS_INDICATORS[progress](cases_to_run)
644  return progress.Run(tasks)
645
646
647def BuildRequirements(context, requirements, mode, scons_flags):
648  command_line = (['scons', '-Y', context.workspace, 'mode=' + ",".join(mode)]
649                  + requirements
650                  + scons_flags)
651  output = ExecuteNoCapture(command_line, context)
652  return output.exit_code == 0
653
654
655# -------------------------------------------
656# --- T e s t   C o n f i g u r a t i o n ---
657# -------------------------------------------
658
659
660SKIP = 'skip'
661FAIL = 'fail'
662PASS = 'pass'
663OKAY = 'okay'
664TIMEOUT = 'timeout'
665CRASH = 'crash'
666SLOW = 'slow'
667
668
669class Expression(object):
670  pass
671
672
673class Constant(Expression):
674
675  def __init__(self, value):
676    self.value = value
677
678  def Evaluate(self, env, defs):
679    return self.value
680
681
682class Variable(Expression):
683
684  def __init__(self, name):
685    self.name = name
686
687  def GetOutcomes(self, env, defs):
688    if self.name in env: return ListSet([env[self.name]])
689    else: return Nothing()
690
691
692class Outcome(Expression):
693
694  def __init__(self, name):
695    self.name = name
696
697  def GetOutcomes(self, env, defs):
698    if self.name in defs:
699      return defs[self.name].GetOutcomes(env, defs)
700    else:
701      return ListSet([self.name])
702
703
704class Set(object):
705  pass
706
707
708class ListSet(Set):
709
710  def __init__(self, elms):
711    self.elms = elms
712
713  def __str__(self):
714    return "ListSet%s" % str(self.elms)
715
716  def Intersect(self, that):
717    if not isinstance(that, ListSet):
718      return that.Intersect(self)
719    return ListSet([ x for x in self.elms if x in that.elms ])
720
721  def Union(self, that):
722    if not isinstance(that, ListSet):
723      return that.Union(self)
724    return ListSet(self.elms + [ x for x in that.elms if x not in self.elms ])
725
726  def IsEmpty(self):
727    return len(self.elms) == 0
728
729
730class Everything(Set):
731
732  def Intersect(self, that):
733    return that
734
735  def Union(self, that):
736    return self
737
738  def IsEmpty(self):
739    return False
740
741
742class Nothing(Set):
743
744  def Intersect(self, that):
745    return self
746
747  def Union(self, that):
748    return that
749
750  def IsEmpty(self):
751    return True
752
753
754class Operation(Expression):
755
756  def __init__(self, left, op, right):
757    self.left = left
758    self.op = op
759    self.right = right
760
761  def Evaluate(self, env, defs):
762    if self.op == '||' or self.op == ',':
763      return self.left.Evaluate(env, defs) or self.right.Evaluate(env, defs)
764    elif self.op == 'if':
765      return False
766    elif self.op == '==':
767      inter = self.left.GetOutcomes(env, defs).Intersect(self.right.GetOutcomes(env, defs))
768      return not inter.IsEmpty()
769    else:
770      assert self.op == '&&'
771      return self.left.Evaluate(env, defs) and self.right.Evaluate(env, defs)
772
773  def GetOutcomes(self, env, defs):
774    if self.op == '||' or self.op == ',':
775      return self.left.GetOutcomes(env, defs).Union(self.right.GetOutcomes(env, defs))
776    elif self.op == 'if':
777      if self.right.Evaluate(env, defs): return self.left.GetOutcomes(env, defs)
778      else: return Nothing()
779    else:
780      assert self.op == '&&'
781      return self.left.GetOutcomes(env, defs).Intersect(self.right.GetOutcomes(env, defs))
782
783
784def IsAlpha(str):
785  for char in str:
786    if not (char.isalpha() or char.isdigit() or char == '_'):
787      return False
788  return True
789
790
791class Tokenizer(object):
792  """A simple string tokenizer that chops expressions into variables,
793  parens and operators"""
794
795  def __init__(self, expr):
796    self.index = 0
797    self.expr = expr
798    self.length = len(expr)
799    self.tokens = None
800
801  def Current(self, length = 1):
802    if not self.HasMore(length): return ""
803    return self.expr[self.index:self.index+length]
804
805  def HasMore(self, length = 1):
806    return self.index < self.length + (length - 1)
807
808  def Advance(self, count = 1):
809    self.index = self.index + count
810
811  def AddToken(self, token):
812    self.tokens.append(token)
813
814  def SkipSpaces(self):
815    while self.HasMore() and self.Current().isspace():
816      self.Advance()
817
818  def Tokenize(self):
819    self.tokens = [ ]
820    while self.HasMore():
821      self.SkipSpaces()
822      if not self.HasMore():
823        return None
824      if self.Current() == '(':
825        self.AddToken('(')
826        self.Advance()
827      elif self.Current() == ')':
828        self.AddToken(')')
829        self.Advance()
830      elif self.Current() == '$':
831        self.AddToken('$')
832        self.Advance()
833      elif self.Current() == ',':
834        self.AddToken(',')
835        self.Advance()
836      elif IsAlpha(self.Current()):
837        buf = ""
838        while self.HasMore() and IsAlpha(self.Current()):
839          buf += self.Current()
840          self.Advance()
841        self.AddToken(buf)
842      elif self.Current(2) == '&&':
843        self.AddToken('&&')
844        self.Advance(2)
845      elif self.Current(2) == '||':
846        self.AddToken('||')
847        self.Advance(2)
848      elif self.Current(2) == '==':
849        self.AddToken('==')
850        self.Advance(2)
851      else:
852        return None
853    return self.tokens
854
855
856class Scanner(object):
857  """A simple scanner that can serve out tokens from a given list"""
858
859  def __init__(self, tokens):
860    self.tokens = tokens
861    self.length = len(tokens)
862    self.index = 0
863
864  def HasMore(self):
865    return self.index < self.length
866
867  def Current(self):
868    return self.tokens[self.index]
869
870  def Advance(self):
871    self.index = self.index + 1
872
873
874def ParseAtomicExpression(scan):
875  if scan.Current() == "true":
876    scan.Advance()
877    return Constant(True)
878  elif scan.Current() == "false":
879    scan.Advance()
880    return Constant(False)
881  elif IsAlpha(scan.Current()):
882    name = scan.Current()
883    scan.Advance()
884    return Outcome(name.lower())
885  elif scan.Current() == '$':
886    scan.Advance()
887    if not IsAlpha(scan.Current()):
888      return None
889    name = scan.Current()
890    scan.Advance()
891    return Variable(name.lower())
892  elif scan.Current() == '(':
893    scan.Advance()
894    result = ParseLogicalExpression(scan)
895    if (not result) or (scan.Current() != ')'):
896      return None
897    scan.Advance()
898    return result
899  else:
900    return None
901
902
903BINARIES = ['==']
904def ParseOperatorExpression(scan):
905  left = ParseAtomicExpression(scan)
906  if not left: return None
907  while scan.HasMore() and (scan.Current() in BINARIES):
908    op = scan.Current()
909    scan.Advance()
910    right = ParseOperatorExpression(scan)
911    if not right:
912      return None
913    left = Operation(left, op, right)
914  return left
915
916
917def ParseConditionalExpression(scan):
918  left = ParseOperatorExpression(scan)
919  if not left: return None
920  while scan.HasMore() and (scan.Current() == 'if'):
921    scan.Advance()
922    right = ParseOperatorExpression(scan)
923    if not right:
924      return None
925    left=  Operation(left, 'if', right)
926  return left
927
928
929LOGICALS = ["&&", "||", ","]
930def ParseLogicalExpression(scan):
931  left = ParseConditionalExpression(scan)
932  if not left: return None
933  while scan.HasMore() and (scan.Current() in LOGICALS):
934    op = scan.Current()
935    scan.Advance()
936    right = ParseConditionalExpression(scan)
937    if not right:
938      return None
939    left = Operation(left, op, right)
940  return left
941
942
943def ParseCondition(expr):
944  """Parses a logical expression into an Expression object"""
945  tokens = Tokenizer(expr).Tokenize()
946  if not tokens:
947    print "Malformed expression: '%s'" % expr
948    return None
949  scan = Scanner(tokens)
950  ast = ParseLogicalExpression(scan)
951  if not ast:
952    print "Malformed expression: '%s'" % expr
953    return None
954  if scan.HasMore():
955    print "Malformed expression: '%s'" % expr
956    return None
957  return ast
958
959
960class ClassifiedTest(object):
961
962  def __init__(self, case, outcomes):
963    self.case = case
964    self.outcomes = outcomes
965
966
967class Configuration(object):
968  """The parsed contents of a configuration file"""
969
970  def __init__(self, sections, defs):
971    self.sections = sections
972    self.defs = defs
973
974  def ClassifyTests(self, cases, env):
975    sections = [s for s in self.sections if s.condition.Evaluate(env, self.defs)]
976    all_rules = reduce(list.__add__, [s.rules for s in sections], [])
977    unused_rules = set(all_rules)
978    result = [ ]
979    all_outcomes = set([])
980    for case in cases:
981      matches = [ r for r in all_rules if r.Contains(case.path) ]
982      outcomes = set([])
983      for rule in matches:
984        outcomes = outcomes.union(rule.GetOutcomes(env, self.defs))
985        unused_rules.discard(rule)
986      if not outcomes:
987        outcomes = [PASS]
988      case.outcomes = outcomes
989      all_outcomes = all_outcomes.union(outcomes)
990      result.append(ClassifiedTest(case, outcomes))
991    return (result, list(unused_rules), all_outcomes)
992
993
994class Section(object):
995  """A section of the configuration file.  Sections are enabled or
996  disabled prior to running the tests, based on their conditions"""
997
998  def __init__(self, condition):
999    self.condition = condition
1000    self.rules = [ ]
1001
1002  def AddRule(self, rule):
1003    self.rules.append(rule)
1004
1005
1006class Rule(object):
1007  """A single rule that specifies the expected outcome for a single
1008  test."""
1009
1010  def __init__(self, raw_path, path, value):
1011    self.raw_path = raw_path
1012    self.path = path
1013    self.value = value
1014
1015  def GetOutcomes(self, env, defs):
1016    set = self.value.GetOutcomes(env, defs)
1017    assert isinstance(set, ListSet)
1018    return set.elms
1019
1020  def Contains(self, path):
1021    if len(self.path) > len(path):
1022      return False
1023    for i in xrange(len(self.path)):
1024      if not self.path[i].match(path[i]):
1025        return False
1026    return True
1027
1028
1029HEADER_PATTERN = re.compile(r'\[([^]]+)\]')
1030RULE_PATTERN = re.compile(r'\s*([^: ]*)\s*:(.*)')
1031DEF_PATTERN = re.compile(r'^def\s*(\w+)\s*=(.*)$')
1032PREFIX_PATTERN = re.compile(r'^\s*prefix\s+([\w\_\.\-\/]+)$')
1033
1034
1035def ReadConfigurationInto(path, sections, defs):
1036  current_section = Section(Constant(True))
1037  sections.append(current_section)
1038  prefix = []
1039  for line in utils.ReadLinesFrom(path):
1040    header_match = HEADER_PATTERN.match(line)
1041    if header_match:
1042      condition_str = header_match.group(1).strip()
1043      condition = ParseCondition(condition_str)
1044      new_section = Section(condition)
1045      sections.append(new_section)
1046      current_section = new_section
1047      continue
1048    rule_match = RULE_PATTERN.match(line)
1049    if rule_match:
1050      path = prefix + SplitPath(rule_match.group(1).strip())
1051      value_str = rule_match.group(2).strip()
1052      value = ParseCondition(value_str)
1053      if not value:
1054        return False
1055      current_section.AddRule(Rule(rule_match.group(1), path, value))
1056      continue
1057    def_match = DEF_PATTERN.match(line)
1058    if def_match:
1059      name = def_match.group(1).lower()
1060      value = ParseCondition(def_match.group(2).strip())
1061      if not value:
1062        return False
1063      defs[name] = value
1064      continue
1065    prefix_match = PREFIX_PATTERN.match(line)
1066    if prefix_match:
1067      prefix = SplitPath(prefix_match.group(1).strip())
1068      continue
1069    print "Malformed line: '%s'." % line
1070    return False
1071  return True
1072
1073
1074# ---------------
1075# --- M a i n ---
1076# ---------------
1077
1078
1079ARCH_GUESS = utils.GuessArchitecture()
1080
1081
1082def BuildOptions():
1083  result = optparse.OptionParser()
1084  result.add_option("-m", "--mode", help="The test modes in which to run (comma-separated)",
1085      default='release')
1086  result.add_option("-v", "--verbose", help="Verbose output",
1087      default=False, action="store_true")
1088  result.add_option("-S", dest="scons_flags", help="Flag to pass through to scons",
1089      default=[], action="append")
1090  result.add_option("-p", "--progress",
1091      help="The style of progress indicator (verbose, dots, color, mono)",
1092      choices=PROGRESS_INDICATORS.keys(), default="mono")
1093  result.add_option("--no-build", help="Don't build requirements",
1094      default=False, action="store_true")
1095  result.add_option("--build-only", help="Only build requirements, don't run the tests",
1096      default=False, action="store_true")
1097  result.add_option("--report", help="Print a summary of the tests to be run",
1098      default=False, action="store_true")
1099  result.add_option("-s", "--suite", help="A test suite",
1100      default=[], action="append")
1101  result.add_option("-t", "--timeout", help="Timeout in seconds",
1102      default=60, type="int")
1103  result.add_option("--arch", help='The architecture to run tests for',
1104      default='none')
1105  result.add_option("--snapshot", help="Run the tests with snapshot turned on",
1106      default=False, action="store_true")
1107  result.add_option("--simulator", help="Run tests with architecture simulator",
1108      default='none')
1109  result.add_option("--special-command", default=None)
1110  result.add_option("--valgrind", help="Run tests through valgrind",
1111      default=False, action="store_true")
1112  result.add_option("--cat", help="Print the source of the tests",
1113      default=False, action="store_true")
1114  result.add_option("--warn-unused", help="Report unused rules",
1115      default=False, action="store_true")
1116  result.add_option("-j", help="The number of parallel tasks to run",
1117      default=1, type="int")
1118  result.add_option("--time", help="Print timing information after running",
1119      default=False, action="store_true")
1120  result.add_option("--suppress-dialogs", help="Suppress Windows dialogs for crashing tests",
1121        dest="suppress_dialogs", default=True, action="store_true")
1122  result.add_option("--no-suppress-dialogs", help="Display Windows dialogs for crashing tests",
1123        dest="suppress_dialogs", action="store_false")
1124  result.add_option("--shell", help="Path to V8 shell", default="shell");
1125  return result
1126
1127
1128def ProcessOptions(options):
1129  global VERBOSE
1130  VERBOSE = options.verbose
1131  options.mode = options.mode.split(',')
1132  for mode in options.mode:
1133    if not mode in ['debug', 'release']:
1134      print "Unknown mode %s" % mode
1135      return False
1136  if options.simulator != 'none':
1137    # Simulator argument was set. Make sure arch and simulator agree.
1138    if options.simulator != options.arch:
1139      if options.arch == 'none':
1140        options.arch = options.simulator
1141      else:
1142        print "Architecture %s does not match sim %s" %(options.arch, options.simulator)
1143        return False
1144    # Ensure that the simulator argument is handed down to scons.
1145    options.scons_flags.append("simulator=" + options.simulator)
1146  else:
1147    # If options.arch is not set by the command line and no simulator setting
1148    # was found, set the arch to the guess.
1149    if options.arch == 'none':
1150      options.arch = ARCH_GUESS
1151    options.scons_flags.append("arch=" + options.arch)
1152  if options.snapshot:
1153    options.scons_flags.append("snapshot=on")
1154  return True
1155
1156
1157REPORT_TEMPLATE = """\
1158Total: %(total)i tests
1159 * %(skipped)4d tests will be skipped
1160 * %(nocrash)4d tests are expected to be flaky but not crash
1161 * %(pass)4d tests are expected to pass
1162 * %(fail_ok)4d tests are expected to fail that we won't fix
1163 * %(fail)4d tests are expected to fail that we should fix\
1164"""
1165
1166def PrintReport(cases):
1167  def IsFlaky(o):
1168    return (PASS in o) and (FAIL in o) and (not CRASH in o) and (not OKAY in o)
1169  def IsFailOk(o):
1170    return (len(o) == 2) and (FAIL in o) and (OKAY in o)
1171  unskipped = [c for c in cases if not SKIP in c.outcomes]
1172  print REPORT_TEMPLATE % {
1173    'total': len(cases),
1174    'skipped': len(cases) - len(unskipped),
1175    'nocrash': len([t for t in unskipped if IsFlaky(t.outcomes)]),
1176    'pass': len([t for t in unskipped if list(t.outcomes) == [PASS]]),
1177    'fail_ok': len([t for t in unskipped if IsFailOk(t.outcomes)]),
1178    'fail': len([t for t in unskipped if list(t.outcomes) == [FAIL]])
1179  }
1180
1181
1182class Pattern(object):
1183
1184  def __init__(self, pattern):
1185    self.pattern = pattern
1186    self.compiled = None
1187
1188  def match(self, str):
1189    if not self.compiled:
1190      pattern = "^" + self.pattern.replace('*', '.*') + "$"
1191      self.compiled = re.compile(pattern)
1192    return self.compiled.match(str)
1193
1194  def __str__(self):
1195    return self.pattern
1196
1197
1198def SplitPath(s):
1199  stripped = [ c.strip() for c in s.split('/') ]
1200  return [ Pattern(s) for s in stripped if len(s) > 0 ]
1201
1202
1203def GetSpecialCommandProcessor(value):
1204  if (not value) or (value.find('@') == -1):
1205    def ExpandCommand(args):
1206      return args
1207    return ExpandCommand
1208  else:
1209    pos = value.find('@')
1210    import urllib
1211    prefix = urllib.unquote(value[:pos]).split()
1212    suffix = urllib.unquote(value[pos+1:]).split()
1213    def ExpandCommand(args):
1214      return prefix + args + suffix
1215    return ExpandCommand
1216
1217
1218BUILT_IN_TESTS = ['mjsunit', 'cctest', 'message']
1219
1220
1221def GetSuites(test_root):
1222  def IsSuite(path):
1223    return isdir(path) and exists(join(path, 'testcfg.py'))
1224  return [ f for f in os.listdir(test_root) if IsSuite(join(test_root, f)) ]
1225
1226
1227def FormatTime(d):
1228  millis = round(d * 1000) % 1000
1229  return time.strftime("%M:%S.", time.gmtime(d)) + ("%03i" % millis)
1230
1231
1232def Main():
1233  parser = BuildOptions()
1234  (options, args) = parser.parse_args()
1235  if not ProcessOptions(options):
1236    parser.print_help()
1237    return 1
1238
1239  workspace = abspath(join(dirname(sys.argv[0]), '..'))
1240  suites = GetSuites(join(workspace, 'test'))
1241  repositories = [TestRepository(join(workspace, 'test', name)) for name in suites]
1242  repositories += [TestRepository(a) for a in options.suite]
1243
1244  root = LiteralTestSuite(repositories)
1245  if len(args) == 0:
1246    paths = [SplitPath(t) for t in BUILT_IN_TESTS]
1247  else:
1248    paths = [ ]
1249    for arg in args:
1250      path = SplitPath(arg)
1251      paths.append(path)
1252
1253  # Check for --valgrind option. If enabled, we overwrite the special
1254  # command flag with a command that uses the run-valgrind.py script.
1255  if options.valgrind:
1256    run_valgrind = join(workspace, "tools", "run-valgrind.py")
1257    options.special_command = "python -u " + run_valgrind + " @"
1258
1259  shell = abspath(options.shell)
1260  buildspace = dirname(shell)
1261  context = Context(workspace, buildspace, VERBOSE,
1262                    shell,
1263                    options.timeout,
1264                    GetSpecialCommandProcessor(options.special_command),
1265                    options.suppress_dialogs)
1266  # First build the required targets
1267  if not options.no_build:
1268    reqs = [ ]
1269    for path in paths:
1270      reqs += root.GetBuildRequirements(path, context)
1271    reqs = list(set(reqs))
1272    if len(reqs) > 0:
1273      if options.j != 1:
1274        options.scons_flags += ['-j', str(options.j)]
1275      if not BuildRequirements(context, reqs, options.mode, options.scons_flags):
1276        return 1
1277
1278  # Just return if we are only building the targets for running the tests.
1279  if options.build_only:
1280    return 0
1281
1282  # Get status for tests
1283  sections = [ ]
1284  defs = { }
1285  root.GetTestStatus(context, sections, defs)
1286  config = Configuration(sections, defs)
1287
1288  # List the tests
1289  all_cases = [ ]
1290  all_unused = [ ]
1291  unclassified_tests = [ ]
1292  globally_unused_rules = None
1293  for path in paths:
1294    for mode in options.mode:
1295      if not exists(context.GetVm(mode)):
1296        print "Can't find shell executable: '%s'" % context.GetVm(mode)
1297        continue
1298      env = {
1299        'mode': mode,
1300        'system': utils.GuessOS(),
1301        'arch': options.arch,
1302        'simulator': options.simulator
1303      }
1304      test_list = root.ListTests([], path, context, mode)
1305      unclassified_tests += test_list
1306      (cases, unused_rules, all_outcomes) = config.ClassifyTests(test_list, env)
1307      if globally_unused_rules is None:
1308        globally_unused_rules = set(unused_rules)
1309      else:
1310        globally_unused_rules = globally_unused_rules.intersection(unused_rules)
1311      all_cases += cases
1312      all_unused.append(unused_rules)
1313
1314  if options.cat:
1315    visited = set()
1316    for test in unclassified_tests:
1317      key = tuple(test.path)
1318      if key in visited:
1319        continue
1320      visited.add(key)
1321      print "--- begin source: %s ---" % test.GetLabel()
1322      source = test.GetSource().strip()
1323      print source
1324      print "--- end source: %s ---" % test.GetLabel()
1325    return 0
1326
1327  if options.warn_unused:
1328    for rule in globally_unused_rules:
1329      print "Rule for '%s' was not used." % '/'.join([str(s) for s in rule.path])
1330
1331  if options.report:
1332    PrintReport(all_cases)
1333
1334  result = None
1335  def DoSkip(case):
1336    return SKIP in case.outcomes or SLOW in case.outcomes
1337  cases_to_run = [ c for c in all_cases if not DoSkip(c) ]
1338  if len(cases_to_run) == 0:
1339    print "No tests to run."
1340    return 0
1341  else:
1342    try:
1343      start = time.time()
1344      if RunTestCases(cases_to_run, options.progress, options.j):
1345        result = 0
1346      else:
1347        result = 1
1348      duration = time.time() - start
1349    except KeyboardInterrupt:
1350      print "Interrupted"
1351      return 1
1352
1353  if options.time:
1354    # Write the times to stderr to make it easy to separate from the
1355    # test output.
1356    print
1357    sys.stderr.write("--- Total time: %s ---\n" % FormatTime(duration))
1358    timed_tests = [ t.case for t in cases_to_run if not t.case.duration is None ]
1359    timed_tests.sort(lambda a, b: a.CompareTime(b))
1360    index = 1
1361    for entry in timed_tests[:20]:
1362      t = FormatTime(entry.duration)
1363      sys.stderr.write("%4i (%s) %s\n" % (index, t, entry.GetLabel()))
1364      index += 1
1365
1366  return result
1367
1368
1369if __name__ == '__main__':
1370  sys.exit(Main())
1371