• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2#
3# Copyright 2008 the V8 project authors. All rights reserved.
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9#       notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11#       copyright notice, this list of conditions and the following
12#       disclaimer in the documentation and/or other materials provided
13#       with the distribution.
14#     * Neither the name of Google Inc. nor the names of its
15#       contributors may be used to endorse or promote products derived
16#       from this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30
31from __future__ import print_function
32import logging
33import optparse
34import os
35import re
36import signal
37import subprocess
38import sys
39import tempfile
40import time
41import threading
42import utils
43import multiprocessing
44import errno
45import copy
46
47
48if sys.version_info >= (3, 5):
49  from importlib import machinery, util
50  def get_module(name, path):
51    loader_details = (machinery.SourceFileLoader, machinery.SOURCE_SUFFIXES)
52    spec = machinery.FileFinder(path, loader_details).find_spec(name)
53    module = util.module_from_spec(spec)
54    spec.loader.exec_module(module)
55    return module
56else:
57  import imp
58  def get_module(name, path):
59    file = None
60    try:
61      (file, pathname, description) = imp.find_module(name, [path])
62      return imp.load_module(name, file, pathname, description)
63    finally:
64      if file:
65        file.close()
66
67
68from io import open
69from os.path import join, dirname, abspath, basename, isdir, exists
70from datetime import datetime
71try:
72    from queue import Queue, Empty  # Python 3
73except ImportError:
74    from Queue import Queue, Empty  # Python 2
75
76from functools import reduce
77
78try:
79  from urllib.parse import unquote    # Python 3
80except ImportError:
81  from urllib import unquote          # Python 2
82
83
84logger = logging.getLogger('testrunner')
85skip_regex = re.compile(r'# SKIP\S*\s+(.*)', re.IGNORECASE)
86
87VERBOSE = False
88
89os.umask(0o022)
90os.environ['NODE_OPTIONS'] = ''
91
92# ---------------------------------------------
93# --- P r o g r e s s   I n d i c a t o r s ---
94# ---------------------------------------------
95
96
97class ProgressIndicator(object):
98
99  def __init__(self, cases, flaky_tests_mode):
100    self.cases = cases
101    self.serial_id = 0
102    self.flaky_tests_mode = flaky_tests_mode
103    self.parallel_queue = Queue(len(cases))
104    self.sequential_queue = Queue(len(cases))
105    for case in cases:
106      if case.parallel:
107        self.parallel_queue.put_nowait(case)
108      else:
109        self.sequential_queue.put_nowait(case)
110    self.succeeded = 0
111    self.remaining = len(cases)
112    self.total = len(cases)
113    self.failed = [ ]
114    self.flaky_failed = [ ]
115    self.crashed = 0
116    self.lock = threading.Lock()
117    self.shutdown_event = threading.Event()
118
119  def GetFailureOutput(self, failure):
120    output = []
121    if failure.output.stderr:
122      output += ["--- stderr ---" ]
123      output += [failure.output.stderr.strip()]
124    if failure.output.stdout:
125      output += ["--- stdout ---"]
126      output += [failure.output.stdout.strip()]
127    output += ["Command: %s" % EscapeCommand(failure.command)]
128    if failure.HasCrashed():
129      output += ["--- %s ---" % PrintCrashed(failure.output.exit_code)]
130    if failure.HasTimedOut():
131      output += ["--- TIMEOUT ---"]
132    output = "\n".join(output)
133    return output
134
135  def PrintFailureOutput(self, failure):
136    print(self.GetFailureOutput(failure))
137
138  def PrintFailureHeader(self, test):
139    if test.IsNegative():
140      negative_marker = '[negative] '
141    else:
142      negative_marker = ''
143    print("=== %(label)s %(negative)s===" % {
144      'label': test.GetLabel(),
145      'negative': negative_marker
146    })
147    print("Path: %s" % "/".join(test.path))
148
149  def Run(self, tasks):
150    self.Starting()
151    threads = []
152    # Spawn N-1 threads and then use this thread as the last one.
153    # That way -j1 avoids threading altogether which is a nice fallback
154    # in case of threading problems.
155    for i in range(tasks - 1):
156      thread = threading.Thread(target=self.RunSingle, args=[True, i + 1])
157      threads.append(thread)
158      thread.start()
159    try:
160      self.RunSingle(False, 0)
161      # Wait for the remaining threads
162      for thread in threads:
163        # Use a timeout so that signals (ctrl-c) will be processed.
164        thread.join(timeout=1000000)
165    except (KeyboardInterrupt, SystemExit):
166      self.shutdown_event.set()
167    except Exception:
168      # If there's an exception we schedule an interruption for any
169      # remaining threads.
170      self.shutdown_event.set()
171      # ...and then reraise the exception to bail out
172      raise
173    self.Done()
174    return not self.failed
175
176  def RunSingle(self, parallel, thread_id):
177    while not self.shutdown_event.is_set():
178      try:
179        test = self.parallel_queue.get_nowait()
180      except Empty:
181        if parallel:
182          return
183        try:
184          test = self.sequential_queue.get_nowait()
185        except Empty:
186          return
187      case = test
188      case.thread_id = thread_id
189      self.lock.acquire()
190      case.serial_id = self.serial_id
191      self.serial_id += 1
192      self.AboutToRun(case)
193      self.lock.release()
194      try:
195        start = datetime.now()
196        output = case.Run()
197        # SmartOS has a bug that causes unexpected ECONNREFUSED errors.
198        # See https://smartos.org/bugview/OS-2767
199        # If ECONNREFUSED on SmartOS, retry the test one time.
200        if (output.UnexpectedOutput() and
201          sys.platform == 'sunos5' and
202          'ECONNREFUSED' in output.output.stderr):
203            output = case.Run()
204            output.diagnostic.append('ECONNREFUSED received, test retried')
205        case.duration = (datetime.now() - start)
206      except IOError:
207        return
208      if self.shutdown_event.is_set():
209        return
210      self.lock.acquire()
211      if output.UnexpectedOutput():
212        if FLAKY in output.test.outcomes and self.flaky_tests_mode == DONTCARE:
213          self.flaky_failed.append(output)
214        else:
215          self.failed.append(output)
216          if output.HasCrashed():
217            self.crashed += 1
218      else:
219        self.succeeded += 1
220      self.remaining -= 1
221      self.HasRun(output)
222      self.lock.release()
223
224
225def EscapeCommand(command):
226  parts = []
227  for part in command:
228    if ' ' in part:
229      # Escape spaces.  We may need to escape more characters for this
230      # to work properly.
231      parts.append('"%s"' % part)
232    else:
233      parts.append(part)
234  return " ".join(parts)
235
236
237class SimpleProgressIndicator(ProgressIndicator):
238
239  def Starting(self):
240    print('Running %i tests' % len(self.cases))
241
242  def Done(self):
243    print()
244    for failed in self.failed:
245      self.PrintFailureHeader(failed.test)
246      self.PrintFailureOutput(failed)
247    if len(self.failed) == 0:
248      print("===")
249      print("=== All tests succeeded")
250      print("===")
251    else:
252      print()
253      print("===")
254      print("=== %i tests failed" % len(self.failed))
255      if self.crashed > 0:
256        print("=== %i tests CRASHED" % self.crashed)
257      print("===")
258
259
260class VerboseProgressIndicator(SimpleProgressIndicator):
261
262  def AboutToRun(self, case):
263    print('Starting %s...' % case.GetLabel())
264    sys.stdout.flush()
265
266  def HasRun(self, output):
267    if output.UnexpectedOutput():
268      if output.HasCrashed():
269        outcome = 'CRASH'
270      else:
271        outcome = 'FAIL'
272    else:
273      outcome = 'pass'
274    print('Done running %s: %s' % (output.test.GetLabel(), outcome))
275
276
277class DotsProgressIndicator(SimpleProgressIndicator):
278
279  def AboutToRun(self, case):
280    pass
281
282  def HasRun(self, output):
283    total = self.succeeded + len(self.failed)
284    if (total > 1) and (total % 50 == 1):
285      sys.stdout.write('\n')
286    if output.UnexpectedOutput():
287      if output.HasCrashed():
288        sys.stdout.write('C')
289        sys.stdout.flush()
290      elif output.HasTimedOut():
291        sys.stdout.write('T')
292        sys.stdout.flush()
293      else:
294        sys.stdout.write('F')
295        sys.stdout.flush()
296    else:
297      sys.stdout.write('.')
298      sys.stdout.flush()
299
300class ActionsAnnotationProgressIndicator(DotsProgressIndicator):
301  def GetAnnotationInfo(self, test, output):
302    traceback = output.stdout + output.stderr
303    find_full_path = re.search(r' +at .*\(.*%s:([0-9]+):([0-9]+)' % test.file, traceback)
304    col = line = 0
305    if find_full_path:
306        line, col = map(int, find_full_path.groups())
307    root_path = abspath(join(dirname(__file__), '../')) + os.sep
308    filename = test.file.replace(root_path, "")
309    return filename, line, col
310
311  def PrintFailureOutput(self, failure):
312    output = self.GetFailureOutput(failure)
313    filename, line, column = self.GetAnnotationInfo(failure.test, failure.output)
314    print("::error file=%s,line=%d,col=%d::%s" % (filename, line, column, output.replace('\n', '%0A')))
315
316class TapProgressIndicator(SimpleProgressIndicator):
317
318  def _printDiagnostic(self):
319    logger.info('  severity: %s', self.severity)
320    self.exitcode and logger.info('  exitcode: %s', self.exitcode)
321    logger.info('  stack: |-')
322
323    for l in self.traceback.splitlines():
324      logger.info('    ' + l)
325
326  def Starting(self):
327    logger.info('TAP version 13')
328    logger.info('1..%i' % len(self.cases))
329    self._done = 0
330
331  def AboutToRun(self, case):
332    pass
333
334  def HasRun(self, output):
335    self._done += 1
336    self.traceback = ''
337    self.severity = 'ok'
338    self.exitcode = ''
339
340    # Print test name as (for example) "parallel/test-assert".  Tests that are
341    # scraped from the addons documentation are all named test.js, making it
342    # hard to decipher what test is running when only the filename is printed.
343    prefix = abspath(join(dirname(__file__), '../test')) + os.sep
344    command = output.command[-1]
345    command = NormalizePath(command, prefix)
346
347    if output.UnexpectedOutput():
348      status_line = 'not ok %i %s' % (self._done, command)
349      self.severity = 'fail'
350      self.exitcode = output.output.exit_code
351      self.traceback = output.output.stdout + output.output.stderr
352
353      if FLAKY in output.test.outcomes and self.flaky_tests_mode == DONTCARE:
354        status_line = status_line + ' # TODO : Fix flaky test'
355        self.severity = 'flaky'
356
357      logger.info(status_line)
358
359      if output.HasCrashed():
360        self.severity = 'crashed'
361
362      elif output.HasTimedOut():
363        self.severity = 'fail'
364
365    else:
366      skip = skip_regex.search(output.output.stdout)
367      if skip:
368        logger.info(
369          'ok %i %s # skip %s' % (self._done, command, skip.group(1)))
370      else:
371        status_line = 'ok %i %s' % (self._done, command)
372        if FLAKY in output.test.outcomes:
373          status_line = status_line + ' # TODO : Fix flaky test'
374        logger.info(status_line)
375
376      if output.diagnostic:
377        self.severity = 'ok'
378        if isinstance(output.diagnostic, list):
379          self.traceback = '\n'.join(output.diagnostic)
380        else:
381          self.traceback = output.diagnostic
382
383
384    duration = output.test.duration
385
386    # total_seconds() was added in 2.7
387    total_seconds = (duration.microseconds +
388      (duration.seconds + duration.days * 24 * 3600) * 10**6) / 10**6
389
390    # duration_ms is measured in seconds and is read as such by TAP parsers.
391    # It should read as "duration including ms" rather than "duration in ms"
392    logger.info('  ---')
393    logger.info('  duration_ms: %d.%d' %
394      (total_seconds, duration.microseconds / 1000))
395    if self.severity != 'ok' or self.traceback != '':
396      if output.HasTimedOut():
397        self.traceback = 'timeout\n' + output.output.stdout + output.output.stderr
398      self._printDiagnostic()
399    logger.info('  ...')
400
401  def Done(self):
402    pass
403
404class DeoptsCheckProgressIndicator(SimpleProgressIndicator):
405
406  def Starting(self):
407    pass
408
409  def AboutToRun(self, case):
410    pass
411
412  def HasRun(self, output):
413    # Print test name as (for example) "parallel/test-assert".  Tests that are
414    # scraped from the addons documentation are all named test.js, making it
415    # hard to decipher what test is running when only the filename is printed.
416    prefix = abspath(join(dirname(__file__), '../test')) + os.sep
417    command = output.command[-1]
418    command = NormalizePath(command, prefix)
419
420    stdout = output.output.stdout.strip()
421    printed_file = False
422    for line in stdout.splitlines():
423      if (
424        (line.startswith("[aborted optimiz") or line.startswith("[disabled optimiz")) and
425        ("because:" in line or "reason:" in line)
426      ):
427        if not printed_file:
428          printed_file = True
429          print('==== %s ====' % command)
430          self.failed.append(output)
431        print('  %s' % line)
432
433  def Done(self):
434    pass
435
436
437class CompactProgressIndicator(ProgressIndicator):
438
439  def __init__(self, cases, flaky_tests_mode, templates):
440    super(CompactProgressIndicator, self).__init__(cases, flaky_tests_mode)
441    self.templates = templates
442    self.last_status_length = 0
443    self.start_time = time.time()
444
445  def Starting(self):
446    pass
447
448  def Done(self):
449    self.PrintProgress('Done\n')
450
451  def AboutToRun(self, case):
452    self.PrintProgress(case.GetLabel())
453
454  def HasRun(self, output):
455    if output.UnexpectedOutput():
456      self.ClearLine(self.last_status_length)
457      self.PrintFailureHeader(output.test)
458      stdout = output.output.stdout.strip()
459      if len(stdout):
460        print(self.templates['stdout'] % stdout)
461      stderr = output.output.stderr.strip()
462      if len(stderr):
463        print(self.templates['stderr'] % stderr)
464      print("Command: %s" % EscapeCommand(output.command))
465      if output.HasCrashed():
466        print("--- %s ---" % PrintCrashed(output.output.exit_code))
467      if output.HasTimedOut():
468        print("--- TIMEOUT ---")
469
470  def Truncate(self, str, length):
471    if length and (len(str) > (length - 3)):
472      return str[:(length-3)] + "..."
473    else:
474      return str
475
476  def PrintProgress(self, name):
477    self.ClearLine(self.last_status_length)
478    elapsed = time.time() - self.start_time
479    status = self.templates['status_line'] % {
480      'passed': self.succeeded,
481      'remaining': (((self.total - self.remaining) * 100) // self.total),
482      'failed': len(self.failed),
483      'test': name,
484      'mins': int(elapsed) / 60,
485      'secs': int(elapsed) % 60
486    }
487    status = self.Truncate(status, 78)
488    self.last_status_length = len(status)
489    print(status, end='')
490    sys.stdout.flush()
491
492
493class ColorProgressIndicator(CompactProgressIndicator):
494
495  def __init__(self, cases, flaky_tests_mode):
496    templates = {
497      'status_line': "[%(mins)02i:%(secs)02i|\033[34m%%%(remaining) 4d\033[0m|\033[32m+%(passed) 4d\033[0m|\033[31m-%(failed) 4d\033[0m]: %(test)s",
498      'stdout': "\033[1m%s\033[0m",
499      'stderr': "\033[31m%s\033[0m",
500    }
501    super(ColorProgressIndicator, self).__init__(cases, flaky_tests_mode, templates)
502
503  def ClearLine(self, last_line_length):
504    print("\033[1K\r", end='')
505
506
507class MonochromeProgressIndicator(CompactProgressIndicator):
508
509  def __init__(self, cases, flaky_tests_mode):
510    templates = {
511      'status_line': "[%(mins)02i:%(secs)02i|%%%(remaining) 4d|+%(passed) 4d|-%(failed) 4d]: %(test)s",
512      'stdout': '%s',
513      'stderr': '%s',
514      'clear': lambda last_line_length: ("\r" + (" " * last_line_length) + "\r"),
515      'max_length': 78
516    }
517    super(MonochromeProgressIndicator, self).__init__(cases, flaky_tests_mode, templates)
518
519  def ClearLine(self, last_line_length):
520    print(("\r" + (" " * last_line_length) + "\r"), end='')
521
522
523PROGRESS_INDICATORS = {
524  'verbose': VerboseProgressIndicator,
525  'dots': DotsProgressIndicator,
526  'actions': ActionsAnnotationProgressIndicator,
527  'color': ColorProgressIndicator,
528  'tap': TapProgressIndicator,
529  'mono': MonochromeProgressIndicator,
530  'deopts': DeoptsCheckProgressIndicator
531}
532
533
534# -------------------------
535# --- F r a m e w o r k ---
536# -------------------------
537
538
539class CommandOutput(object):
540
541  def __init__(self, exit_code, timed_out, stdout, stderr):
542    self.exit_code = exit_code
543    self.timed_out = timed_out
544    self.stdout = stdout
545    self.stderr = stderr
546    self.failed = None
547
548
549class TestCase(object):
550
551  def __init__(self, context, path, arch, mode):
552    self.path = path
553    self.context = context
554    self.duration = None
555    self.arch = arch
556    self.mode = mode
557    self.parallel = False
558    self.disable_core_files = False
559    self.serial_id = 0
560    self.thread_id = 0
561
562  def IsNegative(self):
563    return self.context.expect_fail
564
565  def DidFail(self, output):
566    if output.failed is None:
567      output.failed = self.IsFailureOutput(output)
568    return output.failed
569
570  def IsFailureOutput(self, output):
571    return output.exit_code != 0
572
573  def GetSource(self):
574    return "(no source available)"
575
576  def RunCommand(self, command, env):
577    full_command = self.context.processor(command)
578    output = Execute(full_command,
579                     self.context,
580                     self.context.GetTimeout(self.mode),
581                     env,
582                     disable_core_files = self.disable_core_files)
583    return TestOutput(self,
584                      full_command,
585                      output,
586                      self.context.store_unexpected_output)
587
588  def Run(self):
589    try:
590      result = self.RunCommand(self.GetCommand(), {
591        "TEST_SERIAL_ID": "%d" % self.serial_id,
592        "TEST_THREAD_ID": "%d" % self.thread_id,
593        "TEST_PARALLEL" : "%d" % self.parallel
594      })
595    finally:
596      # Tests can leave the tty in non-blocking mode. If the test runner
597      # tries to print to stdout/stderr after that and the tty buffer is
598      # full, it'll die with a EAGAIN OSError. Ergo, put the tty back in
599      # blocking mode before proceeding.
600      if sys.platform != 'win32':
601        from fcntl import fcntl, F_GETFL, F_SETFL
602        from os import O_NONBLOCK
603        for fd in 0,1,2: fcntl(fd, F_SETFL, ~O_NONBLOCK & fcntl(fd, F_GETFL))
604
605    return result
606
607
608class TestOutput(object):
609
610  def __init__(self, test, command, output, store_unexpected_output):
611    self.test = test
612    self.command = command
613    self.output = output
614    self.store_unexpected_output = store_unexpected_output
615    self.diagnostic = []
616
617  def UnexpectedOutput(self):
618    if self.HasCrashed():
619      outcome = CRASH
620    elif self.HasTimedOut():
621      outcome = TIMEOUT
622    elif self.HasFailed():
623      outcome = FAIL
624    else:
625      outcome = PASS
626    return not outcome in self.test.outcomes
627
628  def HasCrashed(self):
629    if utils.IsWindows():
630      return 0x80000000 & self.output.exit_code and not (0x3FFFFF00 & self.output.exit_code)
631    else:
632      # Timed out tests will have exit_code -signal.SIGTERM.
633      if self.output.timed_out:
634        return False
635      return self.output.exit_code < 0
636
637  def HasTimedOut(self):
638    return self.output.timed_out
639
640  def HasFailed(self):
641    execution_failed = self.test.DidFail(self.output)
642    if self.test.IsNegative():
643      return not execution_failed
644    else:
645      return execution_failed
646
647
648def KillProcessWithID(pid, signal_to_send=signal.SIGTERM):
649  if utils.IsWindows():
650    os.popen('taskkill /T /F /PID %d' % pid)
651  else:
652    os.kill(pid, signal_to_send)
653
654
655MAX_SLEEP_TIME = 0.1
656INITIAL_SLEEP_TIME = 0.0001
657SLEEP_TIME_FACTOR = 1.25
658
659SEM_INVALID_VALUE = -1
660SEM_NOGPFAULTERRORBOX = 0x0002 # Microsoft Platform SDK WinBase.h
661
662def Win32SetErrorMode(mode):
663  prev_error_mode = SEM_INVALID_VALUE
664  try:
665    import ctypes
666    prev_error_mode = ctypes.windll.kernel32.SetErrorMode(mode)
667  except ImportError:
668    pass
669  return prev_error_mode
670
671
672def KillTimedOutProcess(context, pid):
673  signal_to_send = signal.SIGTERM
674  if context.abort_on_timeout:
675    # Using SIGABRT here allows the OS to generate a core dump that can be
676    # looked at post-mortem, which helps for investigating failures that are
677    # difficult to reproduce.
678    signal_to_send = signal.SIGABRT
679  KillProcessWithID(pid, signal_to_send)
680
681
682def RunProcess(context, timeout, args, **rest):
683  if context.verbose: print("#", " ".join(args))
684  popen_args = args
685  prev_error_mode = SEM_INVALID_VALUE
686  if utils.IsWindows():
687    if context.suppress_dialogs:
688      # Try to change the error mode to avoid dialogs on fatal errors. Don't
689      # touch any existing error mode flags by merging the existing error mode.
690      # See http://blogs.msdn.com/oldnewthing/archive/2004/07/27/198410.aspx.
691      error_mode = SEM_NOGPFAULTERRORBOX
692      prev_error_mode = Win32SetErrorMode(error_mode)
693      Win32SetErrorMode(error_mode | prev_error_mode)
694
695  process = subprocess.Popen(
696    args = popen_args,
697    **rest
698  )
699  if utils.IsWindows() and context.suppress_dialogs and prev_error_mode != SEM_INVALID_VALUE:
700    Win32SetErrorMode(prev_error_mode)
701  # Compute the end time - if the process crosses this limit we
702  # consider it timed out.
703  if timeout is None: end_time = None
704  else: end_time = time.time() + timeout
705  timed_out = False
706  # Repeatedly check the exit code from the process in a
707  # loop and keep track of whether or not it times out.
708  exit_code = None
709  sleep_time = INITIAL_SLEEP_TIME
710
711  while exit_code is None:
712    if (not end_time is None) and (time.time() >= end_time):
713      # Kill the process and wait for it to exit.
714      KillTimedOutProcess(context, process.pid)
715      exit_code = process.wait()
716      timed_out = True
717    else:
718      exit_code = process.poll()
719      time.sleep(sleep_time)
720      sleep_time = sleep_time * SLEEP_TIME_FACTOR
721      if sleep_time > MAX_SLEEP_TIME:
722        sleep_time = MAX_SLEEP_TIME
723  return (process, exit_code, timed_out)
724
725
726def PrintError(str):
727  sys.stderr.write(str)
728  sys.stderr.write('\n')
729
730
731def CheckedUnlink(name):
732  while True:
733    try:
734      os.unlink(name)
735    except OSError as e:
736      # On Windows unlink() fails if another process (typically a virus scanner
737      # or the indexing service) has the file open. Those processes keep a
738      # file open for a short time only, so yield and try again; it'll succeed.
739      if sys.platform == 'win32' and e.errno == errno.EACCES:
740        time.sleep(0)
741        continue
742      PrintError("os.unlink() " + str(e))
743    break
744
745def Execute(args, context, timeout=None, env=None, disable_core_files=False, stdin=None):
746  (fd_out, outname) = tempfile.mkstemp()
747  (fd_err, errname) = tempfile.mkstemp()
748
749  if env is None:
750    env = {}
751  env_copy = os.environ.copy()
752
753  # Remove NODE_PATH
754  if "NODE_PATH" in env_copy:
755    del env_copy["NODE_PATH"]
756
757  # Remove NODE_REPL_EXTERNAL_MODULE
758  if "NODE_REPL_EXTERNAL_MODULE" in env_copy:
759    del env_copy["NODE_REPL_EXTERNAL_MODULE"]
760
761  # Extend environment
762  for key, value in env.items():
763    env_copy[key] = value
764
765  preexec_fn = None
766
767  if disable_core_files and not utils.IsWindows():
768    def disableCoreFiles():
769      import resource
770      resource.setrlimit(resource.RLIMIT_CORE, (0,0))
771    preexec_fn = disableCoreFiles
772
773  (process, exit_code, timed_out) = RunProcess(
774    context,
775    timeout,
776    args = args,
777    stdin = stdin,
778    stdout = fd_out,
779    stderr = fd_err,
780    env = env_copy,
781    preexec_fn = preexec_fn
782  )
783  os.close(fd_out)
784  os.close(fd_err)
785  output = open(outname, encoding='utf8').read()
786  errors = open(errname, encoding='utf8').read()
787  CheckedUnlink(outname)
788  CheckedUnlink(errname)
789
790  return CommandOutput(exit_code, timed_out, output, errors)
791
792
793def CarCdr(path):
794  if len(path) == 0:
795    return (None, [ ])
796  else:
797    return (path[0], path[1:])
798
799
800class TestConfiguration(object):
801  def __init__(self, context, root, section):
802    self.context = context
803    self.root = root
804    self.section = section
805
806  def Contains(self, path, file):
807    if len(path) > len(file):
808      return False
809    for i in range(len(path)):
810      if not path[i].match(NormalizePath(file[i])):
811        return False
812    return True
813
814  def GetTestStatus(self, sections, defs):
815    status_file = join(self.root, '%s.status' % self.section)
816    if exists(status_file):
817      ReadConfigurationInto(status_file, sections, defs)
818
819
820class TestSuite(object):
821
822  def __init__(self, name):
823    self.name = name
824
825  def GetName(self):
826    return self.name
827
828
829class TestRepository(TestSuite):
830
831  def __init__(self, path):
832    normalized_path = abspath(path)
833    super(TestRepository, self).__init__(basename(normalized_path))
834    self.path = normalized_path
835    self.is_loaded = False
836    self.config = None
837
838  def GetConfiguration(self, context):
839    if self.is_loaded:
840      return self.config
841    self.is_loaded = True
842
843    module = get_module('testcfg', self.path)
844    self.config = module.GetConfiguration(context, self.path)
845    if hasattr(self.config, 'additional_flags'):
846      self.config.additional_flags += context.node_args
847    else:
848      self.config.additional_flags = context.node_args
849    return self.config
850
851  def GetBuildRequirements(self, path, context):
852    return self.GetConfiguration(context).GetBuildRequirements()
853
854  def AddTestsToList(self, result, current_path, path, context, arch, mode):
855    tests = self.GetConfiguration(context).ListTests(current_path, path,
856                                                     arch, mode)
857    result += tests
858    for i in range(1, context.repeat):
859      result += copy.deepcopy(tests)
860
861  def GetTestStatus(self, context, sections, defs):
862    self.GetConfiguration(context).GetTestStatus(sections, defs)
863
864
865class LiteralTestSuite(TestSuite):
866  def __init__(self, tests_repos, test_root):
867    super(LiteralTestSuite, self).__init__('root')
868    self.tests_repos = tests_repos
869    self.test_root = test_root
870
871  def GetBuildRequirements(self, path, context):
872    (name, rest) = CarCdr(path)
873    result = [ ]
874    for test in self.tests_repos:
875      if not name or name.match(test.GetName()):
876        result += test.GetBuildRequirements(rest, context)
877    return result
878
879  def ListTests(self, current_path, path, context, arch, mode):
880    (name, rest) = CarCdr(path)
881    result = [ ]
882    for test in self.tests_repos:
883      test_name = test.GetName()
884      if not name or name.match(test_name):
885        full_path = current_path + [test_name]
886        test.AddTestsToList(result, full_path, path, context, arch, mode)
887    result.sort(key=lambda x: x.GetName())
888    return result
889
890  def GetTestStatus(self, context, sections, defs):
891    # Just read the test configuration from root_path/root.status.
892    root = TestConfiguration(context, self.test_root, 'root')
893    root.GetTestStatus(sections, defs)
894    for tests_repos in self.tests_repos:
895      tests_repos.GetTestStatus(context, sections, defs)
896
897
898TIMEOUT_SCALEFACTOR = {
899    'armv6' : { 'debug' : 12, 'release' : 3 },  # The ARM buildbots are slow.
900    'arm'   : { 'debug' :  8, 'release' : 2 },
901    'ia32'  : { 'debug' :  4, 'release' : 1 },
902    'ppc'   : { 'debug' :  4, 'release' : 1 },
903    's390'  : { 'debug' :  4, 'release' : 1 } }
904
905
906class Context(object):
907
908  def __init__(self, workspace, verbose, vm, args, expect_fail,
909               timeout, processor, suppress_dialogs,
910               store_unexpected_output, repeat, abort_on_timeout):
911    self.workspace = workspace
912    self.verbose = verbose
913    self.vm = vm
914    self.node_args = args
915    self.expect_fail = expect_fail
916    self.timeout = timeout
917    self.processor = processor
918    self.suppress_dialogs = suppress_dialogs
919    self.store_unexpected_output = store_unexpected_output
920    self.repeat = repeat
921    self.abort_on_timeout = abort_on_timeout
922    self.v8_enable_inspector = True
923    self.node_has_crypto = True
924
925  def GetVm(self, arch, mode):
926    if self.vm is not None:
927      return self.vm
928    if arch == 'none':
929      name = 'out/Debug/node' if mode == 'debug' else 'out/Release/node'
930    else:
931      name = 'out/%s.%s/node' % (arch, mode)
932
933    # Currently GYP does not support output_dir for MSVS.
934    # http://code.google.com/p/gyp/issues/detail?id=40
935    # It will put the builds into Release/node.exe or Debug/node.exe
936    if utils.IsWindows():
937      if not exists(name + '.exe'):
938        name = name.replace('out/', '')
939      name = os.path.abspath(name + '.exe')
940
941    if not exists(name):
942      raise ValueError('Could not find executable. Should be ' + name)
943
944    return name
945
946  def GetTimeout(self, mode):
947    return self.timeout * TIMEOUT_SCALEFACTOR[ARCH_GUESS or 'ia32'][mode]
948
949def RunTestCases(cases_to_run, progress, tasks, flaky_tests_mode):
950  progress = PROGRESS_INDICATORS[progress](cases_to_run, flaky_tests_mode)
951  return progress.Run(tasks)
952
953# -------------------------------------------
954# --- T e s t   C o n f i g u r a t i o n ---
955# -------------------------------------------
956
957
958RUN = 'run'
959SKIP = 'skip'
960FAIL = 'fail'
961PASS = 'pass'
962OKAY = 'okay'
963TIMEOUT = 'timeout'
964CRASH = 'crash'
965SLOW = 'slow'
966FLAKY = 'flaky'
967DONTCARE = 'dontcare'
968
969class Expression(object):
970  pass
971
972
973class Constant(Expression):
974
975  def __init__(self, value):
976    self.value = value
977
978  def Evaluate(self, env, defs):
979    return self.value
980
981
982class Variable(Expression):
983
984  def __init__(self, name):
985    self.name = name
986
987  def GetOutcomes(self, env, defs):
988    if self.name in env: return set([env[self.name]])
989    else: return set()
990
991
992class Outcome(Expression):
993
994  def __init__(self, name):
995    self.name = name
996
997  def GetOutcomes(self, env, defs):
998    if self.name in defs:
999      return defs[self.name].GetOutcomes(env, defs)
1000    else:
1001      return set([self.name])
1002
1003
1004class Operation(Expression):
1005
1006  def __init__(self, left, op, right):
1007    self.left = left
1008    self.op = op
1009    self.right = right
1010
1011  def Evaluate(self, env, defs):
1012    if self.op == '||' or self.op == ',':
1013      return self.left.Evaluate(env, defs) or self.right.Evaluate(env, defs)
1014    elif self.op == 'if':
1015      return False
1016    elif self.op == '==':
1017      inter = self.left.GetOutcomes(env, defs) & self.right.GetOutcomes(env, defs)
1018      return bool(inter)
1019    else:
1020      assert self.op == '&&'
1021      return self.left.Evaluate(env, defs) and self.right.Evaluate(env, defs)
1022
1023  def GetOutcomes(self, env, defs):
1024    if self.op == '||' or self.op == ',':
1025      return self.left.GetOutcomes(env, defs) | self.right.GetOutcomes(env, defs)
1026    elif self.op == 'if':
1027      if self.right.Evaluate(env, defs):
1028        return self.left.GetOutcomes(env, defs)
1029      else:
1030        return set()
1031    else:
1032      assert self.op == '&&'
1033      return self.left.GetOutcomes(env, defs) & self.right.GetOutcomes(env, defs)
1034
1035
1036def IsAlpha(str):
1037  for char in str:
1038    if not (char.isalpha() or char.isdigit() or char == '_'):
1039      return False
1040  return True
1041
1042
1043class Tokenizer(object):
1044  """A simple string tokenizer that chops expressions into variables,
1045  parens and operators"""
1046
1047  def __init__(self, expr):
1048    self.index = 0
1049    self.expr = expr
1050    self.length = len(expr)
1051    self.tokens = None
1052
1053  def Current(self, length = 1):
1054    if not self.HasMore(length): return ""
1055    return self.expr[self.index:self.index+length]
1056
1057  def HasMore(self, length = 1):
1058    return self.index < self.length + (length - 1)
1059
1060  def Advance(self, count = 1):
1061    self.index = self.index + count
1062
1063  def AddToken(self, token):
1064    self.tokens.append(token)
1065
1066  def SkipSpaces(self):
1067    while self.HasMore() and self.Current().isspace():
1068      self.Advance()
1069
1070  def Tokenize(self):
1071    self.tokens = [ ]
1072    while self.HasMore():
1073      self.SkipSpaces()
1074      if not self.HasMore():
1075        return None
1076      if self.Current() == '(':
1077        self.AddToken('(')
1078        self.Advance()
1079      elif self.Current() == ')':
1080        self.AddToken(')')
1081        self.Advance()
1082      elif self.Current() == '$':
1083        self.AddToken('$')
1084        self.Advance()
1085      elif self.Current() == ',':
1086        self.AddToken(',')
1087        self.Advance()
1088      elif IsAlpha(self.Current()):
1089        buf = ""
1090        while self.HasMore() and IsAlpha(self.Current()):
1091          buf += self.Current()
1092          self.Advance()
1093        self.AddToken(buf)
1094      elif self.Current(2) == '&&':
1095        self.AddToken('&&')
1096        self.Advance(2)
1097      elif self.Current(2) == '||':
1098        self.AddToken('||')
1099        self.Advance(2)
1100      elif self.Current(2) == '==':
1101        self.AddToken('==')
1102        self.Advance(2)
1103      else:
1104        return None
1105    return self.tokens
1106
1107
1108class Scanner(object):
1109  """A simple scanner that can serve out tokens from a given list"""
1110
1111  def __init__(self, tokens):
1112    self.tokens = tokens
1113    self.length = len(tokens)
1114    self.index = 0
1115
1116  def HasMore(self):
1117    return self.index < self.length
1118
1119  def Current(self):
1120    return self.tokens[self.index]
1121
1122  def Advance(self):
1123    self.index = self.index + 1
1124
1125
1126def ParseAtomicExpression(scan):
1127  if scan.Current() == "true":
1128    scan.Advance()
1129    return Constant(True)
1130  elif scan.Current() == "false":
1131    scan.Advance()
1132    return Constant(False)
1133  elif IsAlpha(scan.Current()):
1134    name = scan.Current()
1135    scan.Advance()
1136    return Outcome(name.lower())
1137  elif scan.Current() == '$':
1138    scan.Advance()
1139    if not IsAlpha(scan.Current()):
1140      return None
1141    name = scan.Current()
1142    scan.Advance()
1143    return Variable(name.lower())
1144  elif scan.Current() == '(':
1145    scan.Advance()
1146    result = ParseLogicalExpression(scan)
1147    if (not result) or (scan.Current() != ')'):
1148      return None
1149    scan.Advance()
1150    return result
1151  else:
1152    return None
1153
1154
1155BINARIES = ['==']
1156def ParseOperatorExpression(scan):
1157  left = ParseAtomicExpression(scan)
1158  if not left: return None
1159  while scan.HasMore() and (scan.Current() in BINARIES):
1160    op = scan.Current()
1161    scan.Advance()
1162    right = ParseOperatorExpression(scan)
1163    if not right:
1164      return None
1165    left = Operation(left, op, right)
1166  return left
1167
1168
1169def ParseConditionalExpression(scan):
1170  left = ParseOperatorExpression(scan)
1171  if not left: return None
1172  while scan.HasMore() and (scan.Current() == 'if'):
1173    scan.Advance()
1174    right = ParseOperatorExpression(scan)
1175    if not right:
1176      return None
1177    left=  Operation(left, 'if', right)
1178  return left
1179
1180
1181LOGICALS = ["&&", "||", ","]
1182def ParseLogicalExpression(scan):
1183  left = ParseConditionalExpression(scan)
1184  if not left: return None
1185  while scan.HasMore() and (scan.Current() in LOGICALS):
1186    op = scan.Current()
1187    scan.Advance()
1188    right = ParseConditionalExpression(scan)
1189    if not right:
1190      return None
1191    left = Operation(left, op, right)
1192  return left
1193
1194
1195def ParseCondition(expr):
1196  """Parses a logical expression into an Expression object"""
1197  tokens = Tokenizer(expr).Tokenize()
1198  if not tokens:
1199    print("Malformed expression: '%s'" % expr)
1200    return None
1201  scan = Scanner(tokens)
1202  ast = ParseLogicalExpression(scan)
1203  if not ast:
1204    print("Malformed expression: '%s'" % expr)
1205    return None
1206  if scan.HasMore():
1207    print("Malformed expression: '%s'" % expr)
1208    return None
1209  return ast
1210
1211
1212class Configuration(object):
1213  """The parsed contents of a configuration file"""
1214
1215  def __init__(self, sections, defs):
1216    self.sections = sections
1217    self.defs = defs
1218
1219  def ClassifyTests(self, cases, env):
1220    sections = [ s for s in self.sections if s.condition.Evaluate(env, self.defs) ]
1221    all_rules = reduce(list.__add__, [s.rules for s in sections], [])
1222    unused_rules = set(all_rules)
1223    result = []
1224    for case in cases:
1225      matches = [ r for r in all_rules if r.Contains(case.path) ]
1226      outcomes_list = [ r.GetOutcomes(env, self.defs) for r in matches ]
1227      outcomes = reduce(set.union, outcomes_list, set())
1228      unused_rules.difference_update(matches)
1229      case.outcomes = set(outcomes) or set([PASS])
1230      # slow tests may also just pass.
1231      if SLOW in case.outcomes:
1232        case.outcomes.add(PASS)
1233      result.append(case)
1234    return result, unused_rules
1235
1236
1237class Section(object):
1238  """A section of the configuration file.  Sections are enabled or
1239  disabled prior to running the tests, based on their conditions"""
1240
1241  def __init__(self, condition):
1242    self.condition = condition
1243    self.rules = [ ]
1244
1245  def AddRule(self, rule):
1246    self.rules.append(rule)
1247
1248
1249class Rule(object):
1250  """A single rule that specifies the expected outcome for a single
1251  test."""
1252
1253  def __init__(self, raw_path, path, value):
1254    self.raw_path = raw_path
1255    self.path = path
1256    self.value = value
1257
1258  def GetOutcomes(self, env, defs):
1259    return self.value.GetOutcomes(env, defs)
1260
1261  def Contains(self, path):
1262    if len(self.path) > len(path):
1263      return False
1264    for i in range(len(self.path)):
1265      if not self.path[i].match(path[i]):
1266        return False
1267    return True
1268
1269
1270HEADER_PATTERN = re.compile(r'\[([^]]+)\]')
1271RULE_PATTERN = re.compile(r'\s*([^: ]*)\s*:(.*)')
1272DEF_PATTERN = re.compile(r'^def\s*(\w+)\s*=(.*)$')
1273PREFIX_PATTERN = re.compile(r'^\s*prefix\s+([\w_.\-/]+)$')
1274
1275
1276def ReadConfigurationInto(path, sections, defs):
1277  current_section = Section(Constant(True))
1278  sections.append(current_section)
1279  prefix = []
1280  for line in utils.ReadLinesFrom(path):
1281    header_match = HEADER_PATTERN.match(line)
1282    if header_match:
1283      condition_str = header_match.group(1).strip()
1284      condition = ParseCondition(condition_str)
1285      new_section = Section(condition)
1286      sections.append(new_section)
1287      current_section = new_section
1288      continue
1289    rule_match = RULE_PATTERN.match(line)
1290    if rule_match:
1291      path = prefix + SplitPath(rule_match.group(1).strip())
1292      value_str = rule_match.group(2).strip()
1293      value = ParseCondition(value_str)
1294      if not value:
1295        return False
1296      current_section.AddRule(Rule(rule_match.group(1), path, value))
1297      continue
1298    def_match = DEF_PATTERN.match(line)
1299    if def_match:
1300      name = def_match.group(1).lower()
1301      value = ParseCondition(def_match.group(2).strip())
1302      if not value:
1303        return False
1304      defs[name] = value
1305      continue
1306    prefix_match = PREFIX_PATTERN.match(line)
1307    if prefix_match:
1308      prefix = SplitPath(prefix_match.group(1).strip())
1309      continue
1310    raise Exception("Malformed line: '%s'." % line)
1311
1312
1313# ---------------
1314# --- M a i n ---
1315# ---------------
1316
1317
1318ARCH_GUESS = utils.GuessArchitecture()
1319
1320
1321def BuildOptions():
1322  result = optparse.OptionParser()
1323  result.add_option("-m", "--mode", help="The test modes in which to run (comma-separated)",
1324      default='release')
1325  result.add_option("-v", "--verbose", help="Verbose output",
1326      default=False, action="store_true")
1327  result.add_option('--logfile', dest='logfile',
1328      help='write test output to file. NOTE: this only applies the tap progress indicator')
1329  result.add_option("-p", "--progress",
1330      help="The style of progress indicator (%s)" % ", ".join(PROGRESS_INDICATORS.keys()),
1331      choices=list(PROGRESS_INDICATORS.keys()), default="mono")
1332  result.add_option("--report", help="Print a summary of the tests to be run",
1333      default=False, action="store_true")
1334  result.add_option("-s", "--suite", help="A test suite",
1335      default=[], action="append")
1336  result.add_option("-t", "--timeout", help="Timeout in seconds",
1337      default=120, type="int")
1338  result.add_option("--arch", help='The architecture to run tests for',
1339      default='none')
1340  result.add_option("--snapshot", help="Run the tests with snapshot turned on",
1341      default=False, action="store_true")
1342  result.add_option("--special-command", default=None)
1343  result.add_option("--node-args", dest="node_args", help="Args to pass through to Node",
1344      default=[], action="append")
1345  result.add_option("--expect-fail", dest="expect_fail",
1346      help="Expect test cases to fail", default=False, action="store_true")
1347  result.add_option("--valgrind", help="Run tests through valgrind",
1348      default=False, action="store_true")
1349  result.add_option("--worker", help="Run parallel tests inside a worker context",
1350      default=False, action="store_true")
1351  result.add_option("--check-deopts", help="Check tests for permanent deoptimizations",
1352      default=False, action="store_true")
1353  result.add_option("--cat", help="Print the source of the tests",
1354      default=False, action="store_true")
1355  result.add_option("--flaky-tests",
1356      help="Regard tests marked as flaky (run|skip|dontcare)",
1357      default="run")
1358  result.add_option("--skip-tests",
1359      help="Tests that should not be executed (comma-separated)",
1360      default="")
1361  result.add_option("--warn-unused", help="Report unused rules",
1362      default=False, action="store_true")
1363  result.add_option("-j", help="The number of parallel tasks to run",
1364      default=1, type="int")
1365  result.add_option("-J", help="Run tasks in parallel on all cores",
1366      default=False, action="store_true")
1367  result.add_option("--time", help="Print timing information after running",
1368      default=False, action="store_true")
1369  result.add_option("--suppress-dialogs", help="Suppress Windows dialogs for crashing tests",
1370        dest="suppress_dialogs", default=True, action="store_true")
1371  result.add_option("--no-suppress-dialogs", help="Display Windows dialogs for crashing tests",
1372        dest="suppress_dialogs", action="store_false")
1373  result.add_option("--shell", help="Path to node executable", default=None)
1374  result.add_option("--store-unexpected-output",
1375      help="Store the temporary JS files from tests that fails",
1376      dest="store_unexpected_output", default=True, action="store_true")
1377  result.add_option("--no-store-unexpected-output",
1378      help="Deletes the temporary JS files from tests that fails",
1379      dest="store_unexpected_output", action="store_false")
1380  result.add_option("-r", "--run",
1381      help="Divide the tests in m groups (interleaved) and run tests from group n (--run=n,m with n < m)",
1382      default="")
1383  result.add_option('--temp-dir',
1384      help='Optional path to change directory used for tests', default=False)
1385  result.add_option('--test-root',
1386      help='Optional path to change test directory', dest='test_root', default=None)
1387  result.add_option('--repeat',
1388      help='Number of times to repeat given tests',
1389      default=1, type="int")
1390  result.add_option('--abort-on-timeout',
1391      help='Send SIGABRT instead of SIGTERM to kill processes that time out',
1392      default=False, action="store_true", dest="abort_on_timeout")
1393  result.add_option("--type",
1394      help="Type of build (simple, fips, coverage)",
1395      default=None)
1396  return result
1397
1398
1399def ProcessOptions(options):
1400  global VERBOSE
1401  VERBOSE = options.verbose
1402  options.arch = options.arch.split(',')
1403  options.mode = options.mode.split(',')
1404  options.run = options.run.split(',')
1405  # Split at commas and filter out all the empty strings.
1406  options.skip_tests = [test for test in options.skip_tests.split(',') if test]
1407  if options.run == [""]:
1408    options.run = None
1409  elif len(options.run) != 2:
1410    print("The run argument must be two comma-separated integers.")
1411    return False
1412  else:
1413    try:
1414      options.run = [int(level) for level in options.run]
1415    except ValueError:
1416      print("Could not parse the integers from the run argument.")
1417      return False
1418    if options.run[0] < 0 or options.run[1] < 0:
1419      print("The run argument cannot have negative integers.")
1420      return False
1421    if options.run[0] >= options.run[1]:
1422      print("The test group to run (n) must be smaller than number of groups (m).")
1423      return False
1424  if options.J:
1425    # inherit JOBS from environment if provided. some virtualised systems
1426    # tends to exaggerate the number of available cpus/cores.
1427    cores = os.environ.get('JOBS')
1428    options.j = int(cores) if cores is not None else multiprocessing.cpu_count()
1429  if options.flaky_tests not in [RUN, SKIP, DONTCARE]:
1430    print("Unknown flaky-tests mode %s" % options.flaky_tests)
1431    return False
1432  return True
1433
1434
1435REPORT_TEMPLATE = """\
1436Total: %(total)i tests
1437 * %(skipped)4d tests will be skipped
1438 * %(pass)4d tests are expected to pass
1439 * %(fail_ok)4d tests are expected to fail that we won't fix
1440 * %(fail)4d tests are expected to fail that we should fix\
1441"""
1442
1443
1444class Pattern(object):
1445
1446  def __init__(self, pattern):
1447    self.pattern = pattern
1448    self.compiled = None
1449
1450  def match(self, str):
1451    if not self.compiled:
1452      pattern = "^" + self.pattern.replace('*', '.*') + "$"
1453      self.compiled = re.compile(pattern)
1454    return self.compiled.match(str)
1455
1456  def __str__(self):
1457    return self.pattern
1458
1459
1460def SplitPath(path_arg):
1461  stripped = [c.strip() for c in path_arg.split('/')]
1462  return [Pattern(s) for s in stripped if len(s) > 0]
1463
1464def NormalizePath(path, prefix='test/'):
1465  # strip the extra path information of the specified test
1466  prefix = prefix.replace('\\', '/')
1467  path = path.replace('\\', '/')
1468  if path.startswith(prefix):
1469    path = path[len(prefix):]
1470  if path.endswith('.js'):
1471    path = path[:-3]
1472  elif path.endswith('.mjs'):
1473    path = path[:-4]
1474  return path
1475
1476def GetSpecialCommandProcessor(value):
1477  if (not value) or (value.find('@') == -1):
1478    def ExpandCommand(args):
1479      return args
1480    return ExpandCommand
1481  else:
1482    prefix, _, suffix = value.partition('@')
1483    prefix = unquote(prefix).split()
1484    suffix = unquote(suffix).split()
1485    def ExpandCommand(args):
1486      return prefix + args + suffix
1487    return ExpandCommand
1488
1489def GetSuites(test_root):
1490  def IsSuite(path):
1491    return isdir(path) and exists(join(path, 'testcfg.py'))
1492  return [ f for f in os.listdir(test_root) if IsSuite(join(test_root, f)) ]
1493
1494
1495def FormatTime(d):
1496  millis = round(d * 1000) % 1000
1497  return time.strftime("%M:%S.", time.gmtime(d)) + ("%03i" % millis)
1498
1499
1500def FormatTimedelta(td):
1501  if hasattr(td, 'total_seconds'):
1502    d = td.total_seconds()
1503  else: # python2.6 compat
1504    d =  td.seconds + (td.microseconds / 10.0**6)
1505  return FormatTime(d)
1506
1507
1508def PrintCrashed(code):
1509  if utils.IsWindows():
1510    return "CRASHED"
1511  else:
1512    return "CRASHED (Signal: %d)" % -code
1513
1514
1515# these suites represent special cases that should not be run as part of the
1516# default JavaScript test-run, e.g., internet/ requires a network connection,
1517# addons/ requires compilation.
1518IGNORED_SUITES = [
1519  'addons',
1520  'benchmark',
1521  'doctool',
1522  'embedding',
1523  'internet',
1524  'js-native-api',
1525  'node-api',
1526  'pummel',
1527  'tick-processor',
1528  'v8-updates'
1529]
1530
1531
1532def ArgsToTestPaths(test_root, args, suites):
1533  if len(args) == 0 or 'default' in args:
1534    def_suites = [s for s in suites if s not in IGNORED_SUITES]
1535    args = [a for a in args if a != 'default'] + def_suites
1536  subsystem_regex = re.compile(r'^[a-zA-Z-]*$')
1537  check = lambda arg: subsystem_regex.match(arg) and (arg not in suites)
1538  mapped_args = ["*/test*-%s-*" % arg if check(arg) else arg for arg in args]
1539  paths = [SplitPath(NormalizePath(a)) for a in mapped_args]
1540  return paths
1541
1542
1543def get_env_type(vm, options_type, context):
1544  if options_type is not None:
1545    env_type = options_type
1546  else:
1547    # 'simple' is the default value for 'env_type'.
1548    env_type = 'simple'
1549    ssl_ver = Execute([vm, '-p', 'process.versions.openssl'], context).stdout
1550    if 'fips' in ssl_ver:
1551      env_type = 'fips'
1552  return env_type
1553
1554
1555def Main():
1556  parser = BuildOptions()
1557  (options, args) = parser.parse_args()
1558  if not ProcessOptions(options):
1559    parser.print_help()
1560    return 1
1561
1562  ch = logging.StreamHandler(sys.stdout)
1563  logger.addHandler(ch)
1564  logger.setLevel(logging.INFO)
1565  if options.logfile:
1566    fh = logging.FileHandler(options.logfile, encoding='utf-8', mode='w')
1567    logger.addHandler(fh)
1568
1569  workspace = abspath(join(dirname(sys.argv[0]), '..'))
1570  test_root = join(workspace, 'test')
1571  if options.test_root is not None:
1572    test_root = options.test_root
1573  suites = GetSuites(test_root)
1574  repositories = [TestRepository(join(test_root, name)) for name in suites]
1575  repositories += [TestRepository(a) for a in options.suite]
1576
1577  root = LiteralTestSuite(repositories, test_root)
1578  paths = ArgsToTestPaths(test_root, args, suites)
1579
1580  # Check for --valgrind option. If enabled, we overwrite the special
1581  # command flag with a command that uses the run-valgrind.py script.
1582  if options.valgrind:
1583    run_valgrind = join(workspace, "tools", "run-valgrind.py")
1584    options.special_command = "python -u " + run_valgrind + " @"
1585
1586  if options.check_deopts:
1587    options.node_args.append("--trace-opt")
1588    options.node_args.append("--trace-file-names")
1589    # --always-opt is needed because many tests do not run long enough for the
1590    # optimizer to kick in, so this flag will force it to run.
1591    options.node_args.append("--always-opt")
1592    options.progress = "deopts"
1593
1594  if options.worker:
1595    run_worker = join(workspace, "tools", "run-worker.js")
1596    options.node_args.append(run_worker)
1597
1598  processor = GetSpecialCommandProcessor(options.special_command)
1599
1600  context = Context(workspace,
1601                    VERBOSE,
1602                    options.shell,
1603                    options.node_args,
1604                    options.expect_fail,
1605                    options.timeout,
1606                    processor,
1607                    options.suppress_dialogs,
1608                    options.store_unexpected_output,
1609                    options.repeat,
1610                    options.abort_on_timeout)
1611
1612  # Get status for tests
1613  sections = [ ]
1614  defs = { }
1615  root.GetTestStatus(context, sections, defs)
1616  config = Configuration(sections, defs)
1617
1618  # List the tests
1619  all_cases = [ ]
1620  all_unused = [ ]
1621  unclassified_tests = [ ]
1622  globally_unused_rules = None
1623  for path in paths:
1624    for arch in options.arch:
1625      for mode in options.mode:
1626        vm = context.GetVm(arch, mode)
1627        if not exists(vm):
1628          print("Can't find shell executable: '%s'" % vm)
1629          continue
1630        archEngineContext = Execute([vm, "-p", "process.arch"], context)
1631        vmArch = archEngineContext.stdout.rstrip()
1632        if archEngineContext.exit_code != 0 or vmArch == "undefined":
1633          print("Can't determine the arch of: '%s'" % vm)
1634          print(archEngineContext.stderr.rstrip())
1635          continue
1636        env = {
1637          'mode': mode,
1638          'system': utils.GuessOS(),
1639          'arch': vmArch,
1640          'type': get_env_type(vm, options.type, context),
1641        }
1642        test_list = root.ListTests([], path, context, arch, mode)
1643        unclassified_tests += test_list
1644        cases, unused_rules = config.ClassifyTests(test_list, env)
1645        if globally_unused_rules is None:
1646          globally_unused_rules = set(unused_rules)
1647        else:
1648          globally_unused_rules = (
1649              globally_unused_rules.intersection(unused_rules))
1650        all_cases += cases
1651        all_unused.append(unused_rules)
1652
1653  # We want to skip the inspector tests if node was built without the inspector.
1654  has_inspector = Execute([vm,
1655      '-p', 'process.features.inspector'], context)
1656  if has_inspector.stdout.rstrip() == 'false':
1657    context.v8_enable_inspector = False
1658
1659  has_crypto = Execute([vm,
1660      '-p', 'process.versions.openssl'], context)
1661  if has_crypto.stdout.rstrip() == 'undefined':
1662    context.node_has_crypto = False
1663
1664  if options.cat:
1665    visited = set()
1666    for test in unclassified_tests:
1667      key = tuple(test.path)
1668      if key in visited:
1669        continue
1670      visited.add(key)
1671      print("--- begin source: %s ---" % test.GetLabel())
1672      source = test.GetSource().strip()
1673      print(source)
1674      print("--- end source: %s ---" % test.GetLabel())
1675    return 0
1676
1677  if options.warn_unused:
1678    for rule in globally_unused_rules:
1679      print("Rule for '%s' was not used." % '/'.join([str(s) for s in rule.path]))
1680
1681  tempdir = os.environ.get('NODE_TEST_DIR') or options.temp_dir
1682  if tempdir:
1683    os.environ['NODE_TEST_DIR'] = tempdir
1684    try:
1685      os.makedirs(tempdir)
1686    except OSError as exception:
1687      if exception.errno != errno.EEXIST:
1688        print("Could not create the temporary directory", options.temp_dir)
1689        sys.exit(1)
1690
1691  def should_keep(case):
1692    if any((s in case.file) for s in options.skip_tests):
1693      return False
1694    elif SKIP in case.outcomes:
1695      return False
1696    elif (options.flaky_tests == SKIP) and (set([SLOW, FLAKY]) & case.outcomes):
1697      return False
1698    else:
1699      return True
1700
1701  cases_to_run = [
1702    test_case for test_case in all_cases if should_keep(test_case)
1703  ]
1704
1705  if options.report:
1706    print(REPORT_TEMPLATE % {
1707      'total': len(all_cases),
1708      'skipped': len(all_cases) - len(cases_to_run),
1709      'pass': len([t for t in cases_to_run if PASS in t.outcomes]),
1710      'fail_ok': len([t for t in cases_to_run if t.outcomes == set([FAIL, OKAY])]),
1711      'fail': len([t for t in cases_to_run if t.outcomes == set([FAIL])])
1712    })
1713
1714  if options.run is not None:
1715    # Must ensure the list of tests is sorted before selecting, to avoid
1716    # silent errors if this file is changed to list the tests in a way that
1717    # can be different in different machines
1718    cases_to_run.sort(key=lambda c: (c.arch, c.mode, c.file))
1719    cases_to_run = [ cases_to_run[i] for i
1720                     in range(options.run[0],
1721                               len(cases_to_run),
1722                               options.run[1]) ]
1723  if len(cases_to_run) == 0:
1724    print("No tests to run.")
1725    return 1
1726  else:
1727    try:
1728      start = time.time()
1729      if RunTestCases(cases_to_run, options.progress, options.j, options.flaky_tests):
1730        result = 0
1731      else:
1732        result = 1
1733      duration = time.time() - start
1734    except KeyboardInterrupt:
1735      print("Interrupted")
1736      return 1
1737
1738  if options.time:
1739    # Write the times to stderr to make it easy to separate from the
1740    # test output.
1741    print()
1742    sys.stderr.write("--- Total time: %s ---\n" % FormatTime(duration))
1743    timed_tests = [ t for t in cases_to_run if not t.duration is None ]
1744    timed_tests.sort(key=lambda x: x.duration)
1745    for i, entry in enumerate(timed_tests[:20], start=1):
1746      t = FormatTimedelta(entry.duration)
1747      sys.stderr.write("%4i (%s) %s\n" % (i, t, entry.GetLabel()))
1748
1749  return result
1750
1751
1752if __name__ == '__main__':
1753  sys.exit(Main())
1754