• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
3#
4# Use of this source code is governed by a BSD-style license
5# that can be found in the LICENSE file in the root of the source
6# tree. An additional intellectual property rights grant can be found
7# in the file PATENTS.  All contributing project authors may
8# be found in the AUTHORS file in the root of the source tree.
9
10"""Runs an exe through Valgrind and puts the intermediate files in a
11directory.
12"""
13
14import datetime
15import glob
16import logging
17import optparse
18import os
19import re
20import shutil
21import stat
22import subprocess
23import sys
24import tempfile
25
26import common
27
28import memcheck_analyze
29
30class BaseTool(object):
31  """Abstract class for running dynamic error detection tools.
32
33  Always subclass this and implement ToolCommand with framework- and
34  tool-specific stuff.
35  """
36
37  def __init__(self):
38    temp_parent_dir = None
39    self.log_parent_dir = ""
40    if common.IsWindows():
41      # gpu process on Windows Vista+ runs at Low Integrity and can only
42      # write to certain directories (http://crbug.com/119131)
43      #
44      # TODO(bruening): if scripts die in middle and don't clean up temp
45      # dir, we'll accumulate files in profile dir.  should remove
46      # really old files automatically.
47      profile = os.getenv("USERPROFILE")
48      if profile:
49        self.log_parent_dir = profile + "\\AppData\\LocalLow\\"
50        if os.path.exists(self.log_parent_dir):
51          self.log_parent_dir = common.NormalizeWindowsPath(self.log_parent_dir)
52          temp_parent_dir = self.log_parent_dir
53    # Generated every time (even when overridden)
54    self.temp_dir = tempfile.mkdtemp(prefix="vg_logs_", dir=temp_parent_dir)
55    self.log_dir = self.temp_dir # overridable by --keep_logs
56    self.option_parser_hooks = []
57    # TODO(glider): we may not need some of the env vars on some of the
58    # platforms.
59    self._env = {
60      "G_SLICE" : "always-malloc",
61      "NSS_DISABLE_UNLOAD" : "1",
62      "NSS_DISABLE_ARENA_FREE_LIST" : "1",
63      "GTEST_DEATH_TEST_USE_FORK": "1",
64    }
65
66  def ToolName(self):
67    raise NotImplementedError, "This method should be implemented " \
68                               "in the tool-specific subclass"
69
70  def Analyze(self, check_sanity=False):
71    raise NotImplementedError, "This method should be implemented " \
72                               "in the tool-specific subclass"
73
74  def RegisterOptionParserHook(self, hook):
75    # Frameworks and tools can add their own flags to the parser.
76    self.option_parser_hooks.append(hook)
77
78  def CreateOptionParser(self):
79    # Defines Chromium-specific flags.
80    self._parser = optparse.OptionParser("usage: %prog [options] <program to "
81                                         "test>")
82    self._parser.disable_interspersed_args()
83    self._parser.add_option("-t", "--timeout",
84                      dest="timeout", metavar="TIMEOUT", default=10000,
85                      help="timeout in seconds for the run (default 10000)")
86    self._parser.add_option("", "--build-dir",
87                            help="the location of the compiler output")
88    self._parser.add_option("", "--source-dir",
89                            help="path to top of source tree for this build"
90                                 "(used to normalize source paths in baseline)")
91    self._parser.add_option("", "--gtest_filter", default="",
92                            help="which test case to run")
93    self._parser.add_option("", "--gtest_repeat",
94                            help="how many times to run each test")
95    self._parser.add_option("", "--gtest_print_time", action="store_true",
96                            default=False,
97                            help="show how long each test takes")
98    self._parser.add_option("", "--ignore_exit_code", action="store_true",
99                            default=False,
100                            help="ignore exit code of the test "
101                                 "(e.g. test failures)")
102    self._parser.add_option("", "--keep_logs", action="store_true",
103                            default=False,
104                            help="store memory tool logs in the <tool>.logs "
105                                 "directory instead of /tmp.\nThis can be "
106                                 "useful for tool developers/maintainers.\n"
107                                 "Please note that the <tool>.logs directory "
108                                 "will be clobbered on tool startup.")
109
110    # To add framework- or tool-specific flags, please add a hook using
111    # RegisterOptionParserHook in the corresponding subclass.
112    # See ValgrindTool for an example.
113    for hook in self.option_parser_hooks:
114      hook(self, self._parser)
115
116  def ParseArgv(self, args):
117    self.CreateOptionParser()
118
119    # self._tool_flags will store those tool flags which we don't parse
120    # manually in this script.
121    self._tool_flags = []
122    known_args = []
123
124    """ We assume that the first argument not starting with "-" is a program
125    name and all the following flags should be passed to the program.
126    TODO(timurrrr): customize optparse instead
127    """
128    while len(args) > 0 and args[0][:1] == "-":
129      arg = args[0]
130      if (arg == "--"):
131        break
132      if self._parser.has_option(arg.split("=")[0]):
133        known_args += [arg]
134      else:
135        self._tool_flags += [arg]
136      args = args[1:]
137
138    if len(args) > 0:
139      known_args += args
140
141    self._options, self._args = self._parser.parse_args(known_args)
142
143    self._timeout = int(self._options.timeout)
144    self._source_dir = self._options.source_dir
145    if self._options.keep_logs:
146      # log_parent_dir has trailing slash if non-empty
147      self.log_dir = self.log_parent_dir + "%s.logs" % self.ToolName()
148      if os.path.exists(self.log_dir):
149        shutil.rmtree(self.log_dir)
150      os.mkdir(self.log_dir)
151      logging.info("Logs are in " + self.log_dir)
152
153    self._ignore_exit_code = self._options.ignore_exit_code
154    if self._options.gtest_filter != "":
155      self._args.append("--gtest_filter=%s" % self._options.gtest_filter)
156    if self._options.gtest_repeat:
157      self._args.append("--gtest_repeat=%s" % self._options.gtest_repeat)
158    if self._options.gtest_print_time:
159      self._args.append("--gtest_print_time")
160
161    return True
162
163  def Setup(self, args):
164    return self.ParseArgv(args)
165
166  def ToolCommand(self):
167    raise NotImplementedError, "This method should be implemented " \
168                               "in the tool-specific subclass"
169
170  def Cleanup(self):
171    # You may override it in the tool-specific subclass
172    pass
173
174  def Execute(self):
175    """ Execute the app to be tested after successful instrumentation.
176    Full execution command-line provided by subclassers via proc."""
177    logging.info("starting execution...")
178    proc = self.ToolCommand()
179    for var in self._env:
180      common.PutEnvAndLog(var, self._env[var])
181    return common.RunSubprocess(proc, self._timeout)
182
183  def RunTestsAndAnalyze(self, check_sanity):
184    exec_retcode = self.Execute()
185    analyze_retcode = self.Analyze(check_sanity)
186
187    if analyze_retcode:
188      logging.error("Analyze failed.")
189      logging.info("Search the log for '[ERROR]' to see the error reports.")
190      return analyze_retcode
191
192    if exec_retcode:
193      if self._ignore_exit_code:
194        logging.info("Test execution failed, but the exit code is ignored.")
195      else:
196        logging.error("Test execution failed.")
197        return exec_retcode
198    else:
199      logging.info("Test execution completed successfully.")
200
201    if not analyze_retcode:
202      logging.info("Analysis completed successfully.")
203
204    return 0
205
206  def Main(self, args, check_sanity, min_runtime_in_seconds):
207    """Call this to run through the whole process: Setup, Execute, Analyze"""
208    start_time = datetime.datetime.now()
209    retcode = -1
210    if self.Setup(args):
211      retcode = self.RunTestsAndAnalyze(check_sanity)
212      shutil.rmtree(self.temp_dir, ignore_errors=True)
213      self.Cleanup()
214    else:
215      logging.error("Setup failed")
216    end_time = datetime.datetime.now()
217    runtime_in_seconds = (end_time - start_time).seconds
218    hours = runtime_in_seconds / 3600
219    seconds = runtime_in_seconds % 3600
220    minutes = seconds / 60
221    seconds = seconds % 60
222    logging.info("elapsed time: %02d:%02d:%02d" % (hours, minutes, seconds))
223    if (min_runtime_in_seconds > 0 and
224        runtime_in_seconds < min_runtime_in_seconds):
225      logging.error("Layout tests finished too quickly. "
226                    "It should have taken at least %d seconds. "
227                    "Something went wrong?" % min_runtime_in_seconds)
228      retcode = -1
229    return retcode
230
231  def Run(self, args, module, min_runtime_in_seconds=0):
232    MODULES_TO_SANITY_CHECK = ["base"]
233
234    check_sanity = module in MODULES_TO_SANITY_CHECK
235    return self.Main(args, check_sanity, min_runtime_in_seconds)
236
237
238class ValgrindTool(BaseTool):
239  """Abstract class for running Valgrind tools.
240
241  Always subclass this and implement ToolSpecificFlags() and
242  ExtendOptionParser() for tool-specific stuff.
243  """
244  def __init__(self):
245    super(ValgrindTool, self).__init__()
246    self.RegisterOptionParserHook(ValgrindTool.ExtendOptionParser)
247
248  def UseXML(self):
249    # Override if tool prefers nonxml output
250    return True
251
252  def ExtendOptionParser(self, parser):
253    parser.add_option("", "--suppressions", default=[],
254                            action="append",
255                            help="path to a valgrind suppression file")
256    parser.add_option("", "--indirect", action="store_true",
257                            default=False,
258                            help="set BROWSER_WRAPPER rather than "
259                                 "running valgrind directly")
260    parser.add_option("", "--indirect_webkit_layout", action="store_true",
261                            default=False,
262                            help="set --wrapper rather than running Dr. Memory "
263                                 "directly.")
264    parser.add_option("", "--trace_children", action="store_true",
265                            default=False,
266                            help="also trace child processes")
267    parser.add_option("", "--num-callers",
268                            dest="num_callers", default=30,
269                            help="number of callers to show in stack traces")
270    parser.add_option("", "--generate_dsym", action="store_true",
271                          default=False,
272                          help="Generate .dSYM file on Mac if needed. Slow!")
273
274  def Setup(self, args):
275    if not BaseTool.Setup(self, args):
276      return False
277    return True
278
279  def ToolCommand(self):
280    """Get the valgrind command to run."""
281    # Note that self._args begins with the exe to be run.
282    tool_name = self.ToolName()
283
284    # Construct the valgrind command.
285    if 'CHROME_VALGRIND' in os.environ:
286      path = os.path.join(os.environ['CHROME_VALGRIND'], "bin", "valgrind")
287    else:
288      path = "valgrind"
289    proc = [path, "--tool=%s" % tool_name]
290
291    proc += ["--num-callers=%i" % int(self._options.num_callers)]
292
293    if self._options.trace_children:
294      proc += ["--trace-children=yes"]
295      proc += ["--trace-children-skip='*dbus-daemon*'"]
296      proc += ["--trace-children-skip='*dbus-launch*'"]
297      proc += ["--trace-children-skip='*perl*'"]
298      proc += ["--trace-children-skip='*python*'"]
299      # This is really Python, but for some reason Valgrind follows it.
300      proc += ["--trace-children-skip='*lsb_release*'"]
301
302    proc += self.ToolSpecificFlags()
303    proc += self._tool_flags
304
305    suppression_count = 0
306    for suppression_file in self._options.suppressions:
307      if os.path.exists(suppression_file):
308        suppression_count += 1
309        proc += ["--suppressions=%s" % suppression_file]
310
311    if not suppression_count:
312      logging.warning("WARNING: NOT USING SUPPRESSIONS!")
313
314    logfilename = self.log_dir + ("/%s." % tool_name) + "%p"
315    if self.UseXML():
316      proc += ["--xml=yes", "--xml-file=" + logfilename]
317    else:
318      proc += ["--log-file=" + logfilename]
319
320    # The Valgrind command is constructed.
321
322    # Handle --indirect_webkit_layout separately.
323    if self._options.indirect_webkit_layout:
324      # Need to create the wrapper before modifying |proc|.
325      wrapper = self.CreateBrowserWrapper(proc, webkit=True)
326      proc = self._args
327      proc.append("--wrapper")
328      proc.append(wrapper)
329      return proc
330
331    if self._options.indirect:
332      wrapper = self.CreateBrowserWrapper(proc)
333      os.environ["BROWSER_WRAPPER"] = wrapper
334      logging.info('export BROWSER_WRAPPER=' + wrapper)
335      proc = []
336    proc += self._args
337    return proc
338
339  def ToolSpecificFlags(self):
340    raise NotImplementedError, "This method should be implemented " \
341                               "in the tool-specific subclass"
342
343  def CreateBrowserWrapper(self, proc, webkit=False):
344    """The program being run invokes Python or something else that can't stand
345    to be valgrinded, and also invokes the Chrome browser. In this case, use a
346    magic wrapper to only valgrind the Chrome browser. Build the wrapper here.
347    Returns the path to the wrapper. It's up to the caller to use the wrapper
348    appropriately.
349    """
350    command = " ".join(proc)
351    # Add the PID of the browser wrapper to the logfile names so we can
352    # separate log files for different UI tests at the analyze stage.
353    command = command.replace("%p", "$$.%p")
354
355    (fd, indirect_fname) = tempfile.mkstemp(dir=self.log_dir,
356                                            prefix="browser_wrapper.",
357                                            text=True)
358    f = os.fdopen(fd, "w")
359    f.write('#!/bin/bash\n'
360            'echo "Started Valgrind wrapper for this test, PID=$$" >&2\n')
361
362    f.write('DIR=`dirname $0`\n'
363            'TESTNAME_FILE=$DIR/testcase.$$.name\n\n')
364
365    if webkit:
366      # Webkit layout_tests pass the URL as the first line of stdin.
367      f.write('tee $TESTNAME_FILE | %s "$@"\n' % command)
368    else:
369      # Try to get the test case name by looking at the program arguments.
370      # i.e. Chromium ui_tests used --test-name arg.
371      # TODO(timurrrr): This doesn't handle "--test-name Test.Name"
372      # TODO(timurrrr): ui_tests are dead. Where do we use the non-webkit
373      # wrapper now? browser_tests? What do they do?
374      f.write('for arg in $@\ndo\n'
375              '  if [[ "$arg" =~ --test-name=(.*) ]]\n  then\n'
376              '    echo ${BASH_REMATCH[1]} >$TESTNAME_FILE\n'
377              '  fi\n'
378              'done\n\n'
379              '%s "$@"\n' % command)
380
381    f.close()
382    os.chmod(indirect_fname, stat.S_IRUSR|stat.S_IXUSR)
383    return indirect_fname
384
385  def CreateAnalyzer(self):
386    raise NotImplementedError, "This method should be implemented " \
387                               "in the tool-specific subclass"
388
389  def GetAnalyzeResults(self, check_sanity=False):
390    # Glob all the files in the log directory
391    filenames = glob.glob(self.log_dir + "/" + self.ToolName() + ".*")
392
393    # If we have browser wrapper, the logfiles are named as
394    # "toolname.wrapper_PID.valgrind_PID".
395    # Let's extract the list of wrapper_PIDs and name it ppids
396    ppids = set([int(f.split(".")[-2]) \
397                for f in filenames if re.search("\.[0-9]+\.[0-9]+$", f)])
398
399    analyzer = self.CreateAnalyzer()
400    if len(ppids) == 0:
401      # Fast path - no browser wrapper was set.
402      return analyzer.Report(filenames, None, check_sanity)
403
404    ret = 0
405    for ppid in ppids:
406      testcase_name = None
407      try:
408        f = open(self.log_dir + ("/testcase.%d.name" % ppid))
409        testcase_name = f.read().strip()
410        f.close()
411        wk_layout_prefix="third_party/WebKit/LayoutTests/"
412        wk_prefix_at = testcase_name.rfind(wk_layout_prefix)
413        if wk_prefix_at != -1:
414          testcase_name = testcase_name[wk_prefix_at + len(wk_layout_prefix):]
415      except IOError:
416        pass
417      print "====================================================="
418      print " Below is the report for valgrind wrapper PID=%d." % ppid
419      if testcase_name:
420        print " It was used while running the `%s` test." % testcase_name
421      else:
422        print " You can find the corresponding test"
423        print " by searching the above log for 'PID=%d'" % ppid
424      sys.stdout.flush()
425
426      ppid_filenames = [f for f in filenames \
427                        if re.search("\.%d\.[0-9]+$" % ppid, f)]
428      # check_sanity won't work with browser wrappers
429      assert check_sanity == False
430      ret |= analyzer.Report(ppid_filenames, testcase_name)
431      print "====================================================="
432      sys.stdout.flush()
433
434    if ret != 0:
435      print ""
436      print "The Valgrind reports are grouped by test names."
437      print "Each test has its PID printed in the log when the test was run"
438      print "and at the beginning of its Valgrind report."
439      print "Hint: you can search for the reports by Ctrl+F -> `=#`"
440      sys.stdout.flush()
441
442    return ret
443
444
445# TODO(timurrrr): Split into a separate file.
446class Memcheck(ValgrindTool):
447  """Memcheck
448  Dynamic memory error detector for Linux & Mac
449
450  http://valgrind.org/info/tools.html#memcheck
451  """
452
453  def __init__(self):
454    super(Memcheck, self).__init__()
455    self.RegisterOptionParserHook(Memcheck.ExtendOptionParser)
456
457  def ToolName(self):
458    return "memcheck"
459
460  def ExtendOptionParser(self, parser):
461    parser.add_option("--leak-check", "--leak_check", type="string",
462                      default="yes",  # --leak-check=yes is equivalent of =full
463                      help="perform leak checking at the end of the run")
464    parser.add_option("", "--show_all_leaks", action="store_true",
465                      default=False,
466                      help="also show less blatant leaks")
467    parser.add_option("", "--track_origins", action="store_true",
468                      default=False,
469                      help="Show whence uninitialized bytes came. 30% slower.")
470
471  def ToolSpecificFlags(self):
472    ret = ["--gen-suppressions=all", "--demangle=no"]
473    ret += ["--leak-check=%s" % self._options.leak_check]
474
475    if self._options.show_all_leaks:
476      ret += ["--show-reachable=yes"]
477    else:
478      ret += ["--show-possibly-lost=no"]
479
480    if self._options.track_origins:
481      ret += ["--track-origins=yes"]
482
483    # TODO(glider): this is a temporary workaround for http://crbug.com/51716
484    # Let's see whether it helps.
485    if common.IsMac():
486      ret += ["--smc-check=all"]
487
488    return ret
489
490  def CreateAnalyzer(self):
491    use_gdb = common.IsMac()
492    return memcheck_analyze.MemcheckAnalyzer(self._source_dir,
493                                            self._options.show_all_leaks,
494                                            use_gdb=use_gdb)
495
496  def Analyze(self, check_sanity=False):
497    ret = self.GetAnalyzeResults(check_sanity)
498
499    if ret != 0:
500      logging.info("Please see http://dev.chromium.org/developers/how-tos/"
501                   "using-valgrind for the info on Memcheck/Valgrind")
502    return ret
503
504
505class ToolFactory:
506  def Create(self, tool_name):
507    if tool_name == "memcheck":
508      return Memcheck()
509    try:
510      platform_name = common.PlatformNames()[0]
511    except common.NotImplementedError:
512      platform_name = sys.platform + "(Unknown)"
513    raise RuntimeError, "Unknown tool (tool=%s, platform=%s)" % (tool_name,
514                                                                 platform_name)
515
516def CreateTool(tool):
517  return ToolFactory().Create(tool)
518