• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2013 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import datetime
6import glob
7import heapq
8import logging
9import os
10import os.path
11import random
12import re
13import shutil
14import subprocess as subprocess
15import sys
16import tempfile
17import time
18
19from py_utils import cloud_storage  # pylint: disable=import-error
20import dependency_manager  # pylint: disable=import-error
21
22from telemetry.internal.util import binary_manager
23from telemetry.core import exceptions
24from telemetry.core import util
25from telemetry.internal.backends import browser_backend
26from telemetry.internal.backends.chrome import chrome_browser_backend
27from telemetry.internal.util import path
28
29
30def ParseCrashpadDateTime(date_time_str):
31  # Python strptime does not support time zone parsing, strip it.
32  date_time_parts = date_time_str.split()
33  if len(date_time_parts) >= 3:
34    date_time_str = ' '.join(date_time_parts[:2])
35  return datetime.datetime.strptime(date_time_str, '%Y-%m-%d %H:%M:%S')
36
37
38def GetSymbolBinaries(minidump, arch_name, os_name):
39  # Returns binary file where symbols are located.
40  minidump_dump = binary_manager.FetchPath('minidump_dump', arch_name, os_name)
41  assert minidump_dump
42
43  symbol_binaries = []
44
45  minidump_cmd = [minidump_dump, minidump]
46  try:
47    with open(os.devnull, 'wb') as DEVNULL:
48      minidump_output = subprocess.check_output(minidump_cmd, stderr=DEVNULL)
49  except subprocess.CalledProcessError as e:
50    # For some reason minidump_dump always fails despite successful dumping.
51    minidump_output = e.output
52
53  minidump_binary_re = re.compile(r'\W+\(code_file\)\W+=\W\"(.*)\"')
54  for minidump_line in minidump_output.splitlines():
55    line_match = minidump_binary_re.match(minidump_line)
56    if line_match:
57      binary_path = line_match.group(1)
58      if not os.path.isfile(binary_path):
59        continue
60
61      # Filter out system binaries.
62      if (binary_path.startswith('/usr/lib/') or
63          binary_path.startswith('/System/Library/') or
64          binary_path.startswith('/lib/')):
65        continue
66
67      # Filter out other binary file types which have no symbols.
68      if (binary_path.endswith('.pak') or
69          binary_path.endswith('.bin') or
70          binary_path.endswith('.dat') or
71          binary_path.endswith('.ttf')):
72        continue
73
74      symbol_binaries.append(binary_path)
75  return symbol_binaries
76
77
78def GenerateBreakpadSymbols(minidump, arch, os_name, symbols_dir, browser_dir):
79  logging.info('Dumping breakpad symbols.')
80  generate_breakpad_symbols_command = binary_manager.FetchPath(
81      'generate_breakpad_symbols', arch, os_name)
82  if not generate_breakpad_symbols_command:
83    return
84
85  for binary_path in GetSymbolBinaries(minidump, arch, os_name):
86    cmd = [
87        sys.executable,
88        generate_breakpad_symbols_command,
89        '--binary=%s' % binary_path,
90        '--symbols-dir=%s' % symbols_dir,
91        '--build-dir=%s' % browser_dir,
92        ]
93
94    try:
95      subprocess.check_call(cmd, stderr=open(os.devnull, 'w'))
96    except subprocess.CalledProcessError:
97      logging.warning('Failed to execute "%s"' % ' '.join(cmd))
98      return
99
100
101class DesktopBrowserBackend(chrome_browser_backend.ChromeBrowserBackend):
102  """The backend for controlling a locally-executed browser instance, on Linux,
103  Mac or Windows.
104  """
105  def __init__(self, desktop_platform_backend, browser_options, executable,
106               flash_path, is_content_shell, browser_directory):
107    super(DesktopBrowserBackend, self).__init__(
108        desktop_platform_backend,
109        supports_tab_control=not is_content_shell,
110        supports_extensions=not is_content_shell,
111        browser_options=browser_options)
112
113    # Initialize fields so that an explosion during init doesn't break in Close.
114    self._proc = None
115    self._tmp_profile_dir = None
116    self._tmp_output_file = None
117    self._most_recent_symbolized_minidump_paths = set([])
118
119    self._executable = executable
120    if not self._executable:
121      raise Exception('Cannot create browser, no executable found!')
122
123    assert not flash_path or os.path.exists(flash_path)
124    self._flash_path = flash_path
125
126    self._is_content_shell = is_content_shell
127
128    extensions_to_load = browser_options.extensions_to_load
129
130    if len(extensions_to_load) > 0 and is_content_shell:
131      raise browser_backend.ExtensionsNotSupportedException(
132          'Content shell does not support extensions.')
133
134    self._browser_directory = browser_directory
135    self._port = None
136    self._tmp_minidump_dir = tempfile.mkdtemp()
137    if self.is_logging_enabled:
138      self._log_file_path = os.path.join(tempfile.mkdtemp(), 'chrome.log')
139    else:
140      self._log_file_path = None
141
142    self._SetupProfile()
143
144  @property
145  def is_logging_enabled(self):
146    return self.browser_options.logging_verbosity in [
147        self.browser_options.NON_VERBOSE_LOGGING,
148        self.browser_options.VERBOSE_LOGGING]
149
150  @property
151  def log_file_path(self):
152    return self._log_file_path
153
154  @property
155  def supports_uploading_logs(self):
156    return (self.browser_options.logs_cloud_bucket and self.log_file_path and
157            os.path.isfile(self.log_file_path))
158
159  def _SetupProfile(self):
160    if not self.browser_options.dont_override_profile:
161      if self._output_profile_path:
162        self._tmp_profile_dir = self._output_profile_path
163      else:
164        self._tmp_profile_dir = tempfile.mkdtemp()
165
166      profile_dir = self.browser_options.profile_dir
167      if profile_dir:
168        assert self._tmp_profile_dir != profile_dir
169        if self._is_content_shell:
170          logging.critical('Profiles cannot be used with content shell')
171          sys.exit(1)
172        logging.info("Using profile directory:'%s'." % profile_dir)
173        shutil.rmtree(self._tmp_profile_dir)
174        shutil.copytree(profile_dir, self._tmp_profile_dir)
175    # No matter whether we're using an existing profile directory or
176    # creating a new one, always delete the well-known file containing
177    # the active DevTools port number.
178    port_file = self._GetDevToolsActivePortPath()
179    if os.path.isfile(port_file):
180      try:
181        os.remove(port_file)
182      except Exception as e:
183        logging.critical('Unable to remove DevToolsActivePort file: %s' % e)
184        sys.exit(1)
185
186  def _GetDevToolsActivePortPath(self):
187    return os.path.join(self.profile_directory, 'DevToolsActivePort')
188
189  def _GetCdbPath(self):
190    # cdb.exe might have been co-located with the browser's executable
191    # during the build, but that's not a certainty. (This is only done
192    # in Chromium builds on the bots, which is why it's not a hard
193    # requirement.) See if it's available.
194    colocated_cdb = os.path.join(self._browser_directory, 'cdb', 'cdb.exe')
195    if path.IsExecutable(colocated_cdb):
196      return colocated_cdb
197    possible_paths = (
198        # Installed copies of the Windows SDK.
199        os.path.join('Windows Kits', '*', 'Debuggers', 'x86'),
200        os.path.join('Windows Kits', '*', 'Debuggers', 'x64'),
201        # Old copies of the Debugging Tools for Windows.
202        'Debugging Tools For Windows',
203        'Debugging Tools For Windows (x86)',
204        'Debugging Tools For Windows (x64)',
205        # The hermetic copy of the Windows toolchain in depot_tools.
206        os.path.join('win_toolchain', 'vs_files', '*', 'win_sdk',
207                     'Debuggers', 'x86'),
208        os.path.join('win_toolchain', 'vs_files', '*', 'win_sdk',
209                     'Debuggers', 'x64'),
210    )
211    for possible_path in possible_paths:
212      app_path = os.path.join(possible_path, 'cdb.exe')
213      app_path = path.FindInstalledWindowsApplication(app_path)
214      if app_path:
215        return app_path
216    return None
217
218  def HasBrowserFinishedLaunching(self):
219    # In addition to the functional check performed by the base class, quickly
220    # check if the browser process is still alive.
221    if not self.IsBrowserRunning():
222      raise exceptions.ProcessGoneException(
223          "Return code: %d" % self._proc.returncode)
224    # Start DevTools on an ephemeral port and wait for the well-known file
225    # containing the port number to exist.
226    port_file = self._GetDevToolsActivePortPath()
227    if not os.path.isfile(port_file):
228      # File isn't ready yet. Return false. Will retry.
229      return False
230    # Attempt to avoid reading the file until it's populated.
231    got_port = False
232    try:
233      if os.stat(port_file).st_size > 0:
234        with open(port_file) as f:
235          port_string = f.read()
236          self._port = int(port_string)
237          logging.info('Discovered ephemeral port %s' % self._port)
238          got_port = True
239    except Exception:
240      # Both stat and open can throw exceptions.
241      pass
242    if not got_port:
243      # File isn't ready yet. Return false. Will retry.
244      return False
245    return super(DesktopBrowserBackend, self).HasBrowserFinishedLaunching()
246
247  def GetBrowserStartupArgs(self):
248    args = super(DesktopBrowserBackend, self).GetBrowserStartupArgs()
249    self._port = 0
250    logging.info('Requested remote debugging port: %d' % self._port)
251    args.append('--remote-debugging-port=%i' % self._port)
252    args.append('--enable-crash-reporter-for-testing')
253    if not self._is_content_shell:
254      args.append('--window-size=1280,1024')
255      if self._flash_path:
256        args.append('--ppapi-flash-path=%s' % self._flash_path)
257        # Also specify the version of Flash as a large version, so that it is
258        # not overridden by the bundled or component-updated version of Flash.
259        args.append('--ppapi-flash-version=99.9.999.999')
260      if not self.browser_options.dont_override_profile:
261        args.append('--user-data-dir=%s' % self._tmp_profile_dir)
262    else:
263      args.append('--data-path=%s' % self._tmp_profile_dir)
264
265    trace_config_file = (self.platform_backend.tracing_controller_backend
266                         .GetChromeTraceConfigFile())
267    if trace_config_file:
268      args.append('--trace-config-file=%s' % trace_config_file)
269    return args
270
271  def Start(self):
272    assert not self._proc, 'Must call Close() before Start()'
273
274    args = [self._executable]
275    args.extend(self.GetBrowserStartupArgs())
276    if self.browser_options.startup_url:
277      args.append(self.browser_options.startup_url)
278    env = os.environ.copy()
279    env['CHROME_HEADLESS'] = '1'  # Don't upload minidumps.
280    env['BREAKPAD_DUMP_LOCATION'] = self._tmp_minidump_dir
281    if self.is_logging_enabled:
282      sys.stderr.write(
283        'Chrome log file will be saved in %s\n' % self.log_file_path)
284      env['CHROME_LOG_FILE'] = self.log_file_path
285    logging.info('Starting Chrome %s', args)
286    if not self.browser_options.show_stdout:
287      self._tmp_output_file = tempfile.NamedTemporaryFile('w', 0)
288      self._proc = subprocess.Popen(
289          args, stdout=self._tmp_output_file, stderr=subprocess.STDOUT, env=env)
290    else:
291      self._proc = subprocess.Popen(args, env=env)
292
293    try:
294      self._WaitForBrowserToComeUp()
295      # browser is foregrounded by default on Windows and Linux, but not Mac.
296      if self.browser.platform.GetOSName() == 'mac':
297        subprocess.Popen([
298          'osascript', '-e', ('tell application "%s" to activate' %
299                              self._executable)])
300      self._InitDevtoolsClientBackend()
301      if self._supports_extensions:
302        self._WaitForExtensionsToLoad()
303    except:
304      self.Close()
305      raise
306
307  @property
308  def pid(self):
309    if self._proc:
310      return self._proc.pid
311    return None
312
313  @property
314  def browser_directory(self):
315    return self._browser_directory
316
317  @property
318  def profile_directory(self):
319    return self._tmp_profile_dir
320
321  def IsBrowserRunning(self):
322    return self._proc and self._proc.poll() == None
323
324  def GetStandardOutput(self):
325    if not self._tmp_output_file:
326      if self.browser_options.show_stdout:
327        # This can happen in the case that loading the Chrome binary fails.
328        # We print rather than using logging here, because that makes a
329        # recursive call to this function.
330        print >> sys.stderr, "Can't get standard output with --show-stdout"
331      return ''
332    self._tmp_output_file.flush()
333    try:
334      with open(self._tmp_output_file.name) as f:
335        return f.read()
336    except IOError:
337      return ''
338
339  def _GetAllCrashpadMinidumps(self):
340    os_name = self.browser.platform.GetOSName()
341    arch_name = self.browser.platform.GetArchName()
342    try:
343      crashpad_database_util = binary_manager.FetchPath(
344          'crashpad_database_util', arch_name, os_name)
345      if not crashpad_database_util:
346        logging.warning('No crashpad_database_util found')
347        return None
348    except dependency_manager.NoPathFoundError:
349      logging.warning('No path to crashpad_database_util found')
350      return None
351
352    logging.info('Found crashpad_database_util')
353
354    report_output = subprocess.check_output([
355        crashpad_database_util, '--database=' + self._tmp_minidump_dir,
356        '--show-pending-reports', '--show-completed-reports',
357        '--show-all-report-info'])
358
359    last_indentation = -1
360    reports_list = []
361    report_dict = {}
362    for report_line in report_output.splitlines():
363      # Report values are grouped together by the same indentation level.
364      current_indentation = 0
365      for report_char in report_line:
366        if not report_char.isspace():
367          break
368        current_indentation += 1
369
370      # Decrease in indentation level indicates a new report is being printed.
371      if current_indentation >= last_indentation:
372        report_key, report_value = report_line.split(':', 1)
373        if report_value:
374          report_dict[report_key.strip()] = report_value.strip()
375      elif report_dict:
376        try:
377          report_time = ParseCrashpadDateTime(report_dict['Creation time'])
378          report_path = report_dict['Path'].strip()
379          reports_list.append((report_time, report_path))
380        except (ValueError, KeyError) as e:
381          logging.warning('Crashpad report expected valid keys'
382                          ' "Path" and "Creation time": %s', e)
383        finally:
384          report_dict = {}
385
386      last_indentation = current_indentation
387
388    # Include the last report.
389    if report_dict:
390      try:
391        report_time = ParseCrashpadDateTime(report_dict['Creation time'])
392        report_path = report_dict['Path'].strip()
393        reports_list.append((report_time, report_path))
394      except (ValueError, KeyError) as e:
395        logging.warning('Crashpad report expected valid keys'
396                          ' "Path" and "Creation time": %s', e)
397
398    return reports_list
399
400  def _GetMostRecentCrashpadMinidump(self):
401    reports_list = self._GetAllCrashpadMinidumps()
402    if reports_list:
403      _, most_recent_report_path = max(reports_list)
404      return most_recent_report_path
405
406    return None
407
408  def _GetBreakPadMinidumpPaths(self):
409    return glob.glob(os.path.join(self._tmp_minidump_dir, '*.dmp'))
410
411  def _GetMostRecentMinidump(self):
412    # Crashpad dump layout will be the standard eventually, check it first.
413    most_recent_dump = self._GetMostRecentCrashpadMinidump()
414
415    # Typical breakpad format is simply dump files in a folder.
416    if not most_recent_dump:
417      logging.info('No minidump found via crashpad_database_util')
418      dumps = self._GetBreakPadMinidumpPaths()
419      if dumps:
420        most_recent_dump = heapq.nlargest(1, dumps, os.path.getmtime)[0]
421        if most_recent_dump:
422          logging.info('Found minidump via globbing in minidump dir')
423
424    # As a sanity check, make sure the crash dump is recent.
425    if (most_recent_dump and
426        os.path.getmtime(most_recent_dump) < (time.time() - (5 * 60))):
427      logging.warning('Crash dump is older than 5 minutes. May not be correct.')
428
429    return most_recent_dump
430
431  def _IsExecutableStripped(self):
432    if self.browser.platform.GetOSName() == 'mac':
433      try:
434        symbols = subprocess.check_output(['/usr/bin/nm', self._executable])
435      except subprocess.CalledProcessError as err:
436        logging.warning('Error when checking whether executable is stripped: %s'
437                        % err.output)
438        # Just assume that binary is stripped to skip breakpad symbol generation
439        # if this check failed.
440        return True
441      num_symbols = len(symbols.splitlines())
442      # We assume that if there are more than 10 symbols the executable is not
443      # stripped.
444      return num_symbols < 10
445    else:
446      return False
447
448  def _GetStackFromMinidump(self, minidump):
449    os_name = self.browser.platform.GetOSName()
450    if os_name == 'win':
451      cdb = self._GetCdbPath()
452      if not cdb:
453        logging.warning('cdb.exe not found.')
454        return None
455      # Include all the threads' stacks ("~*kb30") in addition to the
456      # ostensibly crashed stack associated with the exception context
457      # record (".ecxr;kb30"). Note that stack dumps, including that
458      # for the crashed thread, may not be as precise as the one
459      # starting from the exception context record.
460      # Specify kb instead of k in order to get four arguments listed, for
461      # easier diagnosis from stacks.
462      output = subprocess.check_output([cdb, '-y', self._browser_directory,
463                                        '-c', '.ecxr;kb30;~*kb30;q',
464                                        '-z', minidump])
465      # cdb output can start the stack with "ChildEBP", "Child-SP", and possibly
466      # other things we haven't seen yet. If we can't find the start of the
467      # stack, include output from the beginning.
468      stack_start = 0
469      stack_start_match = re.search("^Child(?:EBP|-SP)", output, re.MULTILINE)
470      if stack_start_match:
471        stack_start = stack_start_match.start()
472      stack_end = output.find('quit:')
473      return output[stack_start:stack_end]
474
475    arch_name = self.browser.platform.GetArchName()
476    stackwalk = binary_manager.FetchPath(
477        'minidump_stackwalk', arch_name, os_name)
478    if not stackwalk:
479      logging.warning('minidump_stackwalk binary not found.')
480      return None
481
482    with open(minidump, 'rb') as infile:
483      minidump += '.stripped'
484      with open(minidump, 'wb') as outfile:
485        outfile.write(''.join(infile.read().partition('MDMP')[1:]))
486
487    symbols_path = os.path.join(self._tmp_minidump_dir, 'symbols')
488    GenerateBreakpadSymbols(minidump, arch_name, os_name,
489                            symbols_path, self._browser_directory)
490
491    return subprocess.check_output([stackwalk, minidump, symbols_path],
492                                   stderr=open(os.devnull, 'w'))
493
494  def _UploadMinidumpToCloudStorage(self, minidump_path):
495    """ Upload minidump_path to cloud storage and return the cloud storage url.
496    """
497    remote_path = ('minidump-%s-%i.dmp' %
498                   (datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'),
499                    random.randint(0, 1000000)))
500    try:
501      return cloud_storage.Insert(cloud_storage.TELEMETRY_OUTPUT, remote_path,
502                                  minidump_path)
503    except cloud_storage.CloudStorageError as err:
504      logging.error('Cloud storage error while trying to upload dump: %s' %
505                    repr(err))
506      return '<Missing link>'
507
508  def GetStackTrace(self):
509    """Returns a stack trace if a valid minidump is found, will return a tuple
510       (valid, output) where valid will be True if a valid minidump was found
511       and output will contain either an error message or the attempt to
512       symbolize the minidump if one was found.
513    """
514    most_recent_dump = self._GetMostRecentMinidump()
515    if not most_recent_dump:
516      return (False, 'No crash dump found.')
517    logging.info('Minidump found: %s' % most_recent_dump)
518    return self._InternalSymbolizeMinidump(most_recent_dump)
519
520  def GetMostRecentMinidumpPath(self):
521    return self._GetMostRecentMinidump()
522
523  def GetAllMinidumpPaths(self):
524    reports_list = self._GetAllCrashpadMinidumps()
525    if reports_list:
526      return [report[1] for report in reports_list]
527    else:
528      logging.info('No minidump found via crashpad_database_util')
529      dumps = self._GetBreakPadMinidumpPaths()
530      if dumps:
531        logging.info('Found minidump via globbing in minidump dir')
532        return dumps
533      return None
534
535  def GetAllUnsymbolizedMinidumpPaths(self):
536    minidump_paths = set(self.GetAllMinidumpPaths())
537    # If we have already symbolized paths remove them from the list
538    unsymbolized_paths = (minidump_paths
539      - self._most_recent_symbolized_minidump_paths)
540    return list(unsymbolized_paths)
541
542  def SymbolizeMinidump(self, minidump_path):
543    return self._InternalSymbolizeMinidump(minidump_path)
544
545  def _InternalSymbolizeMinidump(self, minidump_path):
546    stack = self._GetStackFromMinidump(minidump_path)
547    if not stack:
548      cloud_storage_link = self._UploadMinidumpToCloudStorage(minidump_path)
549      error_message = ('Failed to symbolize minidump. Raw stack is uploaded to'
550                       ' cloud storage: %s.' % cloud_storage_link)
551      return (False, error_message)
552
553    self._most_recent_symbolized_minidump_paths.add(minidump_path)
554    return (True, stack)
555
556  def __del__(self):
557    self.Close()
558
559  def _TryCooperativeShutdown(self):
560    if self.browser.platform.IsCooperativeShutdownSupported():
561      # Ideally there would be a portable, cooperative shutdown
562      # mechanism for the browser. This seems difficult to do
563      # correctly for all embedders of the content API. The only known
564      # problem with unclean shutdown of the browser process is on
565      # Windows, where suspended child processes frequently leak. For
566      # now, just solve this particular problem. See Issue 424024.
567      if self.browser.platform.CooperativelyShutdown(self._proc, "chrome"):
568        try:
569          util.WaitFor(lambda: not self.IsBrowserRunning(), timeout=5)
570          logging.info('Successfully shut down browser cooperatively')
571        except exceptions.TimeoutException as e:
572          logging.warning('Failed to cooperatively shutdown. ' +
573                          'Proceeding to terminate: ' + str(e))
574
575  def Close(self):
576    super(DesktopBrowserBackend, self).Close()
577
578    # First, try to cooperatively shutdown.
579    if self.IsBrowserRunning():
580      self._TryCooperativeShutdown()
581
582    # Second, try to politely shutdown with SIGTERM.
583    if self.IsBrowserRunning():
584      self._proc.terminate()
585      try:
586        util.WaitFor(lambda: not self.IsBrowserRunning(), timeout=5)
587        self._proc = None
588      except exceptions.TimeoutException:
589        logging.warning('Failed to gracefully shutdown.')
590
591    # Shutdown aggressively if all above failed.
592    if self.IsBrowserRunning():
593      logging.warning('Proceed to kill the browser.')
594      self._proc.kill()
595    self._proc = None
596
597    if self._output_profile_path:
598      # If we need the output then double check that it exists.
599      if not (self._tmp_profile_dir and os.path.exists(self._tmp_profile_dir)):
600        raise Exception("No profile directory generated by Chrome: '%s'." %
601            self._tmp_profile_dir)
602    else:
603      # If we don't need the profile after the run then cleanup.
604      if self._tmp_profile_dir and os.path.exists(self._tmp_profile_dir):
605        shutil.rmtree(self._tmp_profile_dir, ignore_errors=True)
606        self._tmp_profile_dir = None
607
608    if self._tmp_output_file:
609      self._tmp_output_file.close()
610      self._tmp_output_file = None
611
612    if self._tmp_minidump_dir:
613      shutil.rmtree(self._tmp_minidump_dir, ignore_errors=True)
614      self._tmp_minidump_dir = None
615