• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import glob
6import hashlib
7import logging
8import os
9import platform
10import re
11import shutil
12import subprocess
13
14try:
15  import sqlite3
16except ImportError:
17  sqlite3 = None
18
19from telemetry.core import util
20from telemetry.core.platform.profiler import android_prebuilt_profiler_helper
21
22
23_TEXT_SECTION = '.text'
24
25
26def _ElfMachineId(elf_file):
27  headers = subprocess.check_output(['readelf', '-h', elf_file])
28  return re.match(r'.*Machine:\s+(\w+)', headers, re.DOTALL).group(1)
29
30
31def _ElfSectionAsString(elf_file, section):
32  return subprocess.check_output(['readelf', '-p', section, elf_file])
33
34
35def _ElfSectionMd5Sum(elf_file, section):
36  result = subprocess.check_output(
37      'readelf -p%s "%s" | md5sum' % (section, elf_file), shell=True)
38  return result.split(' ', 1)[0]
39
40
41def _FindMatchingUnstrippedLibraryOnHost(device, lib):
42  lib_base = os.path.basename(lib)
43
44  device_md5 = device.RunShellCommand('md5 "%s"' % lib, root=True)[0]
45  device_md5 = device_md5.split(' ', 1)[0]
46
47  def FindMatchingStrippedLibrary(out_path):
48    # First find a matching stripped library on the host. This avoids the need
49    # to pull the stripped library from the device, which can take tens of
50    # seconds.
51    host_lib_pattern = os.path.join(out_path, '*_apk', 'libs', '*', lib_base)
52    for stripped_host_lib in glob.glob(host_lib_pattern):
53      with open(stripped_host_lib) as f:
54        host_md5 = hashlib.md5(f.read()).hexdigest()
55        if host_md5 == device_md5:
56          return stripped_host_lib
57
58  for build_dir, build_type in util.GetBuildDirectories():
59    out_path = os.path.join(build_dir, build_type)
60    stripped_host_lib = FindMatchingStrippedLibrary(out_path)
61    if stripped_host_lib:
62      break
63  else:
64    return None
65
66  # The corresponding unstripped library will be under out/Release/lib.
67  unstripped_host_lib = os.path.join(out_path, 'lib', lib_base)
68
69  # Make sure the unstripped library matches the stripped one. We do this
70  # by comparing the hashes of text sections in both libraries. This isn't an
71  # exact guarantee, but should still give reasonable confidence that the
72  # libraries are compatible.
73  # TODO(skyostil): Check .note.gnu.build-id instead once we're using
74  # --build-id=sha1.
75  # pylint: disable=W0631
76  if (_ElfSectionMd5Sum(unstripped_host_lib, _TEXT_SECTION) !=
77      _ElfSectionMd5Sum(stripped_host_lib, _TEXT_SECTION)):
78    return None
79  return unstripped_host_lib
80
81
82# Ignored directories for libraries that aren't useful for symbolization.
83_IGNORED_LIB_PATHS = [
84  '/data/dalvik-cache',
85  '/tmp'
86]
87
88
89def GetRequiredLibrariesForPerfProfile(profile_file):
90  """Returns the set of libraries necessary to symbolize a given perf profile.
91
92  Args:
93    profile_file: Path to perf profile to analyse.
94
95  Returns:
96    A set of required library file names.
97  """
98  with open(os.devnull, 'w') as dev_null:
99    perf = subprocess.Popen(['perf', 'script', '-i', profile_file],
100                             stdout=dev_null, stderr=subprocess.PIPE)
101    _, output = perf.communicate()
102  missing_lib_re = re.compile(
103      r'^Failed to open (.*), continuing without symbols')
104  libs = set()
105  for line in output.split('\n'):
106    lib = missing_lib_re.match(line)
107    if lib:
108      lib = lib.group(1)
109      path = os.path.dirname(lib)
110      if any(path.startswith(ignored_path)
111             for ignored_path in _IGNORED_LIB_PATHS) or path == '/':
112        continue
113      libs.add(lib)
114  return libs
115
116
117def GetRequiredLibrariesForVTuneProfile(profile_file):
118  """Returns the set of libraries necessary to symbolize a given VTune profile.
119
120  Args:
121    profile_file: Path to VTune profile to analyse.
122
123  Returns:
124    A set of required library file names.
125  """
126  db_file = os.path.join(profile_file, 'sqlite-db', 'dicer.db')
127  conn = sqlite3.connect(db_file)
128
129  try:
130    # The 'dd_module_file' table lists all libraries on the device. Only the
131    # ones with 'bin_located_path' are needed for the profile.
132    query = 'SELECT bin_path, bin_located_path FROM dd_module_file'
133    return set(row[0] for row in conn.cursor().execute(query) if row[1])
134  finally:
135    conn.close()
136
137
138def CreateSymFs(device, symfs_dir, libraries, use_symlinks=True):
139  """Creates a symfs directory to be used for symbolizing profiles.
140
141  Prepares a set of files ("symfs") to be used with profilers such as perf for
142  converting binary addresses into human readable function names.
143
144  Args:
145    device: DeviceUtils instance identifying the target device.
146    symfs_dir: Path where the symfs should be created.
147    libraries: Set of library file names that should be included in the symfs.
148    use_symlinks: If True, link instead of copy unstripped libraries into the
149      symfs. This will speed up the operation, but the resulting symfs will no
150      longer be valid if the linked files are modified, e.g., by rebuilding.
151
152  Returns:
153    The absolute path to the kernel symbols within the created symfs.
154  """
155  logging.info('Building symfs into %s.' % symfs_dir)
156
157  mismatching_files = {}
158  for lib in libraries:
159    device_dir = os.path.dirname(lib)
160    output_dir = os.path.join(symfs_dir, device_dir[1:])
161    if not os.path.exists(output_dir):
162      os.makedirs(output_dir)
163    output_lib = os.path.join(output_dir, os.path.basename(lib))
164
165    if lib.startswith('/data/app-lib/'):
166      # If this is our own library instead of a system one, look for a matching
167      # unstripped library under the out directory.
168      unstripped_host_lib = _FindMatchingUnstrippedLibraryOnHost(device, lib)
169      if not unstripped_host_lib:
170        logging.warning('Could not find symbols for %s.' % lib)
171        logging.warning('Is the correct output directory selected '
172                        '(CHROMIUM_OUT_DIR)? Did you install the APK after '
173                        'building?')
174        continue
175      if use_symlinks:
176        if os.path.lexists(output_lib):
177          os.remove(output_lib)
178        os.symlink(os.path.abspath(unstripped_host_lib), output_lib)
179      # Copy the unstripped library only if it has been changed to avoid the
180      # delay. Add one second to the modification time to guard against file
181      # systems with poor timestamp resolution.
182      elif not os.path.exists(output_lib) or \
183          (os.stat(unstripped_host_lib).st_mtime >
184           os.stat(output_lib).st_mtime + 1):
185        logging.info('Copying %s to %s' % (unstripped_host_lib, output_lib))
186        shutil.copy2(unstripped_host_lib, output_lib)
187    else:
188      # Otherwise save a copy of the stripped system library under the symfs so
189      # the profiler can at least use the public symbols of that library. To
190      # speed things up, only pull files that don't match copies we already
191      # have in the symfs.
192      if not device_dir in mismatching_files:
193        changed_files = device.old_interface.GetFilesChanged(output_dir,
194                                                             device_dir)
195        mismatching_files[device_dir] = [
196            device_path for _, device_path in changed_files]
197
198      if not os.path.exists(output_lib) or lib in mismatching_files[device_dir]:
199        logging.info('Pulling %s to %s' % (lib, output_lib))
200        device.old_interface.PullFileFromDevice(lib, output_lib)
201
202  # Also pull a copy of the kernel symbols.
203  output_kallsyms = os.path.join(symfs_dir, 'kallsyms')
204  if not os.path.exists(output_kallsyms):
205    device.old_interface.PullFileFromDevice('/proc/kallsyms', output_kallsyms)
206  return output_kallsyms
207
208
209def PrepareDeviceForPerf(device):
210  """Set up a device for running perf.
211
212  Args:
213    device: DeviceUtils instance identifying the target device.
214
215  Returns:
216    The path to the installed perf binary on the device.
217  """
218  android_prebuilt_profiler_helper.InstallOnDevice(device, 'perf')
219  # Make sure kernel pointers are not hidden.
220  device.old_interface.SetProtectedFileContents(
221      '/proc/sys/kernel/kptr_restrict', '0')
222  return android_prebuilt_profiler_helper.GetDevicePath('perf')
223
224
225def GetToolchainBinaryPath(library_file, binary_name):
226  """Return the path to an Android toolchain binary on the host.
227
228  Args:
229    library_file: ELF library which is used to identify the used ABI,
230        architecture and toolchain.
231    binary_name: Binary to search for, e.g., 'objdump'
232  Returns:
233    Full path to binary or None if the binary was not found.
234  """
235  # Mapping from ELF machine identifiers to GNU toolchain names.
236  toolchain_configs = {
237    'x86': 'i686-linux-android',
238    'MIPS': 'mipsel-linux-android',
239    'ARM': 'arm-linux-androideabi',
240    'x86-64': 'x86_64-linux-android',
241    'AArch64': 'aarch64-linux-android',
242  }
243  toolchain_config = toolchain_configs[_ElfMachineId(library_file)]
244  host_os = platform.uname()[0].lower()
245  host_machine = platform.uname()[4]
246
247  elf_comment = _ElfSectionAsString(library_file, '.comment')
248  toolchain_version = re.match(r'.*GCC: \(GNU\) ([\w.]+)',
249                               elf_comment, re.DOTALL)
250  if not toolchain_version:
251    return None
252  toolchain_version = toolchain_version.group(1)
253
254  path = os.path.join(util.GetChromiumSrcDir(), 'third_party', 'android_tools',
255                      'ndk', 'toolchains',
256                      '%s-%s' % (toolchain_config, toolchain_version),
257                      'prebuilt', '%s-%s' % (host_os, host_machine), 'bin',
258                      '%s-%s' % (toolchain_config, binary_name))
259  path = os.path.abspath(path)
260  return path if os.path.exists(path) else None
261