• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import glob
6import hashlib
7import logging
8import os
9import platform
10import re
11import shutil
12import subprocess
13
14from telemetry.internal.util import binary_manager
15from telemetry.core import platform as telemetry_platform
16from telemetry.core import util
17from telemetry import decorators
18from telemetry.internal.platform.profiler import android_prebuilt_profiler_helper
19
20from devil.android import md5sum  # pylint: disable=import-error
21
22
23try:
24  import sqlite3
25except ImportError:
26  sqlite3 = None
27
28
29
30_TEXT_SECTION = '.text'
31
32
33def _ElfMachineId(elf_file):
34  headers = subprocess.check_output(['readelf', '-h', elf_file])
35  return re.match(r'.*Machine:\s+(\w+)', headers, re.DOTALL).group(1)
36
37
38def _ElfSectionAsString(elf_file, section):
39  return subprocess.check_output(['readelf', '-p', section, elf_file])
40
41
42def _ElfSectionMd5Sum(elf_file, section):
43  result = subprocess.check_output(
44      'readelf -p%s "%s" | md5sum' % (section, elf_file), shell=True)
45  return result.split(' ', 1)[0]
46
47
48def _FindMatchingUnstrippedLibraryOnHost(device, lib):
49  lib_base = os.path.basename(lib)
50
51  device_md5sums = md5sum.CalculateDeviceMd5Sums([lib], device)
52  if lib not in device_md5sums:
53    return None
54
55  device_md5 = device_md5sums[lib]
56
57  def FindMatchingStrippedLibrary(out_path):
58    # First find a matching stripped library on the host. This avoids the need
59    # to pull the stripped library from the device, which can take tens of
60    # seconds.
61    # Check the GN stripped lib path first, and the GYP ones afterwards.
62    host_lib_path = os.path.join(out_path, lib_base)
63    host_lib_pattern = os.path.join(out_path, '*_apk', 'libs', '*', lib_base)
64    for stripped_host_lib in [host_lib_path] + glob.glob(host_lib_pattern):
65      if os.path.exists(stripped_host_lib):
66        with open(stripped_host_lib) as f:
67          host_md5 = hashlib.md5(f.read()).hexdigest()
68          if host_md5 == device_md5:
69            return stripped_host_lib
70    return None
71
72  out_path = None
73  stripped_host_lib = None
74  for out_path in util.GetBuildDirectories():
75    stripped_host_lib = FindMatchingStrippedLibrary(out_path)
76    if stripped_host_lib:
77      break
78
79  if not stripped_host_lib:
80    return None
81
82  # The corresponding unstripped library will be under lib.unstripped for GN, or
83  # lib for GYP.
84  unstripped_host_lib_paths = [
85      os.path.join(out_path, 'lib.unstripped', lib_base),
86      os.path.join(out_path, 'lib', lib_base)
87  ]
88  unstripped_host_lib = next(
89      (lib for lib in unstripped_host_lib_paths if os.path.exists(lib)), None)
90  if unstripped_host_lib is None:
91    return None
92
93  # Make sure the unstripped library matches the stripped one. We do this
94  # by comparing the hashes of text sections in both libraries. This isn't an
95  # exact guarantee, but should still give reasonable confidence that the
96  # libraries are compatible.
97  # TODO(skyostil): Check .note.gnu.build-id instead once we're using
98  # --build-id=sha1.
99  # pylint: disable=undefined-loop-variable
100  if (_ElfSectionMd5Sum(unstripped_host_lib, _TEXT_SECTION) !=
101      _ElfSectionMd5Sum(stripped_host_lib, _TEXT_SECTION)):
102    return None
103  return unstripped_host_lib
104
105
106@decorators.Cache
107def GetPerfhostName():
108  return 'perfhost_' + telemetry_platform.GetHostPlatform().GetOSVersionName()
109
110
111# Ignored directories for libraries that aren't useful for symbolization.
112_IGNORED_LIB_PATHS = [
113  '/data/dalvik-cache',
114  '/tmp'
115]
116
117
118def GetRequiredLibrariesForPerfProfile(profile_file):
119  """Returns the set of libraries necessary to symbolize a given perf profile.
120
121  Args:
122    profile_file: Path to perf profile to analyse.
123
124  Returns:
125    A set of required library file names.
126  """
127  with open(os.devnull, 'w') as dev_null:
128    perfhost_path = binary_manager.FetchPath(
129        GetPerfhostName(), 'x86_64', 'linux')
130    perf = subprocess.Popen([perfhost_path, 'script', '-i', profile_file],
131                             stdout=dev_null, stderr=subprocess.PIPE)
132    _, output = perf.communicate()
133  missing_lib_re = re.compile(
134      ('^Failed to open (.*), continuing without symbols|'
135       '^(.*[.]so).*not found, continuing without symbols'))
136  libs = set()
137  for line in output.split('\n'):
138    lib = missing_lib_re.match(line)
139    if lib:
140      lib = lib.group(1) or lib.group(2)
141      path = os.path.dirname(lib)
142      if (any(path.startswith(ignored_path)
143              for ignored_path in _IGNORED_LIB_PATHS)
144          or path == '/' or not path):
145        continue
146      libs.add(lib)
147  return libs
148
149
150def GetRequiredLibrariesForVTuneProfile(profile_file):
151  """Returns the set of libraries necessary to symbolize a given VTune profile.
152
153  Args:
154    profile_file: Path to VTune profile to analyse.
155
156  Returns:
157    A set of required library file names.
158  """
159  db_file = os.path.join(profile_file, 'sqlite-db', 'dicer.db')
160  conn = sqlite3.connect(db_file)
161
162  try:
163    # The 'dd_module_file' table lists all libraries on the device. Only the
164    # ones with 'bin_located_path' are needed for the profile.
165    query = 'SELECT bin_path, bin_located_path FROM dd_module_file'
166    return set(row[0] for row in conn.cursor().execute(query) if row[1])
167  finally:
168    conn.close()
169
170
171def _FileMetadataMatches(filea, fileb):
172  """Check if the metadata of two files matches."""
173  assert os.path.exists(filea)
174  if not os.path.exists(fileb):
175    return False
176
177  fields_to_compare = [
178      'st_ctime', 'st_gid', 'st_mode', 'st_mtime', 'st_size', 'st_uid']
179
180  filea_stat = os.stat(filea)
181  fileb_stat = os.stat(fileb)
182  for field in fields_to_compare:
183    # shutil.copy2 doesn't get ctime/mtime identical when the file system
184    # provides sub-second accuracy.
185    if int(getattr(filea_stat, field)) != int(getattr(fileb_stat, field)):
186      return False
187  return True
188
189
190def CreateSymFs(device, symfs_dir, libraries, use_symlinks=True):
191  """Creates a symfs directory to be used for symbolizing profiles.
192
193  Prepares a set of files ("symfs") to be used with profilers such as perf for
194  converting binary addresses into human readable function names.
195
196  Args:
197    device: DeviceUtils instance identifying the target device.
198    symfs_dir: Path where the symfs should be created.
199    libraries: Set of library file names that should be included in the symfs.
200    use_symlinks: If True, link instead of copy unstripped libraries into the
201      symfs. This will speed up the operation, but the resulting symfs will no
202      longer be valid if the linked files are modified, e.g., by rebuilding.
203
204  Returns:
205    The absolute path to the kernel symbols within the created symfs.
206  """
207  logging.info('Building symfs into %s.' % symfs_dir)
208
209  for lib in libraries:
210    device_dir = os.path.dirname(lib)
211    output_dir = os.path.join(symfs_dir, device_dir[1:])
212    if not os.path.exists(output_dir):
213      os.makedirs(output_dir)
214    output_lib = os.path.join(output_dir, os.path.basename(lib))
215
216    if lib.startswith('/data/app'):
217      # If this is our own library instead of a system one, look for a matching
218      # unstripped library under the out directory.
219      unstripped_host_lib = _FindMatchingUnstrippedLibraryOnHost(device, lib)
220      if not unstripped_host_lib:
221        logging.warning('Could not find symbols for %s.' % lib)
222        logging.warning('Is the correct output directory selected '
223                        '(CHROMIUM_OUTPUT_DIR)? Did you install the APK after '
224                        'building?')
225        continue
226      if use_symlinks:
227        if os.path.lexists(output_lib):
228          os.remove(output_lib)
229        os.symlink(os.path.abspath(unstripped_host_lib), output_lib)
230      # Copy the unstripped library only if it has been changed to avoid the
231      # delay.
232      elif not _FileMetadataMatches(unstripped_host_lib, output_lib):
233        logging.info('Copying %s to %s' % (unstripped_host_lib, output_lib))
234        shutil.copy2(unstripped_host_lib, output_lib)
235    else:
236      # Otherwise save a copy of the stripped system library under the symfs so
237      # the profiler can at least use the public symbols of that library. To
238      # speed things up, only pull files that don't match copies we already
239      # have in the symfs.
240      if not os.path.exists(output_lib):
241        pull = True
242      else:
243        host_md5sums = md5sum.CalculateHostMd5Sums([output_lib])
244        device_md5sums = md5sum.CalculateDeviceMd5Sums([lib], device)
245
246        pull = True
247        if host_md5sums and device_md5sums and output_lib in host_md5sums \
248          and lib in device_md5sums:
249          pull = host_md5sums[output_lib] != device_md5sums[lib]
250
251      if pull:
252        logging.info('Pulling %s to %s', lib, output_lib)
253        device.PullFile(lib, output_lib)
254
255  # Also pull a copy of the kernel symbols.
256  output_kallsyms = os.path.join(symfs_dir, 'kallsyms')
257  if not os.path.exists(output_kallsyms):
258    device.PullFile('/proc/kallsyms', output_kallsyms)
259  return output_kallsyms
260
261
262def PrepareDeviceForPerf(device):
263  """Set up a device for running perf.
264
265  Args:
266    device: DeviceUtils instance identifying the target device.
267
268  Returns:
269    The path to the installed perf binary on the device.
270  """
271  android_prebuilt_profiler_helper.InstallOnDevice(device, 'perf')
272  # Make sure kernel pointers are not hidden.
273  device.WriteFile('/proc/sys/kernel/kptr_restrict', '0', as_root=True)
274  return android_prebuilt_profiler_helper.GetDevicePath('perf')
275
276
277def GetToolchainBinaryPath(library_file, binary_name):
278  """Return the path to an Android toolchain binary on the host.
279
280  Args:
281    library_file: ELF library which is used to identify the used ABI,
282        architecture and toolchain.
283    binary_name: Binary to search for, e.g., 'objdump'
284  Returns:
285    Full path to binary or None if the binary was not found.
286  """
287  # Mapping from ELF machine identifiers to GNU toolchain names.
288  toolchain_configs = {
289    'x86': 'i686-linux-android',
290    'MIPS': 'mipsel-linux-android',
291    'ARM': 'arm-linux-androideabi',
292    'x86-64': 'x86_64-linux-android',
293    'AArch64': 'aarch64-linux-android',
294  }
295  toolchain_config = toolchain_configs[_ElfMachineId(library_file)]
296  host_os = platform.uname()[0].lower()
297  host_machine = platform.uname()[4]
298
299  elf_comment = _ElfSectionAsString(library_file, '.comment')
300  toolchain_version = re.match(r'.*GCC: \(GNU\) ([\w.]+)',
301                               elf_comment, re.DOTALL)
302  if not toolchain_version:
303    return None
304  toolchain_version = toolchain_version.group(1)
305  toolchain_version = toolchain_version.replace('.x', '')
306
307  toolchain_path = os.path.abspath(os.path.join(
308      util.GetChromiumSrcDir(), 'third_party', 'android_tools', 'ndk',
309      'toolchains', '%s-%s' % (toolchain_config, toolchain_version)))
310  if not os.path.exists(toolchain_path):
311    logging.warning(
312        'Unable to find toolchain binary %s: toolchain not found at %s',
313        binary_name, toolchain_path)
314    return None
315
316  path = os.path.join(
317      toolchain_path, 'prebuilt', '%s-%s' % (host_os, host_machine), 'bin',
318      '%s-%s' % (toolchain_config, binary_name))
319  if not os.path.exists(path):
320    logging.warning(
321        'Unable to find toolchain binary %s: binary not found at %s',
322        binary_name, path)
323    return None
324
325  return path
326