#!/usr/bin/env python
#
# Copyright (C) 2017 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Generates a human-interpretable view of a native heap dump from 'am dumpheap -n'."""
import logging
import os
import os.path
import re
import subprocess
import sys
import zipfile
class Args:
_usage = """
Usage:
1. Collect a native heap dump from the device. For example:
$ adb shell stop
$ adb shell setprop libc.debug.malloc.program app_process
$ adb shell setprop libc.debug.malloc.options backtrace=64
$ adb shell start
(launch and use app)
$ adb shell am dumpheap -n <pid> /data/local/tmp/native_heap.txt
$ adb pull /data/local/tmp/native_heap.txt
2. Run the viewer:
$ python native_heapdump_viewer.py [options] native_heap.txt
[--verbose]: verbose output
[--html]: interactive html output
[--reverse]: reverse the backtraces (start the tree from the leaves)
[--symbols SYMBOL_DIR] SYMBOL_DIR is the directory containing the .so files with symbols.
Defaults to $ANDROID_PRODUCT_OUT/symbols
[--app-symbols SYMBOL_DIR] SYMBOL_DIR is the directory containing the app APK and so files.
Defaults to the current directory.
This outputs a file with lines of the form:
5831776 29.09% 100.00% 10532 71b07bc0b0 /system/lib64/libandroid_runtime.so Typeface_createFromArray frameworks/base/core/jni/android/graphics/Typeface.cpp:68
5831776 is the total number of bytes allocated at this stack frame, which
is 29.09% of the total number of bytes allocated and 100.00% of the parent
frame's bytes allocated. 10532 is the total number of allocations at this
stack frame. 71b07bc0b0 is the address of the stack frame.
"""
def __init__(self):
self.verbose = False
self.html_output = False
self.reverse_frames = False
product_out = os.getenv("ANDROID_PRODUCT_OUT")
if product_out:
self.symboldir = product_out + "/symbols"
else:
self.symboldir = "./symbols"
self.app_symboldir = ""
i = 1
extra_args = []
while i < len(sys.argv):
if sys.argv[i] == "--symbols":
i += 1
self.symboldir = sys.argv[i] + "/"
elif sys.argv[i] == "--app-symbols":
i += 1
self.app_symboldir = sys.argv[i] + "/"
elif sys.argv[i] == "--verbose":
self.verbose = True
elif sys.argv[i] == "--html":
self.html_output = True
elif sys.argv[i] == "--reverse":
self.reverse_frames = True
elif sys.argv[i][0] == '-':
print("Invalid option %s" % (sys.argv[i]))
else:
extra_args.append(sys.argv[i])
i += 1
if len(extra_args) != 1:
print(self._usage)
sys.exit(1)
self.native_heap = extra_args[0]
class Backtrace:
def __init__(self, is_zygote, size, num_allocs, frames):
self.is_zygote = is_zygote
self.size = size
self.num_allocs = num_allocs
self.frames = frames
class Mapping:
def __init__(self, start, end, offset, name):
self.start = start
self.end = end
self.offset = offset
self.name = name
class FrameDescription:
def __init__(self, function, location, library):
self.function = function
self.location = location
self.library = library
def GetVersion(native_heap):
"""Get the version of the native heap dump."""
re_line = re.compile("Android\s+Native\s+Heap\s+Dump\s+(?P<version>v\d+\.\d+)\s*$")
matched = 0
with open(native_heap, "r") as f:
for line in f:
m = re_line.match(line)
if m:
return m.group('version')
return None
def GetNumFieldValidByParsingLines(native_heap):
"""Determine if the num field is valid by parsing the backtrace lines.
Malloc debug for N incorrectly set the num field to the number of
backtraces instead of the number of allocations with the same size and
backtrace. Read the file and if at least three lines all have the field
set to the number of backtraces values, then consider this generated by
the buggy malloc debug and indicate the num field is not valid.
Returns:
True if the num field is valid.
False if the num field is not valid and should be ignored.
"""
re_backtrace = re.compile("Backtrace\s+size:\s+(?P<backtrace_size>\d+)")
re_line = re.compile("z\s+(?P<zygote>\d+)\s+sz\s+(?P<size>\d+)\s+num\s+(?P<num_allocations>\d+)")
matched = 0
backtrace_size = 0
with open(native_heap, "r") as f:
for line in f:
if backtrace_size == 0:
m = re_backtrace.match(line)
if m:
backtrace_size = int(m.group('backtrace_size'))
parts = line.split()
if len(parts) > 7 and parts[0] == "z" and parts[2] == "sz":
m = re_line.match(line)
if m:
num_allocations = int(m.group('num_allocations'))
if num_allocations == backtrace_size:
# At least three lines must match this pattern before
# considering this the old buggy version of malloc debug.
matched += 1
if matched == 3:
return False
else:
return True
return matched == 0
def GetNumFieldValid(native_heap):
version = GetVersion(native_heap)
if not version or version == "v1.0":
# Version v1.0 was produced by a buggy version of malloc debug where the
# num field was set incorrectly.
# Unfortunately, Android P produced a v1.0 version that does set the
# num field. Do one more check to see if this is the broken version.
return GetNumFieldValidByParsingLines(native_heap)
else:
return True
def GetMappingFromOffset(mapping, app_symboldir):
"""
If the input mapping is a zip file, translate the contained uncompressed files and add mapping
entries.
This is done to handle symbols for the uncompressed .so files inside APKs. With the replaced
mappings, the script looks up the .so files as separate files.
"""
basename = os.path.basename(mapping.name)
zip_name = app_symboldir + basename
if os.path.isfile(zip_name):
opened_zip = zipfile.ZipFile(zip_name)
if opened_zip:
# For all files in the zip, add mappings for the internal files.
for file_info in opened_zip.infolist():
# Only add stored files since it doesn't make sense to have PC into compressed ones.
if file_info.compress_type == zipfile.ZIP_STORED:
zip_header_entry_size = 30
data_offset = (file_info.header_offset
+ zip_header_entry_size
+ len(file_info.filename)
+ len(file_info.extra)
+ len(file_info.comment))
end_offset = data_offset + file_info.file_size
if mapping.offset >= data_offset and mapping.offset < end_offset:
# Round up the data_offset to the nearest page since the .so must be aligned.
so_file_alignment = 4096
data_offset += so_file_alignment - 1;
data_offset -= data_offset % so_file_alignment;
mapping.name = file_info.filename
mapping.offset -= data_offset
break
return mapping
def ParseNativeHeap(native_heap, reverse_frames, num_field_valid, app_symboldir):
"""Parse the native heap into backtraces, maps.
Returns two lists, the first is a list of all of the backtraces, the
second is the sorted list of maps.
"""
backtraces = []
mappings = []
re_map = re.compile("(?P<start>[0-9a-f]+)-(?P<end>[0-9a-f]+) .... (?P<offset>[0-9a-f]+) [0-9a-f]+:[0-9a-f]+ [0-9]+ +(?P<name>.*)")
with open(native_heap, "r") as f:
for line in f:
# Format of line:
# z 0 sz 50 num 1 bt 000000000000a100 000000000000b200
parts = line.split()
if len(parts) > 7 and parts[0] == "z" and parts[2] == "sz":
is_zygote = parts[1] != "1"
size = int(parts[3])
if num_field_valid:
num_allocs = int(parts[5])
else:
num_allocs = 1
frames = list(map(lambda x: int(x, 16), parts[7:]))
if reverse_frames:
frames = list(reversed(frames))
backtraces.append(Backtrace(is_zygote, size, num_allocs, frames))
else:
# Parse map line:
# 720de01000-720ded7000 r-xp 00000000 fd:00 495 /system/lib64/libc.so
m = re_map.match(line)
if m:
# Offset of mapping start
start = int(m.group('start'), 16)
# Offset of mapping end
end = int(m.group('end'), 16)
# Offset within file that is mapped
offset = int(m.group('offset'), 16)
name = m.group('name')
mappings.append(GetMappingFromOffset(Mapping(start, end, offset, name), app_symboldir))
return backtraces, mappings
def FindMapping(mappings, addr):
"""Find the mapping given addr.
Returns the mapping that contains addr.
Returns None if there is no such mapping.
"""
min = 0
max = len(mappings) - 1
while True:
if max < min:
return None
mid = (min + max) // 2
if mappings[mid].end <= addr:
min = mid + 1
elif mappings[mid].start > addr:
max = mid - 1
else:
return mappings[mid]
def ResolveAddrs(html_output, symboldir, app_symboldir, backtraces, mappings):
"""Resolve address libraries and offsets.
addr_offsets maps addr to .so file offset
addrs_by_lib maps library to list of addrs from that library
Resolved addrs maps addr to FrameDescription
Returns the resolved_addrs hash.
"""
addr_offsets = {}
addrs_by_lib = {}
resolved_addrs = {}
empty_frame_description = FrameDescription("???", "???", "???")
for backtrace in backtraces:
for addr in backtrace.frames:
if addr in addr_offsets:
continue
mapping = FindMapping(mappings, addr)
if mapping:
addr_offsets[addr] = addr - mapping.start + mapping.offset
if not (mapping.name in addrs_by_lib):
addrs_by_lib[mapping.name] = []
addrs_by_lib[mapping.name].append(addr)
else:
resolved_addrs[addr] = empty_frame_description
# Resolve functions and line numbers.
if html_output == False:
print("Resolving symbols using directory %s..." % symboldir)
for lib in addrs_by_lib:
sofile = app_symboldir + lib
if not os.path.isfile(sofile):
sofile = symboldir + lib
if os.path.isfile(sofile):
file_offset = 0
result = subprocess.check_output(["objdump", "-w", "-j", ".text", "-h", sofile])
for line in result.split("\n"):
splitted = line.split()
if len(splitted) > 5 and splitted[1] == ".text":
file_offset = int(splitted[5], 16)
break
input_addrs = ""
for addr in addrs_by_lib[lib]:
input_addrs += "%s\n" % hex(addr_offsets[addr] - file_offset)
p = subprocess.Popen(["addr2line", "-C", "-j", ".text", "-e", sofile, "-f"], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
result = p.communicate(input_addrs)[0]
addr2line_rc = p.returncode
if addr2line_rc and (addr2line_rc < 0):
logging.warn("addr2line on " + sofile + " terminated by signal " + str(-1 * addr2line_rc))
splitted = result.split("\n")
for x in range(0, len(addrs_by_lib[lib])):
try:
function = splitted[2*x];
location = splitted[2*x+1];
resolved_addrs[addrs_by_lib[lib][x]] = FrameDescription(function, location, lib)
except Exception:
logging.warn("exception while resolving symbols", exc_info=True)
resolved_addrs[addrs_by_lib[lib][x]] = FrameDescription("---", "---", lib)
else:
if html_output == False:
print("%s not found for symbol resolution" % lib)
fd = FrameDescription("???", "???", lib)
for addr in addrs_by_lib[lib]:
resolved_addrs[addr] = fd
return resolved_addrs
def Addr2Line(resolved_addrs, addr):
if addr == "ZYGOTE" or addr == "APP":
return FrameDescription("", "", "")
return resolved_addrs[int(addr, 16)]
class AddrInfo:
def __init__(self, addr):
self.addr = addr
self.size = 0
self.number = 0
self.num_allocs = 0
self.children = {}
def addStack(self, size, num_allocs, stack):
self.size += size * num_allocs
self.number += num_allocs
if len(stack) > 0:
child = stack[0]
if not (child.addr in self.children):
self.children[child.addr] = child
self.children[child.addr].addStack(size, num_allocs, stack[1:])
def Display(resolved_addrs, indent, total, parent_total, node):
fd = Addr2Line(resolved_addrs, node.addr)
total_percent = 0
if total != 0:
total_percent = 100 * node.size / float(total)
parent_percent = 0
if parent_total != 0:
parent_percent = 100 * node.size / float(parent_total)
print("%9d %6.2f%% %6.2f%% %8d %s%s %s %s %s" % (node.size, total_percent, parent_percent, node.number, indent, node.addr, fd.library, fd.function, fd.location))
children = sorted(node.children.values(), key=lambda x: x.size, reverse=True)
for child in children:
Display(resolved_addrs, indent + " ", total, node.size, child)
def DisplayHtml(verbose, resolved_addrs, total, node, extra, label_count):
fd = Addr2Line(resolved_addrs, node.addr)
if verbose:
lib = fd.library
else:
lib = os.path.basename(fd.library)
total_percent = 0
if total != 0:
total_percent = 100 * node.size / float(total)
label = "%d %6.2f%% %6d %s%s %s %s" % (node.size, total_percent, node.number, extra, lib, fd.function, fd.location)
label = label.replace("&", "&")
label = label.replace("'", "'")
label = label.replace('"', """)
label = label.replace("<", "<")
label = label.replace(">", ">")
children = sorted(node.children.values(), key=lambda x: x.size, reverse=True)
print('<li>')
if len(children) > 0:
print('<label for="' + str(label_count) + '">' + label + '</label>')
print('<input type="checkbox" id="' + str(label_count) + '"/>')
print('<ol>')
label_count += 1
for child in children:
label_count = DisplayHtml(verbose, resolved_addrs, total, child, "", label_count)
print('</ol>')
else:
print(label)
print('</li>')
return label_count
def CreateHtml(verbose, app, zygote, resolved_addrs):
print("""
<!DOCTYPE html>
<html><head><style>
li input {
display: none;
}
li input:checked + ol > li {
display: block;
}
li input + ol > li {
display: none;
}
li {
font-family: Roboto Mono,monospace;
}
label {
font-family: Roboto Mono,monospace;
cursor: pointer
}
</style></head><body>Native allocation HTML viewer<br><br>
Click on an individual line to expand/collapse to see the details of the
allocation data<ol>
""")
label_count = 0
label_count = DisplayHtml(verbose, resolved_addrs, app.size, app, "app ", label_count)
if zygote.size > 0:
DisplayHtml(verbose, resolved_addrs, zygote.size, zygote, "zygote ", label_count)
print("</ol></body></html>")
def main():
args = Args()
num_field_valid = GetNumFieldValid(args.native_heap)
backtraces, mappings = ParseNativeHeap(args.native_heap, args.reverse_frames, num_field_valid,
args.app_symboldir)
# Resolve functions and line numbers
resolved_addrs = ResolveAddrs(args.html_output, args.symboldir, args.app_symboldir, backtraces,
mappings)
app = AddrInfo("APP")
zygote = AddrInfo("ZYGOTE")
for backtrace in backtraces:
stack = []
for addr in backtrace.frames:
stack.append(AddrInfo("%x" % addr))
stack.reverse()
if backtrace.is_zygote:
zygote.addStack(backtrace.size, backtrace.num_allocs, stack)
else:
app.addStack(backtrace.size, backtrace.num_allocs, stack)
if args.html_output:
CreateHtml(args.verbose, app, zygote, resolved_addrs)
else:
print("")
print("%9s %6s %6s %8s %s %s %s %s" % ("BYTES", "%TOTAL", "%PARENT", "COUNT", "ADDR", "LIBRARY", "FUNCTION", "LOCATION"))
Display(resolved_addrs, "", app.size, app.size + zygote.size, app)
print("")
Display(resolved_addrs, "", zygote.size, app.size + zygote.size, zygote)
print("")
if __name__ == '__main__':
main()