• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3# Copyright (c) 2021-2022 Huawei Device Co., Ltd.
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16"""
17A tool to view memory usage reports.
18
19To get a memory usage report build Panda with -DPANDA_TRACK_INTERNAL_ALLOCATIONS=2 cmake option.
20Panda runtime writes memory reports into memdump.bin file at runtime destruction and at SIGUSR2
21signal. This script is aimed to analyse memdump.bin file.
22To view the report run the script as follow:
23    python3 scripts/memdump.py memdump.bin
24The report contains:
25    * total allocated memory (not considering free)
26    * peak allocated memory (maximum amount of allocated memory)
27    * detailed information about each allocation point:
28        * total number of bytes allocated (not considering free)
29        * number of allocations
30        * range of allocation sizes
31        * stacktrace
32
33To view only live allocations (allocations which are not free at the moment of dump) --live
34option is used. If the dump is collected during runtime destruction this report will contains
35memory leaks
36
37It is possible to filter and sort data (run the script with -h option)
38"""
39
40import sys
41import argparse
42import struct
43
44# must be aligned with SpaceType enum in
45# libpandabase/mem/space.h
46SPACES = {
47    1: 'object',
48    2: 'humongous',
49    3: 'nonmovable',
50    4: 'internal',
51    5: 'code',
52    6: 'compiler'
53}
54
55TAG_ALLOC = 1
56TAG_FREE = 2
57
58
59class AllocInfo:
60    """Contains information about all allocated memory"""
61
62    def __init__(self, stacktrace):
63        self.stacktrace = stacktrace
64        self.allocated_size = 0
65        self.sizes = []
66
67    def alloc(self, size):
68        """Handles allocation of size bytes"""
69
70        self.allocated_size += size
71        self.sizes.append(size)
72
73    def free(self, size):
74        """Handles deallocation of size bytes"""
75
76        self.allocated_size -= size
77
78
79# pylint: disable=too-few-public-methods
80class Filter:
81    """Filter by space and substring"""
82
83    def __init__(self, space, strfilter):
84        self.space = space
85        self.strfilter = strfilter
86
87    def filter(self, space, stacktrace):
88        """Checks that space and stacktrace matches filter"""
89
90        if self.space != 'all' and SPACES[space] != self.space:
91            return True
92        if self.strfilter is not None and self.strfilter not in stacktrace:
93            return True
94        return False
95
96
97def validate_space(value):
98    """Validates space value"""
99
100    if value not in SPACES.values():
101        print('Invalid value {} of --space option'.format(value))
102        sys.exit(1)
103
104
105def read_string(file):
106    """Reads string from file"""
107
108    num = struct.unpack('I', file.read(4))[0]
109    return file.read(num).decode('utf-8')
110
111
112def sort(data):
113    """Sorts data by allocated size and number of allocations"""
114
115    return sorted(
116        data, key=lambda info: (info.allocated_size, len(info.sizes)),
117        reverse=True)
118
119
120def pretty_alloc_sizes(sizes):
121    """Prettifies allocatation sizes"""
122
123    min_size = sizes[0]
124    max_size = min_size
125    for size in sizes:
126        if size < min_size:
127            min_size = size
128        elif size > max_size:
129            max_size = size
130
131    if min_size == max_size:
132        return 'all {} bytes'.format(min_size)
133
134    return 'from {} to {} bytes'.format(min_size, max_size)
135
136
137def pretty_stacktrace(stacktrace):
138    """Prettifies stacktrace"""
139
140    if not stacktrace:
141        return "<No stacktrace>"
142    return stacktrace
143
144
145def get_args():
146    """Gets cli arguments"""
147
148    parser = argparse.ArgumentParser()
149    parser.add_argument(
150        '--live', action='store_true', default=False,
151        help='Dump only live allocations (for which "free" is not called)')
152    parser.add_argument(
153        '--space',
154        help='Report only allocations for the specific space. Possible values: {}'.
155        format(', '.join(SPACES.values())))
156    parser.add_argument(
157        '--filter',
158        help='Filter allocations by a string in stacktrace (it may be a function of file and line)')
159    parser.add_argument('input_file', default='memdump.bin')
160
161    args = parser.parse_args()
162
163    return args
164
165
166# pylint: disable=too-many-locals
167def get_allocs(args, space_filter):
168    """Prints and returns statistic: stacktrace -> allocation info"""
169
170    total_allocated = 0
171    max_allocated = 0
172    cur_allocated = 0
173    with open(args.input_file, "rb") as file:
174        num_items, num_stacktraces = struct.unpack('II', file.read(8))
175        stacktraces = {}
176        stacktrace_id = 0
177        while stacktrace_id < num_stacktraces:
178            stacktraces[stacktrace_id] = read_string(file)
179            stacktrace_id += 1
180
181        allocs = {}
182        id2alloc = {}
183        while num_items > 0:
184            tag = struct.unpack_from("I", file.read(4))[0]
185            if tag == TAG_ALLOC:
186                identifier, size, space, stacktrace_id = struct.unpack(
187                    "IIII", file.read(16))
188                stacktrace = stacktraces[stacktrace_id]
189                if not space_filter.filter(space, stacktrace):
190                    info = allocs.setdefault(
191                        stacktrace_id, AllocInfo(stacktrace))
192                    info.alloc(size)
193                    id2alloc[identifier] = (info, size)
194                    total_allocated += size
195                    cur_allocated += size
196                    max_allocated = max(cur_allocated, max_allocated)
197            elif tag == TAG_FREE:
198                alloc_id = struct.unpack("I", file.read(4))[0]
199                res = id2alloc.pop(alloc_id, None)
200                if res is not None:
201                    info = res[0]
202                    size = res[1]
203                    info.free(size)
204                    cur_allocated -= size
205            else:
206                raise Exception("Invalid file format")
207
208            num_items -= 1
209
210    print("Total allocated: {}, peak allocated: {}, current allocated {}".format(
211        total_allocated, max_allocated, cur_allocated))
212
213    return allocs
214
215
216def main():
217    """Script's entrypoint"""
218
219    args = get_args()
220    live_allocs = args.live
221
222    space_filter = 'all'
223    if args.space is not None:
224        validate_space(args.space)
225        space_filter = args.space
226
227    allocs = get_allocs(args, Filter(space_filter, args.filter))
228    allocs = allocs.values()
229    allocs = sort(allocs)
230
231    for info in allocs:
232        if not live_allocs or (live_allocs and info.allocated_size > 0):
233            print("Allocated: {} bytes. {} allocs {} from:\n{}".format(
234                info.allocated_size, len(info.sizes),
235                pretty_alloc_sizes(info.sizes),
236                pretty_stacktrace(info.stacktrace)))
237
238
239if __name__ == "__main__":
240    main()
241