1 /*
2 * Copyright (C) 2020 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "src/trace_processor/importers/proto/memory_tracker_snapshot_parser.h"
18
19 #include "perfetto/ext/base/string_view.h"
20 #include "protos/perfetto/trace/memory_graph.pbzero.h"
21 #include "src/trace_processor/containers/string_pool.h"
22 #include "src/trace_processor/importers/common/args_tracker.h"
23 #include "src/trace_processor/tables/memory_tables_py.h"
24
25 namespace perfetto {
26 namespace trace_processor {
27
MemoryTrackerSnapshotParser(TraceProcessorContext * context)28 MemoryTrackerSnapshotParser::MemoryTrackerSnapshotParser(
29 TraceProcessorContext* context)
30 : context_(context),
31 level_of_detail_ids_{{context_->storage->InternString("background"),
32 context_->storage->InternString("light"),
33 context_->storage->InternString("detailed")}},
34 unit_ids_{{context_->storage->InternString("objects"),
35 context_->storage->InternString("bytes")}},
36 aggregate_raw_nodes_(),
37 last_snapshot_timestamp_(-1),
38 last_snapshot_level_of_detail_(LevelOfDetail::kFirst) {}
39
ParseMemoryTrackerSnapshot(int64_t ts,ConstBytes blob)40 void MemoryTrackerSnapshotParser::ParseMemoryTrackerSnapshot(int64_t ts,
41 ConstBytes blob) {
42 PERFETTO_DCHECK(last_snapshot_timestamp_ <= ts);
43 if (!aggregate_raw_nodes_.empty() && ts != last_snapshot_timestamp_) {
44 GenerateGraphFromRawNodesAndEmitRows();
45 }
46 ReadProtoSnapshot(blob, aggregate_raw_nodes_, last_snapshot_level_of_detail_);
47 last_snapshot_timestamp_ = ts;
48 }
49
NotifyEndOfFile()50 void MemoryTrackerSnapshotParser::NotifyEndOfFile() {
51 if (!aggregate_raw_nodes_.empty()) {
52 GenerateGraphFromRawNodesAndEmitRows();
53 }
54 }
55
ReadProtoSnapshot(ConstBytes blob,RawMemoryNodeMap & raw_nodes,LevelOfDetail & level_of_detail)56 void MemoryTrackerSnapshotParser::ReadProtoSnapshot(
57 ConstBytes blob,
58 RawMemoryNodeMap& raw_nodes,
59 LevelOfDetail& level_of_detail) {
60 protos::pbzero::MemoryTrackerSnapshot::Decoder snapshot(blob.data, blob.size);
61 level_of_detail = LevelOfDetail::kDetailed;
62
63 switch (snapshot.level_of_detail()) {
64 case 0: // FULL
65 level_of_detail = LevelOfDetail::kDetailed;
66 break;
67 case 1: // LIGHT
68 level_of_detail = LevelOfDetail::kLight;
69 break;
70 case 2: // BACKGROUND
71 level_of_detail = LevelOfDetail::kBackground;
72 break;
73 }
74
75 for (auto process_it = snapshot.process_memory_dumps(); process_it;
76 ++process_it) {
77 protos::pbzero::MemoryTrackerSnapshot::ProcessSnapshot::Decoder
78 process_memory_dump(*process_it);
79
80 base::PlatformProcessId pid =
81 static_cast<base::PlatformProcessId>(process_memory_dump.pid());
82
83 RawProcessMemoryNode::MemoryNodesMap nodes_map;
84 RawProcessMemoryNode::AllocatorNodeEdgesMap edges_map;
85
86 for (auto node_it = process_memory_dump.allocator_dumps(); node_it;
87 ++node_it) {
88 protos::pbzero::MemoryTrackerSnapshot::ProcessSnapshot::MemoryNode::
89 Decoder node(*node_it);
90
91 MemoryAllocatorNodeId node_id(node.id());
92 const std::string absolute_name = node.absolute_name().ToStdString();
93 int flags;
94 if (node.weak()) {
95 flags = RawMemoryGraphNode::kWeak;
96 } else {
97 flags = RawMemoryGraphNode::kDefault;
98 }
99
100 std::vector<RawMemoryGraphNode::MemoryNodeEntry> entries;
101
102 if (node.has_size_bytes()) {
103 entries.emplace_back("size", RawMemoryGraphNode::kUnitsBytes,
104 node.size_bytes());
105 }
106
107 for (auto entry_it = node.entries(); entry_it; ++entry_it) {
108 protos::pbzero::MemoryTrackerSnapshot::ProcessSnapshot::MemoryNode::
109 MemoryNodeEntry::Decoder entry(*entry_it);
110
111 std::string unit;
112
113 switch (entry.units()) {
114 case 1: // BYTES
115 unit = RawMemoryGraphNode::kUnitsBytes;
116 break;
117 case 2: // COUNT
118 unit = RawMemoryGraphNode::kUnitsObjects;
119 break;
120 }
121 if (entry.has_value_uint64()) {
122 entries.emplace_back(entry.name().ToStdString(), unit,
123 entry.value_uint64());
124 } else if (entry.has_value_string()) {
125 entries.emplace_back(entry.name().ToStdString(), unit,
126 entry.value_string().ToStdString());
127 } else {
128 context_->storage->IncrementStats(
129 stats::memory_snapshot_parser_failure);
130 }
131 }
132 std::unique_ptr<RawMemoryGraphNode> raw_graph_node(new RawMemoryGraphNode(
133 absolute_name, level_of_detail, node_id, std::move(entries)));
134 raw_graph_node->set_flags(flags);
135 nodes_map.insert(
136 std::make_pair(absolute_name, std::move(raw_graph_node)));
137 }
138
139 for (auto edge_it = process_memory_dump.memory_edges(); edge_it;
140 ++edge_it) {
141 protos::pbzero::MemoryTrackerSnapshot::ProcessSnapshot::MemoryEdge::
142 Decoder edge(*edge_it);
143
144 std::unique_ptr<MemoryGraphEdge> graph_edge(new MemoryGraphEdge(
145 MemoryAllocatorNodeId(edge.source_id()),
146 MemoryAllocatorNodeId(edge.target_id()),
147 static_cast<int>(edge.importance()), edge.overridable()));
148
149 edges_map.insert(std::make_pair(MemoryAllocatorNodeId(edge.source_id()),
150 std::move(graph_edge)));
151 }
152 std::unique_ptr<RawProcessMemoryNode> raw_node(new RawProcessMemoryNode(
153 level_of_detail, std::move(edges_map), std::move(nodes_map)));
154 raw_nodes.insert(std::make_pair(pid, std::move(raw_node)));
155 }
156 }
157
GenerateGraph(RawMemoryNodeMap & raw_nodes)158 std::unique_ptr<GlobalNodeGraph> MemoryTrackerSnapshotParser::GenerateGraph(
159 RawMemoryNodeMap& raw_nodes) {
160 auto graph = GraphProcessor::CreateMemoryGraph(raw_nodes);
161 GraphProcessor::CalculateSizesForGraph(graph.get());
162 return graph;
163 }
164
EmitRows(int64_t ts,GlobalNodeGraph & graph,LevelOfDetail level_of_detail)165 void MemoryTrackerSnapshotParser::EmitRows(int64_t ts,
166 GlobalNodeGraph& graph,
167 LevelOfDetail level_of_detail) {
168 IdNodeMap id_node_map;
169
170 // For now, we use the existing global instant event track for chrome events,
171 // since memory dumps are global.
172 TrackId track_id =
173 context_->track_tracker->GetOrCreateLegacyChromeGlobalInstantTrack();
174
175 tables::MemorySnapshotTable::Row snapshot_row(
176 ts, track_id, level_of_detail_ids_[static_cast<size_t>(level_of_detail)]);
177 tables::MemorySnapshotTable::Id snapshot_row_id =
178 context_->storage->mutable_memory_snapshot_table()
179 ->Insert(snapshot_row)
180 .id;
181
182 for (auto const& it_process : graph.process_node_graphs()) {
183 tables::ProcessMemorySnapshotTable::Row process_row;
184 process_row.upid = context_->process_tracker->GetOrCreateProcess(
185 static_cast<uint32_t>(it_process.first));
186 process_row.snapshot_id = snapshot_row_id;
187 tables::ProcessMemorySnapshotTable::Id proc_snapshot_row_id =
188 context_->storage->mutable_process_memory_snapshot_table()
189 ->Insert(process_row)
190 .id;
191 EmitMemorySnapshotNodeRows(*(it_process.second->root()),
192 proc_snapshot_row_id, id_node_map);
193 }
194
195 // For each snapshot nodes from shared_memory_graph will be associated
196 // with a fabricated process_memory_snapshot entry whose pid == 0.
197 // TODO(mobica-google-contributors@mobica.com): Track the shared memory graph
198 // in a separate table.
199 tables::ProcessMemorySnapshotTable::Row fake_process_row;
200 fake_process_row.upid = context_->process_tracker->GetOrCreateProcess(0u);
201 fake_process_row.snapshot_id = snapshot_row_id;
202 tables::ProcessMemorySnapshotTable::Id fake_proc_snapshot_row_id =
203 context_->storage->mutable_process_memory_snapshot_table()
204 ->Insert(fake_process_row)
205 .id;
206 EmitMemorySnapshotNodeRows(*(graph.shared_memory_graph()->root()),
207 fake_proc_snapshot_row_id, id_node_map);
208
209 for (const auto& edge : graph.edges()) {
210 tables::MemorySnapshotEdgeTable::Row edge_row;
211 auto source_it = id_node_map.find(edge.source()->id());
212 if (source_it == id_node_map.end())
213 continue;
214 edge_row.source_node_id =
215 static_cast<tables::MemorySnapshotNodeTable::Id>(source_it->second);
216 auto target_it = id_node_map.find(edge.target()->id());
217 if (target_it == id_node_map.end())
218 continue;
219 edge_row.target_node_id =
220 static_cast<tables::MemorySnapshotNodeTable::Id>(target_it->second);
221 edge_row.importance = static_cast<uint32_t>(edge.priority());
222 context_->storage->mutable_memory_snapshot_edge_table()->Insert(edge_row);
223 }
224 }
225
EmitMemorySnapshotNodeRows(GlobalNodeGraph::Node & root_node_graph,ProcessMemorySnapshotId & proc_snapshot_row_id,IdNodeMap & id_node_map)226 void MemoryTrackerSnapshotParser::EmitMemorySnapshotNodeRows(
227 GlobalNodeGraph::Node& root_node_graph,
228 ProcessMemorySnapshotId& proc_snapshot_row_id,
229 IdNodeMap& id_node_map) {
230 EmitMemorySnapshotNodeRowsRecursively(root_node_graph, std::string(),
231 std::nullopt, proc_snapshot_row_id,
232 id_node_map);
233 }
234
EmitMemorySnapshotNodeRowsRecursively(GlobalNodeGraph::Node & node,const std::string & path,std::optional<tables::MemorySnapshotNodeTable::Id> parent_node_row_id,ProcessMemorySnapshotId & proc_snapshot_row_id,IdNodeMap & id_node_map)235 void MemoryTrackerSnapshotParser::EmitMemorySnapshotNodeRowsRecursively(
236 GlobalNodeGraph::Node& node,
237 const std::string& path,
238 std::optional<tables::MemorySnapshotNodeTable::Id> parent_node_row_id,
239 ProcessMemorySnapshotId& proc_snapshot_row_id,
240 IdNodeMap& id_node_map) {
241 std::optional<tables::MemorySnapshotNodeTable::Id> node_id;
242 // Skip emitting the root node into the tables - it is not a real node.
243 if (!path.empty()) {
244 node_id = EmitNode(node, path, parent_node_row_id, proc_snapshot_row_id,
245 id_node_map);
246 }
247
248 for (const auto& name_and_child : *node.children()) {
249 std::string child_path = path;
250 if (!child_path.empty())
251 child_path += "/";
252 child_path += name_and_child.first;
253
254 EmitMemorySnapshotNodeRowsRecursively(*(name_and_child.second), child_path,
255 /*parent_node_row_id=*/node_id,
256 proc_snapshot_row_id, id_node_map);
257 }
258 }
259
260 std::optional<tables::MemorySnapshotNodeTable::Id>
EmitNode(const GlobalNodeGraph::Node & node,const std::string & path,std::optional<tables::MemorySnapshotNodeTable::Id> parent_node_row_id,ProcessMemorySnapshotId & proc_snapshot_row_id,IdNodeMap & id_node_map)261 MemoryTrackerSnapshotParser::EmitNode(
262 const GlobalNodeGraph::Node& node,
263 const std::string& path,
264 std::optional<tables::MemorySnapshotNodeTable::Id> parent_node_row_id,
265 ProcessMemorySnapshotId& proc_snapshot_row_id,
266 IdNodeMap& id_node_map) {
267 tables::MemorySnapshotNodeTable::Row node_row;
268 node_row.process_snapshot_id = proc_snapshot_row_id;
269 node_row.parent_node_id = parent_node_row_id;
270 node_row.path = context_->storage->InternString(base::StringView(path));
271
272 tables::MemorySnapshotNodeTable::Id node_row_id =
273 context_->storage->mutable_memory_snapshot_node_table()
274 ->Insert(node_row)
275 .id;
276
277 auto* node_table = context_->storage->mutable_memory_snapshot_node_table();
278 uint32_t node_row_index =
279 static_cast<uint32_t>(*node_table->id().IndexOf(node_row_id));
280 ArgsTracker::BoundInserter args =
281 context_->args_tracker->AddArgsTo(node_row_id);
282
283 for (const auto& entry : node.const_entries()) {
284 switch (entry.second.type) {
285 case GlobalNodeGraph::Node::Entry::Type::kUInt64: {
286 int64_t value_int = static_cast<int64_t>(entry.second.value_uint64);
287
288 if (entry.first == "size") {
289 node_table->mutable_size()->Set(node_row_index, value_int);
290 } else if (entry.first == "effective_size") {
291 node_table->mutable_effective_size()->Set(node_row_index, value_int);
292 } else {
293 args.AddArg(context_->storage->InternString(
294 base::StringView(entry.first + ".value")),
295 Variadic::Integer(value_int));
296 if (entry.second.units < unit_ids_.size()) {
297 args.AddArg(context_->storage->InternString(
298 base::StringView(entry.first + ".unit")),
299 Variadic::String(unit_ids_[entry.second.units]));
300 }
301 }
302 break;
303 }
304 case GlobalNodeGraph::Node::Entry::Type::kString: {
305 args.AddArg(context_->storage->InternString(
306 base::StringView(entry.first + ".value")),
307 Variadic::String(context_->storage->InternString(
308 base::StringView(entry.second.value_string))));
309 break;
310 }
311 }
312 }
313 id_node_map.emplace(std::make_pair(node.id(), node_row_id));
314 return node_row_id;
315 }
316
GenerateGraphFromRawNodesAndEmitRows()317 void MemoryTrackerSnapshotParser::GenerateGraphFromRawNodesAndEmitRows() {
318 std::unique_ptr<GlobalNodeGraph> graph = GenerateGraph(aggregate_raw_nodes_);
319 EmitRows(last_snapshot_timestamp_, *graph, last_snapshot_level_of_detail_);
320 aggregate_raw_nodes_.clear();
321 }
322
323 } // namespace trace_processor
324 } // namespace perfetto
325