1 /*
2 * Copyright (C) 2024 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "src/trace_processor/perfetto_sql/intrinsics/table_functions/winscope_proto_to_args_with_defaults.h"
18
19 #include <cstddef>
20 #include <cstdint>
21 #include <memory>
22 #include <string>
23 #include <unordered_set>
24 #include <utility>
25 #include <vector>
26
27 #include "perfetto/base/logging.h"
28 #include "perfetto/base/status.h"
29 #include "perfetto/ext/base/base64.h"
30 #include "perfetto/ext/base/flat_hash_map.h"
31 #include "perfetto/ext/base/status_or.h"
32 #include "perfetto/ext/base/string_utils.h"
33 #include "perfetto/ext/base/string_view.h"
34 #include "perfetto/protozero/field.h"
35 #include "perfetto/trace_processor/basic_types.h"
36 #include "src/trace_processor/containers/string_pool.h"
37 #include "src/trace_processor/db/table.h"
38 #include "src/trace_processor/perfetto_sql/engine/perfetto_sql_engine.h"
39 #include "src/trace_processor/perfetto_sql/intrinsics/table_functions/tables_py.h"
40 #include "src/trace_processor/storage/trace_storage.h"
41 #include "src/trace_processor/types/trace_processor_context.h"
42 #include "src/trace_processor/util/descriptors.h"
43 #include "src/trace_processor/util/proto_to_args_parser.h"
44 #include "src/trace_processor/util/status_macros.h"
45 #include "src/trace_processor/util/winscope_proto_mapping.h"
46
47 namespace perfetto::trace_processor {
48 namespace tables {
49 WinscopeArgsWithDefaultsTable::~WinscopeArgsWithDefaultsTable() = default;
50 } // namespace tables
51
52 namespace {
53 constexpr char kDeinternError[] = "STRING DE-INTERNING ERROR";
54
55 // Interned data stored in table with columns:
56 // - base64_proto_id
57 // - flat_key
58 // - iid
59 // - deinterned_value
60 // Mapping reconstructed using nested FlatHashMaps to optionally
61 // deintern strings from proto data.
62 using ProtoId = uint32_t;
63 using FlatKey = StringPool::Id;
64 using Iid = uint64_t;
65 using DeinternedValue = StringPool::Id;
66
67 using DeinternedIids = base::FlatHashMap<Iid, DeinternedValue>;
68 using InternedData = base::FlatHashMap<FlatKey, DeinternedIids>;
69 using ProtoToInternedData = base::FlatHashMap<ProtoId, InternedData>;
70
GetProtoToInternedData(const std::string & table_name,TraceStorage * storage,StringPool * pool)71 ProtoToInternedData GetProtoToInternedData(const std::string& table_name,
72 TraceStorage* storage,
73 StringPool* pool) {
74 ProtoToInternedData proto_to_interned_data;
75 auto interned_data_table =
76 util::winscope_proto_mapping::GetInternedDataTable(table_name, storage);
77 if (interned_data_table) {
78 const Table* table = interned_data_table.value();
79 const auto proto_id_idx =
80 table->ColumnIdxFromName("base64_proto_id").value();
81 const auto flat_key_idx = table->ColumnIdxFromName("flat_key").value();
82 const auto iid_idx = table->ColumnIdxFromName("iid").value();
83 const auto deinterned_value_idx =
84 table->ColumnIdxFromName("deinterned_value").value();
85
86 for (auto it = table->IterateRows(); it; ++it) {
87 const auto proto_id =
88 static_cast<uint32_t>(it.Get(proto_id_idx).AsLong());
89 const auto flat_key = pool->InternString(
90 base::StringView(std::string(it.Get(flat_key_idx).AsString())));
91 const auto iid = static_cast<uint64_t>(it.Get(iid_idx).AsLong());
92 const auto deinterned_value = pool->InternString(base::StringView(
93 std::string(it.Get(deinterned_value_idx).AsString())));
94
95 auto& deinterned_iids = proto_to_interned_data[proto_id][flat_key];
96 deinterned_iids.Insert(iid, deinterned_value);
97 }
98 }
99 return proto_to_interned_data;
100 }
101
102 using RowReference = tables::WinscopeArgsWithDefaultsTable::RowReference;
103 using Row = tables::WinscopeArgsWithDefaultsTable::Row;
104 using RowId = tables::WinscopeArgsWithDefaultsTable::Id;
105 using KeyToRowMap = std::unordered_map<StringPool::Id, RowId>;
106
107 class Delegate : public util::ProtoToArgsParser::Delegate {
108 public:
109 using Key = util::ProtoToArgsParser::Key;
Delegate(StringPool * pool,const uint32_t base64_proto_id,tables::WinscopeArgsWithDefaultsTable * table,KeyToRowMap * key_to_row,const InternedData * interned_data)110 explicit Delegate(StringPool* pool,
111 const uint32_t base64_proto_id,
112 tables::WinscopeArgsWithDefaultsTable* table,
113 KeyToRowMap* key_to_row,
114 const InternedData* interned_data)
115 : pool_(pool),
116 base64_proto_id_(base64_proto_id),
117 table_(table),
118 key_to_row_(key_to_row),
119 interned_data_(interned_data) {}
120
AddInteger(const Key & key,int64_t res)121 void AddInteger(const Key& key, int64_t res) override {
122 if (TryAddDeinternedString(key, static_cast<uint64_t>(res))) {
123 return;
124 }
125 RowReference r = GetOrCreateRow(key);
126 r.set_int_value(res);
127 }
AddUnsignedInteger(const Key & key,uint64_t res)128 void AddUnsignedInteger(const Key& key, uint64_t res) override {
129 if (TryAddDeinternedString(key, static_cast<uint64_t>(res))) {
130 return;
131 }
132 RowReference r = GetOrCreateRow(key);
133 r.set_int_value(int64_t(res));
134 }
AddString(const Key & key,const protozero::ConstChars & res)135 void AddString(const Key& key, const protozero::ConstChars& res) override {
136 RowReference r = GetOrCreateRow(key);
137 r.set_string_value(
138 pool_->InternString(base::StringView((res.ToStdString()))));
139 }
AddString(const Key & key,const std::string & res)140 void AddString(const Key& key, const std::string& res) override {
141 RowReference r = GetOrCreateRow(key);
142 r.set_string_value(pool_->InternString(base::StringView(res)));
143 }
AddDouble(const Key & key,double res)144 void AddDouble(const Key& key, double res) override {
145 RowReference r = GetOrCreateRow(key);
146 r.set_real_value(res);
147 }
AddBoolean(const Key & key,bool res)148 void AddBoolean(const Key& key, bool res) override {
149 RowReference r = GetOrCreateRow(key);
150 r.set_int_value(res);
151 }
AddBytes(const Key & key,const protozero::ConstBytes & res)152 void AddBytes(const Key& key, const protozero::ConstBytes& res) override {
153 RowReference r = GetOrCreateRow(key);
154 r.set_string_value(
155 pool_->InternString(base::StringView((res.ToStdString()))));
156 }
AddNull(const Key & key)157 void AddNull(const Key& key) override { GetOrCreateRow(key); }
AddPointer(const Key &,uint64_t)158 void AddPointer(const Key&, uint64_t) override {
159 PERFETTO_FATAL("Unsupported");
160 }
AddJson(const Key &,const protozero::ConstChars &)161 bool AddJson(const Key&, const protozero::ConstChars&) override {
162 PERFETTO_FATAL("Unsupported");
163 }
GetArrayEntryIndex(const std::string &)164 size_t GetArrayEntryIndex(const std::string&) override {
165 PERFETTO_FATAL("Unsupported");
166 }
IncrementArrayEntryIndex(const std::string &)167 size_t IncrementArrayEntryIndex(const std::string&) override {
168 PERFETTO_FATAL("Unsupported");
169 }
seq_state()170 PacketSequenceStateGeneration* seq_state() override { return nullptr; }
171
ShouldAddDefaultArg(const Key & key)172 bool ShouldAddDefaultArg(const Key& key) override {
173 if (!key_to_row_) {
174 return true;
175 }
176 auto key_id = pool_->InternString(base::StringView(key.key));
177 auto pos = key_to_row_->find(key_id);
178 return pos == key_to_row_->end();
179 }
180
181 private:
GetInternedMessageView(uint32_t,uint64_t)182 InternedMessageView* GetInternedMessageView(uint32_t, uint64_t) override {
183 return nullptr;
184 }
185
GetOrCreateRow(const Key & key)186 RowReference GetOrCreateRow(const Key& key) {
187 RowId row_id;
188 if (!key_to_row_) {
189 Row new_row;
190 row_id = table_->Insert(new_row).id;
191 } else {
192 auto key_id = pool_->InternString(base::StringView(key.key));
193 auto pos = key_to_row_->find(key_id);
194 if (pos != key_to_row_->end()) {
195 row_id = pos->second;
196 } else {
197 Row new_row;
198 row_id = table_->Insert(new_row).id;
199 key_to_row_->insert({key_id, row_id});
200 }
201 }
202
203 auto row = table_->FindById(row_id).value();
204 row.set_key(pool_->InternString(base::StringView(key.key)));
205 row.set_flat_key(pool_->InternString(base::StringView(key.flat_key)));
206 row.set_base64_proto_id(base64_proto_id_);
207 return row;
208 }
209
TryAddDeinternedString(const Key & key,uint64_t iid)210 bool TryAddDeinternedString(const Key& key, uint64_t iid) {
211 if (!interned_data_ || !base::EndsWith(key.key, "_iid")) {
212 return false;
213 }
214 const auto deinterned_key =
215 Key{key.flat_key.substr(0, key.flat_key.size() - 4),
216 key.key.substr(0, key.key.size() - 4)};
217 const auto deinterned_value = TryDeinternString(key, iid);
218 if (!deinterned_value) {
219 AddString(deinterned_key,
220 protozero::ConstChars{kDeinternError, sizeof(kDeinternError)});
221 return false;
222 }
223 AddString(deinterned_key, *deinterned_value);
224 return true;
225 }
226
TryDeinternString(const Key & key,uint64_t iid)227 std::optional<std::string> TryDeinternString(const Key& key, uint64_t iid) {
228 DeinternedIids* deinterned_iids = interned_data_->Find(
229 pool_->InternString(base::StringView(key.flat_key)));
230 if (!deinterned_iids) {
231 return std::nullopt;
232 }
233 auto* deinterned_value = deinterned_iids->Find(iid);
234 if (!deinterned_value) {
235 return std::nullopt;
236 }
237 return pool_->Get(*(deinterned_value)).data();
238 }
239
240 StringPool* pool_;
241 const uint32_t base64_proto_id_;
242 tables::WinscopeArgsWithDefaultsTable* table_;
243 KeyToRowMap* key_to_row_;
244 const InternedData* interned_data_;
245 };
246
InsertRows(const Table & static_table,tables::WinscopeArgsWithDefaultsTable * inflated_args_table,const std::string & proto_name,const std::vector<uint32_t> * allowed_fields,const std::string * group_id_col_name,DescriptorPool & descriptor_pool,StringPool * string_pool,const ProtoToInternedData & proto_to_interned_data)247 base::Status InsertRows(
248 const Table& static_table,
249 tables::WinscopeArgsWithDefaultsTable* inflated_args_table,
250 const std::string& proto_name,
251 const std::vector<uint32_t>* allowed_fields,
252 const std::string* group_id_col_name,
253 DescriptorPool& descriptor_pool,
254 StringPool* string_pool,
255 const ProtoToInternedData& proto_to_interned_data) {
256 util::ProtoToArgsParser args_parser{descriptor_pool};
257 const auto base64_proto_id_col_idx =
258 static_table.ColumnIdxFromName("base64_proto_id").value();
259
260 std::optional<uint32_t> group_id_col_idx;
261 if (group_id_col_name) {
262 group_id_col_idx = static_table.ColumnIdxFromName(*group_id_col_name);
263 }
264
265 std::unordered_set<uint32_t> inflated_protos;
266 std::unordered_map<uint32_t, KeyToRowMap> group_id_to_key_row_map;
267 for (auto it = static_table.IterateRows(); it; ++it) {
268 const auto base64_proto_id =
269 static_cast<uint32_t>(it.Get(base64_proto_id_col_idx).AsLong());
270 if (inflated_protos.count(base64_proto_id) > 0) {
271 continue;
272 }
273 inflated_protos.insert(base64_proto_id);
274
275 const auto raw_proto =
276 string_pool->Get(StringPool::Id::Raw(base64_proto_id));
277 const auto blob = *base::Base64Decode(raw_proto);
278 const auto cb = protozero::ConstBytes{
279 reinterpret_cast<const uint8_t*>(blob.data()), blob.size()};
280
281 KeyToRowMap* key_to_row = nullptr;
282 if (group_id_col_idx.has_value()) {
283 auto group_id = static_cast<uint32_t>(it.Get(*group_id_col_idx).AsLong());
284 auto pos = group_id_to_key_row_map.find(group_id);
285 if (pos != group_id_to_key_row_map.end()) {
286 key_to_row = &(pos->second);
287 } else {
288 key_to_row = &(group_id_to_key_row_map[group_id]);
289 }
290 }
291
292 InternedData* interned_data = proto_to_interned_data.Find(base64_proto_id);
293 Delegate delegate(string_pool, base64_proto_id, inflated_args_table,
294 key_to_row, interned_data);
295 RETURN_IF_ERROR(args_parser.ParseMessage(cb, proto_name, allowed_fields,
296 delegate, nullptr, true));
297 }
298 return base::OkStatus();
299 }
300 } // namespace
301
WinscopeProtoToArgsWithDefaults(StringPool * string_pool,const PerfettoSqlEngine * engine,TraceProcessorContext * context)302 WinscopeProtoToArgsWithDefaults::WinscopeProtoToArgsWithDefaults(
303 StringPool* string_pool,
304 const PerfettoSqlEngine* engine,
305 TraceProcessorContext* context)
306 : string_pool_(string_pool), engine_(engine), context_(context) {}
307
308 base::StatusOr<std::unique_ptr<Table>>
ComputeTable(const std::vector<SqlValue> & arguments)309 WinscopeProtoToArgsWithDefaults::ComputeTable(
310 const std::vector<SqlValue>& arguments) {
311 PERFETTO_CHECK(arguments.size() == 1);
312 if (arguments[0].type != SqlValue::kString) {
313 return base::ErrStatus(
314 "__intrinsic_winscope_proto_to_args_with_defaults takes table name as "
315 "a string.");
316 }
317 std::string table_name = arguments[0].AsString();
318
319 const Table* static_table = engine_->GetTableOrNull(table_name);
320 if (!static_table) {
321 return base::ErrStatus("Failed to find %s table.", table_name.c_str());
322 }
323
324 std::string proto_name;
325 ASSIGN_OR_RETURN(proto_name,
326 util::winscope_proto_mapping::GetProtoName(table_name));
327
328 auto table =
329 std::make_unique<tables::WinscopeArgsWithDefaultsTable>(string_pool_);
330 auto allowed_fields =
331 util::winscope_proto_mapping::GetAllowedFields(table_name);
332 auto group_id_col_name =
333 util::winscope_proto_mapping::GetGroupIdColName(table_name);
334 auto proto_to_interned_data =
335 GetProtoToInternedData(table_name, context_->storage.get(), string_pool_);
336
337 RETURN_IF_ERROR(InsertRows(
338 *static_table, table.get(), proto_name,
339 allowed_fields ? &allowed_fields.value() : nullptr,
340 group_id_col_name ? &group_id_col_name.value() : nullptr,
341 *context_->descriptor_pool_, string_pool_, proto_to_interned_data));
342
343 return std::unique_ptr<Table>(std::move(table));
344 }
345
CreateSchema()346 Table::Schema WinscopeProtoToArgsWithDefaults::CreateSchema() {
347 return tables::WinscopeArgsWithDefaultsTable::ComputeStaticSchema();
348 }
349
TableName()350 std::string WinscopeProtoToArgsWithDefaults::TableName() {
351 return tables::WinscopeArgsWithDefaultsTable::Name();
352 }
353
EstimateRowCount()354 uint32_t WinscopeProtoToArgsWithDefaults::EstimateRowCount() {
355 // 100 inflated args per 100 elements per 100 entries
356 return 1000000;
357 }
358 } // namespace perfetto::trace_processor
359