1 /*
2 * Copyright (C) 2022 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "src/trace_processor/trace_summary/summary.h"
18
19 #include <cstddef>
20 #include <cstdint>
21 #include <optional>
22 #include <string>
23 #include <string_view>
24 #include <vector>
25
26 #include "perfetto/base/logging.h"
27 #include "perfetto/base/status.h"
28 #include "perfetto/ext/base/flat_hash_map.h"
29 #include "perfetto/ext/base/status_or.h"
30 #include "perfetto/protozero/field.h"
31 #include "perfetto/protozero/scattered_heap_buffer.h"
32 #include "perfetto/trace_processor/basic_types.h"
33 #include "perfetto/trace_processor/trace_processor.h"
34 #include "src/protozero/text_to_proto/text_to_proto.h"
35 #include "src/trace_processor/perfetto_sql/generator/structured_query_generator.h"
36 #include "src/trace_processor/trace_summary/trace_summary.descriptor.h"
37 #include "src/trace_processor/util/descriptors.h"
38 #include "src/trace_processor/util/protozero_to_text.h"
39 #include "src/trace_processor/util/status_macros.h"
40
41 #include "protos/perfetto/trace_summary/file.pbzero.h"
42 #include "protos/perfetto/trace_summary/v2_metric.pbzero.h"
43
44 namespace perfetto::trace_processor::summary {
45
46 namespace {
47
48 struct Metric {
49 std::string query;
50 protozero::ConstBytes spec;
51 };
52
53 using perfetto_sql::generator::StructuredQueryGenerator;
54
WriteMetadata(TraceProcessor * processor,const std::string & metadata_sql,protos::pbzero::TraceSummary * summary)55 base::Status WriteMetadata(TraceProcessor* processor,
56 const std::string& metadata_sql,
57 protos::pbzero::TraceSummary* summary) {
58 auto it = processor->ExecuteQuery(metadata_sql);
59 RETURN_IF_ERROR(it.Status());
60
61 uint32_t col_count = it.ColumnCount();
62
63 // This can happen if there is no metadata. Just early return in that case.
64 if (col_count == 0) {
65 return base::OkStatus();
66 }
67
68 // Otherwise we expect a strict schema of (key, value).
69 if (col_count != 2 || it.GetColumnName(0) != "key" ||
70 it.GetColumnName(1) != "value") {
71 return base::ErrStatus(
72 "Metadata query did not match schema of (key, value)");
73 }
74 while (it.Next()) {
75 auto key = it.Get(0);
76 if (key.type != SqlValue::kString) {
77 return base::ErrStatus(
78 "Key column in metadata query was not of type string");
79 }
80 // Silently ignore any null values.
81 auto value = it.Get(1);
82 if (value.is_null()) {
83 continue;
84 }
85 if (value.type != SqlValue::kString) {
86 return base::ErrStatus(
87 "Key column in metadata query was not of type string or null");
88 }
89 auto* metadata = summary->add_metadata();
90 metadata->set_key(key.AsString());
91 metadata->set_value(value.AsString());
92 }
93 return it.Status();
94 }
95
CreateQueriesAndComputeMetrics(TraceProcessor * processor,const DescriptorPool & pool,const std::vector<StructuredQueryGenerator::Query> & queries,const base::FlatHashMap<std::string,Metric> & queries_per_metric,const std::optional<std::string> & metadata_sql,std::vector<uint8_t> * output,const TraceSummaryOutputSpec & output_spec)96 base::Status CreateQueriesAndComputeMetrics(
97 TraceProcessor* processor,
98 const DescriptorPool& pool,
99 const std::vector<StructuredQueryGenerator::Query>& queries,
100 const base::FlatHashMap<std::string, Metric>& queries_per_metric,
101 const std::optional<std::string>& metadata_sql,
102 std::vector<uint8_t>* output,
103 const TraceSummaryOutputSpec& output_spec) {
104 for (const auto& query : queries) {
105 auto it = processor->ExecuteQuery("CREATE PERFETTO TABLE " +
106 query.table_name + " " + query.sql);
107 PERFETTO_CHECK(!it.Next());
108 if (!it.Status().ok()) {
109 return base::ErrStatus("Error while executing shared query %s: %s",
110 query.id.c_str(), it.Status().c_message());
111 }
112 }
113 protozero::HeapBuffered<protos::pbzero::TraceSummary> summary;
114 for (auto m = queries_per_metric.GetIterator(); m; ++m) {
115 if (m.value().query.empty()) {
116 return base::ErrStatus("Metric %s was not found in any summary spec",
117 m.key().c_str());
118 }
119 auto* metric = summary->add_metric();
120 metric->AppendBytes(protos::pbzero::TraceMetricV2::kSpecFieldNumber,
121 m.value().spec.data, m.value().spec.size);
122
123 auto it = processor->ExecuteQuery(m.value().query);
124 uint32_t col_count = it.ColumnCount();
125 while (it.Next()) {
126 PERFETTO_CHECK(col_count > 0);
127 const auto& value = it.Get(col_count - 1);
128
129 // Skip null rows.
130 if (value.is_null()) {
131 continue;
132 }
133
134 auto* row = metric->add_row();
135 for (uint32_t i = 0; i < col_count - 1; ++i) {
136 const auto& dim = it.Get(i);
137 switch (dim.type) {
138 case SqlValue::kLong:
139 row->add_dimension()->set_int64_value(dim.AsLong());
140 break;
141 case SqlValue::kDouble:
142 row->add_dimension()->set_double_value(dim.AsDouble());
143 break;
144 case SqlValue::kString:
145 row->add_dimension()->set_string_value(dim.AsString());
146 break;
147 case SqlValue::kNull:
148 row->add_dimension()->set_null_value();
149 break;
150 case SqlValue::kBytes:
151 return base::ErrStatus(
152 "Received bytes for dimension in metric %s: this is not "
153 "supported",
154 m.key().c_str());
155 }
156 }
157
158 switch (value.type) {
159 case SqlValue::kLong:
160 row->set_value(static_cast<double>(value.AsLong()));
161 break;
162 case SqlValue::kDouble:
163 row->set_value(value.AsDouble());
164 break;
165 case SqlValue::kNull:
166 PERFETTO_FATAL("Null value should have been skipped");
167 case SqlValue::kString:
168 return base::ErrStatus(
169 "Received string for metric value in metric %s: this is not "
170 "supported",
171 m.key().c_str());
172 case SqlValue::kBytes:
173 return base::ErrStatus(
174 "Received bytes for metric value in metric %s: this is not "
175 "supported",
176 m.key().c_str());
177 }
178 }
179 RETURN_IF_ERROR(it.Status());
180 }
181 if (metadata_sql) {
182 RETURN_IF_ERROR(WriteMetadata(processor, *metadata_sql, summary.get()));
183 }
184 switch (output_spec.format) {
185 case TraceSummaryOutputSpec::Format::kBinaryProto:
186 *output = summary.SerializeAsArray();
187 break;
188 case TraceSummaryOutputSpec::Format::kTextProto:
189 std::vector<uint8_t> proto = summary.SerializeAsArray();
190 std::string out = protozero_to_text::ProtozeroToText(
191 pool, ".perfetto.protos.TraceSummary",
192 protozero::ConstBytes{proto.data(), proto.size()});
193 *output = std::vector<uint8_t>(out.begin(), out.end());
194 break;
195 }
196 return base::OkStatus();
197 }
198
199 } // namespace
200
Summarize(TraceProcessor * processor,const DescriptorPool & pool,const TraceSummaryComputationSpec & computation,const std::vector<TraceSummarySpecBytes> & specs,std::vector<uint8_t> * output,const TraceSummaryOutputSpec & output_spec)201 base::Status Summarize(TraceProcessor* processor,
202 const DescriptorPool& pool,
203 const TraceSummaryComputationSpec& computation,
204 const std::vector<TraceSummarySpecBytes>& specs,
205 std::vector<uint8_t>* output,
206 const TraceSummaryOutputSpec& output_spec) {
207 std::vector<protos::pbzero::TraceSummarySpec::Decoder> spec_decoders;
208 std::vector<std::vector<uint8_t>> textproto_converted_specs(specs.size());
209 for (uint32_t i = 0; i < specs.size(); ++i) {
210 switch (specs[i].format) {
211 case TraceSummarySpecBytes::Format::kBinaryProto:
212 spec_decoders.emplace_back(specs[i].ptr, specs[i].size);
213 break;
214 case TraceSummarySpecBytes::Format::kTextProto:
215 ASSIGN_OR_RETURN(
216 textproto_converted_specs[i],
217 protozero::TextToProto(
218 kTraceSummaryDescriptor.data(), kTraceSummaryDescriptor.size(),
219 ".perfetto.protos.TraceSummarySpec", "-",
220 std::string_view(reinterpret_cast<const char*>(specs[i].ptr),
221 specs[i].size)));
222 spec_decoders.emplace_back(textproto_converted_specs[i].data(),
223 textproto_converted_specs[i].size());
224 break;
225 }
226 }
227
228 perfetto_sql::generator::StructuredQueryGenerator generator;
229 for (const auto& spec : spec_decoders) {
230 for (auto it = spec.query(); it; ++it) {
231 RETURN_IF_ERROR(generator.AddQuery(it->data(), it->size()));
232 }
233 }
234
235 base::FlatHashMap<std::string, Metric> queries_per_metric;
236 if (!computation.v2_metric_ids.empty()) {
237 for (const auto& id : computation.v2_metric_ids) {
238 queries_per_metric.Insert(id, Metric{});
239 }
240 for (const auto& spec : spec_decoders) {
241 for (auto it = spec.metric_spec(); it; ++it) {
242 protos::pbzero::TraceMetricV2Spec::Decoder m(*it);
243 std::string id = m.id().ToStdString();
244 if (id.empty()) {
245 return base::ErrStatus(
246 "Metric with empty id field: this is not allowed");
247 }
248
249 // If metric ids is empty, we need to compute all metrics. Otherwise
250 // only compute metrics which were populated in the map.
251 Metric* metric = queries_per_metric.Find(id);
252 if (!metric) {
253 continue;
254 }
255 if (!metric->query.empty()) {
256 return base::ErrStatus(
257 "Duplicate definitions for metric %s received: this is not "
258 "allowed",
259 id.c_str());
260 }
261 base::StatusOr<std::string> query_or =
262 generator.Generate(m.query().data, m.query().size);
263 if (!query_or.ok()) {
264 return base::ErrStatus("Unable to build query for metric %s: %s",
265 id.c_str(), query_or.status().c_message());
266 }
267 metric->query = *query_or;
268 metric->spec = protozero::ConstBytes{
269 m.begin(),
270 static_cast<size_t>(m.end() - m.begin()),
271 };
272 }
273 }
274 }
275
276 std::optional<std::string> metadata_sql;
277 if (computation.metadata_query_id) {
278 ASSIGN_OR_RETURN(metadata_sql,
279 generator.GenerateById(*computation.metadata_query_id));
280 }
281
282 for (const auto& module : generator.ComputeReferencedModules()) {
283 auto it = processor->ExecuteQuery("INCLUDE PERFETTO MODULE " + module);
284 PERFETTO_CHECK(!it.Next());
285 RETURN_IF_ERROR(it.Status());
286 }
287
288 auto queries = generator.referenced_queries();
289 base::Status status = CreateQueriesAndComputeMetrics(
290 processor, pool, queries, queries_per_metric, metadata_sql, output,
291 output_spec);
292
293 // Make sure to cleanup all the queries.
294 for (const auto& query : queries) {
295 auto it =
296 processor->ExecuteQuery("DROP TABLE IF EXISTS " + query.table_name);
297 PERFETTO_CHECK(!it.Next());
298 PERFETTO_CHECK(it.Status().ok());
299 }
300 return status;
301 }
302
303 } // namespace perfetto::trace_processor::summary
304