1 /*
2 * Copyright (C) 2021 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "perfetto/ext/base/file_utils.h"
18 #include "perfetto/ext/base/getopt.h"
19 #include "perfetto/ext/base/scoped_file.h"
20 #include "perfetto/ext/base/string_utils.h"
21 #include "perfetto/ext/base/version.h"
22 #include "src/protozero/filtering/filter_util.h"
23 #include "src/protozero/filtering/message_filter.h"
24
25 namespace perfetto {
26 namespace proto_filter {
27 namespace {
28
29 const char kUsage[] =
30 R"(Usage: proto_filter [-s schema_in] [-i message in] [-o message out] [-f filter in] [-F filter out] [-T filter_oct_out] [-d --dedupe] [-I proto include path] [-r root message]
31
32 -s --schema-in: Path to the root .proto file. Required for most operations
33 -I --proto_path: Extra include directory for proto includes. If omitted assumed CWD.
34 -r --root_message: Fully qualified name for the root proto message (e.g. perfetto.protos.Trace)
35 If omitted the first message defined in the schema will be used.
36 -i --msg_in: Path of a binary-encoded proto message which will be filtered.
37 -o --msg_out: Path of the binary-encoded filtered proto message written in output.
38 -f --filter_in: Path of a filter bytecode file previously generated by this tool.
39 -F --filter_out: Path of the filter bytecode file generated from the --schema-in definition.
40 -T --filter_oct_out: Like --filter_out, but emits a octal-escaped C string suitable for .pbtx.
41 -d --dedupe: Minimize filter size by deduping leaf messages with same field ids.
42
43 Example usage:
44
45 # Convert a .proto schema file into a diff-friendly list of messages/fields>
46
47 proto_filter -r perfetto.protos.Trace -s protos/perfetto/trace/trace.proto
48
49 # Generate the filter bytecode from a .proto schema
50
51 proto_filter -r perfetto.protos.Trace -s protos/perfetto/trace/trace.proto \
52 -F /tmp/bytecode [--dedupe] [-x protos.Message:field_to_pass]
53
54 # List the used/filtered fields from a trace file
55
56 proto_filter -r perfetto.protos.Trace -s protos/perfetto/trace/trace.proto \
57 -i test/data/example_android_trace_30s.pb -f /tmp/bytecode
58
59 # Filter a trace using a filter bytecode
60
61 proto_filter -i test/data/example_android_trace_30s.pb -f /tmp/bytecode \
62 -o /tmp/filtered_trace
63
64 # Show which fields are allowed by a filter bytecode
65
66 proto_filter -r perfetto.protos.Trace -s protos/perfetto/trace/trace.proto \
67 -f /tmp/bytecode
68 )";
69
Main(int argc,char ** argv)70 int Main(int argc, char** argv) {
71 static const option long_options[] = {
72 {"help", no_argument, nullptr, 'h'},
73 {"version", no_argument, nullptr, 'v'},
74 {"dedupe", no_argument, nullptr, 'd'},
75 {"proto_path", required_argument, nullptr, 'I'},
76 {"schema_in", required_argument, nullptr, 's'},
77 {"root_message", required_argument, nullptr, 'r'},
78 {"msg_in", required_argument, nullptr, 'i'},
79 {"msg_out", required_argument, nullptr, 'o'},
80 {"filter_in", required_argument, nullptr, 'f'},
81 {"filter_out", required_argument, nullptr, 'F'},
82 {"filter_oct_out", required_argument, nullptr, 'T'},
83 {"passthrough", required_argument, nullptr, 'x'},
84 {nullptr, 0, nullptr, 0}};
85
86 std::string msg_in;
87 std::string msg_out;
88 std::string filter_in;
89 std::string schema_in;
90 std::string filter_out;
91 std::string filter_oct_out;
92 std::string proto_path;
93 std::string root_message_arg;
94 std::set<std::string> passthrough_fields;
95 bool dedupe = false;
96
97 for (;;) {
98 int option =
99 getopt_long(argc, argv, "hvdI:s:r:i:o:f:F:T:x:", long_options, nullptr);
100
101 if (option == -1)
102 break; // EOF.
103
104 if (option == 'v') {
105 printf("%s\n", base::GetVersionString());
106 exit(0);
107 }
108
109 if (option == 'd') {
110 dedupe = true;
111 continue;
112 }
113
114 if (option == 'I') {
115 proto_path = optarg;
116 continue;
117 }
118
119 if (option == 's') {
120 schema_in = optarg;
121 continue;
122 }
123
124 if (option == 'r') {
125 root_message_arg = optarg;
126 continue;
127 }
128
129 if (option == 'i') {
130 msg_in = optarg;
131 continue;
132 }
133
134 if (option == 'o') {
135 msg_out = optarg;
136 continue;
137 }
138
139 if (option == 'f') {
140 filter_in = optarg;
141 continue;
142 }
143
144 if (option == 'F') {
145 filter_out = optarg;
146 continue;
147 }
148
149 if (option == 'T') {
150 filter_oct_out = optarg;
151 continue;
152 }
153
154 if (option == 'x') {
155 passthrough_fields.insert(optarg);
156 continue;
157 }
158
159 if (option == 'h') {
160 fprintf(stdout, kUsage);
161 exit(0);
162 }
163
164 fprintf(stderr, kUsage);
165 exit(1);
166 }
167
168 if (msg_in.empty() && filter_in.empty() && schema_in.empty()) {
169 fprintf(stderr, kUsage);
170 return 1;
171 }
172
173 std::string msg_in_data;
174 if (!msg_in.empty()) {
175 PERFETTO_LOG("Loading proto-encoded message from %s", msg_in.c_str());
176 if (!base::ReadFile(msg_in, &msg_in_data)) {
177 PERFETTO_ELOG("Could not open message file %s", msg_in.c_str());
178 return 1;
179 }
180 }
181
182 protozero::FilterUtil filter;
183 if (!schema_in.empty()) {
184 PERFETTO_LOG("Loading proto schema from %s", schema_in.c_str());
185 if (!filter.LoadMessageDefinition(schema_in, root_message_arg, proto_path,
186 passthrough_fields)) {
187 PERFETTO_ELOG("Failed to parse proto schema from %s", schema_in.c_str());
188 return 1;
189 }
190 if (dedupe)
191 filter.Dedupe();
192 }
193
194 protozero::MessageFilter msg_filter;
195 std::string filter_data;
196 std::string filter_data_src;
197 if (!filter_in.empty()) {
198 PERFETTO_LOG("Loading filter bytecode from %s", filter_in.c_str());
199 if (!base::ReadFile(filter_in, &filter_data)) {
200 PERFETTO_ELOG("Could not open filter file %s", filter_in.c_str());
201 return 1;
202 }
203 filter_data_src = filter_in;
204 } else if (!schema_in.empty()) {
205 PERFETTO_LOG("Generating filter bytecode from %s", schema_in.c_str());
206 filter_data = filter.GenerateFilterBytecode();
207 filter_data_src = schema_in;
208 }
209
210 if (!filter_data.empty()) {
211 const uint8_t* data = reinterpret_cast<const uint8_t*>(filter_data.data());
212 if (!msg_filter.LoadFilterBytecode(data, filter_data.size())) {
213 PERFETTO_ELOG("Failed to parse filter bytecode from %s",
214 filter_data_src.c_str());
215 return 1;
216 }
217 }
218
219 // Write the filter bytecode in output.
220 if (!filter_out.empty()) {
221 auto fd = base::OpenFile(filter_out, O_WRONLY | O_TRUNC | O_CREAT, 0644);
222 if (!fd) {
223 PERFETTO_ELOG("Could not open filter out path %s", filter_out.c_str());
224 return 1;
225 }
226 PERFETTO_LOG("Writing filter bytecode (%zu bytes) into %s",
227 filter_data.size(), filter_out.c_str());
228 base::WriteAll(*fd, filter_data.data(), filter_data.size());
229 }
230
231 if (!filter_oct_out.empty()) {
232 auto fd =
233 base::OpenFile(filter_oct_out, O_WRONLY | O_TRUNC | O_CREAT, 0644);
234 if (!fd) {
235 PERFETTO_ELOG("Could not open filter out path %s",
236 filter_oct_out.c_str());
237 return 1;
238 }
239 std::string oct_str;
240 oct_str.reserve(filter_data.size() * 4 + 64);
241 oct_str.append("trace_filter {\n bytecode: \"");
242 for (char c : filter_data) {
243 uint8_t octect = static_cast<uint8_t>(c);
244 char buf[5]{'\\', '0', '0', '0', 0};
245 for (uint8_t i = 0; i < 3; ++i) {
246 buf[3 - i] = static_cast<char>('0' + static_cast<uint8_t>(octect) % 8);
247 octect /= 8;
248 }
249 oct_str.append(buf);
250 }
251 oct_str.append("\"\n}\n");
252 PERFETTO_LOG("Writing filter bytecode (%zu bytes) into %s", oct_str.size(),
253 filter_oct_out.c_str());
254 base::WriteAll(*fd, oct_str.data(), oct_str.size());
255 }
256
257 // Apply the filter to the input message (if any).
258 std::vector<uint8_t> msg_filtered_data;
259 if (!msg_in.empty()) {
260 PERFETTO_LOG("Applying filter %s to proto message %s",
261 filter_data_src.c_str(), msg_in.c_str());
262 msg_filter.enable_field_usage_tracking(true);
263 auto res = msg_filter.FilterMessage(msg_in_data.data(), msg_in_data.size());
264 if (res.error)
265 PERFETTO_FATAL("Filtering failed");
266 msg_filtered_data.insert(msg_filtered_data.end(), res.data.get(),
267 res.data.get() + res.size);
268 }
269
270 // Write out the filtered message.
271 if (!msg_out.empty()) {
272 PERFETTO_LOG("Writing filtered proto bytes (%zu bytes) into %s",
273 msg_filtered_data.size(), msg_out.c_str());
274 auto fd = base::OpenFile(msg_out, O_WRONLY | O_TRUNC | O_CREAT, 0644);
275 base::WriteAll(*fd, msg_filtered_data.data(), msg_filtered_data.size());
276 }
277
278 if (!msg_in.empty()) {
279 const auto& field_usage_map = msg_filter.field_usage();
280 for (const auto& it : field_usage_map) {
281 const std::string& field_path_varint = it.first;
282 int32_t num_occurrences = it.second;
283 std::string path_str = filter.LookupField(field_path_varint);
284 printf("%-100s %s %d\n", path_str.c_str(),
285 num_occurrences < 0 ? "DROP" : "PASS", std::abs(num_occurrences));
286 }
287 } else if (!schema_in.empty()) {
288 filter.PrintAsText(!filter_data.empty() ? std::make_optional(filter_data)
289 : std::nullopt);
290 }
291
292 if ((!filter_out.empty() || !filter_oct_out.empty()) && !dedupe) {
293 PERFETTO_ELOG(
294 "Warning: looks like you are generating a filter without --dedupe. For "
295 "production use cases, --dedupe can make the output bytecode "
296 "significantly smaller.");
297 }
298 return 0;
299 }
300
301 } // namespace
302 } // namespace proto_filter
303 } // namespace perfetto
304
main(int argc,char ** argv)305 int main(int argc, char** argv) {
306 return perfetto::proto_filter::Main(argc, argv);
307 }
308