1 #include "annotated_binary_text_gen.h"
2
3 #include <algorithm>
4 #include <cstdint>
5 #include <fstream>
6 #include <ostream>
7 #include <sstream>
8 #include <string>
9
10 #include "binary_annotator.h"
11 #include "flatbuffers/base.h"
12 #include "flatbuffers/util.h"
13
14 namespace flatbuffers {
15 namespace {
16
17 struct OutputConfig {
18 size_t largest_type_string = 10;
19
20 size_t largest_value_string = 20;
21
22 size_t max_bytes_per_line = 8;
23
24 size_t offset_max_char = 4;
25
26 char delimiter = '|';
27
28 bool include_vector_contents = true;
29 };
30
ToString(const BinarySectionType type)31 static std::string ToString(const BinarySectionType type) {
32 switch (type) {
33 case BinarySectionType::Header: return "header";
34 case BinarySectionType::Table: return "table";
35 case BinarySectionType::RootTable: return "root_table";
36 case BinarySectionType::VTable: return "vtable";
37 case BinarySectionType::Struct: return "struct";
38 case BinarySectionType::String: return "string";
39 case BinarySectionType::Vector: return "vector";
40 case BinarySectionType::Vector64: return "vector64";
41 case BinarySectionType::Unknown: return "unknown";
42 case BinarySectionType::Union: return "union";
43 case BinarySectionType::Padding: return "padding";
44 default: return "todo";
45 }
46 }
47
IsOffset(const BinaryRegionType type)48 static bool IsOffset(const BinaryRegionType type) {
49 return type == BinaryRegionType::UOffset ||
50 type == BinaryRegionType::SOffset ||
51 type == BinaryRegionType::UOffset64;
52 }
53
ToString(T value)54 template<typename T> std::string ToString(T value) {
55 if (std::is_floating_point<T>::value) {
56 std::stringstream ss;
57 ss << value;
58 return ss.str();
59 } else {
60 return std::to_string(value);
61 }
62 }
63
64 template<typename T>
ToValueString(const BinaryRegion & region,const uint8_t * binary)65 std::string ToValueString(const BinaryRegion ®ion, const uint8_t *binary) {
66 std::string s;
67 s += "0x";
68 const T val = ReadScalar<T>(binary + region.offset);
69 const uint64_t start_index = region.offset + region.length - 1;
70 for (uint64_t i = 0; i < region.length; ++i) {
71 s += ToHex(binary[start_index - i]);
72 }
73 s += " (";
74 s += ToString(val);
75 s += ")";
76 return s;
77 }
78
79 template<>
ToValueString(const BinaryRegion & region,const uint8_t * binary)80 std::string ToValueString<std::string>(const BinaryRegion ®ion,
81 const uint8_t *binary) {
82 return std::string(reinterpret_cast<const char *>(binary + region.offset),
83 static_cast<size_t>(region.array_length));
84 }
85
ToValueString(const BinaryRegion & region,const uint8_t * binary,const OutputConfig & output_config)86 static std::string ToValueString(const BinaryRegion ®ion,
87 const uint8_t *binary,
88 const OutputConfig &output_config) {
89 std::string s;
90
91 if (region.array_length) {
92 if (region.type == BinaryRegionType::Uint8 ||
93 region.type == BinaryRegionType::Unknown) {
94 // Interpret each value as a ASCII to aid debugging
95 for (uint64_t i = 0; i < region.array_length; ++i) {
96 const uint8_t c = *(binary + region.offset + i);
97 s += isprint(c) ? static_cast<char>(c & 0x7F) : '.';
98 }
99 return s;
100 } else if (region.type == BinaryRegionType::Char) {
101 // string value
102 return ToValueString<std::string>(region, binary);
103 }
104 }
105
106 switch (region.type) {
107 case BinaryRegionType::Uint32:
108 return ToValueString<uint32_t>(region, binary);
109 case BinaryRegionType::Int32: return ToValueString<int32_t>(region, binary);
110 case BinaryRegionType::Uint16:
111 return ToValueString<uint16_t>(region, binary);
112 case BinaryRegionType::Int16: return ToValueString<int16_t>(region, binary);
113 case BinaryRegionType::Bool: return ToValueString<bool>(region, binary);
114 case BinaryRegionType::Uint8: return ToValueString<uint8_t>(region, binary);
115 case BinaryRegionType::Char: return ToValueString<char>(region, binary);
116 case BinaryRegionType::Byte:
117 case BinaryRegionType::Int8: return ToValueString<int8_t>(region, binary);
118 case BinaryRegionType::Int64: return ToValueString<int64_t>(region, binary);
119 case BinaryRegionType::Uint64:
120 return ToValueString<uint64_t>(region, binary);
121 case BinaryRegionType::Double: return ToValueString<double>(region, binary);
122 case BinaryRegionType::Float: return ToValueString<float>(region, binary);
123 case BinaryRegionType::UType: return ToValueString<uint8_t>(region, binary);
124
125 // Handle Offsets separately, incase they add additional details.
126 case BinaryRegionType::UOffset64:
127 s += ToValueString<uint64_t>(region, binary);
128 break;
129 case BinaryRegionType::UOffset:
130 s += ToValueString<uint32_t>(region, binary);
131 break;
132 case BinaryRegionType::SOffset:
133 s += ToValueString<int32_t>(region, binary);
134 break;
135 case BinaryRegionType::VOffset:
136 s += ToValueString<uint16_t>(region, binary);
137 break;
138
139 default: break;
140 }
141 // If this is an offset type, include the calculated offset location in the
142 // value.
143 // TODO(dbaileychess): It might be nicer to put this in the comment field.
144 if (IsOffset(region.type)) {
145 s += " Loc: 0x";
146 s += ToHex(region.points_to_offset, output_config.offset_max_char);
147 }
148 return s;
149 }
150
151 struct DocContinuation {
152 // The start column where the value text first starts
153 size_t value_start_column = 0;
154
155 // The remaining part of the doc to print.
156 std::string value;
157 };
158
GenerateTypeString(const BinaryRegion & region)159 static std::string GenerateTypeString(const BinaryRegion ®ion) {
160 return ToString(region.type) +
161 ((region.array_length)
162 ? "[" + std::to_string(region.array_length) + "]"
163 : "");
164 }
165
GenerateComment(const BinaryRegionComment & comment,const BinarySection &)166 static std::string GenerateComment(const BinaryRegionComment &comment,
167 const BinarySection &) {
168 std::string s;
169 switch (comment.type) {
170 case BinaryRegionCommentType::Unknown: s = "unknown"; break;
171 case BinaryRegionCommentType::SizePrefix: s = "size prefix"; break;
172 case BinaryRegionCommentType::RootTableOffset:
173 s = "offset to root table `" + comment.name + "`";
174 break;
175 // TODO(dbaileychess): make this lowercase to follow the convention.
176 case BinaryRegionCommentType::FileIdentifier: s = "File Identifier"; break;
177 case BinaryRegionCommentType::Padding: s = "padding"; break;
178 case BinaryRegionCommentType::VTableSize: s = "size of this vtable"; break;
179 case BinaryRegionCommentType::VTableRefferingTableLength:
180 s = "size of referring table";
181 break;
182 case BinaryRegionCommentType::VTableFieldOffset:
183 s = "offset to field `" + comment.name;
184 break;
185 case BinaryRegionCommentType::VTableUnknownFieldOffset:
186 s = "offset to unknown field (id: " + std::to_string(comment.index) + ")";
187 break;
188
189 case BinaryRegionCommentType::TableVTableOffset:
190 s = "offset to vtable";
191 break;
192 case BinaryRegionCommentType::TableField:
193 s = "table field `" + comment.name;
194 break;
195 case BinaryRegionCommentType::TableUnknownField: s = "unknown field"; break;
196 case BinaryRegionCommentType::TableOffsetField:
197 s = "offset to field `" + comment.name + "`";
198 break;
199 case BinaryRegionCommentType::StructField:
200 s = "struct field `" + comment.name + "`";
201 break;
202 case BinaryRegionCommentType::ArrayField:
203 s = "array field `" + comment.name + "`[" +
204 std::to_string(comment.index) + "]";
205 break;
206 case BinaryRegionCommentType::StringLength: s = "length of string"; break;
207 case BinaryRegionCommentType::StringValue: s = "string literal"; break;
208 case BinaryRegionCommentType::StringTerminator:
209 s = "string terminator";
210 break;
211 case BinaryRegionCommentType::VectorLength:
212 s = "length of vector (# items)";
213 break;
214 case BinaryRegionCommentType::VectorValue:
215 s = "value[" + std::to_string(comment.index) + "]";
216 break;
217 case BinaryRegionCommentType::VectorTableValue:
218 s = "offset to table[" + std::to_string(comment.index) + "]";
219 break;
220 case BinaryRegionCommentType::VectorStringValue:
221 s = "offset to string[" + std::to_string(comment.index) + "]";
222 break;
223 case BinaryRegionCommentType::VectorUnionValue:
224 s = "offset to union[" + std::to_string(comment.index) + "]";
225 break;
226
227 default: break;
228 }
229 if (!comment.default_value.empty()) { s += " " + comment.default_value; }
230
231 switch (comment.status) {
232 case BinaryRegionStatus::OK: break; // no-op
233 case BinaryRegionStatus::WARN: s = "WARN: " + s; break;
234 case BinaryRegionStatus::WARN_NO_REFERENCES:
235 s = "WARN: nothing refers to this section.";
236 break;
237 case BinaryRegionStatus::WARN_CORRUPTED_PADDING:
238 s = "WARN: could be corrupted padding region.";
239 break;
240 case BinaryRegionStatus::WARN_PADDING_LENGTH:
241 s = "WARN: padding is longer than expected.";
242 break;
243 case BinaryRegionStatus::ERROR: s = "ERROR: " + s; break;
244 case BinaryRegionStatus::ERROR_OFFSET_OUT_OF_BINARY:
245 s = "ERROR: " + s + ". Invalid offset, points outside the binary.";
246 break;
247 case BinaryRegionStatus::ERROR_INCOMPLETE_BINARY:
248 s = "ERROR: " + s + ". Incomplete binary, expected to read " +
249 comment.status_message + " bytes.";
250 break;
251 case BinaryRegionStatus::ERROR_LENGTH_TOO_LONG:
252 s = "ERROR: " + s + ". Longer than the binary.";
253 break;
254 case BinaryRegionStatus::ERROR_LENGTH_TOO_SHORT:
255 s = "ERROR: " + s + ". Shorter than the minimum length: ";
256 break;
257 case BinaryRegionStatus::ERROR_REQUIRED_FIELD_NOT_PRESENT:
258 s = "ERROR: " + s + ". Required field is not present.";
259 break;
260 case BinaryRegionStatus::ERROR_INVALID_UNION_TYPE:
261 s = "ERROR: " + s + ". Invalid union type value.";
262 break;
263 case BinaryRegionStatus::ERROR_CYCLE_DETECTED:
264 s = "ERROR: " + s + ". Invalid offset, cycle detected.";
265 break;
266 }
267
268 return s;
269 }
270
GenerateDocumentation(std::ostream & os,const BinaryRegion & region,const BinarySection & section,const uint8_t * binary,DocContinuation & continuation,const OutputConfig & output_config)271 static void GenerateDocumentation(std::ostream &os, const BinaryRegion ®ion,
272 const BinarySection §ion,
273 const uint8_t *binary,
274 DocContinuation &continuation,
275 const OutputConfig &output_config) {
276 // Check if there is a doc continuation that should be prioritized.
277 if (continuation.value_start_column) {
278 os << std::string(continuation.value_start_column - 2, ' ');
279 os << output_config.delimiter << " ";
280
281 os << continuation.value.substr(0, output_config.max_bytes_per_line);
282 continuation.value = continuation.value.substr(
283 std::min(output_config.max_bytes_per_line, continuation.value.size()));
284 return;
285 }
286
287 size_t size_of = 0;
288 {
289 std::stringstream ss;
290 ss << std::setw(static_cast<int>(output_config.largest_type_string))
291 << std::left;
292 ss << GenerateTypeString(region);
293 os << ss.str();
294 size_of = ss.str().size();
295 }
296 os << " " << output_config.delimiter << " ";
297 if (region.array_length) {
298 // Record where the value is first being outputted.
299 continuation.value_start_column = 3 + size_of;
300
301 // Get the full-length value, which we will chunk below.
302 const std::string value = ToValueString(region, binary, output_config);
303
304 std::stringstream ss;
305 ss << std::setw(static_cast<int>(output_config.largest_value_string))
306 << std::left;
307 ss << value.substr(0, output_config.max_bytes_per_line);
308 os << ss.str();
309
310 continuation.value =
311 value.substr(std::min(output_config.max_bytes_per_line, value.size()));
312 } else {
313 std::stringstream ss;
314 ss << std::setw(static_cast<int>(output_config.largest_value_string))
315 << std::left;
316 ss << ToValueString(region, binary, output_config);
317 os << ss.str();
318 }
319
320 os << " " << output_config.delimiter << " ";
321 os << GenerateComment(region.comment, section);
322 }
323
GenerateRegion(std::ostream & os,const BinaryRegion & region,const BinarySection & section,const uint8_t * binary,const OutputConfig & output_config)324 static void GenerateRegion(std::ostream &os, const BinaryRegion ®ion,
325 const BinarySection §ion, const uint8_t *binary,
326 const OutputConfig &output_config) {
327 bool doc_generated = false;
328 DocContinuation doc_continuation;
329 for (uint64_t i = 0; i < region.length; ++i) {
330 if ((i % output_config.max_bytes_per_line) == 0) {
331 // Start a new line of output
332 os << std::endl;
333 os << " +0x" << ToHex(region.offset + i, output_config.offset_max_char);
334 os << " " << output_config.delimiter;
335 }
336
337 // Add each byte
338 os << " " << ToHex(binary[region.offset + i]);
339
340 // Check for end of line or end of region conditions.
341 if (((i + 1) % output_config.max_bytes_per_line == 0) ||
342 i + 1 == region.length) {
343 if (i + 1 == region.length) {
344 // We are out of bytes but haven't the kMaxBytesPerLine, so we need to
345 // zero those out to align everything globally.
346 for (uint64_t j = i + 1; (j % output_config.max_bytes_per_line) != 0;
347 ++j) {
348 os << " ";
349 }
350 }
351 os << " " << output_config.delimiter;
352 // This is the end of the first line or its the last byte of the region,
353 // generate the end-of-line documentation.
354 if (!doc_generated) {
355 os << " ";
356 GenerateDocumentation(os, region, section, binary, doc_continuation,
357 output_config);
358
359 // If we have a value in the doc continuation, that means the doc is
360 // being printed on multiple lines.
361 doc_generated = doc_continuation.value.empty();
362 }
363 }
364 }
365 }
366
GenerateSection(std::ostream & os,const BinarySection & section,const uint8_t * binary,const OutputConfig & output_config)367 static void GenerateSection(std::ostream &os, const BinarySection §ion,
368 const uint8_t *binary,
369 const OutputConfig &output_config) {
370 os << std::endl;
371 os << ToString(section.type);
372 if (!section.name.empty()) { os << " (" + section.name + ")"; }
373 os << ":";
374
375 // As a space saving measure, skip generating every vector element, just put
376 // the first and last elements in the output. Skip the whole thing if there
377 // are only three or fewer elements, as it doesn't save space.
378 if ((section.type == BinarySectionType::Vector ||
379 section.type == BinarySectionType::Vector64) &&
380 !output_config.include_vector_contents && section.regions.size() > 4) {
381 // Generate the length region which should be first.
382 GenerateRegion(os, section.regions[0], section, binary, output_config);
383
384 // Generate the first element.
385 GenerateRegion(os, section.regions[1], section, binary, output_config);
386
387 // Indicate that we omitted elements.
388 os << std::endl
389 << " <" << section.regions.size() - 3 << " regions omitted>";
390
391 // Generate the last element.
392 GenerateRegion(os, section.regions.back(), section, binary, output_config);
393 os << std::endl;
394 return;
395 }
396
397 for (const BinaryRegion ®ion : section.regions) {
398 GenerateRegion(os, region, section, binary, output_config);
399 }
400 os << std::endl;
401 }
402 } // namespace
403
Generate(const std::string & filename,const std::string & schema_filename,const std::string & output_filename)404 bool AnnotatedBinaryTextGenerator::Generate(
405 const std::string &filename, const std::string &schema_filename,
406 const std::string &output_filename) {
407 OutputConfig output_config;
408 output_config.max_bytes_per_line = options_.max_bytes_per_line;
409 output_config.include_vector_contents = options_.include_vector_contents;
410
411 // Given the length of the binary, we can calculate the maximum number of
412 // characters to display in the offset hex: (i.e. 2 would lead to 0XFF being
413 // the max output).
414 output_config.offset_max_char =
415 binary_length_ > 0xFFFFFF
416 ? 8
417 : (binary_length_ > 0xFFFF ? 6 : (binary_length_ > 0xFF ? 4 : 2));
418
419 // Find the largest type string of all the regions in this file, so we can
420 // align the output nicely.
421 output_config.largest_type_string = 0;
422 for (const auto §ion : annotations_) {
423 for (const auto ®ion : section.second.regions) {
424 std::string s = GenerateTypeString(region);
425 if (s.size() > output_config.largest_type_string) {
426 output_config.largest_type_string = s.size();
427 }
428
429 // Don't consider array regions, as they will be split to multiple lines.
430 if (!region.array_length) {
431 s = ToValueString(region, binary_, output_config);
432 if (s.size() > output_config.largest_value_string) {
433 output_config.largest_value_string = s.size();
434 }
435 }
436 }
437 }
438
439 std::string out = output_filename;
440 if (out.empty()) {
441 // Modify the output filename.
442 out = StripExtension(filename);
443 out += options_.output_postfix;
444 out +=
445 "." + (options_.output_extension.empty() ? GetExtension(filename)
446 : options_.output_extension);
447 }
448
449 std::ofstream ofs(out.c_str());
450
451 ofs << "// Annotated Flatbuffer Binary" << std::endl;
452 ofs << "//" << std::endl;
453 if (!schema_filename.empty()) {
454 ofs << "// Schema file: " << schema_filename << std::endl;
455 }
456 ofs << "// Binary file: " << filename << std::endl;
457
458 // Generate each of the binary sections
459 for (const auto §ion : annotations_) {
460 GenerateSection(ofs, section.second, binary_, output_config);
461 }
462
463 ofs.close();
464 return true;
465 }
466
467 } // namespace flatbuffers
468