1 /*
2 * Copyright 2014 Google Inc. All rights reserved.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <algorithm>
18 #include <list>
19 #include <string>
20
21 #include <math.h>
22
23 #include "flatbuffers/idl.h"
24 #include "flatbuffers/util.h"
25
26 namespace flatbuffers {
27
28 const double kPi = 3.14159265358979323846;
29
30 const char *const kTypeNames[] = {
31 // clang-format off
32 #define FLATBUFFERS_TD(ENUM, IDLTYPE, \
33 CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \
34 IDLTYPE,
35 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
36 #undef FLATBUFFERS_TD
37 // clang-format on
38 nullptr
39 };
40
41 const char kTypeSizes[] = {
42 // clang-format off
43 #define FLATBUFFERS_TD(ENUM, IDLTYPE, \
44 CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \
45 sizeof(CTYPE),
46 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
47 #undef FLATBUFFERS_TD
48 // clang-format on
49 };
50
51 // The enums in the reflection schema should match the ones we use internally.
52 // Compare the last element to check if these go out of sync.
53 static_assert(BASE_TYPE_UNION == static_cast<BaseType>(reflection::Union),
54 "enums don't match");
55
56 // Any parsing calls have to be wrapped in this macro, which automates
57 // handling of recursive error checking a bit. It will check the received
58 // CheckedError object, and return straight away on error.
59 #define ECHECK(call) \
60 { \
61 auto ce = (call); \
62 if (ce.Check()) return ce; \
63 }
64
65 // These two functions are called hundreds of times below, so define a short
66 // form:
67 #define NEXT() ECHECK(Next())
68 #define EXPECT(tok) ECHECK(Expect(tok))
69
ValidateUTF8(const std::string & str)70 static bool ValidateUTF8(const std::string &str) {
71 const char *s = &str[0];
72 const char *const sEnd = s + str.length();
73 while (s < sEnd) {
74 if (FromUTF8(&s) < 0) { return false; }
75 }
76 return true;
77 }
78
79 // Convert an underscore_based_indentifier in to camelCase.
80 // Also uppercases the first character if first is true.
MakeCamel(const std::string & in,bool first)81 std::string MakeCamel(const std::string &in, bool first) {
82 std::string s;
83 for (size_t i = 0; i < in.length(); i++) {
84 if (!i && first)
85 s += static_cast<char>(toupper(in[0]));
86 else if (in[i] == '_' && i + 1 < in.length())
87 s += static_cast<char>(toupper(in[++i]));
88 else
89 s += in[i];
90 }
91 return s;
92 }
93
DeserializeDoc(std::vector<std::string> & doc,const Vector<Offset<String>> * documentation)94 void DeserializeDoc( std::vector<std::string> &doc,
95 const Vector<Offset<String>> *documentation) {
96 if (documentation == nullptr) return;
97 for (uoffset_t index = 0; index < documentation->size(); index++)
98 doc.push_back(documentation->Get(index)->str());
99 }
100
Message(const std::string & msg)101 void Parser::Message(const std::string &msg) {
102 error_ = file_being_parsed_.length() ? AbsolutePath(file_being_parsed_) : "";
103 // clang-format off
104 #ifdef _WIN32 // MSVC alike
105 error_ +=
106 "(" + NumToString(line_) + ", " + NumToString(CursorPosition()) + ")";
107 #else // gcc alike
108 if (file_being_parsed_.length()) error_ += ":";
109 error_ += NumToString(line_) + ": " + NumToString(CursorPosition());
110 #endif
111 // clang-format on
112 error_ += ": " + msg;
113 }
114
Warning(const std::string & msg)115 void Parser::Warning(const std::string &msg) { Message("warning: " + msg); }
116
Error(const std::string & msg)117 CheckedError Parser::Error(const std::string &msg) {
118 Message("error: " + msg);
119 return CheckedError(true);
120 }
121
NoError()122 inline CheckedError NoError() { return CheckedError(false); }
123
RecurseError()124 CheckedError Parser::RecurseError() {
125 return Error("maximum parsing recursion of " +
126 NumToString(FLATBUFFERS_MAX_PARSING_DEPTH) + " reached");
127 }
128
Recurse(F f)129 template<typename F> CheckedError Parser::Recurse(F f) {
130 if (recurse_protection_counter >= (FLATBUFFERS_MAX_PARSING_DEPTH))
131 return RecurseError();
132 recurse_protection_counter++;
133 auto ce = f();
134 recurse_protection_counter--;
135 return ce;
136 }
137
InvalidNumber(const char * number,const std::string & msg)138 CheckedError Parser::InvalidNumber(const char *number, const std::string &msg) {
139 return Error("invalid number: \"" + std::string(number) + "\"" + msg);
140 }
141 // atot: templated version of atoi/atof: convert a string to an instance of T.
142 template<typename T>
atot(const char * s,Parser & parser,T * val)143 inline CheckedError atot(const char *s, Parser &parser, T *val) {
144 auto done = StringToNumber(s, val);
145 if (done) return NoError();
146
147 return parser.InvalidNumber(
148 s, (0 == *val)
149 ? ""
150 : (", constant does not fit [" +
151 NumToString(flatbuffers::numeric_limits<T>::lowest()) + "; " +
152 NumToString(flatbuffers::numeric_limits<T>::max()) + "]"));
153 }
154 template<>
atot(const char * s,Parser & parser,Offset<void> * val)155 inline CheckedError atot<Offset<void>>(const char *s, Parser &parser,
156 Offset<void> *val) {
157 (void)parser;
158 *val = Offset<void>(atoi(s));
159 return NoError();
160 }
161
GetFullyQualifiedName(const std::string & name,size_t max_components) const162 std::string Namespace::GetFullyQualifiedName(const std::string &name,
163 size_t max_components) const {
164 // Early exit if we don't have a defined namespace.
165 if (components.empty() || !max_components) { return name; }
166 std::string stream_str;
167 for (size_t i = 0; i < std::min(components.size(), max_components); i++) {
168 if (i) { stream_str += '.'; }
169 stream_str += std::string(components[i]);
170 }
171 if (name.length()) {
172 stream_str += '.';
173 stream_str += name;
174 }
175 return stream_str;
176 }
177
178 // Declare tokens we'll use. Single character tokens are represented by their
179 // ascii character code (e.g. '{'), others above 256.
180 // clang-format off
181 #define FLATBUFFERS_GEN_TOKENS(TD) \
182 TD(Eof, 256, "end of file") \
183 TD(StringConstant, 257, "string constant") \
184 TD(IntegerConstant, 258, "integer constant") \
185 TD(FloatConstant, 259, "float constant") \
186 TD(Identifier, 260, "identifier")
187 #ifdef __GNUC__
188 __extension__ // Stop GCC complaining about trailing comma with -Wpendantic.
189 #endif
190 enum {
191 #define FLATBUFFERS_TOKEN(NAME, VALUE, STRING) kToken ## NAME = VALUE,
192 FLATBUFFERS_GEN_TOKENS(FLATBUFFERS_TOKEN)
193 #undef FLATBUFFERS_TOKEN
194 };
195
TokenToString(int t)196 static std::string TokenToString(int t) {
197 static const char * const tokens[] = {
198 #define FLATBUFFERS_TOKEN(NAME, VALUE, STRING) STRING,
199 FLATBUFFERS_GEN_TOKENS(FLATBUFFERS_TOKEN)
200 #undef FLATBUFFERS_TOKEN
201 #define FLATBUFFERS_TD(ENUM, IDLTYPE, \
202 CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \
203 IDLTYPE,
204 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
205 #undef FLATBUFFERS_TD
206 };
207 if (t < 256) { // A single ascii char token.
208 std::string s;
209 s.append(1, static_cast<char>(t));
210 return s;
211 } else { // Other tokens.
212 return tokens[t - 256];
213 }
214 }
215 // clang-format on
216
TokenToStringId(int t) const217 std::string Parser::TokenToStringId(int t) const {
218 return t == kTokenIdentifier ? attribute_ : TokenToString(t);
219 }
220
221 // Parses exactly nibbles worth of hex digits into a number, or error.
ParseHexNum(int nibbles,uint64_t * val)222 CheckedError Parser::ParseHexNum(int nibbles, uint64_t *val) {
223 FLATBUFFERS_ASSERT(nibbles > 0);
224 for (int i = 0; i < nibbles; i++)
225 if (!is_xdigit(cursor_[i]))
226 return Error("escape code must be followed by " + NumToString(nibbles) +
227 " hex digits");
228 std::string target(cursor_, cursor_ + nibbles);
229 *val = StringToUInt(target.c_str(), 16);
230 cursor_ += nibbles;
231 return NoError();
232 }
233
SkipByteOrderMark()234 CheckedError Parser::SkipByteOrderMark() {
235 if (static_cast<unsigned char>(*cursor_) != 0xef) return NoError();
236 cursor_++;
237 if (static_cast<unsigned char>(*cursor_) != 0xbb)
238 return Error("invalid utf-8 byte order mark");
239 cursor_++;
240 if (static_cast<unsigned char>(*cursor_) != 0xbf)
241 return Error("invalid utf-8 byte order mark");
242 cursor_++;
243 return NoError();
244 }
245
IsIdentifierStart(char c)246 static inline bool IsIdentifierStart(char c) {
247 return is_alpha(c) || (c == '_');
248 }
249
Next()250 CheckedError Parser::Next() {
251 doc_comment_.clear();
252 bool seen_newline = cursor_ == source_;
253 attribute_.clear();
254 attr_is_trivial_ascii_string_ = true;
255 for (;;) {
256 char c = *cursor_++;
257 token_ = c;
258 switch (c) {
259 case '\0':
260 cursor_--;
261 token_ = kTokenEof;
262 return NoError();
263 case ' ':
264 case '\r':
265 case '\t': break;
266 case '\n':
267 MarkNewLine();
268 seen_newline = true;
269 break;
270 case '{':
271 case '}':
272 case '(':
273 case ')':
274 case '[':
275 case ']':
276 case ',':
277 case ':':
278 case ';':
279 case '=': return NoError();
280 case '\"':
281 case '\'': {
282 int unicode_high_surrogate = -1;
283
284 while (*cursor_ != c) {
285 if (*cursor_ < ' ' && static_cast<signed char>(*cursor_) >= 0)
286 return Error("illegal character in string constant");
287 if (*cursor_ == '\\') {
288 attr_is_trivial_ascii_string_ = false; // has escape sequence
289 cursor_++;
290 if (unicode_high_surrogate != -1 && *cursor_ != 'u') {
291 return Error(
292 "illegal Unicode sequence (unpaired high surrogate)");
293 }
294 switch (*cursor_) {
295 case 'n':
296 attribute_ += '\n';
297 cursor_++;
298 break;
299 case 't':
300 attribute_ += '\t';
301 cursor_++;
302 break;
303 case 'r':
304 attribute_ += '\r';
305 cursor_++;
306 break;
307 case 'b':
308 attribute_ += '\b';
309 cursor_++;
310 break;
311 case 'f':
312 attribute_ += '\f';
313 cursor_++;
314 break;
315 case '\"':
316 attribute_ += '\"';
317 cursor_++;
318 break;
319 case '\'':
320 attribute_ += '\'';
321 cursor_++;
322 break;
323 case '\\':
324 attribute_ += '\\';
325 cursor_++;
326 break;
327 case '/':
328 attribute_ += '/';
329 cursor_++;
330 break;
331 case 'x': { // Not in the JSON standard
332 cursor_++;
333 uint64_t val;
334 ECHECK(ParseHexNum(2, &val));
335 attribute_ += static_cast<char>(val);
336 break;
337 }
338 case 'u': {
339 cursor_++;
340 uint64_t val;
341 ECHECK(ParseHexNum(4, &val));
342 if (val >= 0xD800 && val <= 0xDBFF) {
343 if (unicode_high_surrogate != -1) {
344 return Error(
345 "illegal Unicode sequence (multiple high surrogates)");
346 } else {
347 unicode_high_surrogate = static_cast<int>(val);
348 }
349 } else if (val >= 0xDC00 && val <= 0xDFFF) {
350 if (unicode_high_surrogate == -1) {
351 return Error(
352 "illegal Unicode sequence (unpaired low surrogate)");
353 } else {
354 int code_point = 0x10000 +
355 ((unicode_high_surrogate & 0x03FF) << 10) +
356 (val & 0x03FF);
357 ToUTF8(code_point, &attribute_);
358 unicode_high_surrogate = -1;
359 }
360 } else {
361 if (unicode_high_surrogate != -1) {
362 return Error(
363 "illegal Unicode sequence (unpaired high surrogate)");
364 }
365 ToUTF8(static_cast<int>(val), &attribute_);
366 }
367 break;
368 }
369 default: return Error("unknown escape code in string constant");
370 }
371 } else { // printable chars + UTF-8 bytes
372 if (unicode_high_surrogate != -1) {
373 return Error(
374 "illegal Unicode sequence (unpaired high surrogate)");
375 }
376 // reset if non-printable
377 attr_is_trivial_ascii_string_ &= check_in_range(*cursor_, ' ', '~');
378
379 attribute_ += *cursor_++;
380 }
381 }
382 if (unicode_high_surrogate != -1) {
383 return Error("illegal Unicode sequence (unpaired high surrogate)");
384 }
385 cursor_++;
386 if (!attr_is_trivial_ascii_string_ && !opts.allow_non_utf8 &&
387 !ValidateUTF8(attribute_)) {
388 return Error("illegal UTF-8 sequence");
389 }
390 token_ = kTokenStringConstant;
391 return NoError();
392 }
393 case '/':
394 if (*cursor_ == '/') {
395 const char *start = ++cursor_;
396 while (*cursor_ && *cursor_ != '\n' && *cursor_ != '\r') cursor_++;
397 if (*start == '/') { // documentation comment
398 if (!seen_newline)
399 return Error(
400 "a documentation comment should be on a line on its own");
401 doc_comment_.push_back(std::string(start + 1, cursor_));
402 }
403 break;
404 } else if (*cursor_ == '*') {
405 cursor_++;
406 // TODO: make nested.
407 while (*cursor_ != '*' || cursor_[1] != '/') {
408 if (*cursor_ == '\n') MarkNewLine();
409 if (!*cursor_) return Error("end of file in comment");
410 cursor_++;
411 }
412 cursor_ += 2;
413 break;
414 }
415 FLATBUFFERS_FALLTHROUGH(); // else fall thru
416 default:
417 const auto has_sign = (c == '+') || (c == '-');
418 // '-'/'+' and following identifier - can be a predefined constant like:
419 // NAN, INF, PI, etc.
420 if (IsIdentifierStart(c) || (has_sign && IsIdentifierStart(*cursor_))) {
421 // Collect all chars of an identifier:
422 const char *start = cursor_ - 1;
423 while (IsIdentifierStart(*cursor_) || is_digit(*cursor_)) cursor_++;
424 attribute_.append(start, cursor_);
425 token_ = has_sign ? kTokenStringConstant : kTokenIdentifier;
426 return NoError();
427 }
428
429 auto dot_lvl = (c == '.') ? 0 : 1; // dot_lvl==0 <=> exactly one '.' seen
430 if (!dot_lvl && !is_digit(*cursor_)) return NoError(); // enum?
431 // Parser accepts hexadecimal-floating-literal (see C++ 5.13.4).
432 if (is_digit(c) || has_sign || !dot_lvl) {
433 const auto start = cursor_ - 1;
434 auto start_digits = !is_digit(c) ? cursor_ : cursor_ - 1;
435 if (!is_digit(c) && is_digit(*cursor_)){
436 start_digits = cursor_; // see digit in cursor_ position
437 c = *cursor_++;
438 }
439 // hex-float can't begind with '.'
440 auto use_hex = dot_lvl && (c == '0') && is_alpha_char(*cursor_, 'X');
441 if (use_hex) start_digits = ++cursor_; // '0x' is the prefix, skip it
442 // Read an integer number or mantisa of float-point number.
443 do {
444 if (use_hex) {
445 while (is_xdigit(*cursor_)) cursor_++;
446 } else {
447 while (is_digit(*cursor_)) cursor_++;
448 }
449 } while ((*cursor_ == '.') && (++cursor_) && (--dot_lvl >= 0));
450 // Exponent of float-point number.
451 if ((dot_lvl >= 0) && (cursor_ > start_digits)) {
452 // The exponent suffix of hexadecimal float number is mandatory.
453 if (use_hex && !dot_lvl) start_digits = cursor_;
454 if ((use_hex && is_alpha_char(*cursor_, 'P')) ||
455 is_alpha_char(*cursor_, 'E')) {
456 dot_lvl = 0; // Emulate dot to signal about float-point number.
457 cursor_++;
458 if (*cursor_ == '+' || *cursor_ == '-') cursor_++;
459 start_digits = cursor_; // the exponent-part has to have digits
460 // Exponent is decimal integer number
461 while (is_digit(*cursor_)) cursor_++;
462 if (*cursor_ == '.') {
463 cursor_++; // If see a dot treat it as part of invalid number.
464 dot_lvl = -1; // Fall thru to Error().
465 }
466 }
467 }
468 // Finalize.
469 if ((dot_lvl >= 0) && (cursor_ > start_digits)) {
470 attribute_.append(start, cursor_);
471 token_ = dot_lvl ? kTokenIntegerConstant : kTokenFloatConstant;
472 return NoError();
473 } else {
474 return Error("invalid number: " + std::string(start, cursor_));
475 }
476 }
477 std::string ch;
478 ch = c;
479 if (false == check_in_range(c, ' ', '~')) ch = "code: " + NumToString(c);
480 return Error("illegal character: " + ch);
481 }
482 }
483 }
484
485 // Check if a given token is next.
Is(int t) const486 bool Parser::Is(int t) const { return t == token_; }
487
IsIdent(const char * id) const488 bool Parser::IsIdent(const char *id) const {
489 return token_ == kTokenIdentifier && attribute_ == id;
490 }
491
492 // Expect a given token to be next, consume it, or error if not present.
Expect(int t)493 CheckedError Parser::Expect(int t) {
494 if (t != token_) {
495 return Error("expecting: " + TokenToString(t) +
496 " instead got: " + TokenToStringId(token_));
497 }
498 NEXT();
499 return NoError();
500 }
501
ParseNamespacing(std::string * id,std::string * last)502 CheckedError Parser::ParseNamespacing(std::string *id, std::string *last) {
503 while (Is('.')) {
504 NEXT();
505 *id += ".";
506 *id += attribute_;
507 if (last) *last = attribute_;
508 EXPECT(kTokenIdentifier);
509 }
510 return NoError();
511 }
512
LookupEnum(const std::string & id)513 EnumDef *Parser::LookupEnum(const std::string &id) {
514 // Search thru parent namespaces.
515 for (int components = static_cast<int>(current_namespace_->components.size());
516 components >= 0; components--) {
517 auto ed = enums_.Lookup(
518 current_namespace_->GetFullyQualifiedName(id, components));
519 if (ed) return ed;
520 }
521 return nullptr;
522 }
523
LookupStruct(const std::string & id) const524 StructDef *Parser::LookupStruct(const std::string &id) const {
525 auto sd = structs_.Lookup(id);
526 if (sd) sd->refcount++;
527 return sd;
528 }
529
ParseTypeIdent(Type & type)530 CheckedError Parser::ParseTypeIdent(Type &type) {
531 std::string id = attribute_;
532 EXPECT(kTokenIdentifier);
533 ECHECK(ParseNamespacing(&id, nullptr));
534 auto enum_def = LookupEnum(id);
535 if (enum_def) {
536 type = enum_def->underlying_type;
537 if (enum_def->is_union) type.base_type = BASE_TYPE_UNION;
538 } else {
539 type.base_type = BASE_TYPE_STRUCT;
540 type.struct_def = LookupCreateStruct(id);
541 }
542 return NoError();
543 }
544
545 // Parse any IDL type.
ParseType(Type & type)546 CheckedError Parser::ParseType(Type &type) {
547 if (token_ == kTokenIdentifier) {
548 if (IsIdent("bool")) {
549 type.base_type = BASE_TYPE_BOOL;
550 NEXT();
551 } else if (IsIdent("byte") || IsIdent("int8")) {
552 type.base_type = BASE_TYPE_CHAR;
553 NEXT();
554 } else if (IsIdent("ubyte") || IsIdent("uint8")) {
555 type.base_type = BASE_TYPE_UCHAR;
556 NEXT();
557 } else if (IsIdent("short") || IsIdent("int16")) {
558 type.base_type = BASE_TYPE_SHORT;
559 NEXT();
560 } else if (IsIdent("ushort") || IsIdent("uint16")) {
561 type.base_type = BASE_TYPE_USHORT;
562 NEXT();
563 } else if (IsIdent("int") || IsIdent("int32")) {
564 type.base_type = BASE_TYPE_INT;
565 NEXT();
566 } else if (IsIdent("uint") || IsIdent("uint32")) {
567 type.base_type = BASE_TYPE_UINT;
568 NEXT();
569 } else if (IsIdent("long") || IsIdent("int64")) {
570 type.base_type = BASE_TYPE_LONG;
571 NEXT();
572 } else if (IsIdent("ulong") || IsIdent("uint64")) {
573 type.base_type = BASE_TYPE_ULONG;
574 NEXT();
575 } else if (IsIdent("float") || IsIdent("float32")) {
576 type.base_type = BASE_TYPE_FLOAT;
577 NEXT();
578 } else if (IsIdent("double") || IsIdent("float64")) {
579 type.base_type = BASE_TYPE_DOUBLE;
580 NEXT();
581 } else if (IsIdent("string")) {
582 type.base_type = BASE_TYPE_STRING;
583 NEXT();
584 } else {
585 ECHECK(ParseTypeIdent(type));
586 }
587 } else if (token_ == '[') {
588 NEXT();
589 Type subtype;
590 ECHECK(Recurse([&]() { return ParseType(subtype); }));
591 if (subtype.base_type == BASE_TYPE_VECTOR) {
592 // We could support this, but it will complicate things, and it's
593 // easier to work around with a struct around the inner vector.
594 return Error("nested vector types not supported (wrap in table first).");
595 }
596 type = Type(BASE_TYPE_VECTOR, subtype.struct_def, subtype.enum_def);
597 type.element = subtype.base_type;
598 EXPECT(']');
599 } else {
600 return Error("illegal type syntax");
601 }
602 return NoError();
603 }
604
AddField(StructDef & struct_def,const std::string & name,const Type & type,FieldDef ** dest)605 CheckedError Parser::AddField(StructDef &struct_def, const std::string &name,
606 const Type &type, FieldDef **dest) {
607 auto &field = *new FieldDef();
608 field.value.offset =
609 FieldIndexToOffset(static_cast<voffset_t>(struct_def.fields.vec.size()));
610 field.name = name;
611 field.file = struct_def.file;
612 field.value.type = type;
613 if (struct_def.fixed) { // statically compute the field offset
614 auto size = InlineSize(type);
615 auto alignment = InlineAlignment(type);
616 // structs_ need to have a predictable format, so we need to align to
617 // the largest scalar
618 struct_def.minalign = std::max(struct_def.minalign, alignment);
619 struct_def.PadLastField(alignment);
620 field.value.offset = static_cast<voffset_t>(struct_def.bytesize);
621 struct_def.bytesize += size;
622 }
623 if (struct_def.fields.Add(name, &field))
624 return Error("field already exists: " + name);
625 *dest = &field;
626 return NoError();
627 }
628
ParseField(StructDef & struct_def)629 CheckedError Parser::ParseField(StructDef &struct_def) {
630 std::string name = attribute_;
631
632 if (LookupStruct(name))
633 return Error("field name can not be the same as table/struct name");
634
635 std::vector<std::string> dc = doc_comment_;
636 EXPECT(kTokenIdentifier);
637 EXPECT(':');
638 Type type;
639 ECHECK(ParseType(type));
640
641 if (struct_def.fixed && !IsScalar(type.base_type) && !IsStruct(type))
642 return Error("structs_ may contain only scalar or struct fields");
643
644 FieldDef *typefield = nullptr;
645 if (type.base_type == BASE_TYPE_UNION) {
646 // For union fields, add a second auto-generated field to hold the type,
647 // with a special suffix.
648 ECHECK(AddField(struct_def, name + UnionTypeFieldSuffix(),
649 type.enum_def->underlying_type, &typefield));
650 } else if (type.base_type == BASE_TYPE_VECTOR &&
651 type.element == BASE_TYPE_UNION) {
652 // Only cpp, js and ts supports the union vector feature so far.
653 if (!SupportsVectorOfUnions()) {
654 return Error(
655 "Vectors of unions are not yet supported in all "
656 "the specified programming languages.");
657 }
658 // For vector of union fields, add a second auto-generated vector field to
659 // hold the types, with a special suffix.
660 Type union_vector(BASE_TYPE_VECTOR, nullptr, type.enum_def);
661 union_vector.element = BASE_TYPE_UTYPE;
662 ECHECK(AddField(struct_def, name + UnionTypeFieldSuffix(), union_vector,
663 &typefield));
664 }
665
666 FieldDef *field;
667 ECHECK(AddField(struct_def, name, type, &field));
668
669 if (token_ == '=') {
670 NEXT();
671 if (!IsScalar(type.base_type) ||
672 (struct_def.fixed && field->value.constant != "0"))
673 return Error(
674 "default values currently only supported for scalars in tables");
675 ECHECK(ParseSingleValue(&field->name, field->value, true));
676 }
677 if (type.enum_def &&
678 !type.enum_def->is_union &&
679 !type.enum_def->attributes.Lookup("bit_flags") &&
680 !type.enum_def->ReverseLookup(StringToInt(
681 field->value.constant.c_str()))) {
682 return Error("default value of " + field->value.constant + " for field " +
683 name + " is not part of enum " + type.enum_def->name);
684 }
685 // Append .0 if the value has not it (skip hex and scientific floats).
686 // This suffix needed for generated C++ code.
687 if (IsFloat(type.base_type)) {
688 auto &text = field->value.constant;
689 FLATBUFFERS_ASSERT(false == text.empty());
690 auto s = text.c_str();
691 while(*s == ' ') s++;
692 if (*s == '-' || *s == '+') s++;
693 // 1) A float constants (nan, inf, pi, etc) is a kind of identifier.
694 // 2) A float number needn't ".0" at the end if it has exponent.
695 if ((false == IsIdentifierStart(*s)) &&
696 (std::string::npos == field->value.constant.find_first_of(".eEpP"))) {
697 field->value.constant += ".0";
698 }
699 }
700
701 if (type.enum_def && IsScalar(type.base_type) && !struct_def.fixed &&
702 !type.enum_def->attributes.Lookup("bit_flags") &&
703 !type.enum_def->ReverseLookup(StringToInt(
704 field->value.constant.c_str())))
705 Warning("enum " + type.enum_def->name +
706 " does not have a declaration for this field\'s default of " +
707 field->value.constant);
708
709 field->doc_comment = dc;
710 ECHECK(ParseMetaData(&field->attributes));
711 field->deprecated = field->attributes.Lookup("deprecated") != nullptr;
712 auto hash_name = field->attributes.Lookup("hash");
713 if (hash_name) {
714 switch ((type.base_type == BASE_TYPE_VECTOR) ? type.element : type.base_type) {
715 case BASE_TYPE_SHORT:
716 case BASE_TYPE_USHORT: {
717 if (FindHashFunction16(hash_name->constant.c_str()) == nullptr)
718 return Error("Unknown hashing algorithm for 16 bit types: " +
719 hash_name->constant);
720 break;
721 }
722 case BASE_TYPE_INT:
723 case BASE_TYPE_UINT: {
724 if (FindHashFunction32(hash_name->constant.c_str()) == nullptr)
725 return Error("Unknown hashing algorithm for 32 bit types: " +
726 hash_name->constant);
727 break;
728 }
729 case BASE_TYPE_LONG:
730 case BASE_TYPE_ULONG: {
731 if (FindHashFunction64(hash_name->constant.c_str()) == nullptr)
732 return Error("Unknown hashing algorithm for 64 bit types: " +
733 hash_name->constant);
734 break;
735 }
736 default:
737 return Error(
738 "only short, ushort, int, uint, long and ulong data types support hashing.");
739 }
740 }
741 auto cpp_type = field->attributes.Lookup("cpp_type");
742 if (cpp_type) {
743 if (!hash_name)
744 return Error("cpp_type can only be used with a hashed field");
745 /// forcing cpp_ptr_type to 'naked' if unset
746 auto cpp_ptr_type = field->attributes.Lookup("cpp_ptr_type");
747 if (!cpp_ptr_type) {
748 auto val = new Value();
749 val->type = cpp_type->type;
750 val->constant = "naked";
751 field->attributes.Add("cpp_ptr_type", val);
752 }
753 }
754 if (field->deprecated && struct_def.fixed)
755 return Error("can't deprecate fields in a struct");
756 field->required = field->attributes.Lookup("required") != nullptr;
757 if (field->required &&
758 (struct_def.fixed || IsScalar(type.base_type)))
759 return Error("only non-scalar fields in tables may be 'required'");
760 field->key = field->attributes.Lookup("key") != nullptr;
761 if (field->key) {
762 if (struct_def.has_key) return Error("only one field may be set as 'key'");
763 struct_def.has_key = true;
764 if (!IsScalar(type.base_type)) {
765 field->required = true;
766 if (type.base_type != BASE_TYPE_STRING)
767 return Error("'key' field must be string or scalar type");
768 }
769 }
770 field->shared = field->attributes.Lookup("shared") != nullptr;
771 if (field->shared && field->value.type.base_type != BASE_TYPE_STRING)
772 return Error("shared can only be defined on strings");
773
774 auto field_native_custom_alloc =
775 field->attributes.Lookup("native_custom_alloc");
776 if (field_native_custom_alloc)
777 return Error(
778 "native_custom_alloc can only be used with a table or struct "
779 "definition");
780
781 field->native_inline = field->attributes.Lookup("native_inline") != nullptr;
782 if (field->native_inline && !IsStruct(field->value.type))
783 return Error("native_inline can only be defined on structs");
784
785 auto nested = field->attributes.Lookup("nested_flatbuffer");
786 if (nested) {
787 if (nested->type.base_type != BASE_TYPE_STRING)
788 return Error(
789 "nested_flatbuffer attribute must be a string (the root type)");
790 if (type.base_type != BASE_TYPE_VECTOR || type.element != BASE_TYPE_UCHAR)
791 return Error(
792 "nested_flatbuffer attribute may only apply to a vector of ubyte");
793 // This will cause an error if the root type of the nested flatbuffer
794 // wasn't defined elsewhere.
795 LookupCreateStruct(nested->constant);
796
797 // Keep a pointer to StructDef in FieldDef to simplify re-use later
798 auto nested_qualified_name =
799 current_namespace_->GetFullyQualifiedName(nested->constant);
800 field->nested_flatbuffer = LookupStruct(nested_qualified_name);
801 }
802
803 if (field->attributes.Lookup("flexbuffer")) {
804 field->flexbuffer = true;
805 uses_flexbuffers_ = true;
806 if (type.base_type != BASE_TYPE_VECTOR ||
807 type.element != BASE_TYPE_UCHAR)
808 return Error("flexbuffer attribute may only apply to a vector of ubyte");
809 }
810
811 if (typefield) {
812 if (!IsScalar(typefield->value.type.base_type)) {
813 // this is a union vector field
814 typefield->required = field->required;
815 }
816 // If this field is a union, and it has a manually assigned id,
817 // the automatically added type field should have an id as well (of N - 1).
818 auto attr = field->attributes.Lookup("id");
819 if (attr) {
820 auto id = atoi(attr->constant.c_str());
821 auto val = new Value();
822 val->type = attr->type;
823 val->constant = NumToString(id - 1);
824 typefield->attributes.Add("id", val);
825 }
826 }
827
828 EXPECT(';');
829 return NoError();
830 }
831
ParseString(Value & val)832 CheckedError Parser::ParseString(Value &val) {
833 auto s = attribute_;
834 EXPECT(kTokenStringConstant);
835 val.constant = NumToString(builder_.CreateString(s).o);
836 return NoError();
837 }
838
ParseComma()839 CheckedError Parser::ParseComma() {
840 if (!opts.protobuf_ascii_alike) EXPECT(',');
841 return NoError();
842 }
843
ParseAnyValue(Value & val,FieldDef * field,size_t parent_fieldn,const StructDef * parent_struct_def)844 CheckedError Parser::ParseAnyValue(Value &val, FieldDef *field,
845 size_t parent_fieldn,
846 const StructDef *parent_struct_def) {
847 switch (val.type.base_type) {
848 case BASE_TYPE_UNION: {
849 FLATBUFFERS_ASSERT(field);
850 std::string constant;
851 // Find corresponding type field we may have already parsed.
852 for (auto elem = field_stack_.rbegin();
853 elem != field_stack_.rbegin() + parent_fieldn; ++elem) {
854 auto &type = elem->second->value.type;
855 if (type.base_type == BASE_TYPE_UTYPE &&
856 type.enum_def == val.type.enum_def) {
857 constant = elem->first.constant;
858 break;
859 }
860 }
861 if (constant.empty()) {
862 // We haven't seen the type field yet. Sadly a lot of JSON writers
863 // output these in alphabetical order, meaning it comes after this
864 // value. So we scan past the value to find it, then come back here.
865 auto type_name = field->name + UnionTypeFieldSuffix();
866 FLATBUFFERS_ASSERT(parent_struct_def);
867 auto type_field = parent_struct_def->fields.Lookup(type_name);
868 FLATBUFFERS_ASSERT(type_field); // Guaranteed by ParseField().
869 // Remember where we are in the source file, so we can come back here.
870 auto backup = *static_cast<ParserState *>(this);
871 ECHECK(SkipAnyJsonValue()); // The table.
872 ECHECK(ParseComma());
873 auto next_name = attribute_;
874 if (Is(kTokenStringConstant)) {
875 NEXT();
876 } else {
877 EXPECT(kTokenIdentifier);
878 }
879 if (next_name != type_name)
880 return Error("missing type field after this union value: " +
881 type_name);
882 EXPECT(':');
883 Value type_val = type_field->value;
884 ECHECK(ParseAnyValue(type_val, type_field, 0, nullptr));
885 constant = type_val.constant;
886 // Got the information we needed, now rewind:
887 *static_cast<ParserState *>(this) = backup;
888 }
889 uint8_t enum_idx;
890 ECHECK(atot(constant.c_str(), *this, &enum_idx));
891 auto enum_val = val.type.enum_def->ReverseLookup(enum_idx);
892 if (!enum_val) return Error("illegal type id for: " + field->name);
893 if (enum_val->union_type.base_type == BASE_TYPE_STRUCT) {
894 ECHECK(ParseTable(*enum_val->union_type.struct_def, &val.constant,
895 nullptr));
896 if (enum_val->union_type.struct_def->fixed) {
897 // All BASE_TYPE_UNION values are offsets, so turn this into one.
898 SerializeStruct(*enum_val->union_type.struct_def, val);
899 builder_.ClearOffsets();
900 val.constant = NumToString(builder_.GetSize());
901 }
902 } else if (enum_val->union_type.base_type == BASE_TYPE_STRING) {
903 ECHECK(ParseString(val));
904 } else {
905 FLATBUFFERS_ASSERT(false);
906 }
907 break;
908 }
909 case BASE_TYPE_STRUCT:
910 ECHECK(ParseTable(*val.type.struct_def, &val.constant, nullptr));
911 break;
912 case BASE_TYPE_STRING: {
913 ECHECK(ParseString(val));
914 break;
915 }
916 case BASE_TYPE_VECTOR: {
917 uoffset_t off;
918 ECHECK(ParseVector(val.type.VectorType(), &off));
919 val.constant = NumToString(off);
920 break;
921 }
922 case BASE_TYPE_INT:
923 case BASE_TYPE_UINT:
924 case BASE_TYPE_LONG:
925 case BASE_TYPE_ULONG: {
926 if (field && field->attributes.Lookup("hash") &&
927 (token_ == kTokenIdentifier || token_ == kTokenStringConstant)) {
928 ECHECK(ParseHash(val, field));
929 } else {
930 ECHECK(ParseSingleValue(field ? &field->name : nullptr, val, false));
931 }
932 break;
933 }
934 default:
935 ECHECK(ParseSingleValue(field ? &field->name : nullptr, val, false));
936 break;
937 }
938 return NoError();
939 }
940
SerializeStruct(const StructDef & struct_def,const Value & val)941 void Parser::SerializeStruct(const StructDef &struct_def, const Value &val) {
942 FLATBUFFERS_ASSERT(val.constant.length() == struct_def.bytesize);
943 builder_.Align(struct_def.minalign);
944 builder_.PushBytes(reinterpret_cast<const uint8_t *>(val.constant.c_str()),
945 struct_def.bytesize);
946 builder_.AddStructOffset(val.offset, builder_.GetSize());
947 }
948
949 template <typename F>
ParseTableDelimiters(size_t & fieldn,const StructDef * struct_def,F body)950 CheckedError Parser::ParseTableDelimiters(size_t &fieldn,
951 const StructDef *struct_def,
952 F body) {
953 // We allow tables both as JSON object{ .. } with field names
954 // or vector[..] with all fields in order
955 char terminator = '}';
956 bool is_nested_vector = struct_def && Is('[');
957 if (is_nested_vector) {
958 NEXT();
959 terminator = ']';
960 } else {
961 EXPECT('{');
962 }
963 for (;;) {
964 if ((!opts.strict_json || !fieldn) && Is(terminator)) break;
965 std::string name;
966 if (is_nested_vector) {
967 if (fieldn >= struct_def->fields.vec.size()) {
968 return Error("too many unnamed fields in nested array");
969 }
970 name = struct_def->fields.vec[fieldn]->name;
971 } else {
972 name = attribute_;
973 if (Is(kTokenStringConstant)) {
974 NEXT();
975 } else {
976 EXPECT(opts.strict_json ? kTokenStringConstant : kTokenIdentifier);
977 }
978 if (!opts.protobuf_ascii_alike || !(Is('{') || Is('['))) EXPECT(':');
979 }
980 ECHECK(body(name, fieldn, struct_def));
981 if (Is(terminator)) break;
982 ECHECK(ParseComma());
983 }
984 NEXT();
985 if (is_nested_vector && fieldn != struct_def->fields.vec.size()) {
986 return Error("wrong number of unnamed fields in table vector");
987 }
988 return NoError();
989 }
990
ParseTable(const StructDef & struct_def,std::string * value,uoffset_t * ovalue)991 CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
992 uoffset_t *ovalue) {
993 size_t fieldn_outer = 0;
994 auto err = ParseTableDelimiters(
995 fieldn_outer, &struct_def,
996 [&](const std::string &name, size_t &fieldn,
997 const StructDef *struct_def_inner) -> CheckedError {
998 if (name == "$schema") {
999 ECHECK(Expect(kTokenStringConstant));
1000 return NoError();
1001 }
1002 auto field = struct_def_inner->fields.Lookup(name);
1003 if (!field) {
1004 if (!opts.skip_unexpected_fields_in_json) {
1005 return Error("unknown field: " + name);
1006 } else {
1007 ECHECK(SkipAnyJsonValue());
1008 }
1009 } else {
1010 if (IsIdent("null") && !IsScalar(field->value.type.base_type)) {
1011 ECHECK(Next()); // Ignore this field.
1012 } else {
1013 Value val = field->value;
1014 if (field->flexbuffer) {
1015 flexbuffers::Builder builder(1024,
1016 flexbuffers::BUILDER_FLAG_SHARE_ALL);
1017 ECHECK(ParseFlexBufferValue(&builder));
1018 builder.Finish();
1019 // Force alignment for nested flexbuffer
1020 builder_.ForceVectorAlignment(builder.GetSize(), sizeof(uint8_t),
1021 sizeof(largest_scalar_t));
1022 auto off = builder_.CreateVector(builder.GetBuffer());
1023 val.constant = NumToString(off.o);
1024 } else if (field->nested_flatbuffer) {
1025 ECHECK(
1026 ParseNestedFlatbuffer(val, field, fieldn, struct_def_inner));
1027 } else {
1028 ECHECK(Recurse([&]() {
1029 return ParseAnyValue(val, field, fieldn, struct_def_inner);
1030 }));
1031 }
1032 // Hardcoded insertion-sort with error-check.
1033 // If fields are specified in order, then this loop exits
1034 // immediately.
1035 auto elem = field_stack_.rbegin();
1036 for (; elem != field_stack_.rbegin() + fieldn; ++elem) {
1037 auto existing_field = elem->second;
1038 if (existing_field == field)
1039 return Error("field set more than once: " + field->name);
1040 if (existing_field->value.offset < field->value.offset) break;
1041 }
1042 // Note: elem points to before the insertion point, thus .base()
1043 // points to the correct spot.
1044 field_stack_.insert(elem.base(), std::make_pair(val, field));
1045 fieldn++;
1046 }
1047 }
1048 return NoError();
1049 });
1050 ECHECK(err);
1051
1052 // Check if all required fields are parsed.
1053 for (auto field_it = struct_def.fields.vec.begin();
1054 field_it != struct_def.fields.vec.end(); ++field_it) {
1055 auto required_field = *field_it;
1056 if (!required_field->required) { continue; }
1057 bool found = false;
1058 for (auto pf_it = field_stack_.end() - fieldn_outer;
1059 pf_it != field_stack_.end(); ++pf_it) {
1060 auto parsed_field = pf_it->second;
1061 if (parsed_field == required_field) {
1062 found = true;
1063 break;
1064 }
1065 }
1066 if (!found) {
1067 return Error("required field is missing: " + required_field->name +
1068 " in " + struct_def.name);
1069 }
1070 }
1071
1072 if (struct_def.fixed && fieldn_outer != struct_def.fields.vec.size())
1073 return Error("struct: wrong number of initializers: " + struct_def.name);
1074
1075 auto start = struct_def.fixed ? builder_.StartStruct(struct_def.minalign)
1076 : builder_.StartTable();
1077
1078 for (size_t size = struct_def.sortbysize ? sizeof(largest_scalar_t) : 1; size;
1079 size /= 2) {
1080 // Go through elements in reverse, since we're building the data backwards.
1081 for (auto it = field_stack_.rbegin();
1082 it != field_stack_.rbegin() + fieldn_outer; ++it) {
1083 auto &field_value = it->first;
1084 auto field = it->second;
1085 if (!struct_def.sortbysize ||
1086 size == SizeOf(field_value.type.base_type)) {
1087 switch (field_value.type.base_type) {
1088 // clang-format off
1089 #define FLATBUFFERS_TD(ENUM, IDLTYPE, \
1090 CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \
1091 case BASE_TYPE_ ## ENUM: \
1092 builder_.Pad(field->padding); \
1093 if (struct_def.fixed) { \
1094 CTYPE val; \
1095 ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
1096 builder_.PushElement(val); \
1097 } else { \
1098 CTYPE val, valdef; \
1099 ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
1100 ECHECK(atot(field->value.constant.c_str(), *this, &valdef)); \
1101 builder_.AddElement(field_value.offset, val, valdef); \
1102 } \
1103 break;
1104 FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD);
1105 #undef FLATBUFFERS_TD
1106 #define FLATBUFFERS_TD(ENUM, IDLTYPE, \
1107 CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \
1108 case BASE_TYPE_ ## ENUM: \
1109 builder_.Pad(field->padding); \
1110 if (IsStruct(field->value.type)) { \
1111 SerializeStruct(*field->value.type.struct_def, field_value); \
1112 } else { \
1113 CTYPE val; \
1114 ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
1115 builder_.AddOffset(field_value.offset, val); \
1116 } \
1117 break;
1118 FLATBUFFERS_GEN_TYPES_POINTER(FLATBUFFERS_TD);
1119 #undef FLATBUFFERS_TD
1120 // clang-format on
1121 }
1122 }
1123 }
1124 }
1125 for (size_t i = 0; i < fieldn_outer; i++) field_stack_.pop_back();
1126
1127 if (struct_def.fixed) {
1128 builder_.ClearOffsets();
1129 builder_.EndStruct();
1130 FLATBUFFERS_ASSERT(value);
1131 // Temporarily store this struct in the value string, since it is to
1132 // be serialized in-place elsewhere.
1133 value->assign(
1134 reinterpret_cast<const char *>(builder_.GetCurrentBufferPointer()),
1135 struct_def.bytesize);
1136 builder_.PopBytes(struct_def.bytesize);
1137 FLATBUFFERS_ASSERT(!ovalue);
1138 } else {
1139 auto val = builder_.EndTable(start);
1140 if (ovalue) *ovalue = val;
1141 if (value) *value = NumToString(val);
1142 }
1143 return NoError();
1144 }
1145
1146 template <typename F>
ParseVectorDelimiters(size_t & count,F body)1147 CheckedError Parser::ParseVectorDelimiters(size_t &count, F body) {
1148 EXPECT('[');
1149 for (;;) {
1150 if ((!opts.strict_json || !count) && Is(']')) break;
1151 ECHECK(body(count));
1152 count++;
1153 if (Is(']')) break;
1154 ECHECK(ParseComma());
1155 }
1156 NEXT();
1157 return NoError();
1158 }
1159
ParseVector(const Type & type,uoffset_t * ovalue)1160 CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue) {
1161 size_t count = 0;
1162 auto err = ParseVectorDelimiters(count, [&](size_t &) -> CheckedError {
1163 Value val;
1164 val.type = type;
1165 ECHECK(Recurse([&]() { return ParseAnyValue(val, nullptr, 0, nullptr); }));
1166 field_stack_.push_back(std::make_pair(val, nullptr));
1167 return NoError();
1168 });
1169 ECHECK(err);
1170
1171 builder_.StartVector(count * InlineSize(type) / InlineAlignment(type),
1172 InlineAlignment(type));
1173 for (size_t i = 0; i < count; i++) {
1174 // start at the back, since we're building the data backwards.
1175 auto &val = field_stack_.back().first;
1176 switch (val.type.base_type) {
1177 // clang-format off
1178 #define FLATBUFFERS_TD(ENUM, IDLTYPE, \
1179 CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \
1180 case BASE_TYPE_ ## ENUM: \
1181 if (IsStruct(val.type)) SerializeStruct(*val.type.struct_def, val); \
1182 else { \
1183 CTYPE elem; \
1184 ECHECK(atot(val.constant.c_str(), *this, &elem)); \
1185 builder_.PushElement(elem); \
1186 } \
1187 break;
1188 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
1189 #undef FLATBUFFERS_TD
1190 // clang-format on
1191 }
1192 field_stack_.pop_back();
1193 }
1194
1195 builder_.ClearOffsets();
1196 *ovalue = builder_.EndVector(count);
1197 return NoError();
1198 }
1199
ParseNestedFlatbuffer(Value & val,FieldDef * field,size_t fieldn,const StructDef * parent_struct_def)1200 CheckedError Parser::ParseNestedFlatbuffer(Value &val, FieldDef *field,
1201 size_t fieldn,
1202 const StructDef *parent_struct_def) {
1203 if (token_ == '[') { // backwards compat for 'legacy' ubyte buffers
1204 ECHECK(ParseAnyValue(val, field, fieldn, parent_struct_def));
1205 } else {
1206 auto cursor_at_value_begin = cursor_;
1207 ECHECK(SkipAnyJsonValue());
1208 std::string substring(cursor_at_value_begin - 1, cursor_ - 1);
1209
1210 // Create and initialize new parser
1211 Parser nested_parser;
1212 FLATBUFFERS_ASSERT(field->nested_flatbuffer);
1213 nested_parser.root_struct_def_ = field->nested_flatbuffer;
1214 nested_parser.enums_ = enums_;
1215 nested_parser.opts = opts;
1216 nested_parser.uses_flexbuffers_ = uses_flexbuffers_;
1217
1218 // Parse JSON substring into new flatbuffer builder using nested_parser
1219 bool ok = nested_parser.Parse(substring.c_str(), nullptr, nullptr);
1220
1221 // Clean nested_parser to avoid deleting the elements in
1222 // the SymbolTables on destruction
1223 nested_parser.enums_.dict.clear();
1224 nested_parser.enums_.vec.clear();
1225
1226 if (!ok) {
1227 ECHECK(Error(nested_parser.error_));
1228 }
1229 // Force alignment for nested flatbuffer
1230 builder_.ForceVectorAlignment(nested_parser.builder_.GetSize(), sizeof(uint8_t),
1231 nested_parser.builder_.GetBufferMinAlignment());
1232
1233 auto off = builder_.CreateVector(nested_parser.builder_.GetBufferPointer(),
1234 nested_parser.builder_.GetSize());
1235 val.constant = NumToString(off.o);
1236 }
1237 return NoError();
1238 }
1239
ParseMetaData(SymbolTable<Value> * attributes)1240 CheckedError Parser::ParseMetaData(SymbolTable<Value> *attributes) {
1241 if (Is('(')) {
1242 NEXT();
1243 for (;;) {
1244 auto name = attribute_;
1245 if (false == (Is(kTokenIdentifier) || Is(kTokenStringConstant)))
1246 return Error("attribute name must be either identifier or string: " +
1247 name);
1248 if (known_attributes_.find(name) == known_attributes_.end())
1249 return Error("user define attributes must be declared before use: " +
1250 name);
1251 NEXT();
1252 auto e = new Value();
1253 attributes->Add(name, e);
1254 if (Is(':')) {
1255 NEXT();
1256 ECHECK(ParseSingleValue(&name, *e, true));
1257 }
1258 if (Is(')')) {
1259 NEXT();
1260 break;
1261 }
1262 EXPECT(',');
1263 }
1264 }
1265 return NoError();
1266 }
1267
TryTypedValue(const std::string * name,int dtoken,bool check,Value & e,BaseType req,bool * destmatch)1268 CheckedError Parser::TryTypedValue(const std::string *name, int dtoken,
1269 bool check, Value &e, BaseType req,
1270 bool *destmatch) {
1271 bool match = dtoken == token_;
1272 if (match) {
1273 FLATBUFFERS_ASSERT(*destmatch == false);
1274 *destmatch = true;
1275 e.constant = attribute_;
1276 // Check token match
1277 if (!check) {
1278 if (e.type.base_type == BASE_TYPE_NONE) {
1279 e.type.base_type = req;
1280 } else {
1281 return Error(
1282 std::string("type mismatch: expecting: ") +
1283 kTypeNames[e.type.base_type] + ", found: " + kTypeNames[req] +
1284 ", name: " + (name ? *name : "") + ", value: " + e.constant);
1285 }
1286 }
1287 // The exponent suffix of hexadecimal float-point number is mandatory.
1288 // A hex-integer constant is forbidden as an initializer of float number.
1289 if ((kTokenFloatConstant != dtoken) && IsFloat(e.type.base_type)) {
1290 const auto &s = e.constant;
1291 const auto k = s.find_first_of("0123456789.");
1292 if ((std::string::npos != k) && (s.length() > (k + 1)) &&
1293 (s.at(k) == '0' && is_alpha_char(s.at(k + 1), 'X')) &&
1294 (std::string::npos == s.find_first_of("pP", k + 2))) {
1295 return Error(
1296 "invalid number, the exponent suffix of hexadecimal "
1297 "floating-point literals is mandatory: \"" +
1298 s + "\"");
1299 }
1300 }
1301
1302 NEXT();
1303 }
1304 return NoError();
1305 }
1306
ParseEnumFromString(Type & type,int64_t * result)1307 CheckedError Parser::ParseEnumFromString(Type &type, int64_t *result) {
1308 *result = 0;
1309 // Parse one or more enum identifiers, separated by spaces.
1310 const char *next = attribute_.c_str();
1311 do {
1312 const char *divider = strchr(next, ' ');
1313 std::string word;
1314 if (divider) {
1315 word = std::string(next, divider);
1316 next = divider + strspn(divider, " ");
1317 } else {
1318 word = next;
1319 next += word.length();
1320 }
1321 if (type.enum_def) { // The field has an enum type
1322 auto enum_val = type.enum_def->vals.Lookup(word);
1323 if (!enum_val)
1324 return Error("unknown enum value: " + word +
1325 ", for enum: " + type.enum_def->name);
1326 *result |= enum_val->value;
1327 } else { // No enum type, probably integral field.
1328 if (!IsInteger(type.base_type))
1329 return Error("not a valid value for this field: " + word);
1330 // TODO: could check if its a valid number constant here.
1331 const char *dot = strrchr(word.c_str(), '.');
1332 if (!dot)
1333 return Error("enum values need to be qualified by an enum type");
1334 std::string enum_def_str(word.c_str(), dot);
1335 std::string enum_val_str(dot + 1, word.c_str() + word.length());
1336 auto enum_def = LookupEnum(enum_def_str);
1337 if (!enum_def) return Error("unknown enum: " + enum_def_str);
1338 auto enum_val = enum_def->vals.Lookup(enum_val_str);
1339 if (!enum_val) return Error("unknown enum value: " + enum_val_str);
1340 *result |= enum_val->value;
1341 }
1342 } while (*next);
1343 return NoError();
1344 }
1345
ParseHash(Value & e,FieldDef * field)1346 CheckedError Parser::ParseHash(Value &e, FieldDef *field) {
1347 FLATBUFFERS_ASSERT(field);
1348 Value *hash_name = field->attributes.Lookup("hash");
1349 switch (e.type.base_type) {
1350 case BASE_TYPE_SHORT: {
1351 auto hash = FindHashFunction16(hash_name->constant.c_str());
1352 int16_t hashed_value = static_cast<int16_t>(hash(attribute_.c_str()));
1353 e.constant = NumToString(hashed_value);
1354 break;
1355 }
1356 case BASE_TYPE_USHORT: {
1357 auto hash = FindHashFunction16(hash_name->constant.c_str());
1358 uint16_t hashed_value = hash(attribute_.c_str());
1359 e.constant = NumToString(hashed_value);
1360 break;
1361 }
1362 case BASE_TYPE_INT: {
1363 auto hash = FindHashFunction32(hash_name->constant.c_str());
1364 int32_t hashed_value = static_cast<int32_t>(hash(attribute_.c_str()));
1365 e.constant = NumToString(hashed_value);
1366 break;
1367 }
1368 case BASE_TYPE_UINT: {
1369 auto hash = FindHashFunction32(hash_name->constant.c_str());
1370 uint32_t hashed_value = hash(attribute_.c_str());
1371 e.constant = NumToString(hashed_value);
1372 break;
1373 }
1374 case BASE_TYPE_LONG: {
1375 auto hash = FindHashFunction64(hash_name->constant.c_str());
1376 int64_t hashed_value = static_cast<int64_t>(hash(attribute_.c_str()));
1377 e.constant = NumToString(hashed_value);
1378 break;
1379 }
1380 case BASE_TYPE_ULONG: {
1381 auto hash = FindHashFunction64(hash_name->constant.c_str());
1382 uint64_t hashed_value = hash(attribute_.c_str());
1383 e.constant = NumToString(hashed_value);
1384 break;
1385 }
1386 default: FLATBUFFERS_ASSERT(0);
1387 }
1388 NEXT();
1389 return NoError();
1390 }
1391
TokenError()1392 CheckedError Parser::TokenError() {
1393 return Error("cannot parse value starting with: " + TokenToStringId(token_));
1394 }
1395
ParseSingleValue(const std::string * name,Value & e,bool check_now)1396 CheckedError Parser::ParseSingleValue(const std::string *name, Value &e,
1397 bool check_now) {
1398 // First see if this could be a conversion function:
1399 if (token_ == kTokenIdentifier && *cursor_ == '(') {
1400 // todo: Extract processing of conversion functions to ParseFunction.
1401 const auto functionname = attribute_;
1402 if (!IsFloat(e.type.base_type)) {
1403 return Error(functionname + ": type of argument mismatch, expecting: " +
1404 kTypeNames[BASE_TYPE_DOUBLE] +
1405 ", found: " + kTypeNames[e.type.base_type] +
1406 ", name: " + (name ? *name : "") + ", value: " + e.constant);
1407 }
1408 NEXT();
1409 EXPECT('(');
1410 ECHECK(Recurse([&]() { return ParseSingleValue(name, e, false); }));
1411 EXPECT(')');
1412 // calculate with double precision
1413 double x, y = 0.0;
1414 ECHECK(atot(e.constant.c_str(), *this, &x));
1415 auto func_match = false;
1416 // clang-format off
1417 #define FLATBUFFERS_FN_DOUBLE(name, op) \
1418 if (!func_match && functionname == name) { y = op; func_match = true; }
1419 FLATBUFFERS_FN_DOUBLE("deg", x / kPi * 180);
1420 FLATBUFFERS_FN_DOUBLE("rad", x * kPi / 180);
1421 FLATBUFFERS_FN_DOUBLE("sin", sin(x));
1422 FLATBUFFERS_FN_DOUBLE("cos", cos(x));
1423 FLATBUFFERS_FN_DOUBLE("tan", tan(x));
1424 FLATBUFFERS_FN_DOUBLE("asin", asin(x));
1425 FLATBUFFERS_FN_DOUBLE("acos", acos(x));
1426 FLATBUFFERS_FN_DOUBLE("atan", atan(x));
1427 // TODO(wvo): add more useful conversion functions here.
1428 #undef FLATBUFFERS_FN_DOUBLE
1429 // clang-format on
1430 if (true != func_match) {
1431 return Error(std::string("Unknown conversion function: ") + functionname +
1432 ", field name: " + (name ? *name : "") +
1433 ", value: " + e.constant);
1434 }
1435 e.constant = NumToString(y);
1436 return NoError();
1437 }
1438
1439 auto match = false;
1440 // clang-format off
1441 #define TRY_ECHECK(force, dtoken, check, req) \
1442 if (!match && ((check) || IsConstTrue(force))) \
1443 ECHECK(TryTypedValue(name, dtoken, check, e, req, &match))
1444 // clang-format on
1445
1446 if (token_ == kTokenStringConstant || token_ == kTokenIdentifier) {
1447 const auto kTokenStringOrIdent = token_;
1448 // The string type is a most probable type, check it first.
1449 TRY_ECHECK(false, kTokenStringConstant,
1450 e.type.base_type == BASE_TYPE_STRING, BASE_TYPE_STRING);
1451
1452 // avoid escaped and non-ascii in the string
1453 if ((token_ == kTokenStringConstant) && IsScalar(e.type.base_type) &&
1454 !attr_is_trivial_ascii_string_) {
1455 return Error(
1456 std::string("type mismatch or invalid value, an initializer of "
1457 "non-string field must be trivial ASCII string: type: ") +
1458 kTypeNames[e.type.base_type] + ", name: " + (name ? *name : "") +
1459 ", value: " + attribute_);
1460 }
1461
1462 // A boolean as true/false. Boolean as Integer check below.
1463 if (!match && IsBool(e.type.base_type)) {
1464 auto is_true = attribute_ == "true";
1465 if (is_true || attribute_ == "false") {
1466 attribute_ = is_true ? "1" : "0";
1467 // accepts both kTokenStringConstant and kTokenIdentifier
1468 TRY_ECHECK(false, kTokenStringOrIdent, IsBool(e.type.base_type),
1469 BASE_TYPE_BOOL);
1470 }
1471 }
1472 // Check if this could be a string/identifier enum value.
1473 // Enum can have only true integer base type.
1474 if (!match && IsInteger(e.type.base_type) && !IsBool(e.type.base_type) &&
1475 IsIdentifierStart(*attribute_.c_str())) {
1476 int64_t val;
1477 ECHECK(ParseEnumFromString(e.type, &val));
1478 e.constant = NumToString(val);
1479 NEXT();
1480 match = true;
1481 }
1482 // float/integer number in string
1483 if ((token_ == kTokenStringConstant) && IsScalar(e.type.base_type)) {
1484 // remove trailing whitespaces from attribute_
1485 auto last = attribute_.find_last_not_of(' ');
1486 if (std::string::npos != last) // has non-whitespace
1487 attribute_.resize(last + 1);
1488 }
1489 // Float numbers or nan, inf, pi, etc.
1490 TRY_ECHECK(false, kTokenStringOrIdent, IsFloat(e.type.base_type),
1491 BASE_TYPE_FLOAT);
1492 // An integer constant in string.
1493 TRY_ECHECK(false, kTokenStringOrIdent, IsInteger(e.type.base_type),
1494 BASE_TYPE_INT);
1495 // Unknown tokens will be interpreted as string type.
1496 TRY_ECHECK(true, kTokenStringConstant, e.type.base_type == BASE_TYPE_STRING,
1497 BASE_TYPE_STRING);
1498 } else {
1499 // Try a float number.
1500 TRY_ECHECK(false, kTokenFloatConstant, IsFloat(e.type.base_type),
1501 BASE_TYPE_FLOAT);
1502 // Integer token can init any scalar (integer of float).
1503 TRY_ECHECK(true, kTokenIntegerConstant, IsScalar(e.type.base_type),
1504 BASE_TYPE_INT);
1505 }
1506 #undef TRY_ECHECK
1507
1508 if (!match) return TokenError();
1509
1510 // The check_now flag must be true when parse a fbs-schema.
1511 // This flag forces to check default scalar values or metadata of field.
1512 // For JSON parser the flag should be false.
1513 // If it is set for JSON each value will be checked twice (see ParseTable).
1514 if (check_now && IsScalar(e.type.base_type)) {
1515 // "re-pack" an integer scalar to remove any ambiguities like leading zeros
1516 // which can be treated as octal-literal (idl_gen_cpp/GenDefaultConstant).
1517 const auto repack = IsInteger(e.type.base_type);
1518 switch (e.type.base_type) {
1519 // clang-format off
1520 #define FLATBUFFERS_TD(ENUM, IDLTYPE, \
1521 CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \
1522 case BASE_TYPE_ ## ENUM: {\
1523 CTYPE val; \
1524 ECHECK(atot(e.constant.c_str(), *this, &val)); \
1525 if(repack) e.constant = NumToString(val); \
1526 break; }
1527 FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD);
1528 #undef FLATBUFFERS_TD
1529 default: break;
1530 // clang-format on
1531 }
1532 }
1533 return NoError();
1534 }
1535
LookupCreateStruct(const std::string & name,bool create_if_new,bool definition)1536 StructDef *Parser::LookupCreateStruct(const std::string &name,
1537 bool create_if_new, bool definition) {
1538 std::string qualified_name = current_namespace_->GetFullyQualifiedName(name);
1539 // See if it exists pre-declared by an unqualified use.
1540 auto struct_def = LookupStruct(name);
1541 if (struct_def && struct_def->predecl) {
1542 if (definition) {
1543 // Make sure it has the current namespace, and is registered under its
1544 // qualified name.
1545 struct_def->defined_namespace = current_namespace_;
1546 structs_.Move(name, qualified_name);
1547 }
1548 return struct_def;
1549 }
1550 // See if it exists pre-declared by an qualified use.
1551 struct_def = LookupStruct(qualified_name);
1552 if (struct_def && struct_def->predecl) {
1553 if (definition) {
1554 // Make sure it has the current namespace.
1555 struct_def->defined_namespace = current_namespace_;
1556 }
1557 return struct_def;
1558 }
1559 if (!definition) {
1560 // Search thru parent namespaces.
1561 for (size_t components = current_namespace_->components.size();
1562 components && !struct_def; components--) {
1563 struct_def = LookupStruct(
1564 current_namespace_->GetFullyQualifiedName(name, components - 1));
1565 }
1566 }
1567 if (!struct_def && create_if_new) {
1568 struct_def = new StructDef();
1569 if (definition) {
1570 structs_.Add(qualified_name, struct_def);
1571 struct_def->name = name;
1572 struct_def->defined_namespace = current_namespace_;
1573 } else {
1574 // Not a definition.
1575 // Rather than failing, we create a "pre declared" StructDef, due to
1576 // circular references, and check for errors at the end of parsing.
1577 // It is defined in the current namespace, as the best guess what the
1578 // final namespace will be.
1579 structs_.Add(name, struct_def);
1580 struct_def->name = name;
1581 struct_def->defined_namespace = current_namespace_;
1582 struct_def->original_location.reset(
1583 new std::string(file_being_parsed_ + ":" + NumToString(line_)));
1584 }
1585 }
1586 return struct_def;
1587 }
1588
ParseEnum(bool is_union,EnumDef ** dest)1589 CheckedError Parser::ParseEnum(bool is_union, EnumDef **dest) {
1590 std::vector<std::string> enum_comment = doc_comment_;
1591 NEXT();
1592 std::string enum_name = attribute_;
1593 EXPECT(kTokenIdentifier);
1594 EnumDef *enum_def;
1595 ECHECK(StartEnum(enum_name, is_union, &enum_def));
1596 enum_def->doc_comment = enum_comment;
1597 if (!is_union && !opts.proto_mode) {
1598 // Give specialized error message, since this type spec used to
1599 // be optional in the first FlatBuffers release.
1600 if (!Is(':')) {
1601 return Error(
1602 "must specify the underlying integer type for this"
1603 " enum (e.g. \': short\', which was the default).");
1604 } else {
1605 NEXT();
1606 }
1607 // Specify the integer type underlying this enum.
1608 ECHECK(ParseType(enum_def->underlying_type));
1609 if (!IsInteger(enum_def->underlying_type.base_type) ||
1610 IsBool(enum_def->underlying_type.base_type))
1611 return Error("underlying enum type must be integral");
1612 // Make this type refer back to the enum it was derived from.
1613 enum_def->underlying_type.enum_def = enum_def;
1614 }
1615 ECHECK(ParseMetaData(&enum_def->attributes));
1616 EXPECT('{');
1617 if (is_union) enum_def->vals.Add("NONE", new EnumVal("NONE", 0));
1618 std::set<std::pair<BaseType, StructDef*>> union_types;
1619 for (;;) {
1620 if (opts.proto_mode && attribute_ == "option") {
1621 ECHECK(ParseProtoOption());
1622 } else {
1623 auto value_name = attribute_;
1624 auto full_name = value_name;
1625 std::vector<std::string> value_comment = doc_comment_;
1626 EXPECT(kTokenIdentifier);
1627 if (is_union) {
1628 ECHECK(ParseNamespacing(&full_name, &value_name));
1629 if (opts.union_value_namespacing) {
1630 // Since we can't namespace the actual enum identifiers, turn
1631 // namespace parts into part of the identifier.
1632 value_name = full_name;
1633 std::replace(value_name.begin(), value_name.end(), '.', '_');
1634 }
1635 }
1636 auto prevsize = enum_def->vals.vec.size();
1637 auto prevvalue = prevsize > 0 ? enum_def->vals.vec.back()->value : 0;
1638 auto &ev = *new EnumVal(value_name, 0);
1639 if (enum_def->vals.Add(value_name, &ev))
1640 return Error("enum value already exists: " + value_name);
1641 ev.doc_comment = value_comment;
1642 if (is_union) {
1643 if (Is(':')) {
1644 NEXT();
1645 ECHECK(ParseType(ev.union_type));
1646 if (ev.union_type.base_type != BASE_TYPE_STRUCT &&
1647 ev.union_type.base_type != BASE_TYPE_STRING)
1648 return Error("union value type may only be table/struct/string");
1649 } else {
1650 ev.union_type = Type(BASE_TYPE_STRUCT, LookupCreateStruct(full_name));
1651 }
1652 if (!enum_def->uses_multiple_type_instances) {
1653 auto union_type_key = std::make_pair(ev.union_type.base_type, ev.union_type.struct_def);
1654 if (union_types.count(union_type_key) > 0) {
1655 enum_def->uses_multiple_type_instances = true;
1656 } else {
1657 union_types.insert(union_type_key);
1658 }
1659 }
1660 }
1661 if (Is('=')) {
1662 NEXT();
1663 ECHECK(atot(attribute_.c_str(), *this, &ev.value));
1664 EXPECT(kTokenIntegerConstant);
1665 if (!opts.proto_mode && prevsize &&
1666 enum_def->vals.vec[prevsize - 1]->value >= ev.value)
1667 return Error("enum values must be specified in ascending order");
1668 } else if (prevsize == 0) {
1669 // already set to zero
1670 } else if (prevvalue != flatbuffers::numeric_limits<int64_t>::max()) {
1671 ev.value = prevvalue + 1;
1672 } else {
1673 return Error("enum value overflows");
1674 }
1675
1676 // Check that value fits into the underlying type.
1677 switch (enum_def->underlying_type.base_type) {
1678 // clang-format off
1679 #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, \
1680 PTYPE, RTYPE) \
1681 case BASE_TYPE_##ENUM: { \
1682 int64_t min_value = static_cast<int64_t>( \
1683 flatbuffers::numeric_limits<CTYPE>::lowest()); \
1684 int64_t max_value = static_cast<int64_t>( \
1685 flatbuffers::numeric_limits<CTYPE>::max()); \
1686 if (ev.value < min_value || ev.value > max_value) { \
1687 return Error( \
1688 "enum value does not fit [" + NumToString(min_value) + \
1689 "; " + NumToString(max_value) + "]"); \
1690 } \
1691 break; \
1692 }
1693 FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD);
1694 #undef FLATBUFFERS_TD
1695 default: break;
1696 // clang-format on
1697 }
1698
1699 if (opts.proto_mode && Is('[')) {
1700 NEXT();
1701 // ignore attributes on enums.
1702 while (token_ != ']') NEXT();
1703 NEXT();
1704 }
1705 }
1706 if (!Is(opts.proto_mode ? ';' : ',')) break;
1707 NEXT();
1708 if (Is('}')) break;
1709 }
1710 EXPECT('}');
1711 if (enum_def->attributes.Lookup("bit_flags")) {
1712 for (auto it = enum_def->vals.vec.begin(); it != enum_def->vals.vec.end();
1713 ++it) {
1714 if (static_cast<size_t>((*it)->value) >=
1715 SizeOf(enum_def->underlying_type.base_type) * 8)
1716 return Error("bit flag out of range of underlying integral type");
1717 (*it)->value = 1LL << (*it)->value;
1718 }
1719 }
1720 if (dest) *dest = enum_def;
1721 types_.Add(current_namespace_->GetFullyQualifiedName(enum_def->name),
1722 new Type(BASE_TYPE_UNION, nullptr, enum_def));
1723 return NoError();
1724 }
1725
StartStruct(const std::string & name,StructDef ** dest)1726 CheckedError Parser::StartStruct(const std::string &name, StructDef **dest) {
1727 auto &struct_def = *LookupCreateStruct(name, true, true);
1728 if (!struct_def.predecl) return Error("datatype already exists: " + name);
1729 struct_def.predecl = false;
1730 struct_def.name = name;
1731 struct_def.file = file_being_parsed_;
1732 // Move this struct to the back of the vector just in case it was predeclared,
1733 // to preserve declaration order.
1734 *std::remove(structs_.vec.begin(), structs_.vec.end(), &struct_def) =
1735 &struct_def;
1736 *dest = &struct_def;
1737 return NoError();
1738 }
1739
CheckClash(std::vector<FieldDef * > & fields,StructDef * struct_def,const char * suffix,BaseType basetype)1740 CheckedError Parser::CheckClash(std::vector<FieldDef *> &fields,
1741 StructDef *struct_def, const char *suffix,
1742 BaseType basetype) {
1743 auto len = strlen(suffix);
1744 for (auto it = fields.begin(); it != fields.end(); ++it) {
1745 auto &fname = (*it)->name;
1746 if (fname.length() > len &&
1747 fname.compare(fname.length() - len, len, suffix) == 0 &&
1748 (*it)->value.type.base_type != BASE_TYPE_UTYPE) {
1749 auto field =
1750 struct_def->fields.Lookup(fname.substr(0, fname.length() - len));
1751 if (field && field->value.type.base_type == basetype)
1752 return Error("Field " + fname +
1753 " would clash with generated functions for field " +
1754 field->name);
1755 }
1756 }
1757 return NoError();
1758 }
1759
SupportsVectorOfUnions() const1760 bool Parser::SupportsVectorOfUnions() const {
1761 return opts.lang_to_generate != 0 &&
1762 (opts.lang_to_generate & ~(IDLOptions::kCpp | IDLOptions::kJs |
1763 IDLOptions::kTs | IDLOptions::kPhp |
1764 IDLOptions::kJava | IDLOptions::kCSharp)) == 0;
1765 }
1766
UniqueNamespace(Namespace * ns)1767 Namespace *Parser::UniqueNamespace(Namespace *ns) {
1768 for (auto it = namespaces_.begin(); it != namespaces_.end(); ++it) {
1769 if (ns->components == (*it)->components) {
1770 delete ns;
1771 return *it;
1772 }
1773 }
1774 namespaces_.push_back(ns);
1775 return ns;
1776 }
1777
UnqualifiedName(std::string full_qualified_name)1778 std::string Parser::UnqualifiedName(std::string full_qualified_name) {
1779 Namespace *ns = new Namespace();
1780
1781 std::size_t current, previous = 0;
1782 current = full_qualified_name.find('.');
1783 while (current != std::string::npos) {
1784 ns->components.push_back(
1785 full_qualified_name.substr(previous, current - previous));
1786 previous = current + 1;
1787 current = full_qualified_name.find('.', previous);
1788 }
1789 current_namespace_ = UniqueNamespace(ns);
1790 return full_qualified_name.substr(previous, current - previous);
1791 }
1792
compareFieldDefs(const FieldDef * a,const FieldDef * b)1793 static bool compareFieldDefs(const FieldDef *a, const FieldDef *b) {
1794 auto a_id = atoi(a->attributes.Lookup("id")->constant.c_str());
1795 auto b_id = atoi(b->attributes.Lookup("id")->constant.c_str());
1796 return a_id < b_id;
1797 }
1798
ParseDecl()1799 CheckedError Parser::ParseDecl() {
1800 std::vector<std::string> dc = doc_comment_;
1801 bool fixed = IsIdent("struct");
1802 if (!fixed && !IsIdent("table")) return Error("declaration expected");
1803 NEXT();
1804 std::string name = attribute_;
1805 EXPECT(kTokenIdentifier);
1806 StructDef *struct_def;
1807 ECHECK(StartStruct(name, &struct_def));
1808 struct_def->doc_comment = dc;
1809 struct_def->fixed = fixed;
1810 ECHECK(ParseMetaData(&struct_def->attributes));
1811 struct_def->sortbysize =
1812 struct_def->attributes.Lookup("original_order") == nullptr && !fixed;
1813 EXPECT('{');
1814 while (token_ != '}') ECHECK(ParseField(*struct_def));
1815 auto force_align = struct_def->attributes.Lookup("force_align");
1816 if (fixed) {
1817 if (force_align) {
1818 auto align = static_cast<size_t>(atoi(force_align->constant.c_str()));
1819 if (force_align->type.base_type != BASE_TYPE_INT ||
1820 align < struct_def->minalign || align > FLATBUFFERS_MAX_ALIGNMENT ||
1821 align & (align - 1))
1822 return Error(
1823 "force_align must be a power of two integer ranging from the"
1824 "struct\'s natural alignment to " +
1825 NumToString(FLATBUFFERS_MAX_ALIGNMENT));
1826 struct_def->minalign = align;
1827 }
1828 if (!struct_def->bytesize) return Error("size 0 structs not allowed");
1829 }
1830 struct_def->PadLastField(struct_def->minalign);
1831 // Check if this is a table that has manual id assignments
1832 auto &fields = struct_def->fields.vec;
1833 if (!fixed && fields.size()) {
1834 size_t num_id_fields = 0;
1835 for (auto it = fields.begin(); it != fields.end(); ++it) {
1836 if ((*it)->attributes.Lookup("id")) num_id_fields++;
1837 }
1838 // If any fields have ids..
1839 if (num_id_fields) {
1840 // Then all fields must have them.
1841 if (num_id_fields != fields.size())
1842 return Error(
1843 "either all fields or no fields must have an 'id' attribute");
1844 // Simply sort by id, then the fields are the same as if no ids had
1845 // been specified.
1846 std::sort(fields.begin(), fields.end(), compareFieldDefs);
1847 // Verify we have a contiguous set, and reassign vtable offsets.
1848 for (int i = 0; i < static_cast<int>(fields.size()); i++) {
1849 if (i != atoi(fields[i]->attributes.Lookup("id")->constant.c_str()))
1850 return Error("field id\'s must be consecutive from 0, id " +
1851 NumToString(i) + " missing or set twice");
1852 fields[i]->value.offset = FieldIndexToOffset(static_cast<voffset_t>(i));
1853 }
1854 }
1855 }
1856
1857 ECHECK(
1858 CheckClash(fields, struct_def, UnionTypeFieldSuffix(), BASE_TYPE_UNION));
1859 ECHECK(CheckClash(fields, struct_def, "Type", BASE_TYPE_UNION));
1860 ECHECK(CheckClash(fields, struct_def, "_length", BASE_TYPE_VECTOR));
1861 ECHECK(CheckClash(fields, struct_def, "Length", BASE_TYPE_VECTOR));
1862 ECHECK(CheckClash(fields, struct_def, "_byte_vector", BASE_TYPE_STRING));
1863 ECHECK(CheckClash(fields, struct_def, "ByteVector", BASE_TYPE_STRING));
1864 EXPECT('}');
1865 types_.Add(current_namespace_->GetFullyQualifiedName(struct_def->name),
1866 new Type(BASE_TYPE_STRUCT, struct_def, nullptr));
1867 return NoError();
1868 }
1869
ParseService()1870 CheckedError Parser::ParseService() {
1871 std::vector<std::string> service_comment = doc_comment_;
1872 NEXT();
1873 auto service_name = attribute_;
1874 EXPECT(kTokenIdentifier);
1875 auto &service_def = *new ServiceDef();
1876 service_def.name = service_name;
1877 service_def.file = file_being_parsed_;
1878 service_def.doc_comment = service_comment;
1879 service_def.defined_namespace = current_namespace_;
1880 if (services_.Add(current_namespace_->GetFullyQualifiedName(service_name),
1881 &service_def))
1882 return Error("service already exists: " + service_name);
1883 ECHECK(ParseMetaData(&service_def.attributes));
1884 EXPECT('{');
1885 do {
1886 std::vector<std::string> doc_comment = doc_comment_;
1887 auto rpc_name = attribute_;
1888 EXPECT(kTokenIdentifier);
1889 EXPECT('(');
1890 Type reqtype, resptype;
1891 ECHECK(ParseTypeIdent(reqtype));
1892 EXPECT(')');
1893 EXPECT(':');
1894 ECHECK(ParseTypeIdent(resptype));
1895 if (reqtype.base_type != BASE_TYPE_STRUCT || reqtype.struct_def->fixed ||
1896 resptype.base_type != BASE_TYPE_STRUCT || resptype.struct_def->fixed)
1897 return Error("rpc request and response types must be tables");
1898 auto &rpc = *new RPCCall();
1899 rpc.name = rpc_name;
1900 rpc.request = reqtype.struct_def;
1901 rpc.response = resptype.struct_def;
1902 rpc.doc_comment = doc_comment;
1903 if (service_def.calls.Add(rpc_name, &rpc))
1904 return Error("rpc already exists: " + rpc_name);
1905 ECHECK(ParseMetaData(&rpc.attributes));
1906 EXPECT(';');
1907 } while (token_ != '}');
1908 NEXT();
1909 return NoError();
1910 }
1911
SetRootType(const char * name)1912 bool Parser::SetRootType(const char *name) {
1913 root_struct_def_ = LookupStruct(name);
1914 if (!root_struct_def_)
1915 root_struct_def_ =
1916 LookupStruct(current_namespace_->GetFullyQualifiedName(name));
1917 return root_struct_def_ != nullptr;
1918 }
1919
MarkGenerated()1920 void Parser::MarkGenerated() {
1921 // This function marks all existing definitions as having already
1922 // been generated, which signals no code for included files should be
1923 // generated.
1924 for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) {
1925 (*it)->generated = true;
1926 }
1927 for (auto it = structs_.vec.begin(); it != structs_.vec.end(); ++it) {
1928 if (!(*it)->predecl) { (*it)->generated = true; }
1929 }
1930 for (auto it = services_.vec.begin(); it != services_.vec.end(); ++it) {
1931 (*it)->generated = true;
1932 }
1933 }
1934
ParseNamespace()1935 CheckedError Parser::ParseNamespace() {
1936 NEXT();
1937 auto ns = new Namespace();
1938 namespaces_.push_back(ns); // Store it here to not leak upon error.
1939 if (token_ != ';') {
1940 for (;;) {
1941 ns->components.push_back(attribute_);
1942 EXPECT(kTokenIdentifier);
1943 if (Is('.')) NEXT() else break;
1944 }
1945 }
1946 namespaces_.pop_back();
1947 current_namespace_ = UniqueNamespace(ns);
1948 EXPECT(';');
1949 return NoError();
1950 }
1951
compareEnumVals(const EnumVal * a,const EnumVal * b)1952 static bool compareEnumVals(const EnumVal *a, const EnumVal *b) {
1953 return a->value < b->value;
1954 }
1955
1956 // Best effort parsing of .proto declarations, with the aim to turn them
1957 // in the closest corresponding FlatBuffer equivalent.
1958 // We parse everything as identifiers instead of keywords, since we don't
1959 // want protobuf keywords to become invalid identifiers in FlatBuffers.
ParseProtoDecl()1960 CheckedError Parser::ParseProtoDecl() {
1961 bool isextend = IsIdent("extend");
1962 if (IsIdent("package")) {
1963 // These are identical in syntax to FlatBuffer's namespace decl.
1964 ECHECK(ParseNamespace());
1965 } else if (IsIdent("message") || isextend) {
1966 std::vector<std::string> struct_comment = doc_comment_;
1967 NEXT();
1968 StructDef *struct_def = nullptr;
1969 Namespace *parent_namespace = nullptr;
1970 if (isextend) {
1971 if (Is('.')) NEXT(); // qualified names may start with a . ?
1972 auto id = attribute_;
1973 EXPECT(kTokenIdentifier);
1974 ECHECK(ParseNamespacing(&id, nullptr));
1975 struct_def = LookupCreateStruct(id, false);
1976 if (!struct_def)
1977 return Error("cannot extend unknown message type: " + id);
1978 } else {
1979 std::string name = attribute_;
1980 EXPECT(kTokenIdentifier);
1981 ECHECK(StartStruct(name, &struct_def));
1982 // Since message definitions can be nested, we create a new namespace.
1983 auto ns = new Namespace();
1984 // Copy of current namespace.
1985 *ns = *current_namespace_;
1986 // But with current message name.
1987 ns->components.push_back(name);
1988 ns->from_table++;
1989 parent_namespace = current_namespace_;
1990 current_namespace_ = UniqueNamespace(ns);
1991 }
1992 struct_def->doc_comment = struct_comment;
1993 ECHECK(ParseProtoFields(struct_def, isextend, false));
1994 if (!isextend) { current_namespace_ = parent_namespace; }
1995 if (Is(';')) NEXT();
1996 } else if (IsIdent("enum")) {
1997 // These are almost the same, just with different terminator:
1998 EnumDef *enum_def;
1999 ECHECK(ParseEnum(false, &enum_def));
2000 if (Is(';')) NEXT();
2001 // Protobuf allows them to be specified in any order, so sort afterwards.
2002 auto &v = enum_def->vals.vec;
2003 std::sort(v.begin(), v.end(), compareEnumVals);
2004
2005 // Temp: remove any duplicates, as .fbs files can't handle them.
2006 for (auto it = v.begin(); it != v.end();) {
2007 if (it != v.begin() && it[0]->value == it[-1]->value)
2008 it = v.erase(it);
2009 else
2010 ++it;
2011 }
2012 } else if (IsIdent("syntax")) { // Skip these.
2013 NEXT();
2014 EXPECT('=');
2015 EXPECT(kTokenStringConstant);
2016 EXPECT(';');
2017 } else if (IsIdent("option")) { // Skip these.
2018 ECHECK(ParseProtoOption());
2019 EXPECT(';');
2020 } else if (IsIdent("service")) { // Skip these.
2021 NEXT();
2022 EXPECT(kTokenIdentifier);
2023 ECHECK(ParseProtoCurliesOrIdent());
2024 } else {
2025 return Error("don\'t know how to parse .proto declaration starting with " +
2026 TokenToStringId(token_));
2027 }
2028 return NoError();
2029 }
2030
StartEnum(const std::string & enum_name,bool is_union,EnumDef ** dest)2031 CheckedError Parser::StartEnum(const std::string &enum_name, bool is_union,
2032 EnumDef **dest) {
2033 auto &enum_def = *new EnumDef();
2034 enum_def.name = enum_name;
2035 enum_def.file = file_being_parsed_;
2036 enum_def.doc_comment = doc_comment_;
2037 enum_def.is_union = is_union;
2038 enum_def.defined_namespace = current_namespace_;
2039 if (enums_.Add(current_namespace_->GetFullyQualifiedName(enum_name),
2040 &enum_def))
2041 return Error("enum already exists: " + enum_name);
2042 enum_def.underlying_type.base_type = is_union ? BASE_TYPE_UTYPE
2043 : BASE_TYPE_INT;
2044 enum_def.underlying_type.enum_def = &enum_def;
2045 if (dest) *dest = &enum_def;
2046 return NoError();
2047 }
2048
ParseProtoFields(StructDef * struct_def,bool isextend,bool inside_oneof)2049 CheckedError Parser::ParseProtoFields(StructDef *struct_def, bool isextend,
2050 bool inside_oneof) {
2051 EXPECT('{');
2052 while (token_ != '}') {
2053 if (IsIdent("message") || IsIdent("extend") || IsIdent("enum")) {
2054 // Nested declarations.
2055 ECHECK(ParseProtoDecl());
2056 } else if (IsIdent("extensions")) { // Skip these.
2057 NEXT();
2058 EXPECT(kTokenIntegerConstant);
2059 if (Is(kTokenIdentifier)) {
2060 NEXT(); // to
2061 NEXT(); // num
2062 }
2063 EXPECT(';');
2064 } else if (IsIdent("option")) { // Skip these.
2065 ECHECK(ParseProtoOption());
2066 EXPECT(';');
2067 } else if (IsIdent("reserved")) { // Skip these.
2068 NEXT();
2069 while (!Is(';')) { NEXT(); } // A variety of formats, just skip.
2070 NEXT();
2071 } else {
2072 std::vector<std::string> field_comment = doc_comment_;
2073 // Parse the qualifier.
2074 bool required = false;
2075 bool repeated = false;
2076 bool oneof = false;
2077 if (!inside_oneof) {
2078 if (IsIdent("optional")) {
2079 // This is the default.
2080 NEXT();
2081 } else if (IsIdent("required")) {
2082 required = true;
2083 NEXT();
2084 } else if (IsIdent("repeated")) {
2085 repeated = true;
2086 NEXT();
2087 } else if (IsIdent("oneof")) {
2088 oneof = true;
2089 NEXT();
2090 } else {
2091 // can't error, proto3 allows decls without any of the above.
2092 }
2093 }
2094 StructDef *anonymous_struct = nullptr;
2095 EnumDef *oneof_union = nullptr;
2096 Type type;
2097 if (IsIdent("group") || oneof) {
2098 if (!oneof) NEXT();
2099 if (oneof && opts.proto_oneof_union) {
2100 auto name = MakeCamel(attribute_, true) + "Union";
2101 ECHECK(StartEnum(name, true, &oneof_union));
2102 type = Type(BASE_TYPE_UNION, nullptr, oneof_union);
2103 } else {
2104 auto name = "Anonymous" + NumToString(anonymous_counter++);
2105 ECHECK(StartStruct(name, &anonymous_struct));
2106 type = Type(BASE_TYPE_STRUCT, anonymous_struct);
2107 }
2108 } else {
2109 ECHECK(ParseTypeFromProtoType(&type));
2110 }
2111 // Repeated elements get mapped to a vector.
2112 if (repeated) {
2113 type.element = type.base_type;
2114 type.base_type = BASE_TYPE_VECTOR;
2115 if (type.element == BASE_TYPE_VECTOR) {
2116 // We have a vector or vectors, which FlatBuffers doesn't support.
2117 // For now make it a vector of string (since the source is likely
2118 // "repeated bytes").
2119 // TODO(wvo): A better solution would be to wrap this in a table.
2120 type.element = BASE_TYPE_STRING;
2121 }
2122 }
2123 std::string name = attribute_;
2124 EXPECT(kTokenIdentifier);
2125 if (!oneof) {
2126 // Parse the field id. Since we're just translating schemas, not
2127 // any kind of binary compatibility, we can safely ignore these, and
2128 // assign our own.
2129 EXPECT('=');
2130 EXPECT(kTokenIntegerConstant);
2131 }
2132 FieldDef *field = nullptr;
2133 if (isextend) {
2134 // We allow a field to be re-defined when extending.
2135 // TODO: are there situations where that is problematic?
2136 field = struct_def->fields.Lookup(name);
2137 }
2138 if (!field) ECHECK(AddField(*struct_def, name, type, &field));
2139 field->doc_comment = field_comment;
2140 if (!IsScalar(type.base_type)) field->required = required;
2141 // See if there's a default specified.
2142 if (Is('[')) {
2143 NEXT();
2144 for (;;) {
2145 auto key = attribute_;
2146 ECHECK(ParseProtoKey());
2147 EXPECT('=');
2148 auto val = attribute_;
2149 ECHECK(ParseProtoCurliesOrIdent());
2150 if (key == "default") {
2151 // Temp: skip non-numeric defaults (enums).
2152 auto numeric = strpbrk(val.c_str(), "0123456789-+.");
2153 if (IsScalar(type.base_type) && numeric == val.c_str())
2154 field->value.constant = val;
2155 } else if (key == "deprecated") {
2156 field->deprecated = val == "true";
2157 }
2158 if (!Is(',')) break;
2159 NEXT();
2160 }
2161 EXPECT(']');
2162 }
2163 if (anonymous_struct) {
2164 ECHECK(ParseProtoFields(anonymous_struct, false, oneof));
2165 if (Is(';')) NEXT();
2166 } else if (oneof_union) {
2167 // Parse into a temporary StructDef, then transfer fields into an
2168 // EnumDef describing the oneof as a union.
2169 StructDef oneof_struct;
2170 ECHECK(ParseProtoFields(&oneof_struct, false, oneof));
2171 if (Is(';')) NEXT();
2172 for (auto field_it = oneof_struct.fields.vec.begin();
2173 field_it != oneof_struct.fields.vec.end(); ++field_it) {
2174 const auto &oneof_field = **field_it;
2175 const auto &oneof_type = oneof_field.value.type;
2176 if (oneof_type.base_type != BASE_TYPE_STRUCT ||
2177 !oneof_type.struct_def || oneof_type.struct_def->fixed)
2178 return Error("oneof '" + name +
2179 "' cannot be mapped to a union because member '" +
2180 oneof_field.name + "' is not a table type.");
2181 auto enum_val = new EnumVal(oneof_type.struct_def->name,
2182 oneof_union->vals.vec.size());
2183 enum_val->union_type = oneof_type;
2184 enum_val->doc_comment = oneof_field.doc_comment;
2185 oneof_union->vals.Add(oneof_field.name, enum_val);
2186 }
2187 } else {
2188 EXPECT(';');
2189 }
2190 }
2191 }
2192 NEXT();
2193 return NoError();
2194 }
2195
ParseProtoKey()2196 CheckedError Parser::ParseProtoKey() {
2197 if (token_ == '(') {
2198 NEXT();
2199 // Skip "(a.b)" style custom attributes.
2200 while (token_ == '.' || token_ == kTokenIdentifier) NEXT();
2201 EXPECT(')');
2202 while (Is('.')) {
2203 NEXT();
2204 EXPECT(kTokenIdentifier);
2205 }
2206 } else {
2207 EXPECT(kTokenIdentifier);
2208 }
2209 return NoError();
2210 }
2211
ParseProtoCurliesOrIdent()2212 CheckedError Parser::ParseProtoCurliesOrIdent() {
2213 if (Is('{')) {
2214 NEXT();
2215 for (int nesting = 1; nesting;) {
2216 if (token_ == '{')
2217 nesting++;
2218 else if (token_ == '}')
2219 nesting--;
2220 NEXT();
2221 }
2222 } else {
2223 NEXT(); // Any single token.
2224 }
2225 return NoError();
2226 }
2227
ParseProtoOption()2228 CheckedError Parser::ParseProtoOption() {
2229 NEXT();
2230 ECHECK(ParseProtoKey());
2231 EXPECT('=');
2232 ECHECK(ParseProtoCurliesOrIdent());
2233 return NoError();
2234 }
2235
2236 // Parse a protobuf type, and map it to the corresponding FlatBuffer one.
ParseTypeFromProtoType(Type * type)2237 CheckedError Parser::ParseTypeFromProtoType(Type *type) {
2238 struct type_lookup {
2239 const char *proto_type;
2240 BaseType fb_type, element;
2241 };
2242 static type_lookup lookup[] = {
2243 { "float", BASE_TYPE_FLOAT, BASE_TYPE_NONE },
2244 { "double", BASE_TYPE_DOUBLE, BASE_TYPE_NONE },
2245 { "int32", BASE_TYPE_INT, BASE_TYPE_NONE },
2246 { "int64", BASE_TYPE_LONG, BASE_TYPE_NONE },
2247 { "uint32", BASE_TYPE_UINT, BASE_TYPE_NONE },
2248 { "uint64", BASE_TYPE_ULONG, BASE_TYPE_NONE },
2249 { "sint32", BASE_TYPE_INT, BASE_TYPE_NONE },
2250 { "sint64", BASE_TYPE_LONG, BASE_TYPE_NONE },
2251 { "fixed32", BASE_TYPE_UINT, BASE_TYPE_NONE },
2252 { "fixed64", BASE_TYPE_ULONG, BASE_TYPE_NONE },
2253 { "sfixed32", BASE_TYPE_INT, BASE_TYPE_NONE },
2254 { "sfixed64", BASE_TYPE_LONG, BASE_TYPE_NONE },
2255 { "bool", BASE_TYPE_BOOL, BASE_TYPE_NONE },
2256 { "string", BASE_TYPE_STRING, BASE_TYPE_NONE },
2257 { "bytes", BASE_TYPE_VECTOR, BASE_TYPE_UCHAR },
2258 { nullptr, BASE_TYPE_NONE, BASE_TYPE_NONE }
2259 };
2260 for (auto tl = lookup; tl->proto_type; tl++) {
2261 if (attribute_ == tl->proto_type) {
2262 type->base_type = tl->fb_type;
2263 type->element = tl->element;
2264 NEXT();
2265 return NoError();
2266 }
2267 }
2268 if (Is('.')) NEXT(); // qualified names may start with a . ?
2269 ECHECK(ParseTypeIdent(*type));
2270 return NoError();
2271 }
2272
SkipAnyJsonValue()2273 CheckedError Parser::SkipAnyJsonValue() {
2274 switch (token_) {
2275 case '{': {
2276 size_t fieldn_outer = 0;
2277 return ParseTableDelimiters(
2278 fieldn_outer, nullptr,
2279 [&](const std::string &, size_t &fieldn,
2280 const StructDef *) -> CheckedError {
2281 ECHECK(Recurse([&]() { return SkipAnyJsonValue(); }));
2282 fieldn++;
2283 return NoError();
2284 });
2285 }
2286 case '[': {
2287 size_t count = 0;
2288 return ParseVectorDelimiters(count, [&](size_t &) -> CheckedError {
2289 return Recurse([&]() { return SkipAnyJsonValue(); });
2290 });
2291 }
2292 case kTokenStringConstant:
2293 case kTokenIntegerConstant:
2294 case kTokenFloatConstant: NEXT(); break;
2295 default:
2296 if (IsIdent("true") || IsIdent("false") || IsIdent("null")) {
2297 NEXT();
2298 } else
2299 return TokenError();
2300 }
2301 return NoError();
2302 }
2303
ParseFlexBufferValue(flexbuffers::Builder * builder)2304 CheckedError Parser::ParseFlexBufferValue(flexbuffers::Builder *builder) {
2305 switch (token_) {
2306 case '{': {
2307 auto start = builder->StartMap();
2308 size_t fieldn_outer = 0;
2309 auto err =
2310 ParseTableDelimiters(fieldn_outer, nullptr,
2311 [&](const std::string &name, size_t &fieldn,
2312 const StructDef *) -> CheckedError {
2313 builder->Key(name);
2314 ECHECK(ParseFlexBufferValue(builder));
2315 fieldn++;
2316 return NoError();
2317 });
2318 ECHECK(err);
2319 builder->EndMap(start);
2320 break;
2321 }
2322 case '[': {
2323 auto start = builder->StartVector();
2324 size_t count = 0;
2325 ECHECK(ParseVectorDelimiters(count, [&](size_t &) -> CheckedError {
2326 return ParseFlexBufferValue(builder);
2327 }));
2328 builder->EndVector(start, false, false);
2329 break;
2330 }
2331 case kTokenStringConstant:
2332 builder->String(attribute_);
2333 EXPECT(kTokenStringConstant);
2334 break;
2335 case kTokenIntegerConstant:
2336 builder->Int(StringToInt(attribute_.c_str()));
2337 EXPECT(kTokenIntegerConstant);
2338 break;
2339 case kTokenFloatConstant:
2340 builder->Double(strtod(attribute_.c_str(), nullptr));
2341 EXPECT(kTokenFloatConstant);
2342 break;
2343 default:
2344 if (IsIdent("true")) {
2345 builder->Bool(true);
2346 NEXT();
2347 } else if (IsIdent("false")) {
2348 builder->Bool(false);
2349 NEXT();
2350 } else if (IsIdent("null")) {
2351 builder->Null();
2352 NEXT();
2353 } else
2354 return TokenError();
2355 }
2356 return NoError();
2357 }
2358
ParseFlexBuffer(const char * source,const char * source_filename,flexbuffers::Builder * builder)2359 bool Parser::ParseFlexBuffer(const char *source, const char *source_filename,
2360 flexbuffers::Builder *builder) {
2361 auto ok = !StartParseFile(source, source_filename).Check() &&
2362 !ParseFlexBufferValue(builder).Check();
2363 if (ok) builder->Finish();
2364 return ok;
2365 }
2366
Parse(const char * source,const char ** include_paths,const char * source_filename)2367 bool Parser::Parse(const char *source, const char **include_paths,
2368 const char *source_filename) {
2369 FLATBUFFERS_ASSERT(0 == recurse_protection_counter);
2370 auto r = !ParseRoot(source, include_paths, source_filename).Check();
2371 FLATBUFFERS_ASSERT(0 == recurse_protection_counter);
2372 return r;
2373 }
2374
StartParseFile(const char * source,const char * source_filename)2375 CheckedError Parser::StartParseFile(const char *source,
2376 const char *source_filename) {
2377 file_being_parsed_ = source_filename ? source_filename : "";
2378 source_ = source;
2379 ResetState(source_);
2380 error_.clear();
2381 ECHECK(SkipByteOrderMark());
2382 NEXT();
2383 if (Is(kTokenEof)) return Error("input file is empty");
2384 return NoError();
2385 }
2386
ParseRoot(const char * source,const char ** include_paths,const char * source_filename)2387 CheckedError Parser::ParseRoot(const char *source, const char **include_paths,
2388 const char *source_filename) {
2389 ECHECK(DoParse(source, include_paths, source_filename, nullptr));
2390
2391 // Check that all types were defined.
2392 for (auto it = structs_.vec.begin(); it != structs_.vec.end();) {
2393 auto &struct_def = **it;
2394 if (struct_def.predecl) {
2395 if (opts.proto_mode) {
2396 // Protos allow enums to be used before declaration, so check if that
2397 // is the case here.
2398 EnumDef *enum_def = nullptr;
2399 for (size_t components =
2400 struct_def.defined_namespace->components.size() + 1;
2401 components && !enum_def; components--) {
2402 auto qualified_name =
2403 struct_def.defined_namespace->GetFullyQualifiedName(
2404 struct_def.name, components - 1);
2405 enum_def = LookupEnum(qualified_name);
2406 }
2407 if (enum_def) {
2408 // This is pretty slow, but a simple solution for now.
2409 auto initial_count = struct_def.refcount;
2410 for (auto struct_it = structs_.vec.begin();
2411 struct_it != structs_.vec.end(); ++struct_it) {
2412 auto &sd = **struct_it;
2413 for (auto field_it = sd.fields.vec.begin();
2414 field_it != sd.fields.vec.end(); ++field_it) {
2415 auto &field = **field_it;
2416 if (field.value.type.struct_def == &struct_def) {
2417 field.value.type.struct_def = nullptr;
2418 field.value.type.enum_def = enum_def;
2419 auto &bt = field.value.type.base_type == BASE_TYPE_VECTOR
2420 ? field.value.type.element
2421 : field.value.type.base_type;
2422 FLATBUFFERS_ASSERT(bt == BASE_TYPE_STRUCT);
2423 bt = enum_def->underlying_type.base_type;
2424 struct_def.refcount--;
2425 enum_def->refcount++;
2426 }
2427 }
2428 }
2429 if (struct_def.refcount)
2430 return Error("internal: " + NumToString(struct_def.refcount) + "/" +
2431 NumToString(initial_count) +
2432 " use(s) of pre-declaration enum not accounted for: " +
2433 enum_def->name);
2434 structs_.dict.erase(structs_.dict.find(struct_def.name));
2435 it = structs_.vec.erase(it);
2436 delete &struct_def;
2437 continue; // Skip error.
2438 }
2439 }
2440 auto err = "type referenced but not defined (check namespace): " +
2441 struct_def.name;
2442 if (struct_def.original_location)
2443 err += ", originally at: " + *struct_def.original_location;
2444 return Error(err);
2445 }
2446 ++it;
2447 }
2448
2449 // This check has to happen here and not earlier, because only now do we
2450 // know for sure what the type of these are.
2451 for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) {
2452 auto &enum_def = **it;
2453 if (enum_def.is_union) {
2454 for (auto val_it = enum_def.vals.vec.begin();
2455 val_it != enum_def.vals.vec.end(); ++val_it) {
2456 auto &val = **val_it;
2457 if (!SupportsVectorOfUnions() && val.union_type.struct_def &&
2458 val.union_type.struct_def->fixed)
2459 return Error(
2460 "only tables can be union elements in the generated language: " +
2461 val.name);
2462 }
2463 }
2464 }
2465 return NoError();
2466 }
2467
DoParse(const char * source,const char ** include_paths,const char * source_filename,const char * include_filename)2468 CheckedError Parser::DoParse(const char *source, const char **include_paths,
2469 const char *source_filename,
2470 const char *include_filename) {
2471 if (source_filename) {
2472 if (included_files_.find(source_filename) == included_files_.end()) {
2473 included_files_[source_filename] =
2474 include_filename ? include_filename : "";
2475 files_included_per_file_[source_filename] = std::set<std::string>();
2476 } else {
2477 return NoError();
2478 }
2479 }
2480 if (!include_paths) {
2481 static const char *current_directory[] = { "", nullptr };
2482 include_paths = current_directory;
2483 }
2484 field_stack_.clear();
2485 builder_.Clear();
2486 // Start with a blank namespace just in case this file doesn't have one.
2487 current_namespace_ = empty_namespace_;
2488
2489 ECHECK(StartParseFile(source, source_filename));
2490
2491 // Includes must come before type declarations:
2492 for (;;) {
2493 // Parse pre-include proto statements if any:
2494 if (opts.proto_mode && (attribute_ == "option" || attribute_ == "syntax" ||
2495 attribute_ == "package")) {
2496 ECHECK(ParseProtoDecl());
2497 } else if (IsIdent("native_include")) {
2498 NEXT();
2499 vector_emplace_back(&native_included_files_, attribute_);
2500 EXPECT(kTokenStringConstant);
2501 EXPECT(';');
2502 } else if (IsIdent("include") || (opts.proto_mode && IsIdent("import"))) {
2503 NEXT();
2504 if (opts.proto_mode && attribute_ == "public") NEXT();
2505 auto name = flatbuffers::PosixPath(attribute_.c_str());
2506 EXPECT(kTokenStringConstant);
2507 // Look for the file in include_paths.
2508 std::string filepath;
2509 for (auto paths = include_paths; paths && *paths; paths++) {
2510 filepath = flatbuffers::ConCatPathFileName(*paths, name);
2511 if (FileExists(filepath.c_str())) break;
2512 }
2513 if (filepath.empty())
2514 return Error("unable to locate include file: " + name);
2515 if (source_filename)
2516 files_included_per_file_[source_filename].insert(filepath);
2517 if (included_files_.find(filepath) == included_files_.end()) {
2518 // We found an include file that we have not parsed yet.
2519 // Load it and parse it.
2520 std::string contents;
2521 if (!LoadFile(filepath.c_str(), true, &contents))
2522 return Error("unable to load include file: " + name);
2523 ECHECK(DoParse(contents.c_str(), include_paths, filepath.c_str(),
2524 name.c_str()));
2525 // We generally do not want to output code for any included files:
2526 if (!opts.generate_all) MarkGenerated();
2527 // Reset these just in case the included file had them, and the
2528 // parent doesn't.
2529 root_struct_def_ = nullptr;
2530 file_identifier_.clear();
2531 file_extension_.clear();
2532 // This is the easiest way to continue this file after an include:
2533 // instead of saving and restoring all the state, we simply start the
2534 // file anew. This will cause it to encounter the same include
2535 // statement again, but this time it will skip it, because it was
2536 // entered into included_files_.
2537 // This is recursive, but only go as deep as the number of include
2538 // statements.
2539 if (source_filename) {
2540 included_files_.erase(source_filename);
2541 }
2542 return DoParse(source, include_paths, source_filename,
2543 include_filename);
2544 }
2545 EXPECT(';');
2546 } else {
2547 break;
2548 }
2549 }
2550 // Now parse all other kinds of declarations:
2551 while (token_ != kTokenEof) {
2552 if (opts.proto_mode) {
2553 ECHECK(ParseProtoDecl());
2554 } else if (IsIdent("namespace")) {
2555 ECHECK(ParseNamespace());
2556 } else if (token_ == '{') {
2557 if (!root_struct_def_)
2558 return Error("no root type set to parse json with");
2559 if (builder_.GetSize()) {
2560 return Error("cannot have more than one json object in a file");
2561 }
2562 uoffset_t toff;
2563 ECHECK(ParseTable(*root_struct_def_, nullptr, &toff));
2564 if (opts.size_prefixed) {
2565 builder_.FinishSizePrefixed(Offset<Table>(toff), file_identifier_.length()
2566 ? file_identifier_.c_str()
2567 : nullptr);
2568 } else {
2569 builder_.Finish(Offset<Table>(toff), file_identifier_.length()
2570 ? file_identifier_.c_str()
2571 : nullptr);
2572 }
2573 // Check that JSON file doesn't contain more objects or IDL directives.
2574 // Comments after JSON are allowed.
2575 EXPECT(kTokenEof);
2576 } else if (IsIdent("enum")) {
2577 ECHECK(ParseEnum(false, nullptr));
2578 } else if (IsIdent("union")) {
2579 ECHECK(ParseEnum(true, nullptr));
2580 } else if (IsIdent("root_type")) {
2581 NEXT();
2582 auto root_type = attribute_;
2583 EXPECT(kTokenIdentifier);
2584 ECHECK(ParseNamespacing(&root_type, nullptr));
2585 if (opts.root_type.empty()) {
2586 if (!SetRootType(root_type.c_str()))
2587 return Error("unknown root type: " + root_type);
2588 if (root_struct_def_->fixed)
2589 return Error("root type must be a table");
2590 }
2591 EXPECT(';');
2592 } else if (IsIdent("file_identifier")) {
2593 NEXT();
2594 file_identifier_ = attribute_;
2595 EXPECT(kTokenStringConstant);
2596 if (file_identifier_.length() != FlatBufferBuilder::kFileIdentifierLength)
2597 return Error("file_identifier must be exactly " +
2598 NumToString(FlatBufferBuilder::kFileIdentifierLength) +
2599 " characters");
2600 EXPECT(';');
2601 } else if (IsIdent("file_extension")) {
2602 NEXT();
2603 file_extension_ = attribute_;
2604 EXPECT(kTokenStringConstant);
2605 EXPECT(';');
2606 } else if (IsIdent("include")) {
2607 return Error("includes must come before declarations");
2608 } else if (IsIdent("attribute")) {
2609 NEXT();
2610 auto name = attribute_;
2611 if (Is(kTokenIdentifier)) {
2612 NEXT();
2613 } else {
2614 EXPECT(kTokenStringConstant);
2615 }
2616 EXPECT(';');
2617 known_attributes_[name] = false;
2618 } else if (IsIdent("rpc_service")) {
2619 ECHECK(ParseService());
2620 } else {
2621 ECHECK(ParseDecl());
2622 }
2623 }
2624 return NoError();
2625 }
2626
GetIncludedFilesRecursive(const std::string & file_name) const2627 std::set<std::string> Parser::GetIncludedFilesRecursive(
2628 const std::string &file_name) const {
2629 std::set<std::string> included_files;
2630 std::list<std::string> to_process;
2631
2632 if (file_name.empty()) return included_files;
2633 to_process.push_back(file_name);
2634
2635 while (!to_process.empty()) {
2636 std::string current = to_process.front();
2637 to_process.pop_front();
2638 included_files.insert(current);
2639
2640 // Workaround the lack of const accessor in C++98 maps.
2641 auto &new_files =
2642 (*const_cast<std::map<std::string, std::set<std::string>> *>(
2643 &files_included_per_file_))[current];
2644 for (auto it = new_files.begin(); it != new_files.end(); ++it) {
2645 if (included_files.find(*it) == included_files.end())
2646 to_process.push_back(*it);
2647 }
2648 }
2649
2650 return included_files;
2651 }
2652
2653 // Schema serialization functionality:
2654
compareName(const T * a,const T * b)2655 template<typename T> bool compareName(const T *a, const T *b) {
2656 return a->defined_namespace->GetFullyQualifiedName(a->name) <
2657 b->defined_namespace->GetFullyQualifiedName(b->name);
2658 }
2659
AssignIndices(const std::vector<T * > & defvec)2660 template<typename T> void AssignIndices(const std::vector<T *> &defvec) {
2661 // Pre-sort these vectors, such that we can set the correct indices for them.
2662 auto vec = defvec;
2663 std::sort(vec.begin(), vec.end(), compareName<T>);
2664 for (int i = 0; i < static_cast<int>(vec.size()); i++) vec[i]->index = i;
2665 }
2666
Serialize()2667 void Parser::Serialize() {
2668 builder_.Clear();
2669 AssignIndices(structs_.vec);
2670 AssignIndices(enums_.vec);
2671 std::vector<Offset<reflection::Object>> object_offsets;
2672 for (auto it = structs_.vec.begin(); it != structs_.vec.end(); ++it) {
2673 auto offset = (*it)->Serialize(&builder_, *this);
2674 object_offsets.push_back(offset);
2675 (*it)->serialized_location = offset.o;
2676 }
2677 std::vector<Offset<reflection::Enum>> enum_offsets;
2678 for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) {
2679 auto offset = (*it)->Serialize(&builder_, *this);
2680 enum_offsets.push_back(offset);
2681 (*it)->serialized_location = offset.o;
2682 }
2683 std::vector<Offset<reflection::Service>> service_offsets;
2684 for (auto it = services_.vec.begin(); it != services_.vec.end(); ++it) {
2685 auto offset = (*it)->Serialize(&builder_, *this);
2686 service_offsets.push_back(offset);
2687 (*it)->serialized_location = offset.o;
2688 }
2689 auto objs__ = builder_.CreateVectorOfSortedTables(&object_offsets);
2690 auto enum__ = builder_.CreateVectorOfSortedTables(&enum_offsets);
2691 auto fiid__ = builder_.CreateString(file_identifier_);
2692 auto fext__ = builder_.CreateString(file_extension_);
2693 auto serv__ = builder_.CreateVectorOfSortedTables(&service_offsets);
2694 auto schema_offset =
2695 reflection::CreateSchema(builder_, objs__, enum__, fiid__, fext__,
2696 (root_struct_def_ ? root_struct_def_->serialized_location : 0),
2697 serv__);
2698 if (opts.size_prefixed) {
2699 builder_.FinishSizePrefixed(schema_offset, reflection::SchemaIdentifier());
2700 } else {
2701 builder_.Finish(schema_offset, reflection::SchemaIdentifier());
2702 }
2703 }
2704
GetNamespace(const std::string & qualified_name,std::vector<Namespace * > & namespaces,std::map<std::string,Namespace * > & namespaces_index)2705 static Namespace *GetNamespace(
2706 const std::string &qualified_name, std::vector<Namespace *> &namespaces,
2707 std::map<std::string, Namespace *> &namespaces_index) {
2708 size_t dot = qualified_name.find_last_of('.');
2709 std::string namespace_name = (dot != std::string::npos)
2710 ? std::string(qualified_name.c_str(), dot)
2711 : "";
2712 Namespace *&ns = namespaces_index[namespace_name];
2713
2714 if (!ns) {
2715 ns = new Namespace();
2716 namespaces.push_back(ns);
2717
2718 size_t pos = 0;
2719
2720 for (;;) {
2721 dot = qualified_name.find('.', pos);
2722 if (dot == std::string::npos) { break; }
2723 ns->components.push_back(qualified_name.substr(pos, dot-pos));
2724 pos = dot + 1;
2725 }
2726 }
2727
2728 return ns;
2729 }
2730
Serialize(FlatBufferBuilder * builder,const Parser & parser) const2731 Offset<reflection::Object> StructDef::Serialize(FlatBufferBuilder *builder,
2732 const Parser &parser) const {
2733 std::vector<Offset<reflection::Field>> field_offsets;
2734 for (auto it = fields.vec.begin(); it != fields.vec.end(); ++it) {
2735 field_offsets.push_back((*it)->Serialize(
2736 builder, static_cast<uint16_t>(it - fields.vec.begin()), parser));
2737 }
2738 auto qualified_name = defined_namespace->GetFullyQualifiedName(name);
2739 auto name__ = builder->CreateString(qualified_name);
2740 auto flds__ = builder->CreateVectorOfSortedTables(&field_offsets);
2741 auto attr__ = SerializeAttributes(builder, parser);
2742 auto docs__ = parser.opts.binary_schema_comments
2743 ? builder->CreateVectorOfStrings(doc_comment)
2744 : 0;
2745 return reflection::CreateObject(*builder, name__, flds__, fixed,
2746 static_cast<int>(minalign),
2747 static_cast<int>(bytesize),
2748 attr__, docs__);
2749 }
2750
Deserialize(Parser & parser,const reflection::Object * object)2751 bool StructDef::Deserialize(Parser &parser, const reflection::Object *object) {
2752 if (!DeserializeAttributes(parser, object->attributes()))
2753 return false;
2754 DeserializeDoc(doc_comment, object->documentation());
2755 name = parser.UnqualifiedName(object->name()->str());
2756 fixed = object->is_struct();
2757 minalign = object->minalign();
2758 predecl = false;
2759 sortbysize = attributes.Lookup("original_order") == nullptr && !fixed;
2760 std::vector<uoffset_t> indexes =
2761 std::vector<uoffset_t>(object->fields()->size());
2762 for (uoffset_t i = 0; i < object->fields()->size(); i++)
2763 indexes[object->fields()->Get(i)->id()] = i;
2764 for (size_t i = 0; i < indexes.size(); i++) {
2765 auto field = object->fields()->Get(indexes[i]);
2766 auto field_def = new FieldDef();
2767 if (!field_def->Deserialize(parser, field) ||
2768 fields.Add(field_def->name, field_def)) {
2769 delete field_def;
2770 return false;
2771 }
2772 if (fixed) {
2773 // Recompute padding since that's currently not serialized.
2774 auto size = InlineSize(field_def->value.type);
2775 auto next_field =
2776 i + 1 < indexes.size()
2777 ? object->fields()->Get(indexes[i+1])
2778 : nullptr;
2779 bytesize += size;
2780 field_def->padding =
2781 next_field ? (next_field->offset() - field_def->value.offset) - size
2782 : PaddingBytes(bytesize, minalign);
2783 bytesize += field_def->padding;
2784 }
2785 }
2786 FLATBUFFERS_ASSERT(static_cast<int>(bytesize) == object->bytesize());
2787 return true;
2788 }
2789
Serialize(FlatBufferBuilder * builder,uint16_t id,const Parser & parser) const2790 Offset<reflection::Field> FieldDef::Serialize(FlatBufferBuilder *builder,
2791 uint16_t id,
2792 const Parser &parser) const {
2793 auto name__ = builder->CreateString(name);
2794 auto type__ = value.type.Serialize(builder);
2795 auto attr__ = SerializeAttributes(builder, parser);
2796 auto docs__ = parser.opts.binary_schema_comments
2797 ? builder->CreateVectorOfStrings(doc_comment)
2798 : 0;
2799 return reflection::CreateField(*builder, name__, type__, id, value.offset,
2800 // Is uint64>max(int64) tested?
2801 IsInteger(value.type.base_type) ? StringToInt(value.constant.c_str()) : 0,
2802 // result may be platform-dependent if underlying is float (not double)
2803 IsFloat(value.type.base_type) ? strtod(value.constant.c_str(), nullptr)
2804 : 0.0,
2805 deprecated, required, key, attr__, docs__);
2806 // TODO: value.constant is almost always "0", we could save quite a bit of
2807 // space by sharing it. Same for common values of value.type.
2808 }
2809
Deserialize(Parser & parser,const reflection::Field * field)2810 bool FieldDef::Deserialize(Parser &parser, const reflection::Field *field) {
2811 name = parser.UnqualifiedName(field->name()->str());
2812 defined_namespace = parser.current_namespace_;
2813 if (!value.type.Deserialize(parser, field->type()))
2814 return false;
2815 value.offset = field->offset();
2816 if (IsInteger(value.type.base_type)) {
2817 value.constant = NumToString(field->default_integer());
2818 } else if (IsFloat(value.type.base_type)) {
2819 value.constant = FloatToString(field->default_real(), 16);
2820 size_t last_zero = value.constant.find_last_not_of('0');
2821 if (last_zero != std::string::npos && last_zero != 0) {
2822 value.constant.erase(last_zero, std::string::npos);
2823 }
2824 }
2825 deprecated = field->deprecated();
2826 required = field->required();
2827 key = field->key();
2828 if (!DeserializeAttributes(parser, field->attributes()))
2829 return false;
2830 // TODO: this should probably be handled by a separate attribute
2831 if (attributes.Lookup("flexbuffer")) {
2832 flexbuffer = true;
2833 parser.uses_flexbuffers_ = true;
2834 if (value.type.base_type != BASE_TYPE_VECTOR ||
2835 value.type.element != BASE_TYPE_UCHAR)
2836 return false;
2837 }
2838 DeserializeDoc(doc_comment, field->documentation());
2839 return true;
2840 }
2841
Serialize(FlatBufferBuilder * builder,const Parser & parser) const2842 Offset<reflection::RPCCall> RPCCall::Serialize(FlatBufferBuilder *builder,
2843 const Parser &parser) const {
2844 auto name__ = builder->CreateString(name);
2845 auto attr__ = SerializeAttributes(builder, parser);
2846 auto docs__ = parser.opts.binary_schema_comments
2847 ? builder->CreateVectorOfStrings(doc_comment)
2848 : 0;
2849 return reflection::CreateRPCCall(*builder, name__,
2850 request->serialized_location,
2851 response->serialized_location,
2852 attr__, docs__);
2853 }
2854
Deserialize(Parser & parser,const reflection::RPCCall * call)2855 bool RPCCall::Deserialize(Parser &parser, const reflection::RPCCall *call) {
2856 name = call->name()->str();
2857 if (!DeserializeAttributes(parser, call->attributes()))
2858 return false;
2859 DeserializeDoc(doc_comment, call->documentation());
2860 request = parser.structs_.Lookup(call->request()->name()->str());
2861 response = parser.structs_.Lookup(call->response()->name()->str());
2862 if (!request || !response) { return false; }
2863 return true;
2864 }
2865
Serialize(FlatBufferBuilder * builder,const Parser & parser) const2866 Offset<reflection::Service> ServiceDef::Serialize(FlatBufferBuilder *builder,
2867 const Parser &parser) const {
2868 std::vector<Offset<reflection::RPCCall>> servicecall_offsets;
2869 for (auto it = calls.vec.begin(); it != calls.vec.end(); ++it) {
2870 servicecall_offsets.push_back((*it)->Serialize(builder, parser));
2871 }
2872 auto qualified_name = defined_namespace->GetFullyQualifiedName(name);
2873 auto name__ = builder->CreateString(qualified_name);
2874 auto call__ = builder->CreateVector(servicecall_offsets);
2875 auto attr__ = SerializeAttributes(builder, parser);
2876 auto docs__ = parser.opts.binary_schema_comments
2877 ? builder->CreateVectorOfStrings(doc_comment)
2878 : 0;
2879 return reflection::CreateService(*builder, name__, call__, attr__, docs__);
2880 }
2881
Deserialize(Parser & parser,const reflection::Service * service)2882 bool ServiceDef::Deserialize(Parser &parser,
2883 const reflection::Service *service) {
2884 name = parser.UnqualifiedName(service->name()->str());
2885 if (service->calls()) {
2886 for (uoffset_t i = 0; i < service->calls()->size(); ++i) {
2887 auto call = new RPCCall();
2888 if (!call->Deserialize(parser, service->calls()->Get(i)) ||
2889 calls.Add(call->name, call)) {
2890 delete call;
2891 return false;
2892 }
2893 }
2894 }
2895 if (!DeserializeAttributes(parser, service->attributes()))
2896 return false;
2897 DeserializeDoc(doc_comment, service->documentation());
2898 return true;
2899 }
2900
Serialize(FlatBufferBuilder * builder,const Parser & parser) const2901 Offset<reflection::Enum> EnumDef::Serialize(FlatBufferBuilder *builder,
2902 const Parser &parser) const {
2903 std::vector<Offset<reflection::EnumVal>> enumval_offsets;
2904 for (auto it = vals.vec.begin(); it != vals.vec.end(); ++it) {
2905 enumval_offsets.push_back((*it)->Serialize(builder, parser));
2906 }
2907 auto qualified_name = defined_namespace->GetFullyQualifiedName(name);
2908 auto name__ = builder->CreateString(qualified_name);
2909 auto vals__ = builder->CreateVector(enumval_offsets);
2910 auto type__ = underlying_type.Serialize(builder);
2911 auto attr__ = SerializeAttributes(builder, parser);
2912 auto docs__ = parser.opts.binary_schema_comments
2913 ? builder->CreateVectorOfStrings(doc_comment)
2914 : 0;
2915 return reflection::CreateEnum(*builder, name__, vals__, is_union, type__,
2916 attr__, docs__);
2917 }
2918
Deserialize(Parser & parser,const reflection::Enum * _enum)2919 bool EnumDef::Deserialize(Parser &parser, const reflection::Enum *_enum) {
2920 name = parser.UnqualifiedName(_enum->name()->str());
2921 for (uoffset_t i = 0; i < _enum->values()->size(); ++i) {
2922 auto val = new EnumVal();
2923 if (!val->Deserialize(parser, _enum->values()->Get(i)) ||
2924 vals.Add(val->name, val)) {
2925 delete val;
2926 return false;
2927 }
2928 }
2929 is_union = _enum->is_union();
2930 if (!underlying_type.Deserialize(parser, _enum->underlying_type())) {
2931 return false;
2932 }
2933 if (!DeserializeAttributes(parser, _enum->attributes()))
2934 return false;
2935 DeserializeDoc(doc_comment, _enum->documentation());
2936 return true;
2937 }
2938
Serialize(FlatBufferBuilder * builder,const Parser & parser) const2939 Offset<reflection::EnumVal> EnumVal::Serialize(FlatBufferBuilder *builder,
2940 const Parser &parser) const {
2941 auto name__ = builder->CreateString(name);
2942 auto type__ = union_type.Serialize(builder);
2943 auto docs__ = parser.opts.binary_schema_comments
2944 ? builder->CreateVectorOfStrings(doc_comment)
2945 : 0;
2946 return reflection::CreateEnumVal(*builder, name__, value,
2947 union_type.struct_def ? union_type.struct_def->serialized_location : 0,
2948 type__, docs__);
2949 }
2950
Deserialize(const Parser & parser,const reflection::EnumVal * val)2951 bool EnumVal::Deserialize(const Parser &parser,
2952 const reflection::EnumVal *val) {
2953 name = val->name()->str();
2954 value = val->value();
2955 if (!union_type.Deserialize(parser, val->union_type()))
2956 return false;
2957 DeserializeDoc(doc_comment, val->documentation());
2958 return true;
2959 }
2960
Serialize(FlatBufferBuilder * builder) const2961 Offset<reflection::Type> Type::Serialize(FlatBufferBuilder *builder) const {
2962 return reflection::CreateType(
2963 *builder,
2964 static_cast<reflection::BaseType>(base_type),
2965 static_cast<reflection::BaseType>(element),
2966 struct_def ? struct_def->index : (enum_def ? enum_def->index : -1));
2967 }
2968
Deserialize(const Parser & parser,const reflection::Type * type)2969 bool Type::Deserialize(const Parser &parser, const reflection::Type *type) {
2970 if (type == nullptr) return true;
2971 base_type = static_cast<BaseType>(type->base_type());
2972 element = static_cast<BaseType>(type->element());
2973 if (type->index() >= 0) {
2974 if (type->base_type() == reflection::Obj ||
2975 (type->base_type() == reflection::Vector &&
2976 type->element() == reflection::Obj)) {
2977 if (static_cast<size_t>(type->index()) < parser.structs_.vec.size()) {
2978 struct_def = parser.structs_.vec[type->index()];
2979 struct_def->refcount++;
2980 } else {
2981 return false;
2982 }
2983 } else {
2984 if (static_cast<size_t>(type->index()) < parser.enums_.vec.size()) {
2985 enum_def = parser.enums_.vec[type->index()];
2986 } else {
2987 return false;
2988 }
2989 }
2990 }
2991 return true;
2992 }
2993
2994 flatbuffers::Offset<
2995 flatbuffers::Vector<flatbuffers::Offset<reflection::KeyValue>>>
SerializeAttributes(FlatBufferBuilder * builder,const Parser & parser) const2996 Definition::SerializeAttributes(FlatBufferBuilder *builder,
2997 const Parser &parser) const {
2998 std::vector<flatbuffers::Offset<reflection::KeyValue>> attrs;
2999 for (auto kv = attributes.dict.begin(); kv != attributes.dict.end(); ++kv) {
3000 auto it = parser.known_attributes_.find(kv->first);
3001 FLATBUFFERS_ASSERT(it != parser.known_attributes_.end());
3002 if (parser.opts.binary_schema_builtins || !it->second) {
3003 auto key = builder->CreateString(kv->first);
3004 auto val = builder->CreateString(kv->second->constant);
3005 attrs.push_back(reflection::CreateKeyValue(*builder, key, val));
3006 }
3007 }
3008 if (attrs.size()) {
3009 return builder->CreateVectorOfSortedTables(&attrs);
3010 } else {
3011 return 0;
3012 }
3013 }
3014
DeserializeAttributes(Parser & parser,const Vector<Offset<reflection::KeyValue>> * attrs)3015 bool Definition::DeserializeAttributes(
3016 Parser &parser, const Vector<Offset<reflection::KeyValue>> *attrs) {
3017 if (attrs == nullptr)
3018 return true;
3019 for (uoffset_t i = 0; i < attrs->size(); ++i) {
3020 auto kv = attrs->Get(i);
3021 auto value = new Value();
3022 if (kv->value()) { value->constant = kv->value()->str(); }
3023 if (attributes.Add(kv->key()->str(), value)) {
3024 delete value;
3025 return false;
3026 }
3027 parser.known_attributes_[kv->key()->str()];
3028 }
3029 return true;
3030 }
3031
3032 /************************************************************************/
3033 /* DESERIALIZATION */
3034 /************************************************************************/
Deserialize(const uint8_t * buf,const size_t size)3035 bool Parser::Deserialize(const uint8_t *buf, const size_t size) {
3036 flatbuffers::Verifier verifier(reinterpret_cast<const uint8_t *>(buf), size);
3037 bool size_prefixed = false;
3038 if(!reflection::SchemaBufferHasIdentifier(buf)) {
3039 if (!flatbuffers::BufferHasIdentifier(buf, reflection::SchemaIdentifier(),
3040 true))
3041 return false;
3042 else
3043 size_prefixed = true;
3044 }
3045 auto verify_fn = size_prefixed ? &reflection::VerifySizePrefixedSchemaBuffer
3046 : &reflection::VerifySchemaBuffer;
3047 if (!verify_fn(verifier)) {
3048 return false;
3049 }
3050 auto schema = size_prefixed ? reflection::GetSizePrefixedSchema(buf)
3051 : reflection::GetSchema(buf);
3052 return Deserialize(schema);
3053 }
3054
Deserialize(const reflection::Schema * schema)3055 bool Parser::Deserialize(const reflection::Schema *schema) {
3056 file_identifier_ = schema->file_ident() ? schema->file_ident()->str() : "";
3057 file_extension_ = schema->file_ext() ? schema->file_ext()->str() : "";
3058 std::map<std::string, Namespace *> namespaces_index;
3059
3060 // Create defs without deserializing so references from fields to structs and
3061 // enums can be resolved.
3062 for (auto it = schema->objects()->begin(); it != schema->objects()->end();
3063 ++it) {
3064 auto struct_def = new StructDef();
3065 if (structs_.Add(it->name()->str(), struct_def)) {
3066 delete struct_def;
3067 return false;
3068 }
3069 auto type = new Type(BASE_TYPE_STRUCT, struct_def, nullptr);
3070 if (types_.Add(it->name()->str(), type)) {
3071 delete type;
3072 return false;
3073 }
3074 }
3075 for (auto it = schema->enums()->begin(); it != schema->enums()->end(); ++it) {
3076 auto enum_def = new EnumDef();
3077 if (enums_.Add(it->name()->str(), enum_def)) {
3078 delete enum_def;
3079 return false;
3080 }
3081 auto type = new Type(BASE_TYPE_UNION, nullptr, enum_def);
3082 if (types_.Add(it->name()->str(), type)) {
3083 delete type;
3084 return false;
3085 }
3086 }
3087
3088 // Now fields can refer to structs and enums by index.
3089 for (auto it = schema->objects()->begin(); it != schema->objects()->end();
3090 ++it) {
3091 std::string qualified_name = it->name()->str();
3092 auto struct_def = structs_.Lookup(qualified_name);
3093 struct_def->defined_namespace =
3094 GetNamespace(qualified_name, namespaces_, namespaces_index);
3095 if (!struct_def->Deserialize(*this, * it)) { return false; }
3096 if (schema->root_table() == *it) { root_struct_def_ = struct_def; }
3097 }
3098 for (auto it = schema->enums()->begin(); it != schema->enums()->end(); ++it) {
3099 std::string qualified_name = it->name()->str();
3100 auto enum_def = enums_.Lookup(qualified_name);
3101 enum_def->defined_namespace =
3102 GetNamespace(qualified_name, namespaces_, namespaces_index);
3103 if (!enum_def->Deserialize(*this, *it)) { return false; }
3104 }
3105
3106 if (schema->services()) {
3107 for (auto it = schema->services()->begin(); it != schema->services()->end();
3108 ++it) {
3109 std::string qualified_name = it->name()->str();
3110 auto service_def = new ServiceDef();
3111 service_def->defined_namespace =
3112 GetNamespace(qualified_name, namespaces_, namespaces_index);
3113 if (!service_def->Deserialize(*this, *it) ||
3114 services_.Add(qualified_name, service_def)) {
3115 delete service_def;
3116 return false;
3117 }
3118 }
3119 }
3120
3121 return true;
3122 }
3123
ConformTo(const Parser & base)3124 std::string Parser::ConformTo(const Parser &base) {
3125 for (auto sit = structs_.vec.begin(); sit != structs_.vec.end(); ++sit) {
3126 auto &struct_def = **sit;
3127 auto qualified_name =
3128 struct_def.defined_namespace->GetFullyQualifiedName(struct_def.name);
3129 auto struct_def_base = base.LookupStruct(qualified_name);
3130 if (!struct_def_base) continue;
3131 for (auto fit = struct_def.fields.vec.begin();
3132 fit != struct_def.fields.vec.end(); ++fit) {
3133 auto &field = **fit;
3134 auto field_base = struct_def_base->fields.Lookup(field.name);
3135 if (field_base) {
3136 if (field.value.offset != field_base->value.offset)
3137 return "offsets differ for field: " + field.name;
3138 if (field.value.constant != field_base->value.constant)
3139 return "defaults differ for field: " + field.name;
3140 if (!EqualByName(field.value.type, field_base->value.type))
3141 return "types differ for field: " + field.name;
3142 } else {
3143 // Doesn't have to exist, deleting fields is fine.
3144 // But we should check if there is a field that has the same offset
3145 // but is incompatible (in the case of field renaming).
3146 for (auto fbit = struct_def_base->fields.vec.begin();
3147 fbit != struct_def_base->fields.vec.end(); ++fbit) {
3148 field_base = *fbit;
3149 if (field.value.offset == field_base->value.offset) {
3150 if (!EqualByName(field.value.type, field_base->value.type))
3151 return "field renamed to different type: " + field.name;
3152 break;
3153 }
3154 }
3155 }
3156 }
3157 }
3158 for (auto eit = enums_.vec.begin(); eit != enums_.vec.end(); ++eit) {
3159 auto &enum_def = **eit;
3160 auto qualified_name =
3161 enum_def.defined_namespace->GetFullyQualifiedName(enum_def.name);
3162 auto enum_def_base = base.enums_.Lookup(qualified_name);
3163 if (!enum_def_base) continue;
3164 for (auto evit = enum_def.vals.vec.begin(); evit != enum_def.vals.vec.end();
3165 ++evit) {
3166 auto &enum_val = **evit;
3167 auto enum_val_base = enum_def_base->vals.Lookup(enum_val.name);
3168 if (enum_val_base) {
3169 if (enum_val.value != enum_val_base->value)
3170 return "values differ for enum: " + enum_val.name;
3171 }
3172 }
3173 }
3174 return "";
3175 }
3176
3177 } // namespace flatbuffers
3178