1 /*
2 * Copyright 2014 Google Inc. All rights reserved.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <algorithm>
18 #include <list>
19 #include <iostream>
20
21 #include <math.h>
22
23 #include "flatbuffers/idl.h"
24 #include "flatbuffers/util.h"
25
26 namespace flatbuffers {
27
28 const double kPi = 3.14159265358979323846;
29
30 const char *const kTypeNames[] = {
31 #define FLATBUFFERS_TD(ENUM, IDLTYPE, \
32 CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \
33 IDLTYPE,
34 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
35 #undef FLATBUFFERS_TD
36 nullptr
37 };
38
39 const char kTypeSizes[] = {
40 #define FLATBUFFERS_TD(ENUM, IDLTYPE, \
41 CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \
42 sizeof(CTYPE),
43 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
44 #undef FLATBUFFERS_TD
45 };
46
47 // The enums in the reflection schema should match the ones we use internally.
48 // Compare the last element to check if these go out of sync.
49 static_assert(BASE_TYPE_UNION ==
50 static_cast<BaseType>(reflection::Union),
51 "enums don't match");
52
53 // Any parsing calls have to be wrapped in this macro, which automates
54 // handling of recursive error checking a bit. It will check the received
55 // CheckedError object, and return straight away on error.
56 #define ECHECK(call) { auto ce = (call); if (ce.Check()) return ce; }
57
58 // These two functions are called hundreds of times below, so define a short
59 // form:
60 #define NEXT() ECHECK(Next())
61 #define EXPECT(tok) ECHECK(Expect(tok))
62
ValidateUTF8(const std::string & str)63 static bool ValidateUTF8(const std::string &str) {
64 const char *s = &str[0];
65 const char * const sEnd = s + str.length();
66 while (s < sEnd) {
67 if (FromUTF8(&s) < 0) {
68 return false;
69 }
70 }
71 return true;
72 }
73
Message(const std::string & msg)74 void Parser::Message(const std::string &msg) {
75 error_ = file_being_parsed_.length() ? AbsolutePath(file_being_parsed_) : "";
76 #ifdef _WIN32
77 error_ += "(" + NumToString(line_) + ")"; // MSVC alike
78 #else
79 if (file_being_parsed_.length()) error_ += ":";
80 error_ += NumToString(line_) + ":0"; // gcc alike
81 #endif
82 error_ += ": " + msg;
83 }
84
Warning(const std::string & msg)85 void Parser::Warning(const std::string &msg) {
86 Message("warning: " + msg);
87 }
88
Error(const std::string & msg)89 CheckedError Parser::Error(const std::string &msg) {
90 Message("error: " + msg);
91 return CheckedError(true);
92 }
93
NoError()94 inline CheckedError NoError() { return CheckedError(false); }
95
OutOfRangeErrorMsg(int64_t val,const std::string & op,int64_t limit)96 inline std::string OutOfRangeErrorMsg(int64_t val, const std::string &op,
97 int64_t limit) {
98 const std::string cause = NumToString(val) + op + NumToString(limit);
99 return "constant does not fit (" + cause + ")";
100 }
101
102 // Ensure that integer values we parse fit inside the declared integer type.
CheckInRange(int64_t val,int64_t min,int64_t max)103 CheckedError Parser::CheckInRange(int64_t val, int64_t min, int64_t max) {
104 if (val < min)
105 return Error(OutOfRangeErrorMsg(val, " < ", min));
106 else if (val > max)
107 return Error(OutOfRangeErrorMsg(val, " > ", max));
108 else
109 return NoError();
110 }
111
112 // atot: templated version of atoi/atof: convert a string to an instance of T.
atot(const char * s,Parser & parser,T * val)113 template<typename T> inline CheckedError atot(const char *s, Parser &parser,
114 T *val) {
115 int64_t i = StringToInt(s);
116 const int64_t min = flatbuffers::numeric_limits<T>::min();
117 const int64_t max = flatbuffers::numeric_limits<T>::max();
118 ECHECK(parser.CheckInRange(i, min, max));
119 *val = (T)i;
120 return NoError();
121 }
atot(const char * s,Parser & parser,uint64_t * val)122 template<> inline CheckedError atot<uint64_t>(const char *s, Parser &parser,
123 uint64_t *val) {
124 (void)parser;
125 *val = StringToUInt(s);
126 return NoError();
127 }
atot(const char * s,Parser & parser,bool * val)128 template<> inline CheckedError atot<bool>(const char *s, Parser &parser,
129 bool *val) {
130 (void)parser;
131 *val = 0 != atoi(s);
132 return NoError();
133 }
atot(const char * s,Parser & parser,float * val)134 template<> inline CheckedError atot<float>(const char *s, Parser &parser,
135 float *val) {
136 (void)parser;
137 *val = static_cast<float>(strtod(s, nullptr));
138 return NoError();
139 }
atot(const char * s,Parser & parser,double * val)140 template<> inline CheckedError atot<double>(const char *s, Parser &parser,
141 double *val) {
142 (void)parser;
143 *val = strtod(s, nullptr);
144 return NoError();
145 }
146
atot(const char * s,Parser & parser,Offset<void> * val)147 template<> inline CheckedError atot<Offset<void>>(const char *s, Parser &parser,
148 Offset<void> *val) {
149 (void)parser;
150 *val = Offset<void>(atoi(s));
151 return NoError();
152 }
153
GetFullyQualifiedName(const std::string & name,size_t max_components) const154 std::string Namespace::GetFullyQualifiedName(const std::string &name,
155 size_t max_components) const {
156 // Early exit if we don't have a defined namespace.
157 if (components.size() == 0 || !max_components) {
158 return name;
159 }
160 std::stringstream stream;
161 for (size_t i = 0; i < std::min(components.size(), max_components);
162 i++) {
163 if (i) {
164 stream << ".";
165 }
166 stream << components[i];
167 }
168 if (name.length()) stream << "." << name;
169 return stream.str();
170 }
171
172 // Declare tokens we'll use. Single character tokens are represented by their
173 // ascii character code (e.g. '{'), others above 256.
174 #define FLATBUFFERS_GEN_TOKENS(TD) \
175 TD(Eof, 256, "end of file") \
176 TD(StringConstant, 257, "string constant") \
177 TD(IntegerConstant, 258, "integer constant") \
178 TD(FloatConstant, 259, "float constant") \
179 TD(Identifier, 260, "identifier")
180 #ifdef __GNUC__
181 __extension__ // Stop GCC complaining about trailing comma with -Wpendantic.
182 #endif
183 enum {
184 #define FLATBUFFERS_TOKEN(NAME, VALUE, STRING) kToken ## NAME = VALUE,
185 FLATBUFFERS_GEN_TOKENS(FLATBUFFERS_TOKEN)
186 #undef FLATBUFFERS_TOKEN
187 };
188
TokenToString(int t)189 static std::string TokenToString(int t) {
190 static const char *tokens[] = {
191 #define FLATBUFFERS_TOKEN(NAME, VALUE, STRING) STRING,
192 FLATBUFFERS_GEN_TOKENS(FLATBUFFERS_TOKEN)
193 #undef FLATBUFFERS_TOKEN
194 #define FLATBUFFERS_TD(ENUM, IDLTYPE, \
195 CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \
196 IDLTYPE,
197 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
198 #undef FLATBUFFERS_TD
199 };
200 if (t < 256) { // A single ascii char token.
201 std::string s;
202 s.append(1, static_cast<char>(t));
203 return s;
204 } else { // Other tokens.
205 return tokens[t - 256];
206 }
207 }
208
TokenToStringId(int t)209 std::string Parser::TokenToStringId(int t) {
210 return t == kTokenIdentifier ? attribute_ : TokenToString(t);
211 }
212
213 // Parses exactly nibbles worth of hex digits into a number, or error.
ParseHexNum(int nibbles,uint64_t * val)214 CheckedError Parser::ParseHexNum(int nibbles, uint64_t *val) {
215 for (int i = 0; i < nibbles; i++)
216 if (!isxdigit(static_cast<const unsigned char>(cursor_[i])))
217 return Error("escape code must be followed by " + NumToString(nibbles) +
218 " hex digits");
219 std::string target(cursor_, cursor_ + nibbles);
220 *val = StringToUInt(target.c_str(), nullptr, 16);
221 cursor_ += nibbles;
222 return NoError();
223 }
224
SkipByteOrderMark()225 CheckedError Parser::SkipByteOrderMark() {
226 if (static_cast<unsigned char>(*cursor_) != 0xef)
227 return NoError();
228 cursor_++;
229 if (static_cast<unsigned char>(*cursor_) != 0xbb)
230 return Error("invalid utf-8 byte order mark");
231 cursor_++;
232 if (static_cast<unsigned char>(*cursor_) != 0xbf)
233 return Error("invalid utf-8 byte order mark");
234 cursor_++;
235 return NoError();
236 }
237
IsIdentifierStart(char c)238 bool IsIdentifierStart(char c) {
239 return isalpha(static_cast<unsigned char>(c)) || c == '_';
240 }
241
Next()242 CheckedError Parser::Next() {
243 doc_comment_.clear();
244 bool seen_newline = false;
245 attribute_.clear();
246 for (;;) {
247 char c = *cursor_++;
248 token_ = c;
249 switch (c) {
250 case '\0': cursor_--; token_ = kTokenEof; return NoError();
251 case ' ': case '\r': case '\t': break;
252 case '\n': line_++; seen_newline = true; break;
253 case '{': case '}': case '(': case ')': case '[': case ']':
254 case ',': case ':': case ';': case '=': return NoError();
255 case '.':
256 if(!isdigit(static_cast<const unsigned char>(*cursor_))) return NoError();
257 return Error("floating point constant can\'t start with \".\"");
258 case '\"':
259 case '\'': {
260 int unicode_high_surrogate = -1;
261
262 while (*cursor_ != c) {
263 if (*cursor_ < ' ' && *cursor_ >= 0)
264 return Error("illegal character in string constant");
265 if (*cursor_ == '\\') {
266 cursor_++;
267 if (unicode_high_surrogate != -1 &&
268 *cursor_ != 'u') {
269 return Error(
270 "illegal Unicode sequence (unpaired high surrogate)");
271 }
272 switch (*cursor_) {
273 case 'n': attribute_ += '\n'; cursor_++; break;
274 case 't': attribute_ += '\t'; cursor_++; break;
275 case 'r': attribute_ += '\r'; cursor_++; break;
276 case 'b': attribute_ += '\b'; cursor_++; break;
277 case 'f': attribute_ += '\f'; cursor_++; break;
278 case '\"': attribute_ += '\"'; cursor_++; break;
279 case '\'': attribute_ += '\''; cursor_++; break;
280 case '\\': attribute_ += '\\'; cursor_++; break;
281 case '/': attribute_ += '/'; cursor_++; break;
282 case 'x': { // Not in the JSON standard
283 cursor_++;
284 uint64_t val;
285 ECHECK(ParseHexNum(2, &val));
286 attribute_ += static_cast<char>(val);
287 break;
288 }
289 case 'u': {
290 cursor_++;
291 uint64_t val;
292 ECHECK(ParseHexNum(4, &val));
293 if (val >= 0xD800 && val <= 0xDBFF) {
294 if (unicode_high_surrogate != -1) {
295 return Error(
296 "illegal Unicode sequence (multiple high surrogates)");
297 } else {
298 unicode_high_surrogate = static_cast<int>(val);
299 }
300 } else if (val >= 0xDC00 && val <= 0xDFFF) {
301 if (unicode_high_surrogate == -1) {
302 return Error(
303 "illegal Unicode sequence (unpaired low surrogate)");
304 } else {
305 int code_point = 0x10000 +
306 ((unicode_high_surrogate & 0x03FF) << 10) +
307 (val & 0x03FF);
308 ToUTF8(code_point, &attribute_);
309 unicode_high_surrogate = -1;
310 }
311 } else {
312 if (unicode_high_surrogate != -1) {
313 return Error(
314 "illegal Unicode sequence (unpaired high surrogate)");
315 }
316 ToUTF8(static_cast<int>(val), &attribute_);
317 }
318 break;
319 }
320 default: return Error("unknown escape code in string constant");
321 }
322 } else { // printable chars + UTF-8 bytes
323 if (unicode_high_surrogate != -1) {
324 return Error(
325 "illegal Unicode sequence (unpaired high surrogate)");
326 }
327 attribute_ += *cursor_++;
328 }
329 }
330 if (unicode_high_surrogate != -1) {
331 return Error(
332 "illegal Unicode sequence (unpaired high surrogate)");
333 }
334 cursor_++;
335 if (!opts.allow_non_utf8 && !ValidateUTF8(attribute_)) {
336 return Error("illegal UTF-8 sequence");
337 }
338 token_ = kTokenStringConstant;
339 return NoError();
340 }
341 case '/':
342 if (*cursor_ == '/') {
343 const char *start = ++cursor_;
344 while (*cursor_ && *cursor_ != '\n' && *cursor_ != '\r') cursor_++;
345 if (*start == '/') { // documentation comment
346 if (cursor_ != source_ && !seen_newline)
347 return Error(
348 "a documentation comment should be on a line on its own");
349 doc_comment_.push_back(std::string(start + 1, cursor_));
350 }
351 break;
352 } else if (*cursor_ == '*') {
353 cursor_++;
354 // TODO: make nested.
355 while (*cursor_ != '*' || cursor_[1] != '/') {
356 if (*cursor_ == '\n') line_++;
357 if (!*cursor_) return Error("end of file in comment");
358 cursor_++;
359 }
360 cursor_ += 2;
361 break;
362 }
363 // fall thru
364 default:
365 if (IsIdentifierStart(c)) {
366 // Collect all chars of an identifier:
367 const char *start = cursor_ - 1;
368 while (isalnum(static_cast<unsigned char>(*cursor_)) ||
369 *cursor_ == '_')
370 cursor_++;
371 attribute_.append(start, cursor_);
372 token_ = kTokenIdentifier;
373 return NoError();
374 } else if (isdigit(static_cast<unsigned char>(c)) || c == '-') {
375 const char *start = cursor_ - 1;
376 if (c == '-' && *cursor_ == '0' &&
377 (cursor_[1] == 'x' || cursor_[1] == 'X')) {
378 ++start;
379 ++cursor_;
380 attribute_.append(&c, &c + 1);
381 c = '0';
382 }
383 if (c == '0' && (*cursor_ == 'x' || *cursor_ == 'X')) {
384 cursor_++;
385 while (isxdigit(static_cast<unsigned char>(*cursor_))) cursor_++;
386 attribute_.append(start + 2, cursor_);
387 attribute_ = NumToString(static_cast<int64_t>(
388 StringToUInt(attribute_.c_str(), nullptr, 16)));
389 token_ = kTokenIntegerConstant;
390 return NoError();
391 }
392 while (isdigit(static_cast<unsigned char>(*cursor_))) cursor_++;
393 if (*cursor_ == '.' || *cursor_ == 'e' || *cursor_ == 'E') {
394 if (*cursor_ == '.') {
395 cursor_++;
396 while (isdigit(static_cast<unsigned char>(*cursor_))) cursor_++;
397 }
398 // See if this float has a scientific notation suffix. Both JSON
399 // and C++ (through strtod() we use) have the same format:
400 if (*cursor_ == 'e' || *cursor_ == 'E') {
401 cursor_++;
402 if (*cursor_ == '+' || *cursor_ == '-') cursor_++;
403 while (isdigit(static_cast<unsigned char>(*cursor_))) cursor_++;
404 }
405 token_ = kTokenFloatConstant;
406 } else {
407 token_ = kTokenIntegerConstant;
408 }
409 attribute_.append(start, cursor_);
410 return NoError();
411 }
412 std::string ch;
413 ch = c;
414 if (c < ' ' || c > '~') ch = "code: " + NumToString(c);
415 return Error("illegal character: " + ch);
416 }
417 }
418 }
419
420 // Check if a given token is next.
Is(int t)421 bool Parser::Is(int t) {
422 return t == token_;
423 }
424
IsIdent(const char * id)425 bool Parser::IsIdent(const char *id) {
426 return token_ == kTokenIdentifier && attribute_ == id;
427 }
428
429 // Expect a given token to be next, consume it, or error if not present.
Expect(int t)430 CheckedError Parser::Expect(int t) {
431 if (t != token_) {
432 return Error("expecting: " + TokenToString(t) + " instead got: " +
433 TokenToStringId(token_));
434 }
435 NEXT();
436 return NoError();
437 }
438
ParseNamespacing(std::string * id,std::string * last)439 CheckedError Parser::ParseNamespacing(std::string *id, std::string *last) {
440 while (Is('.')) {
441 NEXT();
442 *id += ".";
443 *id += attribute_;
444 if (last) *last = attribute_;
445 EXPECT(kTokenIdentifier);
446 }
447 return NoError();
448 }
449
LookupEnum(const std::string & id)450 EnumDef *Parser::LookupEnum(const std::string &id) {
451 // Search thru parent namespaces.
452 for (int components = static_cast<int>(current_namespace_->components.size());
453 components >= 0; components--) {
454 auto ed = enums_.Lookup(
455 current_namespace_->GetFullyQualifiedName(id, components));
456 if (ed) return ed;
457 }
458 return nullptr;
459 }
460
LookupStruct(const std::string & id) const461 StructDef *Parser::LookupStruct(const std::string &id) const {
462 auto sd = structs_.Lookup(id);
463 if (sd) sd->refcount++;
464 return sd;
465 }
466
ParseTypeIdent(Type & type)467 CheckedError Parser::ParseTypeIdent(Type &type) {
468 std::string id = attribute_;
469 EXPECT(kTokenIdentifier);
470 ECHECK(ParseNamespacing(&id, nullptr));
471 auto enum_def = LookupEnum(id);
472 if (enum_def) {
473 type = enum_def->underlying_type;
474 if (enum_def->is_union) type.base_type = BASE_TYPE_UNION;
475 } else {
476 type.base_type = BASE_TYPE_STRUCT;
477 type.struct_def = LookupCreateStruct(id);
478 }
479 return NoError();
480 }
481
482 // Parse any IDL type.
ParseType(Type & type)483 CheckedError Parser::ParseType(Type &type) {
484 if (token_ == kTokenIdentifier) {
485 if (IsIdent("bool")) {
486 type.base_type = BASE_TYPE_BOOL;
487 NEXT();
488 } else if (IsIdent("byte") || IsIdent("int8")) {
489 type.base_type = BASE_TYPE_CHAR;
490 NEXT();
491 } else if (IsIdent("ubyte") || IsIdent("uint8")) {
492 type.base_type = BASE_TYPE_UCHAR;
493 NEXT();
494 } else if (IsIdent("short") || IsIdent("int16")) {
495 type.base_type = BASE_TYPE_SHORT;
496 NEXT();
497 } else if (IsIdent("ushort") || IsIdent("uint16")) {
498 type.base_type = BASE_TYPE_USHORT;
499 NEXT();
500 } else if (IsIdent("int") || IsIdent("int32")) {
501 type.base_type = BASE_TYPE_INT;
502 NEXT();
503 } else if (IsIdent("uint") || IsIdent("uint32")) {
504 type.base_type = BASE_TYPE_UINT;
505 NEXT();
506 } else if (IsIdent("long") || IsIdent("int64")) {
507 type.base_type = BASE_TYPE_LONG;
508 NEXT();
509 } else if (IsIdent("ulong") || IsIdent("uint64")) {
510 type.base_type = BASE_TYPE_ULONG;
511 NEXT();
512 } else if (IsIdent("float") || IsIdent("float32")) {
513 type.base_type = BASE_TYPE_FLOAT;
514 NEXT();
515 } else if (IsIdent("double") || IsIdent("float64")) {
516 type.base_type = BASE_TYPE_DOUBLE;
517 NEXT();
518 } else if (IsIdent("string")) {
519 type.base_type = BASE_TYPE_STRING;
520 NEXT();
521 } else {
522 ECHECK(ParseTypeIdent(type));
523 }
524 } else if (token_ == '[') {
525 NEXT();
526 Type subtype;
527 ECHECK(ParseType(subtype));
528 if (subtype.base_type == BASE_TYPE_VECTOR) {
529 // We could support this, but it will complicate things, and it's
530 // easier to work around with a struct around the inner vector.
531 return Error(
532 "nested vector types not supported (wrap in table first).");
533 }
534 type = Type(BASE_TYPE_VECTOR, subtype.struct_def, subtype.enum_def);
535 type.element = subtype.base_type;
536 EXPECT(']');
537 } else {
538 return Error("illegal type syntax");
539 }
540 return NoError();
541 }
542
AddField(StructDef & struct_def,const std::string & name,const Type & type,FieldDef ** dest)543 CheckedError Parser::AddField(StructDef &struct_def, const std::string &name,
544 const Type &type, FieldDef **dest) {
545 auto &field = *new FieldDef();
546 field.value.offset =
547 FieldIndexToOffset(static_cast<voffset_t>(struct_def.fields.vec.size()));
548 field.name = name;
549 field.file = struct_def.file;
550 field.value.type = type;
551 if (struct_def.fixed) { // statically compute the field offset
552 auto size = InlineSize(type);
553 auto alignment = InlineAlignment(type);
554 // structs_ need to have a predictable format, so we need to align to
555 // the largest scalar
556 struct_def.minalign = std::max(struct_def.minalign, alignment);
557 struct_def.PadLastField(alignment);
558 field.value.offset = static_cast<voffset_t>(struct_def.bytesize);
559 struct_def.bytesize += size;
560 }
561 if (struct_def.fields.Add(name, &field))
562 return Error("field already exists: " + name);
563 *dest = &field;
564 return NoError();
565 }
566
ParseField(StructDef & struct_def)567 CheckedError Parser::ParseField(StructDef &struct_def) {
568 std::string name = attribute_;
569
570 if (LookupStruct(name))
571 return Error("field name can not be the same as table/struct name");
572
573 std::vector<std::string> dc = doc_comment_;
574 EXPECT(kTokenIdentifier);
575 EXPECT(':');
576 Type type;
577 ECHECK(ParseType(type));
578
579 if (struct_def.fixed && !IsScalar(type.base_type) && !IsStruct(type))
580 return Error("structs_ may contain only scalar or struct fields");
581
582 FieldDef *typefield = nullptr;
583 if (type.base_type == BASE_TYPE_UNION) {
584 // For union fields, add a second auto-generated field to hold the type,
585 // with a special suffix.
586 ECHECK(AddField(struct_def, name + UnionTypeFieldSuffix(),
587 type.enum_def->underlying_type, &typefield));
588 } else if (type.base_type == BASE_TYPE_VECTOR &&
589 type.element == BASE_TYPE_UNION) {
590 // Only cpp, js and ts supports the union vector feature so far.
591 if (!SupportsVectorOfUnions()) {
592 return Error("Vectors of unions are not yet supported in all "
593 "the specified programming languages.");
594 }
595 // For vector of union fields, add a second auto-generated vector field to
596 // hold the types, with a special suffix.
597 Type union_vector(BASE_TYPE_VECTOR, nullptr, type.enum_def);
598 union_vector.element = BASE_TYPE_UTYPE;
599 ECHECK(AddField(struct_def, name + UnionTypeFieldSuffix(),
600 union_vector, &typefield));
601 }
602
603 FieldDef *field;
604 ECHECK(AddField(struct_def, name, type, &field));
605
606 if (token_ == '=') {
607 NEXT();
608 if (!IsScalar(type.base_type))
609 return Error("default values currently only supported for scalars");
610 ECHECK(ParseSingleValue(field->value));
611 }
612 if (IsFloat(field->value.type.base_type)) {
613 if (!strpbrk(field->value.constant.c_str(), ".eE"))
614 field->value.constant += ".0";
615 }
616
617 if (type.enum_def &&
618 IsScalar(type.base_type) &&
619 !struct_def.fixed &&
620 !type.enum_def->attributes.Lookup("bit_flags") &&
621 !type.enum_def->ReverseLookup(static_cast<int>(
622 StringToInt(field->value.constant.c_str()))))
623 Warning("enum " + type.enum_def->name +
624 " does not have a declaration for this field\'s default of " +
625 field->value.constant);
626
627 field->doc_comment = dc;
628 ECHECK(ParseMetaData(&field->attributes));
629 field->deprecated = field->attributes.Lookup("deprecated") != nullptr;
630 auto hash_name = field->attributes.Lookup("hash");
631 if (hash_name) {
632 switch (type.base_type) {
633 case BASE_TYPE_INT:
634 case BASE_TYPE_UINT: {
635 if (FindHashFunction32(hash_name->constant.c_str()) == nullptr)
636 return Error("Unknown hashing algorithm for 32 bit types: " +
637 hash_name->constant);
638 break;
639 }
640 case BASE_TYPE_LONG:
641 case BASE_TYPE_ULONG: {
642 if (FindHashFunction64(hash_name->constant.c_str()) == nullptr)
643 return Error("Unknown hashing algorithm for 64 bit types: " +
644 hash_name->constant);
645 break;
646 }
647 default:
648 return Error(
649 "only int, uint, long and ulong data types support hashing.");
650 }
651 }
652 auto cpp_type = field->attributes.Lookup("cpp_type");
653 if (cpp_type) {
654 if (!hash_name)
655 return Error("cpp_type can only be used with a hashed field");
656 }
657 if (field->deprecated && struct_def.fixed)
658 return Error("can't deprecate fields in a struct");
659 field->required = field->attributes.Lookup("required") != nullptr;
660 if (field->required && (struct_def.fixed ||
661 IsScalar(field->value.type.base_type)))
662 return Error("only non-scalar fields in tables may be 'required'");
663 field->key = field->attributes.Lookup("key") != nullptr;
664 if (field->key) {
665 if (struct_def.has_key)
666 return Error("only one field may be set as 'key'");
667 struct_def.has_key = true;
668 if (!IsScalar(field->value.type.base_type)) {
669 field->required = true;
670 if (field->value.type.base_type != BASE_TYPE_STRING)
671 return Error("'key' field must be string or scalar type");
672 }
673 }
674
675 auto field_native_custom_alloc = field->attributes.Lookup("native_custom_alloc");
676 if (field_native_custom_alloc)
677 return Error("native_custom_alloc can only be used with a table or struct definition");
678
679 field->native_inline = field->attributes.Lookup("native_inline") != nullptr;
680 if (field->native_inline && !IsStruct(field->value.type))
681 return Error("native_inline can only be defined on structs'");
682
683 auto nested = field->attributes.Lookup("nested_flatbuffer");
684 if (nested) {
685 if (nested->type.base_type != BASE_TYPE_STRING)
686 return Error(
687 "nested_flatbuffer attribute must be a string (the root type)");
688 if (field->value.type.base_type != BASE_TYPE_VECTOR ||
689 field->value.type.element != BASE_TYPE_UCHAR)
690 return Error(
691 "nested_flatbuffer attribute may only apply to a vector of ubyte");
692 // This will cause an error if the root type of the nested flatbuffer
693 // wasn't defined elsewhere.
694 LookupCreateStruct(nested->constant);
695
696 // Keep a pointer to StructDef in FieldDef to simplify re-use later
697 auto nested_qualified_name =
698 current_namespace_->GetFullyQualifiedName(nested->constant);
699 field->nested_flatbuffer = LookupStruct(nested_qualified_name);
700 }
701
702 if (field->attributes.Lookup("flexbuffer")) {
703 field->flexbuffer = true;
704 uses_flexbuffers_ = true;
705 if (field->value.type.base_type != BASE_TYPE_VECTOR ||
706 field->value.type.element != BASE_TYPE_UCHAR)
707 return Error(
708 "flexbuffer attribute may only apply to a vector of ubyte");
709 }
710
711 if (typefield) {
712 if (!IsScalar(typefield->value.type.base_type)) {
713 // this is a union vector field
714 typefield->required = field->required;
715 }
716 // If this field is a union, and it has a manually assigned id,
717 // the automatically added type field should have an id as well (of N - 1).
718 auto attr = field->attributes.Lookup("id");
719 if (attr) {
720 auto id = atoi(attr->constant.c_str());
721 auto val = new Value();
722 val->type = attr->type;
723 val->constant = NumToString(id - 1);
724 typefield->attributes.Add("id", val);
725 }
726 }
727
728 EXPECT(';');
729 return NoError();
730 }
731
ParseString(Value & val)732 CheckedError Parser::ParseString(Value &val) {
733 auto s = attribute_;
734 EXPECT(kTokenStringConstant);
735 val.constant = NumToString(builder_.CreateString(s).o);
736 return NoError();
737 }
738
ParseComma()739 CheckedError Parser::ParseComma() {
740 if (!opts.protobuf_ascii_alike) EXPECT(',');
741 return NoError();
742 }
743
ParseAnyValue(Value & val,FieldDef * field,size_t parent_fieldn,const StructDef * parent_struct_def)744 CheckedError Parser::ParseAnyValue(Value &val, FieldDef *field,
745 size_t parent_fieldn,
746 const StructDef *parent_struct_def) {
747 switch (val.type.base_type) {
748 case BASE_TYPE_UNION: {
749 assert(field);
750 std::string constant;
751 // Find corresponding type field we may have already parsed.
752 for (auto elem = field_stack_.rbegin();
753 elem != field_stack_.rbegin() + parent_fieldn; ++elem) {
754 auto &type = elem->second->value.type;
755 if (type.base_type == BASE_TYPE_UTYPE &&
756 type.enum_def == val.type.enum_def) {
757 constant = elem->first.constant;
758 break;
759 }
760 }
761 if (constant.empty()) {
762 // We haven't seen the type field yet. Sadly a lot of JSON writers
763 // output these in alphabetical order, meaning it comes after this
764 // value. So we scan past the value to find it, then come back here.
765 auto type_name = field->name + UnionTypeFieldSuffix();
766 assert(parent_struct_def);
767 auto type_field = parent_struct_def->fields.Lookup(type_name);
768 assert(type_field); // Guaranteed by ParseField().
769 // Remember where we are in the source file, so we can come back here.
770 auto backup = *static_cast<ParserState *>(this);
771 ECHECK(SkipAnyJsonValue()); // The table.
772 ECHECK(ParseComma());
773 auto next_name = attribute_;
774 if (Is(kTokenStringConstant)) {
775 NEXT();
776 } else {
777 EXPECT(kTokenIdentifier);
778 }
779 if (next_name != type_name)
780 return Error("missing type field after this union value: " +
781 type_name);
782 EXPECT(':');
783 Value type_val = type_field->value;
784 ECHECK(ParseAnyValue(type_val, type_field, 0, nullptr));
785 constant = type_val.constant;
786 // Got the information we needed, now rewind:
787 *static_cast<ParserState *>(this) = backup;
788 }
789 uint8_t enum_idx;
790 ECHECK(atot(constant.c_str(), *this, &enum_idx));
791 auto enum_val = val.type.enum_def->ReverseLookup(enum_idx);
792 if (!enum_val) return Error("illegal type id for: " + field->name);
793 if (enum_val->union_type.base_type == BASE_TYPE_STRUCT) {
794 ECHECK(ParseTable(*enum_val->union_type.struct_def, &val.constant,
795 nullptr));
796 if (enum_val->union_type.struct_def->fixed) {
797 // All BASE_TYPE_UNION values are offsets, so turn this into one.
798 SerializeStruct(*enum_val->union_type.struct_def, val);
799 builder_.ClearOffsets();
800 val.constant = NumToString(builder_.GetSize());
801 }
802 } else if (enum_val->union_type.base_type == BASE_TYPE_STRING) {
803 ECHECK(ParseString(val));
804 } else {
805 assert(false);
806 }
807 break;
808 }
809 case BASE_TYPE_STRUCT:
810 ECHECK(ParseTable(*val.type.struct_def, &val.constant, nullptr));
811 break;
812 case BASE_TYPE_STRING: {
813 ECHECK(ParseString(val));
814 break;
815 }
816 case BASE_TYPE_VECTOR: {
817 uoffset_t off;
818 ECHECK(ParseVector(val.type.VectorType(), &off));
819 val.constant = NumToString(off);
820 break;
821 }
822 case BASE_TYPE_INT:
823 case BASE_TYPE_UINT:
824 case BASE_TYPE_LONG:
825 case BASE_TYPE_ULONG: {
826 if (field && field->attributes.Lookup("hash") &&
827 (token_ == kTokenIdentifier || token_ == kTokenStringConstant)) {
828 ECHECK(ParseHash(val, field));
829 } else {
830 ECHECK(ParseSingleValue(val));
831 }
832 break;
833 }
834 default:
835 ECHECK(ParseSingleValue(val));
836 break;
837 }
838 return NoError();
839 }
840
SerializeStruct(const StructDef & struct_def,const Value & val)841 void Parser::SerializeStruct(const StructDef &struct_def, const Value &val) {
842 assert(val.constant.length() == struct_def.bytesize);
843 builder_.Align(struct_def.minalign);
844 builder_.PushBytes(reinterpret_cast<const uint8_t *>(val.constant.c_str()),
845 struct_def.bytesize);
846 builder_.AddStructOffset(val.offset, builder_.GetSize());
847 }
848
ParseTableDelimiters(size_t & fieldn,const StructDef * struct_def,ParseTableDelimitersBody body,void * state)849 CheckedError Parser::ParseTableDelimiters(size_t &fieldn,
850 const StructDef *struct_def,
851 ParseTableDelimitersBody body,
852 void *state) {
853 // We allow tables both as JSON object{ .. } with field names
854 // or vector[..] with all fields in order
855 char terminator = '}';
856 bool is_nested_vector = struct_def && Is('[');
857 if (is_nested_vector) {
858 NEXT();
859 terminator = ']';
860 } else {
861 EXPECT('{');
862 }
863 for (;;) {
864 if ((!opts.strict_json || !fieldn) && Is(terminator)) break;
865 std::string name;
866 if (is_nested_vector) {
867 if (fieldn > struct_def->fields.vec.size()) {
868 return Error("too many unnamed fields in nested array");
869 }
870 name = struct_def->fields.vec[fieldn]->name;
871 } else {
872 name = attribute_;
873 if (Is(kTokenStringConstant)) {
874 NEXT();
875 } else {
876 EXPECT(opts.strict_json ? kTokenStringConstant : kTokenIdentifier);
877 }
878 if (!opts.protobuf_ascii_alike || !(Is('{') || Is('['))) EXPECT(':');
879 }
880 ECHECK(body(name, fieldn, struct_def, state));
881 if (Is(terminator)) break;
882 ECHECK(ParseComma());
883 }
884 NEXT();
885 if (is_nested_vector && fieldn != struct_def->fields.vec.size()) {
886 return Error("wrong number of unnamed fields in table vector");
887 }
888 return NoError();
889 }
890
ParseTable(const StructDef & struct_def,std::string * value,uoffset_t * ovalue)891 CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
892 uoffset_t *ovalue) {
893 size_t fieldn_outer = 0;
894 auto err = ParseTableDelimiters(fieldn_outer, &struct_def,
895 [](const std::string &name, size_t &fieldn,
896 const StructDef *struct_def_inner,
897 void *state) -> CheckedError {
898 Parser *parser = static_cast<Parser *>(state);
899 if (name == "$schema") {
900 ECHECK(parser->Expect(kTokenStringConstant));
901 return NoError();
902 }
903 auto field = struct_def_inner->fields.Lookup(name);
904 if (!field) {
905 if (!parser->opts.skip_unexpected_fields_in_json) {
906 return parser->Error("unknown field: " + name);
907 } else {
908 ECHECK(parser->SkipAnyJsonValue());
909 }
910 } else {
911 if (parser->IsIdent("null")) {
912 ECHECK(parser->Next()); // Ignore this field.
913 } else {
914 Value val = field->value;
915 if (field->flexbuffer) {
916 flexbuffers::Builder builder(1024,
917 flexbuffers::BUILDER_FLAG_SHARE_ALL);
918 ECHECK(parser->ParseFlexBufferValue(&builder));
919 builder.Finish();
920 auto off = parser->builder_.CreateVector(builder.GetBuffer());
921 val.constant = NumToString(off.o);
922 } else if (field->nested_flatbuffer) {
923 ECHECK(parser->ParseNestedFlatbuffer(val, field, fieldn, struct_def_inner));
924 } else {
925 ECHECK(parser->ParseAnyValue(val, field, fieldn, struct_def_inner));
926 }
927 // Hardcoded insertion-sort with error-check.
928 // If fields are specified in order, then this loop exits immediately.
929 auto elem = parser->field_stack_.rbegin();
930 for (; elem != parser->field_stack_.rbegin() + fieldn; ++elem) {
931 auto existing_field = elem->second;
932 if (existing_field == field)
933 return parser->Error("field set more than once: " + field->name);
934 if (existing_field->value.offset < field->value.offset) break;
935 }
936 // Note: elem points to before the insertion point, thus .base() points
937 // to the correct spot.
938 parser->field_stack_.insert(elem.base(),
939 std::make_pair(val, field));
940 fieldn++;
941 }
942 }
943 return NoError();
944 }, this);
945 ECHECK(err);
946
947 // Check if all required fields are parsed.
948 for (auto field_it = struct_def.fields.vec.begin();
949 field_it != struct_def.fields.vec.end();
950 ++field_it) {
951 auto required_field = *field_it;
952 if (!required_field->required) {
953 continue;
954 }
955 bool found = false;
956 for (auto pf_it = field_stack_.end() - fieldn_outer;
957 pf_it != field_stack_.end();
958 ++pf_it) {
959 auto parsed_field = pf_it->second;
960 if (parsed_field == required_field) {
961 found = true;
962 break;
963 }
964 }
965 if (!found) {
966 return Error("required field is missing: " + required_field->name + " in " + struct_def.name);
967 }
968 }
969
970 if (struct_def.fixed && fieldn_outer != struct_def.fields.vec.size())
971 return Error("struct: wrong number of initializers: " + struct_def.name);
972
973 auto start = struct_def.fixed
974 ? builder_.StartStruct(struct_def.minalign)
975 : builder_.StartTable();
976
977 for (size_t size = struct_def.sortbysize ? sizeof(largest_scalar_t) : 1;
978 size;
979 size /= 2) {
980 // Go through elements in reverse, since we're building the data backwards.
981 for (auto it = field_stack_.rbegin(); it != field_stack_.rbegin() +
982 fieldn_outer;
983 ++it) {
984 auto &field_value = it->first;
985 auto field = it->second;
986 if (!struct_def.sortbysize ||
987 size == SizeOf(field_value.type.base_type)) {
988 switch (field_value.type.base_type) {
989 #define FLATBUFFERS_TD(ENUM, IDLTYPE, \
990 CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \
991 case BASE_TYPE_ ## ENUM: \
992 builder_.Pad(field->padding); \
993 if (struct_def.fixed) { \
994 CTYPE val; \
995 ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
996 builder_.PushElement(val); \
997 } else { \
998 CTYPE val, valdef; \
999 ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
1000 ECHECK(atot(field->value.constant.c_str(), *this, &valdef)); \
1001 builder_.AddElement(field_value.offset, val, valdef); \
1002 } \
1003 break;
1004 FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD);
1005 #undef FLATBUFFERS_TD
1006 #define FLATBUFFERS_TD(ENUM, IDLTYPE, \
1007 CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \
1008 case BASE_TYPE_ ## ENUM: \
1009 builder_.Pad(field->padding); \
1010 if (IsStruct(field->value.type)) { \
1011 SerializeStruct(*field->value.type.struct_def, field_value); \
1012 } else { \
1013 CTYPE val; \
1014 ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
1015 builder_.AddOffset(field_value.offset, val); \
1016 } \
1017 break;
1018 FLATBUFFERS_GEN_TYPES_POINTER(FLATBUFFERS_TD);
1019 #undef FLATBUFFERS_TD
1020 }
1021 }
1022 }
1023 }
1024 for (size_t i = 0; i < fieldn_outer; i++) field_stack_.pop_back();
1025
1026 if (struct_def.fixed) {
1027 builder_.ClearOffsets();
1028 builder_.EndStruct();
1029 assert(value);
1030 // Temporarily store this struct in the value string, since it is to
1031 // be serialized in-place elsewhere.
1032 value->assign(
1033 reinterpret_cast<const char *>(builder_.GetCurrentBufferPointer()),
1034 struct_def.bytesize);
1035 builder_.PopBytes(struct_def.bytesize);
1036 assert(!ovalue);
1037 } else {
1038 auto val = builder_.EndTable(start);
1039 if (ovalue) *ovalue = val;
1040 if (value) *value = NumToString(val);
1041 }
1042 return NoError();
1043 }
1044
ParseVectorDelimiters(size_t & count,ParseVectorDelimitersBody body,void * state)1045 CheckedError Parser::ParseVectorDelimiters(size_t &count,
1046 ParseVectorDelimitersBody body,
1047 void *state) {
1048 EXPECT('[');
1049 for (;;) {
1050 if ((!opts.strict_json || !count) && Is(']')) break;
1051 ECHECK(body(count, state));
1052 count++;
1053 if (Is(']')) break;
1054 ECHECK(ParseComma());
1055 }
1056 NEXT();
1057 return NoError();
1058 }
1059
ParseVector(const Type & type,uoffset_t * ovalue)1060 CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue) {
1061 size_t count = 0;
1062 std::pair<Parser *, const Type &> parser_and_type_state(this, type);
1063 auto err = ParseVectorDelimiters(count,
1064 [](size_t &, void *state) -> CheckedError {
1065 auto *parser_and_type =
1066 static_cast<std::pair<Parser *, const Type &> *>(state);
1067 auto *parser = parser_and_type->first;
1068 Value val;
1069 val.type = parser_and_type->second;
1070 ECHECK(parser->ParseAnyValue(val, nullptr, 0, nullptr));
1071 parser->field_stack_.push_back(std::make_pair(val, nullptr));
1072 return NoError();
1073 }, &parser_and_type_state);
1074 ECHECK(err);
1075
1076 builder_.StartVector(count * InlineSize(type) / InlineAlignment(type),
1077 InlineAlignment(type));
1078 for (size_t i = 0; i < count; i++) {
1079 // start at the back, since we're building the data backwards.
1080 auto &val = field_stack_.back().first;
1081 switch (val.type.base_type) {
1082 #define FLATBUFFERS_TD(ENUM, IDLTYPE, \
1083 CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \
1084 case BASE_TYPE_ ## ENUM: \
1085 if (IsStruct(val.type)) SerializeStruct(*val.type.struct_def, val); \
1086 else { \
1087 CTYPE elem; \
1088 ECHECK(atot(val.constant.c_str(), *this, &elem)); \
1089 builder_.PushElement(elem); \
1090 } \
1091 break;
1092 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
1093 #undef FLATBUFFERS_TD
1094 }
1095 field_stack_.pop_back();
1096 }
1097
1098 builder_.ClearOffsets();
1099 *ovalue = builder_.EndVector(count);
1100 return NoError();
1101 }
1102
ParseNestedFlatbuffer(Value & val,FieldDef * field,size_t fieldn,const StructDef * parent_struct_def)1103 CheckedError Parser::ParseNestedFlatbuffer(Value &val, FieldDef *field,
1104 size_t fieldn,
1105 const StructDef *parent_struct_def) {
1106 if (token_ == '[') {// backwards compat for 'legacy' ubyte buffers
1107 ECHECK(ParseAnyValue(val, field, fieldn, parent_struct_def));
1108 } else {
1109 auto cursor_at_value_begin = cursor_;
1110 ECHECK(SkipAnyJsonValue());
1111 std::string substring(cursor_at_value_begin -1 , cursor_ -1);
1112
1113 // Create and initialize new parser
1114 Parser nested_parser;
1115 assert(field->nested_flatbuffer);
1116 nested_parser.root_struct_def_ = field->nested_flatbuffer;
1117 nested_parser.enums_ = enums_;
1118 nested_parser.opts = opts;
1119 nested_parser.uses_flexbuffers_ = uses_flexbuffers_;
1120
1121 // Parse JSON substring into new flatbuffer builder using nested_parser
1122 if (!nested_parser.Parse(substring.c_str(), nullptr, nullptr)) {
1123 ECHECK(Error(nested_parser.error_));
1124 }
1125 auto off = builder_.CreateVector(nested_parser.builder_.GetBufferPointer(), nested_parser.builder_.GetSize());
1126 val.constant = NumToString(off.o);
1127
1128 // Clean nested_parser before destruction to avoid deleting the elements in the SymbolTables
1129 nested_parser.enums_.dict.clear();
1130 nested_parser.enums_.vec.clear();
1131 }
1132 return NoError();
1133 }
1134
ParseMetaData(SymbolTable<Value> * attributes)1135 CheckedError Parser::ParseMetaData(SymbolTable<Value> *attributes) {
1136 if (Is('(')) {
1137 NEXT();
1138 for (;;) {
1139 auto name = attribute_;
1140 EXPECT(kTokenIdentifier);
1141 if (known_attributes_.find(name) == known_attributes_.end())
1142 return Error("user define attributes must be declared before use: " +
1143 name);
1144 auto e = new Value();
1145 attributes->Add(name, e);
1146 if (Is(':')) {
1147 NEXT();
1148 ECHECK(ParseSingleValue(*e));
1149 }
1150 if (Is(')')) { NEXT(); break; }
1151 EXPECT(',');
1152 }
1153 }
1154 return NoError();
1155 }
1156
TryTypedValue(int dtoken,bool check,Value & e,BaseType req,bool * destmatch)1157 CheckedError Parser::TryTypedValue(int dtoken, bool check, Value &e,
1158 BaseType req, bool *destmatch) {
1159 bool match = dtoken == token_;
1160 if (match) {
1161 *destmatch = true;
1162 e.constant = attribute_;
1163 if (!check) {
1164 if (e.type.base_type == BASE_TYPE_NONE) {
1165 e.type.base_type = req;
1166 } else {
1167 return Error(std::string("type mismatch: expecting: ") +
1168 kTypeNames[e.type.base_type] +
1169 ", found: " +
1170 kTypeNames[req]);
1171 }
1172 }
1173 NEXT();
1174 }
1175 return NoError();
1176 }
1177
ParseEnumFromString(Type & type,int64_t * result)1178 CheckedError Parser::ParseEnumFromString(Type &type, int64_t *result) {
1179 *result = 0;
1180 // Parse one or more enum identifiers, separated by spaces.
1181 const char *next = attribute_.c_str();
1182 do {
1183 const char *divider = strchr(next, ' ');
1184 std::string word;
1185 if (divider) {
1186 word = std::string(next, divider);
1187 next = divider + strspn(divider, " ");
1188 } else {
1189 word = next;
1190 next += word.length();
1191 }
1192 if (type.enum_def) { // The field has an enum type
1193 auto enum_val = type.enum_def->vals.Lookup(word);
1194 if (!enum_val)
1195 return Error("unknown enum value: " + word +
1196 ", for enum: " + type.enum_def->name);
1197 *result |= enum_val->value;
1198 } else { // No enum type, probably integral field.
1199 if (!IsInteger(type.base_type))
1200 return Error("not a valid value for this field: " + word);
1201 // TODO: could check if its a valid number constant here.
1202 const char *dot = strrchr(word.c_str(), '.');
1203 if (!dot)
1204 return Error("enum values need to be qualified by an enum type");
1205 std::string enum_def_str(word.c_str(), dot);
1206 std::string enum_val_str(dot + 1, word.c_str() + word.length());
1207 auto enum_def = LookupEnum(enum_def_str);
1208 if (!enum_def) return Error("unknown enum: " + enum_def_str);
1209 auto enum_val = enum_def->vals.Lookup(enum_val_str);
1210 if (!enum_val) return Error("unknown enum value: " + enum_val_str);
1211 *result |= enum_val->value;
1212 }
1213 } while(*next);
1214 return NoError();
1215 }
1216
1217
ParseHash(Value & e,FieldDef * field)1218 CheckedError Parser::ParseHash(Value &e, FieldDef* field) {
1219 assert(field);
1220 Value *hash_name = field->attributes.Lookup("hash");
1221 switch (e.type.base_type) {
1222 case BASE_TYPE_INT: {
1223 auto hash = FindHashFunction32(hash_name->constant.c_str());
1224 int32_t hashed_value = static_cast<int32_t>(hash(attribute_.c_str()));
1225 e.constant = NumToString(hashed_value);
1226 break;
1227 }
1228 case BASE_TYPE_UINT: {
1229 auto hash = FindHashFunction32(hash_name->constant.c_str());
1230 uint32_t hashed_value = hash(attribute_.c_str());
1231 e.constant = NumToString(hashed_value);
1232 break;
1233 }
1234 case BASE_TYPE_LONG: {
1235 auto hash = FindHashFunction64(hash_name->constant.c_str());
1236 int64_t hashed_value = static_cast<int64_t>(hash(attribute_.c_str()));
1237 e.constant = NumToString(hashed_value);
1238 break;
1239 }
1240 case BASE_TYPE_ULONG: {
1241 auto hash = FindHashFunction64(hash_name->constant.c_str());
1242 uint64_t hashed_value = hash(attribute_.c_str());
1243 e.constant = NumToString(hashed_value);
1244 break;
1245 }
1246 default:
1247 assert(0);
1248 }
1249 NEXT();
1250 return NoError();
1251 }
1252
TokenError()1253 CheckedError Parser::TokenError() {
1254 return Error("cannot parse value starting with: " +
1255 TokenToStringId(token_));
1256 }
1257
ParseSingleValue(Value & e)1258 CheckedError Parser::ParseSingleValue(Value &e) {
1259 // First see if this could be a conversion function:
1260 if (token_ == kTokenIdentifier && *cursor_ == '(') {
1261 auto functionname = attribute_;
1262 NEXT();
1263 EXPECT('(');
1264 ECHECK(ParseSingleValue(e));
1265 EXPECT(')');
1266 #define FLATBUFFERS_FN_DOUBLE(name, op) \
1267 if (functionname == name) { \
1268 auto x = strtod(e.constant.c_str(), nullptr); \
1269 e.constant = NumToString(op); \
1270 }
1271 FLATBUFFERS_FN_DOUBLE("deg", x / kPi * 180);
1272 FLATBUFFERS_FN_DOUBLE("rad", x * kPi / 180);
1273 FLATBUFFERS_FN_DOUBLE("sin", sin(x));
1274 FLATBUFFERS_FN_DOUBLE("cos", cos(x));
1275 FLATBUFFERS_FN_DOUBLE("tan", tan(x));
1276 FLATBUFFERS_FN_DOUBLE("asin", asin(x));
1277 FLATBUFFERS_FN_DOUBLE("acos", acos(x));
1278 FLATBUFFERS_FN_DOUBLE("atan", atan(x));
1279 // TODO(wvo): add more useful conversion functions here.
1280 #undef FLATBUFFERS_FN_DOUBLE
1281 // Then check if this could be a string/identifier enum value:
1282 } else if (e.type.base_type != BASE_TYPE_STRING &&
1283 e.type.base_type != BASE_TYPE_BOOL &&
1284 e.type.base_type != BASE_TYPE_NONE &&
1285 (token_ == kTokenIdentifier || token_ == kTokenStringConstant)) {
1286 if (IsIdentifierStart(attribute_[0])) { // Enum value.
1287 int64_t val;
1288 ECHECK(ParseEnumFromString(e.type, &val));
1289 e.constant = NumToString(val);
1290 NEXT();
1291 } else { // Numeric constant in string.
1292 if (IsInteger(e.type.base_type)) {
1293 char *end;
1294 e.constant = NumToString(StringToInt(attribute_.c_str(), &end));
1295 if (*end)
1296 return Error("invalid integer: " + attribute_);
1297 } else if (IsFloat(e.type.base_type)) {
1298 char *end;
1299 e.constant = NumToString(strtod(attribute_.c_str(), &end));
1300 if (*end)
1301 return Error("invalid float: " + attribute_);
1302 } else {
1303 assert(0); // Shouldn't happen, we covered all types.
1304 e.constant = "0";
1305 }
1306 NEXT();
1307 }
1308 } else {
1309 bool match = false;
1310 ECHECK(TryTypedValue(kTokenIntegerConstant,
1311 IsScalar(e.type.base_type),
1312 e,
1313 BASE_TYPE_INT,
1314 &match));
1315 ECHECK(TryTypedValue(kTokenFloatConstant,
1316 IsFloat(e.type.base_type),
1317 e,
1318 BASE_TYPE_FLOAT,
1319 &match));
1320 ECHECK(TryTypedValue(kTokenStringConstant,
1321 e.type.base_type == BASE_TYPE_STRING,
1322 e,
1323 BASE_TYPE_STRING,
1324 &match));
1325 auto istrue = IsIdent("true");
1326 if (istrue || IsIdent("false")) {
1327 attribute_ = NumToString(istrue);
1328 ECHECK(TryTypedValue(kTokenIdentifier,
1329 IsBool(e.type.base_type),
1330 e,
1331 BASE_TYPE_BOOL,
1332 &match));
1333 }
1334 if (!match) return TokenError();
1335 }
1336 return NoError();
1337 }
1338
LookupCreateStruct(const std::string & name,bool create_if_new,bool definition)1339 StructDef *Parser::LookupCreateStruct(const std::string &name,
1340 bool create_if_new, bool definition) {
1341 std::string qualified_name = current_namespace_->GetFullyQualifiedName(name);
1342 // See if it exists pre-declared by an unqualified use.
1343 auto struct_def = LookupStruct(name);
1344 if (struct_def && struct_def->predecl) {
1345 if (definition) {
1346 // Make sure it has the current namespace, and is registered under its
1347 // qualified name.
1348 struct_def->defined_namespace = current_namespace_;
1349 structs_.Move(name, qualified_name);
1350 }
1351 return struct_def;
1352 }
1353 // See if it exists pre-declared by an qualified use.
1354 struct_def = LookupStruct(qualified_name);
1355 if (struct_def && struct_def->predecl) {
1356 if (definition) {
1357 // Make sure it has the current namespace.
1358 struct_def->defined_namespace = current_namespace_;
1359 }
1360 return struct_def;
1361 }
1362 if (!definition) {
1363 // Search thru parent namespaces.
1364 for (size_t components = current_namespace_->components.size();
1365 components && !struct_def; components--) {
1366 struct_def = LookupStruct(
1367 current_namespace_->GetFullyQualifiedName(name, components - 1));
1368 }
1369 }
1370 if (!struct_def && create_if_new) {
1371 struct_def = new StructDef();
1372 if (definition) {
1373 structs_.Add(qualified_name, struct_def);
1374 struct_def->name = name;
1375 struct_def->defined_namespace = current_namespace_;
1376 } else {
1377 // Not a definition.
1378 // Rather than failing, we create a "pre declared" StructDef, due to
1379 // circular references, and check for errors at the end of parsing.
1380 // It is defined in the current namespace, as the best guess what the
1381 // final namespace will be.
1382 structs_.Add(name, struct_def);
1383 struct_def->name = name;
1384 struct_def->defined_namespace = current_namespace_;
1385 struct_def->original_location.reset(new std::string(file_being_parsed_ +
1386 ":" + NumToString(line_)));
1387 }
1388 }
1389 return struct_def;
1390 }
1391
ParseEnum(bool is_union,EnumDef ** dest)1392 CheckedError Parser::ParseEnum(bool is_union, EnumDef **dest) {
1393 std::vector<std::string> enum_comment = doc_comment_;
1394 NEXT();
1395 std::string enum_name = attribute_;
1396 EXPECT(kTokenIdentifier);
1397 auto &enum_def = *new EnumDef();
1398 enum_def.name = enum_name;
1399 enum_def.file = file_being_parsed_;
1400 enum_def.doc_comment = enum_comment;
1401 enum_def.is_union = is_union;
1402 enum_def.defined_namespace = current_namespace_;
1403 if (enums_.Add(current_namespace_->GetFullyQualifiedName(enum_name),
1404 &enum_def))
1405 return Error("enum already exists: " + enum_name);
1406 if (is_union) {
1407 enum_def.underlying_type.base_type = BASE_TYPE_UTYPE;
1408 enum_def.underlying_type.enum_def = &enum_def;
1409 } else {
1410 if (opts.proto_mode) {
1411 enum_def.underlying_type.base_type = BASE_TYPE_INT;
1412 } else {
1413 // Give specialized error message, since this type spec used to
1414 // be optional in the first FlatBuffers release.
1415 if (!Is(':')) {
1416 return Error("must specify the underlying integer type for this"
1417 " enum (e.g. \': short\', which was the default).");
1418 } else {
1419 NEXT();
1420 }
1421 // Specify the integer type underlying this enum.
1422 ECHECK(ParseType(enum_def.underlying_type));
1423 if (!IsInteger(enum_def.underlying_type.base_type))
1424 return Error("underlying enum type must be integral");
1425 }
1426 // Make this type refer back to the enum it was derived from.
1427 enum_def.underlying_type.enum_def = &enum_def;
1428 }
1429 ECHECK(ParseMetaData(&enum_def.attributes));
1430 EXPECT('{');
1431 if (is_union) enum_def.vals.Add("NONE", new EnumVal("NONE", 0));
1432 for (;;) {
1433 if (opts.proto_mode && attribute_ == "option") {
1434 ECHECK(ParseProtoOption());
1435 } else {
1436 auto value_name = attribute_;
1437 auto full_name = value_name;
1438 std::vector<std::string> value_comment = doc_comment_;
1439 EXPECT(kTokenIdentifier);
1440 if (is_union) {
1441 ECHECK(ParseNamespacing(&full_name, &value_name));
1442 if (opts.union_value_namespacing) {
1443 // Since we can't namespace the actual enum identifiers, turn
1444 // namespace parts into part of the identifier.
1445 value_name = full_name;
1446 std::replace(value_name.begin(), value_name.end(), '.', '_');
1447 }
1448 }
1449 auto prevsize = enum_def.vals.vec.size();
1450 auto value = enum_def.vals.vec.size()
1451 ? enum_def.vals.vec.back()->value + 1
1452 : 0;
1453 auto &ev = *new EnumVal(value_name, value);
1454 if (enum_def.vals.Add(value_name, &ev))
1455 return Error("enum value already exists: " + value_name);
1456 ev.doc_comment = value_comment;
1457 if (is_union) {
1458 if (Is(':')) {
1459 NEXT();
1460 ECHECK(ParseType(ev.union_type));
1461 if (ev.union_type.base_type != BASE_TYPE_STRUCT &&
1462 ev.union_type.base_type != BASE_TYPE_STRING)
1463 return Error("union value type may only be table/struct/string");
1464 enum_def.uses_type_aliases = true;
1465 } else {
1466 ev.union_type = Type(BASE_TYPE_STRUCT, LookupCreateStruct(full_name));
1467 }
1468 }
1469 if (Is('=')) {
1470 NEXT();
1471 ev.value = StringToInt(attribute_.c_str());
1472 EXPECT(kTokenIntegerConstant);
1473 if (!opts.proto_mode && prevsize &&
1474 enum_def.vals.vec[prevsize - 1]->value >= ev.value)
1475 return Error("enum values must be specified in ascending order");
1476 }
1477 if (is_union) {
1478 if (ev.value < 0 || ev.value >= 256)
1479 return Error("union enum value must fit in a ubyte");
1480 }
1481 if (opts.proto_mode && Is('[')) {
1482 NEXT();
1483 // ignore attributes on enums.
1484 while (token_ != ']') NEXT();
1485 NEXT();
1486 }
1487 }
1488 if (!Is(opts.proto_mode ? ';' : ',')) break;
1489 NEXT();
1490 if (Is('}')) break;
1491 }
1492 EXPECT('}');
1493 if (enum_def.attributes.Lookup("bit_flags")) {
1494 for (auto it = enum_def.vals.vec.begin(); it != enum_def.vals.vec.end();
1495 ++it) {
1496 if (static_cast<size_t>((*it)->value) >=
1497 SizeOf(enum_def.underlying_type.base_type) * 8)
1498 return Error("bit flag out of range of underlying integral type");
1499 (*it)->value = 1LL << (*it)->value;
1500 }
1501 }
1502 if (dest) *dest = &enum_def;
1503 types_.Add(current_namespace_->GetFullyQualifiedName(enum_def.name),
1504 new Type(BASE_TYPE_UNION, nullptr, &enum_def));
1505 return NoError();
1506 }
1507
StartStruct(const std::string & name,StructDef ** dest)1508 CheckedError Parser::StartStruct(const std::string &name, StructDef **dest) {
1509 auto &struct_def = *LookupCreateStruct(name, true, true);
1510 if (!struct_def.predecl) return Error("datatype already exists: " + name);
1511 struct_def.predecl = false;
1512 struct_def.name = name;
1513 struct_def.file = file_being_parsed_;
1514 // Move this struct to the back of the vector just in case it was predeclared,
1515 // to preserve declaration order.
1516 *std::remove(structs_.vec.begin(), structs_.vec.end(), &struct_def) = &struct_def;
1517 *dest = &struct_def;
1518 return NoError();
1519 }
1520
CheckClash(std::vector<FieldDef * > & fields,StructDef * struct_def,const char * suffix,BaseType basetype)1521 CheckedError Parser::CheckClash(std::vector<FieldDef*> &fields,
1522 StructDef *struct_def,
1523 const char *suffix,
1524 BaseType basetype) {
1525 auto len = strlen(suffix);
1526 for (auto it = fields.begin(); it != fields.end(); ++it) {
1527 auto &fname = (*it)->name;
1528 if (fname.length() > len &&
1529 fname.compare(fname.length() - len, len, suffix) == 0 &&
1530 (*it)->value.type.base_type != BASE_TYPE_UTYPE) {
1531 auto field = struct_def->fields.Lookup(
1532 fname.substr(0, fname.length() - len));
1533 if (field && field->value.type.base_type == basetype)
1534 return Error("Field " + fname +
1535 " would clash with generated functions for field " +
1536 field->name);
1537 }
1538 }
1539 return NoError();
1540 }
1541
SupportsVectorOfUnions() const1542 bool Parser::SupportsVectorOfUnions() const {
1543 return opts.lang_to_generate != 0 && (opts.lang_to_generate &
1544 ~(IDLOptions::kCpp | IDLOptions::kJs | IDLOptions::kTs | IDLOptions::kPhp)) == 0;
1545 }
1546
UniqueNamespace(Namespace * ns)1547 Namespace *Parser::UniqueNamespace(Namespace *ns) {
1548 for (auto it = namespaces_.begin(); it != namespaces_.end(); ++it) {
1549 if (ns->components == (*it)->components) {
1550 delete ns;
1551 return *it;
1552 }
1553 }
1554 namespaces_.push_back(ns);
1555 return ns;
1556 }
1557
compareFieldDefs(const FieldDef * a,const FieldDef * b)1558 static bool compareFieldDefs(const FieldDef *a, const FieldDef *b) {
1559 auto a_id = atoi(a->attributes.Lookup("id")->constant.c_str());
1560 auto b_id = atoi(b->attributes.Lookup("id")->constant.c_str());
1561 return a_id < b_id;
1562 }
1563
ParseDecl()1564 CheckedError Parser::ParseDecl() {
1565 std::vector<std::string> dc = doc_comment_;
1566 bool fixed = IsIdent("struct");
1567 if (!fixed && !IsIdent("table")) return Error("declaration expected");
1568 NEXT();
1569 std::string name = attribute_;
1570 EXPECT(kTokenIdentifier);
1571 StructDef *struct_def;
1572 ECHECK(StartStruct(name, &struct_def));
1573 struct_def->doc_comment = dc;
1574 struct_def->fixed = fixed;
1575 ECHECK(ParseMetaData(&struct_def->attributes));
1576 struct_def->sortbysize =
1577 struct_def->attributes.Lookup("original_order") == nullptr && !fixed;
1578 EXPECT('{');
1579 while (token_ != '}') ECHECK(ParseField(*struct_def));
1580 auto force_align = struct_def->attributes.Lookup("force_align");
1581 if (fixed && force_align) {
1582 auto align = static_cast<size_t>(atoi(force_align->constant.c_str()));
1583 if (force_align->type.base_type != BASE_TYPE_INT ||
1584 align < struct_def->minalign ||
1585 align > FLATBUFFERS_MAX_ALIGNMENT ||
1586 align & (align - 1))
1587 return Error("force_align must be a power of two integer ranging from the"
1588 "struct\'s natural alignment to " +
1589 NumToString(FLATBUFFERS_MAX_ALIGNMENT));
1590 struct_def->minalign = align;
1591 }
1592 struct_def->PadLastField(struct_def->minalign);
1593 // Check if this is a table that has manual id assignments
1594 auto &fields = struct_def->fields.vec;
1595 if (!struct_def->fixed && fields.size()) {
1596 size_t num_id_fields = 0;
1597 for (auto it = fields.begin(); it != fields.end(); ++it) {
1598 if ((*it)->attributes.Lookup("id")) num_id_fields++;
1599 }
1600 // If any fields have ids..
1601 if (num_id_fields) {
1602 // Then all fields must have them.
1603 if (num_id_fields != fields.size())
1604 return Error(
1605 "either all fields or no fields must have an 'id' attribute");
1606 // Simply sort by id, then the fields are the same as if no ids had
1607 // been specified.
1608 std::sort(fields.begin(), fields.end(), compareFieldDefs);
1609 // Verify we have a contiguous set, and reassign vtable offsets.
1610 for (int i = 0; i < static_cast<int>(fields.size()); i++) {
1611 if (i != atoi(fields[i]->attributes.Lookup("id")->constant.c_str()))
1612 return Error("field id\'s must be consecutive from 0, id " +
1613 NumToString(i) + " missing or set twice");
1614 fields[i]->value.offset = FieldIndexToOffset(static_cast<voffset_t>(i));
1615 }
1616 }
1617 }
1618
1619 ECHECK(CheckClash(fields, struct_def, UnionTypeFieldSuffix(),
1620 BASE_TYPE_UNION));
1621 ECHECK(CheckClash(fields, struct_def, "Type", BASE_TYPE_UNION));
1622 ECHECK(CheckClash(fields, struct_def, "_length", BASE_TYPE_VECTOR));
1623 ECHECK(CheckClash(fields, struct_def, "Length", BASE_TYPE_VECTOR));
1624 ECHECK(CheckClash(fields, struct_def, "_byte_vector", BASE_TYPE_STRING));
1625 ECHECK(CheckClash(fields, struct_def, "ByteVector", BASE_TYPE_STRING));
1626 EXPECT('}');
1627 types_.Add(current_namespace_->GetFullyQualifiedName(struct_def->name),
1628 new Type(BASE_TYPE_STRUCT, struct_def, nullptr));
1629 return NoError();
1630 }
1631
ParseService()1632 CheckedError Parser::ParseService() {
1633 std::vector<std::string> service_comment = doc_comment_;
1634 NEXT();
1635 auto service_name = attribute_;
1636 EXPECT(kTokenIdentifier);
1637 auto &service_def = *new ServiceDef();
1638 service_def.name = service_name;
1639 service_def.file = file_being_parsed_;
1640 service_def.doc_comment = service_comment;
1641 service_def.defined_namespace = current_namespace_;
1642 if (services_.Add(current_namespace_->GetFullyQualifiedName(service_name),
1643 &service_def))
1644 return Error("service already exists: " + service_name);
1645 ECHECK(ParseMetaData(&service_def.attributes));
1646 EXPECT('{');
1647 do {
1648 auto rpc_name = attribute_;
1649 EXPECT(kTokenIdentifier);
1650 EXPECT('(');
1651 Type reqtype, resptype;
1652 ECHECK(ParseTypeIdent(reqtype));
1653 EXPECT(')');
1654 EXPECT(':');
1655 ECHECK(ParseTypeIdent(resptype));
1656 if (reqtype.base_type != BASE_TYPE_STRUCT || reqtype.struct_def->fixed ||
1657 resptype.base_type != BASE_TYPE_STRUCT || resptype.struct_def->fixed)
1658 return Error("rpc request and response types must be tables");
1659 auto &rpc = *new RPCCall();
1660 rpc.name = rpc_name;
1661 rpc.request = reqtype.struct_def;
1662 rpc.response = resptype.struct_def;
1663 if (service_def.calls.Add(rpc_name, &rpc))
1664 return Error("rpc already exists: " + rpc_name);
1665 ECHECK(ParseMetaData(&rpc.attributes));
1666 EXPECT(';');
1667 } while (token_ != '}');
1668 NEXT();
1669 return NoError();
1670 }
1671
SetRootType(const char * name)1672 bool Parser::SetRootType(const char *name) {
1673 root_struct_def_ = LookupStruct(name);
1674 if (!root_struct_def_)
1675 root_struct_def_ = LookupStruct(
1676 current_namespace_->GetFullyQualifiedName(name));
1677 return root_struct_def_ != nullptr;
1678 }
1679
MarkGenerated()1680 void Parser::MarkGenerated() {
1681 // This function marks all existing definitions as having already
1682 // been generated, which signals no code for included files should be
1683 // generated.
1684 for (auto it = enums_.vec.begin();
1685 it != enums_.vec.end(); ++it) {
1686 (*it)->generated = true;
1687 }
1688 for (auto it = structs_.vec.begin();
1689 it != structs_.vec.end(); ++it) {
1690 if (!(*it)->predecl) {
1691 (*it)->generated = true;
1692 }
1693 }
1694 for (auto it = services_.vec.begin();
1695 it != services_.vec.end(); ++it) {
1696 (*it)->generated = true;
1697 }
1698 }
1699
ParseNamespace()1700 CheckedError Parser::ParseNamespace() {
1701 NEXT();
1702 auto ns = new Namespace();
1703 namespaces_.push_back(ns); // Store it here to not leak upon error.
1704 if (token_ != ';') {
1705 for (;;) {
1706 ns->components.push_back(attribute_);
1707 EXPECT(kTokenIdentifier);
1708 if (Is('.')) NEXT() else break;
1709 }
1710 }
1711 namespaces_.pop_back();
1712 current_namespace_ = UniqueNamespace(ns);
1713 EXPECT(';');
1714 return NoError();
1715 }
1716
compareEnumVals(const EnumVal * a,const EnumVal * b)1717 static bool compareEnumVals(const EnumVal *a, const EnumVal* b) {
1718 return a->value < b->value;
1719 }
1720
1721 // Best effort parsing of .proto declarations, with the aim to turn them
1722 // in the closest corresponding FlatBuffer equivalent.
1723 // We parse everything as identifiers instead of keywords, since we don't
1724 // want protobuf keywords to become invalid identifiers in FlatBuffers.
ParseProtoDecl()1725 CheckedError Parser::ParseProtoDecl() {
1726 bool isextend = IsIdent("extend");
1727 if (IsIdent("package")) {
1728 // These are identical in syntax to FlatBuffer's namespace decl.
1729 ECHECK(ParseNamespace());
1730 } else if (IsIdent("message") || isextend) {
1731 std::vector<std::string> struct_comment = doc_comment_;
1732 NEXT();
1733 StructDef *struct_def = nullptr;
1734 Namespace *parent_namespace = nullptr;
1735 if (isextend) {
1736 if (Is('.')) NEXT(); // qualified names may start with a . ?
1737 auto id = attribute_;
1738 EXPECT(kTokenIdentifier);
1739 ECHECK(ParseNamespacing(&id, nullptr));
1740 struct_def = LookupCreateStruct(id, false);
1741 if (!struct_def)
1742 return Error("cannot extend unknown message type: " + id);
1743 } else {
1744 std::string name = attribute_;
1745 EXPECT(kTokenIdentifier);
1746 ECHECK(StartStruct(name, &struct_def));
1747 // Since message definitions can be nested, we create a new namespace.
1748 auto ns = new Namespace();
1749 // Copy of current namespace.
1750 *ns = *current_namespace_;
1751 // But with current message name.
1752 ns->components.push_back(name);
1753 ns->from_table++;
1754 parent_namespace = current_namespace_;
1755 current_namespace_ = UniqueNamespace(ns);
1756 }
1757 struct_def->doc_comment = struct_comment;
1758 ECHECK(ParseProtoFields(struct_def, isextend, false));
1759 if (!isextend) {
1760 current_namespace_ = parent_namespace;
1761 }
1762 if (Is(';')) NEXT();
1763 } else if (IsIdent("enum")) {
1764 // These are almost the same, just with different terminator:
1765 EnumDef *enum_def;
1766 ECHECK(ParseEnum(false, &enum_def));
1767 if (Is(';')) NEXT();
1768 // Protobuf allows them to be specified in any order, so sort afterwards.
1769 auto &v = enum_def->vals.vec;
1770 std::sort(v.begin(), v.end(), compareEnumVals);
1771
1772 // Temp: remove any duplicates, as .fbs files can't handle them.
1773 for (auto it = v.begin(); it != v.end(); ) {
1774 if (it != v.begin() && it[0]->value == it[-1]->value) it = v.erase(it);
1775 else ++it;
1776 }
1777 } else if (IsIdent("syntax")) { // Skip these.
1778 NEXT();
1779 EXPECT('=');
1780 EXPECT(kTokenStringConstant);
1781 EXPECT(';');
1782 } else if (IsIdent("option")) { // Skip these.
1783 ECHECK(ParseProtoOption());
1784 EXPECT(';');
1785 } else if (IsIdent("service")) { // Skip these.
1786 NEXT();
1787 EXPECT(kTokenIdentifier);
1788 ECHECK(ParseProtoCurliesOrIdent());
1789 } else {
1790 return Error("don\'t know how to parse .proto declaration starting with " +
1791 TokenToStringId(token_));
1792 }
1793 return NoError();
1794 }
1795
ParseProtoFields(StructDef * struct_def,bool isextend,bool inside_oneof)1796 CheckedError Parser::ParseProtoFields(StructDef *struct_def, bool isextend,
1797 bool inside_oneof) {
1798 EXPECT('{');
1799 while (token_ != '}') {
1800 if (IsIdent("message") || IsIdent("extend") || IsIdent("enum")) {
1801 // Nested declarations.
1802 ECHECK(ParseProtoDecl());
1803 } else if (IsIdent("extensions")) { // Skip these.
1804 NEXT();
1805 EXPECT(kTokenIntegerConstant);
1806 if (Is(kTokenIdentifier)) {
1807 NEXT(); // to
1808 NEXT(); // num
1809 }
1810 EXPECT(';');
1811 } else if (IsIdent("option")) { // Skip these.
1812 ECHECK(ParseProtoOption());
1813 EXPECT(';');
1814 } else if (IsIdent("reserved")) { // Skip these.
1815 NEXT();
1816 while (!Is(';')) { NEXT(); } // A variety of formats, just skip.
1817 NEXT();
1818 } else {
1819 std::vector<std::string> field_comment = doc_comment_;
1820 // Parse the qualifier.
1821 bool required = false;
1822 bool repeated = false;
1823 bool oneof = false;
1824 if (!inside_oneof) {
1825 if (IsIdent("optional")) {
1826 // This is the default.
1827 NEXT();
1828 } else if (IsIdent("required")) {
1829 required = true;
1830 NEXT();
1831 } else if (IsIdent("repeated")) {
1832 repeated = true;
1833 NEXT();
1834 } else if (IsIdent("oneof")) {
1835 oneof = true;
1836 NEXT();
1837 } else {
1838 // can't error, proto3 allows decls without any of the above.
1839 }
1840 }
1841 StructDef *anonymous_struct = nullptr;
1842 Type type;
1843 if (IsIdent("group") || oneof) {
1844 if (!oneof) NEXT();
1845 auto name = "Anonymous" + NumToString(anonymous_counter++);
1846 ECHECK(StartStruct(name, &anonymous_struct));
1847 type = Type(BASE_TYPE_STRUCT, anonymous_struct);
1848 } else {
1849 ECHECK(ParseTypeFromProtoType(&type));
1850 }
1851 // Repeated elements get mapped to a vector.
1852 if (repeated) {
1853 type.element = type.base_type;
1854 type.base_type = BASE_TYPE_VECTOR;
1855 if (type.element == BASE_TYPE_VECTOR) {
1856 // We have a vector or vectors, which FlatBuffers doesn't support.
1857 // For now make it a vector of string (since the source is likely
1858 // "repeated bytes").
1859 // TODO(wvo): A better solution would be to wrap this in a table.
1860 type.element = BASE_TYPE_STRING;
1861 }
1862 }
1863 std::string name = attribute_;
1864 EXPECT(kTokenIdentifier);
1865 if (!oneof) {
1866 // Parse the field id. Since we're just translating schemas, not
1867 // any kind of binary compatibility, we can safely ignore these, and
1868 // assign our own.
1869 EXPECT('=');
1870 EXPECT(kTokenIntegerConstant);
1871 }
1872 FieldDef *field = nullptr;
1873 if (isextend) {
1874 // We allow a field to be re-defined when extending.
1875 // TODO: are there situations where that is problematic?
1876 field = struct_def->fields.Lookup(name);
1877 }
1878 if (!field) ECHECK(AddField(*struct_def, name, type, &field));
1879 field->doc_comment = field_comment;
1880 if (!IsScalar(type.base_type)) field->required = required;
1881 // See if there's a default specified.
1882 if (Is('[')) {
1883 NEXT();
1884 for (;;) {
1885 auto key = attribute_;
1886 ECHECK(ParseProtoKey());
1887 EXPECT('=');
1888 auto val = attribute_;
1889 ECHECK(ParseProtoCurliesOrIdent());
1890 if (key == "default") {
1891 // Temp: skip non-numeric defaults (enums).
1892 auto numeric = strpbrk(val.c_str(), "0123456789-+.");
1893 if (IsScalar(type.base_type) && numeric == val.c_str())
1894 field->value.constant = val;
1895 } else if (key == "deprecated") {
1896 field->deprecated = val == "true";
1897 }
1898 if (!Is(',')) break;
1899 NEXT();
1900 }
1901 EXPECT(']');
1902 }
1903 if (anonymous_struct) {
1904 ECHECK(ParseProtoFields(anonymous_struct, false, oneof));
1905 if (Is(';')) NEXT();
1906 } else {
1907 EXPECT(';');
1908 }
1909 }
1910 }
1911 NEXT();
1912 return NoError();
1913 }
1914
ParseProtoKey()1915 CheckedError Parser::ParseProtoKey() {
1916 if (token_ == '(') {
1917 NEXT();
1918 // Skip "(a.b)" style custom attributes.
1919 while (token_ == '.' || token_ == kTokenIdentifier) NEXT();
1920 EXPECT(')');
1921 while (Is('.')) { NEXT(); EXPECT(kTokenIdentifier); }
1922 } else {
1923 EXPECT(kTokenIdentifier);
1924 }
1925 return NoError();
1926 }
1927
ParseProtoCurliesOrIdent()1928 CheckedError Parser::ParseProtoCurliesOrIdent() {
1929 if (Is('{')) {
1930 NEXT();
1931 for (int nesting = 1; nesting; ) {
1932 if (token_ == '{') nesting++;
1933 else if (token_ == '}') nesting--;
1934 NEXT();
1935 }
1936 } else {
1937 NEXT(); // Any single token.
1938 }
1939 return NoError();
1940 }
1941
ParseProtoOption()1942 CheckedError Parser::ParseProtoOption() {
1943 NEXT();
1944 ECHECK(ParseProtoKey());
1945 EXPECT('=');
1946 ECHECK(ParseProtoCurliesOrIdent());
1947 return NoError();
1948 }
1949
1950 // Parse a protobuf type, and map it to the corresponding FlatBuffer one.
ParseTypeFromProtoType(Type * type)1951 CheckedError Parser::ParseTypeFromProtoType(Type *type) {
1952 struct type_lookup { const char *proto_type; BaseType fb_type, element; };
1953 static type_lookup lookup[] = {
1954 { "float", BASE_TYPE_FLOAT, BASE_TYPE_NONE },
1955 { "double", BASE_TYPE_DOUBLE, BASE_TYPE_NONE },
1956 { "int32", BASE_TYPE_INT, BASE_TYPE_NONE },
1957 { "int64", BASE_TYPE_LONG, BASE_TYPE_NONE },
1958 { "uint32", BASE_TYPE_UINT, BASE_TYPE_NONE },
1959 { "uint64", BASE_TYPE_ULONG, BASE_TYPE_NONE },
1960 { "sint32", BASE_TYPE_INT, BASE_TYPE_NONE },
1961 { "sint64", BASE_TYPE_LONG, BASE_TYPE_NONE },
1962 { "fixed32", BASE_TYPE_UINT, BASE_TYPE_NONE },
1963 { "fixed64", BASE_TYPE_ULONG, BASE_TYPE_NONE },
1964 { "sfixed32", BASE_TYPE_INT, BASE_TYPE_NONE },
1965 { "sfixed64", BASE_TYPE_LONG, BASE_TYPE_NONE },
1966 { "bool", BASE_TYPE_BOOL, BASE_TYPE_NONE },
1967 { "string", BASE_TYPE_STRING, BASE_TYPE_NONE },
1968 { "bytes", BASE_TYPE_VECTOR, BASE_TYPE_UCHAR },
1969 { nullptr, BASE_TYPE_NONE, BASE_TYPE_NONE }
1970 };
1971 for (auto tl = lookup; tl->proto_type; tl++) {
1972 if (attribute_ == tl->proto_type) {
1973 type->base_type = tl->fb_type;
1974 type->element = tl->element;
1975 NEXT();
1976 return NoError();
1977 }
1978 }
1979 if (Is('.')) NEXT(); // qualified names may start with a . ?
1980 ECHECK(ParseTypeIdent(*type));
1981 return NoError();
1982 }
1983
SkipAnyJsonValue()1984 CheckedError Parser::SkipAnyJsonValue() {
1985 switch (token_) {
1986 case '{': {
1987 size_t fieldn_outer = 0;
1988 return ParseTableDelimiters(fieldn_outer, nullptr,
1989 [](const std::string &,
1990 size_t &fieldn, const StructDef *,
1991 void *state) -> CheckedError {
1992 auto *parser = static_cast<Parser *>(state);
1993 ECHECK(parser->SkipAnyJsonValue());
1994 fieldn++;
1995 return NoError();
1996 },
1997 this);
1998 }
1999 case '[': {
2000 size_t count = 0;
2001 return ParseVectorDelimiters(count, [](size_t &,
2002 void *state) -> CheckedError {
2003 return static_cast<Parser *>(state)->SkipAnyJsonValue();
2004 },
2005 this);
2006 }
2007 case kTokenStringConstant:
2008 case kTokenIntegerConstant:
2009 case kTokenFloatConstant:
2010 NEXT();
2011 break;
2012 default:
2013 if (IsIdent("true") || IsIdent("false") || IsIdent("null")) { NEXT(); }
2014 else return TokenError();
2015 }
2016 return NoError();
2017 }
2018
ParseFlexBufferValue(flexbuffers::Builder * builder)2019 CheckedError Parser::ParseFlexBufferValue(flexbuffers::Builder *builder) {
2020 switch (token_) {
2021 case '{': {
2022 std::pair<Parser *, flexbuffers::Builder *> parser_and_builder_state(
2023 this, builder);
2024 auto start = builder->StartMap();
2025 size_t fieldn_outer = 0;
2026 auto err = ParseTableDelimiters(fieldn_outer, nullptr,
2027 [](const std::string &name,
2028 size_t &fieldn, const StructDef *,
2029 void *state) -> CheckedError {
2030 auto *parser_and_builder =
2031 static_cast<std::pair<Parser *, flexbuffers::Builder *> *>(
2032 state);
2033 auto *parser = parser_and_builder->first;
2034 auto *current_builder = parser_and_builder->second;
2035 current_builder->Key(name);
2036 ECHECK(parser->ParseFlexBufferValue(current_builder));
2037 fieldn++;
2038 return NoError();
2039 },
2040 &parser_and_builder_state);
2041 ECHECK(err);
2042 builder->EndMap(start);
2043 break;
2044 }
2045 case '[':{
2046 auto start = builder->StartVector();
2047 size_t count = 0;
2048 std::pair<Parser *, flexbuffers::Builder *> parser_and_builder_state(
2049 this, builder);
2050 ECHECK(ParseVectorDelimiters(count, [](size_t &,
2051 void *state) -> CheckedError {
2052 auto *parser_and_builder =
2053 static_cast<std::pair<Parser *, flexbuffers::Builder *> *>(
2054 state);
2055 return parser_and_builder->first->ParseFlexBufferValue(
2056 parser_and_builder->second);
2057 },
2058 &parser_and_builder_state));
2059 builder->EndVector(start, false, false);
2060 break;
2061 }
2062 case kTokenStringConstant:
2063 builder->String(attribute_);
2064 EXPECT(kTokenStringConstant);
2065 break;
2066 case kTokenIntegerConstant:
2067 builder->Int(StringToInt(attribute_.c_str()));
2068 EXPECT(kTokenIntegerConstant);
2069 break;
2070 case kTokenFloatConstant:
2071 builder->Double(strtod(attribute_.c_str(), nullptr));
2072 EXPECT(kTokenFloatConstant);
2073 break;
2074 default:
2075 if (IsIdent("true")) { builder->Bool(true); NEXT(); }
2076 else if (IsIdent("false")) { builder->Bool(false); NEXT(); }
2077 else if (IsIdent("null")) { builder->Null(); NEXT(); }
2078 else return TokenError();
2079 }
2080 return NoError();
2081 }
2082
ParseFlexBuffer(const char * source,const char * source_filename,flexbuffers::Builder * builder)2083 bool Parser::ParseFlexBuffer(const char *source, const char *source_filename,
2084 flexbuffers::Builder *builder) {
2085 auto ok = !StartParseFile(source, source_filename).Check() &&
2086 !ParseFlexBufferValue(builder).Check();
2087 if (ok) builder->Finish();
2088 return ok;
2089 }
2090
Parse(const char * source,const char ** include_paths,const char * source_filename)2091 bool Parser::Parse(const char *source, const char **include_paths,
2092 const char *source_filename) {
2093 return !ParseRoot(source, include_paths, source_filename).Check();
2094 }
2095
StartParseFile(const char * source,const char * source_filename)2096 CheckedError Parser::StartParseFile(const char *source, const char *source_filename) {
2097 file_being_parsed_ = source_filename ? source_filename : "";
2098 source_ = cursor_ = source;
2099 line_ = 1;
2100 error_.clear();
2101 ECHECK(SkipByteOrderMark());
2102 NEXT();
2103 if (Is(kTokenEof))
2104 return Error("input file is empty");
2105 return NoError();
2106 }
2107
ParseRoot(const char * source,const char ** include_paths,const char * source_filename)2108 CheckedError Parser::ParseRoot(const char *source, const char **include_paths,
2109 const char *source_filename) {
2110 ECHECK(DoParse(source, include_paths, source_filename, nullptr));
2111
2112 // Check that all types were defined.
2113 for (auto it = structs_.vec.begin(); it != structs_.vec.end(); ) {
2114 auto &struct_def = **it;
2115 if (struct_def.predecl) {
2116 if (opts.proto_mode) {
2117 // Protos allow enums to be used before declaration, so check if that
2118 // is the case here.
2119 EnumDef *enum_def = nullptr;
2120 for (size_t components = struct_def.defined_namespace->
2121 components.size() + 1;
2122 components && !enum_def; components--) {
2123 auto qualified_name = struct_def.defined_namespace->
2124 GetFullyQualifiedName(struct_def.name,
2125 components - 1);
2126 enum_def = LookupEnum(qualified_name);
2127 }
2128 if (enum_def) {
2129 // This is pretty slow, but a simple solution for now.
2130 auto initial_count = struct_def.refcount;
2131 for (auto struct_it = structs_.vec.begin();
2132 struct_it != structs_.vec.end();
2133 ++struct_it) {
2134 auto &sd = **struct_it;
2135 for (auto field_it = sd.fields.vec.begin();
2136 field_it != sd.fields.vec.end();
2137 ++field_it) {
2138 auto &field = **field_it;
2139 if (field.value.type.struct_def == &struct_def) {
2140 field.value.type.struct_def = nullptr;
2141 field.value.type.enum_def = enum_def;
2142 auto &bt = field.value.type.base_type == BASE_TYPE_VECTOR
2143 ? field.value.type.element
2144 : field.value.type.base_type;
2145 assert(bt == BASE_TYPE_STRUCT);
2146 bt = enum_def->underlying_type.base_type;
2147 struct_def.refcount--;
2148 enum_def->refcount++;
2149 }
2150 }
2151 }
2152 if (struct_def.refcount)
2153 return Error("internal: " + NumToString(struct_def.refcount) + "/" +
2154 NumToString(initial_count) +
2155 " use(s) of pre-declaration enum not accounted for: "
2156 + enum_def->name);
2157 structs_.dict.erase(structs_.dict.find(struct_def.name));
2158 it = structs_.vec.erase(it);
2159 delete &struct_def;
2160 continue; // Skip error.
2161 }
2162 }
2163 auto err = "type referenced but not defined (check namespace): " +
2164 struct_def.name;
2165 if (struct_def.original_location)
2166 err += ", originally at: " + *struct_def.original_location;
2167 return Error(err);
2168 }
2169 ++it;
2170 }
2171
2172 // This check has to happen here and not earlier, because only now do we
2173 // know for sure what the type of these are.
2174 for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) {
2175 auto &enum_def = **it;
2176 if (enum_def.is_union) {
2177 for (auto val_it = enum_def.vals.vec.begin();
2178 val_it != enum_def.vals.vec.end();
2179 ++val_it) {
2180 auto &val = **val_it;
2181 if (!SupportsVectorOfUnions() &&
2182 val.union_type.struct_def && val.union_type.struct_def->fixed)
2183 return Error(
2184 "only tables can be union elements in the generated language: "
2185 + val.name);
2186 }
2187 }
2188 }
2189 return NoError();
2190 }
2191
DoParse(const char * source,const char ** include_paths,const char * source_filename,const char * include_filename)2192 CheckedError Parser::DoParse(const char *source,
2193 const char **include_paths,
2194 const char *source_filename,
2195 const char *include_filename) {
2196 if (source_filename &&
2197 included_files_.find(source_filename) == included_files_.end()) {
2198 included_files_[source_filename] = include_filename ? include_filename : "";
2199 files_included_per_file_[source_filename] = std::set<std::string>();
2200 }
2201 if (!include_paths) {
2202 static const char *current_directory[] = { "", nullptr };
2203 include_paths = current_directory;
2204 }
2205 field_stack_.clear();
2206 builder_.Clear();
2207 // Start with a blank namespace just in case this file doesn't have one.
2208 current_namespace_ = empty_namespace_;
2209
2210 ECHECK(StartParseFile(source, source_filename));
2211
2212 // Includes must come before type declarations:
2213 for (;;) {
2214 // Parse pre-include proto statements if any:
2215 if (opts.proto_mode &&
2216 (attribute_ == "option" || attribute_ == "syntax" ||
2217 attribute_ == "package")) {
2218 ECHECK(ParseProtoDecl());
2219 } else if (IsIdent("native_include")) {
2220 NEXT();
2221 vector_emplace_back(&native_included_files_, attribute_);
2222 EXPECT(kTokenStringConstant);
2223 } else if (IsIdent("include") || (opts.proto_mode && IsIdent("import"))) {
2224 NEXT();
2225 if (opts.proto_mode && attribute_ == "public") NEXT();
2226 auto name = flatbuffers::PosixPath(attribute_.c_str());
2227 EXPECT(kTokenStringConstant);
2228 // Look for the file in include_paths.
2229 std::string filepath;
2230 for (auto paths = include_paths; paths && *paths; paths++) {
2231 filepath = flatbuffers::ConCatPathFileName(*paths, name);
2232 if(FileExists(filepath.c_str())) break;
2233 }
2234 if (filepath.empty())
2235 return Error("unable to locate include file: " + name);
2236 if (source_filename)
2237 files_included_per_file_[source_filename].insert(filepath);
2238 if (included_files_.find(filepath) == included_files_.end()) {
2239 // We found an include file that we have not parsed yet.
2240 // Load it and parse it.
2241 std::string contents;
2242 if (!LoadFile(filepath.c_str(), true, &contents))
2243 return Error("unable to load include file: " + name);
2244 ECHECK(DoParse(contents.c_str(), include_paths, filepath.c_str(),
2245 name.c_str()));
2246 // We generally do not want to output code for any included files:
2247 if (!opts.generate_all) MarkGenerated();
2248 // Reset these just in case the included file had them, and the
2249 // parent doesn't.
2250 root_struct_def_ = nullptr;
2251 file_identifier_.clear();
2252 file_extension_.clear();
2253 // This is the easiest way to continue this file after an include:
2254 // instead of saving and restoring all the state, we simply start the
2255 // file anew. This will cause it to encounter the same include
2256 // statement again, but this time it will skip it, because it was
2257 // entered into included_files_.
2258 // This is recursive, but only go as deep as the number of include
2259 // statements.
2260 return DoParse(source, include_paths, source_filename, include_filename);
2261 }
2262 EXPECT(';');
2263 } else {
2264 break;
2265 }
2266 }
2267 // Now parse all other kinds of declarations:
2268 while (token_ != kTokenEof) {
2269 if (opts.proto_mode) {
2270 ECHECK(ParseProtoDecl());
2271 } else if (IsIdent("namespace")) {
2272 ECHECK(ParseNamespace());
2273 } else if (token_ == '{') {
2274 if (!root_struct_def_)
2275 return Error("no root type set to parse json with");
2276 if (builder_.GetSize()) {
2277 return Error("cannot have more than one json object in a file");
2278 }
2279 uoffset_t toff;
2280 ECHECK(ParseTable(*root_struct_def_, nullptr, &toff));
2281 builder_.Finish(Offset<Table>(toff),
2282 file_identifier_.length() ? file_identifier_.c_str() : nullptr);
2283 } else if (IsIdent("enum")) {
2284 ECHECK(ParseEnum(false, nullptr));
2285 } else if (IsIdent("union")) {
2286 ECHECK(ParseEnum(true, nullptr));
2287 } else if (IsIdent("root_type")) {
2288 NEXT();
2289 auto root_type = attribute_;
2290 EXPECT(kTokenIdentifier);
2291 ECHECK(ParseNamespacing(&root_type, nullptr));
2292 if (!SetRootType(root_type.c_str()))
2293 return Error("unknown root type: " + root_type);
2294 if (root_struct_def_->fixed)
2295 return Error("root type must be a table");
2296 EXPECT(';');
2297 } else if (IsIdent("file_identifier")) {
2298 NEXT();
2299 file_identifier_ = attribute_;
2300 EXPECT(kTokenStringConstant);
2301 if (file_identifier_.length() !=
2302 FlatBufferBuilder::kFileIdentifierLength)
2303 return Error("file_identifier must be exactly " +
2304 NumToString(FlatBufferBuilder::kFileIdentifierLength) +
2305 " characters");
2306 EXPECT(';');
2307 } else if (IsIdent("file_extension")) {
2308 NEXT();
2309 file_extension_ = attribute_;
2310 EXPECT(kTokenStringConstant);
2311 EXPECT(';');
2312 } else if(IsIdent("include")) {
2313 return Error("includes must come before declarations");
2314 } else if(IsIdent("attribute")) {
2315 NEXT();
2316 auto name = attribute_;
2317 EXPECT(kTokenStringConstant);
2318 EXPECT(';');
2319 known_attributes_[name] = false;
2320 } else if (IsIdent("rpc_service")) {
2321 ECHECK(ParseService());
2322 } else {
2323 ECHECK(ParseDecl());
2324 }
2325 }
2326 return NoError();
2327 }
2328
GetIncludedFilesRecursive(const std::string & file_name) const2329 std::set<std::string> Parser::GetIncludedFilesRecursive(
2330 const std::string &file_name) const {
2331 std::set<std::string> included_files;
2332 std::list<std::string> to_process;
2333
2334 if (file_name.empty()) return included_files;
2335 to_process.push_back(file_name);
2336
2337 while (!to_process.empty()) {
2338 std::string current = to_process.front();
2339 to_process.pop_front();
2340 included_files.insert(current);
2341
2342 // Workaround the lack of const accessor in C++98 maps.
2343 auto &new_files =
2344 (*const_cast<std::map<std::string, std::set<std::string>> *>(
2345 &files_included_per_file_))[current];
2346 for (auto it = new_files.begin(); it != new_files.end(); ++it) {
2347 if (included_files.find(*it) == included_files.end())
2348 to_process.push_back(*it);
2349 }
2350 }
2351
2352 return included_files;
2353 }
2354
2355 // Schema serialization functionality:
2356
compareName(const T * a,const T * b)2357 template<typename T> bool compareName(const T* a, const T* b) {
2358 return a->defined_namespace->GetFullyQualifiedName(a->name)
2359 < b->defined_namespace->GetFullyQualifiedName(b->name);
2360 }
2361
AssignIndices(const std::vector<T * > & defvec)2362 template<typename T> void AssignIndices(const std::vector<T *> &defvec) {
2363 // Pre-sort these vectors, such that we can set the correct indices for them.
2364 auto vec = defvec;
2365 std::sort(vec.begin(), vec.end(), compareName<T>);
2366 for (int i = 0; i < static_cast<int>(vec.size()); i++) vec[i]->index = i;
2367 }
2368
Serialize()2369 void Parser::Serialize() {
2370 builder_.Clear();
2371 AssignIndices(structs_.vec);
2372 AssignIndices(enums_.vec);
2373 std::vector<Offset<reflection::Object>> object_offsets;
2374 for (auto it = structs_.vec.begin(); it != structs_.vec.end(); ++it) {
2375 auto offset = (*it)->Serialize(&builder_, *this);
2376 object_offsets.push_back(offset);
2377 (*it)->serialized_location = offset.o;
2378 }
2379 std::vector<Offset<reflection::Enum>> enum_offsets;
2380 for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) {
2381 auto offset = (*it)->Serialize(&builder_, *this);
2382 enum_offsets.push_back(offset);
2383 (*it)->serialized_location = offset.o;
2384 }
2385 auto schema_offset = reflection::CreateSchema(
2386 builder_,
2387 builder_.CreateVectorOfSortedTables(&object_offsets),
2388 builder_.CreateVectorOfSortedTables(&enum_offsets),
2389 builder_.CreateString(file_identifier_),
2390 builder_.CreateString(file_extension_),
2391 root_struct_def_
2392 ? root_struct_def_->serialized_location
2393 : 0);
2394 builder_.Finish(schema_offset, reflection::SchemaIdentifier());
2395 }
2396
Serialize(FlatBufferBuilder * builder,const Parser & parser) const2397 Offset<reflection::Object> StructDef::Serialize(FlatBufferBuilder *builder,
2398 const Parser &parser) const {
2399 std::vector<Offset<reflection::Field>> field_offsets;
2400 for (auto it = fields.vec.begin(); it != fields.vec.end(); ++it) {
2401 field_offsets.push_back(
2402 (*it)->Serialize(builder,
2403 static_cast<uint16_t>(it - fields.vec.begin()), parser));
2404 }
2405 auto qualified_name = defined_namespace->GetFullyQualifiedName(name);
2406 return reflection::CreateObject(*builder,
2407 builder->CreateString(qualified_name),
2408 builder->CreateVectorOfSortedTables(
2409 &field_offsets),
2410 fixed,
2411 static_cast<int>(minalign),
2412 static_cast<int>(bytesize),
2413 SerializeAttributes(builder, parser),
2414 parser.opts.binary_schema_comments
2415 ? builder->CreateVectorOfStrings(
2416 doc_comment)
2417 : 0);
2418 }
2419
Serialize(FlatBufferBuilder * builder,uint16_t id,const Parser & parser) const2420 Offset<reflection::Field> FieldDef::Serialize(FlatBufferBuilder *builder,
2421 uint16_t id,
2422 const Parser &parser) const {
2423 return reflection::CreateField(*builder,
2424 builder->CreateString(name),
2425 value.type.Serialize(builder),
2426 id,
2427 value.offset,
2428 IsInteger(value.type.base_type)
2429 ? StringToInt(value.constant.c_str())
2430 : 0,
2431 IsFloat(value.type.base_type)
2432 ? strtod(value.constant.c_str(), nullptr)
2433 : 0.0,
2434 deprecated,
2435 required,
2436 key,
2437 SerializeAttributes(builder, parser),
2438 parser.opts.binary_schema_comments
2439 ? builder->CreateVectorOfStrings(doc_comment)
2440 : 0);
2441 // TODO: value.constant is almost always "0", we could save quite a bit of
2442 // space by sharing it. Same for common values of value.type.
2443 }
2444
Serialize(FlatBufferBuilder * builder,const Parser & parser) const2445 Offset<reflection::Enum> EnumDef::Serialize(FlatBufferBuilder *builder,
2446 const Parser &parser) const {
2447 std::vector<Offset<reflection::EnumVal>> enumval_offsets;
2448 for (auto it = vals.vec.begin(); it != vals.vec.end(); ++it) {
2449 enumval_offsets.push_back((*it)->Serialize(builder));
2450 }
2451 auto qualified_name = defined_namespace->GetFullyQualifiedName(name);
2452 return reflection::CreateEnum(*builder,
2453 builder->CreateString(qualified_name),
2454 builder->CreateVector(enumval_offsets),
2455 is_union,
2456 underlying_type.Serialize(builder),
2457 SerializeAttributes(builder, parser),
2458 parser.opts.binary_schema_comments
2459 ? builder->CreateVectorOfStrings(doc_comment)
2460 : 0);
2461 }
2462
Serialize(FlatBufferBuilder * builder) const2463 Offset<reflection::EnumVal> EnumVal::Serialize(FlatBufferBuilder *builder) const
2464 {
2465 return reflection::CreateEnumVal(*builder,
2466 builder->CreateString(name),
2467 value,
2468 union_type.struct_def
2469 ? union_type.struct_def->
2470 serialized_location
2471 : 0,
2472 union_type.Serialize(builder));
2473 }
2474
Serialize(FlatBufferBuilder * builder) const2475 Offset<reflection::Type> Type::Serialize(FlatBufferBuilder *builder) const {
2476 return reflection::CreateType(*builder,
2477 static_cast<reflection::BaseType>(base_type),
2478 static_cast<reflection::BaseType>(element),
2479 struct_def ? struct_def->index :
2480 (enum_def ? enum_def->index : -1));
2481 }
2482
2483 flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<
2484 reflection::KeyValue>>>
SerializeAttributes(FlatBufferBuilder * builder,const Parser & parser) const2485 Definition::SerializeAttributes(FlatBufferBuilder *builder,
2486 const Parser &parser) const {
2487 std::vector<flatbuffers::Offset<reflection::KeyValue>> attrs;
2488 for (auto kv = attributes.dict.begin(); kv != attributes.dict.end(); ++kv) {
2489 auto it = parser.known_attributes_.find(kv->first);
2490 assert(it != parser.known_attributes_.end());
2491 if (!it->second) { // Custom attribute.
2492 attrs.push_back(
2493 reflection::CreateKeyValue(*builder, builder->CreateString(kv->first),
2494 builder->CreateString(
2495 kv->second->constant)));
2496 }
2497 }
2498 if (attrs.size()) {
2499 return builder->CreateVectorOfSortedTables(&attrs);
2500 } else {
2501 return 0;
2502 }
2503 }
2504
ConformTo(const Parser & base)2505 std::string Parser::ConformTo(const Parser &base) {
2506 for (auto sit = structs_.vec.begin(); sit != structs_.vec.end(); ++sit) {
2507 auto &struct_def = **sit;
2508 auto qualified_name =
2509 struct_def.defined_namespace->GetFullyQualifiedName(struct_def.name);
2510 auto struct_def_base = base.LookupStruct(qualified_name);
2511 if (!struct_def_base) continue;
2512 for (auto fit = struct_def.fields.vec.begin();
2513 fit != struct_def.fields.vec.end(); ++fit) {
2514 auto &field = **fit;
2515 auto field_base = struct_def_base->fields.Lookup(field.name);
2516 if (field_base) {
2517 if (field.value.offset != field_base->value.offset)
2518 return "offsets differ for field: " + field.name;
2519 if (field.value.constant != field_base->value.constant)
2520 return "defaults differ for field: " + field.name;
2521 if (!EqualByName(field.value.type, field_base->value.type))
2522 return "types differ for field: " + field.name;
2523 } else {
2524 // Doesn't have to exist, deleting fields is fine.
2525 // But we should check if there is a field that has the same offset
2526 // but is incompatible (in the case of field renaming).
2527 for (auto fbit = struct_def_base->fields.vec.begin();
2528 fbit != struct_def_base->fields.vec.end(); ++fbit) {
2529 field_base = *fbit;
2530 if (field.value.offset == field_base->value.offset) {
2531 if (!EqualByName(field.value.type, field_base->value.type))
2532 return "field renamed to different type: " + field.name;
2533 break;
2534 }
2535 }
2536 }
2537 }
2538 }
2539 for (auto eit = enums_.vec.begin(); eit != enums_.vec.end(); ++eit) {
2540 auto &enum_def = **eit;
2541 auto qualified_name =
2542 enum_def.defined_namespace->GetFullyQualifiedName(enum_def.name);
2543 auto enum_def_base = base.enums_.Lookup(qualified_name);
2544 if (!enum_def_base) continue;
2545 for (auto evit = enum_def.vals.vec.begin();
2546 evit != enum_def.vals.vec.end(); ++evit) {
2547 auto &enum_val = **evit;
2548 auto enum_val_base = enum_def_base->vals.Lookup(enum_val.name);
2549 if (enum_val_base) {
2550 if (enum_val.value != enum_val_base->value)
2551 return "values differ for enum: " + enum_val.name;
2552 }
2553 }
2554 }
2555 return "";
2556 }
2557
2558 } // namespace flatbuffers
2559