1 /*
2 * Copyright 2014 Google Inc. All rights reserved.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <algorithm>
18 #include <cmath>
19 #include <list>
20 #include <string>
21 #include <utility>
22
23 #include "flatbuffers/base.h"
24 #include "flatbuffers/idl.h"
25 #include "flatbuffers/util.h"
26
27 namespace flatbuffers {
28
29 // Reflects the version at the compiling time of binary(lib/dll/so).
FLATBUFFERS_VERSION()30 const char *FLATBUFFERS_VERSION() {
31 // clang-format off
32 return
33 FLATBUFFERS_STRING(FLATBUFFERS_VERSION_MAJOR) "."
34 FLATBUFFERS_STRING(FLATBUFFERS_VERSION_MINOR) "."
35 FLATBUFFERS_STRING(FLATBUFFERS_VERSION_REVISION);
36 // clang-format on
37 }
38
39 namespace {
40
41 static const double kPi = 3.14159265358979323846;
42
43 // The enums in the reflection schema should match the ones we use internally.
44 // Compare the last element to check if these go out of sync.
45 static_assert(BASE_TYPE_UNION == static_cast<BaseType>(reflection::Union),
46 "enums don't match");
47
48 // Any parsing calls have to be wrapped in this macro, which automates
49 // handling of recursive error checking a bit. It will check the received
50 // CheckedError object, and return straight away on error.
51 #define ECHECK(call) \
52 { \
53 auto ce = (call); \
54 if (ce.Check()) return ce; \
55 }
56
57 // These two functions are called hundreds of times below, so define a short
58 // form:
59 #define NEXT() ECHECK(Next())
60 #define EXPECT(tok) ECHECK(Expect(tok))
61
ValidateUTF8(const std::string & str)62 static bool ValidateUTF8(const std::string &str) {
63 const char *s = &str[0];
64 const char *const sEnd = s + str.length();
65 while (s < sEnd) {
66 if (FromUTF8(&s) < 0) { return false; }
67 }
68 return true;
69 }
70
IsLowerSnakeCase(const std::string & str)71 static bool IsLowerSnakeCase(const std::string &str) {
72 for (size_t i = 0; i < str.length(); i++) {
73 char c = str[i];
74 if (!check_ascii_range(c, 'a', 'z') && !is_digit(c) && c != '_') {
75 return false;
76 }
77 }
78 return true;
79 }
80
DeserializeDoc(std::vector<std::string> & doc,const Vector<Offset<String>> * documentation)81 static void DeserializeDoc(std::vector<std::string> &doc,
82 const Vector<Offset<String>> *documentation) {
83 if (documentation == nullptr) return;
84 for (uoffset_t index = 0; index < documentation->size(); index++)
85 doc.push_back(documentation->Get(index)->str());
86 }
87
NoError()88 static CheckedError NoError() { return CheckedError(false); }
89
90 template<typename T>
TypeToIntervalString()91 static std::string TypeToIntervalString() {
92 return "[" + NumToString((flatbuffers::numeric_limits<T>::lowest)()) + "; " +
93 NumToString((flatbuffers::numeric_limits<T>::max)()) + "]";
94 }
95
96
97 // atot: template version of atoi/atof: convert a string to an instance of T.
98 template<typename T>
atot_scalar(const char * s,T * val,bool_constant<false>)99 static bool atot_scalar(const char *s, T *val, bool_constant<false>) {
100 return StringToNumber(s, val);
101 }
102
103 template<typename T>
atot_scalar(const char * s,T * val,bool_constant<true>)104 static bool atot_scalar(const char *s, T *val, bool_constant<true>) {
105 // Normalize NaN parsed from fbs or json to unsigned NaN.
106 if (false == StringToNumber(s, val)) return false;
107 *val = (*val != *val) ? std::fabs(*val) : *val;
108 return true;
109 }
110
111 template<typename T>
atot(const char * s,Parser & parser,T * val)112 static CheckedError atot(const char *s, Parser &parser, T *val) {
113 auto done = atot_scalar(s, val, bool_constant<is_floating_point<T>::value>());
114 if (done) return NoError();
115 if (0 == *val)
116 return parser.Error("invalid number: \"" + std::string(s) + "\"");
117 else
118 return parser.Error("invalid number: \"" + std::string(s) + "\"" +
119 ", constant does not fit " + TypeToIntervalString<T>());
120 }
121 template<>
atot(const char * s,Parser & parser,Offset<void> * val)122 CheckedError atot<Offset<void>>(const char *s, Parser &parser,
123 Offset<void> *val) {
124 (void)parser;
125 *val = Offset<void>(atoi(s));
126 return NoError();
127 }
128
129 template<typename T>
LookupTableByName(const SymbolTable<T> & table,const std::string & name,const Namespace & current_namespace,size_t skip_top)130 static T *LookupTableByName(const SymbolTable<T> &table, const std::string &name,
131 const Namespace ¤t_namespace, size_t skip_top) {
132 const auto &components = current_namespace.components;
133 if (table.dict.empty()) return nullptr;
134 if (components.size() < skip_top) return nullptr;
135 const auto N = components.size() - skip_top;
136 std::string full_name;
137 for (size_t i = 0; i < N; i++) {
138 full_name += components[i];
139 full_name += '.';
140 }
141 for (size_t i = N; i > 0; i--) {
142 full_name += name;
143 auto obj = table.Lookup(full_name);
144 if (obj) return obj;
145 auto len = full_name.size() - components[i - 1].size() - 1 - name.size();
146 full_name.resize(len);
147 }
148 FLATBUFFERS_ASSERT(full_name.empty());
149 return table.Lookup(name); // lookup in global namespace
150 }
151
152 // Declare tokens we'll use. Single character tokens are represented by their
153 // ascii character code (e.g. '{'), others above 256.
154 // clang-format off
155 #define FLATBUFFERS_GEN_TOKENS(TD) \
156 TD(Eof, 256, "end of file") \
157 TD(StringConstant, 257, "string constant") \
158 TD(IntegerConstant, 258, "integer constant") \
159 TD(FloatConstant, 259, "float constant") \
160 TD(Identifier, 260, "identifier")
161 #ifdef __GNUC__
162 __extension__ // Stop GCC complaining about trailing comma with -Wpendantic.
163 #endif
164 enum {
165 #define FLATBUFFERS_TOKEN(NAME, VALUE, STRING) kToken ## NAME = VALUE,
166 FLATBUFFERS_GEN_TOKENS(FLATBUFFERS_TOKEN)
167 #undef FLATBUFFERS_TOKEN
168 };
169
TokenToString(int t)170 static std::string TokenToString(int t) {
171 static const char * const tokens[] = {
172 #define FLATBUFFERS_TOKEN(NAME, VALUE, STRING) STRING,
173 FLATBUFFERS_GEN_TOKENS(FLATBUFFERS_TOKEN)
174 #undef FLATBUFFERS_TOKEN
175 #define FLATBUFFERS_TD(ENUM, IDLTYPE, ...) \
176 IDLTYPE,
177 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
178 #undef FLATBUFFERS_TD
179 };
180 if (t < 256) { // A single ascii char token.
181 std::string s;
182 s.append(1, static_cast<char>(t));
183 return s;
184 } else { // Other tokens.
185 return tokens[t - 256];
186 }
187 }
188 // clang-format on
189
IsIdentifierStart(char c)190 static bool IsIdentifierStart(char c) {
191 return is_alpha(c) || (c == '_');
192 }
193
CompareSerializedScalars(const uint8_t * a,const uint8_t * b,const FieldDef & key)194 static bool CompareSerializedScalars(const uint8_t *a, const uint8_t *b,
195 const FieldDef &key) {
196 switch (key.value.type.base_type) {
197 #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, ...) \
198 case BASE_TYPE_##ENUM: { \
199 CTYPE def = static_cast<CTYPE>(0); \
200 if (!a || !b) { StringToNumber(key.value.constant.c_str(), &def); } \
201 const auto av = a ? ReadScalar<CTYPE>(a) : def; \
202 const auto bv = b ? ReadScalar<CTYPE>(b) : def; \
203 return av < bv; \
204 }
205 FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD)
206 #undef FLATBUFFERS_TD
207 default: {
208 FLATBUFFERS_ASSERT(false && "scalar type expected");
209 return false;
210 }
211 }
212 }
213
CompareTablesByScalarKey(const Offset<Table> * _a,const Offset<Table> * _b,const FieldDef & key)214 static bool CompareTablesByScalarKey(const Offset<Table> *_a,
215 const Offset<Table> *_b,
216 const FieldDef &key) {
217 const voffset_t offset = key.value.offset;
218 // Indirect offset pointer to table pointer.
219 auto a = reinterpret_cast<const uint8_t *>(_a) + ReadScalar<uoffset_t>(_a);
220 auto b = reinterpret_cast<const uint8_t *>(_b) + ReadScalar<uoffset_t>(_b);
221 // Fetch field address from table.
222 a = reinterpret_cast<const Table *>(a)->GetAddressOf(offset);
223 b = reinterpret_cast<const Table *>(b)->GetAddressOf(offset);
224 return CompareSerializedScalars(a, b, key);
225 }
226
CompareTablesByStringKey(const Offset<Table> * _a,const Offset<Table> * _b,const FieldDef & key)227 static bool CompareTablesByStringKey(const Offset<Table> *_a,
228 const Offset<Table> *_b,
229 const FieldDef &key) {
230 const voffset_t offset = key.value.offset;
231 // Indirect offset pointer to table pointer.
232 auto a = reinterpret_cast<const uint8_t *>(_a) + ReadScalar<uoffset_t>(_a);
233 auto b = reinterpret_cast<const uint8_t *>(_b) + ReadScalar<uoffset_t>(_b);
234 // Fetch field address from table.
235 a = reinterpret_cast<const Table *>(a)->GetAddressOf(offset);
236 b = reinterpret_cast<const Table *>(b)->GetAddressOf(offset);
237 if (a && b) {
238 // Indirect offset pointer to string pointer.
239 a += ReadScalar<uoffset_t>(a);
240 b += ReadScalar<uoffset_t>(b);
241 return *reinterpret_cast<const String *>(a) <
242 *reinterpret_cast<const String *>(b);
243 } else {
244 return a ? true : false;
245 }
246 }
247
SwapSerializedTables(Offset<Table> * a,Offset<Table> * b)248 static void SwapSerializedTables(Offset<Table> *a, Offset<Table> *b) {
249 // These are serialized offsets, so are relative where they are
250 // stored in memory, so compute the distance between these pointers:
251 ptrdiff_t diff = (b - a) * sizeof(Offset<Table>);
252 FLATBUFFERS_ASSERT(diff >= 0); // Guaranteed by SimpleQsort.
253 auto udiff = static_cast<uoffset_t>(diff);
254 a->o = EndianScalar(ReadScalar<uoffset_t>(a) - udiff);
255 b->o = EndianScalar(ReadScalar<uoffset_t>(b) + udiff);
256 std::swap(*a, *b);
257 }
258
259 // See below for why we need our own sort :(
260 template<typename T, typename F, typename S>
SimpleQsort(T * begin,T * end,size_t width,F comparator,S swapper)261 static void SimpleQsort(T *begin, T *end, size_t width, F comparator, S swapper) {
262 if (end - begin <= static_cast<ptrdiff_t>(width)) return;
263 auto l = begin + width;
264 auto r = end;
265 while (l < r) {
266 if (comparator(begin, l)) {
267 r -= width;
268 swapper(l, r);
269 } else {
270 l += width;
271 }
272 }
273 l -= width;
274 swapper(begin, l);
275 SimpleQsort(begin, l, width, comparator, swapper);
276 SimpleQsort(r, end, width, comparator, swapper);
277 }
278
279
280 template<typename T>
SingleValueRepack(Value & e,T val)281 static inline void SingleValueRepack(Value &e, T val) {
282 // Remove leading zeros.
283 if (IsInteger(e.type.base_type)) { e.constant = NumToString(val); }
284 }
285
286 #if defined(FLATBUFFERS_HAS_NEW_STRTOD) && (FLATBUFFERS_HAS_NEW_STRTOD > 0)
287 // Normalize defaults NaN to unsigned quiet-NaN(0) if value was parsed from
288 // hex-float literal.
SingleValueRepack(Value & e,float val)289 static void SingleValueRepack(Value &e, float val) {
290 if (val != val) e.constant = "nan";
291 }
SingleValueRepack(Value & e,double val)292 static void SingleValueRepack(Value &e, double val) {
293 if (val != val) e.constant = "nan";
294 }
295 #endif
296
297
EnumDistanceImpl(T e1,T e2)298 template<typename T> static uint64_t EnumDistanceImpl(T e1, T e2) {
299 if (e1 < e2) { std::swap(e1, e2); } // use std for scalars
300 // Signed overflow may occur, use unsigned calculation.
301 // The unsigned overflow is well-defined by C++ standard (modulo 2^n).
302 return static_cast<uint64_t>(e1) - static_cast<uint64_t>(e2);
303 }
304
compareFieldDefs(const FieldDef * a,const FieldDef * b)305 static bool compareFieldDefs(const FieldDef *a, const FieldDef *b) {
306 auto a_id = atoi(a->attributes.Lookup("id")->constant.c_str());
307 auto b_id = atoi(b->attributes.Lookup("id")->constant.c_str());
308 return a_id < b_id;
309 }
310
GetNamespace(const std::string & qualified_name,std::vector<Namespace * > & namespaces,std::map<std::string,Namespace * > & namespaces_index)311 static Namespace *GetNamespace(
312 const std::string &qualified_name, std::vector<Namespace *> &namespaces,
313 std::map<std::string, Namespace *> &namespaces_index) {
314 size_t dot = qualified_name.find_last_of('.');
315 std::string namespace_name = (dot != std::string::npos)
316 ? std::string(qualified_name.c_str(), dot)
317 : "";
318 Namespace *&ns = namespaces_index[namespace_name];
319
320 if (!ns) {
321 ns = new Namespace();
322 namespaces.push_back(ns);
323
324 size_t pos = 0;
325
326 for (;;) {
327 dot = qualified_name.find('.', pos);
328 if (dot == std::string::npos) { break; }
329 ns->components.push_back(qualified_name.substr(pos, dot - pos));
330 pos = dot + 1;
331 }
332 }
333
334 return ns;
335 }
336
337 // Generate a unique hash for a file based on its name and contents (if any).
HashFile(const char * source_filename,const char * source)338 static uint64_t HashFile(const char *source_filename, const char *source) {
339 uint64_t hash = 0;
340
341 if (source_filename)
342 hash = HashFnv1a<uint64_t>(StripPath(source_filename).c_str());
343
344 if (source && *source) hash ^= HashFnv1a<uint64_t>(source);
345
346 return hash;
347 }
348
349 template<typename T>
compareName(const T * a,const T * b)350 static bool compareName(const T *a, const T *b) {
351 return a->defined_namespace->GetFullyQualifiedName(a->name) <
352 b->defined_namespace->GetFullyQualifiedName(b->name);
353 }
354
355 template<typename T>
AssignIndices(const std::vector<T * > & defvec)356 static void AssignIndices(const std::vector<T *> &defvec) {
357 // Pre-sort these vectors, such that we can set the correct indices for them.
358 auto vec = defvec;
359 std::sort(vec.begin(), vec.end(), compareName<T>);
360 for (int i = 0; i < static_cast<int>(vec.size()); i++) vec[i]->index = i;
361 }
362
363 } // namespace
364
365
366
367
368
369
370 // clang-format off
371 const char *const kTypeNames[] = {
372 #define FLATBUFFERS_TD(ENUM, IDLTYPE, ...) \
373 IDLTYPE,
374 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
375 #undef FLATBUFFERS_TD
376 nullptr
377 };
378
379 const char kTypeSizes[] = {
380 #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, ...) \
381 sizeof(CTYPE),
382 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
383 #undef FLATBUFFERS_TD
384 };
385 // clang-format on
386
387
Message(const std::string & msg)388 void Parser::Message(const std::string &msg) {
389 if (!error_.empty()) error_ += "\n"; // log all warnings and errors
390 error_ += file_being_parsed_.length() ? AbsolutePath(file_being_parsed_) : "";
391 // clang-format off
392
393 #ifdef _WIN32 // MSVC alike
394 error_ +=
395 "(" + NumToString(line_) + ", " + NumToString(CursorPosition()) + ")";
396 #else // gcc alike
397 if (file_being_parsed_.length()) error_ += ":";
398 error_ += NumToString(line_) + ": " + NumToString(CursorPosition());
399 #endif
400 // clang-format on
401 error_ += ": " + msg;
402 }
403
Warning(const std::string & msg)404 void Parser::Warning(const std::string &msg) {
405 if (!opts.no_warnings) {
406 Message("warning: " + msg);
407 has_warning_ = true; // for opts.warnings_as_errors
408 }
409 }
410
Error(const std::string & msg)411 CheckedError Parser::Error(const std::string &msg) {
412 Message("error: " + msg);
413 return CheckedError(true);
414 }
415
RecurseError()416 CheckedError Parser::RecurseError() {
417 return Error("maximum parsing depth " + NumToString(parse_depth_counter_) +
418 " reached");
419 }
420
GetPooledString(const std::string & s) const421 const std::string &Parser::GetPooledString(const std::string &s) const {
422 return *(string_cache_.insert(s).first);
423 }
424
425 class Parser::ParseDepthGuard {
426 public:
ParseDepthGuard(Parser * parser_not_null)427 explicit ParseDepthGuard(Parser *parser_not_null)
428 : parser_(*parser_not_null), caller_depth_(parser_.parse_depth_counter_) {
429 FLATBUFFERS_ASSERT(caller_depth_ <= (FLATBUFFERS_MAX_PARSING_DEPTH) &&
430 "Check() must be called to prevent stack overflow");
431 parser_.parse_depth_counter_ += 1;
432 }
433
~ParseDepthGuard()434 ~ParseDepthGuard() { parser_.parse_depth_counter_ -= 1; }
435
Check()436 CheckedError Check() {
437 return caller_depth_ >= (FLATBUFFERS_MAX_PARSING_DEPTH)
438 ? parser_.RecurseError()
439 : CheckedError(false);
440 }
441
442 FLATBUFFERS_DELETE_FUNC(ParseDepthGuard(const ParseDepthGuard &));
443 FLATBUFFERS_DELETE_FUNC(ParseDepthGuard &operator=(const ParseDepthGuard &));
444
445 private:
446 Parser &parser_;
447 const int caller_depth_;
448 };
449
450
GetFullyQualifiedName(const std::string & name,size_t max_components) const451 std::string Namespace::GetFullyQualifiedName(const std::string &name,
452 size_t max_components) const {
453 // Early exit if we don't have a defined namespace.
454 if (components.empty() || !max_components) { return name; }
455 std::string stream_str;
456 for (size_t i = 0; i < std::min(components.size(), max_components); i++) {
457 stream_str += components[i];
458 stream_str += '.';
459 }
460 if (!stream_str.empty()) stream_str.pop_back();
461 if (name.length()) {
462 stream_str += '.';
463 stream_str += name;
464 }
465 return stream_str;
466 }
467
468
TokenToStringId(int t) const469 std::string Parser::TokenToStringId(int t) const {
470 return t == kTokenIdentifier ? attribute_ : TokenToString(t);
471 }
472
473 // Parses exactly nibbles worth of hex digits into a number, or error.
ParseHexNum(int nibbles,uint64_t * val)474 CheckedError Parser::ParseHexNum(int nibbles, uint64_t *val) {
475 FLATBUFFERS_ASSERT(nibbles > 0);
476 for (int i = 0; i < nibbles; i++)
477 if (!is_xdigit(cursor_[i]))
478 return Error("escape code must be followed by " + NumToString(nibbles) +
479 " hex digits");
480 std::string target(cursor_, cursor_ + nibbles);
481 *val = StringToUInt(target.c_str(), 16);
482 cursor_ += nibbles;
483 return NoError();
484 }
485
SkipByteOrderMark()486 CheckedError Parser::SkipByteOrderMark() {
487 if (static_cast<unsigned char>(*cursor_) != 0xef) return NoError();
488 cursor_++;
489 if (static_cast<unsigned char>(*cursor_) != 0xbb)
490 return Error("invalid utf-8 byte order mark");
491 cursor_++;
492 if (static_cast<unsigned char>(*cursor_) != 0xbf)
493 return Error("invalid utf-8 byte order mark");
494 cursor_++;
495 return NoError();
496 }
497
Next()498 CheckedError Parser::Next() {
499 doc_comment_.clear();
500 bool seen_newline = cursor_ == source_;
501 attribute_.clear();
502 attr_is_trivial_ascii_string_ = true;
503 for (;;) {
504 char c = *cursor_++;
505 token_ = c;
506 switch (c) {
507 case '\0':
508 cursor_--;
509 token_ = kTokenEof;
510 return NoError();
511 case ' ':
512 case '\r':
513 case '\t': break;
514 case '\n':
515 MarkNewLine();
516 seen_newline = true;
517 break;
518 case '{':
519 case '}':
520 case '(':
521 case ')':
522 case '[':
523 case ']':
524 case ',':
525 case ':':
526 case ';':
527 case '=': return NoError();
528 case '\"':
529 case '\'': {
530 int unicode_high_surrogate = -1;
531
532 while (*cursor_ != c) {
533 if (*cursor_ < ' ' && static_cast<signed char>(*cursor_) >= 0)
534 return Error("illegal character in string constant");
535 if (*cursor_ == '\\') {
536 attr_is_trivial_ascii_string_ = false; // has escape sequence
537 cursor_++;
538 if (unicode_high_surrogate != -1 && *cursor_ != 'u') {
539 return Error(
540 "illegal Unicode sequence (unpaired high surrogate)");
541 }
542 switch (*cursor_) {
543 case 'n':
544 attribute_ += '\n';
545 cursor_++;
546 break;
547 case 't':
548 attribute_ += '\t';
549 cursor_++;
550 break;
551 case 'r':
552 attribute_ += '\r';
553 cursor_++;
554 break;
555 case 'b':
556 attribute_ += '\b';
557 cursor_++;
558 break;
559 case 'f':
560 attribute_ += '\f';
561 cursor_++;
562 break;
563 case '\"':
564 attribute_ += '\"';
565 cursor_++;
566 break;
567 case '\'':
568 attribute_ += '\'';
569 cursor_++;
570 break;
571 case '\\':
572 attribute_ += '\\';
573 cursor_++;
574 break;
575 case '/':
576 attribute_ += '/';
577 cursor_++;
578 break;
579 case 'x': { // Not in the JSON standard
580 cursor_++;
581 uint64_t val;
582 ECHECK(ParseHexNum(2, &val));
583 attribute_ += static_cast<char>(val);
584 break;
585 }
586 case 'u': {
587 cursor_++;
588 uint64_t val;
589 ECHECK(ParseHexNum(4, &val));
590 if (val >= 0xD800 && val <= 0xDBFF) {
591 if (unicode_high_surrogate != -1) {
592 return Error(
593 "illegal Unicode sequence (multiple high surrogates)");
594 } else {
595 unicode_high_surrogate = static_cast<int>(val);
596 }
597 } else if (val >= 0xDC00 && val <= 0xDFFF) {
598 if (unicode_high_surrogate == -1) {
599 return Error(
600 "illegal Unicode sequence (unpaired low surrogate)");
601 } else {
602 int code_point = 0x10000 +
603 ((unicode_high_surrogate & 0x03FF) << 10) +
604 (val & 0x03FF);
605 ToUTF8(code_point, &attribute_);
606 unicode_high_surrogate = -1;
607 }
608 } else {
609 if (unicode_high_surrogate != -1) {
610 return Error(
611 "illegal Unicode sequence (unpaired high surrogate)");
612 }
613 ToUTF8(static_cast<int>(val), &attribute_);
614 }
615 break;
616 }
617 default: return Error("unknown escape code in string constant");
618 }
619 } else { // printable chars + UTF-8 bytes
620 if (unicode_high_surrogate != -1) {
621 return Error(
622 "illegal Unicode sequence (unpaired high surrogate)");
623 }
624 // reset if non-printable
625 attr_is_trivial_ascii_string_ &=
626 check_ascii_range(*cursor_, ' ', '~');
627
628 attribute_ += *cursor_++;
629 }
630 }
631 if (unicode_high_surrogate != -1) {
632 return Error("illegal Unicode sequence (unpaired high surrogate)");
633 }
634 cursor_++;
635 if (!attr_is_trivial_ascii_string_ && !opts.allow_non_utf8 &&
636 !ValidateUTF8(attribute_)) {
637 return Error("illegal UTF-8 sequence");
638 }
639 token_ = kTokenStringConstant;
640 return NoError();
641 }
642 case '/':
643 if (*cursor_ == '/') {
644 const char *start = ++cursor_;
645 while (*cursor_ && *cursor_ != '\n' && *cursor_ != '\r') cursor_++;
646 if (*start == '/') { // documentation comment
647 if (!seen_newline)
648 return Error(
649 "a documentation comment should be on a line on its own");
650 doc_comment_.push_back(std::string(start + 1, cursor_));
651 }
652 break;
653 } else if (*cursor_ == '*') {
654 cursor_++;
655 // TODO: make nested.
656 while (*cursor_ != '*' || cursor_[1] != '/') {
657 if (*cursor_ == '\n') MarkNewLine();
658 if (!*cursor_) return Error("end of file in comment");
659 cursor_++;
660 }
661 cursor_ += 2;
662 break;
663 }
664 FLATBUFFERS_FALLTHROUGH(); // else fall thru
665 default:
666 if (IsIdentifierStart(c)) {
667 // Collect all chars of an identifier:
668 const char *start = cursor_ - 1;
669 while (IsIdentifierStart(*cursor_) || is_digit(*cursor_)) cursor_++;
670 attribute_.append(start, cursor_);
671 token_ = kTokenIdentifier;
672 return NoError();
673 }
674
675 const auto has_sign = (c == '+') || (c == '-');
676 if (has_sign) {
677 // Check for +/-inf which is considered a float constant.
678 if (strncmp(cursor_, "inf", 3) == 0 &&
679 !(IsIdentifierStart(cursor_[3]) || is_digit(cursor_[3]))) {
680 attribute_.assign(cursor_ - 1, cursor_ + 3);
681 token_ = kTokenFloatConstant;
682 cursor_ += 3;
683 return NoError();
684 }
685
686 if (IsIdentifierStart(*cursor_)) {
687 // '-'/'+' and following identifier - it could be a predefined
688 // constant. Return the sign in token_, see ParseSingleValue.
689 return NoError();
690 }
691 }
692
693 auto dot_lvl =
694 (c == '.') ? 0 : 1; // dot_lvl==0 <=> exactly one '.' seen
695 if (!dot_lvl && !is_digit(*cursor_)) return NoError(); // enum?
696 // Parser accepts hexadecimal-floating-literal (see C++ 5.13.4).
697 if (is_digit(c) || has_sign || !dot_lvl) {
698 const auto start = cursor_ - 1;
699 auto start_digits = !is_digit(c) ? cursor_ : cursor_ - 1;
700 if (!is_digit(c) && is_digit(*cursor_)) {
701 start_digits = cursor_; // see digit in cursor_ position
702 c = *cursor_++;
703 }
704 // hex-float can't begind with '.'
705 auto use_hex = dot_lvl && (c == '0') && is_alpha_char(*cursor_, 'X');
706 if (use_hex) start_digits = ++cursor_; // '0x' is the prefix, skip it
707 // Read an integer number or mantisa of float-point number.
708 do {
709 if (use_hex) {
710 while (is_xdigit(*cursor_)) cursor_++;
711 } else {
712 while (is_digit(*cursor_)) cursor_++;
713 }
714 } while ((*cursor_ == '.') && (++cursor_) && (--dot_lvl >= 0));
715 // Exponent of float-point number.
716 if ((dot_lvl >= 0) && (cursor_ > start_digits)) {
717 // The exponent suffix of hexadecimal float number is mandatory.
718 if (use_hex && !dot_lvl) start_digits = cursor_;
719 if ((use_hex && is_alpha_char(*cursor_, 'P')) ||
720 is_alpha_char(*cursor_, 'E')) {
721 dot_lvl = 0; // Emulate dot to signal about float-point number.
722 cursor_++;
723 if (*cursor_ == '+' || *cursor_ == '-') cursor_++;
724 start_digits = cursor_; // the exponent-part has to have digits
725 // Exponent is decimal integer number
726 while (is_digit(*cursor_)) cursor_++;
727 if (*cursor_ == '.') {
728 cursor_++; // If see a dot treat it as part of invalid number.
729 dot_lvl = -1; // Fall thru to Error().
730 }
731 }
732 }
733 // Finalize.
734 if ((dot_lvl >= 0) && (cursor_ > start_digits)) {
735 attribute_.append(start, cursor_);
736 token_ = dot_lvl ? kTokenIntegerConstant : kTokenFloatConstant;
737 return NoError();
738 } else {
739 return Error("invalid number: " + std::string(start, cursor_));
740 }
741 }
742 std::string ch;
743 ch = c;
744 if (false == check_ascii_range(c, ' ', '~'))
745 ch = "code: " + NumToString(c);
746 return Error("illegal character: " + ch);
747 }
748 }
749 }
750
751 // Check if a given token is next.
Is(int t) const752 bool Parser::Is(int t) const { return t == token_; }
753
IsIdent(const char * id) const754 bool Parser::IsIdent(const char *id) const {
755 return token_ == kTokenIdentifier && attribute_ == id;
756 }
757
758 // Expect a given token to be next, consume it, or error if not present.
Expect(int t)759 CheckedError Parser::Expect(int t) {
760 if (t != token_) {
761 return Error("expecting: " + TokenToString(t) +
762 " instead got: " + TokenToStringId(token_));
763 }
764 NEXT();
765 return NoError();
766 }
767
ParseNamespacing(std::string * id,std::string * last)768 CheckedError Parser::ParseNamespacing(std::string *id, std::string *last) {
769 while (Is('.')) {
770 NEXT();
771 *id += ".";
772 *id += attribute_;
773 if (last) *last = attribute_;
774 EXPECT(kTokenIdentifier);
775 }
776 return NoError();
777 }
778
LookupEnum(const std::string & id)779 EnumDef *Parser::LookupEnum(const std::string &id) {
780 // Search thru parent namespaces.
781 return LookupTableByName(enums_, id, *current_namespace_, 0);
782 }
783
LookupStruct(const std::string & id) const784 StructDef *Parser::LookupStruct(const std::string &id) const {
785 auto sd = structs_.Lookup(id);
786 if (sd) sd->refcount++;
787 return sd;
788 }
789
LookupStructThruParentNamespaces(const std::string & id) const790 StructDef *Parser::LookupStructThruParentNamespaces(
791 const std::string &id) const {
792 auto sd = LookupTableByName(structs_, id, *current_namespace_, 1);
793 if (sd) sd->refcount++;
794 return sd;
795 }
796
ParseTypeIdent(Type & type)797 CheckedError Parser::ParseTypeIdent(Type &type) {
798 std::string id = attribute_;
799 EXPECT(kTokenIdentifier);
800 ECHECK(ParseNamespacing(&id, nullptr));
801 auto enum_def = LookupEnum(id);
802 if (enum_def) {
803 type = enum_def->underlying_type;
804 if (enum_def->is_union) type.base_type = BASE_TYPE_UNION;
805 } else {
806 type.base_type = BASE_TYPE_STRUCT;
807 type.struct_def = LookupCreateStruct(id);
808 }
809 return NoError();
810 }
811
812 // Parse any IDL type.
ParseType(Type & type)813 CheckedError Parser::ParseType(Type &type) {
814 if (token_ == kTokenIdentifier) {
815 if (IsIdent("bool")) {
816 type.base_type = BASE_TYPE_BOOL;
817 NEXT();
818 } else if (IsIdent("byte") || IsIdent("int8")) {
819 type.base_type = BASE_TYPE_CHAR;
820 NEXT();
821 } else if (IsIdent("ubyte") || IsIdent("uint8")) {
822 type.base_type = BASE_TYPE_UCHAR;
823 NEXT();
824 } else if (IsIdent("short") || IsIdent("int16")) {
825 type.base_type = BASE_TYPE_SHORT;
826 NEXT();
827 } else if (IsIdent("ushort") || IsIdent("uint16")) {
828 type.base_type = BASE_TYPE_USHORT;
829 NEXT();
830 } else if (IsIdent("int") || IsIdent("int32")) {
831 type.base_type = BASE_TYPE_INT;
832 NEXT();
833 } else if (IsIdent("uint") || IsIdent("uint32")) {
834 type.base_type = BASE_TYPE_UINT;
835 NEXT();
836 } else if (IsIdent("long") || IsIdent("int64")) {
837 type.base_type = BASE_TYPE_LONG;
838 NEXT();
839 } else if (IsIdent("ulong") || IsIdent("uint64")) {
840 type.base_type = BASE_TYPE_ULONG;
841 NEXT();
842 } else if (IsIdent("float") || IsIdent("float32")) {
843 type.base_type = BASE_TYPE_FLOAT;
844 NEXT();
845 } else if (IsIdent("double") || IsIdent("float64")) {
846 type.base_type = BASE_TYPE_DOUBLE;
847 NEXT();
848 } else if (IsIdent("string")) {
849 type.base_type = BASE_TYPE_STRING;
850 NEXT();
851 } else {
852 ECHECK(ParseTypeIdent(type));
853 }
854 } else if (token_ == '[') {
855 ParseDepthGuard depth_guard(this);
856 ECHECK(depth_guard.Check());
857 NEXT();
858 Type subtype;
859 ECHECK(ParseType(subtype));
860 if (IsSeries(subtype)) {
861 // We could support this, but it will complicate things, and it's
862 // easier to work around with a struct around the inner vector.
863 return Error("nested vector types not supported (wrap in table first)");
864 }
865 if (token_ == ':') {
866 NEXT();
867 if (token_ != kTokenIntegerConstant) {
868 return Error("length of fixed-length array must be an integer value");
869 }
870 uint16_t fixed_length = 0;
871 bool check = StringToNumber(attribute_.c_str(), &fixed_length);
872 if (!check || fixed_length < 1) {
873 return Error(
874 "length of fixed-length array must be positive and fit to "
875 "uint16_t type");
876 }
877 type = Type(BASE_TYPE_ARRAY, subtype.struct_def, subtype.enum_def,
878 fixed_length);
879 NEXT();
880 } else {
881 type = Type(BASE_TYPE_VECTOR, subtype.struct_def, subtype.enum_def);
882 }
883 type.element = subtype.base_type;
884 EXPECT(']');
885 } else {
886 return Error("illegal type syntax");
887 }
888 return NoError();
889 }
890
AddField(StructDef & struct_def,const std::string & name,const Type & type,FieldDef ** dest)891 CheckedError Parser::AddField(StructDef &struct_def, const std::string &name,
892 const Type &type, FieldDef **dest) {
893 auto &field = *new FieldDef();
894 field.value.offset =
895 FieldIndexToOffset(static_cast<voffset_t>(struct_def.fields.vec.size()));
896 field.name = name;
897 field.file = struct_def.file;
898 field.value.type = type;
899 if (struct_def.fixed) { // statically compute the field offset
900 auto size = InlineSize(type);
901 auto alignment = InlineAlignment(type);
902 // structs_ need to have a predictable format, so we need to align to
903 // the largest scalar
904 struct_def.minalign = std::max(struct_def.minalign, alignment);
905 struct_def.PadLastField(alignment);
906 field.value.offset = static_cast<voffset_t>(struct_def.bytesize);
907 struct_def.bytesize += size;
908 }
909 if (struct_def.fields.Add(name, &field))
910 return Error("field already exists: " + name);
911 *dest = &field;
912 return NoError();
913 }
914
ParseField(StructDef & struct_def)915 CheckedError Parser::ParseField(StructDef &struct_def) {
916 std::string name = attribute_;
917
918 if (LookupCreateStruct(name, false, false))
919 return Error("field name can not be the same as table/struct name");
920
921 if (!IsLowerSnakeCase(name)) {
922 Warning("field names should be lowercase snake_case, got: " + name);
923 }
924
925 std::vector<std::string> dc = doc_comment_;
926 EXPECT(kTokenIdentifier);
927 EXPECT(':');
928 Type type;
929 ECHECK(ParseType(type));
930
931 if (struct_def.fixed) {
932 auto valid = IsScalar(type.base_type) || IsStruct(type);
933 if (!valid && IsArray(type)) {
934 const auto &elem_type = type.VectorType();
935 valid |= IsScalar(elem_type.base_type) || IsStruct(elem_type);
936 }
937 if (!valid)
938 return Error("structs may contain only scalar or struct fields");
939 }
940
941 if (!struct_def.fixed && IsArray(type))
942 return Error("fixed-length array in table must be wrapped in struct");
943
944 if (IsArray(type)) {
945 advanced_features_ |= reflection::AdvancedArrayFeatures;
946 if (!SupportsAdvancedArrayFeatures()) {
947 return Error(
948 "Arrays are not yet supported in all "
949 "the specified programming languages.");
950 }
951 }
952
953 FieldDef *typefield = nullptr;
954 if (type.base_type == BASE_TYPE_UNION) {
955 // For union fields, add a second auto-generated field to hold the type,
956 // with a special suffix.
957 ECHECK(AddField(struct_def, name + UnionTypeFieldSuffix(),
958 type.enum_def->underlying_type, &typefield));
959 } else if (IsVector(type) && type.element == BASE_TYPE_UNION) {
960 advanced_features_ |= reflection::AdvancedUnionFeatures;
961 // Only cpp, js and ts supports the union vector feature so far.
962 if (!SupportsAdvancedUnionFeatures()) {
963 return Error(
964 "Vectors of unions are not yet supported in at least one of "
965 "the specified programming languages.");
966 }
967 // For vector of union fields, add a second auto-generated vector field to
968 // hold the types, with a special suffix.
969 Type union_vector(BASE_TYPE_VECTOR, nullptr, type.enum_def);
970 union_vector.element = BASE_TYPE_UTYPE;
971 ECHECK(AddField(struct_def, name + UnionTypeFieldSuffix(), union_vector,
972 &typefield));
973 }
974
975 FieldDef *field;
976 ECHECK(AddField(struct_def, name, type, &field));
977
978 if (token_ == '=') {
979 NEXT();
980 ECHECK(ParseSingleValue(&field->name, field->value, true));
981 if (IsStruct(type) || (struct_def.fixed && field->value.constant != "0"))
982 return Error(
983 "default values are not supported for struct fields, table fields, "
984 "or in structs.");
985 if (IsString(type) || IsVector(type)) {
986 advanced_features_ |= reflection::DefaultVectorsAndStrings;
987 if (field->value.constant != "0" && !SupportsDefaultVectorsAndStrings()) {
988 return Error(
989 "Default values for strings and vectors are not supported in one "
990 "of the specified programming languages");
991 }
992 }
993
994 if (IsVector(type) && field->value.constant != "0" &&
995 field->value.constant != "[]") {
996 return Error("The only supported default for vectors is `[]`.");
997 }
998 }
999
1000 // Append .0 if the value has not it (skip hex and scientific floats).
1001 // This suffix needed for generated C++ code.
1002 if (IsFloat(type.base_type)) {
1003 auto &text = field->value.constant;
1004 FLATBUFFERS_ASSERT(false == text.empty());
1005 auto s = text.c_str();
1006 while (*s == ' ') s++;
1007 if (*s == '-' || *s == '+') s++;
1008 // 1) A float constants (nan, inf, pi, etc) is a kind of identifier.
1009 // 2) A float number needn't ".0" at the end if it has exponent.
1010 if ((false == IsIdentifierStart(*s)) &&
1011 (std::string::npos == field->value.constant.find_first_of(".eEpP"))) {
1012 field->value.constant += ".0";
1013 }
1014 }
1015
1016 field->doc_comment = dc;
1017 ECHECK(ParseMetaData(&field->attributes));
1018 field->deprecated = field->attributes.Lookup("deprecated") != nullptr;
1019 auto hash_name = field->attributes.Lookup("hash");
1020 if (hash_name) {
1021 switch ((IsVector(type)) ? type.element : type.base_type) {
1022 case BASE_TYPE_SHORT:
1023 case BASE_TYPE_USHORT: {
1024 if (FindHashFunction16(hash_name->constant.c_str()) == nullptr)
1025 return Error("Unknown hashing algorithm for 16 bit types: " +
1026 hash_name->constant);
1027 break;
1028 }
1029 case BASE_TYPE_INT:
1030 case BASE_TYPE_UINT: {
1031 if (FindHashFunction32(hash_name->constant.c_str()) == nullptr)
1032 return Error("Unknown hashing algorithm for 32 bit types: " +
1033 hash_name->constant);
1034 break;
1035 }
1036 case BASE_TYPE_LONG:
1037 case BASE_TYPE_ULONG: {
1038 if (FindHashFunction64(hash_name->constant.c_str()) == nullptr)
1039 return Error("Unknown hashing algorithm for 64 bit types: " +
1040 hash_name->constant);
1041 break;
1042 }
1043 default:
1044 return Error(
1045 "only short, ushort, int, uint, long and ulong data types support "
1046 "hashing.");
1047 }
1048 }
1049
1050 // For historical convenience reasons, string keys are assumed required.
1051 // Scalars are kDefault unless otherwise specified.
1052 // Nonscalars are kOptional unless required;
1053 field->key = field->attributes.Lookup("key") != nullptr;
1054 const bool required = field->attributes.Lookup("required") != nullptr ||
1055 (IsString(type) && field->key);
1056 const bool default_str_or_vec =
1057 ((IsString(type) || IsVector(type)) && field->value.constant != "0");
1058 const bool optional = IsScalar(type.base_type)
1059 ? (field->value.constant == "null")
1060 : !(required || default_str_or_vec);
1061 if (required && optional) {
1062 return Error("Fields cannot be both optional and required.");
1063 }
1064 field->presence = FieldDef::MakeFieldPresence(optional, required);
1065
1066 if (required && (struct_def.fixed || IsScalar(type.base_type))) {
1067 return Error("only non-scalar fields in tables may be 'required'");
1068 }
1069 if (field->key) {
1070 if (struct_def.has_key) return Error("only one field may be set as 'key'");
1071 struct_def.has_key = true;
1072 if (!IsScalar(type.base_type) && !IsString(type)) {
1073 return Error("'key' field must be string or scalar type");
1074 }
1075 }
1076
1077 if (field->IsScalarOptional()) {
1078 advanced_features_ |= reflection::OptionalScalars;
1079 if (type.enum_def && type.enum_def->Lookup("null")) {
1080 FLATBUFFERS_ASSERT(IsInteger(type.base_type));
1081 return Error(
1082 "the default 'null' is reserved for declaring optional scalar "
1083 "fields, it conflicts with declaration of enum '" +
1084 type.enum_def->name + "'.");
1085 }
1086 if (field->attributes.Lookup("key")) {
1087 return Error(
1088 "only a non-optional scalar field can be used as a 'key' field");
1089 }
1090 if (!SupportsOptionalScalars()) {
1091 return Error(
1092 "Optional scalars are not yet supported in at least one of "
1093 "the specified programming languages.");
1094 }
1095 }
1096
1097 if (type.enum_def) {
1098 // Verify the enum's type and default value.
1099 const std::string &constant = field->value.constant;
1100 if (type.base_type == BASE_TYPE_UNION) {
1101 if (constant != "0") { return Error("Union defaults must be NONE"); }
1102 } else if (IsVector(type)) {
1103 if (constant != "0" && constant != "[]") {
1104 return Error("Vector defaults may only be `[]`.");
1105 }
1106 } else if (IsArray(type)) {
1107 if (constant != "0") {
1108 return Error("Array defaults are not supported yet.");
1109 }
1110 } else {
1111 if (!IsInteger(type.base_type)) {
1112 return Error("Enums must have integer base types");
1113 }
1114 // Optional and bitflags enums may have default constants that are not
1115 // their specified variants.
1116 if (!field->IsOptional() &&
1117 type.enum_def->attributes.Lookup("bit_flags") == nullptr) {
1118 if (type.enum_def->FindByValue(constant) == nullptr) {
1119 return Error("default value of `" + constant + "` for " + "field `" +
1120 name + "` is not part of enum `" + type.enum_def->name +
1121 "`.");
1122 }
1123 }
1124 }
1125 }
1126
1127 if (field->deprecated && struct_def.fixed)
1128 return Error("can't deprecate fields in a struct");
1129
1130 auto cpp_type = field->attributes.Lookup("cpp_type");
1131 if (cpp_type) {
1132 if (!hash_name)
1133 return Error("cpp_type can only be used with a hashed field");
1134 /// forcing cpp_ptr_type to 'naked' if unset
1135 auto cpp_ptr_type = field->attributes.Lookup("cpp_ptr_type");
1136 if (!cpp_ptr_type) {
1137 auto val = new Value();
1138 val->type = cpp_type->type;
1139 val->constant = "naked";
1140 field->attributes.Add("cpp_ptr_type", val);
1141 }
1142 }
1143
1144 field->shared = field->attributes.Lookup("shared") != nullptr;
1145 if (field->shared && field->value.type.base_type != BASE_TYPE_STRING)
1146 return Error("shared can only be defined on strings");
1147
1148 auto field_native_custom_alloc =
1149 field->attributes.Lookup("native_custom_alloc");
1150 if (field_native_custom_alloc)
1151 return Error(
1152 "native_custom_alloc can only be used with a table or struct "
1153 "definition");
1154
1155 field->native_inline = field->attributes.Lookup("native_inline") != nullptr;
1156 if (field->native_inline && !IsStruct(field->value.type))
1157 return Error("native_inline can only be defined on structs");
1158
1159 auto nested = field->attributes.Lookup("nested_flatbuffer");
1160 if (nested) {
1161 if (nested->type.base_type != BASE_TYPE_STRING)
1162 return Error(
1163 "nested_flatbuffer attribute must be a string (the root type)");
1164 if (type.base_type != BASE_TYPE_VECTOR || type.element != BASE_TYPE_UCHAR)
1165 return Error(
1166 "nested_flatbuffer attribute may only apply to a vector of ubyte");
1167 // This will cause an error if the root type of the nested flatbuffer
1168 // wasn't defined elsewhere.
1169 field->nested_flatbuffer = LookupCreateStruct(nested->constant);
1170 }
1171
1172 if (field->attributes.Lookup("flexbuffer")) {
1173 field->flexbuffer = true;
1174 uses_flexbuffers_ = true;
1175 if (type.base_type != BASE_TYPE_VECTOR || type.element != BASE_TYPE_UCHAR)
1176 return Error("flexbuffer attribute may only apply to a vector of ubyte");
1177 }
1178
1179 if (typefield) {
1180 if (!IsScalar(typefield->value.type.base_type)) {
1181 // this is a union vector field
1182 typefield->presence = field->presence;
1183 }
1184 // If this field is a union, and it has a manually assigned id,
1185 // the automatically added type field should have an id as well (of N - 1).
1186 auto attr = field->attributes.Lookup("id");
1187 if (attr) {
1188 const auto &id_str = attr->constant;
1189 voffset_t id = 0;
1190 const auto done = !atot(id_str.c_str(), *this, &id).Check();
1191 if (done && id > 0) {
1192 auto val = new Value();
1193 val->type = attr->type;
1194 val->constant = NumToString(id - 1);
1195 typefield->attributes.Add("id", val);
1196 } else {
1197 return Error(
1198 "a union type effectively adds two fields with non-negative ids, "
1199 "its id must be that of the second field (the first field is "
1200 "the type field and not explicitly declared in the schema);\n"
1201 "field: " +
1202 field->name + ", id: " + id_str);
1203 }
1204 }
1205 // if this field is a union that is deprecated,
1206 // the automatically added type field should be deprecated as well
1207 if (field->deprecated) { typefield->deprecated = true; }
1208 }
1209
1210 EXPECT(';');
1211 return NoError();
1212 }
1213
ParseString(Value & val,bool use_string_pooling)1214 CheckedError Parser::ParseString(Value &val, bool use_string_pooling) {
1215 auto s = attribute_;
1216 EXPECT(kTokenStringConstant);
1217 if (use_string_pooling) {
1218 val.constant = NumToString(builder_.CreateSharedString(s).o);
1219 } else {
1220 val.constant = NumToString(builder_.CreateString(s).o);
1221 }
1222 return NoError();
1223 }
1224
ParseComma()1225 CheckedError Parser::ParseComma() {
1226 if (!opts.protobuf_ascii_alike) EXPECT(',');
1227 return NoError();
1228 }
1229
ParseAnyValue(Value & val,FieldDef * field,size_t parent_fieldn,const StructDef * parent_struct_def,uoffset_t count,bool inside_vector)1230 CheckedError Parser::ParseAnyValue(Value &val, FieldDef *field,
1231 size_t parent_fieldn,
1232 const StructDef *parent_struct_def,
1233 uoffset_t count, bool inside_vector) {
1234 switch (val.type.base_type) {
1235 case BASE_TYPE_UNION: {
1236 FLATBUFFERS_ASSERT(field);
1237 std::string constant;
1238 Vector<uint8_t> *vector_of_union_types = nullptr;
1239 // Find corresponding type field we may have already parsed.
1240 for (auto elem = field_stack_.rbegin() + count;
1241 elem != field_stack_.rbegin() + parent_fieldn + count; ++elem) {
1242 auto &type = elem->second->value.type;
1243 if (type.enum_def == val.type.enum_def) {
1244 if (inside_vector) {
1245 if (IsVector(type) && type.element == BASE_TYPE_UTYPE) {
1246 // Vector of union type field.
1247 uoffset_t offset;
1248 ECHECK(atot(elem->first.constant.c_str(), *this, &offset));
1249 vector_of_union_types = reinterpret_cast<Vector<uint8_t> *>(
1250 builder_.GetCurrentBufferPointer() + builder_.GetSize() -
1251 offset);
1252 break;
1253 }
1254 } else {
1255 if (type.base_type == BASE_TYPE_UTYPE) {
1256 // Union type field.
1257 constant = elem->first.constant;
1258 break;
1259 }
1260 }
1261 }
1262 }
1263 if (constant.empty() && !inside_vector) {
1264 // We haven't seen the type field yet. Sadly a lot of JSON writers
1265 // output these in alphabetical order, meaning it comes after this
1266 // value. So we scan past the value to find it, then come back here.
1267 // We currently don't do this for vectors of unions because the
1268 // scanning/serialization logic would get very complicated.
1269 auto type_name = field->name + UnionTypeFieldSuffix();
1270 FLATBUFFERS_ASSERT(parent_struct_def);
1271 auto type_field = parent_struct_def->fields.Lookup(type_name);
1272 FLATBUFFERS_ASSERT(type_field); // Guaranteed by ParseField().
1273 // Remember where we are in the source file, so we can come back here.
1274 auto backup = *static_cast<ParserState *>(this);
1275 ECHECK(SkipAnyJsonValue()); // The table.
1276 ECHECK(ParseComma());
1277 auto next_name = attribute_;
1278 if (Is(kTokenStringConstant)) {
1279 NEXT();
1280 } else {
1281 EXPECT(kTokenIdentifier);
1282 }
1283 if (next_name == type_name) {
1284 EXPECT(':');
1285 ParseDepthGuard depth_guard(this);
1286 ECHECK(depth_guard.Check());
1287 Value type_val = type_field->value;
1288 ECHECK(ParseAnyValue(type_val, type_field, 0, nullptr, 0));
1289 constant = type_val.constant;
1290 // Got the information we needed, now rewind:
1291 *static_cast<ParserState *>(this) = backup;
1292 }
1293 }
1294 if (constant.empty() && !vector_of_union_types) {
1295 return Error("missing type field for this union value: " + field->name);
1296 }
1297 uint8_t enum_idx;
1298 if (vector_of_union_types) {
1299 if (vector_of_union_types->size() <= count)
1300 return Error(
1301 "union types vector smaller than union values vector for: " +
1302 field->name);
1303 enum_idx = vector_of_union_types->Get(count);
1304 } else {
1305 ECHECK(atot(constant.c_str(), *this, &enum_idx));
1306 }
1307 auto enum_val = val.type.enum_def->ReverseLookup(enum_idx, true);
1308 if (!enum_val) return Error("illegal type id for: " + field->name);
1309 if (enum_val->union_type.base_type == BASE_TYPE_STRUCT) {
1310 ECHECK(ParseTable(*enum_val->union_type.struct_def, &val.constant,
1311 nullptr));
1312 if (enum_val->union_type.struct_def->fixed) {
1313 // All BASE_TYPE_UNION values are offsets, so turn this into one.
1314 SerializeStruct(*enum_val->union_type.struct_def, val);
1315 builder_.ClearOffsets();
1316 val.constant = NumToString(builder_.GetSize());
1317 }
1318 } else if (IsString(enum_val->union_type)) {
1319 ECHECK(ParseString(val, field->shared));
1320 } else {
1321 FLATBUFFERS_ASSERT(false);
1322 }
1323 break;
1324 }
1325 case BASE_TYPE_STRUCT:
1326 ECHECK(ParseTable(*val.type.struct_def, &val.constant, nullptr));
1327 break;
1328 case BASE_TYPE_STRING: {
1329 ECHECK(ParseString(val, field->shared));
1330 break;
1331 }
1332 case BASE_TYPE_VECTOR: {
1333 uoffset_t off;
1334 ECHECK(ParseVector(val.type.VectorType(), &off, field, parent_fieldn));
1335 val.constant = NumToString(off);
1336 break;
1337 }
1338 case BASE_TYPE_ARRAY: {
1339 ECHECK(ParseArray(val));
1340 break;
1341 }
1342 case BASE_TYPE_INT:
1343 case BASE_TYPE_UINT:
1344 case BASE_TYPE_LONG:
1345 case BASE_TYPE_ULONG: {
1346 if (field && field->attributes.Lookup("hash") &&
1347 (token_ == kTokenIdentifier || token_ == kTokenStringConstant)) {
1348 ECHECK(ParseHash(val, field));
1349 } else {
1350 ECHECK(ParseSingleValue(field ? &field->name : nullptr, val, false));
1351 }
1352 break;
1353 }
1354 default:
1355 ECHECK(ParseSingleValue(field ? &field->name : nullptr, val, false));
1356 break;
1357 }
1358 return NoError();
1359 }
1360
SerializeStruct(const StructDef & struct_def,const Value & val)1361 void Parser::SerializeStruct(const StructDef &struct_def, const Value &val) {
1362 SerializeStruct(builder_, struct_def, val);
1363 }
1364
SerializeStruct(FlatBufferBuilder & builder,const StructDef & struct_def,const Value & val)1365 void Parser::SerializeStruct(FlatBufferBuilder &builder,
1366 const StructDef &struct_def, const Value &val) {
1367 FLATBUFFERS_ASSERT(val.constant.length() == struct_def.bytesize);
1368 builder.Align(struct_def.minalign);
1369 builder.PushBytes(reinterpret_cast<const uint8_t *>(val.constant.c_str()),
1370 struct_def.bytesize);
1371 builder.AddStructOffset(val.offset, builder.GetSize());
1372 }
1373
1374 template<typename F>
ParseTableDelimiters(size_t & fieldn,const StructDef * struct_def,F body)1375 CheckedError Parser::ParseTableDelimiters(size_t &fieldn,
1376 const StructDef *struct_def, F body) {
1377 // We allow tables both as JSON object{ .. } with field names
1378 // or vector[..] with all fields in order
1379 char terminator = '}';
1380 bool is_nested_vector = struct_def && Is('[');
1381 if (is_nested_vector) {
1382 NEXT();
1383 terminator = ']';
1384 } else {
1385 EXPECT('{');
1386 }
1387 for (;;) {
1388 if ((!opts.strict_json || !fieldn) && Is(terminator)) break;
1389 std::string name;
1390 if (is_nested_vector) {
1391 if (fieldn >= struct_def->fields.vec.size()) {
1392 return Error("too many unnamed fields in nested array");
1393 }
1394 name = struct_def->fields.vec[fieldn]->name;
1395 } else {
1396 name = attribute_;
1397 if (Is(kTokenStringConstant)) {
1398 NEXT();
1399 } else {
1400 EXPECT(opts.strict_json ? kTokenStringConstant : kTokenIdentifier);
1401 }
1402 if (!opts.protobuf_ascii_alike || !(Is('{') || Is('['))) EXPECT(':');
1403 }
1404 ECHECK(body(name, fieldn, struct_def));
1405 if (Is(terminator)) break;
1406 ECHECK(ParseComma());
1407 }
1408 NEXT();
1409 if (is_nested_vector && fieldn != struct_def->fields.vec.size()) {
1410 return Error("wrong number of unnamed fields in table vector");
1411 }
1412 return NoError();
1413 }
1414
ParseTable(const StructDef & struct_def,std::string * value,uoffset_t * ovalue)1415 CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
1416 uoffset_t *ovalue) {
1417 ParseDepthGuard depth_guard(this);
1418 ECHECK(depth_guard.Check());
1419
1420 size_t fieldn_outer = 0;
1421 auto err = ParseTableDelimiters(
1422 fieldn_outer, &struct_def,
1423 [&](const std::string &name, size_t &fieldn,
1424 const StructDef *struct_def_inner) -> CheckedError {
1425 if (name == "$schema") {
1426 ECHECK(Expect(kTokenStringConstant));
1427 return NoError();
1428 }
1429 auto field = struct_def_inner->fields.Lookup(name);
1430 if (!field) {
1431 if (!opts.skip_unexpected_fields_in_json) {
1432 return Error("unknown field: " + name);
1433 } else {
1434 ECHECK(SkipAnyJsonValue());
1435 }
1436 } else {
1437 if (IsIdent("null") && !IsScalar(field->value.type.base_type)) {
1438 ECHECK(Next()); // Ignore this field.
1439 } else {
1440 Value val = field->value;
1441 if (field->flexbuffer) {
1442 flexbuffers::Builder builder(1024,
1443 flexbuffers::BUILDER_FLAG_SHARE_ALL);
1444 ECHECK(ParseFlexBufferValue(&builder));
1445 builder.Finish();
1446 // Force alignment for nested flexbuffer
1447 builder_.ForceVectorAlignment(builder.GetSize(), sizeof(uint8_t),
1448 sizeof(largest_scalar_t));
1449 auto off = builder_.CreateVector(builder.GetBuffer());
1450 val.constant = NumToString(off.o);
1451 } else if (field->nested_flatbuffer) {
1452 ECHECK(
1453 ParseNestedFlatbuffer(val, field, fieldn, struct_def_inner));
1454 } else {
1455 ECHECK(ParseAnyValue(val, field, fieldn, struct_def_inner, 0));
1456 }
1457 // Hardcoded insertion-sort with error-check.
1458 // If fields are specified in order, then this loop exits
1459 // immediately.
1460 auto elem = field_stack_.rbegin();
1461 for (; elem != field_stack_.rbegin() + fieldn; ++elem) {
1462 auto existing_field = elem->second;
1463 if (existing_field == field)
1464 return Error("field set more than once: " + field->name);
1465 if (existing_field->value.offset < field->value.offset) break;
1466 }
1467 // Note: elem points to before the insertion point, thus .base()
1468 // points to the correct spot.
1469 field_stack_.insert(elem.base(), std::make_pair(val, field));
1470 fieldn++;
1471 }
1472 }
1473 return NoError();
1474 });
1475 ECHECK(err);
1476
1477 // Check if all required fields are parsed.
1478 for (auto field_it = struct_def.fields.vec.begin();
1479 field_it != struct_def.fields.vec.end(); ++field_it) {
1480 auto required_field = *field_it;
1481 if (!required_field->IsRequired()) { continue; }
1482 bool found = false;
1483 for (auto pf_it = field_stack_.end() - fieldn_outer;
1484 pf_it != field_stack_.end(); ++pf_it) {
1485 auto parsed_field = pf_it->second;
1486 if (parsed_field == required_field) {
1487 found = true;
1488 break;
1489 }
1490 }
1491 if (!found) {
1492 return Error("required field is missing: " + required_field->name +
1493 " in " + struct_def.name);
1494 }
1495 }
1496
1497 if (struct_def.fixed && fieldn_outer != struct_def.fields.vec.size())
1498 return Error("struct: wrong number of initializers: " + struct_def.name);
1499
1500 auto start = struct_def.fixed ? builder_.StartStruct(struct_def.minalign)
1501 : builder_.StartTable();
1502
1503 for (size_t size = struct_def.sortbysize ? sizeof(largest_scalar_t) : 1; size;
1504 size /= 2) {
1505 // Go through elements in reverse, since we're building the data backwards.
1506 for (auto it = field_stack_.rbegin();
1507 it != field_stack_.rbegin() + fieldn_outer; ++it) {
1508 auto &field_value = it->first;
1509 auto field = it->second;
1510 if (!struct_def.sortbysize ||
1511 size == SizeOf(field_value.type.base_type)) {
1512 switch (field_value.type.base_type) {
1513 // clang-format off
1514 #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, ...) \
1515 case BASE_TYPE_ ## ENUM: \
1516 builder_.Pad(field->padding); \
1517 if (struct_def.fixed) { \
1518 CTYPE val; \
1519 ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
1520 builder_.PushElement(val); \
1521 } else { \
1522 if (field->IsScalarOptional()) { \
1523 if (field_value.constant != "null") { \
1524 CTYPE val; \
1525 ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
1526 builder_.AddElement(field_value.offset, val); \
1527 } \
1528 } else { \
1529 CTYPE val, valdef; \
1530 ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
1531 ECHECK(atot(field->value.constant.c_str(), *this, &valdef)); \
1532 builder_.AddElement(field_value.offset, val, valdef); \
1533 } \
1534 } \
1535 break;
1536 FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD)
1537 #undef FLATBUFFERS_TD
1538 #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, ...) \
1539 case BASE_TYPE_ ## ENUM: \
1540 builder_.Pad(field->padding); \
1541 if (IsStruct(field->value.type)) { \
1542 SerializeStruct(*field->value.type.struct_def, field_value); \
1543 } else { \
1544 CTYPE val; \
1545 ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
1546 builder_.AddOffset(field_value.offset, val); \
1547 } \
1548 break;
1549 FLATBUFFERS_GEN_TYPES_POINTER(FLATBUFFERS_TD)
1550 #undef FLATBUFFERS_TD
1551 case BASE_TYPE_ARRAY:
1552 builder_.Pad(field->padding);
1553 builder_.PushBytes(
1554 reinterpret_cast<const uint8_t*>(field_value.constant.c_str()),
1555 InlineSize(field_value.type));
1556 break;
1557 // clang-format on
1558 }
1559 }
1560 }
1561 }
1562 for (size_t i = 0; i < fieldn_outer; i++) field_stack_.pop_back();
1563
1564 if (struct_def.fixed) {
1565 builder_.ClearOffsets();
1566 builder_.EndStruct();
1567 FLATBUFFERS_ASSERT(value);
1568 // Temporarily store this struct in the value string, since it is to
1569 // be serialized in-place elsewhere.
1570 value->assign(
1571 reinterpret_cast<const char *>(builder_.GetCurrentBufferPointer()),
1572 struct_def.bytesize);
1573 builder_.PopBytes(struct_def.bytesize);
1574 FLATBUFFERS_ASSERT(!ovalue);
1575 } else {
1576 auto val = builder_.EndTable(start);
1577 if (ovalue) *ovalue = val;
1578 if (value) *value = NumToString(val);
1579 }
1580 return NoError();
1581 }
1582
1583 template<typename F>
ParseVectorDelimiters(uoffset_t & count,F body)1584 CheckedError Parser::ParseVectorDelimiters(uoffset_t &count, F body) {
1585 EXPECT('[');
1586 for (;;) {
1587 if ((!opts.strict_json || !count) && Is(']')) break;
1588 ECHECK(body(count));
1589 count++;
1590 if (Is(']')) break;
1591 ECHECK(ParseComma());
1592 }
1593 NEXT();
1594 return NoError();
1595 }
1596
1597
ParseAlignAttribute(const std::string & align_constant,size_t min_align,size_t * align)1598 CheckedError Parser::ParseAlignAttribute(const std::string &align_constant,
1599 size_t min_align, size_t *align) {
1600 // Use uint8_t to avoid problems with size_t==`unsigned long` on LP64.
1601 uint8_t align_value;
1602 if (StringToNumber(align_constant.c_str(), &align_value) &&
1603 VerifyAlignmentRequirements(static_cast<size_t>(align_value),
1604 min_align)) {
1605 *align = align_value;
1606 return NoError();
1607 }
1608 return Error("unexpected force_align value '" + align_constant +
1609 "', alignment must be a power of two integer ranging from the "
1610 "type\'s natural alignment " +
1611 NumToString(min_align) + " to " +
1612 NumToString(FLATBUFFERS_MAX_ALIGNMENT));
1613 }
1614
ParseVector(const Type & type,uoffset_t * ovalue,FieldDef * field,size_t fieldn)1615 CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue,
1616 FieldDef *field, size_t fieldn) {
1617 uoffset_t count = 0;
1618 auto err = ParseVectorDelimiters(count, [&](uoffset_t &) -> CheckedError {
1619 Value val;
1620 val.type = type;
1621 ECHECK(ParseAnyValue(val, field, fieldn, nullptr, count, true));
1622 field_stack_.push_back(std::make_pair(val, nullptr));
1623 return NoError();
1624 });
1625 ECHECK(err);
1626
1627 const size_t len = count * InlineSize(type) / InlineAlignment(type);
1628 const size_t elemsize = InlineAlignment(type);
1629 const auto force_align = field->attributes.Lookup("force_align");
1630 if (force_align) {
1631 size_t align;
1632 ECHECK(ParseAlignAttribute(force_align->constant, 1, &align));
1633 if (align > 1) { builder_.ForceVectorAlignment(len, elemsize, align); }
1634 }
1635
1636 builder_.StartVector(len, elemsize);
1637 for (uoffset_t i = 0; i < count; i++) {
1638 // start at the back, since we're building the data backwards.
1639 auto &val = field_stack_.back().first;
1640 switch (val.type.base_type) {
1641 // clang-format off
1642 #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE,...) \
1643 case BASE_TYPE_ ## ENUM: \
1644 if (IsStruct(val.type)) SerializeStruct(*val.type.struct_def, val); \
1645 else { \
1646 CTYPE elem; \
1647 ECHECK(atot(val.constant.c_str(), *this, &elem)); \
1648 builder_.PushElement(elem); \
1649 } \
1650 break;
1651 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
1652 #undef FLATBUFFERS_TD
1653 // clang-format on
1654 }
1655 field_stack_.pop_back();
1656 }
1657
1658 builder_.ClearOffsets();
1659 *ovalue = builder_.EndVector(count);
1660
1661 if (type.base_type == BASE_TYPE_STRUCT && type.struct_def->has_key) {
1662 // We should sort this vector. Find the key first.
1663 const FieldDef *key = nullptr;
1664 for (auto it = type.struct_def->fields.vec.begin();
1665 it != type.struct_def->fields.vec.end(); ++it) {
1666 if ((*it)->key) {
1667 key = (*it);
1668 break;
1669 }
1670 }
1671 FLATBUFFERS_ASSERT(key);
1672 // Now sort it.
1673 // We can't use std::sort because for structs the size is not known at
1674 // compile time, and for tables our iterators dereference offsets, so can't
1675 // be used to swap elements.
1676 // And we can't use C qsort either, since that would force use to use
1677 // globals, making parsing thread-unsafe.
1678 // So for now, we use SimpleQsort above.
1679 // TODO: replace with something better, preferably not recursive.
1680
1681 if (type.struct_def->fixed) {
1682 const voffset_t offset = key->value.offset;
1683 const size_t struct_size = type.struct_def->bytesize;
1684 auto v =
1685 reinterpret_cast<VectorOfAny *>(builder_.GetCurrentBufferPointer());
1686 SimpleQsort<uint8_t>(
1687 v->Data(), v->Data() + v->size() * type.struct_def->bytesize,
1688 type.struct_def->bytesize,
1689 [offset, key](const uint8_t *a, const uint8_t *b) -> bool {
1690 return CompareSerializedScalars(a + offset, b + offset, *key);
1691 },
1692 [struct_size](uint8_t *a, uint8_t *b) {
1693 // FIXME: faster?
1694 for (size_t i = 0; i < struct_size; i++) { std::swap(a[i], b[i]); }
1695 });
1696 } else {
1697 auto v = reinterpret_cast<Vector<Offset<Table>> *>(
1698 builder_.GetCurrentBufferPointer());
1699 // Here also can't use std::sort. We do have an iterator type for it,
1700 // but it is non-standard as it will dereference the offsets, and thus
1701 // can't be used to swap elements.
1702 if (key->value.type.base_type == BASE_TYPE_STRING) {
1703 SimpleQsort<Offset<Table>>(
1704 v->data(), v->data() + v->size(), 1,
1705 [key](const Offset<Table> *_a, const Offset<Table> *_b) -> bool {
1706 return CompareTablesByStringKey(_a, _b, *key);
1707 },
1708 SwapSerializedTables);
1709 } else {
1710 SimpleQsort<Offset<Table>>(
1711 v->data(), v->data() + v->size(), 1,
1712 [key](const Offset<Table> *_a, const Offset<Table> *_b) -> bool {
1713 return CompareTablesByScalarKey(_a, _b, *key);
1714 },
1715 SwapSerializedTables);
1716 }
1717 }
1718 }
1719 return NoError();
1720 }
1721
ParseArray(Value & array)1722 CheckedError Parser::ParseArray(Value &array) {
1723 std::vector<Value> stack;
1724 FlatBufferBuilder builder;
1725 const auto &type = array.type.VectorType();
1726 auto length = array.type.fixed_length;
1727 uoffset_t count = 0;
1728 auto err = ParseVectorDelimiters(count, [&](uoffset_t &) -> CheckedError {
1729 stack.emplace_back(Value());
1730 auto &val = stack.back();
1731 val.type = type;
1732 if (IsStruct(type)) {
1733 ECHECK(ParseTable(*val.type.struct_def, &val.constant, nullptr));
1734 } else {
1735 ECHECK(ParseSingleValue(nullptr, val, false));
1736 }
1737 return NoError();
1738 });
1739 ECHECK(err);
1740 if (length != count) return Error("Fixed-length array size is incorrect.");
1741
1742 for (auto it = stack.rbegin(); it != stack.rend(); ++it) {
1743 auto &val = *it;
1744 // clang-format off
1745 switch (val.type.base_type) {
1746 #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, ...) \
1747 case BASE_TYPE_ ## ENUM: \
1748 if (IsStruct(val.type)) { \
1749 SerializeStruct(builder, *val.type.struct_def, val); \
1750 } else { \
1751 CTYPE elem; \
1752 ECHECK(atot(val.constant.c_str(), *this, &elem)); \
1753 builder.PushElement(elem); \
1754 } \
1755 break;
1756 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
1757 #undef FLATBUFFERS_TD
1758 default: FLATBUFFERS_ASSERT(0);
1759 }
1760 // clang-format on
1761 }
1762
1763 array.constant.assign(
1764 reinterpret_cast<const char *>(builder.GetCurrentBufferPointer()),
1765 InlineSize(array.type));
1766 return NoError();
1767 }
1768
ParseNestedFlatbuffer(Value & val,FieldDef * field,size_t fieldn,const StructDef * parent_struct_def)1769 CheckedError Parser::ParseNestedFlatbuffer(Value &val, FieldDef *field,
1770 size_t fieldn,
1771 const StructDef *parent_struct_def) {
1772 if (token_ == '[') { // backwards compat for 'legacy' ubyte buffers
1773 if (opts.json_nested_legacy_flatbuffers) {
1774 ECHECK(ParseAnyValue(val, field, fieldn, parent_struct_def, 0));
1775 } else {
1776 return Error(
1777 "cannot parse nested_flatbuffer as bytes unless"
1778 " --json-nested-bytes is set");
1779 }
1780 } else {
1781 auto cursor_at_value_begin = cursor_;
1782 ECHECK(SkipAnyJsonValue());
1783 std::string substring(cursor_at_value_begin - 1, cursor_ - 1);
1784
1785 // Create and initialize new parser
1786 Parser nested_parser;
1787 FLATBUFFERS_ASSERT(field->nested_flatbuffer);
1788 nested_parser.root_struct_def_ = field->nested_flatbuffer;
1789 nested_parser.enums_ = enums_;
1790 nested_parser.opts = opts;
1791 nested_parser.uses_flexbuffers_ = uses_flexbuffers_;
1792 nested_parser.parse_depth_counter_ = parse_depth_counter_;
1793 // Parse JSON substring into new flatbuffer builder using nested_parser
1794 bool ok = nested_parser.Parse(substring.c_str(), nullptr, nullptr);
1795
1796 // Clean nested_parser to avoid deleting the elements in
1797 // the SymbolTables on destruction
1798 nested_parser.enums_.dict.clear();
1799 nested_parser.enums_.vec.clear();
1800
1801 if (!ok) { ECHECK(Error(nested_parser.error_)); }
1802 // Force alignment for nested flatbuffer
1803 builder_.ForceVectorAlignment(
1804 nested_parser.builder_.GetSize(), sizeof(uint8_t),
1805 nested_parser.builder_.GetBufferMinAlignment());
1806
1807 auto off = builder_.CreateVector(nested_parser.builder_.GetBufferPointer(),
1808 nested_parser.builder_.GetSize());
1809 val.constant = NumToString(off.o);
1810 }
1811 return NoError();
1812 }
1813
ParseMetaData(SymbolTable<Value> * attributes)1814 CheckedError Parser::ParseMetaData(SymbolTable<Value> *attributes) {
1815 if (Is('(')) {
1816 NEXT();
1817 for (;;) {
1818 auto name = attribute_;
1819 if (false == (Is(kTokenIdentifier) || Is(kTokenStringConstant)))
1820 return Error("attribute name must be either identifier or string: " +
1821 name);
1822 if (known_attributes_.find(name) == known_attributes_.end())
1823 return Error("user define attributes must be declared before use: " +
1824 name);
1825 NEXT();
1826 auto e = new Value();
1827 if (attributes->Add(name, e)) Warning("attribute already found: " + name);
1828 if (Is(':')) {
1829 NEXT();
1830 ECHECK(ParseSingleValue(&name, *e, true));
1831 }
1832 if (Is(')')) {
1833 NEXT();
1834 break;
1835 }
1836 EXPECT(',');
1837 }
1838 }
1839 return NoError();
1840 }
1841
ParseEnumFromString(const Type & type,std::string * result)1842 CheckedError Parser::ParseEnumFromString(const Type &type,
1843 std::string *result) {
1844 const auto base_type =
1845 type.enum_def ? type.enum_def->underlying_type.base_type : type.base_type;
1846 if (!IsInteger(base_type)) return Error("not a valid value for this field");
1847 uint64_t u64 = 0;
1848 for (size_t pos = 0; pos != std::string::npos;) {
1849 const auto delim = attribute_.find_first_of(' ', pos);
1850 const auto last = (std::string::npos == delim);
1851 auto word = attribute_.substr(pos, !last ? delim - pos : std::string::npos);
1852 pos = !last ? delim + 1 : std::string::npos;
1853 const EnumVal *ev = nullptr;
1854 if (type.enum_def) {
1855 ev = type.enum_def->Lookup(word);
1856 } else {
1857 auto dot = word.find_first_of('.');
1858 if (std::string::npos == dot)
1859 return Error("enum values need to be qualified by an enum type");
1860 auto enum_def_str = word.substr(0, dot);
1861 const auto enum_def = LookupEnum(enum_def_str);
1862 if (!enum_def) return Error("unknown enum: " + enum_def_str);
1863 auto enum_val_str = word.substr(dot + 1);
1864 ev = enum_def->Lookup(enum_val_str);
1865 }
1866 if (!ev) return Error("unknown enum value: " + word);
1867 u64 |= ev->GetAsUInt64();
1868 }
1869 *result = IsUnsigned(base_type) ? NumToString(u64)
1870 : NumToString(static_cast<int64_t>(u64));
1871 return NoError();
1872 }
1873
ParseHash(Value & e,FieldDef * field)1874 CheckedError Parser::ParseHash(Value &e, FieldDef *field) {
1875 FLATBUFFERS_ASSERT(field);
1876 Value *hash_name = field->attributes.Lookup("hash");
1877 switch (e.type.base_type) {
1878 case BASE_TYPE_SHORT: {
1879 auto hash = FindHashFunction16(hash_name->constant.c_str());
1880 int16_t hashed_value = static_cast<int16_t>(hash(attribute_.c_str()));
1881 e.constant = NumToString(hashed_value);
1882 break;
1883 }
1884 case BASE_TYPE_USHORT: {
1885 auto hash = FindHashFunction16(hash_name->constant.c_str());
1886 uint16_t hashed_value = hash(attribute_.c_str());
1887 e.constant = NumToString(hashed_value);
1888 break;
1889 }
1890 case BASE_TYPE_INT: {
1891 auto hash = FindHashFunction32(hash_name->constant.c_str());
1892 int32_t hashed_value = static_cast<int32_t>(hash(attribute_.c_str()));
1893 e.constant = NumToString(hashed_value);
1894 break;
1895 }
1896 case BASE_TYPE_UINT: {
1897 auto hash = FindHashFunction32(hash_name->constant.c_str());
1898 uint32_t hashed_value = hash(attribute_.c_str());
1899 e.constant = NumToString(hashed_value);
1900 break;
1901 }
1902 case BASE_TYPE_LONG: {
1903 auto hash = FindHashFunction64(hash_name->constant.c_str());
1904 int64_t hashed_value = static_cast<int64_t>(hash(attribute_.c_str()));
1905 e.constant = NumToString(hashed_value);
1906 break;
1907 }
1908 case BASE_TYPE_ULONG: {
1909 auto hash = FindHashFunction64(hash_name->constant.c_str());
1910 uint64_t hashed_value = hash(attribute_.c_str());
1911 e.constant = NumToString(hashed_value);
1912 break;
1913 }
1914 default: FLATBUFFERS_ASSERT(0);
1915 }
1916 NEXT();
1917 return NoError();
1918 }
1919
TokenError()1920 CheckedError Parser::TokenError() {
1921 return Error("cannot parse value starting with: " + TokenToStringId(token_));
1922 }
1923
ParseFunction(const std::string * name,Value & e)1924 CheckedError Parser::ParseFunction(const std::string *name, Value &e) {
1925 ParseDepthGuard depth_guard(this);
1926 ECHECK(depth_guard.Check());
1927
1928 // Copy name, attribute will be changed on NEXT().
1929 const auto functionname = attribute_;
1930 if (!IsFloat(e.type.base_type)) {
1931 return Error(functionname + ": type of argument mismatch, expecting: " +
1932 kTypeNames[BASE_TYPE_DOUBLE] +
1933 ", found: " + kTypeNames[e.type.base_type] +
1934 ", name: " + (name ? *name : "") + ", value: " + e.constant);
1935 }
1936 NEXT();
1937 EXPECT('(');
1938 ECHECK(ParseSingleValue(name, e, false));
1939 EXPECT(')');
1940 // calculate with double precision
1941 double x, y = 0.0;
1942 ECHECK(atot(e.constant.c_str(), *this, &x));
1943 // clang-format off
1944 auto func_match = false;
1945 #define FLATBUFFERS_FN_DOUBLE(name, op) \
1946 if (!func_match && functionname == name) { y = op; func_match = true; }
1947 FLATBUFFERS_FN_DOUBLE("deg", x / kPi * 180);
1948 FLATBUFFERS_FN_DOUBLE("rad", x * kPi / 180);
1949 FLATBUFFERS_FN_DOUBLE("sin", sin(x));
1950 FLATBUFFERS_FN_DOUBLE("cos", cos(x));
1951 FLATBUFFERS_FN_DOUBLE("tan", tan(x));
1952 FLATBUFFERS_FN_DOUBLE("asin", asin(x));
1953 FLATBUFFERS_FN_DOUBLE("acos", acos(x));
1954 FLATBUFFERS_FN_DOUBLE("atan", atan(x));
1955 // TODO(wvo): add more useful conversion functions here.
1956 #undef FLATBUFFERS_FN_DOUBLE
1957 // clang-format on
1958 if (true != func_match) {
1959 return Error(std::string("Unknown conversion function: ") + functionname +
1960 ", field name: " + (name ? *name : "") +
1961 ", value: " + e.constant);
1962 }
1963 e.constant = NumToString(y);
1964 return NoError();
1965 }
1966
TryTypedValue(const std::string * name,int dtoken,bool check,Value & e,BaseType req,bool * destmatch)1967 CheckedError Parser::TryTypedValue(const std::string *name, int dtoken,
1968 bool check, Value &e, BaseType req,
1969 bool *destmatch) {
1970 FLATBUFFERS_ASSERT(*destmatch == false && dtoken == token_);
1971 *destmatch = true;
1972 e.constant = attribute_;
1973 // Check token match
1974 if (!check) {
1975 if (e.type.base_type == BASE_TYPE_NONE) {
1976 e.type.base_type = req;
1977 } else {
1978 return Error(std::string("type mismatch: expecting: ") +
1979 kTypeNames[e.type.base_type] +
1980 ", found: " + kTypeNames[req] +
1981 ", name: " + (name ? *name : "") + ", value: " + e.constant);
1982 }
1983 }
1984 // The exponent suffix of hexadecimal float-point number is mandatory.
1985 // A hex-integer constant is forbidden as an initializer of float number.
1986 if ((kTokenFloatConstant != dtoken) && IsFloat(e.type.base_type)) {
1987 const auto &s = e.constant;
1988 const auto k = s.find_first_of("0123456789.");
1989 if ((std::string::npos != k) && (s.length() > (k + 1)) &&
1990 (s[k] == '0' && is_alpha_char(s[k + 1], 'X')) &&
1991 (std::string::npos == s.find_first_of("pP", k + 2))) {
1992 return Error(
1993 "invalid number, the exponent suffix of hexadecimal "
1994 "floating-point literals is mandatory: \"" +
1995 s + "\"");
1996 }
1997 }
1998 NEXT();
1999 return NoError();
2000 }
2001
ParseSingleValue(const std::string * name,Value & e,bool check_now)2002 CheckedError Parser::ParseSingleValue(const std::string *name, Value &e,
2003 bool check_now) {
2004 if (token_ == '+' || token_ == '-') {
2005 const char sign = static_cast<char>(token_);
2006 // Get an indentifier: NAN, INF, or function name like cos/sin/deg.
2007 NEXT();
2008 if (token_ != kTokenIdentifier) return Error("constant name expected");
2009 attribute_.insert(0, 1, sign);
2010 }
2011
2012 const auto in_type = e.type.base_type;
2013 const auto is_tok_ident = (token_ == kTokenIdentifier);
2014 const auto is_tok_string = (token_ == kTokenStringConstant);
2015
2016 // First see if this could be a conversion function.
2017 if (is_tok_ident && *cursor_ == '(') { return ParseFunction(name, e); }
2018
2019 // clang-format off
2020 auto match = false;
2021
2022 #define IF_ECHECK_(force, dtoken, check, req) \
2023 if (!match && ((dtoken) == token_) && ((check) || IsConstTrue(force))) \
2024 ECHECK(TryTypedValue(name, dtoken, check, e, req, &match))
2025 #define TRY_ECHECK(dtoken, check, req) IF_ECHECK_(false, dtoken, check, req)
2026 #define FORCE_ECHECK(dtoken, check, req) IF_ECHECK_(true, dtoken, check, req)
2027 // clang-format on
2028
2029 if (is_tok_ident || is_tok_string) {
2030 const auto kTokenStringOrIdent = token_;
2031 // The string type is a most probable type, check it first.
2032 TRY_ECHECK(kTokenStringConstant, in_type == BASE_TYPE_STRING,
2033 BASE_TYPE_STRING);
2034
2035 // avoid escaped and non-ascii in the string
2036 if (!match && is_tok_string && IsScalar(in_type) &&
2037 !attr_is_trivial_ascii_string_) {
2038 return Error(
2039 std::string("type mismatch or invalid value, an initializer of "
2040 "non-string field must be trivial ASCII string: type: ") +
2041 kTypeNames[in_type] + ", name: " + (name ? *name : "") +
2042 ", value: " + attribute_);
2043 }
2044
2045 // A boolean as true/false. Boolean as Integer check below.
2046 if (!match && IsBool(in_type)) {
2047 auto is_true = attribute_ == "true";
2048 if (is_true || attribute_ == "false") {
2049 attribute_ = is_true ? "1" : "0";
2050 // accepts both kTokenStringConstant and kTokenIdentifier
2051 TRY_ECHECK(kTokenStringOrIdent, IsBool(in_type), BASE_TYPE_BOOL);
2052 }
2053 }
2054 // Check for optional scalars.
2055 if (!match && IsScalar(in_type) && attribute_ == "null") {
2056 e.constant = "null";
2057 NEXT();
2058 match = true;
2059 }
2060 // Check if this could be a string/identifier enum value.
2061 // Enum can have only true integer base type.
2062 if (!match && IsInteger(in_type) && !IsBool(in_type) &&
2063 IsIdentifierStart(*attribute_.c_str())) {
2064 ECHECK(ParseEnumFromString(e.type, &e.constant));
2065 NEXT();
2066 match = true;
2067 }
2068 // Parse a float/integer number from the string.
2069 // A "scalar-in-string" value needs extra checks.
2070 if (!match && is_tok_string && IsScalar(in_type)) {
2071 // Strip trailing whitespaces from attribute_.
2072 auto last_non_ws = attribute_.find_last_not_of(' ');
2073 if (std::string::npos != last_non_ws) attribute_.resize(last_non_ws + 1);
2074 if (IsFloat(e.type.base_type)) {
2075 // The functions strtod() and strtof() accept both 'nan' and
2076 // 'nan(number)' literals. While 'nan(number)' is rejected by the parser
2077 // as an unsupported function if is_tok_ident is true.
2078 if (attribute_.find_last_of(')') != std::string::npos) {
2079 return Error("invalid number: " + attribute_);
2080 }
2081 }
2082 }
2083 // Float numbers or nan, inf, pi, etc.
2084 TRY_ECHECK(kTokenStringOrIdent, IsFloat(in_type), BASE_TYPE_FLOAT);
2085 // An integer constant in string.
2086 TRY_ECHECK(kTokenStringOrIdent, IsInteger(in_type), BASE_TYPE_INT);
2087 // Unknown tokens will be interpreted as string type.
2088 // An attribute value may be a scalar or string constant.
2089 FORCE_ECHECK(kTokenStringConstant, in_type == BASE_TYPE_STRING,
2090 BASE_TYPE_STRING);
2091 } else {
2092 // Try a float number.
2093 TRY_ECHECK(kTokenFloatConstant, IsFloat(in_type), BASE_TYPE_FLOAT);
2094 // Integer token can init any scalar (integer of float).
2095 FORCE_ECHECK(kTokenIntegerConstant, IsScalar(in_type), BASE_TYPE_INT);
2096 }
2097 // Match empty vectors for default-empty-vectors.
2098 if (!match && IsVector(e.type) && token_ == '[') {
2099 NEXT();
2100 if (token_ != ']') { return Error("Expected `]` in vector default"); }
2101 NEXT();
2102 match = true;
2103 e.constant = "[]";
2104 }
2105
2106 #undef FORCE_ECHECK
2107 #undef TRY_ECHECK
2108 #undef IF_ECHECK_
2109
2110 if (!match) {
2111 std::string msg;
2112 msg += "Cannot assign token starting with '" + TokenToStringId(token_) +
2113 "' to value of <" + std::string(kTypeNames[in_type]) + "> type.";
2114 return Error(msg);
2115 }
2116 const auto match_type = e.type.base_type; // may differ from in_type
2117 // The check_now flag must be true when parse a fbs-schema.
2118 // This flag forces to check default scalar values or metadata of field.
2119 // For JSON parser the flag should be false.
2120 // If it is set for JSON each value will be checked twice (see ParseTable).
2121 // Special case 'null' since atot can't handle that.
2122 if (check_now && IsScalar(match_type) && e.constant != "null") {
2123 // clang-format off
2124 switch (match_type) {
2125 #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, ...) \
2126 case BASE_TYPE_ ## ENUM: {\
2127 CTYPE val; \
2128 ECHECK(atot(e.constant.c_str(), *this, &val)); \
2129 SingleValueRepack(e, val); \
2130 break; }
2131 FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD)
2132 #undef FLATBUFFERS_TD
2133 default: break;
2134 }
2135 // clang-format on
2136 }
2137 return NoError();
2138 }
2139
LookupCreateStruct(const std::string & name,bool create_if_new,bool definition)2140 StructDef *Parser::LookupCreateStruct(const std::string &name,
2141 bool create_if_new, bool definition) {
2142 std::string qualified_name = current_namespace_->GetFullyQualifiedName(name);
2143 // See if it exists pre-declared by an unqualified use.
2144 auto struct_def = LookupStruct(name);
2145 if (struct_def && struct_def->predecl) {
2146 if (definition) {
2147 // Make sure it has the current namespace, and is registered under its
2148 // qualified name.
2149 struct_def->defined_namespace = current_namespace_;
2150 structs_.Move(name, qualified_name);
2151 }
2152 return struct_def;
2153 }
2154 // See if it exists pre-declared by an qualified use.
2155 struct_def = LookupStruct(qualified_name);
2156 if (struct_def && struct_def->predecl) {
2157 if (definition) {
2158 // Make sure it has the current namespace.
2159 struct_def->defined_namespace = current_namespace_;
2160 }
2161 return struct_def;
2162 }
2163 if (!definition && !struct_def) {
2164 struct_def = LookupStructThruParentNamespaces(name);
2165 }
2166 if (!struct_def && create_if_new) {
2167 struct_def = new StructDef();
2168 if (definition) {
2169 structs_.Add(qualified_name, struct_def);
2170 struct_def->name = name;
2171 struct_def->defined_namespace = current_namespace_;
2172 } else {
2173 // Not a definition.
2174 // Rather than failing, we create a "pre declared" StructDef, due to
2175 // circular references, and check for errors at the end of parsing.
2176 // It is defined in the current namespace, as the best guess what the
2177 // final namespace will be.
2178 structs_.Add(name, struct_def);
2179 struct_def->name = name;
2180 struct_def->defined_namespace = current_namespace_;
2181 struct_def->original_location.reset(
2182 new std::string(file_being_parsed_ + ":" + NumToString(line_)));
2183 }
2184 }
2185 return struct_def;
2186 }
2187
MinValue() const2188 const EnumVal *EnumDef::MinValue() const {
2189 return vals.vec.empty() ? nullptr : vals.vec.front();
2190 }
MaxValue() const2191 const EnumVal *EnumDef::MaxValue() const {
2192 return vals.vec.empty() ? nullptr : vals.vec.back();
2193 }
2194
Distance(const EnumVal * v1,const EnumVal * v2) const2195 uint64_t EnumDef::Distance(const EnumVal *v1, const EnumVal *v2) const {
2196 return IsUInt64() ? EnumDistanceImpl(v1->GetAsUInt64(), v2->GetAsUInt64())
2197 : EnumDistanceImpl(v1->GetAsInt64(), v2->GetAsInt64());
2198 }
2199
AllFlags() const2200 std::string EnumDef::AllFlags() const {
2201 FLATBUFFERS_ASSERT(attributes.Lookup("bit_flags"));
2202 uint64_t u64 = 0;
2203 for (auto it = Vals().begin(); it != Vals().end(); ++it) {
2204 u64 |= (*it)->GetAsUInt64();
2205 }
2206 return IsUInt64() ? NumToString(u64) : NumToString(static_cast<int64_t>(u64));
2207 }
2208
ReverseLookup(int64_t enum_idx,bool skip_union_default) const2209 EnumVal *EnumDef::ReverseLookup(int64_t enum_idx,
2210 bool skip_union_default) const {
2211 auto skip_first = static_cast<int>(is_union && skip_union_default);
2212 for (auto it = Vals().begin() + skip_first; it != Vals().end(); ++it) {
2213 if ((*it)->GetAsInt64() == enum_idx) { return *it; }
2214 }
2215 return nullptr;
2216 }
2217
FindByValue(const std::string & constant) const2218 EnumVal *EnumDef::FindByValue(const std::string &constant) const {
2219 int64_t i64;
2220 auto done = false;
2221 if (IsUInt64()) {
2222 uint64_t u64; // avoid reinterpret_cast of pointers
2223 done = StringToNumber(constant.c_str(), &u64);
2224 i64 = static_cast<int64_t>(u64);
2225 } else {
2226 done = StringToNumber(constant.c_str(), &i64);
2227 }
2228 FLATBUFFERS_ASSERT(done);
2229 if (!done) return nullptr;
2230 return ReverseLookup(i64, false);
2231 }
2232
SortByValue()2233 void EnumDef::SortByValue() {
2234 auto &v = vals.vec;
2235 if (IsUInt64())
2236 std::sort(v.begin(), v.end(), [](const EnumVal *e1, const EnumVal *e2) {
2237 if (e1->GetAsUInt64() == e2->GetAsUInt64()) {
2238 return e1->name < e2->name;
2239 }
2240 return e1->GetAsUInt64() < e2->GetAsUInt64();
2241 });
2242 else
2243 std::sort(v.begin(), v.end(), [](const EnumVal *e1, const EnumVal *e2) {
2244 if (e1->GetAsInt64() == e2->GetAsInt64()) { return e1->name < e2->name; }
2245 return e1->GetAsInt64() < e2->GetAsInt64();
2246 });
2247 }
2248
RemoveDuplicates()2249 void EnumDef::RemoveDuplicates() {
2250 // This method depends form SymbolTable implementation!
2251 // 1) vals.vec - owner (raw pointer)
2252 // 2) vals.dict - access map
2253 auto first = vals.vec.begin();
2254 auto last = vals.vec.end();
2255 if (first == last) return;
2256 auto result = first;
2257 while (++first != last) {
2258 if ((*result)->value != (*first)->value) {
2259 *(++result) = *first;
2260 } else {
2261 auto ev = *first;
2262 for (auto it = vals.dict.begin(); it != vals.dict.end(); ++it) {
2263 if (it->second == ev) it->second = *result; // reassign
2264 }
2265 delete ev; // delete enum value
2266 *first = nullptr;
2267 }
2268 }
2269 vals.vec.erase(++result, last);
2270 }
2271
ChangeEnumValue(EnumVal * ev,T new_value)2272 template<typename T> void EnumDef::ChangeEnumValue(EnumVal *ev, T new_value) {
2273 ev->value = static_cast<int64_t>(new_value);
2274 }
2275
2276 namespace EnumHelper {
2277 template<BaseType E> struct EnumValType { typedef int64_t type; };
2278 template<> struct EnumValType<BASE_TYPE_ULONG> { typedef uint64_t type; };
2279 } // namespace EnumHelper
2280
2281 struct EnumValBuilder {
CreateEnumeratorflatbuffers::EnumValBuilder2282 EnumVal *CreateEnumerator(const std::string &ev_name) {
2283 FLATBUFFERS_ASSERT(!temp);
2284 auto first = enum_def.vals.vec.empty();
2285 user_value = first;
2286 temp = new EnumVal(ev_name, first ? 0 : enum_def.vals.vec.back()->value);
2287 return temp;
2288 }
2289
CreateEnumeratorflatbuffers::EnumValBuilder2290 EnumVal *CreateEnumerator(const std::string &ev_name, int64_t val) {
2291 FLATBUFFERS_ASSERT(!temp);
2292 user_value = true;
2293 temp = new EnumVal(ev_name, val);
2294 return temp;
2295 }
2296
AcceptEnumeratorflatbuffers::EnumValBuilder2297 FLATBUFFERS_CHECKED_ERROR AcceptEnumerator(const std::string &name) {
2298 FLATBUFFERS_ASSERT(temp);
2299 ECHECK(ValidateValue(&temp->value, false == user_value));
2300 FLATBUFFERS_ASSERT((temp->union_type.enum_def == nullptr) ||
2301 (temp->union_type.enum_def == &enum_def));
2302 auto not_unique = enum_def.vals.Add(name, temp);
2303 temp = nullptr;
2304 if (not_unique) return parser.Error("enum value already exists: " + name);
2305 return NoError();
2306 }
2307
AcceptEnumeratorflatbuffers::EnumValBuilder2308 FLATBUFFERS_CHECKED_ERROR AcceptEnumerator() {
2309 return AcceptEnumerator(temp->name);
2310 }
2311
AssignEnumeratorValueflatbuffers::EnumValBuilder2312 FLATBUFFERS_CHECKED_ERROR AssignEnumeratorValue(const std::string &value) {
2313 user_value = true;
2314 auto fit = false;
2315 if (enum_def.IsUInt64()) {
2316 uint64_t u64;
2317 fit = StringToNumber(value.c_str(), &u64);
2318 temp->value = static_cast<int64_t>(u64); // well-defined since C++20.
2319 } else {
2320 int64_t i64;
2321 fit = StringToNumber(value.c_str(), &i64);
2322 temp->value = i64;
2323 }
2324 if (!fit) return parser.Error("enum value does not fit, \"" + value + "\"");
2325 return NoError();
2326 }
2327
2328 template<BaseType E, typename CTYPE>
ValidateImplflatbuffers::EnumValBuilder2329 inline FLATBUFFERS_CHECKED_ERROR ValidateImpl(int64_t *ev, int m) {
2330 typedef typename EnumHelper::EnumValType<E>::type T; // int64_t or uint64_t
2331 static_assert(sizeof(T) == sizeof(int64_t), "invalid EnumValType");
2332 const auto v = static_cast<T>(*ev);
2333 auto up = static_cast<T>((flatbuffers::numeric_limits<CTYPE>::max)());
2334 auto dn = static_cast<T>((flatbuffers::numeric_limits<CTYPE>::lowest)());
2335 if (v < dn || v > (up - m)) {
2336 return parser.Error("enum value does not fit, \"" + NumToString(v) +
2337 (m ? " + 1\"" : "\"") + " out of " +
2338 TypeToIntervalString<CTYPE>());
2339 }
2340 *ev = static_cast<int64_t>(v + m); // well-defined since C++20.
2341 return NoError();
2342 }
2343
ValidateValueflatbuffers::EnumValBuilder2344 FLATBUFFERS_CHECKED_ERROR ValidateValue(int64_t *ev, bool next) {
2345 // clang-format off
2346 switch (enum_def.underlying_type.base_type) {
2347 #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, ...) \
2348 case BASE_TYPE_##ENUM: { \
2349 if (!IsInteger(BASE_TYPE_##ENUM)) break; \
2350 return ValidateImpl<BASE_TYPE_##ENUM, CTYPE>(ev, next ? 1 : 0); \
2351 }
2352 FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD)
2353 #undef FLATBUFFERS_TD
2354 default: break;
2355 }
2356 // clang-format on
2357 return parser.Error("fatal: invalid enum underlying type");
2358 }
2359
EnumValBuilderflatbuffers::EnumValBuilder2360 EnumValBuilder(Parser &_parser, EnumDef &_enum_def)
2361 : parser(_parser),
2362 enum_def(_enum_def),
2363 temp(nullptr),
2364 user_value(false) {}
2365
~EnumValBuilderflatbuffers::EnumValBuilder2366 ~EnumValBuilder() { delete temp; }
2367
2368 Parser &parser;
2369 EnumDef &enum_def;
2370 EnumVal *temp;
2371 bool user_value;
2372 };
2373
ParseEnum(const bool is_union,EnumDef ** dest,const char * filename)2374 CheckedError Parser::ParseEnum(const bool is_union, EnumDef **dest,
2375 const char *filename) {
2376 std::vector<std::string> enum_comment = doc_comment_;
2377 NEXT();
2378 std::string enum_name = attribute_;
2379 EXPECT(kTokenIdentifier);
2380 EnumDef *enum_def;
2381 ECHECK(StartEnum(enum_name, is_union, &enum_def));
2382 if (filename != nullptr && !opts.project_root.empty()) {
2383 enum_def->declaration_file =
2384 &GetPooledString(RelativeToRootPath(opts.project_root, filename));
2385 }
2386 enum_def->doc_comment = enum_comment;
2387 if (!is_union && !opts.proto_mode) {
2388 // Give specialized error message, since this type spec used to
2389 // be optional in the first FlatBuffers release.
2390 if (!Is(':')) {
2391 return Error(
2392 "must specify the underlying integer type for this"
2393 " enum (e.g. \': short\', which was the default).");
2394 } else {
2395 NEXT();
2396 }
2397 // Specify the integer type underlying this enum.
2398 ECHECK(ParseType(enum_def->underlying_type));
2399 if (!IsInteger(enum_def->underlying_type.base_type) ||
2400 IsBool(enum_def->underlying_type.base_type))
2401 return Error("underlying enum type must be integral");
2402 // Make this type refer back to the enum it was derived from.
2403 enum_def->underlying_type.enum_def = enum_def;
2404 }
2405 ECHECK(ParseMetaData(&enum_def->attributes));
2406 const auto underlying_type = enum_def->underlying_type.base_type;
2407 if (enum_def->attributes.Lookup("bit_flags") &&
2408 !IsUnsigned(underlying_type)) {
2409 // todo: Convert to the Error in the future?
2410 Warning("underlying type of bit_flags enum must be unsigned");
2411 }
2412 EnumValBuilder evb(*this, *enum_def);
2413 EXPECT('{');
2414 // A lot of code generatos expect that an enum is not-empty.
2415 if ((is_union || Is('}')) && !opts.proto_mode) {
2416 evb.CreateEnumerator("NONE");
2417 ECHECK(evb.AcceptEnumerator());
2418 }
2419 std::set<std::pair<BaseType, StructDef *>> union_types;
2420 while (!Is('}')) {
2421 if (opts.proto_mode && attribute_ == "option") {
2422 ECHECK(ParseProtoOption());
2423 } else {
2424 auto &ev = *evb.CreateEnumerator(attribute_);
2425 auto full_name = ev.name;
2426 ev.doc_comment = doc_comment_;
2427 EXPECT(kTokenIdentifier);
2428 if (is_union) {
2429 ECHECK(ParseNamespacing(&full_name, &ev.name));
2430 if (opts.union_value_namespacing) {
2431 // Since we can't namespace the actual enum identifiers, turn
2432 // namespace parts into part of the identifier.
2433 ev.name = full_name;
2434 std::replace(ev.name.begin(), ev.name.end(), '.', '_');
2435 }
2436 if (Is(':')) {
2437 NEXT();
2438 ECHECK(ParseType(ev.union_type));
2439 if (ev.union_type.base_type != BASE_TYPE_STRUCT &&
2440 ev.union_type.base_type != BASE_TYPE_STRING)
2441 return Error("union value type may only be table/struct/string");
2442 } else {
2443 ev.union_type = Type(BASE_TYPE_STRUCT, LookupCreateStruct(full_name));
2444 }
2445 if (!enum_def->uses_multiple_type_instances) {
2446 auto ins = union_types.insert(std::make_pair(
2447 ev.union_type.base_type, ev.union_type.struct_def));
2448 enum_def->uses_multiple_type_instances = (false == ins.second);
2449 }
2450 }
2451
2452 if (Is('=')) {
2453 NEXT();
2454 ECHECK(evb.AssignEnumeratorValue(attribute_));
2455 EXPECT(kTokenIntegerConstant);
2456 }
2457
2458 ECHECK(evb.AcceptEnumerator());
2459
2460 if (opts.proto_mode && Is('[')) {
2461 NEXT();
2462 // ignore attributes on enums.
2463 while (token_ != ']') NEXT();
2464 NEXT();
2465 }
2466 }
2467 if (!Is(opts.proto_mode ? ';' : ',')) break;
2468 NEXT();
2469 }
2470 EXPECT('}');
2471
2472 // At this point, the enum can be empty if input is invalid proto-file.
2473 if (!enum_def->size())
2474 return Error("incomplete enum declaration, values not found");
2475
2476 if (enum_def->attributes.Lookup("bit_flags")) {
2477 const auto base_width = static_cast<uint64_t>(8 * SizeOf(underlying_type));
2478 for (auto it = enum_def->Vals().begin(); it != enum_def->Vals().end();
2479 ++it) {
2480 auto ev = *it;
2481 const auto u = ev->GetAsUInt64();
2482 // Stop manipulations with the sign.
2483 if (!IsUnsigned(underlying_type) && u == (base_width - 1))
2484 return Error("underlying type of bit_flags enum must be unsigned");
2485 if (u >= base_width)
2486 return Error("bit flag out of range of underlying integral type");
2487 enum_def->ChangeEnumValue(ev, 1ULL << u);
2488 }
2489 }
2490
2491 enum_def->SortByValue(); // Must be sorted to use MinValue/MaxValue.
2492
2493 // Ensure enum value uniqueness.
2494 auto prev_it = enum_def->Vals().begin();
2495 for (auto it = prev_it + 1; it != enum_def->Vals().end(); ++it) {
2496 auto prev_ev = *prev_it;
2497 auto ev = *it;
2498 if (prev_ev->GetAsUInt64() == ev->GetAsUInt64())
2499 return Error("all enum values must be unique: " + prev_ev->name +
2500 " and " + ev->name + " are both " +
2501 NumToString(ev->GetAsInt64()));
2502 }
2503
2504 if (dest) *dest = enum_def;
2505 const auto qualified_name =
2506 current_namespace_->GetFullyQualifiedName(enum_def->name);
2507 if (types_.Add(qualified_name, new Type(BASE_TYPE_UNION, nullptr, enum_def)))
2508 return Error("datatype already exists: " + qualified_name);
2509 return NoError();
2510 }
2511
StartStruct(const std::string & name,StructDef ** dest)2512 CheckedError Parser::StartStruct(const std::string &name, StructDef **dest) {
2513 auto &struct_def = *LookupCreateStruct(name, true, true);
2514 if (!struct_def.predecl)
2515 return Error("datatype already exists: " +
2516 current_namespace_->GetFullyQualifiedName(name));
2517 struct_def.predecl = false;
2518 struct_def.name = name;
2519 struct_def.file = file_being_parsed_;
2520 // Move this struct to the back of the vector just in case it was predeclared,
2521 // to preserve declaration order.
2522 *std::remove(structs_.vec.begin(), structs_.vec.end(), &struct_def) =
2523 &struct_def;
2524 *dest = &struct_def;
2525 return NoError();
2526 }
2527
CheckClash(std::vector<FieldDef * > & fields,StructDef * struct_def,const char * suffix,BaseType basetype)2528 CheckedError Parser::CheckClash(std::vector<FieldDef *> &fields,
2529 StructDef *struct_def, const char *suffix,
2530 BaseType basetype) {
2531 auto len = strlen(suffix);
2532 for (auto it = fields.begin(); it != fields.end(); ++it) {
2533 auto &fname = (*it)->name;
2534 if (fname.length() > len &&
2535 fname.compare(fname.length() - len, len, suffix) == 0 &&
2536 (*it)->value.type.base_type != BASE_TYPE_UTYPE) {
2537 auto field =
2538 struct_def->fields.Lookup(fname.substr(0, fname.length() - len));
2539 if (field && field->value.type.base_type == basetype)
2540 return Error("Field " + fname +
2541 " would clash with generated functions for field " +
2542 field->name);
2543 }
2544 }
2545 return NoError();
2546 }
2547
GetIncludedFiles() const2548 std::vector<IncludedFile> Parser::GetIncludedFiles() const {
2549 const auto it = files_included_per_file_.find(file_being_parsed_);
2550 if (it == files_included_per_file_.end()) { return {}; }
2551
2552 return { it->second.cbegin(), it->second.cend() };
2553 }
2554
SupportsOptionalScalars(const flatbuffers::IDLOptions & opts)2555 bool Parser::SupportsOptionalScalars(const flatbuffers::IDLOptions &opts) {
2556 static FLATBUFFERS_CONSTEXPR unsigned long supported_langs =
2557 IDLOptions::kRust | IDLOptions::kSwift | IDLOptions::kLobster |
2558 IDLOptions::kKotlin | IDLOptions::kCpp | IDLOptions::kJava |
2559 IDLOptions::kCSharp | IDLOptions::kTs | IDLOptions::kBinary |
2560 IDLOptions::kGo | IDLOptions::kPython | IDLOptions::kJson;
2561 unsigned long langs = opts.lang_to_generate;
2562 return (langs > 0 && langs < IDLOptions::kMAX) && !(langs & ~supported_langs);
2563 }
SupportsOptionalScalars() const2564 bool Parser::SupportsOptionalScalars() const {
2565 // Check in general if a language isn't specified.
2566 return opts.lang_to_generate == 0 || SupportsOptionalScalars(opts);
2567 }
2568
SupportsDefaultVectorsAndStrings() const2569 bool Parser::SupportsDefaultVectorsAndStrings() const {
2570 static FLATBUFFERS_CONSTEXPR unsigned long supported_langs =
2571 IDLOptions::kRust | IDLOptions::kSwift;
2572 return !(opts.lang_to_generate & ~supported_langs);
2573 }
2574
SupportsAdvancedUnionFeatures() const2575 bool Parser::SupportsAdvancedUnionFeatures() const {
2576 return (opts.lang_to_generate &
2577 ~(IDLOptions::kCpp | IDLOptions::kTs | IDLOptions::kPhp |
2578 IDLOptions::kJava | IDLOptions::kCSharp | IDLOptions::kKotlin |
2579 IDLOptions::kBinary | IDLOptions::kSwift)) == 0;
2580 }
2581
SupportsAdvancedArrayFeatures() const2582 bool Parser::SupportsAdvancedArrayFeatures() const {
2583 return (opts.lang_to_generate &
2584 ~(IDLOptions::kCpp | IDLOptions::kPython | IDLOptions::kJava |
2585 IDLOptions::kCSharp | IDLOptions::kJsonSchema | IDLOptions::kJson |
2586 IDLOptions::kBinary | IDLOptions::kRust)) == 0;
2587 }
2588
UniqueNamespace(Namespace * ns)2589 Namespace *Parser::UniqueNamespace(Namespace *ns) {
2590 for (auto it = namespaces_.begin(); it != namespaces_.end(); ++it) {
2591 if (ns->components == (*it)->components) {
2592 delete ns;
2593 return *it;
2594 }
2595 }
2596 namespaces_.push_back(ns);
2597 return ns;
2598 }
2599
UnqualifiedName(const std::string & full_qualified_name)2600 std::string Parser::UnqualifiedName(const std::string &full_qualified_name) {
2601 Namespace *ns = new Namespace();
2602
2603 std::size_t current, previous = 0;
2604 current = full_qualified_name.find('.');
2605 while (current != std::string::npos) {
2606 ns->components.push_back(
2607 full_qualified_name.substr(previous, current - previous));
2608 previous = current + 1;
2609 current = full_qualified_name.find('.', previous);
2610 }
2611 current_namespace_ = UniqueNamespace(ns);
2612 return full_qualified_name.substr(previous, current - previous);
2613 }
2614
ParseDecl(const char * filename)2615 CheckedError Parser::ParseDecl(const char *filename) {
2616 std::vector<std::string> dc = doc_comment_;
2617 bool fixed = IsIdent("struct");
2618 if (!fixed && !IsIdent("table")) return Error("declaration expected");
2619 NEXT();
2620 std::string name = attribute_;
2621 EXPECT(kTokenIdentifier);
2622 StructDef *struct_def;
2623 ECHECK(StartStruct(name, &struct_def));
2624 struct_def->doc_comment = dc;
2625 struct_def->fixed = fixed;
2626 if (filename && !opts.project_root.empty()) {
2627 struct_def->declaration_file =
2628 &GetPooledString(RelativeToRootPath(opts.project_root, filename));
2629 }
2630 ECHECK(ParseMetaData(&struct_def->attributes));
2631 struct_def->sortbysize =
2632 struct_def->attributes.Lookup("original_order") == nullptr && !fixed;
2633 EXPECT('{');
2634 while (token_ != '}') ECHECK(ParseField(*struct_def));
2635 if (fixed) {
2636 const auto force_align = struct_def->attributes.Lookup("force_align");
2637 if (force_align) {
2638 size_t align;
2639 ECHECK(ParseAlignAttribute(force_align->constant, struct_def->minalign,
2640 &align));
2641 struct_def->minalign = align;
2642 }
2643 if (!struct_def->bytesize) return Error("size 0 structs not allowed");
2644 }
2645 struct_def->PadLastField(struct_def->minalign);
2646 // Check if this is a table that has manual id assignments
2647 auto &fields = struct_def->fields.vec;
2648 if (!fixed && fields.size()) {
2649 size_t num_id_fields = 0;
2650 for (auto it = fields.begin(); it != fields.end(); ++it) {
2651 if ((*it)->attributes.Lookup("id")) num_id_fields++;
2652 }
2653 // If any fields have ids..
2654 if (num_id_fields || opts.require_explicit_ids) {
2655 // Then all fields must have them.
2656 if (num_id_fields != fields.size()) {
2657 if (opts.require_explicit_ids) {
2658 return Error(
2659 "all fields must have an 'id' attribute when "
2660 "--require-explicit-ids is used");
2661 } else {
2662 return Error(
2663 "either all fields or no fields must have an 'id' attribute");
2664 }
2665 }
2666 // Simply sort by id, then the fields are the same as if no ids had
2667 // been specified.
2668 std::sort(fields.begin(), fields.end(), compareFieldDefs);
2669 // Verify we have a contiguous set, and reassign vtable offsets.
2670 FLATBUFFERS_ASSERT(fields.size() <=
2671 flatbuffers::numeric_limits<voffset_t>::max());
2672 for (voffset_t i = 0; i < static_cast<voffset_t>(fields.size()); i++) {
2673 auto &field = *fields[i];
2674 const auto &id_str = field.attributes.Lookup("id")->constant;
2675 // Metadata values have a dynamic type, they can be `float`, 'int', or
2676 // 'string`.
2677 // The FieldIndexToOffset(i) expects the voffset_t so `id` is limited by
2678 // this type.
2679 voffset_t id = 0;
2680 const auto done = !atot(id_str.c_str(), *this, &id).Check();
2681 if (!done)
2682 return Error("field id\'s must be non-negative number, field: " +
2683 field.name + ", id: " + id_str);
2684 if (i != id)
2685 return Error("field id\'s must be consecutive from 0, id " +
2686 NumToString(i) + " missing or set twice, field: " +
2687 field.name + ", id: " + id_str);
2688 field.value.offset = FieldIndexToOffset(i);
2689 }
2690 }
2691 }
2692
2693 ECHECK(
2694 CheckClash(fields, struct_def, UnionTypeFieldSuffix(), BASE_TYPE_UNION));
2695 ECHECK(CheckClash(fields, struct_def, "Type", BASE_TYPE_UNION));
2696 ECHECK(CheckClash(fields, struct_def, "_length", BASE_TYPE_VECTOR));
2697 ECHECK(CheckClash(fields, struct_def, "Length", BASE_TYPE_VECTOR));
2698 ECHECK(CheckClash(fields, struct_def, "_byte_vector", BASE_TYPE_STRING));
2699 ECHECK(CheckClash(fields, struct_def, "ByteVector", BASE_TYPE_STRING));
2700 EXPECT('}');
2701 const auto qualified_name =
2702 current_namespace_->GetFullyQualifiedName(struct_def->name);
2703 if (types_.Add(qualified_name,
2704 new Type(BASE_TYPE_STRUCT, struct_def, nullptr)))
2705 return Error("datatype already exists: " + qualified_name);
2706 return NoError();
2707 }
2708
ParseService(const char * filename)2709 CheckedError Parser::ParseService(const char *filename) {
2710 std::vector<std::string> service_comment = doc_comment_;
2711 NEXT();
2712 auto service_name = attribute_;
2713 EXPECT(kTokenIdentifier);
2714 auto &service_def = *new ServiceDef();
2715 service_def.name = service_name;
2716 service_def.file = file_being_parsed_;
2717 service_def.doc_comment = service_comment;
2718 service_def.defined_namespace = current_namespace_;
2719 if (filename != nullptr && !opts.project_root.empty()) {
2720 service_def.declaration_file =
2721 &GetPooledString(RelativeToRootPath(opts.project_root, filename));
2722 }
2723 if (services_.Add(current_namespace_->GetFullyQualifiedName(service_name),
2724 &service_def))
2725 return Error("service already exists: " + service_name);
2726 ECHECK(ParseMetaData(&service_def.attributes));
2727 EXPECT('{');
2728 do {
2729 std::vector<std::string> doc_comment = doc_comment_;
2730 auto rpc_name = attribute_;
2731 EXPECT(kTokenIdentifier);
2732 EXPECT('(');
2733 Type reqtype, resptype;
2734 ECHECK(ParseTypeIdent(reqtype));
2735 EXPECT(')');
2736 EXPECT(':');
2737 ECHECK(ParseTypeIdent(resptype));
2738 if (reqtype.base_type != BASE_TYPE_STRUCT || reqtype.struct_def->fixed ||
2739 resptype.base_type != BASE_TYPE_STRUCT || resptype.struct_def->fixed)
2740 return Error("rpc request and response types must be tables");
2741 auto &rpc = *new RPCCall();
2742 rpc.name = rpc_name;
2743 rpc.request = reqtype.struct_def;
2744 rpc.response = resptype.struct_def;
2745 rpc.doc_comment = doc_comment;
2746 if (service_def.calls.Add(rpc_name, &rpc))
2747 return Error("rpc already exists: " + rpc_name);
2748 ECHECK(ParseMetaData(&rpc.attributes));
2749 EXPECT(';');
2750 } while (token_ != '}');
2751 NEXT();
2752 return NoError();
2753 }
2754
SetRootType(const char * name)2755 bool Parser::SetRootType(const char *name) {
2756 root_struct_def_ = LookupStruct(name);
2757 if (!root_struct_def_)
2758 root_struct_def_ =
2759 LookupStruct(current_namespace_->GetFullyQualifiedName(name));
2760 return root_struct_def_ != nullptr;
2761 }
2762
MarkGenerated()2763 void Parser::MarkGenerated() {
2764 // This function marks all existing definitions as having already
2765 // been generated, which signals no code for included files should be
2766 // generated.
2767 for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) {
2768 (*it)->generated = true;
2769 }
2770 for (auto it = structs_.vec.begin(); it != structs_.vec.end(); ++it) {
2771 if (!(*it)->predecl) { (*it)->generated = true; }
2772 }
2773 for (auto it = services_.vec.begin(); it != services_.vec.end(); ++it) {
2774 (*it)->generated = true;
2775 }
2776 }
2777
ParseNamespace()2778 CheckedError Parser::ParseNamespace() {
2779 NEXT();
2780 auto ns = new Namespace();
2781 namespaces_.push_back(ns); // Store it here to not leak upon error.
2782 if (token_ != ';') {
2783 for (;;) {
2784 ns->components.push_back(attribute_);
2785 EXPECT(kTokenIdentifier);
2786 if (Is('.')) NEXT() else break;
2787 }
2788 }
2789 namespaces_.pop_back();
2790 current_namespace_ = UniqueNamespace(ns);
2791 EXPECT(';');
2792 return NoError();
2793 }
2794
2795 // Best effort parsing of .proto declarations, with the aim to turn them
2796 // in the closest corresponding FlatBuffer equivalent.
2797 // We parse everything as identifiers instead of keywords, since we don't
2798 // want protobuf keywords to become invalid identifiers in FlatBuffers.
ParseProtoDecl()2799 CheckedError Parser::ParseProtoDecl() {
2800 bool isextend = IsIdent("extend");
2801 if (IsIdent("package")) {
2802 // These are identical in syntax to FlatBuffer's namespace decl.
2803 ECHECK(ParseNamespace());
2804 } else if (IsIdent("message") || isextend) {
2805 std::vector<std::string> struct_comment = doc_comment_;
2806 NEXT();
2807 StructDef *struct_def = nullptr;
2808 Namespace *parent_namespace = nullptr;
2809 if (isextend) {
2810 if (Is('.')) NEXT(); // qualified names may start with a . ?
2811 auto id = attribute_;
2812 EXPECT(kTokenIdentifier);
2813 ECHECK(ParseNamespacing(&id, nullptr));
2814 struct_def = LookupCreateStruct(id, false);
2815 if (!struct_def)
2816 return Error("cannot extend unknown message type: " + id);
2817 } else {
2818 std::string name = attribute_;
2819 EXPECT(kTokenIdentifier);
2820 ECHECK(StartStruct(name, &struct_def));
2821 // Since message definitions can be nested, we create a new namespace.
2822 auto ns = new Namespace();
2823 // Copy of current namespace.
2824 *ns = *current_namespace_;
2825 // But with current message name.
2826 ns->components.push_back(name);
2827 ns->from_table++;
2828 parent_namespace = current_namespace_;
2829 current_namespace_ = UniqueNamespace(ns);
2830 }
2831 struct_def->doc_comment = struct_comment;
2832 ECHECK(ParseProtoFields(struct_def, isextend, false));
2833 if (!isextend) { current_namespace_ = parent_namespace; }
2834 if (Is(';')) NEXT();
2835 } else if (IsIdent("enum")) {
2836 // These are almost the same, just with different terminator:
2837 EnumDef *enum_def;
2838 ECHECK(ParseEnum(false, &enum_def, nullptr));
2839 if (Is(';')) NEXT();
2840 // Temp: remove any duplicates, as .fbs files can't handle them.
2841 enum_def->RemoveDuplicates();
2842 } else if (IsIdent("syntax")) { // Skip these.
2843 NEXT();
2844 EXPECT('=');
2845 EXPECT(kTokenStringConstant);
2846 EXPECT(';');
2847 } else if (IsIdent("option")) { // Skip these.
2848 ECHECK(ParseProtoOption());
2849 EXPECT(';');
2850 } else if (IsIdent("service")) { // Skip these.
2851 NEXT();
2852 EXPECT(kTokenIdentifier);
2853 ECHECK(ParseProtoCurliesOrIdent());
2854 } else {
2855 return Error("don\'t know how to parse .proto declaration starting with " +
2856 TokenToStringId(token_));
2857 }
2858 return NoError();
2859 }
2860
StartEnum(const std::string & name,bool is_union,EnumDef ** dest)2861 CheckedError Parser::StartEnum(const std::string &name, bool is_union,
2862 EnumDef **dest) {
2863 auto &enum_def = *new EnumDef();
2864 enum_def.name = name;
2865 enum_def.file = file_being_parsed_;
2866 enum_def.doc_comment = doc_comment_;
2867 enum_def.is_union = is_union;
2868 enum_def.defined_namespace = current_namespace_;
2869 const auto qualified_name = current_namespace_->GetFullyQualifiedName(name);
2870 if (enums_.Add(qualified_name, &enum_def))
2871 return Error("enum already exists: " + qualified_name);
2872 enum_def.underlying_type.base_type =
2873 is_union ? BASE_TYPE_UTYPE : BASE_TYPE_INT;
2874 enum_def.underlying_type.enum_def = &enum_def;
2875 if (dest) *dest = &enum_def;
2876 return NoError();
2877 }
2878
ParseProtoFields(StructDef * struct_def,bool isextend,bool inside_oneof)2879 CheckedError Parser::ParseProtoFields(StructDef *struct_def, bool isextend,
2880 bool inside_oneof) {
2881 EXPECT('{');
2882 while (token_ != '}') {
2883 if (IsIdent("message") || IsIdent("extend") || IsIdent("enum")) {
2884 // Nested declarations.
2885 ECHECK(ParseProtoDecl());
2886 } else if (IsIdent("extensions")) { // Skip these.
2887 NEXT();
2888 EXPECT(kTokenIntegerConstant);
2889 if (Is(kTokenIdentifier)) {
2890 NEXT(); // to
2891 NEXT(); // num
2892 }
2893 EXPECT(';');
2894 } else if (IsIdent("option")) { // Skip these.
2895 ECHECK(ParseProtoOption());
2896 EXPECT(';');
2897 } else if (IsIdent("reserved")) { // Skip these.
2898 NEXT();
2899 while (!Is(';')) { NEXT(); } // A variety of formats, just skip.
2900 NEXT();
2901 } else {
2902 std::vector<std::string> field_comment = doc_comment_;
2903 // Parse the qualifier.
2904 bool required = false;
2905 bool repeated = false;
2906 bool oneof = false;
2907 if (!inside_oneof) {
2908 if (IsIdent("optional")) {
2909 // This is the default.
2910 NEXT();
2911 } else if (IsIdent("required")) {
2912 required = true;
2913 NEXT();
2914 } else if (IsIdent("repeated")) {
2915 repeated = true;
2916 NEXT();
2917 } else if (IsIdent("oneof")) {
2918 oneof = true;
2919 NEXT();
2920 } else {
2921 // can't error, proto3 allows decls without any of the above.
2922 }
2923 }
2924 StructDef *anonymous_struct = nullptr;
2925 EnumDef *oneof_union = nullptr;
2926 Type type;
2927 if (IsIdent("group") || oneof) {
2928 if (!oneof) NEXT();
2929 if (oneof && opts.proto_oneof_union) {
2930 auto name = ConvertCase(attribute_, Case::kUpperCamel) + "Union";
2931 ECHECK(StartEnum(name, true, &oneof_union));
2932 type = Type(BASE_TYPE_UNION, nullptr, oneof_union);
2933 } else {
2934 auto name = "Anonymous" + NumToString(anonymous_counter_++);
2935 ECHECK(StartStruct(name, &anonymous_struct));
2936 type = Type(BASE_TYPE_STRUCT, anonymous_struct);
2937 }
2938 } else {
2939 ECHECK(ParseTypeFromProtoType(&type));
2940 }
2941 // Repeated elements get mapped to a vector.
2942 if (repeated) {
2943 type.element = type.base_type;
2944 type.base_type = BASE_TYPE_VECTOR;
2945 if (type.element == BASE_TYPE_VECTOR) {
2946 // We have a vector or vectors, which FlatBuffers doesn't support.
2947 // For now make it a vector of string (since the source is likely
2948 // "repeated bytes").
2949 // TODO(wvo): A better solution would be to wrap this in a table.
2950 type.element = BASE_TYPE_STRING;
2951 }
2952 }
2953 std::string name = attribute_;
2954 EXPECT(kTokenIdentifier);
2955 if (!oneof) {
2956 // Parse the field id. Since we're just translating schemas, not
2957 // any kind of binary compatibility, we can safely ignore these, and
2958 // assign our own.
2959 EXPECT('=');
2960 EXPECT(kTokenIntegerConstant);
2961 }
2962 FieldDef *field = nullptr;
2963 if (isextend) {
2964 // We allow a field to be re-defined when extending.
2965 // TODO: are there situations where that is problematic?
2966 field = struct_def->fields.Lookup(name);
2967 }
2968 if (!field) ECHECK(AddField(*struct_def, name, type, &field));
2969 field->doc_comment = field_comment;
2970 if (!IsScalar(type.base_type) && required) {
2971 field->presence = FieldDef::kRequired;
2972 }
2973 // See if there's a default specified.
2974 if (Is('[')) {
2975 NEXT();
2976 for (;;) {
2977 auto key = attribute_;
2978 ECHECK(ParseProtoKey());
2979 EXPECT('=');
2980 auto val = attribute_;
2981 ECHECK(ParseProtoCurliesOrIdent());
2982 if (key == "default") {
2983 // Temp: skip non-numeric and non-boolean defaults (enums).
2984 auto numeric = strpbrk(val.c_str(), "0123456789-+.");
2985 if (IsFloat(type.base_type) &&
2986 (val == "inf" || val == "+inf" || val == "-inf")) {
2987 // Prefer to be explicit with +inf.
2988 field->value.constant = val == "inf" ? "+inf" : val;
2989 } else if (IsScalar(type.base_type) && numeric == val.c_str()) {
2990 field->value.constant = val;
2991 } else if (val == "true") {
2992 field->value.constant = val;
2993 } // "false" is default, no need to handle explicitly.
2994 } else if (key == "deprecated") {
2995 field->deprecated = val == "true";
2996 }
2997 if (!Is(',')) break;
2998 NEXT();
2999 }
3000 EXPECT(']');
3001 }
3002 if (anonymous_struct) {
3003 ECHECK(ParseProtoFields(anonymous_struct, false, oneof));
3004 if (Is(';')) NEXT();
3005 } else if (oneof_union) {
3006 // Parse into a temporary StructDef, then transfer fields into an
3007 // EnumDef describing the oneof as a union.
3008 StructDef oneof_struct;
3009 ECHECK(ParseProtoFields(&oneof_struct, false, oneof));
3010 if (Is(';')) NEXT();
3011 for (auto field_it = oneof_struct.fields.vec.begin();
3012 field_it != oneof_struct.fields.vec.end(); ++field_it) {
3013 const auto &oneof_field = **field_it;
3014 const auto &oneof_type = oneof_field.value.type;
3015 if (oneof_type.base_type != BASE_TYPE_STRUCT ||
3016 !oneof_type.struct_def || oneof_type.struct_def->fixed)
3017 return Error("oneof '" + name +
3018 "' cannot be mapped to a union because member '" +
3019 oneof_field.name + "' is not a table type.");
3020 EnumValBuilder evb(*this, *oneof_union);
3021 auto ev = evb.CreateEnumerator(oneof_type.struct_def->name);
3022 ev->union_type = oneof_type;
3023 ev->doc_comment = oneof_field.doc_comment;
3024 ECHECK(evb.AcceptEnumerator(oneof_field.name));
3025 }
3026 } else {
3027 EXPECT(';');
3028 }
3029 }
3030 }
3031 NEXT();
3032 return NoError();
3033 }
3034
ParseProtoKey()3035 CheckedError Parser::ParseProtoKey() {
3036 if (token_ == '(') {
3037 NEXT();
3038 // Skip "(a.b)" style custom attributes.
3039 while (token_ == '.' || token_ == kTokenIdentifier) NEXT();
3040 EXPECT(')');
3041 while (Is('.')) {
3042 NEXT();
3043 EXPECT(kTokenIdentifier);
3044 }
3045 } else {
3046 EXPECT(kTokenIdentifier);
3047 }
3048 return NoError();
3049 }
3050
ParseProtoCurliesOrIdent()3051 CheckedError Parser::ParseProtoCurliesOrIdent() {
3052 if (Is('{')) {
3053 NEXT();
3054 for (int nesting = 1; nesting;) {
3055 if (token_ == '{')
3056 nesting++;
3057 else if (token_ == '}')
3058 nesting--;
3059 NEXT();
3060 }
3061 } else {
3062 NEXT(); // Any single token.
3063 }
3064 return NoError();
3065 }
3066
ParseProtoOption()3067 CheckedError Parser::ParseProtoOption() {
3068 NEXT();
3069 ECHECK(ParseProtoKey());
3070 EXPECT('=');
3071 ECHECK(ParseProtoCurliesOrIdent());
3072 return NoError();
3073 }
3074
3075 // Parse a protobuf type, and map it to the corresponding FlatBuffer one.
ParseTypeFromProtoType(Type * type)3076 CheckedError Parser::ParseTypeFromProtoType(Type *type) {
3077 struct type_lookup {
3078 const char *proto_type;
3079 BaseType fb_type, element;
3080 };
3081 static type_lookup lookup[] = {
3082 { "float", BASE_TYPE_FLOAT, BASE_TYPE_NONE },
3083 { "double", BASE_TYPE_DOUBLE, BASE_TYPE_NONE },
3084 { "int32", BASE_TYPE_INT, BASE_TYPE_NONE },
3085 { "int64", BASE_TYPE_LONG, BASE_TYPE_NONE },
3086 { "uint32", BASE_TYPE_UINT, BASE_TYPE_NONE },
3087 { "uint64", BASE_TYPE_ULONG, BASE_TYPE_NONE },
3088 { "sint32", BASE_TYPE_INT, BASE_TYPE_NONE },
3089 { "sint64", BASE_TYPE_LONG, BASE_TYPE_NONE },
3090 { "fixed32", BASE_TYPE_UINT, BASE_TYPE_NONE },
3091 { "fixed64", BASE_TYPE_ULONG, BASE_TYPE_NONE },
3092 { "sfixed32", BASE_TYPE_INT, BASE_TYPE_NONE },
3093 { "sfixed64", BASE_TYPE_LONG, BASE_TYPE_NONE },
3094 { "bool", BASE_TYPE_BOOL, BASE_TYPE_NONE },
3095 { "string", BASE_TYPE_STRING, BASE_TYPE_NONE },
3096 { "bytes", BASE_TYPE_VECTOR, BASE_TYPE_UCHAR },
3097 { nullptr, BASE_TYPE_NONE, BASE_TYPE_NONE }
3098 };
3099 for (auto tl = lookup; tl->proto_type; tl++) {
3100 if (attribute_ == tl->proto_type) {
3101 type->base_type = tl->fb_type;
3102 type->element = tl->element;
3103 NEXT();
3104 return NoError();
3105 }
3106 }
3107 if (Is('.')) NEXT(); // qualified names may start with a . ?
3108 ECHECK(ParseTypeIdent(*type));
3109 return NoError();
3110 }
3111
SkipAnyJsonValue()3112 CheckedError Parser::SkipAnyJsonValue() {
3113 ParseDepthGuard depth_guard(this);
3114 ECHECK(depth_guard.Check());
3115
3116 switch (token_) {
3117 case '{': {
3118 size_t fieldn_outer = 0;
3119 return ParseTableDelimiters(fieldn_outer, nullptr,
3120 [&](const std::string &, size_t &fieldn,
3121 const StructDef *) -> CheckedError {
3122 ECHECK(SkipAnyJsonValue());
3123 fieldn++;
3124 return NoError();
3125 });
3126 }
3127 case '[': {
3128 uoffset_t count = 0;
3129 return ParseVectorDelimiters(count, [&](uoffset_t &) -> CheckedError {
3130 return SkipAnyJsonValue();
3131 });
3132 }
3133 case kTokenStringConstant:
3134 case kTokenIntegerConstant:
3135 case kTokenFloatConstant: NEXT(); break;
3136 default:
3137 if (IsIdent("true") || IsIdent("false") || IsIdent("null") ||
3138 IsIdent("inf")) {
3139 NEXT();
3140 } else
3141 return TokenError();
3142 }
3143 return NoError();
3144 }
3145
ParseFlexBufferNumericConstant(flexbuffers::Builder * builder)3146 CheckedError Parser::ParseFlexBufferNumericConstant(
3147 flexbuffers::Builder *builder) {
3148 double d;
3149 if (!StringToNumber(attribute_.c_str(), &d))
3150 return Error("unexpected floating-point constant: " + attribute_);
3151 builder->Double(d);
3152 return NoError();
3153 }
3154
ParseFlexBufferValue(flexbuffers::Builder * builder)3155 CheckedError Parser::ParseFlexBufferValue(flexbuffers::Builder *builder) {
3156 ParseDepthGuard depth_guard(this);
3157 ECHECK(depth_guard.Check());
3158
3159 switch (token_) {
3160 case '{': {
3161 auto start = builder->StartMap();
3162 size_t fieldn_outer = 0;
3163 auto err =
3164 ParseTableDelimiters(fieldn_outer, nullptr,
3165 [&](const std::string &name, size_t &fieldn,
3166 const StructDef *) -> CheckedError {
3167 builder->Key(name);
3168 ECHECK(ParseFlexBufferValue(builder));
3169 fieldn++;
3170 return NoError();
3171 });
3172 ECHECK(err);
3173 builder->EndMap(start);
3174 if (builder->HasDuplicateKeys())
3175 return Error("FlexBuffers map has duplicate keys");
3176 break;
3177 }
3178 case '[': {
3179 auto start = builder->StartVector();
3180 uoffset_t count = 0;
3181 ECHECK(ParseVectorDelimiters(count, [&](uoffset_t &) -> CheckedError {
3182 return ParseFlexBufferValue(builder);
3183 }));
3184 builder->EndVector(start, false, false);
3185 break;
3186 }
3187 case kTokenStringConstant:
3188 builder->String(attribute_);
3189 EXPECT(kTokenStringConstant);
3190 break;
3191 case kTokenIntegerConstant:
3192 builder->Int(StringToInt(attribute_.c_str()));
3193 EXPECT(kTokenIntegerConstant);
3194 break;
3195 case kTokenFloatConstant: {
3196 double d;
3197 StringToNumber(attribute_.c_str(), &d);
3198 builder->Double(d);
3199 EXPECT(kTokenFloatConstant);
3200 break;
3201 }
3202 case '-':
3203 case '+': {
3204 // `[-+]?(nan|inf|infinity)`, see ParseSingleValue().
3205 const auto sign = static_cast<char>(token_);
3206 NEXT();
3207 if (token_ != kTokenIdentifier)
3208 return Error("floating-point constant expected");
3209 attribute_.insert(0, 1, sign);
3210 ECHECK(ParseFlexBufferNumericConstant(builder));
3211 NEXT();
3212 break;
3213 }
3214 default:
3215 if (IsIdent("true")) {
3216 builder->Bool(true);
3217 NEXT();
3218 } else if (IsIdent("false")) {
3219 builder->Bool(false);
3220 NEXT();
3221 } else if (IsIdent("null")) {
3222 builder->Null();
3223 NEXT();
3224 } else if (IsIdent("inf") || IsIdent("infinity") || IsIdent("nan")) {
3225 ECHECK(ParseFlexBufferNumericConstant(builder));
3226 NEXT();
3227 } else
3228 return TokenError();
3229 }
3230 return NoError();
3231 }
3232
ParseFlexBuffer(const char * source,const char * source_filename,flexbuffers::Builder * builder)3233 bool Parser::ParseFlexBuffer(const char *source, const char *source_filename,
3234 flexbuffers::Builder *builder) {
3235 const auto initial_depth = parse_depth_counter_;
3236 (void)initial_depth;
3237 auto ok = !StartParseFile(source, source_filename).Check() &&
3238 !ParseFlexBufferValue(builder).Check();
3239 if (ok) builder->Finish();
3240 FLATBUFFERS_ASSERT(initial_depth == parse_depth_counter_);
3241 return ok;
3242 }
3243
Parse(const char * source,const char ** include_paths,const char * source_filename)3244 bool Parser::Parse(const char *source, const char **include_paths,
3245 const char *source_filename) {
3246 const auto initial_depth = parse_depth_counter_;
3247 (void)initial_depth;
3248 bool r;
3249
3250 if (opts.use_flexbuffers) {
3251 r = ParseFlexBuffer(source, source_filename, &flex_builder_);
3252 } else {
3253 r = !ParseRoot(source, include_paths, source_filename).Check();
3254 }
3255 FLATBUFFERS_ASSERT(initial_depth == parse_depth_counter_);
3256 return r;
3257 }
3258
ParseJson(const char * json,const char * json_filename)3259 bool Parser::ParseJson(const char *json, const char *json_filename) {
3260 const auto initial_depth = parse_depth_counter_;
3261 (void)initial_depth;
3262 builder_.Clear();
3263 const auto done =
3264 !StartParseFile(json, json_filename).Check() && !DoParseJson().Check();
3265 FLATBUFFERS_ASSERT(initial_depth == parse_depth_counter_);
3266 return done;
3267 }
3268
StartParseFile(const char * source,const char * source_filename)3269 CheckedError Parser::StartParseFile(const char *source,
3270 const char *source_filename) {
3271 file_being_parsed_ = source_filename ? source_filename : "";
3272 source_ = source;
3273 ResetState(source_);
3274 error_.clear();
3275 ECHECK(SkipByteOrderMark());
3276 NEXT();
3277 if (Is(kTokenEof)) return Error("input file is empty");
3278 return NoError();
3279 }
3280
ParseRoot(const char * source,const char ** include_paths,const char * source_filename)3281 CheckedError Parser::ParseRoot(const char *source, const char **include_paths,
3282 const char *source_filename) {
3283 ECHECK(DoParse(source, include_paths, source_filename, nullptr));
3284
3285 // Check that all types were defined.
3286 for (auto it = structs_.vec.begin(); it != structs_.vec.end();) {
3287 auto &struct_def = **it;
3288 if (struct_def.predecl) {
3289 if (opts.proto_mode) {
3290 // Protos allow enums to be used before declaration, so check if that
3291 // is the case here.
3292 EnumDef *enum_def = nullptr;
3293 for (size_t components =
3294 struct_def.defined_namespace->components.size() + 1;
3295 components && !enum_def; components--) {
3296 auto qualified_name =
3297 struct_def.defined_namespace->GetFullyQualifiedName(
3298 struct_def.name, components - 1);
3299 enum_def = LookupEnum(qualified_name);
3300 }
3301 if (enum_def) {
3302 // This is pretty slow, but a simple solution for now.
3303 auto initial_count = struct_def.refcount;
3304 for (auto struct_it = structs_.vec.begin();
3305 struct_it != structs_.vec.end(); ++struct_it) {
3306 auto &sd = **struct_it;
3307 for (auto field_it = sd.fields.vec.begin();
3308 field_it != sd.fields.vec.end(); ++field_it) {
3309 auto &field = **field_it;
3310 if (field.value.type.struct_def == &struct_def) {
3311 field.value.type.struct_def = nullptr;
3312 field.value.type.enum_def = enum_def;
3313 auto &bt = IsVector(field.value.type)
3314 ? field.value.type.element
3315 : field.value.type.base_type;
3316 FLATBUFFERS_ASSERT(bt == BASE_TYPE_STRUCT);
3317 bt = enum_def->underlying_type.base_type;
3318 struct_def.refcount--;
3319 enum_def->refcount++;
3320 }
3321 }
3322 }
3323 if (struct_def.refcount)
3324 return Error("internal: " + NumToString(struct_def.refcount) + "/" +
3325 NumToString(initial_count) +
3326 " use(s) of pre-declaration enum not accounted for: " +
3327 enum_def->name);
3328 structs_.dict.erase(structs_.dict.find(struct_def.name));
3329 it = structs_.vec.erase(it);
3330 delete &struct_def;
3331 continue; // Skip error.
3332 }
3333 }
3334 auto err = "type referenced but not defined (check namespace): " +
3335 struct_def.name;
3336 if (struct_def.original_location)
3337 err += ", originally at: " + *struct_def.original_location;
3338 return Error(err);
3339 }
3340 ++it;
3341 }
3342
3343 // This check has to happen here and not earlier, because only now do we
3344 // know for sure what the type of these are.
3345 for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) {
3346 auto &enum_def = **it;
3347 if (enum_def.is_union) {
3348 for (auto val_it = enum_def.Vals().begin();
3349 val_it != enum_def.Vals().end(); ++val_it) {
3350 auto &val = **val_it;
3351
3352 if (!(opts.lang_to_generate != 0 && SupportsAdvancedUnionFeatures()) &&
3353 (IsStruct(val.union_type) || IsString(val.union_type)))
3354
3355 return Error(
3356 "only tables can be union elements in the generated language: " +
3357 val.name);
3358 }
3359 }
3360 }
3361
3362 auto err = CheckPrivateLeak();
3363 if (err.Check()) return err;
3364
3365 // Parse JSON object only if the scheme has been parsed.
3366 if (token_ == '{') { ECHECK(DoParseJson()); }
3367 return NoError();
3368 }
3369
CheckPrivateLeak()3370 CheckedError Parser::CheckPrivateLeak() {
3371 if (!opts.no_leak_private_annotations) return NoError();
3372 // Iterate over all structs/tables to validate we arent leaking
3373 // any private (structs/tables/enums)
3374 for (auto it = structs_.vec.begin(); it != structs_.vec.end(); it++) {
3375 auto &struct_def = **it;
3376 for (auto fld_it = struct_def.fields.vec.begin();
3377 fld_it != struct_def.fields.vec.end(); ++fld_it) {
3378 auto &field = **fld_it;
3379
3380 if (field.value.type.enum_def) {
3381 auto err =
3382 CheckPrivatelyLeakedFields(struct_def, *field.value.type.enum_def);
3383 if (err.Check()) { return err; }
3384 } else if (field.value.type.struct_def) {
3385 auto err = CheckPrivatelyLeakedFields(struct_def,
3386 *field.value.type.struct_def);
3387 if (err.Check()) { return err; }
3388 }
3389 }
3390 }
3391 // Iterate over all enums to validate we arent leaking
3392 // any private (structs/tables)
3393 for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) {
3394 auto &enum_def = **it;
3395 if (enum_def.is_union) {
3396 for (auto val_it = enum_def.Vals().begin();
3397 val_it != enum_def.Vals().end(); ++val_it) {
3398 auto &val = **val_it;
3399 if (val.union_type.struct_def) {
3400 auto err =
3401 CheckPrivatelyLeakedFields(enum_def, *val.union_type.struct_def);
3402 if (err.Check()) { return err; }
3403 }
3404 }
3405 }
3406 }
3407 return NoError();
3408 }
3409
CheckPrivatelyLeakedFields(const Definition & def,const Definition & value_type)3410 CheckedError Parser::CheckPrivatelyLeakedFields(const Definition &def,
3411 const Definition &value_type) {
3412 if (!opts.no_leak_private_annotations) return NoError();
3413 const auto is_private = def.attributes.Lookup("private");
3414 const auto is_field_private = value_type.attributes.Lookup("private");
3415 if (!is_private && is_field_private) {
3416 return Error(
3417 "Leaking private implementation, verify all objects have similar "
3418 "annotations");
3419 }
3420 return NoError();
3421 }
3422
3423
DoParse(const char * source,const char ** include_paths,const char * source_filename,const char * include_filename)3424 CheckedError Parser::DoParse(const char *source, const char **include_paths,
3425 const char *source_filename,
3426 const char *include_filename) {
3427 uint64_t source_hash = 0;
3428 if (source_filename) {
3429 // If the file is in-memory, don't include its contents in the hash as we
3430 // won't be able to load them later.
3431 if (FileExists(source_filename))
3432 source_hash = HashFile(source_filename, source);
3433 else
3434 source_hash = HashFile(source_filename, nullptr);
3435
3436 if (included_files_.find(source_hash) == included_files_.end()) {
3437 included_files_[source_hash] = include_filename ? include_filename : "";
3438 files_included_per_file_[source_filename] = std::set<IncludedFile>();
3439 } else {
3440 return NoError();
3441 }
3442 }
3443 if (!include_paths) {
3444 static const char *current_directory[] = { "", nullptr };
3445 include_paths = current_directory;
3446 }
3447 field_stack_.clear();
3448 builder_.Clear();
3449 // Start with a blank namespace just in case this file doesn't have one.
3450 current_namespace_ = empty_namespace_;
3451
3452 ECHECK(StartParseFile(source, source_filename));
3453
3454 // Includes must come before type declarations:
3455 for (;;) {
3456 // Parse pre-include proto statements if any:
3457 if (opts.proto_mode && (attribute_ == "option" || attribute_ == "syntax" ||
3458 attribute_ == "package")) {
3459 ECHECK(ParseProtoDecl());
3460 } else if (IsIdent("native_include")) {
3461 NEXT();
3462 native_included_files_.emplace_back(attribute_);
3463 EXPECT(kTokenStringConstant);
3464 EXPECT(';');
3465 } else if (IsIdent("include") || (opts.proto_mode && IsIdent("import"))) {
3466 NEXT();
3467 if (opts.proto_mode && attribute_ == "public") NEXT();
3468 auto name = flatbuffers::PosixPath(attribute_.c_str());
3469 EXPECT(kTokenStringConstant);
3470 // Look for the file relative to the directory of the current file.
3471 std::string filepath;
3472 if (source_filename) {
3473 auto source_file_directory =
3474 flatbuffers::StripFileName(source_filename);
3475 filepath = flatbuffers::ConCatPathFileName(source_file_directory, name);
3476 }
3477 if (filepath.empty() || !FileExists(filepath.c_str())) {
3478 // Look for the file in include_paths.
3479 for (auto paths = include_paths; paths && *paths; paths++) {
3480 filepath = flatbuffers::ConCatPathFileName(*paths, name);
3481 if (FileExists(filepath.c_str())) break;
3482 }
3483 }
3484 if (filepath.empty())
3485 return Error("unable to locate include file: " + name);
3486 if (source_filename) {
3487 IncludedFile included_file;
3488 included_file.filename = filepath;
3489 included_file.schema_name = name;
3490 files_included_per_file_[source_filename].insert(included_file);
3491 }
3492
3493 std::string contents;
3494 bool file_loaded = LoadFile(filepath.c_str(), true, &contents);
3495 if (included_files_.find(HashFile(filepath.c_str(), contents.c_str())) ==
3496 included_files_.end()) {
3497 // We found an include file that we have not parsed yet.
3498 // Parse it.
3499 if (!file_loaded) return Error("unable to load include file: " + name);
3500 ECHECK(DoParse(contents.c_str(), include_paths, filepath.c_str(),
3501 name.c_str()));
3502 // We generally do not want to output code for any included files:
3503 if (!opts.generate_all) MarkGenerated();
3504 // Reset these just in case the included file had them, and the
3505 // parent doesn't.
3506 root_struct_def_ = nullptr;
3507 file_identifier_.clear();
3508 file_extension_.clear();
3509 // This is the easiest way to continue this file after an include:
3510 // instead of saving and restoring all the state, we simply start the
3511 // file anew. This will cause it to encounter the same include
3512 // statement again, but this time it will skip it, because it was
3513 // entered into included_files_.
3514 // This is recursive, but only go as deep as the number of include
3515 // statements.
3516 included_files_.erase(source_hash);
3517 return DoParse(source, include_paths, source_filename,
3518 include_filename);
3519 }
3520 EXPECT(';');
3521 } else {
3522 break;
3523 }
3524 }
3525 // Now parse all other kinds of declarations:
3526 while (token_ != kTokenEof) {
3527 if (opts.proto_mode) {
3528 ECHECK(ParseProtoDecl());
3529 } else if (IsIdent("namespace")) {
3530 ECHECK(ParseNamespace());
3531 } else if (token_ == '{') {
3532 return NoError();
3533 } else if (IsIdent("enum")) {
3534 ECHECK(ParseEnum(false, nullptr, source_filename));
3535 } else if (IsIdent("union")) {
3536 ECHECK(ParseEnum(true, nullptr, source_filename));
3537 } else if (IsIdent("root_type")) {
3538 NEXT();
3539 auto root_type = attribute_;
3540 EXPECT(kTokenIdentifier);
3541 ECHECK(ParseNamespacing(&root_type, nullptr));
3542 if (opts.root_type.empty()) {
3543 if (!SetRootType(root_type.c_str()))
3544 return Error("unknown root type: " + root_type);
3545 if (root_struct_def_->fixed) return Error("root type must be a table");
3546 }
3547 EXPECT(';');
3548 } else if (IsIdent("file_identifier")) {
3549 NEXT();
3550 file_identifier_ = attribute_;
3551 EXPECT(kTokenStringConstant);
3552 if (file_identifier_.length() != flatbuffers::kFileIdentifierLength)
3553 return Error("file_identifier must be exactly " +
3554 NumToString(flatbuffers::kFileIdentifierLength) +
3555 " characters");
3556 EXPECT(';');
3557 } else if (IsIdent("file_extension")) {
3558 NEXT();
3559 file_extension_ = attribute_;
3560 EXPECT(kTokenStringConstant);
3561 EXPECT(';');
3562 } else if (IsIdent("include")) {
3563 return Error("includes must come before declarations");
3564 } else if (IsIdent("attribute")) {
3565 NEXT();
3566 auto name = attribute_;
3567 if (Is(kTokenIdentifier)) {
3568 NEXT();
3569 } else {
3570 EXPECT(kTokenStringConstant);
3571 }
3572 EXPECT(';');
3573 known_attributes_[name] = false;
3574 } else if (IsIdent("rpc_service")) {
3575 ECHECK(ParseService(source_filename));
3576 } else {
3577 ECHECK(ParseDecl(source_filename));
3578 }
3579 }
3580 EXPECT(kTokenEof);
3581 if (opts.warnings_as_errors && has_warning_) {
3582 return Error("treating warnings as errors, failed due to above warnings");
3583 }
3584 return NoError();
3585 }
3586
DoParseJson()3587 CheckedError Parser::DoParseJson() {
3588 if (token_ != '{') {
3589 EXPECT('{');
3590 } else {
3591 if (!root_struct_def_) return Error("no root type set to parse json with");
3592 if (builder_.GetSize()) {
3593 return Error("cannot have more than one json object in a file");
3594 }
3595 uoffset_t toff;
3596 ECHECK(ParseTable(*root_struct_def_, nullptr, &toff));
3597 if (opts.size_prefixed) {
3598 builder_.FinishSizePrefixed(
3599 Offset<Table>(toff),
3600 file_identifier_.length() ? file_identifier_.c_str() : nullptr);
3601 } else {
3602 builder_.Finish(Offset<Table>(toff), file_identifier_.length()
3603 ? file_identifier_.c_str()
3604 : nullptr);
3605 }
3606 }
3607 // Check that JSON file doesn't contain more objects or IDL directives.
3608 // Comments after JSON are allowed.
3609 EXPECT(kTokenEof);
3610 return NoError();
3611 }
3612
GetIncludedFilesRecursive(const std::string & file_name) const3613 std::set<std::string> Parser::GetIncludedFilesRecursive(
3614 const std::string &file_name) const {
3615 std::set<std::string> included_files;
3616 std::list<std::string> to_process;
3617
3618 if (file_name.empty()) return included_files;
3619 to_process.push_back(file_name);
3620
3621 while (!to_process.empty()) {
3622 std::string current = to_process.front();
3623 to_process.pop_front();
3624 included_files.insert(current);
3625
3626 // Workaround the lack of const accessor in C++98 maps.
3627 auto &new_files =
3628 (*const_cast<std::map<std::string, std::set<IncludedFile>> *>(
3629 &files_included_per_file_))[current];
3630 for (auto it = new_files.begin(); it != new_files.end(); ++it) {
3631 if (included_files.find(it->filename) == included_files.end())
3632 to_process.push_back(it->filename);
3633 }
3634 }
3635
3636 return included_files;
3637 }
3638
3639 // Schema serialization functionality:
3640
Serialize()3641 void Parser::Serialize() {
3642 builder_.Clear();
3643 AssignIndices(structs_.vec);
3644 AssignIndices(enums_.vec);
3645 std::vector<Offset<reflection::Object>> object_offsets;
3646 std::set<std::string> files;
3647 for (auto it = structs_.vec.begin(); it != structs_.vec.end(); ++it) {
3648 auto offset = (*it)->Serialize(&builder_, *this);
3649 object_offsets.push_back(offset);
3650 (*it)->serialized_location = offset.o;
3651 const std::string *file = (*it)->declaration_file;
3652 if (file) files.insert(*file);
3653 }
3654 std::vector<Offset<reflection::Enum>> enum_offsets;
3655 for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) {
3656 auto offset = (*it)->Serialize(&builder_, *this);
3657 enum_offsets.push_back(offset);
3658 const std::string *file = (*it)->declaration_file;
3659 if (file) files.insert(*file);
3660 }
3661 std::vector<Offset<reflection::Service>> service_offsets;
3662 for (auto it = services_.vec.begin(); it != services_.vec.end(); ++it) {
3663 auto offset = (*it)->Serialize(&builder_, *this);
3664 service_offsets.push_back(offset);
3665 const std::string *file = (*it)->declaration_file;
3666 if (file) files.insert(*file);
3667 }
3668
3669 // Create Schemafiles vector of tables.
3670 flatbuffers::Offset<
3671 flatbuffers::Vector<flatbuffers::Offset<reflection::SchemaFile>>>
3672 schema_files__;
3673 if (!opts.project_root.empty()) {
3674 std::vector<Offset<reflection::SchemaFile>> schema_files;
3675 std::vector<Offset<flatbuffers::String>> included_files;
3676 for (auto f = files_included_per_file_.begin();
3677 f != files_included_per_file_.end(); f++) {
3678 const auto filename__ = builder_.CreateSharedString(
3679 RelativeToRootPath(opts.project_root, f->first));
3680 for (auto i = f->second.begin(); i != f->second.end(); i++) {
3681 included_files.push_back(builder_.CreateSharedString(
3682 RelativeToRootPath(opts.project_root, i->filename)));
3683 }
3684 const auto included_files__ = builder_.CreateVector(included_files);
3685 included_files.clear();
3686
3687 schema_files.push_back(
3688 reflection::CreateSchemaFile(builder_, filename__, included_files__));
3689 }
3690 schema_files__ = builder_.CreateVectorOfSortedTables(&schema_files);
3691 }
3692
3693 const auto objs__ = builder_.CreateVectorOfSortedTables(&object_offsets);
3694 const auto enum__ = builder_.CreateVectorOfSortedTables(&enum_offsets);
3695 const auto fiid__ = builder_.CreateString(file_identifier_);
3696 const auto fext__ = builder_.CreateString(file_extension_);
3697 const auto serv__ = builder_.CreateVectorOfSortedTables(&service_offsets);
3698 const auto schema_offset = reflection::CreateSchema(
3699 builder_, objs__, enum__, fiid__, fext__,
3700 (root_struct_def_ ? root_struct_def_->serialized_location : 0), serv__,
3701 static_cast<reflection::AdvancedFeatures>(advanced_features_),
3702 schema_files__);
3703 if (opts.size_prefixed) {
3704 builder_.FinishSizePrefixed(schema_offset, reflection::SchemaIdentifier());
3705 } else {
3706 builder_.Finish(schema_offset, reflection::SchemaIdentifier());
3707 }
3708 }
3709
Serialize(FlatBufferBuilder * builder,const Parser & parser) const3710 Offset<reflection::Object> StructDef::Serialize(FlatBufferBuilder *builder,
3711 const Parser &parser) const {
3712 std::vector<Offset<reflection::Field>> field_offsets;
3713 for (auto it = fields.vec.begin(); it != fields.vec.end(); ++it) {
3714 field_offsets.push_back((*it)->Serialize(
3715 builder, static_cast<uint16_t>(it - fields.vec.begin()), parser));
3716 }
3717 const auto qualified_name = defined_namespace->GetFullyQualifiedName(name);
3718 const auto name__ = builder->CreateString(qualified_name);
3719 const auto flds__ = builder->CreateVectorOfSortedTables(&field_offsets);
3720 const auto attr__ = SerializeAttributes(builder, parser);
3721 const auto docs__ = parser.opts.binary_schema_comments
3722 ? builder->CreateVectorOfStrings(doc_comment)
3723 : 0;
3724 std::string decl_file_in_project = declaration_file ? *declaration_file : "";
3725 const auto file__ = builder->CreateSharedString(decl_file_in_project);
3726 return reflection::CreateObject(
3727 *builder, name__, flds__, fixed, static_cast<int>(minalign),
3728 static_cast<int>(bytesize), attr__, docs__, file__);
3729 }
3730
Deserialize(Parser & parser,const reflection::Object * object)3731 bool StructDef::Deserialize(Parser &parser, const reflection::Object *object) {
3732 if (!DeserializeAttributes(parser, object->attributes())) return false;
3733 DeserializeDoc(doc_comment, object->documentation());
3734 name = parser.UnqualifiedName(object->name()->str());
3735 predecl = false;
3736 sortbysize = attributes.Lookup("original_order") == nullptr && !fixed;
3737 const auto &of = *(object->fields());
3738 auto indexes = std::vector<uoffset_t>(of.size());
3739 for (uoffset_t i = 0; i < of.size(); i++) indexes[of.Get(i)->id()] = i;
3740 size_t tmp_struct_size = 0;
3741 for (size_t i = 0; i < indexes.size(); i++) {
3742 auto field = of.Get(indexes[i]);
3743 auto field_def = new FieldDef();
3744 if (!field_def->Deserialize(parser, field) ||
3745 fields.Add(field_def->name, field_def)) {
3746 delete field_def;
3747 return false;
3748 }
3749 if (field_def->key) {
3750 if (has_key) {
3751 // only one field may be set as key
3752 delete field_def;
3753 return false;
3754 }
3755 has_key = true;
3756 }
3757 if (fixed) {
3758 // Recompute padding since that's currently not serialized.
3759 auto size = InlineSize(field_def->value.type);
3760 auto next_field =
3761 i + 1 < indexes.size() ? of.Get(indexes[i + 1]) : nullptr;
3762 tmp_struct_size += size;
3763 field_def->padding =
3764 next_field ? (next_field->offset() - field_def->value.offset) - size
3765 : PaddingBytes(tmp_struct_size, minalign);
3766 tmp_struct_size += field_def->padding;
3767 }
3768 }
3769 FLATBUFFERS_ASSERT(static_cast<int>(tmp_struct_size) == object->bytesize());
3770 return true;
3771 }
3772
Serialize(FlatBufferBuilder * builder,uint16_t id,const Parser & parser) const3773 Offset<reflection::Field> FieldDef::Serialize(FlatBufferBuilder *builder,
3774 uint16_t id,
3775 const Parser &parser) const {
3776 auto name__ = builder->CreateString(name);
3777 auto type__ = value.type.Serialize(builder);
3778 auto attr__ = SerializeAttributes(builder, parser);
3779 auto docs__ = parser.opts.binary_schema_comments
3780 ? builder->CreateVectorOfStrings(doc_comment)
3781 : 0;
3782 double d;
3783 StringToNumber(value.constant.c_str(), &d);
3784 return reflection::CreateField(
3785 *builder, name__, type__, id, value.offset,
3786 // Is uint64>max(int64) tested?
3787 IsInteger(value.type.base_type) ? StringToInt(value.constant.c_str()) : 0,
3788 // result may be platform-dependent if underlying is float (not double)
3789 IsFloat(value.type.base_type) ? d : 0.0, deprecated, IsRequired(), key,
3790 attr__, docs__, IsOptional(), static_cast<uint16_t>(padding));
3791 // TODO: value.constant is almost always "0", we could save quite a bit of
3792 // space by sharing it. Same for common values of value.type.
3793 }
3794
Deserialize(Parser & parser,const reflection::Field * field)3795 bool FieldDef::Deserialize(Parser &parser, const reflection::Field *field) {
3796 name = field->name()->str();
3797 defined_namespace = parser.current_namespace_;
3798 if (!value.type.Deserialize(parser, field->type())) return false;
3799 value.offset = field->offset();
3800 if (IsInteger(value.type.base_type)) {
3801 value.constant = NumToString(field->default_integer());
3802 } else if (IsFloat(value.type.base_type)) {
3803 value.constant = FloatToString(field->default_real(), 16);
3804 }
3805 presence = FieldDef::MakeFieldPresence(field->optional(), field->required());
3806 padding = field->padding();
3807 key = field->key();
3808 if (!DeserializeAttributes(parser, field->attributes())) return false;
3809 // TODO: this should probably be handled by a separate attribute
3810 if (attributes.Lookup("flexbuffer")) {
3811 flexbuffer = true;
3812 parser.uses_flexbuffers_ = true;
3813 if (value.type.base_type != BASE_TYPE_VECTOR ||
3814 value.type.element != BASE_TYPE_UCHAR)
3815 return false;
3816 }
3817 if (auto nested = attributes.Lookup("nested_flatbuffer")) {
3818 auto nested_qualified_name =
3819 parser.current_namespace_->GetFullyQualifiedName(nested->constant);
3820 nested_flatbuffer = parser.LookupStruct(nested_qualified_name);
3821 if (!nested_flatbuffer) return false;
3822 }
3823 shared = attributes.Lookup("shared") != nullptr;
3824 DeserializeDoc(doc_comment, field->documentation());
3825 return true;
3826 }
3827
Serialize(FlatBufferBuilder * builder,const Parser & parser) const3828 Offset<reflection::RPCCall> RPCCall::Serialize(FlatBufferBuilder *builder,
3829 const Parser &parser) const {
3830 auto name__ = builder->CreateString(name);
3831 auto attr__ = SerializeAttributes(builder, parser);
3832 auto docs__ = parser.opts.binary_schema_comments
3833 ? builder->CreateVectorOfStrings(doc_comment)
3834 : 0;
3835 return reflection::CreateRPCCall(
3836 *builder, name__, request->serialized_location,
3837 response->serialized_location, attr__, docs__);
3838 }
3839
Deserialize(Parser & parser,const reflection::RPCCall * call)3840 bool RPCCall::Deserialize(Parser &parser, const reflection::RPCCall *call) {
3841 name = call->name()->str();
3842 if (!DeserializeAttributes(parser, call->attributes())) return false;
3843 DeserializeDoc(doc_comment, call->documentation());
3844 request = parser.structs_.Lookup(call->request()->name()->str());
3845 response = parser.structs_.Lookup(call->response()->name()->str());
3846 if (!request || !response) { return false; }
3847 return true;
3848 }
3849
Serialize(FlatBufferBuilder * builder,const Parser & parser) const3850 Offset<reflection::Service> ServiceDef::Serialize(FlatBufferBuilder *builder,
3851 const Parser &parser) const {
3852 std::vector<Offset<reflection::RPCCall>> servicecall_offsets;
3853 for (auto it = calls.vec.begin(); it != calls.vec.end(); ++it) {
3854 servicecall_offsets.push_back((*it)->Serialize(builder, parser));
3855 }
3856 const auto qualified_name = defined_namespace->GetFullyQualifiedName(name);
3857 const auto name__ = builder->CreateString(qualified_name);
3858 const auto call__ = builder->CreateVector(servicecall_offsets);
3859 const auto attr__ = SerializeAttributes(builder, parser);
3860 const auto docs__ = parser.opts.binary_schema_comments
3861 ? builder->CreateVectorOfStrings(doc_comment)
3862 : 0;
3863 std::string decl_file_in_project = declaration_file ? *declaration_file : "";
3864 const auto file__ = builder->CreateSharedString(decl_file_in_project);
3865 return reflection::CreateService(*builder, name__, call__, attr__, docs__,
3866 file__);
3867 }
3868
Deserialize(Parser & parser,const reflection::Service * service)3869 bool ServiceDef::Deserialize(Parser &parser,
3870 const reflection::Service *service) {
3871 name = parser.UnqualifiedName(service->name()->str());
3872 if (service->calls()) {
3873 for (uoffset_t i = 0; i < service->calls()->size(); ++i) {
3874 auto call = new RPCCall();
3875 if (!call->Deserialize(parser, service->calls()->Get(i)) ||
3876 calls.Add(call->name, call)) {
3877 delete call;
3878 return false;
3879 }
3880 }
3881 }
3882 if (!DeserializeAttributes(parser, service->attributes())) return false;
3883 DeserializeDoc(doc_comment, service->documentation());
3884 return true;
3885 }
3886
Serialize(FlatBufferBuilder * builder,const Parser & parser) const3887 Offset<reflection::Enum> EnumDef::Serialize(FlatBufferBuilder *builder,
3888 const Parser &parser) const {
3889 std::vector<Offset<reflection::EnumVal>> enumval_offsets;
3890 for (auto it = vals.vec.begin(); it != vals.vec.end(); ++it) {
3891 enumval_offsets.push_back((*it)->Serialize(builder, parser));
3892 }
3893 const auto qualified_name = defined_namespace->GetFullyQualifiedName(name);
3894 const auto name__ = builder->CreateString(qualified_name);
3895 const auto vals__ = builder->CreateVector(enumval_offsets);
3896 const auto type__ = underlying_type.Serialize(builder);
3897 const auto attr__ = SerializeAttributes(builder, parser);
3898 const auto docs__ = parser.opts.binary_schema_comments
3899 ? builder->CreateVectorOfStrings(doc_comment)
3900 : 0;
3901 std::string decl_file_in_project = declaration_file ? *declaration_file : "";
3902 const auto file__ = builder->CreateSharedString(decl_file_in_project);
3903 return reflection::CreateEnum(*builder, name__, vals__, is_union, type__,
3904 attr__, docs__, file__);
3905 }
3906
Deserialize(Parser & parser,const reflection::Enum * _enum)3907 bool EnumDef::Deserialize(Parser &parser, const reflection::Enum *_enum) {
3908 name = parser.UnqualifiedName(_enum->name()->str());
3909 for (uoffset_t i = 0; i < _enum->values()->size(); ++i) {
3910 auto val = new EnumVal();
3911 if (!val->Deserialize(parser, _enum->values()->Get(i)) ||
3912 vals.Add(val->name, val)) {
3913 delete val;
3914 return false;
3915 }
3916 }
3917 is_union = _enum->is_union();
3918 if (!underlying_type.Deserialize(parser, _enum->underlying_type())) {
3919 return false;
3920 }
3921 if (!DeserializeAttributes(parser, _enum->attributes())) return false;
3922 DeserializeDoc(doc_comment, _enum->documentation());
3923 return true;
3924 }
3925
Serialize(FlatBufferBuilder * builder,const Parser & parser) const3926 Offset<reflection::EnumVal> EnumVal::Serialize(FlatBufferBuilder *builder,
3927 const Parser &parser) const {
3928 auto name__ = builder->CreateString(name);
3929 auto type__ = union_type.Serialize(builder);
3930 auto docs__ = parser.opts.binary_schema_comments
3931 ? builder->CreateVectorOfStrings(doc_comment)
3932 : 0;
3933 return reflection::CreateEnumVal(*builder, name__, value, type__, docs__);
3934 }
3935
Deserialize(const Parser & parser,const reflection::EnumVal * val)3936 bool EnumVal::Deserialize(const Parser &parser,
3937 const reflection::EnumVal *val) {
3938 name = val->name()->str();
3939 value = val->value();
3940 if (!union_type.Deserialize(parser, val->union_type())) return false;
3941 DeserializeDoc(doc_comment, val->documentation());
3942 return true;
3943 }
3944
Serialize(FlatBufferBuilder * builder) const3945 Offset<reflection::Type> Type::Serialize(FlatBufferBuilder *builder) const {
3946 return reflection::CreateType(
3947 *builder, static_cast<reflection::BaseType>(base_type),
3948 static_cast<reflection::BaseType>(element),
3949 struct_def ? struct_def->index : (enum_def ? enum_def->index : -1),
3950 fixed_length, static_cast<uint32_t>(SizeOf(base_type)),
3951 static_cast<uint32_t>(SizeOf(element)));
3952 }
3953
Deserialize(const Parser & parser,const reflection::Type * type)3954 bool Type::Deserialize(const Parser &parser, const reflection::Type *type) {
3955 if (type == nullptr) return true;
3956 base_type = static_cast<BaseType>(type->base_type());
3957 element = static_cast<BaseType>(type->element());
3958 fixed_length = type->fixed_length();
3959 if (type->index() >= 0) {
3960 bool is_series = type->base_type() == reflection::Vector ||
3961 type->base_type() == reflection::Array;
3962 if (type->base_type() == reflection::Obj ||
3963 (is_series && type->element() == reflection::Obj)) {
3964 if (static_cast<size_t>(type->index()) < parser.structs_.vec.size()) {
3965 struct_def = parser.structs_.vec[type->index()];
3966 struct_def->refcount++;
3967 } else {
3968 return false;
3969 }
3970 } else {
3971 if (static_cast<size_t>(type->index()) < parser.enums_.vec.size()) {
3972 enum_def = parser.enums_.vec[type->index()];
3973 } else {
3974 return false;
3975 }
3976 }
3977 }
3978 return true;
3979 }
3980
3981 flatbuffers::Offset<
3982 flatbuffers::Vector<flatbuffers::Offset<reflection::KeyValue>>>
SerializeAttributes(FlatBufferBuilder * builder,const Parser & parser) const3983 Definition::SerializeAttributes(FlatBufferBuilder *builder,
3984 const Parser &parser) const {
3985 std::vector<flatbuffers::Offset<reflection::KeyValue>> attrs;
3986 for (auto kv = attributes.dict.begin(); kv != attributes.dict.end(); ++kv) {
3987 auto it = parser.known_attributes_.find(kv->first);
3988 FLATBUFFERS_ASSERT(it != parser.known_attributes_.end());
3989 if (parser.opts.binary_schema_builtins || !it->second) {
3990 auto key = builder->CreateString(kv->first);
3991 auto val = builder->CreateString(kv->second->constant);
3992 attrs.push_back(reflection::CreateKeyValue(*builder, key, val));
3993 }
3994 }
3995 if (attrs.size()) {
3996 return builder->CreateVectorOfSortedTables(&attrs);
3997 } else {
3998 return 0;
3999 }
4000 }
4001
DeserializeAttributes(Parser & parser,const Vector<Offset<reflection::KeyValue>> * attrs)4002 bool Definition::DeserializeAttributes(
4003 Parser &parser, const Vector<Offset<reflection::KeyValue>> *attrs) {
4004 if (attrs == nullptr) return true;
4005 for (uoffset_t i = 0; i < attrs->size(); ++i) {
4006 auto kv = attrs->Get(i);
4007 auto value = new Value();
4008 if (kv->value()) { value->constant = kv->value()->str(); }
4009 if (attributes.Add(kv->key()->str(), value)) {
4010 delete value;
4011 return false;
4012 }
4013 parser.known_attributes_[kv->key()->str()];
4014 }
4015 return true;
4016 }
4017
4018 /************************************************************************/
4019 /* DESERIALIZATION */
4020 /************************************************************************/
Deserialize(const uint8_t * buf,const size_t size)4021 bool Parser::Deserialize(const uint8_t *buf, const size_t size) {
4022 flatbuffers::Verifier verifier(reinterpret_cast<const uint8_t *>(buf), size);
4023 bool size_prefixed = false;
4024 if (!reflection::SchemaBufferHasIdentifier(buf)) {
4025 if (!flatbuffers::BufferHasIdentifier(buf, reflection::SchemaIdentifier(),
4026 true))
4027 return false;
4028 else
4029 size_prefixed = true;
4030 }
4031 auto verify_fn = size_prefixed ? &reflection::VerifySizePrefixedSchemaBuffer
4032 : &reflection::VerifySchemaBuffer;
4033 if (!verify_fn(verifier)) { return false; }
4034 auto schema = size_prefixed ? reflection::GetSizePrefixedSchema(buf)
4035 : reflection::GetSchema(buf);
4036 return Deserialize(schema);
4037 }
4038
Deserialize(const reflection::Schema * schema)4039 bool Parser::Deserialize(const reflection::Schema *schema) {
4040 file_identifier_ = schema->file_ident() ? schema->file_ident()->str() : "";
4041 file_extension_ = schema->file_ext() ? schema->file_ext()->str() : "";
4042 std::map<std::string, Namespace *> namespaces_index;
4043
4044 // Create defs without deserializing so references from fields to structs and
4045 // enums can be resolved.
4046 for (auto it = schema->objects()->begin(); it != schema->objects()->end();
4047 ++it) {
4048 auto struct_def = new StructDef();
4049 struct_def->bytesize = it->bytesize();
4050 struct_def->fixed = it->is_struct();
4051 struct_def->minalign = it->minalign();
4052 if (structs_.Add(it->name()->str(), struct_def)) {
4053 delete struct_def;
4054 return false;
4055 }
4056 auto type = new Type(BASE_TYPE_STRUCT, struct_def, nullptr);
4057 if (types_.Add(it->name()->str(), type)) {
4058 delete type;
4059 return false;
4060 }
4061 }
4062 for (auto it = schema->enums()->begin(); it != schema->enums()->end(); ++it) {
4063 auto enum_def = new EnumDef();
4064 if (enums_.Add(it->name()->str(), enum_def)) {
4065 delete enum_def;
4066 return false;
4067 }
4068 auto type = new Type(BASE_TYPE_UNION, nullptr, enum_def);
4069 if (types_.Add(it->name()->str(), type)) {
4070 delete type;
4071 return false;
4072 }
4073 }
4074
4075 // Now fields can refer to structs and enums by index.
4076 for (auto it = schema->objects()->begin(); it != schema->objects()->end();
4077 ++it) {
4078 std::string qualified_name = it->name()->str();
4079 auto struct_def = structs_.Lookup(qualified_name);
4080 struct_def->defined_namespace =
4081 GetNamespace(qualified_name, namespaces_, namespaces_index);
4082 if (!struct_def->Deserialize(*this, *it)) { return false; }
4083 if (schema->root_table() == *it) { root_struct_def_ = struct_def; }
4084 }
4085 for (auto it = schema->enums()->begin(); it != schema->enums()->end(); ++it) {
4086 std::string qualified_name = it->name()->str();
4087 auto enum_def = enums_.Lookup(qualified_name);
4088 enum_def->defined_namespace =
4089 GetNamespace(qualified_name, namespaces_, namespaces_index);
4090 if (!enum_def->Deserialize(*this, *it)) { return false; }
4091 }
4092
4093 if (schema->services()) {
4094 for (auto it = schema->services()->begin(); it != schema->services()->end();
4095 ++it) {
4096 std::string qualified_name = it->name()->str();
4097 auto service_def = new ServiceDef();
4098 service_def->defined_namespace =
4099 GetNamespace(qualified_name, namespaces_, namespaces_index);
4100 if (!service_def->Deserialize(*this, *it) ||
4101 services_.Add(qualified_name, service_def)) {
4102 delete service_def;
4103 return false;
4104 }
4105 }
4106 }
4107 advanced_features_ = schema->advanced_features();
4108
4109 if (schema->fbs_files())
4110 for (auto s = schema->fbs_files()->begin(); s != schema->fbs_files()->end();
4111 ++s) {
4112 for (auto f = s->included_filenames()->begin();
4113 f != s->included_filenames()->end(); ++f) {
4114 IncludedFile included_file;
4115 included_file.filename = f->str();
4116 files_included_per_file_[s->filename()->str()].insert(included_file);
4117 }
4118 }
4119
4120 return true;
4121 }
4122
ConformTo(const Parser & base)4123 std::string Parser::ConformTo(const Parser &base) {
4124 for (auto sit = structs_.vec.begin(); sit != structs_.vec.end(); ++sit) {
4125 auto &struct_def = **sit;
4126 auto qualified_name =
4127 struct_def.defined_namespace->GetFullyQualifiedName(struct_def.name);
4128 auto struct_def_base = base.LookupStruct(qualified_name);
4129 if (!struct_def_base) continue;
4130 for (auto fit = struct_def.fields.vec.begin();
4131 fit != struct_def.fields.vec.end(); ++fit) {
4132 auto &field = **fit;
4133 auto field_base = struct_def_base->fields.Lookup(field.name);
4134 if (field_base) {
4135 if (field.value.offset != field_base->value.offset)
4136 return "offsets differ for field: " + field.name;
4137 if (field.value.constant != field_base->value.constant)
4138 return "defaults differ for field: " + field.name;
4139 if (!EqualByName(field.value.type, field_base->value.type))
4140 return "types differ for field: " + field.name;
4141 } else {
4142 // Doesn't have to exist, deleting fields is fine.
4143 // But we should check if there is a field that has the same offset
4144 // but is incompatible (in the case of field renaming).
4145 for (auto fbit = struct_def_base->fields.vec.begin();
4146 fbit != struct_def_base->fields.vec.end(); ++fbit) {
4147 field_base = *fbit;
4148 if (field.value.offset == field_base->value.offset) {
4149 if (!EqualByName(field.value.type, field_base->value.type))
4150 return "field renamed to different type: " + field.name;
4151 break;
4152 }
4153 }
4154 }
4155 }
4156 }
4157 for (auto eit = enums_.vec.begin(); eit != enums_.vec.end(); ++eit) {
4158 auto &enum_def = **eit;
4159 auto qualified_name =
4160 enum_def.defined_namespace->GetFullyQualifiedName(enum_def.name);
4161 auto enum_def_base = base.enums_.Lookup(qualified_name);
4162 if (!enum_def_base) continue;
4163 for (auto evit = enum_def.Vals().begin(); evit != enum_def.Vals().end();
4164 ++evit) {
4165 auto &enum_val = **evit;
4166 auto enum_val_base = enum_def_base->Lookup(enum_val.name);
4167 if (enum_val_base) {
4168 if (enum_val != *enum_val_base)
4169 return "values differ for enum: " + enum_val.name;
4170 }
4171 }
4172 }
4173 return "";
4174 }
4175
4176 } // namespace flatbuffers
4177