1 /*
2 * Copyright 2014 Google Inc. All rights reserved.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 // independent from idl_parser, since this code is not needed for most clients
18
19 #include "idl_gen_python.h"
20
21 #include <cctype>
22 #include <set>
23 #include <string>
24 #include <unordered_set>
25 #include <vector>
26
27 #include "flatbuffers/code_generators.h"
28 #include "flatbuffers/flatbuffers.h"
29 #include "flatbuffers/idl.h"
30 #include "flatbuffers/util.h"
31 #include "idl_namer.h"
32
33 namespace flatbuffers {
34 namespace python {
35
36 namespace {
37
38 typedef std::pair<std::string, std::string> ImportMapEntry;
39 typedef std::set<ImportMapEntry> ImportMap;
40
PythonKeywords()41 static std::set<std::string> PythonKeywords() {
42 return { "False", "None", "True", "and", "as", "assert",
43 "break", "class", "continue", "def", "del", "elif",
44 "else", "except", "finally", "for", "from", "global",
45 "if", "import", "in", "is", "lambda", "nonlocal",
46 "not", "or", "pass", "raise", "return", "try",
47 "while", "with", "yield" };
48 }
49
PythonDefaultConfig()50 static Namer::Config PythonDefaultConfig() {
51 return { /*types=*/Case::kKeep,
52 /*constants=*/Case::kScreamingSnake,
53 /*methods=*/Case::kUpperCamel,
54 /*functions=*/Case::kUpperCamel,
55 /*fields=*/Case::kLowerCamel,
56 /*variable=*/Case::kLowerCamel,
57 /*variants=*/Case::kKeep,
58 /*enum_variant_seperator=*/".",
59 /*escape_keywords=*/Namer::Config::Escape::BeforeConvertingCase,
60 /*namespaces=*/Case::kKeep, // Packages in python.
61 /*namespace_seperator=*/".",
62 /*object_prefix=*/"",
63 /*object_suffix=*/"T",
64 /*keyword_prefix=*/"",
65 /*keyword_suffix=*/"_",
66 /*filenames=*/Case::kKeep,
67 /*directories=*/Case::kKeep,
68 /*output_path=*/"",
69 /*filename_suffix=*/"",
70 /*filename_extension=*/".py" };
71 }
72
73 // Hardcode spaces per indentation.
74 static const CommentConfig def_comment = { nullptr, "#", nullptr };
75 static const std::string Indent = " ";
76
77 } // namespace
78
79 class PythonGenerator : public BaseGenerator {
80 public:
PythonGenerator(const Parser & parser,const std::string & path,const std::string & file_name)81 PythonGenerator(const Parser &parser, const std::string &path,
82 const std::string &file_name)
83 : BaseGenerator(parser, path, file_name, "" /* not used */,
84 "" /* not used */, "py"),
85 float_const_gen_("float('nan')", "float('inf')", "float('-inf')"),
86 namer_(WithFlagOptions(PythonDefaultConfig(), parser.opts, path),
87 PythonKeywords()) {}
88
89 // Most field accessors need to retrieve and test the field offset first,
90 // this is the prefix code for that.
OffsetPrefix(const FieldDef & field,bool new_line=true) const91 std::string OffsetPrefix(const FieldDef &field, bool new_line = true) const {
92 return "\n" + Indent + Indent +
93 "o = flatbuffers.number_types.UOffsetTFlags.py_type" +
94 "(self._tab.Offset(" + NumToString(field.value.offset) + "))\n" +
95 Indent + Indent + "if o != 0:" + (new_line ? "\n" : "");
96 }
97
98 // Begin a class declaration.
BeginClass(const StructDef & struct_def,std::string * code_ptr) const99 void BeginClass(const StructDef &struct_def, std::string *code_ptr) const {
100 auto &code = *code_ptr;
101 code += "class " + namer_.Type(struct_def) + "(object):\n";
102 code += Indent + "__slots__ = ['_tab']";
103 code += "\n\n";
104 }
105
106 // Begin enum code with a class declaration.
BeginEnum(const EnumDef & enum_def,std::string * code_ptr) const107 void BeginEnum(const EnumDef &enum_def, std::string *code_ptr) const {
108 auto &code = *code_ptr;
109 code += "class " + namer_.Type(enum_def) + "(object):\n";
110 }
111
112 // Starts a new line and then indents.
GenIndents(int num) const113 std::string GenIndents(int num) const {
114 return "\n" + std::string(num * Indent.length(), ' ');
115 }
116
117 // A single enum member.
EnumMember(const EnumDef & enum_def,const EnumVal & ev,std::string * code_ptr) const118 void EnumMember(const EnumDef &enum_def, const EnumVal &ev,
119 std::string *code_ptr) const {
120 auto &code = *code_ptr;
121 code += Indent;
122 code += namer_.Variant(ev);
123 code += " = ";
124 code += enum_def.ToString(ev) + "\n";
125 }
126
127 // Initialize a new struct or table from existing data.
NewRootTypeFromBuffer(const StructDef & struct_def,std::string * code_ptr) const128 void NewRootTypeFromBuffer(const StructDef &struct_def,
129 std::string *code_ptr) const {
130 auto &code = *code_ptr;
131 const std::string struct_type = namer_.Type(struct_def);
132
133 code += Indent + "@classmethod\n";
134 code += Indent + "def GetRootAs";
135 if (parser_.opts.python_typing) {
136 code += "(cls, buf, offset: int = 0):";
137 } else {
138 code += "(cls, buf, offset=0):";
139 }
140 code += "\n";
141 code += Indent + Indent;
142 code += "n = flatbuffers.encode.Get";
143 code += "(flatbuffers.packer.uoffset, buf, offset)\n";
144 code += Indent + Indent + "x = " + struct_type + "()\n";
145 code += Indent + Indent + "x.Init(buf, n + offset)\n";
146 code += Indent + Indent + "return x\n";
147 code += "\n";
148
149 if (!parser_.opts.python_no_type_prefix_suffix) {
150 // Add an alias with the old name
151 code += Indent + "@classmethod\n";
152 code +=
153 Indent + "def GetRootAs" + struct_type + "(cls, buf, offset=0):\n";
154 code += Indent + Indent +
155 "\"\"\"This method is deprecated. Please switch to "
156 "GetRootAs.\"\"\"\n";
157 code += Indent + Indent + "return cls.GetRootAs(buf, offset)\n";
158 }
159 }
160
161 // Initialize an existing object with other data, to avoid an allocation.
InitializeExisting(const StructDef & struct_def,std::string * code_ptr) const162 void InitializeExisting(const StructDef &struct_def,
163 std::string *code_ptr) const {
164 auto &code = *code_ptr;
165
166 GenReceiver(struct_def, code_ptr);
167 if (parser_.opts.python_typing) {
168 code += "Init(self, buf: bytes, pos: int):\n";
169 } else {
170 code += "Init(self, buf, pos):\n";
171 }
172 code += Indent + Indent + "self._tab = flatbuffers.table.Table(buf, pos)\n";
173 code += "\n";
174 }
175
176 // Get the length of a vector.
GetVectorLen(const StructDef & struct_def,const FieldDef & field,std::string * code_ptr) const177 void GetVectorLen(const StructDef &struct_def, const FieldDef &field,
178 std::string *code_ptr) const {
179 auto &code = *code_ptr;
180
181 GenReceiver(struct_def, code_ptr);
182 code += namer_.Method(field) + "Length(self)";
183 if (parser_.opts.python_typing) { code += " -> int"; }
184 code += ":";
185 if (!IsArray(field.value.type)) {
186 code += OffsetPrefix(field, false);
187 code += GenIndents(3) + "return self._tab.VectorLen(o)";
188 code += GenIndents(2) + "return 0\n\n";
189 } else {
190 code += GenIndents(2) + "return " +
191 NumToString(field.value.type.fixed_length) + "\n\n";
192 }
193 }
194
195 // Determines whether a vector is none or not.
GetVectorIsNone(const StructDef & struct_def,const FieldDef & field,std::string * code_ptr) const196 void GetVectorIsNone(const StructDef &struct_def, const FieldDef &field,
197 std::string *code_ptr) const {
198 auto &code = *code_ptr;
199
200 GenReceiver(struct_def, code_ptr);
201 code += namer_.Method(field) + "IsNone(self)";
202 if (parser_.opts.python_typing) { code += " -> bool"; }
203 code += ":";
204 if (!IsArray(field.value.type)) {
205 code += GenIndents(2) +
206 "o = flatbuffers.number_types.UOffsetTFlags.py_type" +
207 "(self._tab.Offset(" + NumToString(field.value.offset) + "))";
208 code += GenIndents(2) + "return o == 0";
209 } else {
210 // assume that we always have an array as memory is preassigned
211 code += GenIndents(2) + "return False";
212 }
213 code += "\n\n";
214 }
215
216 // Get the value of a struct's scalar.
GetScalarFieldOfStruct(const StructDef & struct_def,const FieldDef & field,std::string * code_ptr) const217 void GetScalarFieldOfStruct(const StructDef &struct_def,
218 const FieldDef &field,
219 std::string *code_ptr) const {
220 auto &code = *code_ptr;
221 std::string getter = GenGetter(field.value.type);
222 GenReceiver(struct_def, code_ptr);
223 code += namer_.Method(field);
224 code += "(self): return " + getter;
225 code += "self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(";
226 code += NumToString(field.value.offset) + "))\n";
227 }
228
229 // Get the value of a table's scalar.
GetScalarFieldOfTable(const StructDef & struct_def,const FieldDef & field,std::string * code_ptr) const230 void GetScalarFieldOfTable(const StructDef &struct_def, const FieldDef &field,
231 std::string *code_ptr) const {
232 auto &code = *code_ptr;
233 std::string getter = GenGetter(field.value.type);
234 GenReceiver(struct_def, code_ptr);
235 code += namer_.Method(field);
236 code += "(self):";
237 code += OffsetPrefix(field);
238 getter += "o + self._tab.Pos)";
239 auto is_bool = IsBool(field.value.type.base_type);
240 if (is_bool) { getter = "bool(" + getter + ")"; }
241 code += Indent + Indent + Indent + "return " + getter + "\n";
242 std::string default_value;
243 if (field.IsScalarOptional()) {
244 default_value = "None";
245 } else if (is_bool) {
246 default_value = field.value.constant == "0" ? "False" : "True";
247 } else {
248 default_value = IsFloat(field.value.type.base_type)
249 ? float_const_gen_.GenFloatConstant(field)
250 : field.value.constant;
251 }
252 code += Indent + Indent + "return " + default_value + "\n\n";
253 }
254
255 // Get a struct by initializing an existing struct.
256 // Specific to Struct.
GetStructFieldOfStruct(const StructDef & struct_def,const FieldDef & field,std::string * code_ptr) const257 void GetStructFieldOfStruct(const StructDef &struct_def,
258 const FieldDef &field,
259 std::string *code_ptr) const {
260 auto &code = *code_ptr;
261 GenReceiver(struct_def, code_ptr);
262 code += namer_.Method(field);
263 code += "(self, obj):\n";
264 code += Indent + Indent + "obj.Init(self._tab.Bytes, self._tab.Pos + ";
265 code += NumToString(field.value.offset) + ")";
266 code += "\n" + Indent + Indent + "return obj\n\n";
267 }
268
269 // Get the value of a fixed size array.
GetArrayOfStruct(const StructDef & struct_def,const FieldDef & field,std::string * code_ptr,ImportMap & imports) const270 void GetArrayOfStruct(const StructDef &struct_def, const FieldDef &field,
271 std::string *code_ptr, ImportMap &imports) const {
272 auto &code = *code_ptr;
273 const auto vec_type = field.value.type.VectorType();
274 GenReceiver(struct_def, code_ptr);
275 code += namer_.Method(field);
276
277 const ImportMapEntry import_entry = {
278 GenPackageReference(field.value.type), TypeName(field)
279 };
280
281 if (parser_.opts.python_typing) {
282 const std::string return_type = ReturnType(struct_def, field);
283 code += "(self, i: int)";
284 code += " -> " + return_type + ":";
285
286 imports.insert(import_entry);
287 } else {
288 code += "(self, i):";
289 }
290
291 if (parser_.opts.include_dependence_headers &&
292 !parser_.opts.python_typing) {
293 code += GenIndents(2);
294 code += "from " + import_entry.first + " import " + import_entry.second +
295 "\n";
296 }
297
298 code += GenIndents(2) + "obj = " + TypeName(field) + "()";
299 code += GenIndents(2) + "obj.Init(self._tab.Bytes, self._tab.Pos + ";
300 code += NumToString(field.value.offset) + " + i * ";
301 code += NumToString(InlineSize(vec_type));
302 code += ")" + GenIndents(2) + "return obj\n\n";
303 }
304
305 // Get the value of a vector's non-struct member. Uses a named return
306 // argument to conveniently set the zero value for the result.
GetArrayOfNonStruct(const StructDef & struct_def,const FieldDef & field,std::string * code_ptr) const307 void GetArrayOfNonStruct(const StructDef &struct_def, const FieldDef &field,
308 std::string *code_ptr) const {
309 auto &code = *code_ptr;
310 GenReceiver(struct_def, code_ptr);
311 code += namer_.Method(field);
312 code += "(self, j = None):";
313 code += GenIndents(2) + "if j is None:";
314 code += GenIndents(3) + "return [" + GenGetter(field.value.type);
315 code += "self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(";
316 code += NumToString(field.value.offset) + " + i * ";
317 code += NumToString(InlineSize(field.value.type.VectorType()));
318 code += ")) for i in range(";
319 code += "self." + namer_.Method(field) + "Length()" + ")]";
320 code += GenIndents(2) + "elif j >= 0 and j < self." + namer_.Method(field) +
321 "Length():";
322 code += GenIndents(3) + "return " + GenGetter(field.value.type);
323 code += "self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(";
324 code += NumToString(field.value.offset) + " + j * ";
325 code += NumToString(InlineSize(field.value.type.VectorType()));
326 code += "))";
327 code += GenIndents(2) + "else:";
328 code += GenIndents(3) + "return None\n\n";
329 }
330
331 // Get a struct by initializing an existing struct.
332 // Specific to Table.
GetStructFieldOfTable(const StructDef & struct_def,const FieldDef & field,std::string * code_ptr,ImportMap & imports) const333 void GetStructFieldOfTable(const StructDef &struct_def, const FieldDef &field,
334 std::string *code_ptr, ImportMap &imports) const {
335 auto &code = *code_ptr;
336 GenReceiver(struct_def, code_ptr);
337 code += namer_.Method(field) + "(self)";
338
339 const ImportMapEntry import_entry = {
340 GenPackageReference(field.value.type), TypeName(field)
341 };
342
343 if (parser_.opts.python_typing) {
344 const std::string return_type = ReturnType(struct_def, field);
345 code += " -> Optional[" + return_type + "]";
346 imports.insert(ImportMapEntry{ "typing", "Optional" });
347 imports.insert(import_entry);
348 }
349 code += ":";
350 code += OffsetPrefix(field);
351 if (field.value.type.struct_def->fixed) {
352 code += Indent + Indent + Indent + "x = o + self._tab.Pos\n";
353 } else {
354 code += Indent + Indent + Indent;
355 code += "x = self._tab.Indirect(o + self._tab.Pos)\n";
356 }
357
358 if (parser_.opts.include_dependence_headers &&
359 !parser_.opts.python_typing) {
360 code += Indent + Indent + Indent;
361 code += "from " + import_entry.first + " import " + import_entry.second +
362 "\n";
363 }
364 code += Indent + Indent + Indent + "obj = " + TypeName(field) + "()\n";
365 code += Indent + Indent + Indent + "obj.Init(self._tab.Bytes, x)\n";
366 code += Indent + Indent + Indent + "return obj\n";
367 code += Indent + Indent + "return None\n\n";
368 }
369
370 // Get the value of a string.
GetStringField(const StructDef & struct_def,const FieldDef & field,std::string * code_ptr,ImportMap & imports) const371 void GetStringField(const StructDef &struct_def, const FieldDef &field,
372 std::string *code_ptr, ImportMap &imports) const {
373 auto &code = *code_ptr;
374 GenReceiver(struct_def, code_ptr);
375 code += namer_.Method(field);
376
377 if (parser_.opts.python_typing) {
378 code += "(self) -> Optional[str]:";
379 imports.insert(ImportMapEntry{ "typing", "Optional" });
380 } else {
381 code += "(self):";
382 }
383
384 code += OffsetPrefix(field);
385 code += Indent + Indent + Indent + "return " + GenGetter(field.value.type);
386 code += "o + self._tab.Pos)\n";
387 code += Indent + Indent + "return None\n\n";
388 }
389
390 // Get the value of a union from an object.
GetUnionField(const StructDef & struct_def,const FieldDef & field,std::string * code_ptr,ImportMap & imports) const391 void GetUnionField(const StructDef &struct_def, const FieldDef &field,
392 std::string *code_ptr, ImportMap &imports) const {
393 auto &code = *code_ptr;
394 GenReceiver(struct_def, code_ptr);
395 std::string return_ty = "flatbuffers.table.Table";
396
397 bool is_native_table = TypeName(field) == "*flatbuffers.Table";
398 ImportMapEntry import_entry;
399 if (is_native_table) {
400 import_entry = ImportMapEntry{ "flatbuffers.table", "Table" };
401 } else {
402 return_ty = TypeName(field);
403 import_entry = ImportMapEntry{ GenPackageReference(field.value.type),
404 TypeName(field) };
405 }
406
407 code += namer_.Method(field) + "(self)";
408 if (parser_.opts.python_typing) {
409 code += " -> Optional[" + return_ty + "]";
410 imports.insert(ImportMapEntry{ "typing", "Optional" });
411 imports.insert(import_entry);
412 }
413 code += ":";
414 code += OffsetPrefix(field);
415
416 if (!parser_.opts.python_typing) {
417 code += Indent + Indent + Indent;
418 code += "from " + import_entry.first + " import " + import_entry.second +
419 "\n";
420 }
421 code += Indent + Indent + Indent + "obj = Table(bytearray(), 0)\n";
422 code += Indent + Indent + Indent + GenGetter(field.value.type);
423 code += "obj, o)\n" + Indent + Indent + Indent + "return obj\n";
424 code += Indent + Indent + "return None\n\n";
425 }
426
427 // Generate the package reference when importing a struct or enum from its
428 // module.
GenPackageReference(const Type & type) const429 std::string GenPackageReference(const Type &type) const {
430 if (type.struct_def) {
431 return namer_.NamespacedType(*type.struct_def);
432 } else if (type.enum_def) {
433 return namer_.NamespacedType(*type.enum_def);
434 } else {
435 return "." + GenTypeGet(type);
436 }
437 }
438
439 // Get the value of a vector's struct member.
GetMemberOfVectorOfStruct(const StructDef & struct_def,const FieldDef & field,std::string * code_ptr,ImportMap & imports) const440 void GetMemberOfVectorOfStruct(const StructDef &struct_def,
441 const FieldDef &field, std::string *code_ptr,
442 ImportMap &imports) const {
443 auto &code = *code_ptr;
444 auto vectortype = field.value.type.VectorType();
445
446 GenReceiver(struct_def, code_ptr);
447 code += namer_.Method(field);
448 const ImportMapEntry import_entry = {
449 GenPackageReference(field.value.type), TypeName(field)
450 };
451
452 if (parser_.opts.python_typing) {
453 const std::string return_type = ReturnType(struct_def, field);
454 code += "(self, j: int) -> Optional[" + return_type + "]";
455 imports.insert(ImportMapEntry{ "typing", "Optional" });
456 imports.insert(import_entry);
457 } else {
458 code += "(self, j)";
459 }
460 code += ":" + OffsetPrefix(field);
461 code += Indent + Indent + Indent + "x = self._tab.Vector(o)\n";
462 code += Indent + Indent + Indent;
463 code += "x += flatbuffers.number_types.UOffsetTFlags.py_type(j) * ";
464 code += NumToString(InlineSize(vectortype)) + "\n";
465 if (!(vectortype.struct_def->fixed)) {
466 code += Indent + Indent + Indent + "x = self._tab.Indirect(x)\n";
467 }
468 if (parser_.opts.include_dependence_headers &&
469 !parser_.opts.python_typing) {
470 code += Indent + Indent + Indent;
471 code += "from " + import_entry.first + " import " + import_entry.second +
472 "\n";
473 }
474 code += Indent + Indent + Indent + "obj = " + TypeName(field) + "()\n";
475 code += Indent + Indent + Indent + "obj.Init(self._tab.Bytes, x)\n";
476 code += Indent + Indent + Indent + "return obj\n";
477 code += Indent + Indent + "return None\n\n";
478 }
479
480 // Get the value of a vector's non-struct member. Uses a named return
481 // argument to conveniently set the zero value for the result.
GetMemberOfVectorOfNonStruct(const StructDef & struct_def,const FieldDef & field,std::string * code_ptr) const482 void GetMemberOfVectorOfNonStruct(const StructDef &struct_def,
483 const FieldDef &field,
484 std::string *code_ptr) const {
485 auto &code = *code_ptr;
486 auto vectortype = field.value.type.VectorType();
487
488 GenReceiver(struct_def, code_ptr);
489 code += namer_.Method(field);
490 if (parser_.opts.python_typing) {
491 code += "(self, j: int)";
492 } else {
493 code += "(self, j)";
494 }
495 code += ":";
496 code += OffsetPrefix(field);
497 code += Indent + Indent + Indent + "a = self._tab.Vector(o)\n";
498 code += Indent + Indent + Indent;
499 code += "return " + GenGetter(field.value.type);
500 code += "a + flatbuffers.number_types.UOffsetTFlags.py_type(j * ";
501 code += NumToString(InlineSize(vectortype)) + "))\n";
502 if (IsString(vectortype)) {
503 code += Indent + Indent + "return \"\"\n";
504 } else {
505 code += Indent + Indent + "return 0\n";
506 }
507 code += "\n";
508 }
509
510 // Returns a non-struct vector as a numpy array. Much faster
511 // than iterating over the vector element by element.
GetVectorOfNonStructAsNumpy(const StructDef & struct_def,const FieldDef & field,std::string * code_ptr) const512 void GetVectorOfNonStructAsNumpy(const StructDef &struct_def,
513 const FieldDef &field,
514 std::string *code_ptr) const {
515 auto &code = *code_ptr;
516 auto vectortype = field.value.type.VectorType();
517
518 // Currently, we only support accessing as numpy array if
519 // the vector type is a scalar.
520 if (!(IsScalar(vectortype.base_type))) { return; }
521
522 GenReceiver(struct_def, code_ptr);
523 code += namer_.Method(field) + "AsNumpy(self):";
524 if (!IsArray(field.value.type)) {
525 code += OffsetPrefix(field, false);
526
527 code += GenIndents(3);
528 code += "return ";
529 code += "self._tab.GetVectorAsNumpy(flatbuffers.number_types.";
530 code += namer_.Method(GenTypeGet(field.value.type));
531 code += "Flags, o)";
532
533 if (IsString(vectortype)) {
534 code += GenIndents(2) + "return \"\"\n";
535 } else {
536 code += GenIndents(2) + "return 0\n";
537 }
538 } else {
539 code += GenIndents(2) + "return ";
540 code += "self._tab.GetArrayAsNumpy(flatbuffers.number_types.";
541 code += namer_.Method(GenTypeGet(field.value.type.VectorType()));
542 code += "Flags, self._tab.Pos + " + NumToString(field.value.offset) +
543 ", " + NumToString("self." + namer_.Method(field) + "Length()") +
544 ")\n";
545 }
546 code += "\n";
547 }
548
NestedFlatbufferType(std::string unqualified_name) const549 std::string NestedFlatbufferType(std::string unqualified_name) const {
550 StructDef *nested_root = parser_.LookupStruct(unqualified_name);
551 std::string qualified_name;
552 if (nested_root == nullptr) {
553 qualified_name = namer_.NamespacedType(
554 parser_.current_namespace_->components, unqualified_name);
555 // Double check qualified name just to be sure it exists.
556 nested_root = parser_.LookupStruct(qualified_name);
557 }
558 FLATBUFFERS_ASSERT(nested_root); // Guaranteed to exist by parser.
559 return qualified_name;
560 }
561
562 // Returns a nested flatbuffer as itself.
GetVectorAsNestedFlatbuffer(const StructDef & struct_def,const FieldDef & field,std::string * code_ptr,ImportMap & imports) const563 void GetVectorAsNestedFlatbuffer(const StructDef &struct_def,
564 const FieldDef &field, std::string *code_ptr,
565 ImportMap &imports) const {
566 auto nested = field.attributes.Lookup("nested_flatbuffer");
567 if (!nested) { return; } // There is no nested flatbuffer.
568
569 const std::string unqualified_name = nested->constant;
570 std::string qualified_name = NestedFlatbufferType(unqualified_name);
571 if (qualified_name.empty()) { qualified_name = nested->constant; }
572
573 const ImportMapEntry import_entry = { qualified_name,
574 unqualified_name };
575
576 auto &code = *code_ptr;
577 GenReceiver(struct_def, code_ptr);
578 code += namer_.Method(field) + "NestedRoot(self)";
579 if (parser_.opts.python_typing) {
580 code += " -> Union[" + unqualified_name + ", int]";
581 imports.insert(ImportMapEntry{ "typing", "Union" });
582 imports.insert(import_entry);
583 }
584 code += ":";
585
586 code += OffsetPrefix(field);
587
588 if (!parser_.opts.python_typing) {
589 code += Indent + Indent + Indent;
590 code += "from " + import_entry.first + " import " + import_entry.second +
591 "\n";
592 }
593 code += Indent + Indent + Indent + "return " + unqualified_name;
594 code += ".GetRootAs";
595 code += "(self._tab.Bytes, self._tab.Vector(o))\n";
596 code += Indent + Indent + "return 0\n";
597 code += "\n";
598 }
599
600 // Begin the creator function signature.
BeginBuilderArgs(const StructDef & struct_def,std::string * code_ptr) const601 void BeginBuilderArgs(const StructDef &struct_def,
602 std::string *code_ptr) const {
603 auto &code = *code_ptr;
604
605 code += "\n";
606 code += "def Create" + namer_.Type(struct_def);
607 code += "(builder";
608 }
609
610 // Recursively generate arguments for a constructor, to deal with nested
611 // structs.
StructBuilderArgs(const StructDef & struct_def,const std::string nameprefix,const std::string namesuffix,bool has_field_name,const std::string fieldname_suffix,std::string * code_ptr) const612 void StructBuilderArgs(const StructDef &struct_def,
613 const std::string nameprefix,
614 const std::string namesuffix, bool has_field_name,
615 const std::string fieldname_suffix,
616 std::string *code_ptr) const {
617 for (auto it = struct_def.fields.vec.begin();
618 it != struct_def.fields.vec.end(); ++it) {
619 auto &field = **it;
620 const auto &field_type = field.value.type;
621 const auto &type =
622 IsArray(field_type) ? field_type.VectorType() : field_type;
623 if (IsStruct(type)) {
624 // Generate arguments for a struct inside a struct. To ensure names
625 // don't clash, and to make it obvious these arguments are constructing
626 // a nested struct, prefix the name with the field name.
627 auto subprefix = nameprefix;
628 if (has_field_name) {
629 subprefix += namer_.Field(field) + fieldname_suffix;
630 }
631 StructBuilderArgs(*field.value.type.struct_def, subprefix, namesuffix,
632 has_field_name, fieldname_suffix, code_ptr);
633 } else {
634 auto &code = *code_ptr;
635 code += std::string(", ") + nameprefix;
636 if (has_field_name) { code += namer_.Field(field); }
637 code += namesuffix;
638 }
639 }
640 }
641
642 // End the creator function signature.
EndBuilderArgs(std::string * code_ptr) const643 void EndBuilderArgs(std::string *code_ptr) const {
644 auto &code = *code_ptr;
645 code += "):\n";
646 }
647
648 // Recursively generate struct construction statements and instert manual
649 // padding.
StructBuilderBody(const StructDef & struct_def,const char * nameprefix,std::string * code_ptr,size_t index=0,bool in_array=false) const650 void StructBuilderBody(const StructDef &struct_def, const char *nameprefix,
651 std::string *code_ptr, size_t index = 0,
652 bool in_array = false) const {
653 auto &code = *code_ptr;
654 std::string indent(index * 4, ' ');
655 code +=
656 indent + " builder.Prep(" + NumToString(struct_def.minalign) + ", ";
657 code += NumToString(struct_def.bytesize) + ")\n";
658 for (auto it = struct_def.fields.vec.rbegin();
659 it != struct_def.fields.vec.rend(); ++it) {
660 auto &field = **it;
661 const auto &field_type = field.value.type;
662 const auto &type =
663 IsArray(field_type) ? field_type.VectorType() : field_type;
664 if (field.padding)
665 code +=
666 indent + " builder.Pad(" + NumToString(field.padding) + ")\n";
667 if (IsStruct(field_type)) {
668 StructBuilderBody(*field_type.struct_def,
669 (nameprefix + (namer_.Field(field) + "_")).c_str(),
670 code_ptr, index, in_array);
671 } else {
672 const auto index_var = "_idx" + NumToString(index);
673 if (IsArray(field_type)) {
674 code += indent + " for " + index_var + " in range(";
675 code += NumToString(field_type.fixed_length);
676 code += " , 0, -1):\n";
677 in_array = true;
678 }
679 if (IsStruct(type)) {
680 StructBuilderBody(*field_type.struct_def,
681 (nameprefix + (namer_.Field(field) + "_")).c_str(),
682 code_ptr, index + 1, in_array);
683 } else {
684 code += IsArray(field_type) ? " " : "";
685 code += indent + " builder.Prepend" + GenMethod(field) + "(";
686 code += nameprefix + namer_.Variable(field);
687 size_t array_cnt = index + (IsArray(field_type) ? 1 : 0);
688 for (size_t i = 0; in_array && i < array_cnt; i++) {
689 code += "[_idx" + NumToString(i) + "-1]";
690 }
691 code += ")\n";
692 }
693 }
694 }
695 }
696
EndBuilderBody(std::string * code_ptr) const697 void EndBuilderBody(std::string *code_ptr) const {
698 auto &code = *code_ptr;
699 code += " return builder.Offset()\n";
700 }
701
702 // Get the value of a table's starting offset.
GetStartOfTable(const StructDef & struct_def,std::string * code_ptr) const703 void GetStartOfTable(const StructDef &struct_def,
704 std::string *code_ptr) const {
705 auto &code = *code_ptr;
706 const auto struct_type = namer_.Type(struct_def);
707 // Generate method with struct name.
708
709 const auto name = parser_.opts.python_no_type_prefix_suffix
710 ? "Start"
711 : struct_type + "Start";
712
713 code += "def " + name;
714 if (parser_.opts.python_typing) {
715 code += "(builder: flatbuffers.Builder):\n";
716 } else {
717 code += "(builder):\n";
718 }
719
720 code += Indent + "builder.StartObject(";
721 code += NumToString(struct_def.fields.vec.size());
722 code += ")\n\n";
723
724 if (!parser_.opts.one_file && !parser_.opts.python_no_type_prefix_suffix) {
725 // Generate method without struct name.
726 if (parser_.opts.python_typing) {
727 code += "def Start(builder: flatbuffers.Builder):\n";
728 } else {
729 code += "def Start(builder):\n";
730 }
731 code += Indent + struct_type + "Start(builder)\n\n";
732 }
733 }
734
735 // Set the value of a table's field.
BuildFieldOfTable(const StructDef & struct_def,const FieldDef & field,const size_t offset,std::string * code_ptr) const736 void BuildFieldOfTable(const StructDef &struct_def, const FieldDef &field,
737 const size_t offset, std::string *code_ptr) const {
738 auto &code = *code_ptr;
739 const std::string field_var = namer_.Variable(field);
740 const std::string field_method = namer_.Method(field);
741 const std::string field_ty = GenFieldTy(field);
742
743 const auto name = parser_.opts.python_no_type_prefix_suffix
744 ? "Add" + field_method
745 : namer_.Type(struct_def) + "Add" + field_method;
746
747 // Generate method with struct name.
748 code += "def " + name;
749 if (parser_.opts.python_typing) {
750 code += "(builder: flatbuffers.Builder, " + field_var + ": " + field_ty;
751 } else {
752 code += "(builder, " + field_var;
753 }
754 code += "):\n";
755 code += Indent + "builder.Prepend";
756 code += GenMethod(field) + "Slot(";
757 code += NumToString(offset) + ", ";
758 if (!IsScalar(field.value.type.base_type) && (!struct_def.fixed)) {
759 code += "flatbuffers.number_types.UOffsetTFlags.py_type";
760 code += "(" + field_var + ")";
761 } else {
762 code += field_var;
763 }
764 code += ", ";
765 if (field.IsScalarOptional()) {
766 code += "None";
767 } else if (IsFloat(field.value.type.base_type)) {
768 code += float_const_gen_.GenFloatConstant(field);
769 } else {
770 code += field.value.constant;
771 }
772 code += ")\n\n";
773
774 if (!parser_.opts.one_file && !parser_.opts.python_no_type_prefix_suffix) {
775 // Generate method without struct name.
776 code += "def Add" + field_method;
777 if (parser_.opts.python_typing) {
778 code += "(builder: flatbuffers.Builder, " + field_var + ": " + field_ty;
779 } else {
780 code += "(builder, " + field_var;
781 }
782 code += "):\n";
783 code += Indent + namer_.Type(struct_def) + "Add" + field_method;
784 code += "(builder, ";
785 code += field_var;
786 code += ")\n\n";
787 }
788 }
789
790 // Set the value of one of the members of a table's vector.
BuildVectorOfTable(const StructDef & struct_def,const FieldDef & field,std::string * code_ptr) const791 void BuildVectorOfTable(const StructDef &struct_def, const FieldDef &field,
792 std::string *code_ptr) const {
793 auto &code = *code_ptr;
794 const std::string struct_type = namer_.Type(struct_def);
795 const std::string field_method = namer_.Method(field);
796
797 // Generate method with struct name.
798 const auto name = parser_.opts.python_no_type_prefix_suffix
799 ? "Start" + field_method
800 : struct_type + "Start" + field_method;
801 code += "def " + name;
802 if (parser_.opts.python_typing) {
803 code += "Vector(builder, numElems: int) -> int:\n";
804 } else {
805 code += "Vector(builder, numElems):\n";
806 }
807
808 code += Indent + "return builder.StartVector(";
809 auto vector_type = field.value.type.VectorType();
810 auto alignment = InlineAlignment(vector_type);
811 auto elem_size = InlineSize(vector_type);
812 code += NumToString(elem_size);
813 code += ", numElems, " + NumToString(alignment);
814 code += ")\n\n";
815
816 if (!parser_.opts.one_file && !parser_.opts.python_no_type_prefix_suffix) {
817 // Generate method without struct name.
818 if (parser_.opts.python_typing) {
819 code += "def Start" + field_method +
820 "Vector(builder, numElems: int) -> int:\n";
821 } else {
822 code += "def Start" + field_method + "Vector(builder, numElems):\n";
823 }
824 code += Indent + "return " + struct_type + "Start";
825 code += field_method + "Vector(builder, numElems)\n\n";
826 }
827 }
828
829 // Set the value of one of the members of a table's vector and fills in the
830 // elements from a bytearray. This is for simplifying the use of nested
831 // flatbuffers.
BuildVectorOfTableFromBytes(const StructDef & struct_def,const FieldDef & field,std::string * code_ptr) const832 void BuildVectorOfTableFromBytes(const StructDef &struct_def,
833 const FieldDef &field,
834 std::string *code_ptr) const {
835 auto nested = field.attributes.Lookup("nested_flatbuffer");
836 if (!nested) { return; } // There is no nested flatbuffer.
837
838 auto &code = *code_ptr;
839 const std::string field_method = namer_.Method(field);
840 const std::string struct_type = namer_.Type(struct_def);
841
842 // Generate method with struct and field name.
843 code += "def " + struct_type + "Make" + field_method;
844 code += "VectorFromBytes(builder, bytes):\n";
845 code += Indent + "builder.StartVector(";
846 auto vector_type = field.value.type.VectorType();
847 auto alignment = InlineAlignment(vector_type);
848 auto elem_size = InlineSize(vector_type);
849 code += NumToString(elem_size);
850 code += ", len(bytes), " + NumToString(alignment);
851 code += ")\n";
852 code += Indent + "builder.head = builder.head - len(bytes)\n";
853 code += Indent + "builder.Bytes[builder.head : builder.head + len(bytes)]";
854 code += " = bytes\n";
855 code += Indent + "return builder.EndVector()\n";
856
857 if (!parser_.opts.one_file) {
858 // Generate method without struct and field name.
859 code += "def Make" + field_method + "VectorFromBytes(builder, bytes):\n";
860 code += Indent + "return " + struct_type + "Make" + field_method +
861 "VectorFromBytes(builder, bytes)\n";
862 }
863 }
864
865 // Get the offset of the end of a table.
GetEndOffsetOnTable(const StructDef & struct_def,std::string * code_ptr) const866 void GetEndOffsetOnTable(const StructDef &struct_def,
867 std::string *code_ptr) const {
868 auto &code = *code_ptr;
869
870 const auto name = parser_.opts.python_no_type_prefix_suffix
871 ? "End"
872 : namer_.Type(struct_def) + "End";
873 // Generate method with struct name.
874 if (parser_.opts.python_typing) {
875 code += "def " + name + "(builder: flatbuffers.Builder) -> int:\n";
876 } else {
877 code += "def " + name + "(builder):\n";
878 }
879 code += Indent + "return builder.EndObject()\n\n";
880
881 if (!parser_.opts.one_file && !parser_.opts.python_no_type_prefix_suffix) {
882 // Generate method without struct name.
883 if (parser_.opts.python_typing) {
884 code += "def End(builder: flatbuffers.Builder) -> int:\n";
885 } else {
886 code += "def End(builder):\n";
887 }
888 code += Indent + "return " + namer_.Type(struct_def) + "End(builder)";
889 code += "\n";
890 }
891 }
892
893 // Generate the receiver for function signatures.
GenReceiver(const StructDef & struct_def,std::string * code_ptr) const894 void GenReceiver(const StructDef &struct_def, std::string *code_ptr) const {
895 auto &code = *code_ptr;
896 code += Indent + "# " + namer_.Type(struct_def) + "\n";
897 code += Indent + "def ";
898 }
899
900 // Generate a struct field, conditioned on its child type(s).
GenStructAccessor(const StructDef & struct_def,const FieldDef & field,std::string * code_ptr,ImportMap & imports) const901 void GenStructAccessor(const StructDef &struct_def, const FieldDef &field,
902 std::string *code_ptr, ImportMap &imports) const {
903 GenComment(field.doc_comment, code_ptr, &def_comment, Indent.c_str());
904 if (IsScalar(field.value.type.base_type)) {
905 if (struct_def.fixed) {
906 GetScalarFieldOfStruct(struct_def, field, code_ptr);
907 } else {
908 GetScalarFieldOfTable(struct_def, field, code_ptr);
909 }
910 } else {
911 switch (field.value.type.base_type) {
912 case BASE_TYPE_STRUCT:
913 if (struct_def.fixed) {
914 GetStructFieldOfStruct(struct_def, field, code_ptr);
915 } else {
916 GetStructFieldOfTable(struct_def, field, code_ptr, imports);
917 }
918 break;
919 case BASE_TYPE_STRING:
920 GetStringField(struct_def, field, code_ptr, imports);
921 break;
922 case BASE_TYPE_VECTOR: {
923 auto vectortype = field.value.type.VectorType();
924 if (vectortype.base_type == BASE_TYPE_STRUCT) {
925 GetMemberOfVectorOfStruct(struct_def, field, code_ptr, imports);
926 } else {
927 GetMemberOfVectorOfNonStruct(struct_def, field, code_ptr);
928 GetVectorOfNonStructAsNumpy(struct_def, field, code_ptr);
929 GetVectorAsNestedFlatbuffer(struct_def, field, code_ptr, imports);
930 }
931 break;
932 }
933 case BASE_TYPE_ARRAY: {
934 auto vectortype = field.value.type.VectorType();
935 if (vectortype.base_type == BASE_TYPE_STRUCT) {
936 GetArrayOfStruct(struct_def, field, code_ptr, imports);
937 } else {
938 GetArrayOfNonStruct(struct_def, field, code_ptr);
939 GetVectorOfNonStructAsNumpy(struct_def, field, code_ptr);
940 GetVectorAsNestedFlatbuffer(struct_def, field, code_ptr, imports);
941 }
942 break;
943 }
944 case BASE_TYPE_UNION:
945 GetUnionField(struct_def, field, code_ptr, imports);
946 break;
947 default: FLATBUFFERS_ASSERT(0);
948 }
949 }
950 if (IsVector(field.value.type) || IsArray(field.value.type)) {
951 GetVectorLen(struct_def, field, code_ptr);
952 GetVectorIsNone(struct_def, field, code_ptr);
953 }
954 }
955
956 // Generate struct sizeof.
GenStructSizeOf(const StructDef & struct_def,std::string * code_ptr) const957 void GenStructSizeOf(const StructDef &struct_def,
958 std::string *code_ptr) const {
959 auto &code = *code_ptr;
960 code += Indent + "@classmethod\n";
961 if (parser_.opts.python_typing) {
962 code += Indent + "def SizeOf(cls) -> int:\n";
963 } else {
964 code += Indent + "def SizeOf(cls):\n";
965 }
966 code +=
967 Indent + Indent + "return " + NumToString(struct_def.bytesize) + "\n";
968 code += "\n";
969 }
970
971 // Generate table constructors, conditioned on its members' types.
GenTableBuilders(const StructDef & struct_def,std::string * code_ptr) const972 void GenTableBuilders(const StructDef &struct_def,
973 std::string *code_ptr) const {
974 GetStartOfTable(struct_def, code_ptr);
975
976 for (auto it = struct_def.fields.vec.begin();
977 it != struct_def.fields.vec.end(); ++it) {
978 auto &field = **it;
979 if (field.deprecated) continue;
980
981 auto offset = it - struct_def.fields.vec.begin();
982 BuildFieldOfTable(struct_def, field, offset, code_ptr);
983 if (IsVector(field.value.type)) {
984 BuildVectorOfTable(struct_def, field, code_ptr);
985 BuildVectorOfTableFromBytes(struct_def, field, code_ptr);
986 }
987 }
988
989 GetEndOffsetOnTable(struct_def, code_ptr);
990 }
991
992 // Generate function to check for proper file identifier
GenHasFileIdentifier(const StructDef & struct_def,std::string * code_ptr) const993 void GenHasFileIdentifier(const StructDef &struct_def,
994 std::string *code_ptr) const {
995 auto &code = *code_ptr;
996 std::string escapedID;
997 // In the event any of file_identifier characters are special(NULL, \, etc),
998 // problems occur. To prevent this, convert all chars to their hex-escaped
999 // equivalent.
1000 for (auto it = parser_.file_identifier_.begin();
1001 it != parser_.file_identifier_.end(); ++it) {
1002 escapedID += "\\x" + IntToStringHex(*it, 2);
1003 }
1004
1005 code += Indent + "@classmethod\n";
1006 code += Indent + "def " + namer_.Type(struct_def);
1007 code += "BufferHasIdentifier(cls, buf, offset, size_prefixed=False):";
1008 code += "\n";
1009 code += Indent + Indent;
1010 code += "return flatbuffers.util.BufferHasIdentifier(buf, offset, b\"";
1011 code += escapedID;
1012 code += "\", size_prefixed=size_prefixed)\n";
1013 code += "\n";
1014 }
1015
1016 // Generates struct or table methods.
GenStruct(const StructDef & struct_def,std::string * code_ptr,ImportMap & imports) const1017 void GenStruct(const StructDef &struct_def, std::string *code_ptr,
1018 ImportMap &imports) const {
1019 if (struct_def.generated) return;
1020
1021 GenComment(struct_def.doc_comment, code_ptr, &def_comment);
1022 BeginClass(struct_def, code_ptr);
1023 if (!struct_def.fixed) {
1024 // Generate a special accessor for the table that has been declared as
1025 // the root type.
1026 NewRootTypeFromBuffer(struct_def, code_ptr);
1027 if (parser_.file_identifier_.length()) {
1028 // Generate a special function to test file_identifier
1029 GenHasFileIdentifier(struct_def, code_ptr);
1030 }
1031 } else {
1032 // Generates the SizeOf method for all structs.
1033 GenStructSizeOf(struct_def, code_ptr);
1034 }
1035 // Generates the Init method that sets the field in a pre-existing
1036 // accessor object. This is to allow object reuse.
1037 InitializeExisting(struct_def, code_ptr);
1038 for (auto it = struct_def.fields.vec.begin();
1039 it != struct_def.fields.vec.end(); ++it) {
1040 auto &field = **it;
1041 if (field.deprecated) continue;
1042
1043 GenStructAccessor(struct_def, field, code_ptr, imports);
1044 }
1045
1046 if (struct_def.fixed) {
1047 // creates a struct constructor function
1048 GenStructBuilder(struct_def, code_ptr);
1049 } else {
1050 // Creates a set of functions that allow table construction.
1051 GenTableBuilders(struct_def, code_ptr);
1052 }
1053 }
1054
GenReceiverForObjectAPI(const StructDef & struct_def,std::string * code_ptr) const1055 void GenReceiverForObjectAPI(const StructDef &struct_def,
1056 std::string *code_ptr) const {
1057 auto &code = *code_ptr;
1058 code += GenIndents(1) + "# " + namer_.ObjectType(struct_def);
1059 code += GenIndents(1) + "def ";
1060 }
1061
BeginClassForObjectAPI(const StructDef & struct_def,std::string * code_ptr) const1062 void BeginClassForObjectAPI(const StructDef &struct_def,
1063 std::string *code_ptr) const {
1064 auto &code = *code_ptr;
1065 code += "\n";
1066 code += "class " + namer_.ObjectType(struct_def) + "(object):";
1067 code += "\n";
1068 }
1069
1070 // Gets the accoresponding python builtin type of a BaseType for scalars and
1071 // string.
GetBasePythonTypeForScalarAndString(const BaseType & base_type) const1072 std::string GetBasePythonTypeForScalarAndString(
1073 const BaseType &base_type) const {
1074 if (IsBool(base_type)) {
1075 return "bool";
1076 } else if (IsFloat(base_type)) {
1077 return "float";
1078 } else if (IsInteger(base_type)) {
1079 return "int";
1080 } else if (base_type == BASE_TYPE_STRING) {
1081 return "str";
1082 } else {
1083 FLATBUFFERS_ASSERT(false && "base_type is not a scalar or string type.");
1084 return "";
1085 }
1086 }
1087
GetDefaultValue(const FieldDef & field) const1088 std::string GetDefaultValue(const FieldDef &field) const {
1089 BaseType base_type = field.value.type.base_type;
1090 if (field.IsScalarOptional()) {
1091 return "None";
1092 } else if (IsBool(base_type)) {
1093 return field.value.constant == "0" ? "False" : "True";
1094 } else if (IsFloat(base_type)) {
1095 return float_const_gen_.GenFloatConstant(field);
1096 } else if (IsInteger(base_type)) {
1097 return field.value.constant;
1098 } else {
1099 // For string, struct, and table.
1100 return "None";
1101 }
1102 }
1103
GenUnionInit(const FieldDef & field,std::string * field_types_ptr,std::set<std::string> * import_list,std::set<std::string> * import_typing_list) const1104 void GenUnionInit(const FieldDef &field, std::string *field_types_ptr,
1105 std::set<std::string> *import_list,
1106 std::set<std::string> *import_typing_list) const {
1107 // Gets all possible types in the union.
1108 import_typing_list->insert("Union");
1109 auto &field_types = *field_types_ptr;
1110 field_types = "Union[";
1111
1112 std::string separator_string = ", ";
1113 auto enum_def = field.value.type.enum_def;
1114 for (auto it = enum_def->Vals().begin(); it != enum_def->Vals().end();
1115 ++it) {
1116 auto &ev = **it;
1117 // Union only supports string and table.
1118 std::string field_type;
1119 switch (ev.union_type.base_type) {
1120 case BASE_TYPE_STRUCT:
1121 field_type = namer_.ObjectType(*ev.union_type.struct_def);
1122 if (parser_.opts.include_dependence_headers) {
1123 auto package_reference = GenPackageReference(ev.union_type);
1124 field_type = package_reference + "." + field_type;
1125 import_list->insert("import " + package_reference);
1126 }
1127 break;
1128 case BASE_TYPE_STRING: field_type += "str"; break;
1129 case BASE_TYPE_NONE: field_type += "None"; break;
1130 default: break;
1131 }
1132 field_types += field_type + separator_string;
1133 }
1134
1135 // Removes the last separator_string.
1136 field_types.erase(field_types.length() - separator_string.size());
1137 field_types += "]";
1138
1139 // Gets the import lists for the union.
1140 if (parser_.opts.include_dependence_headers) {
1141 const auto package_reference = GenPackageReference(field.value.type);
1142 import_list->insert("import " + package_reference);
1143 }
1144 }
1145
GenStructInit(const FieldDef & field,std::string * out_ptr,std::set<std::string> * import_list,std::set<std::string> * import_typing_list) const1146 void GenStructInit(const FieldDef &field, std::string *out_ptr,
1147 std::set<std::string> *import_list,
1148 std::set<std::string> *import_typing_list) const {
1149 import_typing_list->insert("Optional");
1150 auto &output = *out_ptr;
1151 const Type &type = field.value.type;
1152 const std::string object_type = namer_.ObjectType(*type.struct_def);
1153 if (parser_.opts.include_dependence_headers) {
1154 auto package_reference = GenPackageReference(type);
1155 output = package_reference + "." + object_type + "]";
1156 import_list->insert("import " + package_reference);
1157 } else {
1158 output = object_type + "]";
1159 }
1160 output = "Optional[" + output;
1161 }
1162
GenVectorInit(const FieldDef & field,std::string * field_type_ptr,std::set<std::string> * import_list,std::set<std::string> * import_typing_list) const1163 void GenVectorInit(const FieldDef &field, std::string *field_type_ptr,
1164 std::set<std::string> *import_list,
1165 std::set<std::string> *import_typing_list) const {
1166 import_typing_list->insert("List");
1167 auto &field_type = *field_type_ptr;
1168 const Type &vector_type = field.value.type.VectorType();
1169 const BaseType base_type = vector_type.base_type;
1170 if (base_type == BASE_TYPE_STRUCT) {
1171 const std::string object_type =
1172 namer_.ObjectType(*vector_type.struct_def);
1173 field_type = object_type + "]";
1174 if (parser_.opts.include_dependence_headers) {
1175 auto package_reference = GenPackageReference(vector_type);
1176 field_type = package_reference + "." + object_type + "]";
1177 import_list->insert("import " + package_reference);
1178 }
1179 field_type = "List[" + field_type;
1180 } else {
1181 field_type =
1182 "List[" + GetBasePythonTypeForScalarAndString(base_type) + "]";
1183 }
1184 }
1185
GenInitialize(const StructDef & struct_def,std::string * code_ptr,std::set<std::string> * import_list) const1186 void GenInitialize(const StructDef &struct_def, std::string *code_ptr,
1187 std::set<std::string> *import_list) const {
1188 std::string code;
1189 std::set<std::string> import_typing_list;
1190 for (auto it = struct_def.fields.vec.begin();
1191 it != struct_def.fields.vec.end(); ++it) {
1192 auto &field = **it;
1193 if (field.deprecated) continue;
1194
1195 // Determines field type, default value, and typing imports.
1196 auto base_type = field.value.type.base_type;
1197 std::string field_type;
1198 switch (base_type) {
1199 case BASE_TYPE_UNION: {
1200 GenUnionInit(field, &field_type, import_list, &import_typing_list);
1201 break;
1202 }
1203 case BASE_TYPE_STRUCT: {
1204 GenStructInit(field, &field_type, import_list, &import_typing_list);
1205 break;
1206 }
1207 case BASE_TYPE_VECTOR:
1208 case BASE_TYPE_ARRAY: {
1209 GenVectorInit(field, &field_type, import_list, &import_typing_list);
1210 break;
1211 }
1212 default:
1213 // Scalar or sting fields.
1214 field_type = GetBasePythonTypeForScalarAndString(base_type);
1215 if (field.IsScalarOptional()) {
1216 field_type = "Optional[" + field_type + "]";
1217 }
1218 break;
1219 }
1220
1221 const auto default_value = GetDefaultValue(field);
1222 // Wrties the init statement.
1223 const auto field_field = namer_.Field(field);
1224 code += GenIndents(2) + "self." + field_field + " = " + default_value +
1225 " # type: " + field_type;
1226 }
1227
1228 // Writes __init__ method.
1229 auto &code_base = *code_ptr;
1230 GenReceiverForObjectAPI(struct_def, code_ptr);
1231 code_base += "__init__(self):";
1232 if (code.empty()) {
1233 code_base += GenIndents(2) + "pass";
1234 } else {
1235 code_base += code;
1236 }
1237 code_base += "\n";
1238
1239 // Merges the typing imports into import_list.
1240 if (!import_typing_list.empty()) {
1241 // Adds the try statement.
1242 std::string typing_imports = "try:";
1243 typing_imports += GenIndents(1) + "from typing import ";
1244 std::string separator_string = ", ";
1245 for (auto it = import_typing_list.begin(); it != import_typing_list.end();
1246 ++it) {
1247 const std::string &im = *it;
1248 typing_imports += im + separator_string;
1249 }
1250 // Removes the last separator_string.
1251 typing_imports.erase(typing_imports.length() - separator_string.size());
1252
1253 // Adds the except statement.
1254 typing_imports += "\n";
1255 typing_imports += "except:";
1256 typing_imports += GenIndents(1) + "pass";
1257 import_list->insert(typing_imports);
1258 }
1259
1260 // Removes the import of the struct itself, if applied.
1261 auto struct_import = "import " + namer_.NamespacedType(struct_def);
1262 import_list->erase(struct_import);
1263 }
1264
InitializeFromBuf(const StructDef & struct_def,std::string * code_ptr) const1265 void InitializeFromBuf(const StructDef &struct_def,
1266 std::string *code_ptr) const {
1267 auto &code = *code_ptr;
1268 const auto struct_var = namer_.Variable(struct_def);
1269 const auto struct_type = namer_.Type(struct_def);
1270
1271 code += GenIndents(1) + "@classmethod";
1272 code += GenIndents(1) + "def InitFromBuf(cls, buf, pos):";
1273 code += GenIndents(2) + struct_var + " = " + struct_type + "()";
1274 code += GenIndents(2) + struct_var + ".Init(buf, pos)";
1275 code += GenIndents(2) + "return cls.InitFromObj(" + struct_var + ")";
1276 code += "\n";
1277 }
1278
InitializeFromPackedBuf(const StructDef & struct_def,std::string * code_ptr) const1279 void InitializeFromPackedBuf(const StructDef &struct_def,
1280 std::string *code_ptr) const {
1281 auto &code = *code_ptr;
1282 const auto struct_var = namer_.Variable(struct_def);
1283 const auto struct_type = namer_.Type(struct_def);
1284
1285 code += GenIndents(1) + "@classmethod";
1286 code += GenIndents(1) + "def InitFromPackedBuf(cls, buf, pos=0):";
1287 code += GenIndents(2) +
1288 "n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, pos)";
1289 code += GenIndents(2) + "return cls.InitFromBuf(buf, pos+n)";
1290 code += "\n";
1291 }
1292
InitializeFromObjForObject(const StructDef & struct_def,std::string * code_ptr) const1293 void InitializeFromObjForObject(const StructDef &struct_def,
1294 std::string *code_ptr) const {
1295 auto &code = *code_ptr;
1296 const auto struct_var = namer_.Variable(struct_def);
1297 const auto struct_object = namer_.ObjectType(struct_def);
1298
1299 code += GenIndents(1) + "@classmethod";
1300 code += GenIndents(1) + "def InitFromObj(cls, " + struct_var + "):";
1301 code += GenIndents(2) + "x = " + struct_object + "()";
1302 code += GenIndents(2) + "x._UnPack(" + struct_var + ")";
1303 code += GenIndents(2) + "return x";
1304 code += "\n";
1305 }
1306
GenCompareOperator(const StructDef & struct_def,std::string * code_ptr) const1307 void GenCompareOperator(const StructDef &struct_def,
1308 std::string *code_ptr) const {
1309 auto &code = *code_ptr;
1310 code += GenIndents(1) + "def __eq__(self, other):";
1311 code += GenIndents(2) + "return type(self) == type(other)";
1312 for (auto it = struct_def.fields.vec.begin();
1313 it != struct_def.fields.vec.end(); ++it) {
1314 auto &field = **it;
1315 if (field.deprecated) continue;
1316
1317 // Wrties the comparison statement for this field.
1318 const auto field_field = namer_.Field(field);
1319 code += " and \\" + GenIndents(3) + "self." + field_field +
1320 " == " + "other." + field_field;
1321 }
1322 code += "\n";
1323 }
1324
GenUnPackForStruct(const StructDef & struct_def,const FieldDef & field,std::string * code_ptr) const1325 void GenUnPackForStruct(const StructDef &struct_def, const FieldDef &field,
1326 std::string *code_ptr) const {
1327 auto &code = *code_ptr;
1328 const auto struct_var = namer_.Variable(struct_def);
1329 const auto field_field = namer_.Field(field);
1330 const auto field_method = namer_.Method(field);
1331 auto field_type = TypeName(field);
1332
1333 if (parser_.opts.include_dependence_headers) {
1334 auto package_reference = GenPackageReference(field.value.type);
1335 field_type = package_reference + "." + TypeName(field);
1336 }
1337
1338 code += GenIndents(2) + "if " + struct_var + "." + field_method + "(";
1339 // if field is a struct, we need to create an instance for it first.
1340 if (struct_def.fixed && field.value.type.base_type == BASE_TYPE_STRUCT) {
1341 code += field_type + "()";
1342 }
1343 code += ") is not None:";
1344 code += GenIndents(3) + "self." + field_field + " = " +
1345 namer_.ObjectType(field_type) + +".InitFromObj(" + struct_var +
1346 "." + field_method + "(";
1347 // A struct's accessor requires a struct buf instance.
1348 if (struct_def.fixed && field.value.type.base_type == BASE_TYPE_STRUCT) {
1349 code += field_type + "()";
1350 }
1351 code += "))";
1352 }
1353
GenUnPackForUnion(const StructDef & struct_def,const FieldDef & field,std::string * code_ptr) const1354 void GenUnPackForUnion(const StructDef &struct_def, const FieldDef &field,
1355 std::string *code_ptr) const {
1356 auto &code = *code_ptr;
1357 const auto field_field = namer_.Field(field);
1358 const auto field_method = namer_.Method(field);
1359 const auto struct_var = namer_.Variable(struct_def);
1360 const EnumDef &enum_def = *field.value.type.enum_def;
1361 auto union_type = namer_.Type(enum_def);
1362
1363 if (parser_.opts.include_dependence_headers) {
1364 union_type = namer_.NamespacedType(enum_def) + "." + union_type;
1365 }
1366 code += GenIndents(2) + "self." + field_field + " = " + union_type +
1367 "Creator(" + "self." + field_field + "Type, " + struct_var + "." +
1368 field_method + "())";
1369 }
1370
GenUnPackForStructVector(const StructDef & struct_def,const FieldDef & field,std::string * code_ptr) const1371 void GenUnPackForStructVector(const StructDef &struct_def,
1372 const FieldDef &field,
1373 std::string *code_ptr) const {
1374 auto &code = *code_ptr;
1375 const auto field_field = namer_.Field(field);
1376 const auto field_method = namer_.Method(field);
1377 const auto struct_var = namer_.Variable(struct_def);
1378
1379 code += GenIndents(2) + "if not " + struct_var + "." + field_method +
1380 "IsNone():";
1381 code += GenIndents(3) + "self." + field_field + " = []";
1382 code += GenIndents(3) + "for i in range(" + struct_var + "." +
1383 field_method + "Length()):";
1384
1385 auto field_type = TypeName(field);
1386 auto one_instance = field_type + "_";
1387 one_instance[0] = CharToLower(one_instance[0]);
1388 if (parser_.opts.include_dependence_headers) {
1389 auto package_reference = GenPackageReference(field.value.type);
1390 field_type = package_reference + "." + TypeName(field);
1391 }
1392 code += GenIndents(4) + "if " + struct_var + "." + field_method +
1393 "(i) is None:";
1394 code += GenIndents(5) + "self." + field_field + ".append(None)";
1395 code += GenIndents(4) + "else:";
1396 code += GenIndents(5) + one_instance + " = " +
1397 namer_.ObjectType(field_type) + ".InitFromObj(" + struct_var + "." +
1398 field_method + "(i))";
1399 code +=
1400 GenIndents(5) + "self." + field_field + ".append(" + one_instance + ")";
1401 }
1402
GenUnpackForTableVector(const StructDef & struct_def,const FieldDef & field,std::string * code_ptr) const1403 void GenUnpackForTableVector(const StructDef &struct_def,
1404 const FieldDef &field,
1405 std::string *code_ptr) const {
1406 auto &code = *code_ptr;
1407 const auto field_field = namer_.Field(field);
1408 const auto field_method = namer_.Method(field);
1409 const auto struct_var = namer_.Variable(struct_def);
1410
1411 code += GenIndents(2) + "if not " + struct_var + "." + field_method +
1412 "IsNone():";
1413 code += GenIndents(3) + "self." + field_field + " = []";
1414 code += GenIndents(3) + "for i in range(" + struct_var + "." +
1415 field_method + "Length()):";
1416
1417 auto field_type = TypeName(field);
1418 auto one_instance = field_type + "_";
1419 one_instance[0] = CharToLower(one_instance[0]);
1420 if (parser_.opts.include_dependence_headers) {
1421 auto package_reference = GenPackageReference(field.value.type);
1422 field_type = package_reference + "." + TypeName(field);
1423 }
1424 code += GenIndents(4) + "if " + struct_var + "." + field_method +
1425 "(i) is None:";
1426 code += GenIndents(5) + "self." + field_field + ".append(None)";
1427 code += GenIndents(4) + "else:";
1428 code += GenIndents(5) + one_instance + " = " +
1429 namer_.ObjectType(field_type) + ".InitFromObj(" + struct_var + "." +
1430 field_method + "(i))";
1431 code +=
1432 GenIndents(5) + "self." + field_field + ".append(" + one_instance + ")";
1433 }
1434
GenUnpackforScalarVectorHelper(const StructDef & struct_def,const FieldDef & field,std::string * code_ptr,int indents) const1435 void GenUnpackforScalarVectorHelper(const StructDef &struct_def,
1436 const FieldDef &field,
1437 std::string *code_ptr,
1438 int indents) const {
1439 auto &code = *code_ptr;
1440 const auto field_field = namer_.Field(field);
1441 const auto field_method = namer_.Method(field);
1442 const auto struct_var = namer_.Variable(struct_def);
1443
1444 code += GenIndents(indents) + "self." + field_field + " = []";
1445 code += GenIndents(indents) + "for i in range(" + struct_var + "." +
1446 field_method + "Length()):";
1447 code += GenIndents(indents + 1) + "self." + field_field + ".append(" +
1448 struct_var + "." + field_method + "(i))";
1449 }
1450
GenUnPackForScalarVector(const StructDef & struct_def,const FieldDef & field,std::string * code_ptr) const1451 void GenUnPackForScalarVector(const StructDef &struct_def,
1452 const FieldDef &field,
1453 std::string *code_ptr) const {
1454 auto &code = *code_ptr;
1455 const auto field_field = namer_.Field(field);
1456 const auto field_method = namer_.Method(field);
1457 const auto struct_var = namer_.Variable(struct_def);
1458
1459 code += GenIndents(2) + "if not " + struct_var + "." + field_method +
1460 "IsNone():";
1461
1462 // String does not have the AsNumpy method.
1463 if (!(IsScalar(field.value.type.VectorType().base_type))) {
1464 GenUnpackforScalarVectorHelper(struct_def, field, code_ptr, 3);
1465 return;
1466 }
1467
1468 code += GenIndents(3) + "if np is None:";
1469 GenUnpackforScalarVectorHelper(struct_def, field, code_ptr, 4);
1470
1471 // If numpy exists, use the AsNumpy method to optimize the unpack speed.
1472 code += GenIndents(3) + "else:";
1473 code += GenIndents(4) + "self." + field_field + " = " + struct_var + "." +
1474 field_method + "AsNumpy()";
1475 }
1476
GenUnPackForScalar(const StructDef & struct_def,const FieldDef & field,std::string * code_ptr) const1477 void GenUnPackForScalar(const StructDef &struct_def, const FieldDef &field,
1478 std::string *code_ptr) const {
1479 auto &code = *code_ptr;
1480 const auto field_field = namer_.Field(field);
1481 const auto field_method = namer_.Method(field);
1482 const auto struct_var = namer_.Variable(struct_def);
1483
1484 code += GenIndents(2) + "self." + field_field + " = " + struct_var + "." +
1485 field_method + "()";
1486 }
1487
1488 // Generates the UnPack method for the object class.
GenUnPack(const StructDef & struct_def,std::string * code_ptr) const1489 void GenUnPack(const StructDef &struct_def, std::string *code_ptr) const {
1490 std::string code;
1491 // Items that needs to be imported. No duplicate modules will be imported.
1492 std::set<std::string> import_list;
1493
1494 for (auto it = struct_def.fields.vec.begin();
1495 it != struct_def.fields.vec.end(); ++it) {
1496 auto &field = **it;
1497 if (field.deprecated) continue;
1498
1499 auto field_type = TypeName(field);
1500 switch (field.value.type.base_type) {
1501 case BASE_TYPE_STRUCT: {
1502 GenUnPackForStruct(struct_def, field, &code);
1503 break;
1504 }
1505 case BASE_TYPE_UNION: {
1506 GenUnPackForUnion(struct_def, field, &code);
1507 break;
1508 }
1509 case BASE_TYPE_ARRAY:
1510 case BASE_TYPE_VECTOR: {
1511 auto vectortype = field.value.type.VectorType();
1512 if (vectortype.base_type == BASE_TYPE_STRUCT) {
1513 GenUnPackForStructVector(struct_def, field, &code);
1514 } else {
1515 GenUnPackForScalarVector(struct_def, field, &code);
1516 }
1517 break;
1518 }
1519 default: GenUnPackForScalar(struct_def, field, &code);
1520 }
1521 }
1522
1523 // Writes import statements and code into the generated file.
1524 auto &code_base = *code_ptr;
1525 const auto struct_var = namer_.Variable(struct_def);
1526
1527 GenReceiverForObjectAPI(struct_def, code_ptr);
1528 code_base += "_UnPack(self, " + struct_var + "):";
1529 code_base += GenIndents(2) + "if " + struct_var + " is None:";
1530 code_base += GenIndents(3) + "return";
1531
1532 // Write the import statements.
1533 for (std::set<std::string>::iterator it = import_list.begin();
1534 it != import_list.end(); ++it) {
1535 code_base += GenIndents(2) + *it;
1536 }
1537
1538 // Write the code.
1539 code_base += code;
1540 code_base += "\n";
1541 }
1542
GenPackForStruct(const StructDef & struct_def,std::string * code_ptr) const1543 void GenPackForStruct(const StructDef &struct_def,
1544 std::string *code_ptr) const {
1545 auto &code = *code_ptr;
1546 const auto struct_fn = namer_.Function(struct_def);
1547
1548 GenReceiverForObjectAPI(struct_def, code_ptr);
1549 code += "Pack(self, builder):";
1550 code += GenIndents(2) + "return Create" + struct_fn + "(builder";
1551
1552 StructBuilderArgs(struct_def,
1553 /* nameprefix = */ "self.",
1554 /* namesuffix = */ "",
1555 /* has_field_name = */ true,
1556 /* fieldname_suffix = */ ".", code_ptr);
1557 code += ")\n";
1558 }
1559
GenPackForStructVectorField(const StructDef & struct_def,const FieldDef & field,std::string * code_prefix_ptr,std::string * code_ptr) const1560 void GenPackForStructVectorField(const StructDef &struct_def,
1561 const FieldDef &field,
1562 std::string *code_prefix_ptr,
1563 std::string *code_ptr) const {
1564 auto &code_prefix = *code_prefix_ptr;
1565 auto &code = *code_ptr;
1566 const auto field_field = namer_.Field(field);
1567 const auto struct_type = namer_.Type(struct_def);
1568 const auto field_method = namer_.Method(field);
1569
1570 // Creates the field.
1571 code_prefix += GenIndents(2) + "if self." + field_field + " is not None:";
1572 if (field.value.type.struct_def->fixed) {
1573 code_prefix += GenIndents(3) + struct_type + "Start" + field_method +
1574 "Vector(builder, len(self." + field_field + "))";
1575 code_prefix += GenIndents(3) + "for i in reversed(range(len(self." +
1576 field_field + "))):";
1577 code_prefix +=
1578 GenIndents(4) + "self." + field_field + "[i].Pack(builder)";
1579 code_prefix += GenIndents(3) + field_field + " = builder.EndVector()";
1580 } else {
1581 // If the vector is a struct vector, we need to first build accessor for
1582 // each struct element.
1583 code_prefix += GenIndents(3) + field_field + "list = []";
1584 code_prefix += GenIndents(3);
1585 code_prefix += "for i in range(len(self." + field_field + ")):";
1586 code_prefix += GenIndents(4) + field_field + "list.append(self." +
1587 field_field + "[i].Pack(builder))";
1588
1589 code_prefix += GenIndents(3) + struct_type + "Start" + field_method +
1590 "Vector(builder, len(self." + field_field + "))";
1591 code_prefix += GenIndents(3) + "for i in reversed(range(len(self." +
1592 field_field + "))):";
1593 code_prefix += GenIndents(4) + "builder.PrependUOffsetTRelative" + "(" +
1594 field_field + "list[i])";
1595 code_prefix += GenIndents(3) + field_field + " = builder.EndVector()";
1596 }
1597
1598 // Adds the field into the struct.
1599 code += GenIndents(2) + "if self." + field_field + " is not None:";
1600 code += GenIndents(3) + struct_type + "Add" + field_method + "(builder, " +
1601 field_field + ")";
1602 }
1603
GenPackForScalarVectorFieldHelper(const StructDef & struct_def,const FieldDef & field,std::string * code_ptr,int indents) const1604 void GenPackForScalarVectorFieldHelper(const StructDef &struct_def,
1605 const FieldDef &field,
1606 std::string *code_ptr,
1607 int indents) const {
1608 auto &code = *code_ptr;
1609 const auto field_field = namer_.Field(field);
1610 const auto field_method = namer_.Method(field);
1611 const auto struct_type = namer_.Type(struct_def);
1612 const auto vectortype = field.value.type.VectorType();
1613
1614 code += GenIndents(indents) + struct_type + "Start" + field_method +
1615 "Vector(builder, len(self." + field_field + "))";
1616 code += GenIndents(indents) + "for i in reversed(range(len(self." +
1617 field_field + "))):";
1618 code += GenIndents(indents + 1) + "builder.Prepend";
1619
1620 std::string type_name;
1621 switch (vectortype.base_type) {
1622 case BASE_TYPE_BOOL: type_name = "Bool"; break;
1623 case BASE_TYPE_CHAR: type_name = "Byte"; break;
1624 case BASE_TYPE_UCHAR: type_name = "Uint8"; break;
1625 case BASE_TYPE_SHORT: type_name = "Int16"; break;
1626 case BASE_TYPE_USHORT: type_name = "Uint16"; break;
1627 case BASE_TYPE_INT: type_name = "Int32"; break;
1628 case BASE_TYPE_UINT: type_name = "Uint32"; break;
1629 case BASE_TYPE_LONG: type_name = "Int64"; break;
1630 case BASE_TYPE_ULONG: type_name = "Uint64"; break;
1631 case BASE_TYPE_FLOAT: type_name = "Float32"; break;
1632 case BASE_TYPE_DOUBLE: type_name = "Float64"; break;
1633 case BASE_TYPE_STRING: type_name = "UOffsetTRelative"; break;
1634 default: type_name = "VOffsetT"; break;
1635 }
1636 code += type_name;
1637 }
1638
GenPackForScalarVectorField(const StructDef & struct_def,const FieldDef & field,std::string * code_prefix_ptr,std::string * code_ptr) const1639 void GenPackForScalarVectorField(const StructDef &struct_def,
1640 const FieldDef &field,
1641 std::string *code_prefix_ptr,
1642 std::string *code_ptr) const {
1643 auto &code = *code_ptr;
1644 auto &code_prefix = *code_prefix_ptr;
1645 const auto field_field = namer_.Field(field);
1646 const auto field_method = namer_.Method(field);
1647 const auto struct_type = namer_.Type(struct_def);
1648
1649 // Adds the field into the struct.
1650 code += GenIndents(2) + "if self." + field_field + " is not None:";
1651 code += GenIndents(3) + struct_type + "Add" + field_method + "(builder, " +
1652 field_field + ")";
1653
1654 // Creates the field.
1655 code_prefix += GenIndents(2) + "if self." + field_field + " is not None:";
1656 // If the vector is a string vector, we need to first build accessor for
1657 // each string element. And this generated code, needs to be
1658 // placed ahead of code_prefix.
1659 auto vectortype = field.value.type.VectorType();
1660 if (IsString(vectortype)) {
1661 code_prefix += GenIndents(3) + field_field + "list = []";
1662 code_prefix +=
1663 GenIndents(3) + "for i in range(len(self." + field_field + ")):";
1664 code_prefix += GenIndents(4) + field_field +
1665 "list.append(builder.CreateString(self." + field_field +
1666 "[i]))";
1667 GenPackForScalarVectorFieldHelper(struct_def, field, code_prefix_ptr, 3);
1668 code_prefix += "(" + field_field + "list[i])";
1669 code_prefix += GenIndents(3) + field_field + " = builder.EndVector()";
1670 return;
1671 }
1672
1673 code_prefix += GenIndents(3) + "if np is not None and type(self." +
1674 field_field + ") is np.ndarray:";
1675 code_prefix += GenIndents(4) + field_field +
1676 " = builder.CreateNumpyVector(self." + field_field + ")";
1677 code_prefix += GenIndents(3) + "else:";
1678 GenPackForScalarVectorFieldHelper(struct_def, field, code_prefix_ptr, 4);
1679 code_prefix += "(self." + field_field + "[i])";
1680 code_prefix += GenIndents(4) + field_field + " = builder.EndVector()";
1681 }
1682
GenPackForStructField(const StructDef & struct_def,const FieldDef & field,std::string * code_prefix_ptr,std::string * code_ptr) const1683 void GenPackForStructField(const StructDef &struct_def, const FieldDef &field,
1684 std::string *code_prefix_ptr,
1685 std::string *code_ptr) const {
1686 auto &code_prefix = *code_prefix_ptr;
1687 auto &code = *code_ptr;
1688 const auto field_field = namer_.Field(field);
1689 const auto field_method = namer_.Method(field);
1690 const auto struct_type = namer_.Type(struct_def);
1691
1692 if (field.value.type.struct_def->fixed) {
1693 // Pure struct fields need to be created along with their parent
1694 // structs.
1695 code += GenIndents(2) + "if self." + field_field + " is not None:";
1696 code += GenIndents(3) + field_field + " = self." + field_field +
1697 ".Pack(builder)";
1698 } else {
1699 // Tables need to be created before their parent structs are created.
1700 code_prefix += GenIndents(2) + "if self." + field_field + " is not None:";
1701 code_prefix += GenIndents(3) + field_field + " = self." + field_field +
1702 ".Pack(builder)";
1703 code += GenIndents(2) + "if self." + field_field + " is not None:";
1704 }
1705
1706 code += GenIndents(3) + struct_type + "Add" + field_method + "(builder, " +
1707 field_field + ")";
1708 }
1709
GenPackForUnionField(const StructDef & struct_def,const FieldDef & field,std::string * code_prefix_ptr,std::string * code_ptr) const1710 void GenPackForUnionField(const StructDef &struct_def, const FieldDef &field,
1711 std::string *code_prefix_ptr,
1712 std::string *code_ptr) const {
1713 auto &code_prefix = *code_prefix_ptr;
1714 auto &code = *code_ptr;
1715 const auto field_field = namer_.Field(field);
1716 const auto field_method = namer_.Method(field);
1717 const auto struct_type = namer_.Type(struct_def);
1718
1719 // TODO(luwa): TypeT should be moved under the None check as well.
1720 code_prefix += GenIndents(2) + "if self." + field_field + " is not None:";
1721 code_prefix += GenIndents(3) + field_field + " = self." + field_field +
1722 ".Pack(builder)";
1723 code += GenIndents(2) + "if self." + field_field + " is not None:";
1724 code += GenIndents(3) + struct_type + "Add" + field_method + "(builder, " +
1725 field_field + ")";
1726 }
1727
GenPackForTable(const StructDef & struct_def,std::string * code_ptr) const1728 void GenPackForTable(const StructDef &struct_def,
1729 std::string *code_ptr) const {
1730 auto &code_base = *code_ptr;
1731 std::string code, code_prefix;
1732 const auto struct_var = namer_.Variable(struct_def);
1733 const auto struct_type = namer_.Type(struct_def);
1734
1735 GenReceiverForObjectAPI(struct_def, code_ptr);
1736 code_base += "Pack(self, builder):";
1737 code += GenIndents(2) + struct_type + "Start(builder)";
1738 for (auto it = struct_def.fields.vec.begin();
1739 it != struct_def.fields.vec.end(); ++it) {
1740 auto &field = **it;
1741 if (field.deprecated) continue;
1742
1743 const auto field_method = namer_.Method(field);
1744 const auto field_field = namer_.Field(field);
1745
1746 switch (field.value.type.base_type) {
1747 case BASE_TYPE_STRUCT: {
1748 GenPackForStructField(struct_def, field, &code_prefix, &code);
1749 break;
1750 }
1751 case BASE_TYPE_UNION: {
1752 GenPackForUnionField(struct_def, field, &code_prefix, &code);
1753 break;
1754 }
1755 case BASE_TYPE_ARRAY:
1756 case BASE_TYPE_VECTOR: {
1757 auto vectortype = field.value.type.VectorType();
1758 if (vectortype.base_type == BASE_TYPE_STRUCT) {
1759 GenPackForStructVectorField(struct_def, field, &code_prefix, &code);
1760 } else {
1761 GenPackForScalarVectorField(struct_def, field, &code_prefix, &code);
1762 }
1763 break;
1764 }
1765 case BASE_TYPE_STRING: {
1766 code_prefix +=
1767 GenIndents(2) + "if self." + field_field + " is not None:";
1768 code_prefix += GenIndents(3) + field_field +
1769 " = builder.CreateString(self." + field_field + ")";
1770 code += GenIndents(2) + "if self." + field_field + " is not None:";
1771 code += GenIndents(3) + struct_type + "Add" + field_method +
1772 "(builder, " + field_field + ")";
1773 break;
1774 }
1775 default:
1776 // Generates code for scalar values. If the value equals to the
1777 // default value, builder will automatically ignore it. So we don't
1778 // need to check the value ahead.
1779 code += GenIndents(2) + struct_type + "Add" + field_method +
1780 "(builder, self." + field_field + ")";
1781 break;
1782 }
1783 }
1784
1785 code += GenIndents(2) + struct_var + " = " + struct_type + "End(builder)";
1786 code += GenIndents(2) + "return " + struct_var;
1787
1788 code_base += code_prefix + code;
1789 code_base += "\n";
1790 }
1791
GenStructForObjectAPI(const StructDef & struct_def,std::string * code_ptr) const1792 void GenStructForObjectAPI(const StructDef &struct_def,
1793 std::string *code_ptr) const {
1794 if (struct_def.generated) return;
1795
1796 std::set<std::string> import_list;
1797 std::string code;
1798
1799 // Creates an object class for a struct or a table
1800 BeginClassForObjectAPI(struct_def, &code);
1801
1802 GenInitialize(struct_def, &code, &import_list);
1803
1804 InitializeFromBuf(struct_def, &code);
1805
1806 InitializeFromPackedBuf(struct_def, &code);
1807
1808 InitializeFromObjForObject(struct_def, &code);
1809
1810 if (parser_.opts.gen_compare) { GenCompareOperator(struct_def, &code); }
1811
1812 GenUnPack(struct_def, &code);
1813
1814 if (struct_def.fixed) {
1815 GenPackForStruct(struct_def, &code);
1816 } else {
1817 GenPackForTable(struct_def, &code);
1818 }
1819
1820 // Adds the imports at top.
1821 auto &code_base = *code_ptr;
1822 code_base += "\n";
1823 for (auto it = import_list.begin(); it != import_list.end(); it++) {
1824 auto im = *it;
1825 code_base += im + "\n";
1826 }
1827 code_base += code;
1828 }
1829
GenUnionCreatorForStruct(const EnumDef & enum_def,const EnumVal & ev,std::string * code_ptr) const1830 void GenUnionCreatorForStruct(const EnumDef &enum_def, const EnumVal &ev,
1831 std::string *code_ptr) const {
1832 auto &code = *code_ptr;
1833 const auto union_type = namer_.Type(enum_def);
1834 const auto variant = namer_.Variant(ev);
1835 auto field_type = namer_.ObjectType(*ev.union_type.struct_def);
1836
1837 code +=
1838 GenIndents(1) + "if unionType == " + union_type + "()." + variant + ":";
1839 if (parser_.opts.include_dependence_headers) {
1840 auto package_reference = GenPackageReference(ev.union_type);
1841 code += GenIndents(2) + "import " + package_reference;
1842 field_type = package_reference + "." + field_type;
1843 }
1844 code += GenIndents(2) + "return " + field_type +
1845 ".InitFromBuf(table.Bytes, table.Pos)";
1846 }
1847
GenUnionCreatorForString(const EnumDef & enum_def,const EnumVal & ev,std::string * code_ptr) const1848 void GenUnionCreatorForString(const EnumDef &enum_def, const EnumVal &ev,
1849 std::string *code_ptr) const {
1850 auto &code = *code_ptr;
1851 const auto union_type = namer_.Type(enum_def);
1852 const auto variant = namer_.Variant(ev);
1853
1854 code +=
1855 GenIndents(1) + "if unionType == " + union_type + "()." + variant + ":";
1856 code += GenIndents(2) + "tab = Table(table.Bytes, table.Pos)";
1857 code += GenIndents(2) + "union = tab.String(table.Pos)";
1858 code += GenIndents(2) + "return union";
1859 }
1860
1861 // Creates an union object based on union type.
GenUnionCreator(const EnumDef & enum_def,std::string * code_ptr) const1862 void GenUnionCreator(const EnumDef &enum_def, std::string *code_ptr) const {
1863 if (enum_def.generated) return;
1864
1865 auto &code = *code_ptr;
1866 const auto enum_fn = namer_.Function(enum_def);
1867
1868 code += "\n";
1869 code += "def " + enum_fn + "Creator(unionType, table):";
1870 code += GenIndents(1) + "from flatbuffers.table import Table";
1871 code += GenIndents(1) + "if not isinstance(table, Table):";
1872 code += GenIndents(2) + "return None";
1873
1874 for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); ++it) {
1875 auto &ev = **it;
1876 // Union only supports string and table.
1877 switch (ev.union_type.base_type) {
1878 case BASE_TYPE_STRUCT:
1879 GenUnionCreatorForStruct(enum_def, ev, &code);
1880 break;
1881 case BASE_TYPE_STRING:
1882 GenUnionCreatorForString(enum_def, ev, &code);
1883 break;
1884 default: break;
1885 }
1886 }
1887 code += GenIndents(1) + "return None";
1888 code += "\n";
1889 }
1890
1891 // Generate enum declarations.
GenEnum(const EnumDef & enum_def,std::string * code_ptr) const1892 void GenEnum(const EnumDef &enum_def, std::string *code_ptr) const {
1893 if (enum_def.generated) return;
1894
1895 GenComment(enum_def.doc_comment, code_ptr, &def_comment);
1896 BeginEnum(enum_def, code_ptr);
1897 for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); ++it) {
1898 auto &ev = **it;
1899 GenComment(ev.doc_comment, code_ptr, &def_comment, Indent.c_str());
1900 EnumMember(enum_def, ev, code_ptr);
1901 }
1902 }
1903
1904 // Returns the function name that is able to read a value of the given type.
GenGetter(const Type & type) const1905 std::string GenGetter(const Type &type) const {
1906 switch (type.base_type) {
1907 case BASE_TYPE_STRING: return "self._tab.String(";
1908 case BASE_TYPE_UNION: return "self._tab.Union(";
1909 case BASE_TYPE_VECTOR: return GenGetter(type.VectorType());
1910 default:
1911 return "self._tab.Get(flatbuffers.number_types." +
1912 namer_.Method(GenTypeGet(type)) + "Flags, ";
1913 }
1914 }
1915
GenFieldTy(const FieldDef & field) const1916 std::string GenFieldTy(const FieldDef &field) const {
1917 if (IsScalar(field.value.type.base_type) || IsArray(field.value.type)) {
1918 const std::string ty = GenTypeBasic(field.value.type);
1919 if (ty.find("int") != std::string::npos) { return "int"; }
1920
1921 if (ty.find("float") != std::string::npos) { return "float"; }
1922
1923 if (ty == "bool") { return "bool"; }
1924
1925 return "Any";
1926 } else {
1927 if (IsStruct(field.value.type)) {
1928 return "Any";
1929 } else {
1930 return "int";
1931 }
1932 }
1933 }
1934
1935 // Returns the method name for use with add/put calls.
GenMethod(const FieldDef & field) const1936 std::string GenMethod(const FieldDef &field) const {
1937 return (IsScalar(field.value.type.base_type) || IsArray(field.value.type))
1938 ? namer_.Method(GenTypeBasic(field.value.type))
1939 : (IsStruct(field.value.type) ? "Struct" : "UOffsetTRelative");
1940 }
1941
GenTypeBasic(const Type & type) const1942 std::string GenTypeBasic(const Type &type) const {
1943 // clang-format off
1944 static const char *ctypename[] = {
1945 #define FLATBUFFERS_TD(ENUM, IDLTYPE, \
1946 CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, ...) \
1947 #PTYPE,
1948 FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
1949 #undef FLATBUFFERS_TD
1950 };
1951 // clang-format on
1952 return ctypename[IsArray(type) ? type.VectorType().base_type
1953 : type.base_type];
1954 }
1955
GenTypePointer(const Type & type) const1956 std::string GenTypePointer(const Type &type) const {
1957 switch (type.base_type) {
1958 case BASE_TYPE_STRING: return "string";
1959 case BASE_TYPE_VECTOR:
1960 // fall through
1961 case BASE_TYPE_ARRAY: return GenTypeGet(type.VectorType());
1962 case BASE_TYPE_STRUCT: return type.struct_def->name;
1963 case BASE_TYPE_UNION:
1964 // fall through
1965 default: return "*flatbuffers.Table";
1966 }
1967 }
1968
GenTypeGet(const Type & type) const1969 std::string GenTypeGet(const Type &type) const {
1970 return IsScalar(type.base_type) ? GenTypeBasic(type) : GenTypePointer(type);
1971 }
1972
TypeName(const FieldDef & field) const1973 std::string TypeName(const FieldDef &field) const {
1974 return GenTypeGet(field.value.type);
1975 }
1976
ReturnType(const StructDef & struct_def,const FieldDef & field) const1977 std::string ReturnType(const StructDef &struct_def,
1978 const FieldDef &field) const {
1979 // If we have a class member that returns an instance of the same class,
1980 // for example:
1981 // class Field(object):
1982 // def Children(self, j: int) -> Optional[Field]:
1983 // pass
1984 //
1985 // we need to quote the return type:
1986 // class Field(object):
1987 // def Children(self, j: int) -> Optional['Field']:
1988 // pass
1989 //
1990 // because Python is unable to resolve the name during parse and will return
1991 // an error.
1992 // (see PEP 484 under forward references:
1993 // https://peps.python.org/pep-0484/#forward-references)
1994 const std::string self_type = struct_def.name;
1995 std::string field_type = TypeName(field);
1996
1997 if (self_type == field_type) { field_type = "'" + field_type + "'"; }
1998
1999 return field_type;
2000 }
2001
2002 // Create a struct with a builder and the struct's arguments.
GenStructBuilder(const StructDef & struct_def,std::string * code_ptr) const2003 void GenStructBuilder(const StructDef &struct_def,
2004 std::string *code_ptr) const {
2005 BeginBuilderArgs(struct_def, code_ptr);
2006 StructBuilderArgs(struct_def,
2007 /* nameprefix = */ "",
2008 /* namesuffix = */ "",
2009 /* has_field_name = */ true,
2010 /* fieldname_suffix = */ "_", code_ptr);
2011 EndBuilderArgs(code_ptr);
2012
2013 StructBuilderBody(struct_def, "", code_ptr);
2014 EndBuilderBody(code_ptr);
2015 }
2016
generate()2017 bool generate() {
2018 std::string one_file_code;
2019 ImportMap one_file_imports;
2020 if (!generateEnums(&one_file_code)) return false;
2021 if (!generateStructs(&one_file_code, one_file_imports)) return false;
2022
2023 if (parser_.opts.one_file) {
2024 const std::string mod = file_name_ + "_generated";
2025
2026 // Legacy file format uses keep casing.
2027 return SaveType(mod + ".py", *parser_.current_namespace_, one_file_code,
2028 one_file_imports, mod, true);
2029 }
2030
2031 return true;
2032 }
2033
2034 private:
generateEnums(std::string * one_file_code) const2035 bool generateEnums(std::string *one_file_code) const {
2036 for (auto it = parser_.enums_.vec.begin(); it != parser_.enums_.vec.end();
2037 ++it) {
2038 auto &enum_def = **it;
2039 std::string enumcode;
2040 GenEnum(enum_def, &enumcode);
2041 if (parser_.opts.generate_object_based_api & enum_def.is_union) {
2042 GenUnionCreator(enum_def, &enumcode);
2043 }
2044
2045 if (parser_.opts.one_file && !enumcode.empty()) {
2046 *one_file_code += enumcode + "\n\n";
2047 } else {
2048 ImportMap imports;
2049 const std::string mod =
2050 namer_.File(enum_def, SkipFile::SuffixAndExtension);
2051
2052 if (!SaveType(namer_.File(enum_def, SkipFile::Suffix),
2053 *enum_def.defined_namespace, enumcode, imports, mod,
2054 false))
2055 return false;
2056 }
2057 }
2058 return true;
2059 }
2060
generateStructs(std::string * one_file_code,ImportMap & one_file_imports) const2061 bool generateStructs(std::string *one_file_code,
2062 ImportMap &one_file_imports) const {
2063 for (auto it = parser_.structs_.vec.begin();
2064 it != parser_.structs_.vec.end(); ++it) {
2065 auto &struct_def = **it;
2066 std::string declcode;
2067 ImportMap imports;
2068 GenStruct(struct_def, &declcode, imports);
2069 if (parser_.opts.generate_object_based_api) {
2070 GenStructForObjectAPI(struct_def, &declcode);
2071 }
2072
2073 if (parser_.opts.one_file) {
2074 if (!declcode.empty()) { *one_file_code += declcode + "\n\n"; }
2075
2076 for (auto import_str : imports) { one_file_imports.insert(import_str); }
2077 } else {
2078 const std::string mod =
2079 namer_.File(struct_def, SkipFile::SuffixAndExtension);
2080 if (!SaveType(namer_.File(struct_def, SkipFile::Suffix),
2081 *struct_def.defined_namespace, declcode, imports, mod,
2082 true))
2083 return false;
2084 }
2085 }
2086 return true;
2087 }
2088
2089 // Begin by declaring namespace and imports.
BeginFile(const std::string & name_space_name,const bool needs_imports,std::string * code_ptr,const std::string & mod,const ImportMap & imports) const2090 void BeginFile(const std::string &name_space_name, const bool needs_imports,
2091 std::string *code_ptr, const std::string &mod,
2092 const ImportMap &imports) const {
2093 auto &code = *code_ptr;
2094 code = code + "# " + FlatBuffersGeneratedWarning() + "\n\n";
2095 code += "# namespace: " + name_space_name + "\n\n";
2096
2097 if (needs_imports) {
2098 const std::string local_import = "." + mod;
2099
2100 code += "import flatbuffers\n";
2101 code += "from flatbuffers.compat import import_numpy\n";
2102 if (parser_.opts.python_typing) {
2103 code += "from typing import Any\n";
2104
2105 for (auto import_entry : imports) {
2106 // If we have a file called, say, "MyType.py" and in it we have a
2107 // class "MyType", we can generate imports -- usually when we
2108 // have a type that contains arrays of itself -- of the type
2109 // "from .MyType import MyType", which Python can't resolve. So
2110 // if we are trying to import ourself, we skip.
2111 if (import_entry.first != local_import) {
2112 code += "from " + import_entry.first + " import " +
2113 import_entry.second + "\n";
2114 }
2115 }
2116 }
2117 code += "np = import_numpy()\n\n";
2118 }
2119 }
2120
2121 // Save out the generated code for a Python Table type.
SaveType(const std::string & defname,const Namespace & ns,const std::string & classcode,const ImportMap & imports,const std::string & mod,bool needs_imports) const2122 bool SaveType(const std::string &defname, const Namespace &ns,
2123 const std::string &classcode, const ImportMap &imports,
2124 const std::string &mod, bool needs_imports) const {
2125 if (!classcode.length()) return true;
2126
2127 std::string code = "";
2128 BeginFile(LastNamespacePart(ns), needs_imports, &code, mod, imports);
2129 code += classcode;
2130
2131 const std::string directories =
2132 parser_.opts.one_file ? path_ : namer_.Directories(ns.components);
2133 EnsureDirExists(directories);
2134
2135 for (size_t i = path_.size() + 1; i != std::string::npos;
2136 i = directories.find(kPathSeparator, i + 1)) {
2137 const std::string init_py =
2138 directories.substr(0, i) + kPathSeparator + "__init__.py";
2139 SaveFile(init_py.c_str(), "", false);
2140 }
2141
2142 const std::string filename = directories + defname;
2143 return SaveFile(filename.c_str(), code, false);
2144 }
2145
2146 private:
2147 const SimpleFloatConstantGenerator float_const_gen_;
2148 const IdlNamer namer_;
2149 };
2150
2151 } // namespace python
2152
GeneratePython(const Parser & parser,const std::string & path,const std::string & file_name)2153 static bool GeneratePython(const Parser &parser, const std::string &path,
2154 const std::string &file_name) {
2155 python::PythonGenerator generator(parser, path, file_name);
2156 return generator.generate();
2157 }
2158
2159 namespace {
2160
2161 class PythonCodeGenerator : public CodeGenerator {
2162 public:
GenerateCode(const Parser & parser,const std::string & path,const std::string & filename)2163 Status GenerateCode(const Parser &parser, const std::string &path,
2164 const std::string &filename) override {
2165 if (!GeneratePython(parser, path, filename)) { return Status::ERROR; }
2166 return Status::OK;
2167 }
2168
GenerateCode(const uint8_t *,int64_t,const CodeGenOptions &)2169 Status GenerateCode(const uint8_t *, int64_t,
2170 const CodeGenOptions &) override {
2171 return Status::NOT_IMPLEMENTED;
2172 }
2173
GenerateMakeRule(const Parser & parser,const std::string & path,const std::string & filename,std::string & output)2174 Status GenerateMakeRule(const Parser &parser, const std::string &path,
2175 const std::string &filename,
2176 std::string &output) override {
2177 (void)parser;
2178 (void)path;
2179 (void)filename;
2180 (void)output;
2181 return Status::NOT_IMPLEMENTED;
2182 }
2183
GenerateGrpcCode(const Parser & parser,const std::string & path,const std::string & filename)2184 Status GenerateGrpcCode(const Parser &parser, const std::string &path,
2185 const std::string &filename) override {
2186 if (!GeneratePythonGRPC(parser, path, filename)) { return Status::ERROR; }
2187 return Status::OK;
2188 }
2189
GenerateRootFile(const Parser & parser,const std::string & path)2190 Status GenerateRootFile(const Parser &parser,
2191 const std::string &path) override {
2192 (void)parser;
2193 (void)path;
2194 return Status::NOT_IMPLEMENTED;
2195 }
2196
IsSchemaOnly() const2197 bool IsSchemaOnly() const override { return true; }
2198
SupportsBfbsGeneration() const2199 bool SupportsBfbsGeneration() const override { return false; }
SupportsRootFileGeneration() const2200 bool SupportsRootFileGeneration() const override { return false; }
2201
Language() const2202 IDLOptions::Language Language() const override { return IDLOptions::kPython; }
2203
LanguageName() const2204 std::string LanguageName() const override { return "Python"; }
2205 };
2206 } // namespace
2207
NewPythonCodeGenerator()2208 std::unique_ptr<CodeGenerator> NewPythonCodeGenerator() {
2209 return std::unique_ptr<PythonCodeGenerator>(new PythonCodeGenerator());
2210 }
2211
2212 } // namespace flatbuffers
2213