1 // Protocol Buffers - Google's data interrdchange format
2 // Copyright 2023 Google LLC. All rights reserved.
3 //
4 // Use of this source code is governed by a BSD-style
5 // license that can be found in the LICENSE file or at
6 // https://developers.google.com/open-source/licenses/bsd
7
8 #include "upb_generator/minitable/generator.h"
9
10 #include <cstdint>
11 #include <map>
12 #include <string>
13 #include <utility>
14 #include <vector>
15
16 #include "absl/container/flat_hash_set.h"
17 #include "absl/log/absl_check.h"
18 #include "absl/strings/str_cat.h"
19 #include "absl/strings/string_view.h"
20 #include "absl/strings/substitute.h"
21 #include "upb/mini_table/enum.h"
22 #include "upb/mini_table/field.h"
23 #include "upb/mini_table/internal/field.h"
24 #include "upb/mini_table/message.h"
25 #include "upb/reflection/def.hpp"
26 #include "upb_generator/common.h"
27 #include "upb_generator/common/names.h"
28 #include "upb_generator/file_layout.h"
29 #include "upb_generator/minitable/fasttable.h"
30 #include "upb_generator/minitable/names.h"
31 #include "upb_generator/minitable/names_internal.h"
32 #include "upb_generator/plugin.h"
33
34 // Must be last.
35 #include "upb/port/def.inc"
36
37 namespace upb {
38 namespace generator {
39 namespace {
40
41 // Some local convenience aliases for MiniTable variable names.
42
MessageVarName(upb::MessageDefPtr message)43 std::string MessageVarName(upb::MessageDefPtr message) {
44 return MiniTableMessageVarName(message.full_name());
45 }
46
MessagePtrVarName(upb::MessageDefPtr message)47 std::string MessagePtrVarName(upb::MessageDefPtr message) {
48 return MiniTableMessagePtrVarName(message.full_name());
49 }
50
EnumVarName(upb::EnumDefPtr e)51 std::string EnumVarName(upb::EnumDefPtr e) {
52 return MiniTableEnumVarName(e.full_name());
53 }
54
ExtensionVarName(upb::FieldDefPtr ext)55 std::string ExtensionVarName(upb::FieldDefPtr ext) {
56 return MiniTableExtensionVarName(ext.full_name());
57 }
58
FileVarName(upb::FileDefPtr file)59 std::string FileVarName(upb::FileDefPtr file) {
60 return MiniTableFileVarName(file.name());
61 }
62
HeaderFilename(upb::FileDefPtr file,bool bootstrap)63 std::string HeaderFilename(upb::FileDefPtr file, bool bootstrap) {
64 return MiniTableHeaderFilename(file.name(), bootstrap);
65 }
66
ArchDependentSize(int64_t size32,int64_t size64)67 std::string ArchDependentSize(int64_t size32, int64_t size64) {
68 if (size32 == size64) return absl::StrCat(size32);
69 return absl::Substitute("UPB_SIZE($0, $1)", size32, size64);
70 }
71
FieldInitializer(const DefPoolPair & pools,upb::FieldDefPtr field)72 std::string FieldInitializer(const DefPoolPair& pools, upb::FieldDefPtr field) {
73 return upb::generator::FieldInitializer(field, pools.GetField64(field),
74 pools.GetField32(field));
75 }
76
77 // Writes a single field into a .upb.c source file.
WriteMessageField(upb::FieldDefPtr field,const upb_MiniTableField * field64,const upb_MiniTableField * field32,Output & output)78 void WriteMessageField(upb::FieldDefPtr field,
79 const upb_MiniTableField* field64,
80 const upb_MiniTableField* field32, Output& output) {
81 output(" $0,\n", upb::generator::FieldInitializer(field, field64, field32));
82 }
83
GetSub(upb::FieldDefPtr field,bool is_extension)84 std::string GetSub(upb::FieldDefPtr field, bool is_extension) {
85 if (auto message_def = field.message_type()) {
86 return absl::Substitute("{.UPB_PRIVATE(submsg) = &$0}",
87 is_extension ? MessageVarName(message_def)
88 : MessagePtrVarName(message_def));
89 }
90
91 if (auto enum_def = field.enum_subdef()) {
92 if (enum_def.is_closed()) {
93 return absl::Substitute("{.UPB_PRIVATE(subenum) = &$0}",
94 MiniTableEnumVarName(enum_def.full_name()));
95 }
96 }
97
98 return std::string("{.UPB_PRIVATE(submsg) = NULL}");
99 }
100
IsCrossFile(upb::FieldDefPtr field)101 bool IsCrossFile(upb::FieldDefPtr field) {
102 return field.message_type() != field.containing_type();
103 }
104
105 // Writes a single message into a .upb.c source file.
WriteMessage(upb::MessageDefPtr message,const DefPoolPair & pools,const MiniTableOptions & options,Output & output)106 void WriteMessage(upb::MessageDefPtr message, const DefPoolPair& pools,
107 const MiniTableOptions& options, Output& output) {
108 std::string fields_array_ref = "NULL";
109 std::string submsgs_array_ref = "NULL";
110 std::string subenums_array_ref = "NULL";
111 const upb_MiniTable* mt_32 = pools.GetMiniTable32(message);
112 const upb_MiniTable* mt_64 = pools.GetMiniTable64(message);
113 std::map<int, std::string> subs;
114 absl::flat_hash_set<const upb_MiniTable*> seen;
115
116 // Construct map of sub messages by field number.
117 for (int i = 0; i < mt_64->UPB_PRIVATE(field_count); i++) {
118 const upb_MiniTableField* f = &mt_64->UPB_PRIVATE(fields)[i];
119 uint32_t index = f->UPB_PRIVATE(submsg_index);
120 if (index != kUpb_NoSub) {
121 const int f_number = upb_MiniTableField_Number(f);
122 upb::FieldDefPtr field = message.FindFieldByNumber(f_number);
123 auto pair = subs.emplace(index, GetSub(field, false));
124 ABSL_CHECK(pair.second);
125 if (options.one_output_per_message && field.IsSubMessage() &&
126 IsCrossFile(field) && !upb_MiniTableField_IsMap(f)) {
127 if (seen.insert(pools.GetMiniTable64(field.message_type())).second) {
128 output(
129 "__attribute__((weak)) const upb_MiniTable* $0 ="
130 " &UPB_PRIVATE(_kUpb_MiniTable_StaticallyTreeShaken);\n",
131 MessagePtrVarName(field.message_type()));
132 }
133 }
134 }
135 }
136 // Write upb_MiniTableSubInternal table for sub messages referenced from
137 // fields.
138 if (!subs.empty()) {
139 std::string submsgs_array_name =
140 MiniTableSubMessagesVarName(message.full_name());
141 submsgs_array_ref = "&" + submsgs_array_name + "[0]";
142 output("static const upb_MiniTableSubInternal $0[$1] = {\n",
143 submsgs_array_name, subs.size());
144
145 int i = 0;
146 for (const auto& pair : subs) {
147 ABSL_CHECK(pair.first == i++);
148 output(" $0,\n", pair.second);
149 }
150
151 output("};\n\n");
152 }
153
154 // Write upb_MiniTableField table.
155 if (mt_64->UPB_PRIVATE(field_count) > 0) {
156 std::string fields_array_name = MiniTableFieldsVarName(message.full_name());
157 fields_array_ref = "&" + fields_array_name + "[0]";
158 output("static const upb_MiniTableField $0[$1] = {\n", fields_array_name,
159 mt_64->UPB_PRIVATE(field_count));
160 for (int i = 0; i < mt_64->UPB_PRIVATE(field_count); i++) {
161 WriteMessageField(message.FindFieldByNumber(
162 mt_64->UPB_PRIVATE(fields)[i].UPB_PRIVATE(number)),
163 &mt_64->UPB_PRIVATE(fields)[i],
164 &mt_32->UPB_PRIVATE(fields)[i], output);
165 }
166 output("};\n\n");
167 }
168
169 std::vector<TableEntry> table;
170 uint8_t table_mask = ~0;
171
172 table = FastDecodeTable(message, pools);
173
174 if (table.size() > 1) {
175 UPB_ASSERT((table.size() & (table.size() - 1)) == 0);
176 table_mask = (table.size() - 1) << 3;
177 }
178
179 std::string msgext = "kUpb_ExtMode_NonExtendable";
180
181 if (message.extension_range_count()) {
182 if (UPB_DESC(MessageOptions_message_set_wire_format)(message.options())) {
183 msgext = "kUpb_ExtMode_IsMessageSet";
184 } else {
185 msgext = "kUpb_ExtMode_Extendable";
186 }
187 }
188
189 output("const upb_MiniTable $0 = {\n", MessageVarName(message));
190 output(" $0,\n", submsgs_array_ref);
191 output(" $0,\n", fields_array_ref);
192 output(" $0, $1, $2, $3, UPB_FASTTABLE_MASK($4), $5,\n",
193 ArchDependentSize(mt_32->UPB_PRIVATE(size), mt_64->UPB_PRIVATE(size)),
194 mt_64->UPB_PRIVATE(field_count), msgext,
195 mt_64->UPB_PRIVATE(dense_below), table_mask,
196 mt_64->UPB_PRIVATE(required_count));
197 output("#ifdef UPB_TRACING_ENABLED\n");
198 output(" \"$0\",\n", message.full_name());
199 output("#endif\n");
200 if (!table.empty()) {
201 output(" UPB_FASTTABLE_INIT({\n");
202 for (const auto& ent : table) {
203 output(" {0x$1, &$0},\n", ent.first,
204 absl::StrCat(absl::Hex(ent.second, absl::kZeroPad16)));
205 }
206 output(" })\n");
207 }
208 output("};\n\n");
209 output("const upb_MiniTable* $0 = &$1;\n", MessagePtrVarName(message),
210 MessageVarName(message));
211 }
212
WriteEnum(upb::EnumDefPtr e,Output & output)213 void WriteEnum(upb::EnumDefPtr e, Output& output) {
214 std::string values_init = "{\n";
215 const upb_MiniTableEnum* mt = e.mini_table();
216 uint32_t value_count =
217 (mt->UPB_PRIVATE(mask_limit) / 32) + mt->UPB_PRIVATE(value_count);
218 for (uint32_t i = 0; i < value_count; i++) {
219 absl::StrAppend(&values_init, " 0x",
220 absl::Hex(mt->UPB_PRIVATE(data)[i]), ",\n");
221 }
222 values_init += " }";
223
224 output(
225 R"cc(
226 const upb_MiniTableEnum $0 = {
227 $1,
228 $2,
229 $3,
230 };
231 )cc",
232 EnumVarName(e), mt->UPB_PRIVATE(mask_limit), mt->UPB_PRIVATE(value_count),
233 values_init);
234 output("\n");
235 }
236
WriteExtension(const DefPoolPair & pools,upb::FieldDefPtr ext,Output & output)237 void WriteExtension(const DefPoolPair& pools, upb::FieldDefPtr ext,
238 Output& output) {
239 output("UPB_LINKARR_APPEND(upb_AllExts)\n");
240 output("const upb_MiniTableExtension $0 = {\n ", ExtensionVarName(ext));
241 output("$0,\n", FieldInitializer(pools, ext));
242 output(" &$0,\n", MessageVarName(ext.containing_type()));
243 output(" $0,\n", GetSub(ext, true));
244 output("\n};\n");
245 }
246
247 } // namespace
248
WriteMiniTableHeader(const DefPoolPair & pools,upb::FileDefPtr file,const MiniTableOptions & options,Output & output)249 void WriteMiniTableHeader(const DefPoolPair& pools, upb::FileDefPtr file,
250 const MiniTableOptions& options, Output& output) {
251 output(FileWarning(file.name()));
252 output(
253 "#ifndef $0_UPB_MINITABLE_H_\n"
254 "#define $0_UPB_MINITABLE_H_\n\n"
255 "#include \"upb/generated_code_support.h\"\n",
256 IncludeGuard(file.name()));
257
258 for (int i = 0; i < file.public_dependency_count(); i++) {
259 if (i == 0) {
260 output("/* Public Imports. */\n");
261 }
262 output("#include \"$0\"\n",
263 HeaderFilename(file.public_dependency(i), options.bootstrap));
264 if (i == file.public_dependency_count() - 1) {
265 output("\n");
266 }
267 }
268
269 output(
270 "\n"
271 "// Must be last.\n"
272 "#include \"upb/port/def.inc\"\n"
273 "\n"
274 "#ifdef __cplusplus\n"
275 "extern \"C\" {\n"
276 "#endif\n"
277 "\n");
278
279 const std::vector<upb::MessageDefPtr> this_file_messages =
280 SortedMessages(file);
281 const std::vector<upb::FieldDefPtr> this_file_exts = SortedExtensions(file);
282
283 for (auto message : this_file_messages) {
284 output("extern const upb_MiniTable $0;\n", MessageVarName(message));
285 output("extern const upb_MiniTable* $0;\n", MessagePtrVarName(message));
286 }
287 for (auto ext : this_file_exts) {
288 output("extern const upb_MiniTableExtension $0;\n", ExtensionVarName(ext));
289 }
290
291 output("\n");
292
293 std::vector<upb::EnumDefPtr> this_file_enums =
294 SortedEnums(file, kClosedEnums);
295
296 for (const auto enumdesc : this_file_enums) {
297 output("extern const upb_MiniTableEnum $0;\n", EnumVarName(enumdesc));
298 }
299
300 output("extern const upb_MiniTableFile $0;\n\n", FileVarName(file));
301
302 output(
303 "#ifdef __cplusplus\n"
304 "} /* extern \"C\" */\n"
305 "#endif\n"
306 "\n"
307 "#include \"upb/port/undef.inc\"\n"
308 "\n"
309 "#endif /* $0_UPB_MINITABLE_H_ */\n",
310 IncludeGuard(file.name()));
311 }
312
WriteMiniTableSourceIncludes(upb::FileDefPtr file,const MiniTableOptions & options,Output & output)313 void WriteMiniTableSourceIncludes(upb::FileDefPtr file,
314 const MiniTableOptions& options,
315 Output& output) {
316 output(FileWarning(file.name()));
317
318 output(
319 "#include <stddef.h>\n"
320 "#include \"upb/generated_code_support.h\"\n"
321 "#include \"$0\"\n",
322 HeaderFilename(file, options.bootstrap));
323
324 for (int i = 0; i < file.dependency_count(); i++) {
325 if (options.strip_nonfunctional_codegen &&
326 google::protobuf::compiler::IsKnownFeatureProto(file.dependency(i).name())) {
327 // Strip feature imports for editions codegen tests.
328 continue;
329 }
330 output("#include \"$0\"\n",
331 HeaderFilename(file.dependency(i), options.bootstrap));
332 }
333
334 output(
335 "\n"
336 "// Must be last.\n"
337 "#include \"upb/port/def.inc\"\n"
338 "\n");
339
340 output(
341 "extern const struct upb_MiniTable "
342 "UPB_PRIVATE(_kUpb_MiniTable_StaticallyTreeShaken);\n");
343 }
344
WriteMiniTableSource(const DefPoolPair & pools,upb::FileDefPtr file,const MiniTableOptions & options,Output & output)345 void WriteMiniTableSource(const DefPoolPair& pools, upb::FileDefPtr file,
346 const MiniTableOptions& options, Output& output) {
347 WriteMiniTableSourceIncludes(file, options, output);
348
349 std::vector<upb::MessageDefPtr> messages = SortedMessages(file);
350 std::vector<upb::FieldDefPtr> extensions = SortedExtensions(file);
351 std::vector<upb::EnumDefPtr> enums = SortedEnums(file, kClosedEnums);
352
353 if (options.one_output_per_message) {
354 for (auto message : messages) {
355 output("extern const upb_MiniTable* $0;\n", MessagePtrVarName(message));
356 }
357 for (const auto e : enums) {
358 output("extern const upb_MiniTableEnum $0;\n", EnumVarName(e));
359 }
360 for (const auto ext : extensions) {
361 output("extern const upb_MiniTableExtension $0;\n",
362 ExtensionVarName(ext));
363 }
364 } else {
365 for (auto message : messages) {
366 WriteMessage(message, pools, options, output);
367 }
368 for (const auto e : enums) {
369 WriteEnum(e, output);
370 }
371 for (const auto ext : extensions) {
372 WriteExtension(pools, ext, output);
373 }
374 }
375
376 // Messages.
377 if (!messages.empty()) {
378 output("static const upb_MiniTable *$0[$1] = {\n", kMessagesInit,
379 messages.size());
380 for (auto message : messages) {
381 output(" &$0,\n", MessageVarName(message));
382 }
383 output("};\n");
384 output("\n");
385 }
386
387 // Enums.
388 if (!enums.empty()) {
389 output("static const upb_MiniTableEnum *$0[$1] = {\n", kEnumsInit,
390 enums.size());
391 for (const auto e : enums) {
392 output(" &$0,\n", EnumVarName(e));
393 }
394 output("};\n");
395 output("\n");
396 }
397
398 if (!extensions.empty()) {
399 // Extensions.
400 output(
401 "\n"
402 "static const upb_MiniTableExtension *$0[$1] = {\n",
403 kExtensionsInit, extensions.size());
404
405 for (auto ext : extensions) {
406 output(" &$0,\n", ExtensionVarName(ext));
407 }
408
409 output(
410 "};\n"
411 "\n");
412 }
413
414 output("const upb_MiniTableFile $0 = {\n", FileVarName(file));
415 output(" $0,\n", messages.empty() ? "NULL" : kMessagesInit);
416 output(" $0,\n", enums.empty() ? "NULL" : kEnumsInit);
417 output(" $0,\n", extensions.empty() ? "NULL" : kExtensionsInit);
418 output(" $0,\n", messages.size());
419 output(" $0,\n", enums.size());
420 output(" $0,\n", extensions.size());
421 output("};\n\n");
422
423 output("#include \"upb/port/undef.inc\"\n");
424 output("\n");
425 }
426
MultipleSourceFilename(upb::FileDefPtr file,absl::string_view full_name,int * i)427 std::string MultipleSourceFilename(upb::FileDefPtr file,
428 absl::string_view full_name, int* i) {
429 *i += 1;
430 return absl::StrCat(StripExtension(file.name()), ".upb_weak_minitables/",
431 *i, ".upb.c");
432 }
433
WriteMiniTableMultipleSources(const DefPoolPair & pools,upb::FileDefPtr file,const MiniTableOptions & options,Plugin * plugin)434 void WriteMiniTableMultipleSources(const DefPoolPair& pools,
435 upb::FileDefPtr file,
436 const MiniTableOptions& options,
437 Plugin* plugin) {
438 std::vector<upb::MessageDefPtr> messages = SortedMessages(file);
439 std::vector<upb::FieldDefPtr> extensions = SortedExtensions(file);
440 std::vector<upb::EnumDefPtr> enums = SortedEnums(file, kClosedEnums);
441 int i = 0;
442
443 for (auto message : messages) {
444 Output output;
445 WriteMiniTableSourceIncludes(file, options, output);
446 WriteMessage(message, pools, options, output);
447 plugin->AddOutputFile(MultipleSourceFilename(file, message.full_name(), &i),
448 output.output());
449 }
450 for (const auto e : enums) {
451 Output output;
452 WriteMiniTableSourceIncludes(file, options, output);
453 WriteEnum(e, output);
454 plugin->AddOutputFile(MultipleSourceFilename(file, e.full_name(), &i),
455 output.output());
456 }
457 for (const auto ext : extensions) {
458 Output output;
459 WriteMiniTableSourceIncludes(file, options, output);
460 WriteExtension(pools, ext, output);
461 plugin->AddOutputFile(MultipleSourceFilename(file, ext.full_name(), &i),
462 output.output());
463 }
464 }
465
466 } // namespace generator
467 } // namespace upb
468