1 // Protocol Buffers - Google's data interchange format
2 // Copyright 2023 Google LLC. All rights reserved.
3 //
4 // Use of this source code is governed by a BSD-style
5 // license that can be found in the LICENSE file or at
6 // https://developers.google.com/open-source/licenses/bsd
7
8 // Fast decoder: ~3x the speed of decode.c, but requires x86-64/ARM64.
9 // Also the table size grows by 2x.
10 //
11 // Could potentially be ported to other 64-bit archs that pass at least six
12 // arguments in registers and have 8 unused high bits in pointers.
13 //
14 // The overall design is to create specialized functions for every possible
15 // field type (eg. oneof boolean field with a 1 byte tag) and then dispatch
16 // to the specialized function as quickly as possible.
17
18 #include "upb/wire/internal/decode_fast.h"
19
20 #include "upb/message/array.h"
21 #include "upb/message/internal/array.h"
22 #include "upb/mini_table/sub.h"
23 #include "upb/wire/internal/decoder.h"
24
25 // Must be last.
26 #include "upb/port/def.inc"
27
28 #if UPB_FASTTABLE
29
30 // The standard set of arguments passed to each parsing function.
31 // Thanks to x86-64 calling conventions, these will stay in registers.
32 #define UPB_PARSE_PARAMS \
33 upb_Decoder *d, const char *ptr, upb_Message *msg, intptr_t table, \
34 uint64_t hasbits, uint64_t data
35
36 #define UPB_PARSE_ARGS d, ptr, msg, table, hasbits, data
37
38 #define RETURN_GENERIC(m) \
39 /* Uncomment either of these for debugging purposes. */ \
40 /* fprintf(stderr, m); */ \
41 /*__builtin_trap(); */ \
42 return _upb_FastDecoder_DecodeGeneric(d, ptr, msg, table, hasbits, 0);
43
44 typedef enum {
45 CARD_s = 0, /* Singular (optional, non-repeated) */
46 CARD_o = 1, /* Oneof */
47 CARD_r = 2, /* Repeated */
48 CARD_p = 3 /* Packed Repeated */
49 } upb_card;
50
51 UPB_NOINLINE
fastdecode_isdonefallback(UPB_PARSE_PARAMS)52 static const char* fastdecode_isdonefallback(UPB_PARSE_PARAMS) {
53 int overrun = data;
54 ptr = _upb_EpsCopyInputStream_IsDoneFallbackInline(
55 &d->input, ptr, overrun, _upb_Decoder_BufferFlipCallback);
56 data = _upb_FastDecoder_LoadTag(ptr);
57 UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS);
58 }
59
60 UPB_FORCEINLINE
fastdecode_dispatch(UPB_PARSE_PARAMS)61 const char* fastdecode_dispatch(UPB_PARSE_PARAMS) {
62 int overrun;
63 switch (upb_EpsCopyInputStream_IsDoneStatus(&d->input, ptr, &overrun)) {
64 case kUpb_IsDoneStatus_Done:
65 ((uint32_t*)msg)[2] |= hasbits; // Sync hasbits.
66 const upb_MiniTable* m = decode_totablep(table);
67 return UPB_UNLIKELY(m->UPB_PRIVATE(required_count))
68 ? _upb_Decoder_CheckRequired(d, ptr, msg, m)
69 : ptr;
70 case kUpb_IsDoneStatus_NotDone:
71 break;
72 case kUpb_IsDoneStatus_NeedFallback:
73 data = overrun;
74 UPB_MUSTTAIL return fastdecode_isdonefallback(UPB_PARSE_ARGS);
75 }
76
77 // Read two bytes of tag data (for a one-byte tag, the high byte is junk).
78 data = _upb_FastDecoder_LoadTag(ptr);
79 UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS);
80 }
81
82 UPB_FORCEINLINE
fastdecode_checktag(uint16_t data,int tagbytes)83 bool fastdecode_checktag(uint16_t data, int tagbytes) {
84 if (tagbytes == 1) {
85 return (data & 0xff) == 0;
86 } else {
87 return data == 0;
88 }
89 }
90
91 UPB_FORCEINLINE
fastdecode_longsize(const char * ptr,int * size)92 const char* fastdecode_longsize(const char* ptr, int* size) {
93 int i;
94 UPB_ASSERT(*size & 0x80);
95 *size &= 0xff;
96 for (i = 0; i < 3; i++) {
97 ptr++;
98 size_t byte = (uint8_t)ptr[-1];
99 *size += (byte - 1) << (7 + 7 * i);
100 if (UPB_LIKELY((byte & 0x80) == 0)) return ptr;
101 }
102 ptr++;
103 size_t byte = (uint8_t)ptr[-1];
104 // len is limited by 2gb not 4gb, hence 8 and not 16 as normally expected
105 // for a 32 bit varint.
106 if (UPB_UNLIKELY(byte >= 8)) return NULL;
107 *size += (byte - 1) << 28;
108 return ptr;
109 }
110
111 UPB_FORCEINLINE
fastdecode_delimited(upb_Decoder * d,const char * ptr,upb_EpsCopyInputStream_ParseDelimitedFunc * func,void * ctx)112 const char* fastdecode_delimited(
113 upb_Decoder* d, const char* ptr,
114 upb_EpsCopyInputStream_ParseDelimitedFunc* func, void* ctx) {
115 ptr++;
116
117 // Sign-extend so varint greater than one byte becomes negative, causing
118 // fast delimited parse to fail.
119 int len = (int8_t)ptr[-1];
120
121 if (!upb_EpsCopyInputStream_TryParseDelimitedFast(&d->input, &ptr, len, func,
122 ctx)) {
123 // Slow case: Sub-message is >=128 bytes and/or exceeds the current buffer.
124 // If it exceeds the buffer limit, limit/limit_ptr will change during
125 // sub-message parsing, so we need to preserve delta, not limit.
126 if (UPB_UNLIKELY(len & 0x80)) {
127 // Size varint >1 byte (length >= 128).
128 ptr = fastdecode_longsize(ptr, &len);
129 if (!ptr) {
130 // Corrupt wire format: size exceeded INT_MAX.
131 return NULL;
132 }
133 }
134 if (!upb_EpsCopyInputStream_CheckSize(&d->input, ptr, len)) {
135 // Corrupt wire format: invalid limit.
136 return NULL;
137 }
138 int delta = upb_EpsCopyInputStream_PushLimit(&d->input, ptr, len);
139 ptr = func(&d->input, ptr, ctx);
140 upb_EpsCopyInputStream_PopLimit(&d->input, ptr, delta);
141 }
142 return ptr;
143 }
144
145 /* singular, oneof, repeated field handling ***********************************/
146
147 typedef struct {
148 upb_Array* arr;
149 void* end;
150 } fastdecode_arr;
151
152 typedef enum {
153 FD_NEXT_ATLIMIT,
154 FD_NEXT_SAMEFIELD,
155 FD_NEXT_OTHERFIELD
156 } fastdecode_next;
157
158 typedef struct {
159 void* dst;
160 fastdecode_next next;
161 uint32_t tag;
162 } fastdecode_nextret;
163
164 UPB_FORCEINLINE
fastdecode_resizearr(upb_Decoder * d,void * dst,fastdecode_arr * farr,int valbytes)165 void* fastdecode_resizearr(upb_Decoder* d, void* dst, fastdecode_arr* farr,
166 int valbytes) {
167 if (UPB_UNLIKELY(dst == farr->end)) {
168 size_t old_capacity = farr->arr->UPB_PRIVATE(capacity);
169 size_t old_bytes = old_capacity * valbytes;
170 size_t new_capacity = old_capacity * 2;
171 size_t new_bytes = new_capacity * valbytes;
172 char* old_ptr = upb_Array_MutableDataPtr(farr->arr);
173 char* new_ptr = upb_Arena_Realloc(&d->arena, old_ptr, old_bytes, new_bytes);
174 uint8_t elem_size_lg2 = __builtin_ctz(valbytes);
175 UPB_PRIVATE(_upb_Array_SetTaggedPtr)(farr->arr, new_ptr, elem_size_lg2);
176 farr->arr->UPB_PRIVATE(capacity) = new_capacity;
177 dst = (void*)(new_ptr + (old_capacity * valbytes));
178 farr->end = (void*)(new_ptr + (new_capacity * valbytes));
179 }
180 return dst;
181 }
182
183 UPB_FORCEINLINE
fastdecode_tagmatch(uint32_t tag,uint64_t data,int tagbytes)184 bool fastdecode_tagmatch(uint32_t tag, uint64_t data, int tagbytes) {
185 if (tagbytes == 1) {
186 return (uint8_t)tag == (uint8_t)data;
187 } else {
188 return (uint16_t)tag == (uint16_t)data;
189 }
190 }
191
192 UPB_FORCEINLINE
fastdecode_commitarr(void * dst,fastdecode_arr * farr,int valbytes)193 void fastdecode_commitarr(void* dst, fastdecode_arr* farr, int valbytes) {
194 farr->arr->UPB_PRIVATE(size) =
195 (size_t)((char*)dst - (char*)upb_Array_MutableDataPtr(farr->arr)) /
196 valbytes;
197 }
198
199 UPB_FORCEINLINE
fastdecode_nextrepeated(upb_Decoder * d,void * dst,const char ** ptr,fastdecode_arr * farr,uint64_t data,int tagbytes,int valbytes)200 fastdecode_nextret fastdecode_nextrepeated(upb_Decoder* d, void* dst,
201 const char** ptr,
202 fastdecode_arr* farr, uint64_t data,
203 int tagbytes, int valbytes) {
204 fastdecode_nextret ret;
205 dst = (char*)dst + valbytes;
206
207 if (UPB_LIKELY(!_upb_Decoder_IsDone(d, ptr))) {
208 ret.tag = _upb_FastDecoder_LoadTag(*ptr);
209 if (fastdecode_tagmatch(ret.tag, data, tagbytes)) {
210 ret.next = FD_NEXT_SAMEFIELD;
211 } else {
212 fastdecode_commitarr(dst, farr, valbytes);
213 ret.next = FD_NEXT_OTHERFIELD;
214 }
215 } else {
216 fastdecode_commitarr(dst, farr, valbytes);
217 ret.next = FD_NEXT_ATLIMIT;
218 }
219
220 ret.dst = dst;
221 return ret;
222 }
223
224 UPB_FORCEINLINE
fastdecode_fieldmem(upb_Message * msg,uint64_t data)225 void* fastdecode_fieldmem(upb_Message* msg, uint64_t data) {
226 size_t ofs = data >> 48;
227 return (char*)msg + ofs;
228 }
229
230 UPB_FORCEINLINE
fastdecode_getfield(upb_Decoder * d,const char * ptr,upb_Message * msg,uint64_t * data,uint64_t * hasbits,fastdecode_arr * farr,int valbytes,upb_card card)231 void* fastdecode_getfield(upb_Decoder* d, const char* ptr, upb_Message* msg,
232 uint64_t* data, uint64_t* hasbits,
233 fastdecode_arr* farr, int valbytes, upb_card card) {
234 UPB_ASSERT(!upb_Message_IsFrozen(msg));
235 switch (card) {
236 case CARD_s: {
237 uint8_t hasbit_index = *data >> 24;
238 // Set hasbit and return pointer to scalar field.
239 *hasbits |= 1ull << hasbit_index;
240 return fastdecode_fieldmem(msg, *data);
241 }
242 case CARD_o: {
243 uint16_t case_ofs = *data >> 32;
244 uint32_t* oneof_case = UPB_PTR_AT(msg, case_ofs, uint32_t);
245 uint8_t field_number = *data >> 24;
246 *oneof_case = field_number;
247 return fastdecode_fieldmem(msg, *data);
248 }
249 case CARD_r: {
250 // Get pointer to upb_Array and allocate/expand if necessary.
251 uint8_t elem_size_lg2 = __builtin_ctz(valbytes);
252 upb_Array** arr_p = fastdecode_fieldmem(msg, *data);
253 char* begin;
254 ((uint32_t*)msg)[2] |= *hasbits;
255 *hasbits = 0;
256 if (UPB_LIKELY(!*arr_p)) {
257 farr->arr = UPB_PRIVATE(_upb_Array_New)(&d->arena, 8, elem_size_lg2);
258 *arr_p = farr->arr;
259 } else {
260 farr->arr = *arr_p;
261 }
262 begin = upb_Array_MutableDataPtr(farr->arr);
263 farr->end = begin + (farr->arr->UPB_PRIVATE(capacity) * valbytes);
264 *data = _upb_FastDecoder_LoadTag(ptr);
265 return begin + (farr->arr->UPB_PRIVATE(size) * valbytes);
266 }
267 default:
268 UPB_UNREACHABLE();
269 }
270 }
271
272 UPB_FORCEINLINE
fastdecode_flippacked(uint64_t * data,int tagbytes)273 bool fastdecode_flippacked(uint64_t* data, int tagbytes) {
274 *data ^= (0x2 ^ 0x0); // Patch data to match packed wiretype.
275 return fastdecode_checktag(*data, tagbytes);
276 }
277
278 #define FASTDECODE_CHECKPACKED(tagbytes, card, func) \
279 if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \
280 if (card == CARD_r && fastdecode_flippacked(&data, tagbytes)) { \
281 UPB_MUSTTAIL return func(UPB_PARSE_ARGS); \
282 } \
283 RETURN_GENERIC("packed check tag mismatch\n"); \
284 }
285
286 /* varint fields **************************************************************/
287
288 UPB_FORCEINLINE
fastdecode_munge(uint64_t val,int valbytes,bool zigzag)289 uint64_t fastdecode_munge(uint64_t val, int valbytes, bool zigzag) {
290 if (valbytes == 1) {
291 return val != 0;
292 } else if (zigzag) {
293 if (valbytes == 4) {
294 uint32_t n = val;
295 return (n >> 1) ^ -(int32_t)(n & 1);
296 } else if (valbytes == 8) {
297 return (val >> 1) ^ -(int64_t)(val & 1);
298 }
299 UPB_UNREACHABLE();
300 }
301 return val;
302 }
303
304 UPB_FORCEINLINE
fastdecode_varint64(const char * ptr,uint64_t * val)305 const char* fastdecode_varint64(const char* ptr, uint64_t* val) {
306 ptr++;
307 *val = (uint8_t)ptr[-1];
308 if (UPB_UNLIKELY(*val & 0x80)) {
309 int i;
310 for (i = 0; i < 8; i++) {
311 ptr++;
312 uint64_t byte = (uint8_t)ptr[-1];
313 *val += (byte - 1) << (7 + 7 * i);
314 if (UPB_LIKELY((byte & 0x80) == 0)) goto done;
315 }
316 ptr++;
317 uint64_t byte = (uint8_t)ptr[-1];
318 if (byte > 1) {
319 return NULL;
320 }
321 *val += (byte - 1) << 63;
322 }
323 done:
324 UPB_ASSUME(ptr != NULL);
325 return ptr;
326 }
327
328 #define FASTDECODE_UNPACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \
329 valbytes, card, zigzag, packed) \
330 uint64_t val; \
331 void* dst; \
332 fastdecode_arr farr; \
333 \
334 FASTDECODE_CHECKPACKED(tagbytes, card, packed); \
335 \
336 dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, valbytes, \
337 card); \
338 if (card == CARD_r) { \
339 if (UPB_UNLIKELY(!dst)) { \
340 RETURN_GENERIC("need array resize\n"); \
341 } \
342 } \
343 \
344 again: \
345 if (card == CARD_r) { \
346 dst = fastdecode_resizearr(d, dst, &farr, valbytes); \
347 } \
348 \
349 ptr += tagbytes; \
350 ptr = fastdecode_varint64(ptr, &val); \
351 if (ptr == NULL) _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \
352 val = fastdecode_munge(val, valbytes, zigzag); \
353 memcpy(dst, &val, valbytes); \
354 \
355 if (card == CARD_r) { \
356 fastdecode_nextret ret = fastdecode_nextrepeated( \
357 d, dst, &ptr, &farr, data, tagbytes, valbytes); \
358 switch (ret.next) { \
359 case FD_NEXT_SAMEFIELD: \
360 dst = ret.dst; \
361 goto again; \
362 case FD_NEXT_OTHERFIELD: \
363 data = ret.tag; \
364 UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \
365 case FD_NEXT_ATLIMIT: \
366 return ptr; \
367 } \
368 } \
369 \
370 UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS);
371
372 typedef struct {
373 uint8_t valbytes;
374 bool zigzag;
375 void* dst;
376 fastdecode_arr farr;
377 } fastdecode_varintdata;
378
379 UPB_FORCEINLINE
fastdecode_topackedvarint(upb_EpsCopyInputStream * e,const char * ptr,void * ctx)380 const char* fastdecode_topackedvarint(upb_EpsCopyInputStream* e,
381 const char* ptr, void* ctx) {
382 upb_Decoder* d = (upb_Decoder*)e;
383 fastdecode_varintdata* data = ctx;
384 void* dst = data->dst;
385 uint64_t val;
386
387 while (!_upb_Decoder_IsDone(d, &ptr)) {
388 dst = fastdecode_resizearr(d, dst, &data->farr, data->valbytes);
389 ptr = fastdecode_varint64(ptr, &val);
390 if (ptr == NULL) return NULL;
391 val = fastdecode_munge(val, data->valbytes, data->zigzag);
392 memcpy(dst, &val, data->valbytes);
393 dst = (char*)dst + data->valbytes;
394 }
395
396 fastdecode_commitarr(dst, &data->farr, data->valbytes);
397 return ptr;
398 }
399
400 #define FASTDECODE_PACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \
401 valbytes, zigzag, unpacked) \
402 fastdecode_varintdata ctx = {valbytes, zigzag}; \
403 \
404 FASTDECODE_CHECKPACKED(tagbytes, CARD_r, unpacked); \
405 \
406 ctx.dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &ctx.farr, \
407 valbytes, CARD_r); \
408 if (UPB_UNLIKELY(!ctx.dst)) { \
409 RETURN_GENERIC("need array resize\n"); \
410 } \
411 \
412 ptr += tagbytes; \
413 ptr = fastdecode_delimited(d, ptr, &fastdecode_topackedvarint, &ctx); \
414 \
415 if (UPB_UNLIKELY(ptr == NULL)) { \
416 _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \
417 } \
418 \
419 UPB_MUSTTAIL return fastdecode_dispatch(d, ptr, msg, table, hasbits, 0);
420
421 #define FASTDECODE_VARINT(d, ptr, msg, table, hasbits, data, tagbytes, \
422 valbytes, card, zigzag, unpacked, packed) \
423 if (card == CARD_p) { \
424 FASTDECODE_PACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \
425 valbytes, zigzag, unpacked); \
426 } else { \
427 FASTDECODE_UNPACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \
428 valbytes, card, zigzag, packed); \
429 }
430
431 #define z_ZZ true
432 #define b_ZZ false
433 #define v_ZZ false
434
435 /* Generate all combinations:
436 * {s,o,r,p} x {b1,v4,z4,v8,z8} x {1bt,2bt} */
437
438 #define F(card, type, valbytes, tagbytes) \
439 UPB_NOINLINE \
440 const char* upb_p##card##type##valbytes##_##tagbytes##bt(UPB_PARSE_PARAMS) { \
441 FASTDECODE_VARINT(d, ptr, msg, table, hasbits, data, tagbytes, valbytes, \
442 CARD_##card, type##_ZZ, \
443 upb_pr##type##valbytes##_##tagbytes##bt, \
444 upb_pp##type##valbytes##_##tagbytes##bt); \
445 }
446
447 #define TYPES(card, tagbytes) \
448 F(card, b, 1, tagbytes) \
449 F(card, v, 4, tagbytes) \
450 F(card, v, 8, tagbytes) \
451 F(card, z, 4, tagbytes) \
452 F(card, z, 8, tagbytes)
453
454 #define TAGBYTES(card) \
455 TYPES(card, 1) \
456 TYPES(card, 2)
457
458 TAGBYTES(s)
459 TAGBYTES(o)
460 TAGBYTES(r)
461 TAGBYTES(p)
462
463 #undef z_ZZ
464 #undef b_ZZ
465 #undef v_ZZ
466 #undef o_ONEOF
467 #undef s_ONEOF
468 #undef r_ONEOF
469 #undef F
470 #undef TYPES
471 #undef TAGBYTES
472 #undef FASTDECODE_UNPACKEDVARINT
473 #undef FASTDECODE_PACKEDVARINT
474 #undef FASTDECODE_VARINT
475
476 /* fixed fields ***************************************************************/
477
478 #define FASTDECODE_UNPACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \
479 valbytes, card, packed) \
480 void* dst; \
481 fastdecode_arr farr; \
482 \
483 FASTDECODE_CHECKPACKED(tagbytes, card, packed) \
484 \
485 dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, valbytes, \
486 card); \
487 if (card == CARD_r) { \
488 if (UPB_UNLIKELY(!dst)) { \
489 RETURN_GENERIC("couldn't allocate array in arena\n"); \
490 } \
491 } \
492 \
493 again: \
494 if (card == CARD_r) { \
495 dst = fastdecode_resizearr(d, dst, &farr, valbytes); \
496 } \
497 \
498 ptr += tagbytes; \
499 memcpy(dst, ptr, valbytes); \
500 ptr += valbytes; \
501 \
502 if (card == CARD_r) { \
503 fastdecode_nextret ret = fastdecode_nextrepeated( \
504 d, dst, &ptr, &farr, data, tagbytes, valbytes); \
505 switch (ret.next) { \
506 case FD_NEXT_SAMEFIELD: \
507 dst = ret.dst; \
508 goto again; \
509 case FD_NEXT_OTHERFIELD: \
510 data = ret.tag; \
511 UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \
512 case FD_NEXT_ATLIMIT: \
513 return ptr; \
514 } \
515 } \
516 \
517 UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS);
518
519 #define FASTDECODE_PACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \
520 valbytes, unpacked) \
521 FASTDECODE_CHECKPACKED(tagbytes, CARD_r, unpacked) \
522 \
523 ptr += tagbytes; \
524 int size = (uint8_t)ptr[0]; \
525 ptr++; \
526 if (size & 0x80) { \
527 ptr = fastdecode_longsize(ptr, &size); \
528 } \
529 \
530 if (UPB_UNLIKELY(!upb_EpsCopyInputStream_CheckDataSizeAvailable( \
531 &d->input, ptr, size) || \
532 (size % valbytes) != 0)) { \
533 _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \
534 } \
535 \
536 upb_Array** arr_p = fastdecode_fieldmem(msg, data); \
537 upb_Array* arr = *arr_p; \
538 uint8_t elem_size_lg2 = __builtin_ctz(valbytes); \
539 int elems = size / valbytes; \
540 \
541 if (UPB_LIKELY(!arr)) { \
542 *arr_p = arr = \
543 UPB_PRIVATE(_upb_Array_New)(&d->arena, elems, elem_size_lg2); \
544 if (!arr) { \
545 _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \
546 } \
547 } else { \
548 UPB_PRIVATE(_upb_Array_ResizeUninitialized)(arr, elems, &d->arena); \
549 } \
550 \
551 char* dst = upb_Array_MutableDataPtr(arr); \
552 memcpy(dst, ptr, size); \
553 arr->UPB_PRIVATE(size) = elems; \
554 \
555 ptr += size; \
556 UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS);
557
558 #define FASTDECODE_FIXED(d, ptr, msg, table, hasbits, data, tagbytes, \
559 valbytes, card, unpacked, packed) \
560 if (card == CARD_p) { \
561 FASTDECODE_PACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \
562 valbytes, unpacked); \
563 } else { \
564 FASTDECODE_UNPACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \
565 valbytes, card, packed); \
566 }
567
568 /* Generate all combinations:
569 * {s,o,r,p} x {f4,f8} x {1bt,2bt} */
570
571 #define F(card, valbytes, tagbytes) \
572 UPB_NOINLINE \
573 const char* upb_p##card##f##valbytes##_##tagbytes##bt(UPB_PARSE_PARAMS) { \
574 FASTDECODE_FIXED(d, ptr, msg, table, hasbits, data, tagbytes, valbytes, \
575 CARD_##card, upb_ppf##valbytes##_##tagbytes##bt, \
576 upb_prf##valbytes##_##tagbytes##bt); \
577 }
578
579 #define TYPES(card, tagbytes) \
580 F(card, 4, tagbytes) \
581 F(card, 8, tagbytes)
582
583 #define TAGBYTES(card) \
584 TYPES(card, 1) \
585 TYPES(card, 2)
586
587 TAGBYTES(s)
588 TAGBYTES(o)
589 TAGBYTES(r)
590 TAGBYTES(p)
591
592 #undef F
593 #undef TYPES
594 #undef TAGBYTES
595 #undef FASTDECODE_UNPACKEDFIXED
596 #undef FASTDECODE_PACKEDFIXED
597
598 /* string fields **************************************************************/
599
600 typedef const char* fastdecode_copystr_func(struct upb_Decoder* d,
601 const char* ptr, upb_Message* msg,
602 const upb_MiniTable* table,
603 uint64_t hasbits,
604 upb_StringView* dst);
605
606 UPB_NOINLINE
fastdecode_verifyutf8(upb_Decoder * d,const char * ptr,upb_Message * msg,intptr_t table,uint64_t hasbits,uint64_t data)607 static const char* fastdecode_verifyutf8(upb_Decoder* d, const char* ptr,
608 upb_Message* msg, intptr_t table,
609 uint64_t hasbits, uint64_t data) {
610 UPB_ASSERT(!upb_Message_IsFrozen(msg));
611 upb_StringView* dst = (upb_StringView*)data;
612 if (!_upb_Decoder_VerifyUtf8Inline(dst->data, dst->size)) {
613 _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8);
614 }
615 UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS);
616 }
617
618 #define FASTDECODE_LONGSTRING(d, ptr, msg, table, hasbits, dst, validate_utf8) \
619 int size = (uint8_t)ptr[0]; /* Could plumb through hasbits. */ \
620 ptr++; \
621 if (size & 0x80) { \
622 ptr = fastdecode_longsize(ptr, &size); \
623 } \
624 \
625 if (UPB_UNLIKELY(!upb_EpsCopyInputStream_CheckSize(&d->input, ptr, size))) { \
626 dst->size = 0; \
627 _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \
628 } \
629 \
630 const char* s_ptr = ptr; \
631 ptr = upb_EpsCopyInputStream_ReadString(&d->input, &s_ptr, size, &d->arena); \
632 if (!ptr) _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); \
633 dst->data = s_ptr; \
634 dst->size = size; \
635 \
636 if (validate_utf8) { \
637 data = (uint64_t)dst; \
638 UPB_MUSTTAIL return fastdecode_verifyutf8(UPB_PARSE_ARGS); \
639 } else { \
640 UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); \
641 }
642
643 UPB_NOINLINE
fastdecode_longstring_utf8(struct upb_Decoder * d,const char * ptr,upb_Message * msg,intptr_t table,uint64_t hasbits,uint64_t data)644 static const char* fastdecode_longstring_utf8(struct upb_Decoder* d,
645 const char* ptr, upb_Message* msg,
646 intptr_t table, uint64_t hasbits,
647 uint64_t data) {
648 upb_StringView* dst = (upb_StringView*)data;
649 FASTDECODE_LONGSTRING(d, ptr, msg, table, hasbits, dst, true);
650 }
651
652 UPB_NOINLINE
fastdecode_longstring_noutf8(struct upb_Decoder * d,const char * ptr,upb_Message * msg,intptr_t table,uint64_t hasbits,uint64_t data)653 static const char* fastdecode_longstring_noutf8(
654 struct upb_Decoder* d, const char* ptr, upb_Message* msg, intptr_t table,
655 uint64_t hasbits, uint64_t data) {
656 UPB_ASSERT(!upb_Message_IsFrozen(msg));
657 upb_StringView* dst = (upb_StringView*)data;
658 FASTDECODE_LONGSTRING(d, ptr, msg, table, hasbits, dst, false);
659 }
660
661 UPB_FORCEINLINE
fastdecode_docopy(upb_Decoder * d,const char * ptr,uint32_t size,int copy,char * data,size_t data_offset,upb_StringView * dst)662 void fastdecode_docopy(upb_Decoder* d, const char* ptr, uint32_t size, int copy,
663 char* data, size_t data_offset, upb_StringView* dst) {
664 d->arena.UPB_PRIVATE(ptr) += copy;
665 dst->data = data + data_offset;
666 UPB_UNPOISON_MEMORY_REGION(data, copy);
667 memcpy(data, ptr, copy);
668 UPB_POISON_MEMORY_REGION(data + data_offset + size,
669 copy - data_offset - size);
670 }
671
672 #define FASTDECODE_COPYSTRING(d, ptr, msg, table, hasbits, data, tagbytes, \
673 card, validate_utf8) \
674 upb_StringView* dst; \
675 fastdecode_arr farr; \
676 int64_t size; \
677 size_t arena_has; \
678 size_t common_has; \
679 char* buf; \
680 \
681 UPB_ASSERT(!upb_EpsCopyInputStream_AliasingAvailable(&d->input, ptr, 0)); \
682 UPB_ASSERT(fastdecode_checktag(data, tagbytes)); \
683 \
684 dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, \
685 sizeof(upb_StringView), card); \
686 \
687 again: \
688 if (card == CARD_r) { \
689 dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_StringView)); \
690 } \
691 \
692 size = (uint8_t)ptr[tagbytes]; \
693 ptr += tagbytes + 1; \
694 dst->size = size; \
695 \
696 buf = d->arena.UPB_PRIVATE(ptr); \
697 arena_has = UPB_PRIVATE(_upb_ArenaHas)(&d->arena); \
698 common_has = UPB_MIN(arena_has, \
699 upb_EpsCopyInputStream_BytesAvailable(&d->input, ptr)); \
700 \
701 if (UPB_LIKELY(size <= 15 - tagbytes)) { \
702 if (arena_has < 16) goto longstr; \
703 fastdecode_docopy(d, ptr - tagbytes - 1, size, 16, buf, tagbytes + 1, \
704 dst); \
705 } else if (UPB_LIKELY(size <= 32)) { \
706 if (UPB_UNLIKELY(common_has < 32)) goto longstr; \
707 fastdecode_docopy(d, ptr, size, 32, buf, 0, dst); \
708 } else if (UPB_LIKELY(size <= 64)) { \
709 if (UPB_UNLIKELY(common_has < 64)) goto longstr; \
710 fastdecode_docopy(d, ptr, size, 64, buf, 0, dst); \
711 } else if (UPB_LIKELY(size < 128)) { \
712 if (UPB_UNLIKELY(common_has < 128)) goto longstr; \
713 fastdecode_docopy(d, ptr, size, 128, buf, 0, dst); \
714 } else { \
715 goto longstr; \
716 } \
717 \
718 ptr += size; \
719 \
720 if (card == CARD_r) { \
721 if (validate_utf8 && \
722 !_upb_Decoder_VerifyUtf8Inline(dst->data, dst->size)) { \
723 _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8); \
724 } \
725 fastdecode_nextret ret = fastdecode_nextrepeated( \
726 d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_StringView)); \
727 switch (ret.next) { \
728 case FD_NEXT_SAMEFIELD: \
729 dst = ret.dst; \
730 goto again; \
731 case FD_NEXT_OTHERFIELD: \
732 data = ret.tag; \
733 UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \
734 case FD_NEXT_ATLIMIT: \
735 return ptr; \
736 } \
737 } \
738 \
739 if (card != CARD_r && validate_utf8) { \
740 data = (uint64_t)dst; \
741 UPB_MUSTTAIL return fastdecode_verifyutf8(UPB_PARSE_ARGS); \
742 } \
743 \
744 UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); \
745 \
746 longstr: \
747 if (card == CARD_r) { \
748 fastdecode_commitarr(dst + 1, &farr, sizeof(upb_StringView)); \
749 } \
750 ptr--; \
751 if (validate_utf8) { \
752 UPB_MUSTTAIL return fastdecode_longstring_utf8(d, ptr, msg, table, \
753 hasbits, (uint64_t)dst); \
754 } else { \
755 UPB_MUSTTAIL return fastdecode_longstring_noutf8(d, ptr, msg, table, \
756 hasbits, (uint64_t)dst); \
757 }
758
759 #define FASTDECODE_STRING(d, ptr, msg, table, hasbits, data, tagbytes, card, \
760 copyfunc, validate_utf8) \
761 upb_StringView* dst; \
762 fastdecode_arr farr; \
763 int64_t size; \
764 \
765 if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \
766 RETURN_GENERIC("string field tag mismatch\n"); \
767 } \
768 \
769 if (UPB_UNLIKELY( \
770 !upb_EpsCopyInputStream_AliasingAvailable(&d->input, ptr, 0))) { \
771 UPB_MUSTTAIL return copyfunc(UPB_PARSE_ARGS); \
772 } \
773 \
774 dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, \
775 sizeof(upb_StringView), card); \
776 \
777 again: \
778 if (card == CARD_r) { \
779 dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_StringView)); \
780 } \
781 \
782 size = (int8_t)ptr[tagbytes]; \
783 ptr += tagbytes + 1; \
784 \
785 if (UPB_UNLIKELY( \
786 !upb_EpsCopyInputStream_AliasingAvailable(&d->input, ptr, size))) { \
787 ptr--; \
788 if (validate_utf8) { \
789 return fastdecode_longstring_utf8(d, ptr, msg, table, hasbits, \
790 (uint64_t)dst); \
791 } else { \
792 return fastdecode_longstring_noutf8(d, ptr, msg, table, hasbits, \
793 (uint64_t)dst); \
794 } \
795 } \
796 \
797 dst->data = ptr; \
798 dst->size = size; \
799 ptr = upb_EpsCopyInputStream_ReadStringAliased(&d->input, &dst->data, \
800 dst->size); \
801 \
802 if (card == CARD_r) { \
803 if (validate_utf8 && \
804 !_upb_Decoder_VerifyUtf8Inline(dst->data, dst->size)) { \
805 _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8); \
806 } \
807 fastdecode_nextret ret = fastdecode_nextrepeated( \
808 d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_StringView)); \
809 switch (ret.next) { \
810 case FD_NEXT_SAMEFIELD: \
811 dst = ret.dst; \
812 goto again; \
813 case FD_NEXT_OTHERFIELD: \
814 data = ret.tag; \
815 UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \
816 case FD_NEXT_ATLIMIT: \
817 return ptr; \
818 } \
819 } \
820 \
821 if (card != CARD_r && validate_utf8) { \
822 data = (uint64_t)dst; \
823 UPB_MUSTTAIL return fastdecode_verifyutf8(UPB_PARSE_ARGS); \
824 } \
825 \
826 UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS);
827
828 /* Generate all combinations:
829 * {p,c} x {s,o,r} x {s, b} x {1bt,2bt} */
830
831 #define s_VALIDATE true
832 #define b_VALIDATE false
833
834 #define F(card, tagbytes, type) \
835 UPB_NOINLINE \
836 const char* upb_c##card##type##_##tagbytes##bt(UPB_PARSE_PARAMS) { \
837 FASTDECODE_COPYSTRING(d, ptr, msg, table, hasbits, data, tagbytes, \
838 CARD_##card, type##_VALIDATE); \
839 } \
840 const char* upb_p##card##type##_##tagbytes##bt(UPB_PARSE_PARAMS) { \
841 FASTDECODE_STRING(d, ptr, msg, table, hasbits, data, tagbytes, \
842 CARD_##card, upb_c##card##type##_##tagbytes##bt, \
843 type##_VALIDATE); \
844 }
845
846 #define UTF8(card, tagbytes) \
847 F(card, tagbytes, s) \
848 F(card, tagbytes, b)
849
850 #define TAGBYTES(card) \
851 UTF8(card, 1) \
852 UTF8(card, 2)
853
854 TAGBYTES(s)
TAGBYTES(o)855 TAGBYTES(o)
856 TAGBYTES(r)
857
858 #undef s_VALIDATE
859 #undef b_VALIDATE
860 #undef F
861 #undef TAGBYTES
862 #undef FASTDECODE_LONGSTRING
863 #undef FASTDECODE_COPYSTRING
864 #undef FASTDECODE_STRING
865
866 /* message fields *************************************************************/
867
868 UPB_INLINE
869 upb_Message* decode_newmsg_ceil(upb_Decoder* d, const upb_MiniTable* m,
870 int msg_ceil_bytes) {
871 size_t size = m->UPB_PRIVATE(size);
872 char* msg_data;
873 if (UPB_LIKELY(msg_ceil_bytes > 0 &&
874 UPB_PRIVATE(_upb_ArenaHas)(&d->arena) >= msg_ceil_bytes)) {
875 UPB_ASSERT(size <= (size_t)msg_ceil_bytes);
876 msg_data = d->arena.UPB_PRIVATE(ptr);
877 d->arena.UPB_PRIVATE(ptr) += size;
878 UPB_UNPOISON_MEMORY_REGION(msg_data, msg_ceil_bytes);
879 memset(msg_data, 0, msg_ceil_bytes);
880 UPB_POISON_MEMORY_REGION(msg_data + size, msg_ceil_bytes - size);
881 } else {
882 msg_data = (char*)upb_Arena_Malloc(&d->arena, size);
883 memset(msg_data, 0, size);
884 }
885 return (upb_Message*)msg_data;
886 }
887
888 typedef struct {
889 intptr_t table;
890 upb_Message* msg;
891 } fastdecode_submsgdata;
892
893 UPB_FORCEINLINE
fastdecode_tosubmsg(upb_EpsCopyInputStream * e,const char * ptr,void * ctx)894 const char* fastdecode_tosubmsg(upb_EpsCopyInputStream* e, const char* ptr,
895 void* ctx) {
896 upb_Decoder* d = (upb_Decoder*)e;
897 fastdecode_submsgdata* submsg = ctx;
898 ptr = fastdecode_dispatch(d, ptr, submsg->msg, submsg->table, 0, 0);
899 UPB_ASSUME(ptr != NULL);
900 return ptr;
901 }
902
903 #define FASTDECODE_SUBMSG(d, ptr, msg, table, hasbits, data, tagbytes, \
904 msg_ceil_bytes, card) \
905 \
906 if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \
907 RETURN_GENERIC("submessage field tag mismatch\n"); \
908 } \
909 \
910 if (--d->depth == 0) { \
911 _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_MaxDepthExceeded); \
912 } \
913 \
914 upb_Message** dst; \
915 uint32_t submsg_idx = (data >> 16) & 0xff; \
916 const upb_MiniTable* tablep = decode_totablep(table); \
917 const upb_MiniTable* subtablep = upb_MiniTableSub_Message( \
918 *UPB_PRIVATE(_upb_MiniTable_GetSubByIndex)(tablep, submsg_idx)); \
919 fastdecode_submsgdata submsg = {decode_totable(subtablep)}; \
920 fastdecode_arr farr; \
921 \
922 if (subtablep->UPB_PRIVATE(table_mask) == (uint8_t)-1) { \
923 d->depth++; \
924 RETURN_GENERIC("submessage doesn't have fast tables."); \
925 } \
926 \
927 dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, \
928 sizeof(upb_Message*), card); \
929 \
930 if (card == CARD_s) { \
931 ((uint32_t*)msg)[2] |= hasbits; \
932 hasbits = 0; \
933 } \
934 \
935 again: \
936 if (card == CARD_r) { \
937 dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_Message*)); \
938 } \
939 \
940 submsg.msg = *dst; \
941 \
942 if (card == CARD_r || UPB_LIKELY(!submsg.msg)) { \
943 *dst = submsg.msg = decode_newmsg_ceil(d, subtablep, msg_ceil_bytes); \
944 } \
945 \
946 ptr += tagbytes; \
947 ptr = fastdecode_delimited(d, ptr, fastdecode_tosubmsg, &submsg); \
948 \
949 if (UPB_UNLIKELY(ptr == NULL || d->end_group != DECODE_NOGROUP)) { \
950 _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \
951 } \
952 \
953 if (card == CARD_r) { \
954 fastdecode_nextret ret = fastdecode_nextrepeated( \
955 d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_Message*)); \
956 switch (ret.next) { \
957 case FD_NEXT_SAMEFIELD: \
958 dst = ret.dst; \
959 goto again; \
960 case FD_NEXT_OTHERFIELD: \
961 d->depth++; \
962 data = ret.tag; \
963 UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \
964 case FD_NEXT_ATLIMIT: \
965 d->depth++; \
966 return ptr; \
967 } \
968 } \
969 \
970 d->depth++; \
971 UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS);
972
973 #define F(card, tagbytes, size_ceil, ceil_arg) \
974 const char* upb_p##card##m_##tagbytes##bt_max##size_ceil##b( \
975 UPB_PARSE_PARAMS) { \
976 FASTDECODE_SUBMSG(d, ptr, msg, table, hasbits, data, tagbytes, ceil_arg, \
977 CARD_##card); \
978 }
979
980 #define SIZES(card, tagbytes) \
981 F(card, tagbytes, 64, 64) \
982 F(card, tagbytes, 128, 128) \
983 F(card, tagbytes, 192, 192) \
984 F(card, tagbytes, 256, 256) \
985 F(card, tagbytes, max, -1)
986
987 #define TAGBYTES(card) \
988 SIZES(card, 1) \
989 SIZES(card, 2)
990
991 TAGBYTES(s)
992 TAGBYTES(o)
993 TAGBYTES(r)
994
995 #undef TAGBYTES
996 #undef SIZES
997 #undef F
998 #undef FASTDECODE_SUBMSG
999
1000 #endif /* UPB_FASTTABLE */
1001