• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // Copyright (C) 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
3 /*
4 *******************************************************************************
5 *
6 *   Copyright (C) 2000-2015, International Business Machines
7 *   Corporation and others.  All Rights Reserved.
8 *
9 *******************************************************************************
10 *
11 * File reslist.cpp
12 *
13 * Modification History:
14 *
15 *   Date        Name        Description
16 *   02/21/00    weiv        Creation.
17 *******************************************************************************
18 */
19 
20 // Safer use of UnicodeString.
21 #ifndef UNISTR_FROM_CHAR_EXPLICIT
22 #   define UNISTR_FROM_CHAR_EXPLICIT explicit
23 #endif
24 
25 // Less important, but still a good idea.
26 #ifndef UNISTR_FROM_STRING_EXPLICIT
27 #   define UNISTR_FROM_STRING_EXPLICIT explicit
28 #endif
29 
30 #include <assert.h>
31 #include <stdio.h>
32 #include "unicode/localpointer.h"
33 #include "reslist.h"
34 #include "unewdata.h"
35 #include "unicode/ures.h"
36 #include "unicode/putil.h"
37 #include "errmsg.h"
38 
39 #include "uarrsort.h"
40 #include "uelement.h"
41 #include "uhash.h"
42 #include "uinvchar.h"
43 #include "ustr_imp.h"
44 #include "unicode/utf16.h"
45 /*
46  * Align binary data at a 16-byte offset from the start of the resource bundle,
47  * to be safe for any data type it may contain.
48  */
49 #define BIN_ALIGNMENT 16
50 
51 // This numeric constant must be at least 1.
52 // If StringResource.fNumUnitsSaved == 0 then the string occurs only once,
53 // and it makes no sense to move it to the pool bundle.
54 // The larger the threshold for fNumUnitsSaved
55 // the smaller the savings, and the smaller the pool bundle.
56 // We trade some total size reduction to reduce the pool bundle a bit,
57 // so that one can reasonably save data size by
58 // removing bundle files without rebuilding the pool bundle.
59 // This can also help to keep the pool and total (pool+local) string indexes
60 // within 16 bits, that is, within range of Table16 and Array16 containers.
61 #ifndef GENRB_MIN_16BIT_UNITS_SAVED_FOR_POOL_STRING
62 #   define GENRB_MIN_16BIT_UNITS_SAVED_FOR_POOL_STRING 10
63 #endif
64 
65 U_NAMESPACE_USE
66 
67 static UBool gIncludeCopyright = FALSE;
68 static UBool gUsePoolBundle = FALSE;
69 static UBool gIsDefaultFormatVersion = TRUE;
70 static int32_t gFormatVersion = 3;
71 
72 /* How do we store string values? */
73 enum {
74     STRINGS_UTF16_V1,   /* formatVersion 1: int length + UChars + NUL + padding to 4 bytes */
75     STRINGS_UTF16_V2    /* formatVersion 2 & up: optional length in 1..3 UChars + UChars + NUL */
76 };
77 
78 static const int32_t MAX_IMPLICIT_STRING_LENGTH = 40;  /* do not store the length explicitly for such strings */
79 
80 static const ResFile kNoPoolBundle;
81 
82 /*
83  * res_none() returns the address of kNoResource,
84  * for use in non-error cases when no resource is to be added to the bundle.
85  * (NULL is used in error cases.)
86  */
87 static SResource kNoResource;  // TODO: const
88 
89 static UDataInfo dataInfo= {
90     sizeof(UDataInfo),
91     0,
92 
93     U_IS_BIG_ENDIAN,
94     U_CHARSET_FAMILY,
95     sizeof(UChar),
96     0,
97 
98     {0x52, 0x65, 0x73, 0x42},     /* dataFormat="ResB" */
99     {1, 3, 0, 0},                 /* formatVersion */
100     {1, 4, 0, 0}                  /* dataVersion take a look at version inside parsed resb*/
101 };
102 
103 static const UVersionInfo gFormatVersions[4] = {  /* indexed by a major-formatVersion integer */
104     { 0, 0, 0, 0 },
105     { 1, 3, 0, 0 },
106     { 2, 0, 0, 0 },
107     { 3, 0, 0, 0 }
108 };
109 // Remember to update genrb.h GENRB_VERSION when changing the data format.
110 // (Or maybe we should remove GENRB_VERSION and report the ICU version number?)
111 
calcPadding(uint32_t size)112 static uint8_t calcPadding(uint32_t size) {
113     /* returns space we need to pad */
114     return (uint8_t) ((size % sizeof(uint32_t)) ? (sizeof(uint32_t) - (size % sizeof(uint32_t))) : 0);
115 
116 }
117 
setIncludeCopyright(UBool val)118 void setIncludeCopyright(UBool val){
119     gIncludeCopyright=val;
120 }
121 
getIncludeCopyright(void)122 UBool getIncludeCopyright(void){
123     return gIncludeCopyright;
124 }
125 
setFormatVersion(int32_t formatVersion)126 void setFormatVersion(int32_t formatVersion) {
127     gIsDefaultFormatVersion = FALSE;
128     gFormatVersion = formatVersion;
129 }
130 
getFormatVersion()131 int32_t getFormatVersion() {
132     return gFormatVersion;
133 }
134 
setUsePoolBundle(UBool use)135 void setUsePoolBundle(UBool use) {
136     gUsePoolBundle = use;
137 }
138 
139 // TODO: return const pointer, or find another way to express "none"
res_none()140 struct SResource* res_none() {
141     return &kNoResource;
142 }
143 
SResource()144 SResource::SResource()
145         : fType(URES_NONE), fWritten(FALSE), fRes(RES_BOGUS), fRes16(-1), fKey(-1), fKey16(-1),
146           line(0), fNext(NULL) {
147     ustr_init(&fComment);
148 }
149 
SResource(SRBRoot * bundle,const char * tag,int8_t type,const UString * comment,UErrorCode & errorCode)150 SResource::SResource(SRBRoot *bundle, const char *tag, int8_t type, const UString* comment,
151                      UErrorCode &errorCode)
152         : fType(type), fWritten(FALSE), fRes(RES_BOGUS), fRes16(-1),
153           fKey(bundle != NULL ? bundle->addTag(tag, errorCode) : -1), fKey16(-1),
154           line(0), fNext(NULL) {
155     ustr_init(&fComment);
156     if(comment != NULL) {
157         ustr_cpy(&fComment, comment, &errorCode);
158     }
159 }
160 
~SResource()161 SResource::~SResource() {
162     ustr_deinit(&fComment);
163 }
164 
~ContainerResource()165 ContainerResource::~ContainerResource() {
166     SResource *current = fFirst;
167     while (current != NULL) {
168         SResource *next = current->fNext;
169         delete current;
170         current = next;
171     }
172 }
173 
~TableResource()174 TableResource::~TableResource() {}
175 
176 // TODO: clarify that containers adopt new items, even in error cases; use LocalPointer
add(SResource * res,int linenumber,UErrorCode & errorCode)177 void TableResource::add(SResource *res, int linenumber, UErrorCode &errorCode) {
178     if (U_FAILURE(errorCode) || res == NULL || res == &kNoResource) {
179         return;
180     }
181 
182     /* remember this linenumber to report to the user if there is a duplicate key */
183     res->line = linenumber;
184 
185     /* here we need to traverse the list */
186     ++fCount;
187 
188     /* is the list still empty? */
189     if (fFirst == NULL) {
190         fFirst = res;
191         res->fNext = NULL;
192         return;
193     }
194 
195     const char *resKeyString = fRoot->fKeys + res->fKey;
196 
197     SResource *current = fFirst;
198 
199     SResource *prev = NULL;
200     while (current != NULL) {
201         const char *currentKeyString = fRoot->fKeys + current->fKey;
202         int diff;
203         /*
204          * formatVersion 1: compare key strings in native-charset order
205          * formatVersion 2 and up: compare key strings in ASCII order
206          */
207         if (gFormatVersion == 1 || U_CHARSET_FAMILY == U_ASCII_FAMILY) {
208             diff = uprv_strcmp(currentKeyString, resKeyString);
209         } else {
210             diff = uprv_compareInvCharsAsAscii(currentKeyString, resKeyString);
211         }
212         if (diff < 0) {
213             prev    = current;
214             current = current->fNext;
215         } else if (diff > 0) {
216             /* we're either in front of the list, or in the middle */
217             if (prev == NULL) {
218                 /* front of the list */
219                 fFirst = res;
220             } else {
221                 /* middle of the list */
222                 prev->fNext = res;
223             }
224 
225             res->fNext = current;
226             return;
227         } else {
228             /* Key already exists! ERROR! */
229             error(linenumber, "duplicate key '%s' in table, first appeared at line %d", currentKeyString, current->line);
230             errorCode = U_UNSUPPORTED_ERROR;
231             return;
232         }
233     }
234 
235     /* end of list */
236     prev->fNext = res;
237     res->fNext  = NULL;
238 }
239 
~ArrayResource()240 ArrayResource::~ArrayResource() {}
241 
add(SResource * res)242 void ArrayResource::add(SResource *res) {
243     if (res != NULL && res != &kNoResource) {
244         if (fFirst == NULL) {
245             fFirst = res;
246         } else {
247             fLast->fNext = res;
248         }
249         fLast = res;
250         ++fCount;
251     }
252 }
253 
~PseudoListResource()254 PseudoListResource::~PseudoListResource() {}
255 
add(SResource * res)256 void PseudoListResource::add(SResource *res) {
257     if (res != NULL && res != &kNoResource) {
258         res->fNext = fFirst;
259         fFirst = res;
260         ++fCount;
261     }
262 }
263 
StringBaseResource(SRBRoot * bundle,const char * tag,int8_t type,const UChar * value,int32_t len,const UString * comment,UErrorCode & errorCode)264 StringBaseResource::StringBaseResource(SRBRoot *bundle, const char *tag, int8_t type,
265                                        const UChar *value, int32_t len,
266                                        const UString* comment, UErrorCode &errorCode)
267         : SResource(bundle, tag, type, comment, errorCode) {
268     if (len == 0 && gFormatVersion > 1) {
269         fRes = URES_MAKE_EMPTY_RESOURCE(type);
270         fWritten = TRUE;
271         return;
272     }
273 
274     fString.setTo(value, len);
275     fString.getTerminatedBuffer();  // Some code relies on NUL-termination.
276     if (U_SUCCESS(errorCode) && fString.isBogus()) {
277         errorCode = U_MEMORY_ALLOCATION_ERROR;
278     }
279 }
280 
StringBaseResource(SRBRoot * bundle,int8_t type,const icu::UnicodeString & value,UErrorCode & errorCode)281 StringBaseResource::StringBaseResource(SRBRoot *bundle, int8_t type,
282                                        const icu::UnicodeString &value, UErrorCode &errorCode)
283         : SResource(bundle, NULL, type, NULL, errorCode), fString(value) {
284     if (value.isEmpty() && gFormatVersion > 1) {
285         fRes = URES_MAKE_EMPTY_RESOURCE(type);
286         fWritten = TRUE;
287         return;
288     }
289 
290     fString.getTerminatedBuffer();  // Some code relies on NUL-termination.
291     if (U_SUCCESS(errorCode) && fString.isBogus()) {
292         errorCode = U_MEMORY_ALLOCATION_ERROR;
293     }
294 }
295 
296 // Pool bundle string, alias the buffer. Guaranteed NUL-terminated and not empty.
StringBaseResource(int8_t type,const UChar * value,int32_t len,UErrorCode & errorCode)297 StringBaseResource::StringBaseResource(int8_t type, const UChar *value, int32_t len,
298                                        UErrorCode &errorCode)
299         : SResource(NULL, NULL, type, NULL, errorCode), fString(TRUE, value, len) {
300     assert(len > 0);
301     assert(!fString.isBogus());
302 }
303 
~StringBaseResource()304 StringBaseResource::~StringBaseResource() {}
305 
306 static int32_t U_CALLCONV
string_hash(const UElement key)307 string_hash(const UElement key) {
308     const StringResource *res = static_cast<const StringResource *>(key.pointer);
309     return res->fString.hashCode();
310 }
311 
312 static UBool U_CALLCONV
string_comp(const UElement key1,const UElement key2)313 string_comp(const UElement key1, const UElement key2) {
314     const StringResource *res1 = static_cast<const StringResource *>(key1.pointer);
315     const StringResource *res2 = static_cast<const StringResource *>(key2.pointer);
316     return res1->fString == res2->fString;
317 }
318 
~StringResource()319 StringResource::~StringResource() {}
320 
~AliasResource()321 AliasResource::~AliasResource() {}
322 
IntResource(SRBRoot * bundle,const char * tag,int32_t value,const UString * comment,UErrorCode & errorCode)323 IntResource::IntResource(SRBRoot *bundle, const char *tag, int32_t value,
324                          const UString* comment, UErrorCode &errorCode)
325         : SResource(bundle, tag, URES_INT, comment, errorCode) {
326     fValue = value;
327     fRes = URES_MAKE_RESOURCE(URES_INT, value & RES_MAX_OFFSET);
328     fWritten = TRUE;
329 }
330 
~IntResource()331 IntResource::~IntResource() {}
332 
IntVectorResource(SRBRoot * bundle,const char * tag,const UString * comment,UErrorCode & errorCode)333 IntVectorResource::IntVectorResource(SRBRoot *bundle, const char *tag,
334                   const UString* comment, UErrorCode &errorCode)
335         : SResource(bundle, tag, URES_INT_VECTOR, comment, errorCode),
336           fCount(0), fArray(new uint32_t[RESLIST_MAX_INT_VECTOR]) {
337     if (fArray == NULL) {
338         errorCode = U_MEMORY_ALLOCATION_ERROR;
339         return;
340     }
341 }
342 
~IntVectorResource()343 IntVectorResource::~IntVectorResource() {
344     delete[] fArray;
345 }
346 
add(int32_t value,UErrorCode & errorCode)347 void IntVectorResource::add(int32_t value, UErrorCode &errorCode) {
348     if (U_SUCCESS(errorCode)) {
349         fArray[fCount++] = value;
350     }
351 }
352 
BinaryResource(SRBRoot * bundle,const char * tag,uint32_t length,uint8_t * data,const char * fileName,const UString * comment,UErrorCode & errorCode)353 BinaryResource::BinaryResource(SRBRoot *bundle, const char *tag,
354                                uint32_t length, uint8_t *data, const char* fileName,
355                                const UString* comment, UErrorCode &errorCode)
356         : SResource(bundle, tag, URES_BINARY, comment, errorCode),
357           fLength(length), fData(NULL), fFileName(NULL) {
358     if (U_FAILURE(errorCode)) {
359         return;
360     }
361     if (fileName != NULL && *fileName != 0){
362         fFileName = new char[uprv_strlen(fileName)+1];
363         if (fFileName == NULL) {
364             errorCode = U_MEMORY_ALLOCATION_ERROR;
365             return;
366         }
367         uprv_strcpy(fFileName, fileName);
368     }
369     if (length > 0) {
370         fData = new uint8_t[length];
371         if (fData == NULL) {
372             errorCode = U_MEMORY_ALLOCATION_ERROR;
373             return;
374         }
375         uprv_memcpy(fData, data, length);
376     } else {
377         if (gFormatVersion > 1) {
378             fRes = URES_MAKE_EMPTY_RESOURCE(URES_BINARY);
379             fWritten = TRUE;
380         }
381     }
382 }
383 
~BinaryResource()384 BinaryResource::~BinaryResource() {
385     delete[] fData;
386     delete[] fFileName;
387 }
388 
389 /* Writing Functions */
390 
391 void
handlePreflightStrings(SRBRoot * bundle,UHashtable * stringSet,UErrorCode & errorCode)392 StringResource::handlePreflightStrings(SRBRoot *bundle, UHashtable *stringSet,
393                                        UErrorCode &errorCode) {
394     assert(fSame == NULL);
395     fSame = static_cast<StringResource *>(uhash_get(stringSet, this));
396     if (fSame != NULL) {
397         // This is a duplicate of a pool bundle string or of an earlier-visited string.
398         if (++fSame->fNumCopies == 1) {
399             assert(fSame->fWritten);
400             int32_t poolStringIndex = (int32_t)RES_GET_OFFSET(fSame->fRes);
401             if (poolStringIndex >= bundle->fPoolStringIndexLimit) {
402                 bundle->fPoolStringIndexLimit = poolStringIndex + 1;
403             }
404         }
405         return;
406     }
407     /* Put this string into the set for finding duplicates. */
408     fNumCopies = 1;
409     uhash_put(stringSet, this, this, &errorCode);
410 
411     if (bundle->fStringsForm != STRINGS_UTF16_V1) {
412         int32_t len = length();
413         if (len <= MAX_IMPLICIT_STRING_LENGTH &&
414                 !U16_IS_TRAIL(fString[0]) && fString.indexOf((UChar)0) < 0) {
415             /*
416              * This string will be stored without an explicit length.
417              * Runtime will detect !U16_IS_TRAIL(s[0]) and call u_strlen().
418              */
419             fNumCharsForLength = 0;
420         } else if (len <= 0x3ee) {
421             fNumCharsForLength = 1;
422         } else if (len <= 0xfffff) {
423             fNumCharsForLength = 2;
424         } else {
425             fNumCharsForLength = 3;
426         }
427         bundle->f16BitStringsLength += fNumCharsForLength + len + 1;  /* +1 for the NUL */
428     }
429 }
430 
431 void
handlePreflightStrings(SRBRoot * bundle,UHashtable * stringSet,UErrorCode & errorCode)432 ContainerResource::handlePreflightStrings(SRBRoot *bundle, UHashtable *stringSet,
433                                           UErrorCode &errorCode) {
434     for (SResource *current = fFirst; current != NULL; current = current->fNext) {
435         current->preflightStrings(bundle, stringSet, errorCode);
436     }
437 }
438 
439 void
preflightStrings(SRBRoot * bundle,UHashtable * stringSet,UErrorCode & errorCode)440 SResource::preflightStrings(SRBRoot *bundle, UHashtable *stringSet, UErrorCode &errorCode) {
441     if (U_FAILURE(errorCode)) {
442         return;
443     }
444     if (fRes != RES_BOGUS) {
445         /*
446          * The resource item word was already precomputed, which means
447          * no further data needs to be written.
448          * This might be an integer, or an empty string/binary/etc.
449          */
450         return;
451     }
452     handlePreflightStrings(bundle, stringSet, errorCode);
453 }
454 
455 void
handlePreflightStrings(SRBRoot *,UHashtable *,UErrorCode &)456 SResource::handlePreflightStrings(SRBRoot * /*bundle*/, UHashtable * /*stringSet*/,
457                                   UErrorCode & /*errorCode*/) {
458     /* Neither a string nor a container. */
459 }
460 
461 int32_t
makeRes16(uint32_t resWord) const462 SRBRoot::makeRes16(uint32_t resWord) const {
463     if (resWord == 0) {
464         return 0;  /* empty string */
465     }
466     uint32_t type = RES_GET_TYPE(resWord);
467     int32_t offset = (int32_t)RES_GET_OFFSET(resWord);
468     if (type == URES_STRING_V2) {
469         assert(offset > 0);
470         if (offset < fPoolStringIndexLimit) {
471             if (offset < fPoolStringIndex16Limit) {
472                 return offset;
473             }
474         } else {
475             offset = offset - fPoolStringIndexLimit + fPoolStringIndex16Limit;
476             if (offset <= 0xffff) {
477                 return offset;
478             }
479         }
480     }
481     return -1;
482 }
483 
484 int32_t
mapKey(int32_t oldpos) const485 SRBRoot::mapKey(int32_t oldpos) const {
486     const KeyMapEntry *map = fKeyMap;
487     if (map == NULL) {
488         return oldpos;
489     }
490     int32_t i, start, limit;
491 
492     /* do a binary search for the old, pre-compactKeys() key offset */
493     start = fUsePoolBundle->fKeysCount;
494     limit = start + fKeysCount;
495     while (start < limit - 1) {
496         i = (start + limit) / 2;
497         if (oldpos < map[i].oldpos) {
498             limit = i;
499         } else {
500             start = i;
501         }
502     }
503     assert(oldpos == map[start].oldpos);
504     return map[start].newpos;
505 }
506 
507 /*
508  * Only called for UTF-16 v1 strings and duplicate UTF-16 v2 strings.
509  * For unique UTF-16 v2 strings, write16() sees fRes != RES_BOGUS
510  * and exits early.
511  */
512 void
handleWrite16(SRBRoot *)513 StringResource::handleWrite16(SRBRoot * /*bundle*/) {
514     SResource *same;
515     if ((same = fSame) != NULL) {
516         /* This is a duplicate. */
517         assert(same->fRes != RES_BOGUS && same->fWritten);
518         fRes = same->fRes;
519         fWritten = same->fWritten;
520     }
521 }
522 
523 void
writeAllRes16(SRBRoot * bundle)524 ContainerResource::writeAllRes16(SRBRoot *bundle) {
525     for (SResource *current = fFirst; current != NULL; current = current->fNext) {
526         bundle->f16BitUnits.append((UChar)current->fRes16);
527     }
528     fWritten = TRUE;
529 }
530 
531 void
handleWrite16(SRBRoot * bundle)532 ArrayResource::handleWrite16(SRBRoot *bundle) {
533     if (fCount == 0 && gFormatVersion > 1) {
534         fRes = URES_MAKE_EMPTY_RESOURCE(URES_ARRAY);
535         fWritten = TRUE;
536         return;
537     }
538 
539     int32_t res16 = 0;
540     for (SResource *current = fFirst; current != NULL; current = current->fNext) {
541         current->write16(bundle);
542         res16 |= current->fRes16;
543     }
544     if (fCount <= 0xffff && res16 >= 0 && gFormatVersion > 1) {
545         fRes = URES_MAKE_RESOURCE(URES_ARRAY16, bundle->f16BitUnits.length());
546         bundle->f16BitUnits.append((UChar)fCount);
547         writeAllRes16(bundle);
548     }
549 }
550 
551 void
handleWrite16(SRBRoot * bundle)552 TableResource::handleWrite16(SRBRoot *bundle) {
553     if (fCount == 0 && gFormatVersion > 1) {
554         fRes = URES_MAKE_EMPTY_RESOURCE(URES_TABLE);
555         fWritten = TRUE;
556         return;
557     }
558     /* Find the smallest table type that fits the data. */
559     int32_t key16 = 0;
560     int32_t res16 = 0;
561     for (SResource *current = fFirst; current != NULL; current = current->fNext) {
562         current->write16(bundle);
563         key16 |= current->fKey16;
564         res16 |= current->fRes16;
565     }
566     if(fCount > (uint32_t)bundle->fMaxTableLength) {
567         bundle->fMaxTableLength = fCount;
568     }
569     if (fCount <= 0xffff && key16 >= 0) {
570         if (res16 >= 0 && gFormatVersion > 1) {
571             /* 16-bit count, key offsets and values */
572             fRes = URES_MAKE_RESOURCE(URES_TABLE16, bundle->f16BitUnits.length());
573             bundle->f16BitUnits.append((UChar)fCount);
574             for (SResource *current = fFirst; current != NULL; current = current->fNext) {
575                 bundle->f16BitUnits.append((UChar)current->fKey16);
576             }
577             writeAllRes16(bundle);
578         } else {
579             /* 16-bit count, 16-bit key offsets, 32-bit values */
580             fTableType = URES_TABLE;
581         }
582     } else {
583         /* 32-bit count, key offsets and values */
584         fTableType = URES_TABLE32;
585     }
586 }
587 
588 void
handleWrite16(SRBRoot *)589 PseudoListResource::handleWrite16(SRBRoot * /*bundle*/) {
590     fRes = URES_MAKE_EMPTY_RESOURCE(URES_TABLE);
591     fWritten = TRUE;
592 }
593 
594 void
write16(SRBRoot * bundle)595 SResource::write16(SRBRoot *bundle) {
596     if (fKey >= 0) {
597         // A tagged resource has a non-negative key index into the parsed key strings.
598         // compactKeys() built a map from parsed key index to the final key index.
599         // After the mapping, negative key indexes are used for shared pool bundle keys.
600         fKey = bundle->mapKey(fKey);
601         // If the key index fits into a Key16 for a Table or Table16,
602         // then set the fKey16 field accordingly.
603         // Otherwise keep it at -1.
604         if (fKey >= 0) {
605             if (fKey < bundle->fLocalKeyLimit) {
606                 fKey16 = fKey;
607             }
608         } else {
609             int32_t poolKeyIndex = fKey & 0x7fffffff;
610             if (poolKeyIndex <= 0xffff) {
611                 poolKeyIndex += bundle->fLocalKeyLimit;
612                 if (poolKeyIndex <= 0xffff) {
613                     fKey16 = poolKeyIndex;
614                 }
615             }
616         }
617     }
618     /*
619      * fRes != RES_BOGUS:
620      * The resource item word was already precomputed, which means
621      * no further data needs to be written.
622      * This might be an integer, or an empty or UTF-16 v2 string,
623      * an empty binary, etc.
624      */
625     if (fRes == RES_BOGUS) {
626         handleWrite16(bundle);
627     }
628     // Compute fRes16 for precomputed as well as just-computed fRes.
629     fRes16 = bundle->makeRes16(fRes);
630 }
631 
632 void
handleWrite16(SRBRoot *)633 SResource::handleWrite16(SRBRoot * /*bundle*/) {
634     /* Only a few resource types write 16-bit units. */
635 }
636 
637 /*
638  * Only called for UTF-16 v1 strings, and for aliases.
639  * For UTF-16 v2 strings, preWrite() sees fRes != RES_BOGUS
640  * and exits early.
641  */
642 void
handlePreWrite(uint32_t * byteOffset)643 StringBaseResource::handlePreWrite(uint32_t *byteOffset) {
644     /* Write the UTF-16 v1 string. */
645     fRes = URES_MAKE_RESOURCE(fType, *byteOffset >> 2);
646     *byteOffset += 4 + (length() + 1) * U_SIZEOF_UCHAR;
647 }
648 
649 void
handlePreWrite(uint32_t * byteOffset)650 IntVectorResource::handlePreWrite(uint32_t *byteOffset) {
651     if (fCount == 0 && gFormatVersion > 1) {
652         fRes = URES_MAKE_EMPTY_RESOURCE(URES_INT_VECTOR);
653         fWritten = TRUE;
654     } else {
655         fRes = URES_MAKE_RESOURCE(URES_INT_VECTOR, *byteOffset >> 2);
656         *byteOffset += (1 + fCount) * 4;
657     }
658 }
659 
660 void
handlePreWrite(uint32_t * byteOffset)661 BinaryResource::handlePreWrite(uint32_t *byteOffset) {
662     uint32_t pad       = 0;
663     uint32_t dataStart = *byteOffset + sizeof(fLength);
664 
665     if (dataStart % BIN_ALIGNMENT) {
666         pad = (BIN_ALIGNMENT - dataStart % BIN_ALIGNMENT);
667         *byteOffset += pad;  /* pad == 4 or 8 or 12 */
668     }
669     fRes = URES_MAKE_RESOURCE(URES_BINARY, *byteOffset >> 2);
670     *byteOffset += 4 + fLength;
671 }
672 
673 void
preWriteAllRes(uint32_t * byteOffset)674 ContainerResource::preWriteAllRes(uint32_t *byteOffset) {
675     for (SResource *current = fFirst; current != NULL; current = current->fNext) {
676         current->preWrite(byteOffset);
677     }
678 }
679 
680 void
handlePreWrite(uint32_t * byteOffset)681 ArrayResource::handlePreWrite(uint32_t *byteOffset) {
682     preWriteAllRes(byteOffset);
683     fRes = URES_MAKE_RESOURCE(URES_ARRAY, *byteOffset >> 2);
684     *byteOffset += (1 + fCount) * 4;
685 }
686 
687 void
handlePreWrite(uint32_t * byteOffset)688 TableResource::handlePreWrite(uint32_t *byteOffset) {
689     preWriteAllRes(byteOffset);
690     if (fTableType == URES_TABLE) {
691         /* 16-bit count, 16-bit key offsets, 32-bit values */
692         fRes = URES_MAKE_RESOURCE(URES_TABLE, *byteOffset >> 2);
693         *byteOffset += 2 + fCount * 6;
694     } else {
695         /* 32-bit count, key offsets and values */
696         fRes = URES_MAKE_RESOURCE(URES_TABLE32, *byteOffset >> 2);
697         *byteOffset += 4 + fCount * 8;
698     }
699 }
700 
701 void
preWrite(uint32_t * byteOffset)702 SResource::preWrite(uint32_t *byteOffset) {
703     if (fRes != RES_BOGUS) {
704         /*
705          * The resource item word was already precomputed, which means
706          * no further data needs to be written.
707          * This might be an integer, or an empty or UTF-16 v2 string,
708          * an empty binary, etc.
709          */
710         return;
711     }
712     handlePreWrite(byteOffset);
713     *byteOffset += calcPadding(*byteOffset);
714 }
715 
716 void
handlePreWrite(uint32_t *)717 SResource::handlePreWrite(uint32_t * /*byteOffset*/) {
718     assert(FALSE);
719 }
720 
721 /*
722  * Only called for UTF-16 v1 strings, and for aliases. For UTF-16 v2 strings,
723  * write() sees fWritten and exits early.
724  */
725 void
handleWrite(UNewDataMemory * mem,uint32_t * byteOffset)726 StringBaseResource::handleWrite(UNewDataMemory *mem, uint32_t *byteOffset) {
727     /* Write the UTF-16 v1 string. */
728     int32_t len = length();
729     udata_write32(mem, len);
730     udata_writeUString(mem, getBuffer(), len + 1);
731     *byteOffset += 4 + (len + 1) * U_SIZEOF_UCHAR;
732     fWritten = TRUE;
733 }
734 
735 void
writeAllRes(UNewDataMemory * mem,uint32_t * byteOffset)736 ContainerResource::writeAllRes(UNewDataMemory *mem, uint32_t *byteOffset) {
737     uint32_t i = 0;
738     for (SResource *current = fFirst; current != NULL; ++i, current = current->fNext) {
739         current->write(mem, byteOffset);
740     }
741     assert(i == fCount);
742 }
743 
744 void
writeAllRes32(UNewDataMemory * mem,uint32_t * byteOffset)745 ContainerResource::writeAllRes32(UNewDataMemory *mem, uint32_t *byteOffset) {
746     for (SResource *current = fFirst; current != NULL; current = current->fNext) {
747         udata_write32(mem, current->fRes);
748     }
749     *byteOffset += fCount * 4;
750 }
751 
752 void
handleWrite(UNewDataMemory * mem,uint32_t * byteOffset)753 ArrayResource::handleWrite(UNewDataMemory *mem, uint32_t *byteOffset) {
754     writeAllRes(mem, byteOffset);
755     udata_write32(mem, fCount);
756     *byteOffset += 4;
757     writeAllRes32(mem, byteOffset);
758 }
759 
760 void
handleWrite(UNewDataMemory * mem,uint32_t * byteOffset)761 IntVectorResource::handleWrite(UNewDataMemory *mem, uint32_t *byteOffset) {
762     udata_write32(mem, fCount);
763     for(uint32_t i = 0; i < fCount; ++i) {
764       udata_write32(mem, fArray[i]);
765     }
766     *byteOffset += (1 + fCount) * 4;
767 }
768 
769 void
handleWrite(UNewDataMemory * mem,uint32_t * byteOffset)770 BinaryResource::handleWrite(UNewDataMemory *mem, uint32_t *byteOffset) {
771     uint32_t pad       = 0;
772     uint32_t dataStart = *byteOffset + sizeof(fLength);
773 
774     if (dataStart % BIN_ALIGNMENT) {
775         pad = (BIN_ALIGNMENT - dataStart % BIN_ALIGNMENT);
776         udata_writePadding(mem, pad);  /* pad == 4 or 8 or 12 */
777         *byteOffset += pad;
778     }
779 
780     udata_write32(mem, fLength);
781     if (fLength > 0) {
782         udata_writeBlock(mem, fData, fLength);
783     }
784     *byteOffset += 4 + fLength;
785 }
786 
787 void
handleWrite(UNewDataMemory * mem,uint32_t * byteOffset)788 TableResource::handleWrite(UNewDataMemory *mem, uint32_t *byteOffset) {
789     writeAllRes(mem, byteOffset);
790     if(fTableType == URES_TABLE) {
791         udata_write16(mem, (uint16_t)fCount);
792         for (SResource *current = fFirst; current != NULL; current = current->fNext) {
793             udata_write16(mem, current->fKey16);
794         }
795         *byteOffset += (1 + fCount)* 2;
796         if ((fCount & 1) == 0) {
797             /* 16-bit count and even number of 16-bit key offsets need padding before 32-bit resource items */
798             udata_writePadding(mem, 2);
799             *byteOffset += 2;
800         }
801     } else /* URES_TABLE32 */ {
802         udata_write32(mem, fCount);
803         for (SResource *current = fFirst; current != NULL; current = current->fNext) {
804             udata_write32(mem, (uint32_t)current->fKey);
805         }
806         *byteOffset += (1 + fCount)* 4;
807     }
808     writeAllRes32(mem, byteOffset);
809 }
810 
811 void
write(UNewDataMemory * mem,uint32_t * byteOffset)812 SResource::write(UNewDataMemory *mem, uint32_t *byteOffset) {
813     if (fWritten) {
814         assert(fRes != RES_BOGUS);
815         return;
816     }
817     handleWrite(mem, byteOffset);
818     uint8_t paddingSize = calcPadding(*byteOffset);
819     if (paddingSize > 0) {
820         udata_writePadding(mem, paddingSize);
821         *byteOffset += paddingSize;
822     }
823     fWritten = TRUE;
824 }
825 
826 void
handleWrite(UNewDataMemory *,uint32_t *)827 SResource::handleWrite(UNewDataMemory * /*mem*/, uint32_t * /*byteOffset*/) {
828     assert(FALSE);
829 }
830 
write(const char * outputDir,const char * outputPkg,char * writtenFilename,int writtenFilenameLen,UErrorCode & errorCode)831 void SRBRoot::write(const char *outputDir, const char *outputPkg,
832                     char *writtenFilename, int writtenFilenameLen,
833                     UErrorCode &errorCode) {
834     UNewDataMemory *mem        = NULL;
835     uint32_t        byteOffset = 0;
836     uint32_t        top, size;
837     char            dataName[1024];
838     int32_t         indexes[URES_INDEX_TOP];
839 
840     compactKeys(errorCode);
841     /*
842      * Add padding bytes to fKeys so that fKeysTop is 4-aligned.
843      * Safe because the capacity is a multiple of 4.
844      */
845     while (fKeysTop & 3) {
846         fKeys[fKeysTop++] = (char)0xaa;
847     }
848     /*
849      * In URES_TABLE, use all local key offsets that fit into 16 bits,
850      * and use the remaining 16-bit offsets for pool key offsets
851      * if there are any.
852      * If there are no local keys, then use the whole 16-bit space
853      * for pool key offsets.
854      * Note: This cannot be changed without changing the major formatVersion.
855      */
856     if (fKeysBottom < fKeysTop) {
857         if (fKeysTop <= 0x10000) {
858             fLocalKeyLimit = fKeysTop;
859         } else {
860             fLocalKeyLimit = 0x10000;
861         }
862     } else {
863         fLocalKeyLimit = 0;
864     }
865 
866     UHashtable *stringSet;
867     if (gFormatVersion > 1) {
868         stringSet = uhash_open(string_hash, string_comp, string_comp, &errorCode);
869         if (U_SUCCESS(errorCode) &&
870                 fUsePoolBundle != NULL && fUsePoolBundle->fStrings != NULL) {
871             for (SResource *current = fUsePoolBundle->fStrings->fFirst;
872                     current != NULL;
873                     current = current->fNext) {
874                 StringResource *sr = static_cast<StringResource *>(current);
875                 sr->fNumCopies = 0;
876                 sr->fNumUnitsSaved = 0;
877                 uhash_put(stringSet, sr, sr, &errorCode);
878             }
879         }
880         fRoot->preflightStrings(this, stringSet, errorCode);
881     } else {
882         stringSet = NULL;
883     }
884     if (fStringsForm == STRINGS_UTF16_V2 && f16BitStringsLength > 0) {
885         compactStringsV2(stringSet, errorCode);
886     }
887     uhash_close(stringSet);
888     if (U_FAILURE(errorCode)) {
889         return;
890     }
891 
892     int32_t formatVersion = gFormatVersion;
893     if (fPoolStringIndexLimit != 0) {
894         int32_t sum = fPoolStringIndexLimit + fLocalStringIndexLimit;
895         if ((sum - 1) > RES_MAX_OFFSET) {
896             errorCode = U_BUFFER_OVERFLOW_ERROR;
897             return;
898         }
899         if (fPoolStringIndexLimit < 0x10000 && sum <= 0x10000) {
900             // 16-bit indexes work for all pool + local strings.
901             fPoolStringIndex16Limit = fPoolStringIndexLimit;
902         } else {
903             // Set the pool index threshold so that 16-bit indexes work
904             // for some pool strings and some local strings.
905             fPoolStringIndex16Limit = (int32_t)(
906                     ((int64_t)fPoolStringIndexLimit * 0xffff) / sum);
907         }
908     } else if (gIsDefaultFormatVersion && formatVersion == 3 && !fIsPoolBundle) {
909         // If we just default to formatVersion 3
910         // but there are no pool bundle strings to share
911         // and we do not write a pool bundle,
912         // then write formatVersion 2 which is just as good.
913         formatVersion = 2;
914     }
915 
916     fRoot->write16(this);
917     if (f16BitUnits.isBogus()) {
918         errorCode = U_MEMORY_ALLOCATION_ERROR;
919         return;
920     }
921     if (f16BitUnits.length() & 1) {
922         f16BitUnits.append((UChar)0xaaaa);  /* pad to multiple of 4 bytes */
923     }
924     /* all keys have been mapped */
925     uprv_free(fKeyMap);
926     fKeyMap = NULL;
927 
928     byteOffset = fKeysTop + f16BitUnits.length() * 2;
929     fRoot->preWrite(&byteOffset);
930 
931     /* total size including the root item */
932     top = byteOffset;
933 
934     if (writtenFilename && writtenFilenameLen) {
935         *writtenFilename = 0;
936     }
937 
938     if (writtenFilename) {
939        int32_t off = 0, len = 0;
940        if (outputDir) {
941            len = (int32_t)uprv_strlen(outputDir);
942            if (len > writtenFilenameLen) {
943                len = writtenFilenameLen;
944            }
945            uprv_strncpy(writtenFilename, outputDir, len);
946        }
947        if (writtenFilenameLen -= len) {
948            off += len;
949            writtenFilename[off] = U_FILE_SEP_CHAR;
950            if (--writtenFilenameLen) {
951                ++off;
952                if(outputPkg != NULL)
953                {
954                    uprv_strcpy(writtenFilename+off, outputPkg);
955                    off += (int32_t)uprv_strlen(outputPkg);
956                    writtenFilename[off] = '_';
957                    ++off;
958                }
959 
960                len = (int32_t)uprv_strlen(fLocale);
961                if (len > writtenFilenameLen) {
962                    len = writtenFilenameLen;
963                }
964                uprv_strncpy(writtenFilename + off, fLocale, len);
965                if (writtenFilenameLen -= len) {
966                    off += len;
967                    len = 5;
968                    if (len > writtenFilenameLen) {
969                        len = writtenFilenameLen;
970                    }
971                    uprv_strncpy(writtenFilename +  off, ".res", len);
972                }
973            }
974        }
975     }
976 
977     if(outputPkg)
978     {
979         uprv_strcpy(dataName, outputPkg);
980         uprv_strcat(dataName, "_");
981         uprv_strcat(dataName, fLocale);
982     }
983     else
984     {
985         uprv_strcpy(dataName, fLocale);
986     }
987 
988     uprv_memcpy(dataInfo.formatVersion, gFormatVersions + formatVersion, sizeof(UVersionInfo));
989 
990     mem = udata_create(outputDir, "res", dataName,
991                        &dataInfo, (gIncludeCopyright==TRUE)? U_COPYRIGHT_STRING:NULL, &errorCode);
992     if(U_FAILURE(errorCode)){
993         return;
994     }
995 
996     /* write the root item */
997     udata_write32(mem, fRoot->fRes);
998 
999     /*
1000      * formatVersion 1.1 (ICU 2.8):
1001      * write int32_t indexes[] after root and before the key strings
1002      * to make it easier to parse resource bundles in icuswap or from Java etc.
1003      */
1004     uprv_memset(indexes, 0, sizeof(indexes));
1005     indexes[URES_INDEX_LENGTH]=             fIndexLength;
1006     indexes[URES_INDEX_KEYS_TOP]=           fKeysTop>>2;
1007     indexes[URES_INDEX_RESOURCES_TOP]=      (int32_t)(top>>2);
1008     indexes[URES_INDEX_BUNDLE_TOP]=         indexes[URES_INDEX_RESOURCES_TOP];
1009     indexes[URES_INDEX_MAX_TABLE_LENGTH]=   fMaxTableLength;
1010 
1011     /*
1012      * formatVersion 1.2 (ICU 3.6):
1013      * write indexes[URES_INDEX_ATTRIBUTES] with URES_ATT_NO_FALLBACK set or not set
1014      * the memset() above initialized all indexes[] to 0
1015      */
1016     if (fNoFallback) {
1017         indexes[URES_INDEX_ATTRIBUTES]=URES_ATT_NO_FALLBACK;
1018     }
1019     /*
1020      * formatVersion 2.0 (ICU 4.4):
1021      * more compact string value storage, optional pool bundle
1022      */
1023     if (URES_INDEX_16BIT_TOP < fIndexLength) {
1024         indexes[URES_INDEX_16BIT_TOP] = (fKeysTop>>2) + (f16BitUnits.length()>>1);
1025     }
1026     if (URES_INDEX_POOL_CHECKSUM < fIndexLength) {
1027         if (fIsPoolBundle) {
1028             indexes[URES_INDEX_ATTRIBUTES] |= URES_ATT_IS_POOL_BUNDLE | URES_ATT_NO_FALLBACK;
1029             uint32_t checksum = computeCRC((const char *)(fKeys + fKeysBottom),
1030                                            (uint32_t)(fKeysTop - fKeysBottom), 0);
1031             if (f16BitUnits.length() <= 1) {
1032                 // no pool strings to checksum
1033             } else if (U_IS_BIG_ENDIAN) {
1034                 checksum = computeCRC((const char *)f16BitUnits.getBuffer(),
1035                                       (uint32_t)f16BitUnits.length() * 2, checksum);
1036             } else {
1037                 // Swap to big-endian so we get the same checksum on all platforms
1038                 // (except for charset family, due to the key strings).
1039                 UnicodeString s(f16BitUnits);
1040                 s.append((UChar)1);  // Ensure that we own this buffer.
1041                 assert(!s.isBogus());
1042                 uint16_t *p = (uint16_t *)s.getBuffer();
1043                 for (int32_t count = f16BitUnits.length(); count > 0; --count) {
1044                     uint16_t x = *p;
1045                     *p++ = (uint16_t)((x << 8) | (x >> 8));
1046                 }
1047                 checksum = computeCRC((const char *)p,
1048                                       (uint32_t)f16BitUnits.length() * 2, checksum);
1049             }
1050             indexes[URES_INDEX_POOL_CHECKSUM] = (int32_t)checksum;
1051         } else if (gUsePoolBundle) {
1052             indexes[URES_INDEX_ATTRIBUTES] |= URES_ATT_USES_POOL_BUNDLE;
1053             indexes[URES_INDEX_POOL_CHECKSUM] = fUsePoolBundle->fChecksum;
1054         }
1055     }
1056     // formatVersion 3 (ICU 56):
1057     // share string values via pool bundle strings
1058     indexes[URES_INDEX_LENGTH] |= fPoolStringIndexLimit << 8;  // bits 23..0 -> 31..8
1059     indexes[URES_INDEX_ATTRIBUTES] |= (fPoolStringIndexLimit >> 12) & 0xf000;  // bits 27..24 -> 15..12
1060     indexes[URES_INDEX_ATTRIBUTES] |= fPoolStringIndex16Limit << 16;
1061 
1062     /* write the indexes[] */
1063     udata_writeBlock(mem, indexes, fIndexLength*4);
1064 
1065     /* write the table key strings */
1066     udata_writeBlock(mem, fKeys+fKeysBottom,
1067                           fKeysTop-fKeysBottom);
1068 
1069     /* write the v2 UTF-16 strings, URES_TABLE16 and URES_ARRAY16 */
1070     udata_writeBlock(mem, f16BitUnits.getBuffer(), f16BitUnits.length()*2);
1071 
1072     /* write all of the bundle contents: the root item and its children */
1073     byteOffset = fKeysTop + f16BitUnits.length() * 2;
1074     fRoot->write(mem, &byteOffset);
1075     assert(byteOffset == top);
1076 
1077     size = udata_finish(mem, &errorCode);
1078     if(top != size) {
1079         fprintf(stderr, "genrb error: wrote %u bytes but counted %u\n",
1080                 (int)size, (int)top);
1081         errorCode = U_INTERNAL_PROGRAM_ERROR;
1082     }
1083 }
1084 
1085 /* Opening Functions */
1086 
table_open(struct SRBRoot * bundle,const char * tag,const struct UString * comment,UErrorCode * status)1087 TableResource* table_open(struct SRBRoot *bundle, const char *tag, const struct UString* comment, UErrorCode *status) {
1088     LocalPointer<TableResource> res(new TableResource(bundle, tag, comment, *status), *status);
1089     return U_SUCCESS(*status) ? res.orphan() : NULL;
1090 }
1091 
array_open(struct SRBRoot * bundle,const char * tag,const struct UString * comment,UErrorCode * status)1092 ArrayResource* array_open(struct SRBRoot *bundle, const char *tag, const struct UString* comment, UErrorCode *status) {
1093     LocalPointer<ArrayResource> res(new ArrayResource(bundle, tag, comment, *status), *status);
1094     return U_SUCCESS(*status) ? res.orphan() : NULL;
1095 }
1096 
string_open(struct SRBRoot * bundle,const char * tag,const UChar * value,int32_t len,const struct UString * comment,UErrorCode * status)1097 struct SResource *string_open(struct SRBRoot *bundle, const char *tag, const UChar *value, int32_t len, const struct UString* comment, UErrorCode *status) {
1098     LocalPointer<SResource> res(
1099             new StringResource(bundle, tag, value, len, comment, *status), *status);
1100     return U_SUCCESS(*status) ? res.orphan() : NULL;
1101 }
1102 
alias_open(struct SRBRoot * bundle,const char * tag,UChar * value,int32_t len,const struct UString * comment,UErrorCode * status)1103 struct SResource *alias_open(struct SRBRoot *bundle, const char *tag, UChar *value, int32_t len, const struct UString* comment, UErrorCode *status) {
1104     LocalPointer<SResource> res(
1105             new AliasResource(bundle, tag, value, len, comment, *status), *status);
1106     return U_SUCCESS(*status) ? res.orphan() : NULL;
1107 }
1108 
intvector_open(struct SRBRoot * bundle,const char * tag,const struct UString * comment,UErrorCode * status)1109 IntVectorResource *intvector_open(struct SRBRoot *bundle, const char *tag, const struct UString* comment, UErrorCode *status) {
1110     LocalPointer<IntVectorResource> res(
1111             new IntVectorResource(bundle, tag, comment, *status), *status);
1112     return U_SUCCESS(*status) ? res.orphan() : NULL;
1113 }
1114 
int_open(struct SRBRoot * bundle,const char * tag,int32_t value,const struct UString * comment,UErrorCode * status)1115 struct SResource *int_open(struct SRBRoot *bundle, const char *tag, int32_t value, const struct UString* comment, UErrorCode *status) {
1116     LocalPointer<SResource> res(new IntResource(bundle, tag, value, comment, *status), *status);
1117     return U_SUCCESS(*status) ? res.orphan() : NULL;
1118 }
1119 
bin_open(struct SRBRoot * bundle,const char * tag,uint32_t length,uint8_t * data,const char * fileName,const struct UString * comment,UErrorCode * status)1120 struct SResource *bin_open(struct SRBRoot *bundle, const char *tag, uint32_t length, uint8_t *data, const char* fileName, const struct UString* comment, UErrorCode *status) {
1121     LocalPointer<SResource> res(
1122             new BinaryResource(bundle, tag, length, data, fileName, comment, *status), *status);
1123     return U_SUCCESS(*status) ? res.orphan() : NULL;
1124 }
1125 
SRBRoot(const UString * comment,UBool isPoolBundle,UErrorCode & errorCode)1126 SRBRoot::SRBRoot(const UString *comment, UBool isPoolBundle, UErrorCode &errorCode)
1127         : fRoot(NULL), fLocale(NULL), fIndexLength(0), fMaxTableLength(0), fNoFallback(FALSE),
1128           fStringsForm(STRINGS_UTF16_V1), fIsPoolBundle(isPoolBundle),
1129           fKeys(NULL), fKeyMap(NULL),
1130           fKeysBottom(0), fKeysTop(0), fKeysCapacity(0), fKeysCount(0), fLocalKeyLimit(0),
1131           f16BitUnits(), f16BitStringsLength(0),
1132           fUsePoolBundle(&kNoPoolBundle),
1133           fPoolStringIndexLimit(0), fPoolStringIndex16Limit(0), fLocalStringIndexLimit(0),
1134           fWritePoolBundle(NULL) {
1135     if (U_FAILURE(errorCode)) {
1136         return;
1137     }
1138 
1139     if (gFormatVersion > 1) {
1140         // f16BitUnits must start with a zero for empty resources.
1141         // We might be able to omit it if there are no empty 16-bit resources.
1142         f16BitUnits.append((UChar)0);
1143     }
1144 
1145     fKeys = (char *) uprv_malloc(sizeof(char) * KEY_SPACE_SIZE);
1146     if (isPoolBundle) {
1147         fRoot = new PseudoListResource(this, errorCode);
1148     } else {
1149         fRoot = new TableResource(this, NULL, comment, errorCode);
1150     }
1151     if (fKeys == NULL || fRoot == NULL || U_FAILURE(errorCode)) {
1152         if (U_SUCCESS(errorCode)) {
1153             errorCode = U_MEMORY_ALLOCATION_ERROR;
1154         }
1155         return;
1156     }
1157 
1158     fKeysCapacity = KEY_SPACE_SIZE;
1159     /* formatVersion 1.1 and up: start fKeysTop after the root item and indexes[] */
1160     if (gUsePoolBundle || isPoolBundle) {
1161         fIndexLength = URES_INDEX_POOL_CHECKSUM + 1;
1162     } else if (gFormatVersion >= 2) {
1163         fIndexLength = URES_INDEX_16BIT_TOP + 1;
1164     } else /* formatVersion 1 */ {
1165         fIndexLength = URES_INDEX_ATTRIBUTES + 1;
1166     }
1167     fKeysBottom = (1 /* root */ + fIndexLength) * 4;
1168     uprv_memset(fKeys, 0, fKeysBottom);
1169     fKeysTop = fKeysBottom;
1170 
1171     if (gFormatVersion == 1) {
1172         fStringsForm = STRINGS_UTF16_V1;
1173     } else {
1174         fStringsForm = STRINGS_UTF16_V2;
1175     }
1176 }
1177 
1178 /* Closing Functions */
1179 
res_close(struct SResource * res)1180 void res_close(struct SResource *res) {
1181     delete res;
1182 }
1183 
~SRBRoot()1184 SRBRoot::~SRBRoot() {
1185     delete fRoot;
1186     uprv_free(fLocale);
1187     uprv_free(fKeys);
1188     uprv_free(fKeyMap);
1189 }
1190 
1191 /* Misc Functions */
1192 
setLocale(UChar * locale,UErrorCode & errorCode)1193 void SRBRoot::setLocale(UChar *locale, UErrorCode &errorCode) {
1194     if(U_FAILURE(errorCode)) {
1195         return;
1196     }
1197 
1198     uprv_free(fLocale);
1199     fLocale = (char*) uprv_malloc(sizeof(char) * (u_strlen(locale)+1));
1200     if(fLocale == NULL) {
1201         errorCode = U_MEMORY_ALLOCATION_ERROR;
1202         return;
1203     }
1204 
1205     u_UCharsToChars(locale, fLocale, u_strlen(locale)+1);
1206 }
1207 
1208 const char *
getKeyString(int32_t key) const1209 SRBRoot::getKeyString(int32_t key) const {
1210     if (key < 0) {
1211         return fUsePoolBundle->fKeys + (key & 0x7fffffff);
1212     } else {
1213         return fKeys + key;
1214     }
1215 }
1216 
1217 const char *
getKeyString(const SRBRoot * bundle) const1218 SResource::getKeyString(const SRBRoot *bundle) const {
1219     if (fKey == -1) {
1220         return NULL;
1221     }
1222     return bundle->getKeyString(fKey);
1223 }
1224 
1225 const char *
getKeyBytes(int32_t * pLength) const1226 SRBRoot::getKeyBytes(int32_t *pLength) const {
1227     *pLength = fKeysTop - fKeysBottom;
1228     return fKeys + fKeysBottom;
1229 }
1230 
1231 int32_t
addKeyBytes(const char * keyBytes,int32_t length,UErrorCode & errorCode)1232 SRBRoot::addKeyBytes(const char *keyBytes, int32_t length, UErrorCode &errorCode) {
1233     int32_t keypos;
1234 
1235     if (U_FAILURE(errorCode)) {
1236         return -1;
1237     }
1238     if (length < 0 || (keyBytes == NULL && length != 0)) {
1239         errorCode = U_ILLEGAL_ARGUMENT_ERROR;
1240         return -1;
1241     }
1242     if (length == 0) {
1243         return fKeysTop;
1244     }
1245 
1246     keypos = fKeysTop;
1247     fKeysTop += length;
1248     if (fKeysTop >= fKeysCapacity) {
1249         /* overflow - resize the keys buffer */
1250         fKeysCapacity += KEY_SPACE_SIZE;
1251         fKeys = static_cast<char *>(uprv_realloc(fKeys, fKeysCapacity));
1252         if(fKeys == NULL) {
1253             errorCode = U_MEMORY_ALLOCATION_ERROR;
1254             return -1;
1255         }
1256     }
1257 
1258     uprv_memcpy(fKeys + keypos, keyBytes, length);
1259 
1260     return keypos;
1261 }
1262 
1263 int32_t
addTag(const char * tag,UErrorCode & errorCode)1264 SRBRoot::addTag(const char *tag, UErrorCode &errorCode) {
1265     int32_t keypos;
1266 
1267     if (U_FAILURE(errorCode)) {
1268         return -1;
1269     }
1270 
1271     if (tag == NULL) {
1272         /* no error: the root table and array items have no keys */
1273         return -1;
1274     }
1275 
1276     keypos = addKeyBytes(tag, (int32_t)(uprv_strlen(tag) + 1), errorCode);
1277     if (U_SUCCESS(errorCode)) {
1278         ++fKeysCount;
1279     }
1280     return keypos;
1281 }
1282 
1283 static int32_t
compareInt32(int32_t lPos,int32_t rPos)1284 compareInt32(int32_t lPos, int32_t rPos) {
1285     /*
1286      * Compare possibly-negative key offsets. Don't just return lPos - rPos
1287      * because that is prone to negative-integer underflows.
1288      */
1289     if (lPos < rPos) {
1290         return -1;
1291     } else if (lPos > rPos) {
1292         return 1;
1293     } else {
1294         return 0;
1295     }
1296 }
1297 
1298 static int32_t U_CALLCONV
compareKeySuffixes(const void * context,const void * l,const void * r)1299 compareKeySuffixes(const void *context, const void *l, const void *r) {
1300     const struct SRBRoot *bundle=(const struct SRBRoot *)context;
1301     int32_t lPos = ((const KeyMapEntry *)l)->oldpos;
1302     int32_t rPos = ((const KeyMapEntry *)r)->oldpos;
1303     const char *lStart = bundle->getKeyString(lPos);
1304     const char *lLimit = lStart;
1305     const char *rStart = bundle->getKeyString(rPos);
1306     const char *rLimit = rStart;
1307     int32_t diff;
1308     while (*lLimit != 0) { ++lLimit; }
1309     while (*rLimit != 0) { ++rLimit; }
1310     /* compare keys in reverse character order */
1311     while (lStart < lLimit && rStart < rLimit) {
1312         diff = (int32_t)(uint8_t)*--lLimit - (int32_t)(uint8_t)*--rLimit;
1313         if (diff != 0) {
1314             return diff;
1315         }
1316     }
1317     /* sort equal suffixes by descending key length */
1318     diff = (int32_t)(rLimit - rStart) - (int32_t)(lLimit - lStart);
1319     if (diff != 0) {
1320         return diff;
1321     }
1322     /* Sort pool bundle keys first (negative oldpos), and otherwise keys in parsing order. */
1323     return compareInt32(lPos, rPos);
1324 }
1325 
1326 static int32_t U_CALLCONV
compareKeyNewpos(const void *,const void * l,const void * r)1327 compareKeyNewpos(const void * /*context*/, const void *l, const void *r) {
1328     return compareInt32(((const KeyMapEntry *)l)->newpos, ((const KeyMapEntry *)r)->newpos);
1329 }
1330 
1331 static int32_t U_CALLCONV
compareKeyOldpos(const void *,const void * l,const void * r)1332 compareKeyOldpos(const void * /*context*/, const void *l, const void *r) {
1333     return compareInt32(((const KeyMapEntry *)l)->oldpos, ((const KeyMapEntry *)r)->oldpos);
1334 }
1335 
1336 void
compactKeys(UErrorCode & errorCode)1337 SRBRoot::compactKeys(UErrorCode &errorCode) {
1338     KeyMapEntry *map;
1339     char *keys;
1340     int32_t i;
1341     int32_t keysCount = fUsePoolBundle->fKeysCount + fKeysCount;
1342     if (U_FAILURE(errorCode) || fKeysCount == 0 || fKeyMap != NULL) {
1343         return;
1344     }
1345     map = (KeyMapEntry *)uprv_malloc(keysCount * sizeof(KeyMapEntry));
1346     if (map == NULL) {
1347         errorCode = U_MEMORY_ALLOCATION_ERROR;
1348         return;
1349     }
1350     keys = (char *)fUsePoolBundle->fKeys;
1351     for (i = 0; i < fUsePoolBundle->fKeysCount; ++i) {
1352         map[i].oldpos =
1353             (int32_t)(keys - fUsePoolBundle->fKeys) | 0x80000000;  /* negative oldpos */
1354         map[i].newpos = 0;
1355         while (*keys != 0) { ++keys; }  /* skip the key */
1356         ++keys;  /* skip the NUL */
1357     }
1358     keys = fKeys + fKeysBottom;
1359     for (; i < keysCount; ++i) {
1360         map[i].oldpos = (int32_t)(keys - fKeys);
1361         map[i].newpos = 0;
1362         while (*keys != 0) { ++keys; }  /* skip the key */
1363         ++keys;  /* skip the NUL */
1364     }
1365     /* Sort the keys so that each one is immediately followed by all of its suffixes. */
1366     uprv_sortArray(map, keysCount, (int32_t)sizeof(KeyMapEntry),
1367                    compareKeySuffixes, this, FALSE, &errorCode);
1368     /*
1369      * Make suffixes point into earlier, longer strings that contain them
1370      * and mark the old, now unused suffix bytes as deleted.
1371      */
1372     if (U_SUCCESS(errorCode)) {
1373         keys = fKeys;
1374         for (i = 0; i < keysCount;) {
1375             /*
1376              * This key is not a suffix of the previous one;
1377              * keep this one and delete the following ones that are
1378              * suffixes of this one.
1379              */
1380             const char *key;
1381             const char *keyLimit;
1382             int32_t j = i + 1;
1383             map[i].newpos = map[i].oldpos;
1384             if (j < keysCount && map[j].oldpos < 0) {
1385                 /* Key string from the pool bundle, do not delete. */
1386                 i = j;
1387                 continue;
1388             }
1389             key = getKeyString(map[i].oldpos);
1390             for (keyLimit = key; *keyLimit != 0; ++keyLimit) {}
1391             for (; j < keysCount && map[j].oldpos >= 0; ++j) {
1392                 const char *k;
1393                 char *suffix;
1394                 const char *suffixLimit;
1395                 int32_t offset;
1396                 suffix = keys + map[j].oldpos;
1397                 for (suffixLimit = suffix; *suffixLimit != 0; ++suffixLimit) {}
1398                 offset = (int32_t)(keyLimit - key) - (suffixLimit - suffix);
1399                 if (offset < 0) {
1400                     break;  /* suffix cannot be longer than the original */
1401                 }
1402                 /* Is it a suffix of the earlier, longer key? */
1403                 for (k = keyLimit; suffix < suffixLimit && *--k == *--suffixLimit;) {}
1404                 if (suffix == suffixLimit && *k == *suffixLimit) {
1405                     map[j].newpos = map[i].oldpos + offset;  /* yes, point to the earlier key */
1406                     /* mark the suffix as deleted */
1407                     while (*suffix != 0) { *suffix++ = 1; }
1408                     *suffix = 1;
1409                 } else {
1410                     break;  /* not a suffix, restart from here */
1411                 }
1412             }
1413             i = j;
1414         }
1415         /*
1416          * Re-sort by newpos, then modify the key characters array in-place
1417          * to squeeze out unused bytes, and readjust the newpos offsets.
1418          */
1419         uprv_sortArray(map, keysCount, (int32_t)sizeof(KeyMapEntry),
1420                        compareKeyNewpos, NULL, FALSE, &errorCode);
1421         if (U_SUCCESS(errorCode)) {
1422             int32_t oldpos, newpos, limit;
1423             oldpos = newpos = fKeysBottom;
1424             limit = fKeysTop;
1425             /* skip key offsets that point into the pool bundle rather than this new bundle */
1426             for (i = 0; i < keysCount && map[i].newpos < 0; ++i) {}
1427             if (i < keysCount) {
1428                 while (oldpos < limit) {
1429                     if (keys[oldpos] == 1) {
1430                         ++oldpos;  /* skip unused bytes */
1431                     } else {
1432                         /* adjust the new offsets for keys starting here */
1433                         while (i < keysCount && map[i].newpos == oldpos) {
1434                             map[i++].newpos = newpos;
1435                         }
1436                         /* move the key characters to their new position */
1437                         keys[newpos++] = keys[oldpos++];
1438                     }
1439                 }
1440                 assert(i == keysCount);
1441             }
1442             fKeysTop = newpos;
1443             /* Re-sort once more, by old offsets for binary searching. */
1444             uprv_sortArray(map, keysCount, (int32_t)sizeof(KeyMapEntry),
1445                            compareKeyOldpos, NULL, FALSE, &errorCode);
1446             if (U_SUCCESS(errorCode)) {
1447                 /* key size reduction by limit - newpos */
1448                 fKeyMap = map;
1449                 map = NULL;
1450             }
1451         }
1452     }
1453     uprv_free(map);
1454 }
1455 
1456 static int32_t U_CALLCONV
compareStringSuffixes(const void *,const void * l,const void * r)1457 compareStringSuffixes(const void * /*context*/, const void *l, const void *r) {
1458     const StringResource *left = *((const StringResource **)l);
1459     const StringResource *right = *((const StringResource **)r);
1460     const UChar *lStart = left->getBuffer();
1461     const UChar *lLimit = lStart + left->length();
1462     const UChar *rStart = right->getBuffer();
1463     const UChar *rLimit = rStart + right->length();
1464     int32_t diff;
1465     /* compare keys in reverse character order */
1466     while (lStart < lLimit && rStart < rLimit) {
1467         diff = (int32_t)*--lLimit - (int32_t)*--rLimit;
1468         if (diff != 0) {
1469             return diff;
1470         }
1471     }
1472     /* sort equal suffixes by descending string length */
1473     return right->length() - left->length();
1474 }
1475 
1476 static int32_t U_CALLCONV
compareStringLengths(const void *,const void * l,const void * r)1477 compareStringLengths(const void * /*context*/, const void *l, const void *r) {
1478     const StringResource *left = *((const StringResource **)l);
1479     const StringResource *right = *((const StringResource **)r);
1480     int32_t diff;
1481     /* Make "is suffix of another string" compare greater than a non-suffix. */
1482     diff = (int)(left->fSame != NULL) - (int)(right->fSame != NULL);
1483     if (diff != 0) {
1484         return diff;
1485     }
1486     /* sort by ascending string length */
1487     diff = left->length() - right->length();
1488     if (diff != 0) {
1489         return diff;
1490     }
1491     // sort by descending size reduction
1492     diff = right->fNumUnitsSaved - left->fNumUnitsSaved;
1493     if (diff != 0) {
1494         return diff;
1495     }
1496     // sort lexically
1497     return left->fString.compare(right->fString);
1498 }
1499 
1500 void
writeUTF16v2(int32_t base,UnicodeString & dest)1501 StringResource::writeUTF16v2(int32_t base, UnicodeString &dest) {
1502     int32_t len = length();
1503     fRes = URES_MAKE_RESOURCE(URES_STRING_V2, base + dest.length());
1504     fWritten = TRUE;
1505     switch(fNumCharsForLength) {
1506     case 0:
1507         break;
1508     case 1:
1509         dest.append((UChar)(0xdc00 + len));
1510         break;
1511     case 2:
1512         dest.append((UChar)(0xdfef + (len >> 16)));
1513         dest.append((UChar)len);
1514         break;
1515     case 3:
1516         dest.append((UChar)0xdfff);
1517         dest.append((UChar)(len >> 16));
1518         dest.append((UChar)len);
1519         break;
1520     default:
1521         break;  /* will not occur */
1522     }
1523     dest.append(fString);
1524     dest.append((UChar)0);
1525 }
1526 
1527 void
compactStringsV2(UHashtable * stringSet,UErrorCode & errorCode)1528 SRBRoot::compactStringsV2(UHashtable *stringSet, UErrorCode &errorCode) {
1529     if (U_FAILURE(errorCode)) {
1530         return;
1531     }
1532     // Store the StringResource pointers in an array for
1533     // easy sorting and processing.
1534     // We enumerate a set of strings, so there are no duplicates.
1535     int32_t count = uhash_count(stringSet);
1536     LocalArray<StringResource *> array(new StringResource *[count], errorCode);
1537     if (U_FAILURE(errorCode)) {
1538         return;
1539     }
1540     for (int32_t pos = UHASH_FIRST, i = 0; i < count; ++i) {
1541         array[i] = (StringResource *)uhash_nextElement(stringSet, &pos)->key.pointer;
1542     }
1543     /* Sort the strings so that each one is immediately followed by all of its suffixes. */
1544     uprv_sortArray(array.getAlias(), count, (int32_t)sizeof(struct SResource **),
1545                    compareStringSuffixes, NULL, FALSE, &errorCode);
1546     if (U_FAILURE(errorCode)) {
1547         return;
1548     }
1549     /*
1550      * Make suffixes point into earlier, longer strings that contain them.
1551      * Temporarily use fSame and fSuffixOffset for suffix strings to
1552      * refer to the remaining ones.
1553      */
1554     for (int32_t i = 0; i < count;) {
1555         /*
1556          * This string is not a suffix of the previous one;
1557          * write this one and subsume the following ones that are
1558          * suffixes of this one.
1559          */
1560         StringResource *res = array[i];
1561         res->fNumUnitsSaved = (res->fNumCopies - 1) * res->get16BitStringsLength();
1562         // Whole duplicates of pool strings are already account for in fPoolStringIndexLimit,
1563         // see StringResource::handlePreflightStrings().
1564         int32_t j;
1565         for (j = i + 1; j < count; ++j) {
1566             StringResource *suffixRes = array[j];
1567             /* Is it a suffix of the earlier, longer string? */
1568             if (res->fString.endsWith(suffixRes->fString)) {
1569                 assert(res->length() != suffixRes->length());  // Set strings are unique.
1570                 if (suffixRes->fWritten) {
1571                     // Pool string, skip.
1572                 } else if (suffixRes->fNumCharsForLength == 0) {
1573                     /* yes, point to the earlier string */
1574                     suffixRes->fSame = res;
1575                     suffixRes->fSuffixOffset = res->length() - suffixRes->length();
1576                     if (res->fWritten) {
1577                         // Suffix-share res which is a pool string.
1578                         // Compute the resource word and collect the maximum.
1579                         suffixRes->fRes =
1580                                 res->fRes + res->fNumCharsForLength + suffixRes->fSuffixOffset;
1581                         int32_t poolStringIndex = (int32_t)RES_GET_OFFSET(suffixRes->fRes);
1582                         if (poolStringIndex >= fPoolStringIndexLimit) {
1583                             fPoolStringIndexLimit = poolStringIndex + 1;
1584                         }
1585                         suffixRes->fWritten = TRUE;
1586                     }
1587                     res->fNumUnitsSaved += suffixRes->fNumCopies * suffixRes->get16BitStringsLength();
1588                 } else {
1589                     /* write the suffix by itself if we need explicit length */
1590                 }
1591             } else {
1592                 break;  /* not a suffix, restart from here */
1593             }
1594         }
1595         i = j;
1596     }
1597     /*
1598      * Re-sort the strings by ascending length (except suffixes last)
1599      * to optimize for URES_TABLE16 and URES_ARRAY16:
1600      * Keep as many as possible within reach of 16-bit offsets.
1601      */
1602     uprv_sortArray(array.getAlias(), count, (int32_t)sizeof(struct SResource **),
1603                    compareStringLengths, NULL, FALSE, &errorCode);
1604     if (U_FAILURE(errorCode)) {
1605         return;
1606     }
1607     if (fIsPoolBundle) {
1608         // Write strings that are sufficiently shared.
1609         // Avoid writing other strings.
1610         int32_t numStringsWritten = 0;
1611         int32_t numUnitsSaved = 0;
1612         int32_t numUnitsNotSaved = 0;
1613         for (int32_t i = 0; i < count; ++i) {
1614             StringResource *res = array[i];
1615             // Maximum pool string index when suffix-sharing the last character.
1616             int32_t maxStringIndex =
1617                     f16BitUnits.length() + res->fNumCharsForLength + res->length() - 1;
1618             if (res->fNumUnitsSaved >= GENRB_MIN_16BIT_UNITS_SAVED_FOR_POOL_STRING &&
1619                     maxStringIndex < RES_MAX_OFFSET) {
1620                 res->writeUTF16v2(0, f16BitUnits);
1621                 ++numStringsWritten;
1622                 numUnitsSaved += res->fNumUnitsSaved;
1623             } else {
1624                 numUnitsNotSaved += res->fNumUnitsSaved;
1625                 res->fRes = URES_MAKE_EMPTY_RESOURCE(URES_STRING);
1626                 res->fWritten = TRUE;
1627             }
1628         }
1629         if (f16BitUnits.isBogus()) {
1630             errorCode = U_MEMORY_ALLOCATION_ERROR;
1631         }
1632         if (getShowWarning()) {  // not quiet
1633             printf("number of shared strings: %d\n", (int)numStringsWritten);
1634             printf("16-bit units for strings: %6d = %6d bytes\n",
1635                    (int)f16BitUnits.length(), (int)f16BitUnits.length() * 2);
1636             printf("16-bit units saved:       %6d = %6d bytes\n",
1637                    (int)numUnitsSaved, (int)numUnitsSaved * 2);
1638             printf("16-bit units not saved:   %6d = %6d bytes\n",
1639                    (int)numUnitsNotSaved, (int)numUnitsNotSaved * 2);
1640         }
1641     } else {
1642         assert(fPoolStringIndexLimit <= fUsePoolBundle->fStringIndexLimit);
1643         /* Write the non-suffix strings. */
1644         int32_t i;
1645         for (i = 0; i < count && array[i]->fSame == NULL; ++i) {
1646             StringResource *res = array[i];
1647             if (!res->fWritten) {
1648                 int32_t localStringIndex = f16BitUnits.length();
1649                 if (localStringIndex >= fLocalStringIndexLimit) {
1650                     fLocalStringIndexLimit = localStringIndex + 1;
1651                 }
1652                 res->writeUTF16v2(fPoolStringIndexLimit, f16BitUnits);
1653             }
1654         }
1655         if (f16BitUnits.isBogus()) {
1656             errorCode = U_MEMORY_ALLOCATION_ERROR;
1657             return;
1658         }
1659         if (fWritePoolBundle != NULL && gFormatVersion >= 3) {
1660             PseudoListResource *poolStrings =
1661                     static_cast<PseudoListResource *>(fWritePoolBundle->fRoot);
1662             for (i = 0; i < count && array[i]->fSame == NULL; ++i) {
1663                 assert(!array[i]->fString.isEmpty());
1664                 StringResource *poolString =
1665                         new StringResource(fWritePoolBundle, array[i]->fString, errorCode);
1666                 if (poolString == NULL) {
1667                     errorCode = U_MEMORY_ALLOCATION_ERROR;
1668                     break;
1669                 }
1670                 poolStrings->add(poolString);
1671             }
1672         }
1673         /* Write the suffix strings. Make each point to the real string. */
1674         for (; i < count; ++i) {
1675             StringResource *res = array[i];
1676             if (res->fWritten) {
1677                 continue;
1678             }
1679             StringResource *same = res->fSame;
1680             assert(res->length() != same->length());  // Set strings are unique.
1681             res->fRes = same->fRes + same->fNumCharsForLength + res->fSuffixOffset;
1682             int32_t localStringIndex = (int32_t)RES_GET_OFFSET(res->fRes) - fPoolStringIndexLimit;
1683             // Suffixes of pool strings have been set already.
1684             assert(localStringIndex >= 0);
1685             if (localStringIndex >= fLocalStringIndexLimit) {
1686                 fLocalStringIndexLimit = localStringIndex + 1;
1687             }
1688             res->fWritten = TRUE;
1689         }
1690     }
1691     // +1 to account for the initial zero in f16BitUnits
1692     assert(f16BitUnits.length() <= (f16BitStringsLength + 1));
1693 }
1694