1 /*
2 *******************************************************************************
3 *
4 * Copyright (C) 2001-2010, International Business Machines
5 * Corporation and others. All Rights Reserved.
6 *
7 *******************************************************************************
8 * file name: ucaelems.cpp
9 * encoding: US-ASCII
10 * tab size: 8 (not used)
11 * indentation:4
12 *
13 * created 02/22/2001
14 * created by: Vladimir Weinstein
15 *
16 * This program reads the Franctional UCA table and generates
17 * internal format for UCA table as well as inverse UCA table.
18 * It then writes binary files containing the data: ucadata.dat
19 * & invuca.dat
20 *
21 * date name comments
22 * 03/02/2001 synwee added setMaxExpansion
23 * 03/07/2001 synwee merged UCA's maxexpansion and tailoring's
24 */
25
26 #include "unicode/utypes.h"
27
28 #if !UCONFIG_NO_COLLATION
29
30 #include "unicode/uchar.h"
31 #include "unicode/unistr.h"
32 #include "unicode/ucoleitr.h"
33 #include "unicode/normlzr.h"
34 #include "normalizer2impl.h"
35 #include "ucol_elm.h"
36 #include "ucol_tok.h"
37 #include "ucol_cnt.h"
38 #include "unicode/caniter.h"
39 #include "cmemory.h"
40
41 U_NAMESPACE_USE
42
43 static uint32_t uprv_uca_processContraction(CntTable *contractions, UCAElements *element, uint32_t existingCE, UErrorCode *status);
44
45 U_CDECL_BEGIN
46 static int32_t U_CALLCONV
prefixLookupHash(const UHashTok e)47 prefixLookupHash(const UHashTok e) {
48 UCAElements *element = (UCAElements *)e.pointer;
49 UChar buf[256];
50 UHashTok key;
51 key.pointer = buf;
52 uprv_memcpy(buf, element->cPoints, element->cSize*sizeof(UChar));
53 buf[element->cSize] = 0;
54 //key.pointer = element->cPoints;
55 //element->cPoints[element->cSize] = 0;
56 return uhash_hashUChars(key);
57 }
58
59 static int8_t U_CALLCONV
prefixLookupComp(const UHashTok e1,const UHashTok e2)60 prefixLookupComp(const UHashTok e1, const UHashTok e2) {
61 UCAElements *element1 = (UCAElements *)e1.pointer;
62 UCAElements *element2 = (UCAElements *)e2.pointer;
63
64 UChar buf1[256];
65 UHashTok key1;
66 key1.pointer = buf1;
67 uprv_memcpy(buf1, element1->cPoints, element1->cSize*sizeof(UChar));
68 buf1[element1->cSize] = 0;
69
70 UChar buf2[256];
71 UHashTok key2;
72 key2.pointer = buf2;
73 uprv_memcpy(buf2, element2->cPoints, element2->cSize*sizeof(UChar));
74 buf2[element2->cSize] = 0;
75
76 return uhash_compareUChars(key1, key2);
77 }
78 U_CDECL_END
79
uprv_uca_addExpansion(ExpansionTable * expansions,uint32_t value,UErrorCode * status)80 static int32_t uprv_uca_addExpansion(ExpansionTable *expansions, uint32_t value, UErrorCode *status) {
81 if(U_FAILURE(*status)) {
82 return 0;
83 }
84 if(expansions->CEs == NULL) {
85 expansions->CEs = (uint32_t *)uprv_malloc(INIT_EXP_TABLE_SIZE*sizeof(uint32_t));
86 /* test for NULL */
87 if (expansions->CEs == NULL) {
88 *status = U_MEMORY_ALLOCATION_ERROR;
89 return 0;
90 }
91 expansions->size = INIT_EXP_TABLE_SIZE;
92 expansions->position = 0;
93 }
94
95 if(expansions->position == expansions->size) {
96 uint32_t *newData = (uint32_t *)uprv_realloc(expansions->CEs, 2*expansions->size*sizeof(uint32_t));
97 if(newData == NULL) {
98 #ifdef UCOL_DEBUG
99 fprintf(stderr, "out of memory for expansions\n");
100 #endif
101 *status = U_MEMORY_ALLOCATION_ERROR;
102 return -1;
103 }
104 expansions->CEs = newData;
105 expansions->size *= 2;
106 }
107
108 expansions->CEs[expansions->position] = value;
109 return(expansions->position++);
110 }
111
112 U_CAPI tempUCATable* U_EXPORT2
uprv_uca_initTempTable(UCATableHeader * image,UColOptionSet * opts,const UCollator * UCA,UColCETags initTag,UColCETags supplementaryInitTag,UErrorCode * status)113 uprv_uca_initTempTable(UCATableHeader *image, UColOptionSet *opts, const UCollator *UCA, UColCETags initTag, UColCETags supplementaryInitTag, UErrorCode *status) {
114 MaxJamoExpansionTable *maxjet;
115 MaxExpansionTable *maxet;
116 tempUCATable *t = (tempUCATable *)uprv_malloc(sizeof(tempUCATable));
117 /* test for NULL */
118 if (t == NULL) {
119 *status = U_MEMORY_ALLOCATION_ERROR;
120 return NULL;
121 }
122 uprv_memset(t, 0, sizeof(tempUCATable));
123
124 maxet = (MaxExpansionTable *)uprv_malloc(sizeof(MaxExpansionTable));
125 if (maxet == NULL) {
126 goto allocation_failure;
127 }
128 uprv_memset(maxet, 0, sizeof(MaxExpansionTable));
129 t->maxExpansions = maxet;
130
131 maxjet = (MaxJamoExpansionTable *)uprv_malloc(sizeof(MaxJamoExpansionTable));
132 if (maxjet == NULL) {
133 goto allocation_failure;
134 }
135 uprv_memset(maxjet, 0, sizeof(MaxJamoExpansionTable));
136 t->maxJamoExpansions = maxjet;
137
138 t->image = image;
139 t->options = opts;
140
141 t->UCA = UCA;
142 t->expansions = (ExpansionTable *)uprv_malloc(sizeof(ExpansionTable));
143 /* test for NULL */
144 if (t->expansions == NULL) {
145 goto allocation_failure;
146 }
147 uprv_memset(t->expansions, 0, sizeof(ExpansionTable));
148
149 t->mapping = utrie_open(NULL, NULL, UCOL_ELM_TRIE_CAPACITY,
150 UCOL_SPECIAL_FLAG | (initTag<<24),
151 UCOL_SPECIAL_FLAG | (supplementaryInitTag << 24),
152 TRUE); // Do your own mallocs for the structure, array and have linear Latin 1
153 if (U_FAILURE(*status)) {
154 goto allocation_failure;
155 }
156 t->prefixLookup = uhash_open(prefixLookupHash, prefixLookupComp, NULL, status);
157 if (U_FAILURE(*status)) {
158 goto allocation_failure;
159 }
160 uhash_setValueDeleter(t->prefixLookup, uhash_freeBlock);
161
162 t->contractions = uprv_cnttab_open(t->mapping, status);
163 if (U_FAILURE(*status)) {
164 goto cleanup;
165 }
166
167 /* copy UCA's maxexpansion and merge as we go along */
168 if (UCA != NULL) {
169 /* adding an extra initial value for easier manipulation */
170 maxet->size = (int32_t)(UCA->lastEndExpansionCE - UCA->endExpansionCE) + 2;
171 maxet->position = maxet->size - 1;
172 maxet->endExpansionCE =
173 (uint32_t *)uprv_malloc(sizeof(uint32_t) * maxet->size);
174 /* test for NULL */
175 if (maxet->endExpansionCE == NULL) {
176 goto allocation_failure;
177 }
178 maxet->expansionCESize =
179 (uint8_t *)uprv_malloc(sizeof(uint8_t) * maxet->size);
180 /* test for NULL */
181 if (maxet->expansionCESize == NULL) {
182 goto allocation_failure;
183 }
184 /* initialized value */
185 *(maxet->endExpansionCE) = 0;
186 *(maxet->expansionCESize) = 0;
187 uprv_memcpy(maxet->endExpansionCE + 1, UCA->endExpansionCE,
188 sizeof(uint32_t) * (maxet->size - 1));
189 uprv_memcpy(maxet->expansionCESize + 1, UCA->expansionCESize,
190 sizeof(uint8_t) * (maxet->size - 1));
191 }
192 else {
193 maxet->size = 0;
194 }
195 maxjet->endExpansionCE = NULL;
196 maxjet->isV = NULL;
197 maxjet->size = 0;
198 maxjet->position = 0;
199 maxjet->maxLSize = 1;
200 maxjet->maxVSize = 1;
201 maxjet->maxTSize = 1;
202
203 t->unsafeCP = (uint8_t *)uprv_malloc(UCOL_UNSAFECP_TABLE_SIZE);
204 /* test for NULL */
205 if (t->unsafeCP == NULL) {
206 goto allocation_failure;
207 }
208 t->contrEndCP = (uint8_t *)uprv_malloc(UCOL_UNSAFECP_TABLE_SIZE);
209 /* test for NULL */
210 if (t->contrEndCP == NULL) {
211 goto allocation_failure;
212 }
213 uprv_memset(t->unsafeCP, 0, UCOL_UNSAFECP_TABLE_SIZE);
214 uprv_memset(t->contrEndCP, 0, UCOL_UNSAFECP_TABLE_SIZE);
215 t->cmLookup = NULL;
216 return t;
217
218 allocation_failure:
219 *status = U_MEMORY_ALLOCATION_ERROR;
220 cleanup:
221 uprv_uca_closeTempTable(t);
222 return NULL;
223 }
224
225 static tempUCATable* U_EXPORT2
uprv_uca_cloneTempTable(tempUCATable * t,UErrorCode * status)226 uprv_uca_cloneTempTable(tempUCATable *t, UErrorCode *status) {
227 if(U_FAILURE(*status)) {
228 return NULL;
229 }
230
231 tempUCATable *r = (tempUCATable *)uprv_malloc(sizeof(tempUCATable));
232 /* test for NULL */
233 if (r == NULL) {
234 *status = U_MEMORY_ALLOCATION_ERROR;
235 return NULL;
236 }
237 uprv_memset(r, 0, sizeof(tempUCATable));
238
239 /* mapping */
240 if(t->mapping != NULL) {
241 /*r->mapping = ucmpe32_clone(t->mapping, status);*/
242 r->mapping = utrie_clone(NULL, t->mapping, NULL, 0);
243 }
244
245 // a hashing clone function would be very nice. We have none currently...
246 // However, we should be good, as closing should not produce any prefixed elements.
247 r->prefixLookup = NULL; // prefixes are not used in closing
248
249 /* expansions */
250 if(t->expansions != NULL) {
251 r->expansions = (ExpansionTable *)uprv_malloc(sizeof(ExpansionTable));
252 /* test for NULL */
253 if (r->expansions == NULL) {
254 *status = U_MEMORY_ALLOCATION_ERROR;
255 goto cleanup;
256 }
257 r->expansions->position = t->expansions->position;
258 r->expansions->size = t->expansions->size;
259 if(t->expansions->CEs != NULL) {
260 r->expansions->CEs = (uint32_t *)uprv_malloc(sizeof(uint32_t)*t->expansions->size);
261 /* test for NULL */
262 if (r->expansions->CEs == NULL) {
263 *status = U_MEMORY_ALLOCATION_ERROR;
264 goto cleanup;
265 }
266 uprv_memcpy(r->expansions->CEs, t->expansions->CEs, sizeof(uint32_t)*t->expansions->position);
267 } else {
268 r->expansions->CEs = NULL;
269 }
270 }
271
272 if(t->contractions != NULL) {
273 r->contractions = uprv_cnttab_clone(t->contractions, status);
274 // Check for cloning failure.
275 if (r->contractions == NULL) {
276 *status = U_MEMORY_ALLOCATION_ERROR;
277 goto cleanup;
278 }
279 r->contractions->mapping = r->mapping;
280 }
281
282 if(t->maxExpansions != NULL) {
283 r->maxExpansions = (MaxExpansionTable *)uprv_malloc(sizeof(MaxExpansionTable));
284 /* test for NULL */
285 if (r->maxExpansions == NULL) {
286 *status = U_MEMORY_ALLOCATION_ERROR;
287 goto cleanup;
288 }
289 r->maxExpansions->size = t->maxExpansions->size;
290 r->maxExpansions->position = t->maxExpansions->position;
291 if(t->maxExpansions->endExpansionCE != NULL) {
292 r->maxExpansions->endExpansionCE = (uint32_t *)uprv_malloc(sizeof(uint32_t)*t->maxExpansions->size);
293 /* test for NULL */
294 if (r->maxExpansions->endExpansionCE == NULL) {
295 *status = U_MEMORY_ALLOCATION_ERROR;
296 goto cleanup;
297 }
298 uprv_memset(r->maxExpansions->endExpansionCE, 0xDB, sizeof(uint32_t)*t->maxExpansions->size);
299 uprv_memcpy(r->maxExpansions->endExpansionCE, t->maxExpansions->endExpansionCE, t->maxExpansions->position*sizeof(uint32_t));
300 } else {
301 r->maxExpansions->endExpansionCE = NULL;
302 }
303 if(t->maxExpansions->expansionCESize != NULL) {
304 r->maxExpansions->expansionCESize = (uint8_t *)uprv_malloc(sizeof(uint8_t)*t->maxExpansions->size);
305 /* test for NULL */
306 if (r->maxExpansions->expansionCESize == NULL) {
307 *status = U_MEMORY_ALLOCATION_ERROR;
308 goto cleanup;
309 }
310 uprv_memset(r->maxExpansions->expansionCESize, 0xDB, sizeof(uint8_t)*t->maxExpansions->size);
311 uprv_memcpy(r->maxExpansions->expansionCESize, t->maxExpansions->expansionCESize, t->maxExpansions->position*sizeof(uint8_t));
312 } else {
313 r->maxExpansions->expansionCESize = NULL;
314 }
315 }
316
317 if(t->maxJamoExpansions != NULL) {
318 r->maxJamoExpansions = (MaxJamoExpansionTable *)uprv_malloc(sizeof(MaxJamoExpansionTable));
319 /* test for NULL */
320 if (r->maxJamoExpansions == NULL) {
321 *status = U_MEMORY_ALLOCATION_ERROR;
322 goto cleanup;
323 }
324 r->maxJamoExpansions->size = t->maxJamoExpansions->size;
325 r->maxJamoExpansions->position = t->maxJamoExpansions->position;
326 r->maxJamoExpansions->maxLSize = t->maxJamoExpansions->maxLSize;
327 r->maxJamoExpansions->maxVSize = t->maxJamoExpansions->maxVSize;
328 r->maxJamoExpansions->maxTSize = t->maxJamoExpansions->maxTSize;
329 if(t->maxJamoExpansions->size != 0) {
330 r->maxJamoExpansions->endExpansionCE = (uint32_t *)uprv_malloc(sizeof(uint32_t)*t->maxJamoExpansions->size);
331 /* test for NULL */
332 if (r->maxJamoExpansions->endExpansionCE == NULL) {
333 *status = U_MEMORY_ALLOCATION_ERROR;
334 goto cleanup;
335 }
336 uprv_memcpy(r->maxJamoExpansions->endExpansionCE, t->maxJamoExpansions->endExpansionCE, t->maxJamoExpansions->position*sizeof(uint32_t));
337 r->maxJamoExpansions->isV = (UBool *)uprv_malloc(sizeof(UBool)*t->maxJamoExpansions->size);
338 /* test for NULL */
339 if (r->maxJamoExpansions->isV == NULL) {
340 *status = U_MEMORY_ALLOCATION_ERROR;
341 goto cleanup;
342 }
343 uprv_memcpy(r->maxJamoExpansions->isV, t->maxJamoExpansions->isV, t->maxJamoExpansions->position*sizeof(UBool));
344 } else {
345 r->maxJamoExpansions->endExpansionCE = NULL;
346 r->maxJamoExpansions->isV = NULL;
347 }
348 }
349
350 if(t->unsafeCP != NULL) {
351 r->unsafeCP = (uint8_t *)uprv_malloc(UCOL_UNSAFECP_TABLE_SIZE);
352 /* test for NULL */
353 if (r->unsafeCP == NULL) {
354 *status = U_MEMORY_ALLOCATION_ERROR;
355 goto cleanup;
356 }
357 uprv_memcpy(r->unsafeCP, t->unsafeCP, UCOL_UNSAFECP_TABLE_SIZE);
358 }
359
360 if(t->contrEndCP != NULL) {
361 r->contrEndCP = (uint8_t *)uprv_malloc(UCOL_UNSAFECP_TABLE_SIZE);
362 /* test for NULL */
363 if (r->contrEndCP == NULL) {
364 *status = U_MEMORY_ALLOCATION_ERROR;
365 goto cleanup;
366 }
367 uprv_memcpy(r->contrEndCP, t->contrEndCP, UCOL_UNSAFECP_TABLE_SIZE);
368 }
369
370 r->UCA = t->UCA;
371 r->image = t->image;
372 r->options = t->options;
373
374 return r;
375 cleanup:
376 uprv_uca_closeTempTable(t);
377 return NULL;
378 }
379
380
381 U_CAPI void U_EXPORT2
uprv_uca_closeTempTable(tempUCATable * t)382 uprv_uca_closeTempTable(tempUCATable *t) {
383 if(t != NULL) {
384 if (t->expansions != NULL) {
385 uprv_free(t->expansions->CEs);
386 uprv_free(t->expansions);
387 }
388 if(t->contractions != NULL) {
389 uprv_cnttab_close(t->contractions);
390 }
391 if (t->mapping != NULL) {
392 utrie_close(t->mapping);
393 }
394
395 if(t->prefixLookup != NULL) {
396 uhash_close(t->prefixLookup);
397 }
398
399 if (t->maxExpansions != NULL) {
400 uprv_free(t->maxExpansions->endExpansionCE);
401 uprv_free(t->maxExpansions->expansionCESize);
402 uprv_free(t->maxExpansions);
403 }
404
405 if (t->maxJamoExpansions->size > 0) {
406 uprv_free(t->maxJamoExpansions->endExpansionCE);
407 uprv_free(t->maxJamoExpansions->isV);
408 }
409 uprv_free(t->maxJamoExpansions);
410
411 uprv_free(t->unsafeCP);
412 uprv_free(t->contrEndCP);
413
414 if (t->cmLookup != NULL) {
415 uprv_free(t->cmLookup->cPoints);
416 uprv_free(t->cmLookup);
417 }
418
419 uprv_free(t);
420 }
421 }
422
423 /**
424 * Looks for the maximum length of all expansion sequences ending with the same
425 * collation element. The size required for maxexpansion and maxsize is
426 * returned if the arrays are too small.
427 * @param endexpansion the last expansion collation element to be added
428 * @param expansionsize size of the expansion
429 * @param maxexpansion data structure to store the maximum expansion data.
430 * @param status error status
431 * @returns size of the maxexpansion and maxsize used.
432 */
uprv_uca_setMaxExpansion(uint32_t endexpansion,uint8_t expansionsize,MaxExpansionTable * maxexpansion,UErrorCode * status)433 static int uprv_uca_setMaxExpansion(uint32_t endexpansion,
434 uint8_t expansionsize,
435 MaxExpansionTable *maxexpansion,
436 UErrorCode *status)
437 {
438 if (maxexpansion->size == 0) {
439 /* we'll always make the first element 0, for easier manipulation */
440 maxexpansion->endExpansionCE =
441 (uint32_t *)uprv_malloc(INIT_EXP_TABLE_SIZE * sizeof(int32_t));
442 /* test for NULL */
443 if (maxexpansion->endExpansionCE == NULL) {
444 *status = U_MEMORY_ALLOCATION_ERROR;
445 return 0;
446 }
447 *(maxexpansion->endExpansionCE) = 0;
448 maxexpansion->expansionCESize =
449 (uint8_t *)uprv_malloc(INIT_EXP_TABLE_SIZE * sizeof(uint8_t));
450 /* test for NULL */;
451 if (maxexpansion->expansionCESize == NULL) {
452 *status = U_MEMORY_ALLOCATION_ERROR;
453 return 0;
454 }
455 *(maxexpansion->expansionCESize) = 0;
456 maxexpansion->size = INIT_EXP_TABLE_SIZE;
457 maxexpansion->position = 0;
458 }
459
460 if (maxexpansion->position + 1 == maxexpansion->size) {
461 uint32_t *neweece = (uint32_t *)uprv_realloc(maxexpansion->endExpansionCE,
462 2 * maxexpansion->size * sizeof(uint32_t));
463 if (neweece == NULL) {
464 *status = U_MEMORY_ALLOCATION_ERROR;
465 return 0;
466 }
467 maxexpansion->endExpansionCE = neweece;
468
469 uint8_t *neweces = (uint8_t *)uprv_realloc(maxexpansion->expansionCESize,
470 2 * maxexpansion->size * sizeof(uint8_t));
471 if (neweces == NULL) {
472 *status = U_MEMORY_ALLOCATION_ERROR;
473 return 0;
474 }
475 maxexpansion->expansionCESize = neweces;
476 maxexpansion->size *= 2;
477 }
478
479 uint32_t *pendexpansionce = maxexpansion->endExpansionCE;
480 uint8_t *pexpansionsize = maxexpansion->expansionCESize;
481 int pos = maxexpansion->position;
482
483 uint32_t *start = pendexpansionce;
484 uint32_t *limit = pendexpansionce + pos;
485
486 /* using binary search to determine if last expansion element is
487 already in the array */
488 uint32_t *mid;
489 int result = -1;
490 while (start < limit - 1) {
491 mid = start + ((limit - start) >> 1);
492 if (endexpansion <= *mid) {
493 limit = mid;
494 }
495 else {
496 start = mid;
497 }
498 }
499
500 if (*start == endexpansion) {
501 result = (int)(start - pendexpansionce);
502 }
503 else if (*limit == endexpansion) {
504 result = (int)(limit - pendexpansionce);
505 }
506
507 if (result > -1) {
508 /* found the ce in expansion, we'll just modify the size if it is
509 smaller */
510 uint8_t *currentsize = pexpansionsize + result;
511 if (*currentsize < expansionsize) {
512 *currentsize = expansionsize;
513 }
514 }
515 else {
516 /* we'll need to squeeze the value into the array.
517 initial implementation. */
518 /* shifting the subarray down by 1 */
519 int shiftsize = (int)((pendexpansionce + pos) - start);
520 uint32_t *shiftpos = start + 1;
521 uint8_t *sizeshiftpos = pexpansionsize + (shiftpos - pendexpansionce);
522
523 /* okay need to rearrange the array into sorted order */
524 if (shiftsize == 0 /*|| *(pendexpansionce + pos) < endexpansion*/) { /* the commented part is actually both redundant and dangerous */
525 *(pendexpansionce + pos + 1) = endexpansion;
526 *(pexpansionsize + pos + 1) = expansionsize;
527 }
528 else {
529 uprv_memmove(shiftpos + 1, shiftpos, shiftsize * sizeof(int32_t));
530 uprv_memmove(sizeshiftpos + 1, sizeshiftpos,
531 shiftsize * sizeof(uint8_t));
532 *shiftpos = endexpansion;
533 *sizeshiftpos = expansionsize;
534 }
535 maxexpansion->position ++;
536
537 #ifdef UCOL_DEBUG
538 int temp;
539 UBool found = FALSE;
540 for (temp = 0; temp < maxexpansion->position; temp ++) {
541 if (pendexpansionce[temp] >= pendexpansionce[temp + 1]) {
542 fprintf(stderr, "expansions %d\n", temp);
543 }
544 if (pendexpansionce[temp] == endexpansion) {
545 found =TRUE;
546 if (pexpansionsize[temp] < expansionsize) {
547 fprintf(stderr, "expansions size %d\n", temp);
548 }
549 }
550 }
551 if (pendexpansionce[temp] == endexpansion) {
552 found =TRUE;
553 if (pexpansionsize[temp] < expansionsize) {
554 fprintf(stderr, "expansions size %d\n", temp);
555 }
556 }
557 if (!found)
558 fprintf(stderr, "expansion not found %d\n", temp);
559 #endif
560 }
561
562 return maxexpansion->position;
563 }
564
565 /**
566 * Sets the maximum length of all jamo expansion sequences ending with the same
567 * collation element. The size required for maxexpansion and maxsize is
568 * returned if the arrays are too small.
569 * @param ch the jamo codepoint
570 * @param endexpansion the last expansion collation element to be added
571 * @param expansionsize size of the expansion
572 * @param maxexpansion data structure to store the maximum expansion data.
573 * @param status error status
574 * @returns size of the maxexpansion and maxsize used.
575 */
uprv_uca_setMaxJamoExpansion(UChar ch,uint32_t endexpansion,uint8_t expansionsize,MaxJamoExpansionTable * maxexpansion,UErrorCode * status)576 static int uprv_uca_setMaxJamoExpansion(UChar ch,
577 uint32_t endexpansion,
578 uint8_t expansionsize,
579 MaxJamoExpansionTable *maxexpansion,
580 UErrorCode *status)
581 {
582 UBool isV = TRUE;
583 if (((uint32_t)ch - 0x1100) <= (0x1112 - 0x1100)) {
584 /* determines L for Jamo, doesn't need to store this since it is never
585 at the end of a expansion */
586 if (maxexpansion->maxLSize < expansionsize) {
587 maxexpansion->maxLSize = expansionsize;
588 }
589 return maxexpansion->position;
590 }
591
592 if (((uint32_t)ch - 0x1161) <= (0x1175 - 0x1161)) {
593 /* determines V for Jamo */
594 if (maxexpansion->maxVSize < expansionsize) {
595 maxexpansion->maxVSize = expansionsize;
596 }
597 }
598
599 if (((uint32_t)ch - 0x11A8) <= (0x11C2 - 0x11A8)) {
600 isV = FALSE;
601 /* determines T for Jamo */
602 if (maxexpansion->maxTSize < expansionsize) {
603 maxexpansion->maxTSize = expansionsize;
604 }
605 }
606
607 if (maxexpansion->size == 0) {
608 /* we'll always make the first element 0, for easier manipulation */
609 maxexpansion->endExpansionCE =
610 (uint32_t *)uprv_malloc(INIT_EXP_TABLE_SIZE * sizeof(uint32_t));
611 /* test for NULL */;
612 if (maxexpansion->endExpansionCE == NULL) {
613 *status = U_MEMORY_ALLOCATION_ERROR;
614 return 0;
615 }
616 *(maxexpansion->endExpansionCE) = 0;
617 maxexpansion->isV =
618 (UBool *)uprv_malloc(INIT_EXP_TABLE_SIZE * sizeof(UBool));
619 /* test for NULL */;
620 if (maxexpansion->isV == NULL) {
621 *status = U_MEMORY_ALLOCATION_ERROR;
622 uprv_free(maxexpansion->endExpansionCE);
623 maxexpansion->endExpansionCE = NULL;
624 return 0;
625 }
626 *(maxexpansion->isV) = 0;
627 maxexpansion->size = INIT_EXP_TABLE_SIZE;
628 maxexpansion->position = 0;
629 }
630
631 if (maxexpansion->position + 1 == maxexpansion->size) {
632 maxexpansion->size *= 2;
633 maxexpansion->endExpansionCE = (uint32_t *)uprv_realloc(maxexpansion->endExpansionCE,
634 maxexpansion->size * sizeof(uint32_t));
635 if (maxexpansion->endExpansionCE == NULL) {
636 #ifdef UCOL_DEBUG
637 fprintf(stderr, "out of memory for maxExpansions\n");
638 #endif
639 *status = U_MEMORY_ALLOCATION_ERROR;
640 return 0;
641 }
642 maxexpansion->isV = (UBool *)uprv_realloc(maxexpansion->isV,
643 maxexpansion->size * sizeof(UBool));
644 if (maxexpansion->isV == NULL) {
645 #ifdef UCOL_DEBUG
646 fprintf(stderr, "out of memory for maxExpansions\n");
647 #endif
648 *status = U_MEMORY_ALLOCATION_ERROR;
649 uprv_free(maxexpansion->endExpansionCE);
650 maxexpansion->endExpansionCE = NULL;
651 return 0;
652 }
653 }
654
655 uint32_t *pendexpansionce = maxexpansion->endExpansionCE;
656 int pos = maxexpansion->position;
657
658 while (pos > 0) {
659 pos --;
660 if (*(pendexpansionce + pos) == endexpansion) {
661 return maxexpansion->position;
662 }
663 }
664
665 *(pendexpansionce + maxexpansion->position) = endexpansion;
666 *(maxexpansion->isV + maxexpansion->position) = isV;
667 maxexpansion->position ++;
668
669 return maxexpansion->position;
670 }
671
672
ContrEndCPSet(uint8_t * table,UChar c)673 static void ContrEndCPSet(uint8_t *table, UChar c) {
674 uint32_t hash;
675 uint8_t *htByte;
676
677 hash = c;
678 if (hash >= UCOL_UNSAFECP_TABLE_SIZE*8) {
679 hash = (hash & UCOL_UNSAFECP_TABLE_MASK) + 256;
680 }
681 htByte = &table[hash>>3];
682 *htByte |= (1 << (hash & 7));
683 }
684
685
unsafeCPSet(uint8_t * table,UChar c)686 static void unsafeCPSet(uint8_t *table, UChar c) {
687 uint32_t hash;
688 uint8_t *htByte;
689
690 hash = c;
691 if (hash >= UCOL_UNSAFECP_TABLE_SIZE*8) {
692 if (hash >= 0xd800 && hash <= 0xf8ff) {
693 /* Part of a surrogate, or in private use area. */
694 /* These don't go in the table */
695 return;
696 }
697 hash = (hash & UCOL_UNSAFECP_TABLE_MASK) + 256;
698 }
699 htByte = &table[hash>>3];
700 *htByte |= (1 << (hash & 7));
701 }
702
703 static void
uprv_uca_createCMTable(tempUCATable * t,int32_t noOfCM,UErrorCode * status)704 uprv_uca_createCMTable(tempUCATable *t, int32_t noOfCM, UErrorCode *status) {
705 t->cmLookup = (CombinClassTable *)uprv_malloc(sizeof(CombinClassTable));
706 if (t->cmLookup==NULL) {
707 *status = U_MEMORY_ALLOCATION_ERROR;
708 return;
709 }
710 t->cmLookup->cPoints=(UChar *)uprv_malloc(noOfCM*sizeof(UChar));
711 if (t->cmLookup->cPoints ==NULL) {
712 uprv_free(t->cmLookup);
713 t->cmLookup = NULL;
714 *status = U_MEMORY_ALLOCATION_ERROR;
715 return;
716 }
717
718 t->cmLookup->size=noOfCM;
719 uprv_memset(t->cmLookup->index, 0, sizeof(t->cmLookup->index));
720
721 return;
722 }
723
724 static void
uprv_uca_copyCMTable(tempUCATable * t,UChar * cm,uint16_t * index)725 uprv_uca_copyCMTable(tempUCATable *t, UChar *cm, uint16_t *index) {
726 int32_t count=0;
727
728 for (int32_t i=0; i<256; ++i) {
729 if (index[i]>0) {
730 // cPoints is ordered by combining class value.
731 uprv_memcpy(t->cmLookup->cPoints+count, cm+(i<<8), index[i]*sizeof(UChar));
732 count += index[i];
733 }
734 t->cmLookup->index[i]=count;
735 }
736 return;
737 }
738
739 /* 1. to the UnsafeCP hash table, add all chars with combining class != 0 */
740 /* 2. build combining marks table for all chars with combining class != 0 */
uprv_uca_unsafeCPAddCCNZ(tempUCATable * t,UErrorCode * status)741 static void uprv_uca_unsafeCPAddCCNZ(tempUCATable *t, UErrorCode *status) {
742
743 UChar c;
744 uint16_t fcd; // Hi byte is lead combining class.
745 // lo byte is trailing combing class.
746 const uint16_t *fcdTrieIndex;
747 UChar32 fcdHighStart;
748 UBool buildCMTable = (t->cmLookup==NULL); // flag for building combining class table
749 UChar *cm=NULL;
750 uint16_t index[256];
751 int32_t count=0;
752 fcdTrieIndex = unorm_getFCDTrieIndex(fcdHighStart, status);
753 if (U_FAILURE(*status)) {
754 return;
755 }
756
757 if (buildCMTable) {
758 if (cm==NULL) {
759 cm = (UChar *)uprv_malloc(sizeof(UChar)*UCOL_MAX_CM_TAB);
760 if (cm==NULL) {
761 *status = U_MEMORY_ALLOCATION_ERROR;
762 return;
763 }
764 }
765 uprv_memset(index, 0, sizeof(index));
766 }
767 for (c=0; c<0xffff; c++) {
768 fcd = unorm_getFCD16(fcdTrieIndex, c);
769 if (fcd >= 0x100 || // if the leading combining class(c) > 0 ||
770 (UTF_IS_LEAD(c) && fcd != 0)) {// c is a leading surrogate with some FCD data
771 if (buildCMTable) {
772 uint32_t cClass = fcd & 0xff;
773 //uint32_t temp=(cClass<<8)+index[cClass];
774 cm[(cClass<<8)+index[cClass]] = c; //
775 index[cClass]++;
776 count++;
777 }
778 unsafeCPSet(t->unsafeCP, c);
779 }
780 }
781
782 // copy to cm table
783 if (buildCMTable) {
784 uprv_uca_createCMTable(t, count, status);
785 if(U_FAILURE(*status)) {
786 if (cm!=NULL) {
787 uprv_free(cm);
788 }
789 return;
790 }
791 uprv_uca_copyCMTable(t, cm, index);
792 }
793
794 if(t->prefixLookup != NULL) {
795 int32_t i = -1;
796 const UHashElement *e = NULL;
797 UCAElements *element = NULL;
798 UChar NFCbuf[256];
799 uint32_t NFCbufLen = 0;
800 while((e = uhash_nextElement(t->prefixLookup, &i)) != NULL) {
801 element = (UCAElements *)e->value.pointer;
802 // codepoints here are in the NFD form. We need to add the
803 // first code point of the NFC form to unsafe, because
804 // strcoll needs to backup over them.
805 NFCbufLen = unorm_normalize(element->cPoints, element->cSize, UNORM_NFC, 0,
806 NFCbuf, 256, status);
807 unsafeCPSet(t->unsafeCP, NFCbuf[0]);
808 }
809 }
810
811 if (cm!=NULL) {
812 uprv_free(cm);
813 }
814 }
815
uprv_uca_addPrefix(tempUCATable * t,uint32_t CE,UCAElements * element,UErrorCode * status)816 static uint32_t uprv_uca_addPrefix(tempUCATable *t, uint32_t CE,
817 UCAElements *element, UErrorCode *status)
818 {
819 // currently the longest prefix we're supporting in Japanese is two characters
820 // long. Although this table could quite easily mimic complete contraction stuff
821 // there is no good reason to make a general solution, as it would require some
822 // error prone messing.
823 CntTable *contractions = t->contractions;
824 UChar32 cp;
825 uint32_t cpsize = 0;
826 UChar *oldCP = element->cPoints;
827 uint32_t oldCPSize = element->cSize;
828
829
830 contractions->currentTag = SPEC_PROC_TAG;
831
832 // here, we will normalize & add prefix to the table.
833 uint32_t j = 0;
834 #ifdef UCOL_DEBUG
835 for(j=0; j<element->cSize; j++) {
836 fprintf(stdout, "CP: %04X ", element->cPoints[j]);
837 }
838 fprintf(stdout, "El: %08X Pref: ", CE);
839 for(j=0; j<element->prefixSize; j++) {
840 fprintf(stdout, "%04X ", element->prefix[j]);
841 }
842 fprintf(stdout, "%08X ", element->mapCE);
843 #endif
844
845 for (j = 1; j<element->prefixSize; j++) { /* First add NFD prefix chars to unsafe CP hash table */
846 // Unless it is a trail surrogate, which is handled algoritmically and
847 // shouldn't take up space in the table.
848 if(!(UTF_IS_TRAIL(element->prefix[j]))) {
849 unsafeCPSet(t->unsafeCP, element->prefix[j]);
850 }
851 }
852
853 UChar tempPrefix = 0;
854
855 for(j = 0; j < /*nfcSize*/element->prefixSize/2; j++) { // prefixes are going to be looked up backwards
856 // therefore, we will promptly reverse the prefix buffer...
857 tempPrefix = *(/*nfcBuffer*/element->prefix+element->prefixSize-j-1);
858 *(/*nfcBuffer*/element->prefix+element->prefixSize-j-1) = element->prefix[j];
859 element->prefix[j] = tempPrefix;
860 }
861
862 #ifdef UCOL_DEBUG
863 fprintf(stdout, "Reversed: ");
864 for(j=0; j<element->prefixSize; j++) {
865 fprintf(stdout, "%04X ", element->prefix[j]);
866 }
867 fprintf(stdout, "%08X\n", element->mapCE);
868 #endif
869
870 // the first codepoint is also unsafe, as it forms a 'contraction' with the prefix
871 if(!(UTF_IS_TRAIL(element->cPoints[0]))) {
872 unsafeCPSet(t->unsafeCP, element->cPoints[0]);
873 }
874
875 // Maybe we need this... To handle prefixes completely in the forward direction...
876 //if(element->cSize == 1) {
877 // if(!(UTF_IS_TRAIL(element->cPoints[0]))) {
878 // ContrEndCPSet(t->contrEndCP, element->cPoints[0]);
879 // }
880 //}
881
882 element->cPoints = element->prefix;
883 element->cSize = element->prefixSize;
884
885 // Add the last char of the contraction to the contraction-end hash table.
886 // unless it is a trail surrogate, which is handled algorithmically and
887 // shouldn't be in the table
888 if(!(UTF_IS_TRAIL(element->cPoints[element->cSize -1]))) {
889 ContrEndCPSet(t->contrEndCP, element->cPoints[element->cSize -1]);
890 }
891
892 // First we need to check if contractions starts with a surrogate
893 UTF_NEXT_CHAR(element->cPoints, cpsize, element->cSize, cp);
894
895 // If there are any Jamos in the contraction, we should turn on special
896 // processing for Jamos
897 if(UCOL_ISJAMO(element->prefix[0])) {
898 t->image->jamoSpecial = TRUE;
899 }
900 /* then we need to deal with it */
901 /* we could aready have something in table - or we might not */
902
903 if(!isPrefix(CE)) {
904 /* if it wasn't contraction, we wouldn't end up here*/
905 int32_t firstContractionOffset = 0;
906 firstContractionOffset = uprv_cnttab_addContraction(contractions, UPRV_CNTTAB_NEWELEMENT, 0, CE, status);
907 uint32_t newCE = uprv_uca_processContraction(contractions, element, UCOL_NOT_FOUND, status);
908 uprv_cnttab_addContraction(contractions, firstContractionOffset, *element->prefix, newCE, status);
909 uprv_cnttab_addContraction(contractions, firstContractionOffset, 0xFFFF, CE, status);
910 CE = constructContractCE(SPEC_PROC_TAG, firstContractionOffset);
911 } else { /* we are adding to existing contraction */
912 /* there were already some elements in the table, so we need to add a new contraction */
913 /* Two things can happen here: either the codepoint is already in the table, or it is not */
914 int32_t position = uprv_cnttab_findCP(contractions, CE, *element->prefix, status);
915 if(position > 0) { /* if it is we just continue down the chain */
916 uint32_t eCE = uprv_cnttab_getCE(contractions, CE, position, status);
917 uint32_t newCE = uprv_uca_processContraction(contractions, element, eCE, status);
918 uprv_cnttab_setContraction(contractions, CE, position, *(element->prefix), newCE, status);
919 } else { /* if it isn't, we will have to create a new sequence */
920 uprv_uca_processContraction(contractions, element, UCOL_NOT_FOUND, status);
921 uprv_cnttab_insertContraction(contractions, CE, *(element->prefix), element->mapCE, status);
922 }
923 }
924
925 element->cPoints = oldCP;
926 element->cSize = oldCPSize;
927
928 return CE;
929 }
930
931 // Note regarding surrogate handling: We are interested only in the single
932 // or leading surrogates in a contraction. If a surrogate is somewhere else
933 // in the contraction, it is going to be handled as a pair of code units,
934 // as it doesn't affect the performance AND handling surrogates specially
935 // would complicate code way too much.
uprv_uca_addContraction(tempUCATable * t,uint32_t CE,UCAElements * element,UErrorCode * status)936 static uint32_t uprv_uca_addContraction(tempUCATable *t, uint32_t CE,
937 UCAElements *element, UErrorCode *status)
938 {
939 CntTable *contractions = t->contractions;
940 UChar32 cp;
941 uint32_t cpsize = 0;
942
943 contractions->currentTag = CONTRACTION_TAG;
944
945 // First we need to check if contractions starts with a surrogate
946 UTF_NEXT_CHAR(element->cPoints, cpsize, element->cSize, cp);
947
948 if(cpsize<element->cSize) { // This is a real contraction, if there are other characters after the first
949 uint32_t j = 0;
950 for (j=1; j<element->cSize; j++) { /* First add contraction chars to unsafe CP hash table */
951 // Unless it is a trail surrogate, which is handled algoritmically and
952 // shouldn't take up space in the table.
953 if(!(UTF_IS_TRAIL(element->cPoints[j]))) {
954 unsafeCPSet(t->unsafeCP, element->cPoints[j]);
955 }
956 }
957 // Add the last char of the contraction to the contraction-end hash table.
958 // unless it is a trail surrogate, which is handled algorithmically and
959 // shouldn't be in the table
960 if(!(UTF_IS_TRAIL(element->cPoints[element->cSize -1]))) {
961 ContrEndCPSet(t->contrEndCP, element->cPoints[element->cSize -1]);
962 }
963
964 // If there are any Jamos in the contraction, we should turn on special
965 // processing for Jamos
966 if(UCOL_ISJAMO(element->cPoints[0])) {
967 t->image->jamoSpecial = TRUE;
968 }
969 /* then we need to deal with it */
970 /* we could aready have something in table - or we might not */
971 element->cPoints+=cpsize;
972 element->cSize-=cpsize;
973 if(!isContraction(CE)) {
974 /* if it wasn't contraction, we wouldn't end up here*/
975 int32_t firstContractionOffset = 0;
976 firstContractionOffset = uprv_cnttab_addContraction(contractions, UPRV_CNTTAB_NEWELEMENT, 0, CE, status);
977 uint32_t newCE = uprv_uca_processContraction(contractions, element, UCOL_NOT_FOUND, status);
978 uprv_cnttab_addContraction(contractions, firstContractionOffset, *element->cPoints, newCE, status);
979 uprv_cnttab_addContraction(contractions, firstContractionOffset, 0xFFFF, CE, status);
980 CE = constructContractCE(CONTRACTION_TAG, firstContractionOffset);
981 } else { /* we are adding to existing contraction */
982 /* there were already some elements in the table, so we need to add a new contraction */
983 /* Two things can happen here: either the codepoint is already in the table, or it is not */
984 int32_t position = uprv_cnttab_findCP(contractions, CE, *element->cPoints, status);
985 if(position > 0) { /* if it is we just continue down the chain */
986 uint32_t eCE = uprv_cnttab_getCE(contractions, CE, position, status);
987 uint32_t newCE = uprv_uca_processContraction(contractions, element, eCE, status);
988 uprv_cnttab_setContraction(contractions, CE, position, *(element->cPoints), newCE, status);
989 } else { /* if it isn't, we will have to create a new sequence */
990 uint32_t newCE = uprv_uca_processContraction(contractions, element, UCOL_NOT_FOUND, status);
991 uprv_cnttab_insertContraction(contractions, CE, *(element->cPoints), newCE, status);
992 }
993 }
994 element->cPoints-=cpsize;
995 element->cSize+=cpsize;
996 /*ucmpe32_set(t->mapping, cp, CE);*/
997 utrie_set32(t->mapping, cp, CE);
998 } else if(!isContraction(CE)) { /* this is just a surrogate, and there is no contraction */
999 /*ucmpe32_set(t->mapping, cp, element->mapCE);*/
1000 utrie_set32(t->mapping, cp, element->mapCE);
1001 } else { /* fill out the first stage of the contraction with the surrogate CE */
1002 uprv_cnttab_changeContraction(contractions, CE, 0, element->mapCE, status);
1003 uprv_cnttab_changeContraction(contractions, CE, 0xFFFF, element->mapCE, status);
1004 }
1005 return CE;
1006 }
1007
1008
uprv_uca_processContraction(CntTable * contractions,UCAElements * element,uint32_t existingCE,UErrorCode * status)1009 static uint32_t uprv_uca_processContraction(CntTable *contractions, UCAElements *element, uint32_t existingCE, UErrorCode *status) {
1010 int32_t firstContractionOffset = 0;
1011 // uint32_t contractionElement = UCOL_NOT_FOUND;
1012
1013 if(U_FAILURE(*status)) {
1014 return UCOL_NOT_FOUND;
1015 }
1016
1017 /* end of recursion */
1018 if(element->cSize == 1) {
1019 if(isCntTableElement(existingCE) && ((UColCETags)getCETag(existingCE) == contractions->currentTag)) {
1020 uprv_cnttab_changeContraction(contractions, existingCE, 0, element->mapCE, status);
1021 uprv_cnttab_changeContraction(contractions, existingCE, 0xFFFF, element->mapCE, status);
1022 return existingCE;
1023 } else {
1024 return element->mapCE; /*can't do just that. existingCe might be a contraction, meaning that we need to do another step */
1025 }
1026 }
1027
1028 /* this recursion currently feeds on the only element we have... We will have to copy it in order to accomodate */
1029 /* for both backward and forward cycles */
1030
1031 /* we encountered either an empty space or a non-contraction element */
1032 /* this means we are constructing a new contraction sequence */
1033 element->cPoints++;
1034 element->cSize--;
1035 if(!isCntTableElement(existingCE)) {
1036 /* if it wasn't contraction, we wouldn't end up here*/
1037 firstContractionOffset = uprv_cnttab_addContraction(contractions, UPRV_CNTTAB_NEWELEMENT, 0, existingCE, status);
1038 uint32_t newCE = uprv_uca_processContraction(contractions, element, UCOL_NOT_FOUND, status);
1039 uprv_cnttab_addContraction(contractions, firstContractionOffset, *element->cPoints, newCE, status);
1040 uprv_cnttab_addContraction(contractions, firstContractionOffset, 0xFFFF, existingCE, status);
1041 existingCE = constructContractCE(contractions->currentTag, firstContractionOffset);
1042 } else { /* we are adding to existing contraction */
1043 /* there were already some elements in the table, so we need to add a new contraction */
1044 /* Two things can happen here: either the codepoint is already in the table, or it is not */
1045 int32_t position = uprv_cnttab_findCP(contractions, existingCE, *element->cPoints, status);
1046 if(position > 0) { /* if it is we just continue down the chain */
1047 uint32_t eCE = uprv_cnttab_getCE(contractions, existingCE, position, status);
1048 uint32_t newCE = uprv_uca_processContraction(contractions, element, eCE, status);
1049 uprv_cnttab_setContraction(contractions, existingCE, position, *(element->cPoints), newCE, status);
1050 } else { /* if it isn't, we will have to create a new sequence */
1051 uint32_t newCE = uprv_uca_processContraction(contractions, element, UCOL_NOT_FOUND, status);
1052 uprv_cnttab_insertContraction(contractions, existingCE, *(element->cPoints), newCE, status);
1053 }
1054 }
1055 element->cPoints--;
1056 element->cSize++;
1057 return existingCE;
1058 }
1059
uprv_uca_finalizeAddition(tempUCATable * t,UCAElements * element,UErrorCode * status)1060 static uint32_t uprv_uca_finalizeAddition(tempUCATable *t, UCAElements *element, UErrorCode *status) {
1061 uint32_t CE = UCOL_NOT_FOUND;
1062 // This should add a completely ignorable element to the
1063 // unsafe table, so that backward iteration will skip
1064 // over it when treating contractions.
1065 uint32_t i = 0;
1066 if(element->mapCE == 0) {
1067 for(i = 0; i < element->cSize; i++) {
1068 if(!UTF_IS_TRAIL(element->cPoints[i])) {
1069 unsafeCPSet(t->unsafeCP, element->cPoints[i]);
1070 }
1071 }
1072 }
1073 if(element->cSize > 1) { /* we're adding a contraction */
1074 uint32_t i = 0;
1075 UChar32 cp;
1076
1077 UTF_NEXT_CHAR(element->cPoints, i, element->cSize, cp);
1078 /*CE = ucmpe32_get(t->mapping, cp);*/
1079 CE = utrie_get32(t->mapping, cp, NULL);
1080
1081 CE = uprv_uca_addContraction(t, CE, element, status);
1082 } else { /* easy case, */
1083 /*CE = ucmpe32_get(t->mapping, element->cPoints[0]);*/
1084 CE = utrie_get32(t->mapping, element->cPoints[0], NULL);
1085
1086 if( CE != UCOL_NOT_FOUND) {
1087 if(isCntTableElement(CE) /*isContraction(CE)*/) { /* adding a non contraction element (thai, expansion, single) to already existing contraction */
1088 if(!isPrefix(element->mapCE)) { // we cannot reenter prefix elements - as we are going to create a dead loop
1089 // Only expansions and regular CEs can go here... Contractions will never happen in this place
1090 uprv_cnttab_setContraction(t->contractions, CE, 0, 0, element->mapCE, status);
1091 /* This loop has to change the CE at the end of contraction REDO!*/
1092 uprv_cnttab_changeLastCE(t->contractions, CE, element->mapCE, status);
1093 }
1094 } else {
1095 /*ucmpe32_set(t->mapping, element->cPoints[0], element->mapCE);*/
1096 utrie_set32(t->mapping, element->cPoints[0], element->mapCE);
1097 if ((element->prefixSize!=0) && (getCETag(CE)!=IMPLICIT_TAG)) {
1098 UCAElements *origElem = (UCAElements *)uprv_malloc(sizeof(UCAElements));
1099 /* test for NULL */
1100 if (origElem== NULL) {
1101 *status = U_MEMORY_ALLOCATION_ERROR;
1102 return 0;
1103 }
1104 /* copy the original UCA value */
1105 origElem->prefixSize = 0;
1106 origElem->prefix = NULL;
1107 origElem->cPoints = origElem->uchars;
1108 origElem->cPoints[0] = element->cPoints[0];
1109 origElem->cSize = 1;
1110 origElem->CEs[0]=CE;
1111 origElem->mapCE=CE;
1112 origElem->noOfCEs=1;
1113 uprv_uca_finalizeAddition(t, origElem, status);
1114 uprv_free(origElem);
1115 }
1116 #ifdef UCOL_DEBUG
1117 fprintf(stderr, "Warning - trying to overwrite existing data %08X for cp %04X with %08X\n", CE, element->cPoints[0], element->CEs[0]);
1118 //*status = U_ILLEGAL_ARGUMENT_ERROR;
1119 #endif
1120 }
1121 } else {
1122 /*ucmpe32_set(t->mapping, element->cPoints[0], element->mapCE);*/
1123 utrie_set32(t->mapping, element->cPoints[0], element->mapCE);
1124 }
1125 }
1126 return CE;
1127 }
1128
1129 /* This adds a read element, while testing for existence */
1130 U_CAPI uint32_t U_EXPORT2
uprv_uca_addAnElement(tempUCATable * t,UCAElements * element,UErrorCode * status)1131 uprv_uca_addAnElement(tempUCATable *t, UCAElements *element, UErrorCode *status) {
1132 U_NAMESPACE_USE
1133
1134 ExpansionTable *expansions = t->expansions;
1135
1136 uint32_t i = 1;
1137 uint32_t expansion = 0;
1138 uint32_t CE;
1139
1140 if(U_FAILURE(*status)) {
1141 return 0xFFFF;
1142 }
1143
1144 element->mapCE = 0; // clear mapCE so that we can catch expansions
1145
1146 if(element->noOfCEs == 1) {
1147 element->mapCE = element->CEs[0];
1148 } else {
1149 /* ICU 2.1 long primaries */
1150 /* unfortunately, it looks like we have to look for a long primary here */
1151 /* since in canonical closure we are going to hit some long primaries from */
1152 /* the first phase, and they will come back as continuations/expansions */
1153 /* destroying the effect of the previous opitimization */
1154 /* A long primary is a three byte primary with starting secondaries and tertiaries */
1155 /* It can appear in long runs of only primary differences (like east Asian tailorings) */
1156 /* also, it should not be an expansion, as expansions would break with this */
1157 // This part came in from ucol_bld.cpp
1158 //if(tok->expansion == 0
1159 //&& noOfBytes[0] == 3 && noOfBytes[1] == 1 && noOfBytes[2] == 1
1160 //&& CEparts[1] == (UCOL_BYTE_COMMON << 24) && CEparts[2] == (UCOL_BYTE_COMMON << 24)) {
1161 /* we will construct a special CE that will go unchanged to the table */
1162 if(element->noOfCEs == 2 // a two CE expansion
1163 && isContinuation(element->CEs[1]) // which is a continuation
1164 && (element->CEs[1] & (~(0xFF << 24 | UCOL_CONTINUATION_MARKER))) == 0 // that has only primaries in continuation,
1165 && (((element->CEs[0]>>8) & 0xFF) == UCOL_BYTE_COMMON) // a common secondary
1166 && ((element->CEs[0] & 0xFF) == UCOL_BYTE_COMMON) // and a common tertiary
1167 )
1168 {
1169 #ifdef UCOL_DEBUG
1170 fprintf(stdout, "Long primary %04X\n", element->cPoints[0]);
1171 #endif
1172 element->mapCE = UCOL_SPECIAL_FLAG | (LONG_PRIMARY_TAG<<24) // a long primary special
1173 | ((element->CEs[0]>>8) & 0xFFFF00) // first and second byte of primary
1174 | ((element->CEs[1]>>24) & 0xFF); // third byte of primary
1175 }
1176 else {
1177 expansion = (uint32_t)(UCOL_SPECIAL_FLAG | (EXPANSION_TAG<<UCOL_TAG_SHIFT)
1178 | ((uprv_uca_addExpansion(expansions, element->CEs[0], status)+(headersize>>2))<<4)
1179 & 0xFFFFF0);
1180
1181 for(i = 1; i<element->noOfCEs; i++) {
1182 uprv_uca_addExpansion(expansions, element->CEs[i], status);
1183 }
1184 if(element->noOfCEs <= 0xF) {
1185 expansion |= element->noOfCEs;
1186 } else {
1187 uprv_uca_addExpansion(expansions, 0, status);
1188 }
1189 element->mapCE = expansion;
1190 uprv_uca_setMaxExpansion(element->CEs[element->noOfCEs - 1],
1191 (uint8_t)element->noOfCEs,
1192 t->maxExpansions,
1193 status);
1194 if(UCOL_ISJAMO(element->cPoints[0])) {
1195 t->image->jamoSpecial = TRUE;
1196 uprv_uca_setMaxJamoExpansion(element->cPoints[0],
1197 element->CEs[element->noOfCEs - 1],
1198 (uint8_t)element->noOfCEs,
1199 t->maxJamoExpansions,
1200 status);
1201 }
1202 if (U_FAILURE(*status)) {
1203 return 0;
1204 }
1205 }
1206 }
1207
1208 // We treat digits differently - they are "uber special" and should be
1209 // processed differently if numeric collation is on.
1210 UChar32 uniChar = 0;
1211 //printElement(element);
1212 if ((element->cSize == 2) && U16_IS_LEAD(element->cPoints[0])){
1213 uniChar = U16_GET_SUPPLEMENTARY(element->cPoints[0], element->cPoints[1]);
1214 } else if (element->cSize == 1){
1215 uniChar = element->cPoints[0];
1216 }
1217
1218 // Here, we either have one normal CE OR mapCE is set. Therefore, we stuff only
1219 // one element to the expansion buffer. When we encounter a digit and we don't
1220 // do numeric collation, we will just pick the CE we have and break out of case
1221 // (see ucol.cpp ucol_prv_getSpecialCE && ucol_prv_getSpecialPrevCE). If we picked
1222 // a special, further processing will occur. If it's a simple CE, we'll return due
1223 // to how the loop is constructed.
1224 if (uniChar != 0 && u_isdigit(uniChar)){
1225 expansion = (uint32_t)(UCOL_SPECIAL_FLAG | (DIGIT_TAG<<UCOL_TAG_SHIFT) | 1); // prepare the element
1226 if(element->mapCE) { // if there is an expansion, we'll pick it here
1227 expansion |= ((uprv_uca_addExpansion(expansions, element->mapCE, status)+(headersize>>2))<<4);
1228 } else {
1229 expansion |= ((uprv_uca_addExpansion(expansions, element->CEs[0], status)+(headersize>>2))<<4);
1230 }
1231 element->mapCE = expansion;
1232
1233 // Need to go back to the beginning of the digit string if in the middle!
1234 if(uniChar <= 0xFFFF) { // supplementaries are always unsafe. API takes UChars
1235 unsafeCPSet(t->unsafeCP, (UChar)uniChar);
1236 }
1237 }
1238
1239 // here we want to add the prefix structure.
1240 // I will try to process it as a reverse contraction, if possible.
1241 // prefix buffer is already reversed.
1242
1243 if(element->prefixSize!=0) {
1244 // We keep the seen prefix starter elements in a hashtable
1245 // we need it to be able to distinguish between the simple
1246 // codepoints and prefix starters. Also, we need to use it
1247 // for canonical closure.
1248
1249 UCAElements *composed = (UCAElements *)uprv_malloc(sizeof(UCAElements));
1250 /* test for NULL */
1251 if (composed == NULL) {
1252 *status = U_MEMORY_ALLOCATION_ERROR;
1253 return 0;
1254 }
1255 uprv_memcpy(composed, element, sizeof(UCAElements));
1256 composed->cPoints = composed->uchars;
1257 composed->prefix = composed->prefixChars;
1258
1259 composed->prefixSize = unorm_normalize(element->prefix, element->prefixSize, UNORM_NFC, 0, composed->prefix, 128, status);
1260
1261
1262 if(t->prefixLookup != NULL) {
1263 UCAElements *uCE = (UCAElements *)uhash_get(t->prefixLookup, element);
1264 if(uCE != NULL) { // there is already a set of code points here
1265 element->mapCE = uprv_uca_addPrefix(t, uCE->mapCE, element, status);
1266 } else { // no code points, so this spot is clean
1267 element->mapCE = uprv_uca_addPrefix(t, UCOL_NOT_FOUND, element, status);
1268 uCE = (UCAElements *)uprv_malloc(sizeof(UCAElements));
1269 /* test for NULL */
1270 if (uCE == NULL) {
1271 *status = U_MEMORY_ALLOCATION_ERROR;
1272 return 0;
1273 }
1274 uprv_memcpy(uCE, element, sizeof(UCAElements));
1275 uCE->cPoints = uCE->uchars;
1276 uhash_put(t->prefixLookup, uCE, uCE, status);
1277 }
1278 if(composed->prefixSize != element->prefixSize || uprv_memcmp(composed->prefix, element->prefix, element->prefixSize)) {
1279 // do it!
1280 composed->mapCE = uprv_uca_addPrefix(t, element->mapCE, composed, status);
1281 }
1282 }
1283 uprv_free(composed);
1284 }
1285
1286 // We need to use the canonical iterator here
1287 // the way we do it is to generate the canonically equivalent strings
1288 // for the contraction and then add the sequences that pass FCD check
1289 if(element->cSize > 1 && !(element->cSize==2 && UTF16_IS_LEAD(element->cPoints[0]) && UTF16_IS_TRAIL(element->cPoints[1]))) { // this is a contraction, we should check whether a composed form should also be included
1290 UnicodeString source(element->cPoints, element->cSize);
1291 CanonicalIterator it(source, *status);
1292 source = it.next();
1293 while(!source.isBogus()) {
1294 if(Normalizer::quickCheck(source, UNORM_FCD, *status) != UNORM_NO) {
1295 element->cSize = source.extract(element->cPoints, 128, *status);
1296 uprv_uca_finalizeAddition(t, element, status);
1297 }
1298 source = it.next();
1299 }
1300 CE = element->mapCE;
1301 } else {
1302 CE = uprv_uca_finalizeAddition(t, element, status);
1303 }
1304
1305 return CE;
1306 }
1307
1308
1309 /*void uprv_uca_getMaxExpansionJamo(CompactEIntArray *mapping, */
uprv_uca_getMaxExpansionJamo(UNewTrie * mapping,MaxExpansionTable * maxexpansion,MaxJamoExpansionTable * maxjamoexpansion,UBool jamospecial,UErrorCode * status)1310 static void uprv_uca_getMaxExpansionJamo(UNewTrie *mapping,
1311 MaxExpansionTable *maxexpansion,
1312 MaxJamoExpansionTable *maxjamoexpansion,
1313 UBool jamospecial,
1314 UErrorCode *status)
1315 {
1316 const uint32_t VBASE = 0x1161;
1317 const uint32_t TBASE = 0x11A8;
1318 const uint32_t VCOUNT = 21;
1319 const uint32_t TCOUNT = 28;
1320
1321 uint32_t v = VBASE + VCOUNT - 1;
1322 uint32_t t = TBASE + TCOUNT - 1;
1323 uint32_t ce;
1324
1325 while (v >= VBASE) {
1326 /*ce = ucmpe32_get(mapping, v);*/
1327 ce = utrie_get32(mapping, v, NULL);
1328 if (ce < UCOL_SPECIAL_FLAG) {
1329 uprv_uca_setMaxExpansion(ce, 2, maxexpansion, status);
1330 }
1331 v --;
1332 }
1333
1334 while (t >= TBASE)
1335 {
1336 /*ce = ucmpe32_get(mapping, t);*/
1337 ce = utrie_get32(mapping, t, NULL);
1338 if (ce < UCOL_SPECIAL_FLAG) {
1339 uprv_uca_setMaxExpansion(ce, 3, maxexpansion, status);
1340 }
1341 t --;
1342 }
1343 /* According to the docs, 99% of the time, the Jamo will not be special */
1344 if (jamospecial) {
1345 /* gets the max expansion in all unicode characters */
1346 int count = maxjamoexpansion->position;
1347 uint8_t maxTSize = (uint8_t)(maxjamoexpansion->maxLSize +
1348 maxjamoexpansion->maxVSize +
1349 maxjamoexpansion->maxTSize);
1350 uint8_t maxVSize = (uint8_t)(maxjamoexpansion->maxLSize +
1351 maxjamoexpansion->maxVSize);
1352
1353 while (count > 0) {
1354 count --;
1355 if (*(maxjamoexpansion->isV + count) == TRUE) {
1356 uprv_uca_setMaxExpansion(
1357 *(maxjamoexpansion->endExpansionCE + count),
1358 maxVSize, maxexpansion, status);
1359 }
1360 else {
1361 uprv_uca_setMaxExpansion(
1362 *(maxjamoexpansion->endExpansionCE + count),
1363 maxTSize, maxexpansion, status);
1364 }
1365 }
1366 }
1367 }
1368
1369 U_CDECL_BEGIN
1370 static inline uint32_t U_CALLCONV
getFoldedValue(UNewTrie * trie,UChar32 start,int32_t offset)1371 getFoldedValue(UNewTrie *trie, UChar32 start, int32_t offset)
1372 {
1373 uint32_t value;
1374 uint32_t tag;
1375 UChar32 limit;
1376 UBool inBlockZero;
1377
1378 limit=start+0x400;
1379 while(start<limit) {
1380 value=utrie_get32(trie, start, &inBlockZero);
1381 tag = getCETag(value);
1382 if(inBlockZero == TRUE) {
1383 start+=UTRIE_DATA_BLOCK_LENGTH;
1384 } else if(!(isSpecial(value) && (tag == IMPLICIT_TAG || tag == NOT_FOUND_TAG))) {
1385 /* These are values that are starting in either UCA (IMPLICIT_TAG) or in the
1386 * tailorings (NOT_FOUND_TAG). Presence of these tags means that there is
1387 * nothing in this position and that it should be skipped.
1388 */
1389 #ifdef UCOL_DEBUG
1390 static int32_t count = 1;
1391 fprintf(stdout, "%i, Folded %08X, value %08X\n", count++, start, value);
1392 #endif
1393 return (uint32_t)(UCOL_SPECIAL_FLAG | (SURROGATE_TAG<<24) | offset);
1394 } else {
1395 ++start;
1396 }
1397 }
1398 return 0;
1399 }
1400 U_CDECL_END
1401
1402 #ifdef UCOL_DEBUG
1403 // This is a debug function to print the contents of a trie.
1404 // It is used in conjuction with the code around utrie_unserialize call
enumRange(const void * context,UChar32 start,UChar32 limit,uint32_t value)1405 void enumRange(const void *context, UChar32 start, UChar32 limit, uint32_t value) {
1406 if(start<0x10000) {
1407 fprintf(stdout, "%08X, %08X, %08X\n", start, limit, value);
1408 } else {
1409 fprintf(stdout, "%08X=%04X %04X, %08X=%04X %04X, %08X\n", start, UTF16_LEAD(start), UTF16_TRAIL(start), limit, UTF16_LEAD(limit), UTF16_TRAIL(limit), value);
1410 }
1411 }
1412
1413 int32_t
myGetFoldingOffset(uint32_t data)1414 myGetFoldingOffset(uint32_t data) {
1415 if(data > UCOL_NOT_FOUND && getCETag(data) == SURROGATE_TAG) {
1416 return (data&0xFFFFFF);
1417 } else {
1418 return 0;
1419 }
1420 }
1421 #endif
1422
1423 U_CAPI UCATableHeader* U_EXPORT2
uprv_uca_assembleTable(tempUCATable * t,UErrorCode * status)1424 uprv_uca_assembleTable(tempUCATable *t, UErrorCode *status) {
1425 /*CompactEIntArray *mapping = t->mapping;*/
1426 UNewTrie *mapping = t->mapping;
1427 ExpansionTable *expansions = t->expansions;
1428 CntTable *contractions = t->contractions;
1429 MaxExpansionTable *maxexpansion = t->maxExpansions;
1430
1431 if(U_FAILURE(*status)) {
1432 return NULL;
1433 }
1434
1435 uint32_t beforeContractions = (uint32_t)((headersize+paddedsize(expansions->position*sizeof(uint32_t)))/sizeof(UChar));
1436
1437 int32_t contractionsSize = 0;
1438 contractionsSize = uprv_cnttab_constructTable(contractions, beforeContractions, status);
1439
1440 /* the following operation depends on the trie data. Therefore, we have to do it before */
1441 /* the trie is compacted */
1442 /* sets jamo expansions */
1443 uprv_uca_getMaxExpansionJamo(mapping, maxexpansion, t->maxJamoExpansions,
1444 t->image->jamoSpecial, status);
1445
1446 /*ucmpe32_compact(mapping);*/
1447 /*UMemoryStream *ms = uprv_mstrm_openNew(8192);*/
1448 /*int32_t mappingSize = ucmpe32_flattenMem(mapping, ms);*/
1449 /*const uint8_t *flattened = uprv_mstrm_getBuffer(ms, &mappingSize);*/
1450
1451 // After setting the jamo expansions, compact the trie and get the needed size
1452 int32_t mappingSize = utrie_serialize(mapping, NULL, 0, getFoldedValue /*getFoldedValue*/, FALSE, status);
1453
1454 uint32_t tableOffset = 0;
1455 uint8_t *dataStart;
1456
1457 /* TODO: LATIN1 array is now in the utrie - it should be removed from the calculation */
1458
1459 uint32_t toAllocate =(uint32_t)(headersize+
1460 paddedsize(expansions->position*sizeof(uint32_t))+
1461 paddedsize(mappingSize)+
1462 paddedsize(contractionsSize*(sizeof(UChar)+sizeof(uint32_t)))+
1463 //paddedsize(0x100*sizeof(uint32_t)) /* Latin1 is now included in the trie */
1464 /* maxexpansion array */
1465 + paddedsize(maxexpansion->position * sizeof(uint32_t)) +
1466 /* maxexpansion size array */
1467 paddedsize(maxexpansion->position * sizeof(uint8_t)) +
1468 paddedsize(UCOL_UNSAFECP_TABLE_SIZE) + /* Unsafe chars */
1469 paddedsize(UCOL_UNSAFECP_TABLE_SIZE)); /* Contraction Ending chars */
1470
1471
1472 dataStart = (uint8_t *)uprv_malloc(toAllocate);
1473 /* test for NULL */
1474 if (dataStart == NULL) {
1475 *status = U_MEMORY_ALLOCATION_ERROR;
1476 return NULL;
1477 }
1478
1479 UCATableHeader *myData = (UCATableHeader *)dataStart;
1480 // Please, do reset all the fields!
1481 uprv_memset(dataStart, 0, toAllocate);
1482 // Make sure we know this is reset
1483 myData->magic = UCOL_HEADER_MAGIC;
1484 myData->isBigEndian = U_IS_BIG_ENDIAN;
1485 myData->charSetFamily = U_CHARSET_FAMILY;
1486 myData->formatVersion[0] = UCA_FORMAT_VERSION_0;
1487 myData->formatVersion[1] = UCA_FORMAT_VERSION_1;
1488 myData->formatVersion[2] = UCA_FORMAT_VERSION_2;
1489 myData->formatVersion[3] = UCA_FORMAT_VERSION_3;
1490 myData->jamoSpecial = t->image->jamoSpecial;
1491
1492 // Don't copy stuff from UCA header!
1493 //uprv_memcpy(myData, t->image, sizeof(UCATableHeader));
1494
1495 myData->contractionSize = contractionsSize;
1496
1497 tableOffset += (uint32_t)(paddedsize(sizeof(UCATableHeader)));
1498
1499 myData->options = tableOffset;
1500 uprv_memcpy(dataStart+tableOffset, t->options, sizeof(UColOptionSet));
1501 tableOffset += (uint32_t)(paddedsize(sizeof(UColOptionSet)));
1502
1503 /* copy expansions */
1504 /*myData->expansion = (uint32_t *)dataStart+tableOffset;*/
1505 myData->expansion = tableOffset;
1506 uprv_memcpy(dataStart+tableOffset, expansions->CEs, expansions->position*sizeof(uint32_t));
1507 tableOffset += (uint32_t)(paddedsize(expansions->position*sizeof(uint32_t)));
1508
1509 /* contractions block */
1510 if(contractionsSize != 0) {
1511 /* copy contraction index */
1512 /*myData->contractionIndex = (UChar *)(dataStart+tableOffset);*/
1513 myData->contractionIndex = tableOffset;
1514 uprv_memcpy(dataStart+tableOffset, contractions->codePoints, contractionsSize*sizeof(UChar));
1515 tableOffset += (uint32_t)(paddedsize(contractionsSize*sizeof(UChar)));
1516
1517 /* copy contraction collation elements */
1518 /*myData->contractionCEs = (uint32_t *)(dataStart+tableOffset);*/
1519 myData->contractionCEs = tableOffset;
1520 uprv_memcpy(dataStart+tableOffset, contractions->CEs, contractionsSize*sizeof(uint32_t));
1521 tableOffset += (uint32_t)(paddedsize(contractionsSize*sizeof(uint32_t)));
1522 } else {
1523 myData->contractionIndex = 0;
1524 myData->contractionCEs = 0;
1525 }
1526
1527 /* copy mapping table */
1528 /*myData->mappingPosition = dataStart+tableOffset;*/
1529 /*myData->mappingPosition = tableOffset;*/
1530 /*uprv_memcpy(dataStart+tableOffset, flattened, mappingSize);*/
1531
1532 myData->mappingPosition = tableOffset;
1533 utrie_serialize(mapping, dataStart+tableOffset, toAllocate-tableOffset, getFoldedValue, FALSE, status);
1534 #ifdef UCOL_DEBUG
1535 // This is debug code to dump the contents of the trie. It needs two functions defined above
1536 {
1537 UTrie UCAt = { 0 };
1538 uint32_t trieWord;
1539 utrie_unserialize(&UCAt, dataStart+tableOffset, 9999999, status);
1540 UCAt.getFoldingOffset = myGetFoldingOffset;
1541 if(U_SUCCESS(*status)) {
1542 utrie_enum(&UCAt, NULL, enumRange, NULL);
1543 }
1544 trieWord = UTRIE_GET32_FROM_LEAD(UCAt, 0xDC01)
1545 }
1546 #endif
1547 tableOffset += paddedsize(mappingSize);
1548
1549
1550 int32_t i = 0;
1551
1552 /* copy max expansion table */
1553 myData->endExpansionCE = tableOffset;
1554 myData->endExpansionCECount = maxexpansion->position - 1;
1555 /* not copying the first element which is a dummy */
1556 uprv_memcpy(dataStart + tableOffset, maxexpansion->endExpansionCE + 1,
1557 (maxexpansion->position - 1) * sizeof(uint32_t));
1558 tableOffset += (uint32_t)(paddedsize((maxexpansion->position)* sizeof(uint32_t)));
1559 myData->expansionCESize = tableOffset;
1560 uprv_memcpy(dataStart + tableOffset, maxexpansion->expansionCESize + 1,
1561 (maxexpansion->position - 1) * sizeof(uint8_t));
1562 tableOffset += (uint32_t)(paddedsize((maxexpansion->position)* sizeof(uint8_t)));
1563
1564 /* Unsafe chars table. Finish it off, then copy it. */
1565 uprv_uca_unsafeCPAddCCNZ(t, status);
1566 if (t->UCA != 0) { /* Or in unsafebits from UCA, making a combined table. */
1567 for (i=0; i<UCOL_UNSAFECP_TABLE_SIZE; i++) {
1568 t->unsafeCP[i] |= t->UCA->unsafeCP[i];
1569 }
1570 }
1571 myData->unsafeCP = tableOffset;
1572 uprv_memcpy(dataStart + tableOffset, t->unsafeCP, UCOL_UNSAFECP_TABLE_SIZE);
1573 tableOffset += paddedsize(UCOL_UNSAFECP_TABLE_SIZE);
1574
1575
1576 /* Finish building Contraction Ending chars hash table and then copy it out. */
1577 if (t->UCA != 0) { /* Or in unsafebits from UCA, making a combined table. */
1578 for (i=0; i<UCOL_UNSAFECP_TABLE_SIZE; i++) {
1579 t->contrEndCP[i] |= t->UCA->contrEndCP[i];
1580 }
1581 }
1582 myData->contrEndCP = tableOffset;
1583 uprv_memcpy(dataStart + tableOffset, t->contrEndCP, UCOL_UNSAFECP_TABLE_SIZE);
1584 tableOffset += paddedsize(UCOL_UNSAFECP_TABLE_SIZE);
1585
1586 if(tableOffset != toAllocate) {
1587 #ifdef UCOL_DEBUG
1588 fprintf(stderr, "calculation screwup!!! Expected to write %i but wrote %i instead!!!\n", toAllocate, tableOffset);
1589 #endif
1590 *status = U_INTERNAL_PROGRAM_ERROR;
1591 uprv_free(dataStart);
1592 return 0;
1593 }
1594
1595 myData->size = tableOffset;
1596 /* This should happen upon ressurection */
1597 /*const uint8_t *mapPosition = (uint8_t*)myData+myData->mappingPosition;*/
1598 /*uprv_mstrm_close(ms);*/
1599 return myData;
1600 }
1601
1602
1603 struct enumStruct {
1604 tempUCATable *t;
1605 UCollator *tempColl;
1606 UCollationElements* colEl;
1607 const Normalizer2Impl *nfcImpl;
1608 int32_t noOfClosures;
1609 UErrorCode *status;
1610 };
1611 U_CDECL_BEGIN
1612 static UBool U_CALLCONV
_enumCategoryRangeClosureCategory(const void * context,UChar32 start,UChar32 limit,UCharCategory type)1613 _enumCategoryRangeClosureCategory(const void *context, UChar32 start, UChar32 limit, UCharCategory type) {
1614
1615 if (type != U_UNASSIGNED && type != U_PRIVATE_USE_CHAR) { // if the range is assigned - we might ommit more categories later
1616 UErrorCode *status = ((enumStruct *)context)->status;
1617 tempUCATable *t = ((enumStruct *)context)->t;
1618 UCollator *tempColl = ((enumStruct *)context)->tempColl;
1619 UCollationElements* colEl = ((enumStruct *)context)->colEl;
1620 UCAElements el;
1621 UChar decompBuffer[4];
1622 const UChar *decomp;
1623 int32_t noOfDec = 0;
1624
1625 UChar32 u32 = 0;
1626 UChar comp[2];
1627 uint32_t len = 0;
1628
1629 for(u32 = start; u32 < limit; u32++) {
1630 decomp = ((enumStruct *)context)->nfcImpl->
1631 getDecomposition(u32, decompBuffer, noOfDec);
1632 //if((noOfDec = unorm_normalize(comp, len, UNORM_NFD, 0, decomp, 256, status)) > 1
1633 //|| (noOfDec == 1 && *decomp != (UChar)u32))
1634 if(decomp != NULL)
1635 {
1636 len = 0;
1637 U16_APPEND_UNSAFE(comp, len, u32);
1638 if(ucol_strcoll(tempColl, comp, len, decomp, noOfDec) != UCOL_EQUAL) {
1639 #ifdef UCOL_DEBUG
1640 fprintf(stderr, "Closure: %08X -> ", u32);
1641 uint32_t i = 0;
1642 for(i = 0; i<noOfDec; i++) {
1643 fprintf(stderr, "%04X ", decomp[i]);
1644 }
1645 fprintf(stderr, "\n");
1646 #endif
1647 ((enumStruct *)context)->noOfClosures++;
1648 el.cPoints = (UChar *)decomp;
1649 el.cSize = noOfDec;
1650 el.noOfCEs = 0;
1651 el.prefix = el.prefixChars;
1652 el.prefixSize = 0;
1653
1654 UCAElements *prefix=(UCAElements *)uhash_get(t->prefixLookup, &el);
1655 el.cPoints = comp;
1656 el.cSize = len;
1657 el.prefix = el.prefixChars;
1658 el.prefixSize = 0;
1659 if(prefix == NULL) {
1660 el.noOfCEs = 0;
1661 ucol_setText(colEl, decomp, noOfDec, status);
1662 while((el.CEs[el.noOfCEs] = ucol_next(colEl, status)) != (uint32_t)UCOL_NULLORDER) {
1663 el.noOfCEs++;
1664 }
1665 } else {
1666 el.noOfCEs = 1;
1667 el.CEs[0] = prefix->mapCE;
1668 // This character uses a prefix. We have to add it
1669 // to the unsafe table, as it decomposed form is already
1670 // in. In Japanese, this happens for \u309e & \u30fe
1671 // Since unsafeCPSet is static in ucol_elm, we are going
1672 // to wrap it up in the uprv_uca_unsafeCPAddCCNZ function
1673 }
1674 uprv_uca_addAnElement(t, &el, status);
1675 }
1676 }
1677 }
1678 }
1679 return TRUE;
1680 }
1681 U_CDECL_END
1682
1683 static void
uprv_uca_setMapCE(tempUCATable * t,UCAElements * element,UErrorCode * status)1684 uprv_uca_setMapCE(tempUCATable *t, UCAElements *element, UErrorCode *status) {
1685 uint32_t expansion = 0;
1686 int32_t j;
1687
1688 ExpansionTable *expansions = t->expansions;
1689 if(element->noOfCEs == 2 // a two CE expansion
1690 && isContinuation(element->CEs[1]) // which is a continuation
1691 && (element->CEs[1] & (~(0xFF << 24 | UCOL_CONTINUATION_MARKER))) == 0 // that has only primaries in continuation,
1692 && (((element->CEs[0]>>8) & 0xFF) == UCOL_BYTE_COMMON) // a common secondary
1693 && ((element->CEs[0] & 0xFF) == UCOL_BYTE_COMMON) // and a common tertiary
1694 ) {
1695 element->mapCE = UCOL_SPECIAL_FLAG | (LONG_PRIMARY_TAG<<24) // a long primary special
1696 | ((element->CEs[0]>>8) & 0xFFFF00) // first and second byte of primary
1697 | ((element->CEs[1]>>24) & 0xFF); // third byte of primary
1698 } else {
1699 expansion = (uint32_t)(UCOL_SPECIAL_FLAG | (EXPANSION_TAG<<UCOL_TAG_SHIFT)
1700 | ((uprv_uca_addExpansion(expansions, element->CEs[0], status)+(headersize>>2))<<4)
1701 & 0xFFFFF0);
1702
1703 for(j = 1; j<(int32_t)element->noOfCEs; j++) {
1704 uprv_uca_addExpansion(expansions, element->CEs[j], status);
1705 }
1706 if(element->noOfCEs <= 0xF) {
1707 expansion |= element->noOfCEs;
1708 } else {
1709 uprv_uca_addExpansion(expansions, 0, status);
1710 }
1711 element->mapCE = expansion;
1712 uprv_uca_setMaxExpansion(element->CEs[element->noOfCEs - 1],
1713 (uint8_t)element->noOfCEs,
1714 t->maxExpansions,
1715 status);
1716 }
1717 }
1718
1719 static void
uprv_uca_addFCD4AccentedContractions(tempUCATable * t,UCollationElements * colEl,UChar * data,int32_t len,UCAElements * el,UErrorCode * status)1720 uprv_uca_addFCD4AccentedContractions(tempUCATable *t,
1721 UCollationElements* colEl,
1722 UChar *data,
1723 int32_t len,
1724 UCAElements *el,
1725 UErrorCode *status) {
1726 UChar decomp[256], comp[256];
1727 int32_t decLen, compLen;
1728
1729 decLen = unorm_normalize(data, len, UNORM_NFD, 0, decomp, 256, status);
1730 compLen = unorm_normalize(data, len, UNORM_NFC, 0, comp, 256, status);
1731 decomp[decLen] = comp[compLen] = 0;
1732
1733 el->cPoints = decomp;
1734 el->cSize = decLen;
1735 el->noOfCEs = 0;
1736 el->prefixSize = 0;
1737 el->prefix = el->prefixChars;
1738
1739 UCAElements *prefix=(UCAElements *)uhash_get(t->prefixLookup, el);
1740 el->cPoints = comp;
1741 el->cSize = compLen;
1742 el->prefix = el->prefixChars;
1743 el->prefixSize = 0;
1744 if(prefix == NULL) {
1745 el->noOfCEs = 0;
1746 ucol_setText(colEl, decomp, decLen, status);
1747 while((el->CEs[el->noOfCEs] = ucol_next(colEl, status)) != (uint32_t)UCOL_NULLORDER) {
1748 el->noOfCEs++;
1749 }
1750 uprv_uca_setMapCE(t, el, status);
1751 uprv_uca_addAnElement(t, el, status);
1752 }
1753 }
1754
1755 static void
uprv_uca_addMultiCMContractions(tempUCATable * t,UCollationElements * colEl,tempTailorContext * c,UCAElements * el,UErrorCode * status)1756 uprv_uca_addMultiCMContractions(tempUCATable *t,
1757 UCollationElements* colEl,
1758 tempTailorContext *c,
1759 UCAElements *el,
1760 UErrorCode *status) {
1761 CombinClassTable *cmLookup = t->cmLookup;
1762 UChar newDecomp[256];
1763 int32_t maxComp, newDecLen;
1764 UChar32 fcdHighStart;
1765 const uint16_t *fcdTrieIndex = unorm_getFCDTrieIndex(fcdHighStart, status);
1766 if (U_FAILURE(*status)) {
1767 return;
1768 }
1769 int16_t curClass = (unorm_getFCD16(fcdTrieIndex, c->tailoringCM) & 0xff);
1770 CompData *precomp = c->precomp;
1771 int32_t compLen = c->compLen;
1772 UChar *comp = c->comp;
1773 maxComp = c->precompLen;
1774
1775 for (int32_t j=0; j < maxComp; j++) {
1776 int32_t count=0;
1777 do {
1778 if ( count == 0 ) { // Decompose the saved precomposed char.
1779 UChar temp[2];
1780 temp[0]=precomp[j].cp;
1781 temp[1]=0;
1782 newDecLen = unorm_normalize(temp, 1, UNORM_NFD, 0,
1783 newDecomp, sizeof(newDecomp)/sizeof(UChar), status);
1784 newDecomp[newDecLen++] = cmLookup->cPoints[c->cmPos];
1785 }
1786 else { // swap 2 combining marks when they are equal.
1787 uprv_memcpy(newDecomp, c->decomp, sizeof(UChar)*(c->decompLen));
1788 newDecLen = c->decompLen;
1789 newDecomp[newDecLen++] = precomp[j].cClass;
1790 }
1791 newDecomp[newDecLen] = 0;
1792 compLen = unorm_normalize(newDecomp, newDecLen, UNORM_NFC, 0,
1793 comp, 256, status);
1794 if (compLen==1) {
1795 comp[compLen++] = newDecomp[newDecLen++] = c->tailoringCM;
1796 comp[compLen] = newDecomp[newDecLen] = 0;
1797 el->cPoints = newDecomp;
1798 el->cSize = newDecLen;
1799
1800 UCAElements *prefix=(UCAElements *)uhash_get(t->prefixLookup, el);
1801 el->cPoints = c->comp;
1802 el->cSize = compLen;
1803 el->prefix = el->prefixChars;
1804 el->prefixSize = 0;
1805 if(prefix == NULL) {
1806 el->noOfCEs = 0;
1807 ucol_setText(colEl, newDecomp, newDecLen, status);
1808 while((el->CEs[el->noOfCEs] = ucol_next(colEl, status)) != (uint32_t)UCOL_NULLORDER) {
1809 el->noOfCEs++;
1810 }
1811 uprv_uca_setMapCE(t, el, status);
1812 uprv_uca_finalizeAddition(t, el, status);
1813
1814 // Save the current precomposed char and its class to find any
1815 // other combining mark combinations.
1816 precomp[c->precompLen].cp=comp[0];
1817 precomp[c->precompLen].cClass = curClass;
1818 c->precompLen++;
1819 }
1820 }
1821 } while (++count<2 && (precomp[j].cClass == curClass));
1822 }
1823
1824 }
1825
1826 static void
uprv_uca_addTailCanonicalClosures(tempUCATable * t,UCollationElements * colEl,UChar baseCh,UChar cMark,UCAElements * el,UErrorCode * status)1827 uprv_uca_addTailCanonicalClosures(tempUCATable *t,
1828 UCollationElements* colEl,
1829 UChar baseCh,
1830 UChar cMark,
1831 UCAElements *el,
1832 UErrorCode *status) {
1833 CombinClassTable *cmLookup = t->cmLookup;
1834 UChar32 fcdHighStart;
1835 const uint16_t *fcdTrieIndex = unorm_getFCDTrieIndex(fcdHighStart, status);
1836 if (U_FAILURE(*status)) {
1837 return;
1838 }
1839 int16_t maxIndex = (unorm_getFCD16(fcdTrieIndex, cMark) & 0xff );
1840 UCAElements element;
1841 uint16_t *index;
1842 UChar decomp[256];
1843 UChar comp[256];
1844 CompData precomp[256]; // precomposed array
1845 int32_t precompLen = 0; // count for precomp
1846 int32_t i, len, decompLen, curClass, replacedPos;
1847 tempTailorContext c;
1848
1849 if ( cmLookup == NULL ) {
1850 return;
1851 }
1852 index = cmLookup->index;
1853 int32_t cClass=(unorm_getFCD16(fcdTrieIndex, cMark) & 0xff);
1854 maxIndex = (int32_t)index[(unorm_getFCD16(fcdTrieIndex, cMark) & 0xff)-1];
1855 c.comp = comp;
1856 c.decomp = decomp;
1857 c.precomp = precomp;
1858 c.tailoringCM = cMark;
1859
1860 if (cClass>0) {
1861 maxIndex = (int32_t)index[cClass-1];
1862 }
1863 else {
1864 maxIndex=0;
1865 }
1866 decomp[0]=baseCh;
1867 for ( i=0; i<maxIndex ; i++ ) {
1868 decomp[1] = cmLookup->cPoints[i];
1869 decomp[2]=0;
1870 decompLen=2;
1871 len = unorm_normalize(decomp, decompLen, UNORM_NFC, 0, comp, 256, status);
1872 if (len==1) {
1873 // Save the current precomposed char and its class to find any
1874 // other combining mark combinations.
1875 precomp[precompLen].cp=comp[0];
1876 curClass = precomp[precompLen].cClass =
1877 index[unorm_getFCD16(fcdTrieIndex, decomp[1]) & 0xff];
1878 precompLen++;
1879 replacedPos=0;
1880 for (decompLen=0; decompLen< (int32_t)el->cSize; decompLen++) {
1881 decomp[decompLen] = el->cPoints[decompLen];
1882 if (decomp[decompLen]==cMark) {
1883 replacedPos = decompLen; // record the position for later use
1884 }
1885 }
1886 if ( replacedPos != 0 ) {
1887 decomp[replacedPos]=cmLookup->cPoints[i];
1888 }
1889 decomp[decompLen] = 0;
1890 len = unorm_normalize(decomp, decompLen, UNORM_NFC, 0, comp, 256, status);
1891 comp[len++] = decomp[decompLen++] = cMark;
1892 comp[len] = decomp[decompLen] = 0;
1893 element.cPoints = decomp;
1894 element.cSize = decompLen;
1895 element.noOfCEs = 0;
1896 element.prefix = el->prefixChars;
1897 element.prefixSize = 0;
1898
1899 UCAElements *prefix=(UCAElements *)uhash_get(t->prefixLookup, &element);
1900 element.cPoints = comp;
1901 element.cSize = len;
1902 element.prefix = el->prefixChars;
1903 element.prefixSize = 0;
1904 if(prefix == NULL) {
1905 element.noOfCEs = 0;
1906 ucol_setText(colEl, decomp, decompLen, status);
1907 while((element.CEs[element.noOfCEs] = ucol_next(colEl, status)) != (uint32_t)UCOL_NULLORDER) {
1908 element.noOfCEs++;
1909 }
1910 uprv_uca_setMapCE(t, &element, status);
1911 uprv_uca_finalizeAddition(t, &element, status);
1912 }
1913
1914 // This is a fix for tailoring contractions with accented
1915 // character at the end of contraction string.
1916 if ((len>2) &&
1917 (unorm_getFCD16(fcdTrieIndex, comp[len-2]) & 0xff00)==0) {
1918 uprv_uca_addFCD4AccentedContractions(t, colEl, comp, len, &element, status);
1919 }
1920
1921 if (precompLen >1) {
1922 c.compLen = len;
1923 c.decompLen = decompLen;
1924 c.precompLen = precompLen;
1925 c.cmPos = i;
1926 uprv_uca_addMultiCMContractions(t, colEl, &c, &element, status);
1927 precompLen = c.precompLen;
1928 }
1929 }
1930 }
1931 }
1932
1933 U_CFUNC int32_t U_EXPORT2
uprv_uca_canonicalClosure(tempUCATable * t,UColTokenParser * src,UErrorCode * status)1934 uprv_uca_canonicalClosure(tempUCATable *t,
1935 UColTokenParser *src,
1936 UErrorCode *status)
1937 {
1938 enumStruct context;
1939 context.noOfClosures = 0;
1940 UCAElements el;
1941 UColToken *tok;
1942 uint32_t i = 0, j = 0;
1943 UChar baseChar, firstCM;
1944 UChar32 fcdHighStart;
1945 const uint16_t *fcdTrieIndex = unorm_getFCDTrieIndex(fcdHighStart, status);
1946 context.nfcImpl=Normalizer2Factory::getNFCImpl(*status);
1947 if(U_FAILURE(*status)) {
1948 return 0;
1949 }
1950
1951 UCollator *tempColl = NULL;
1952 tempUCATable *tempTable = uprv_uca_cloneTempTable(t, status);
1953 // Check for null pointer
1954 if (U_FAILURE(*status)) {
1955 return 0;
1956 }
1957
1958 UCATableHeader *tempData = uprv_uca_assembleTable(tempTable, status);
1959 tempColl = ucol_initCollator(tempData, 0, t->UCA, status);
1960 if ( tempTable->cmLookup != NULL ) {
1961 t->cmLookup = tempTable->cmLookup; // copy over to t
1962 tempTable->cmLookup = NULL;
1963 }
1964 uprv_uca_closeTempTable(tempTable);
1965
1966 if(U_SUCCESS(*status)) {
1967 tempColl->ucaRules = NULL;
1968 tempColl->actualLocale = NULL;
1969 tempColl->validLocale = NULL;
1970 tempColl->requestedLocale = NULL;
1971 tempColl->hasRealData = TRUE;
1972 tempColl->freeImageOnClose = TRUE;
1973 } else if(tempData != 0) {
1974 uprv_free(tempData);
1975 }
1976
1977 /* produce canonical closure */
1978 UCollationElements* colEl = ucol_openElements(tempColl, NULL, 0, status);
1979 // Check for null pointer
1980 if (U_FAILURE(*status)) {
1981 return 0;
1982 }
1983 context.t = t;
1984 context.tempColl = tempColl;
1985 context.colEl = colEl;
1986 context.status = status;
1987 u_enumCharTypes(_enumCategoryRangeClosureCategory, &context);
1988
1989 if ( (src==NULL) || !src->buildCCTabFlag ) {
1990 ucol_closeElements(colEl);
1991 ucol_close(tempColl);
1992 return context.noOfClosures; // no extra contraction needed to add
1993 }
1994
1995 for (i=0; i < src->resultLen; i++) {
1996 baseChar = firstCM= (UChar)0;
1997 tok = src->lh[i].first;
1998 while (tok != NULL && U_SUCCESS(*status)) {
1999 el.prefix = el.prefixChars;
2000 el.cPoints = el.uchars;
2001 if(tok->prefix != 0) {
2002 el.prefixSize = tok->prefix>>24;
2003 uprv_memcpy(el.prefix, src->source + (tok->prefix & 0x00FFFFFF), el.prefixSize*sizeof(UChar));
2004
2005 el.cSize = (tok->source >> 24)-(tok->prefix>>24);
2006 uprv_memcpy(el.uchars, (tok->source & 0x00FFFFFF)+(tok->prefix>>24) + src->source, el.cSize*sizeof(UChar));
2007 } else {
2008 el.prefixSize = 0;
2009 *el.prefix = 0;
2010
2011 el.cSize = (tok->source >> 24);
2012 uprv_memcpy(el.uchars, (tok->source & 0x00FFFFFF) + src->source, el.cSize*sizeof(UChar));
2013 }
2014 if(src->UCA != NULL) {
2015 for(j = 0; j<el.cSize; j++) {
2016 int16_t fcd = unorm_getFCD16(fcdTrieIndex, el.cPoints[j]);
2017 if ( (fcd & 0xff) == 0 ) {
2018 baseChar = el.cPoints[j]; // last base character
2019 firstCM=0; // reset combining mark value
2020 }
2021 else {
2022 if ( (baseChar!=0) && (firstCM==0) ) {
2023 firstCM = el.cPoints[j]; // first combining mark
2024 }
2025 }
2026 }
2027 }
2028 if ( (baseChar!= (UChar)0) && (firstCM != (UChar)0) ) {
2029 // find all the canonical rules
2030 uprv_uca_addTailCanonicalClosures(t, colEl, baseChar, firstCM, &el, status);
2031 }
2032 tok = tok->next;
2033 }
2034 }
2035 ucol_closeElements(colEl);
2036 ucol_close(tempColl);
2037
2038 return context.noOfClosures;
2039 }
2040
2041 #endif /* #if !UCONFIG_NO_COLLATION */
2042