1 // © 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
3 /*
4 *******************************************************************************
5 *
6 * Copyright (C) 2003-2013, International Business Machines
7 * Corporation and others. All Rights Reserved.
8 *
9 *******************************************************************************
10 * file name: ucm.c
11 * encoding: UTF-8
12 * tab size: 8 (not used)
13 * indentation:4
14 *
15 * created on: 2003jun20
16 * created by: Markus W. Scherer
17 *
18 * This file reads a .ucm file, stores its mappings and sorts them.
19 * It implements handling of Unicode conversion mappings from .ucm files
20 * for makeconv, canonucm, rptp2ucm, etc.
21 *
22 * Unicode code point sequences with a length of more than 1,
23 * as well as byte sequences with more than 4 bytes or more than one complete
24 * character sequence are handled to support m:n mappings.
25 */
26
27 #include "unicode/utypes.h"
28 #include "unicode/ustring.h"
29 #include "cstring.h"
30 #include "cmemory.h"
31 #include "filestrm.h"
32 #include "uarrsort.h"
33 #include "ucnvmbcs.h"
34 #include "ucnv_bld.h"
35 #include "ucnv_ext.h"
36 #include "uparse.h"
37 #include "ucm.h"
38 #include <stdio.h>
39
40 #if !UCONFIG_NO_CONVERSION
41
42 /* -------------------------------------------------------------------------- */
43
44 static void
printMapping(UCMapping * m,UChar32 * codePoints,uint8_t * bytes,FILE * f)45 printMapping(UCMapping *m, UChar32 *codePoints, uint8_t *bytes, FILE *f) {
46 int32_t j;
47
48 for(j=0; j<m->uLen; ++j) {
49 fprintf(f, "<U%04lX>", (long)codePoints[j]);
50 }
51
52 fputc(' ', f);
53
54 for(j=0; j<m->bLen; ++j) {
55 fprintf(f, "\\x%02X", bytes[j]);
56 }
57
58 if(m->f>=0) {
59 fprintf(f, " |%u\n", m->f);
60 } else {
61 fputs("\n", f);
62 }
63 }
64
65 U_CAPI void U_EXPORT2
ucm_printMapping(UCMTable * table,UCMapping * m,FILE * f)66 ucm_printMapping(UCMTable *table, UCMapping *m, FILE *f) {
67 printMapping(m, UCM_GET_CODE_POINTS(table, m), UCM_GET_BYTES(table, m), f);
68 }
69
70 U_CAPI void U_EXPORT2
ucm_printTable(UCMTable * table,FILE * f,UBool byUnicode)71 ucm_printTable(UCMTable *table, FILE *f, UBool byUnicode) {
72 UCMapping *m;
73 int32_t i, length;
74
75 m=table->mappings;
76 length=table->mappingsLength;
77 if(byUnicode) {
78 for(i=0; i<length; ++m, ++i) {
79 ucm_printMapping(table, m, f);
80 }
81 } else {
82 const int32_t *map=table->reverseMap;
83 for(i=0; i<length; ++i) {
84 ucm_printMapping(table, m+map[i], f);
85 }
86 }
87 }
88
89 /* mapping comparisons ------------------------------------------------------ */
90
91 static int32_t
compareUnicode(UCMTable * lTable,const UCMapping * l,UCMTable * rTable,const UCMapping * r)92 compareUnicode(UCMTable *lTable, const UCMapping *l,
93 UCMTable *rTable, const UCMapping *r) {
94 const UChar32 *lu, *ru;
95 int32_t result, i, length;
96
97 if(l->uLen==1 && r->uLen==1) {
98 /* compare two single code points */
99 return l->u-r->u;
100 }
101
102 /* get pointers to the code point sequences */
103 lu=UCM_GET_CODE_POINTS(lTable, l);
104 ru=UCM_GET_CODE_POINTS(rTable, r);
105
106 /* get the minimum length */
107 if(l->uLen<=r->uLen) {
108 length=l->uLen;
109 } else {
110 length=r->uLen;
111 }
112
113 /* compare the code points */
114 for(i=0; i<length; ++i) {
115 result=lu[i]-ru[i];
116 if(result!=0) {
117 return result;
118 }
119 }
120
121 /* compare the lengths */
122 return l->uLen-r->uLen;
123 }
124
125 static int32_t
compareBytes(UCMTable * lTable,const UCMapping * l,UCMTable * rTable,const UCMapping * r,UBool lexical)126 compareBytes(UCMTable *lTable, const UCMapping *l,
127 UCMTable *rTable, const UCMapping *r,
128 UBool lexical) {
129 const uint8_t *lb, *rb;
130 int32_t result, i, length;
131
132 /*
133 * A lexical comparison is used for sorting in the builder, to allow
134 * an efficient search for a byte sequence that could be a prefix
135 * of a previously entered byte sequence.
136 *
137 * Comparing by lengths first is for compatibility with old .ucm tools
138 * like canonucm and rptp2ucm.
139 */
140 if(lexical) {
141 /* get the minimum length and continue */
142 if(l->bLen<=r->bLen) {
143 length=l->bLen;
144 } else {
145 length=r->bLen;
146 }
147 } else {
148 /* compare lengths first */
149 result=l->bLen-r->bLen;
150 if(result!=0) {
151 return result;
152 } else {
153 length=l->bLen;
154 }
155 }
156
157 /* get pointers to the byte sequences */
158 lb=UCM_GET_BYTES(lTable, l);
159 rb=UCM_GET_BYTES(rTable, r);
160
161 /* compare the bytes */
162 for(i=0; i<length; ++i) {
163 result=lb[i]-rb[i];
164 if(result!=0) {
165 return result;
166 }
167 }
168
169 /* compare the lengths */
170 return l->bLen-r->bLen;
171 }
172
173 /* compare UCMappings for sorting */
174 static int32_t
compareMappings(UCMTable * lTable,const UCMapping * l,UCMTable * rTable,const UCMapping * r,UBool uFirst)175 compareMappings(UCMTable *lTable, const UCMapping *l,
176 UCMTable *rTable, const UCMapping *r,
177 UBool uFirst) {
178 int32_t result;
179
180 /* choose which side to compare first */
181 if(uFirst) {
182 /* Unicode then bytes */
183 result=compareUnicode(lTable, l, rTable, r);
184 if(result==0) {
185 result=compareBytes(lTable, l, rTable, r, FALSE); /* not lexically, like canonucm */
186 }
187 } else {
188 /* bytes then Unicode */
189 result=compareBytes(lTable, l, rTable, r, TRUE); /* lexically, for builder */
190 if(result==0) {
191 result=compareUnicode(lTable, l, rTable, r);
192 }
193 }
194
195 if(result!=0) {
196 return result;
197 }
198
199 /* compare the flags */
200 return l->f-r->f;
201 }
202 U_CDECL_BEGIN
203 /* sorting by Unicode first sorts mappings directly */
204 static int32_t U_CALLCONV
compareMappingsUnicodeFirst(const void * context,const void * left,const void * right)205 compareMappingsUnicodeFirst(const void *context, const void *left, const void *right) {
206 return compareMappings(
207 (UCMTable *)context, (const UCMapping *)left,
208 (UCMTable *)context, (const UCMapping *)right, TRUE);
209 }
210
211 /* sorting by bytes first sorts the reverseMap; use indirection to mappings */
212 static int32_t U_CALLCONV
compareMappingsBytesFirst(const void * context,const void * left,const void * right)213 compareMappingsBytesFirst(const void *context, const void *left, const void *right) {
214 UCMTable *table=(UCMTable *)context;
215 int32_t l=*(const int32_t *)left, r=*(const int32_t *)right;
216 return compareMappings(
217 table, table->mappings+l,
218 table, table->mappings+r, FALSE);
219 }
220 U_CDECL_END
221
222 U_CAPI void U_EXPORT2
ucm_sortTable(UCMTable * t)223 ucm_sortTable(UCMTable *t) {
224 UErrorCode errorCode;
225 int32_t i;
226
227 if(t->isSorted) {
228 return;
229 }
230
231 errorCode=U_ZERO_ERROR;
232
233 /* 1. sort by Unicode first */
234 uprv_sortArray(t->mappings, t->mappingsLength, sizeof(UCMapping),
235 compareMappingsUnicodeFirst, t,
236 FALSE, &errorCode);
237
238 /* build the reverseMap */
239 if(t->reverseMap==NULL) {
240 /*
241 * allocate mappingsCapacity instead of mappingsLength so that
242 * if mappings are added, the reverseMap need not be
243 * reallocated each time
244 * (see ucm_moveMappings() and ucm_addMapping())
245 */
246 t->reverseMap=(int32_t *)uprv_malloc(t->mappingsCapacity*sizeof(int32_t));
247 if(t->reverseMap==NULL) {
248 fprintf(stderr, "ucm error: unable to allocate reverseMap\n");
249 exit(U_MEMORY_ALLOCATION_ERROR);
250 }
251 }
252 for(i=0; i<t->mappingsLength; ++i) {
253 t->reverseMap[i]=i;
254 }
255
256 /* 2. sort reverseMap by mappings bytes first */
257 uprv_sortArray(t->reverseMap, t->mappingsLength, sizeof(int32_t),
258 compareMappingsBytesFirst, t,
259 FALSE, &errorCode);
260
261 if(U_FAILURE(errorCode)) {
262 fprintf(stderr, "ucm error: sortTable()/uprv_sortArray() fails - %s\n",
263 u_errorName(errorCode));
264 exit(errorCode);
265 }
266
267 t->isSorted=TRUE;
268 }
269
270 /*
271 * remove mappings with their move flag set from the base table
272 * and move some of them (with UCM_MOVE_TO_EXT) to the extension table
273 */
274 U_CAPI void U_EXPORT2
ucm_moveMappings(UCMTable * base,UCMTable * ext)275 ucm_moveMappings(UCMTable *base, UCMTable *ext) {
276 UCMapping *mb, *mbLimit;
277 int8_t flag;
278
279 mb=base->mappings;
280 mbLimit=mb+base->mappingsLength;
281
282 while(mb<mbLimit) {
283 flag=mb->moveFlag;
284 if(flag!=0) {
285 /* reset the move flag */
286 mb->moveFlag=0;
287
288 if(ext!=NULL && (flag&UCM_MOVE_TO_EXT)) {
289 /* add the mapping to the extension table */
290 ucm_addMapping(ext, mb, UCM_GET_CODE_POINTS(base, mb), UCM_GET_BYTES(base, mb));
291 }
292
293 /* remove this mapping: move the last base mapping down and overwrite the current one */
294 if(mb<(mbLimit-1)) {
295 uprv_memcpy(mb, mbLimit-1, sizeof(UCMapping));
296 }
297 --mbLimit;
298 --base->mappingsLength;
299 base->isSorted=FALSE;
300 } else {
301 ++mb;
302 }
303 }
304 }
305
306 enum {
307 NEEDS_MOVE=1,
308 HAS_ERRORS=2
309 };
310
311 static uint8_t
checkBaseExtUnicode(UCMStates * baseStates,UCMTable * base,UCMTable * ext,UBool moveToExt,UBool intersectBase)312 checkBaseExtUnicode(UCMStates *baseStates, UCMTable *base, UCMTable *ext,
313 UBool moveToExt, UBool intersectBase) {
314 (void)baseStates;
315
316 UCMapping *mb, *me, *mbLimit, *meLimit;
317 int32_t cmp;
318 uint8_t result;
319
320 mb=base->mappings;
321 mbLimit=mb+base->mappingsLength;
322
323 me=ext->mappings;
324 meLimit=me+ext->mappingsLength;
325
326 result=0;
327
328 for(;;) {
329 /* skip irrelevant mappings on both sides */
330 for(;;) {
331 if(mb==mbLimit) {
332 return result;
333 }
334
335 if((0<=mb->f && mb->f<=2) || mb->f==4) {
336 break;
337 }
338
339 ++mb;
340 }
341
342 for(;;) {
343 if(me==meLimit) {
344 return result;
345 }
346
347 if((0<=me->f && me->f<=2) || me->f==4) {
348 break;
349 }
350
351 ++me;
352 }
353
354 /* compare the base and extension mappings */
355 cmp=compareUnicode(base, mb, ext, me);
356 if(cmp<0) {
357 if(intersectBase && (intersectBase!=2 || mb->bLen>1)) {
358 /*
359 * mapping in base but not in ext, move it
360 *
361 * if ext is DBCS, move DBCS mappings here
362 * and check SBCS ones for Unicode prefix below
363 */
364 mb->moveFlag|=UCM_MOVE_TO_EXT;
365 result|=NEEDS_MOVE;
366
367 /* does mb map from an input sequence that is a prefix of me's? */
368 } else if( mb->uLen<me->uLen &&
369 0==uprv_memcmp(UCM_GET_CODE_POINTS(base, mb), UCM_GET_CODE_POINTS(ext, me), 4*mb->uLen)
370 ) {
371 if(moveToExt) {
372 /* mark this mapping to be moved to the extension table */
373 mb->moveFlag|=UCM_MOVE_TO_EXT;
374 result|=NEEDS_MOVE;
375 } else {
376 fprintf(stderr,
377 "ucm error: the base table contains a mapping whose input sequence\n"
378 " is a prefix of the input sequence of an extension mapping\n");
379 ucm_printMapping(base, mb, stderr);
380 ucm_printMapping(ext, me, stderr);
381 result|=HAS_ERRORS;
382 }
383 }
384
385 ++mb;
386 } else if(cmp==0) {
387 /*
388 * same output: remove the extension mapping,
389 * otherwise treat as an error
390 */
391 if( mb->f==me->f && mb->bLen==me->bLen &&
392 0==uprv_memcmp(UCM_GET_BYTES(base, mb), UCM_GET_BYTES(ext, me), mb->bLen)
393 ) {
394 me->moveFlag|=UCM_REMOVE_MAPPING;
395 result|=NEEDS_MOVE;
396 } else if(intersectBase) {
397 /* mapping in base but not in ext, move it */
398 mb->moveFlag|=UCM_MOVE_TO_EXT;
399 result|=NEEDS_MOVE;
400 } else {
401 fprintf(stderr,
402 "ucm error: the base table contains a mapping whose input sequence\n"
403 " is the same as the input sequence of an extension mapping\n"
404 " but it maps differently\n");
405 ucm_printMapping(base, mb, stderr);
406 ucm_printMapping(ext, me, stderr);
407 result|=HAS_ERRORS;
408 }
409
410 ++mb;
411 } else /* cmp>0 */ {
412 ++me;
413 }
414 }
415 }
416
417 static uint8_t
checkBaseExtBytes(UCMStates * baseStates,UCMTable * base,UCMTable * ext,UBool moveToExt,UBool intersectBase)418 checkBaseExtBytes(UCMStates *baseStates, UCMTable *base, UCMTable *ext,
419 UBool moveToExt, UBool intersectBase) {
420 UCMapping *mb, *me;
421 int32_t *baseMap, *extMap;
422 int32_t b, e, bLimit, eLimit, cmp;
423 uint8_t result;
424 UBool isSISO;
425
426 baseMap=base->reverseMap;
427 extMap=ext->reverseMap;
428
429 b=e=0;
430 bLimit=base->mappingsLength;
431 eLimit=ext->mappingsLength;
432
433 result=0;
434
435 isSISO=(UBool)(baseStates->outputType==MBCS_OUTPUT_2_SISO);
436
437 for(;;) {
438 /* skip irrelevant mappings on both sides */
439 for(;; ++b) {
440 if(b==bLimit) {
441 return result;
442 }
443 mb=base->mappings+baseMap[b];
444
445 if(intersectBase==2 && mb->bLen==1) {
446 /*
447 * comparing a base against a DBCS extension:
448 * leave SBCS base mappings alone
449 */
450 continue;
451 }
452
453 if(mb->f==0 || mb->f==3) {
454 break;
455 }
456 }
457
458 for(;;) {
459 if(e==eLimit) {
460 return result;
461 }
462 me=ext->mappings+extMap[e];
463
464 if(me->f==0 || me->f==3) {
465 break;
466 }
467
468 ++e;
469 }
470
471 /* compare the base and extension mappings */
472 cmp=compareBytes(base, mb, ext, me, TRUE);
473 if(cmp<0) {
474 if(intersectBase) {
475 /* mapping in base but not in ext, move it */
476 mb->moveFlag|=UCM_MOVE_TO_EXT;
477 result|=NEEDS_MOVE;
478
479 /*
480 * does mb map from an input sequence that is a prefix of me's?
481 * for SI/SO tables, a single byte is never a prefix because it
482 * occurs in a separate single-byte state
483 */
484 } else if( mb->bLen<me->bLen &&
485 (!isSISO || mb->bLen>1) &&
486 0==uprv_memcmp(UCM_GET_BYTES(base, mb), UCM_GET_BYTES(ext, me), mb->bLen)
487 ) {
488 if(moveToExt) {
489 /* mark this mapping to be moved to the extension table */
490 mb->moveFlag|=UCM_MOVE_TO_EXT;
491 result|=NEEDS_MOVE;
492 } else {
493 fprintf(stderr,
494 "ucm error: the base table contains a mapping whose input sequence\n"
495 " is a prefix of the input sequence of an extension mapping\n");
496 ucm_printMapping(base, mb, stderr);
497 ucm_printMapping(ext, me, stderr);
498 result|=HAS_ERRORS;
499 }
500 }
501
502 ++b;
503 } else if(cmp==0) {
504 /*
505 * same output: remove the extension mapping,
506 * otherwise treat as an error
507 */
508 if( mb->f==me->f && mb->uLen==me->uLen &&
509 0==uprv_memcmp(UCM_GET_CODE_POINTS(base, mb), UCM_GET_CODE_POINTS(ext, me), 4*mb->uLen)
510 ) {
511 me->moveFlag|=UCM_REMOVE_MAPPING;
512 result|=NEEDS_MOVE;
513 } else if(intersectBase) {
514 /* mapping in base but not in ext, move it */
515 mb->moveFlag|=UCM_MOVE_TO_EXT;
516 result|=NEEDS_MOVE;
517 } else {
518 fprintf(stderr,
519 "ucm error: the base table contains a mapping whose input sequence\n"
520 " is the same as the input sequence of an extension mapping\n"
521 " but it maps differently\n");
522 ucm_printMapping(base, mb, stderr);
523 ucm_printMapping(ext, me, stderr);
524 result|=HAS_ERRORS;
525 }
526
527 ++b;
528 } else /* cmp>0 */ {
529 ++e;
530 }
531 }
532 }
533
534 U_CAPI UBool U_EXPORT2
ucm_checkValidity(UCMTable * table,UCMStates * baseStates)535 ucm_checkValidity(UCMTable *table, UCMStates *baseStates) {
536 UCMapping *m, *mLimit;
537 int32_t count;
538 UBool isOK;
539
540 m=table->mappings;
541 mLimit=m+table->mappingsLength;
542 isOK=TRUE;
543
544 while(m<mLimit) {
545 count=ucm_countChars(baseStates, UCM_GET_BYTES(table, m), m->bLen);
546 if(count<1) {
547 ucm_printMapping(table, m, stderr);
548 isOK=FALSE;
549 }
550 ++m;
551 }
552
553 return isOK;
554 }
555
556 U_CAPI UBool U_EXPORT2
ucm_checkBaseExt(UCMStates * baseStates,UCMTable * base,UCMTable * ext,UCMTable * moveTarget,UBool intersectBase)557 ucm_checkBaseExt(UCMStates *baseStates,
558 UCMTable *base, UCMTable *ext, UCMTable *moveTarget,
559 UBool intersectBase) {
560 uint8_t result;
561
562 /* if we have an extension table, we must always use precision flags */
563 if(base->flagsType&UCM_FLAGS_IMPLICIT) {
564 fprintf(stderr, "ucm error: the base table contains mappings without precision flags\n");
565 return FALSE;
566 }
567 if(ext->flagsType&UCM_FLAGS_IMPLICIT) {
568 fprintf(stderr, "ucm error: extension table contains mappings without precision flags\n");
569 return FALSE;
570 }
571
572 /* checking requires both tables to be sorted */
573 ucm_sortTable(base);
574 ucm_sortTable(ext);
575
576 /* check */
577 result=
578 checkBaseExtUnicode(baseStates, base, ext, (UBool)(moveTarget!=NULL), intersectBase)|
579 checkBaseExtBytes(baseStates, base, ext, (UBool)(moveTarget!=NULL), intersectBase);
580
581 if(result&HAS_ERRORS) {
582 return FALSE;
583 }
584
585 if(result&NEEDS_MOVE) {
586 ucm_moveMappings(ext, NULL);
587 ucm_moveMappings(base, moveTarget);
588 ucm_sortTable(base);
589 ucm_sortTable(ext);
590 if(moveTarget!=NULL) {
591 ucm_sortTable(moveTarget);
592 }
593 }
594
595 return TRUE;
596 }
597
598 /* merge tables for rptp2ucm ------------------------------------------------ */
599
600 U_CAPI void U_EXPORT2
ucm_mergeTables(UCMTable * fromUTable,UCMTable * toUTable,const uint8_t * subchar,int32_t subcharLength,uint8_t subchar1)601 ucm_mergeTables(UCMTable *fromUTable, UCMTable *toUTable,
602 const uint8_t *subchar, int32_t subcharLength,
603 uint8_t subchar1) {
604 UCMapping *fromUMapping, *toUMapping;
605 int32_t fromUIndex, toUIndex, fromUTop, toUTop, cmp;
606
607 ucm_sortTable(fromUTable);
608 ucm_sortTable(toUTable);
609
610 fromUMapping=fromUTable->mappings;
611 toUMapping=toUTable->mappings;
612
613 fromUTop=fromUTable->mappingsLength;
614 toUTop=toUTable->mappingsLength;
615
616 fromUIndex=toUIndex=0;
617
618 while(fromUIndex<fromUTop && toUIndex<toUTop) {
619 cmp=compareMappings(fromUTable, fromUMapping, toUTable, toUMapping, TRUE);
620 if(cmp==0) {
621 /* equal: roundtrip, nothing to do (flags are initially 0) */
622 ++fromUMapping;
623 ++toUMapping;
624
625 ++fromUIndex;
626 ++toUIndex;
627 } else if(cmp<0) {
628 /*
629 * the fromU mapping does not have a toU counterpart:
630 * fallback Unicode->codepage
631 */
632 if( (fromUMapping->bLen==subcharLength &&
633 0==uprv_memcmp(UCM_GET_BYTES(fromUTable, fromUMapping), subchar, subcharLength)) ||
634 (subchar1!=0 && fromUMapping->bLen==1 && fromUMapping->b.bytes[0]==subchar1)
635 ) {
636 fromUMapping->f=2; /* SUB mapping */
637 } else {
638 fromUMapping->f=1; /* normal fallback */
639 }
640
641 ++fromUMapping;
642 ++fromUIndex;
643 } else {
644 /*
645 * the toU mapping does not have a fromU counterpart:
646 * (reverse) fallback codepage->Unicode, copy it to the fromU table
647 */
648
649 /* ignore reverse fallbacks to Unicode SUB */
650 if(!(toUMapping->uLen==1 && (toUMapping->u==0xfffd || toUMapping->u==0x1a))) {
651 toUMapping->f=3; /* reverse fallback */
652 ucm_addMapping(fromUTable, toUMapping, UCM_GET_CODE_POINTS(toUTable, toUMapping), UCM_GET_BYTES(toUTable, toUMapping));
653
654 /* the table may have been reallocated */
655 fromUMapping=fromUTable->mappings+fromUIndex;
656 }
657
658 ++toUMapping;
659 ++toUIndex;
660 }
661 }
662
663 /* either one or both tables are exhausted */
664 while(fromUIndex<fromUTop) {
665 /* leftover fromU mappings are fallbacks */
666 if( (fromUMapping->bLen==subcharLength &&
667 0==uprv_memcmp(UCM_GET_BYTES(fromUTable, fromUMapping), subchar, subcharLength)) ||
668 (subchar1!=0 && fromUMapping->bLen==1 && fromUMapping->b.bytes[0]==subchar1)
669 ) {
670 fromUMapping->f=2; /* SUB mapping */
671 } else {
672 fromUMapping->f=1; /* normal fallback */
673 }
674
675 ++fromUMapping;
676 ++fromUIndex;
677 }
678
679 while(toUIndex<toUTop) {
680 /* leftover toU mappings are reverse fallbacks */
681
682 /* ignore reverse fallbacks to Unicode SUB */
683 if(!(toUMapping->uLen==1 && (toUMapping->u==0xfffd || toUMapping->u==0x1a))) {
684 toUMapping->f=3; /* reverse fallback */
685 ucm_addMapping(fromUTable, toUMapping, UCM_GET_CODE_POINTS(toUTable, toUMapping), UCM_GET_BYTES(toUTable, toUMapping));
686 }
687
688 ++toUMapping;
689 ++toUIndex;
690 }
691
692 fromUTable->isSorted=FALSE;
693 }
694
695 /* separate extension mappings out of base table for rptp2ucm --------------- */
696
697 U_CAPI UBool U_EXPORT2
ucm_separateMappings(UCMFile * ucm,UBool isSISO)698 ucm_separateMappings(UCMFile *ucm, UBool isSISO) {
699 UCMTable *table;
700 UCMapping *m, *mLimit;
701 int32_t type;
702 UBool needsMove, isOK;
703
704 table=ucm->base;
705 m=table->mappings;
706 mLimit=m+table->mappingsLength;
707
708 needsMove=FALSE;
709 isOK=TRUE;
710
711 for(; m<mLimit; ++m) {
712 if(isSISO && m->bLen==1 && (m->b.bytes[0]==0xe || m->b.bytes[0]==0xf)) {
713 fprintf(stderr, "warning: removing illegal mapping from an SI/SO-stateful table\n");
714 ucm_printMapping(table, m, stderr);
715 m->moveFlag|=UCM_REMOVE_MAPPING;
716 needsMove=TRUE;
717 continue;
718 }
719
720 type=ucm_mappingType(
721 &ucm->states, m,
722 UCM_GET_CODE_POINTS(table, m), UCM_GET_BYTES(table, m));
723 if(type<0) {
724 /* illegal byte sequence */
725 printMapping(m, UCM_GET_CODE_POINTS(table, m), UCM_GET_BYTES(table, m), stderr);
726 isOK=FALSE;
727 } else if(type>0) {
728 m->moveFlag|=UCM_MOVE_TO_EXT;
729 needsMove=TRUE;
730 }
731 }
732
733 if(!isOK) {
734 return FALSE;
735 }
736 if(needsMove) {
737 ucm_moveMappings(ucm->base, ucm->ext);
738 return ucm_checkBaseExt(&ucm->states, ucm->base, ucm->ext, ucm->ext, FALSE);
739 } else {
740 ucm_sortTable(ucm->base);
741 return TRUE;
742 }
743 }
744
745 /* ucm parser --------------------------------------------------------------- */
746
747 U_CAPI int8_t U_EXPORT2
ucm_parseBytes(uint8_t bytes[UCNV_EXT_MAX_BYTES],const char * line,const char ** ps)748 ucm_parseBytes(uint8_t bytes[UCNV_EXT_MAX_BYTES], const char *line, const char **ps) {
749 const char *s=*ps;
750 char *end;
751 uint8_t byte;
752 int8_t bLen;
753
754 bLen=0;
755 for(;;) {
756 /* skip an optional plus sign */
757 if(bLen>0 && *s=='+') {
758 ++s;
759 }
760 if(*s!='\\') {
761 break;
762 }
763
764 if( s[1]!='x' ||
765 (byte=(uint8_t)uprv_strtoul(s+2, &end, 16), end)!=s+4
766 ) {
767 fprintf(stderr, "ucm error: byte must be formatted as \\xXX (2 hex digits) - \"%s\"\n", line);
768 return -1;
769 }
770
771 if(bLen==UCNV_EXT_MAX_BYTES) {
772 fprintf(stderr, "ucm error: too many bytes on \"%s\"\n", line);
773 return -1;
774 }
775 bytes[bLen++]=byte;
776 s=end;
777 }
778
779 *ps=s;
780 return bLen;
781 }
782
783 /* parse a mapping line; must not be empty */
784 U_CAPI UBool U_EXPORT2
ucm_parseMappingLine(UCMapping * m,UChar32 codePoints[UCNV_EXT_MAX_UCHARS],uint8_t bytes[UCNV_EXT_MAX_BYTES],const char * line)785 ucm_parseMappingLine(UCMapping *m,
786 UChar32 codePoints[UCNV_EXT_MAX_UCHARS],
787 uint8_t bytes[UCNV_EXT_MAX_BYTES],
788 const char *line) {
789 const char *s;
790 char *end;
791 UChar32 cp;
792 int32_t u16Length;
793 int8_t uLen, bLen, f;
794
795 s=line;
796 uLen=bLen=0;
797
798 /* parse code points */
799 for(;;) {
800 /* skip an optional plus sign */
801 if(uLen>0 && *s=='+') {
802 ++s;
803 }
804 if(*s!='<') {
805 break;
806 }
807
808 if( s[1]!='U' ||
809 (cp=(UChar32)uprv_strtoul(s+2, &end, 16), end)==s+2 ||
810 *end!='>'
811 ) {
812 fprintf(stderr, "ucm error: Unicode code point must be formatted as <UXXXX> (1..6 hex digits) - \"%s\"\n", line);
813 return FALSE;
814 }
815 if((uint32_t)cp>0x10ffff || U_IS_SURROGATE(cp)) {
816 fprintf(stderr, "ucm error: Unicode code point must be 0..d7ff or e000..10ffff - \"%s\"\n", line);
817 return FALSE;
818 }
819
820 if(uLen==UCNV_EXT_MAX_UCHARS) {
821 fprintf(stderr, "ucm error: too many code points on \"%s\"\n", line);
822 return FALSE;
823 }
824 codePoints[uLen++]=cp;
825 s=end+1;
826 }
827
828 if(uLen==0) {
829 fprintf(stderr, "ucm error: no Unicode code points on \"%s\"\n", line);
830 return FALSE;
831 } else if(uLen==1) {
832 m->u=codePoints[0];
833 } else {
834 UErrorCode errorCode=U_ZERO_ERROR;
835 u_strFromUTF32(NULL, 0, &u16Length, codePoints, uLen, &errorCode);
836 if( (U_FAILURE(errorCode) && errorCode!=U_BUFFER_OVERFLOW_ERROR) ||
837 u16Length>UCNV_EXT_MAX_UCHARS
838 ) {
839 fprintf(stderr, "ucm error: too many UChars on \"%s\"\n", line);
840 return FALSE;
841 }
842 }
843
844 s=u_skipWhitespace(s);
845
846 /* parse bytes */
847 bLen=ucm_parseBytes(bytes, line, &s);
848
849 if(bLen<0) {
850 return FALSE;
851 } else if(bLen==0) {
852 fprintf(stderr, "ucm error: no bytes on \"%s\"\n", line);
853 return FALSE;
854 } else if(bLen<=4) {
855 uprv_memcpy(m->b.bytes, bytes, bLen);
856 }
857
858 /* skip everything until the fallback indicator, even the start of a comment */
859 for(;;) {
860 if(*s==0) {
861 f=-1; /* no fallback indicator */
862 break;
863 } else if(*s=='|') {
864 f=(int8_t)(s[1]-'0');
865 if((uint8_t)f>4) {
866 fprintf(stderr, "ucm error: fallback indicator must be |0..|4 - \"%s\"\n", line);
867 return FALSE;
868 }
869 break;
870 }
871 ++s;
872 }
873
874 m->uLen=uLen;
875 m->bLen=bLen;
876 m->f=f;
877 return TRUE;
878 }
879
880 /* general APIs ------------------------------------------------------------- */
881
882 U_CAPI UCMTable * U_EXPORT2
ucm_openTable()883 ucm_openTable() {
884 UCMTable *table=(UCMTable *)uprv_malloc(sizeof(UCMTable));
885 if(table==NULL) {
886 fprintf(stderr, "ucm error: unable to allocate a UCMTable\n");
887 exit(U_MEMORY_ALLOCATION_ERROR);
888 }
889
890 memset(table, 0, sizeof(UCMTable));
891 return table;
892 }
893
894 U_CAPI void U_EXPORT2
ucm_closeTable(UCMTable * table)895 ucm_closeTable(UCMTable *table) {
896 if(table!=NULL) {
897 uprv_free(table->mappings);
898 uprv_free(table->codePoints);
899 uprv_free(table->bytes);
900 uprv_free(table->reverseMap);
901 uprv_free(table);
902 }
903 }
904
905 U_CAPI void U_EXPORT2
ucm_resetTable(UCMTable * table)906 ucm_resetTable(UCMTable *table) {
907 if(table!=NULL) {
908 table->mappingsLength=0;
909 table->flagsType=0;
910 table->unicodeMask=0;
911 table->bytesLength=table->codePointsLength=0;
912 table->isSorted=FALSE;
913 }
914 }
915
916 U_CAPI void U_EXPORT2
ucm_addMapping(UCMTable * table,UCMapping * m,UChar32 codePoints[UCNV_EXT_MAX_UCHARS],uint8_t bytes[UCNV_EXT_MAX_BYTES])917 ucm_addMapping(UCMTable *table,
918 UCMapping *m,
919 UChar32 codePoints[UCNV_EXT_MAX_UCHARS],
920 uint8_t bytes[UCNV_EXT_MAX_BYTES]) {
921 UCMapping *tm;
922 UChar32 c;
923 int32_t idx;
924
925 if(table->mappingsLength>=table->mappingsCapacity) {
926 /* make the mappings array larger */
927 if(table->mappingsCapacity==0) {
928 table->mappingsCapacity=1000;
929 } else {
930 table->mappingsCapacity*=10;
931 }
932 table->mappings=(UCMapping *)uprv_realloc(table->mappings,
933 table->mappingsCapacity*sizeof(UCMapping));
934 if(table->mappings==NULL) {
935 fprintf(stderr, "ucm error: unable to allocate %d UCMappings\n",
936 (int)table->mappingsCapacity);
937 exit(U_MEMORY_ALLOCATION_ERROR);
938 }
939
940 if(table->reverseMap!=NULL) {
941 /* the reverseMap must be reallocated in a new sort */
942 uprv_free(table->reverseMap);
943 table->reverseMap=NULL;
944 }
945 }
946
947 if(m->uLen>1 && table->codePointsCapacity==0) {
948 table->codePointsCapacity=10000;
949 table->codePoints=(UChar32 *)uprv_malloc(table->codePointsCapacity*4);
950 if(table->codePoints==NULL) {
951 fprintf(stderr, "ucm error: unable to allocate %d UChar32s\n",
952 (int)table->codePointsCapacity);
953 exit(U_MEMORY_ALLOCATION_ERROR);
954 }
955 }
956
957 if(m->bLen>4 && table->bytesCapacity==0) {
958 table->bytesCapacity=10000;
959 table->bytes=(uint8_t *)uprv_malloc(table->bytesCapacity);
960 if(table->bytes==NULL) {
961 fprintf(stderr, "ucm error: unable to allocate %d bytes\n",
962 (int)table->bytesCapacity);
963 exit(U_MEMORY_ALLOCATION_ERROR);
964 }
965 }
966
967 if(m->uLen>1) {
968 idx=table->codePointsLength;
969 table->codePointsLength+=m->uLen;
970 if(table->codePointsLength>table->codePointsCapacity) {
971 fprintf(stderr, "ucm error: too many code points in multiple-code point mappings\n");
972 exit(U_MEMORY_ALLOCATION_ERROR);
973 }
974
975 uprv_memcpy(table->codePoints+idx, codePoints, (size_t)m->uLen*4);
976 m->u=idx;
977 }
978
979 if(m->bLen>4) {
980 idx=table->bytesLength;
981 table->bytesLength+=m->bLen;
982 if(table->bytesLength>table->bytesCapacity) {
983 fprintf(stderr, "ucm error: too many bytes in mappings with >4 charset bytes\n");
984 exit(U_MEMORY_ALLOCATION_ERROR);
985 }
986
987 uprv_memcpy(table->bytes+idx, bytes, m->bLen);
988 m->b.idx=idx;
989 }
990
991 /* set unicodeMask */
992 for(idx=0; idx<m->uLen; ++idx) {
993 c=codePoints[idx];
994 if(c>=0x10000) {
995 table->unicodeMask|=UCNV_HAS_SUPPLEMENTARY; /* there are supplementary code points */
996 } else if(U_IS_SURROGATE(c)) {
997 table->unicodeMask|=UCNV_HAS_SURROGATES; /* there are surrogate code points */
998 }
999 }
1000
1001 /* set flagsType */
1002 if(m->f<0) {
1003 table->flagsType|=UCM_FLAGS_IMPLICIT;
1004 } else {
1005 table->flagsType|=UCM_FLAGS_EXPLICIT;
1006 }
1007
1008 tm=table->mappings+table->mappingsLength++;
1009 uprv_memcpy(tm, m, sizeof(UCMapping));
1010
1011 table->isSorted=FALSE;
1012 }
1013
1014 U_CAPI UCMFile * U_EXPORT2
ucm_open()1015 ucm_open() {
1016 UCMFile *ucm=(UCMFile *)uprv_malloc(sizeof(UCMFile));
1017 if(ucm==NULL) {
1018 fprintf(stderr, "ucm error: unable to allocate a UCMFile\n");
1019 exit(U_MEMORY_ALLOCATION_ERROR);
1020 }
1021
1022 memset(ucm, 0, sizeof(UCMFile));
1023
1024 ucm->base=ucm_openTable();
1025 ucm->ext=ucm_openTable();
1026
1027 ucm->states.stateFlags[0]=MBCS_STATE_FLAG_DIRECT;
1028 ucm->states.conversionType=UCNV_UNSUPPORTED_CONVERTER;
1029 ucm->states.outputType=-1;
1030 ucm->states.minCharLength=ucm->states.maxCharLength=1;
1031
1032 return ucm;
1033 }
1034
1035 U_CAPI void U_EXPORT2
ucm_close(UCMFile * ucm)1036 ucm_close(UCMFile *ucm) {
1037 if(ucm!=NULL) {
1038 ucm_closeTable(ucm->base);
1039 ucm_closeTable(ucm->ext);
1040 uprv_free(ucm);
1041 }
1042 }
1043
1044 U_CAPI int32_t U_EXPORT2
ucm_mappingType(UCMStates * baseStates,UCMapping * m,UChar32 codePoints[UCNV_EXT_MAX_UCHARS],uint8_t bytes[UCNV_EXT_MAX_BYTES])1045 ucm_mappingType(UCMStates *baseStates,
1046 UCMapping *m,
1047 UChar32 codePoints[UCNV_EXT_MAX_UCHARS],
1048 uint8_t bytes[UCNV_EXT_MAX_BYTES]) {
1049 (void)codePoints;
1050 /* check validity of the bytes and count the characters in them */
1051 int32_t count=ucm_countChars(baseStates, bytes, m->bLen);
1052 if(count<1) {
1053 /* illegal byte sequence */
1054 return -1;
1055 }
1056
1057 /*
1058 * Suitable for an ICU conversion base table means:
1059 * - a 1:1 mapping (1 Unicode code point : 1 byte sequence)
1060 * - precision flag 0..3
1061 * - SBCS: any 1:1 mapping
1062 * (the table stores additional bits to distinguish mapping types)
1063 * - MBCS: not a |2 SUB mapping for <subchar1>
1064 * - MBCS: not a |1 fallback to 0x00
1065 * - MBCS: not a multi-byte mapping with leading 0x00 bytes
1066 *
1067 * Further restrictions for fromUnicode tables
1068 * are enforced in makeconv (MBCSOkForBaseFromUnicode()).
1069 *
1070 * All of the MBCS fromUnicode specific tests could be removed from here,
1071 * but the ones above are for unusual mappings, and removing the tests
1072 * from here would change canonucm output which seems gratuitous.
1073 * (Markus Scherer 2006-nov-28)
1074 *
1075 * Exception: All implicit mappings (f<0) that need to be moved
1076 * because of fromUnicode restrictions _must_ be moved here because
1077 * makeconv uses a hack for moving mappings only for the fromUnicode table
1078 * that only works with non-negative values of f.
1079 */
1080 if( m->uLen==1 && count==1 && m->f<=3 &&
1081 (baseStates->maxCharLength==1 ||
1082 !((m->f==2 && m->bLen==1) ||
1083 (m->f==1 && bytes[0]==0) ||
1084 (m->f<=1 && m->bLen>1 && bytes[0]==0)))
1085 ) {
1086 return 0; /* suitable for a base table */
1087 } else {
1088 return 1; /* needs to go into an extension table */
1089 }
1090 }
1091
1092 U_CAPI UBool U_EXPORT2
ucm_addMappingAuto(UCMFile * ucm,UBool forBase,UCMStates * baseStates,UCMapping * m,UChar32 codePoints[UCNV_EXT_MAX_UCHARS],uint8_t bytes[UCNV_EXT_MAX_BYTES])1093 ucm_addMappingAuto(UCMFile *ucm, UBool forBase, UCMStates *baseStates,
1094 UCMapping *m,
1095 UChar32 codePoints[UCNV_EXT_MAX_UCHARS],
1096 uint8_t bytes[UCNV_EXT_MAX_BYTES]) {
1097 int32_t type;
1098
1099 if(m->f==2 && m->uLen>1) {
1100 fprintf(stderr, "ucm error: illegal <subchar1> |2 mapping from multiple code points\n");
1101 printMapping(m, codePoints, bytes, stderr);
1102 return FALSE;
1103 }
1104
1105 if(baseStates!=NULL) {
1106 /* check validity of the bytes and count the characters in them */
1107 type=ucm_mappingType(baseStates, m, codePoints, bytes);
1108 if(type<0) {
1109 /* illegal byte sequence */
1110 printMapping(m, codePoints, bytes, stderr);
1111 return FALSE;
1112 }
1113 } else {
1114 /* not used - adding a mapping for an extension-only table before its base table is read */
1115 type=1;
1116 }
1117
1118 /*
1119 * Add the mapping to the base table if this is requested and suitable.
1120 * Otherwise, add it to the extension table.
1121 */
1122 if(forBase && type==0) {
1123 ucm_addMapping(ucm->base, m, codePoints, bytes);
1124 } else {
1125 ucm_addMapping(ucm->ext, m, codePoints, bytes);
1126 }
1127
1128 return TRUE;
1129 }
1130
1131 U_CAPI UBool U_EXPORT2
ucm_addMappingFromLine(UCMFile * ucm,const char * line,UBool forBase,UCMStates * baseStates)1132 ucm_addMappingFromLine(UCMFile *ucm, const char *line, UBool forBase, UCMStates *baseStates) {
1133 UCMapping m={ 0, {0}, 0, 0, 0, 0 };
1134 UChar32 codePoints[UCNV_EXT_MAX_UCHARS];
1135 uint8_t bytes[UCNV_EXT_MAX_BYTES];
1136
1137 const char *s;
1138
1139 /* ignore empty and comment lines */
1140 if(line[0]=='#' || *(s=u_skipWhitespace(line))==0 || *s=='\n' || *s=='\r') {
1141 return TRUE;
1142 }
1143
1144 return
1145 ucm_parseMappingLine(&m, codePoints, bytes, line) &&
1146 ucm_addMappingAuto(ucm, forBase, baseStates, &m, codePoints, bytes);
1147 }
1148
1149 U_CAPI void U_EXPORT2
ucm_readTable(UCMFile * ucm,FileStream * convFile,UBool forBase,UCMStates * baseStates,UErrorCode * pErrorCode)1150 ucm_readTable(UCMFile *ucm, FileStream* convFile,
1151 UBool forBase, UCMStates *baseStates,
1152 UErrorCode *pErrorCode) {
1153 char line[500];
1154 char *end;
1155 UBool isOK;
1156
1157 if(U_FAILURE(*pErrorCode)) {
1158 return;
1159 }
1160
1161 isOK=TRUE;
1162
1163 for(;;) {
1164 /* read the next line */
1165 if(!T_FileStream_readLine(convFile, line, sizeof(line))) {
1166 fprintf(stderr, "incomplete charmap section\n");
1167 isOK=FALSE;
1168 break;
1169 }
1170
1171 /* remove CR LF */
1172 end=uprv_strchr(line, 0);
1173 while(line<end && (*(end-1)=='\r' || *(end-1)=='\n')) {
1174 --end;
1175 }
1176 *end=0;
1177
1178 /* ignore empty and comment lines */
1179 if(line[0]==0 || line[0]=='#') {
1180 continue;
1181 }
1182
1183 /* stop at the end of the mapping table */
1184 if(0==uprv_strcmp(line, "END CHARMAP")) {
1185 break;
1186 }
1187
1188 isOK&=ucm_addMappingFromLine(ucm, line, forBase, baseStates);
1189 }
1190
1191 if(!isOK) {
1192 *pErrorCode=U_INVALID_TABLE_FORMAT;
1193 }
1194 }
1195 #endif
1196