• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // Copyright (C) 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
3 /*
4 ******************************************************************************
5 *
6 *   Copyright (C) 2003-2016, International Business Machines
7 *   Corporation and others.  All Rights Reserved.
8 *
9 ******************************************************************************
10 *   file name:  ucnv_ext.cpp
11 *   encoding:   US-ASCII
12 *   tab size:   8 (not used)
13 *   indentation:4
14 *
15 *   created on: 2003jun13
16 *   created by: Markus W. Scherer
17 *
18 *   Conversion extensions
19 */
20 
21 #include "unicode/utypes.h"
22 
23 #if !UCONFIG_NO_CONVERSION && !UCONFIG_NO_LEGACY_CONVERSION
24 
25 #include "unicode/uset.h"
26 #include "ucnv_bld.h"
27 #include "ucnv_cnv.h"
28 #include "ucnv_ext.h"
29 #include "cmemory.h"
30 #include "uassert.h"
31 
32 /* to Unicode --------------------------------------------------------------- */
33 
34 /*
35  * @return lookup value for the byte, if found; else 0
36  */
37 static inline uint32_t
ucnv_extFindToU(const uint32_t * toUSection,int32_t length,uint8_t byte)38 ucnv_extFindToU(const uint32_t *toUSection, int32_t length, uint8_t byte) {
39     uint32_t word0, word;
40     int32_t i, start, limit;
41 
42     /* check the input byte against the lowest and highest section bytes */
43     start=(int32_t)UCNV_EXT_TO_U_GET_BYTE(toUSection[0]);
44     limit=(int32_t)UCNV_EXT_TO_U_GET_BYTE(toUSection[length-1]);
45     if(byte<start || limit<byte) {
46         return 0; /* the byte is out of range */
47     }
48 
49     if(length==((limit-start)+1)) {
50         /* direct access on a linear array */
51         return UCNV_EXT_TO_U_GET_VALUE(toUSection[byte-start]); /* could be 0 */
52     }
53 
54     /* word0 is suitable for <=toUSection[] comparison, word for <toUSection[] */
55     word0=UCNV_EXT_TO_U_MAKE_WORD(byte, 0);
56 
57     /*
58      * Shift byte once instead of each section word and add 0xffffff.
59      * We will compare the shifted/added byte (bbffffff) against
60      * section words which have byte values in the same bit position.
61      * If and only if byte bb < section byte ss then bbffffff<ssvvvvvv
62      * for all v=0..f
63      * so we need not mask off the lower 24 bits of each section word.
64      */
65     word=word0|UCNV_EXT_TO_U_VALUE_MASK;
66 
67     /* binary search */
68     start=0;
69     limit=length;
70     for(;;) {
71         i=limit-start;
72         if(i<=1) {
73             break; /* done */
74         }
75         /* start<limit-1 */
76 
77         if(i<=4) {
78             /* linear search for the last part */
79             if(word0<=toUSection[start]) {
80                 break;
81             }
82             if(++start<limit && word0<=toUSection[start]) {
83                 break;
84             }
85             if(++start<limit && word0<=toUSection[start]) {
86                 break;
87             }
88             /* always break at start==limit-1 */
89             ++start;
90             break;
91         }
92 
93         i=(start+limit)/2;
94         if(word<toUSection[i]) {
95             limit=i;
96         } else {
97             start=i;
98         }
99     }
100 
101     /* did we really find it? */
102     if(start<limit && byte==UCNV_EXT_TO_U_GET_BYTE(word=toUSection[start])) {
103         return UCNV_EXT_TO_U_GET_VALUE(word); /* never 0 */
104     } else {
105         return 0; /* not found */
106     }
107 }
108 
109 /*
110  * TRUE if not an SI/SO stateful converter,
111  * or if the match length fits with the current converter state
112  */
113 #define UCNV_EXT_TO_U_VERIFY_SISO_MATCH(sisoState, match) \
114     ((sisoState)<0 || ((sisoState)==0) == (match==1))
115 
116 /*
117  * this works like ucnv_extMatchFromU() except
118  * - the first character is in pre
119  * - no trie is used
120  * - the returned matchLength is not offset by 2
121  */
122 static int32_t
ucnv_extMatchToU(const int32_t * cx,int8_t sisoState,const char * pre,int32_t preLength,const char * src,int32_t srcLength,uint32_t * pMatchValue,UBool,UBool flush)123 ucnv_extMatchToU(const int32_t *cx, int8_t sisoState,
124                  const char *pre, int32_t preLength,
125                  const char *src, int32_t srcLength,
126                  uint32_t *pMatchValue,
127                  UBool /*useFallback*/, UBool flush) {
128     const uint32_t *toUTable, *toUSection;
129 
130     uint32_t value, matchValue;
131     int32_t i, j, idx, length, matchLength;
132     uint8_t b;
133 
134     if(cx==NULL || cx[UCNV_EXT_TO_U_LENGTH]<=0) {
135         return 0; /* no extension data, no match */
136     }
137 
138     /* initialize */
139     toUTable=UCNV_EXT_ARRAY(cx, UCNV_EXT_TO_U_INDEX, uint32_t);
140     idx=0;
141 
142     matchValue=0;
143     i=j=matchLength=0;
144 
145     if(sisoState==0) {
146         /* SBCS state of an SI/SO stateful converter, look at only exactly 1 byte */
147         if(preLength>1) {
148             return 0; /* no match of a DBCS sequence in SBCS mode */
149         } else if(preLength==1) {
150             srcLength=0;
151         } else /* preLength==0 */ {
152             if(srcLength>1) {
153                 srcLength=1;
154             }
155         }
156         flush=TRUE;
157     }
158 
159     /* we must not remember fallback matches when not using fallbacks */
160 
161     /* match input units until there is a full match or the input is consumed */
162     for(;;) {
163         /* go to the next section */
164         toUSection=toUTable+idx;
165 
166         /* read first pair of the section */
167         value=*toUSection++;
168         length=UCNV_EXT_TO_U_GET_BYTE(value);
169         value=UCNV_EXT_TO_U_GET_VALUE(value);
170         if( value!=0 &&
171             (UCNV_EXT_TO_U_IS_ROUNDTRIP(value) ||
172              TO_U_USE_FALLBACK(useFallback)) &&
173             UCNV_EXT_TO_U_VERIFY_SISO_MATCH(sisoState, i+j)
174         ) {
175             /* remember longest match so far */
176             matchValue=value;
177             matchLength=i+j;
178         }
179 
180         /* match pre[] then src[] */
181         if(i<preLength) {
182             b=(uint8_t)pre[i++];
183         } else if(j<srcLength) {
184             b=(uint8_t)src[j++];
185         } else {
186             /* all input consumed, partial match */
187             if(flush || (length=(i+j))>UCNV_EXT_MAX_BYTES) {
188                 /*
189                  * end of the entire input stream, stop with the longest match so far
190                  * or: partial match must not be longer than UCNV_EXT_MAX_BYTES
191                  * because it must fit into state buffers
192                  */
193                 break;
194             } else {
195                 /* continue with more input next time */
196                 return -length;
197             }
198         }
199 
200         /* search for the current UChar */
201         value=ucnv_extFindToU(toUSection, length, b);
202         if(value==0) {
203             /* no match here, stop with the longest match so far */
204             break;
205         } else {
206             if(UCNV_EXT_TO_U_IS_PARTIAL(value)) {
207                 /* partial match, continue */
208                 idx=(int32_t)UCNV_EXT_TO_U_GET_PARTIAL_INDEX(value);
209             } else {
210                 if( (UCNV_EXT_TO_U_IS_ROUNDTRIP(value) ||
211                      TO_U_USE_FALLBACK(useFallback)) &&
212                     UCNV_EXT_TO_U_VERIFY_SISO_MATCH(sisoState, i+j)
213                 ) {
214                     /* full match, stop with result */
215                     matchValue=value;
216                     matchLength=i+j;
217                 } else {
218                     /* full match on fallback not taken, stop with the longest match so far */
219                 }
220                 break;
221             }
222         }
223     }
224 
225     if(matchLength==0) {
226         /* no match at all */
227         return 0;
228     }
229 
230     /* return result */
231     *pMatchValue=UCNV_EXT_TO_U_MASK_ROUNDTRIP(matchValue);
232     return matchLength;
233 }
234 
235 static inline void
ucnv_extWriteToU(UConverter * cnv,const int32_t * cx,uint32_t value,UChar ** target,const UChar * targetLimit,int32_t ** offsets,int32_t srcIndex,UErrorCode * pErrorCode)236 ucnv_extWriteToU(UConverter *cnv, const int32_t *cx,
237                  uint32_t value,
238                  UChar **target, const UChar *targetLimit,
239                  int32_t **offsets, int32_t srcIndex,
240                  UErrorCode *pErrorCode) {
241     /* output the result */
242     if(UCNV_EXT_TO_U_IS_CODE_POINT(value)) {
243         /* output a single code point */
244         ucnv_toUWriteCodePoint(
245             cnv, UCNV_EXT_TO_U_GET_CODE_POINT(value),
246             target, targetLimit,
247             offsets, srcIndex,
248             pErrorCode);
249     } else {
250         /* output a string - with correct data we have resultLength>0 */
251         ucnv_toUWriteUChars(
252             cnv,
253             UCNV_EXT_ARRAY(cx, UCNV_EXT_TO_U_UCHARS_INDEX, UChar)+
254                 UCNV_EXT_TO_U_GET_INDEX(value),
255             UCNV_EXT_TO_U_GET_LENGTH(value),
256             target, targetLimit,
257             offsets, srcIndex,
258             pErrorCode);
259     }
260 }
261 
262 /*
263  * get the SI/SO toU state (state 0 is for SBCS, 1 for DBCS),
264  * or 1 for DBCS-only,
265  * or -1 if the converter is not SI/SO stateful
266  *
267  * Note: For SI/SO stateful converters getting here,
268  * cnv->mode==0 is equivalent to firstLength==1.
269  */
270 #define UCNV_SISO_STATE(cnv) \
271     ((cnv)->sharedData->mbcs.outputType==MBCS_OUTPUT_2_SISO ? (int8_t)(cnv)->mode : \
272      (cnv)->sharedData->mbcs.outputType==MBCS_OUTPUT_DBCS_ONLY ? 1 : -1)
273 
274 /*
275  * target<targetLimit; set error code for overflow
276  */
277 U_CFUNC UBool
ucnv_extInitialMatchToU(UConverter * cnv,const int32_t * cx,int32_t firstLength,const char ** src,const char * srcLimit,UChar ** target,const UChar * targetLimit,int32_t ** offsets,int32_t srcIndex,UBool flush,UErrorCode * pErrorCode)278 ucnv_extInitialMatchToU(UConverter *cnv, const int32_t *cx,
279                         int32_t firstLength,
280                         const char **src, const char *srcLimit,
281                         UChar **target, const UChar *targetLimit,
282                         int32_t **offsets, int32_t srcIndex,
283                         UBool flush,
284                         UErrorCode *pErrorCode) {
285     uint32_t value = 0;  /* initialize output-only param to 0 to silence gcc */
286     int32_t match;
287 
288     /* try to match */
289     match=ucnv_extMatchToU(cx, (int8_t)UCNV_SISO_STATE(cnv),
290                            (const char *)cnv->toUBytes, firstLength,
291                            *src, (int32_t)(srcLimit-*src),
292                            &value,
293                            cnv->useFallback, flush);
294     if(match>0) {
295         /* advance src pointer for the consumed input */
296         *src+=match-firstLength;
297 
298         /* write result to target */
299         ucnv_extWriteToU(cnv, cx,
300                          value,
301                          target, targetLimit,
302                          offsets, srcIndex,
303                          pErrorCode);
304         return TRUE;
305     } else if(match<0) {
306         /* save state for partial match */
307         const char *s;
308         int32_t j;
309 
310         /* copy the first code point */
311         s=(const char *)cnv->toUBytes;
312         cnv->preToUFirstLength=(int8_t)firstLength;
313         for(j=0; j<firstLength; ++j) {
314             cnv->preToU[j]=*s++;
315         }
316 
317         /* now copy the newly consumed input */
318         s=*src;
319         match=-match;
320         for(; j<match; ++j) {
321             cnv->preToU[j]=*s++;
322         }
323         *src=s; /* same as *src=srcLimit; because we reached the end of input */
324         cnv->preToULength=(int8_t)match;
325         return TRUE;
326     } else /* match==0 no match */ {
327         return FALSE;
328     }
329 }
330 
331 U_CFUNC UChar32
ucnv_extSimpleMatchToU(const int32_t * cx,const char * source,int32_t length,UBool useFallback)332 ucnv_extSimpleMatchToU(const int32_t *cx,
333                        const char *source, int32_t length,
334                        UBool useFallback) {
335     uint32_t value = 0;  /* initialize output-only param to 0 to silence gcc */
336     int32_t match;
337 
338     if(length<=0) {
339         return 0xffff;
340     }
341 
342     /* try to match */
343     match=ucnv_extMatchToU(cx, -1,
344                            source, length,
345                            NULL, 0,
346                            &value,
347                            useFallback, TRUE);
348     if(match==length) {
349         /* write result for simple, single-character conversion */
350         if(UCNV_EXT_TO_U_IS_CODE_POINT(value)) {
351             return UCNV_EXT_TO_U_GET_CODE_POINT(value);
352         }
353     }
354 
355     /*
356      * return no match because
357      * - match>0 && value points to string: simple conversion cannot handle multiple code points
358      * - match>0 && match!=length: not all input consumed, forbidden for this function
359      * - match==0: no match found in the first place
360      * - match<0: partial match, not supported for simple conversion (and flush==TRUE)
361      */
362     return 0xfffe;
363 }
364 
365 /*
366  * continue partial match with new input
367  * never called for simple, single-character conversion
368  */
369 U_CFUNC void
ucnv_extContinueMatchToU(UConverter * cnv,UConverterToUnicodeArgs * pArgs,int32_t srcIndex,UErrorCode * pErrorCode)370 ucnv_extContinueMatchToU(UConverter *cnv,
371                          UConverterToUnicodeArgs *pArgs, int32_t srcIndex,
372                          UErrorCode *pErrorCode) {
373     uint32_t value = 0;  /* initialize output-only param to 0 to silence gcc */
374     int32_t match, length;
375 
376     match=ucnv_extMatchToU(cnv->sharedData->mbcs.extIndexes, (int8_t)UCNV_SISO_STATE(cnv),
377                            cnv->preToU, cnv->preToULength,
378                            pArgs->source, (int32_t)(pArgs->sourceLimit-pArgs->source),
379                            &value,
380                            cnv->useFallback, pArgs->flush);
381     if(match>0) {
382         if(match>=cnv->preToULength) {
383             /* advance src pointer for the consumed input */
384             pArgs->source+=match-cnv->preToULength;
385             cnv->preToULength=0;
386         } else {
387             /* the match did not use all of preToU[] - keep the rest for replay */
388             length=cnv->preToULength-match;
389             uprv_memmove(cnv->preToU, cnv->preToU+match, length);
390             cnv->preToULength=(int8_t)-length;
391         }
392 
393         /* write result */
394         ucnv_extWriteToU(cnv, cnv->sharedData->mbcs.extIndexes,
395                          value,
396                          &pArgs->target, pArgs->targetLimit,
397                          &pArgs->offsets, srcIndex,
398                          pErrorCode);
399     } else if(match<0) {
400         /* save state for partial match */
401         const char *s;
402         int32_t j;
403 
404         /* just _append_ the newly consumed input to preToU[] */
405         s=pArgs->source;
406         match=-match;
407         for(j=cnv->preToULength; j<match; ++j) {
408             cnv->preToU[j]=*s++;
409         }
410         pArgs->source=s; /* same as *src=srcLimit; because we reached the end of input */
411         cnv->preToULength=(int8_t)match;
412     } else /* match==0 */ {
413         /*
414          * no match
415          *
416          * We need to split the previous input into two parts:
417          *
418          * 1. The first codepage character is unmappable - that's how we got into
419          *    trying the extension data in the first place.
420          *    We need to move it from the preToU buffer
421          *    to the error buffer, set an error code,
422          *    and prepare the rest of the previous input for 2.
423          *
424          * 2. The rest of the previous input must be converted once we
425          *    come back from the callback for the first character.
426          *    At that time, we have to try again from scratch to convert
427          *    these input characters.
428          *    The replay will be handled by the ucnv.c conversion code.
429          */
430 
431         /* move the first codepage character to the error field */
432         uprv_memcpy(cnv->toUBytes, cnv->preToU, cnv->preToUFirstLength);
433         cnv->toULength=cnv->preToUFirstLength;
434 
435         /* move the rest up inside the buffer */
436         length=cnv->preToULength-cnv->preToUFirstLength;
437         if(length>0) {
438             uprv_memmove(cnv->preToU, cnv->preToU+cnv->preToUFirstLength, length);
439         }
440 
441         /* mark preToU for replay */
442         cnv->preToULength=(int8_t)-length;
443 
444         /* set the error code for unassigned */
445         *pErrorCode=U_INVALID_CHAR_FOUND;
446     }
447 }
448 
449 /* from Unicode ------------------------------------------------------------- */
450 
451 // Use roundtrips, "good one-way" mappings, and some normal fallbacks.
452 static inline UBool
extFromUUseMapping(UBool useFallback,uint32_t value,UChar32 firstCP)453 extFromUUseMapping(UBool useFallback, uint32_t value, UChar32 firstCP) {
454     return
455         ((value&UCNV_EXT_FROM_U_STATUS_MASK)!=0 ||
456             FROM_U_USE_FALLBACK(useFallback, firstCP)) &&
457         (value&UCNV_EXT_FROM_U_RESERVED_MASK)==0;
458 }
459 
460 /*
461  * @return index of the UChar, if found; else <0
462  */
463 static inline int32_t
ucnv_extFindFromU(const UChar * fromUSection,int32_t length,UChar u)464 ucnv_extFindFromU(const UChar *fromUSection, int32_t length, UChar u) {
465     int32_t i, start, limit;
466 
467     /* binary search */
468     start=0;
469     limit=length;
470     for(;;) {
471         i=limit-start;
472         if(i<=1) {
473             break; /* done */
474         }
475         /* start<limit-1 */
476 
477         if(i<=4) {
478             /* linear search for the last part */
479             if(u<=fromUSection[start]) {
480                 break;
481             }
482             if(++start<limit && u<=fromUSection[start]) {
483                 break;
484             }
485             if(++start<limit && u<=fromUSection[start]) {
486                 break;
487             }
488             /* always break at start==limit-1 */
489             ++start;
490             break;
491         }
492 
493         i=(start+limit)/2;
494         if(u<fromUSection[i]) {
495             limit=i;
496         } else {
497             start=i;
498         }
499     }
500 
501     /* did we really find it? */
502     if(start<limit && u==fromUSection[start]) {
503         return start;
504     } else {
505         return -1; /* not found */
506     }
507 }
508 
509 /*
510  * @param cx pointer to extension data; if NULL, returns 0
511  * @param firstCP the first code point before all the other UChars
512  * @param pre UChars that must match; !initialMatch: partial match with them
513  * @param preLength length of pre, >=0
514  * @param src UChars that can be used to complete a match
515  * @param srcLength length of src, >=0
516  * @param pMatchValue [out] output result value for the match from the data structure
517  * @param useFallback "use fallback" flag, usually from cnv->useFallback
518  * @param flush TRUE if the end of the input stream is reached
519  * @return >1: matched, return value=total match length (number of input units matched)
520  *          1: matched, no mapping but request for <subchar1>
521  *             (only for the first code point)
522  *          0: no match
523  *         <0: partial match, return value=negative total match length
524  *             (partial matches are never returned for flush==TRUE)
525  *             (partial matches are never returned as being longer than UCNV_EXT_MAX_UCHARS)
526  *         the matchLength is 2 if only firstCP matched, and >2 if firstCP and
527  *         further code units matched
528  */
529 static int32_t
ucnv_extMatchFromU(const int32_t * cx,UChar32 firstCP,const UChar * pre,int32_t preLength,const UChar * src,int32_t srcLength,uint32_t * pMatchValue,UBool useFallback,UBool flush)530 ucnv_extMatchFromU(const int32_t *cx,
531                    UChar32 firstCP,
532                    const UChar *pre, int32_t preLength,
533                    const UChar *src, int32_t srcLength,
534                    uint32_t *pMatchValue,
535                    UBool useFallback, UBool flush) {
536     const uint16_t *stage12, *stage3;
537     const uint32_t *stage3b;
538 
539     const UChar *fromUTableUChars, *fromUSectionUChars;
540     const uint32_t *fromUTableValues, *fromUSectionValues;
541 
542     uint32_t value, matchValue;
543     int32_t i, j, idx, length, matchLength;
544     UChar c;
545 
546     if(cx==NULL) {
547         return 0; /* no extension data, no match */
548     }
549 
550     /* trie lookup of firstCP */
551     idx=firstCP>>10; /* stage 1 index */
552     if(idx>=cx[UCNV_EXT_FROM_U_STAGE_1_LENGTH]) {
553         return 0; /* the first code point is outside the trie */
554     }
555 
556     stage12=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_12_INDEX, uint16_t);
557     stage3=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_3_INDEX, uint16_t);
558     idx=UCNV_EXT_FROM_U(stage12, stage3, idx, firstCP);
559 
560     stage3b=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_3B_INDEX, uint32_t);
561     value=stage3b[idx];
562     if(value==0) {
563         return 0;
564     }
565 
566     /*
567      * Tests for (value&UCNV_EXT_FROM_U_RESERVED_MASK)==0:
568      * Do not interpret values with reserved bits used, for forward compatibility,
569      * and do not even remember intermediate results with reserved bits used.
570      */
571 
572     if(UCNV_EXT_TO_U_IS_PARTIAL(value)) {
573         /* partial match, enter the loop below */
574         idx=(int32_t)UCNV_EXT_FROM_U_GET_PARTIAL_INDEX(value);
575 
576         /* initialize */
577         fromUTableUChars=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_UCHARS_INDEX, UChar);
578         fromUTableValues=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_VALUES_INDEX, uint32_t);
579 
580         matchValue=0;
581         i=j=matchLength=0;
582 
583         /* we must not remember fallback matches when not using fallbacks */
584 
585         /* match input units until there is a full match or the input is consumed */
586         for(;;) {
587             /* go to the next section */
588             fromUSectionUChars=fromUTableUChars+idx;
589             fromUSectionValues=fromUTableValues+idx;
590 
591             /* read first pair of the section */
592             length=*fromUSectionUChars++;
593             value=*fromUSectionValues++;
594             if(value!=0 && extFromUUseMapping(useFallback, value, firstCP)) {
595                 /* remember longest match so far */
596                 matchValue=value;
597                 matchLength=2+i+j;
598             }
599 
600             /* match pre[] then src[] */
601             if(i<preLength) {
602                 c=pre[i++];
603             } else if(j<srcLength) {
604                 c=src[j++];
605             } else {
606                 /* all input consumed, partial match */
607                 if(flush || (length=(i+j))>UCNV_EXT_MAX_UCHARS) {
608                     /*
609                      * end of the entire input stream, stop with the longest match so far
610                      * or: partial match must not be longer than UCNV_EXT_MAX_UCHARS
611                      * because it must fit into state buffers
612                      */
613                     break;
614                 } else {
615                     /* continue with more input next time */
616                     return -(2+length);
617                 }
618             }
619 
620             /* search for the current UChar */
621             idx=ucnv_extFindFromU(fromUSectionUChars, length, c);
622             if(idx<0) {
623                 /* no match here, stop with the longest match so far */
624                 break;
625             } else {
626                 value=fromUSectionValues[idx];
627                 if(UCNV_EXT_FROM_U_IS_PARTIAL(value)) {
628                     /* partial match, continue */
629                     idx=(int32_t)UCNV_EXT_FROM_U_GET_PARTIAL_INDEX(value);
630                 } else {
631                     if(extFromUUseMapping(useFallback, value, firstCP)) {
632                         /* full match, stop with result */
633                         matchValue=value;
634                         matchLength=2+i+j;
635                     } else {
636                         /* full match on fallback not taken, stop with the longest match so far */
637                     }
638                     break;
639                 }
640             }
641         }
642 
643         if(matchLength==0) {
644             /* no match at all */
645             return 0;
646         }
647     } else /* result from firstCP trie lookup */ {
648         if(extFromUUseMapping(useFallback, value, firstCP)) {
649             /* full match, stop with result */
650             matchValue=value;
651             matchLength=2;
652         } else {
653             /* fallback not taken */
654             return 0;
655         }
656     }
657 
658     /* return result */
659     if(matchValue==UCNV_EXT_FROM_U_SUBCHAR1) {
660         return 1; /* assert matchLength==2 */
661     }
662 
663     *pMatchValue=matchValue;
664     return matchLength;
665 }
666 
667 /*
668  * @param value fromUnicode mapping table value; ignores roundtrip and reserved bits
669  */
670 static inline void
ucnv_extWriteFromU(UConverter * cnv,const int32_t * cx,uint32_t value,char ** target,const char * targetLimit,int32_t ** offsets,int32_t srcIndex,UErrorCode * pErrorCode)671 ucnv_extWriteFromU(UConverter *cnv, const int32_t *cx,
672                    uint32_t value,
673                    char **target, const char *targetLimit,
674                    int32_t **offsets, int32_t srcIndex,
675                    UErrorCode *pErrorCode) {
676     uint8_t buffer[1+UCNV_EXT_MAX_BYTES];
677     const uint8_t *result;
678     int32_t length, prevLength;
679 
680     length=UCNV_EXT_FROM_U_GET_LENGTH(value);
681     value=(uint32_t)UCNV_EXT_FROM_U_GET_DATA(value);
682 
683     /* output the result */
684     if(length<=UCNV_EXT_FROM_U_MAX_DIRECT_LENGTH) {
685         /*
686          * Generate a byte array and then write it below.
687          * This is not the fastest possible way, but it should be ok for
688          * extension mappings, and it is much simpler.
689          * Offset and overflow handling are only done once this way.
690          */
691         uint8_t *p=buffer+1; /* reserve buffer[0] for shiftByte below */
692         switch(length) {
693         case 3:
694             *p++=(uint8_t)(value>>16);
695             U_FALLTHROUGH;
696         case 2:
697             *p++=(uint8_t)(value>>8);
698             U_FALLTHROUGH;
699         case 1:
700             *p++=(uint8_t)value;
701             U_FALLTHROUGH;
702         default:
703             break; /* will never occur */
704         }
705         result=buffer+1;
706     } else {
707         result=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_BYTES_INDEX, uint8_t)+value;
708     }
709 
710     /* with correct data we have length>0 */
711 
712     if((prevLength=cnv->fromUnicodeStatus)!=0) {
713         /* handle SI/SO stateful output */
714         uint8_t shiftByte;
715 
716         if(prevLength>1 && length==1) {
717             /* change from double-byte mode to single-byte */
718             shiftByte=(uint8_t)UCNV_SI;
719             cnv->fromUnicodeStatus=1;
720         } else if(prevLength==1 && length>1) {
721             /* change from single-byte mode to double-byte */
722             shiftByte=(uint8_t)UCNV_SO;
723             cnv->fromUnicodeStatus=2;
724         } else {
725             shiftByte=0;
726         }
727 
728         if(shiftByte!=0) {
729             /* prepend the shift byte to the result bytes */
730             buffer[0]=shiftByte;
731             if(result!=buffer+1) {
732                 uprv_memcpy(buffer+1, result, length);
733             }
734             result=buffer;
735             ++length;
736         }
737     }
738 
739     ucnv_fromUWriteBytes(cnv, (const char *)result, length,
740                          target, targetLimit,
741                          offsets, srcIndex,
742                          pErrorCode);
743 }
744 
745 /*
746  * target<targetLimit; set error code for overflow
747  */
748 U_CFUNC UBool
ucnv_extInitialMatchFromU(UConverter * cnv,const int32_t * cx,UChar32 cp,const UChar ** src,const UChar * srcLimit,char ** target,const char * targetLimit,int32_t ** offsets,int32_t srcIndex,UBool flush,UErrorCode * pErrorCode)749 ucnv_extInitialMatchFromU(UConverter *cnv, const int32_t *cx,
750                           UChar32 cp,
751                           const UChar **src, const UChar *srcLimit,
752                           char **target, const char *targetLimit,
753                           int32_t **offsets, int32_t srcIndex,
754                           UBool flush,
755                           UErrorCode *pErrorCode) {
756     uint32_t value = 0;  /* initialize output-only param to 0 to silence gcc */
757     int32_t match;
758 
759     /* try to match */
760     match=ucnv_extMatchFromU(cx, cp,
761                              NULL, 0,
762                              *src, (int32_t)(srcLimit-*src),
763                              &value,
764                              cnv->useFallback, flush);
765 
766     /* reject a match if the result is a single byte for DBCS-only */
767     if( match>=2 &&
768         !(UCNV_EXT_FROM_U_GET_LENGTH(value)==1 &&
769           cnv->sharedData->mbcs.outputType==MBCS_OUTPUT_DBCS_ONLY)
770     ) {
771         /* advance src pointer for the consumed input */
772         *src+=match-2; /* remove 2 for the initial code point */
773 
774         /* write result to target */
775         ucnv_extWriteFromU(cnv, cx,
776                            value,
777                            target, targetLimit,
778                            offsets, srcIndex,
779                            pErrorCode);
780         return TRUE;
781     } else if(match<0) {
782         /* save state for partial match */
783         const UChar *s;
784         int32_t j;
785 
786         /* copy the first code point */
787         cnv->preFromUFirstCP=cp;
788 
789         /* now copy the newly consumed input */
790         s=*src;
791         match=-match-2; /* remove 2 for the initial code point */
792         for(j=0; j<match; ++j) {
793             cnv->preFromU[j]=*s++;
794         }
795         *src=s; /* same as *src=srcLimit; because we reached the end of input */
796         cnv->preFromULength=(int8_t)match;
797         return TRUE;
798     } else if(match==1) {
799         /* matched, no mapping but request for <subchar1> */
800         cnv->useSubChar1=TRUE;
801         return FALSE;
802     } else /* match==0 no match */ {
803         return FALSE;
804     }
805 }
806 
807 /*
808  * Used by ISO 2022 implementation.
809  * @return number of bytes in *pValue; negative number if fallback; 0 for no mapping
810  */
811 U_CFUNC int32_t
ucnv_extSimpleMatchFromU(const int32_t * cx,UChar32 cp,uint32_t * pValue,UBool useFallback)812 ucnv_extSimpleMatchFromU(const int32_t *cx,
813                          UChar32 cp, uint32_t *pValue,
814                          UBool useFallback) {
815     uint32_t value;
816     int32_t match;
817 
818     /* try to match */
819     match=ucnv_extMatchFromU(cx,
820                              cp,
821                              NULL, 0,
822                              NULL, 0,
823                              &value,
824                              useFallback, TRUE);
825     if(match>=2) {
826         /* write result for simple, single-character conversion */
827         int32_t length;
828         int isRoundtrip;
829 
830         isRoundtrip=UCNV_EXT_FROM_U_IS_ROUNDTRIP(value);
831         length=UCNV_EXT_FROM_U_GET_LENGTH(value);
832         value=(uint32_t)UCNV_EXT_FROM_U_GET_DATA(value);
833 
834         if(length<=UCNV_EXT_FROM_U_MAX_DIRECT_LENGTH) {
835             *pValue=value;
836             return isRoundtrip ? length : -length;
837 #if 0 /* not currently used */
838         } else if(length==4) {
839             /* de-serialize a 4-byte result */
840             const uint8_t *result=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_BYTES_INDEX, uint8_t)+value;
841             *pValue=
842                 ((uint32_t)result[0]<<24)|
843                 ((uint32_t)result[1]<<16)|
844                 ((uint32_t)result[2]<<8)|
845                 result[3];
846             return isRoundtrip ? 4 : -4;
847 #endif
848         }
849     }
850 
851     /*
852      * return no match because
853      * - match>1 && resultLength>4: result too long for simple conversion
854      * - match==1: no match found, <subchar1> preferred
855      * - match==0: no match found in the first place
856      * - match<0: partial match, not supported for simple conversion (and flush==TRUE)
857      */
858     return 0;
859 }
860 
861 /*
862  * continue partial match with new input, requires cnv->preFromUFirstCP>=0
863  * never called for simple, single-character conversion
864  */
865 U_CFUNC void
ucnv_extContinueMatchFromU(UConverter * cnv,UConverterFromUnicodeArgs * pArgs,int32_t srcIndex,UErrorCode * pErrorCode)866 ucnv_extContinueMatchFromU(UConverter *cnv,
867                            UConverterFromUnicodeArgs *pArgs, int32_t srcIndex,
868                            UErrorCode *pErrorCode) {
869     uint32_t value = 0;  /* initialize output-only param to 0 to silence gcc */
870     int32_t match;
871 
872     match=ucnv_extMatchFromU(cnv->sharedData->mbcs.extIndexes,
873                              cnv->preFromUFirstCP,
874                              cnv->preFromU, cnv->preFromULength,
875                              pArgs->source, (int32_t)(pArgs->sourceLimit-pArgs->source),
876                              &value,
877                              cnv->useFallback, pArgs->flush);
878     if(match>=2) {
879         match-=2; /* remove 2 for the initial code point */
880 
881         if(match>=cnv->preFromULength) {
882             /* advance src pointer for the consumed input */
883             pArgs->source+=match-cnv->preFromULength;
884             cnv->preFromULength=0;
885         } else {
886             /* the match did not use all of preFromU[] - keep the rest for replay */
887             int32_t length=cnv->preFromULength-match;
888             u_memmove(cnv->preFromU, cnv->preFromU+match, length);
889             cnv->preFromULength=(int8_t)-length;
890         }
891 
892         /* finish the partial match */
893         cnv->preFromUFirstCP=U_SENTINEL;
894 
895         /* write result */
896         ucnv_extWriteFromU(cnv, cnv->sharedData->mbcs.extIndexes,
897                            value,
898                            &pArgs->target, pArgs->targetLimit,
899                            &pArgs->offsets, srcIndex,
900                            pErrorCode);
901     } else if(match<0) {
902         /* save state for partial match */
903         const UChar *s;
904         int32_t j;
905 
906         /* just _append_ the newly consumed input to preFromU[] */
907         s=pArgs->source;
908         match=-match-2; /* remove 2 for the initial code point */
909         for(j=cnv->preFromULength; j<match; ++j) {
910             U_ASSERT(j>=0);
911             cnv->preFromU[j]=*s++;
912         }
913         pArgs->source=s; /* same as *src=srcLimit; because we reached the end of input */
914         cnv->preFromULength=(int8_t)match;
915     } else /* match==0 or 1 */ {
916         /*
917          * no match
918          *
919          * We need to split the previous input into two parts:
920          *
921          * 1. The first code point is unmappable - that's how we got into
922          *    trying the extension data in the first place.
923          *    We need to move it from the preFromU buffer
924          *    to the error buffer, set an error code,
925          *    and prepare the rest of the previous input for 2.
926          *
927          * 2. The rest of the previous input must be converted once we
928          *    come back from the callback for the first code point.
929          *    At that time, we have to try again from scratch to convert
930          *    these input characters.
931          *    The replay will be handled by the ucnv.c conversion code.
932          */
933 
934         if(match==1) {
935             /* matched, no mapping but request for <subchar1> */
936             cnv->useSubChar1=TRUE;
937         }
938 
939         /* move the first code point to the error field */
940         cnv->fromUChar32=cnv->preFromUFirstCP;
941         cnv->preFromUFirstCP=U_SENTINEL;
942 
943         /* mark preFromU for replay */
944         cnv->preFromULength=-cnv->preFromULength;
945 
946         /* set the error code for unassigned */
947         *pErrorCode=U_INVALID_CHAR_FOUND;
948     }
949 }
950 
951 static UBool
extSetUseMapping(UConverterUnicodeSet which,int32_t minLength,uint32_t value)952 extSetUseMapping(UConverterUnicodeSet which, int32_t minLength, uint32_t value) {
953     if(which==UCNV_ROUNDTRIP_SET) {
954         // Add only code points for which the roundtrip flag is set.
955         // Do not add any fallbacks, even if ucnv_fromUnicode() would use them
956         // (fallbacks from PUA). See the API docs for ucnv_getUnicodeSet().
957         //
958         // By analogy, also do not add "good one-way" mappings.
959         //
960         // Do not add entries with reserved bits set.
961         if(((value&(UCNV_EXT_FROM_U_ROUNDTRIP_FLAG|UCNV_EXT_FROM_U_RESERVED_MASK))!=
962                 UCNV_EXT_FROM_U_ROUNDTRIP_FLAG)) {
963             return FALSE;
964         }
965     } else /* UCNV_ROUNDTRIP_AND_FALLBACK_SET */ {
966         // Do not add entries with reserved bits set.
967         if((value&UCNV_EXT_FROM_U_RESERVED_MASK)!=0) {
968             return FALSE;
969         }
970     }
971     // Do not add <subchar1> entries or other (future?) pseudo-entries
972     // with an output length of 0.
973     return UCNV_EXT_FROM_U_GET_LENGTH(value)>=minLength;
974 }
975 
976 static void
ucnv_extGetUnicodeSetString(const UConverterSharedData * sharedData,const int32_t * cx,const USetAdder * sa,UConverterUnicodeSet which,int32_t minLength,UChar32 firstCP,UChar s[UCNV_EXT_MAX_UCHARS],int32_t length,int32_t sectionIndex,UErrorCode * pErrorCode)977 ucnv_extGetUnicodeSetString(const UConverterSharedData *sharedData,
978                             const int32_t *cx,
979                             const USetAdder *sa,
980                             UConverterUnicodeSet which,
981                             int32_t minLength,
982                             UChar32 firstCP,
983                             UChar s[UCNV_EXT_MAX_UCHARS], int32_t length,
984                             int32_t sectionIndex,
985                             UErrorCode *pErrorCode) {
986     const UChar *fromUSectionUChars;
987     const uint32_t *fromUSectionValues;
988 
989     uint32_t value;
990     int32_t i, count;
991 
992     fromUSectionUChars=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_UCHARS_INDEX, UChar)+sectionIndex;
993     fromUSectionValues=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_VALUES_INDEX, uint32_t)+sectionIndex;
994 
995     /* read first pair of the section */
996     count=*fromUSectionUChars++;
997     value=*fromUSectionValues++;
998 
999     if(extSetUseMapping(which, minLength, value)) {
1000         if(length==U16_LENGTH(firstCP)) {
1001             /* add the initial code point */
1002             sa->add(sa->set, firstCP);
1003         } else {
1004             /* add the string so far */
1005             sa->addString(sa->set, s, length);
1006         }
1007     }
1008 
1009     for(i=0; i<count; ++i) {
1010         /* append this code unit and recurse or add the string */
1011         s[length]=fromUSectionUChars[i];
1012         value=fromUSectionValues[i];
1013 
1014         if(value==0) {
1015             /* no mapping, do nothing */
1016         } else if(UCNV_EXT_FROM_U_IS_PARTIAL(value)) {
1017             ucnv_extGetUnicodeSetString(
1018                 sharedData, cx, sa, which, minLength,
1019                 firstCP, s, length+1,
1020                 (int32_t)UCNV_EXT_FROM_U_GET_PARTIAL_INDEX(value),
1021                 pErrorCode);
1022         } else if(extSetUseMapping(which, minLength, value)) {
1023             sa->addString(sa->set, s, length+1);
1024         }
1025     }
1026 }
1027 
1028 U_CFUNC void
ucnv_extGetUnicodeSet(const UConverterSharedData * sharedData,const USetAdder * sa,UConverterUnicodeSet which,UConverterSetFilter filter,UErrorCode * pErrorCode)1029 ucnv_extGetUnicodeSet(const UConverterSharedData *sharedData,
1030                       const USetAdder *sa,
1031                       UConverterUnicodeSet which,
1032                       UConverterSetFilter filter,
1033                       UErrorCode *pErrorCode) {
1034     const int32_t *cx;
1035     const uint16_t *stage12, *stage3, *ps2, *ps3;
1036     const uint32_t *stage3b;
1037 
1038     uint32_t value;
1039     int32_t st1, stage1Length, st2, st3, minLength;
1040 
1041     UChar s[UCNV_EXT_MAX_UCHARS];
1042     UChar32 c;
1043     int32_t length;
1044 
1045     cx=sharedData->mbcs.extIndexes;
1046     if(cx==NULL) {
1047         return;
1048     }
1049 
1050     stage12=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_12_INDEX, uint16_t);
1051     stage3=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_3_INDEX, uint16_t);
1052     stage3b=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_3B_INDEX, uint32_t);
1053 
1054     stage1Length=cx[UCNV_EXT_FROM_U_STAGE_1_LENGTH];
1055 
1056     /* enumerate the from-Unicode trie table */
1057     c=0; /* keep track of the current code point while enumerating */
1058 
1059     if(filter==UCNV_SET_FILTER_2022_CN) {
1060         minLength=3;
1061     } else if( sharedData->mbcs.outputType==MBCS_OUTPUT_DBCS_ONLY ||
1062                filter!=UCNV_SET_FILTER_NONE
1063     ) {
1064         /* DBCS-only, ignore single-byte results */
1065         minLength=2;
1066     } else {
1067         minLength=1;
1068     }
1069 
1070     /*
1071      * the trie enumeration is almost the same as
1072      * in MBCSGetUnicodeSet() for MBCS_OUTPUT_1
1073      */
1074     for(st1=0; st1<stage1Length; ++st1) {
1075         st2=stage12[st1];
1076         if(st2>stage1Length) {
1077             ps2=stage12+st2;
1078             for(st2=0; st2<64; ++st2) {
1079                 if((st3=(int32_t)ps2[st2]<<UCNV_EXT_STAGE_2_LEFT_SHIFT)!=0) {
1080                     /* read the stage 3 block */
1081                     ps3=stage3+st3;
1082 
1083                     do {
1084                         value=stage3b[*ps3++];
1085                         if(value==0) {
1086                             /* no mapping, do nothing */
1087                         } else if(UCNV_EXT_FROM_U_IS_PARTIAL(value)) {
1088                             // Recurse for partial results.
1089                             length=0;
1090                             U16_APPEND_UNSAFE(s, length, c);
1091                             ucnv_extGetUnicodeSetString(
1092                                 sharedData, cx, sa, which, minLength,
1093                                 c, s, length,
1094                                 (int32_t)UCNV_EXT_FROM_U_GET_PARTIAL_INDEX(value),
1095                                 pErrorCode);
1096                         } else if(extSetUseMapping(which, minLength, value)) {
1097                             switch(filter) {
1098                             case UCNV_SET_FILTER_2022_CN:
1099                                 if(!(UCNV_EXT_FROM_U_GET_LENGTH(value)==3 && UCNV_EXT_FROM_U_GET_DATA(value)<=0x82ffff)) {
1100                                     continue;
1101                                 }
1102                                 break;
1103                             case UCNV_SET_FILTER_SJIS:
1104                                 if(!(UCNV_EXT_FROM_U_GET_LENGTH(value)==2 && (value=UCNV_EXT_FROM_U_GET_DATA(value))>=0x8140 && value<=0xeffc)) {
1105                                     continue;
1106                                 }
1107                                 break;
1108                             case UCNV_SET_FILTER_GR94DBCS:
1109                                 if(!(UCNV_EXT_FROM_U_GET_LENGTH(value)==2 &&
1110                                      (uint16_t)((value=UCNV_EXT_FROM_U_GET_DATA(value))-0xa1a1)<=(0xfefe - 0xa1a1) &&
1111                                      (uint8_t)(value-0xa1)<=(0xfe - 0xa1))) {
1112                                     continue;
1113                                 }
1114                                 break;
1115                             case UCNV_SET_FILTER_HZ:
1116                                 if(!(UCNV_EXT_FROM_U_GET_LENGTH(value)==2 &&
1117                                      (uint16_t)((value=UCNV_EXT_FROM_U_GET_DATA(value))-0xa1a1)<=(0xfdfe - 0xa1a1) &&
1118                                      (uint8_t)(value-0xa1)<=(0xfe - 0xa1))) {
1119                                     continue;
1120                                 }
1121                                 break;
1122                             default:
1123                                 /*
1124                                  * UCNV_SET_FILTER_NONE,
1125                                  * or UCNV_SET_FILTER_DBCS_ONLY which is handled via minLength
1126                                  */
1127                                 break;
1128                             }
1129                             sa->add(sa->set, c);
1130                         }
1131                     } while((++c&0xf)!=0);
1132                 } else {
1133                     c+=16; /* empty stage 3 block */
1134                 }
1135             }
1136         } else {
1137             c+=1024; /* empty stage 2 block */
1138         }
1139     }
1140 }
1141 
1142 #endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */
1143