1 /*
2 *******************************************************************************
3 *
4 * Copyright (C) 1998-2007, International Business Machines
5 * Corporation and others. All Rights Reserved.
6 *
7 *******************************************************************************
8 *
9 * Private implementation header for C collation
10 * file name: ucol_imp.h
11 * encoding: US-ASCII
12 * tab size: 8 (not used)
13 * indentation:4
14 *
15 * created on: 2000dec11
16 * created by: Vladimir Weinstein
17 *
18 * Modification history
19 * Date Name Comments
20 * 02/16/2001 synwee Added UCOL_GETPREVCE for the use in ucoleitr
21 * 02/27/2001 synwee Added getMaxExpansion data structure in UCollator
22 * 03/02/2001 synwee Added UCOL_IMPLICIT_CE
23 * 03/12/2001 synwee Added pointer start to collIterate.
24 */
25
26 #ifndef UCOL_IMP_H
27 #define UCOL_IMP_H
28
29 #include "unicode/utypes.h"
30
31 #define UCA_DATA_TYPE "icu"
32 #define UCA_DATA_NAME "ucadata"
33 #define INVC_DATA_TYPE "icu"
34 #define INVC_DATA_NAME "invuca"
35
36 /**
37 * Convenience string denoting the Collation data tree
38 * @internal ICU 3.0
39 */
40 #define U_ICUDATA_COLL U_ICUDATA_NAME U_TREE_SEPARATOR_STRING "coll"
41
42 #if !UCONFIG_NO_COLLATION
43
44 #include "unicode/ucol.h"
45 #include "utrie.h"
46 #include "uresimp.h"
47 #include "unicode/udata.h"
48 #include "unicode/uiter.h"
49
50 /* This is the internal header file which contains important declarations for
51 * the collation framework.
52 * Ready to use collators are stored as binary images. Both UCA and tailorings
53 * share the same binary format. Individual files (currently only UCA) have a
54 * udata header in front of the image and should be opened using udata_open.
55 * Tailoring images are currently stored inside resource bundles and are intialized
56 * through ucol_open API.
57 *
58 * The following describes the formats for collation binaries
59 * (UCA & tailorings) and for the inverse UCA table.
60 * Substructures are described in the collation design document at
61 * http://source.icu-project.org/repos/icu/icuhtml/trunk/design/collation/ICU_collation_design.htm
62 *
63 * -------------------------------------------------------------
64 *
65 * Here is the format of binary collation image.
66 *
67 * Physical order of structures:
68 * - header (UCATableHeader)
69 * - options (UColOptionSet)
70 * - expansions (CE[])
71 * - contractions (UChar[contractionSize] + CE[contractionSize])
72 * - serialized UTrie with mappings of code points to CEs
73 * - max expansion tables (CE[endExpansionCECount] + uint8_t[endExpansionCECount])
74 * - two bit sets for backward processing in strcoll (identical prefixes)
75 * and for backward CE iteration (each set is uint8_t[UCOL_UNSAFECP_TABLE_SIZE])
76 * - UCA constants (UCAConstants)
77 * - UCA contractions (UChar[contractionUCACombosSize][contractionUCACombosWidth])
78 *
79 * UCATableHeader fields:
80 *
81 * int32_t size; - image size in bytes
82 *
83 * Offsets to interesting data. All offsets are in bytes.
84 * to get the address add to the header address and cast properly.
85 * Some offsets are zero if the corresponding structures are empty.
86 *
87 * Tailoring binaries that only set options and contain no mappings etc.
88 * will have all offsets 0 except for the options and expansion offsets,
89 * which give the position and length of the options array.
90 *
91 * uint32_t options; - offset to default collator options (UColOptionSet *),
92 * a set of 32-bit values. See declaration of UColOptionSet for more details
93 *
94 * uint32_t UCAConsts; - only used (!=0) in UCA image - structure which holds values for indirect positioning and implicit ranges
95 * See declaration of UCAConstants structure. This is a set of unsigned 32-bit values used to store
96 * important constant values that are defined in the UCA and used for building and runtime.
97 *
98 * uint32_t contractionUCACombos; - only used (!=0) in UCA image - list of UCA contractions. This is a zero terminated array of UChar[contractionUCACombosWidth],
99 * containing contractions from the UCA. These are needed in the build process to copy UCA contractions
100 * in case the base contraction symbol is tailored.
101 *
102 * uint32_t magic; - must contain UCOL_HEADER_MAGIC (formatVersion 2.3)
103 *
104 * uint32_t mappingPosition; - offset to UTrie (const uint8_t *mappingPosition). This is a serialized UTrie and should be treated as such.
105 * Used as a primary lookup table for collation elements.
106 *
107 * uint32_t expansion; - offset to expansion table (uint32_t *expansion). This is an array of expansion CEs. Never 0.
108 *
109 * uint32_t contractionIndex; - offset to contraction table (UChar *contractionIndex). Used to look up contraction sequences. Contents
110 * are aligned with the contents of contractionCEs table. 0 if no contractions.
111 *
112 * uint32_t contractionCEs; - offset to resulting contraction CEs (uint32_t *contractionCEs). When a contraction is resolved in the
113 * in the contractionIndex table, the resulting index is used to look up corresponding CE in this table.
114 * 0 if no contractions.
115 * uint32_t contractionSize; - size of contraction table in elements (both Index and CEs).
116 *
117 * Tables described below are used for Boyer-Moore searching algorithm - they define the size of longest expansion
118 * and last CEs in expansions.
119 * uint32_t endExpansionCE; - offset to array of last collation element in expansion (uint32_t *).
120 * Never 0.
121 * uint32_t expansionCESize; - array of maximum expansion sizes (uint8_t *)
122 * int32_t endExpansionCECount; - size of endExpansionCE. See UCOL_GETMAXEXPANSION
123 * for the usage model
124 *
125 * These two offsets point to byte tables that are used in the backup heuristics.
126 * uint32_t unsafeCP; - hash table of unsafe code points (uint8_t *). See ucol_unsafeCP function.
127 * uint32_t contrEndCP; - hash table of final code points in contractions (uint8_t *). See ucol_contractionEndCP.
128 *
129 * int32_t contractionUCACombosSize; - number of UChar[contractionUCACombosWidth] in contractionUCACombos
130 * (formatVersion 2.3)
131 * UBool jamoSpecial; - Jamo special indicator (uint8_t). If TRUE, Jamos are special, so we cannot use simple Hangul decomposition.
132 * UBool isBigEndian; - endianness of this collation binary (formatVersion 2.3)
133 * uint8_t charSetFamily; - charset family of this collation binary (formatVersion 2.3)
134 * uint8_t contractionUCACombosWidth; - number of UChars per UCA contraction in contractionUCACombos (formatVersion 2.3)
135 *
136 * Various version fields
137 * UVersionInfo version; - version 4 uint8_t
138 * UVersionInfo UCAVersion; - version 4 uint8_t
139 * UVersionInfo UCDVersion; - version 4 uint8_t
140 * UVersionInfo formatVersion; - version of the format of the collation binary
141 * same formatVersion as in ucadata.icu's UDataInfo header
142 * (formatVersion 2.3)
143 *
144 * uint8_t reserved[84]; - currently unused
145 *
146 * -------------------------------------------------------------
147 *
148 * Inverse UCA is used for constructing collators from rules. It is always an individual file
149 * and always has a UDataInfo header.
150 * here is the structure:
151 *
152 * uint32_t byteSize; - size of inverse UCA image in bytes
153 * uint32_t tableSize; - length of inverse table (number of uint32_t[3] rows)
154 * uint32_t contsSize; - size of continuation table (number of UChars in table)
155 *
156 * uint32_t table; - offset to inverse table (uint32_t *)
157 * Inverse table contains of rows of 3 uint32_t values. First two values are CE and a possible continuation
158 * the third value is either a code unit (if there is only one code unit for element) or an index to continuation
159 * (number of code units combined with an index).
160 * table. If more than one codepoint have the same CE, continuation table contains code units separated by FFFF and final
161 * code unit sequence for a CE is terminated by FFFE.
162 * uint32_t conts; - offset to continuation table (uint16_t *). Contains code units that transform to a same CE.
163 *
164 * UVersionInfo UCAVersion; - version of the UCA, read from file 4 uint8_t
165 * uint8_t padding[8]; - padding 8 uint8_t
166 * Header is followed by the table and continuation table.
167 */
168
169 /* let us know whether reserved fields are reset to zero or junked */
170 #define UCOL_HEADER_MAGIC 0x20030618
171
172 /* UDataInfo for UCA mapping table */
173 /* dataFormat="UCol" */
174 #define UCA_DATA_FORMAT_0 ((uint8_t)0x55)
175 #define UCA_DATA_FORMAT_1 ((uint8_t)0x43)
176 #define UCA_DATA_FORMAT_2 ((uint8_t)0x6f)
177 #define UCA_DATA_FORMAT_3 ((uint8_t)0x6c)
178
179 #define UCA_FORMAT_VERSION_0 ((uint8_t)2)
180 #define UCA_FORMAT_VERSION_1 ((uint8_t)3)
181 #define UCA_FORMAT_VERSION_2 ((uint8_t)0)
182 #define UCA_FORMAT_VERSION_3 ((uint8_t)0)
183
184 /* UDataInfo for inverse UCA table */
185 /* dataFormat="InvC" */
186 #define INVUCA_DATA_FORMAT_0 ((uint8_t)0x49)
187 #define INVUCA_DATA_FORMAT_1 ((uint8_t)0x6E)
188 #define INVUCA_DATA_FORMAT_2 ((uint8_t)0x76)
189 #define INVUCA_DATA_FORMAT_3 ((uint8_t)0x43)
190
191 #define INVUCA_FORMAT_VERSION_0 ((uint8_t)2)
192 #define INVUCA_FORMAT_VERSION_1 ((uint8_t)1)
193 #define INVUCA_FORMAT_VERSION_2 ((uint8_t)0)
194 #define INVUCA_FORMAT_VERSION_3 ((uint8_t)0)
195
196 /* This is the size of the stack allocated buffer for sortkey generation and similar operations */
197 /* if it is too small, heap allocation will occur.*/
198 /* you can change this value if you need memory - it will affect the performance, though, since we're going to malloc */
199 #define UCOL_MAX_BUFFER 128
200 #define UCOL_PRIMARY_MAX_BUFFER 8*UCOL_MAX_BUFFER
201 #define UCOL_SECONDARY_MAX_BUFFER UCOL_MAX_BUFFER
202 #define UCOL_TERTIARY_MAX_BUFFER UCOL_MAX_BUFFER
203 #define UCOL_CASE_MAX_BUFFER UCOL_MAX_BUFFER/4
204 #define UCOL_QUAD_MAX_BUFFER 2*UCOL_MAX_BUFFER
205
206 #define UCOL_NORMALIZATION_GROWTH 2
207 #define UCOL_NORMALIZATION_MAX_BUFFER UCOL_MAX_BUFFER*UCOL_NORMALIZATION_GROWTH
208
209 /* This writable buffer is used if we encounter Thai and need to reorder the string on the fly */
210 /* Sometimes we already have a writable buffer (like in case of normalized strings). */
211 /*
212 you can change this value to any value >= 4 if you need memory -
213 it will affect the performance, though, since we're going to malloc.
214 Note 3 is the minimum value for Thai collation and 4 is the
215 minimum number for special Jamo
216 */
217 #define UCOL_WRITABLE_BUFFER_SIZE 256
218
219 /* This is the size of the buffer for expansion CE's */
220 /* In reality we should not have to deal with expm sequences longer then 16 */
221 /* you can change this value if you need memory */
222 /* WARNING THIS BUFFER DOES NOT HAVE MALLOC FALLBACK. If you make it too small, you'll get in trouble */
223 /* Reasonable small value is around 10, if you don't do Arabic or other funky collations that have long expansion sequence */
224 /* This is the longest expansion sequence we can handle without bombing out */
225 #define UCOL_EXPAND_CE_BUFFER_SIZE 512 /* synwee :TODO revert back 64*/
226
227
228 /* Unsafe UChar hash table table size. */
229 /* size is 32 bytes for 1 bit for each latin 1 char + some power of two for */
230 /* hashing the rest of the chars. Size in bytes */
231 #define UCOL_UNSAFECP_TABLE_SIZE 1056
232 /* mask value down to "some power of two"-1 */
233 /* number of bits, not num of bytes. */
234 #define UCOL_UNSAFECP_TABLE_MASK 0x1fff
235
236
237 /* flags bits for collIterate.flags */
238 /* */
239 /* NORM - set for incremental normalize of source string */
240 #define UCOL_ITER_NORM 1
241
242 #define UCOL_ITER_HASLEN 2
243
244 /* UCOL_ITER_INNORMBUF - set if the "pos" is in */
245 /* the writable side buffer, handling */
246 /* incrementally normalized characters. */
247 #define UCOL_ITER_INNORMBUF 4
248
249 /* UCOL_ITER_ALLOCATED - set if this iterator has */
250 /* malloced storage to expand a buffer. */
251 #define UCOL_ITER_ALLOCATED 8
252 /* UCOL_HIRAGANA_Q - note if the codepoint was hiragana */
253 #define UCOL_HIRAGANA_Q 16
254 /* UCOL_WAS_HIRAGANA - set to TRUE if there was a Hiragana */
255 /* otherwise set to false */
256 #define UCOL_WAS_HIRAGANA 32
257 /* UCOL_USE_ITERATOR - set this if collIterate uses a */
258 /* character iterator instead of simply accessing string */
259 /* by index */
260 #define UCOL_USE_ITERATOR 64
261
262 #define NFC_ZERO_CC_BLOCK_LIMIT_ 0x300
263
264 typedef struct collIterate {
265 UChar *string; /* Original string */
266 /* UChar *start; Pointer to the start of the source string. Either points to string
267 or to writableBuffer */
268 UChar *endp; /* string end ptr. Is undefined for null terminated strings */
269 UChar *pos; /* This is position in the string. Can be to original or writable buf */
270
271 uint32_t *toReturn; /* This is the CE from CEs buffer that should be returned */
272 uint32_t *CEpos; /* This is the position to which we have stored processed CEs */
273 UChar *writableBuffer;
274 uint32_t writableBufSize;
275 UChar *fcdPosition; /* Position in the original string to continue FCD check from. */
276 const UCollator *coll;
277 uint8_t flags;
278 uint8_t origFlags;
279 uint32_t CEs[UCOL_EXPAND_CE_BUFFER_SIZE]; /* This is where we store CEs */
280 UChar stackWritableBuffer[UCOL_WRITABLE_BUFFER_SIZE]; /* A writable buffer. */
281 UCharIterator *iterator;
282 /*int32_t iteratorIndex;*/
283 } collIterate;
284
285 /*
286 struct used internally in getSpecial*CE.
287 data similar to collIterate.
288 */
289 struct collIterateState {
290 UChar *pos; /* This is position in the string. Can be to original or writable buf */
291 UChar *fcdPosition; /* Position in the original string to continue FCD check from. */
292 UChar *bufferaddress; /* address of the normalization buffer */
293 uint32_t buffersize;
294 uint8_t flags;
295 uint8_t origFlags;
296 uint32_t iteratorIndex;
297 int32_t iteratorMove;
298 };
299
300 U_CAPI void U_EXPORT2
301 uprv_init_collIterate(const UCollator *collator, const UChar *sourceString, int32_t sourceLen, collIterate *s);
302
303
304 struct UCollationElements
305 {
306 /**
307 * Struct wrapper for source data
308 */
309 collIterate iteratordata_;
310 /**
311 * Indicates if this data has been reset.
312 */
313 UBool reset_;
314 /**
315 * Indicates if the data should be deleted.
316 */
317 UBool isWritable;
318 };
319
320
321 #define UCOL_LEVELTERMINATOR 1
322
323 /* mask off anything but primary order */
324 #define UCOL_PRIMARYORDERMASK 0xffff0000
325 /* mask off anything but secondary order */
326 #define UCOL_SECONDARYORDERMASK 0x0000ff00
327 /* mask off anything but tertiary order */
328 #define UCOL_TERTIARYORDERMASK 0x000000ff
329 /* primary order shift */
330 #define UCOL_PRIMARYORDERSHIFT 16
331 /* secondary order shift */
332 #define UCOL_SECONDARYORDERSHIFT 8
333
334 #define UCOL_BYTE_SIZE_MASK 0xFF
335
336 #define UCOL_CASE_BYTE_START 0x80
337 #define UCOL_CASE_SHIFT_START 7
338
339 #define UCOL_IGNORABLE 0
340
341 /* get weights from a CE */
342 #define UCOL_PRIMARYORDER(order) (((order) & UCOL_PRIMARYORDERMASK)>> UCOL_PRIMARYORDERSHIFT)
343 #define UCOL_SECONDARYORDER(order) (((order) & UCOL_SECONDARYORDERMASK)>> UCOL_SECONDARYORDERSHIFT)
344 #define UCOL_TERTIARYORDER(order) ((order) & UCOL_TERTIARYORDERMASK)
345
346 /**
347 * Determine if a character is a Thai vowel (which sorts after
348 * its base consonant).
349 */
350 #define UCOL_ISTHAIPREVOWEL(ch) ((((uint32_t)(ch) - 0xe40) <= (0xe44 - 0xe40)) || \
351 (((uint32_t)(ch) - 0xec0) <= (0xec4 - 0xec0)))
352
353 /**
354 * Determine if a character is a Thai base consonant
355 */
356 #define UCOL_ISTHAIBASECONSONANT(ch) ((uint32_t)(ch) - 0xe01) <= (0xe2e - 0xe01)
357
358 #define UCOL_ISJAMO(ch) ((((uint32_t)(ch) - 0x1100) <= (0x1112 - 0x1100)) || \
359 (((uint32_t)(ch) - 0x1161) <= (0x1175 - 0x1161)) || \
360 (((uint32_t)(ch) - 0x11A8) <= (0x11C2 - 0x11A8)))
361
362
363
364 #if 0
365 /* initializes collIterate structure */
366 /* made as macro to speed up things */
367 #define init_collIterate(collator, sourceString, sourceLen, s) { \
368 (s)->start = (s)->string = (s)->pos = (UChar *)(sourceString); \
369 (s)->endp = (sourceLen) == -1 ? NULL :(UChar *)(sourceString)+(sourceLen); \
370 (s)->CEpos = (s)->toReturn = (s)->CEs; \
371 (s)->isThai = TRUE; \
372 (s)->writableBuffer = (s)->stackWritableBuffer; \
373 (s)->writableBufSize = UCOL_WRITABLE_BUFFER_SIZE; \
374 (s)->coll = (collator); \
375 (s)->fcdPosition = 0; \
376 (s)->flags = 0; \
377 if(((collator)->normalizationMode == UCOL_ON)) (s)->flags |= UCOL_ITER_NORM; \
378 }
379 #endif
380
381
382
383 /*
384 * Macro to get the maximum size of an expansion ending with the argument ce.
385 * Used in the Boyer Moore algorithm.
386 * Note for tailoring, the UCA maxexpansion table has been merged.
387 * Hence we only have to search the tailored collator only.
388 * @param coll const UCollator pointer
389 * @param order last collation element of the expansion sequence
390 * @param result size of the longest expansion with argument collation element
391 * as the last element
392 */
393 #define UCOL_GETMAXEXPANSION(coll, order, result) { \
394 const uint32_t *start; \
395 const uint32_t *limit; \
396 const uint32_t *mid; \
397 start = (coll)->endExpansionCE; \
398 limit = (coll)->lastEndExpansionCE; \
399 while (start < limit - 1) { \
400 mid = start + ((limit - start) >> 1); \
401 if ((order) <= *mid) { \
402 limit = mid; \
403 } \
404 else { \
405 start = mid; \
406 } \
407 } \
408 if (*start == order) { \
409 result = *((coll)->expansionCESize + (start - (coll)->endExpansionCE)); \
410 } \
411 else if (*limit == order) { \
412 result = *(coll->expansionCESize + (limit - coll->endExpansionCE)); \
413 } \
414 else if ((order & 0xFFFF) == 0x00C0) { \
415 result = 2; \
416 } \
417 else { \
418 result = 1; \
419 } \
420 }
421
422 U_CFUNC
423 uint32_t ucol_prv_getSpecialCE(const UCollator *coll, UChar ch, uint32_t CE, collIterate *source, UErrorCode *status);
424
425 U_CFUNC
426 uint32_t ucol_prv_getSpecialPrevCE(const UCollator *coll, UChar ch, uint32_t CE,
427 collIterate *source, UErrorCode *status);
428 U_CAPI uint32_t U_EXPORT2 ucol_getNextCE(const UCollator *coll, collIterate *collationSource, UErrorCode *status);
429 U_CFUNC uint32_t U_EXPORT2 ucol_getPrevCE(const UCollator *coll,
430 collIterate *collationSource,
431 UErrorCode *status);
432 /* function used by C++ getCollationKey to prevent restarting the calculation */
433 U_CFUNC int32_t
434 ucol_getSortKeyWithAllocation(const UCollator *coll,
435 const UChar *source, int32_t sourceLength,
436 uint8_t **pResult,
437 UErrorCode *pErrorCode);
438
439 /* get some memory */
440 void *ucol_getABuffer(const UCollator *coll, uint32_t size);
441
442 /* worker function for generating sortkeys */
443 U_CFUNC
444 int32_t U_CALLCONV
445 ucol_calcSortKey(const UCollator *coll,
446 const UChar *source,
447 int32_t sourceLength,
448 uint8_t **result,
449 uint32_t resultLength,
450 UBool allocatePrimary,
451 UErrorCode *status);
452
453 U_CFUNC
454 int32_t U_CALLCONV
455 ucol_calcSortKeySimpleTertiary(const UCollator *coll,
456 const UChar *source,
457 int32_t sourceLength,
458 uint8_t **result,
459 uint32_t resultLength,
460 UBool allocatePrimary,
461 UErrorCode *status);
462
463 U_CFUNC
464 int32_t
465 ucol_getSortKeySize(const UCollator *coll, collIterate *s,
466 int32_t currentSize, UColAttributeValue strength,
467 int32_t len);
468 /**
469 * Makes a copy of the Collator's rule data. The format is
470 * that of .col files.
471 *
472 * @param length returns the length of the data, in bytes.
473 * @param status the error status
474 * @return memory, owned by the caller, of size 'length' bytes.
475 * @internal INTERNAL USE ONLY
476 */
477 U_CAPI uint8_t* U_EXPORT2
478 ucol_cloneRuleData(const UCollator *coll, int32_t *length, UErrorCode *status);
479
480 /**
481 * Used to set requested and valid locales on a collator returned by the collator
482 * service.
483 */
484 U_CFUNC void U_EXPORT2
485 ucol_setReqValidLocales(UCollator *coll, char *requestedLocaleToAdopt, char *validLocaleToAdopt);
486
487 #define UCOL_SPECIAL_FLAG 0xF0000000
488 #define UCOL_TAG_SHIFT 24
489 #define UCOL_TAG_MASK 0x0F000000
490 #define INIT_EXP_TABLE_SIZE 1024
491 #define UCOL_NOT_FOUND 0xF0000000
492 #define UCOL_EXPANSION 0xF1000000
493 #define UCOL_CONTRACTION 0xF2000000
494 #define UCOL_THAI 0xF3000000
495 #define UCOL_UNMARKED 0x03
496 #define UCOL_NEW_TERTIARYORDERMASK 0x0000003f
497
498 /* Bit mask for primary collation strength. */
499 #define UCOL_PRIMARYMASK 0xFFFF0000
500
501 /* Bit mask for secondary collation strength. */
502 #define UCOL_SECONDARYMASK 0x0000FF00
503
504 /* Bit mask for tertiary collation strength. */
505 #define UCOL_TERTIARYMASK 0x000000FF
506
507 /**
508 * Internal.
509 * This indicates the last element in a UCollationElements has been consumed.
510 * Compare with the UCOL_NULLORDER, UCOL_NULLORDER is returned if error occurs.
511 */
512 #define UCOL_NO_MORE_CES 0x00010101
513 #define UCOL_NO_MORE_CES_PRIMARY 0x00010000
514 #define UCOL_NO_MORE_CES_SECONDARY 0x00000100
515 #define UCOL_NO_MORE_CES_TERTIARY 0x00000001
516
517 #define isSpecial(CE) ((((CE)&UCOL_SPECIAL_FLAG)>>28)==0xF)
518
519 #define UCOL_UPPER_CASE 0x80
520 #define UCOL_MIXED_CASE 0x40
521 #define UCOL_LOWER_CASE 0x00
522
523 #define UCOL_CONTINUATION_MARKER 0xC0
524 #define UCOL_REMOVE_CONTINUATION 0xFFFFFF3F
525
526 #define isContinuation(CE) (((CE) & UCOL_CONTINUATION_MARKER) == UCOL_CONTINUATION_MARKER)
527 #define isFlagged(CE) (((CE) & 0x80) == 0x80)
528 #define isLongPrimary(CE) (((CE) & 0xC0) == 0xC0)
529
530 #define getCETag(CE) (((CE)&UCOL_TAG_MASK)>>UCOL_TAG_SHIFT)
531 #define isContraction(CE) (isSpecial((CE)) && (getCETag((CE)) == CONTRACTION_TAG))
532 #define isPrefix(CE) (isSpecial((CE)) && (getCETag((CE)) == SPEC_PROC_TAG))
533 #define constructContractCE(tag, CE) (UCOL_SPECIAL_FLAG | ((tag)<<UCOL_TAG_SHIFT) | ((CE))&0xFFFFFF)
534 #define constructSpecProcCE(CE) (UCOL_SPECIAL_FLAG | (SPEC_PROC_TAG<<UCOL_TAG_SHIFT) | ((CE))&0xFFFFFF)
535 #define getContractOffset(CE) ((CE)&0xFFFFFF)
536 #define getExpansionOffset(CE) (((CE)&0x00FFFFF0)>>4)
537 #define getExpansionCount(CE) ((CE)&0xF)
538 #define isCEIgnorable(CE) (((CE) & 0xFFFFFFBF) == 0)
539
540 /* StringSearch internal use */
541 #define inNormBuf(coleiter) ((coleiter)->iteratordata_.flags & UCOL_ITER_INNORMBUF)
542 #define isFCDPointerNull(coleiter) ((coleiter)->iteratordata_.fcdPosition == NULL)
543 #define hasExpansion(coleiter) ((coleiter)->iteratordata_.CEpos != (coleiter)->iteratordata_.CEs)
544 #define getExpansionPrefix(coleiter) ((coleiter)->iteratordata_.toReturn - (coleiter)->iteratordata_.CEs)
545 #define setExpansionPrefix(coleiter, offset) ((coleiter)->iteratordata_.CEs + offset)
546 #define getExpansionSuffix(coleiter) ((coleiter)->iteratordata_.CEpos - (coleiter)->iteratordata_.toReturn)
547 #define setExpansionSuffix(coleiter, offset) ((coleiter)->iteratordata_.toReturn = (coleiter)->iteratordata_.CEpos - leftoverces)
548
549 /* This is an enum that lists magic special byte values from the fractional UCA */
550 /* TODO: all the #defines that refer to special byte values from the UCA should be changed to point here */
551
552 enum {
553 UCOL_BYTE_ZERO = 0x00,
554 UCOL_BYTE_LEVEL_SEPARATOR = 0x01,
555 UCOL_BYTE_SORTKEY_GLUE = 0x02,
556 UCOL_BYTE_SHIFT_PREFIX = 0x03,
557 UCOL_BYTE_UNSHIFTED_MIN = UCOL_BYTE_SHIFT_PREFIX,
558 UCOL_BYTE_FIRST_TAILORED = 0x04,
559 UCOL_BYTE_COMMON = 0x05,
560 UCOL_BYTE_FIRST_UCA = UCOL_BYTE_COMMON,
561 UCOL_CODAN_PLACEHOLDER = 0x27,
562 UCOL_BYTE_LAST_LATIN_PRIMARY = 0x4C,
563 UCOL_BYTE_FIRST_NON_LATIN_PRIMARY = 0x4D,
564 UCOL_BYTE_UNSHIFTED_MAX = 0xFF
565 };
566
567 #if 0
568 #define UCOL_RESET_TOP_VALUE 0x9F000303
569 #define UCOL_FIRST_PRIMARY_IGNORABLE 0x00008705
570 #define UCOL_LAST_PRIMARY_IGNORABLE 0x0000DD05
571 #define UCOL_LAST_PRIMARY_IGNORABLE_CONT 0x000051C0
572 #define UCOL_FIRST_SECONDARY_IGNORABLE 0x00000000
573 #define UCOL_LAST_SECONDARY_IGNORABLE 0x00000500
574 #define UCOL_FIRST_TERTIARY_IGNORABLE 0x00000000
575 #define UCOL_LAST_TERTIARY_IGNORABLE 0x00000000
576 #define UCOL_FIRST_VARIABLE 0x05070505
577 #define UCOL_LAST_VARIABLE 0x179B0505
578 #define UCOL_FIRST_NON_VARIABLE 0x1A200505
579 #define UCOL_LAST_NON_VARIABLE 0x7B41058F
580
581 #define UCOL_NEXT_TOP_VALUE 0xE8960303
582 #define UCOL_NEXT_FIRST_PRIMARY_IGNORABLE 0x00008905
583 #define UCOL_NEXT_LAST_PRIMARY_IGNORABLE 0x03000303
584 #define UCOL_NEXT_FIRST_SECONDARY_IGNORABLE 0x00008705
585 #define UCOL_NEXT_LAST_SECONDARY_IGNORABLE 0x00000500
586 #define UCOL_NEXT_FIRST_TERTIARY_IGNORABLE 0x00000000
587 #define UCOL_NEXT_LAST_TERTIARY_IGNORABLE 0x00000000
588 #define UCOL_NEXT_FIRST_VARIABLE 0x05090505
589 #define UCOL_NEXT_LAST_VARIABLE 0x1A200505
590
591 #define PRIMARY_IMPLICIT_MIN 0xE8000000
592 #define PRIMARY_IMPLICIT_MAX 0xF0000000
593 #endif
594
595 /* These constants can be changed - sortkey size is affected by them */
596 #define UCOL_PROPORTION2 0.5
597 #define UCOL_PROPORTION3 0.667
598
599 /* These values come from the UCA */
600 #define UCOL_COMMON_BOT2 UCOL_BYTE_COMMON
601 #define UCOL_COMMON_TOP2 0x86u
602 #define UCOL_TOTAL2 (UCOL_COMMON_TOP2-UCOL_COMMON_BOT2-1)
603
604 #define UCOL_FLAG_BIT_MASK_CASE_SW_OFF 0x80
605 #define UCOL_FLAG_BIT_MASK_CASE_SW_ON 0x40
606 #define UCOL_COMMON_TOP3_CASE_SW_OFF 0x85
607 #define UCOL_COMMON_TOP3_CASE_SW_LOWER 0x45
608 #define UCOL_COMMON_TOP3_CASE_SW_UPPER 0xC5
609
610 /* These values come from the UCA */
611 #define UCOL_COMMON_BOT3 0x05
612
613 #define UCOL_COMMON_BOTTOM3_CASE_SW_UPPER 0x86;
614 #define UCOL_COMMON_BOTTOM3_CASE_SW_LOWER UCOL_COMMON_BOT3;
615
616 #define UCOL_TOP_COUNT2 (UCOL_PROPORTION2*UCOL_TOTAL2)
617 #define UCOL_BOT_COUNT2 (UCOL_TOTAL2-UCOL_TOP_COUNT2)
618
619
620 #define UCOL_COMMON2 UCOL_COMMON_BOT2
621 #define UCOL_COMMON3_UPPERFIRST 0xC5
622 #define UCOL_COMMON3_NORMAL UCOL_COMMON_BOT3
623
624 #define UCOL_COMMON4 0xFF
625
626 /* constants for case level/case first handling */
627 /* used to instantiate UCollators fields in ucol_updateInternalState */
628 #define UCOL_CASE_SWITCH 0xC0
629 #define UCOL_NO_CASE_SWITCH 0x00
630
631 #define UCOL_REMOVE_CASE 0x3F
632 #define UCOL_KEEP_CASE 0xFF
633
634 #define UCOL_CASE_BIT_MASK 0xC0
635
636 #define UCOL_TERT_CASE_MASK 0xFF
637
638 #define UCOL_ENDOFLATINONERANGE 0xFF
639 #define UCOL_LATINONETABLELEN (UCOL_ENDOFLATINONERANGE+50)
640 #define UCOL_BAIL_OUT_CE 0xFF000000
641
642
643 typedef enum {
644 NOT_FOUND_TAG = 0,
645 EXPANSION_TAG = 1, /* This code point results in an expansion */
646 CONTRACTION_TAG = 2, /* Start of a contraction */
647 THAI_TAG = 3, /* Thai character - do the reordering */
648 CHARSET_TAG = 4, /* Charset processing, not yet implemented */
649 SURROGATE_TAG = 5, /* Lead surrogate that is tailored and doesn't start a contraction */
650 HANGUL_SYLLABLE_TAG = 6, /* AC00-D7AF*/
651 LEAD_SURROGATE_TAG = 7, /* D800-DBFF*/
652 TRAIL_SURROGATE_TAG = 8, /* DC00-DFFF*/
653 CJK_IMPLICIT_TAG = 9, /* 0x3400-0x4DB5, 0x4E00-0x9FA5, 0xF900-0xFA2D*/
654 IMPLICIT_TAG = 10,
655 SPEC_PROC_TAG = 11,
656 /* ICU 2.1 */
657 LONG_PRIMARY_TAG = 12, /* This is a three byte primary with starting secondaries and tertiaries */
658 /* It fits in a single 32 bit CE and is used instead of expansion to save */
659 /* space without affecting the performance (hopefully) */
660
661 DIGIT_TAG = 13, /* COllate Digits As Numbers (CODAN) implementation */
662
663 CE_TAGS_COUNT
664 } UColCETags;
665
666 /*
667 *****************************************************************************************
668 * set to zero
669 * NON_CHARACTER FDD0 - FDEF, FFFE, FFFF, 1FFFE, 1FFFF, 2FFFE, 2FFFF,...e.g. **FFFE, **FFFF
670 ******************************************************************************************
671 */
672
673 typedef struct {
674 uint32_t variableTopValue;
675 /*UColAttributeValue*/ int32_t frenchCollation;
676 /*UColAttributeValue*/ int32_t alternateHandling; /* attribute for handling variable elements*/
677 /*UColAttributeValue*/ int32_t caseFirst; /* who goes first, lower case or uppercase */
678 /*UColAttributeValue*/ int32_t caseLevel; /* do we have an extra case level */
679 /*UColAttributeValue*/ int32_t normalizationMode; /* attribute for normalization */
680 /*UColAttributeValue*/ int32_t strength; /* attribute for strength */
681 /*UColAttributeValue*/ int32_t hiraganaQ; /* attribute for special Hiragana */
682 /*UColAttributeValue*/ int32_t numericCollation; /* attribute for numeric collation */
683 uint32_t reserved[15]; /* for future use */
684 } UColOptionSet;
685
686 typedef struct {
687 uint32_t UCA_FIRST_TERTIARY_IGNORABLE[2]; /*0x00000000*/
688 uint32_t UCA_LAST_TERTIARY_IGNORABLE[2]; /*0x00000000*/
689 uint32_t UCA_FIRST_PRIMARY_IGNORABLE[2]; /*0x00008705*/
690 uint32_t UCA_FIRST_SECONDARY_IGNORABLE[2]; /*0x00000000*/
691 uint32_t UCA_LAST_SECONDARY_IGNORABLE[2]; /*0x00000500*/
692 uint32_t UCA_LAST_PRIMARY_IGNORABLE[2]; /*0x0000DD05*/
693 uint32_t UCA_FIRST_VARIABLE[2]; /*0x05070505*/
694 uint32_t UCA_LAST_VARIABLE[2]; /*0x13CF0505*/
695 uint32_t UCA_FIRST_NON_VARIABLE[2]; /*0x16200505*/
696 uint32_t UCA_LAST_NON_VARIABLE[2]; /*0x767C0505*/
697 uint32_t UCA_RESET_TOP_VALUE[2]; /*0x9F000303*/
698 uint32_t UCA_FIRST_IMPLICIT[2];
699 uint32_t UCA_LAST_IMPLICIT[2];
700 uint32_t UCA_FIRST_TRAILING[2];
701 uint32_t UCA_LAST_TRAILING[2];
702
703 #if 0
704 uint32_t UCA_NEXT_TOP_VALUE[2]; /*0xE8960303*/
705 uint32_t UCA_NEXT_FIRST_PRIMARY_IGNORABLE; /*0x00008905*/
706 uint32_t UCA_NEXT_LAST_PRIMARY_IGNORABLE; /*0x03000303*/
707 uint32_t UCA_NEXT_FIRST_SECONDARY_IGNORABLE; /*0x00008705*/
708 uint32_t UCA_NEXT_LAST_SECONDARY_IGNORABLE; /*0x00000500*/
709 uint32_t UCA_NEXT_FIRST_TERTIARY_IGNORABLE; /*0x00000000*/
710 uint32_t UCA_NEXT_LAST_TERTIARY_IGNORABLE; /*0x00000000*/
711 uint32_t UCA_NEXT_FIRST_VARIABLE; /*0x05090505*/
712 uint32_t UCA_NEXT_LAST_VARIABLE; /*0x16200505*/
713 #endif
714
715 uint32_t UCA_PRIMARY_TOP_MIN;
716 uint32_t UCA_PRIMARY_IMPLICIT_MIN; /*0xE8000000*/
717 uint32_t UCA_PRIMARY_IMPLICIT_MAX; /*0xF0000000*/
718 uint32_t UCA_PRIMARY_TRAILING_MIN; /*0xE8000000*/
719 uint32_t UCA_PRIMARY_TRAILING_MAX; /*0xF0000000*/
720 uint32_t UCA_PRIMARY_SPECIAL_MIN; /*0xE8000000*/
721 uint32_t UCA_PRIMARY_SPECIAL_MAX; /*0xF0000000*/
722 } UCAConstants;
723
724 typedef struct {
725 int32_t size;
726 /* all the offsets are in bytes */
727 /* to get the address add to the header address and cast properly */
728 uint32_t options; /* these are the default options for the collator */
729 uint32_t UCAConsts; /* structure which holds values for indirect positioning and implicit ranges */
730 uint32_t contractionUCACombos; /* this one is needed only for UCA, to copy the appropriate contractions */
731 uint32_t magic; /* magic number - lets us know whether reserved data is reset or junked */
732 uint32_t mappingPosition; /* const uint8_t *mappingPosition; */
733 uint32_t expansion; /* uint32_t *expansion; */
734 uint32_t contractionIndex; /* UChar *contractionIndex; */
735 uint32_t contractionCEs; /* uint32_t *contractionCEs; */
736 uint32_t contractionSize; /* needed for various closures */
737 /*int32_t latinOneMapping;*/ /* this is now handled in the trie itself *//* fast track to latin1 chars */
738
739 uint32_t endExpansionCE; /* array of last collation element in
740 expansion */
741 uint32_t expansionCESize; /* array of maximum expansion size
742 corresponding to the expansion
743 collation elements with last element
744 in endExpansionCE*/
745 int32_t endExpansionCECount; /* size of endExpansionCE */
746 uint32_t unsafeCP; /* hash table of unsafe code points */
747 uint32_t contrEndCP; /* hash table of final code points */
748 /* in contractions. */
749
750 int32_t contractionUCACombosSize; /* number of UCA contraction items. */
751 /*Length is contractionUCACombosSize*contractionUCACombosWidth*sizeof(UChar) */
752 UBool jamoSpecial; /* is jamoSpecial */
753 UBool isBigEndian; /* is this data big endian? from the UDataInfo header*/
754 uint8_t charSetFamily; /* what is the charset family of this data from the UDataInfo header*/
755 uint8_t contractionUCACombosWidth; /* width of UCA combos field */
756 UVersionInfo version;
757 UVersionInfo UCAVersion; /* version of the UCA, read from file */
758 UVersionInfo UCDVersion; /* UCD version, obtained by u_getUnicodeVersion */
759 UVersionInfo formatVersion; /* format version from the UDataInfo header */
760 uint8_t reserved[84]; /* for future use */
761 } UCATableHeader;
762
763 #define U_UNKNOWN_STATE 0
764 #define U_COLLATOR_STATE 0x01
765 #define U_STATE_LIMIT 0x02
766
767 /* This is the first structure in a state */
768 /* it should be machine independent */
769 typedef struct {
770 /* this structure is supposed to be readable on all the platforms.*/
771 /* first 2 fields hold the size of the structure in a platform independent way */
772 uint8_t sizeLo;
773 uint8_t sizeHi;
774 /* identifying the writing platform */
775 uint8_t isBigEndian;
776 /* see U_CHARSET_FAMILY values in utypes.h */
777 uint8_t charsetFamily;
778 /* version of ICU this state structure comes from */
779 uint8_t icuVersion[4];
780 /* What is the data following this state */
781 uint8_t type;
782 /* more stuff to come, keep it on 16 byte boundary */
783 uint8_t reserved[7];
784 } UStateStruct;
785
786 /* This structure follows UStatusStruct */
787 /* and contains data specific for the collators */
788 /* Endianess needs to be decided before accessing this structure */
789 /* However, it's size IS endianess independent */
790 typedef struct {
791 /* size of this structure */
792 uint8_t sizeLo;
793 uint8_t sizeHi;
794 /* This state is followed by the frozen tailoring */
795 uint8_t containsTailoring;
796 /* This state is followed by the frozen UCA */
797 uint8_t containsUCA;
798 /* Version info - the same one */
799 uint8_t versionInfo[4];
800
801 /* for charset CEs */
802 uint8_t charsetName[32];
803 /* this is the resolved locale name*/
804 uint8_t locale[32];
805
806 /* Attributes. Open ended */
807 /* all the following will be moved to uint32_t because of portability */
808 /* variable top value */
809 uint32_t variableTopValue;
810 /* attribute for handling variable elements*/
811 uint32_t /*UColAttributeValue*/ alternateHandling;
812 /* how to handle secondary weights */
813 uint32_t /*UColAttributeValue*/ frenchCollation;
814 /* who goes first, lower case or uppercase */
815 uint32_t /*UColAttributeValue*/ caseFirst;
816 /* do we have an extra case level */
817 uint32_t /*UColAttributeValue*/ caseLevel;
818 /* attribute for normalization */
819 uint32_t /*UColAttributeValue*/ normalizationMode;
820 /* attribute for strength */
821 uint32_t /*UColAttributeValue*/ strength;
822 /* to be immediately 16 byte aligned */
823 uint8_t reserved[12];
824 } UColStateStruct;
825
826 #define UCOL_INV_SIZEMASK 0xFFF00000
827 #define UCOL_INV_OFFSETMASK 0x000FFFFF
828 #define UCOL_INV_SHIFTVALUE 20
829
830 U_CDECL_BEGIN
831
832 typedef struct {
833 uint32_t byteSize;
834 uint32_t tableSize;
835 uint32_t contsSize;
836 uint32_t table;
837 uint32_t conts;
838 UVersionInfo UCAVersion; /* version of the UCA, read from file */
839 uint8_t padding[8];
840 } InverseUCATableHeader;
841
842 typedef int32_t U_CALLCONV
843 SortKeyGenerator(const UCollator *coll,
844 const UChar *source,
845 int32_t sourceLength,
846 uint8_t **result,
847 uint32_t resultLength,
848 UBool allocatePrimary,
849 UErrorCode *status);
850
851 typedef void U_CALLCONV
852 ResourceCleaner(UCollator *coll);
853
854
855 struct UCollator {
856 UColOptionSet *options;
857 SortKeyGenerator *sortKeyGen;
858 uint32_t *latinOneCEs;
859 char* validLocale;
860 char* requestedLocale;
861 const UChar *rules;
862 const UCollator *UCA;
863 ResourceCleaner *resCleaner;
864 UResourceBundle *rb;
865 UResourceBundle *elements;
866 const UCATableHeader *image;
867 UTrie mapping;
868 const uint32_t *latinOneMapping;
869 const uint32_t *expansion;
870 const UChar *contractionIndex;
871 const uint32_t *contractionCEs;
872 /*const uint8_t *scriptOrder;*/
873
874 const uint32_t *endExpansionCE; /* array of last ces in an expansion ce.
875 corresponds to expansionCESize */
876 const uint32_t *lastEndExpansionCE;/* pointer to the last element in endExpansionCE */
877 const uint8_t *expansionCESize; /* array of the maximum size of a
878 expansion ce with the last ce
879 corresponding to endExpansionCE,
880 terminated with a null */
881 const uint8_t *unsafeCP; /* unsafe code points hashtable */
882 const uint8_t *contrEndCP; /* Contraction ending chars hash table */
883 UChar minUnsafeCP; /* Smallest unsafe Code Point. */
884 UChar minContrEndCP; /* Smallest code point at end of a contraction */
885
886 int32_t rulesLength;
887 int32_t latinOneTableLen;
888
889 uint32_t variableTopValue;
890 UColAttributeValue frenchCollation;
891 UColAttributeValue alternateHandling; /* attribute for handling variable elements*/
892 UColAttributeValue caseFirst; /* who goes first, lower case or uppercase */
893 UColAttributeValue caseLevel; /* do we have an extra case level */
894 UColAttributeValue normalizationMode; /* attribute for normalization */
895 UColAttributeValue strength; /* attribute for strength */
896 UColAttributeValue hiraganaQ; /* attribute for Hiragana */
897 UColAttributeValue numericCollation;
898 UBool variableTopValueisDefault;
899 UBool frenchCollationisDefault;
900 UBool alternateHandlingisDefault; /* attribute for handling variable elements*/
901 UBool caseFirstisDefault; /* who goes first, lower case or uppercase */
902 UBool caseLevelisDefault; /* do we have an extra case level */
903 UBool normalizationModeisDefault; /* attribute for normalization */
904 UBool strengthisDefault; /* attribute for strength */
905 UBool hiraganaQisDefault; /* attribute for Hiragana */
906 UBool numericCollationisDefault;
907 UBool hasRealData; /* some collators have only options, like French, no rules */
908 /* to speed up things, we use the UCA image, but we don't want it */
909 /* to run around */
910
911 UBool freeOnClose;
912 UBool freeOptionsOnClose;
913 UBool freeRulesOnClose;
914 UBool freeImageOnClose;
915
916 UBool latinOneUse;
917 UBool latinOneRegenTable;
918 UBool latinOneFailed;
919
920 int32_t tertiaryAddition; /* when switching case, we need to add or subtract different values */
921 uint8_t caseSwitch;
922 uint8_t tertiaryCommon;
923 uint8_t tertiaryMask;
924 uint8_t tertiaryTop; /* Upper range when compressing */
925 uint8_t tertiaryBottom; /* Upper range when compressing */
926 uint8_t tertiaryTopCount;
927 uint8_t tertiaryBottomCount;
928
929 UVersionInfo dataVersion; /* Data info of UCA table */
930 };
931
932 U_CDECL_END
933
934 /* various internal functions */
935
936 /* do not close UCA returned by ucol_initUCA! */
937 U_CFUNC
938 UCollator* ucol_initUCA(UErrorCode *status);
939
940 U_CFUNC
941 UCollator* ucol_initCollator(const UCATableHeader *image, UCollator *fillIn, const UCollator *UCA, UErrorCode *status);
942
943 U_CFUNC
944 void ucol_setOptionsFromHeader(UCollator* result, UColOptionSet * opts, UErrorCode *status);
945
946 U_CFUNC
947 UCollator* ucol_open_internal(const char* loc, UErrorCode* status);
948
949 #if 0
950 U_CFUNC
951 void ucol_putOptionsToHeader(UCollator* result, UColOptionSet * opts, UErrorCode *status);
952 #endif
953
954 U_CFUNC
955 void ucol_updateInternalState(UCollator *coll, UErrorCode *status);
956
957 U_CFUNC uint32_t U_EXPORT2 ucol_getFirstCE(const UCollator *coll, UChar u, UErrorCode *status);
958 U_CAPI UBool U_EXPORT2 ucol_isTailored(const UCollator *coll, const UChar u, UErrorCode *status);
959
960 U_CAPI const InverseUCATableHeader* U_EXPORT2 ucol_initInverseUCA(UErrorCode *status);
961
962 U_CAPI void U_EXPORT2
963 uprv_uca_initImplicitConstants(int32_t minPrimary, int32_t maxPrimary, UErrorCode *status);
964
965 U_CAPI uint32_t U_EXPORT2
966 uprv_uca_getImplicitFromRaw(UChar32 cp);
967
968 /*U_CFUNC uint32_t U_EXPORT2
969 uprv_uca_getImplicitPrimary(UChar32 cp);*/
970
971 U_CAPI UChar32 U_EXPORT2
972 uprv_uca_getRawFromImplicit(uint32_t implicit);
973
974 U_CAPI UChar32 U_EXPORT2
975 uprv_uca_getRawFromCodePoint(UChar32 i);
976
977 U_CAPI UChar32 U_EXPORT2
978 uprv_uca_getCodePointFromRaw(UChar32 i);
979
980
981
982 #ifdef XP_CPLUSPLUS
983 /*
984 * Test whether a character is potentially "unsafe" for use as a collation
985 * starting point. Unsafe chars are those with combining class != 0 plus
986 * those that are the 2nd thru nth character in a contraction sequence.
987 *
988 * Function is in header file because it's used in both collation and string search,
989 * and needs to be inline for performance.
990 */
ucol_unsafeCP(UChar c,const UCollator * coll)991 static inline UBool ucol_unsafeCP(UChar c, const UCollator *coll) {
992 int32_t hash;
993 uint8_t htbyte;
994
995 if (c < coll->minUnsafeCP) {
996 return FALSE;
997 }
998
999 hash = c;
1000 if (hash >= UCOL_UNSAFECP_TABLE_SIZE*8) {
1001 if(UTF_IS_LEAD(c) || UTF_IS_TRAIL(c)) {
1002 /* Trail surrogate */
1003 /* These are always considered unsafe. */
1004 return TRUE;
1005 }
1006 hash = (hash & UCOL_UNSAFECP_TABLE_MASK) + 256;
1007 }
1008 htbyte = coll->unsafeCP[hash>>3];
1009 return (((htbyte >> (hash & 7)) & 1) == 1);
1010 }
1011 #endif /* XP_CPLUSPLUS */
1012
1013
1014 #endif /* #if !UCONFIG_NO_COLLATION */
1015
1016 #endif
1017