1 /*
2 *******************************************************************************
3 *
4 * Copyright (C) 1998-2011, International Business Machines
5 * Corporation and others. All Rights Reserved.
6 *
7 *******************************************************************************
8 *
9 * Private implementation header for C collation
10 * file name: ucol_imp.h
11 * encoding: US-ASCII
12 * tab size: 8 (not used)
13 * indentation:4
14 *
15 * created on: 2000dec11
16 * created by: Vladimir Weinstein
17 *
18 * Modification history
19 * Date Name Comments
20 * 02/16/2001 synwee Added UCOL_GETPREVCE for the use in ucoleitr
21 * 02/27/2001 synwee Added getMaxExpansion data structure in UCollator
22 * 03/02/2001 synwee Added UCOL_IMPLICIT_CE
23 * 03/12/2001 synwee Added pointer start to collIterate.
24 */
25
26 #ifndef UCOL_IMP_H
27 #define UCOL_IMP_H
28
29 #include "unicode/utypes.h"
30
31 #define UCA_DATA_TYPE "icu"
32 #define UCA_DATA_NAME "ucadata"
33 #define INVC_DATA_TYPE "icu"
34 #define INVC_DATA_NAME "invuca"
35
36 /**
37 * Convenience string denoting the Collation data tree
38 * @internal ICU 3.0
39 */
40 #define U_ICUDATA_COLL U_ICUDATA_NAME U_TREE_SEPARATOR_STRING "coll"
41
42 #if !UCONFIG_NO_COLLATION
43
44 #ifdef XP_CPLUSPLUS
45 #include "unicode/normalizer2.h"
46 #include "unicode/unistr.h"
47 #endif
48 #include "unicode/ucol.h"
49 #include "utrie.h"
50 #include "cmemory.h"
51
52 /* This is the internal header file which contains important declarations for
53 * the collation framework.
54 * Ready to use collators are stored as binary images. Both UCA and tailorings
55 * share the same binary format. Individual files (currently only UCA) have a
56 * udata header in front of the image and should be opened using udata_open.
57 * Tailoring images are currently stored inside resource bundles and are intialized
58 * through ucol_open API.
59 *
60 * The following describes the formats for collation binaries
61 * (UCA & tailorings) and for the inverse UCA table.
62 * Substructures are described in the collation design document at
63 * http://source.icu-project.org/repos/icu/icuhtml/trunk/design/collation/ICU_collation_design.htm
64 *
65 * -------------------------------------------------------------
66 *
67 * Here is the format of binary collation image.
68 *
69 * Physical order of structures:
70 * - header (UCATableHeader)
71 * - options (UColOptionSet)
72 * - expansions (CE[])
73 * - contractions (UChar[contractionSize] + CE[contractionSize])
74 * - serialized UTrie with mappings of code points to CEs
75 * - max expansion tables (CE[endExpansionCECount] + uint8_t[endExpansionCECount])
76 * - two bit sets for backward processing in strcoll (identical prefixes)
77 * and for backward CE iteration (each set is uint8_t[UCOL_UNSAFECP_TABLE_SIZE])
78 * - UCA constants (UCAConstants)
79 * - UCA contractions (UChar[contractionUCACombosSize][contractionUCACombosWidth])
80 *
81 * UCATableHeader fields:
82 *
83 * int32_t size; - image size in bytes
84 *
85 * Offsets to interesting data. All offsets are in bytes.
86 * to get the address add to the header address and cast properly.
87 * Some offsets are zero if the corresponding structures are empty.
88 *
89 * Tailoring binaries that only set options and contain no mappings etc.
90 * will have all offsets 0 except for the options and expansion offsets,
91 * which give the position and length of the options array.
92 *
93 * uint32_t options; - offset to default collator options (UColOptionSet *),
94 * a set of 32-bit values. See declaration of UColOptionSet for more details
95 *
96 * uint32_t UCAConsts; - only used (!=0) in UCA image - structure which holds values for indirect positioning and implicit ranges
97 * See declaration of UCAConstants structure. This is a set of unsigned 32-bit values used to store
98 * important constant values that are defined in the UCA and used for building and runtime.
99 *
100 * uint32_t contractionUCACombos; - only used (!=0) in UCA image - list of UCA contractions. This is a zero terminated array of UChar[contractionUCACombosWidth],
101 * containing contractions from the UCA. These are needed in the build process to copy UCA contractions
102 * in case the base contraction symbol is tailored.
103 *
104 * uint32_t magic; - must contain UCOL_HEADER_MAGIC (formatVersion 2.3)
105 *
106 * uint32_t mappingPosition; - offset to UTrie (const uint8_t *mappingPosition). This is a serialized UTrie and should be treated as such.
107 * Used as a primary lookup table for collation elements.
108 *
109 * uint32_t expansion; - offset to expansion table (uint32_t *expansion). This is an array of expansion CEs. Never 0.
110 *
111 * uint32_t contractionIndex; - offset to contraction table (UChar *contractionIndex). Used to look up contraction sequences. Contents
112 * are aligned with the contents of contractionCEs table. 0 if no contractions.
113 *
114 * uint32_t contractionCEs; - offset to resulting contraction CEs (uint32_t *contractionCEs). When a contraction is resolved in the
115 * in the contractionIndex table, the resulting index is used to look up corresponding CE in this table.
116 * 0 if no contractions.
117 * uint32_t contractionSize; - size of contraction table in elements (both Index and CEs).
118 *
119 * Tables described below are used for Boyer-Moore searching algorithm - they define the size of longest expansion
120 * and last CEs in expansions.
121 * uint32_t endExpansionCE; - offset to array of last collation element in expansion (uint32_t *).
122 * Never 0.
123 * uint32_t expansionCESize; - array of maximum expansion sizes (uint8_t *)
124 * int32_t endExpansionCECount; - size of endExpansionCE. See UCOL_GETMAXEXPANSION
125 * for the usage model
126 *
127 * These two offsets point to byte tables that are used in the backup heuristics.
128 * uint32_t unsafeCP; - hash table of unsafe code points (uint8_t *). See ucol_unsafeCP function.
129 * uint32_t contrEndCP; - hash table of final code points in contractions (uint8_t *). See ucol_contractionEndCP.
130 *
131 * int32_t contractionUCACombosSize; - number of UChar[contractionUCACombosWidth] in contractionUCACombos
132 * (formatVersion 2.3)
133 * UBool jamoSpecial; - Jamo special indicator (uint8_t). If TRUE, Jamos are special, so we cannot use simple Hangul decomposition.
134 * UBool isBigEndian; - endianness of this collation binary (formatVersion 2.3)
135 * uint8_t charSetFamily; - charset family of this collation binary (formatVersion 2.3)
136 * uint8_t contractionUCACombosWidth; - number of UChars per UCA contraction in contractionUCACombos (formatVersion 2.3)
137 *
138 * Various version fields
139 * UVersionInfo version; - version 4 uint8_t
140 * UVersionInfo UCAVersion; - version 4 uint8_t
141 * UVersionInfo UCDVersion; - version 4 uint8_t
142 * UVersionInfo formatVersion; - version of the format of the collation binary
143 * same formatVersion as in ucadata.icu's UDataInfo header
144 * (formatVersion 2.3)
145 *
146 * uint32_t offset to the reordering code to lead CE byte remapping table
147 * uint32_t offset to the lead CE byte to reordering code mapping table
148 *
149 * uint8_t reserved[76]; - currently unused
150 *
151 * -------------------------------------------------------------
152 *
153 * Inverse UCA is used for constructing collators from rules. It is always an individual file
154 * and always has a UDataInfo header.
155 * here is the structure:
156 *
157 * uint32_t byteSize; - size of inverse UCA image in bytes
158 * uint32_t tableSize; - length of inverse table (number of uint32_t[3] rows)
159 * uint32_t contsSize; - size of continuation table (number of UChars in table)
160 *
161 * uint32_t table; - offset to inverse table (uint32_t *)
162 * Inverse table contains of rows of 3 uint32_t values. First two values are CE and a possible continuation
163 * the third value is either a code unit (if there is only one code unit for element) or an index to continuation
164 * (number of code units combined with an index).
165 * table. If more than one codepoint have the same CE, continuation table contains code units separated by FFFF and final
166 * code unit sequence for a CE is terminated by FFFE.
167 * uint32_t conts; - offset to continuation table (uint16_t *). Contains code units that transform to a same CE.
168 *
169 * UVersionInfo UCAVersion; - version of the UCA, read from file 4 uint8_t
170 * uint8_t padding[8]; - padding 8 uint8_t
171 * Header is followed by the table and continuation table.
172 */
173
174 /* let us know whether reserved fields are reset to zero or junked */
175 #define UCOL_HEADER_MAGIC 0x20030618
176
177 /* UDataInfo for UCA mapping table */
178 /* dataFormat="UCol" */
179 #define UCA_DATA_FORMAT_0 ((uint8_t)0x55)
180 #define UCA_DATA_FORMAT_1 ((uint8_t)0x43)
181 #define UCA_DATA_FORMAT_2 ((uint8_t)0x6f)
182 #define UCA_DATA_FORMAT_3 ((uint8_t)0x6c)
183
184 #define UCA_FORMAT_VERSION_0 ((uint8_t)3)
185 #define UCA_FORMAT_VERSION_1 0
186 #define UCA_FORMAT_VERSION_2 ((uint8_t)0)
187 #define UCA_FORMAT_VERSION_3 ((uint8_t)0)
188
189 /* UDataInfo for inverse UCA table */
190 /* dataFormat="InvC" */
191 #define INVUCA_DATA_FORMAT_0 ((uint8_t)0x49)
192 #define INVUCA_DATA_FORMAT_1 ((uint8_t)0x6E)
193 #define INVUCA_DATA_FORMAT_2 ((uint8_t)0x76)
194 #define INVUCA_DATA_FORMAT_3 ((uint8_t)0x43)
195
196 #define INVUCA_FORMAT_VERSION_0 ((uint8_t)2)
197 #define INVUCA_FORMAT_VERSION_1 ((uint8_t)1)
198 #define INVUCA_FORMAT_VERSION_2 ((uint8_t)0)
199 #define INVUCA_FORMAT_VERSION_3 ((uint8_t)0)
200
201 /* This is the size of the stack allocated buffer for sortkey generation and similar operations */
202 /* if it is too small, heap allocation will occur.*/
203 /* you can change this value if you need memory - it will affect the performance, though, since we're going to malloc */
204 #define UCOL_MAX_BUFFER 128
205 #define UCOL_PRIMARY_MAX_BUFFER 8*UCOL_MAX_BUFFER
206 #define UCOL_SECONDARY_MAX_BUFFER UCOL_MAX_BUFFER
207 #define UCOL_TERTIARY_MAX_BUFFER UCOL_MAX_BUFFER
208 /*
209 #define UCOL_CASE_MAX_BUFFER UCOL_MAX_BUFFER/4
210
211 UCOL_CASE_MAX_BUFFER as previously defined above was too small. A single collation element can
212 generate two caseShift values, and UCOL_CASE_SHIFT_START (=7) caseShift values are compressed into
213 one byte. UCOL_MAX_BUFFER should effectively be multipled by 2/UCOL_CASE_SHIFT_START (2/7), not 1/4.
214 Perhaps UCOL_CASE_SHIFT_START used to be 8; then this would have been correct. We should dynamically
215 define UCOL_CASE_MAX_BUFFER in terms of both UCOL_MAX_BUFFER and UCOL_CASE_SHIFT_START. Since
216 UCOL_CASE_SHIFT_START is defined lower down, we move the real definition of UCOL_CASE_MAX_BUFFER
217 after it, further down.
218 */
219 #define UCOL_QUAD_MAX_BUFFER 2*UCOL_MAX_BUFFER
220
221 #define UCOL_NORMALIZATION_GROWTH 2
222 #define UCOL_NORMALIZATION_MAX_BUFFER UCOL_MAX_BUFFER*UCOL_NORMALIZATION_GROWTH
223
224 /* This writable buffer is used if we encounter Thai and need to reorder the string on the fly */
225 /* Sometimes we already have a writable buffer (like in case of normalized strings). */
226 /*
227 you can change this value to any value >= 4 if you need memory -
228 it will affect the performance, though, since we're going to malloc.
229 Note 3 is the minimum value for Thai collation and 4 is the
230 minimum number for special Jamo
231 */
232 #define UCOL_WRITABLE_BUFFER_SIZE 256
233
234 /* This is the size of the buffer for expansion CE's */
235 /* In reality we should not have to deal with expm sequences longer then 16 */
236 /* you can change this value if you need memory */
237 /* WARNING THIS BUFFER DOES HAVE MALLOC FALLBACK. If you make it too small, you'll get into performance trouble */
238 /* Reasonable small value is around 10, if you don't do Arabic or other funky collations that have long expansion sequence */
239 /* This is the longest expansion sequence we can handle without bombing out */
240 #define UCOL_EXPAND_CE_BUFFER_SIZE 64
241
242 /* This is the size to increase the buffer for expansion CE's */
243 #define UCOL_EXPAND_CE_BUFFER_EXTEND_SIZE 64
244
245
246 /* Unsafe UChar hash table table size. */
247 /* size is 32 bytes for 1 bit for each latin 1 char + some power of two for */
248 /* hashing the rest of the chars. Size in bytes */
249 #define UCOL_UNSAFECP_TABLE_SIZE 1056
250 /* mask value down to "some power of two"-1 */
251 /* number of bits, not num of bytes. */
252 #define UCOL_UNSAFECP_TABLE_MASK 0x1fff
253
254
255 /* flags bits for collIterate.flags */
256 /* */
257 /* NORM - set for incremental normalize of source string */
258 #define UCOL_ITER_NORM 1
259
260 #define UCOL_ITER_HASLEN 2
261
262 /* UCOL_ITER_INNORMBUF - set if the "pos" is in */
263 /* the writable side buffer, handling */
264 /* incrementally normalized characters. */
265 #define UCOL_ITER_INNORMBUF 4
266
267 /* UCOL_ITER_ALLOCATED - set if this iterator has */
268 /* malloced storage to expand a buffer. */
269 #define UCOL_ITER_ALLOCATED 8
270 /* UCOL_HIRAGANA_Q - note if the codepoint was hiragana */
271 #define UCOL_HIRAGANA_Q 16
272 /* UCOL_WAS_HIRAGANA - set to TRUE if there was a Hiragana */
273 /* otherwise set to false */
274 #define UCOL_WAS_HIRAGANA 32
275 /* UCOL_USE_ITERATOR - set this if collIterate uses a */
276 /* character iterator instead of simply accessing string */
277 /* by index */
278 #define UCOL_USE_ITERATOR 64
279
280 #define UCOL_FORCE_HAN_IMPLICIT 128
281
282 #define NFC_ZERO_CC_BLOCK_LIMIT_ 0x300
283
284 #ifdef XP_CPLUSPLUS
285
286 U_NAMESPACE_BEGIN
287
288 typedef struct collIterate : public UMemory {
289 const UChar *string; /* Original string */
290 /* UChar *start; Pointer to the start of the source string. Either points to string
291 or to writableBuffer */
292 const UChar *endp; /* string end ptr. Is undefined for null terminated strings */
293 const UChar *pos; /* This is position in the string. Can be to original or writable buf */
294
295 uint32_t *toReturn; /* This is the CE from CEs buffer that should be returned */
296 uint32_t *CEpos; /* This is the position to which we have stored processed CEs */
297
298 int32_t *offsetReturn; /* This is the offset to return, if non-NULL */
299 int32_t *offsetStore; /* This is the pointer for storing offsets */
300 int32_t offsetRepeatCount; /* Repeat stored offset if non-zero */
301 int32_t offsetRepeatValue; /* offset value to repeat */
302
303 UnicodeString writableBuffer;
304 const UChar *fcdPosition; /* Position in the original string to continue FCD check from. */
305 const UCollator *coll;
306 const Normalizer2 *nfd;
307 uint8_t flags;
308 uint8_t origFlags;
309 uint32_t *extendCEs; /* This is use if CEs is not big enough */
310 int32_t extendCEsSize; /* Holds the size of the dynamic CEs buffer */
311 uint32_t CEs[UCOL_EXPAND_CE_BUFFER_SIZE]; /* This is where we store CEs */
312
313 int32_t *offsetBuffer; /* A dynamic buffer to hold offsets */
314 int32_t offsetBufferSize; /* The size of the offset buffer */
315
316 UCharIterator *iterator;
317 /*int32_t iteratorIndex;*/
318
319 // The offsetBuffer should probably be a UVector32, but helper functions
320 // are an improvement over duplicated code.
321 void appendOffset(int32_t offset, UErrorCode &errorCode);
322 } collIterate;
323
324 U_NAMESPACE_END
325
326 #else
327
328 typedef struct collIterate collIterate;
329
330 #endif
331
332 #define paddedsize(something) ((something)+((((something)%4)!=0)?(4-(something)%4):0))
333 #define headersize (paddedsize(sizeof(UCATableHeader))+paddedsize(sizeof(UColOptionSet)))
334
335 /*
336 struct used internally in getSpecial*CE.
337 data similar to collIterate.
338 */
339 struct collIterateState {
340 const UChar *pos; /* This is position in the string. Can be to original or writable buf */
341 const UChar *returnPos;
342 const UChar *fcdPosition; /* Position in the original string to continue FCD check from. */
343 const UChar *bufferaddress; /* address of the normalization buffer */
344 int32_t buffersize;
345 uint8_t flags;
346 uint8_t origFlags;
347 uint32_t iteratorIndex;
348 int32_t iteratorMove;
349 };
350
351 U_CAPI void U_EXPORT2
352 uprv_init_collIterate(const UCollator *collator,
353 const UChar *sourceString, int32_t sourceLen,
354 U_NAMESPACE_QUALIFIER collIterate *s, UErrorCode *status);
355
356 /* Internal functions for C test code. */
357 U_CAPI U_NAMESPACE_QUALIFIER collIterate * U_EXPORT2
358 uprv_new_collIterate(UErrorCode *status);
359
360 U_CAPI void U_EXPORT2
361 uprv_delete_collIterate(U_NAMESPACE_QUALIFIER collIterate *s);
362
363 /* @return s->pos == s->endp */
364 U_CAPI UBool U_EXPORT2
365 uprv_collIterateAtEnd(U_NAMESPACE_QUALIFIER collIterate *s);
366
367 #ifdef XP_CPLUSPLUS
368
369 U_NAMESPACE_BEGIN
370
371 struct UCollationPCE;
372 typedef struct UCollationPCE UCollationPCE;
373
374 U_NAMESPACE_END
375
376 struct UCollationElements : public U_NAMESPACE_QUALIFIER UMemory
377 {
378 /**
379 * Struct wrapper for source data
380 */
381 U_NAMESPACE_QUALIFIER collIterate iteratordata_;
382 /**
383 * Indicates if this data has been reset.
384 */
385 UBool reset_;
386 /**
387 * Indicates if the data should be deleted.
388 */
389 UBool isWritable;
390
391 /**
392 * Data for getNextProcessed, getPreviousProcessed.
393 */
394 U_NAMESPACE_QUALIFIER UCollationPCE *pce;
395 };
396
397 #else
398 /*opaque type*/
399 struct UCollationElements;
400 #endif
401
402 U_CAPI void U_EXPORT2
403 uprv_init_pce(const struct UCollationElements *elems);
404
405 #define UCOL_LEVELTERMINATOR 1
406
407 /* mask off anything but primary order */
408 #define UCOL_PRIMARYORDERMASK 0xffff0000
409 /* mask off anything but secondary order */
410 #define UCOL_SECONDARYORDERMASK 0x0000ff00
411 /* mask off anything but tertiary order */
412 #define UCOL_TERTIARYORDERMASK 0x000000ff
413 /* primary order shift */
414 #define UCOL_PRIMARYORDERSHIFT 16
415 /* secondary order shift */
416 #define UCOL_SECONDARYORDERSHIFT 8
417
418 #define UCOL_BYTE_SIZE_MASK 0xFF
419
420 #define UCOL_CASE_BYTE_START 0x80
421 #define UCOL_CASE_SHIFT_START 7
422
423 /*
424 The definition of UCOL_CASE_MAX_BUFFER is moved down here so it can use UCOL_CASE_SHIFT_START.
425
426 A single collation element can generate two caseShift values, and UCOL_CASE_SHIFT_START caseShift
427 values are compressed into one byte. The UCOL_CASE_MAX_BUFFER should effectively be UCOL_MAX_BUFFER
428 multipled by 2/UCOL_CASE_SHIFT_START, with suitable rounding up.
429 */
430 #define UCOL_CASE_MAX_BUFFER (((2*UCOL_MAX_BUFFER) + UCOL_CASE_SHIFT_START - 1)/UCOL_CASE_SHIFT_START)
431
432 #define UCOL_IGNORABLE 0
433
434 /* get weights from a CE */
435 #define UCOL_PRIMARYORDER(order) (((order) & UCOL_PRIMARYORDERMASK)>> UCOL_PRIMARYORDERSHIFT)
436 #define UCOL_SECONDARYORDER(order) (((order) & UCOL_SECONDARYORDERMASK)>> UCOL_SECONDARYORDERSHIFT)
437 #define UCOL_TERTIARYORDER(order) ((order) & UCOL_TERTIARYORDERMASK)
438
439 /**
440 * Determine if a character is a Thai vowel (which sorts after
441 * its base consonant).
442 */
443 #define UCOL_ISTHAIPREVOWEL(ch) ((((uint32_t)(ch) - 0xe40) <= (0xe44 - 0xe40)) || \
444 (((uint32_t)(ch) - 0xec0) <= (0xec4 - 0xec0)))
445
446 /**
447 * Determine if a character is a Thai base consonant
448 */
449 #define UCOL_ISTHAIBASECONSONANT(ch) ((uint32_t)(ch) - 0xe01) <= (0xe2e - 0xe01)
450
451 #define UCOL_ISJAMO(ch) ((((uint32_t)(ch) - 0x1100) <= (0x1112 - 0x1100)) || \
452 (((uint32_t)(ch) - 0x1161) <= (0x1175 - 0x1161)) || \
453 (((uint32_t)(ch) - 0x11A8) <= (0x11C2 - 0x11A8)))
454
455 /* Han character ranges */
456 #define UCOL_FIRST_HAN 0x4E00
457 #define UCOL_LAST_HAN 0x9FFF
458 #define UCOL_FIRST_HAN_A 0x3400
459 #define UCOL_LAST_HAN_A 0x4DBF
460 #define UCOL_FIRST_HAN_COMPAT 0xFAE0
461 #define UCOL_LAST_HAN_COMPAT 0xFA2F
462
463 /* Han extension B is in plane 2 */
464 #define UCOL_FIRST_HAN_B 0x20000
465 #define UCOL_LAST_HAN_B 0x2A6DF
466
467 /* Hangul range */
468 #define UCOL_FIRST_HANGUL 0xAC00
469 #define UCOL_LAST_HANGUL 0xD7AF
470
471 /* Jamo ranges */
472 #define UCOL_FIRST_L_JAMO 0x1100
473 #define UCOL_FIRST_V_JAMO 0x1161
474 #define UCOL_FIRST_T_JAMO 0x11A8
475 #define UCOL_LAST_T_JAMO 0x11F9
476
477
478 #if 0
479 /* initializes collIterate structure */
480 /* made as macro to speed up things */
481 #define init_collIterate(collator, sourceString, sourceLen, s) { \
482 (s)->start = (s)->string = (s)->pos = (UChar *)(sourceString); \
483 (s)->endp = (sourceLen) == -1 ? NULL :(UChar *)(sourceString)+(sourceLen); \
484 (s)->CEpos = (s)->toReturn = (s)->CEs; \
485 (s)->isThai = TRUE; \
486 (s)->writableBuffer = (s)->stackWritableBuffer; \
487 (s)->writableBufSize = UCOL_WRITABLE_BUFFER_SIZE; \
488 (s)->coll = (collator); \
489 (s)->fcdPosition = 0; \
490 (s)->flags = 0; \
491 if(((collator)->normalizationMode == UCOL_ON)) (s)->flags |= UCOL_ITER_NORM; \
492 }
493 #endif
494
495
496
497 /*
498 * Macro to get the maximum size of an expansion ending with the argument ce.
499 * Used in the Boyer Moore algorithm.
500 * Note for tailoring, the UCA maxexpansion table has been merged.
501 * Hence we only have to search the tailored collator only.
502 * @param coll const UCollator pointer
503 * @param order last collation element of the expansion sequence
504 * @param result size of the longest expansion with argument collation element
505 * as the last element
506 */
507 #define UCOL_GETMAXEXPANSION(coll, order, result) { \
508 const uint32_t *start; \
509 const uint32_t *limit; \
510 const uint32_t *mid; \
511 start = (coll)->endExpansionCE; \
512 limit = (coll)->lastEndExpansionCE; \
513 while (start < limit - 1) { \
514 mid = start + ((limit - start) >> 1); \
515 if ((order) <= *mid) { \
516 limit = mid; \
517 } \
518 else { \
519 start = mid; \
520 } \
521 } \
522 if (*start == order) { \
523 result = *((coll)->expansionCESize + (start - (coll)->endExpansionCE)); \
524 } \
525 else if (*limit == order) { \
526 result = *(coll->expansionCESize + (limit - coll->endExpansionCE)); \
527 } \
528 else if ((order & 0xFFFF) == 0x00C0) { \
529 result = 2; \
530 } \
531 else { \
532 result = 1; \
533 } \
534 }
535
536 U_CFUNC
537 uint32_t ucol_prv_getSpecialCE(const UCollator *coll, UChar ch, uint32_t CE,
538 U_NAMESPACE_QUALIFIER collIterate *source, UErrorCode *status);
539
540 U_CFUNC
541 uint32_t ucol_prv_getSpecialPrevCE(const UCollator *coll, UChar ch, uint32_t CE,
542 U_NAMESPACE_QUALIFIER collIterate *source, UErrorCode *status);
543 U_CAPI uint32_t U_EXPORT2 ucol_getNextCE(const UCollator *coll,
544 U_NAMESPACE_QUALIFIER collIterate *collationSource, UErrorCode *status);
545 U_CFUNC uint32_t U_EXPORT2 ucol_getPrevCE(const UCollator *coll,
546 U_NAMESPACE_QUALIFIER collIterate *collationSource,
547 UErrorCode *status);
548 /* get some memory */
549 void *ucol_getABuffer(const UCollator *coll, uint32_t size);
550
551 #ifdef XP_CPLUSPLUS
552
553 U_NAMESPACE_BEGIN
554
555 class SortKeyByteSink;
556
557 U_NAMESPACE_END
558
559 /* function used by C++ getCollationKey to prevent restarting the calculation */
560 U_CFUNC int32_t
561 ucol_getSortKeyWithAllocation(const UCollator *coll,
562 const UChar *source, int32_t sourceLength,
563 uint8_t *&result, int32_t &resultCapacity,
564 UErrorCode *pErrorCode);
565
566 typedef void U_CALLCONV
567 SortKeyGenerator(const UCollator *coll,
568 const UChar *source,
569 int32_t sourceLength,
570 U_NAMESPACE_QUALIFIER SortKeyByteSink &result,
571 UErrorCode *status);
572
573 /* worker function for generating sortkeys */
574 U_CFUNC
575 void U_CALLCONV
576 ucol_calcSortKey(const UCollator *coll,
577 const UChar *source,
578 int32_t sourceLength,
579 U_NAMESPACE_QUALIFIER SortKeyByteSink &result,
580 UErrorCode *status);
581
582 U_CFUNC
583 void U_CALLCONV
584 ucol_calcSortKeySimpleTertiary(const UCollator *coll,
585 const UChar *source,
586 int32_t sourceLength,
587 U_NAMESPACE_QUALIFIER SortKeyByteSink &result,
588 UErrorCode *status);
589
590 #else
591
592 typedef void U_CALLCONV
593 SortKeyGenerator(const UCollator *coll,
594 const UChar *source,
595 int32_t sourceLength,
596 void *result,
597 UErrorCode *status);
598
599 #endif
600
601 /**
602 * Makes a copy of the Collator's rule data. The format is
603 * that of .col files.
604 *
605 * @param length returns the length of the data, in bytes.
606 * @param status the error status
607 * @return memory, owned by the caller, of size 'length' bytes.
608 * @internal INTERNAL USE ONLY
609 */
610 U_CFUNC uint8_t* U_EXPORT2
611 ucol_cloneRuleData(const UCollator *coll, int32_t *length, UErrorCode *status);
612
613 /**
614 * Used to set requested and valid locales on a collator returned by the collator
615 * service.
616 */
617 U_CFUNC void U_EXPORT2
618 ucol_setReqValidLocales(UCollator *coll, char *requestedLocaleToAdopt, char *validLocaleToAdopt, char *actualLocaleToAdopt);
619
620 #define UCOL_SPECIAL_FLAG 0xF0000000
621 #define UCOL_TAG_SHIFT 24
622 #define UCOL_TAG_MASK 0x0F000000
623 #define INIT_EXP_TABLE_SIZE 1024
624 #define UCOL_NOT_FOUND 0xF0000000
625 #define UCOL_EXPANSION 0xF1000000
626 #define UCOL_CONTRACTION 0xF2000000
627 #define UCOL_THAI 0xF3000000
628 #define UCOL_UNMARKED 0x03
629 #define UCOL_NEW_TERTIARYORDERMASK 0x0000003f
630
631 /* Bit mask for primary collation strength. */
632 #define UCOL_PRIMARYMASK 0xFFFF0000
633
634 /* Bit mask for secondary collation strength. */
635 #define UCOL_SECONDARYMASK 0x0000FF00
636
637 /* Bit mask for tertiary collation strength. */
638 #define UCOL_TERTIARYMASK 0x000000FF
639
640 /**
641 * Internal.
642 * This indicates the last element in a UCollationElements has been consumed.
643 * Compare with the UCOL_NULLORDER, UCOL_NULLORDER is returned if error occurs.
644 */
645 #define UCOL_NO_MORE_CES 0x00010101
646 #define UCOL_NO_MORE_CES_PRIMARY 0x00010000
647 #define UCOL_NO_MORE_CES_SECONDARY 0x00000100
648 #define UCOL_NO_MORE_CES_TERTIARY 0x00000001
649
650 #define isSpecial(CE) ((((CE)&UCOL_SPECIAL_FLAG)>>28)==0xF)
651
652 #define UCOL_UPPER_CASE 0x80
653 #define UCOL_MIXED_CASE 0x40
654 #define UCOL_LOWER_CASE 0x00
655
656 #define UCOL_CONTINUATION_MARKER 0xC0
657 #define UCOL_REMOVE_CONTINUATION 0xFFFFFF3F
658
659 #define isContinuation(CE) (((CE) & UCOL_CONTINUATION_MARKER) == UCOL_CONTINUATION_MARKER)
660 #define isFlagged(CE) (((CE) & 0x80) == 0x80)
661 #define isLongPrimary(CE) (((CE) & 0xC0) == 0xC0)
662
663 #define getCETag(CE) (((CE)&UCOL_TAG_MASK)>>UCOL_TAG_SHIFT)
664 #define isContraction(CE) (isSpecial((CE)) && (getCETag((CE)) == CONTRACTION_TAG))
665 #define isPrefix(CE) (isSpecial((CE)) && (getCETag((CE)) == SPEC_PROC_TAG))
666 #define constructContractCE(tag, CE) (UCOL_SPECIAL_FLAG | ((tag)<<UCOL_TAG_SHIFT) | ((CE)&0xFFFFFF))
667 #define constructSpecProcCE(CE) (UCOL_SPECIAL_FLAG | (SPEC_PROC_TAG<<UCOL_TAG_SHIFT) | ((CE)&0xFFFFFF))
668 #define getContractOffset(CE) ((CE)&0xFFFFFF)
669 #define getExpansionOffset(CE) (((CE)&0x00FFFFF0)>>4)
670 #define getExpansionCount(CE) ((CE)&0xF)
671 #define isCEIgnorable(CE) (((CE) & 0xFFFFFFBF) == 0)
672
673 /* StringSearch internal use */
674 #define inNormBuf(coleiter) ((coleiter)->iteratordata_.flags & UCOL_ITER_INNORMBUF)
675 #define isFCDPointerNull(coleiter) ((coleiter)->iteratordata_.fcdPosition == NULL)
676 #define hasExpansion(coleiter) ((coleiter)->iteratordata_.CEpos != (coleiter)->iteratordata_.CEs)
677 #define getExpansionPrefix(coleiter) ((coleiter)->iteratordata_.toReturn - (coleiter)->iteratordata_.CEs)
678 #define setExpansionPrefix(coleiter, offset) ((coleiter)->iteratordata_.CEs + offset)
679 #define getExpansionSuffix(coleiter) ((coleiter)->iteratordata_.CEpos - (coleiter)->iteratordata_.toReturn)
680 #define setExpansionSuffix(coleiter, offset) ((coleiter)->iteratordata_.toReturn = (coleiter)->iteratordata_.CEpos - leftoverces)
681
682 /* This is an enum that lists magic special byte values from the fractional UCA.
683 * See also http://site.icu-project.org/design/collation/bytes */
684 /* TODO: all the #defines that refer to special byte values from the UCA should be changed to point here */
685
686 enum {
687 UCOL_BYTE_ZERO = 0x00,
688 UCOL_BYTE_LEVEL_SEPARATOR = 0x01,
689 UCOL_BYTE_SORTKEY_GLUE = 0x02,
690 UCOL_BYTE_SHIFT_PREFIX = 0x03,
691 UCOL_BYTE_UNSHIFTED_MIN = UCOL_BYTE_SHIFT_PREFIX,
692 UCOL_BYTE_FIRST_TAILORED = 0x04,
693 UCOL_BYTE_COMMON = 0x05,
694 UCOL_BYTE_FIRST_UCA = UCOL_BYTE_COMMON,
695 /* TODO: Make the following values dynamic since they change with almost every UCA version. */
696 UCOL_CODAN_PLACEHOLDER = 0x12,
697 UCOL_BYTE_FIRST_NON_LATIN_PRIMARY = 0x5B,
698 UCOL_BYTE_UNSHIFTED_MAX = 0xFF
699 };
700
701 #if 0
702 #define UCOL_RESET_TOP_VALUE 0x9F000303
703 #define UCOL_FIRST_PRIMARY_IGNORABLE 0x00008705
704 #define UCOL_LAST_PRIMARY_IGNORABLE 0x0000DD05
705 #define UCOL_LAST_PRIMARY_IGNORABLE_CONT 0x000051C0
706 #define UCOL_FIRST_SECONDARY_IGNORABLE 0x00000000
707 #define UCOL_LAST_SECONDARY_IGNORABLE 0x00000500
708 #define UCOL_FIRST_TERTIARY_IGNORABLE 0x00000000
709 #define UCOL_LAST_TERTIARY_IGNORABLE 0x00000000
710 #define UCOL_FIRST_VARIABLE 0x05070505
711 #define UCOL_LAST_VARIABLE 0x179B0505
712 #define UCOL_FIRST_NON_VARIABLE 0x1A200505
713 #define UCOL_LAST_NON_VARIABLE 0x7B41058F
714
715 #define UCOL_NEXT_TOP_VALUE 0xE8960303
716 #define UCOL_NEXT_FIRST_PRIMARY_IGNORABLE 0x00008905
717 #define UCOL_NEXT_LAST_PRIMARY_IGNORABLE 0x03000303
718 #define UCOL_NEXT_FIRST_SECONDARY_IGNORABLE 0x00008705
719 #define UCOL_NEXT_LAST_SECONDARY_IGNORABLE 0x00000500
720 #define UCOL_NEXT_FIRST_TERTIARY_IGNORABLE 0x00000000
721 #define UCOL_NEXT_LAST_TERTIARY_IGNORABLE 0x00000000
722 #define UCOL_NEXT_FIRST_VARIABLE 0x05090505
723 #define UCOL_NEXT_LAST_VARIABLE 0x1A200505
724
725 #define PRIMARY_IMPLICIT_MIN 0xE8000000
726 #define PRIMARY_IMPLICIT_MAX 0xF0000000
727 #endif
728
729 /* These constants can be changed - sortkey size is affected by them */
730 #define UCOL_PROPORTION2 0.5
731 #define UCOL_PROPORTION3 0.667
732
733 /* These values come from the UCA */
734 #define UCOL_COMMON_BOT2 UCOL_BYTE_COMMON
735 #define UCOL_COMMON_TOP2 0x86u
736 #define UCOL_TOTAL2 (UCOL_COMMON_TOP2-UCOL_COMMON_BOT2-1)
737
738 #define UCOL_FLAG_BIT_MASK_CASE_SW_OFF 0x80
739 #define UCOL_FLAG_BIT_MASK_CASE_SW_ON 0x40
740 #define UCOL_COMMON_TOP3_CASE_SW_OFF 0x85
741 #define UCOL_COMMON_TOP3_CASE_SW_LOWER 0x45
742 #define UCOL_COMMON_TOP3_CASE_SW_UPPER 0xC5
743
744 /* These values come from the UCA */
745 #define UCOL_COMMON_BOT3 0x05
746
747 #define UCOL_COMMON_BOTTOM3_CASE_SW_UPPER 0x86;
748 #define UCOL_COMMON_BOTTOM3_CASE_SW_LOWER UCOL_COMMON_BOT3;
749
750 #define UCOL_TOP_COUNT2 (UCOL_PROPORTION2*UCOL_TOTAL2)
751 #define UCOL_BOT_COUNT2 (UCOL_TOTAL2-UCOL_TOP_COUNT2)
752
753
754 #define UCOL_COMMON2 UCOL_COMMON_BOT2
755 #define UCOL_COMMON3_UPPERFIRST 0xC5
756 #define UCOL_COMMON3_NORMAL UCOL_COMMON_BOT3
757
758 #define UCOL_COMMON4 0xFF
759
760 /* constants for case level/case first handling */
761 /* used to instantiate UCollators fields in ucol_updateInternalState */
762 #define UCOL_CASE_SWITCH 0xC0
763 #define UCOL_NO_CASE_SWITCH 0x00
764
765 #define UCOL_REMOVE_CASE 0x3F
766 #define UCOL_KEEP_CASE 0xFF
767
768 #define UCOL_CASE_BIT_MASK 0xC0
769
770 #define UCOL_TERT_CASE_MASK 0xFF
771
772 #define UCOL_ENDOFLATINONERANGE 0xFF
773 #define UCOL_LATINONETABLELEN (UCOL_ENDOFLATINONERANGE+50)
774 #define UCOL_BAIL_OUT_CE 0xFF000000
775
776
777 typedef enum {
778 NOT_FOUND_TAG = 0,
779 EXPANSION_TAG = 1, /* This code point results in an expansion */
780 CONTRACTION_TAG = 2, /* Start of a contraction */
781 THAI_TAG = 3, /* Thai character - do the reordering */
782 CHARSET_TAG = 4, /* Charset processing, not yet implemented */
783 SURROGATE_TAG = 5, /* Lead surrogate that is tailored and doesn't start a contraction */
784 HANGUL_SYLLABLE_TAG = 6, /* AC00-D7AF*/
785 LEAD_SURROGATE_TAG = 7, /* D800-DBFF*/
786 TRAIL_SURROGATE_TAG = 8, /* DC00-DFFF*/
787 CJK_IMPLICIT_TAG = 9, /* 0x3400-0x4DB5, 0x4E00-0x9FA5, 0xF900-0xFA2D*/
788 IMPLICIT_TAG = 10,
789 SPEC_PROC_TAG = 11,
790 /* ICU 2.1 */
791 LONG_PRIMARY_TAG = 12, /* This is a three byte primary with starting secondaries and tertiaries */
792 /* It fits in a single 32 bit CE and is used instead of expansion to save */
793 /* space without affecting the performance (hopefully) */
794
795 DIGIT_TAG = 13, /* COllate Digits As Numbers (CODAN) implementation */
796
797 CE_TAGS_COUNT
798 } UColCETags;
799
800 /*
801 *****************************************************************************************
802 * set to zero
803 * NON_CHARACTER FDD0 - FDEF, FFFE, FFFF, 1FFFE, 1FFFF, 2FFFE, 2FFFF,...e.g. **FFFE, **FFFF
804 ******************************************************************************************
805 */
806
807 typedef struct {
808 uint32_t variableTopValue;
809 /*UColAttributeValue*/ int32_t frenchCollation;
810 /*UColAttributeValue*/ int32_t alternateHandling; /* attribute for handling variable elements*/
811 /*UColAttributeValue*/ int32_t caseFirst; /* who goes first, lower case or uppercase */
812 /*UColAttributeValue*/ int32_t caseLevel; /* do we have an extra case level */
813 /*UColAttributeValue*/ int32_t normalizationMode; /* attribute for normalization */
814 /*UColAttributeValue*/ int32_t strength; /* attribute for strength */
815 /*UColAttributeValue*/ int32_t hiraganaQ; /* attribute for special Hiragana */
816 /*UColAttributeValue*/ int32_t numericCollation; /* attribute for numeric collation */
817 uint32_t reserved[15]; /* for future use */
818 } UColOptionSet;
819
820 typedef struct {
821 uint32_t UCA_FIRST_TERTIARY_IGNORABLE[2]; /*0x00000000*/
822 uint32_t UCA_LAST_TERTIARY_IGNORABLE[2]; /*0x00000000*/
823 uint32_t UCA_FIRST_PRIMARY_IGNORABLE[2]; /*0x00008705*/
824 uint32_t UCA_FIRST_SECONDARY_IGNORABLE[2]; /*0x00000000*/
825 uint32_t UCA_LAST_SECONDARY_IGNORABLE[2]; /*0x00000500*/
826 uint32_t UCA_LAST_PRIMARY_IGNORABLE[2]; /*0x0000DD05*/
827 uint32_t UCA_FIRST_VARIABLE[2]; /*0x05070505*/
828 uint32_t UCA_LAST_VARIABLE[2]; /*0x13CF0505*/
829 uint32_t UCA_FIRST_NON_VARIABLE[2]; /*0x16200505*/
830 uint32_t UCA_LAST_NON_VARIABLE[2]; /*0x767C0505*/
831 uint32_t UCA_RESET_TOP_VALUE[2]; /*0x9F000303*/
832 uint32_t UCA_FIRST_IMPLICIT[2];
833 uint32_t UCA_LAST_IMPLICIT[2];
834 uint32_t UCA_FIRST_TRAILING[2];
835 uint32_t UCA_LAST_TRAILING[2];
836
837 #if 0
838 uint32_t UCA_NEXT_TOP_VALUE[2]; /*0xE8960303*/
839 uint32_t UCA_NEXT_FIRST_PRIMARY_IGNORABLE; /*0x00008905*/
840 uint32_t UCA_NEXT_LAST_PRIMARY_IGNORABLE; /*0x03000303*/
841 uint32_t UCA_NEXT_FIRST_SECONDARY_IGNORABLE; /*0x00008705*/
842 uint32_t UCA_NEXT_LAST_SECONDARY_IGNORABLE; /*0x00000500*/
843 uint32_t UCA_NEXT_FIRST_TERTIARY_IGNORABLE; /*0x00000000*/
844 uint32_t UCA_NEXT_LAST_TERTIARY_IGNORABLE; /*0x00000000*/
845 uint32_t UCA_NEXT_FIRST_VARIABLE; /*0x05090505*/
846 uint32_t UCA_NEXT_LAST_VARIABLE; /*0x16200505*/
847 #endif
848
849 uint32_t UCA_PRIMARY_TOP_MIN;
850 uint32_t UCA_PRIMARY_IMPLICIT_MIN; /*0xE8000000*/
851 uint32_t UCA_PRIMARY_IMPLICIT_MAX; /*0xF0000000*/
852 uint32_t UCA_PRIMARY_TRAILING_MIN; /*0xE8000000*/
853 uint32_t UCA_PRIMARY_TRAILING_MAX; /*0xF0000000*/
854 uint32_t UCA_PRIMARY_SPECIAL_MIN; /*0xE8000000*/
855 uint32_t UCA_PRIMARY_SPECIAL_MAX; /*0xF0000000*/
856 } UCAConstants;
857
858 typedef struct {
859 int32_t size;
860 /* all the offsets are in bytes */
861 /* to get the address add to the header address and cast properly */
862 uint32_t options; /* these are the default options for the collator */
863 uint32_t UCAConsts; /* structure which holds values for indirect positioning and implicit ranges */
864 uint32_t contractionUCACombos; /* this one is needed only for UCA, to copy the appropriate contractions */
865 uint32_t magic; /* magic number - lets us know whether reserved data is reset or junked */
866 uint32_t mappingPosition; /* const uint8_t *mappingPosition; */
867 uint32_t expansion; /* uint32_t *expansion; */
868 uint32_t contractionIndex; /* UChar *contractionIndex; */
869 uint32_t contractionCEs; /* uint32_t *contractionCEs; */
870 uint32_t contractionSize; /* needed for various closures */
871 /*int32_t latinOneMapping;*/ /* this is now handled in the trie itself *//* fast track to latin1 chars */
872
873 uint32_t endExpansionCE; /* array of last collation element in
874 expansion */
875 uint32_t expansionCESize; /* array of maximum expansion size
876 corresponding to the expansion
877 collation elements with last element
878 in endExpansionCE*/
879 int32_t endExpansionCECount; /* size of endExpansionCE */
880 uint32_t unsafeCP; /* hash table of unsafe code points */
881 uint32_t contrEndCP; /* hash table of final code points */
882 /* in contractions. */
883
884 int32_t contractionUCACombosSize; /* number of UCA contraction items. */
885 /*Length is contractionUCACombosSize*contractionUCACombosWidth*sizeof(UChar) */
886 UBool jamoSpecial; /* is jamoSpecial */
887 UBool isBigEndian; /* is this data big endian? from the UDataInfo header*/
888 uint8_t charSetFamily; /* what is the charset family of this data from the UDataInfo header*/
889 uint8_t contractionUCACombosWidth; /* width of UCA combos field */
890 UVersionInfo version;
891 UVersionInfo UCAVersion; /* version of the UCA, read from file */
892 UVersionInfo UCDVersion; /* UCD version, obtained by u_getUnicodeVersion */
893 UVersionInfo formatVersion; /* format version from the UDataInfo header */
894 uint32_t scriptToLeadByte; /* offset to script to lead collation byte mapping data */
895 uint32_t leadByteToScript; /* offset to lead collation byte to script mapping data */
896 uint8_t reserved[76]; /* for future use */
897 } UCATableHeader;
898
899 #define U_UNKNOWN_STATE 0
900 #define U_COLLATOR_STATE 0x01
901 #define U_STATE_LIMIT 0x02
902
903 /* This is the first structure in a state */
904 /* it should be machine independent */
905 typedef struct {
906 /* this structure is supposed to be readable on all the platforms.*/
907 /* first 2 fields hold the size of the structure in a platform independent way */
908 uint8_t sizeLo;
909 uint8_t sizeHi;
910 /* identifying the writing platform */
911 uint8_t isBigEndian;
912 /* see U_CHARSET_FAMILY values in utypes.h */
913 uint8_t charsetFamily;
914 /* version of ICU this state structure comes from */
915 uint8_t icuVersion[4];
916 /* What is the data following this state */
917 uint8_t type;
918 /* more stuff to come, keep it on 16 byte boundary */
919 uint8_t reserved[7];
920 } UStateStruct;
921
922 /* This structure follows UStatusStruct */
923 /* and contains data specific for the collators */
924 /* Endianess needs to be decided before accessing this structure */
925 /* However, it's size IS endianess independent */
926 typedef struct {
927 /* size of this structure */
928 uint8_t sizeLo;
929 uint8_t sizeHi;
930 /* This state is followed by the frozen tailoring */
931 uint8_t containsTailoring;
932 /* This state is followed by the frozen UCA */
933 uint8_t containsUCA;
934 /* Version info - the same one */
935 uint8_t versionInfo[4];
936
937 /* for charset CEs */
938 uint8_t charsetName[32];
939 /* this is the resolved locale name*/
940 uint8_t locale[32];
941
942 /* Attributes. Open ended */
943 /* all the following will be moved to uint32_t because of portability */
944 /* variable top value */
945 uint32_t variableTopValue;
946 /* attribute for handling variable elements*/
947 uint32_t /*UColAttributeValue*/ alternateHandling;
948 /* how to handle secondary weights */
949 uint32_t /*UColAttributeValue*/ frenchCollation;
950 /* who goes first, lower case or uppercase */
951 uint32_t /*UColAttributeValue*/ caseFirst;
952 /* do we have an extra case level */
953 uint32_t /*UColAttributeValue*/ caseLevel;
954 /* attribute for normalization */
955 uint32_t /*UColAttributeValue*/ normalizationMode;
956 /* attribute for strength */
957 uint32_t /*UColAttributeValue*/ strength;
958 /* to be immediately 16 byte aligned */
959 uint8_t reserved[12];
960 } UColStateStruct;
961
962 #define UCOL_INV_SIZEMASK 0xFFF00000
963 #define UCOL_INV_OFFSETMASK 0x000FFFFF
964 #define UCOL_INV_SHIFTVALUE 20
965
966 U_CDECL_BEGIN
967
968 typedef struct {
969 uint32_t byteSize;
970 uint32_t tableSize;
971 uint32_t contsSize;
972 uint32_t table;
973 uint32_t conts;
974 UVersionInfo UCAVersion; /* version of the UCA, read from file */
975 uint8_t padding[8];
976 } InverseUCATableHeader;
977
978 typedef void U_CALLCONV
979 ResourceCleaner(UCollator *coll);
980
981
982 struct UCollator {
983 UColOptionSet *options;
984 SortKeyGenerator *sortKeyGen;
985 uint32_t *latinOneCEs;
986 char* actualLocale;
987 char* validLocale;
988 char* requestedLocale;
989 const UChar *rules;
990 const UChar *ucaRules;
991 const UCollator *UCA;
992 const UCATableHeader *image;
993 UTrie mapping;
994 const uint32_t *latinOneMapping;
995 const uint32_t *expansion;
996 const UChar *contractionIndex;
997 const uint32_t *contractionCEs;
998
999 const uint32_t *endExpansionCE; /* array of last ces in an expansion ce.
1000 corresponds to expansionCESize */
1001 const uint32_t *lastEndExpansionCE;/* pointer to the last element in endExpansionCE */
1002 const uint8_t *expansionCESize; /* array of the maximum size of a
1003 expansion ce with the last ce
1004 corresponding to endExpansionCE,
1005 terminated with a null */
1006 const uint8_t *unsafeCP; /* unsafe code points hashtable */
1007 const uint8_t *contrEndCP; /* Contraction ending chars hash table */
1008 UChar minUnsafeCP; /* Smallest unsafe Code Point. */
1009 UChar minContrEndCP; /* Smallest code point at end of a contraction */
1010
1011 int32_t rulesLength;
1012 int32_t latinOneTableLen;
1013
1014 uint32_t variableTopValue;
1015 UColAttributeValue frenchCollation;
1016 UColAttributeValue alternateHandling; /* attribute for handling variable elements*/
1017 UColAttributeValue caseFirst; /* who goes first, lower case or uppercase */
1018 UColAttributeValue caseLevel; /* do we have an extra case level */
1019 UColAttributeValue normalizationMode; /* attribute for normalization */
1020 UColAttributeValue strength; /* attribute for strength */
1021 UColAttributeValue hiraganaQ; /* attribute for Hiragana */
1022 UColAttributeValue numericCollation;
1023 UBool variableTopValueisDefault;
1024 UBool frenchCollationisDefault;
1025 UBool alternateHandlingisDefault; /* attribute for handling variable elements*/
1026 UBool caseFirstisDefault; /* who goes first, lower case or uppercase */
1027 UBool caseLevelisDefault; /* do we have an extra case level */
1028 UBool normalizationModeisDefault; /* attribute for normalization */
1029 UBool strengthisDefault; /* attribute for strength */
1030 UBool hiraganaQisDefault; /* attribute for Hiragana */
1031 UBool numericCollationisDefault;
1032 UBool hasRealData; /* some collators have only options, like French, no rules */
1033 /* to speed up things, we use the UCA image, but we don't want it */
1034 /* to run around */
1035
1036 UBool freeOnClose;
1037 UBool freeOptionsOnClose;
1038 UBool freeRulesOnClose;
1039 UBool freeImageOnClose;
1040 UBool freeDefaultReorderCodesOnClose;
1041 UBool freeReorderCodesOnClose;
1042 UBool freeLeadBytePermutationTableOnClose;
1043
1044 UBool latinOneUse;
1045 UBool latinOneRegenTable;
1046 UBool latinOneFailed;
1047
1048 int8_t tertiaryAddition; /* when switching case, we need to add or subtract different values */
1049 uint8_t caseSwitch;
1050 uint8_t tertiaryCommon;
1051 uint8_t tertiaryMask;
1052 uint8_t tertiaryTop; /* Upper range when compressing */
1053 uint8_t tertiaryBottom; /* Upper range when compressing */
1054 uint8_t tertiaryTopCount;
1055 uint8_t tertiaryBottomCount;
1056
1057 UVersionInfo dataVersion; /* Data info of UCA table */
1058 int32_t* defaultReorderCodes;
1059 int32_t defaultReorderCodesLength;
1060 int32_t* reorderCodes;
1061 int32_t reorderCodesLength;
1062 uint8_t* leadBytePermutationTable;
1063 };
1064
1065 U_CDECL_END
1066
1067 /* various internal functions */
1068
1069 /* do not close UCA returned by ucol_initUCA! */
1070 U_CFUNC
1071 UCollator* ucol_initUCA(UErrorCode *status);
1072
1073 U_CFUNC
1074 UCollator* ucol_initCollator(const UCATableHeader *image, UCollator *fillIn, const UCollator *UCA, UErrorCode *status);
1075
1076 U_CFUNC
1077 void ucol_setOptionsFromHeader(UCollator* result, UColOptionSet * opts, UErrorCode *status);
1078
1079 U_CFUNC
1080 UCollator* ucol_open_internal(const char* loc, UErrorCode* status);
1081
1082 #if 0
1083 U_CFUNC
1084 void ucol_putOptionsToHeader(UCollator* result, UColOptionSet * opts, UErrorCode *status);
1085 #endif
1086
1087 U_CFUNC
1088 void ucol_updateInternalState(UCollator *coll, UErrorCode *status);
1089
1090 U_CFUNC uint32_t U_EXPORT2 ucol_getFirstCE(const UCollator *coll, UChar u, UErrorCode *status);
1091 U_CAPI UBool U_EXPORT2 ucol_isTailored(const UCollator *coll, const UChar u, UErrorCode *status);
1092
1093 U_CAPI const InverseUCATableHeader* U_EXPORT2 ucol_initInverseUCA(UErrorCode *status);
1094
1095 U_CAPI void U_EXPORT2
1096 uprv_uca_initImplicitConstants(UErrorCode *status);
1097
1098 U_CAPI uint32_t U_EXPORT2
1099 uprv_uca_getImplicitFromRaw(UChar32 cp);
1100
1101 /*U_CFUNC uint32_t U_EXPORT2
1102 uprv_uca_getImplicitPrimary(UChar32 cp);*/
1103
1104 U_CAPI UChar32 U_EXPORT2
1105 uprv_uca_getRawFromImplicit(uint32_t implicit);
1106
1107 U_CAPI UChar32 U_EXPORT2
1108 uprv_uca_getRawFromCodePoint(UChar32 i);
1109
1110 U_CAPI UChar32 U_EXPORT2
1111 uprv_uca_getCodePointFromRaw(UChar32 i);
1112
1113 typedef const UChar* GetCollationRulesFunction(void* context, const char* locale, const char* type, int32_t* pLength, UErrorCode* status);
1114
1115 U_CAPI UCollator* U_EXPORT2
1116 ucol_openRulesForImport( const UChar *rules,
1117 int32_t rulesLength,
1118 UColAttributeValue normalizationMode,
1119 UCollationStrength strength,
1120 UParseError *parseError,
1121 GetCollationRulesFunction importFunc,
1122 void* context,
1123 UErrorCode *status);
1124
1125
1126 U_CFUNC void U_EXPORT2
1127 ucol_buildPermutationTable(UCollator *coll, UErrorCode *status);
1128
1129 U_CFUNC int U_EXPORT2
1130 ucol_getLeadBytesForReorderCode(const UCollator *uca, int reorderCode, uint16_t* returnLeadBytes, int returnCapacity);
1131
1132 U_CFUNC int U_EXPORT2
1133 ucol_getReorderCodesForLeadByte(const UCollator *uca, int leadByte, int16_t* returnReorderCodes, int returnCapacity);
1134
1135 #ifdef XP_CPLUSPLUS
1136 /*
1137 * Test whether a character is potentially "unsafe" for use as a collation
1138 * starting point. Unsafe chars are those with combining class != 0 plus
1139 * those that are the 2nd thru nth character in a contraction sequence.
1140 *
1141 * Function is in header file because it's used in both collation and string search,
1142 * and needs to be inline for performance.
1143 */
ucol_unsafeCP(UChar c,const UCollator * coll)1144 static inline UBool ucol_unsafeCP(UChar c, const UCollator *coll) {
1145 int32_t hash;
1146 uint8_t htbyte;
1147
1148 if (c < coll->minUnsafeCP) {
1149 return FALSE;
1150 }
1151
1152 hash = c;
1153 if (hash >= UCOL_UNSAFECP_TABLE_SIZE*8) {
1154 if(UTF_IS_SURROGATE(c)) {
1155 /* Lead or trail surrogate */
1156 /* These are always considered unsafe. */
1157 return TRUE;
1158 }
1159 hash = (hash & UCOL_UNSAFECP_TABLE_MASK) + 256;
1160 }
1161 htbyte = coll->unsafeCP[hash>>3];
1162 return ((htbyte >> (hash & 7)) & 1);
1163 }
1164 #endif /* XP_CPLUSPLUS */
1165
1166 /* The offsetBuffer in collIterate might need to be freed to avoid memory leaks. */
1167 void ucol_freeOffsetBuffer(U_NAMESPACE_QUALIFIER collIterate *s);
1168
1169 #endif /* #if !UCONFIG_NO_COLLATION */
1170
1171 #endif
1172