1 /*
2 *******************************************************************************
3 *
4 * Copyright (C) 1998-2012, International Business Machines
5 * Corporation and others. All Rights Reserved.
6 *
7 *******************************************************************************
8 *
9 * Private implementation header for C collation
10 * file name: ucol_imp.h
11 * encoding: US-ASCII
12 * tab size: 8 (not used)
13 * indentation:4
14 *
15 * created on: 2000dec11
16 * created by: Vladimir Weinstein
17 *
18 * Modification history
19 * Date Name Comments
20 * 02/16/2001 synwee Added UCOL_GETPREVCE for the use in ucoleitr
21 * 02/27/2001 synwee Added getMaxExpansion data structure in UCollator
22 * 03/02/2001 synwee Added UCOL_IMPLICIT_CE
23 * 03/12/2001 synwee Added pointer start to collIterate.
24 */
25
26 #ifndef UCOL_IMP_H
27 #define UCOL_IMP_H
28
29 #include "unicode/utypes.h"
30 #ifdef __cplusplus
31 # include "unicode/utf16.h"
32 #endif
33
34 #define UCA_DATA_TYPE "icu"
35 #define UCA_DATA_NAME "ucadata"
36 #define INVC_DATA_TYPE "icu"
37 #define INVC_DATA_NAME "invuca"
38
39 /**
40 * Convenience string denoting the Collation data tree
41 * @internal ICU 3.0
42 */
43 #define U_ICUDATA_COLL U_ICUDATA_NAME U_TREE_SEPARATOR_STRING "coll"
44
45 #if !UCONFIG_NO_COLLATION
46
47 #ifdef __cplusplus
48 #include "unicode/normalizer2.h"
49 #include "unicode/unistr.h"
50 #endif
51 #include "unicode/ucol.h"
52 #include "ucol_data.h"
53 #include "utrie.h"
54 #include "cmemory.h"
55
56 /* This is the internal header file which contains important declarations for
57 * the collation framework.
58 * Ready to use collators are stored as binary images. Both UCA and tailorings
59 * share the same binary format. Individual files (currently only UCA) have a
60 * udata header in front of the image and should be opened using udata_open.
61 * Tailoring images are currently stored inside resource bundles and are intialized
62 * through ucol_open API.
63 *
64 * The following describes the formats for collation binaries
65 * (UCA & tailorings) and for the inverse UCA table.
66 * Substructures are described in the collation design document at
67 * http://source.icu-project.org/repos/icu/icuhtml/trunk/design/collation/ICU_collation_design.htm
68 *
69 * -------------------------------------------------------------
70 *
71 * Here is the format of binary collation image.
72 *
73 * Physical order of structures:
74 * - header (UCATableHeader)
75 * - options (UColOptionSet)
76 * - expansions (CE[])
77 * - contractions (UChar[contractionSize] + CE[contractionSize])
78 * - serialized UTrie with mappings of code points to CEs
79 * - max expansion tables (CE[endExpansionCECount] + uint8_t[endExpansionCECount])
80 * - two bit sets for backward processing in strcoll (identical prefixes)
81 * and for backward CE iteration (each set is uint8_t[UCOL_UNSAFECP_TABLE_SIZE])
82 * - UCA constants (UCAConstants)
83 * - UCA contractions (UChar[contractionUCACombosSize][contractionUCACombosWidth])
84 *
85 * UCATableHeader fields:
86 *
87 * int32_t size; - image size in bytes
88 *
89 * Offsets to interesting data. All offsets are in bytes.
90 * to get the address add to the header address and cast properly.
91 * Some offsets are zero if the corresponding structures are empty.
92 *
93 * Tailoring binaries that only set options and contain no mappings etc.
94 * will have all offsets 0 except for the options and expansion offsets,
95 * which give the position and length of the options array.
96 *
97 * uint32_t options; - offset to default collator options (UColOptionSet *),
98 * a set of 32-bit values. See declaration of UColOptionSet for more details
99 *
100 * uint32_t UCAConsts; - only used (!=0) in UCA image - structure which holds values for indirect positioning and implicit ranges
101 * See declaration of UCAConstants structure. This is a set of unsigned 32-bit values used to store
102 * important constant values that are defined in the UCA and used for building and runtime.
103 *
104 * uint32_t contractionUCACombos; - only used (!=0) in UCA image - list of UCA contractions. This is a zero terminated array of UChar[contractionUCACombosWidth],
105 * containing contractions from the UCA. These are needed in the build process to copy UCA contractions
106 * in case the base contraction symbol is tailored.
107 *
108 * uint32_t magic; - must contain UCOL_HEADER_MAGIC (formatVersion 2.3)
109 *
110 * uint32_t mappingPosition; - offset to UTrie (const uint8_t *mappingPosition). This is a serialized UTrie and should be treated as such.
111 * Used as a primary lookup table for collation elements.
112 *
113 * uint32_t expansion; - offset to expansion table (uint32_t *expansion). This is an array of expansion CEs. Never 0.
114 *
115 * uint32_t contractionIndex; - offset to contraction table (UChar *contractionIndex). Used to look up contraction sequences. Contents
116 * are aligned with the contents of contractionCEs table. 0 if no contractions.
117 *
118 * uint32_t contractionCEs; - offset to resulting contraction CEs (uint32_t *contractionCEs). When a contraction is resolved in the
119 * in the contractionIndex table, the resulting index is used to look up corresponding CE in this table.
120 * 0 if no contractions.
121 * uint32_t contractionSize; - size of contraction table in elements (both Index and CEs).
122 *
123 * Tables described below are used for Boyer-Moore searching algorithm - they define the size of longest expansion
124 * and last CEs in expansions.
125 * uint32_t endExpansionCE; - offset to array of last collation element in expansion (uint32_t *).
126 * Never 0.
127 * uint32_t expansionCESize; - array of maximum expansion sizes (uint8_t *)
128 * int32_t endExpansionCECount; - size of endExpansionCE. See UCOL_GETMAXEXPANSION
129 * for the usage model
130 *
131 * These two offsets point to byte tables that are used in the backup heuristics.
132 * uint32_t unsafeCP; - hash table of unsafe code points (uint8_t *). See ucol_unsafeCP function.
133 * uint32_t contrEndCP; - hash table of final code points in contractions (uint8_t *). See ucol_contractionEndCP.
134 *
135 * int32_t contractionUCACombosSize; - number of UChar[contractionUCACombosWidth] in contractionUCACombos
136 * (formatVersion 2.3)
137 * UBool jamoSpecial; - Jamo special indicator (uint8_t). If TRUE, Jamos are special, so we cannot use simple Hangul decomposition.
138 * UBool isBigEndian; - endianness of this collation binary (formatVersion 2.3)
139 * uint8_t charSetFamily; - charset family of this collation binary (formatVersion 2.3)
140 * uint8_t contractionUCACombosWidth; - number of UChars per UCA contraction in contractionUCACombos (formatVersion 2.3)
141 *
142 * Various version fields
143 * UVersionInfo version; - version 4 uint8_t
144 * UVersionInfo UCAVersion; - version 4 uint8_t
145 * UVersionInfo UCDVersion; - version 4 uint8_t
146 * UVersionInfo formatVersion; - version of the format of the collation binary
147 * same formatVersion as in ucadata.icu's UDataInfo header
148 * (formatVersion 2.3)
149 *
150 * uint32_t offset to the reordering code to lead CE byte remapping table
151 * uint32_t offset to the lead CE byte to reordering code mapping table
152 *
153 * uint8_t reserved[76]; - currently unused
154 *
155 * -------------------------------------------------------------
156 *
157 * Inverse UCA is used for constructing collators from rules. It is always an individual file
158 * and always has a UDataInfo header.
159 * here is the structure:
160 *
161 * uint32_t byteSize; - size of inverse UCA image in bytes
162 * uint32_t tableSize; - length of inverse table (number of uint32_t[3] rows)
163 * uint32_t contsSize; - size of continuation table (number of UChars in table)
164 *
165 * uint32_t table; - offset to inverse table (uint32_t *)
166 * Inverse table contains of rows of 3 uint32_t values. First two values are CE and a possible continuation
167 * the third value is either a code unit (if there is only one code unit for element) or an index to continuation
168 * (number of code units combined with an index).
169 * table. If more than one codepoint have the same CE, continuation table contains code units separated by FFFF and final
170 * code unit sequence for a CE is terminated by FFFE.
171 * uint32_t conts; - offset to continuation table (uint16_t *). Contains code units that transform to a same CE.
172 *
173 * UVersionInfo UCAVersion; - version of the UCA, read from file 4 uint8_t
174 * uint8_t padding[8]; - padding 8 uint8_t
175 * Header is followed by the table and continuation table.
176 */
177
178 /* definition of UCOL_HEADER_MAGIC moved to common/ucol_data.h */
179
180 /* UDataInfo for UCA mapping table */
181 /* dataFormat="UCol" */
182 #define UCA_DATA_FORMAT_0 ((uint8_t)0x55)
183 #define UCA_DATA_FORMAT_1 ((uint8_t)0x43)
184 #define UCA_DATA_FORMAT_2 ((uint8_t)0x6f)
185 #define UCA_DATA_FORMAT_3 ((uint8_t)0x6c)
186
187 #define UCA_FORMAT_VERSION_0 ((uint8_t)3)
188 #define UCA_FORMAT_VERSION_1 0
189 #define UCA_FORMAT_VERSION_2 ((uint8_t)0)
190 #define UCA_FORMAT_VERSION_3 ((uint8_t)0)
191
192 /* UDataInfo for inverse UCA table */
193 /* dataFormat="InvC" */
194 #define INVUCA_DATA_FORMAT_0 ((uint8_t)0x49)
195 #define INVUCA_DATA_FORMAT_1 ((uint8_t)0x6E)
196 #define INVUCA_DATA_FORMAT_2 ((uint8_t)0x76)
197 #define INVUCA_DATA_FORMAT_3 ((uint8_t)0x43)
198
199 #define INVUCA_FORMAT_VERSION_0 ((uint8_t)2)
200 #define INVUCA_FORMAT_VERSION_1 ((uint8_t)1)
201 #define INVUCA_FORMAT_VERSION_2 ((uint8_t)0)
202 #define INVUCA_FORMAT_VERSION_3 ((uint8_t)0)
203
204 /* This is the size of the stack allocated buffer for sortkey generation and similar operations */
205 /* if it is too small, heap allocation will occur.*/
206 /* you can change this value if you need memory - it will affect the performance, though, since we're going to malloc */
207 #define UCOL_MAX_BUFFER 128
208
209 #define UCOL_NORMALIZATION_GROWTH 2
210 #define UCOL_NORMALIZATION_MAX_BUFFER UCOL_MAX_BUFFER*UCOL_NORMALIZATION_GROWTH
211
212 /* This writable buffer is used if we encounter Thai and need to reorder the string on the fly */
213 /* Sometimes we already have a writable buffer (like in case of normalized strings). */
214 /*
215 you can change this value to any value >= 4 if you need memory -
216 it will affect the performance, though, since we're going to malloc.
217 Note 3 is the minimum value for Thai collation and 4 is the
218 minimum number for special Jamo
219 */
220 #define UCOL_WRITABLE_BUFFER_SIZE 256
221
222 /* This is the size of the buffer for expansion CE's */
223 /* In reality we should not have to deal with expm sequences longer then 16 */
224 /* you can change this value if you need memory */
225 /* WARNING THIS BUFFER DOES HAVE MALLOC FALLBACK. If you make it too small, you'll get into performance trouble */
226 /* Reasonable small value is around 10, if you don't do Arabic or other funky collations that have long expansion sequence */
227 /* This is the longest expansion sequence we can handle without bombing out */
228 #define UCOL_EXPAND_CE_BUFFER_SIZE 64
229
230 /* This is the size to increase the buffer for expansion CE's */
231 #define UCOL_EXPAND_CE_BUFFER_EXTEND_SIZE 64
232
233
234 /* Unsafe UChar hash table table size. */
235 /* size is 32 bytes for 1 bit for each latin 1 char + some power of two for */
236 /* hashing the rest of the chars. Size in bytes */
237 #define UCOL_UNSAFECP_TABLE_SIZE 1056
238 /* mask value down to "some power of two"-1 */
239 /* number of bits, not num of bytes. */
240 #define UCOL_UNSAFECP_TABLE_MASK 0x1fff
241
242
243 /* flags bits for collIterate.flags */
244 /* */
245 /* NORM - set for incremental normalize of source string */
246 #define UCOL_ITER_NORM 1
247
248 #define UCOL_ITER_HASLEN 2
249
250 /* UCOL_ITER_INNORMBUF - set if the "pos" is in */
251 /* the writable side buffer, handling */
252 /* incrementally normalized characters. */
253 #define UCOL_ITER_INNORMBUF 4
254
255 /* UCOL_ITER_ALLOCATED - set if this iterator has */
256 /* malloced storage to expand a buffer. */
257 #define UCOL_ITER_ALLOCATED 8
258 /* UCOL_HIRAGANA_Q - note if the codepoint was hiragana */
259 #define UCOL_HIRAGANA_Q 16
260 /* UCOL_WAS_HIRAGANA - set to TRUE if there was a Hiragana */
261 /* otherwise set to false */
262 #define UCOL_WAS_HIRAGANA 32
263 /* UCOL_USE_ITERATOR - set this if collIterate uses a */
264 /* character iterator instead of simply accessing string */
265 /* by index */
266 #define UCOL_USE_ITERATOR 64
267
268 #define UCOL_FORCE_HAN_IMPLICIT 128
269
270 #define NFC_ZERO_CC_BLOCK_LIMIT_ 0x300
271
272 #ifdef __cplusplus
273
274 U_NAMESPACE_BEGIN
275
276 typedef struct collIterate : public UMemory {
277 const UChar *string; /* Original string */
278 /* UChar *start; Pointer to the start of the source string. Either points to string
279 or to writableBuffer */
280 const UChar *endp; /* string end ptr. Is undefined for null terminated strings */
281 const UChar *pos; /* This is position in the string. Can be to original or writable buf */
282
283 uint32_t *toReturn; /* This is the CE from CEs buffer that should be returned */
284 uint32_t *CEpos; /* This is the position to which we have stored processed CEs */
285
286 int32_t *offsetReturn; /* This is the offset to return, if non-NULL */
287 int32_t *offsetStore; /* This is the pointer for storing offsets */
288 int32_t offsetRepeatCount; /* Repeat stored offset if non-zero */
289 int32_t offsetRepeatValue; /* offset value to repeat */
290
291 UnicodeString writableBuffer;
292 const UChar *fcdPosition; /* Position in the original string to continue FCD check from. */
293 const UCollator *coll;
294 const Normalizer2 *nfd;
295 uint8_t flags;
296 uint8_t origFlags;
297 uint32_t *extendCEs; /* This is use if CEs is not big enough */
298 int32_t extendCEsSize; /* Holds the size of the dynamic CEs buffer */
299 uint32_t CEs[UCOL_EXPAND_CE_BUFFER_SIZE]; /* This is where we store CEs */
300
301 int32_t *offsetBuffer; /* A dynamic buffer to hold offsets */
302 int32_t offsetBufferSize; /* The size of the offset buffer */
303
304 UCharIterator *iterator;
305 /*int32_t iteratorIndex;*/
306
307 // The offsetBuffer should probably be a UVector32, but helper functions
308 // are an improvement over duplicated code.
309 void appendOffset(int32_t offset, UErrorCode &errorCode);
310 } collIterate;
311
312 U_NAMESPACE_END
313
314 #else
315
316 typedef struct collIterate collIterate;
317
318 #endif
319
320 #define paddedsize(something) ((something)+((((something)%4)!=0)?(4-(something)%4):0))
321 #define headersize (paddedsize(sizeof(UCATableHeader))+paddedsize(sizeof(UColOptionSet)))
322
323 /*
324 struct used internally in getSpecial*CE.
325 data similar to collIterate.
326 */
327 struct collIterateState {
328 const UChar *pos; /* This is position in the string. Can be to original or writable buf */
329 const UChar *returnPos;
330 const UChar *fcdPosition; /* Position in the original string to continue FCD check from. */
331 const UChar *bufferaddress; /* address of the normalization buffer */
332 int32_t buffersize;
333 uint8_t flags;
334 uint8_t origFlags;
335 uint32_t iteratorIndex;
336 int32_t iteratorMove;
337 };
338
339 U_CAPI void U_EXPORT2
340 uprv_init_collIterate(const UCollator *collator,
341 const UChar *sourceString, int32_t sourceLen,
342 U_NAMESPACE_QUALIFIER collIterate *s, UErrorCode *status);
343
344 /* Internal functions for C test code. */
345 U_CAPI U_NAMESPACE_QUALIFIER collIterate * U_EXPORT2
346 uprv_new_collIterate(UErrorCode *status);
347
348 U_CAPI void U_EXPORT2
349 uprv_delete_collIterate(U_NAMESPACE_QUALIFIER collIterate *s);
350
351 /* @return s->pos == s->endp */
352 U_CAPI UBool U_EXPORT2
353 uprv_collIterateAtEnd(U_NAMESPACE_QUALIFIER collIterate *s);
354
355 #ifdef __cplusplus
356
357 U_NAMESPACE_BEGIN
358
359 struct UCollationPCE;
360 typedef struct UCollationPCE UCollationPCE;
361
362 U_NAMESPACE_END
363
364 struct UCollationElements : public icu::UMemory
365 {
366 /**
367 * Struct wrapper for source data
368 */
369 icu::collIterate iteratordata_;
370 /**
371 * Indicates if this data has been reset.
372 */
373 UBool reset_;
374 /**
375 * Indicates if the data should be deleted.
376 */
377 UBool isWritable;
378
379 /**
380 * Data for getNextProcessed, getPreviousProcessed.
381 */
382 icu::UCollationPCE *pce;
383 };
384
385 #else
386 /*opaque type*/
387 struct UCollationElements;
388 #endif
389
390 U_CAPI void U_EXPORT2
391 uprv_init_pce(const struct UCollationElements *elems);
392
393 #define UCOL_LEVELTERMINATOR 1
394
395 /* mask off anything but primary order */
396 #define UCOL_PRIMARYORDERMASK 0xffff0000
397 /* mask off anything but secondary order */
398 #define UCOL_SECONDARYORDERMASK 0x0000ff00
399 /* mask off anything but tertiary order */
400 #define UCOL_TERTIARYORDERMASK 0x000000ff
401 /* primary order shift */
402 #define UCOL_PRIMARYORDERSHIFT 16
403 /* secondary order shift */
404 #define UCOL_SECONDARYORDERSHIFT 8
405
406 #define UCOL_BYTE_SIZE_MASK 0xFF
407
408 #define UCOL_CASE_BYTE_START 0x80
409 #define UCOL_CASE_SHIFT_START 7
410
411 #define UCOL_IGNORABLE 0
412
413 /* get weights from a CE */
414 #define UCOL_PRIMARYORDER(order) (((order) & UCOL_PRIMARYORDERMASK)>> UCOL_PRIMARYORDERSHIFT)
415 #define UCOL_SECONDARYORDER(order) (((order) & UCOL_SECONDARYORDERMASK)>> UCOL_SECONDARYORDERSHIFT)
416 #define UCOL_TERTIARYORDER(order) ((order) & UCOL_TERTIARYORDERMASK)
417
418 /**
419 * Determine if a character is a Thai vowel (which sorts after
420 * its base consonant).
421 */
422 #define UCOL_ISTHAIPREVOWEL(ch) ((((uint32_t)(ch) - 0xe40) <= (0xe44 - 0xe40)) || \
423 (((uint32_t)(ch) - 0xec0) <= (0xec4 - 0xec0)))
424
425 /**
426 * Determine if a character is a Thai base consonant
427 */
428 #define UCOL_ISTHAIBASECONSONANT(ch) ((uint32_t)(ch) - 0xe01) <= (0xe2e - 0xe01)
429
430 #define UCOL_ISJAMO(ch) ((((uint32_t)(ch) - 0x1100) <= (0x1112 - 0x1100)) || \
431 (((uint32_t)(ch) - 0x1161) <= (0x1175 - 0x1161)) || \
432 (((uint32_t)(ch) - 0x11A8) <= (0x11C2 - 0x11A8)))
433
434 /* Han character ranges */
435 #define UCOL_FIRST_HAN 0x4E00
436 #define UCOL_LAST_HAN 0x9FFF
437 #define UCOL_FIRST_HAN_A 0x3400
438 #define UCOL_LAST_HAN_A 0x4DBF
439 #define UCOL_FIRST_HAN_COMPAT 0xFAE0
440 #define UCOL_LAST_HAN_COMPAT 0xFA2F
441
442 /* Han extension B is in plane 2 */
443 #define UCOL_FIRST_HAN_B 0x20000
444 #define UCOL_LAST_HAN_B 0x2A6DF
445
446 /* Hangul range */
447 #define UCOL_FIRST_HANGUL 0xAC00
448 #define UCOL_LAST_HANGUL 0xD7AF
449
450 /* Jamo ranges */
451 #define UCOL_FIRST_L_JAMO 0x1100
452 #define UCOL_FIRST_V_JAMO 0x1161
453 #define UCOL_FIRST_T_JAMO 0x11A8
454 #define UCOL_LAST_T_JAMO 0x11F9
455
456
457 #if 0
458 /* initializes collIterate structure */
459 /* made as macro to speed up things */
460 #define init_collIterate(collator, sourceString, sourceLen, s) { \
461 (s)->start = (s)->string = (s)->pos = (UChar *)(sourceString); \
462 (s)->endp = (sourceLen) == -1 ? NULL :(UChar *)(sourceString)+(sourceLen); \
463 (s)->CEpos = (s)->toReturn = (s)->CEs; \
464 (s)->isThai = TRUE; \
465 (s)->writableBuffer = (s)->stackWritableBuffer; \
466 (s)->writableBufSize = UCOL_WRITABLE_BUFFER_SIZE; \
467 (s)->coll = (collator); \
468 (s)->fcdPosition = 0; \
469 (s)->flags = 0; \
470 if(((collator)->normalizationMode == UCOL_ON)) (s)->flags |= UCOL_ITER_NORM; \
471 }
472 #endif
473
474
475
476 /*
477 * Macro to get the maximum size of an expansion ending with the argument ce.
478 * Used in the Boyer Moore algorithm.
479 * Note for tailoring, the UCA maxexpansion table has been merged.
480 * Hence we only have to search the tailored collator only.
481 * @param coll const UCollator pointer
482 * @param order last collation element of the expansion sequence
483 * @param result size of the longest expansion with argument collation element
484 * as the last element
485 */
486 #define UCOL_GETMAXEXPANSION(coll, order, result) { \
487 const uint32_t *start; \
488 const uint32_t *limit; \
489 const uint32_t *mid; \
490 start = (coll)->endExpansionCE; \
491 limit = (coll)->lastEndExpansionCE; \
492 while (start < limit - 1) { \
493 mid = start + ((limit - start) >> 1); \
494 if ((order) <= *mid) { \
495 limit = mid; \
496 } \
497 else { \
498 start = mid; \
499 } \
500 } \
501 if (*start == order) { \
502 result = *((coll)->expansionCESize + (start - (coll)->endExpansionCE)); \
503 } \
504 else if (*limit == order) { \
505 result = *(coll->expansionCESize + (limit - coll->endExpansionCE)); \
506 } \
507 else if ((order & 0xFFFF) == 0x00C0) { \
508 result = 2; \
509 } \
510 else { \
511 result = 1; \
512 } \
513 }
514
515 U_CFUNC
516 uint32_t ucol_prv_getSpecialCE(const UCollator *coll, UChar ch, uint32_t CE,
517 U_NAMESPACE_QUALIFIER collIterate *source, UErrorCode *status);
518
519 U_CFUNC
520 uint32_t ucol_prv_getSpecialPrevCE(const UCollator *coll, UChar ch, uint32_t CE,
521 U_NAMESPACE_QUALIFIER collIterate *source, UErrorCode *status);
522 U_CAPI uint32_t U_EXPORT2 ucol_getNextCE(const UCollator *coll,
523 U_NAMESPACE_QUALIFIER collIterate *collationSource, UErrorCode *status);
524 U_CFUNC uint32_t U_EXPORT2 ucol_getPrevCE(const UCollator *coll,
525 U_NAMESPACE_QUALIFIER collIterate *collationSource,
526 UErrorCode *status);
527 /* get some memory */
528 void *ucol_getABuffer(const UCollator *coll, uint32_t size);
529
530 #ifdef __cplusplus
531
532 U_NAMESPACE_BEGIN
533
534 class CollationKey;
535 class SortKeyByteSink;
536
537 U_NAMESPACE_END
538
539 /* function used by C++ getCollationKey to prevent restarting the calculation */
540 U_CFUNC int32_t
541 ucol_getCollationKey(const UCollator *coll,
542 const UChar *source, int32_t sourceLength,
543 icu::CollationKey &key,
544 UErrorCode &errorCode);
545
546 typedef void U_CALLCONV
547 SortKeyGenerator(const UCollator *coll,
548 const UChar *source,
549 int32_t sourceLength,
550 icu::SortKeyByteSink &result,
551 UErrorCode *status);
552
553 /* worker function for generating sortkeys */
554 U_CFUNC
555 void U_CALLCONV
556 ucol_calcSortKey(const UCollator *coll,
557 const UChar *source,
558 int32_t sourceLength,
559 icu::SortKeyByteSink &result,
560 UErrorCode *status);
561
562 U_CFUNC
563 void U_CALLCONV
564 ucol_calcSortKeySimpleTertiary(const UCollator *coll,
565 const UChar *source,
566 int32_t sourceLength,
567 icu::SortKeyByteSink &result,
568 UErrorCode *status);
569
570 #else
571
572 typedef void U_CALLCONV
573 SortKeyGenerator(const UCollator *coll,
574 const UChar *source,
575 int32_t sourceLength,
576 void *result,
577 UErrorCode *status);
578
579 #endif
580
581 /**
582 * Makes a copy of the Collator's rule data. The format is
583 * that of .col files.
584 *
585 * @param length returns the length of the data, in bytes.
586 * @param status the error status
587 * @return memory, owned by the caller, of size 'length' bytes.
588 * @internal INTERNAL USE ONLY
589 */
590 U_CFUNC uint8_t* U_EXPORT2
591 ucol_cloneRuleData(const UCollator *coll, int32_t *length, UErrorCode *status);
592
593 /**
594 * Used to set requested and valid locales on a collator returned by the collator
595 * service.
596 */
597 U_CFUNC void U_EXPORT2
598 ucol_setReqValidLocales(UCollator *coll, char *requestedLocaleToAdopt, char *validLocaleToAdopt, char *actualLocaleToAdopt);
599
600 #define UCOL_SPECIAL_FLAG 0xF0000000
601 #define UCOL_TAG_SHIFT 24
602 #define UCOL_TAG_MASK 0x0F000000
603 #define INIT_EXP_TABLE_SIZE 1024
604 #define UCOL_NOT_FOUND 0xF0000000
605 #define UCOL_EXPANSION 0xF1000000
606 #define UCOL_CONTRACTION 0xF2000000
607 #define UCOL_THAI 0xF3000000
608 #define UCOL_UNMARKED 0x03
609 #define UCOL_NEW_TERTIARYORDERMASK 0x0000003f
610
611 /* Bit mask for primary collation strength. */
612 #define UCOL_PRIMARYMASK 0xFFFF0000
613
614 /* Bit mask for secondary collation strength. */
615 #define UCOL_SECONDARYMASK 0x0000FF00
616
617 /* Bit mask for tertiary collation strength. */
618 #define UCOL_TERTIARYMASK 0x000000FF
619
620 /**
621 * Internal.
622 * This indicates the last element in a UCollationElements has been consumed.
623 * Compare with the UCOL_NULLORDER, UCOL_NULLORDER is returned if error occurs.
624 */
625 #define UCOL_NO_MORE_CES 0x00010101
626 #define UCOL_NO_MORE_CES_PRIMARY 0x00010000
627 #define UCOL_NO_MORE_CES_SECONDARY 0x00000100
628 #define UCOL_NO_MORE_CES_TERTIARY 0x00000001
629
630 #define isSpecial(CE) ((((CE)&UCOL_SPECIAL_FLAG)>>28)==0xF)
631
632 #define UCOL_UPPER_CASE 0x80
633 #define UCOL_MIXED_CASE 0x40
634 #define UCOL_LOWER_CASE 0x00
635
636 #define UCOL_CONTINUATION_MARKER 0xC0
637 #define UCOL_REMOVE_CONTINUATION 0xFFFFFF3F
638
639 #define isContinuation(CE) (((CE) & UCOL_CONTINUATION_MARKER) == UCOL_CONTINUATION_MARKER)
640 #define isFlagged(CE) (((CE) & 0x80) == 0x80)
641 #define isLongPrimary(CE) (((CE) & 0xC0) == 0xC0)
642
643 #define getCETag(CE) (((CE)&UCOL_TAG_MASK)>>UCOL_TAG_SHIFT)
644 #define isContraction(CE) (isSpecial((CE)) && (getCETag((CE)) == CONTRACTION_TAG))
645 #define isPrefix(CE) (isSpecial((CE)) && (getCETag((CE)) == SPEC_PROC_TAG))
646 #define constructContractCE(tag, CE) (UCOL_SPECIAL_FLAG | ((tag)<<UCOL_TAG_SHIFT) | ((CE)&0xFFFFFF))
647 #define constructSpecProcCE(CE) (UCOL_SPECIAL_FLAG | (SPEC_PROC_TAG<<UCOL_TAG_SHIFT) | ((CE)&0xFFFFFF))
648 #define getContractOffset(CE) ((CE)&0xFFFFFF)
649 #define getExpansionOffset(CE) (((CE)&0x00FFFFF0)>>4)
650 #define getExpansionCount(CE) ((CE)&0xF)
651 #define isCEIgnorable(CE) (((CE) & 0xFFFFFFBF) == 0)
652
653 /* StringSearch internal use */
654 #define inNormBuf(coleiter) ((coleiter)->iteratordata_.flags & UCOL_ITER_INNORMBUF)
655 #define isFCDPointerNull(coleiter) ((coleiter)->iteratordata_.fcdPosition == NULL)
656 #define hasExpansion(coleiter) ((coleiter)->iteratordata_.CEpos != (coleiter)->iteratordata_.CEs)
657 #define getExpansionPrefix(coleiter) ((coleiter)->iteratordata_.toReturn - (coleiter)->iteratordata_.CEs)
658 #define setExpansionPrefix(coleiter, offset) ((coleiter)->iteratordata_.CEs + offset)
659 #define getExpansionSuffix(coleiter) ((coleiter)->iteratordata_.CEpos - (coleiter)->iteratordata_.toReturn)
660 #define setExpansionSuffix(coleiter, offset) ((coleiter)->iteratordata_.toReturn = (coleiter)->iteratordata_.CEpos - leftoverces)
661
662 /* This is an enum that lists magic special byte values from the fractional UCA.
663 * See also http://site.icu-project.org/design/collation/bytes */
664 /* TODO: all the #defines that refer to special byte values from the UCA should be changed to point here */
665
666 enum {
667 UCOL_BYTE_ZERO = 0x00,
668 UCOL_BYTE_LEVEL_SEPARATOR = 0x01,
669 UCOL_BYTE_SORTKEY_GLUE = 0x02,
670 UCOL_BYTE_SHIFT_PREFIX = 0x03,
671 UCOL_BYTE_UNSHIFTED_MIN = UCOL_BYTE_SHIFT_PREFIX,
672 UCOL_BYTE_FIRST_TAILORED = 0x04,
673 UCOL_BYTE_COMMON = 0x05,
674 UCOL_BYTE_FIRST_UCA = UCOL_BYTE_COMMON,
675 /* TODO: Make the following values dynamic since they change with almost every UCA version. */
676 UCOL_CODAN_PLACEHOLDER = 0x12,
677 UCOL_BYTE_FIRST_NON_LATIN_PRIMARY = 0x5B,
678 UCOL_BYTE_UNSHIFTED_MAX = 0xFF
679 };
680
681 #if 0
682 #define UCOL_RESET_TOP_VALUE 0x9F000303
683 #define UCOL_FIRST_PRIMARY_IGNORABLE 0x00008705
684 #define UCOL_LAST_PRIMARY_IGNORABLE 0x0000DD05
685 #define UCOL_LAST_PRIMARY_IGNORABLE_CONT 0x000051C0
686 #define UCOL_FIRST_SECONDARY_IGNORABLE 0x00000000
687 #define UCOL_LAST_SECONDARY_IGNORABLE 0x00000500
688 #define UCOL_FIRST_TERTIARY_IGNORABLE 0x00000000
689 #define UCOL_LAST_TERTIARY_IGNORABLE 0x00000000
690 #define UCOL_FIRST_VARIABLE 0x05070505
691 #define UCOL_LAST_VARIABLE 0x179B0505
692 #define UCOL_FIRST_NON_VARIABLE 0x1A200505
693 #define UCOL_LAST_NON_VARIABLE 0x7B41058F
694
695 #define UCOL_NEXT_TOP_VALUE 0xE8960303
696 #define UCOL_NEXT_FIRST_PRIMARY_IGNORABLE 0x00008905
697 #define UCOL_NEXT_LAST_PRIMARY_IGNORABLE 0x03000303
698 #define UCOL_NEXT_FIRST_SECONDARY_IGNORABLE 0x00008705
699 #define UCOL_NEXT_LAST_SECONDARY_IGNORABLE 0x00000500
700 #define UCOL_NEXT_FIRST_TERTIARY_IGNORABLE 0x00000000
701 #define UCOL_NEXT_LAST_TERTIARY_IGNORABLE 0x00000000
702 #define UCOL_NEXT_FIRST_VARIABLE 0x05090505
703 #define UCOL_NEXT_LAST_VARIABLE 0x1A200505
704
705 #define PRIMARY_IMPLICIT_MIN 0xE8000000
706 #define PRIMARY_IMPLICIT_MAX 0xF0000000
707 #endif
708
709 /* These constants can be changed - sortkey size is affected by them */
710 #define UCOL_PROPORTION2 0.5
711 #define UCOL_PROPORTION3 0.667
712
713 /* These values come from the UCA */
714 #define UCOL_COMMON_BOT2 UCOL_BYTE_COMMON
715 #define UCOL_COMMON_TOP2 0x86u
716 #define UCOL_TOTAL2 (UCOL_COMMON_TOP2-UCOL_COMMON_BOT2-1)
717
718 #define UCOL_FLAG_BIT_MASK_CASE_SW_OFF 0x80
719 #define UCOL_FLAG_BIT_MASK_CASE_SW_ON 0x40
720 #define UCOL_COMMON_TOP3_CASE_SW_OFF 0x85
721 #define UCOL_COMMON_TOP3_CASE_SW_LOWER 0x45
722 #define UCOL_COMMON_TOP3_CASE_SW_UPPER 0xC5
723
724 /* These values come from the UCA */
725 #define UCOL_COMMON_BOT3 0x05
726
727 #define UCOL_COMMON_BOTTOM3_CASE_SW_UPPER 0x86;
728 #define UCOL_COMMON_BOTTOM3_CASE_SW_LOWER UCOL_COMMON_BOT3;
729
730 #define UCOL_TOP_COUNT2 (UCOL_PROPORTION2*UCOL_TOTAL2)
731 #define UCOL_BOT_COUNT2 (UCOL_TOTAL2-UCOL_TOP_COUNT2)
732
733
734 #define UCOL_COMMON2 UCOL_COMMON_BOT2
735 #define UCOL_COMMON3_UPPERFIRST 0xC5
736 #define UCOL_COMMON3_NORMAL UCOL_COMMON_BOT3
737
738 #define UCOL_COMMON4 0xFF
739
740 /* constants for case level/case first handling */
741 /* used to instantiate UCollators fields in ucol_updateInternalState */
742 #define UCOL_CASE_SWITCH 0xC0
743 #define UCOL_NO_CASE_SWITCH 0x00
744
745 #define UCOL_REMOVE_CASE 0x3F
746 #define UCOL_KEEP_CASE 0xFF
747
748 #define UCOL_CASE_BIT_MASK 0xC0
749
750 #define UCOL_TERT_CASE_MASK 0xFF
751
752 #define UCOL_ENDOFLATINONERANGE 0xFF
753 #define UCOL_LATINONETABLELEN (UCOL_ENDOFLATINONERANGE+50)
754 #define UCOL_BAIL_OUT_CE 0xFF000000
755
756
757 typedef enum {
758 NOT_FOUND_TAG = 0,
759 EXPANSION_TAG = 1, /* This code point results in an expansion */
760 CONTRACTION_TAG = 2, /* Start of a contraction */
761 THAI_TAG = 3, /* Thai character - do the reordering */
762 CHARSET_TAG = 4, /* Charset processing, not yet implemented */
763 SURROGATE_TAG = 5, /* Lead surrogate that is tailored and doesn't start a contraction */
764 HANGUL_SYLLABLE_TAG = 6, /* AC00-D7AF*/
765 LEAD_SURROGATE_TAG = 7, /* D800-DBFF*/
766 TRAIL_SURROGATE_TAG = 8, /* DC00-DFFF*/
767 CJK_IMPLICIT_TAG = 9, /* 0x3400-0x4DB5, 0x4E00-0x9FA5, 0xF900-0xFA2D*/
768 IMPLICIT_TAG = 10,
769 SPEC_PROC_TAG = 11,
770 /* ICU 2.1 */
771 LONG_PRIMARY_TAG = 12, /* This is a three byte primary with starting secondaries and tertiaries */
772 /* It fits in a single 32 bit CE and is used instead of expansion to save */
773 /* space without affecting the performance (hopefully) */
774
775 DIGIT_TAG = 13, /* COllate Digits As Numbers (CODAN) implementation */
776
777 CE_TAGS_COUNT
778 } UColCETags;
779
780 /*
781 *****************************************************************************************
782 * set to zero
783 * NON_CHARACTER FDD0 - FDEF, FFFE, FFFF, 1FFFE, 1FFFF, 2FFFE, 2FFFF,...e.g. **FFFE, **FFFF
784 ******************************************************************************************
785 */
786
787 typedef struct {
788 uint32_t variableTopValue;
789 /*UColAttributeValue*/ int32_t frenchCollation;
790 /*UColAttributeValue*/ int32_t alternateHandling; /* attribute for handling variable elements*/
791 /*UColAttributeValue*/ int32_t caseFirst; /* who goes first, lower case or uppercase */
792 /*UColAttributeValue*/ int32_t caseLevel; /* do we have an extra case level */
793 /*UColAttributeValue*/ int32_t normalizationMode; /* attribute for normalization */
794 /*UColAttributeValue*/ int32_t strength; /* attribute for strength */
795 /*UColAttributeValue*/ int32_t hiraganaQ; /* attribute for special Hiragana */
796 /*UColAttributeValue*/ int32_t numericCollation; /* attribute for numeric collation */
797 uint32_t reserved[15]; /* for future use */
798 } UColOptionSet;
799
800 typedef struct {
801 uint32_t UCA_FIRST_TERTIARY_IGNORABLE[2]; /*0x00000000*/
802 uint32_t UCA_LAST_TERTIARY_IGNORABLE[2]; /*0x00000000*/
803 uint32_t UCA_FIRST_PRIMARY_IGNORABLE[2]; /*0x00008705*/
804 uint32_t UCA_FIRST_SECONDARY_IGNORABLE[2]; /*0x00000000*/
805 uint32_t UCA_LAST_SECONDARY_IGNORABLE[2]; /*0x00000500*/
806 uint32_t UCA_LAST_PRIMARY_IGNORABLE[2]; /*0x0000DD05*/
807 uint32_t UCA_FIRST_VARIABLE[2]; /*0x05070505*/
808 uint32_t UCA_LAST_VARIABLE[2]; /*0x13CF0505*/
809 uint32_t UCA_FIRST_NON_VARIABLE[2]; /*0x16200505*/
810 uint32_t UCA_LAST_NON_VARIABLE[2]; /*0x767C0505*/
811 uint32_t UCA_RESET_TOP_VALUE[2]; /*0x9F000303*/
812 uint32_t UCA_FIRST_IMPLICIT[2];
813 uint32_t UCA_LAST_IMPLICIT[2];
814 uint32_t UCA_FIRST_TRAILING[2];
815 uint32_t UCA_LAST_TRAILING[2];
816
817 #if 0
818 uint32_t UCA_NEXT_TOP_VALUE[2]; /*0xE8960303*/
819 uint32_t UCA_NEXT_FIRST_PRIMARY_IGNORABLE; /*0x00008905*/
820 uint32_t UCA_NEXT_LAST_PRIMARY_IGNORABLE; /*0x03000303*/
821 uint32_t UCA_NEXT_FIRST_SECONDARY_IGNORABLE; /*0x00008705*/
822 uint32_t UCA_NEXT_LAST_SECONDARY_IGNORABLE; /*0x00000500*/
823 uint32_t UCA_NEXT_FIRST_TERTIARY_IGNORABLE; /*0x00000000*/
824 uint32_t UCA_NEXT_LAST_TERTIARY_IGNORABLE; /*0x00000000*/
825 uint32_t UCA_NEXT_FIRST_VARIABLE; /*0x05090505*/
826 uint32_t UCA_NEXT_LAST_VARIABLE; /*0x16200505*/
827 #endif
828
829 uint32_t UCA_PRIMARY_TOP_MIN;
830 uint32_t UCA_PRIMARY_IMPLICIT_MIN; /*0xE8000000*/
831 uint32_t UCA_PRIMARY_IMPLICIT_MAX; /*0xF0000000*/
832 uint32_t UCA_PRIMARY_TRAILING_MIN; /*0xE8000000*/
833 uint32_t UCA_PRIMARY_TRAILING_MAX; /*0xF0000000*/
834 uint32_t UCA_PRIMARY_SPECIAL_MIN; /*0xE8000000*/
835 uint32_t UCA_PRIMARY_SPECIAL_MAX; /*0xF0000000*/
836 } UCAConstants;
837
838 /* definition of UCATableHeader moved to common/ucol_data.h */
839
840 #define U_UNKNOWN_STATE 0
841 #define U_COLLATOR_STATE 0x01
842 #define U_STATE_LIMIT 0x02
843
844 /* This is the first structure in a state */
845 /* it should be machine independent */
846 typedef struct {
847 /* this structure is supposed to be readable on all the platforms.*/
848 /* first 2 fields hold the size of the structure in a platform independent way */
849 uint8_t sizeLo;
850 uint8_t sizeHi;
851 /* identifying the writing platform */
852 uint8_t isBigEndian;
853 /* see U_CHARSET_FAMILY values in utypes.h */
854 uint8_t charsetFamily;
855 /* version of ICU this state structure comes from */
856 uint8_t icuVersion[4];
857 /* What is the data following this state */
858 uint8_t type;
859 /* more stuff to come, keep it on 16 byte boundary */
860 uint8_t reserved[7];
861 } UStateStruct;
862
863 /* This structure follows UStatusStruct */
864 /* and contains data specific for the collators */
865 /* Endianess needs to be decided before accessing this structure */
866 /* However, it's size IS endianess independent */
867 typedef struct {
868 /* size of this structure */
869 uint8_t sizeLo;
870 uint8_t sizeHi;
871 /* This state is followed by the frozen tailoring */
872 uint8_t containsTailoring;
873 /* This state is followed by the frozen UCA */
874 uint8_t containsUCA;
875 /* Version info - the same one */
876 uint8_t versionInfo[4];
877
878 /* for charset CEs */
879 uint8_t charsetName[32];
880 /* this is the resolved locale name*/
881 uint8_t locale[32];
882
883 /* Attributes. Open ended */
884 /* all the following will be moved to uint32_t because of portability */
885 /* variable top value */
886 uint32_t variableTopValue;
887 /* attribute for handling variable elements*/
888 uint32_t /*UColAttributeValue*/ alternateHandling;
889 /* how to handle secondary weights */
890 uint32_t /*UColAttributeValue*/ frenchCollation;
891 /* who goes first, lower case or uppercase */
892 uint32_t /*UColAttributeValue*/ caseFirst;
893 /* do we have an extra case level */
894 uint32_t /*UColAttributeValue*/ caseLevel;
895 /* attribute for normalization */
896 uint32_t /*UColAttributeValue*/ normalizationMode;
897 /* attribute for strength */
898 uint32_t /*UColAttributeValue*/ strength;
899 /* to be immediately 16 byte aligned */
900 uint8_t reserved[12];
901 } UColStateStruct;
902
903 #define UCOL_INV_SIZEMASK 0xFFF00000
904 #define UCOL_INV_OFFSETMASK 0x000FFFFF
905 #define UCOL_INV_SHIFTVALUE 20
906
907 U_CDECL_BEGIN
908
909 /* definition of InverseUCATableHeader moved to common/ucol_data.h */
910
911 typedef void U_CALLCONV
912 ResourceCleaner(UCollator *coll);
913
914
915 struct UCollator {
916 UColOptionSet *options;
917 SortKeyGenerator *sortKeyGen;
918 uint32_t *latinOneCEs;
919 char* actualLocale;
920 char* validLocale;
921 char* requestedLocale;
922 const UChar *rules;
923 const UChar *ucaRules;
924 const UCollator *UCA;
925 const UCATableHeader *image;
926 UTrie mapping;
927 const uint32_t *latinOneMapping;
928 const uint32_t *expansion;
929 const UChar *contractionIndex;
930 const uint32_t *contractionCEs;
931
932 const uint32_t *endExpansionCE; /* array of last ces in an expansion ce.
933 corresponds to expansionCESize */
934 const uint32_t *lastEndExpansionCE;/* pointer to the last element in endExpansionCE */
935 const uint8_t *expansionCESize; /* array of the maximum size of a
936 expansion ce with the last ce
937 corresponding to endExpansionCE,
938 terminated with a null */
939 const uint8_t *unsafeCP; /* unsafe code points hashtable */
940 const uint8_t *contrEndCP; /* Contraction ending chars hash table */
941 UChar minUnsafeCP; /* Smallest unsafe Code Point. */
942 UChar minContrEndCP; /* Smallest code point at end of a contraction */
943
944 int32_t rulesLength;
945 int32_t latinOneTableLen;
946
947 uint32_t variableTopValue;
948 UColAttributeValue frenchCollation;
949 UColAttributeValue alternateHandling; /* attribute for handling variable elements*/
950 UColAttributeValue caseFirst; /* who goes first, lower case or uppercase */
951 UColAttributeValue caseLevel; /* do we have an extra case level */
952 UColAttributeValue normalizationMode; /* attribute for normalization */
953 UColAttributeValue strength; /* attribute for strength */
954 UColAttributeValue hiraganaQ; /* attribute for Hiragana */
955 UColAttributeValue numericCollation;
956 UBool variableTopValueisDefault;
957 UBool frenchCollationisDefault;
958 UBool alternateHandlingisDefault; /* attribute for handling variable elements*/
959 UBool caseFirstisDefault; /* who goes first, lower case or uppercase */
960 UBool caseLevelisDefault; /* do we have an extra case level */
961 UBool normalizationModeisDefault; /* attribute for normalization */
962 UBool strengthisDefault; /* attribute for strength */
963 UBool hiraganaQisDefault; /* attribute for Hiragana */
964 UBool numericCollationisDefault;
965 UBool hasRealData; /* some collators have only options, like French, no rules */
966 /* to speed up things, we use the UCA image, but we don't want it */
967 /* to run around */
968
969 UBool freeOnClose;
970 UBool freeOptionsOnClose;
971 UBool freeRulesOnClose;
972 UBool freeImageOnClose;
973 UBool freeDefaultReorderCodesOnClose;
974 UBool freeReorderCodesOnClose;
975 UBool freeLeadBytePermutationTableOnClose;
976
977 UBool latinOneUse;
978 UBool latinOneRegenTable;
979 UBool latinOneFailed;
980
981 int8_t tertiaryAddition; /* when switching case, we need to add or subtract different values */
982 uint8_t caseSwitch;
983 uint8_t tertiaryCommon;
984 uint8_t tertiaryMask;
985 uint8_t tertiaryTop; /* Upper range when compressing */
986 uint8_t tertiaryBottom; /* Upper range when compressing */
987 uint8_t tertiaryTopCount;
988 uint8_t tertiaryBottomCount;
989
990 UVersionInfo dataVersion; /* Data info of UCA table */
991 int32_t* defaultReorderCodes;
992 int32_t defaultReorderCodesLength;
993 int32_t* reorderCodes;
994 int32_t reorderCodesLength;
995 uint8_t* leadBytePermutationTable;
996 void *delegate; /* if non-null: C++ object to delegate all API calls to. */
997 };
998
999 U_CDECL_END
1000
1001 /* various internal functions */
1002
1003 /* do not close UCA returned by ucol_initUCA! */
1004 U_CFUNC
1005 UCollator* ucol_initUCA(UErrorCode *status);
1006
1007 U_CFUNC
1008 UCollator* ucol_initCollator(const UCATableHeader *image, UCollator *fillIn, const UCollator *UCA, UErrorCode *status);
1009
1010 U_CFUNC
1011 void ucol_setOptionsFromHeader(UCollator* result, UColOptionSet * opts, UErrorCode *status);
1012
1013 U_CFUNC
1014 UCollator* ucol_open_internal(const char* loc, UErrorCode* status);
1015
1016 #if 0
1017 U_CFUNC
1018 void ucol_putOptionsToHeader(UCollator* result, UColOptionSet * opts, UErrorCode *status);
1019 #endif
1020
1021 U_CFUNC
1022 void ucol_updateInternalState(UCollator *coll, UErrorCode *status);
1023
1024 U_CFUNC uint32_t U_EXPORT2 ucol_getFirstCE(const UCollator *coll, UChar u, UErrorCode *status);
1025 U_CAPI UBool U_EXPORT2 ucol_isTailored(const UCollator *coll, const UChar u, UErrorCode *status);
1026
1027 U_CAPI const InverseUCATableHeader* U_EXPORT2 ucol_initInverseUCA(UErrorCode *status);
1028
1029 U_CAPI void U_EXPORT2
1030 uprv_uca_initImplicitConstants(UErrorCode *status);
1031
1032 U_CAPI uint32_t U_EXPORT2
1033 uprv_uca_getImplicitFromRaw(UChar32 cp);
1034
1035 /*U_CFUNC uint32_t U_EXPORT2
1036 uprv_uca_getImplicitPrimary(UChar32 cp);*/
1037
1038 U_CAPI UChar32 U_EXPORT2
1039 uprv_uca_getRawFromImplicit(uint32_t implicit);
1040
1041 U_CAPI UChar32 U_EXPORT2
1042 uprv_uca_getRawFromCodePoint(UChar32 i);
1043
1044 U_CAPI UChar32 U_EXPORT2
1045 uprv_uca_getCodePointFromRaw(UChar32 i);
1046
1047 typedef const UChar* GetCollationRulesFunction(void* context, const char* locale, const char* type, int32_t* pLength, UErrorCode* status);
1048
1049 U_CAPI UCollator* U_EXPORT2
1050 ucol_openRulesForImport( const UChar *rules,
1051 int32_t rulesLength,
1052 UColAttributeValue normalizationMode,
1053 UCollationStrength strength,
1054 UParseError *parseError,
1055 GetCollationRulesFunction importFunc,
1056 void* context,
1057 UErrorCode *status);
1058
1059
1060 U_CFUNC void U_EXPORT2
1061 ucol_buildPermutationTable(UCollator *coll, UErrorCode *status);
1062
1063 U_CFUNC int U_EXPORT2
1064 ucol_getLeadBytesForReorderCode(const UCollator *uca, int reorderCode, uint16_t* returnLeadBytes, int returnCapacity);
1065
1066 U_CFUNC int U_EXPORT2
1067 ucol_getReorderCodesForLeadByte(const UCollator *uca, int leadByte, int16_t* returnReorderCodes, int returnCapacity);
1068
1069 #ifdef __cplusplus
1070 /*
1071 * Test whether a character is potentially "unsafe" for use as a collation
1072 * starting point. Unsafe chars are those with combining class != 0 plus
1073 * those that are the 2nd thru nth character in a contraction sequence.
1074 *
1075 * Function is in header file because it's used in both collation and string search,
1076 * and needs to be inline for performance.
1077 */
ucol_unsafeCP(UChar c,const UCollator * coll)1078 static inline UBool ucol_unsafeCP(UChar c, const UCollator *coll) {
1079 int32_t hash;
1080 uint8_t htbyte;
1081
1082 if (c < coll->minUnsafeCP) {
1083 return FALSE;
1084 }
1085
1086 hash = c;
1087 if (hash >= UCOL_UNSAFECP_TABLE_SIZE*8) {
1088 if(U16_IS_SURROGATE(c)) {
1089 /* Lead or trail surrogate */
1090 /* These are always considered unsafe. */
1091 return TRUE;
1092 }
1093 hash = (hash & UCOL_UNSAFECP_TABLE_MASK) + 256;
1094 }
1095 htbyte = coll->unsafeCP[hash>>3];
1096 return ((htbyte >> (hash & 7)) & 1);
1097 }
1098 #endif /* __cplusplus */
1099
1100 /* The offsetBuffer in collIterate might need to be freed to avoid memory leaks. */
1101 void ucol_freeOffsetBuffer(U_NAMESPACE_QUALIFIER collIterate *s);
1102
1103 #endif /* #if !UCONFIG_NO_COLLATION */
1104
1105 #endif
1106