1 // © 2016 and later: Unicode, Inc. and others. 2 // License & terms of use: http://www.unicode.org/copyright.html 3 /* 4 ******************************************************************************* 5 * Copyright (c) 1996-2016, International Business Machines Corporation 6 * and others. All Rights Reserved. 7 ******************************************************************************* 8 * File unorm.h 9 * 10 * Created by: Vladimir Weinstein 12052000 11 * 12 * Modification history : 13 * 14 * Date Name Description 15 * 02/01/01 synwee Added normalization quickcheck enum and method. 16 */ 17 #ifndef UNORM_H 18 #define UNORM_H 19 20 #include "unicode/utypes.h" 21 22 #if !UCONFIG_NO_NORMALIZATION 23 24 #include "unicode/uiter.h" 25 #include "unicode/unorm2.h" 26 27 /** 28 * \file 29 * \brief C API: Unicode Normalization 30 * 31 * Old Unicode normalization API. 32 * 33 * This API has been replaced by the unorm2.h API and is only available 34 * for backward compatibility. The functions here simply delegate to the 35 * unorm2.h functions, for example unorm2_getInstance() and unorm2_normalize(). 36 * There is one exception: The new API does not provide a replacement for unorm_compare(). 37 * Its declaration has been moved to unorm2.h. 38 * 39 * <code>unorm_normalize</code> transforms Unicode text into an equivalent composed or 40 * decomposed form, allowing for easier sorting and searching of text. 41 * <code>unorm_normalize</code> supports the standard normalization forms described in 42 * <a href="http://www.unicode.org/unicode/reports/tr15/" target="unicode"> 43 * Unicode Standard Annex #15: Unicode Normalization Forms</a>. 44 * 45 * Characters with accents or other adornments can be encoded in 46 * several different ways in Unicode. For example, take the character A-acute. 47 * In Unicode, this can be encoded as a single character (the 48 * "composed" form): 49 * 50 * \code 51 * 00C1 LATIN CAPITAL LETTER A WITH ACUTE 52 * \endcode 53 * 54 * or as two separate characters (the "decomposed" form): 55 * 56 * \code 57 * 0041 LATIN CAPITAL LETTER A 58 * 0301 COMBINING ACUTE ACCENT 59 * \endcode 60 * 61 * To a user of your program, however, both of these sequences should be 62 * treated as the same "user-level" character "A with acute accent". When you are searching or 63 * comparing text, you must ensure that these two sequences are treated 64 * equivalently. In addition, you must handle characters with more than one 65 * accent. Sometimes the order of a character's combining accents is 66 * significant, while in other cases accent sequences in different orders are 67 * really equivalent. 68 * 69 * Similarly, the string "ffi" can be encoded as three separate letters: 70 * 71 * \code 72 * 0066 LATIN SMALL LETTER F 73 * 0066 LATIN SMALL LETTER F 74 * 0069 LATIN SMALL LETTER I 75 * \endcode 76 * 77 * or as the single character 78 * 79 * \code 80 * FB03 LATIN SMALL LIGATURE FFI 81 * \endcode 82 * 83 * The ffi ligature is not a distinct semantic character, and strictly speaking 84 * it shouldn't be in Unicode at all, but it was included for compatibility 85 * with existing character sets that already provided it. The Unicode standard 86 * identifies such characters by giving them "compatibility" decompositions 87 * into the corresponding semantic characters. When sorting and searching, you 88 * will often want to use these mappings. 89 * 90 * <code>unorm_normalize</code> helps solve these problems by transforming text into the 91 * canonical composed and decomposed forms as shown in the first example above. 92 * In addition, you can have it perform compatibility decompositions so that 93 * you can treat compatibility characters the same as their equivalents. 94 * Finally, <code>unorm_normalize</code> rearranges accents into the proper canonical 95 * order, so that you do not have to worry about accent rearrangement on your 96 * own. 97 * 98 * Form FCD, "Fast C or D", is also designed for collation. 99 * It allows to work on strings that are not necessarily normalized 100 * with an algorithm (like in collation) that works under "canonical closure", i.e., it treats precomposed 101 * characters and their decomposed equivalents the same. 102 * 103 * It is not a normalization form because it does not provide for uniqueness of representation. Multiple strings 104 * may be canonically equivalent (their NFDs are identical) and may all conform to FCD without being identical 105 * themselves. 106 * 107 * The form is defined such that the "raw decomposition", the recursive canonical decomposition of each character, 108 * results in a string that is canonically ordered. This means that precomposed characters are allowed for as long 109 * as their decompositions do not need canonical reordering. 110 * 111 * Its advantage for a process like collation is that all NFD and most NFC texts - and many unnormalized texts - 112 * already conform to FCD and do not need to be normalized (NFD) for such a process. The FCD quick check will 113 * return UNORM_YES for most strings in practice. 114 * 115 * unorm_normalize(UNORM_FCD) may be implemented with UNORM_NFD. 116 * 117 * For more details on FCD see the collation design document: 118 * http://source.icu-project.org/repos/icu/icuhtml/trunk/design/collation/ICU_collation_design.htm 119 * 120 * ICU collation performs either NFD or FCD normalization automatically if normalization 121 * is turned on for the collator object. 122 * Beyond collation and string search, normalized strings may be useful for string equivalence comparisons, 123 * transliteration/transcription, unique representations, etc. 124 * 125 * The W3C generally recommends to exchange texts in NFC. 126 * Note also that most legacy character encodings use only precomposed forms and often do not 127 * encode any combining marks by themselves. For conversion to such character encodings the 128 * Unicode text needs to be normalized to NFC. 129 * For more usage examples, see the Unicode Standard Annex. 130 */ 131 132 // Do not conditionalize the following enum with #ifndef U_HIDE_DEPRECATED_API, 133 // it is needed for layout of Normalizer object. 134 /** 135 * Constants for normalization modes. 136 * @deprecated ICU 56 Use unorm2.h instead. 137 */ 138 typedef enum { 139 /** No decomposition/composition. @deprecated ICU 56 Use unorm2.h instead. */ 140 UNORM_NONE = 1, 141 /** Canonical decomposition. @deprecated ICU 56 Use unorm2.h instead. */ 142 UNORM_NFD = 2, 143 /** Compatibility decomposition. @deprecated ICU 56 Use unorm2.h instead. */ 144 UNORM_NFKD = 3, 145 /** Canonical decomposition followed by canonical composition. @deprecated ICU 56 Use unorm2.h instead. */ 146 UNORM_NFC = 4, 147 /** Default normalization. @deprecated ICU 56 Use unorm2.h instead. */ 148 UNORM_DEFAULT = UNORM_NFC, 149 /** Compatibility decomposition followed by canonical composition. @deprecated ICU 56 Use unorm2.h instead. */ 150 UNORM_NFKC =5, 151 /** "Fast C or D" form. @deprecated ICU 56 Use unorm2.h instead. */ 152 UNORM_FCD = 6, 153 154 /** One more than the highest normalization mode constant. @deprecated ICU 56 Use unorm2.h instead. */ 155 UNORM_MODE_COUNT 156 } UNormalizationMode; 157 158 #ifndef U_HIDE_DEPRECATED_API 159 160 /** 161 * Constants for options flags for normalization. 162 * Use 0 for default options, 163 * including normalization according to the Unicode version 164 * that is currently supported by ICU (see u_getUnicodeVersion). 165 * @deprecated ICU 56 Use unorm2.h instead. 166 */ 167 enum { 168 /** 169 * Options bit set value to select Unicode 3.2 normalization 170 * (except NormalizationCorrections). 171 * At most one Unicode version can be selected at a time. 172 * @deprecated ICU 56 Use unorm2.h instead. 173 */ 174 UNORM_UNICODE_3_2=0x20 175 }; 176 177 /** 178 * Lowest-order bit number of unorm_compare() options bits corresponding to 179 * normalization options bits. 180 * 181 * The options parameter for unorm_compare() uses most bits for 182 * itself and for various comparison and folding flags. 183 * The most significant bits, however, are shifted down and passed on 184 * to the normalization implementation. 185 * (That is, from unorm_compare(..., options, ...), 186 * options>>UNORM_COMPARE_NORM_OPTIONS_SHIFT will be passed on to the 187 * internal normalization functions.) 188 * 189 * @see unorm_compare 190 * @deprecated ICU 56 Use unorm2.h instead. 191 */ 192 #define UNORM_COMPARE_NORM_OPTIONS_SHIFT 20 193 194 /** 195 * Normalize a string. 196 * The string will be normalized according the specified normalization mode 197 * and options. 198 * The source and result buffers must not be the same, nor overlap. 199 * 200 * @param source The string to normalize. 201 * @param sourceLength The length of source, or -1 if NUL-terminated. 202 * @param mode The normalization mode; one of UNORM_NONE, 203 * UNORM_NFD, UNORM_NFC, UNORM_NFKC, UNORM_NFKD, UNORM_DEFAULT. 204 * @param options The normalization options, ORed together (0 for no options). 205 * @param result A pointer to a buffer to receive the result string. 206 * The result string is NUL-terminated if possible. 207 * @param resultLength The maximum size of result. 208 * @param status A pointer to a UErrorCode to receive any errors. 209 * @return The total buffer size needed; if greater than resultLength, 210 * the output was truncated, and the error code is set to U_BUFFER_OVERFLOW_ERROR. 211 * @deprecated ICU 56 Use unorm2.h instead. 212 */ 213 U_DEPRECATED int32_t U_EXPORT2 214 unorm_normalize(const UChar *source, int32_t sourceLength, 215 UNormalizationMode mode, int32_t options, 216 UChar *result, int32_t resultLength, 217 UErrorCode *status); 218 219 /** 220 * Performing quick check on a string, to quickly determine if the string is 221 * in a particular normalization format. 222 * Three types of result can be returned UNORM_YES, UNORM_NO or 223 * UNORM_MAYBE. Result UNORM_YES indicates that the argument 224 * string is in the desired normalized format, UNORM_NO determines that 225 * argument string is not in the desired normalized format. A 226 * UNORM_MAYBE result indicates that a more thorough check is required, 227 * the user may have to put the string in its normalized form and compare the 228 * results. 229 * 230 * @param source string for determining if it is in a normalized format 231 * @param sourcelength length of source to test, or -1 if NUL-terminated 232 * @param mode which normalization form to test for 233 * @param status a pointer to a UErrorCode to receive any errors 234 * @return UNORM_YES, UNORM_NO or UNORM_MAYBE 235 * 236 * @see unorm_isNormalized 237 * @deprecated ICU 56 Use unorm2.h instead. 238 */ 239 U_DEPRECATED UNormalizationCheckResult U_EXPORT2 240 unorm_quickCheck(const UChar *source, int32_t sourcelength, 241 UNormalizationMode mode, 242 UErrorCode *status); 243 244 /** 245 * Performing quick check on a string; same as unorm_quickCheck but 246 * takes an extra options parameter like most normalization functions. 247 * 248 * @param src String that is to be tested if it is in a normalization format. 249 * @param srcLength Length of source to test, or -1 if NUL-terminated. 250 * @param mode Which normalization form to test for. 251 * @param options The normalization options, ORed together (0 for no options). 252 * @param pErrorCode ICU error code in/out parameter. 253 * Must fulfill U_SUCCESS before the function call. 254 * @return UNORM_YES, UNORM_NO or UNORM_MAYBE 255 * 256 * @see unorm_quickCheck 257 * @see unorm_isNormalized 258 * @deprecated ICU 56 Use unorm2.h instead. 259 */ 260 U_DEPRECATED UNormalizationCheckResult U_EXPORT2 261 unorm_quickCheckWithOptions(const UChar *src, int32_t srcLength, 262 UNormalizationMode mode, int32_t options, 263 UErrorCode *pErrorCode); 264 265 /** 266 * Test if a string is in a given normalization form. 267 * This is semantically equivalent to source.equals(normalize(source, mode)) . 268 * 269 * Unlike unorm_quickCheck(), this function returns a definitive result, 270 * never a "maybe". 271 * For NFD, NFKD, and FCD, both functions work exactly the same. 272 * For NFC and NFKC where quickCheck may return "maybe", this function will 273 * perform further tests to arrive at a TRUE/FALSE result. 274 * 275 * @param src String that is to be tested if it is in a normalization format. 276 * @param srcLength Length of source to test, or -1 if NUL-terminated. 277 * @param mode Which normalization form to test for. 278 * @param pErrorCode ICU error code in/out parameter. 279 * Must fulfill U_SUCCESS before the function call. 280 * @return Boolean value indicating whether the source string is in the 281 * "mode" normalization form. 282 * 283 * @see unorm_quickCheck 284 * @deprecated ICU 56 Use unorm2.h instead. 285 */ 286 U_DEPRECATED UBool U_EXPORT2 287 unorm_isNormalized(const UChar *src, int32_t srcLength, 288 UNormalizationMode mode, 289 UErrorCode *pErrorCode); 290 291 /** 292 * Test if a string is in a given normalization form; same as unorm_isNormalized but 293 * takes an extra options parameter like most normalization functions. 294 * 295 * @param src String that is to be tested if it is in a normalization format. 296 * @param srcLength Length of source to test, or -1 if NUL-terminated. 297 * @param mode Which normalization form to test for. 298 * @param options The normalization options, ORed together (0 for no options). 299 * @param pErrorCode ICU error code in/out parameter. 300 * Must fulfill U_SUCCESS before the function call. 301 * @return Boolean value indicating whether the source string is in the 302 * "mode/options" normalization form. 303 * 304 * @see unorm_quickCheck 305 * @see unorm_isNormalized 306 * @deprecated ICU 56 Use unorm2.h instead. 307 */ 308 U_DEPRECATED UBool U_EXPORT2 309 unorm_isNormalizedWithOptions(const UChar *src, int32_t srcLength, 310 UNormalizationMode mode, int32_t options, 311 UErrorCode *pErrorCode); 312 313 /** 314 * Iterative normalization forward. 315 * This function (together with unorm_previous) is somewhat 316 * similar to the C++ Normalizer class (see its non-static functions). 317 * 318 * Iterative normalization is useful when only a small portion of a longer 319 * string/text needs to be processed. 320 * 321 * For example, the likelihood may be high that processing the first 10% of some 322 * text will be sufficient to find certain data. 323 * Another example: When one wants to concatenate two normalized strings and get a 324 * normalized result, it is much more efficient to normalize just a small part of 325 * the result around the concatenation place instead of re-normalizing everything. 326 * 327 * The input text is an instance of the C character iteration API UCharIterator. 328 * It may wrap around a simple string, a CharacterIterator, a Replaceable, or any 329 * other kind of text object. 330 * 331 * If a buffer overflow occurs, then the caller needs to reset the iterator to the 332 * old index and call the function again with a larger buffer - if the caller cares 333 * for the actual output. 334 * Regardless of the output buffer, the iterator will always be moved to the next 335 * normalization boundary. 336 * 337 * This function (like unorm_previous) serves two purposes: 338 * 339 * 1) To find the next boundary so that the normalization of the part of the text 340 * from the current position to that boundary does not affect and is not affected 341 * by the part of the text beyond that boundary. 342 * 343 * 2) To normalize the text up to the boundary. 344 * 345 * The second step is optional, per the doNormalize parameter. 346 * It is omitted for operations like string concatenation, where the two adjacent 347 * string ends need to be normalized together. 348 * In such a case, the output buffer will just contain a copy of the text up to the 349 * boundary. 350 * 351 * pNeededToNormalize is an output-only parameter. Its output value is only defined 352 * if normalization was requested (doNormalize) and successful (especially, no 353 * buffer overflow). 354 * It is useful for operations like a normalizing transliterator, where one would 355 * not want to replace a piece of text if it is not modified. 356 * 357 * If doNormalize==TRUE and pNeededToNormalize!=NULL then *pNeeded... is set TRUE 358 * if the normalization was necessary. 359 * 360 * If doNormalize==FALSE then *pNeededToNormalize will be set to FALSE. 361 * 362 * If the buffer overflows, then *pNeededToNormalize will be undefined; 363 * essentially, whenever U_FAILURE is true (like in buffer overflows), this result 364 * will be undefined. 365 * 366 * @param src The input text in the form of a C character iterator. 367 * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting. 368 * @param destCapacity The number of UChars that fit into dest. 369 * @param mode The normalization mode. 370 * @param options The normalization options, ORed together (0 for no options). 371 * @param doNormalize Indicates if the source text up to the next boundary 372 * is to be normalized (TRUE) or just copied (FALSE). 373 * @param pNeededToNormalize Output flag indicating if the normalization resulted in 374 * different text from the input. 375 * Not defined if an error occurs including buffer overflow. 376 * Always FALSE if !doNormalize. 377 * @param pErrorCode ICU error code in/out parameter. 378 * Must fulfill U_SUCCESS before the function call. 379 * @return Length of output (number of UChars) when successful or buffer overflow. 380 * 381 * @see unorm_previous 382 * @see unorm_normalize 383 * 384 * @deprecated ICU 56 Use unorm2.h instead. 385 */ 386 U_DEPRECATED int32_t U_EXPORT2 387 unorm_next(UCharIterator *src, 388 UChar *dest, int32_t destCapacity, 389 UNormalizationMode mode, int32_t options, 390 UBool doNormalize, UBool *pNeededToNormalize, 391 UErrorCode *pErrorCode); 392 393 /** 394 * Iterative normalization backward. 395 * This function (together with unorm_next) is somewhat 396 * similar to the C++ Normalizer class (see its non-static functions). 397 * For all details see unorm_next. 398 * 399 * @param src The input text in the form of a C character iterator. 400 * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting. 401 * @param destCapacity The number of UChars that fit into dest. 402 * @param mode The normalization mode. 403 * @param options The normalization options, ORed together (0 for no options). 404 * @param doNormalize Indicates if the source text up to the next boundary 405 * is to be normalized (TRUE) or just copied (FALSE). 406 * @param pNeededToNormalize Output flag indicating if the normalization resulted in 407 * different text from the input. 408 * Not defined if an error occurs including buffer overflow. 409 * Always FALSE if !doNormalize. 410 * @param pErrorCode ICU error code in/out parameter. 411 * Must fulfill U_SUCCESS before the function call. 412 * @return Length of output (number of UChars) when successful or buffer overflow. 413 * 414 * @see unorm_next 415 * @see unorm_normalize 416 * 417 * @deprecated ICU 56 Use unorm2.h instead. 418 */ 419 U_DEPRECATED int32_t U_EXPORT2 420 unorm_previous(UCharIterator *src, 421 UChar *dest, int32_t destCapacity, 422 UNormalizationMode mode, int32_t options, 423 UBool doNormalize, UBool *pNeededToNormalize, 424 UErrorCode *pErrorCode); 425 426 /** 427 * Concatenate normalized strings, making sure that the result is normalized as well. 428 * 429 * If both the left and the right strings are in 430 * the normalization form according to "mode/options", 431 * then the result will be 432 * 433 * \code 434 * dest=normalize(left+right, mode, options) 435 * \endcode 436 * 437 * With the input strings already being normalized, 438 * this function will use unorm_next() and unorm_previous() 439 * to find the adjacent end pieces of the input strings. 440 * Only the concatenation of these end pieces will be normalized and 441 * then concatenated with the remaining parts of the input strings. 442 * 443 * It is allowed to have dest==left to avoid copying the entire left string. 444 * 445 * @param left Left source string, may be same as dest. 446 * @param leftLength Length of left source string, or -1 if NUL-terminated. 447 * @param right Right source string. Must not be the same as dest, nor overlap. 448 * @param rightLength Length of right source string, or -1 if NUL-terminated. 449 * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting. 450 * @param destCapacity The number of UChars that fit into dest. 451 * @param mode The normalization mode. 452 * @param options The normalization options, ORed together (0 for no options). 453 * @param pErrorCode ICU error code in/out parameter. 454 * Must fulfill U_SUCCESS before the function call. 455 * @return Length of output (number of UChars) when successful or buffer overflow. 456 * 457 * @see unorm_normalize 458 * @see unorm_next 459 * @see unorm_previous 460 * 461 * @deprecated ICU 56 Use unorm2.h instead. 462 */ 463 U_DEPRECATED int32_t U_EXPORT2 464 unorm_concatenate(const UChar *left, int32_t leftLength, 465 const UChar *right, int32_t rightLength, 466 UChar *dest, int32_t destCapacity, 467 UNormalizationMode mode, int32_t options, 468 UErrorCode *pErrorCode); 469 470 #endif /* U_HIDE_DEPRECATED_API */ 471 #endif /* #if !UCONFIG_NO_NORMALIZATION */ 472 #endif 473