• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // © 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
3 /*
4 *******************************************************************************
5 *
6 *   Copyright (C) 1999-2011, International Business Machines
7 *   Corporation and others.  All Rights Reserved.
8 *
9 *******************************************************************************
10 *   file name:  utf.h
11 *   encoding:   UTF-8
12 *   tab size:   8 (not used)
13 *   indentation:4
14 *
15 *   created on: 1999sep09
16 *   created by: Markus W. Scherer
17 */
18 
19 /**
20  * \file
21  * \brief C API: Code point macros
22  *
23  * This file defines macros for checking whether a code point is
24  * a surrogate or a non-character etc.
25  *
26  * If U_NO_DEFAULT_INCLUDE_UTF_HEADERS is 0 then utf.h is included by utypes.h
27  * and itself includes utf8.h and utf16.h after some
28  * common definitions.
29  * If U_NO_DEFAULT_INCLUDE_UTF_HEADERS is 1 then each of these headers must be
30  * included explicitly if their definitions are used.
31  *
32  * utf8.h and utf16.h define macros for efficiently getting code points
33  * in and out of UTF-8/16 strings.
34  * utf16.h macros have "U16_" prefixes.
35  * utf8.h defines similar macros with "U8_" prefixes for UTF-8 string handling.
36  *
37  * ICU mostly processes 16-bit Unicode strings.
38  * Most of the time, such strings are well-formed UTF-16.
39  * Single, unpaired surrogates must be handled as well, and are treated in ICU
40  * like regular code points where possible.
41  * (Pairs of surrogate code points are indistinguishable from supplementary
42  * code points encoded as pairs of supplementary code units.)
43  *
44  * In fact, almost all Unicode code points in normal text (>99%)
45  * are on the BMP (<=U+ffff) and even <=U+d7ff.
46  * ICU functions handle supplementary code points (U+10000..U+10ffff)
47  * but are optimized for the much more frequently occurring BMP code points.
48  *
49  * umachine.h defines UChar to be an unsigned 16-bit integer.
50  * Since ICU 59, ICU uses char16_t in C++, UChar only in C,
51  * and defines UChar=char16_t by default. See the UChar API docs for details.
52  *
53  * UChar32 is defined to be a signed 32-bit integer (int32_t), large enough for a 21-bit
54  * Unicode code point (Unicode scalar value, 0..0x10ffff) and U_SENTINEL (-1).
55  * Before ICU 2.4, the definition of UChar32 was similarly platform-dependent as
56  * the definition of UChar. For details see the documentation for UChar32 itself.
57  *
58  * utf.h defines a small number of C macros for single Unicode code points.
59  * These are simple checks for surrogates and non-characters.
60  * For actual Unicode character properties see uchar.h.
61  *
62  * By default, string operations must be done with error checking in case
63  * a string is not well-formed UTF-16 or UTF-8.
64  *
65  * The U16_ macros detect if a surrogate code unit is unpaired
66  * (lead unit without trail unit or vice versa) and just return the unit itself
67  * as the code point.
68  *
69  * The U8_ macros detect illegal byte sequences and return a negative value.
70  * Starting with ICU 60, the observable length of a single illegal byte sequence
71  * skipped by one of these macros follows the Unicode 6+ recommendation
72  * which is consistent with the W3C Encoding Standard.
73  *
74  * There are ..._OR_FFFD versions of both U16_ and U8_ macros
75  * that return U+FFFD for illegal code unit sequences.
76  *
77  * The regular "safe" macros require that the initial, passed-in string index
78  * is within bounds. They only check the index when they read more than one
79  * code unit. This is usually done with code similar to the following loop:
80  * <pre>while(i<length) {
81  *   U16_NEXT(s, i, length, c);
82  *   // use c
83  * }</pre>
84  *
85  * When it is safe to assume that text is well-formed UTF-16
86  * (does not contain single, unpaired surrogates), then one can use
87  * U16_..._UNSAFE macros.
88  * These do not check for proper code unit sequences or truncated text and may
89  * yield wrong results or even cause a crash if they are used with "malformed"
90  * text.
91  * In practice, U16_..._UNSAFE macros will produce slightly less code but
92  * should not be faster because the processing is only different when a
93  * surrogate code unit is detected, which will be rare.
94  *
95  * Similarly for UTF-8, there are "safe" macros without a suffix,
96  * and U8_..._UNSAFE versions.
97  * The performance differences are much larger here because UTF-8 provides so
98  * many opportunities for malformed sequences.
99  * The unsafe UTF-8 macros are entirely implemented inside the macro definitions
100  * and are fast, while the safe UTF-8 macros call functions for some complicated cases.
101  *
102  * Unlike with UTF-16, malformed sequences cannot be expressed with distinct
103  * code point values (0..U+10ffff). They are indicated with negative values instead.
104  *
105  * For more information see the ICU User Guide Strings chapter
106  * (http://userguide.icu-project.org/strings).
107  *
108  * <em>Usage:</em>
109  * ICU coding guidelines for if() statements should be followed when using these macros.
110  * Compound statements (curly braces {}) must be used  for if-else-while...
111  * bodies and all macro statements should be terminated with semicolon.
112  *
113  * @stable ICU 2.4
114  */
115 
116 #ifndef __UTF_H__
117 #define __UTF_H__
118 
119 #include "unicode/umachine.h"
120 /* include the utfXX.h after the following definitions */
121 
122 /* single-code point definitions -------------------------------------------- */
123 
124 /**
125  * Is this code point a Unicode noncharacter?
126  * @param c 32-bit code point
127  * @return TRUE or FALSE
128  * @stable ICU 2.4
129  */
130 #define U_IS_UNICODE_NONCHAR(c) \
131     ((c)>=0xfdd0 && \
132      ((c)<=0xfdef || ((c)&0xfffe)==0xfffe) && (c)<=0x10ffff)
133 
134 /**
135  * Is c a Unicode code point value (0..U+10ffff)
136  * that can be assigned a character?
137  *
138  * Code points that are not characters include:
139  * - single surrogate code points (U+d800..U+dfff, 2048 code points)
140  * - the last two code points on each plane (U+__fffe and U+__ffff, 34 code points)
141  * - U+fdd0..U+fdef (new with Unicode 3.1, 32 code points)
142  * - the highest Unicode code point value is U+10ffff
143  *
144  * This means that all code points below U+d800 are character code points,
145  * and that boundary is tested first for performance.
146  *
147  * @param c 32-bit code point
148  * @return TRUE or FALSE
149  * @stable ICU 2.4
150  */
151 #define U_IS_UNICODE_CHAR(c) \
152     ((uint32_t)(c)<0xd800 || \
153         (0xdfff<(c) && (c)<=0x10ffff && !U_IS_UNICODE_NONCHAR(c)))
154 
155 /**
156  * Is this code point a BMP code point (U+0000..U+ffff)?
157  * @param c 32-bit code point
158  * @return TRUE or FALSE
159  * @stable ICU 2.8
160  */
161 #define U_IS_BMP(c) ((uint32_t)(c)<=0xffff)
162 
163 /**
164  * Is this code point a supplementary code point (U+10000..U+10ffff)?
165  * @param c 32-bit code point
166  * @return TRUE or FALSE
167  * @stable ICU 2.8
168  */
169 #define U_IS_SUPPLEMENTARY(c) ((uint32_t)((c)-0x10000)<=0xfffff)
170 
171 /**
172  * Is this code point a lead surrogate (U+d800..U+dbff)?
173  * @param c 32-bit code point
174  * @return TRUE or FALSE
175  * @stable ICU 2.4
176  */
177 #define U_IS_LEAD(c) (((c)&0xfffffc00)==0xd800)
178 
179 /**
180  * Is this code point a trail surrogate (U+dc00..U+dfff)?
181  * @param c 32-bit code point
182  * @return TRUE or FALSE
183  * @stable ICU 2.4
184  */
185 #define U_IS_TRAIL(c) (((c)&0xfffffc00)==0xdc00)
186 
187 /**
188  * Is this code point a surrogate (U+d800..U+dfff)?
189  * @param c 32-bit code point
190  * @return TRUE or FALSE
191  * @stable ICU 2.4
192  */
193 #define U_IS_SURROGATE(c) (((c)&0xfffff800)==0xd800)
194 
195 /**
196  * Assuming c is a surrogate code point (U_IS_SURROGATE(c)),
197  * is it a lead surrogate?
198  * @param c 32-bit code point
199  * @return TRUE or FALSE
200  * @stable ICU 2.4
201  */
202 #define U_IS_SURROGATE_LEAD(c) (((c)&0x400)==0)
203 
204 /**
205  * Assuming c is a surrogate code point (U_IS_SURROGATE(c)),
206  * is it a trail surrogate?
207  * @param c 32-bit code point
208  * @return TRUE or FALSE
209  * @stable ICU 4.2
210  */
211 #define U_IS_SURROGATE_TRAIL(c) (((c)&0x400)!=0)
212 
213 /* include the utfXX.h ------------------------------------------------------ */
214 
215 #if !U_NO_DEFAULT_INCLUDE_UTF_HEADERS
216 
217 #include "unicode/utf8.h"
218 #include "unicode/utf16.h"
219 
220 /* utf_old.h contains deprecated, pre-ICU 2.4 definitions */
221 #include "unicode/utf_old.h"
222 
223 #endif  /* !U_NO_DEFAULT_INCLUDE_UTF_HEADERS */
224 
225 #endif  /* __UTF_H__ */
226