• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2    Unicode character type helpers.
3 
4    Written by Marc-Andre Lemburg (mal@lemburg.com).
5    Modified for Python 2.0 by Fredrik Lundh (fredrik@pythonware.com)
6 
7    Copyright (c) Corporation for National Research Initiatives.
8 
9 */
10 
11 #include "Python.h"
12 
13 #define ALPHA_MASK 0x01
14 #define DECIMAL_MASK 0x02
15 #define DIGIT_MASK 0x04
16 #define LOWER_MASK 0x08
17 #define LINEBREAK_MASK 0x10
18 #define SPACE_MASK 0x20
19 #define TITLE_MASK 0x40
20 #define UPPER_MASK 0x80
21 #define XID_START_MASK 0x100
22 #define XID_CONTINUE_MASK 0x200
23 #define PRINTABLE_MASK 0x400
24 #define NUMERIC_MASK 0x800
25 #define CASE_IGNORABLE_MASK 0x1000
26 #define CASED_MASK 0x2000
27 #define EXTENDED_CASE_MASK 0x4000
28 
29 typedef struct {
30     /*
31        These are either deltas to the character or offsets in
32        _PyUnicode_ExtendedCase.
33     */
34     const int upper;
35     const int lower;
36     const int title;
37     /* Note if more flag space is needed, decimal and digit could be unified. */
38     const unsigned char decimal;
39     const unsigned char digit;
40     const unsigned short flags;
41 } _PyUnicode_TypeRecord;
42 
43 #include "unicodetype_db.h"
44 
45 static const _PyUnicode_TypeRecord *
gettyperecord(Py_UCS4 code)46 gettyperecord(Py_UCS4 code)
47 {
48     int index;
49 
50     if (code >= 0x110000)
51         index = 0;
52     else
53     {
54         index = index1[(code>>SHIFT)];
55         index = index2[(index<<SHIFT)+(code&((1<<SHIFT)-1))];
56     }
57 
58     return &_PyUnicode_TypeRecords[index];
59 }
60 
61 /* Returns the titlecase Unicode characters corresponding to ch or just
62    ch if no titlecase mapping is known. */
63 
_PyUnicode_ToTitlecase(Py_UCS4 ch)64 Py_UCS4 _PyUnicode_ToTitlecase(Py_UCS4 ch)
65 {
66     const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
67 
68     if (ctype->flags & EXTENDED_CASE_MASK)
69         return _PyUnicode_ExtendedCase[ctype->title & 0xFFFF];
70     return ch + ctype->title;
71 }
72 
73 /* Returns 1 for Unicode characters having the category 'Lt', 0
74    otherwise. */
75 
_PyUnicode_IsTitlecase(Py_UCS4 ch)76 int _PyUnicode_IsTitlecase(Py_UCS4 ch)
77 {
78     const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
79 
80     return (ctype->flags & TITLE_MASK) != 0;
81 }
82 
83 /* Returns 1 for Unicode characters having the XID_Start property, 0
84    otherwise. */
85 
_PyUnicode_IsXidStart(Py_UCS4 ch)86 int _PyUnicode_IsXidStart(Py_UCS4 ch)
87 {
88     const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
89 
90     return (ctype->flags & XID_START_MASK) != 0;
91 }
92 
93 /* Returns 1 for Unicode characters having the XID_Continue property,
94    0 otherwise. */
95 
_PyUnicode_IsXidContinue(Py_UCS4 ch)96 int _PyUnicode_IsXidContinue(Py_UCS4 ch)
97 {
98     const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
99 
100     return (ctype->flags & XID_CONTINUE_MASK) != 0;
101 }
102 
103 /* Returns the integer decimal (0-9) for Unicode characters having
104    this property, -1 otherwise. */
105 
_PyUnicode_ToDecimalDigit(Py_UCS4 ch)106 int _PyUnicode_ToDecimalDigit(Py_UCS4 ch)
107 {
108     const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
109 
110     return (ctype->flags & DECIMAL_MASK) ? ctype->decimal : -1;
111 }
112 
_PyUnicode_IsDecimalDigit(Py_UCS4 ch)113 int _PyUnicode_IsDecimalDigit(Py_UCS4 ch)
114 {
115     if (_PyUnicode_ToDecimalDigit(ch) < 0)
116         return 0;
117     return 1;
118 }
119 
120 /* Returns the integer digit (0-9) for Unicode characters having
121    this property, -1 otherwise. */
122 
_PyUnicode_ToDigit(Py_UCS4 ch)123 int _PyUnicode_ToDigit(Py_UCS4 ch)
124 {
125     const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
126 
127     return (ctype->flags & DIGIT_MASK) ? ctype->digit : -1;
128 }
129 
_PyUnicode_IsDigit(Py_UCS4 ch)130 int _PyUnicode_IsDigit(Py_UCS4 ch)
131 {
132     if (_PyUnicode_ToDigit(ch) < 0)
133         return 0;
134     return 1;
135 }
136 
137 /* Returns the numeric value as double for Unicode characters having
138    this property, -1.0 otherwise. */
139 
_PyUnicode_IsNumeric(Py_UCS4 ch)140 int _PyUnicode_IsNumeric(Py_UCS4 ch)
141 {
142     const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
143 
144     return (ctype->flags & NUMERIC_MASK) != 0;
145 }
146 
147 /* Returns 1 for Unicode characters to be hex-escaped when repr()ed,
148    0 otherwise.
149    All characters except those characters defined in the Unicode character
150    database as following categories are considered printable.
151       * Cc (Other, Control)
152       * Cf (Other, Format)
153       * Cs (Other, Surrogate)
154       * Co (Other, Private Use)
155       * Cn (Other, Not Assigned)
156       * Zl Separator, Line ('\u2028', LINE SEPARATOR)
157       * Zp Separator, Paragraph ('\u2029', PARAGRAPH SEPARATOR)
158       * Zs (Separator, Space) other than ASCII space('\x20').
159 */
_PyUnicode_IsPrintable(Py_UCS4 ch)160 int _PyUnicode_IsPrintable(Py_UCS4 ch)
161 {
162     const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
163 
164     return (ctype->flags & PRINTABLE_MASK) != 0;
165 }
166 
167 /* Returns 1 for Unicode characters having the category 'Ll', 0
168    otherwise. */
169 
_PyUnicode_IsLowercase(Py_UCS4 ch)170 int _PyUnicode_IsLowercase(Py_UCS4 ch)
171 {
172     const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
173 
174     return (ctype->flags & LOWER_MASK) != 0;
175 }
176 
177 /* Returns 1 for Unicode characters having the category 'Lu', 0
178    otherwise. */
179 
_PyUnicode_IsUppercase(Py_UCS4 ch)180 int _PyUnicode_IsUppercase(Py_UCS4 ch)
181 {
182     const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
183 
184     return (ctype->flags & UPPER_MASK) != 0;
185 }
186 
187 /* Returns the uppercase Unicode characters corresponding to ch or just
188    ch if no uppercase mapping is known. */
189 
_PyUnicode_ToUppercase(Py_UCS4 ch)190 Py_UCS4 _PyUnicode_ToUppercase(Py_UCS4 ch)
191 {
192     const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
193 
194     if (ctype->flags & EXTENDED_CASE_MASK)
195         return _PyUnicode_ExtendedCase[ctype->upper & 0xFFFF];
196     return ch + ctype->upper;
197 }
198 
199 /* Returns the lowercase Unicode characters corresponding to ch or just
200    ch if no lowercase mapping is known. */
201 
_PyUnicode_ToLowercase(Py_UCS4 ch)202 Py_UCS4 _PyUnicode_ToLowercase(Py_UCS4 ch)
203 {
204     const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
205 
206     if (ctype->flags & EXTENDED_CASE_MASK)
207         return _PyUnicode_ExtendedCase[ctype->lower & 0xFFFF];
208     return ch + ctype->lower;
209 }
210 
_PyUnicode_ToLowerFull(Py_UCS4 ch,Py_UCS4 * res)211 int _PyUnicode_ToLowerFull(Py_UCS4 ch, Py_UCS4 *res)
212 {
213     const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
214 
215     if (ctype->flags & EXTENDED_CASE_MASK) {
216         int index = ctype->lower & 0xFFFF;
217         int n = ctype->lower >> 24;
218         int i;
219         for (i = 0; i < n; i++)
220             res[i] = _PyUnicode_ExtendedCase[index + i];
221         return n;
222     }
223     res[0] = ch + ctype->lower;
224     return 1;
225 }
226 
_PyUnicode_ToTitleFull(Py_UCS4 ch,Py_UCS4 * res)227 int _PyUnicode_ToTitleFull(Py_UCS4 ch, Py_UCS4 *res)
228 {
229     const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
230 
231     if (ctype->flags & EXTENDED_CASE_MASK) {
232         int index = ctype->title & 0xFFFF;
233         int n = ctype->title >> 24;
234         int i;
235         for (i = 0; i < n; i++)
236             res[i] = _PyUnicode_ExtendedCase[index + i];
237         return n;
238     }
239     res[0] = ch + ctype->title;
240     return 1;
241 }
242 
_PyUnicode_ToUpperFull(Py_UCS4 ch,Py_UCS4 * res)243 int _PyUnicode_ToUpperFull(Py_UCS4 ch, Py_UCS4 *res)
244 {
245     const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
246 
247     if (ctype->flags & EXTENDED_CASE_MASK) {
248         int index = ctype->upper & 0xFFFF;
249         int n = ctype->upper >> 24;
250         int i;
251         for (i = 0; i < n; i++)
252             res[i] = _PyUnicode_ExtendedCase[index + i];
253         return n;
254     }
255     res[0] = ch + ctype->upper;
256     return 1;
257 }
258 
_PyUnicode_ToFoldedFull(Py_UCS4 ch,Py_UCS4 * res)259 int _PyUnicode_ToFoldedFull(Py_UCS4 ch, Py_UCS4 *res)
260 {
261     const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
262 
263     if (ctype->flags & EXTENDED_CASE_MASK && (ctype->lower >> 20) & 7) {
264         int index = (ctype->lower & 0xFFFF) + (ctype->lower >> 24);
265         int n = (ctype->lower >> 20) & 7;
266         int i;
267         for (i = 0; i < n; i++)
268             res[i] = _PyUnicode_ExtendedCase[index + i];
269         return n;
270     }
271     return _PyUnicode_ToLowerFull(ch, res);
272 }
273 
_PyUnicode_IsCased(Py_UCS4 ch)274 int _PyUnicode_IsCased(Py_UCS4 ch)
275 {
276     const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
277 
278     return (ctype->flags & CASED_MASK) != 0;
279 }
280 
_PyUnicode_IsCaseIgnorable(Py_UCS4 ch)281 int _PyUnicode_IsCaseIgnorable(Py_UCS4 ch)
282 {
283     const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
284 
285     return (ctype->flags & CASE_IGNORABLE_MASK) != 0;
286 }
287 
288 /* Returns 1 for Unicode characters having the category 'Ll', 'Lu', 'Lt',
289    'Lo' or 'Lm',  0 otherwise. */
290 
_PyUnicode_IsAlpha(Py_UCS4 ch)291 int _PyUnicode_IsAlpha(Py_UCS4 ch)
292 {
293     const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
294 
295     return (ctype->flags & ALPHA_MASK) != 0;
296 }
297 
298