• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 #ifndef Py_UNICODEOBJECT_H
2 #define Py_UNICODEOBJECT_H
3 
4 #include <stdarg.h>
5 
6 /*
7 
8 Unicode implementation based on original code by Fredrik Lundh,
9 modified by Marc-Andre Lemburg (mal@lemburg.com) according to the
10 Unicode Integration Proposal (see file Misc/unicode.txt).
11 
12 Copyright (c) Corporation for National Research Initiatives.
13 
14 
15  Original header:
16  --------------------------------------------------------------------
17 
18  * Yet another Unicode string type for Python.  This type supports the
19  * 16-bit Basic Multilingual Plane (BMP) only.
20  *
21  * Written by Fredrik Lundh, January 1999.
22  *
23  * Copyright (c) 1999 by Secret Labs AB.
24  * Copyright (c) 1999 by Fredrik Lundh.
25  *
26  * fredrik@pythonware.com
27  * http://www.pythonware.com
28  *
29  * --------------------------------------------------------------------
30  * This Unicode String Type is
31  *
32  * Copyright (c) 1999 by Secret Labs AB
33  * Copyright (c) 1999 by Fredrik Lundh
34  *
35  * By obtaining, using, and/or copying this software and/or its
36  * associated documentation, you agree that you have read, understood,
37  * and will comply with the following terms and conditions:
38  *
39  * Permission to use, copy, modify, and distribute this software and its
40  * associated documentation for any purpose and without fee is hereby
41  * granted, provided that the above copyright notice appears in all
42  * copies, and that both that copyright notice and this permission notice
43  * appear in supporting documentation, and that the name of Secret Labs
44  * AB or the author not be used in advertising or publicity pertaining to
45  * distribution of the software without specific, written prior
46  * permission.
47  *
48  * SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO
49  * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
50  * FITNESS.  IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR BE LIABLE FOR
51  * ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
52  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
53  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
54  * OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
55  * -------------------------------------------------------------------- */
56 
57 #include <ctype.h>
58 
59 /* === Internal API ======================================================= */
60 
61 /* --- Internal Unicode Format -------------------------------------------- */
62 
63 #ifndef Py_USING_UNICODE
64 
65 #define PyUnicode_Check(op)                 0
66 #define PyUnicode_CheckExact(op)            0
67 
68 #else
69 
70 /* FIXME: MvL's new implementation assumes that Py_UNICODE_SIZE is
71    properly set, but the default rules below doesn't set it.  I'll
72    sort this out some other day -- fredrik@pythonware.com */
73 
74 #ifndef Py_UNICODE_SIZE
75 #error Must define Py_UNICODE_SIZE
76 #endif
77 
78 /* Setting Py_UNICODE_WIDE enables UCS-4 storage.  Otherwise, Unicode
79    strings are stored as UCS-2 (with limited support for UTF-16) */
80 
81 #if Py_UNICODE_SIZE >= 4
82 #define Py_UNICODE_WIDE
83 #endif
84 
85 /* Set these flags if the platform has "wchar.h", "wctype.h" and the
86    wchar_t type is a 16-bit unsigned type */
87 /* #define HAVE_WCHAR_H */
88 /* #define HAVE_USABLE_WCHAR_T */
89 
90 /* Defaults for various platforms */
91 #ifndef PY_UNICODE_TYPE
92 
93 /* Windows has a usable wchar_t type (unless we're using UCS-4) */
94 # if defined(MS_WIN32) && Py_UNICODE_SIZE == 2
95 #  define HAVE_USABLE_WCHAR_T
96 #  define PY_UNICODE_TYPE wchar_t
97 # endif
98 
99 # if defined(Py_UNICODE_WIDE)
100 #  define PY_UNICODE_TYPE Py_UCS4
101 # endif
102 
103 #endif
104 
105 /* If the compiler provides a wchar_t type we try to support it
106    through the interface functions PyUnicode_FromWideChar() and
107    PyUnicode_AsWideChar(). */
108 
109 #ifdef HAVE_USABLE_WCHAR_T
110 # ifndef HAVE_WCHAR_H
111 #  define HAVE_WCHAR_H
112 # endif
113 #endif
114 
115 #ifdef HAVE_WCHAR_H
116 /* Work around a cosmetic bug in BSDI 4.x wchar.h; thanks to Thomas Wouters */
117 # ifdef _HAVE_BSDI
118 #  include <time.h>
119 # endif
120 #  include <wchar.h>
121 #endif
122 
123 /*
124  * Use this typedef when you need to represent a UTF-16 surrogate pair
125  * as single unsigned integer.
126  */
127 #if SIZEOF_INT >= 4
128 typedef unsigned int Py_UCS4;
129 #elif SIZEOF_LONG >= 4
130 typedef unsigned long Py_UCS4;
131 #endif
132 
133 /* Py_UNICODE is the native Unicode storage format (code unit) used by
134    Python and represents a single Unicode element in the Unicode
135    type. */
136 
137 typedef PY_UNICODE_TYPE Py_UNICODE;
138 
139 /* --- UCS-2/UCS-4 Name Mangling ------------------------------------------ */
140 
141 /* Unicode API names are mangled to assure that UCS-2 and UCS-4 builds
142    produce different external names and thus cause import errors in
143    case Python interpreters and extensions with mixed compiled in
144    Unicode width assumptions are combined. */
145 
146 #ifndef Py_UNICODE_WIDE
147 
148 # define PyUnicode_AsASCIIString PyUnicodeUCS2_AsASCIIString
149 # define PyUnicode_AsCharmapString PyUnicodeUCS2_AsCharmapString
150 # define PyUnicode_AsEncodedObject PyUnicodeUCS2_AsEncodedObject
151 # define PyUnicode_AsEncodedString PyUnicodeUCS2_AsEncodedString
152 # define PyUnicode_AsLatin1String PyUnicodeUCS2_AsLatin1String
153 # define PyUnicode_AsRawUnicodeEscapeString PyUnicodeUCS2_AsRawUnicodeEscapeString
154 # define PyUnicode_AsUTF32String PyUnicodeUCS2_AsUTF32String
155 # define PyUnicode_AsUTF16String PyUnicodeUCS2_AsUTF16String
156 # define PyUnicode_AsUTF8String PyUnicodeUCS2_AsUTF8String
157 # define PyUnicode_AsUnicode PyUnicodeUCS2_AsUnicode
158 # define PyUnicode_AsUnicodeEscapeString PyUnicodeUCS2_AsUnicodeEscapeString
159 # define PyUnicode_AsWideChar PyUnicodeUCS2_AsWideChar
160 # define PyUnicode_ClearFreeList PyUnicodeUCS2_ClearFreelist
161 # define PyUnicode_Compare PyUnicodeUCS2_Compare
162 # define PyUnicode_Concat PyUnicodeUCS2_Concat
163 # define PyUnicode_Contains PyUnicodeUCS2_Contains
164 # define PyUnicode_Count PyUnicodeUCS2_Count
165 # define PyUnicode_Decode PyUnicodeUCS2_Decode
166 # define PyUnicode_DecodeASCII PyUnicodeUCS2_DecodeASCII
167 # define PyUnicode_DecodeCharmap PyUnicodeUCS2_DecodeCharmap
168 # define PyUnicode_DecodeLatin1 PyUnicodeUCS2_DecodeLatin1
169 # define PyUnicode_DecodeRawUnicodeEscape PyUnicodeUCS2_DecodeRawUnicodeEscape
170 # define PyUnicode_DecodeUTF32 PyUnicodeUCS2_DecodeUTF32
171 # define PyUnicode_DecodeUTF32Stateful PyUnicodeUCS2_DecodeUTF32Stateful
172 # define PyUnicode_DecodeUTF16 PyUnicodeUCS2_DecodeUTF16
173 # define PyUnicode_DecodeUTF16Stateful PyUnicodeUCS2_DecodeUTF16Stateful
174 # define PyUnicode_DecodeUTF8 PyUnicodeUCS2_DecodeUTF8
175 # define PyUnicode_DecodeUTF8Stateful PyUnicodeUCS2_DecodeUTF8Stateful
176 # define PyUnicode_DecodeUnicodeEscape PyUnicodeUCS2_DecodeUnicodeEscape
177 # define PyUnicode_Encode PyUnicodeUCS2_Encode
178 # define PyUnicode_EncodeASCII PyUnicodeUCS2_EncodeASCII
179 # define PyUnicode_EncodeCharmap PyUnicodeUCS2_EncodeCharmap
180 # define PyUnicode_EncodeDecimal PyUnicodeUCS2_EncodeDecimal
181 # define PyUnicode_EncodeLatin1 PyUnicodeUCS2_EncodeLatin1
182 # define PyUnicode_EncodeRawUnicodeEscape PyUnicodeUCS2_EncodeRawUnicodeEscape
183 # define PyUnicode_EncodeUTF32 PyUnicodeUCS2_EncodeUTF32
184 # define PyUnicode_EncodeUTF16 PyUnicodeUCS2_EncodeUTF16
185 # define PyUnicode_EncodeUTF8 PyUnicodeUCS2_EncodeUTF8
186 # define PyUnicode_EncodeUnicodeEscape PyUnicodeUCS2_EncodeUnicodeEscape
187 # define PyUnicode_Find PyUnicodeUCS2_Find
188 # define PyUnicode_Format PyUnicodeUCS2_Format
189 # define PyUnicode_FromEncodedObject PyUnicodeUCS2_FromEncodedObject
190 # define PyUnicode_FromFormat PyUnicodeUCS2_FromFormat
191 # define PyUnicode_FromFormatV PyUnicodeUCS2_FromFormatV
192 # define PyUnicode_FromObject PyUnicodeUCS2_FromObject
193 # define PyUnicode_FromOrdinal PyUnicodeUCS2_FromOrdinal
194 # define PyUnicode_FromString PyUnicodeUCS2_FromString
195 # define PyUnicode_FromStringAndSize PyUnicodeUCS2_FromStringAndSize
196 # define PyUnicode_FromUnicode PyUnicodeUCS2_FromUnicode
197 # define PyUnicode_FromWideChar PyUnicodeUCS2_FromWideChar
198 # define PyUnicode_GetDefaultEncoding PyUnicodeUCS2_GetDefaultEncoding
199 # define PyUnicode_GetMax PyUnicodeUCS2_GetMax
200 # define PyUnicode_GetSize PyUnicodeUCS2_GetSize
201 # define PyUnicode_Join PyUnicodeUCS2_Join
202 # define PyUnicode_Partition PyUnicodeUCS2_Partition
203 # define PyUnicode_RPartition PyUnicodeUCS2_RPartition
204 # define PyUnicode_RSplit PyUnicodeUCS2_RSplit
205 # define PyUnicode_Replace PyUnicodeUCS2_Replace
206 # define PyUnicode_Resize PyUnicodeUCS2_Resize
207 # define PyUnicode_RichCompare PyUnicodeUCS2_RichCompare
208 # define PyUnicode_SetDefaultEncoding PyUnicodeUCS2_SetDefaultEncoding
209 # define PyUnicode_Split PyUnicodeUCS2_Split
210 # define PyUnicode_Splitlines PyUnicodeUCS2_Splitlines
211 # define PyUnicode_Tailmatch PyUnicodeUCS2_Tailmatch
212 # define PyUnicode_Translate PyUnicodeUCS2_Translate
213 # define PyUnicode_TranslateCharmap PyUnicodeUCS2_TranslateCharmap
214 # define _PyUnicode_AsDefaultEncodedString _PyUnicodeUCS2_AsDefaultEncodedString
215 # define _PyUnicode_Fini _PyUnicodeUCS2_Fini
216 # define _PyUnicode_Init _PyUnicodeUCS2_Init
217 # define _PyUnicode_IsAlpha _PyUnicodeUCS2_IsAlpha
218 # define _PyUnicode_IsDecimalDigit _PyUnicodeUCS2_IsDecimalDigit
219 # define _PyUnicode_IsDigit _PyUnicodeUCS2_IsDigit
220 # define _PyUnicode_IsLinebreak _PyUnicodeUCS2_IsLinebreak
221 # define _PyUnicode_IsLowercase _PyUnicodeUCS2_IsLowercase
222 # define _PyUnicode_IsNumeric _PyUnicodeUCS2_IsNumeric
223 # define _PyUnicode_IsTitlecase _PyUnicodeUCS2_IsTitlecase
224 # define _PyUnicode_IsUppercase _PyUnicodeUCS2_IsUppercase
225 # define _PyUnicode_IsWhitespace _PyUnicodeUCS2_IsWhitespace
226 # define _PyUnicode_ToDecimalDigit _PyUnicodeUCS2_ToDecimalDigit
227 # define _PyUnicode_ToDigit _PyUnicodeUCS2_ToDigit
228 # define _PyUnicode_ToLowercase _PyUnicodeUCS2_ToLowercase
229 # define _PyUnicode_ToNumeric _PyUnicodeUCS2_ToNumeric
230 # define _PyUnicode_ToTitlecase _PyUnicodeUCS2_ToTitlecase
231 # define _PyUnicode_ToUppercase _PyUnicodeUCS2_ToUppercase
232 
233 #else
234 
235 # define PyUnicode_AsASCIIString PyUnicodeUCS4_AsASCIIString
236 # define PyUnicode_AsCharmapString PyUnicodeUCS4_AsCharmapString
237 # define PyUnicode_AsEncodedObject PyUnicodeUCS4_AsEncodedObject
238 # define PyUnicode_AsEncodedString PyUnicodeUCS4_AsEncodedString
239 # define PyUnicode_AsLatin1String PyUnicodeUCS4_AsLatin1String
240 # define PyUnicode_AsRawUnicodeEscapeString PyUnicodeUCS4_AsRawUnicodeEscapeString
241 # define PyUnicode_AsUTF32String PyUnicodeUCS4_AsUTF32String
242 # define PyUnicode_AsUTF16String PyUnicodeUCS4_AsUTF16String
243 # define PyUnicode_AsUTF8String PyUnicodeUCS4_AsUTF8String
244 # define PyUnicode_AsUnicode PyUnicodeUCS4_AsUnicode
245 # define PyUnicode_AsUnicodeEscapeString PyUnicodeUCS4_AsUnicodeEscapeString
246 # define PyUnicode_AsWideChar PyUnicodeUCS4_AsWideChar
247 # define PyUnicode_ClearFreeList PyUnicodeUCS4_ClearFreelist
248 # define PyUnicode_Compare PyUnicodeUCS4_Compare
249 # define PyUnicode_Concat PyUnicodeUCS4_Concat
250 # define PyUnicode_Contains PyUnicodeUCS4_Contains
251 # define PyUnicode_Count PyUnicodeUCS4_Count
252 # define PyUnicode_Decode PyUnicodeUCS4_Decode
253 # define PyUnicode_DecodeASCII PyUnicodeUCS4_DecodeASCII
254 # define PyUnicode_DecodeCharmap PyUnicodeUCS4_DecodeCharmap
255 # define PyUnicode_DecodeLatin1 PyUnicodeUCS4_DecodeLatin1
256 # define PyUnicode_DecodeRawUnicodeEscape PyUnicodeUCS4_DecodeRawUnicodeEscape
257 # define PyUnicode_DecodeUTF32 PyUnicodeUCS4_DecodeUTF32
258 # define PyUnicode_DecodeUTF32Stateful PyUnicodeUCS4_DecodeUTF32Stateful
259 # define PyUnicode_DecodeUTF16 PyUnicodeUCS4_DecodeUTF16
260 # define PyUnicode_DecodeUTF16Stateful PyUnicodeUCS4_DecodeUTF16Stateful
261 # define PyUnicode_DecodeUTF8 PyUnicodeUCS4_DecodeUTF8
262 # define PyUnicode_DecodeUTF8Stateful PyUnicodeUCS4_DecodeUTF8Stateful
263 # define PyUnicode_DecodeUnicodeEscape PyUnicodeUCS4_DecodeUnicodeEscape
264 # define PyUnicode_Encode PyUnicodeUCS4_Encode
265 # define PyUnicode_EncodeASCII PyUnicodeUCS4_EncodeASCII
266 # define PyUnicode_EncodeCharmap PyUnicodeUCS4_EncodeCharmap
267 # define PyUnicode_EncodeDecimal PyUnicodeUCS4_EncodeDecimal
268 # define PyUnicode_EncodeLatin1 PyUnicodeUCS4_EncodeLatin1
269 # define PyUnicode_EncodeRawUnicodeEscape PyUnicodeUCS4_EncodeRawUnicodeEscape
270 # define PyUnicode_EncodeUTF32 PyUnicodeUCS4_EncodeUTF32
271 # define PyUnicode_EncodeUTF16 PyUnicodeUCS4_EncodeUTF16
272 # define PyUnicode_EncodeUTF8 PyUnicodeUCS4_EncodeUTF8
273 # define PyUnicode_EncodeUnicodeEscape PyUnicodeUCS4_EncodeUnicodeEscape
274 # define PyUnicode_Find PyUnicodeUCS4_Find
275 # define PyUnicode_Format PyUnicodeUCS4_Format
276 # define PyUnicode_FromEncodedObject PyUnicodeUCS4_FromEncodedObject
277 # define PyUnicode_FromFormat PyUnicodeUCS4_FromFormat
278 # define PyUnicode_FromFormatV PyUnicodeUCS4_FromFormatV
279 # define PyUnicode_FromObject PyUnicodeUCS4_FromObject
280 # define PyUnicode_FromOrdinal PyUnicodeUCS4_FromOrdinal
281 # define PyUnicode_FromString PyUnicodeUCS4_FromString
282 # define PyUnicode_FromStringAndSize PyUnicodeUCS4_FromStringAndSize
283 # define PyUnicode_FromUnicode PyUnicodeUCS4_FromUnicode
284 # define PyUnicode_FromWideChar PyUnicodeUCS4_FromWideChar
285 # define PyUnicode_GetDefaultEncoding PyUnicodeUCS4_GetDefaultEncoding
286 # define PyUnicode_GetMax PyUnicodeUCS4_GetMax
287 # define PyUnicode_GetSize PyUnicodeUCS4_GetSize
288 # define PyUnicode_Join PyUnicodeUCS4_Join
289 # define PyUnicode_Partition PyUnicodeUCS4_Partition
290 # define PyUnicode_RPartition PyUnicodeUCS4_RPartition
291 # define PyUnicode_RSplit PyUnicodeUCS4_RSplit
292 # define PyUnicode_Replace PyUnicodeUCS4_Replace
293 # define PyUnicode_Resize PyUnicodeUCS4_Resize
294 # define PyUnicode_RichCompare PyUnicodeUCS4_RichCompare
295 # define PyUnicode_SetDefaultEncoding PyUnicodeUCS4_SetDefaultEncoding
296 # define PyUnicode_Split PyUnicodeUCS4_Split
297 # define PyUnicode_Splitlines PyUnicodeUCS4_Splitlines
298 # define PyUnicode_Tailmatch PyUnicodeUCS4_Tailmatch
299 # define PyUnicode_Translate PyUnicodeUCS4_Translate
300 # define PyUnicode_TranslateCharmap PyUnicodeUCS4_TranslateCharmap
301 # define _PyUnicode_AsDefaultEncodedString _PyUnicodeUCS4_AsDefaultEncodedString
302 # define _PyUnicode_Fini _PyUnicodeUCS4_Fini
303 # define _PyUnicode_Init _PyUnicodeUCS4_Init
304 # define _PyUnicode_IsAlpha _PyUnicodeUCS4_IsAlpha
305 # define _PyUnicode_IsDecimalDigit _PyUnicodeUCS4_IsDecimalDigit
306 # define _PyUnicode_IsDigit _PyUnicodeUCS4_IsDigit
307 # define _PyUnicode_IsLinebreak _PyUnicodeUCS4_IsLinebreak
308 # define _PyUnicode_IsLowercase _PyUnicodeUCS4_IsLowercase
309 # define _PyUnicode_IsNumeric _PyUnicodeUCS4_IsNumeric
310 # define _PyUnicode_IsTitlecase _PyUnicodeUCS4_IsTitlecase
311 # define _PyUnicode_IsUppercase _PyUnicodeUCS4_IsUppercase
312 # define _PyUnicode_IsWhitespace _PyUnicodeUCS4_IsWhitespace
313 # define _PyUnicode_ToDecimalDigit _PyUnicodeUCS4_ToDecimalDigit
314 # define _PyUnicode_ToDigit _PyUnicodeUCS4_ToDigit
315 # define _PyUnicode_ToLowercase _PyUnicodeUCS4_ToLowercase
316 # define _PyUnicode_ToNumeric _PyUnicodeUCS4_ToNumeric
317 # define _PyUnicode_ToTitlecase _PyUnicodeUCS4_ToTitlecase
318 # define _PyUnicode_ToUppercase _PyUnicodeUCS4_ToUppercase
319 
320 
321 #endif
322 
323 /* --- Internal Unicode Operations ---------------------------------------- */
324 
325 /* If you want Python to use the compiler's wctype.h functions instead
326    of the ones supplied with Python, define WANT_WCTYPE_FUNCTIONS or
327    configure Python using --with-wctype-functions.  This reduces the
328    interpreter's code size. */
329 
330 #if defined(HAVE_USABLE_WCHAR_T) && defined(WANT_WCTYPE_FUNCTIONS)
331 
332 #include <wctype.h>
333 
334 #define Py_UNICODE_ISSPACE(ch) iswspace(ch)
335 
336 #define Py_UNICODE_ISLOWER(ch) iswlower(ch)
337 #define Py_UNICODE_ISUPPER(ch) iswupper(ch)
338 #define Py_UNICODE_ISTITLE(ch) _PyUnicode_IsTitlecase(ch)
339 #define Py_UNICODE_ISLINEBREAK(ch) _PyUnicode_IsLinebreak(ch)
340 
341 #define Py_UNICODE_TOLOWER(ch) towlower(ch)
342 #define Py_UNICODE_TOUPPER(ch) towupper(ch)
343 #define Py_UNICODE_TOTITLE(ch) _PyUnicode_ToTitlecase(ch)
344 
345 #define Py_UNICODE_ISDECIMAL(ch) _PyUnicode_IsDecimalDigit(ch)
346 #define Py_UNICODE_ISDIGIT(ch) _PyUnicode_IsDigit(ch)
347 #define Py_UNICODE_ISNUMERIC(ch) _PyUnicode_IsNumeric(ch)
348 
349 #define Py_UNICODE_TODECIMAL(ch) _PyUnicode_ToDecimalDigit(ch)
350 #define Py_UNICODE_TODIGIT(ch) _PyUnicode_ToDigit(ch)
351 #define Py_UNICODE_TONUMERIC(ch) _PyUnicode_ToNumeric(ch)
352 
353 #define Py_UNICODE_ISALPHA(ch) iswalpha(ch)
354 
355 #else
356 
357 /* Since splitting on whitespace is an important use case, and
358    whitespace in most situations is solely ASCII whitespace, we
359    optimize for the common case by using a quick look-up table
360    _Py_ascii_whitespace (see below) with an inlined check.
361 
362  */
363 #define Py_UNICODE_ISSPACE(ch) \
364     ((ch) < 128U ? _Py_ascii_whitespace[(ch)] : _PyUnicode_IsWhitespace(ch))
365 
366 #define Py_UNICODE_ISLOWER(ch) _PyUnicode_IsLowercase(ch)
367 #define Py_UNICODE_ISUPPER(ch) _PyUnicode_IsUppercase(ch)
368 #define Py_UNICODE_ISTITLE(ch) _PyUnicode_IsTitlecase(ch)
369 #define Py_UNICODE_ISLINEBREAK(ch) _PyUnicode_IsLinebreak(ch)
370 
371 #define Py_UNICODE_TOLOWER(ch) _PyUnicode_ToLowercase(ch)
372 #define Py_UNICODE_TOUPPER(ch) _PyUnicode_ToUppercase(ch)
373 #define Py_UNICODE_TOTITLE(ch) _PyUnicode_ToTitlecase(ch)
374 
375 #define Py_UNICODE_ISDECIMAL(ch) _PyUnicode_IsDecimalDigit(ch)
376 #define Py_UNICODE_ISDIGIT(ch) _PyUnicode_IsDigit(ch)
377 #define Py_UNICODE_ISNUMERIC(ch) _PyUnicode_IsNumeric(ch)
378 
379 #define Py_UNICODE_TODECIMAL(ch) _PyUnicode_ToDecimalDigit(ch)
380 #define Py_UNICODE_TODIGIT(ch) _PyUnicode_ToDigit(ch)
381 #define Py_UNICODE_TONUMERIC(ch) _PyUnicode_ToNumeric(ch)
382 
383 #define Py_UNICODE_ISALPHA(ch) _PyUnicode_IsAlpha(ch)
384 
385 #endif
386 
387 #define Py_UNICODE_ISALNUM(ch) \
388        (Py_UNICODE_ISALPHA(ch) || \
389     Py_UNICODE_ISDECIMAL(ch) || \
390     Py_UNICODE_ISDIGIT(ch) || \
391     Py_UNICODE_ISNUMERIC(ch))
392 
393 #define Py_UNICODE_COPY(target, source, length)                         \
394     Py_MEMCPY((target), (source), (length)*sizeof(Py_UNICODE))
395 
396 #define Py_UNICODE_FILL(target, value, length) \
397     do {Py_ssize_t i_; Py_UNICODE *t_ = (target); Py_UNICODE v_ = (value);\
398         for (i_ = 0; i_ < (length); i_++) t_[i_] = v_;\
399     } while (0)
400 
401 /* Check if substring matches at given offset.  the offset must be
402    valid, and the substring must not be empty */
403 
404 #define Py_UNICODE_MATCH(string, offset, substring) \
405     ((*((string)->str + (offset)) == *((substring)->str)) && \
406     ((*((string)->str + (offset) + (substring)->length-1) == *((substring)->str + (substring)->length-1))) && \
407      !memcmp((string)->str + (offset), (substring)->str, (substring)->length*sizeof(Py_UNICODE)))
408 
409 #ifdef __cplusplus
410 extern "C" {
411 #endif
412 
413 /* --- Unicode Type ------------------------------------------------------- */
414 
415 typedef struct {
416     PyObject_HEAD
417     Py_ssize_t length;          /* Length of raw Unicode data in buffer */
418     Py_UNICODE *str;            /* Raw Unicode buffer */
419     long hash;                  /* Hash value; -1 if not set */
420     PyObject *defenc;           /* (Default) Encoded version as Python
421                                    string, or NULL; this is used for
422                                    implementing the buffer protocol */
423 } PyUnicodeObject;
424 
425 PyAPI_DATA(PyTypeObject) PyUnicode_Type;
426 
427 #define PyUnicode_Check(op) \
428                  PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_UNICODE_SUBCLASS)
429 #define PyUnicode_CheckExact(op) (Py_TYPE(op) == &PyUnicode_Type)
430 
431 /* Fast access macros */
432 #define PyUnicode_GET_SIZE(op) \
433     (((PyUnicodeObject *)(op))->length)
434 #define PyUnicode_GET_DATA_SIZE(op) \
435     (((PyUnicodeObject *)(op))->length * sizeof(Py_UNICODE))
436 #define PyUnicode_AS_UNICODE(op) \
437     (((PyUnicodeObject *)(op))->str)
438 #define PyUnicode_AS_DATA(op) \
439     ((const char *)((PyUnicodeObject *)(op))->str)
440 
441 /* --- Constants ---------------------------------------------------------- */
442 
443 /* This Unicode character will be used as replacement character during
444    decoding if the errors argument is set to "replace". Note: the
445    Unicode character U+FFFD is the official REPLACEMENT CHARACTER in
446    Unicode 3.0. */
447 
448 #define Py_UNICODE_REPLACEMENT_CHARACTER ((Py_UNICODE) 0xFFFD)
449 
450 /* === Public API ========================================================= */
451 
452 /* --- Plain Py_UNICODE --------------------------------------------------- */
453 
454 /* Create a Unicode Object from the Py_UNICODE buffer u of the given
455    size.
456 
457    u may be NULL which causes the contents to be undefined. It is the
458    user's responsibility to fill in the needed data afterwards. Note
459    that modifying the Unicode object contents after construction is
460    only allowed if u was set to NULL.
461 
462    The buffer is copied into the new object. */
463 
464 PyAPI_FUNC(PyObject*) PyUnicode_FromUnicode(
465     const Py_UNICODE *u,        /* Unicode buffer */
466     Py_ssize_t size             /* size of buffer */
467     );
468 
469 /* Similar to PyUnicode_FromUnicode(), but u points to Latin-1 encoded bytes */
470 PyAPI_FUNC(PyObject*) PyUnicode_FromStringAndSize(
471     const char *u,        /* char buffer */
472     Py_ssize_t size       /* size of buffer */
473     );
474 
475 /* Similar to PyUnicode_FromUnicode(), but u points to null-terminated
476    Latin-1 encoded bytes */
477 PyAPI_FUNC(PyObject*) PyUnicode_FromString(
478     const char *u        /* string */
479     );
480 
481 /* Return a read-only pointer to the Unicode object's internal
482    Py_UNICODE buffer. */
483 
484 PyAPI_FUNC(Py_UNICODE *) PyUnicode_AsUnicode(
485     PyObject *unicode           /* Unicode object */
486     );
487 
488 /* Get the length of the Unicode object. */
489 
490 PyAPI_FUNC(Py_ssize_t) PyUnicode_GetSize(
491     PyObject *unicode           /* Unicode object */
492     );
493 
494 /* Get the maximum ordinal for a Unicode character. */
495 PyAPI_FUNC(Py_UNICODE) PyUnicode_GetMax(void);
496 
497 /* Resize an already allocated Unicode object to the new size length.
498 
499    *unicode is modified to point to the new (resized) object and 0
500    returned on success.
501 
502    This API may only be called by the function which also called the
503    Unicode constructor. The refcount on the object must be 1. Otherwise,
504    an error is returned.
505 
506    Error handling is implemented as follows: an exception is set, -1
507    is returned and *unicode left untouched.
508 
509 */
510 
511 PyAPI_FUNC(int) PyUnicode_Resize(
512     PyObject **unicode,         /* Pointer to the Unicode object */
513     Py_ssize_t length           /* New length */
514     );
515 
516 /* Coerce obj to a Unicode object and return a reference with
517    *incremented* refcount.
518 
519    Coercion is done in the following way:
520 
521    1. String and other char buffer compatible objects are decoded
522       under the assumptions that they contain data using the current
523       default encoding. Decoding is done in "strict" mode.
524 
525    2. All other objects (including Unicode objects) raise an
526       exception.
527 
528    The API returns NULL in case of an error. The caller is responsible
529    for decref'ing the returned objects.
530 
531 */
532 
533 PyAPI_FUNC(PyObject*) PyUnicode_FromEncodedObject(
534     register PyObject *obj,     /* Object */
535     const char *encoding,       /* encoding */
536     const char *errors          /* error handling */
537     );
538 
539 /* Coerce obj to a Unicode object and return a reference with
540    *incremented* refcount.
541 
542    Unicode objects are passed back as-is (subclasses are converted to
543    true Unicode objects), all other objects are delegated to
544    PyUnicode_FromEncodedObject(obj, NULL, "strict") which results in
545    using the default encoding as basis for decoding the object.
546 
547    The API returns NULL in case of an error. The caller is responsible
548    for decref'ing the returned objects.
549 
550 */
551 
552 PyAPI_FUNC(PyObject*) PyUnicode_FromObject(
553     register PyObject *obj      /* Object */
554     );
555 
556 PyAPI_FUNC(PyObject *) PyUnicode_FromFormatV(const char*, va_list);
557 PyAPI_FUNC(PyObject *) PyUnicode_FromFormat(const char*, ...);
558 
559 /* Format the object based on the format_spec, as defined in PEP 3101
560    (Advanced String Formatting). */
561 PyAPI_FUNC(PyObject *) _PyUnicode_FormatAdvanced(PyObject *obj,
562                                                  Py_UNICODE *format_spec,
563                                                  Py_ssize_t format_spec_len);
564 
565 /* --- wchar_t support for platforms which support it --------------------- */
566 
567 #ifdef HAVE_WCHAR_H
568 
569 /* Create a Unicode Object from the whcar_t buffer w of the given
570    size.
571 
572    The buffer is copied into the new object. */
573 
574 PyAPI_FUNC(PyObject*) PyUnicode_FromWideChar(
575     register const wchar_t *w,  /* wchar_t buffer */
576     Py_ssize_t size             /* size of buffer */
577     );
578 
579 /* Copies the Unicode Object contents into the wchar_t buffer w.  At
580    most size wchar_t characters are copied.
581 
582    Note that the resulting wchar_t string may or may not be
583    0-terminated.  It is the responsibility of the caller to make sure
584    that the wchar_t string is 0-terminated in case this is required by
585    the application.
586 
587    Returns the number of wchar_t characters copied (excluding a
588    possibly trailing 0-termination character) or -1 in case of an
589    error. */
590 
591 PyAPI_FUNC(Py_ssize_t) PyUnicode_AsWideChar(
592     PyUnicodeObject *unicode,   /* Unicode object */
593     register wchar_t *w,        /* wchar_t buffer */
594     Py_ssize_t size             /* size of buffer */
595     );
596 
597 #endif
598 
599 /* --- Unicode ordinals --------------------------------------------------- */
600 
601 /* Create a Unicode Object from the given Unicode code point ordinal.
602 
603    The ordinal must be in range(0x10000) on narrow Python builds
604    (UCS2), and range(0x110000) on wide builds (UCS4). A ValueError is
605    raised in case it is not.
606 
607 */
608 
609 PyAPI_FUNC(PyObject*) PyUnicode_FromOrdinal(int ordinal);
610 
611 /* --- Free-list management ----------------------------------------------- */
612 
613 /* Clear the free list used by the Unicode implementation.
614 
615    This can be used to release memory used for objects on the free
616    list back to the Python memory allocator.
617 
618 */
619 
620 PyAPI_FUNC(int) PyUnicode_ClearFreeList(void);
621 
622 /* === Builtin Codecs =====================================================
623 
624    Many of these APIs take two arguments encoding and errors. These
625    parameters encoding and errors have the same semantics as the ones
626    of the builtin unicode() API.
627 
628    Setting encoding to NULL causes the default encoding to be used.
629 
630    Error handling is set by errors which may also be set to NULL
631    meaning to use the default handling defined for the codec. Default
632    error handling for all builtin codecs is "strict" (ValueErrors are
633    raised).
634 
635    The codecs all use a similar interface. Only deviation from the
636    generic ones are documented.
637 
638 */
639 
640 /* --- Manage the default encoding ---------------------------------------- */
641 
642 /* Return a Python string holding the default encoded value of the
643    Unicode object.
644 
645    The resulting string is cached in the Unicode object for subsequent
646    usage by this function. The cached version is needed to implement
647    the character buffer interface and will live (at least) as long as
648    the Unicode object itself.
649 
650    The refcount of the string is *not* incremented.
651 
652    *** Exported for internal use by the interpreter only !!! ***
653 
654 */
655 
656 PyAPI_FUNC(PyObject *) _PyUnicode_AsDefaultEncodedString(
657     PyObject *, const char *);
658 
659 /* Returns the currently active default encoding.
660 
661    The default encoding is currently implemented as run-time settable
662    process global.  This may change in future versions of the
663    interpreter to become a parameter which is managed on a per-thread
664    basis.
665 
666  */
667 
668 PyAPI_FUNC(const char*) PyUnicode_GetDefaultEncoding(void);
669 
670 /* Sets the currently active default encoding.
671 
672    Returns 0 on success, -1 in case of an error.
673 
674  */
675 
676 PyAPI_FUNC(int) PyUnicode_SetDefaultEncoding(
677     const char *encoding        /* Encoding name in standard form */
678     );
679 
680 /* --- Generic Codecs ----------------------------------------------------- */
681 
682 /* Create a Unicode object by decoding the encoded string s of the
683    given size. */
684 
685 PyAPI_FUNC(PyObject*) PyUnicode_Decode(
686     const char *s,              /* encoded string */
687     Py_ssize_t size,            /* size of buffer */
688     const char *encoding,       /* encoding */
689     const char *errors          /* error handling */
690     );
691 
692 /* Encodes a Py_UNICODE buffer of the given size and returns a
693    Python string object. */
694 
695 PyAPI_FUNC(PyObject*) PyUnicode_Encode(
696     const Py_UNICODE *s,        /* Unicode char buffer */
697     Py_ssize_t size,            /* number of Py_UNICODE chars to encode */
698     const char *encoding,       /* encoding */
699     const char *errors          /* error handling */
700     );
701 
702 /* Encodes a Unicode object and returns the result as Python
703    object. */
704 
705 PyAPI_FUNC(PyObject*) PyUnicode_AsEncodedObject(
706     PyObject *unicode,          /* Unicode object */
707     const char *encoding,       /* encoding */
708     const char *errors          /* error handling */
709     );
710 
711 /* Encodes a Unicode object and returns the result as Python string
712    object. */
713 
714 PyAPI_FUNC(PyObject*) PyUnicode_AsEncodedString(
715     PyObject *unicode,          /* Unicode object */
716     const char *encoding,       /* encoding */
717     const char *errors          /* error handling */
718     );
719 
720 PyAPI_FUNC(PyObject*) PyUnicode_BuildEncodingMap(
721     PyObject* string            /* 256 character map */
722    );
723 
724 
725 /* --- UTF-7 Codecs ------------------------------------------------------- */
726 
727 PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF7(
728     const char *string,         /* UTF-7 encoded string */
729     Py_ssize_t length,          /* size of string */
730     const char *errors          /* error handling */
731     );
732 
733 PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF7Stateful(
734     const char *string,         /* UTF-7 encoded string */
735     Py_ssize_t length,          /* size of string */
736     const char *errors,         /* error handling */
737     Py_ssize_t *consumed        /* bytes consumed */
738     );
739 
740 PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF7(
741     const Py_UNICODE *data,     /* Unicode char buffer */
742     Py_ssize_t length,                  /* number of Py_UNICODE chars to encode */
743     int base64SetO,             /* Encode RFC2152 Set O characters in base64 */
744     int base64WhiteSpace,       /* Encode whitespace (sp, ht, nl, cr) in base64 */
745     const char *errors          /* error handling */
746     );
747 
748 /* --- UTF-8 Codecs ------------------------------------------------------- */
749 
750 PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF8(
751     const char *string,         /* UTF-8 encoded string */
752     Py_ssize_t length,          /* size of string */
753     const char *errors          /* error handling */
754     );
755 
756 PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF8Stateful(
757     const char *string,         /* UTF-8 encoded string */
758     Py_ssize_t length,          /* size of string */
759     const char *errors,         /* error handling */
760     Py_ssize_t *consumed                /* bytes consumed */
761     );
762 
763 PyAPI_FUNC(PyObject*) PyUnicode_AsUTF8String(
764     PyObject *unicode           /* Unicode object */
765     );
766 
767 PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF8(
768     const Py_UNICODE *data,     /* Unicode char buffer */
769     Py_ssize_t length,                  /* number of Py_UNICODE chars to encode */
770     const char *errors          /* error handling */
771     );
772 
773 /* --- UTF-32 Codecs ------------------------------------------------------ */
774 
775 /* Decodes length bytes from a UTF-32 encoded buffer string and returns
776    the corresponding Unicode object.
777 
778    errors (if non-NULL) defines the error handling. It defaults
779    to "strict".
780 
781    If byteorder is non-NULL, the decoder starts decoding using the
782    given byte order:
783 
784     *byteorder == -1: little endian
785     *byteorder == 0:  native order
786     *byteorder == 1:  big endian
787 
788    In native mode, the first four bytes of the stream are checked for a
789    BOM mark. If found, the BOM mark is analysed, the byte order
790    adjusted and the BOM skipped.  In the other modes, no BOM mark
791    interpretation is done. After completion, *byteorder is set to the
792    current byte order at the end of input data.
793 
794    If byteorder is NULL, the codec starts in native order mode.
795 
796 */
797 
798 PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF32(
799     const char *string,         /* UTF-32 encoded string */
800     Py_ssize_t length,          /* size of string */
801     const char *errors,         /* error handling */
802     int *byteorder              /* pointer to byteorder to use
803                                    0=native;-1=LE,1=BE; updated on
804                                    exit */
805     );
806 
807 PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF32Stateful(
808     const char *string,         /* UTF-32 encoded string */
809     Py_ssize_t length,          /* size of string */
810     const char *errors,         /* error handling */
811     int *byteorder,             /* pointer to byteorder to use
812                                    0=native;-1=LE,1=BE; updated on
813                                    exit */
814     Py_ssize_t *consumed        /* bytes consumed */
815     );
816 
817 /* Returns a Python string using the UTF-32 encoding in native byte
818    order. The string always starts with a BOM mark.  */
819 
820 PyAPI_FUNC(PyObject*) PyUnicode_AsUTF32String(
821     PyObject *unicode           /* Unicode object */
822     );
823 
824 /* Returns a Python string object holding the UTF-32 encoded value of
825    the Unicode data.
826 
827    If byteorder is not 0, output is written according to the following
828    byte order:
829 
830    byteorder == -1: little endian
831    byteorder == 0:  native byte order (writes a BOM mark)
832    byteorder == 1:  big endian
833 
834    If byteorder is 0, the output string will always start with the
835    Unicode BOM mark (U+FEFF). In the other two modes, no BOM mark is
836    prepended.
837 
838 */
839 
840 PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF32(
841     const Py_UNICODE *data,     /* Unicode char buffer */
842     Py_ssize_t length,          /* number of Py_UNICODE chars to encode */
843     const char *errors,         /* error handling */
844     int byteorder               /* byteorder to use 0=BOM+native;-1=LE,1=BE */
845     );
846 
847 /* --- UTF-16 Codecs ------------------------------------------------------ */
848 
849 /* Decodes length bytes from a UTF-16 encoded buffer string and returns
850    the corresponding Unicode object.
851 
852    errors (if non-NULL) defines the error handling. It defaults
853    to "strict".
854 
855    If byteorder is non-NULL, the decoder starts decoding using the
856    given byte order:
857 
858     *byteorder == -1: little endian
859     *byteorder == 0:  native order
860     *byteorder == 1:  big endian
861 
862    In native mode, the first two bytes of the stream are checked for a
863    BOM mark. If found, the BOM mark is analysed, the byte order
864    adjusted and the BOM skipped.  In the other modes, no BOM mark
865    interpretation is done. After completion, *byteorder is set to the
866    current byte order at the end of input data.
867 
868    If byteorder is NULL, the codec starts in native order mode.
869 
870 */
871 
872 PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF16(
873     const char *string,         /* UTF-16 encoded string */
874     Py_ssize_t length,          /* size of string */
875     const char *errors,         /* error handling */
876     int *byteorder              /* pointer to byteorder to use
877                                    0=native;-1=LE,1=BE; updated on
878                                    exit */
879     );
880 
881 PyAPI_FUNC(PyObject*) PyUnicode_DecodeUTF16Stateful(
882     const char *string,         /* UTF-16 encoded string */
883     Py_ssize_t length,          /* size of string */
884     const char *errors,         /* error handling */
885     int *byteorder,             /* pointer to byteorder to use
886                                    0=native;-1=LE,1=BE; updated on
887                                    exit */
888     Py_ssize_t *consumed                /* bytes consumed */
889     );
890 
891 /* Returns a Python string using the UTF-16 encoding in native byte
892    order. The string always starts with a BOM mark.  */
893 
894 PyAPI_FUNC(PyObject*) PyUnicode_AsUTF16String(
895     PyObject *unicode           /* Unicode object */
896     );
897 
898 /* Returns a Python string object holding the UTF-16 encoded value of
899    the Unicode data.
900 
901    If byteorder is not 0, output is written according to the following
902    byte order:
903 
904    byteorder == -1: little endian
905    byteorder == 0:  native byte order (writes a BOM mark)
906    byteorder == 1:  big endian
907 
908    If byteorder is 0, the output string will always start with the
909    Unicode BOM mark (U+FEFF). In the other two modes, no BOM mark is
910    prepended.
911 
912    Note that Py_UNICODE data is being interpreted as UTF-16 reduced to
913    UCS-2. This trick makes it possible to add full UTF-16 capabilities
914    at a later point without compromising the APIs.
915 
916 */
917 
918 PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF16(
919     const Py_UNICODE *data,     /* Unicode char buffer */
920     Py_ssize_t length,                  /* number of Py_UNICODE chars to encode */
921     const char *errors,         /* error handling */
922     int byteorder               /* byteorder to use 0=BOM+native;-1=LE,1=BE */
923     );
924 
925 /* --- Unicode-Escape Codecs ---------------------------------------------- */
926 
927 PyAPI_FUNC(PyObject*) PyUnicode_DecodeUnicodeEscape(
928     const char *string,         /* Unicode-Escape encoded string */
929     Py_ssize_t length,          /* size of string */
930     const char *errors          /* error handling */
931     );
932 
933 PyAPI_FUNC(PyObject*) PyUnicode_AsUnicodeEscapeString(
934     PyObject *unicode           /* Unicode object */
935     );
936 
937 PyAPI_FUNC(PyObject*) PyUnicode_EncodeUnicodeEscape(
938     const Py_UNICODE *data,     /* Unicode char buffer */
939     Py_ssize_t length                   /* Number of Py_UNICODE chars to encode */
940     );
941 
942 /* --- Raw-Unicode-Escape Codecs ------------------------------------------ */
943 
944 PyAPI_FUNC(PyObject*) PyUnicode_DecodeRawUnicodeEscape(
945     const char *string,         /* Raw-Unicode-Escape encoded string */
946     Py_ssize_t length,          /* size of string */
947     const char *errors          /* error handling */
948     );
949 
950 PyAPI_FUNC(PyObject*) PyUnicode_AsRawUnicodeEscapeString(
951     PyObject *unicode           /* Unicode object */
952     );
953 
954 PyAPI_FUNC(PyObject*) PyUnicode_EncodeRawUnicodeEscape(
955     const Py_UNICODE *data,     /* Unicode char buffer */
956     Py_ssize_t length                   /* Number of Py_UNICODE chars to encode */
957     );
958 
959 /* --- Unicode Internal Codec ---------------------------------------------
960 
961     Only for internal use in _codecsmodule.c */
962 
963 PyObject *_PyUnicode_DecodeUnicodeInternal(
964     const char *string,
965     Py_ssize_t length,
966     const char *errors
967     );
968 
969 /* --- Latin-1 Codecs -----------------------------------------------------
970 
971    Note: Latin-1 corresponds to the first 256 Unicode ordinals.
972 
973 */
974 
975 PyAPI_FUNC(PyObject*) PyUnicode_DecodeLatin1(
976     const char *string,         /* Latin-1 encoded string */
977     Py_ssize_t length,          /* size of string */
978     const char *errors          /* error handling */
979     );
980 
981 PyAPI_FUNC(PyObject*) PyUnicode_AsLatin1String(
982     PyObject *unicode           /* Unicode object */
983     );
984 
985 PyAPI_FUNC(PyObject*) PyUnicode_EncodeLatin1(
986     const Py_UNICODE *data,     /* Unicode char buffer */
987     Py_ssize_t length,                  /* Number of Py_UNICODE chars to encode */
988     const char *errors          /* error handling */
989     );
990 
991 /* --- ASCII Codecs -------------------------------------------------------
992 
993    Only 7-bit ASCII data is excepted. All other codes generate errors.
994 
995 */
996 
997 PyAPI_FUNC(PyObject*) PyUnicode_DecodeASCII(
998     const char *string,         /* ASCII encoded string */
999     Py_ssize_t length,          /* size of string */
1000     const char *errors          /* error handling */
1001     );
1002 
1003 PyAPI_FUNC(PyObject*) PyUnicode_AsASCIIString(
1004     PyObject *unicode           /* Unicode object */
1005     );
1006 
1007 PyAPI_FUNC(PyObject*) PyUnicode_EncodeASCII(
1008     const Py_UNICODE *data,     /* Unicode char buffer */
1009     Py_ssize_t length,                  /* Number of Py_UNICODE chars to encode */
1010     const char *errors          /* error handling */
1011     );
1012 
1013 /* --- Character Map Codecs -----------------------------------------------
1014 
1015    This codec uses mappings to encode and decode characters.
1016 
1017    Decoding mappings must map single string characters to single
1018    Unicode characters, integers (which are then interpreted as Unicode
1019    ordinals) or None (meaning "undefined mapping" and causing an
1020    error).
1021 
1022    Encoding mappings must map single Unicode characters to single
1023    string characters, integers (which are then interpreted as Latin-1
1024    ordinals) or None (meaning "undefined mapping" and causing an
1025    error).
1026 
1027    If a character lookup fails with a LookupError, the character is
1028    copied as-is meaning that its ordinal value will be interpreted as
1029    Unicode or Latin-1 ordinal resp. Because of this mappings only need
1030    to contain those mappings which map characters to different code
1031    points.
1032 
1033 */
1034 
1035 PyAPI_FUNC(PyObject*) PyUnicode_DecodeCharmap(
1036     const char *string,         /* Encoded string */
1037     Py_ssize_t length,          /* size of string */
1038     PyObject *mapping,          /* character mapping
1039                                    (char ordinal -> unicode ordinal) */
1040     const char *errors          /* error handling */
1041     );
1042 
1043 PyAPI_FUNC(PyObject*) PyUnicode_AsCharmapString(
1044     PyObject *unicode,          /* Unicode object */
1045     PyObject *mapping           /* character mapping
1046                                    (unicode ordinal -> char ordinal) */
1047     );
1048 
1049 PyAPI_FUNC(PyObject*) PyUnicode_EncodeCharmap(
1050     const Py_UNICODE *data,     /* Unicode char buffer */
1051     Py_ssize_t length,          /* Number of Py_UNICODE chars to encode */
1052     PyObject *mapping,          /* character mapping
1053                                    (unicode ordinal -> char ordinal) */
1054     const char *errors          /* error handling */
1055     );
1056 
1057 /* Translate a Py_UNICODE buffer of the given length by applying a
1058    character mapping table to it and return the resulting Unicode
1059    object.
1060 
1061    The mapping table must map Unicode ordinal integers to Unicode
1062    ordinal integers or None (causing deletion of the character).
1063 
1064    Mapping tables may be dictionaries or sequences. Unmapped character
1065    ordinals (ones which cause a LookupError) are left untouched and
1066    are copied as-is.
1067 
1068 */
1069 
1070 PyAPI_FUNC(PyObject *) PyUnicode_TranslateCharmap(
1071     const Py_UNICODE *data,     /* Unicode char buffer */
1072     Py_ssize_t length,                  /* Number of Py_UNICODE chars to encode */
1073     PyObject *table,            /* Translate table */
1074     const char *errors          /* error handling */
1075     );
1076 
1077 #ifdef MS_WIN32
1078 
1079 /* --- MBCS codecs for Windows -------------------------------------------- */
1080 
1081 PyAPI_FUNC(PyObject*) PyUnicode_DecodeMBCS(
1082     const char *string,         /* MBCS encoded string */
1083     Py_ssize_t length,              /* size of string */
1084     const char *errors          /* error handling */
1085     );
1086 
1087 PyAPI_FUNC(PyObject*) PyUnicode_DecodeMBCSStateful(
1088     const char *string,         /* MBCS encoded string */
1089     Py_ssize_t length,          /* size of string */
1090     const char *errors,         /* error handling */
1091     Py_ssize_t *consumed        /* bytes consumed */
1092     );
1093 
1094 PyAPI_FUNC(PyObject*) PyUnicode_AsMBCSString(
1095     PyObject *unicode           /* Unicode object */
1096     );
1097 
1098 PyAPI_FUNC(PyObject*) PyUnicode_EncodeMBCS(
1099     const Py_UNICODE *data,     /* Unicode char buffer */
1100     Py_ssize_t length,              /* Number of Py_UNICODE chars to encode */
1101     const char *errors          /* error handling */
1102     );
1103 
1104 #endif /* MS_WIN32 */
1105 
1106 /* --- Decimal Encoder ---------------------------------------------------- */
1107 
1108 /* Takes a Unicode string holding a decimal value and writes it into
1109    an output buffer using standard ASCII digit codes.
1110 
1111    The output buffer has to provide at least length+1 bytes of storage
1112    area. The output string is 0-terminated.
1113 
1114    The encoder converts whitespace to ' ', decimal characters to their
1115    corresponding ASCII digit and all other Latin-1 characters except
1116    \0 as-is. Characters outside this range (Unicode ordinals 1-256)
1117    are treated as errors. This includes embedded NULL bytes.
1118 
1119    Error handling is defined by the errors argument:
1120 
1121       NULL or "strict": raise a ValueError
1122       "ignore": ignore the wrong characters (these are not copied to the
1123                 output buffer)
1124       "replace": replaces illegal characters with '?'
1125 
1126    Returns 0 on success, -1 on failure.
1127 
1128 */
1129 
1130 PyAPI_FUNC(int) PyUnicode_EncodeDecimal(
1131     Py_UNICODE *s,              /* Unicode buffer */
1132     Py_ssize_t length,                  /* Number of Py_UNICODE chars to encode */
1133     char *output,               /* Output buffer; must have size >= length */
1134     const char *errors          /* error handling */
1135     );
1136 
1137 /* --- Methods & Slots ----------------------------------------------------
1138 
1139    These are capable of handling Unicode objects and strings on input
1140    (we refer to them as strings in the descriptions) and return
1141    Unicode objects or integers as appropriate. */
1142 
1143 /* Concat two strings giving a new Unicode string. */
1144 
1145 PyAPI_FUNC(PyObject*) PyUnicode_Concat(
1146     PyObject *left,             /* Left string */
1147     PyObject *right             /* Right string */
1148     );
1149 
1150 /* Split a string giving a list of Unicode strings.
1151 
1152    If sep is NULL, splitting will be done at all whitespace
1153    substrings. Otherwise, splits occur at the given separator.
1154 
1155    At most maxsplit splits will be done. If negative, no limit is set.
1156 
1157    Separators are not included in the resulting list.
1158 
1159 */
1160 
1161 PyAPI_FUNC(PyObject*) PyUnicode_Split(
1162     PyObject *s,                /* String to split */
1163     PyObject *sep,              /* String separator */
1164     Py_ssize_t maxsplit         /* Maxsplit count */
1165     );
1166 
1167 /* Dito, but split at line breaks.
1168 
1169    CRLF is considered to be one line break. Line breaks are not
1170    included in the resulting list. */
1171 
1172 PyAPI_FUNC(PyObject*) PyUnicode_Splitlines(
1173     PyObject *s,                /* String to split */
1174     int keepends                /* If true, line end markers are included */
1175     );
1176 
1177 /* Partition a string using a given separator. */
1178 
1179 PyAPI_FUNC(PyObject*) PyUnicode_Partition(
1180     PyObject *s,                /* String to partition */
1181     PyObject *sep               /* String separator */
1182     );
1183 
1184 /* Partition a string using a given separator, searching from the end of the
1185    string. */
1186 
1187 PyAPI_FUNC(PyObject*) PyUnicode_RPartition(
1188     PyObject *s,                /* String to partition */
1189     PyObject *sep               /* String separator */
1190     );
1191 
1192 /* Split a string giving a list of Unicode strings.
1193 
1194    If sep is NULL, splitting will be done at all whitespace
1195    substrings. Otherwise, splits occur at the given separator.
1196 
1197    At most maxsplit splits will be done. But unlike PyUnicode_Split
1198    PyUnicode_RSplit splits from the end of the string. If negative,
1199    no limit is set.
1200 
1201    Separators are not included in the resulting list.
1202 
1203 */
1204 
1205 PyAPI_FUNC(PyObject*) PyUnicode_RSplit(
1206     PyObject *s,                /* String to split */
1207     PyObject *sep,              /* String separator */
1208     Py_ssize_t maxsplit         /* Maxsplit count */
1209     );
1210 
1211 /* Translate a string by applying a character mapping table to it and
1212    return the resulting Unicode object.
1213 
1214    The mapping table must map Unicode ordinal integers to Unicode
1215    ordinal integers or None (causing deletion of the character).
1216 
1217    Mapping tables may be dictionaries or sequences. Unmapped character
1218    ordinals (ones which cause a LookupError) are left untouched and
1219    are copied as-is.
1220 
1221 */
1222 
1223 PyAPI_FUNC(PyObject *) PyUnicode_Translate(
1224     PyObject *str,              /* String */
1225     PyObject *table,            /* Translate table */
1226     const char *errors          /* error handling */
1227     );
1228 
1229 /* Join a sequence of strings using the given separator and return
1230    the resulting Unicode string. */
1231 
1232 PyAPI_FUNC(PyObject*) PyUnicode_Join(
1233     PyObject *separator,        /* Separator string */
1234     PyObject *seq               /* Sequence object */
1235     );
1236 
1237 /* Return 1 if substr matches str[start:end] at the given tail end, 0
1238    otherwise. */
1239 
1240 PyAPI_FUNC(Py_ssize_t) PyUnicode_Tailmatch(
1241     PyObject *str,              /* String */
1242     PyObject *substr,           /* Prefix or Suffix string */
1243     Py_ssize_t start,           /* Start index */
1244     Py_ssize_t end,             /* Stop index */
1245     int direction               /* Tail end: -1 prefix, +1 suffix */
1246     );
1247 
1248 /* Return the first position of substr in str[start:end] using the
1249    given search direction or -1 if not found. -2 is returned in case
1250    an error occurred and an exception is set. */
1251 
1252 PyAPI_FUNC(Py_ssize_t) PyUnicode_Find(
1253     PyObject *str,              /* String */
1254     PyObject *substr,           /* Substring to find */
1255     Py_ssize_t start,           /* Start index */
1256     Py_ssize_t end,             /* Stop index */
1257     int direction               /* Find direction: +1 forward, -1 backward */
1258     );
1259 
1260 /* Count the number of occurrences of substr in str[start:end]. */
1261 
1262 PyAPI_FUNC(Py_ssize_t) PyUnicode_Count(
1263     PyObject *str,              /* String */
1264     PyObject *substr,           /* Substring to count */
1265     Py_ssize_t start,           /* Start index */
1266     Py_ssize_t end              /* Stop index */
1267     );
1268 
1269 /* Replace at most maxcount occurrences of substr in str with replstr
1270    and return the resulting Unicode object. */
1271 
1272 PyAPI_FUNC(PyObject *) PyUnicode_Replace(
1273     PyObject *str,              /* String */
1274     PyObject *substr,           /* Substring to find */
1275     PyObject *replstr,          /* Substring to replace */
1276     Py_ssize_t maxcount         /* Max. number of replacements to apply;
1277                                    -1 = all */
1278     );
1279 
1280 /* Compare two strings and return -1, 0, 1 for less than, equal,
1281    greater than resp. */
1282 
1283 PyAPI_FUNC(int) PyUnicode_Compare(
1284     PyObject *left,             /* Left string */
1285     PyObject *right             /* Right string */
1286     );
1287 
1288 /* Rich compare two strings and return one of the following:
1289 
1290    - NULL in case an exception was raised
1291    - Py_True or Py_False for successful comparisons
1292    - Py_NotImplemented in case the type combination is unknown
1293 
1294    Note that Py_EQ and Py_NE comparisons can cause a UnicodeWarning in
1295    case the conversion of the arguments to Unicode fails with a
1296    UnicodeDecodeError.
1297 
1298    Possible values for op:
1299 
1300      Py_GT, Py_GE, Py_EQ, Py_NE, Py_LT, Py_LE
1301 
1302 */
1303 
1304 PyAPI_FUNC(PyObject *) PyUnicode_RichCompare(
1305     PyObject *left,             /* Left string */
1306     PyObject *right,            /* Right string */
1307     int op                      /* Operation: Py_EQ, Py_NE, Py_GT, etc. */
1308     );
1309 
1310 /* Apply an argument tuple or dictionary to a format string and return
1311    the resulting Unicode string. */
1312 
1313 PyAPI_FUNC(PyObject *) PyUnicode_Format(
1314     PyObject *format,           /* Format string */
1315     PyObject *args              /* Argument tuple or dictionary */
1316     );
1317 
1318 /* Checks whether element is contained in container and return 1/0
1319    accordingly.
1320 
1321    element has to coerce to a one element Unicode string. -1 is
1322    returned in case of an error. */
1323 
1324 PyAPI_FUNC(int) PyUnicode_Contains(
1325     PyObject *container,        /* Container string */
1326     PyObject *element           /* Element string */
1327     );
1328 
1329 /* Externally visible for str.strip(unicode) */
1330 PyAPI_FUNC(PyObject *) _PyUnicode_XStrip(
1331     PyUnicodeObject *self,
1332     int striptype,
1333     PyObject *sepobj
1334     );
1335 
1336 /* === Characters Type APIs =============================================== */
1337 
1338 /* Helper array used by Py_UNICODE_ISSPACE(). */
1339 
1340 PyAPI_DATA(const unsigned char) _Py_ascii_whitespace[];
1341 
1342 /* These should not be used directly. Use the Py_UNICODE_IS* and
1343    Py_UNICODE_TO* macros instead.
1344 
1345    These APIs are implemented in Objects/unicodectype.c.
1346 
1347 */
1348 
1349 PyAPI_FUNC(int) _PyUnicode_IsLowercase(
1350     Py_UNICODE ch       /* Unicode character */
1351     );
1352 
1353 PyAPI_FUNC(int) _PyUnicode_IsUppercase(
1354     Py_UNICODE ch       /* Unicode character */
1355     );
1356 
1357 PyAPI_FUNC(int) _PyUnicode_IsTitlecase(
1358     Py_UNICODE ch       /* Unicode character */
1359     );
1360 
1361 PyAPI_FUNC(int) _PyUnicode_IsWhitespace(
1362     const Py_UNICODE ch         /* Unicode character */
1363     );
1364 
1365 PyAPI_FUNC(int) _PyUnicode_IsLinebreak(
1366     const Py_UNICODE ch         /* Unicode character */
1367     );
1368 
1369 PyAPI_FUNC(Py_UNICODE) _PyUnicode_ToLowercase(
1370     Py_UNICODE ch       /* Unicode character */
1371     );
1372 
1373 PyAPI_FUNC(Py_UNICODE) _PyUnicode_ToUppercase(
1374     Py_UNICODE ch       /* Unicode character */
1375     );
1376 
1377 PyAPI_FUNC(Py_UNICODE) _PyUnicode_ToTitlecase(
1378     Py_UNICODE ch       /* Unicode character */
1379     );
1380 
1381 PyAPI_FUNC(int) _PyUnicode_ToDecimalDigit(
1382     Py_UNICODE ch       /* Unicode character */
1383     );
1384 
1385 PyAPI_FUNC(int) _PyUnicode_ToDigit(
1386     Py_UNICODE ch       /* Unicode character */
1387     );
1388 
1389 PyAPI_FUNC(double) _PyUnicode_ToNumeric(
1390     Py_UNICODE ch       /* Unicode character */
1391     );
1392 
1393 PyAPI_FUNC(int) _PyUnicode_IsDecimalDigit(
1394     Py_UNICODE ch       /* Unicode character */
1395     );
1396 
1397 PyAPI_FUNC(int) _PyUnicode_IsDigit(
1398     Py_UNICODE ch       /* Unicode character */
1399     );
1400 
1401 PyAPI_FUNC(int) _PyUnicode_IsNumeric(
1402     Py_UNICODE ch       /* Unicode character */
1403     );
1404 
1405 PyAPI_FUNC(int) _PyUnicode_IsAlpha(
1406     Py_UNICODE ch       /* Unicode character */
1407     );
1408 
1409 #ifdef __cplusplus
1410 }
1411 #endif
1412 #endif /* Py_USING_UNICODE */
1413 #endif /* !Py_UNICODEOBJECT_H */
1414