1 /*
2 * cjkcodecs.h: common header for cjkcodecs
3 *
4 * Written by Hye-Shik Chang <perky@FreeBSD.org>
5 */
6
7 #ifndef _CJKCODECS_H_
8 #define _CJKCODECS_H_
9
10 #define PY_SSIZE_T_CLEAN
11 #include "Python.h"
12 #include "multibytecodec.h"
13
14
15 /* a unicode "undefined" code point */
16 #define UNIINV 0xFFFE
17
18 /* internal-use DBCS code points which aren't used by any charsets */
19 #define NOCHAR 0xFFFF
20 #define MULTIC 0xFFFE
21 #define DBCINV 0xFFFD
22
23 /* shorter macros to save source size of mapping tables */
24 #define U UNIINV
25 #define N NOCHAR
26 #define M MULTIC
27 #define D DBCINV
28
29 struct dbcs_index {
30 const ucs2_t *map;
31 unsigned char bottom, top;
32 };
33 typedef struct dbcs_index decode_map;
34
35 struct widedbcs_index {
36 const Py_UCS4 *map;
37 unsigned char bottom, top;
38 };
39 typedef struct widedbcs_index widedecode_map;
40
41 struct unim_index {
42 const DBCHAR *map;
43 unsigned char bottom, top;
44 };
45 typedef struct unim_index encode_map;
46
47 struct unim_index_bytebased {
48 const unsigned char *map;
49 unsigned char bottom, top;
50 };
51
52 struct dbcs_map {
53 const char *charset;
54 const struct unim_index *encmap;
55 const struct dbcs_index *decmap;
56 };
57
58 struct pair_encodemap {
59 Py_UCS4 uniseq;
60 DBCHAR code;
61 };
62
63 static const MultibyteCodec *codec_list;
64 static const struct dbcs_map *mapping_list;
65
66 #define CODEC_INIT(encoding) \
67 static int encoding##_codec_init(const void *config)
68
69 #define ENCODER_INIT(encoding) \
70 static int encoding##_encode_init( \
71 MultibyteCodec_State *state, const void *config)
72 #define ENCODER(encoding) \
73 static Py_ssize_t encoding##_encode( \
74 MultibyteCodec_State *state, const void *config, \
75 int kind, void *data, \
76 Py_ssize_t *inpos, Py_ssize_t inlen, \
77 unsigned char **outbuf, Py_ssize_t outleft, int flags)
78 #define ENCODER_RESET(encoding) \
79 static Py_ssize_t encoding##_encode_reset( \
80 MultibyteCodec_State *state, const void *config, \
81 unsigned char **outbuf, Py_ssize_t outleft)
82
83 #define DECODER_INIT(encoding) \
84 static int encoding##_decode_init( \
85 MultibyteCodec_State *state, const void *config)
86 #define DECODER(encoding) \
87 static Py_ssize_t encoding##_decode( \
88 MultibyteCodec_State *state, const void *config, \
89 const unsigned char **inbuf, Py_ssize_t inleft, \
90 _PyUnicodeWriter *writer)
91 #define DECODER_RESET(encoding) \
92 static Py_ssize_t encoding##_decode_reset( \
93 MultibyteCodec_State *state, const void *config)
94
95 #define NEXT_IN(i) \
96 do { \
97 (*inbuf) += (i); \
98 (inleft) -= (i); \
99 } while (0)
100 #define NEXT_INCHAR(i) \
101 do { \
102 (*inpos) += (i); \
103 } while (0)
104 #define NEXT_OUT(o) \
105 do { \
106 (*outbuf) += (o); \
107 (outleft) -= (o); \
108 } while (0)
109 #define NEXT(i, o) \
110 do { \
111 NEXT_INCHAR(i); \
112 NEXT_OUT(o); \
113 } while (0)
114
115 #define REQUIRE_INBUF(n) \
116 do { \
117 if (inleft < (n)) \
118 return MBERR_TOOFEW; \
119 } while (0)
120
121 #define REQUIRE_OUTBUF(n) \
122 do { \
123 if (outleft < (n)) \
124 return MBERR_TOOSMALL; \
125 } while (0)
126
127 #define INBYTE1 ((*inbuf)[0])
128 #define INBYTE2 ((*inbuf)[1])
129 #define INBYTE3 ((*inbuf)[2])
130 #define INBYTE4 ((*inbuf)[3])
131
132 #define INCHAR1 (PyUnicode_READ(kind, data, *inpos))
133 #define INCHAR2 (PyUnicode_READ(kind, data, *inpos + 1))
134
135 #define OUTCHAR(c) \
136 do { \
137 if (_PyUnicodeWriter_WriteChar(writer, (c)) < 0) \
138 return MBERR_EXCEPTION; \
139 } while (0)
140
141 #define OUTCHAR2(c1, c2) \
142 do { \
143 Py_UCS4 _c1 = (c1); \
144 Py_UCS4 _c2 = (c2); \
145 if (_PyUnicodeWriter_Prepare(writer, 2, Py_MAX(_c1, c2)) < 0) \
146 return MBERR_EXCEPTION; \
147 PyUnicode_WRITE(writer->kind, writer->data, writer->pos, _c1); \
148 PyUnicode_WRITE(writer->kind, writer->data, writer->pos + 1, _c2); \
149 writer->pos += 2; \
150 } while (0)
151
152 #define OUTBYTE1(c) \
153 do { ((*outbuf)[0]) = (c); } while (0)
154 #define OUTBYTE2(c) \
155 do { ((*outbuf)[1]) = (c); } while (0)
156 #define OUTBYTE3(c) \
157 do { ((*outbuf)[2]) = (c); } while (0)
158 #define OUTBYTE4(c) \
159 do { ((*outbuf)[3]) = (c); } while (0)
160
161 #define WRITEBYTE1(c1) \
162 do { \
163 REQUIRE_OUTBUF(1); \
164 (*outbuf)[0] = (c1); \
165 } while (0)
166 #define WRITEBYTE2(c1, c2) \
167 do { \
168 REQUIRE_OUTBUF(2); \
169 (*outbuf)[0] = (c1); \
170 (*outbuf)[1] = (c2); \
171 } while (0)
172 #define WRITEBYTE3(c1, c2, c3) \
173 do { \
174 REQUIRE_OUTBUF(3); \
175 (*outbuf)[0] = (c1); \
176 (*outbuf)[1] = (c2); \
177 (*outbuf)[2] = (c3); \
178 } while (0)
179 #define WRITEBYTE4(c1, c2, c3, c4) \
180 do { \
181 REQUIRE_OUTBUF(4); \
182 (*outbuf)[0] = (c1); \
183 (*outbuf)[1] = (c2); \
184 (*outbuf)[2] = (c3); \
185 (*outbuf)[3] = (c4); \
186 } while (0)
187
188 #define _TRYMAP_ENC(m, assi, val) \
189 ((m)->map != NULL && (val) >= (m)->bottom && \
190 (val)<= (m)->top && ((assi) = (m)->map[(val) - \
191 (m)->bottom]) != NOCHAR)
192 #define TRYMAP_ENC(charset, assi, uni) \
193 _TRYMAP_ENC(&charset##_encmap[(uni) >> 8], assi, (uni) & 0xff)
194
195 #define _TRYMAP_DEC(m, assi, val) \
196 ((m)->map != NULL && \
197 (val) >= (m)->bottom && \
198 (val)<= (m)->top && \
199 ((assi) = (m)->map[(val) - (m)->bottom]) != UNIINV)
200 #define TRYMAP_DEC(charset, assi, c1, c2) \
201 _TRYMAP_DEC(&charset##_decmap[c1], assi, c2)
202
203 #define BEGIN_MAPPINGS_LIST static const struct dbcs_map _mapping_list[] = {
204 #define MAPPING_ENCONLY(enc) {#enc, (void*)enc##_encmap, NULL},
205 #define MAPPING_DECONLY(enc) {#enc, NULL, (void*)enc##_decmap},
206 #define MAPPING_ENCDEC(enc) {#enc, (void*)enc##_encmap, (void*)enc##_decmap},
207 #define END_MAPPINGS_LIST \
208 {"", NULL, NULL} }; \
209 static const struct dbcs_map *mapping_list = \
210 (const struct dbcs_map *)_mapping_list;
211
212 #define BEGIN_CODECS_LIST static const MultibyteCodec _codec_list[] = {
213 #define _STATEFUL_METHODS(enc) \
214 enc##_encode, \
215 enc##_encode_init, \
216 enc##_encode_reset, \
217 enc##_decode, \
218 enc##_decode_init, \
219 enc##_decode_reset,
220 #define _STATELESS_METHODS(enc) \
221 enc##_encode, NULL, NULL, \
222 enc##_decode, NULL, NULL,
223 #define CODEC_STATEFUL(enc) { \
224 #enc, NULL, NULL, \
225 _STATEFUL_METHODS(enc) \
226 },
227 #define CODEC_STATELESS(enc) { \
228 #enc, NULL, NULL, \
229 _STATELESS_METHODS(enc) \
230 },
231 #define CODEC_STATELESS_WINIT(enc) { \
232 #enc, NULL, \
233 enc##_codec_init, \
234 _STATELESS_METHODS(enc) \
235 },
236 #define END_CODECS_LIST \
237 {"", NULL,} }; \
238 static const MultibyteCodec *codec_list = \
239 (const MultibyteCodec *)_codec_list;
240
241
242
243 static PyObject *
getmultibytecodec(void)244 getmultibytecodec(void)
245 {
246 static PyObject *cofunc = NULL;
247
248 if (cofunc == NULL) {
249 PyObject *mod = PyImport_ImportModuleNoBlock("_multibytecodec");
250 if (mod == NULL)
251 return NULL;
252 cofunc = PyObject_GetAttrString(mod, "__create_codec");
253 Py_DECREF(mod);
254 }
255 return cofunc;
256 }
257
258 static PyObject *
getcodec(PyObject * self,PyObject * encoding)259 getcodec(PyObject *self, PyObject *encoding)
260 {
261 PyObject *codecobj, *r, *cofunc;
262 const MultibyteCodec *codec;
263 const char *enc;
264
265 if (!PyUnicode_Check(encoding)) {
266 PyErr_SetString(PyExc_TypeError,
267 "encoding name must be a string.");
268 return NULL;
269 }
270 enc = PyUnicode_AsUTF8(encoding);
271 if (enc == NULL)
272 return NULL;
273
274 cofunc = getmultibytecodec();
275 if (cofunc == NULL)
276 return NULL;
277
278 for (codec = codec_list; codec->encoding[0]; codec++)
279 if (strcmp(codec->encoding, enc) == 0)
280 break;
281
282 if (codec->encoding[0] == '\0') {
283 PyErr_SetString(PyExc_LookupError,
284 "no such codec is supported.");
285 return NULL;
286 }
287
288 codecobj = PyCapsule_New((void *)codec, PyMultibyteCodec_CAPSULE_NAME, NULL);
289 if (codecobj == NULL)
290 return NULL;
291
292 r = PyObject_CallFunctionObjArgs(cofunc, codecobj, NULL);
293 Py_DECREF(codecobj);
294
295 return r;
296 }
297
298 static struct PyMethodDef __methods[] = {
299 {"getcodec", (PyCFunction)getcodec, METH_O, ""},
300 {NULL, NULL},
301 };
302
303 static int
register_maps(PyObject * module)304 register_maps(PyObject *module)
305 {
306 const struct dbcs_map *h;
307
308 for (h = mapping_list; h->charset[0] != '\0'; h++) {
309 char mhname[256] = "__map_";
310 int r;
311 strcpy(mhname + sizeof("__map_") - 1, h->charset);
312 r = PyModule_AddObject(module, mhname,
313 PyCapsule_New((void *)h, PyMultibyteCodec_CAPSULE_NAME, NULL));
314 if (r == -1)
315 return -1;
316 }
317 return 0;
318 }
319
320 #ifdef USING_BINARY_PAIR_SEARCH
321 static DBCHAR
find_pairencmap(ucs2_t body,ucs2_t modifier,const struct pair_encodemap * haystack,int haystacksize)322 find_pairencmap(ucs2_t body, ucs2_t modifier,
323 const struct pair_encodemap *haystack, int haystacksize)
324 {
325 int pos, min, max;
326 Py_UCS4 value = body << 16 | modifier;
327
328 min = 0;
329 max = haystacksize;
330
331 for (pos = haystacksize >> 1; min != max; pos = (min + max) >> 1) {
332 if (value < haystack[pos].uniseq) {
333 if (max != pos) {
334 max = pos;
335 continue;
336 }
337 }
338 else if (value > haystack[pos].uniseq) {
339 if (min != pos) {
340 min = pos;
341 continue;
342 }
343 }
344 break;
345 }
346
347 if (value == haystack[pos].uniseq) {
348 return haystack[pos].code;
349 }
350 return DBCINV;
351 }
352 #endif
353
354 #ifdef USING_IMPORTED_MAPS
355 #define IMPORT_MAP(locale, charset, encmap, decmap) \
356 importmap("_codecs_" #locale, "__map_" #charset, \
357 (const void**)encmap, (const void**)decmap)
358
359 static int
importmap(const char * modname,const char * symbol,const void ** encmap,const void ** decmap)360 importmap(const char *modname, const char *symbol,
361 const void **encmap, const void **decmap)
362 {
363 PyObject *o, *mod;
364
365 mod = PyImport_ImportModule(modname);
366 if (mod == NULL)
367 return -1;
368
369 o = PyObject_GetAttrString(mod, symbol);
370 if (o == NULL)
371 goto errorexit;
372 else if (!PyCapsule_IsValid(o, PyMultibyteCodec_CAPSULE_NAME)) {
373 PyErr_SetString(PyExc_ValueError,
374 "map data must be a Capsule.");
375 goto errorexit;
376 }
377 else {
378 struct dbcs_map *map;
379 map = PyCapsule_GetPointer(o, PyMultibyteCodec_CAPSULE_NAME);
380 if (encmap != NULL)
381 *encmap = map->encmap;
382 if (decmap != NULL)
383 *decmap = map->decmap;
384 Py_DECREF(o);
385 }
386
387 Py_DECREF(mod);
388 return 0;
389
390 errorexit:
391 Py_DECREF(mod);
392 return -1;
393 }
394 #endif
395
396 #define I_AM_A_MODULE_FOR(loc) \
397 static struct PyModuleDef __module = { \
398 PyModuleDef_HEAD_INIT, \
399 "_codecs_"#loc, \
400 NULL, \
401 0, \
402 __methods, \
403 NULL, \
404 NULL, \
405 NULL, \
406 NULL \
407 }; \
408 PyMODINIT_FUNC \
409 PyInit__codecs_##loc(void) \
410 { \
411 PyObject *m = PyModule_Create(&__module); \
412 if (m != NULL) \
413 (void)register_maps(m); \
414 return m; \
415 }
416
417 #endif
418