1 /*
2 * multibytecodec.c: Common Multibyte Codec Implementation
3 *
4 * Written by Hye-Shik Chang <perky@FreeBSD.org>
5 */
6
7 #define PY_SSIZE_T_CLEAN
8 #include "Python.h"
9 #include "structmember.h"
10 #include "multibytecodec.h"
11
12 typedef struct {
13 const Py_UNICODE *inbuf, *inbuf_top, *inbuf_end;
14 unsigned char *outbuf, *outbuf_end;
15 PyObject *excobj, *outobj;
16 } MultibyteEncodeBuffer;
17
18 typedef struct {
19 const unsigned char *inbuf, *inbuf_top, *inbuf_end;
20 Py_UNICODE *outbuf, *outbuf_end;
21 PyObject *excobj, *outobj;
22 } MultibyteDecodeBuffer;
23
24 PyDoc_STRVAR(MultibyteCodec_Encode__doc__,
25 "I.encode(unicode[, errors]) -> (string, length consumed)\n\
26 \n\
27 Return an encoded string version of `unicode'. errors may be given to\n\
28 set a different error handling scheme. Default is 'strict' meaning that\n\
29 encoding errors raise a UnicodeEncodeError. Other possible values are\n\
30 'ignore', 'replace' and 'xmlcharrefreplace' as well as any other name\n\
31 registered with codecs.register_error that can handle UnicodeEncodeErrors.");
32
33 PyDoc_STRVAR(MultibyteCodec_Decode__doc__,
34 "I.decode(string[, errors]) -> (unicodeobject, length consumed)\n\
35 \n\
36 Decodes `string' using I, an MultibyteCodec instance. errors may be given\n\
37 to set a different error handling scheme. Default is 'strict' meaning\n\
38 that encoding errors raise a UnicodeDecodeError. Other possible values\n\
39 are 'ignore' and 'replace' as well as any other name registered with\n\
40 codecs.register_error that is able to handle UnicodeDecodeErrors.");
41
42 static char *codeckwarglist[] = {"input", "errors", NULL};
43 static char *incnewkwarglist[] = {"errors", NULL};
44 static char *incrementalkwarglist[] = {"input", "final", NULL};
45 static char *streamkwarglist[] = {"stream", "errors", NULL};
46
47 static PyObject *multibytecodec_encode(MultibyteCodec *,
48 MultibyteCodec_State *, const Py_UNICODE **, Py_ssize_t,
49 PyObject *, int);
50
51 #define MBENC_RESET MBENC_MAX<<1 /* reset after an encoding session */
52
53 static PyObject *
make_tuple(PyObject * object,Py_ssize_t len)54 make_tuple(PyObject *object, Py_ssize_t len)
55 {
56 PyObject *v, *w;
57
58 if (object == NULL)
59 return NULL;
60
61 v = PyTuple_New(2);
62 if (v == NULL) {
63 Py_DECREF(object);
64 return NULL;
65 }
66 PyTuple_SET_ITEM(v, 0, object);
67
68 w = PyInt_FromSsize_t(len);
69 if (w == NULL) {
70 Py_DECREF(v);
71 return NULL;
72 }
73 PyTuple_SET_ITEM(v, 1, w);
74
75 return v;
76 }
77
78 static PyObject *
internal_error_callback(const char * errors)79 internal_error_callback(const char *errors)
80 {
81 if (errors == NULL || strcmp(errors, "strict") == 0)
82 return ERROR_STRICT;
83 else if (strcmp(errors, "ignore") == 0)
84 return ERROR_IGNORE;
85 else if (strcmp(errors, "replace") == 0)
86 return ERROR_REPLACE;
87 else
88 return PyString_FromString(errors);
89 }
90
91 static PyObject *
call_error_callback(PyObject * errors,PyObject * exc)92 call_error_callback(PyObject *errors, PyObject *exc)
93 {
94 PyObject *args, *cb, *r;
95
96 assert(PyString_Check(errors));
97 cb = PyCodec_LookupError(PyString_AS_STRING(errors));
98 if (cb == NULL)
99 return NULL;
100
101 args = PyTuple_New(1);
102 if (args == NULL) {
103 Py_DECREF(cb);
104 return NULL;
105 }
106
107 PyTuple_SET_ITEM(args, 0, exc);
108 Py_INCREF(exc);
109
110 r = PyObject_CallObject(cb, args);
111 Py_DECREF(args);
112 Py_DECREF(cb);
113 return r;
114 }
115
116 static PyObject *
codecctx_errors_get(MultibyteStatefulCodecContext * self)117 codecctx_errors_get(MultibyteStatefulCodecContext *self)
118 {
119 const char *errors;
120
121 if (self->errors == ERROR_STRICT)
122 errors = "strict";
123 else if (self->errors == ERROR_IGNORE)
124 errors = "ignore";
125 else if (self->errors == ERROR_REPLACE)
126 errors = "replace";
127 else {
128 Py_INCREF(self->errors);
129 return self->errors;
130 }
131
132 return PyString_FromString(errors);
133 }
134
135 static int
codecctx_errors_set(MultibyteStatefulCodecContext * self,PyObject * value,void * closure)136 codecctx_errors_set(MultibyteStatefulCodecContext *self, PyObject *value,
137 void *closure)
138 {
139 PyObject *cb;
140
141 if (!PyString_Check(value)) {
142 PyErr_SetString(PyExc_TypeError, "errors must be a string");
143 return -1;
144 }
145
146 cb = internal_error_callback(PyString_AS_STRING(value));
147 if (cb == NULL)
148 return -1;
149
150 ERROR_DECREF(self->errors);
151 self->errors = cb;
152 return 0;
153 }
154
155 /* This getset handlers list is used by all the stateful codec objects */
156 static PyGetSetDef codecctx_getsets[] = {
157 {"errors", (getter)codecctx_errors_get,
158 (setter)codecctx_errors_set,
159 PyDoc_STR("how to treat errors")},
160 {NULL,}
161 };
162
163 static int
expand_encodebuffer(MultibyteEncodeBuffer * buf,Py_ssize_t esize)164 expand_encodebuffer(MultibyteEncodeBuffer *buf, Py_ssize_t esize)
165 {
166 Py_ssize_t orgpos, orgsize, incsize;
167
168 orgpos = (Py_ssize_t)((char *)buf->outbuf -
169 PyString_AS_STRING(buf->outobj));
170 orgsize = PyString_GET_SIZE(buf->outobj);
171 incsize = (esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize);
172
173 if (orgsize > PY_SSIZE_T_MAX - incsize) {
174 PyErr_NoMemory();
175 return -1;
176 }
177
178 if (_PyString_Resize(&buf->outobj, orgsize + incsize) == -1)
179 return -1;
180
181 buf->outbuf = (unsigned char *)PyString_AS_STRING(buf->outobj) +orgpos;
182 buf->outbuf_end = (unsigned char *)PyString_AS_STRING(buf->outobj)
183 + PyString_GET_SIZE(buf->outobj);
184
185 return 0;
186 }
187 #define REQUIRE_ENCODEBUFFER(buf, s) do { \
188 if ((s) < 0 || (s) > (buf)->outbuf_end - (buf)->outbuf) \
189 if (expand_encodebuffer(buf, s) == -1) \
190 goto errorexit; \
191 } while(0)
192
193 static int
expand_decodebuffer(MultibyteDecodeBuffer * buf,Py_ssize_t esize)194 expand_decodebuffer(MultibyteDecodeBuffer *buf, Py_ssize_t esize)
195 {
196 Py_ssize_t orgpos, orgsize;
197
198 orgpos = (Py_ssize_t)(buf->outbuf - PyUnicode_AS_UNICODE(buf->outobj));
199 orgsize = PyUnicode_GET_SIZE(buf->outobj);
200 if (PyUnicode_Resize(&buf->outobj, orgsize + (
201 esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize)) == -1)
202 return -1;
203
204 buf->outbuf = PyUnicode_AS_UNICODE(buf->outobj) + orgpos;
205 buf->outbuf_end = PyUnicode_AS_UNICODE(buf->outobj)
206 + PyUnicode_GET_SIZE(buf->outobj);
207
208 return 0;
209 }
210 #define REQUIRE_DECODEBUFFER(buf, s) do { \
211 if ((s) < 0 || (s) > (buf)->outbuf_end - (buf)->outbuf) \
212 if (expand_decodebuffer(buf, s) == -1) \
213 goto errorexit; \
214 } while(0)
215
216
217 /**
218 * MultibyteCodec object
219 */
220
221 static int
multibytecodec_encerror(MultibyteCodec * codec,MultibyteCodec_State * state,MultibyteEncodeBuffer * buf,PyObject * errors,Py_ssize_t e)222 multibytecodec_encerror(MultibyteCodec *codec,
223 MultibyteCodec_State *state,
224 MultibyteEncodeBuffer *buf,
225 PyObject *errors, Py_ssize_t e)
226 {
227 PyObject *retobj = NULL, *retstr = NULL, *tobj;
228 Py_ssize_t retstrsize, newpos;
229 Py_ssize_t esize, start, end;
230 const char *reason;
231
232 if (e > 0) {
233 reason = "illegal multibyte sequence";
234 esize = e;
235 }
236 else {
237 switch (e) {
238 case MBERR_TOOSMALL:
239 REQUIRE_ENCODEBUFFER(buf, -1);
240 return 0; /* retry it */
241 case MBERR_TOOFEW:
242 reason = "incomplete multibyte sequence";
243 esize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
244 break;
245 case MBERR_INTERNAL:
246 PyErr_SetString(PyExc_RuntimeError,
247 "internal codec error");
248 return -1;
249 default:
250 PyErr_SetString(PyExc_RuntimeError,
251 "unknown runtime error");
252 return -1;
253 }
254 }
255
256 if (errors == ERROR_REPLACE) {
257 const Py_UNICODE replchar = '?', *inbuf = &replchar;
258 Py_ssize_t r;
259
260 for (;;) {
261 Py_ssize_t outleft;
262
263 outleft = (Py_ssize_t)(buf->outbuf_end - buf->outbuf);
264 r = codec->encode(state, codec->config, &inbuf, 1,
265 &buf->outbuf, outleft, 0);
266 if (r == MBERR_TOOSMALL) {
267 REQUIRE_ENCODEBUFFER(buf, -1);
268 continue;
269 }
270 else
271 break;
272 }
273
274 if (r != 0) {
275 REQUIRE_ENCODEBUFFER(buf, 1);
276 *buf->outbuf++ = '?';
277 }
278 }
279 if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) {
280 buf->inbuf += esize;
281 return 0;
282 }
283
284 start = (Py_ssize_t)(buf->inbuf - buf->inbuf_top);
285 end = start + esize;
286
287 /* use cached exception object if available */
288 if (buf->excobj == NULL) {
289 buf->excobj = PyUnicodeEncodeError_Create(codec->encoding,
290 buf->inbuf_top,
291 buf->inbuf_end - buf->inbuf_top,
292 start, end, reason);
293 if (buf->excobj == NULL)
294 goto errorexit;
295 }
296 else
297 if (PyUnicodeEncodeError_SetStart(buf->excobj, start) != 0 ||
298 PyUnicodeEncodeError_SetEnd(buf->excobj, end) != 0 ||
299 PyUnicodeEncodeError_SetReason(buf->excobj, reason) != 0)
300 goto errorexit;
301
302 if (errors == ERROR_STRICT) {
303 PyCodec_StrictErrors(buf->excobj);
304 goto errorexit;
305 }
306
307 retobj = call_error_callback(errors, buf->excobj);
308 if (retobj == NULL)
309 goto errorexit;
310
311 if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 ||
312 !PyUnicode_Check((tobj = PyTuple_GET_ITEM(retobj, 0))) ||
313 !(PyInt_Check(PyTuple_GET_ITEM(retobj, 1)) ||
314 PyLong_Check(PyTuple_GET_ITEM(retobj, 1)))) {
315 PyErr_SetString(PyExc_TypeError,
316 "encoding error handler must return "
317 "(unicode, int) tuple");
318 goto errorexit;
319 }
320
321 {
322 const Py_UNICODE *uraw = PyUnicode_AS_UNICODE(tobj);
323
324 retstr = multibytecodec_encode(codec, state, &uraw,
325 PyUnicode_GET_SIZE(tobj), ERROR_STRICT,
326 MBENC_FLUSH);
327 if (retstr == NULL)
328 goto errorexit;
329 }
330
331 retstrsize = PyString_GET_SIZE(retstr);
332 if (retstrsize > 0) {
333 REQUIRE_ENCODEBUFFER(buf, retstrsize);
334 memcpy(buf->outbuf, PyString_AS_STRING(retstr), retstrsize);
335 buf->outbuf += retstrsize;
336 }
337
338 newpos = PyInt_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));
339 if (newpos < 0 && !PyErr_Occurred())
340 newpos += (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top);
341 if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) {
342 PyErr_Clear();
343 PyErr_Format(PyExc_IndexError,
344 "position %zd from error handler out of bounds",
345 newpos);
346 goto errorexit;
347 }
348 buf->inbuf = buf->inbuf_top + newpos;
349
350 Py_DECREF(retobj);
351 Py_DECREF(retstr);
352 return 0;
353
354 errorexit:
355 Py_XDECREF(retobj);
356 Py_XDECREF(retstr);
357 return -1;
358 }
359
360 static int
multibytecodec_decerror(MultibyteCodec * codec,MultibyteCodec_State * state,MultibyteDecodeBuffer * buf,PyObject * errors,Py_ssize_t e)361 multibytecodec_decerror(MultibyteCodec *codec,
362 MultibyteCodec_State *state,
363 MultibyteDecodeBuffer *buf,
364 PyObject *errors, Py_ssize_t e)
365 {
366 PyObject *retobj = NULL, *retuni = NULL;
367 Py_ssize_t retunisize, newpos;
368 const char *reason;
369 Py_ssize_t esize, start, end;
370
371 if (e > 0) {
372 reason = "illegal multibyte sequence";
373 esize = e;
374 }
375 else {
376 switch (e) {
377 case MBERR_TOOSMALL:
378 REQUIRE_DECODEBUFFER(buf, -1);
379 return 0; /* retry it */
380 case MBERR_TOOFEW:
381 reason = "incomplete multibyte sequence";
382 esize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
383 break;
384 case MBERR_INTERNAL:
385 PyErr_SetString(PyExc_RuntimeError,
386 "internal codec error");
387 return -1;
388 default:
389 PyErr_SetString(PyExc_RuntimeError,
390 "unknown runtime error");
391 return -1;
392 }
393 }
394
395 if (errors == ERROR_REPLACE) {
396 REQUIRE_DECODEBUFFER(buf, 1);
397 *buf->outbuf++ = Py_UNICODE_REPLACEMENT_CHARACTER;
398 }
399 if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) {
400 buf->inbuf += esize;
401 return 0;
402 }
403
404 start = (Py_ssize_t)(buf->inbuf - buf->inbuf_top);
405 end = start + esize;
406
407 /* use cached exception object if available */
408 if (buf->excobj == NULL) {
409 buf->excobj = PyUnicodeDecodeError_Create(codec->encoding,
410 (const char *)buf->inbuf_top,
411 (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top),
412 start, end, reason);
413 if (buf->excobj == NULL)
414 goto errorexit;
415 }
416 else
417 if (PyUnicodeDecodeError_SetStart(buf->excobj, start) ||
418 PyUnicodeDecodeError_SetEnd(buf->excobj, end) ||
419 PyUnicodeDecodeError_SetReason(buf->excobj, reason))
420 goto errorexit;
421
422 if (errors == ERROR_STRICT) {
423 PyCodec_StrictErrors(buf->excobj);
424 goto errorexit;
425 }
426
427 retobj = call_error_callback(errors, buf->excobj);
428 if (retobj == NULL)
429 goto errorexit;
430
431 if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 ||
432 !PyUnicode_Check((retuni = PyTuple_GET_ITEM(retobj, 0))) ||
433 !(PyInt_Check(PyTuple_GET_ITEM(retobj, 1)) ||
434 PyLong_Check(PyTuple_GET_ITEM(retobj, 1)))) {
435 PyErr_SetString(PyExc_TypeError,
436 "decoding error handler must return "
437 "(unicode, int) tuple");
438 goto errorexit;
439 }
440
441 retunisize = PyUnicode_GET_SIZE(retuni);
442 if (retunisize > 0) {
443 REQUIRE_DECODEBUFFER(buf, retunisize);
444 memcpy((char *)buf->outbuf, PyUnicode_AS_DATA(retuni),
445 retunisize * Py_UNICODE_SIZE);
446 buf->outbuf += retunisize;
447 }
448
449 newpos = PyInt_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));
450 if (newpos < 0 && !PyErr_Occurred())
451 newpos += (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top);
452 if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) {
453 PyErr_Clear();
454 PyErr_Format(PyExc_IndexError,
455 "position %zd from error handler out of bounds",
456 newpos);
457 goto errorexit;
458 }
459 buf->inbuf = buf->inbuf_top + newpos;
460 Py_DECREF(retobj);
461 return 0;
462
463 errorexit:
464 Py_XDECREF(retobj);
465 return -1;
466 }
467
468 static PyObject *
multibytecodec_encode(MultibyteCodec * codec,MultibyteCodec_State * state,const Py_UNICODE ** data,Py_ssize_t datalen,PyObject * errors,int flags)469 multibytecodec_encode(MultibyteCodec *codec,
470 MultibyteCodec_State *state,
471 const Py_UNICODE **data, Py_ssize_t datalen,
472 PyObject *errors, int flags)
473 {
474 MultibyteEncodeBuffer buf;
475 Py_ssize_t finalsize, r = 0;
476
477 if (datalen == 0 && !(flags & MBENC_RESET))
478 return PyString_FromString("");
479
480 buf.excobj = NULL;
481 buf.outobj = NULL;
482 buf.inbuf = buf.inbuf_top = *data;
483 buf.inbuf_end = buf.inbuf_top + datalen;
484
485 if (datalen > (PY_SSIZE_T_MAX - 16) / 2) {
486 PyErr_NoMemory();
487 goto errorexit;
488 }
489
490 buf.outobj = PyString_FromStringAndSize(NULL, datalen * 2 + 16);
491 if (buf.outobj == NULL)
492 goto errorexit;
493 buf.outbuf = (unsigned char *)PyString_AS_STRING(buf.outobj);
494 buf.outbuf_end = buf.outbuf + PyString_GET_SIZE(buf.outobj);
495
496 while (buf.inbuf < buf.inbuf_end) {
497 Py_ssize_t inleft, outleft;
498
499 /* we don't reuse inleft and outleft here.
500 * error callbacks can relocate the cursor anywhere on buffer*/
501 inleft = (Py_ssize_t)(buf.inbuf_end - buf.inbuf);
502 outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);
503 r = codec->encode(state, codec->config, &buf.inbuf, inleft,
504 &buf.outbuf, outleft, flags);
505 if ((r == 0) || (r == MBERR_TOOFEW && !(flags & MBENC_FLUSH)))
506 break;
507 else if (multibytecodec_encerror(codec, state, &buf, errors,r))
508 goto errorexit;
509 else if (r == MBERR_TOOFEW)
510 break;
511 }
512
513 if (codec->encreset != NULL && (flags & MBENC_RESET))
514 for (;;) {
515 Py_ssize_t outleft;
516
517 outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);
518 r = codec->encreset(state, codec->config, &buf.outbuf,
519 outleft);
520 if (r == 0)
521 break;
522 else if (multibytecodec_encerror(codec, state,
523 &buf, errors, r))
524 goto errorexit;
525 }
526
527 finalsize = (Py_ssize_t)((char *)buf.outbuf -
528 PyString_AS_STRING(buf.outobj));
529
530 if (finalsize != PyString_GET_SIZE(buf.outobj))
531 if (_PyString_Resize(&buf.outobj, finalsize) == -1)
532 goto errorexit;
533
534 *data = buf.inbuf;
535 Py_XDECREF(buf.excobj);
536 return buf.outobj;
537
538 errorexit:
539 Py_XDECREF(buf.excobj);
540 Py_XDECREF(buf.outobj);
541 return NULL;
542 }
543
544 static PyObject *
MultibyteCodec_Encode(MultibyteCodecObject * self,PyObject * args,PyObject * kwargs)545 MultibyteCodec_Encode(MultibyteCodecObject *self,
546 PyObject *args, PyObject *kwargs)
547 {
548 MultibyteCodec_State state;
549 Py_UNICODE *data;
550 PyObject *errorcb, *r, *arg, *ucvt;
551 const char *errors = NULL;
552 Py_ssize_t datalen;
553
554 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|z:encode",
555 codeckwarglist, &arg, &errors))
556 return NULL;
557
558 if (PyUnicode_Check(arg))
559 ucvt = NULL;
560 else {
561 arg = ucvt = PyObject_Unicode(arg);
562 if (arg == NULL)
563 return NULL;
564 else if (!PyUnicode_Check(arg)) {
565 PyErr_SetString(PyExc_TypeError,
566 "couldn't convert the object to unicode.");
567 Py_DECREF(ucvt);
568 return NULL;
569 }
570 }
571
572 data = PyUnicode_AS_UNICODE(arg);
573 datalen = PyUnicode_GET_SIZE(arg);
574
575 errorcb = internal_error_callback(errors);
576 if (errorcb == NULL) {
577 Py_XDECREF(ucvt);
578 return NULL;
579 }
580
581 if (self->codec->encinit != NULL &&
582 self->codec->encinit(&state, self->codec->config) != 0)
583 goto errorexit;
584 r = multibytecodec_encode(self->codec, &state,
585 (const Py_UNICODE **)&data, datalen, errorcb,
586 MBENC_FLUSH | MBENC_RESET);
587 if (r == NULL)
588 goto errorexit;
589
590 ERROR_DECREF(errorcb);
591 Py_XDECREF(ucvt);
592 return make_tuple(r, datalen);
593
594 errorexit:
595 ERROR_DECREF(errorcb);
596 Py_XDECREF(ucvt);
597 return NULL;
598 }
599
600 static PyObject *
MultibyteCodec_Decode(MultibyteCodecObject * self,PyObject * args,PyObject * kwargs)601 MultibyteCodec_Decode(MultibyteCodecObject *self,
602 PyObject *args, PyObject *kwargs)
603 {
604 MultibyteCodec_State state;
605 MultibyteDecodeBuffer buf;
606 PyObject *errorcb;
607 Py_buffer pdata;
608 const char *data, *errors = NULL;
609 Py_ssize_t datalen, finalsize;
610
611 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*|z:decode",
612 codeckwarglist, &pdata, &errors))
613 return NULL;
614 data = pdata.buf;
615 datalen = pdata.len;
616
617 errorcb = internal_error_callback(errors);
618 if (errorcb == NULL) {
619 PyBuffer_Release(&pdata);
620 return NULL;
621 }
622
623 if (datalen == 0) {
624 PyBuffer_Release(&pdata);
625 ERROR_DECREF(errorcb);
626 return make_tuple(PyUnicode_FromUnicode(NULL, 0), 0);
627 }
628
629 buf.excobj = NULL;
630 buf.inbuf = buf.inbuf_top = (unsigned char *)data;
631 buf.inbuf_end = buf.inbuf_top + datalen;
632 buf.outobj = PyUnicode_FromUnicode(NULL, datalen);
633 if (buf.outobj == NULL)
634 goto errorexit;
635 buf.outbuf = PyUnicode_AS_UNICODE(buf.outobj);
636 buf.outbuf_end = buf.outbuf + PyUnicode_GET_SIZE(buf.outobj);
637
638 if (self->codec->decinit != NULL &&
639 self->codec->decinit(&state, self->codec->config) != 0)
640 goto errorexit;
641
642 while (buf.inbuf < buf.inbuf_end) {
643 Py_ssize_t inleft, outleft, r;
644
645 inleft = (Py_ssize_t)(buf.inbuf_end - buf.inbuf);
646 outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);
647
648 r = self->codec->decode(&state, self->codec->config,
649 &buf.inbuf, inleft, &buf.outbuf, outleft);
650 if (r == 0)
651 break;
652 else if (multibytecodec_decerror(self->codec, &state,
653 &buf, errorcb, r))
654 goto errorexit;
655 }
656
657 finalsize = (Py_ssize_t)(buf.outbuf -
658 PyUnicode_AS_UNICODE(buf.outobj));
659
660 if (finalsize != PyUnicode_GET_SIZE(buf.outobj))
661 if (PyUnicode_Resize(&buf.outobj, finalsize) == -1)
662 goto errorexit;
663
664 PyBuffer_Release(&pdata);
665 Py_XDECREF(buf.excobj);
666 ERROR_DECREF(errorcb);
667 return make_tuple(buf.outobj, datalen);
668
669 errorexit:
670 PyBuffer_Release(&pdata);
671 ERROR_DECREF(errorcb);
672 Py_XDECREF(buf.excobj);
673 Py_XDECREF(buf.outobj);
674
675 return NULL;
676 }
677
678 static struct PyMethodDef multibytecodec_methods[] = {
679 {"encode", (PyCFunction)MultibyteCodec_Encode,
680 METH_VARARGS | METH_KEYWORDS,
681 MultibyteCodec_Encode__doc__},
682 {"decode", (PyCFunction)MultibyteCodec_Decode,
683 METH_VARARGS | METH_KEYWORDS,
684 MultibyteCodec_Decode__doc__},
685 {NULL, NULL},
686 };
687
688 static void
multibytecodec_dealloc(MultibyteCodecObject * self)689 multibytecodec_dealloc(MultibyteCodecObject *self)
690 {
691 PyObject_Del(self);
692 }
693
694 static PyTypeObject MultibyteCodec_Type = {
695 PyVarObject_HEAD_INIT(NULL, 0)
696 "MultibyteCodec", /* tp_name */
697 sizeof(MultibyteCodecObject), /* tp_basicsize */
698 0, /* tp_itemsize */
699 /* methods */
700 (destructor)multibytecodec_dealloc, /* tp_dealloc */
701 0, /* tp_print */
702 0, /* tp_getattr */
703 0, /* tp_setattr */
704 0, /* tp_compare */
705 0, /* tp_repr */
706 0, /* tp_as_number */
707 0, /* tp_as_sequence */
708 0, /* tp_as_mapping */
709 0, /* tp_hash */
710 0, /* tp_call */
711 0, /* tp_str */
712 PyObject_GenericGetAttr, /* tp_getattro */
713 0, /* tp_setattro */
714 0, /* tp_as_buffer */
715 Py_TPFLAGS_DEFAULT, /* tp_flags */
716 0, /* tp_doc */
717 0, /* tp_traverse */
718 0, /* tp_clear */
719 0, /* tp_richcompare */
720 0, /* tp_weaklistoffset */
721 0, /* tp_iter */
722 0, /* tp_iterext */
723 multibytecodec_methods, /* tp_methods */
724 };
725
726
727 /**
728 * Utility functions for stateful codec mechanism
729 */
730
731 #define STATEFUL_DCTX(o) ((MultibyteStatefulDecoderContext *)(o))
732 #define STATEFUL_ECTX(o) ((MultibyteStatefulEncoderContext *)(o))
733
734 static PyObject *
encoder_encode_stateful(MultibyteStatefulEncoderContext * ctx,PyObject * unistr,int final)735 encoder_encode_stateful(MultibyteStatefulEncoderContext *ctx,
736 PyObject *unistr, int final)
737 {
738 PyObject *ucvt, *r = NULL;
739 Py_UNICODE *inbuf, *inbuf_end, *inbuf_tmp = NULL;
740 Py_ssize_t datalen, origpending;
741
742 if (PyUnicode_Check(unistr))
743 ucvt = NULL;
744 else {
745 unistr = ucvt = PyObject_Unicode(unistr);
746 if (unistr == NULL)
747 return NULL;
748 else if (!PyUnicode_Check(unistr)) {
749 PyErr_SetString(PyExc_TypeError,
750 "couldn't convert the object to unicode.");
751 Py_DECREF(ucvt);
752 return NULL;
753 }
754 }
755
756 datalen = PyUnicode_GET_SIZE(unistr);
757 origpending = ctx->pendingsize;
758
759 if (origpending > 0) {
760 if (datalen > PY_SSIZE_T_MAX - ctx->pendingsize) {
761 PyErr_NoMemory();
762 /* inbuf_tmp == NULL */
763 goto errorexit;
764 }
765 inbuf_tmp = PyMem_New(Py_UNICODE, datalen + ctx->pendingsize);
766 if (inbuf_tmp == NULL)
767 goto errorexit;
768 memcpy(inbuf_tmp, ctx->pending,
769 Py_UNICODE_SIZE * ctx->pendingsize);
770 memcpy(inbuf_tmp + ctx->pendingsize,
771 PyUnicode_AS_UNICODE(unistr),
772 Py_UNICODE_SIZE * datalen);
773 datalen += ctx->pendingsize;
774 ctx->pendingsize = 0;
775 inbuf = inbuf_tmp;
776 }
777 else
778 inbuf = (Py_UNICODE *)PyUnicode_AS_UNICODE(unistr);
779
780 inbuf_end = inbuf + datalen;
781
782 r = multibytecodec_encode(ctx->codec, &ctx->state,
783 (const Py_UNICODE **)&inbuf, datalen,
784 ctx->errors, final ? MBENC_FLUSH | MBENC_RESET : 0);
785 if (r == NULL) {
786 /* recover the original pending buffer */
787 if (origpending > 0)
788 memcpy(ctx->pending, inbuf_tmp,
789 Py_UNICODE_SIZE * origpending);
790 ctx->pendingsize = origpending;
791 goto errorexit;
792 }
793
794 if (inbuf < inbuf_end) {
795 ctx->pendingsize = (Py_ssize_t)(inbuf_end - inbuf);
796 if (ctx->pendingsize > MAXENCPENDING) {
797 /* normal codecs can't reach here */
798 ctx->pendingsize = 0;
799 PyErr_SetString(PyExc_UnicodeError,
800 "pending buffer overflow");
801 goto errorexit;
802 }
803 memcpy(ctx->pending, inbuf,
804 ctx->pendingsize * Py_UNICODE_SIZE);
805 }
806
807 if (inbuf_tmp != NULL)
808 PyMem_Del(inbuf_tmp);
809 Py_XDECREF(ucvt);
810 return r;
811
812 errorexit:
813 if (inbuf_tmp != NULL)
814 PyMem_Del(inbuf_tmp);
815 Py_XDECREF(r);
816 Py_XDECREF(ucvt);
817 return NULL;
818 }
819
820 static int
decoder_append_pending(MultibyteStatefulDecoderContext * ctx,MultibyteDecodeBuffer * buf)821 decoder_append_pending(MultibyteStatefulDecoderContext *ctx,
822 MultibyteDecodeBuffer *buf)
823 {
824 Py_ssize_t npendings;
825
826 npendings = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
827 if (npendings + ctx->pendingsize > MAXDECPENDING ||
828 npendings > PY_SSIZE_T_MAX - ctx->pendingsize) {
829 PyErr_SetString(PyExc_UnicodeError, "pending buffer overflow");
830 return -1;
831 }
832 memcpy(ctx->pending + ctx->pendingsize, buf->inbuf, npendings);
833 ctx->pendingsize += npendings;
834 return 0;
835 }
836
837 static int
decoder_prepare_buffer(MultibyteDecodeBuffer * buf,const char * data,Py_ssize_t size)838 decoder_prepare_buffer(MultibyteDecodeBuffer *buf, const char *data,
839 Py_ssize_t size)
840 {
841 buf->inbuf = buf->inbuf_top = (const unsigned char *)data;
842 buf->inbuf_end = buf->inbuf_top + size;
843 if (buf->outobj == NULL) { /* only if outobj is not allocated yet */
844 buf->outobj = PyUnicode_FromUnicode(NULL, size);
845 if (buf->outobj == NULL)
846 return -1;
847 buf->outbuf = PyUnicode_AS_UNICODE(buf->outobj);
848 buf->outbuf_end = buf->outbuf +
849 PyUnicode_GET_SIZE(buf->outobj);
850 }
851
852 return 0;
853 }
854
855 static int
decoder_feed_buffer(MultibyteStatefulDecoderContext * ctx,MultibyteDecodeBuffer * buf)856 decoder_feed_buffer(MultibyteStatefulDecoderContext *ctx,
857 MultibyteDecodeBuffer *buf)
858 {
859 while (buf->inbuf < buf->inbuf_end) {
860 Py_ssize_t inleft, outleft;
861 Py_ssize_t r;
862
863 inleft = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
864 outleft = (Py_ssize_t)(buf->outbuf_end - buf->outbuf);
865
866 r = ctx->codec->decode(&ctx->state, ctx->codec->config,
867 &buf->inbuf, inleft, &buf->outbuf, outleft);
868 if (r == 0 || r == MBERR_TOOFEW)
869 break;
870 else if (multibytecodec_decerror(ctx->codec, &ctx->state,
871 buf, ctx->errors, r))
872 return -1;
873 }
874 return 0;
875 }
876
877
878 /**
879 * MultibyteIncrementalEncoder object
880 */
881
882 static PyObject *
mbiencoder_encode(MultibyteIncrementalEncoderObject * self,PyObject * args,PyObject * kwargs)883 mbiencoder_encode(MultibyteIncrementalEncoderObject *self,
884 PyObject *args, PyObject *kwargs)
885 {
886 PyObject *data;
887 int final = 0;
888
889 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|i:encode",
890 incrementalkwarglist, &data, &final))
891 return NULL;
892
893 return encoder_encode_stateful(STATEFUL_ECTX(self), data, final);
894 }
895
896 static PyObject *
mbiencoder_reset(MultibyteIncrementalEncoderObject * self)897 mbiencoder_reset(MultibyteIncrementalEncoderObject *self)
898 {
899 if (self->codec->decreset != NULL &&
900 self->codec->decreset(&self->state, self->codec->config) != 0)
901 return NULL;
902 self->pendingsize = 0;
903
904 Py_RETURN_NONE;
905 }
906
907 static struct PyMethodDef mbiencoder_methods[] = {
908 {"encode", (PyCFunction)mbiencoder_encode,
909 METH_VARARGS | METH_KEYWORDS, NULL},
910 {"reset", (PyCFunction)mbiencoder_reset,
911 METH_NOARGS, NULL},
912 {NULL, NULL},
913 };
914
915 static PyObject *
mbiencoder_new(PyTypeObject * type,PyObject * args,PyObject * kwds)916 mbiencoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
917 {
918 MultibyteIncrementalEncoderObject *self;
919 PyObject *codec = NULL;
920 char *errors = NULL;
921
922 if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s:IncrementalEncoder",
923 incnewkwarglist, &errors))
924 return NULL;
925
926 self = (MultibyteIncrementalEncoderObject *)type->tp_alloc(type, 0);
927 if (self == NULL)
928 return NULL;
929
930 codec = PyObject_GetAttrString((PyObject *)type, "codec");
931 if (codec == NULL)
932 goto errorexit;
933 if (!MultibyteCodec_Check(codec)) {
934 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
935 goto errorexit;
936 }
937
938 self->codec = ((MultibyteCodecObject *)codec)->codec;
939 self->pendingsize = 0;
940 self->errors = internal_error_callback(errors);
941 if (self->errors == NULL)
942 goto errorexit;
943 if (self->codec->encinit != NULL &&
944 self->codec->encinit(&self->state, self->codec->config) != 0)
945 goto errorexit;
946
947 Py_DECREF(codec);
948 return (PyObject *)self;
949
950 errorexit:
951 Py_XDECREF(self);
952 Py_XDECREF(codec);
953 return NULL;
954 }
955
956 static int
mbiencoder_init(PyObject * self,PyObject * args,PyObject * kwds)957 mbiencoder_init(PyObject *self, PyObject *args, PyObject *kwds)
958 {
959 return 0;
960 }
961
962 static int
mbiencoder_traverse(MultibyteIncrementalEncoderObject * self,visitproc visit,void * arg)963 mbiencoder_traverse(MultibyteIncrementalEncoderObject *self,
964 visitproc visit, void *arg)
965 {
966 if (ERROR_ISCUSTOM(self->errors))
967 Py_VISIT(self->errors);
968 return 0;
969 }
970
971 static void
mbiencoder_dealloc(MultibyteIncrementalEncoderObject * self)972 mbiencoder_dealloc(MultibyteIncrementalEncoderObject *self)
973 {
974 PyObject_GC_UnTrack(self);
975 ERROR_DECREF(self->errors);
976 Py_TYPE(self)->tp_free(self);
977 }
978
979 static PyTypeObject MultibyteIncrementalEncoder_Type = {
980 PyVarObject_HEAD_INIT(NULL, 0)
981 "MultibyteIncrementalEncoder", /* tp_name */
982 sizeof(MultibyteIncrementalEncoderObject), /* tp_basicsize */
983 0, /* tp_itemsize */
984 /* methods */
985 (destructor)mbiencoder_dealloc, /* tp_dealloc */
986 0, /* tp_print */
987 0, /* tp_getattr */
988 0, /* tp_setattr */
989 0, /* tp_compare */
990 0, /* tp_repr */
991 0, /* tp_as_number */
992 0, /* tp_as_sequence */
993 0, /* tp_as_mapping */
994 0, /* tp_hash */
995 0, /* tp_call */
996 0, /* tp_str */
997 PyObject_GenericGetAttr, /* tp_getattro */
998 0, /* tp_setattro */
999 0, /* tp_as_buffer */
1000 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC
1001 | Py_TPFLAGS_BASETYPE, /* tp_flags */
1002 0, /* tp_doc */
1003 (traverseproc)mbiencoder_traverse, /* tp_traverse */
1004 0, /* tp_clear */
1005 0, /* tp_richcompare */
1006 0, /* tp_weaklistoffset */
1007 0, /* tp_iter */
1008 0, /* tp_iterext */
1009 mbiencoder_methods, /* tp_methods */
1010 0, /* tp_members */
1011 codecctx_getsets, /* tp_getset */
1012 0, /* tp_base */
1013 0, /* tp_dict */
1014 0, /* tp_descr_get */
1015 0, /* tp_descr_set */
1016 0, /* tp_dictoffset */
1017 mbiencoder_init, /* tp_init */
1018 0, /* tp_alloc */
1019 mbiencoder_new, /* tp_new */
1020 };
1021
1022
1023 /**
1024 * MultibyteIncrementalDecoder object
1025 */
1026
1027 static PyObject *
mbidecoder_decode(MultibyteIncrementalDecoderObject * self,PyObject * args,PyObject * kwargs)1028 mbidecoder_decode(MultibyteIncrementalDecoderObject *self,
1029 PyObject *args, PyObject *kwargs)
1030 {
1031 MultibyteDecodeBuffer buf;
1032 char *data, *wdata = NULL;
1033 Py_buffer pdata;
1034 Py_ssize_t wsize, finalsize = 0, size, origpending;
1035 int final = 0;
1036
1037 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*|i:decode",
1038 incrementalkwarglist, &pdata, &final))
1039 return NULL;
1040 data = pdata.buf;
1041 size = pdata.len;
1042
1043 buf.outobj = buf.excobj = NULL;
1044 origpending = self->pendingsize;
1045
1046 if (self->pendingsize == 0) {
1047 wsize = size;
1048 wdata = data;
1049 }
1050 else {
1051 if (size > PY_SSIZE_T_MAX - self->pendingsize) {
1052 PyErr_NoMemory();
1053 goto errorexit;
1054 }
1055 wsize = size + self->pendingsize;
1056 wdata = PyMem_Malloc(wsize);
1057 if (wdata == NULL)
1058 goto errorexit;
1059 memcpy(wdata, self->pending, self->pendingsize);
1060 memcpy(wdata + self->pendingsize, data, size);
1061 self->pendingsize = 0;
1062 }
1063
1064 if (decoder_prepare_buffer(&buf, wdata, wsize) != 0)
1065 goto errorexit;
1066
1067 if (decoder_feed_buffer(STATEFUL_DCTX(self), &buf))
1068 goto errorexit;
1069
1070 if (final && buf.inbuf < buf.inbuf_end) {
1071 if (multibytecodec_decerror(self->codec, &self->state,
1072 &buf, self->errors, MBERR_TOOFEW)) {
1073 /* recover the original pending buffer */
1074 memcpy(self->pending, wdata, origpending);
1075 self->pendingsize = origpending;
1076 goto errorexit;
1077 }
1078 }
1079
1080 if (buf.inbuf < buf.inbuf_end) { /* pending sequence still exists */
1081 if (decoder_append_pending(STATEFUL_DCTX(self), &buf) != 0)
1082 goto errorexit;
1083 }
1084
1085 finalsize = (Py_ssize_t)(buf.outbuf - PyUnicode_AS_UNICODE(buf.outobj));
1086 if (finalsize != PyUnicode_GET_SIZE(buf.outobj))
1087 if (PyUnicode_Resize(&buf.outobj, finalsize) == -1)
1088 goto errorexit;
1089
1090 PyBuffer_Release(&pdata);
1091 if (wdata != data)
1092 PyMem_Del(wdata);
1093 Py_XDECREF(buf.excobj);
1094 return buf.outobj;
1095
1096 errorexit:
1097 PyBuffer_Release(&pdata);
1098 if (wdata != NULL && wdata != data)
1099 PyMem_Del(wdata);
1100 Py_XDECREF(buf.excobj);
1101 Py_XDECREF(buf.outobj);
1102 return NULL;
1103 }
1104
1105 static PyObject *
mbidecoder_reset(MultibyteIncrementalDecoderObject * self)1106 mbidecoder_reset(MultibyteIncrementalDecoderObject *self)
1107 {
1108 if (self->codec->decreset != NULL &&
1109 self->codec->decreset(&self->state, self->codec->config) != 0)
1110 return NULL;
1111 self->pendingsize = 0;
1112
1113 Py_RETURN_NONE;
1114 }
1115
1116 static struct PyMethodDef mbidecoder_methods[] = {
1117 {"decode", (PyCFunction)mbidecoder_decode,
1118 METH_VARARGS | METH_KEYWORDS, NULL},
1119 {"reset", (PyCFunction)mbidecoder_reset,
1120 METH_NOARGS, NULL},
1121 {NULL, NULL},
1122 };
1123
1124 static PyObject *
mbidecoder_new(PyTypeObject * type,PyObject * args,PyObject * kwds)1125 mbidecoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1126 {
1127 MultibyteIncrementalDecoderObject *self;
1128 PyObject *codec = NULL;
1129 char *errors = NULL;
1130
1131 if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s:IncrementalDecoder",
1132 incnewkwarglist, &errors))
1133 return NULL;
1134
1135 self = (MultibyteIncrementalDecoderObject *)type->tp_alloc(type, 0);
1136 if (self == NULL)
1137 return NULL;
1138
1139 codec = PyObject_GetAttrString((PyObject *)type, "codec");
1140 if (codec == NULL)
1141 goto errorexit;
1142 if (!MultibyteCodec_Check(codec)) {
1143 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
1144 goto errorexit;
1145 }
1146
1147 self->codec = ((MultibyteCodecObject *)codec)->codec;
1148 self->pendingsize = 0;
1149 self->errors = internal_error_callback(errors);
1150 if (self->errors == NULL)
1151 goto errorexit;
1152 if (self->codec->decinit != NULL &&
1153 self->codec->decinit(&self->state, self->codec->config) != 0)
1154 goto errorexit;
1155
1156 Py_DECREF(codec);
1157 return (PyObject *)self;
1158
1159 errorexit:
1160 Py_XDECREF(self);
1161 Py_XDECREF(codec);
1162 return NULL;
1163 }
1164
1165 static int
mbidecoder_init(PyObject * self,PyObject * args,PyObject * kwds)1166 mbidecoder_init(PyObject *self, PyObject *args, PyObject *kwds)
1167 {
1168 return 0;
1169 }
1170
1171 static int
mbidecoder_traverse(MultibyteIncrementalDecoderObject * self,visitproc visit,void * arg)1172 mbidecoder_traverse(MultibyteIncrementalDecoderObject *self,
1173 visitproc visit, void *arg)
1174 {
1175 if (ERROR_ISCUSTOM(self->errors))
1176 Py_VISIT(self->errors);
1177 return 0;
1178 }
1179
1180 static void
mbidecoder_dealloc(MultibyteIncrementalDecoderObject * self)1181 mbidecoder_dealloc(MultibyteIncrementalDecoderObject *self)
1182 {
1183 PyObject_GC_UnTrack(self);
1184 ERROR_DECREF(self->errors);
1185 Py_TYPE(self)->tp_free(self);
1186 }
1187
1188 static PyTypeObject MultibyteIncrementalDecoder_Type = {
1189 PyVarObject_HEAD_INIT(NULL, 0)
1190 "MultibyteIncrementalDecoder", /* tp_name */
1191 sizeof(MultibyteIncrementalDecoderObject), /* tp_basicsize */
1192 0, /* tp_itemsize */
1193 /* methods */
1194 (destructor)mbidecoder_dealloc, /* tp_dealloc */
1195 0, /* tp_print */
1196 0, /* tp_getattr */
1197 0, /* tp_setattr */
1198 0, /* tp_compare */
1199 0, /* tp_repr */
1200 0, /* tp_as_number */
1201 0, /* tp_as_sequence */
1202 0, /* tp_as_mapping */
1203 0, /* tp_hash */
1204 0, /* tp_call */
1205 0, /* tp_str */
1206 PyObject_GenericGetAttr, /* tp_getattro */
1207 0, /* tp_setattro */
1208 0, /* tp_as_buffer */
1209 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC
1210 | Py_TPFLAGS_BASETYPE, /* tp_flags */
1211 0, /* tp_doc */
1212 (traverseproc)mbidecoder_traverse, /* tp_traverse */
1213 0, /* tp_clear */
1214 0, /* tp_richcompare */
1215 0, /* tp_weaklistoffset */
1216 0, /* tp_iter */
1217 0, /* tp_iterext */
1218 mbidecoder_methods, /* tp_methods */
1219 0, /* tp_members */
1220 codecctx_getsets, /* tp_getset */
1221 0, /* tp_base */
1222 0, /* tp_dict */
1223 0, /* tp_descr_get */
1224 0, /* tp_descr_set */
1225 0, /* tp_dictoffset */
1226 mbidecoder_init, /* tp_init */
1227 0, /* tp_alloc */
1228 mbidecoder_new, /* tp_new */
1229 };
1230
1231
1232 /**
1233 * MultibyteStreamReader object
1234 */
1235
1236 static PyObject *
mbstreamreader_iread(MultibyteStreamReaderObject * self,const char * method,Py_ssize_t sizehint)1237 mbstreamreader_iread(MultibyteStreamReaderObject *self,
1238 const char *method, Py_ssize_t sizehint)
1239 {
1240 MultibyteDecodeBuffer buf;
1241 PyObject *cres;
1242 Py_ssize_t rsize, finalsize = 0;
1243
1244 if (sizehint == 0)
1245 return PyUnicode_FromUnicode(NULL, 0);
1246
1247 buf.outobj = buf.excobj = NULL;
1248 cres = NULL;
1249
1250 for (;;) {
1251 int endoffile;
1252
1253 if (sizehint < 0)
1254 cres = PyObject_CallMethod(self->stream,
1255 (char *)method, NULL);
1256 else
1257 cres = PyObject_CallMethod(self->stream,
1258 (char *)method, "i", sizehint);
1259 if (cres == NULL)
1260 goto errorexit;
1261
1262 if (!PyString_Check(cres)) {
1263 PyErr_SetString(PyExc_TypeError,
1264 "stream function returned a "
1265 "non-string object");
1266 goto errorexit;
1267 }
1268
1269 endoffile = (PyString_GET_SIZE(cres) == 0);
1270
1271 if (self->pendingsize > 0) {
1272 PyObject *ctr;
1273 char *ctrdata;
1274
1275 if (PyString_GET_SIZE(cres) > PY_SSIZE_T_MAX - self->pendingsize) {
1276 PyErr_NoMemory();
1277 goto errorexit;
1278 }
1279 rsize = PyString_GET_SIZE(cres) + self->pendingsize;
1280 ctr = PyString_FromStringAndSize(NULL, rsize);
1281 if (ctr == NULL)
1282 goto errorexit;
1283 ctrdata = PyString_AS_STRING(ctr);
1284 memcpy(ctrdata, self->pending, self->pendingsize);
1285 memcpy(ctrdata + self->pendingsize,
1286 PyString_AS_STRING(cres),
1287 PyString_GET_SIZE(cres));
1288 Py_DECREF(cres);
1289 cres = ctr;
1290 self->pendingsize = 0;
1291 }
1292
1293 rsize = PyString_GET_SIZE(cres);
1294 if (decoder_prepare_buffer(&buf, PyString_AS_STRING(cres),
1295 rsize) != 0)
1296 goto errorexit;
1297
1298 if (rsize > 0 && decoder_feed_buffer(
1299 (MultibyteStatefulDecoderContext *)self, &buf))
1300 goto errorexit;
1301
1302 if (endoffile || sizehint < 0) {
1303 if (buf.inbuf < buf.inbuf_end &&
1304 multibytecodec_decerror(self->codec, &self->state,
1305 &buf, self->errors, MBERR_TOOFEW))
1306 goto errorexit;
1307 }
1308
1309 if (buf.inbuf < buf.inbuf_end) { /* pending sequence exists */
1310 if (decoder_append_pending(STATEFUL_DCTX(self),
1311 &buf) != 0)
1312 goto errorexit;
1313 }
1314
1315 finalsize = (Py_ssize_t)(buf.outbuf -
1316 PyUnicode_AS_UNICODE(buf.outobj));
1317 Py_DECREF(cres);
1318 cres = NULL;
1319
1320 if (sizehint < 0 || finalsize != 0 || rsize == 0)
1321 break;
1322
1323 sizehint = 1; /* read 1 more byte and retry */
1324 }
1325
1326 if (finalsize != PyUnicode_GET_SIZE(buf.outobj))
1327 if (PyUnicode_Resize(&buf.outobj, finalsize) == -1)
1328 goto errorexit;
1329
1330 Py_XDECREF(cres);
1331 Py_XDECREF(buf.excobj);
1332 return buf.outobj;
1333
1334 errorexit:
1335 Py_XDECREF(cres);
1336 Py_XDECREF(buf.excobj);
1337 Py_XDECREF(buf.outobj);
1338 return NULL;
1339 }
1340
1341 static PyObject *
mbstreamreader_read(MultibyteStreamReaderObject * self,PyObject * args)1342 mbstreamreader_read(MultibyteStreamReaderObject *self, PyObject *args)
1343 {
1344 PyObject *sizeobj = NULL;
1345 Py_ssize_t size;
1346
1347 if (!PyArg_UnpackTuple(args, "read", 0, 1, &sizeobj))
1348 return NULL;
1349
1350 if (sizeobj == Py_None || sizeobj == NULL)
1351 size = -1;
1352 else if (PyInt_Check(sizeobj))
1353 size = PyInt_AsSsize_t(sizeobj);
1354 else {
1355 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");
1356 return NULL;
1357 }
1358
1359 return mbstreamreader_iread(self, "read", size);
1360 }
1361
1362 static PyObject *
mbstreamreader_readline(MultibyteStreamReaderObject * self,PyObject * args)1363 mbstreamreader_readline(MultibyteStreamReaderObject *self, PyObject *args)
1364 {
1365 PyObject *sizeobj = NULL;
1366 Py_ssize_t size;
1367
1368 if (!PyArg_UnpackTuple(args, "readline", 0, 1, &sizeobj))
1369 return NULL;
1370
1371 if (sizeobj == Py_None || sizeobj == NULL)
1372 size = -1;
1373 else if (PyInt_Check(sizeobj))
1374 size = PyInt_AsSsize_t(sizeobj);
1375 else {
1376 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");
1377 return NULL;
1378 }
1379
1380 return mbstreamreader_iread(self, "readline", size);
1381 }
1382
1383 static PyObject *
mbstreamreader_readlines(MultibyteStreamReaderObject * self,PyObject * args)1384 mbstreamreader_readlines(MultibyteStreamReaderObject *self, PyObject *args)
1385 {
1386 PyObject *sizehintobj = NULL, *r, *sr;
1387 Py_ssize_t sizehint;
1388
1389 if (!PyArg_UnpackTuple(args, "readlines", 0, 1, &sizehintobj))
1390 return NULL;
1391
1392 if (sizehintobj == Py_None || sizehintobj == NULL)
1393 sizehint = -1;
1394 else if (PyInt_Check(sizehintobj))
1395 sizehint = PyInt_AsSsize_t(sizehintobj);
1396 else {
1397 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");
1398 return NULL;
1399 }
1400
1401 r = mbstreamreader_iread(self, "read", sizehint);
1402 if (r == NULL)
1403 return NULL;
1404
1405 sr = PyUnicode_Splitlines(r, 1);
1406 Py_DECREF(r);
1407 return sr;
1408 }
1409
1410 static PyObject *
mbstreamreader_reset(MultibyteStreamReaderObject * self)1411 mbstreamreader_reset(MultibyteStreamReaderObject *self)
1412 {
1413 if (self->codec->decreset != NULL &&
1414 self->codec->decreset(&self->state, self->codec->config) != 0)
1415 return NULL;
1416 self->pendingsize = 0;
1417
1418 Py_RETURN_NONE;
1419 }
1420
1421 static struct PyMethodDef mbstreamreader_methods[] = {
1422 {"read", (PyCFunction)mbstreamreader_read,
1423 METH_VARARGS, NULL},
1424 {"readline", (PyCFunction)mbstreamreader_readline,
1425 METH_VARARGS, NULL},
1426 {"readlines", (PyCFunction)mbstreamreader_readlines,
1427 METH_VARARGS, NULL},
1428 {"reset", (PyCFunction)mbstreamreader_reset,
1429 METH_NOARGS, NULL},
1430 {NULL, NULL},
1431 };
1432
1433 static PyMemberDef mbstreamreader_members[] = {
1434 {"stream", T_OBJECT,
1435 offsetof(MultibyteStreamReaderObject, stream),
1436 READONLY, NULL},
1437 {NULL,}
1438 };
1439
1440 static PyObject *
mbstreamreader_new(PyTypeObject * type,PyObject * args,PyObject * kwds)1441 mbstreamreader_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1442 {
1443 MultibyteStreamReaderObject *self;
1444 PyObject *stream, *codec = NULL;
1445 char *errors = NULL;
1446
1447 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s:StreamReader",
1448 streamkwarglist, &stream, &errors))
1449 return NULL;
1450
1451 self = (MultibyteStreamReaderObject *)type->tp_alloc(type, 0);
1452 if (self == NULL)
1453 return NULL;
1454
1455 codec = PyObject_GetAttrString((PyObject *)type, "codec");
1456 if (codec == NULL)
1457 goto errorexit;
1458 if (!MultibyteCodec_Check(codec)) {
1459 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
1460 goto errorexit;
1461 }
1462
1463 self->codec = ((MultibyteCodecObject *)codec)->codec;
1464 self->stream = stream;
1465 Py_INCREF(stream);
1466 self->pendingsize = 0;
1467 self->errors = internal_error_callback(errors);
1468 if (self->errors == NULL)
1469 goto errorexit;
1470 if (self->codec->decinit != NULL &&
1471 self->codec->decinit(&self->state, self->codec->config) != 0)
1472 goto errorexit;
1473
1474 Py_DECREF(codec);
1475 return (PyObject *)self;
1476
1477 errorexit:
1478 Py_XDECREF(self);
1479 Py_XDECREF(codec);
1480 return NULL;
1481 }
1482
1483 static int
mbstreamreader_init(PyObject * self,PyObject * args,PyObject * kwds)1484 mbstreamreader_init(PyObject *self, PyObject *args, PyObject *kwds)
1485 {
1486 return 0;
1487 }
1488
1489 static int
mbstreamreader_traverse(MultibyteStreamReaderObject * self,visitproc visit,void * arg)1490 mbstreamreader_traverse(MultibyteStreamReaderObject *self,
1491 visitproc visit, void *arg)
1492 {
1493 if (ERROR_ISCUSTOM(self->errors))
1494 Py_VISIT(self->errors);
1495 Py_VISIT(self->stream);
1496 return 0;
1497 }
1498
1499 static void
mbstreamreader_dealloc(MultibyteStreamReaderObject * self)1500 mbstreamreader_dealloc(MultibyteStreamReaderObject *self)
1501 {
1502 PyObject_GC_UnTrack(self);
1503 ERROR_DECREF(self->errors);
1504 Py_XDECREF(self->stream);
1505 Py_TYPE(self)->tp_free(self);
1506 }
1507
1508 static PyTypeObject MultibyteStreamReader_Type = {
1509 PyVarObject_HEAD_INIT(NULL, 0)
1510 "MultibyteStreamReader", /* tp_name */
1511 sizeof(MultibyteStreamReaderObject), /* tp_basicsize */
1512 0, /* tp_itemsize */
1513 /* methods */
1514 (destructor)mbstreamreader_dealloc, /* tp_dealloc */
1515 0, /* tp_print */
1516 0, /* tp_getattr */
1517 0, /* tp_setattr */
1518 0, /* tp_compare */
1519 0, /* tp_repr */
1520 0, /* tp_as_number */
1521 0, /* tp_as_sequence */
1522 0, /* tp_as_mapping */
1523 0, /* tp_hash */
1524 0, /* tp_call */
1525 0, /* tp_str */
1526 PyObject_GenericGetAttr, /* tp_getattro */
1527 0, /* tp_setattro */
1528 0, /* tp_as_buffer */
1529 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC
1530 | Py_TPFLAGS_BASETYPE, /* tp_flags */
1531 0, /* tp_doc */
1532 (traverseproc)mbstreamreader_traverse, /* tp_traverse */
1533 0, /* tp_clear */
1534 0, /* tp_richcompare */
1535 0, /* tp_weaklistoffset */
1536 0, /* tp_iter */
1537 0, /* tp_iterext */
1538 mbstreamreader_methods, /* tp_methods */
1539 mbstreamreader_members, /* tp_members */
1540 codecctx_getsets, /* tp_getset */
1541 0, /* tp_base */
1542 0, /* tp_dict */
1543 0, /* tp_descr_get */
1544 0, /* tp_descr_set */
1545 0, /* tp_dictoffset */
1546 mbstreamreader_init, /* tp_init */
1547 0, /* tp_alloc */
1548 mbstreamreader_new, /* tp_new */
1549 };
1550
1551
1552 /**
1553 * MultibyteStreamWriter object
1554 */
1555
1556 static int
mbstreamwriter_iwrite(MultibyteStreamWriterObject * self,PyObject * unistr)1557 mbstreamwriter_iwrite(MultibyteStreamWriterObject *self,
1558 PyObject *unistr)
1559 {
1560 PyObject *str, *wr;
1561
1562 str = encoder_encode_stateful(STATEFUL_ECTX(self), unistr, 0);
1563 if (str == NULL)
1564 return -1;
1565
1566 wr = PyObject_CallMethod(self->stream, "write", "O", str);
1567 Py_DECREF(str);
1568 if (wr == NULL)
1569 return -1;
1570
1571 Py_DECREF(wr);
1572 return 0;
1573 }
1574
1575 static PyObject *
mbstreamwriter_write(MultibyteStreamWriterObject * self,PyObject * strobj)1576 mbstreamwriter_write(MultibyteStreamWriterObject *self, PyObject *strobj)
1577 {
1578 if (mbstreamwriter_iwrite(self, strobj))
1579 return NULL;
1580 else
1581 Py_RETURN_NONE;
1582 }
1583
1584 static PyObject *
mbstreamwriter_writelines(MultibyteStreamWriterObject * self,PyObject * lines)1585 mbstreamwriter_writelines(MultibyteStreamWriterObject *self, PyObject *lines)
1586 {
1587 PyObject *strobj;
1588 int i, r;
1589
1590 if (!PySequence_Check(lines)) {
1591 PyErr_SetString(PyExc_TypeError,
1592 "arg must be a sequence object");
1593 return NULL;
1594 }
1595
1596 for (i = 0; i < PySequence_Length(lines); i++) {
1597 /* length can be changed even within this loop */
1598 strobj = PySequence_GetItem(lines, i);
1599 if (strobj == NULL)
1600 return NULL;
1601
1602 r = mbstreamwriter_iwrite(self, strobj);
1603 Py_DECREF(strobj);
1604 if (r == -1)
1605 return NULL;
1606 }
1607
1608 Py_RETURN_NONE;
1609 }
1610
1611 static PyObject *
mbstreamwriter_reset(MultibyteStreamWriterObject * self)1612 mbstreamwriter_reset(MultibyteStreamWriterObject *self)
1613 {
1614 const Py_UNICODE *pending;
1615 PyObject *pwrt;
1616
1617 pending = self->pending;
1618 pwrt = multibytecodec_encode(self->codec, &self->state,
1619 &pending, self->pendingsize, self->errors,
1620 MBENC_FLUSH | MBENC_RESET);
1621 /* some pending buffer can be truncated when UnicodeEncodeError is
1622 * raised on 'strict' mode. but, 'reset' method is designed to
1623 * reset the pending buffer or states so failed string sequence
1624 * ought to be missed */
1625 self->pendingsize = 0;
1626 if (pwrt == NULL)
1627 return NULL;
1628
1629 if (PyString_Size(pwrt) > 0) {
1630 PyObject *wr;
1631 wr = PyObject_CallMethod(self->stream, "write", "O", pwrt);
1632 if (wr == NULL) {
1633 Py_DECREF(pwrt);
1634 return NULL;
1635 }
1636 }
1637 Py_DECREF(pwrt);
1638
1639 Py_RETURN_NONE;
1640 }
1641
1642 static PyObject *
mbstreamwriter_new(PyTypeObject * type,PyObject * args,PyObject * kwds)1643 mbstreamwriter_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1644 {
1645 MultibyteStreamWriterObject *self;
1646 PyObject *stream, *codec = NULL;
1647 char *errors = NULL;
1648
1649 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s:StreamWriter",
1650 streamkwarglist, &stream, &errors))
1651 return NULL;
1652
1653 self = (MultibyteStreamWriterObject *)type->tp_alloc(type, 0);
1654 if (self == NULL)
1655 return NULL;
1656
1657 codec = PyObject_GetAttrString((PyObject *)type, "codec");
1658 if (codec == NULL)
1659 goto errorexit;
1660 if (!MultibyteCodec_Check(codec)) {
1661 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
1662 goto errorexit;
1663 }
1664
1665 self->codec = ((MultibyteCodecObject *)codec)->codec;
1666 self->stream = stream;
1667 Py_INCREF(stream);
1668 self->pendingsize = 0;
1669 self->errors = internal_error_callback(errors);
1670 if (self->errors == NULL)
1671 goto errorexit;
1672 if (self->codec->encinit != NULL &&
1673 self->codec->encinit(&self->state, self->codec->config) != 0)
1674 goto errorexit;
1675
1676 Py_DECREF(codec);
1677 return (PyObject *)self;
1678
1679 errorexit:
1680 Py_XDECREF(self);
1681 Py_XDECREF(codec);
1682 return NULL;
1683 }
1684
1685 static int
mbstreamwriter_init(PyObject * self,PyObject * args,PyObject * kwds)1686 mbstreamwriter_init(PyObject *self, PyObject *args, PyObject *kwds)
1687 {
1688 return 0;
1689 }
1690
1691 static int
mbstreamwriter_traverse(MultibyteStreamWriterObject * self,visitproc visit,void * arg)1692 mbstreamwriter_traverse(MultibyteStreamWriterObject *self,
1693 visitproc visit, void *arg)
1694 {
1695 if (ERROR_ISCUSTOM(self->errors))
1696 Py_VISIT(self->errors);
1697 Py_VISIT(self->stream);
1698 return 0;
1699 }
1700
1701 static void
mbstreamwriter_dealloc(MultibyteStreamWriterObject * self)1702 mbstreamwriter_dealloc(MultibyteStreamWriterObject *self)
1703 {
1704 PyObject_GC_UnTrack(self);
1705 ERROR_DECREF(self->errors);
1706 Py_XDECREF(self->stream);
1707 Py_TYPE(self)->tp_free(self);
1708 }
1709
1710 static struct PyMethodDef mbstreamwriter_methods[] = {
1711 {"write", (PyCFunction)mbstreamwriter_write,
1712 METH_O, NULL},
1713 {"writelines", (PyCFunction)mbstreamwriter_writelines,
1714 METH_O, NULL},
1715 {"reset", (PyCFunction)mbstreamwriter_reset,
1716 METH_NOARGS, NULL},
1717 {NULL, NULL},
1718 };
1719
1720 static PyMemberDef mbstreamwriter_members[] = {
1721 {"stream", T_OBJECT,
1722 offsetof(MultibyteStreamWriterObject, stream),
1723 READONLY, NULL},
1724 {NULL,}
1725 };
1726
1727 static PyTypeObject MultibyteStreamWriter_Type = {
1728 PyVarObject_HEAD_INIT(NULL, 0)
1729 "MultibyteStreamWriter", /* tp_name */
1730 sizeof(MultibyteStreamWriterObject), /* tp_basicsize */
1731 0, /* tp_itemsize */
1732 /* methods */
1733 (destructor)mbstreamwriter_dealloc, /* tp_dealloc */
1734 0, /* tp_print */
1735 0, /* tp_getattr */
1736 0, /* tp_setattr */
1737 0, /* tp_compare */
1738 0, /* tp_repr */
1739 0, /* tp_as_number */
1740 0, /* tp_as_sequence */
1741 0, /* tp_as_mapping */
1742 0, /* tp_hash */
1743 0, /* tp_call */
1744 0, /* tp_str */
1745 PyObject_GenericGetAttr, /* tp_getattro */
1746 0, /* tp_setattro */
1747 0, /* tp_as_buffer */
1748 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC
1749 | Py_TPFLAGS_BASETYPE, /* tp_flags */
1750 0, /* tp_doc */
1751 (traverseproc)mbstreamwriter_traverse, /* tp_traverse */
1752 0, /* tp_clear */
1753 0, /* tp_richcompare */
1754 0, /* tp_weaklistoffset */
1755 0, /* tp_iter */
1756 0, /* tp_iterext */
1757 mbstreamwriter_methods, /* tp_methods */
1758 mbstreamwriter_members, /* tp_members */
1759 codecctx_getsets, /* tp_getset */
1760 0, /* tp_base */
1761 0, /* tp_dict */
1762 0, /* tp_descr_get */
1763 0, /* tp_descr_set */
1764 0, /* tp_dictoffset */
1765 mbstreamwriter_init, /* tp_init */
1766 0, /* tp_alloc */
1767 mbstreamwriter_new, /* tp_new */
1768 };
1769
1770
1771 /**
1772 * Exposed factory function
1773 */
1774
1775 static PyObject *
__create_codec(PyObject * ignore,PyObject * arg)1776 __create_codec(PyObject *ignore, PyObject *arg)
1777 {
1778 MultibyteCodecObject *self;
1779 MultibyteCodec *codec;
1780
1781 if (!PyCapsule_IsValid(arg, PyMultibyteCodec_CAPSULE_NAME)) {
1782 PyErr_SetString(PyExc_ValueError, "argument type invalid");
1783 return NULL;
1784 }
1785
1786 codec = PyCapsule_GetPointer(arg, PyMultibyteCodec_CAPSULE_NAME);
1787 if (codec->codecinit != NULL && codec->codecinit(codec->config) != 0)
1788 return NULL;
1789
1790 self = PyObject_New(MultibyteCodecObject, &MultibyteCodec_Type);
1791 if (self == NULL)
1792 return NULL;
1793 self->codec = codec;
1794
1795 return (PyObject *)self;
1796 }
1797
1798 static struct PyMethodDef __methods[] = {
1799 {"__create_codec", (PyCFunction)__create_codec, METH_O},
1800 {NULL, NULL},
1801 };
1802
1803 PyMODINIT_FUNC
init_multibytecodec(void)1804 init_multibytecodec(void)
1805 {
1806 int i;
1807 PyObject *m;
1808 PyTypeObject *typelist[] = {
1809 &MultibyteIncrementalEncoder_Type,
1810 &MultibyteIncrementalDecoder_Type,
1811 &MultibyteStreamReader_Type,
1812 &MultibyteStreamWriter_Type,
1813 NULL
1814 };
1815
1816 if (PyType_Ready(&MultibyteCodec_Type) < 0)
1817 return;
1818
1819 m = Py_InitModule("_multibytecodec", __methods);
1820 if (m == NULL)
1821 return;
1822
1823 for (i = 0; typelist[i] != NULL; i++) {
1824 if (PyType_Ready(typelist[i]) < 0)
1825 return;
1826 Py_INCREF(typelist[i]);
1827 PyModule_AddObject(m, typelist[i]->tp_name,
1828 (PyObject *)typelist[i]);
1829 }
1830
1831 if (PyErr_Occurred())
1832 Py_FatalError("can't initialize the _multibytecodec module");
1833 }
1834