1 /*
2 * multibytecodec.c: Common Multibyte Codec Implementation
3 *
4 * Written by Hye-Shik Chang <perky@FreeBSD.org>
5 */
6
7 #define PY_SSIZE_T_CLEAN
8 #include "Python.h"
9 #include "structmember.h"
10 #include "multibytecodec.h"
11
12 typedef struct {
13 const Py_UNICODE *inbuf, *inbuf_top, *inbuf_end;
14 unsigned char *outbuf, *outbuf_end;
15 PyObject *excobj, *outobj;
16 } MultibyteEncodeBuffer;
17
18 typedef struct {
19 const unsigned char *inbuf, *inbuf_top, *inbuf_end;
20 Py_UNICODE *outbuf, *outbuf_end;
21 PyObject *excobj, *outobj;
22 } MultibyteDecodeBuffer;
23
24 PyDoc_STRVAR(MultibyteCodec_Encode__doc__,
25 "I.encode(unicode[, errors]) -> (string, length consumed)\n\
26 \n\
27 Return an encoded string version of `unicode'. errors may be given to\n\
28 set a different error handling scheme. Default is 'strict' meaning that\n\
29 encoding errors raise a UnicodeEncodeError. Other possible values are\n\
30 'ignore', 'replace' and 'xmlcharrefreplace' as well as any other name\n\
31 registered with codecs.register_error that can handle UnicodeEncodeErrors.");
32
33 PyDoc_STRVAR(MultibyteCodec_Decode__doc__,
34 "I.decode(string[, errors]) -> (unicodeobject, length consumed)\n\
35 \n\
36 Decodes `string' using I, an MultibyteCodec instance. errors may be given\n\
37 to set a different error handling scheme. Default is 'strict' meaning\n\
38 that encoding errors raise a UnicodeDecodeError. Other possible values\n\
39 are 'ignore' and 'replace' as well as any other name registered with\n\
40 codecs.register_error that is able to handle UnicodeDecodeErrors.");
41
42 static char *codeckwarglist[] = {"input", "errors", NULL};
43 static char *incnewkwarglist[] = {"errors", NULL};
44 static char *incrementalkwarglist[] = {"input", "final", NULL};
45 static char *streamkwarglist[] = {"stream", "errors", NULL};
46
47 static PyObject *multibytecodec_encode(MultibyteCodec *,
48 MultibyteCodec_State *, const Py_UNICODE **, Py_ssize_t,
49 PyObject *, int);
50
51 #define MBENC_RESET MBENC_MAX<<1 /* reset after an encoding session */
52
53 static PyObject *
make_tuple(PyObject * object,Py_ssize_t len)54 make_tuple(PyObject *object, Py_ssize_t len)
55 {
56 PyObject *v, *w;
57
58 if (object == NULL)
59 return NULL;
60
61 v = PyTuple_New(2);
62 if (v == NULL) {
63 Py_DECREF(object);
64 return NULL;
65 }
66 PyTuple_SET_ITEM(v, 0, object);
67
68 w = PyInt_FromSsize_t(len);
69 if (w == NULL) {
70 Py_DECREF(v);
71 return NULL;
72 }
73 PyTuple_SET_ITEM(v, 1, w);
74
75 return v;
76 }
77
78 static PyObject *
internal_error_callback(const char * errors)79 internal_error_callback(const char *errors)
80 {
81 if (errors == NULL || strcmp(errors, "strict") == 0)
82 return ERROR_STRICT;
83 else if (strcmp(errors, "ignore") == 0)
84 return ERROR_IGNORE;
85 else if (strcmp(errors, "replace") == 0)
86 return ERROR_REPLACE;
87 else
88 return PyString_FromString(errors);
89 }
90
91 static PyObject *
call_error_callback(PyObject * errors,PyObject * exc)92 call_error_callback(PyObject *errors, PyObject *exc)
93 {
94 PyObject *args, *cb, *r;
95
96 assert(PyString_Check(errors));
97 cb = PyCodec_LookupError(PyString_AS_STRING(errors));
98 if (cb == NULL)
99 return NULL;
100
101 args = PyTuple_New(1);
102 if (args == NULL) {
103 Py_DECREF(cb);
104 return NULL;
105 }
106
107 PyTuple_SET_ITEM(args, 0, exc);
108 Py_INCREF(exc);
109
110 r = PyObject_CallObject(cb, args);
111 Py_DECREF(args);
112 Py_DECREF(cb);
113 return r;
114 }
115
116 static PyObject *
codecctx_errors_get(MultibyteStatefulCodecContext * self)117 codecctx_errors_get(MultibyteStatefulCodecContext *self)
118 {
119 const char *errors;
120
121 if (self->errors == ERROR_STRICT)
122 errors = "strict";
123 else if (self->errors == ERROR_IGNORE)
124 errors = "ignore";
125 else if (self->errors == ERROR_REPLACE)
126 errors = "replace";
127 else {
128 Py_INCREF(self->errors);
129 return self->errors;
130 }
131
132 return PyString_FromString(errors);
133 }
134
135 static int
codecctx_errors_set(MultibyteStatefulCodecContext * self,PyObject * value,void * closure)136 codecctx_errors_set(MultibyteStatefulCodecContext *self, PyObject *value,
137 void *closure)
138 {
139 PyObject *cb;
140
141 if (!PyString_Check(value)) {
142 PyErr_SetString(PyExc_TypeError, "errors must be a string");
143 return -1;
144 }
145
146 cb = internal_error_callback(PyString_AS_STRING(value));
147 if (cb == NULL)
148 return -1;
149
150 ERROR_DECREF(self->errors);
151 self->errors = cb;
152 return 0;
153 }
154
155 /* This getset handlers list is used by all the stateful codec objects */
156 static PyGetSetDef codecctx_getsets[] = {
157 {"errors", (getter)codecctx_errors_get,
158 (setter)codecctx_errors_set,
159 PyDoc_STR("how to treat errors")},
160 {NULL,}
161 };
162
163 static int
expand_encodebuffer(MultibyteEncodeBuffer * buf,Py_ssize_t esize)164 expand_encodebuffer(MultibyteEncodeBuffer *buf, Py_ssize_t esize)
165 {
166 Py_ssize_t orgpos, orgsize, incsize;
167
168 orgpos = (Py_ssize_t)((char *)buf->outbuf -
169 PyString_AS_STRING(buf->outobj));
170 orgsize = PyString_GET_SIZE(buf->outobj);
171 incsize = (esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize);
172
173 if (orgsize > PY_SSIZE_T_MAX - incsize) {
174 PyErr_NoMemory();
175 return -1;
176 }
177
178 if (_PyString_Resize(&buf->outobj, orgsize + incsize) == -1)
179 return -1;
180
181 buf->outbuf = (unsigned char *)PyString_AS_STRING(buf->outobj) +orgpos;
182 buf->outbuf_end = (unsigned char *)PyString_AS_STRING(buf->outobj)
183 + PyString_GET_SIZE(buf->outobj);
184
185 return 0;
186 }
187 #define REQUIRE_ENCODEBUFFER(buf, s) do { \
188 if ((s) < 0 || (s) > (buf)->outbuf_end - (buf)->outbuf) \
189 if (expand_encodebuffer(buf, s) == -1) \
190 goto errorexit; \
191 } while(0)
192
193 static int
expand_decodebuffer(MultibyteDecodeBuffer * buf,Py_ssize_t esize)194 expand_decodebuffer(MultibyteDecodeBuffer *buf, Py_ssize_t esize)
195 {
196 Py_ssize_t orgpos, orgsize;
197
198 orgpos = (Py_ssize_t)(buf->outbuf - PyUnicode_AS_UNICODE(buf->outobj));
199 orgsize = PyUnicode_GET_SIZE(buf->outobj);
200 if (PyUnicode_Resize(&buf->outobj, orgsize + (
201 esize < (orgsize >> 1) ? (orgsize >> 1) | 1 : esize)) == -1)
202 return -1;
203
204 buf->outbuf = PyUnicode_AS_UNICODE(buf->outobj) + orgpos;
205 buf->outbuf_end = PyUnicode_AS_UNICODE(buf->outobj)
206 + PyUnicode_GET_SIZE(buf->outobj);
207
208 return 0;
209 }
210 #define REQUIRE_DECODEBUFFER(buf, s) do { \
211 if ((s) < 0 || (s) > (buf)->outbuf_end - (buf)->outbuf) \
212 if (expand_decodebuffer(buf, s) == -1) \
213 goto errorexit; \
214 } while(0)
215
216
217 /**
218 * MultibyteCodec object
219 */
220
221 static int
multibytecodec_encerror(MultibyteCodec * codec,MultibyteCodec_State * state,MultibyteEncodeBuffer * buf,PyObject * errors,Py_ssize_t e)222 multibytecodec_encerror(MultibyteCodec *codec,
223 MultibyteCodec_State *state,
224 MultibyteEncodeBuffer *buf,
225 PyObject *errors, Py_ssize_t e)
226 {
227 PyObject *retobj = NULL, *retstr = NULL, *tobj;
228 Py_ssize_t retstrsize, newpos;
229 Py_ssize_t esize, start, end;
230 const char *reason;
231
232 if (e > 0) {
233 reason = "illegal multibyte sequence";
234 esize = e;
235 }
236 else {
237 switch (e) {
238 case MBERR_TOOSMALL:
239 REQUIRE_ENCODEBUFFER(buf, -1);
240 return 0; /* retry it */
241 case MBERR_TOOFEW:
242 reason = "incomplete multibyte sequence";
243 esize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
244 break;
245 case MBERR_INTERNAL:
246 PyErr_SetString(PyExc_RuntimeError,
247 "internal codec error");
248 return -1;
249 default:
250 PyErr_SetString(PyExc_RuntimeError,
251 "unknown runtime error");
252 return -1;
253 }
254 }
255
256 if (errors == ERROR_REPLACE) {
257 const Py_UNICODE replchar = '?', *inbuf = &replchar;
258 Py_ssize_t r;
259
260 for (;;) {
261 Py_ssize_t outleft;
262
263 outleft = (Py_ssize_t)(buf->outbuf_end - buf->outbuf);
264 r = codec->encode(state, codec->config, &inbuf, 1,
265 &buf->outbuf, outleft, 0);
266 if (r == MBERR_TOOSMALL) {
267 REQUIRE_ENCODEBUFFER(buf, -1);
268 continue;
269 }
270 else
271 break;
272 }
273
274 if (r != 0) {
275 REQUIRE_ENCODEBUFFER(buf, 1);
276 *buf->outbuf++ = '?';
277 }
278 }
279 if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) {
280 buf->inbuf += esize;
281 return 0;
282 }
283
284 start = (Py_ssize_t)(buf->inbuf - buf->inbuf_top);
285 end = start + esize;
286
287 /* use cached exception object if available */
288 if (buf->excobj == NULL) {
289 buf->excobj = PyUnicodeEncodeError_Create(codec->encoding,
290 buf->inbuf_top,
291 buf->inbuf_end - buf->inbuf_top,
292 start, end, reason);
293 if (buf->excobj == NULL)
294 goto errorexit;
295 }
296 else
297 if (PyUnicodeEncodeError_SetStart(buf->excobj, start) != 0 ||
298 PyUnicodeEncodeError_SetEnd(buf->excobj, end) != 0 ||
299 PyUnicodeEncodeError_SetReason(buf->excobj, reason) != 0)
300 goto errorexit;
301
302 if (errors == ERROR_STRICT) {
303 PyCodec_StrictErrors(buf->excobj);
304 goto errorexit;
305 }
306
307 retobj = call_error_callback(errors, buf->excobj);
308 if (retobj == NULL)
309 goto errorexit;
310
311 if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 ||
312 !PyUnicode_Check((tobj = PyTuple_GET_ITEM(retobj, 0))) ||
313 !_PyAnyInt_Check(PyTuple_GET_ITEM(retobj, 1))) {
314 PyErr_SetString(PyExc_TypeError,
315 "encoding error handler must return "
316 "(unicode, int) tuple");
317 goto errorexit;
318 }
319
320 {
321 const Py_UNICODE *uraw = PyUnicode_AS_UNICODE(tobj);
322
323 retstr = multibytecodec_encode(codec, state, &uraw,
324 PyUnicode_GET_SIZE(tobj), ERROR_STRICT,
325 MBENC_FLUSH);
326 if (retstr == NULL)
327 goto errorexit;
328 }
329
330 retstrsize = PyString_GET_SIZE(retstr);
331 if (retstrsize > 0) {
332 REQUIRE_ENCODEBUFFER(buf, retstrsize);
333 memcpy(buf->outbuf, PyString_AS_STRING(retstr), retstrsize);
334 buf->outbuf += retstrsize;
335 }
336
337 newpos = PyInt_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));
338 if (newpos < 0 && !PyErr_Occurred())
339 newpos += (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top);
340 if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) {
341 PyErr_Clear();
342 PyErr_Format(PyExc_IndexError,
343 "position %zd from error handler out of bounds",
344 newpos);
345 goto errorexit;
346 }
347 buf->inbuf = buf->inbuf_top + newpos;
348
349 Py_DECREF(retobj);
350 Py_DECREF(retstr);
351 return 0;
352
353 errorexit:
354 Py_XDECREF(retobj);
355 Py_XDECREF(retstr);
356 return -1;
357 }
358
359 static int
multibytecodec_decerror(MultibyteCodec * codec,MultibyteCodec_State * state,MultibyteDecodeBuffer * buf,PyObject * errors,Py_ssize_t e)360 multibytecodec_decerror(MultibyteCodec *codec,
361 MultibyteCodec_State *state,
362 MultibyteDecodeBuffer *buf,
363 PyObject *errors, Py_ssize_t e)
364 {
365 PyObject *retobj = NULL, *retuni = NULL;
366 Py_ssize_t retunisize, newpos;
367 const char *reason;
368 Py_ssize_t esize, start, end;
369
370 if (e > 0) {
371 reason = "illegal multibyte sequence";
372 esize = e;
373 }
374 else {
375 switch (e) {
376 case MBERR_TOOSMALL:
377 REQUIRE_DECODEBUFFER(buf, -1);
378 return 0; /* retry it */
379 case MBERR_TOOFEW:
380 reason = "incomplete multibyte sequence";
381 esize = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
382 break;
383 case MBERR_INTERNAL:
384 PyErr_SetString(PyExc_RuntimeError,
385 "internal codec error");
386 return -1;
387 default:
388 PyErr_SetString(PyExc_RuntimeError,
389 "unknown runtime error");
390 return -1;
391 }
392 }
393
394 if (errors == ERROR_REPLACE) {
395 REQUIRE_DECODEBUFFER(buf, 1);
396 *buf->outbuf++ = Py_UNICODE_REPLACEMENT_CHARACTER;
397 }
398 if (errors == ERROR_IGNORE || errors == ERROR_REPLACE) {
399 buf->inbuf += esize;
400 return 0;
401 }
402
403 start = (Py_ssize_t)(buf->inbuf - buf->inbuf_top);
404 end = start + esize;
405
406 /* use cached exception object if available */
407 if (buf->excobj == NULL) {
408 buf->excobj = PyUnicodeDecodeError_Create(codec->encoding,
409 (const char *)buf->inbuf_top,
410 (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top),
411 start, end, reason);
412 if (buf->excobj == NULL)
413 goto errorexit;
414 }
415 else
416 if (PyUnicodeDecodeError_SetStart(buf->excobj, start) ||
417 PyUnicodeDecodeError_SetEnd(buf->excobj, end) ||
418 PyUnicodeDecodeError_SetReason(buf->excobj, reason))
419 goto errorexit;
420
421 if (errors == ERROR_STRICT) {
422 PyCodec_StrictErrors(buf->excobj);
423 goto errorexit;
424 }
425
426 retobj = call_error_callback(errors, buf->excobj);
427 if (retobj == NULL)
428 goto errorexit;
429
430 if (!PyTuple_Check(retobj) || PyTuple_GET_SIZE(retobj) != 2 ||
431 !PyUnicode_Check((retuni = PyTuple_GET_ITEM(retobj, 0))) ||
432 !_PyAnyInt_Check(PyTuple_GET_ITEM(retobj, 1))) {
433 PyErr_SetString(PyExc_TypeError,
434 "decoding error handler must return "
435 "(unicode, int) tuple");
436 goto errorexit;
437 }
438
439 retunisize = PyUnicode_GET_SIZE(retuni);
440 if (retunisize > 0) {
441 REQUIRE_DECODEBUFFER(buf, retunisize);
442 memcpy((char *)buf->outbuf, PyUnicode_AS_DATA(retuni),
443 retunisize * Py_UNICODE_SIZE);
444 buf->outbuf += retunisize;
445 }
446
447 newpos = PyInt_AsSsize_t(PyTuple_GET_ITEM(retobj, 1));
448 if (newpos < 0 && !PyErr_Occurred())
449 newpos += (Py_ssize_t)(buf->inbuf_end - buf->inbuf_top);
450 if (newpos < 0 || buf->inbuf_top + newpos > buf->inbuf_end) {
451 PyErr_Clear();
452 PyErr_Format(PyExc_IndexError,
453 "position %zd from error handler out of bounds",
454 newpos);
455 goto errorexit;
456 }
457 buf->inbuf = buf->inbuf_top + newpos;
458 Py_DECREF(retobj);
459 return 0;
460
461 errorexit:
462 Py_XDECREF(retobj);
463 return -1;
464 }
465
466 static PyObject *
multibytecodec_encode(MultibyteCodec * codec,MultibyteCodec_State * state,const Py_UNICODE ** data,Py_ssize_t datalen,PyObject * errors,int flags)467 multibytecodec_encode(MultibyteCodec *codec,
468 MultibyteCodec_State *state,
469 const Py_UNICODE **data, Py_ssize_t datalen,
470 PyObject *errors, int flags)
471 {
472 MultibyteEncodeBuffer buf;
473 Py_ssize_t finalsize, r = 0;
474
475 if (datalen == 0 && !(flags & MBENC_RESET))
476 return PyString_FromString("");
477
478 buf.excobj = NULL;
479 buf.outobj = NULL;
480 buf.inbuf = buf.inbuf_top = *data;
481 buf.inbuf_end = buf.inbuf_top + datalen;
482
483 if (datalen > (PY_SSIZE_T_MAX - 16) / 2) {
484 PyErr_NoMemory();
485 goto errorexit;
486 }
487
488 buf.outobj = PyString_FromStringAndSize(NULL, datalen * 2 + 16);
489 if (buf.outobj == NULL)
490 goto errorexit;
491 buf.outbuf = (unsigned char *)PyString_AS_STRING(buf.outobj);
492 buf.outbuf_end = buf.outbuf + PyString_GET_SIZE(buf.outobj);
493
494 while (buf.inbuf < buf.inbuf_end) {
495 Py_ssize_t inleft, outleft;
496
497 /* we don't reuse inleft and outleft here.
498 * error callbacks can relocate the cursor anywhere on buffer*/
499 inleft = (Py_ssize_t)(buf.inbuf_end - buf.inbuf);
500 outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);
501 r = codec->encode(state, codec->config, &buf.inbuf, inleft,
502 &buf.outbuf, outleft, flags);
503 if ((r == 0) || (r == MBERR_TOOFEW && !(flags & MBENC_FLUSH)))
504 break;
505 else if (multibytecodec_encerror(codec, state, &buf, errors,r))
506 goto errorexit;
507 else if (r == MBERR_TOOFEW)
508 break;
509 }
510
511 if (codec->encreset != NULL && (flags & MBENC_RESET))
512 for (;;) {
513 Py_ssize_t outleft;
514
515 outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);
516 r = codec->encreset(state, codec->config, &buf.outbuf,
517 outleft);
518 if (r == 0)
519 break;
520 else if (multibytecodec_encerror(codec, state,
521 &buf, errors, r))
522 goto errorexit;
523 }
524
525 finalsize = (Py_ssize_t)((char *)buf.outbuf -
526 PyString_AS_STRING(buf.outobj));
527
528 if (finalsize != PyString_GET_SIZE(buf.outobj))
529 if (_PyString_Resize(&buf.outobj, finalsize) == -1)
530 goto errorexit;
531
532 *data = buf.inbuf;
533 Py_XDECREF(buf.excobj);
534 return buf.outobj;
535
536 errorexit:
537 Py_XDECREF(buf.excobj);
538 Py_XDECREF(buf.outobj);
539 return NULL;
540 }
541
542 static PyObject *
MultibyteCodec_Encode(MultibyteCodecObject * self,PyObject * args,PyObject * kwargs)543 MultibyteCodec_Encode(MultibyteCodecObject *self,
544 PyObject *args, PyObject *kwargs)
545 {
546 MultibyteCodec_State state;
547 Py_UNICODE *data;
548 PyObject *errorcb, *r, *arg, *ucvt;
549 const char *errors = NULL;
550 Py_ssize_t datalen;
551
552 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|z:encode",
553 codeckwarglist, &arg, &errors))
554 return NULL;
555
556 if (PyUnicode_Check(arg))
557 ucvt = NULL;
558 else {
559 arg = ucvt = PyObject_Unicode(arg);
560 if (arg == NULL)
561 return NULL;
562 else if (!PyUnicode_Check(arg)) {
563 PyErr_SetString(PyExc_TypeError,
564 "couldn't convert the object to unicode.");
565 Py_DECREF(ucvt);
566 return NULL;
567 }
568 }
569
570 data = PyUnicode_AS_UNICODE(arg);
571 datalen = PyUnicode_GET_SIZE(arg);
572
573 errorcb = internal_error_callback(errors);
574 if (errorcb == NULL) {
575 Py_XDECREF(ucvt);
576 return NULL;
577 }
578
579 if (self->codec->encinit != NULL &&
580 self->codec->encinit(&state, self->codec->config) != 0)
581 goto errorexit;
582 r = multibytecodec_encode(self->codec, &state,
583 (const Py_UNICODE **)&data, datalen, errorcb,
584 MBENC_FLUSH | MBENC_RESET);
585 if (r == NULL)
586 goto errorexit;
587
588 ERROR_DECREF(errorcb);
589 Py_XDECREF(ucvt);
590 return make_tuple(r, datalen);
591
592 errorexit:
593 ERROR_DECREF(errorcb);
594 Py_XDECREF(ucvt);
595 return NULL;
596 }
597
598 static PyObject *
MultibyteCodec_Decode(MultibyteCodecObject * self,PyObject * args,PyObject * kwargs)599 MultibyteCodec_Decode(MultibyteCodecObject *self,
600 PyObject *args, PyObject *kwargs)
601 {
602 MultibyteCodec_State state;
603 MultibyteDecodeBuffer buf;
604 PyObject *errorcb;
605 Py_buffer pdata;
606 const char *data, *errors = NULL;
607 Py_ssize_t datalen, finalsize;
608
609 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*|z:decode",
610 codeckwarglist, &pdata, &errors))
611 return NULL;
612 data = pdata.buf;
613 datalen = pdata.len;
614
615 errorcb = internal_error_callback(errors);
616 if (errorcb == NULL) {
617 PyBuffer_Release(&pdata);
618 return NULL;
619 }
620
621 if (datalen == 0) {
622 PyBuffer_Release(&pdata);
623 ERROR_DECREF(errorcb);
624 return make_tuple(PyUnicode_FromUnicode(NULL, 0), 0);
625 }
626
627 buf.excobj = NULL;
628 buf.inbuf = buf.inbuf_top = (unsigned char *)data;
629 buf.inbuf_end = buf.inbuf_top + datalen;
630 buf.outobj = PyUnicode_FromUnicode(NULL, datalen);
631 if (buf.outobj == NULL)
632 goto errorexit;
633 buf.outbuf = PyUnicode_AS_UNICODE(buf.outobj);
634 buf.outbuf_end = buf.outbuf + PyUnicode_GET_SIZE(buf.outobj);
635
636 if (self->codec->decinit != NULL &&
637 self->codec->decinit(&state, self->codec->config) != 0)
638 goto errorexit;
639
640 while (buf.inbuf < buf.inbuf_end) {
641 Py_ssize_t inleft, outleft, r;
642
643 inleft = (Py_ssize_t)(buf.inbuf_end - buf.inbuf);
644 outleft = (Py_ssize_t)(buf.outbuf_end - buf.outbuf);
645
646 r = self->codec->decode(&state, self->codec->config,
647 &buf.inbuf, inleft, &buf.outbuf, outleft);
648 if (r == 0)
649 break;
650 else if (multibytecodec_decerror(self->codec, &state,
651 &buf, errorcb, r))
652 goto errorexit;
653 }
654
655 finalsize = (Py_ssize_t)(buf.outbuf -
656 PyUnicode_AS_UNICODE(buf.outobj));
657
658 if (finalsize != PyUnicode_GET_SIZE(buf.outobj))
659 if (PyUnicode_Resize(&buf.outobj, finalsize) == -1)
660 goto errorexit;
661
662 PyBuffer_Release(&pdata);
663 Py_XDECREF(buf.excobj);
664 ERROR_DECREF(errorcb);
665 return make_tuple(buf.outobj, datalen);
666
667 errorexit:
668 PyBuffer_Release(&pdata);
669 ERROR_DECREF(errorcb);
670 Py_XDECREF(buf.excobj);
671 Py_XDECREF(buf.outobj);
672
673 return NULL;
674 }
675
676 static struct PyMethodDef multibytecodec_methods[] = {
677 {"encode", (PyCFunction)MultibyteCodec_Encode,
678 METH_VARARGS | METH_KEYWORDS,
679 MultibyteCodec_Encode__doc__},
680 {"decode", (PyCFunction)MultibyteCodec_Decode,
681 METH_VARARGS | METH_KEYWORDS,
682 MultibyteCodec_Decode__doc__},
683 {NULL, NULL},
684 };
685
686 static void
multibytecodec_dealloc(MultibyteCodecObject * self)687 multibytecodec_dealloc(MultibyteCodecObject *self)
688 {
689 PyObject_Del(self);
690 }
691
692 static PyTypeObject MultibyteCodec_Type = {
693 PyVarObject_HEAD_INIT(NULL, 0)
694 "MultibyteCodec", /* tp_name */
695 sizeof(MultibyteCodecObject), /* tp_basicsize */
696 0, /* tp_itemsize */
697 /* methods */
698 (destructor)multibytecodec_dealloc, /* tp_dealloc */
699 0, /* tp_print */
700 0, /* tp_getattr */
701 0, /* tp_setattr */
702 0, /* tp_compare */
703 0, /* tp_repr */
704 0, /* tp_as_number */
705 0, /* tp_as_sequence */
706 0, /* tp_as_mapping */
707 0, /* tp_hash */
708 0, /* tp_call */
709 0, /* tp_str */
710 PyObject_GenericGetAttr, /* tp_getattro */
711 0, /* tp_setattro */
712 0, /* tp_as_buffer */
713 Py_TPFLAGS_DEFAULT, /* tp_flags */
714 0, /* tp_doc */
715 0, /* tp_traverse */
716 0, /* tp_clear */
717 0, /* tp_richcompare */
718 0, /* tp_weaklistoffset */
719 0, /* tp_iter */
720 0, /* tp_iterext */
721 multibytecodec_methods, /* tp_methods */
722 };
723
724
725 /**
726 * Utility functions for stateful codec mechanism
727 */
728
729 #define STATEFUL_DCTX(o) ((MultibyteStatefulDecoderContext *)(o))
730 #define STATEFUL_ECTX(o) ((MultibyteStatefulEncoderContext *)(o))
731
732 static PyObject *
encoder_encode_stateful(MultibyteStatefulEncoderContext * ctx,PyObject * unistr,int final)733 encoder_encode_stateful(MultibyteStatefulEncoderContext *ctx,
734 PyObject *unistr, int final)
735 {
736 PyObject *ucvt, *r = NULL;
737 Py_UNICODE *inbuf, *inbuf_end, *inbuf_tmp = NULL;
738 Py_ssize_t datalen, origpending;
739
740 if (PyUnicode_Check(unistr))
741 ucvt = NULL;
742 else {
743 unistr = ucvt = PyObject_Unicode(unistr);
744 if (unistr == NULL)
745 return NULL;
746 else if (!PyUnicode_Check(unistr)) {
747 PyErr_SetString(PyExc_TypeError,
748 "couldn't convert the object to unicode.");
749 Py_DECREF(ucvt);
750 return NULL;
751 }
752 }
753
754 datalen = PyUnicode_GET_SIZE(unistr);
755 origpending = ctx->pendingsize;
756
757 if (origpending > 0) {
758 if (datalen > PY_SSIZE_T_MAX - ctx->pendingsize) {
759 PyErr_NoMemory();
760 /* inbuf_tmp == NULL */
761 goto errorexit;
762 }
763 inbuf_tmp = PyMem_New(Py_UNICODE, datalen + ctx->pendingsize);
764 if (inbuf_tmp == NULL)
765 goto errorexit;
766 memcpy(inbuf_tmp, ctx->pending,
767 Py_UNICODE_SIZE * ctx->pendingsize);
768 memcpy(inbuf_tmp + ctx->pendingsize,
769 PyUnicode_AS_UNICODE(unistr),
770 Py_UNICODE_SIZE * datalen);
771 datalen += ctx->pendingsize;
772 ctx->pendingsize = 0;
773 inbuf = inbuf_tmp;
774 }
775 else
776 inbuf = (Py_UNICODE *)PyUnicode_AS_UNICODE(unistr);
777
778 inbuf_end = inbuf + datalen;
779
780 r = multibytecodec_encode(ctx->codec, &ctx->state,
781 (const Py_UNICODE **)&inbuf, datalen,
782 ctx->errors, final ? MBENC_FLUSH | MBENC_RESET : 0);
783 if (r == NULL) {
784 /* recover the original pending buffer */
785 if (origpending > 0)
786 memcpy(ctx->pending, inbuf_tmp,
787 Py_UNICODE_SIZE * origpending);
788 ctx->pendingsize = origpending;
789 goto errorexit;
790 }
791
792 if (inbuf < inbuf_end) {
793 ctx->pendingsize = (Py_ssize_t)(inbuf_end - inbuf);
794 if (ctx->pendingsize > MAXENCPENDING) {
795 /* normal codecs can't reach here */
796 ctx->pendingsize = 0;
797 PyErr_SetString(PyExc_UnicodeError,
798 "pending buffer overflow");
799 goto errorexit;
800 }
801 memcpy(ctx->pending, inbuf,
802 ctx->pendingsize * Py_UNICODE_SIZE);
803 }
804
805 if (inbuf_tmp != NULL)
806 PyMem_Del(inbuf_tmp);
807 Py_XDECREF(ucvt);
808 return r;
809
810 errorexit:
811 if (inbuf_tmp != NULL)
812 PyMem_Del(inbuf_tmp);
813 Py_XDECREF(r);
814 Py_XDECREF(ucvt);
815 return NULL;
816 }
817
818 static int
decoder_append_pending(MultibyteStatefulDecoderContext * ctx,MultibyteDecodeBuffer * buf)819 decoder_append_pending(MultibyteStatefulDecoderContext *ctx,
820 MultibyteDecodeBuffer *buf)
821 {
822 Py_ssize_t npendings;
823
824 npendings = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
825 if (npendings + ctx->pendingsize > MAXDECPENDING ||
826 npendings > PY_SSIZE_T_MAX - ctx->pendingsize) {
827 PyErr_SetString(PyExc_UnicodeError, "pending buffer overflow");
828 return -1;
829 }
830 memcpy(ctx->pending + ctx->pendingsize, buf->inbuf, npendings);
831 ctx->pendingsize += npendings;
832 return 0;
833 }
834
835 static int
decoder_prepare_buffer(MultibyteDecodeBuffer * buf,const char * data,Py_ssize_t size)836 decoder_prepare_buffer(MultibyteDecodeBuffer *buf, const char *data,
837 Py_ssize_t size)
838 {
839 buf->inbuf = buf->inbuf_top = (const unsigned char *)data;
840 buf->inbuf_end = buf->inbuf_top + size;
841 if (buf->outobj == NULL) { /* only if outobj is not allocated yet */
842 buf->outobj = PyUnicode_FromUnicode(NULL, size);
843 if (buf->outobj == NULL)
844 return -1;
845 buf->outbuf = PyUnicode_AS_UNICODE(buf->outobj);
846 buf->outbuf_end = buf->outbuf +
847 PyUnicode_GET_SIZE(buf->outobj);
848 }
849
850 return 0;
851 }
852
853 static int
decoder_feed_buffer(MultibyteStatefulDecoderContext * ctx,MultibyteDecodeBuffer * buf)854 decoder_feed_buffer(MultibyteStatefulDecoderContext *ctx,
855 MultibyteDecodeBuffer *buf)
856 {
857 while (buf->inbuf < buf->inbuf_end) {
858 Py_ssize_t inleft, outleft;
859 Py_ssize_t r;
860
861 inleft = (Py_ssize_t)(buf->inbuf_end - buf->inbuf);
862 outleft = (Py_ssize_t)(buf->outbuf_end - buf->outbuf);
863
864 r = ctx->codec->decode(&ctx->state, ctx->codec->config,
865 &buf->inbuf, inleft, &buf->outbuf, outleft);
866 if (r == 0 || r == MBERR_TOOFEW)
867 break;
868 else if (multibytecodec_decerror(ctx->codec, &ctx->state,
869 buf, ctx->errors, r))
870 return -1;
871 }
872 return 0;
873 }
874
875
876 /**
877 * MultibyteIncrementalEncoder object
878 */
879
880 static PyObject *
mbiencoder_encode(MultibyteIncrementalEncoderObject * self,PyObject * args,PyObject * kwargs)881 mbiencoder_encode(MultibyteIncrementalEncoderObject *self,
882 PyObject *args, PyObject *kwargs)
883 {
884 PyObject *data;
885 int final = 0;
886
887 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|i:encode",
888 incrementalkwarglist, &data, &final))
889 return NULL;
890
891 return encoder_encode_stateful(STATEFUL_ECTX(self), data, final);
892 }
893
894 static PyObject *
mbiencoder_reset(MultibyteIncrementalEncoderObject * self)895 mbiencoder_reset(MultibyteIncrementalEncoderObject *self)
896 {
897 if (self->codec->decreset != NULL &&
898 self->codec->decreset(&self->state, self->codec->config) != 0)
899 return NULL;
900 self->pendingsize = 0;
901
902 Py_RETURN_NONE;
903 }
904
905 static struct PyMethodDef mbiencoder_methods[] = {
906 {"encode", (PyCFunction)mbiencoder_encode,
907 METH_VARARGS | METH_KEYWORDS, NULL},
908 {"reset", (PyCFunction)mbiencoder_reset,
909 METH_NOARGS, NULL},
910 {NULL, NULL},
911 };
912
913 static PyObject *
mbiencoder_new(PyTypeObject * type,PyObject * args,PyObject * kwds)914 mbiencoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
915 {
916 MultibyteIncrementalEncoderObject *self;
917 PyObject *codec = NULL;
918 char *errors = NULL;
919
920 if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s:IncrementalEncoder",
921 incnewkwarglist, &errors))
922 return NULL;
923
924 self = (MultibyteIncrementalEncoderObject *)type->tp_alloc(type, 0);
925 if (self == NULL)
926 return NULL;
927
928 codec = PyObject_GetAttrString((PyObject *)type, "codec");
929 if (codec == NULL)
930 goto errorexit;
931 if (!MultibyteCodec_Check(codec)) {
932 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
933 goto errorexit;
934 }
935
936 self->codec = ((MultibyteCodecObject *)codec)->codec;
937 self->pendingsize = 0;
938 self->errors = internal_error_callback(errors);
939 if (self->errors == NULL)
940 goto errorexit;
941 if (self->codec->encinit != NULL &&
942 self->codec->encinit(&self->state, self->codec->config) != 0)
943 goto errorexit;
944
945 Py_DECREF(codec);
946 return (PyObject *)self;
947
948 errorexit:
949 Py_XDECREF(self);
950 Py_XDECREF(codec);
951 return NULL;
952 }
953
954 static int
mbiencoder_init(PyObject * self,PyObject * args,PyObject * kwds)955 mbiencoder_init(PyObject *self, PyObject *args, PyObject *kwds)
956 {
957 return 0;
958 }
959
960 static int
mbiencoder_traverse(MultibyteIncrementalEncoderObject * self,visitproc visit,void * arg)961 mbiencoder_traverse(MultibyteIncrementalEncoderObject *self,
962 visitproc visit, void *arg)
963 {
964 if (ERROR_ISCUSTOM(self->errors))
965 Py_VISIT(self->errors);
966 return 0;
967 }
968
969 static void
mbiencoder_dealloc(MultibyteIncrementalEncoderObject * self)970 mbiencoder_dealloc(MultibyteIncrementalEncoderObject *self)
971 {
972 PyObject_GC_UnTrack(self);
973 ERROR_DECREF(self->errors);
974 Py_TYPE(self)->tp_free(self);
975 }
976
977 static PyTypeObject MultibyteIncrementalEncoder_Type = {
978 PyVarObject_HEAD_INIT(NULL, 0)
979 "MultibyteIncrementalEncoder", /* tp_name */
980 sizeof(MultibyteIncrementalEncoderObject), /* tp_basicsize */
981 0, /* tp_itemsize */
982 /* methods */
983 (destructor)mbiencoder_dealloc, /* tp_dealloc */
984 0, /* tp_print */
985 0, /* tp_getattr */
986 0, /* tp_setattr */
987 0, /* tp_compare */
988 0, /* tp_repr */
989 0, /* tp_as_number */
990 0, /* tp_as_sequence */
991 0, /* tp_as_mapping */
992 0, /* tp_hash */
993 0, /* tp_call */
994 0, /* tp_str */
995 PyObject_GenericGetAttr, /* tp_getattro */
996 0, /* tp_setattro */
997 0, /* tp_as_buffer */
998 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC
999 | Py_TPFLAGS_BASETYPE, /* tp_flags */
1000 0, /* tp_doc */
1001 (traverseproc)mbiencoder_traverse, /* tp_traverse */
1002 0, /* tp_clear */
1003 0, /* tp_richcompare */
1004 0, /* tp_weaklistoffset */
1005 0, /* tp_iter */
1006 0, /* tp_iterext */
1007 mbiencoder_methods, /* tp_methods */
1008 0, /* tp_members */
1009 codecctx_getsets, /* tp_getset */
1010 0, /* tp_base */
1011 0, /* tp_dict */
1012 0, /* tp_descr_get */
1013 0, /* tp_descr_set */
1014 0, /* tp_dictoffset */
1015 mbiencoder_init, /* tp_init */
1016 0, /* tp_alloc */
1017 mbiencoder_new, /* tp_new */
1018 };
1019
1020
1021 /**
1022 * MultibyteIncrementalDecoder object
1023 */
1024
1025 static PyObject *
mbidecoder_decode(MultibyteIncrementalDecoderObject * self,PyObject * args,PyObject * kwargs)1026 mbidecoder_decode(MultibyteIncrementalDecoderObject *self,
1027 PyObject *args, PyObject *kwargs)
1028 {
1029 MultibyteDecodeBuffer buf;
1030 char *data, *wdata = NULL;
1031 Py_buffer pdata;
1032 Py_ssize_t wsize, finalsize = 0, size, origpending;
1033 int final = 0;
1034
1035 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s*|i:decode",
1036 incrementalkwarglist, &pdata, &final))
1037 return NULL;
1038 data = pdata.buf;
1039 size = pdata.len;
1040
1041 buf.outobj = buf.excobj = NULL;
1042 origpending = self->pendingsize;
1043
1044 if (self->pendingsize == 0) {
1045 wsize = size;
1046 wdata = data;
1047 }
1048 else {
1049 if (size > PY_SSIZE_T_MAX - self->pendingsize) {
1050 PyErr_NoMemory();
1051 goto errorexit;
1052 }
1053 wsize = size + self->pendingsize;
1054 wdata = PyMem_Malloc(wsize);
1055 if (wdata == NULL)
1056 goto errorexit;
1057 memcpy(wdata, self->pending, self->pendingsize);
1058 memcpy(wdata + self->pendingsize, data, size);
1059 self->pendingsize = 0;
1060 }
1061
1062 if (decoder_prepare_buffer(&buf, wdata, wsize) != 0)
1063 goto errorexit;
1064
1065 if (decoder_feed_buffer(STATEFUL_DCTX(self), &buf))
1066 goto errorexit;
1067
1068 if (final && buf.inbuf < buf.inbuf_end) {
1069 if (multibytecodec_decerror(self->codec, &self->state,
1070 &buf, self->errors, MBERR_TOOFEW)) {
1071 /* recover the original pending buffer */
1072 memcpy(self->pending, wdata, origpending);
1073 self->pendingsize = origpending;
1074 goto errorexit;
1075 }
1076 }
1077
1078 if (buf.inbuf < buf.inbuf_end) { /* pending sequence still exists */
1079 if (decoder_append_pending(STATEFUL_DCTX(self), &buf) != 0)
1080 goto errorexit;
1081 }
1082
1083 finalsize = (Py_ssize_t)(buf.outbuf - PyUnicode_AS_UNICODE(buf.outobj));
1084 if (finalsize != PyUnicode_GET_SIZE(buf.outobj))
1085 if (PyUnicode_Resize(&buf.outobj, finalsize) == -1)
1086 goto errorexit;
1087
1088 PyBuffer_Release(&pdata);
1089 if (wdata != data)
1090 PyMem_Del(wdata);
1091 Py_XDECREF(buf.excobj);
1092 return buf.outobj;
1093
1094 errorexit:
1095 PyBuffer_Release(&pdata);
1096 if (wdata != NULL && wdata != data)
1097 PyMem_Del(wdata);
1098 Py_XDECREF(buf.excobj);
1099 Py_XDECREF(buf.outobj);
1100 return NULL;
1101 }
1102
1103 static PyObject *
mbidecoder_reset(MultibyteIncrementalDecoderObject * self)1104 mbidecoder_reset(MultibyteIncrementalDecoderObject *self)
1105 {
1106 if (self->codec->decreset != NULL &&
1107 self->codec->decreset(&self->state, self->codec->config) != 0)
1108 return NULL;
1109 self->pendingsize = 0;
1110
1111 Py_RETURN_NONE;
1112 }
1113
1114 static struct PyMethodDef mbidecoder_methods[] = {
1115 {"decode", (PyCFunction)mbidecoder_decode,
1116 METH_VARARGS | METH_KEYWORDS, NULL},
1117 {"reset", (PyCFunction)mbidecoder_reset,
1118 METH_NOARGS, NULL},
1119 {NULL, NULL},
1120 };
1121
1122 static PyObject *
mbidecoder_new(PyTypeObject * type,PyObject * args,PyObject * kwds)1123 mbidecoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1124 {
1125 MultibyteIncrementalDecoderObject *self;
1126 PyObject *codec = NULL;
1127 char *errors = NULL;
1128
1129 if (!PyArg_ParseTupleAndKeywords(args, kwds, "|s:IncrementalDecoder",
1130 incnewkwarglist, &errors))
1131 return NULL;
1132
1133 self = (MultibyteIncrementalDecoderObject *)type->tp_alloc(type, 0);
1134 if (self == NULL)
1135 return NULL;
1136
1137 codec = PyObject_GetAttrString((PyObject *)type, "codec");
1138 if (codec == NULL)
1139 goto errorexit;
1140 if (!MultibyteCodec_Check(codec)) {
1141 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
1142 goto errorexit;
1143 }
1144
1145 self->codec = ((MultibyteCodecObject *)codec)->codec;
1146 self->pendingsize = 0;
1147 self->errors = internal_error_callback(errors);
1148 if (self->errors == NULL)
1149 goto errorexit;
1150 if (self->codec->decinit != NULL &&
1151 self->codec->decinit(&self->state, self->codec->config) != 0)
1152 goto errorexit;
1153
1154 Py_DECREF(codec);
1155 return (PyObject *)self;
1156
1157 errorexit:
1158 Py_XDECREF(self);
1159 Py_XDECREF(codec);
1160 return NULL;
1161 }
1162
1163 static int
mbidecoder_init(PyObject * self,PyObject * args,PyObject * kwds)1164 mbidecoder_init(PyObject *self, PyObject *args, PyObject *kwds)
1165 {
1166 return 0;
1167 }
1168
1169 static int
mbidecoder_traverse(MultibyteIncrementalDecoderObject * self,visitproc visit,void * arg)1170 mbidecoder_traverse(MultibyteIncrementalDecoderObject *self,
1171 visitproc visit, void *arg)
1172 {
1173 if (ERROR_ISCUSTOM(self->errors))
1174 Py_VISIT(self->errors);
1175 return 0;
1176 }
1177
1178 static void
mbidecoder_dealloc(MultibyteIncrementalDecoderObject * self)1179 mbidecoder_dealloc(MultibyteIncrementalDecoderObject *self)
1180 {
1181 PyObject_GC_UnTrack(self);
1182 ERROR_DECREF(self->errors);
1183 Py_TYPE(self)->tp_free(self);
1184 }
1185
1186 static PyTypeObject MultibyteIncrementalDecoder_Type = {
1187 PyVarObject_HEAD_INIT(NULL, 0)
1188 "MultibyteIncrementalDecoder", /* tp_name */
1189 sizeof(MultibyteIncrementalDecoderObject), /* tp_basicsize */
1190 0, /* tp_itemsize */
1191 /* methods */
1192 (destructor)mbidecoder_dealloc, /* tp_dealloc */
1193 0, /* tp_print */
1194 0, /* tp_getattr */
1195 0, /* tp_setattr */
1196 0, /* tp_compare */
1197 0, /* tp_repr */
1198 0, /* tp_as_number */
1199 0, /* tp_as_sequence */
1200 0, /* tp_as_mapping */
1201 0, /* tp_hash */
1202 0, /* tp_call */
1203 0, /* tp_str */
1204 PyObject_GenericGetAttr, /* tp_getattro */
1205 0, /* tp_setattro */
1206 0, /* tp_as_buffer */
1207 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC
1208 | Py_TPFLAGS_BASETYPE, /* tp_flags */
1209 0, /* tp_doc */
1210 (traverseproc)mbidecoder_traverse, /* tp_traverse */
1211 0, /* tp_clear */
1212 0, /* tp_richcompare */
1213 0, /* tp_weaklistoffset */
1214 0, /* tp_iter */
1215 0, /* tp_iterext */
1216 mbidecoder_methods, /* tp_methods */
1217 0, /* tp_members */
1218 codecctx_getsets, /* tp_getset */
1219 0, /* tp_base */
1220 0, /* tp_dict */
1221 0, /* tp_descr_get */
1222 0, /* tp_descr_set */
1223 0, /* tp_dictoffset */
1224 mbidecoder_init, /* tp_init */
1225 0, /* tp_alloc */
1226 mbidecoder_new, /* tp_new */
1227 };
1228
1229
1230 /**
1231 * MultibyteStreamReader object
1232 */
1233
1234 static PyObject *
mbstreamreader_iread(MultibyteStreamReaderObject * self,const char * method,Py_ssize_t sizehint)1235 mbstreamreader_iread(MultibyteStreamReaderObject *self,
1236 const char *method, Py_ssize_t sizehint)
1237 {
1238 MultibyteDecodeBuffer buf;
1239 PyObject *cres;
1240 Py_ssize_t rsize, finalsize = 0;
1241
1242 if (sizehint == 0)
1243 return PyUnicode_FromUnicode(NULL, 0);
1244
1245 buf.outobj = buf.excobj = NULL;
1246 cres = NULL;
1247
1248 for (;;) {
1249 int endoffile;
1250
1251 if (sizehint < 0)
1252 cres = PyObject_CallMethod(self->stream,
1253 (char *)method, NULL);
1254 else
1255 cres = PyObject_CallMethod(self->stream,
1256 (char *)method, "i", sizehint);
1257 if (cres == NULL)
1258 goto errorexit;
1259
1260 if (!PyString_Check(cres)) {
1261 PyErr_SetString(PyExc_TypeError,
1262 "stream function returned a "
1263 "non-string object");
1264 goto errorexit;
1265 }
1266
1267 endoffile = (PyString_GET_SIZE(cres) == 0);
1268
1269 if (self->pendingsize > 0) {
1270 PyObject *ctr;
1271 char *ctrdata;
1272
1273 if (PyString_GET_SIZE(cres) > PY_SSIZE_T_MAX - self->pendingsize) {
1274 PyErr_NoMemory();
1275 goto errorexit;
1276 }
1277 rsize = PyString_GET_SIZE(cres) + self->pendingsize;
1278 ctr = PyString_FromStringAndSize(NULL, rsize);
1279 if (ctr == NULL)
1280 goto errorexit;
1281 ctrdata = PyString_AS_STRING(ctr);
1282 memcpy(ctrdata, self->pending, self->pendingsize);
1283 memcpy(ctrdata + self->pendingsize,
1284 PyString_AS_STRING(cres),
1285 PyString_GET_SIZE(cres));
1286 Py_DECREF(cres);
1287 cres = ctr;
1288 self->pendingsize = 0;
1289 }
1290
1291 rsize = PyString_GET_SIZE(cres);
1292 if (decoder_prepare_buffer(&buf, PyString_AS_STRING(cres),
1293 rsize) != 0)
1294 goto errorexit;
1295
1296 if (rsize > 0 && decoder_feed_buffer(
1297 (MultibyteStatefulDecoderContext *)self, &buf))
1298 goto errorexit;
1299
1300 if (endoffile || sizehint < 0) {
1301 if (buf.inbuf < buf.inbuf_end &&
1302 multibytecodec_decerror(self->codec, &self->state,
1303 &buf, self->errors, MBERR_TOOFEW))
1304 goto errorexit;
1305 }
1306
1307 if (buf.inbuf < buf.inbuf_end) { /* pending sequence exists */
1308 if (decoder_append_pending(STATEFUL_DCTX(self),
1309 &buf) != 0)
1310 goto errorexit;
1311 }
1312
1313 finalsize = (Py_ssize_t)(buf.outbuf -
1314 PyUnicode_AS_UNICODE(buf.outobj));
1315 Py_DECREF(cres);
1316 cres = NULL;
1317
1318 if (sizehint < 0 || finalsize != 0 || rsize == 0)
1319 break;
1320
1321 sizehint = 1; /* read 1 more byte and retry */
1322 }
1323
1324 if (finalsize != PyUnicode_GET_SIZE(buf.outobj))
1325 if (PyUnicode_Resize(&buf.outobj, finalsize) == -1)
1326 goto errorexit;
1327
1328 Py_XDECREF(cres);
1329 Py_XDECREF(buf.excobj);
1330 return buf.outobj;
1331
1332 errorexit:
1333 Py_XDECREF(cres);
1334 Py_XDECREF(buf.excobj);
1335 Py_XDECREF(buf.outobj);
1336 return NULL;
1337 }
1338
1339 static PyObject *
mbstreamreader_read(MultibyteStreamReaderObject * self,PyObject * args)1340 mbstreamreader_read(MultibyteStreamReaderObject *self, PyObject *args)
1341 {
1342 PyObject *sizeobj = NULL;
1343 Py_ssize_t size;
1344
1345 if (!PyArg_UnpackTuple(args, "read", 0, 1, &sizeobj))
1346 return NULL;
1347
1348 if (sizeobj == Py_None || sizeobj == NULL)
1349 size = -1;
1350 else if (PyInt_Check(sizeobj))
1351 size = PyInt_AsSsize_t(sizeobj);
1352 else {
1353 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");
1354 return NULL;
1355 }
1356
1357 return mbstreamreader_iread(self, "read", size);
1358 }
1359
1360 static PyObject *
mbstreamreader_readline(MultibyteStreamReaderObject * self,PyObject * args)1361 mbstreamreader_readline(MultibyteStreamReaderObject *self, PyObject *args)
1362 {
1363 PyObject *sizeobj = NULL;
1364 Py_ssize_t size;
1365
1366 if (!PyArg_UnpackTuple(args, "readline", 0, 1, &sizeobj))
1367 return NULL;
1368
1369 if (sizeobj == Py_None || sizeobj == NULL)
1370 size = -1;
1371 else if (PyInt_Check(sizeobj))
1372 size = PyInt_AsSsize_t(sizeobj);
1373 else {
1374 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");
1375 return NULL;
1376 }
1377
1378 return mbstreamreader_iread(self, "readline", size);
1379 }
1380
1381 static PyObject *
mbstreamreader_readlines(MultibyteStreamReaderObject * self,PyObject * args)1382 mbstreamreader_readlines(MultibyteStreamReaderObject *self, PyObject *args)
1383 {
1384 PyObject *sizehintobj = NULL, *r, *sr;
1385 Py_ssize_t sizehint;
1386
1387 if (!PyArg_UnpackTuple(args, "readlines", 0, 1, &sizehintobj))
1388 return NULL;
1389
1390 if (sizehintobj == Py_None || sizehintobj == NULL)
1391 sizehint = -1;
1392 else if (PyInt_Check(sizehintobj))
1393 sizehint = PyInt_AsSsize_t(sizehintobj);
1394 else {
1395 PyErr_SetString(PyExc_TypeError, "arg 1 must be an integer");
1396 return NULL;
1397 }
1398
1399 r = mbstreamreader_iread(self, "read", sizehint);
1400 if (r == NULL)
1401 return NULL;
1402
1403 sr = PyUnicode_Splitlines(r, 1);
1404 Py_DECREF(r);
1405 return sr;
1406 }
1407
1408 static PyObject *
mbstreamreader_reset(MultibyteStreamReaderObject * self)1409 mbstreamreader_reset(MultibyteStreamReaderObject *self)
1410 {
1411 if (self->codec->decreset != NULL &&
1412 self->codec->decreset(&self->state, self->codec->config) != 0)
1413 return NULL;
1414 self->pendingsize = 0;
1415
1416 Py_RETURN_NONE;
1417 }
1418
1419 static struct PyMethodDef mbstreamreader_methods[] = {
1420 {"read", (PyCFunction)mbstreamreader_read,
1421 METH_VARARGS, NULL},
1422 {"readline", (PyCFunction)mbstreamreader_readline,
1423 METH_VARARGS, NULL},
1424 {"readlines", (PyCFunction)mbstreamreader_readlines,
1425 METH_VARARGS, NULL},
1426 {"reset", (PyCFunction)mbstreamreader_reset,
1427 METH_NOARGS, NULL},
1428 {NULL, NULL},
1429 };
1430
1431 static PyMemberDef mbstreamreader_members[] = {
1432 {"stream", T_OBJECT,
1433 offsetof(MultibyteStreamReaderObject, stream),
1434 READONLY, NULL},
1435 {NULL,}
1436 };
1437
1438 static PyObject *
mbstreamreader_new(PyTypeObject * type,PyObject * args,PyObject * kwds)1439 mbstreamreader_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1440 {
1441 MultibyteStreamReaderObject *self;
1442 PyObject *stream, *codec = NULL;
1443 char *errors = NULL;
1444
1445 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s:StreamReader",
1446 streamkwarglist, &stream, &errors))
1447 return NULL;
1448
1449 self = (MultibyteStreamReaderObject *)type->tp_alloc(type, 0);
1450 if (self == NULL)
1451 return NULL;
1452
1453 codec = PyObject_GetAttrString((PyObject *)type, "codec");
1454 if (codec == NULL)
1455 goto errorexit;
1456 if (!MultibyteCodec_Check(codec)) {
1457 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
1458 goto errorexit;
1459 }
1460
1461 self->codec = ((MultibyteCodecObject *)codec)->codec;
1462 self->stream = stream;
1463 Py_INCREF(stream);
1464 self->pendingsize = 0;
1465 self->errors = internal_error_callback(errors);
1466 if (self->errors == NULL)
1467 goto errorexit;
1468 if (self->codec->decinit != NULL &&
1469 self->codec->decinit(&self->state, self->codec->config) != 0)
1470 goto errorexit;
1471
1472 Py_DECREF(codec);
1473 return (PyObject *)self;
1474
1475 errorexit:
1476 Py_XDECREF(self);
1477 Py_XDECREF(codec);
1478 return NULL;
1479 }
1480
1481 static int
mbstreamreader_init(PyObject * self,PyObject * args,PyObject * kwds)1482 mbstreamreader_init(PyObject *self, PyObject *args, PyObject *kwds)
1483 {
1484 return 0;
1485 }
1486
1487 static int
mbstreamreader_traverse(MultibyteStreamReaderObject * self,visitproc visit,void * arg)1488 mbstreamreader_traverse(MultibyteStreamReaderObject *self,
1489 visitproc visit, void *arg)
1490 {
1491 if (ERROR_ISCUSTOM(self->errors))
1492 Py_VISIT(self->errors);
1493 Py_VISIT(self->stream);
1494 return 0;
1495 }
1496
1497 static void
mbstreamreader_dealloc(MultibyteStreamReaderObject * self)1498 mbstreamreader_dealloc(MultibyteStreamReaderObject *self)
1499 {
1500 PyObject_GC_UnTrack(self);
1501 ERROR_DECREF(self->errors);
1502 Py_XDECREF(self->stream);
1503 Py_TYPE(self)->tp_free(self);
1504 }
1505
1506 static PyTypeObject MultibyteStreamReader_Type = {
1507 PyVarObject_HEAD_INIT(NULL, 0)
1508 "MultibyteStreamReader", /* tp_name */
1509 sizeof(MultibyteStreamReaderObject), /* tp_basicsize */
1510 0, /* tp_itemsize */
1511 /* methods */
1512 (destructor)mbstreamreader_dealloc, /* tp_dealloc */
1513 0, /* tp_print */
1514 0, /* tp_getattr */
1515 0, /* tp_setattr */
1516 0, /* tp_compare */
1517 0, /* tp_repr */
1518 0, /* tp_as_number */
1519 0, /* tp_as_sequence */
1520 0, /* tp_as_mapping */
1521 0, /* tp_hash */
1522 0, /* tp_call */
1523 0, /* tp_str */
1524 PyObject_GenericGetAttr, /* tp_getattro */
1525 0, /* tp_setattro */
1526 0, /* tp_as_buffer */
1527 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC
1528 | Py_TPFLAGS_BASETYPE, /* tp_flags */
1529 0, /* tp_doc */
1530 (traverseproc)mbstreamreader_traverse, /* tp_traverse */
1531 0, /* tp_clear */
1532 0, /* tp_richcompare */
1533 0, /* tp_weaklistoffset */
1534 0, /* tp_iter */
1535 0, /* tp_iterext */
1536 mbstreamreader_methods, /* tp_methods */
1537 mbstreamreader_members, /* tp_members */
1538 codecctx_getsets, /* tp_getset */
1539 0, /* tp_base */
1540 0, /* tp_dict */
1541 0, /* tp_descr_get */
1542 0, /* tp_descr_set */
1543 0, /* tp_dictoffset */
1544 mbstreamreader_init, /* tp_init */
1545 0, /* tp_alloc */
1546 mbstreamreader_new, /* tp_new */
1547 };
1548
1549
1550 /**
1551 * MultibyteStreamWriter object
1552 */
1553
1554 static int
mbstreamwriter_iwrite(MultibyteStreamWriterObject * self,PyObject * unistr)1555 mbstreamwriter_iwrite(MultibyteStreamWriterObject *self,
1556 PyObject *unistr)
1557 {
1558 PyObject *str, *wr;
1559
1560 str = encoder_encode_stateful(STATEFUL_ECTX(self), unistr, 0);
1561 if (str == NULL)
1562 return -1;
1563
1564 wr = PyObject_CallMethod(self->stream, "write", "O", str);
1565 Py_DECREF(str);
1566 if (wr == NULL)
1567 return -1;
1568
1569 Py_DECREF(wr);
1570 return 0;
1571 }
1572
1573 static PyObject *
mbstreamwriter_write(MultibyteStreamWriterObject * self,PyObject * strobj)1574 mbstreamwriter_write(MultibyteStreamWriterObject *self, PyObject *strobj)
1575 {
1576 if (mbstreamwriter_iwrite(self, strobj))
1577 return NULL;
1578 else
1579 Py_RETURN_NONE;
1580 }
1581
1582 static PyObject *
mbstreamwriter_writelines(MultibyteStreamWriterObject * self,PyObject * lines)1583 mbstreamwriter_writelines(MultibyteStreamWriterObject *self, PyObject *lines)
1584 {
1585 PyObject *strobj;
1586 int i, r;
1587
1588 if (!PySequence_Check(lines)) {
1589 PyErr_SetString(PyExc_TypeError,
1590 "arg must be a sequence object");
1591 return NULL;
1592 }
1593
1594 for (i = 0; i < PySequence_Length(lines); i++) {
1595 /* length can be changed even within this loop */
1596 strobj = PySequence_GetItem(lines, i);
1597 if (strobj == NULL)
1598 return NULL;
1599
1600 r = mbstreamwriter_iwrite(self, strobj);
1601 Py_DECREF(strobj);
1602 if (r == -1)
1603 return NULL;
1604 }
1605 /* PySequence_Length() can fail */
1606 if (PyErr_Occurred())
1607 return NULL;
1608
1609 Py_RETURN_NONE;
1610 }
1611
1612 static PyObject *
mbstreamwriter_reset(MultibyteStreamWriterObject * self)1613 mbstreamwriter_reset(MultibyteStreamWriterObject *self)
1614 {
1615 const Py_UNICODE *pending;
1616 PyObject *pwrt;
1617
1618 pending = self->pending;
1619 pwrt = multibytecodec_encode(self->codec, &self->state,
1620 &pending, self->pendingsize, self->errors,
1621 MBENC_FLUSH | MBENC_RESET);
1622 /* some pending buffer can be truncated when UnicodeEncodeError is
1623 * raised on 'strict' mode. but, 'reset' method is designed to
1624 * reset the pending buffer or states so failed string sequence
1625 * ought to be missed */
1626 self->pendingsize = 0;
1627 if (pwrt == NULL)
1628 return NULL;
1629
1630 if (PyString_Size(pwrt) > 0) {
1631 PyObject *wr;
1632 wr = PyObject_CallMethod(self->stream, "write", "O", pwrt);
1633 if (wr == NULL) {
1634 Py_DECREF(pwrt);
1635 return NULL;
1636 }
1637 }
1638 Py_DECREF(pwrt);
1639
1640 Py_RETURN_NONE;
1641 }
1642
1643 static PyObject *
mbstreamwriter_new(PyTypeObject * type,PyObject * args,PyObject * kwds)1644 mbstreamwriter_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
1645 {
1646 MultibyteStreamWriterObject *self;
1647 PyObject *stream, *codec = NULL;
1648 char *errors = NULL;
1649
1650 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s:StreamWriter",
1651 streamkwarglist, &stream, &errors))
1652 return NULL;
1653
1654 self = (MultibyteStreamWriterObject *)type->tp_alloc(type, 0);
1655 if (self == NULL)
1656 return NULL;
1657
1658 codec = PyObject_GetAttrString((PyObject *)type, "codec");
1659 if (codec == NULL)
1660 goto errorexit;
1661 if (!MultibyteCodec_Check(codec)) {
1662 PyErr_SetString(PyExc_TypeError, "codec is unexpected type");
1663 goto errorexit;
1664 }
1665
1666 self->codec = ((MultibyteCodecObject *)codec)->codec;
1667 self->stream = stream;
1668 Py_INCREF(stream);
1669 self->pendingsize = 0;
1670 self->errors = internal_error_callback(errors);
1671 if (self->errors == NULL)
1672 goto errorexit;
1673 if (self->codec->encinit != NULL &&
1674 self->codec->encinit(&self->state, self->codec->config) != 0)
1675 goto errorexit;
1676
1677 Py_DECREF(codec);
1678 return (PyObject *)self;
1679
1680 errorexit:
1681 Py_XDECREF(self);
1682 Py_XDECREF(codec);
1683 return NULL;
1684 }
1685
1686 static int
mbstreamwriter_init(PyObject * self,PyObject * args,PyObject * kwds)1687 mbstreamwriter_init(PyObject *self, PyObject *args, PyObject *kwds)
1688 {
1689 return 0;
1690 }
1691
1692 static int
mbstreamwriter_traverse(MultibyteStreamWriterObject * self,visitproc visit,void * arg)1693 mbstreamwriter_traverse(MultibyteStreamWriterObject *self,
1694 visitproc visit, void *arg)
1695 {
1696 if (ERROR_ISCUSTOM(self->errors))
1697 Py_VISIT(self->errors);
1698 Py_VISIT(self->stream);
1699 return 0;
1700 }
1701
1702 static void
mbstreamwriter_dealloc(MultibyteStreamWriterObject * self)1703 mbstreamwriter_dealloc(MultibyteStreamWriterObject *self)
1704 {
1705 PyObject_GC_UnTrack(self);
1706 ERROR_DECREF(self->errors);
1707 Py_XDECREF(self->stream);
1708 Py_TYPE(self)->tp_free(self);
1709 }
1710
1711 static struct PyMethodDef mbstreamwriter_methods[] = {
1712 {"write", (PyCFunction)mbstreamwriter_write,
1713 METH_O, NULL},
1714 {"writelines", (PyCFunction)mbstreamwriter_writelines,
1715 METH_O, NULL},
1716 {"reset", (PyCFunction)mbstreamwriter_reset,
1717 METH_NOARGS, NULL},
1718 {NULL, NULL},
1719 };
1720
1721 static PyMemberDef mbstreamwriter_members[] = {
1722 {"stream", T_OBJECT,
1723 offsetof(MultibyteStreamWriterObject, stream),
1724 READONLY, NULL},
1725 {NULL,}
1726 };
1727
1728 static PyTypeObject MultibyteStreamWriter_Type = {
1729 PyVarObject_HEAD_INIT(NULL, 0)
1730 "MultibyteStreamWriter", /* tp_name */
1731 sizeof(MultibyteStreamWriterObject), /* tp_basicsize */
1732 0, /* tp_itemsize */
1733 /* methods */
1734 (destructor)mbstreamwriter_dealloc, /* tp_dealloc */
1735 0, /* tp_print */
1736 0, /* tp_getattr */
1737 0, /* tp_setattr */
1738 0, /* tp_compare */
1739 0, /* tp_repr */
1740 0, /* tp_as_number */
1741 0, /* tp_as_sequence */
1742 0, /* tp_as_mapping */
1743 0, /* tp_hash */
1744 0, /* tp_call */
1745 0, /* tp_str */
1746 PyObject_GenericGetAttr, /* tp_getattro */
1747 0, /* tp_setattro */
1748 0, /* tp_as_buffer */
1749 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC
1750 | Py_TPFLAGS_BASETYPE, /* tp_flags */
1751 0, /* tp_doc */
1752 (traverseproc)mbstreamwriter_traverse, /* tp_traverse */
1753 0, /* tp_clear */
1754 0, /* tp_richcompare */
1755 0, /* tp_weaklistoffset */
1756 0, /* tp_iter */
1757 0, /* tp_iterext */
1758 mbstreamwriter_methods, /* tp_methods */
1759 mbstreamwriter_members, /* tp_members */
1760 codecctx_getsets, /* tp_getset */
1761 0, /* tp_base */
1762 0, /* tp_dict */
1763 0, /* tp_descr_get */
1764 0, /* tp_descr_set */
1765 0, /* tp_dictoffset */
1766 mbstreamwriter_init, /* tp_init */
1767 0, /* tp_alloc */
1768 mbstreamwriter_new, /* tp_new */
1769 };
1770
1771
1772 /**
1773 * Exposed factory function
1774 */
1775
1776 static PyObject *
__create_codec(PyObject * ignore,PyObject * arg)1777 __create_codec(PyObject *ignore, PyObject *arg)
1778 {
1779 MultibyteCodecObject *self;
1780 MultibyteCodec *codec;
1781
1782 if (!PyCapsule_IsValid(arg, PyMultibyteCodec_CAPSULE_NAME)) {
1783 PyErr_SetString(PyExc_ValueError, "argument type invalid");
1784 return NULL;
1785 }
1786
1787 codec = PyCapsule_GetPointer(arg, PyMultibyteCodec_CAPSULE_NAME);
1788 if (codec->codecinit != NULL && codec->codecinit(codec->config) != 0)
1789 return NULL;
1790
1791 self = PyObject_New(MultibyteCodecObject, &MultibyteCodec_Type);
1792 if (self == NULL)
1793 return NULL;
1794 self->codec = codec;
1795
1796 return (PyObject *)self;
1797 }
1798
1799 static struct PyMethodDef __methods[] = {
1800 {"__create_codec", (PyCFunction)__create_codec, METH_O},
1801 {NULL, NULL},
1802 };
1803
1804 PyMODINIT_FUNC
init_multibytecodec(void)1805 init_multibytecodec(void)
1806 {
1807 int i;
1808 PyObject *m;
1809 PyTypeObject *typelist[] = {
1810 &MultibyteIncrementalEncoder_Type,
1811 &MultibyteIncrementalDecoder_Type,
1812 &MultibyteStreamReader_Type,
1813 &MultibyteStreamWriter_Type,
1814 NULL
1815 };
1816
1817 if (PyType_Ready(&MultibyteCodec_Type) < 0)
1818 return;
1819
1820 m = Py_InitModule("_multibytecodec", __methods);
1821 if (m == NULL)
1822 return;
1823
1824 for (i = 0; typelist[i] != NULL; i++) {
1825 if (PyType_Ready(typelist[i]) < 0)
1826 return;
1827 Py_INCREF(typelist[i]);
1828 PyModule_AddObject(m, typelist[i]->tp_name,
1829 (PyObject *)typelist[i]);
1830 }
1831
1832 if (PyErr_Occurred())
1833 Py_FatalError("can't initialize the _multibytecodec module");
1834 }
1835