1 #include "Python.h"
2 #include "cStringIO.h"
3 #include "structmember.h"
4
5 PyDoc_STRVAR(cPickle_module_documentation,
6 "C implementation and optimization of the Python pickle module.");
7
8 #ifndef Py_eval_input
9 #include <graminit.h>
10 #define Py_eval_input eval_input
11 #endif /* Py_eval_input */
12
13 #define DEL_LIST_SLICE(list, from, to) (PyList_SetSlice(list, from, to, NULL))
14
15 #define WRITE_BUF_SIZE 256
16
17 /* Bump this when new opcodes are added to the pickle protocol. */
18 #define HIGHEST_PROTOCOL 2
19
20 /*
21 * Note: The UNICODE macro controls the TCHAR meaning of the win32 API. Since
22 * all headers have already been included here, we can safely redefine it.
23 */
24 #ifdef UNICODE
25 # undef UNICODE
26 #endif
27
28 /*
29 * Pickle opcodes. These must be kept in synch with pickle.py. Extensive
30 * docs are in pickletools.py.
31 */
32 #define MARK '('
33 #define STOP '.'
34 #define POP '0'
35 #define POP_MARK '1'
36 #define DUP '2'
37 #define FLOAT 'F'
38 #define BINFLOAT 'G'
39 #define INT 'I'
40 #define BININT 'J'
41 #define BININT1 'K'
42 #define LONG 'L'
43 #define BININT2 'M'
44 #define NONE 'N'
45 #define PERSID 'P'
46 #define BINPERSID 'Q'
47 #define REDUCE 'R'
48 #define STRING 'S'
49 #define BINSTRING 'T'
50 #define SHORT_BINSTRING 'U'
51 #define UNICODE 'V'
52 #define BINUNICODE 'X'
53 #define APPEND 'a'
54 #define BUILD 'b'
55 #define GLOBAL 'c'
56 #define DICT 'd'
57 #define EMPTY_DICT '}'
58 #define APPENDS 'e'
59 #define GET 'g'
60 #define BINGET 'h'
61 #define INST 'i'
62 #define LONG_BINGET 'j'
63 #define LIST 'l'
64 #define EMPTY_LIST ']'
65 #define OBJ 'o'
66 #define PUT 'p'
67 #define BINPUT 'q'
68 #define LONG_BINPUT 'r'
69 #define SETITEM 's'
70 #define TUPLE 't'
71 #define EMPTY_TUPLE ')'
72 #define SETITEMS 'u'
73
74 /* Protocol 2. */
75 #define PROTO '\x80' /* identify pickle protocol */
76 #define NEWOBJ '\x81' /* build object by applying cls.__new__ to argtuple */
77 #define EXT1 '\x82' /* push object from extension registry; 1-byte index */
78 #define EXT2 '\x83' /* ditto, but 2-byte index */
79 #define EXT4 '\x84' /* ditto, but 4-byte index */
80 #define TUPLE1 '\x85' /* build 1-tuple from stack top */
81 #define TUPLE2 '\x86' /* build 2-tuple from two topmost stack items */
82 #define TUPLE3 '\x87' /* build 3-tuple from three topmost stack items */
83 #define NEWTRUE '\x88' /* push True */
84 #define NEWFALSE '\x89' /* push False */
85 #define LONG1 '\x8a' /* push long from < 256 bytes */
86 #define LONG4 '\x8b' /* push really big long */
87
88 /* There aren't opcodes -- they're ways to pickle bools before protocol 2,
89 * so that unpicklers written before bools were introduced unpickle them
90 * as ints, but unpicklers after can recognize that bools were intended.
91 * Note that protocol 2 added direct ways to pickle bools.
92 */
93 #undef TRUE
94 #define TRUE "I01\n"
95 #undef FALSE
96 #define FALSE "I00\n"
97
98 /* Keep in synch with pickle.Pickler._BATCHSIZE. This is how many elements
99 * batch_list/dict() pumps out before doing APPENDS/SETITEMS. Nothing will
100 * break if this gets out of synch with pickle.py, but it's unclear that
101 * would help anything either.
102 */
103 #define BATCHSIZE 1000
104
105 static char MARKv = MARK;
106
107 static PyObject *PickleError;
108 static PyObject *PicklingError;
109 static PyObject *UnpickleableError;
110 static PyObject *UnpicklingError;
111 static PyObject *BadPickleGet;
112
113 /* As the name says, an empty tuple. */
114 static PyObject *empty_tuple;
115
116 /* copy_reg.dispatch_table, {type_object: pickling_function} */
117 static PyObject *dispatch_table;
118
119 /* For EXT[124] opcodes. */
120 /* copy_reg._extension_registry, {(module_name, function_name): code} */
121 static PyObject *extension_registry;
122 /* copy_reg._inverted_registry, {code: (module_name, function_name)} */
123 static PyObject *inverted_registry;
124 /* copy_reg._extension_cache, {code: object} */
125 static PyObject *extension_cache;
126
127 /* For looking up name pairs in copy_reg._extension_registry. */
128 static PyObject *two_tuple;
129
130 static PyObject *__class___str, *__getinitargs___str, *__dict___str,
131 *__getstate___str, *__setstate___str, *__name___str, *__reduce___str,
132 *__reduce_ex___str,
133 *write_str, *append_str,
134 *read_str, *readline_str, *__main___str,
135 *dispatch_table_str;
136
137 /*************************************************************************
138 Internal Data type for pickle data. */
139
140 typedef struct {
141 PyObject_HEAD
142 Py_ssize_t length; /* number of initial slots in data currently used */
143 Py_ssize_t size; /* number of slots in data allocated */
144 PyObject **data;
145 } Pdata;
146
147 static void
Pdata_dealloc(Pdata * self)148 Pdata_dealloc(Pdata *self)
149 {
150 Py_ssize_t i;
151 PyObject **p;
152
153 for (i = self->length, p = self->data; --i >= 0; p++) {
154 Py_DECREF(*p);
155 }
156 if (self->data)
157 free(self->data);
158 PyObject_Del(self);
159 }
160
161 static PyTypeObject PdataType = {
162 PyVarObject_HEAD_INIT(NULL, 0) "cPickle.Pdata", sizeof(Pdata), 0,
163 (destructor)Pdata_dealloc,
164 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0L,0L,0L,0L, ""
165 };
166
167 #define Pdata_Check(O) (Py_TYPE(O) == &PdataType)
168
169 static PyObject *
Pdata_New(void)170 Pdata_New(void)
171 {
172 Pdata *self;
173
174 if (!(self = PyObject_New(Pdata, &PdataType)))
175 return NULL;
176 self->size = 8;
177 self->length = 0;
178 self->data = malloc(self->size * sizeof(PyObject*));
179 if (self->data)
180 return (PyObject*)self;
181 Py_DECREF(self);
182 return PyErr_NoMemory();
183 }
184
185 static int
stackUnderflow(void)186 stackUnderflow(void)
187 {
188 PyErr_SetString(UnpicklingError, "unpickling stack underflow");
189 return -1;
190 }
191
192 /* Retain only the initial clearto items. If clearto >= the current
193 * number of items, this is a (non-erroneous) NOP.
194 */
195 static int
Pdata_clear(Pdata * self,Py_ssize_t clearto)196 Pdata_clear(Pdata *self, Py_ssize_t clearto)
197 {
198 Py_ssize_t i;
199 PyObject **p;
200
201 if (clearto < 0) return stackUnderflow();
202 if (clearto >= self->length) return 0;
203
204 for (i = self->length, p = self->data + clearto;
205 --i >= clearto;
206 p++) {
207 Py_CLEAR(*p);
208 }
209 self->length = clearto;
210
211 return 0;
212 }
213
214 static int
Pdata_grow(Pdata * self)215 Pdata_grow(Pdata *self)
216 {
217 Py_ssize_t bigger;
218 Py_ssize_t nbytes;
219
220 PyObject **tmp;
221
222 if (self->size > (PY_SSIZE_T_MAX >> 1))
223 goto nomemory;
224 bigger = self->size << 1;
225 if (bigger > (PY_SSIZE_T_MAX / sizeof(PyObject *)))
226 goto nomemory;
227 nbytes = bigger * sizeof(PyObject *);
228 tmp = realloc(self->data, nbytes);
229 if (tmp == NULL)
230 goto nomemory;
231 self->data = tmp;
232 self->size = bigger;
233 return 0;
234
235 nomemory:
236 PyErr_NoMemory();
237 return -1;
238 }
239
240 /* D is a Pdata*. Pop the topmost element and store it into V, which
241 * must be an lvalue holding PyObject*. On stack underflow, UnpicklingError
242 * is raised and V is set to NULL. D and V may be evaluated several times.
243 */
244 #define PDATA_POP(D, V) { \
245 if ((D)->length) \
246 (V) = (D)->data[--((D)->length)]; \
247 else { \
248 PyErr_SetString(UnpicklingError, "bad pickle data"); \
249 (V) = NULL; \
250 } \
251 }
252
253 /* PDATA_PUSH and PDATA_APPEND both push rvalue PyObject* O on to Pdata*
254 * D. If the Pdata stack can't be grown to hold the new value, both
255 * raise MemoryError and execute "return ER". The difference is in ownership
256 * of O after: _PUSH transfers ownership of O from the caller to the stack
257 * (no incref of O is done, and in case of error O is decrefed), while
258 * _APPEND pushes a new reference.
259 */
260
261 /* Push O on stack D, giving ownership of O to the stack. */
262 #define PDATA_PUSH(D, O, ER) { \
263 if (((Pdata*)(D))->length == ((Pdata*)(D))->size && \
264 Pdata_grow((Pdata*)(D)) < 0) { \
265 Py_DECREF(O); \
266 return ER; \
267 } \
268 ((Pdata*)(D))->data[((Pdata*)(D))->length++] = (O); \
269 }
270
271 /* Push O on stack D, pushing a new reference. */
272 #define PDATA_APPEND(D, O, ER) { \
273 if (((Pdata*)(D))->length == ((Pdata*)(D))->size && \
274 Pdata_grow((Pdata*)(D)) < 0) \
275 return ER; \
276 Py_INCREF(O); \
277 ((Pdata*)(D))->data[((Pdata*)(D))->length++] = (O); \
278 }
279
280
281 static PyObject *
Pdata_popTuple(Pdata * self,Py_ssize_t start)282 Pdata_popTuple(Pdata *self, Py_ssize_t start)
283 {
284 PyObject *r;
285 Py_ssize_t i, j, l;
286
287 l = self->length-start;
288 r = PyTuple_New(l);
289 if (r == NULL)
290 return NULL;
291 for (i = start, j = 0 ; j < l; i++, j++)
292 PyTuple_SET_ITEM(r, j, self->data[i]);
293
294 self->length = start;
295 return r;
296 }
297
298 static PyObject *
Pdata_popList(Pdata * self,Py_ssize_t start)299 Pdata_popList(Pdata *self, Py_ssize_t start)
300 {
301 PyObject *r;
302 Py_ssize_t i, j, l;
303
304 l=self->length-start;
305 if (!( r=PyList_New(l))) return NULL;
306 for (i=start, j=0 ; j < l; i++, j++)
307 PyList_SET_ITEM(r, j, self->data[i]);
308
309 self->length=start;
310 return r;
311 }
312
313 /*************************************************************************/
314
315 #define ARG_TUP(self, o) { \
316 if (self->arg || (self->arg=PyTuple_New(1))) { \
317 Py_XDECREF(PyTuple_GET_ITEM(self->arg,0)); \
318 PyTuple_SET_ITEM(self->arg,0,o); \
319 } \
320 else { \
321 Py_DECREF(o); \
322 } \
323 }
324
325 #define FREE_ARG_TUP(self) { \
326 if (Py_REFCNT(self->arg) > 1) { \
327 Py_CLEAR(self->arg); \
328 } \
329 }
330
331 typedef struct Picklerobject {
332 PyObject_HEAD
333 FILE *fp;
334 PyObject *write;
335 PyObject *file;
336 PyObject *memo;
337 PyObject *arg;
338 PyObject *pers_func;
339 PyObject *inst_pers_func;
340
341 /* pickle protocol number, >= 0 */
342 int proto;
343
344 /* bool, true if proto > 0 */
345 int bin;
346
347 int fast; /* Fast mode doesn't save in memo, don't use if circ ref */
348 Py_ssize_t (*write_func)(struct Picklerobject *, const char *, Py_ssize_t);
349 char *write_buf;
350 Py_ssize_t buf_size;
351 PyObject *dispatch_table;
352 int fast_container; /* count nested container dumps */
353 PyObject *fast_memo;
354 } Picklerobject;
355
356 #ifndef PY_CPICKLE_FAST_LIMIT
357 #define PY_CPICKLE_FAST_LIMIT 50
358 #endif
359
360 static PyTypeObject Picklertype;
361
362 typedef struct Unpicklerobject {
363 PyObject_HEAD
364 FILE *fp;
365 PyObject *file;
366 PyObject *readline;
367 PyObject *read;
368 PyObject *memo;
369 PyObject *arg;
370 Pdata *stack;
371 PyObject *mark;
372 PyObject *pers_func;
373 PyObject *last_string;
374 Py_ssize_t *marks;
375 Py_ssize_t num_marks;
376 Py_ssize_t marks_size;
377 Py_ssize_t (*read_func)(struct Unpicklerobject *, char **, Py_ssize_t);
378 Py_ssize_t (*readline_func)(struct Unpicklerobject *, char **);
379 Py_ssize_t buf_size;
380 char *buf;
381 PyObject *find_class;
382 } Unpicklerobject;
383
384 static PyTypeObject Unpicklertype;
385
386 /* Forward decls that need the above structs */
387 static int save(Picklerobject *, PyObject *, int);
388 static int put2(Picklerobject *, PyObject *);
389
390 static
391 PyObject *
cPickle_ErrFormat(PyObject * ErrType,char * stringformat,char * format,...)392 cPickle_ErrFormat(PyObject *ErrType, char *stringformat, char *format, ...)
393 {
394 va_list va;
395 PyObject *args=0, *retval=0;
396 va_start(va, format);
397
398 if (format) args = Py_VaBuildValue(format, va);
399 va_end(va);
400 if (format && ! args) return NULL;
401 if (stringformat && !(retval=PyString_FromString(stringformat)))
402 return NULL;
403
404 if (retval) {
405 if (args) {
406 PyObject *v;
407 v=PyString_Format(retval, args);
408 Py_DECREF(retval);
409 Py_DECREF(args);
410 if (! v) return NULL;
411 retval=v;
412 }
413 }
414 else
415 if (args) retval=args;
416 else {
417 PyErr_SetObject(ErrType,Py_None);
418 return NULL;
419 }
420 PyErr_SetObject(ErrType,retval);
421 Py_DECREF(retval);
422 return NULL;
423 }
424
425 static Py_ssize_t
write_file(Picklerobject * self,const char * s,Py_ssize_t n)426 write_file(Picklerobject *self, const char *s, Py_ssize_t n)
427 {
428 size_t nbyteswritten;
429
430 if (s == NULL) {
431 return 0;
432 }
433
434 PyFile_IncUseCount((PyFileObject *)self->file);
435 Py_BEGIN_ALLOW_THREADS
436 nbyteswritten = fwrite(s, sizeof(char), n, self->fp);
437 Py_END_ALLOW_THREADS
438 PyFile_DecUseCount((PyFileObject *)self->file);
439 if (nbyteswritten != (size_t)n) {
440 PyErr_SetFromErrno(PyExc_IOError);
441 return -1;
442 }
443
444 return n;
445 }
446
447 static Py_ssize_t
write_cStringIO(Picklerobject * self,const char * s,Py_ssize_t n)448 write_cStringIO(Picklerobject *self, const char *s, Py_ssize_t n)
449 {
450 Py_ssize_t len = n;
451
452 if (s == NULL) {
453 return 0;
454 }
455
456 while (n > INT_MAX) {
457 if (PycStringIO->cwrite((PyObject *)self->file, s, INT_MAX) != INT_MAX) {
458 return -1;
459 }
460 n -= INT_MAX;
461 }
462
463 if (PycStringIO->cwrite((PyObject *)self->file, s, n) != n) {
464 return -1;
465 }
466
467 return len;
468 }
469
470 static Py_ssize_t
write_none(Picklerobject * self,const char * s,Py_ssize_t n)471 write_none(Picklerobject *self, const char *s, Py_ssize_t n)
472 {
473 if (s == NULL) return 0;
474 return n;
475 }
476
477 static Py_ssize_t
write_other(Picklerobject * self,const char * s,Py_ssize_t n)478 write_other(Picklerobject *self, const char *s, Py_ssize_t n)
479 {
480 PyObject *py_str = 0, *junk = 0;
481
482 if (s == NULL) {
483 if (!( self->buf_size )) return 0;
484 py_str = PyString_FromStringAndSize(self->write_buf,
485 self->buf_size);
486 if (!py_str)
487 return -1;
488 }
489 else {
490 if (self->buf_size && n > WRITE_BUF_SIZE - self->buf_size) {
491 if (write_other(self, NULL, 0) < 0)
492 return -1;
493 }
494
495 if (n > WRITE_BUF_SIZE) {
496 if (!( py_str =
497 PyString_FromStringAndSize(s, n)))
498 return -1;
499 }
500 else {
501 memcpy(self->write_buf + self->buf_size, s, n);
502 self->buf_size += n;
503 return n;
504 }
505 }
506
507 if (self->write) {
508 /* object with write method */
509 ARG_TUP(self, py_str);
510 if (self->arg) {
511 junk = PyObject_Call(self->write, self->arg, NULL);
512 FREE_ARG_TUP(self);
513 }
514 if (junk) Py_DECREF(junk);
515 else return -1;
516 }
517 else
518 PDATA_PUSH(self->file, py_str, -1);
519
520 self->buf_size = 0;
521 return n;
522 }
523
524
525 static Py_ssize_t
read_file(Unpicklerobject * self,char ** s,Py_ssize_t n)526 read_file(Unpicklerobject *self, char **s, Py_ssize_t n)
527 {
528 size_t nbytesread;
529
530 if (self->buf_size == 0) {
531 Py_ssize_t size;
532
533 size = ((n < 32) ? 32 : n);
534 if (!( self->buf = (char *)malloc(size))) {
535 PyErr_NoMemory();
536 return -1;
537 }
538
539 self->buf_size = size;
540 }
541 else if (n > self->buf_size) {
542 char *newbuf = (char *)realloc(self->buf, n);
543 if (!newbuf) {
544 PyErr_NoMemory();
545 return -1;
546 }
547 self->buf = newbuf;
548 self->buf_size = n;
549 }
550
551 PyFile_IncUseCount((PyFileObject *)self->file);
552 Py_BEGIN_ALLOW_THREADS
553 nbytesread = fread(self->buf, sizeof(char), n, self->fp);
554 Py_END_ALLOW_THREADS
555 PyFile_DecUseCount((PyFileObject *)self->file);
556 if (nbytesread != (size_t)n) {
557 if (feof(self->fp)) {
558 PyErr_SetNone(PyExc_EOFError);
559 return -1;
560 }
561
562 PyErr_SetFromErrno(PyExc_IOError);
563 return -1;
564 }
565
566 *s = self->buf;
567
568 return n;
569 }
570
571
572 static Py_ssize_t
readline_file(Unpicklerobject * self,char ** s)573 readline_file(Unpicklerobject *self, char **s)
574 {
575 Py_ssize_t i;
576
577 if (self->buf_size == 0) {
578 if (!( self->buf = (char *)malloc(40))) {
579 PyErr_NoMemory();
580 return -1;
581 }
582 self->buf_size = 40;
583 }
584
585 i = 0;
586 while (1) {
587 Py_ssize_t bigger;
588 char *newbuf;
589 for (; i < (self->buf_size - 1); i++) {
590 if (feof(self->fp) ||
591 (self->buf[i] = getc(self->fp)) == '\n') {
592 self->buf[i + 1] = '\0';
593 *s = self->buf;
594 return i + 1;
595 }
596 }
597 if (self->buf_size > (PY_SSIZE_T_MAX >> 1)) {
598 PyErr_NoMemory();
599 return -1;
600 }
601 bigger = self->buf_size << 1;
602 newbuf = (char *)realloc(self->buf, bigger);
603 if (newbuf == NULL) {
604 PyErr_NoMemory();
605 return -1;
606 }
607 self->buf = newbuf;
608 self->buf_size = bigger;
609 }
610 }
611
612
613 static Py_ssize_t
read_cStringIO(Unpicklerobject * self,char ** s,Py_ssize_t n)614 read_cStringIO(Unpicklerobject *self, char **s, Py_ssize_t n)
615 {
616 Py_ssize_t len = n;
617 char *start, *end = NULL;
618
619 while (1) {
620 int k;
621 char *ptr;
622 if (n > INT_MAX)
623 k = INT_MAX;
624 else
625 k = (int)n;
626 if (PycStringIO->cread((PyObject *)self->file, &ptr, k) != k) {
627 PyErr_SetNone(PyExc_EOFError);
628 return -1;
629 }
630 if (end == NULL)
631 start = ptr;
632 else if (ptr != end) {
633 /* non-continuous area */
634 return -1;
635 }
636 if (n <= INT_MAX)
637 break;
638 end = ptr + INT_MAX;
639 n -= INT_MAX;
640 }
641
642 *s = start;
643
644 return len;
645 }
646
647
648 static Py_ssize_t
readline_cStringIO(Unpicklerobject * self,char ** s)649 readline_cStringIO(Unpicklerobject *self, char **s)
650 {
651 Py_ssize_t n = 0;
652 char *start = NULL, *end = NULL;
653
654 while (1) {
655 int k;
656 char *ptr;
657 if ((k = PycStringIO->creadline((PyObject *)self->file, &ptr)) < 0) {
658 return -1;
659 }
660 n += k;
661 if (end == NULL)
662 start = ptr;
663 else if (ptr != end) {
664 /* non-continuous area */
665 return -1;
666 }
667 if (k == 0 || ptr[k - 1] == '\n')
668 break;
669 end = ptr + k;
670 }
671
672 *s = start;
673
674 return n;
675 }
676
677
678 static Py_ssize_t
read_other(Unpicklerobject * self,char ** s,Py_ssize_t n)679 read_other(Unpicklerobject *self, char **s, Py_ssize_t n)
680 {
681 PyObject *bytes, *str=0;
682
683 if (!( bytes = PyInt_FromSsize_t(n))) return -1;
684
685 ARG_TUP(self, bytes);
686 if (self->arg) {
687 str = PyObject_Call(self->read, self->arg, NULL);
688 FREE_ARG_TUP(self);
689 }
690 if (! str) return -1;
691
692 Py_XSETREF(self->last_string, str);
693
694 if (! (*s = PyString_AsString(str))) return -1;
695
696 if (PyString_GET_SIZE(str) != n) {
697 PyErr_SetNone(PyExc_EOFError);
698 return -1;
699 }
700
701 return n;
702 }
703
704
705 static Py_ssize_t
readline_other(Unpicklerobject * self,char ** s)706 readline_other(Unpicklerobject *self, char **s)
707 {
708 PyObject *str;
709 Py_ssize_t str_size;
710
711 if (!( str = PyObject_CallObject(self->readline, empty_tuple))) {
712 return -1;
713 }
714
715 if ((str_size = PyString_Size(str)) < 0)
716 return -1;
717
718 Py_XSETREF(self->last_string, str);
719
720 if (! (*s = PyString_AsString(str)))
721 return -1;
722
723 return str_size;
724 }
725
726 /* Copy the first n bytes from s into newly malloc'ed memory, plus a
727 * trailing 0 byte. Return a pointer to that, or NULL if out of memory.
728 * The caller is responsible for free()'ing the return value.
729 */
730 static char *
pystrndup(const char * s,Py_ssize_t n)731 pystrndup(const char *s, Py_ssize_t n)
732 {
733 char *r = (char *)malloc(n+1);
734 if (r == NULL)
735 return (char*)PyErr_NoMemory();
736 memcpy(r, s, n);
737 r[n] = 0;
738 return r;
739 }
740
741
742 static int
get(Picklerobject * self,PyObject * id)743 get(Picklerobject *self, PyObject *id)
744 {
745 PyObject *value, *mv;
746 Py_ssize_t c_value;
747 char s[30];
748 size_t len;
749
750 if (!( mv = PyDict_GetItem(self->memo, id))) {
751 PyErr_SetObject(PyExc_KeyError, id);
752 return -1;
753 }
754
755 if (!( value = PyTuple_GetItem(mv, 0)))
756 return -1;
757
758 if (!( PyInt_Check(value))) {
759 PyErr_SetString(PicklingError, "no int where int expected in memo");
760 return -1;
761 }
762 c_value = PyInt_AS_LONG((PyIntObject*)value);
763
764 if (!self->bin) {
765 s[0] = GET;
766 PyOS_snprintf(s + 1, sizeof(s) - 1,
767 "%" PY_FORMAT_SIZE_T "d\n", c_value);
768 len = strlen(s);
769 }
770 else if (Pdata_Check(self->file)) {
771 if (write_other(self, NULL, 0) < 0) return -1;
772 PDATA_APPEND(self->file, mv, -1);
773 return 0;
774 }
775 else {
776 if (c_value < 256) {
777 s[0] = BINGET;
778 s[1] = (int)(c_value & 0xff);
779 len = 2;
780 }
781 else {
782 s[0] = LONG_BINGET;
783 s[1] = (int)(c_value & 0xff);
784 s[2] = (int)((c_value >> 8) & 0xff);
785 s[3] = (int)((c_value >> 16) & 0xff);
786 s[4] = (int)((c_value >> 24) & 0xff);
787 len = 5;
788 }
789 }
790
791 if (self->write_func(self, s, len) < 0)
792 return -1;
793
794 return 0;
795 }
796
797
798 static int
put(Picklerobject * self,PyObject * ob)799 put(Picklerobject *self, PyObject *ob)
800 {
801 if (Py_REFCNT(ob) < 2 || self->fast)
802 return 0;
803
804 return put2(self, ob);
805 }
806
807
808 static int
put2(Picklerobject * self,PyObject * ob)809 put2(Picklerobject *self, PyObject *ob)
810 {
811 char c_str[30];
812 Py_ssize_t len, p;
813 int res = -1;
814 PyObject *py_ob_id = 0, *memo_len = 0, *t = 0;
815
816 if (self->fast)
817 return 0;
818
819 if ((p = PyDict_Size(self->memo)) < 0)
820 goto finally;
821
822 /* Make sure memo keys are positive! */
823 /* XXX Why?
824 * XXX And does "positive" really mean non-negative?
825 * XXX pickle.py starts with PUT index 0, not 1. This makes for
826 * XXX gratuitous differences between the pickling modules.
827 */
828 p++;
829
830 if (!( py_ob_id = PyLong_FromVoidPtr(ob)))
831 goto finally;
832
833 if (!( memo_len = PyInt_FromLong(p)))
834 goto finally;
835
836 if (!( t = PyTuple_New(2)))
837 goto finally;
838
839 PyTuple_SET_ITEM(t, 0, memo_len);
840 Py_INCREF(memo_len);
841 PyTuple_SET_ITEM(t, 1, ob);
842 Py_INCREF(ob);
843
844 if (PyDict_SetItem(self->memo, py_ob_id, t) < 0)
845 goto finally;
846
847 if (!self->bin) {
848 c_str[0] = PUT;
849 PyOS_snprintf(c_str + 1, sizeof(c_str) - 1,
850 "%" PY_FORMAT_SIZE_T "d\n", p);
851 len = strlen(c_str);
852 }
853 else if (Pdata_Check(self->file)) {
854 if (write_other(self, NULL, 0) < 0) return -1;
855 PDATA_APPEND(self->file, memo_len, -1);
856 res=0; /* Job well done ;) */
857 goto finally;
858 }
859 else {
860 if (p >= 256) {
861 c_str[0] = LONG_BINPUT;
862 c_str[1] = (int)(p & 0xff);
863 c_str[2] = (int)((p >> 8) & 0xff);
864 c_str[3] = (int)((p >> 16) & 0xff);
865 c_str[4] = (int)((p >> 24) & 0xff);
866 len = 5;
867 }
868 else {
869 c_str[0] = BINPUT;
870 c_str[1] = p;
871 len = 2;
872 }
873 }
874
875 if (self->write_func(self, c_str, len) < 0)
876 goto finally;
877
878 res = 0;
879
880 finally:
881 Py_XDECREF(py_ob_id);
882 Py_XDECREF(memo_len);
883 Py_XDECREF(t);
884
885 return res;
886 }
887
888 static PyObject *
whichmodule(PyObject * global,PyObject * global_name)889 whichmodule(PyObject *global, PyObject *global_name)
890 {
891 Py_ssize_t i, j;
892 PyObject *module = 0, *modules_dict = 0,
893 *global_name_attr = 0, *name = 0;
894
895 module = PyObject_GetAttrString(global, "__module__");
896 if (module)
897 return module;
898 if (PyErr_ExceptionMatches(PyExc_AttributeError))
899 PyErr_Clear();
900 else
901 return NULL;
902
903 if (!( modules_dict = PySys_GetObject("modules")))
904 return NULL;
905
906 i = 0;
907 while ((j = PyDict_Next(modules_dict, &i, &name, &module))) {
908
909 if (PyObject_Compare(name, __main___str)==0) continue;
910
911 global_name_attr = PyObject_GetAttr(module, global_name);
912 if (!global_name_attr) {
913 if (PyErr_ExceptionMatches(PyExc_AttributeError))
914 PyErr_Clear();
915 else
916 return NULL;
917 continue;
918 }
919
920 if (global_name_attr != global) {
921 Py_DECREF(global_name_attr);
922 continue;
923 }
924
925 Py_DECREF(global_name_attr);
926
927 break;
928 }
929
930 /* The following implements the rule in pickle.py added in 1.5
931 that used __main__ if no module is found. I don't actually
932 like this rule. jlf
933 */
934 if (!j) {
935 name=__main___str;
936 }
937
938 Py_INCREF(name);
939 return name;
940 }
941
942
943 static int
fast_save_enter(Picklerobject * self,PyObject * obj)944 fast_save_enter(Picklerobject *self, PyObject *obj)
945 {
946 /* if fast_container < 0, we're doing an error exit. */
947 if (++self->fast_container >= PY_CPICKLE_FAST_LIMIT) {
948 PyObject *key = NULL;
949 if (self->fast_memo == NULL) {
950 self->fast_memo = PyDict_New();
951 if (self->fast_memo == NULL) {
952 self->fast_container = -1;
953 return 0;
954 }
955 }
956 key = PyLong_FromVoidPtr(obj);
957 if (key == NULL)
958 return 0;
959 if (PyDict_GetItem(self->fast_memo, key)) {
960 Py_DECREF(key);
961 PyErr_Format(PyExc_ValueError,
962 "fast mode: can't pickle cyclic objects "
963 "including object type %s at %p",
964 Py_TYPE(obj)->tp_name, obj);
965 self->fast_container = -1;
966 return 0;
967 }
968 if (PyDict_SetItem(self->fast_memo, key, Py_None) < 0) {
969 Py_DECREF(key);
970 self->fast_container = -1;
971 return 0;
972 }
973 Py_DECREF(key);
974 }
975 return 1;
976 }
977
978 int
fast_save_leave(Picklerobject * self,PyObject * obj)979 fast_save_leave(Picklerobject *self, PyObject *obj)
980 {
981 if (self->fast_container-- >= PY_CPICKLE_FAST_LIMIT) {
982 PyObject *key = PyLong_FromVoidPtr(obj);
983 if (key == NULL)
984 return 0;
985 if (PyDict_DelItem(self->fast_memo, key) < 0) {
986 Py_DECREF(key);
987 return 0;
988 }
989 Py_DECREF(key);
990 }
991 return 1;
992 }
993
994 static int
save_none(Picklerobject * self,PyObject * args)995 save_none(Picklerobject *self, PyObject *args)
996 {
997 static char none = NONE;
998 if (self->write_func(self, &none, 1) < 0)
999 return -1;
1000
1001 return 0;
1002 }
1003
1004 static int
save_bool(Picklerobject * self,PyObject * args)1005 save_bool(Picklerobject *self, PyObject *args)
1006 {
1007 static const char *buf[2] = {FALSE, TRUE};
1008 static char len[2] = {sizeof(FALSE)-1, sizeof(TRUE)-1};
1009 long l = PyInt_AS_LONG((PyIntObject *)args);
1010
1011 if (self->proto >= 2) {
1012 char opcode = l ? NEWTRUE : NEWFALSE;
1013 if (self->write_func(self, &opcode, 1) < 0)
1014 return -1;
1015 }
1016 else if (self->write_func(self, buf[l], len[l]) < 0)
1017 return -1;
1018 return 0;
1019 }
1020
1021 static int
save_int(Picklerobject * self,PyObject * args)1022 save_int(Picklerobject *self, PyObject *args)
1023 {
1024 char c_str[32];
1025 long l = PyInt_AS_LONG((PyIntObject *)args);
1026 Py_ssize_t len = 0;
1027
1028 if (!self->bin
1029 #if SIZEOF_LONG > 4
1030 || l > 0x7fffffffL
1031 || l < -0x80000000L
1032 #endif
1033 ) {
1034 /* Text-mode pickle, or long too big to fit in the 4-byte
1035 * signed BININT format: store as a string.
1036 */
1037 c_str[0] = INT;
1038 PyOS_snprintf(c_str + 1, sizeof(c_str) - 1, "%ld\n", l);
1039 if (self->write_func(self, c_str, strlen(c_str)) < 0)
1040 return -1;
1041 }
1042 else {
1043 /* Binary pickle and l fits in a signed 4-byte int. */
1044 c_str[1] = (int)( l & 0xff);
1045 c_str[2] = (int)((l >> 8) & 0xff);
1046 c_str[3] = (int)((l >> 16) & 0xff);
1047 c_str[4] = (int)((l >> 24) & 0xff);
1048
1049 if ((c_str[4] == 0) && (c_str[3] == 0)) {
1050 if (c_str[2] == 0) {
1051 c_str[0] = BININT1;
1052 len = 2;
1053 }
1054 else {
1055 c_str[0] = BININT2;
1056 len = 3;
1057 }
1058 }
1059 else {
1060 c_str[0] = BININT;
1061 len = 5;
1062 }
1063
1064 if (self->write_func(self, c_str, len) < 0)
1065 return -1;
1066 }
1067
1068 return 0;
1069 }
1070
1071
1072 static int
save_long(Picklerobject * self,PyObject * args)1073 save_long(Picklerobject *self, PyObject *args)
1074 {
1075 Py_ssize_t size;
1076 int res = -1;
1077 PyObject *repr = NULL;
1078
1079 static char l = LONG;
1080
1081 if (self->proto >= 2) {
1082 /* Linear-time pickling. */
1083 size_t nbits;
1084 size_t nbytes;
1085 unsigned char *pdata;
1086 char c_str[5];
1087 int i;
1088 int sign = _PyLong_Sign(args);
1089
1090 if (sign == 0) {
1091 /* It's 0 -- an empty bytestring. */
1092 c_str[0] = LONG1;
1093 c_str[1] = 0;
1094 i = self->write_func(self, c_str, 2);
1095 if (i < 0) goto finally;
1096 res = 0;
1097 goto finally;
1098 }
1099 nbits = _PyLong_NumBits(args);
1100 if (nbits == (size_t)-1 && PyErr_Occurred())
1101 goto finally;
1102 /* How many bytes do we need? There are nbits >> 3 full
1103 * bytes of data, and nbits & 7 leftover bits. If there
1104 * are any leftover bits, then we clearly need another
1105 * byte. Wnat's not so obvious is that we *probably*
1106 * need another byte even if there aren't any leftovers:
1107 * the most-significant bit of the most-significant byte
1108 * acts like a sign bit, and it's usually got a sense
1109 * opposite of the one we need. The exception is longs
1110 * of the form -(2**(8*j-1)) for j > 0. Such a long is
1111 * its own 256's-complement, so has the right sign bit
1112 * even without the extra byte. That's a pain to check
1113 * for in advance, though, so we always grab an extra
1114 * byte at the start, and cut it back later if possible.
1115 */
1116 nbytes = (nbits >> 3) + 1;
1117 if (nbytes > INT_MAX) {
1118 PyErr_SetString(PyExc_OverflowError, "long too large "
1119 "to pickle");
1120 goto finally;
1121 }
1122 repr = PyString_FromStringAndSize(NULL, (int)nbytes);
1123 if (repr == NULL) goto finally;
1124 pdata = (unsigned char *)PyString_AS_STRING(repr);
1125 i = _PyLong_AsByteArray((PyLongObject *)args,
1126 pdata, nbytes,
1127 1 /* little endian */, 1 /* signed */);
1128 if (i < 0) goto finally;
1129 /* If the long is negative, this may be a byte more than
1130 * needed. This is so iff the MSB is all redundant sign
1131 * bits.
1132 */
1133 if (sign < 0 && nbytes > 1 && pdata[nbytes - 1] == 0xff &&
1134 (pdata[nbytes - 2] & 0x80) != 0)
1135 --nbytes;
1136
1137 if (nbytes < 256) {
1138 c_str[0] = LONG1;
1139 c_str[1] = (char)nbytes;
1140 size = 2;
1141 }
1142 else {
1143 c_str[0] = LONG4;
1144 size = (int)nbytes;
1145 for (i = 1; i < 5; i++) {
1146 c_str[i] = (char)(size & 0xff);
1147 size >>= 8;
1148 }
1149 size = 5;
1150 }
1151 i = self->write_func(self, c_str, size);
1152 if (i < 0) goto finally;
1153 i = self->write_func(self, (char *)pdata, (int)nbytes);
1154 if (i < 0) goto finally;
1155 res = 0;
1156 goto finally;
1157 }
1158
1159 /* proto < 2: write the repr and newline. This is quadratic-time
1160 * (in the number of digits), in both directions.
1161 */
1162 if (!( repr = PyObject_Repr(args)))
1163 goto finally;
1164
1165 if ((size = PyString_Size(repr)) < 0)
1166 goto finally;
1167
1168 if (self->write_func(self, &l, 1) < 0)
1169 goto finally;
1170
1171 if (self->write_func(self,
1172 PyString_AS_STRING((PyStringObject *)repr),
1173 size) < 0)
1174 goto finally;
1175
1176 if (self->write_func(self, "\n", 1) < 0)
1177 goto finally;
1178
1179 res = 0;
1180
1181 finally:
1182 Py_XDECREF(repr);
1183 return res;
1184 }
1185
1186
1187 static int
save_float(Picklerobject * self,PyObject * args)1188 save_float(Picklerobject *self, PyObject *args)
1189 {
1190 double x = PyFloat_AS_DOUBLE((PyFloatObject *)args);
1191
1192 if (self->bin) {
1193 char str[9];
1194 str[0] = BINFLOAT;
1195 if (_PyFloat_Pack8(x, (unsigned char *)&str[1], 0) < 0)
1196 return -1;
1197 if (self->write_func(self, str, 9) < 0)
1198 return -1;
1199 }
1200 else {
1201 int result = -1;
1202 char *buf = NULL;
1203 char op = FLOAT;
1204
1205 if (self->write_func(self, &op, 1) < 0)
1206 goto done;
1207
1208 buf = PyOS_double_to_string(x, 'g', 17, 0, NULL);
1209 if (!buf) {
1210 PyErr_NoMemory();
1211 goto done;
1212 }
1213
1214 if (self->write_func(self, buf, strlen(buf)) < 0)
1215 goto done;
1216
1217 if (self->write_func(self, "\n", 1) < 0)
1218 goto done;
1219
1220 result = 0;
1221 done:
1222 PyMem_Free(buf);
1223 return result;
1224 }
1225
1226 return 0;
1227 }
1228
1229
1230 static int
save_string(Picklerobject * self,PyObject * args,int doput)1231 save_string(Picklerobject *self, PyObject *args, int doput)
1232 {
1233 Py_ssize_t size, len;
1234 PyObject *repr=0;
1235
1236 if ((size = PyString_Size(args)) < 0)
1237 return -1;
1238
1239 if (!self->bin) {
1240 char *repr_str;
1241
1242 static char string = STRING;
1243
1244 if (!( repr = PyObject_Repr(args)))
1245 return -1;
1246
1247 if ((len = PyString_Size(repr)) < 0)
1248 goto err;
1249 repr_str = PyString_AS_STRING((PyStringObject *)repr);
1250
1251 if (self->write_func(self, &string, 1) < 0)
1252 goto err;
1253
1254 if (self->write_func(self, repr_str, len) < 0)
1255 goto err;
1256
1257 if (self->write_func(self, "\n", 1) < 0)
1258 goto err;
1259
1260 Py_XDECREF(repr);
1261 }
1262 else {
1263 int i;
1264 char c_str[5];
1265
1266 if (size < 256) {
1267 c_str[0] = SHORT_BINSTRING;
1268 c_str[1] = size;
1269 len = 2;
1270 }
1271 else if (size <= INT_MAX) {
1272 c_str[0] = BINSTRING;
1273 for (i = 1; i < 5; i++)
1274 c_str[i] = (int)(size >> ((i - 1) * 8));
1275 len = 5;
1276 }
1277 else
1278 return -1; /* string too large */
1279
1280 if (self->write_func(self, c_str, len) < 0)
1281 return -1;
1282
1283 if (size > 128 && Pdata_Check(self->file)) {
1284 if (write_other(self, NULL, 0) < 0) return -1;
1285 PDATA_APPEND(self->file, args, -1);
1286 }
1287 else {
1288 if (self->write_func(self,
1289 PyString_AS_STRING(
1290 (PyStringObject *)args),
1291 size) < 0)
1292 return -1;
1293 }
1294 }
1295
1296 if (doput)
1297 if (put(self, args) < 0)
1298 return -1;
1299
1300 return 0;
1301
1302 err:
1303 Py_XDECREF(repr);
1304 return -1;
1305 }
1306
1307
1308 #ifdef Py_USING_UNICODE
1309 /* A copy of PyUnicode_EncodeRawUnicodeEscape() that also translates
1310 backslash and newline characters to \uXXXX escapes. */
1311 static PyObject *
modified_EncodeRawUnicodeEscape(const Py_UNICODE * s,Py_ssize_t size)1312 modified_EncodeRawUnicodeEscape(const Py_UNICODE *s, Py_ssize_t size)
1313 {
1314 PyObject *repr;
1315 char *p;
1316 char *q;
1317
1318 static const char *hexdigit = "0123456789abcdef";
1319 #ifdef Py_UNICODE_WIDE
1320 const Py_ssize_t expandsize = 10;
1321 #else
1322 const Py_ssize_t expandsize = 6;
1323 #endif
1324
1325 if (size > PY_SSIZE_T_MAX / expandsize)
1326 return PyErr_NoMemory();
1327
1328 repr = PyString_FromStringAndSize(NULL, expandsize * size);
1329 if (repr == NULL)
1330 return NULL;
1331 if (size == 0)
1332 return repr;
1333
1334 p = q = PyString_AS_STRING(repr);
1335 while (size-- > 0) {
1336 Py_UNICODE ch = *s++;
1337 #ifdef Py_UNICODE_WIDE
1338 /* Map 32-bit characters to '\Uxxxxxxxx' */
1339 if (ch >= 0x10000) {
1340 *p++ = '\\';
1341 *p++ = 'U';
1342 *p++ = hexdigit[(ch >> 28) & 0xf];
1343 *p++ = hexdigit[(ch >> 24) & 0xf];
1344 *p++ = hexdigit[(ch >> 20) & 0xf];
1345 *p++ = hexdigit[(ch >> 16) & 0xf];
1346 *p++ = hexdigit[(ch >> 12) & 0xf];
1347 *p++ = hexdigit[(ch >> 8) & 0xf];
1348 *p++ = hexdigit[(ch >> 4) & 0xf];
1349 *p++ = hexdigit[ch & 15];
1350 }
1351 else
1352 #else
1353 /* Map UTF-16 surrogate pairs to '\U00xxxxxx' */
1354 if (ch >= 0xD800 && ch < 0xDC00) {
1355 Py_UNICODE ch2;
1356 Py_UCS4 ucs;
1357
1358 ch2 = *s++;
1359 size--;
1360 if (ch2 >= 0xDC00 && ch2 <= 0xDFFF) {
1361 ucs = (((ch & 0x03FF) << 10) | (ch2 & 0x03FF)) + 0x00010000;
1362 *p++ = '\\';
1363 *p++ = 'U';
1364 *p++ = hexdigit[(ucs >> 28) & 0xf];
1365 *p++ = hexdigit[(ucs >> 24) & 0xf];
1366 *p++ = hexdigit[(ucs >> 20) & 0xf];
1367 *p++ = hexdigit[(ucs >> 16) & 0xf];
1368 *p++ = hexdigit[(ucs >> 12) & 0xf];
1369 *p++ = hexdigit[(ucs >> 8) & 0xf];
1370 *p++ = hexdigit[(ucs >> 4) & 0xf];
1371 *p++ = hexdigit[ucs & 0xf];
1372 continue;
1373 }
1374 /* Fall through: isolated surrogates are copied as-is */
1375 s--;
1376 size++;
1377 }
1378 #endif
1379 /* Map 16-bit characters to '\uxxxx' */
1380 if (ch >= 256 || ch == '\\' || ch == '\n') {
1381 *p++ = '\\';
1382 *p++ = 'u';
1383 *p++ = hexdigit[(ch >> 12) & 0xf];
1384 *p++ = hexdigit[(ch >> 8) & 0xf];
1385 *p++ = hexdigit[(ch >> 4) & 0xf];
1386 *p++ = hexdigit[ch & 15];
1387 }
1388 /* Copy everything else as-is */
1389 else
1390 *p++ = (char) ch;
1391 }
1392 *p = '\0';
1393 _PyString_Resize(&repr, p - q);
1394 return repr;
1395 }
1396
1397 static int
save_unicode(Picklerobject * self,PyObject * args,int doput)1398 save_unicode(Picklerobject *self, PyObject *args, int doput)
1399 {
1400 Py_ssize_t size, len;
1401 PyObject *repr=0;
1402
1403 if (!PyUnicode_Check(args))
1404 return -1;
1405
1406 if (!self->bin) {
1407 char *repr_str;
1408 static char string = UNICODE;
1409
1410 repr = modified_EncodeRawUnicodeEscape(
1411 PyUnicode_AS_UNICODE(args), PyUnicode_GET_SIZE(args));
1412 if (!repr)
1413 return -1;
1414
1415 if ((len = PyString_Size(repr)) < 0)
1416 goto err;
1417 repr_str = PyString_AS_STRING((PyStringObject *)repr);
1418
1419 if (self->write_func(self, &string, 1) < 0)
1420 goto err;
1421
1422 if (self->write_func(self, repr_str, len) < 0)
1423 goto err;
1424
1425 if (self->write_func(self, "\n", 1) < 0)
1426 goto err;
1427
1428 Py_XDECREF(repr);
1429 }
1430 else {
1431 int i;
1432 char c_str[5];
1433
1434 if (!( repr = PyUnicode_AsUTF8String(args)))
1435 return -1;
1436
1437 if ((size = PyString_Size(repr)) < 0)
1438 goto err;
1439 if (size > INT_MAX)
1440 return -1; /* string too large */
1441
1442 c_str[0] = BINUNICODE;
1443 for (i = 1; i < 5; i++)
1444 c_str[i] = (int)(size >> ((i - 1) * 8));
1445 len = 5;
1446
1447 if (self->write_func(self, c_str, len) < 0)
1448 goto err;
1449
1450 if (size > 128 && Pdata_Check(self->file)) {
1451 if (write_other(self, NULL, 0) < 0)
1452 goto err;
1453 PDATA_APPEND(self->file, repr, -1);
1454 }
1455 else {
1456 if (self->write_func(self, PyString_AS_STRING(repr),
1457 size) < 0)
1458 goto err;
1459 }
1460
1461 Py_DECREF(repr);
1462 }
1463
1464 if (doput)
1465 if (put(self, args) < 0)
1466 return -1;
1467
1468 return 0;
1469
1470 err:
1471 Py_XDECREF(repr);
1472 return -1;
1473 }
1474 #endif
1475
1476 /* A helper for save_tuple. Push the len elements in tuple t on the stack. */
1477 static int
store_tuple_elements(Picklerobject * self,PyObject * t,int len)1478 store_tuple_elements(Picklerobject *self, PyObject *t, int len)
1479 {
1480 Py_ssize_t i;
1481 int res = -1; /* guilty until proved innocent */
1482
1483 assert(PyTuple_Size(t) == len);
1484
1485 for (i = 0; i < len; i++) {
1486 PyObject *element = PyTuple_GET_ITEM(t, i);
1487
1488 if (element == NULL)
1489 goto finally;
1490 if (save(self, element, 0) < 0)
1491 goto finally;
1492 }
1493 res = 0;
1494
1495 finally:
1496 return res;
1497 }
1498
1499 /* Tuples are ubiquitous in the pickle protocols, so many techniques are
1500 * used across protocols to minimize the space needed to pickle them.
1501 * Tuples are also the only builtin immutable type that can be recursive
1502 * (a tuple can be reached from itself), and that requires some subtle
1503 * magic so that it works in all cases. IOW, this is a long routine.
1504 */
1505 static int
save_tuple(Picklerobject * self,PyObject * args)1506 save_tuple(Picklerobject *self, PyObject *args)
1507 {
1508 PyObject *py_tuple_id = NULL;
1509 Py_ssize_t len, i;
1510 int res = -1;
1511
1512 static char tuple = TUPLE;
1513 static char pop = POP;
1514 static char pop_mark = POP_MARK;
1515 static char len2opcode[] = {EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3};
1516
1517 if ((len = PyTuple_Size(args)) < 0)
1518 goto finally;
1519
1520 if (len == 0) {
1521 char c_str[2];
1522
1523 if (self->proto) {
1524 c_str[0] = EMPTY_TUPLE;
1525 len = 1;
1526 }
1527 else {
1528 c_str[0] = MARK;
1529 c_str[1] = TUPLE;
1530 len = 2;
1531 }
1532 if (self->write_func(self, c_str, len) >= 0)
1533 res = 0;
1534 /* Don't memoize an empty tuple. */
1535 goto finally;
1536 }
1537
1538 /* A non-empty tuple. */
1539
1540 /* id(tuple) isn't in the memo now. If it shows up there after
1541 * saving the tuple elements, the tuple must be recursive, in
1542 * which case we'll pop everything we put on the stack, and fetch
1543 * its value from the memo.
1544 */
1545 py_tuple_id = PyLong_FromVoidPtr(args);
1546 if (py_tuple_id == NULL)
1547 goto finally;
1548
1549 if (len <= 3 && self->proto >= 2) {
1550 /* Use TUPLE{1,2,3} opcodes. */
1551 if (store_tuple_elements(self, args, len) < 0)
1552 goto finally;
1553 if (PyDict_GetItem(self->memo, py_tuple_id)) {
1554 /* pop the len elements */
1555 for (i = 0; i < len; ++i)
1556 if (self->write_func(self, &pop, 1) < 0)
1557 goto finally;
1558 /* fetch from memo */
1559 if (get(self, py_tuple_id) < 0)
1560 goto finally;
1561 res = 0;
1562 goto finally;
1563 }
1564 /* Not recursive. */
1565 if (self->write_func(self, len2opcode + len, 1) < 0)
1566 goto finally;
1567 goto memoize;
1568 }
1569
1570 /* proto < 2 and len > 0, or proto >= 2 and len > 3.
1571 * Generate MARK elt1 elt2 ... TUPLE
1572 */
1573 if (self->write_func(self, &MARKv, 1) < 0)
1574 goto finally;
1575
1576 if (store_tuple_elements(self, args, len) < 0)
1577 goto finally;
1578
1579 if (PyDict_GetItem(self->memo, py_tuple_id)) {
1580 /* pop the stack stuff we pushed */
1581 if (self->bin) {
1582 if (self->write_func(self, &pop_mark, 1) < 0)
1583 goto finally;
1584 }
1585 else {
1586 /* Note that we pop one more than len, to remove
1587 * the MARK too.
1588 */
1589 for (i = 0; i <= len; i++)
1590 if (self->write_func(self, &pop, 1) < 0)
1591 goto finally;
1592 }
1593 /* fetch from memo */
1594 if (get(self, py_tuple_id) >= 0)
1595 res = 0;
1596 goto finally;
1597 }
1598
1599 /* Not recursive. */
1600 if (self->write_func(self, &tuple, 1) < 0)
1601 goto finally;
1602
1603 memoize:
1604 if (put(self, args) >= 0)
1605 res = 0;
1606
1607 finally:
1608 Py_XDECREF(py_tuple_id);
1609 return res;
1610 }
1611
1612 /* iter is an iterator giving items, and we batch up chunks of
1613 * MARK item item ... item APPENDS
1614 * opcode sequences. Calling code should have arranged to first create an
1615 * empty list, or list-like object, for the APPENDS to operate on.
1616 * Returns 0 on success, <0 on error.
1617 */
1618 static int
batch_list(Picklerobject * self,PyObject * iter)1619 batch_list(Picklerobject *self, PyObject *iter)
1620 {
1621 PyObject *obj = NULL;
1622 PyObject *firstitem = NULL;
1623 int i, n;
1624
1625 static char append = APPEND;
1626 static char appends = APPENDS;
1627
1628 assert(iter != NULL);
1629
1630 if (self->proto == 0) {
1631 /* APPENDS isn't available; do one at a time. */
1632 for (;;) {
1633 obj = PyIter_Next(iter);
1634 if (obj == NULL) {
1635 if (PyErr_Occurred())
1636 return -1;
1637 break;
1638 }
1639 i = save(self, obj, 0);
1640 Py_DECREF(obj);
1641 if (i < 0)
1642 return -1;
1643 if (self->write_func(self, &append, 1) < 0)
1644 return -1;
1645 }
1646 return 0;
1647 }
1648
1649 /* proto > 0: write in batches of BATCHSIZE. */
1650 do {
1651 /* Get first item */
1652 firstitem = PyIter_Next(iter);
1653 if (firstitem == NULL) {
1654 if (PyErr_Occurred())
1655 goto BatchFailed;
1656
1657 /* nothing more to add */
1658 break;
1659 }
1660
1661 /* Try to get a second item */
1662 obj = PyIter_Next(iter);
1663 if (obj == NULL) {
1664 if (PyErr_Occurred())
1665 goto BatchFailed;
1666
1667 /* Only one item to write */
1668 if (save(self, firstitem, 0) < 0)
1669 goto BatchFailed;
1670 if (self->write_func(self, &append, 1) < 0)
1671 goto BatchFailed;
1672 Py_CLEAR(firstitem);
1673 break;
1674 }
1675
1676 /* More than one item to write */
1677
1678 /* Pump out MARK, items, APPENDS. */
1679 if (self->write_func(self, &MARKv, 1) < 0)
1680 goto BatchFailed;
1681
1682 if (save(self, firstitem, 0) < 0)
1683 goto BatchFailed;
1684 Py_CLEAR(firstitem);
1685 n = 1;
1686
1687 /* Fetch and save up to BATCHSIZE items */
1688 while (obj) {
1689 if (save(self, obj, 0) < 0)
1690 goto BatchFailed;
1691 Py_CLEAR(obj);
1692 n += 1;
1693
1694 if (n == BATCHSIZE)
1695 break;
1696
1697 obj = PyIter_Next(iter);
1698 if (obj == NULL) {
1699 if (PyErr_Occurred())
1700 goto BatchFailed;
1701 break;
1702 }
1703 }
1704
1705 if (self->write_func(self, &appends, 1) < 0)
1706 goto BatchFailed;
1707
1708 } while (n == BATCHSIZE);
1709 return 0;
1710
1711 BatchFailed:
1712 Py_XDECREF(firstitem);
1713 Py_XDECREF(obj);
1714 return -1;
1715 }
1716
1717 static int
save_list(Picklerobject * self,PyObject * args)1718 save_list(Picklerobject *self, PyObject *args)
1719 {
1720 int res = -1;
1721 char s[3];
1722 Py_ssize_t len;
1723 PyObject *iter;
1724
1725 if (self->fast && !fast_save_enter(self, args))
1726 goto finally;
1727
1728 /* Create an empty list. */
1729 if (self->bin) {
1730 s[0] = EMPTY_LIST;
1731 len = 1;
1732 }
1733 else {
1734 s[0] = MARK;
1735 s[1] = LIST;
1736 len = 2;
1737 }
1738
1739 if (self->write_func(self, s, len) < 0)
1740 goto finally;
1741
1742 /* Get list length, and bow out early if empty. */
1743 if ((len = PyList_Size(args)) < 0)
1744 goto finally;
1745
1746 /* Memoize. */
1747 if (len == 0) {
1748 if (put(self, args) >= 0)
1749 res = 0;
1750 goto finally;
1751 }
1752 if (put2(self, args) < 0)
1753 goto finally;
1754
1755 /* Materialize the list elements. */
1756 iter = PyObject_GetIter(args);
1757 if (iter == NULL)
1758 goto finally;
1759
1760 if (Py_EnterRecursiveCall(" while pickling an object") == 0)
1761 {
1762 res = batch_list(self, iter);
1763 Py_LeaveRecursiveCall();
1764 }
1765 Py_DECREF(iter);
1766
1767 finally:
1768 if (self->fast && !fast_save_leave(self, args))
1769 res = -1;
1770
1771 return res;
1772 }
1773
1774
1775 /* iter is an iterator giving (key, value) pairs, and we batch up chunks of
1776 * MARK key value ... key value SETITEMS
1777 * opcode sequences. Calling code should have arranged to first create an
1778 * empty dict, or dict-like object, for the SETITEMS to operate on.
1779 * Returns 0 on success, <0 on error.
1780 *
1781 * This is very much like batch_list(). The difference between saving
1782 * elements directly, and picking apart two-tuples, is so long-winded at
1783 * the C level, though, that attempts to combine these routines were too
1784 * ugly to bear.
1785 */
1786 static int
batch_dict(Picklerobject * self,PyObject * iter)1787 batch_dict(Picklerobject *self, PyObject *iter)
1788 {
1789 PyObject *p = NULL;
1790 PyObject *firstitem = NULL;
1791 int i, n;
1792
1793 static char setitem = SETITEM;
1794 static char setitems = SETITEMS;
1795
1796 assert(iter != NULL);
1797
1798 if (self->proto == 0) {
1799 /* SETITEMS isn't available; do one at a time. */
1800 for (;;) {
1801 p = PyIter_Next(iter);
1802 if (p == NULL) {
1803 if (PyErr_Occurred())
1804 return -1;
1805 break;
1806 }
1807 if (!PyTuple_Check(p) || PyTuple_Size(p) != 2) {
1808 PyErr_SetString(PyExc_TypeError, "dict items "
1809 "iterator must return 2-tuples");
1810 return -1;
1811 }
1812 i = save(self, PyTuple_GET_ITEM(p, 0), 0);
1813 if (i >= 0)
1814 i = save(self, PyTuple_GET_ITEM(p, 1), 0);
1815 Py_DECREF(p);
1816 if (i < 0)
1817 return -1;
1818 if (self->write_func(self, &setitem, 1) < 0)
1819 return -1;
1820 }
1821 return 0;
1822 }
1823
1824 /* proto > 0: write in batches of BATCHSIZE. */
1825 do {
1826 /* Get first item */
1827 firstitem = PyIter_Next(iter);
1828 if (firstitem == NULL) {
1829 if (PyErr_Occurred())
1830 goto BatchFailed;
1831
1832 /* nothing more to add */
1833 break;
1834 }
1835 if (!PyTuple_Check(firstitem) || PyTuple_Size(firstitem) != 2) {
1836 PyErr_SetString(PyExc_TypeError, "dict items "
1837 "iterator must return 2-tuples");
1838 goto BatchFailed;
1839 }
1840
1841 /* Try to get a second item */
1842 p = PyIter_Next(iter);
1843 if (p == NULL) {
1844 if (PyErr_Occurred())
1845 goto BatchFailed;
1846
1847 /* Only one item to write */
1848 if (save(self, PyTuple_GET_ITEM(firstitem, 0), 0) < 0)
1849 goto BatchFailed;
1850 if (save(self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0)
1851 goto BatchFailed;
1852 if (self->write_func(self, &setitem, 1) < 0)
1853 goto BatchFailed;
1854 Py_CLEAR(firstitem);
1855 break;
1856 }
1857
1858 /* More than one item to write */
1859
1860 /* Pump out MARK, items, SETITEMS. */
1861 if (self->write_func(self, &MARKv, 1) < 0)
1862 goto BatchFailed;
1863
1864 if (save(self, PyTuple_GET_ITEM(firstitem, 0), 0) < 0)
1865 goto BatchFailed;
1866 if (save(self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0)
1867 goto BatchFailed;
1868 Py_CLEAR(firstitem);
1869 n = 1;
1870
1871 /* Fetch and save up to BATCHSIZE items */
1872 while (p) {
1873 if (!PyTuple_Check(p) || PyTuple_Size(p) != 2) {
1874 PyErr_SetString(PyExc_TypeError, "dict items "
1875 "iterator must return 2-tuples");
1876 goto BatchFailed;
1877 }
1878 if (save(self, PyTuple_GET_ITEM(p, 0), 0) < 0)
1879 goto BatchFailed;
1880 if (save(self, PyTuple_GET_ITEM(p, 1), 0) < 0)
1881 goto BatchFailed;
1882 Py_CLEAR(p);
1883 n += 1;
1884
1885 if (n == BATCHSIZE)
1886 break;
1887
1888 p = PyIter_Next(iter);
1889 if (p == NULL) {
1890 if (PyErr_Occurred())
1891 goto BatchFailed;
1892 break;
1893 }
1894 }
1895
1896 if (self->write_func(self, &setitems, 1) < 0)
1897 goto BatchFailed;
1898
1899 } while (n == BATCHSIZE);
1900 return 0;
1901
1902 BatchFailed:
1903 Py_XDECREF(firstitem);
1904 Py_XDECREF(p);
1905 return -1;
1906 }
1907
1908 /* This is a variant of batch_dict() above that specializes for dicts, with no
1909 * support for dict subclasses. Like batch_dict(), we batch up chunks of
1910 * MARK key value ... key value SETITEMS
1911 * opcode sequences. Calling code should have arranged to first create an
1912 * empty dict, or dict-like object, for the SETITEMS to operate on.
1913 * Returns 0 on success, -1 on error.
1914 *
1915 * Note that this currently doesn't work for protocol 0.
1916 */
1917 static int
batch_dict_exact(Picklerobject * self,PyObject * obj)1918 batch_dict_exact(Picklerobject *self, PyObject *obj)
1919 {
1920 PyObject *key = NULL, *value = NULL;
1921 int i;
1922 Py_ssize_t dict_size, ppos = 0;
1923
1924 static char setitem = SETITEM;
1925 static char setitems = SETITEMS;
1926
1927 assert(obj != NULL);
1928 assert(self->proto > 0);
1929
1930 dict_size = PyDict_Size(obj);
1931
1932 /* Special-case len(d) == 1 to save space. */
1933 if (dict_size == 1) {
1934 PyDict_Next(obj, &ppos, &key, &value);
1935 if (save(self, key, 0) < 0)
1936 return -1;
1937 if (save(self, value, 0) < 0)
1938 return -1;
1939 if (self->write_func(self, &setitem, 1) < 0)
1940 return -1;
1941 return 0;
1942 }
1943
1944 /* Write in batches of BATCHSIZE. */
1945 do {
1946 i = 0;
1947 if (self->write_func(self, &MARKv, 1) < 0)
1948 return -1;
1949 while (PyDict_Next(obj, &ppos, &key, &value)) {
1950 if (save(self, key, 0) < 0)
1951 return -1;
1952 if (save(self, value, 0) < 0)
1953 return -1;
1954 if (++i == BATCHSIZE)
1955 break;
1956 }
1957 if (self->write_func(self, &setitems, 1) < 0)
1958 return -1;
1959 if (PyDict_Size(obj) != dict_size) {
1960 PyErr_Format(
1961 PyExc_RuntimeError,
1962 "dictionary changed size during iteration");
1963 return -1;
1964 }
1965
1966 } while (i == BATCHSIZE);
1967 return 0;
1968 }
1969
1970 static int
save_dict(Picklerobject * self,PyObject * args)1971 save_dict(Picklerobject *self, PyObject *args)
1972 {
1973 int res = -1;
1974 char s[3];
1975 Py_ssize_t len;
1976
1977 if (self->fast && !fast_save_enter(self, args))
1978 goto finally;
1979
1980 /* Create an empty dict. */
1981 if (self->bin) {
1982 s[0] = EMPTY_DICT;
1983 len = 1;
1984 }
1985 else {
1986 s[0] = MARK;
1987 s[1] = DICT;
1988 len = 2;
1989 }
1990
1991 if (self->write_func(self, s, len) < 0)
1992 goto finally;
1993
1994 /* Get dict size, and bow out early if empty. */
1995 if ((len = PyDict_Size(args)) < 0)
1996 goto finally;
1997
1998 if (len == 0) {
1999 if (put(self, args) >= 0)
2000 res = 0;
2001 goto finally;
2002 }
2003 if (put2(self, args) < 0)
2004 goto finally;
2005
2006 /* Materialize the dict items. */
2007 if (PyDict_CheckExact(args) && self->proto > 0) {
2008 /* We can take certain shortcuts if we know this is a dict and
2009 not a dict subclass. */
2010 if (Py_EnterRecursiveCall(" while pickling an object") == 0) {
2011 res = batch_dict_exact(self, args);
2012 Py_LeaveRecursiveCall();
2013 }
2014 } else {
2015 PyObject *iter = PyObject_CallMethod(args, "iteritems", "()");
2016 if (iter == NULL)
2017 goto finally;
2018 if (Py_EnterRecursiveCall(" while pickling an object") == 0) {
2019 res = batch_dict(self, iter);
2020 Py_LeaveRecursiveCall();
2021 }
2022 Py_DECREF(iter);
2023 }
2024
2025 finally:
2026 if (self->fast && !fast_save_leave(self, args))
2027 res = -1;
2028
2029 return res;
2030 }
2031
2032
2033 static int
save_inst(Picklerobject * self,PyObject * args)2034 save_inst(Picklerobject *self, PyObject *args)
2035 {
2036 PyObject *class = 0, *module = 0, *name = 0, *state = 0,
2037 *getinitargs_func = 0, *getstate_func = 0, *class_args = 0;
2038 char *module_str, *name_str;
2039 int module_size, name_size, res = -1;
2040
2041 static char inst = INST, obj = OBJ, build = BUILD;
2042
2043 if (self->fast && !fast_save_enter(self, args))
2044 goto finally;
2045
2046 if (self->write_func(self, &MARKv, 1) < 0)
2047 goto finally;
2048
2049 if (!( class = PyObject_GetAttr(args, __class___str)))
2050 goto finally;
2051
2052 if (self->bin) {
2053 if (save(self, class, 0) < 0)
2054 goto finally;
2055 }
2056
2057 if ((getinitargs_func = PyObject_GetAttr(args, __getinitargs___str))) {
2058 PyObject *element = 0;
2059 Py_ssize_t i, len;
2060
2061 if (!( class_args =
2062 PyObject_Call(getinitargs_func, empty_tuple, NULL)))
2063 goto finally;
2064
2065 if ((len = PyObject_Size(class_args)) < 0)
2066 goto finally;
2067
2068 for (i = 0; i < len; i++) {
2069 if (!( element = PySequence_GetItem(class_args, i)))
2070 goto finally;
2071
2072 if (save(self, element, 0) < 0) {
2073 Py_DECREF(element);
2074 goto finally;
2075 }
2076
2077 Py_DECREF(element);
2078 }
2079 }
2080 else {
2081 if (PyErr_ExceptionMatches(PyExc_AttributeError))
2082 PyErr_Clear();
2083 else
2084 goto finally;
2085 }
2086
2087 if (!self->bin) {
2088 if (!( name = ((PyClassObject *)class)->cl_name )) {
2089 PyErr_SetString(PicklingError, "class has no name");
2090 goto finally;
2091 }
2092
2093 if (!( module = whichmodule(class, name)))
2094 goto finally;
2095
2096
2097 if ((module_size = PyString_Size(module)) < 0 ||
2098 (name_size = PyString_Size(name)) < 0)
2099 goto finally;
2100
2101 module_str = PyString_AS_STRING((PyStringObject *)module);
2102 name_str = PyString_AS_STRING((PyStringObject *)name);
2103
2104 if (self->write_func(self, &inst, 1) < 0)
2105 goto finally;
2106
2107 if (self->write_func(self, module_str, module_size) < 0)
2108 goto finally;
2109
2110 if (self->write_func(self, "\n", 1) < 0)
2111 goto finally;
2112
2113 if (self->write_func(self, name_str, name_size) < 0)
2114 goto finally;
2115
2116 if (self->write_func(self, "\n", 1) < 0)
2117 goto finally;
2118 }
2119 else if (self->write_func(self, &obj, 1) < 0) {
2120 goto finally;
2121 }
2122
2123 if ((getstate_func = PyObject_GetAttr(args, __getstate___str))) {
2124 state = PyObject_Call(getstate_func, empty_tuple, NULL);
2125 if (!state)
2126 goto finally;
2127 }
2128 else {
2129 if (PyErr_ExceptionMatches(PyExc_AttributeError))
2130 PyErr_Clear();
2131 else
2132 goto finally;
2133
2134 if (!( state = PyObject_GetAttr(args, __dict___str))) {
2135 if (PyErr_ExceptionMatches(PyExc_AttributeError))
2136 PyErr_Clear();
2137 else
2138 goto finally;
2139 res = 0;
2140 goto finally;
2141 }
2142 }
2143
2144 if (!PyDict_Check(state)) {
2145 if (put2(self, args) < 0)
2146 goto finally;
2147 }
2148 else {
2149 if (put(self, args) < 0)
2150 goto finally;
2151 }
2152
2153 if (save(self, state, 0) < 0)
2154 goto finally;
2155
2156 if (self->write_func(self, &build, 1) < 0)
2157 goto finally;
2158
2159 res = 0;
2160
2161 finally:
2162 if (self->fast && !fast_save_leave(self, args))
2163 res = -1;
2164
2165 Py_XDECREF(module);
2166 Py_XDECREF(class);
2167 Py_XDECREF(state);
2168 Py_XDECREF(getinitargs_func);
2169 Py_XDECREF(getstate_func);
2170 Py_XDECREF(class_args);
2171
2172 return res;
2173 }
2174
2175
2176 static int
save_global(Picklerobject * self,PyObject * args,PyObject * name)2177 save_global(Picklerobject *self, PyObject *args, PyObject *name)
2178 {
2179 PyObject *global_name = 0, *module = 0, *mod = 0, *klass = 0;
2180 char *name_str, *module_str;
2181 int module_size, name_size, res = -1;
2182
2183 static char global = GLOBAL;
2184
2185 if (name) {
2186 global_name = name;
2187 Py_INCREF(global_name);
2188 }
2189 else {
2190 if (!( global_name = PyObject_GetAttr(args, __name___str)))
2191 goto finally;
2192 }
2193
2194 if (!( module = whichmodule(args, global_name)))
2195 goto finally;
2196
2197 if ((module_size = PyString_Size(module)) < 0 ||
2198 (name_size = PyString_Size(global_name)) < 0)
2199 goto finally;
2200
2201 module_str = PyString_AS_STRING((PyStringObject *)module);
2202 name_str = PyString_AS_STRING((PyStringObject *)global_name);
2203
2204 /* XXX This can be doing a relative import. Clearly it shouldn't,
2205 but I don't know how to stop it. :-( */
2206 mod = PyImport_ImportModule(module_str);
2207 if (mod == NULL) {
2208 cPickle_ErrFormat(PicklingError,
2209 "Can't pickle %s: import of module %s "
2210 "failed",
2211 "OS", args, module);
2212 goto finally;
2213 }
2214 klass = PyObject_GetAttrString(mod, name_str);
2215 if (klass == NULL) {
2216 cPickle_ErrFormat(PicklingError,
2217 "Can't pickle %s: attribute lookup %s.%s "
2218 "failed",
2219 "OSS", args, module, global_name);
2220 goto finally;
2221 }
2222 if (klass != args) {
2223 Py_DECREF(klass);
2224 cPickle_ErrFormat(PicklingError,
2225 "Can't pickle %s: it's not the same object "
2226 "as %s.%s",
2227 "OSS", args, module, global_name);
2228 goto finally;
2229 }
2230 Py_DECREF(klass);
2231
2232 if (self->proto >= 2) {
2233 /* See whether this is in the extension registry, and if
2234 * so generate an EXT opcode.
2235 */
2236 PyObject *py_code; /* extension code as Python object */
2237 long code; /* extension code as C value */
2238 char c_str[5];
2239 int n;
2240
2241 PyTuple_SET_ITEM(two_tuple, 0, module);
2242 PyTuple_SET_ITEM(two_tuple, 1, global_name);
2243 py_code = PyDict_GetItem(extension_registry, two_tuple);
2244 if (py_code == NULL)
2245 goto gen_global; /* not registered */
2246
2247 /* Verify py_code has the right type and value. */
2248 if (!PyInt_Check(py_code)) {
2249 cPickle_ErrFormat(PicklingError, "Can't pickle %s: "
2250 "extension code %s isn't an integer",
2251 "OO", args, py_code);
2252 goto finally;
2253 }
2254 code = PyInt_AS_LONG(py_code);
2255 if (code <= 0 || code > 0x7fffffffL) {
2256 cPickle_ErrFormat(PicklingError, "Can't pickle %s: "
2257 "extension code %ld is out of range",
2258 "Ol", args, code);
2259 goto finally;
2260 }
2261
2262 /* Generate an EXT opcode. */
2263 if (code <= 0xff) {
2264 c_str[0] = EXT1;
2265 c_str[1] = (char)code;
2266 n = 2;
2267 }
2268 else if (code <= 0xffff) {
2269 c_str[0] = EXT2;
2270 c_str[1] = (char)(code & 0xff);
2271 c_str[2] = (char)((code >> 8) & 0xff);
2272 n = 3;
2273 }
2274 else {
2275 c_str[0] = EXT4;
2276 c_str[1] = (char)(code & 0xff);
2277 c_str[2] = (char)((code >> 8) & 0xff);
2278 c_str[3] = (char)((code >> 16) & 0xff);
2279 c_str[4] = (char)((code >> 24) & 0xff);
2280 n = 5;
2281 }
2282
2283 if (self->write_func(self, c_str, n) >= 0)
2284 res = 0;
2285 goto finally; /* and don't memoize */
2286 }
2287
2288 gen_global:
2289 if (self->write_func(self, &global, 1) < 0)
2290 goto finally;
2291
2292 if (self->write_func(self, module_str, module_size) < 0)
2293 goto finally;
2294
2295 if (self->write_func(self, "\n", 1) < 0)
2296 goto finally;
2297
2298 if (self->write_func(self, name_str, name_size) < 0)
2299 goto finally;
2300
2301 if (self->write_func(self, "\n", 1) < 0)
2302 goto finally;
2303
2304 if (put(self, args) < 0)
2305 goto finally;
2306
2307 res = 0;
2308
2309 finally:
2310 Py_XDECREF(module);
2311 Py_XDECREF(global_name);
2312 Py_XDECREF(mod);
2313
2314 return res;
2315 }
2316
2317 static int
save_pers(Picklerobject * self,PyObject * args,PyObject * f)2318 save_pers(Picklerobject *self, PyObject *args, PyObject *f)
2319 {
2320 PyObject *pid = 0;
2321 Py_ssize_t size;
2322 int res = -1;
2323
2324 static char persid = PERSID, binpersid = BINPERSID;
2325
2326 Py_INCREF(args);
2327 ARG_TUP(self, args);
2328 if (self->arg) {
2329 pid = PyObject_Call(f, self->arg, NULL);
2330 FREE_ARG_TUP(self);
2331 }
2332 if (! pid) return -1;
2333
2334 if (pid != Py_None) {
2335 if (!self->bin) {
2336 if (!PyString_Check(pid)) {
2337 PyErr_SetString(PicklingError,
2338 "persistent id must be string");
2339 goto finally;
2340 }
2341
2342 if (self->write_func(self, &persid, 1) < 0)
2343 goto finally;
2344
2345 if ((size = PyString_Size(pid)) < 0)
2346 goto finally;
2347
2348 if (self->write_func(self,
2349 PyString_AS_STRING(
2350 (PyStringObject *)pid),
2351 size) < 0)
2352 goto finally;
2353
2354 if (self->write_func(self, "\n", 1) < 0)
2355 goto finally;
2356
2357 res = 1;
2358 goto finally;
2359 }
2360 else if (save(self, pid, 1) >= 0) {
2361 if (self->write_func(self, &binpersid, 1) < 0)
2362 res = -1;
2363 else
2364 res = 1;
2365 }
2366
2367 goto finally;
2368 }
2369
2370 res = 0;
2371
2372 finally:
2373 Py_XDECREF(pid);
2374
2375 return res;
2376 }
2377
2378 /* We're saving ob, and args is the 2-thru-5 tuple returned by the
2379 * appropriate __reduce__ method for ob.
2380 */
2381 static int
save_reduce(Picklerobject * self,PyObject * args,PyObject * fn,PyObject * ob)2382 save_reduce(Picklerobject *self, PyObject *args, PyObject *fn, PyObject *ob)
2383 {
2384 PyObject *callable;
2385 PyObject *argtup;
2386 PyObject *state = NULL;
2387 PyObject *listitems = Py_None;
2388 PyObject *dictitems = Py_None;
2389 Py_ssize_t size;
2390
2391 int use_newobj = self->proto >= 2;
2392
2393 static char reduce = REDUCE;
2394 static char build = BUILD;
2395 static char newobj = NEWOBJ;
2396
2397 size = PyTuple_Size(args);
2398 if (size < 2 || size > 5) {
2399 cPickle_ErrFormat(PicklingError, "tuple returned by "
2400 "%s must contain 2 through 5 elements",
2401 "O", fn);
2402 return -1;
2403 }
2404
2405 if (! PyArg_UnpackTuple(args, "save_reduce", 2, 5,
2406 &callable,
2407 &argtup,
2408 &state,
2409 &listitems,
2410 &dictitems))
2411 return -1;
2412
2413 if (!PyTuple_Check(argtup)) {
2414 cPickle_ErrFormat(PicklingError, "Second element of "
2415 "tuple returned by %s must be a tuple",
2416 "O", fn);
2417 return -1;
2418 }
2419
2420 if (state == Py_None)
2421 state = NULL;
2422
2423 if (listitems == Py_None)
2424 listitems = NULL;
2425 else if (!PyIter_Check(listitems)) {
2426 cPickle_ErrFormat(PicklingError, "Fourth element of "
2427 "tuple returned by %s must be an iterator, not %s",
2428 "Os", fn, Py_TYPE(listitems)->tp_name);
2429 return -1;
2430 }
2431
2432 if (dictitems == Py_None)
2433 dictitems = NULL;
2434 else if (!PyIter_Check(dictitems)) {
2435 cPickle_ErrFormat(PicklingError, "Fifth element of "
2436 "tuple returned by %s must be an iterator, not %s",
2437 "Os", fn, Py_TYPE(dictitems)->tp_name);
2438 return -1;
2439 }
2440
2441 /* Protocol 2 special case: if callable's name is __newobj__, use
2442 * NEWOBJ. This consumes a lot of code.
2443 */
2444 if (use_newobj) {
2445 PyObject *temp = PyObject_GetAttr(callable, __name___str);
2446
2447 if (temp == NULL) {
2448 if (PyErr_ExceptionMatches(PyExc_AttributeError))
2449 PyErr_Clear();
2450 else
2451 return -1;
2452 use_newobj = 0;
2453 }
2454 else {
2455 use_newobj = PyString_Check(temp) &&
2456 strcmp(PyString_AS_STRING(temp),
2457 "__newobj__") == 0;
2458 Py_DECREF(temp);
2459 }
2460 }
2461 if (use_newobj) {
2462 PyObject *cls;
2463 PyObject *newargtup;
2464 Py_ssize_t n, i;
2465
2466 /* Sanity checks. */
2467 n = PyTuple_Size(argtup);
2468 if (n < 1) {
2469 PyErr_SetString(PicklingError, "__newobj__ arglist "
2470 "is empty");
2471 return -1;
2472 }
2473
2474 cls = PyTuple_GET_ITEM(argtup, 0);
2475 if (! PyObject_HasAttrString(cls, "__new__")) {
2476 PyErr_SetString(PicklingError, "args[0] from "
2477 "__newobj__ args has no __new__");
2478 return -1;
2479 }
2480
2481 /* XXX How could ob be NULL? */
2482 if (ob != NULL) {
2483 PyObject *ob_dot_class;
2484
2485 ob_dot_class = PyObject_GetAttr(ob, __class___str);
2486 if (ob_dot_class == NULL) {
2487 if (PyErr_ExceptionMatches(
2488 PyExc_AttributeError))
2489 PyErr_Clear();
2490 else
2491 return -1;
2492 }
2493 i = ob_dot_class != cls; /* true iff a problem */
2494 Py_XDECREF(ob_dot_class);
2495 if (i) {
2496 PyErr_SetString(PicklingError, "args[0] from "
2497 "__newobj__ args has the wrong class");
2498 return -1;
2499 }
2500 }
2501
2502 /* Save the class and its __new__ arguments. */
2503 if (save(self, cls, 0) < 0)
2504 return -1;
2505
2506 newargtup = PyTuple_New(n-1); /* argtup[1:] */
2507 if (newargtup == NULL)
2508 return -1;
2509 for (i = 1; i < n; ++i) {
2510 PyObject *temp = PyTuple_GET_ITEM(argtup, i);
2511 Py_INCREF(temp);
2512 PyTuple_SET_ITEM(newargtup, i-1, temp);
2513 }
2514 i = save(self, newargtup, 0);
2515 Py_DECREF(newargtup);
2516 if (i < 0)
2517 return -1;
2518
2519 /* Add NEWOBJ opcode. */
2520 if (self->write_func(self, &newobj, 1) < 0)
2521 return -1;
2522 }
2523 else {
2524 /* Not using NEWOBJ. */
2525 if (save(self, callable, 0) < 0 ||
2526 save(self, argtup, 0) < 0 ||
2527 self->write_func(self, &reduce, 1) < 0)
2528 return -1;
2529 }
2530
2531 /* Memoize. */
2532 /* XXX How can ob be NULL? */
2533 if (ob != NULL) {
2534 /* If the object is already in the memo, this means it is
2535 recursive. In this case, throw away everything we put on the
2536 stack, and fetch the object back from the memo. */
2537 if (Py_REFCNT(ob) > 1 && !self->fast) {
2538 PyObject *py_ob_id = PyLong_FromVoidPtr(ob);
2539 if (!py_ob_id)
2540 return -1;
2541 if (PyDict_GetItem(self->memo, py_ob_id)) {
2542 const char pop_op = POP;
2543 if (self->write_func(self, &pop_op, 1) < 0 ||
2544 get(self, py_ob_id) < 0) {
2545 Py_DECREF(py_ob_id);
2546 return -1;
2547 }
2548 Py_DECREF(py_ob_id);
2549 return 0;
2550 }
2551 Py_DECREF(py_ob_id);
2552 if (PyErr_Occurred())
2553 return -1;
2554 }
2555 if (state && !PyDict_Check(state)) {
2556 if (put2(self, ob) < 0)
2557 return -1;
2558 }
2559 else if (put(self, ob) < 0)
2560 return -1;
2561 }
2562
2563
2564 if (listitems && batch_list(self, listitems) < 0)
2565 return -1;
2566
2567 if (dictitems && batch_dict(self, dictitems) < 0)
2568 return -1;
2569
2570 if (state) {
2571 if (save(self, state, 0) < 0 ||
2572 self->write_func(self, &build, 1) < 0)
2573 return -1;
2574 }
2575
2576 return 0;
2577 }
2578
2579 static int
save(Picklerobject * self,PyObject * args,int pers_save)2580 save(Picklerobject *self, PyObject *args, int pers_save)
2581 {
2582 PyTypeObject *type;
2583 PyObject *py_ob_id = 0, *__reduce__ = 0, *t = 0;
2584 int res = -1;
2585 int tmp;
2586
2587 if (Py_EnterRecursiveCall(" while pickling an object"))
2588 return -1;
2589
2590 if (!pers_save && self->pers_func) {
2591 if ((tmp = save_pers(self, args, self->pers_func)) != 0) {
2592 res = tmp;
2593 goto finally;
2594 }
2595 }
2596
2597 if (args == Py_None) {
2598 res = save_none(self, args);
2599 goto finally;
2600 }
2601
2602 type = Py_TYPE(args);
2603
2604 switch (type->tp_name[0]) {
2605 case 'b':
2606 if (args == Py_False || args == Py_True) {
2607 res = save_bool(self, args);
2608 goto finally;
2609 }
2610 break;
2611 case 'i':
2612 if (type == &PyInt_Type) {
2613 res = save_int(self, args);
2614 goto finally;
2615 }
2616 break;
2617
2618 case 'l':
2619 if (type == &PyLong_Type) {
2620 res = save_long(self, args);
2621 goto finally;
2622 }
2623 break;
2624
2625 case 'f':
2626 if (type == &PyFloat_Type) {
2627 res = save_float(self, args);
2628 goto finally;
2629 }
2630 break;
2631
2632 case 't':
2633 if (type == &PyTuple_Type && PyTuple_Size(args) == 0) {
2634 res = save_tuple(self, args);
2635 goto finally;
2636 }
2637 break;
2638
2639 case 's':
2640 if ((type == &PyString_Type) && (PyString_GET_SIZE(args) < 2)) {
2641 res = save_string(self, args, 0);
2642 goto finally;
2643 }
2644 break;
2645
2646 #ifdef Py_USING_UNICODE
2647 case 'u':
2648 if ((type == &PyUnicode_Type) && (PyString_GET_SIZE(args) < 2)) {
2649 res = save_unicode(self, args, 0);
2650 goto finally;
2651 }
2652 break;
2653 #endif
2654 }
2655
2656 if (Py_REFCNT(args) > 1) {
2657 if (!( py_ob_id = PyLong_FromVoidPtr(args)))
2658 goto finally;
2659
2660 if (PyDict_GetItem(self->memo, py_ob_id)) {
2661 if (get(self, py_ob_id) < 0)
2662 goto finally;
2663
2664 res = 0;
2665 goto finally;
2666 }
2667 }
2668
2669 switch (type->tp_name[0]) {
2670 case 's':
2671 if (type == &PyString_Type) {
2672 res = save_string(self, args, 1);
2673 goto finally;
2674 }
2675 break;
2676
2677 #ifdef Py_USING_UNICODE
2678 case 'u':
2679 if (type == &PyUnicode_Type) {
2680 res = save_unicode(self, args, 1);
2681 goto finally;
2682 }
2683 break;
2684 #endif
2685
2686 case 't':
2687 if (type == &PyTuple_Type) {
2688 res = save_tuple(self, args);
2689 goto finally;
2690 }
2691 if (type == &PyType_Type) {
2692 res = save_global(self, args, NULL);
2693 goto finally;
2694 }
2695 break;
2696
2697 case 'l':
2698 if (type == &PyList_Type) {
2699 res = save_list(self, args);
2700 goto finally;
2701 }
2702 break;
2703
2704 case 'd':
2705 if (type == &PyDict_Type) {
2706 res = save_dict(self, args);
2707 goto finally;
2708 }
2709 break;
2710
2711 case 'i':
2712 if (type == &PyInstance_Type) {
2713 res = save_inst(self, args);
2714 goto finally;
2715 }
2716 break;
2717
2718 case 'c':
2719 if (type == &PyClass_Type) {
2720 res = save_global(self, args, NULL);
2721 goto finally;
2722 }
2723 break;
2724
2725 case 'f':
2726 if (type == &PyFunction_Type) {
2727 res = save_global(self, args, NULL);
2728 if (res && PyErr_ExceptionMatches(PickleError)) {
2729 /* fall back to reduce */
2730 PyErr_Clear();
2731 break;
2732 }
2733 goto finally;
2734 }
2735 break;
2736
2737 case 'b':
2738 if (type == &PyCFunction_Type) {
2739 res = save_global(self, args, NULL);
2740 goto finally;
2741 }
2742 }
2743
2744 if (!pers_save && self->inst_pers_func) {
2745 if ((tmp = save_pers(self, args, self->inst_pers_func)) != 0) {
2746 res = tmp;
2747 goto finally;
2748 }
2749 }
2750
2751 /* Get a reduction callable, and call it. This may come from
2752 * copy_reg.dispatch_table, the object's __reduce_ex__ method,
2753 * or the object's __reduce__ method.
2754 */
2755 __reduce__ = PyDict_GetItem(dispatch_table, (PyObject *)type);
2756 if (__reduce__ != NULL) {
2757 Py_INCREF(__reduce__);
2758 Py_INCREF(args);
2759 ARG_TUP(self, args);
2760 if (self->arg) {
2761 t = PyObject_Call(__reduce__, self->arg, NULL);
2762 FREE_ARG_TUP(self);
2763 }
2764 }
2765 else {
2766 if (PyType_IsSubtype(type, &PyType_Type)) {
2767 res = save_global(self, args, NULL);
2768 goto finally;
2769 }
2770
2771 /* Check for a __reduce_ex__ method. */
2772 __reduce__ = PyObject_GetAttr(args, __reduce_ex___str);
2773 if (__reduce__ != NULL) {
2774 t = PyInt_FromLong(self->proto);
2775 if (t != NULL) {
2776 ARG_TUP(self, t);
2777 t = NULL;
2778 if (self->arg) {
2779 t = PyObject_Call(__reduce__,
2780 self->arg, NULL);
2781 FREE_ARG_TUP(self);
2782 }
2783 }
2784 }
2785 else {
2786 if (PyErr_ExceptionMatches(PyExc_AttributeError))
2787 PyErr_Clear();
2788 else
2789 goto finally;
2790 /* Check for a __reduce__ method. */
2791 __reduce__ = PyObject_GetAttr(args, __reduce___str);
2792 if (__reduce__ != NULL) {
2793 t = PyObject_Call(__reduce__,
2794 empty_tuple, NULL);
2795 }
2796 else {
2797 PyErr_SetObject(UnpickleableError, args);
2798 goto finally;
2799 }
2800 }
2801 }
2802
2803 if (t == NULL)
2804 goto finally;
2805
2806 if (PyString_Check(t)) {
2807 res = save_global(self, args, t);
2808 goto finally;
2809 }
2810
2811 if (!PyTuple_Check(t)) {
2812 cPickle_ErrFormat(PicklingError, "Value returned by "
2813 "%s must be string or tuple",
2814 "O", __reduce__);
2815 goto finally;
2816 }
2817
2818 res = save_reduce(self, t, __reduce__, args);
2819
2820 finally:
2821 Py_LeaveRecursiveCall();
2822 Py_XDECREF(py_ob_id);
2823 Py_XDECREF(__reduce__);
2824 Py_XDECREF(t);
2825
2826 return res;
2827 }
2828
2829
2830 static int
dump(Picklerobject * self,PyObject * args)2831 dump(Picklerobject *self, PyObject *args)
2832 {
2833 static char stop = STOP;
2834
2835 if (self->proto >= 2) {
2836 char bytes[2];
2837
2838 bytes[0] = PROTO;
2839 assert(self->proto >= 0 && self->proto < 256);
2840 bytes[1] = (char)self->proto;
2841 if (self->write_func(self, bytes, 2) < 0)
2842 return -1;
2843 }
2844
2845 if (save(self, args, 0) < 0)
2846 return -1;
2847
2848 if (self->write_func(self, &stop, 1) < 0)
2849 return -1;
2850
2851 if (self->write_func(self, NULL, 0) < 0)
2852 return -1;
2853
2854 return 0;
2855 }
2856
2857 static PyObject *
Pickle_clear_memo(Picklerobject * self,PyObject * args)2858 Pickle_clear_memo(Picklerobject *self, PyObject *args)
2859 {
2860 if (self->memo)
2861 PyDict_Clear(self->memo);
2862 Py_INCREF(Py_None);
2863 return Py_None;
2864 }
2865
2866 static PyObject *
Pickle_getvalue(Picklerobject * self,PyObject * args)2867 Pickle_getvalue(Picklerobject *self, PyObject *args)
2868 {
2869 Py_ssize_t l, i, rsize, ssize, clear=1, lm;
2870 long ik;
2871 PyObject *k, *r;
2872 char *s, *p, *have_get;
2873 Pdata *data;
2874
2875 /* Can be called by Python code or C code */
2876 if (args && !PyArg_ParseTuple(args, "|i:getvalue", &clear))
2877 return NULL;
2878
2879 /* Check to make sure we are based on a list */
2880 if (! Pdata_Check(self->file)) {
2881 PyErr_SetString(PicklingError,
2882 "Attempt to getvalue() a non-list-based pickler");
2883 return NULL;
2884 }
2885
2886 /* flush write buffer */
2887 if (write_other(self, NULL, 0) < 0) return NULL;
2888
2889 data=(Pdata*)self->file;
2890 l=data->length;
2891
2892 /* set up an array to hold get/put status */
2893 lm = PyDict_Size(self->memo);
2894 if (lm < 0) return NULL;
2895 lm++;
2896 have_get = malloc(lm);
2897 if (have_get == NULL) return PyErr_NoMemory();
2898 memset(have_get, 0, lm);
2899
2900 /* Scan for gets. */
2901 for (rsize = 0, i = l; --i >= 0; ) {
2902 k = data->data[i];
2903
2904 if (PyString_Check(k))
2905 rsize += PyString_GET_SIZE(k);
2906
2907 else if (PyInt_Check(k)) { /* put */
2908 ik = PyInt_AS_LONG((PyIntObject*)k);
2909 if (ik >= lm || ik == 0) {
2910 PyErr_SetString(PicklingError,
2911 "Invalid get data");
2912 goto err;
2913 }
2914 if (have_get[ik]) /* with matching get */
2915 rsize += ik < 256 ? 2 : 5;
2916 }
2917
2918 else if (! (PyTuple_Check(k) &&
2919 PyTuple_GET_SIZE(k) == 2 &&
2920 PyInt_Check((k = PyTuple_GET_ITEM(k, 0))))
2921 ) {
2922 PyErr_SetString(PicklingError,
2923 "Unexpected data in internal list");
2924 goto err;
2925 }
2926
2927 else { /* put */
2928 ik = PyInt_AS_LONG((PyIntObject *)k);
2929 if (ik >= lm || ik == 0) {
2930 PyErr_SetString(PicklingError,
2931 "Invalid get data");
2932 goto err;
2933 }
2934 have_get[ik] = 1;
2935 rsize += ik < 256 ? 2 : 5;
2936 }
2937 }
2938
2939 /* Now generate the result */
2940 r = PyString_FromStringAndSize(NULL, rsize);
2941 if (r == NULL) goto err;
2942 s = PyString_AS_STRING((PyStringObject *)r);
2943
2944 for (i = 0; i < l; i++) {
2945 k = data->data[i];
2946
2947 if (PyString_Check(k)) {
2948 ssize = PyString_GET_SIZE(k);
2949 if (ssize) {
2950 p=PyString_AS_STRING((PyStringObject *)k);
2951 while (--ssize >= 0)
2952 *s++ = *p++;
2953 }
2954 }
2955
2956 else if (PyTuple_Check(k)) { /* get */
2957 ik = PyInt_AS_LONG((PyIntObject *)
2958 PyTuple_GET_ITEM(k, 0));
2959 if (ik < 256) {
2960 *s++ = BINGET;
2961 *s++ = (int)(ik & 0xff);
2962 }
2963 else {
2964 *s++ = LONG_BINGET;
2965 *s++ = (int)(ik & 0xff);
2966 *s++ = (int)((ik >> 8) & 0xff);
2967 *s++ = (int)((ik >> 16) & 0xff);
2968 *s++ = (int)((ik >> 24) & 0xff);
2969 }
2970 }
2971
2972 else { /* put */
2973 ik = PyInt_AS_LONG((PyIntObject*)k);
2974
2975 if (have_get[ik]) { /* with matching get */
2976 if (ik < 256) {
2977 *s++ = BINPUT;
2978 *s++ = (int)(ik & 0xff);
2979 }
2980 else {
2981 *s++ = LONG_BINPUT;
2982 *s++ = (int)(ik & 0xff);
2983 *s++ = (int)((ik >> 8) & 0xff);
2984 *s++ = (int)((ik >> 16) & 0xff);
2985 *s++ = (int)((ik >> 24) & 0xff);
2986 }
2987 }
2988 }
2989 }
2990
2991 if (clear) {
2992 PyDict_Clear(self->memo);
2993 Pdata_clear(data, 0);
2994 }
2995
2996 free(have_get);
2997 return r;
2998 err:
2999 free(have_get);
3000 return NULL;
3001 }
3002
3003 static PyObject *
Pickler_dump(Picklerobject * self,PyObject * args)3004 Pickler_dump(Picklerobject *self, PyObject *args)
3005 {
3006 PyObject *ob;
3007 int get=0;
3008
3009 if (!( PyArg_ParseTuple(args, "O|i:dump", &ob, &get)))
3010 return NULL;
3011
3012 if (dump(self, ob) < 0)
3013 return NULL;
3014
3015 if (get) return Pickle_getvalue(self, NULL);
3016
3017 /* XXX Why does dump() return self? */
3018 Py_INCREF(self);
3019 return (PyObject*)self;
3020 }
3021
3022
3023 static struct PyMethodDef Pickler_methods[] =
3024 {
3025 {"dump", (PyCFunction)Pickler_dump, METH_VARARGS,
3026 PyDoc_STR("dump(object) -- "
3027 "Write an object in pickle format to the object's pickle stream")},
3028 {"clear_memo", (PyCFunction)Pickle_clear_memo, METH_NOARGS,
3029 PyDoc_STR("clear_memo() -- Clear the picklers memo")},
3030 {"getvalue", (PyCFunction)Pickle_getvalue, METH_VARARGS,
3031 PyDoc_STR("getvalue() -- Finish picking a list-based pickle")},
3032 {NULL, NULL} /* sentinel */
3033 };
3034
3035
3036 static Picklerobject *
newPicklerobject(PyObject * file,int proto)3037 newPicklerobject(PyObject *file, int proto)
3038 {
3039 Picklerobject *self;
3040
3041 if (proto < 0)
3042 proto = HIGHEST_PROTOCOL;
3043 if (proto > HIGHEST_PROTOCOL) {
3044 PyErr_Format(PyExc_ValueError, "pickle protocol %d asked for; "
3045 "the highest available protocol is %d",
3046 proto, HIGHEST_PROTOCOL);
3047 return NULL;
3048 }
3049
3050 self = PyObject_GC_New(Picklerobject, &Picklertype);
3051 if (self == NULL)
3052 return NULL;
3053 self->proto = proto;
3054 self->bin = proto > 0;
3055 self->fp = NULL;
3056 self->write = NULL;
3057 self->memo = NULL;
3058 self->arg = NULL;
3059 self->pers_func = NULL;
3060 self->inst_pers_func = NULL;
3061 self->write_buf = NULL;
3062 self->fast = 0;
3063 self->fast_container = 0;
3064 self->fast_memo = NULL;
3065 self->buf_size = 0;
3066 self->dispatch_table = NULL;
3067
3068 self->file = NULL;
3069 if (file)
3070 Py_INCREF(file);
3071 else {
3072 file = Pdata_New();
3073 if (file == NULL)
3074 goto err;
3075 }
3076 self->file = file;
3077
3078 if (!( self->memo = PyDict_New()))
3079 goto err;
3080
3081 if (PyFile_Check(file)) {
3082 self->fp = PyFile_AsFile(file);
3083 if (self->fp == NULL) {
3084 PyErr_SetString(PyExc_ValueError,
3085 "I/O operation on closed file");
3086 goto err;
3087 }
3088 self->write_func = write_file;
3089 }
3090 else if (PycStringIO_OutputCheck(file)) {
3091 self->write_func = write_cStringIO;
3092 }
3093 else if (file == Py_None) {
3094 self->write_func = write_none;
3095 }
3096 else {
3097 self->write_func = write_other;
3098
3099 if (! Pdata_Check(file)) {
3100 self->write = PyObject_GetAttr(file, write_str);
3101 if (!self->write) {
3102 PyErr_Clear();
3103 PyErr_SetString(PyExc_TypeError,
3104 "argument must have 'write' "
3105 "attribute");
3106 goto err;
3107 }
3108 }
3109
3110 self->write_buf = (char *)PyMem_Malloc(WRITE_BUF_SIZE);
3111 if (self->write_buf == NULL) {
3112 PyErr_NoMemory();
3113 goto err;
3114 }
3115 }
3116
3117 if (PyEval_GetRestricted()) {
3118 /* Restricted execution, get private tables */
3119 PyObject *m = PyImport_ImportModule("copy_reg");
3120
3121 if (m == NULL)
3122 goto err;
3123 self->dispatch_table = PyObject_GetAttr(m, dispatch_table_str);
3124 Py_DECREF(m);
3125 if (self->dispatch_table == NULL)
3126 goto err;
3127 }
3128 else {
3129 self->dispatch_table = dispatch_table;
3130 Py_INCREF(dispatch_table);
3131 }
3132 PyObject_GC_Track(self);
3133
3134 return self;
3135
3136 err:
3137 Py_DECREF(self);
3138 return NULL;
3139 }
3140
3141
3142 static PyObject *
get_Pickler(PyObject * self,PyObject * args,PyObject * kwds)3143 get_Pickler(PyObject *self, PyObject *args, PyObject *kwds)
3144 {
3145 static char *kwlist[] = {"file", "protocol", NULL};
3146 PyObject *file = NULL;
3147 int proto = 0;
3148
3149 /* XXX
3150 * The documented signature is Pickler(file, protocol=0), but this
3151 * accepts Pickler() and Pickler(integer) too. The meaning then
3152 * is clear as mud, undocumented, and not supported by pickle.py.
3153 * I'm told Zope uses this, but I haven't traced into this code
3154 * far enough to figure out what it means.
3155 */
3156 if (!PyArg_ParseTuple(args, "|i:Pickler", &proto)) {
3157 PyErr_Clear();
3158 proto = 0;
3159 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|i:Pickler",
3160 kwlist, &file, &proto))
3161 return NULL;
3162 }
3163 return (PyObject *)newPicklerobject(file, proto);
3164 }
3165
3166
3167 static void
Pickler_dealloc(Picklerobject * self)3168 Pickler_dealloc(Picklerobject *self)
3169 {
3170 PyObject_GC_UnTrack(self);
3171 Py_XDECREF(self->write);
3172 Py_XDECREF(self->memo);
3173 Py_XDECREF(self->fast_memo);
3174 Py_XDECREF(self->arg);
3175 Py_XDECREF(self->file);
3176 Py_XDECREF(self->pers_func);
3177 Py_XDECREF(self->inst_pers_func);
3178 Py_XDECREF(self->dispatch_table);
3179 PyMem_Free(self->write_buf);
3180 Py_TYPE(self)->tp_free((PyObject *)self);
3181 }
3182
3183 static int
Pickler_traverse(Picklerobject * self,visitproc visit,void * arg)3184 Pickler_traverse(Picklerobject *self, visitproc visit, void *arg)
3185 {
3186 Py_VISIT(self->write);
3187 Py_VISIT(self->memo);
3188 Py_VISIT(self->fast_memo);
3189 Py_VISIT(self->arg);
3190 Py_VISIT(self->file);
3191 Py_VISIT(self->pers_func);
3192 Py_VISIT(self->inst_pers_func);
3193 Py_VISIT(self->dispatch_table);
3194 return 0;
3195 }
3196
3197 static int
Pickler_clear(Picklerobject * self)3198 Pickler_clear(Picklerobject *self)
3199 {
3200 Py_CLEAR(self->write);
3201 Py_CLEAR(self->memo);
3202 Py_CLEAR(self->fast_memo);
3203 Py_CLEAR(self->arg);
3204 Py_CLEAR(self->file);
3205 Py_CLEAR(self->pers_func);
3206 Py_CLEAR(self->inst_pers_func);
3207 Py_CLEAR(self->dispatch_table);
3208 return 0;
3209 }
3210
3211 static PyObject *
Pickler_get_pers_func(Picklerobject * p)3212 Pickler_get_pers_func(Picklerobject *p)
3213 {
3214 if (p->pers_func == NULL)
3215 PyErr_SetString(PyExc_AttributeError, "persistent_id");
3216 else
3217 Py_INCREF(p->pers_func);
3218 return p->pers_func;
3219 }
3220
3221 static int
Pickler_set_pers_func(Picklerobject * p,PyObject * v)3222 Pickler_set_pers_func(Picklerobject *p, PyObject *v)
3223 {
3224 if (v == NULL) {
3225 PyErr_SetString(PyExc_TypeError,
3226 "attribute deletion is not supported");
3227 return -1;
3228 }
3229 Py_INCREF(v);
3230 Py_XSETREF(p->pers_func, v);
3231 return 0;
3232 }
3233
3234 static int
Pickler_set_inst_pers_func(Picklerobject * p,PyObject * v)3235 Pickler_set_inst_pers_func(Picklerobject *p, PyObject *v)
3236 {
3237 if (v == NULL) {
3238 PyErr_SetString(PyExc_TypeError,
3239 "attribute deletion is not supported");
3240 return -1;
3241 }
3242 Py_INCREF(v);
3243 Py_XSETREF(p->inst_pers_func, v);
3244 return 0;
3245 }
3246
3247 static PyObject *
Pickler_get_memo(Picklerobject * p)3248 Pickler_get_memo(Picklerobject *p)
3249 {
3250 if (p->memo == NULL)
3251 PyErr_SetString(PyExc_AttributeError, "memo");
3252 else
3253 Py_INCREF(p->memo);
3254 return p->memo;
3255 }
3256
3257 static int
Pickler_set_memo(Picklerobject * p,PyObject * v)3258 Pickler_set_memo(Picklerobject *p, PyObject *v)
3259 {
3260 if (v == NULL) {
3261 PyErr_SetString(PyExc_TypeError,
3262 "attribute deletion is not supported");
3263 return -1;
3264 }
3265 if (!PyDict_Check(v)) {
3266 PyErr_SetString(PyExc_TypeError, "memo must be a dictionary");
3267 return -1;
3268 }
3269 Py_INCREF(v);
3270 Py_XSETREF(p->memo, v);
3271 return 0;
3272 }
3273
3274 static PyObject *
Pickler_get_error(Picklerobject * p)3275 Pickler_get_error(Picklerobject *p)
3276 {
3277 /* why is this an attribute on the Pickler? */
3278 Py_INCREF(PicklingError);
3279 return PicklingError;
3280 }
3281
3282 static PyMemberDef Pickler_members[] = {
3283 {"binary", T_INT, offsetof(Picklerobject, bin)},
3284 {"fast", T_INT, offsetof(Picklerobject, fast)},
3285 {NULL}
3286 };
3287
3288 static PyGetSetDef Pickler_getsets[] = {
3289 {"persistent_id", (getter)Pickler_get_pers_func,
3290 (setter)Pickler_set_pers_func},
3291 {"inst_persistent_id", NULL, (setter)Pickler_set_inst_pers_func},
3292 {"memo", (getter)Pickler_get_memo, (setter)Pickler_set_memo},
3293 {"PicklingError", (getter)Pickler_get_error, NULL},
3294 {NULL}
3295 };
3296
3297 PyDoc_STRVAR(Picklertype__doc__,
3298 "Objects that know how to pickle objects\n");
3299
3300 static PyTypeObject Picklertype = {
3301 PyVarObject_HEAD_INIT(NULL, 0)
3302 "cPickle.Pickler", /*tp_name*/
3303 sizeof(Picklerobject), /*tp_basicsize*/
3304 0,
3305 (destructor)Pickler_dealloc, /* tp_dealloc */
3306 0, /* tp_print */
3307 0, /* tp_getattr */
3308 0, /* tp_setattr */
3309 0, /* tp_compare */
3310 0, /* tp_repr */
3311 0, /* tp_as_number */
3312 0, /* tp_as_sequence */
3313 0, /* tp_as_mapping */
3314 0, /* tp_hash */
3315 0, /* tp_call */
3316 0, /* tp_str */
3317 PyObject_GenericGetAttr, /* tp_getattro */
3318 PyObject_GenericSetAttr, /* tp_setattro */
3319 0, /* tp_as_buffer */
3320 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
3321 Picklertype__doc__, /* tp_doc */
3322 (traverseproc)Pickler_traverse, /* tp_traverse */
3323 (inquiry)Pickler_clear, /* tp_clear */
3324 0, /* tp_richcompare */
3325 0, /* tp_weaklistoffset */
3326 0, /* tp_iter */
3327 0, /* tp_iternext */
3328 Pickler_methods, /* tp_methods */
3329 Pickler_members, /* tp_members */
3330 Pickler_getsets, /* tp_getset */
3331 };
3332
3333 static PyObject *
find_class(PyObject * py_module_name,PyObject * py_global_name,PyObject * fc)3334 find_class(PyObject *py_module_name, PyObject *py_global_name, PyObject *fc)
3335 {
3336 PyObject *global = 0, *module;
3337
3338 if (fc) {
3339 if (fc==Py_None) {
3340 PyErr_SetString(UnpicklingError, "Global and instance "
3341 "pickles are not supported.");
3342 return NULL;
3343 }
3344 return PyObject_CallFunctionObjArgs(fc, py_module_name,
3345 py_global_name, NULL);
3346 }
3347
3348 module = PySys_GetObject("modules");
3349 if (module == NULL)
3350 return NULL;
3351
3352 module = PyDict_GetItem(module, py_module_name);
3353 if (module == NULL) {
3354 module = PyImport_Import(py_module_name);
3355 if (!module)
3356 return NULL;
3357 global = PyObject_GetAttr(module, py_global_name);
3358 Py_DECREF(module);
3359 }
3360 else
3361 global = PyObject_GetAttr(module, py_global_name);
3362 return global;
3363 }
3364
3365 static Py_ssize_t
marker(Unpicklerobject * self)3366 marker(Unpicklerobject *self)
3367 {
3368 if (self->num_marks < 1) {
3369 PyErr_SetString(UnpicklingError, "could not find MARK");
3370 return -1;
3371 }
3372
3373 return self->marks[--self->num_marks];
3374 }
3375
3376
3377 static int
load_none(Unpicklerobject * self)3378 load_none(Unpicklerobject *self)
3379 {
3380 PDATA_APPEND(self->stack, Py_None, -1);
3381 return 0;
3382 }
3383
3384 static int
bad_readline(void)3385 bad_readline(void)
3386 {
3387 PyErr_SetString(UnpicklingError, "pickle data was truncated");
3388 return -1;
3389 }
3390
3391 static int
load_int(Unpicklerobject * self)3392 load_int(Unpicklerobject *self)
3393 {
3394 PyObject *py_int = 0;
3395 char *endptr, *s;
3396 Py_ssize_t len;
3397 int res = -1;
3398 long l;
3399
3400 if ((len = self->readline_func(self, &s)) < 0) return -1;
3401 if (len < 2) return bad_readline();
3402 if (!( s=pystrndup(s,len))) return -1;
3403
3404 errno = 0;
3405 l = strtol(s, &endptr, 0);
3406
3407 if (errno || (*endptr != '\n') || (endptr[1] != '\0')) {
3408 /* Hm, maybe we've got something long. Let's try reading
3409 it as a Python long object. */
3410 errno = 0;
3411 py_int = PyLong_FromString(s, NULL, 0);
3412 if (py_int == NULL) {
3413 PyErr_SetString(PyExc_ValueError,
3414 "could not convert string to int");
3415 goto finally;
3416 }
3417 }
3418 else {
3419 if (len == 3 && (l == 0 || l == 1)) {
3420 if (!( py_int = PyBool_FromLong(l))) goto finally;
3421 }
3422 else {
3423 if (!( py_int = PyInt_FromLong(l))) goto finally;
3424 }
3425 }
3426
3427 free(s);
3428 PDATA_PUSH(self->stack, py_int, -1);
3429 return 0;
3430
3431 finally:
3432 free(s);
3433
3434 return res;
3435 }
3436
3437 static int
load_bool(Unpicklerobject * self,PyObject * boolean)3438 load_bool(Unpicklerobject *self, PyObject *boolean)
3439 {
3440 assert(boolean == Py_True || boolean == Py_False);
3441 PDATA_APPEND(self->stack, boolean, -1);
3442 return 0;
3443 }
3444
3445 /* s contains x bytes of a little-endian integer. Return its value as a
3446 * C int. Obscure: when x is 1 or 2, this is an unsigned little-endian
3447 * int, but when x is 4 it's a signed one. This is a historical source
3448 * of x-platform bugs.
3449 */
3450 static long
calc_binint(char * s,int x)3451 calc_binint(char *s, int x)
3452 {
3453 unsigned char c;
3454 int i;
3455 long l;
3456
3457 for (i = 0, l = 0L; i < x; i++) {
3458 c = (unsigned char)s[i];
3459 l |= (long)c << (i * 8);
3460 }
3461 #if SIZEOF_LONG > 4
3462 /* Unlike BININT1 and BININT2, BININT (more accurately BININT4)
3463 * is signed, so on a box with longs bigger than 4 bytes we need
3464 * to extend a BININT's sign bit to the full width.
3465 */
3466 if (x == 4 && l & (1L << 31))
3467 l |= (~0UL) << 32;
3468 #endif
3469 return l;
3470 }
3471
3472
3473 static int
load_binintx(Unpicklerobject * self,char * s,int x)3474 load_binintx(Unpicklerobject *self, char *s, int x)
3475 {
3476 PyObject *py_int = 0;
3477 long l;
3478
3479 l = calc_binint(s, x);
3480
3481 if (!( py_int = PyInt_FromLong(l)))
3482 return -1;
3483
3484 PDATA_PUSH(self->stack, py_int, -1);
3485 return 0;
3486 }
3487
3488
3489 static int
load_binint(Unpicklerobject * self)3490 load_binint(Unpicklerobject *self)
3491 {
3492 char *s;
3493
3494 if (self->read_func(self, &s, 4) < 0)
3495 return -1;
3496
3497 return load_binintx(self, s, 4);
3498 }
3499
3500
3501 static int
load_binint1(Unpicklerobject * self)3502 load_binint1(Unpicklerobject *self)
3503 {
3504 char *s;
3505
3506 if (self->read_func(self, &s, 1) < 0)
3507 return -1;
3508
3509 return load_binintx(self, s, 1);
3510 }
3511
3512
3513 static int
load_binint2(Unpicklerobject * self)3514 load_binint2(Unpicklerobject *self)
3515 {
3516 char *s;
3517
3518 if (self->read_func(self, &s, 2) < 0)
3519 return -1;
3520
3521 return load_binintx(self, s, 2);
3522 }
3523
3524 static int
load_long(Unpicklerobject * self)3525 load_long(Unpicklerobject *self)
3526 {
3527 PyObject *l = 0;
3528 char *end, *s;
3529 Py_ssize_t len;
3530 int res = -1;
3531
3532 if ((len = self->readline_func(self, &s)) < 0) return -1;
3533 if (len < 2) return bad_readline();
3534 if (!( s=pystrndup(s,len))) return -1;
3535
3536 if (!( l = PyLong_FromString(s, &end, 0)))
3537 goto finally;
3538
3539 free(s);
3540 PDATA_PUSH(self->stack, l, -1);
3541 return 0;
3542
3543 finally:
3544 free(s);
3545
3546 return res;
3547 }
3548
3549 /* 'size' bytes contain the # of bytes of little-endian 256's-complement
3550 * data following.
3551 */
3552 static int
load_counted_long(Unpicklerobject * self,int size)3553 load_counted_long(Unpicklerobject *self, int size)
3554 {
3555 Py_ssize_t i;
3556 char *nbytes;
3557 unsigned char *pdata;
3558 PyObject *along;
3559
3560 assert(size == 1 || size == 4);
3561 i = self->read_func(self, &nbytes, size);
3562 if (i < 0) return -1;
3563
3564 size = calc_binint(nbytes, size);
3565 if (size < 0) {
3566 /* Corrupt or hostile pickle -- we never write one like
3567 * this.
3568 */
3569 PyErr_SetString(UnpicklingError, "LONG pickle has negative "
3570 "byte count");
3571 return -1;
3572 }
3573
3574 if (size == 0)
3575 along = PyLong_FromLong(0L);
3576 else {
3577 /* Read the raw little-endian bytes & convert. */
3578 i = self->read_func(self, (char **)&pdata, size);
3579 if (i < 0) return -1;
3580 along = _PyLong_FromByteArray(pdata, (size_t)size,
3581 1 /* little endian */, 1 /* signed */);
3582 }
3583 if (along == NULL)
3584 return -1;
3585 PDATA_PUSH(self->stack, along, -1);
3586 return 0;
3587 }
3588
3589 static int
load_float(Unpicklerobject * self)3590 load_float(Unpicklerobject *self)
3591 {
3592 PyObject *py_float = 0;
3593 char *endptr, *s;
3594 Py_ssize_t len;
3595 int res = -1;
3596 double d;
3597
3598 if ((len = self->readline_func(self, &s)) < 0) return -1;
3599 if (len < 2) return bad_readline();
3600 if (!( s=pystrndup(s,len))) return -1;
3601
3602 d = PyOS_string_to_double(s, &endptr, PyExc_OverflowError);
3603
3604 if (d == -1.0 && PyErr_Occurred()) {
3605 goto finally;
3606 } else if ((endptr[0] != '\n') || (endptr[1] != '\0')) {
3607 PyErr_SetString(PyExc_ValueError,
3608 "could not convert string to float");
3609 goto finally;
3610 }
3611
3612 if (!( py_float = PyFloat_FromDouble(d)))
3613 goto finally;
3614
3615 free(s);
3616 PDATA_PUSH(self->stack, py_float, -1);
3617 return 0;
3618
3619 finally:
3620 free(s);
3621
3622 return res;
3623 }
3624
3625 static int
load_binfloat(Unpicklerobject * self)3626 load_binfloat(Unpicklerobject *self)
3627 {
3628 PyObject *py_float;
3629 double x;
3630 char *p;
3631
3632 if (self->read_func(self, &p, 8) < 0)
3633 return -1;
3634
3635 x = _PyFloat_Unpack8((unsigned char *)p, 0);
3636 if (x == -1.0 && PyErr_Occurred())
3637 return -1;
3638
3639 py_float = PyFloat_FromDouble(x);
3640 if (py_float == NULL)
3641 return -1;
3642
3643 PDATA_PUSH(self->stack, py_float, -1);
3644 return 0;
3645 }
3646
3647 static int
load_string(Unpicklerobject * self)3648 load_string(Unpicklerobject *self)
3649 {
3650 PyObject *str = 0;
3651 Py_ssize_t len;
3652 int res = -1;
3653 char *s, *p;
3654
3655 if ((len = self->readline_func(self, &s)) < 0) return -1;
3656 if (len < 2) return bad_readline();
3657 if (!( s=pystrndup(s,len))) return -1;
3658
3659
3660 /* Strip outermost quotes */
3661 while (len > 0 && s[len-1] <= ' ')
3662 len--;
3663 if (len > 1 && s[0]=='"' && s[len-1]=='"') {
3664 s[len-1] = '\0';
3665 p = s + 1 ;
3666 len -= 2;
3667 }
3668 else if (len > 1 && s[0]=='\'' && s[len-1]=='\'') {
3669 s[len-1] = '\0';
3670 p = s + 1 ;
3671 len -= 2;
3672 }
3673 else
3674 goto insecure;
3675 /********************************************/
3676
3677 str = PyString_DecodeEscape(p, len, NULL, 0, NULL);
3678 free(s);
3679 if (str) {
3680 PDATA_PUSH(self->stack, str, -1);
3681 res = 0;
3682 }
3683 return res;
3684
3685 insecure:
3686 free(s);
3687 PyErr_SetString(PyExc_ValueError,"insecure string pickle");
3688 return -1;
3689 }
3690
3691
3692 static int
load_binstring(Unpicklerobject * self)3693 load_binstring(Unpicklerobject *self)
3694 {
3695 PyObject *py_string = 0;
3696 Py_ssize_t l;
3697 char *s;
3698
3699 if (self->read_func(self, &s, 4) < 0) return -1;
3700
3701 l = calc_binint(s, 4);
3702 if (l < 0) {
3703 /* Corrupt or hostile pickle -- we never write one like
3704 * this.
3705 */
3706 PyErr_SetString(UnpicklingError,
3707 "BINSTRING pickle has negative byte count");
3708 return -1;
3709 }
3710
3711 if (self->read_func(self, &s, l) < 0)
3712 return -1;
3713
3714 if (!( py_string = PyString_FromStringAndSize(s, l)))
3715 return -1;
3716
3717 PDATA_PUSH(self->stack, py_string, -1);
3718 return 0;
3719 }
3720
3721
3722 static int
load_short_binstring(Unpicklerobject * self)3723 load_short_binstring(Unpicklerobject *self)
3724 {
3725 PyObject *py_string = 0;
3726 unsigned char l;
3727 char *s;
3728
3729 if (self->read_func(self, &s, 1) < 0)
3730 return -1;
3731
3732 l = (unsigned char)s[0];
3733
3734 if (self->read_func(self, &s, l) < 0) return -1;
3735
3736 if (!( py_string = PyString_FromStringAndSize(s, l))) return -1;
3737
3738 PDATA_PUSH(self->stack, py_string, -1);
3739 return 0;
3740 }
3741
3742
3743 #ifdef Py_USING_UNICODE
3744 static int
load_unicode(Unpicklerobject * self)3745 load_unicode(Unpicklerobject *self)
3746 {
3747 PyObject *str = 0;
3748 Py_ssize_t len;
3749 char *s;
3750
3751 if ((len = self->readline_func(self, &s)) < 0) return -1;
3752 if (len < 1) return bad_readline();
3753
3754 if (!( str = PyUnicode_DecodeRawUnicodeEscape(s, len - 1, NULL)))
3755 return -1;
3756
3757 PDATA_PUSH(self->stack, str, -1);
3758 return 0;
3759 }
3760 #endif
3761
3762
3763 #ifdef Py_USING_UNICODE
3764 static int
load_binunicode(Unpicklerobject * self)3765 load_binunicode(Unpicklerobject *self)
3766 {
3767 PyObject *unicode;
3768 Py_ssize_t l;
3769 char *s;
3770
3771 if (self->read_func(self, &s, 4) < 0) return -1;
3772
3773 l = calc_binint(s, 4);
3774 if (l < 0) {
3775 /* Corrupt or hostile pickle -- we never write one like
3776 * this.
3777 */
3778 PyErr_SetString(UnpicklingError,
3779 "BINUNICODE pickle has negative byte count");
3780 return -1;
3781 }
3782
3783 if (self->read_func(self, &s, l) < 0)
3784 return -1;
3785
3786 if (!( unicode = PyUnicode_DecodeUTF8(s, l, NULL)))
3787 return -1;
3788
3789 PDATA_PUSH(self->stack, unicode, -1);
3790 return 0;
3791 }
3792 #endif
3793
3794
3795 static int
load_counted_tuple(Unpicklerobject * self,int len)3796 load_counted_tuple(Unpicklerobject *self, int len)
3797 {
3798 PyObject *tup;
3799
3800 if (self->stack->length < len)
3801 return stackUnderflow();
3802
3803 if (!(tup = Pdata_popTuple(self->stack, self->stack->length - len)))
3804 return -1;
3805 PDATA_PUSH(self->stack, tup, -1);
3806 return 0;
3807 }
3808
3809 static int
load_tuple(Unpicklerobject * self)3810 load_tuple(Unpicklerobject *self)
3811 {
3812 Py_ssize_t i;
3813
3814 if ((i = marker(self)) < 0) return -1;
3815 return load_counted_tuple(self, self->stack->length - i);
3816 }
3817
3818 static int
load_empty_list(Unpicklerobject * self)3819 load_empty_list(Unpicklerobject *self)
3820 {
3821 PyObject *list;
3822
3823 if (!( list=PyList_New(0))) return -1;
3824 PDATA_PUSH(self->stack, list, -1);
3825 return 0;
3826 }
3827
3828 static int
load_empty_dict(Unpicklerobject * self)3829 load_empty_dict(Unpicklerobject *self)
3830 {
3831 PyObject *dict;
3832
3833 if (!( dict=PyDict_New())) return -1;
3834 PDATA_PUSH(self->stack, dict, -1);
3835 return 0;
3836 }
3837
3838
3839 static int
load_list(Unpicklerobject * self)3840 load_list(Unpicklerobject *self)
3841 {
3842 PyObject *list = 0;
3843 Py_ssize_t i;
3844
3845 if ((i = marker(self)) < 0) return -1;
3846 if (!( list=Pdata_popList(self->stack, i))) return -1;
3847 PDATA_PUSH(self->stack, list, -1);
3848 return 0;
3849 }
3850
3851 static int
load_dict(Unpicklerobject * self)3852 load_dict(Unpicklerobject *self)
3853 {
3854 PyObject *dict, *key, *value;
3855 Py_ssize_t i, j, k;
3856
3857 if ((i = marker(self)) < 0) return -1;
3858 j=self->stack->length;
3859
3860 if (!( dict = PyDict_New())) return -1;
3861
3862 for (k = i+1; k < j; k += 2) {
3863 key =self->stack->data[k-1];
3864 value=self->stack->data[k ];
3865 if (PyDict_SetItem(dict, key, value) < 0) {
3866 Py_DECREF(dict);
3867 return -1;
3868 }
3869 }
3870 Pdata_clear(self->stack, i);
3871 PDATA_PUSH(self->stack, dict, -1);
3872 return 0;
3873 }
3874
3875 static PyObject *
Instance_New(PyObject * cls,PyObject * args)3876 Instance_New(PyObject *cls, PyObject *args)
3877 {
3878 PyObject *r = 0;
3879
3880 if (PyClass_Check(cls)) {
3881 int l;
3882
3883 if ((l=PyObject_Size(args)) < 0) goto err;
3884 if (!( l )) {
3885 PyObject *__getinitargs__;
3886
3887 __getinitargs__ = PyObject_GetAttr(cls,
3888 __getinitargs___str);
3889 if (!__getinitargs__) {
3890 /* We have a class with no __getinitargs__,
3891 so bypass usual construction */
3892 PyObject *inst;
3893
3894 PyErr_Clear();
3895 if (!( inst=PyInstance_NewRaw(cls, NULL)))
3896 goto err;
3897 return inst;
3898 }
3899 Py_DECREF(__getinitargs__);
3900 }
3901
3902 if ((r=PyInstance_New(cls, args, NULL))) return r;
3903 else goto err;
3904 }
3905
3906 if ((r=PyObject_CallObject(cls, args))) return r;
3907
3908 err:
3909 {
3910 PyObject *tp, *v, *tb, *tmp_value;
3911
3912 PyErr_Fetch(&tp, &v, &tb);
3913 tmp_value = v;
3914 /* NULL occurs when there was a KeyboardInterrupt */
3915 if (tmp_value == NULL)
3916 tmp_value = Py_None;
3917 if ((r = PyTuple_Pack(3, tmp_value, cls, args))) {
3918 Py_XDECREF(v);
3919 v=r;
3920 }
3921 PyErr_Restore(tp,v,tb);
3922 }
3923 return NULL;
3924 }
3925
3926
3927 static int
load_obj(Unpicklerobject * self)3928 load_obj(Unpicklerobject *self)
3929 {
3930 PyObject *class, *tup, *obj=0;
3931 Py_ssize_t i;
3932
3933 if ((i = marker(self)) < 0) return -1;
3934
3935 if (self->stack->length - i < 1)
3936 return stackUnderflow();
3937
3938 if (!( tup=Pdata_popTuple(self->stack, i+1))) return -1;
3939 PDATA_POP(self->stack, class);
3940 if (class) {
3941 obj = Instance_New(class, tup);
3942 Py_DECREF(class);
3943 }
3944 Py_DECREF(tup);
3945
3946 if (! obj) return -1;
3947 PDATA_PUSH(self->stack, obj, -1);
3948 return 0;
3949 }
3950
3951
3952 static int
load_inst(Unpicklerobject * self)3953 load_inst(Unpicklerobject *self)
3954 {
3955 PyObject *tup, *class=0, *obj=0, *module_name, *class_name;
3956 Py_ssize_t i, len;
3957 char *s;
3958
3959 if ((i = marker(self)) < 0) return -1;
3960
3961 if ((len = self->readline_func(self, &s)) < 0) return -1;
3962 if (len < 2) return bad_readline();
3963 module_name = PyString_FromStringAndSize(s, len - 1);
3964 if (!module_name) return -1;
3965
3966 if ((len = self->readline_func(self, &s)) >= 0) {
3967 if (len < 2) {
3968 Py_DECREF(module_name);
3969 return bad_readline();
3970 }
3971 if ((class_name = PyString_FromStringAndSize(s, len - 1))) {
3972 class = find_class(module_name, class_name,
3973 self->find_class);
3974 Py_DECREF(class_name);
3975 }
3976 }
3977 Py_DECREF(module_name);
3978
3979 if (! class) return -1;
3980
3981 if ((tup=Pdata_popTuple(self->stack, i))) {
3982 obj = Instance_New(class, tup);
3983 Py_DECREF(tup);
3984 }
3985 Py_DECREF(class);
3986
3987 if (! obj) return -1;
3988
3989 PDATA_PUSH(self->stack, obj, -1);
3990 return 0;
3991 }
3992
3993 static int
load_newobj(Unpicklerobject * self)3994 load_newobj(Unpicklerobject *self)
3995 {
3996 PyObject *args = NULL;
3997 PyObject *clsraw = NULL;
3998 PyTypeObject *cls; /* clsraw cast to its true type */
3999 PyObject *obj;
4000
4001 /* Stack is ... cls argtuple, and we want to call
4002 * cls.__new__(cls, *argtuple).
4003 */
4004 PDATA_POP(self->stack, args);
4005 if (args == NULL) goto Fail;
4006 if (! PyTuple_Check(args)) {
4007 PyErr_SetString(UnpicklingError, "NEWOBJ expected an arg "
4008 "tuple.");
4009 goto Fail;
4010 }
4011
4012 PDATA_POP(self->stack, clsraw);
4013 cls = (PyTypeObject *)clsraw;
4014 if (cls == NULL) goto Fail;
4015 if (! PyType_Check(cls)) {
4016 PyErr_SetString(UnpicklingError, "NEWOBJ class argument "
4017 "isn't a type object");
4018 goto Fail;
4019 }
4020 if (cls->tp_new == NULL) {
4021 PyErr_SetString(UnpicklingError, "NEWOBJ class argument "
4022 "has NULL tp_new");
4023 goto Fail;
4024 }
4025
4026 /* Call __new__. */
4027 obj = cls->tp_new(cls, args, NULL);
4028 if (obj == NULL) goto Fail;
4029
4030 Py_DECREF(args);
4031 Py_DECREF(clsraw);
4032 PDATA_PUSH(self->stack, obj, -1);
4033 return 0;
4034
4035 Fail:
4036 Py_XDECREF(args);
4037 Py_XDECREF(clsraw);
4038 return -1;
4039 }
4040
4041 static int
load_global(Unpicklerobject * self)4042 load_global(Unpicklerobject *self)
4043 {
4044 PyObject *class = 0, *module_name = 0, *class_name = 0;
4045 Py_ssize_t len;
4046 char *s;
4047
4048 if ((len = self->readline_func(self, &s)) < 0) return -1;
4049 if (len < 2) return bad_readline();
4050 module_name = PyString_FromStringAndSize(s, len - 1);
4051 if (!module_name) return -1;
4052
4053 if ((len = self->readline_func(self, &s)) >= 0) {
4054 if (len < 2) {
4055 Py_DECREF(module_name);
4056 return bad_readline();
4057 }
4058 if ((class_name = PyString_FromStringAndSize(s, len - 1))) {
4059 class = find_class(module_name, class_name,
4060 self->find_class);
4061 Py_DECREF(class_name);
4062 }
4063 }
4064 Py_DECREF(module_name);
4065
4066 if (! class) return -1;
4067 PDATA_PUSH(self->stack, class, -1);
4068 return 0;
4069 }
4070
4071
4072 static int
load_persid(Unpicklerobject * self)4073 load_persid(Unpicklerobject *self)
4074 {
4075 PyObject *pid = 0;
4076 Py_ssize_t len;
4077 char *s;
4078
4079 if (self->pers_func) {
4080 if ((len = self->readline_func(self, &s)) < 0) return -1;
4081 if (len < 2) return bad_readline();
4082
4083 pid = PyString_FromStringAndSize(s, len - 1);
4084 if (!pid) return -1;
4085
4086 if (PyList_Check(self->pers_func)) {
4087 if (PyList_Append(self->pers_func, pid) < 0) {
4088 Py_DECREF(pid);
4089 return -1;
4090 }
4091 }
4092 else {
4093 ARG_TUP(self, pid);
4094 if (self->arg) {
4095 pid = PyObject_Call(self->pers_func, self->arg,
4096 NULL);
4097 FREE_ARG_TUP(self);
4098 }
4099 }
4100
4101 if (! pid) return -1;
4102
4103 PDATA_PUSH(self->stack, pid, -1);
4104 return 0;
4105 }
4106 else {
4107 PyErr_SetString(UnpicklingError,
4108 "A load persistent id instruction was encountered,\n"
4109 "but no persistent_load function was specified.");
4110 return -1;
4111 }
4112 }
4113
4114 static int
load_binpersid(Unpicklerobject * self)4115 load_binpersid(Unpicklerobject *self)
4116 {
4117 PyObject *pid = 0;
4118
4119 if (self->pers_func) {
4120 PDATA_POP(self->stack, pid);
4121 if (! pid) return -1;
4122
4123 if (PyList_Check(self->pers_func)) {
4124 if (PyList_Append(self->pers_func, pid) < 0) {
4125 Py_DECREF(pid);
4126 return -1;
4127 }
4128 }
4129 else {
4130 ARG_TUP(self, pid);
4131 if (self->arg) {
4132 pid = PyObject_Call(self->pers_func, self->arg,
4133 NULL);
4134 FREE_ARG_TUP(self);
4135 }
4136 if (! pid) return -1;
4137 }
4138
4139 PDATA_PUSH(self->stack, pid, -1);
4140 return 0;
4141 }
4142 else {
4143 PyErr_SetString(UnpicklingError,
4144 "A load persistent id instruction was encountered,\n"
4145 "but no persistent_load function was specified.");
4146 return -1;
4147 }
4148 }
4149
4150
4151 static int
load_pop(Unpicklerobject * self)4152 load_pop(Unpicklerobject *self)
4153 {
4154 Py_ssize_t len = self->stack->length;
4155
4156 /* Note that we split the (pickle.py) stack into two stacks,
4157 an object stack and a mark stack. We have to be clever and
4158 pop the right one. We do this by looking at the top of the
4159 mark stack first, and only signalling a stack underflow if
4160 the object stack is empty and the mark stack doesn't match
4161 our expectations.
4162 */
4163 if (self->num_marks > 0 && self->marks[self->num_marks - 1] == len) {
4164 self->num_marks--;
4165 } else if (len > 0) {
4166 len--;
4167 Py_DECREF(self->stack->data[len]);
4168 self->stack->length = len;
4169 } else {
4170 return stackUnderflow();
4171 }
4172 return 0;
4173 }
4174
4175
4176 static int
load_pop_mark(Unpicklerobject * self)4177 load_pop_mark(Unpicklerobject *self)
4178 {
4179 Py_ssize_t i;
4180
4181 if ((i = marker(self)) < 0)
4182 return -1;
4183
4184 Pdata_clear(self->stack, i);
4185
4186 return 0;
4187 }
4188
4189
4190 static int
load_dup(Unpicklerobject * self)4191 load_dup(Unpicklerobject *self)
4192 {
4193 PyObject *last;
4194 Py_ssize_t len;
4195
4196 if ((len = self->stack->length) <= 0) return stackUnderflow();
4197 last=self->stack->data[len-1];
4198 Py_INCREF(last);
4199 PDATA_PUSH(self->stack, last, -1);
4200 return 0;
4201 }
4202
4203
4204 static int
load_get(Unpicklerobject * self)4205 load_get(Unpicklerobject *self)
4206 {
4207 PyObject *py_str = 0, *value = 0;
4208 Py_ssize_t len;
4209 char *s;
4210 int rc;
4211
4212 if ((len = self->readline_func(self, &s)) < 0) return -1;
4213 if (len < 2) return bad_readline();
4214
4215 if (!( py_str = PyString_FromStringAndSize(s, len - 1))) return -1;
4216
4217 value = PyDict_GetItem(self->memo, py_str);
4218 if (! value) {
4219 PyErr_SetObject(BadPickleGet, py_str);
4220 rc = -1;
4221 }
4222 else {
4223 PDATA_APPEND(self->stack, value, -1);
4224 rc = 0;
4225 }
4226
4227 Py_DECREF(py_str);
4228 return rc;
4229 }
4230
4231
4232 static int
load_binget(Unpicklerobject * self)4233 load_binget(Unpicklerobject *self)
4234 {
4235 PyObject *py_key = 0, *value = 0;
4236 unsigned char key;
4237 char *s;
4238 int rc;
4239
4240 if (self->read_func(self, &s, 1) < 0) return -1;
4241
4242 key = (unsigned char)s[0];
4243 if (!( py_key = PyInt_FromLong((long)key))) return -1;
4244
4245 value = PyDict_GetItem(self->memo, py_key);
4246 if (! value) {
4247 PyErr_SetObject(BadPickleGet, py_key);
4248 rc = -1;
4249 }
4250 else {
4251 PDATA_APPEND(self->stack, value, -1);
4252 rc = 0;
4253 }
4254
4255 Py_DECREF(py_key);
4256 return rc;
4257 }
4258
4259
4260 static int
load_long_binget(Unpicklerobject * self)4261 load_long_binget(Unpicklerobject *self)
4262 {
4263 PyObject *py_key = 0, *value = 0;
4264 unsigned char c;
4265 char *s;
4266 Py_ssize_t key;
4267 int rc;
4268
4269 if (self->read_func(self, &s, 4) < 0) return -1;
4270
4271 c = (unsigned char)s[0];
4272 key = (long)c;
4273 c = (unsigned char)s[1];
4274 key |= (long)c << 8;
4275 c = (unsigned char)s[2];
4276 key |= (long)c << 16;
4277 c = (unsigned char)s[3];
4278 key |= (long)c << 24;
4279
4280 if (!( py_key = PyInt_FromLong((long)key))) return -1;
4281
4282 value = PyDict_GetItem(self->memo, py_key);
4283 if (! value) {
4284 PyErr_SetObject(BadPickleGet, py_key);
4285 rc = -1;
4286 }
4287 else {
4288 PDATA_APPEND(self->stack, value, -1);
4289 rc = 0;
4290 }
4291
4292 Py_DECREF(py_key);
4293 return rc;
4294 }
4295
4296 /* Push an object from the extension registry (EXT[124]). nbytes is
4297 * the number of bytes following the opcode, holding the index (code) value.
4298 */
4299 static int
load_extension(Unpicklerobject * self,int nbytes)4300 load_extension(Unpicklerobject *self, int nbytes)
4301 {
4302 char *codebytes; /* the nbytes bytes after the opcode */
4303 long code; /* calc_binint returns long */
4304 PyObject *py_code; /* code as a Python int */
4305 PyObject *obj; /* the object to push */
4306 PyObject *pair; /* (module_name, class_name) */
4307 PyObject *module_name, *class_name;
4308
4309 assert(nbytes == 1 || nbytes == 2 || nbytes == 4);
4310 if (self->read_func(self, &codebytes, nbytes) < 0) return -1;
4311 code = calc_binint(codebytes, nbytes);
4312 if (code <= 0) { /* note that 0 is forbidden */
4313 /* Corrupt or hostile pickle. */
4314 PyErr_SetString(UnpicklingError, "EXT specifies code <= 0");
4315 return -1;
4316 }
4317
4318 /* Look for the code in the cache. */
4319 py_code = PyInt_FromLong(code);
4320 if (py_code == NULL) return -1;
4321 obj = PyDict_GetItem(extension_cache, py_code);
4322 if (obj != NULL) {
4323 /* Bingo. */
4324 Py_DECREF(py_code);
4325 PDATA_APPEND(self->stack, obj, -1);
4326 return 0;
4327 }
4328
4329 /* Look up the (module_name, class_name) pair. */
4330 pair = PyDict_GetItem(inverted_registry, py_code);
4331 if (pair == NULL) {
4332 Py_DECREF(py_code);
4333 PyErr_Format(PyExc_ValueError, "unregistered extension "
4334 "code %ld", code);
4335 return -1;
4336 }
4337 /* Since the extension registry is manipulable via Python code,
4338 * confirm that pair is really a 2-tuple of strings.
4339 */
4340 if (!PyTuple_Check(pair) || PyTuple_Size(pair) != 2 ||
4341 !PyString_Check(module_name = PyTuple_GET_ITEM(pair, 0)) ||
4342 !PyString_Check(class_name = PyTuple_GET_ITEM(pair, 1))) {
4343 Py_DECREF(py_code);
4344 PyErr_Format(PyExc_ValueError, "_inverted_registry[%ld] "
4345 "isn't a 2-tuple of strings", code);
4346 return -1;
4347 }
4348 /* Load the object. */
4349 obj = find_class(module_name, class_name, self->find_class);
4350 if (obj == NULL) {
4351 Py_DECREF(py_code);
4352 return -1;
4353 }
4354 /* Cache code -> obj. */
4355 code = PyDict_SetItem(extension_cache, py_code, obj);
4356 Py_DECREF(py_code);
4357 if (code < 0) {
4358 Py_DECREF(obj);
4359 return -1;
4360 }
4361 PDATA_PUSH(self->stack, obj, -1);
4362 return 0;
4363 }
4364
4365 static int
load_put(Unpicklerobject * self)4366 load_put(Unpicklerobject *self)
4367 {
4368 PyObject *py_str = 0, *value = 0;
4369 Py_ssize_t len, l;
4370 char *s;
4371
4372 if ((l = self->readline_func(self, &s)) < 0) return -1;
4373 if (l < 2) return bad_readline();
4374 if (!( len=self->stack->length )) return stackUnderflow();
4375 if (!( py_str = PyString_FromStringAndSize(s, l - 1))) return -1;
4376 value=self->stack->data[len-1];
4377 l=PyDict_SetItem(self->memo, py_str, value);
4378 Py_DECREF(py_str);
4379 return l;
4380 }
4381
4382
4383 static int
load_binput(Unpicklerobject * self)4384 load_binput(Unpicklerobject *self)
4385 {
4386 PyObject *py_key = 0, *value = 0;
4387 unsigned char key;
4388 char *s;
4389 Py_ssize_t len;
4390
4391 if (self->read_func(self, &s, 1) < 0) return -1;
4392 if (!( (len=self->stack->length) > 0 )) return stackUnderflow();
4393
4394 key = (unsigned char)s[0];
4395
4396 if (!( py_key = PyInt_FromLong((long)key))) return -1;
4397 value=self->stack->data[len-1];
4398 len=PyDict_SetItem(self->memo, py_key, value);
4399 Py_DECREF(py_key);
4400 return len;
4401 }
4402
4403
4404 static int
load_long_binput(Unpicklerobject * self)4405 load_long_binput(Unpicklerobject *self)
4406 {
4407 PyObject *py_key = 0, *value = 0;
4408 Py_ssize_t key;
4409 unsigned char c;
4410 char *s;
4411 Py_ssize_t len;
4412
4413 if (self->read_func(self, &s, 4) < 0) return -1;
4414 if (!( len=self->stack->length )) return stackUnderflow();
4415
4416 c = (unsigned char)s[0];
4417 key = (long)c;
4418 c = (unsigned char)s[1];
4419 key |= (long)c << 8;
4420 c = (unsigned char)s[2];
4421 key |= (long)c << 16;
4422 c = (unsigned char)s[3];
4423 key |= (long)c << 24;
4424
4425 if (!( py_key = PyInt_FromLong(key))) return -1;
4426 value=self->stack->data[len-1];
4427 len=PyDict_SetItem(self->memo, py_key, value);
4428 Py_DECREF(py_key);
4429 return len;
4430 }
4431
4432
4433 static int
do_append(Unpicklerobject * self,Py_ssize_t x)4434 do_append(Unpicklerobject *self, Py_ssize_t x)
4435 {
4436 PyObject *value = 0, *list = 0, *append_method = 0;
4437 Py_ssize_t len, i;
4438
4439 len=self->stack->length;
4440 if (!( len >= x && x > 0 )) return stackUnderflow();
4441 /* nothing to do */
4442 if (len==x) return 0;
4443
4444 list=self->stack->data[x-1];
4445
4446 if (PyList_Check(list)) {
4447 PyObject *slice;
4448 int list_len;
4449
4450 slice=Pdata_popList(self->stack, x);
4451 if (! slice) return -1;
4452 list_len = PyList_GET_SIZE(list);
4453 i=PyList_SetSlice(list, list_len, list_len, slice);
4454 Py_DECREF(slice);
4455 return i;
4456 }
4457 else {
4458
4459 if (!( append_method = PyObject_GetAttr(list, append_str)))
4460 return -1;
4461
4462 for (i = x; i < len; i++) {
4463 PyObject *junk;
4464
4465 value=self->stack->data[i];
4466 junk=0;
4467 ARG_TUP(self, value);
4468 if (self->arg) {
4469 junk = PyObject_Call(append_method, self->arg,
4470 NULL);
4471 FREE_ARG_TUP(self);
4472 }
4473 if (! junk) {
4474 Pdata_clear(self->stack, i+1);
4475 self->stack->length=x;
4476 Py_DECREF(append_method);
4477 return -1;
4478 }
4479 Py_DECREF(junk);
4480 }
4481 self->stack->length=x;
4482 Py_DECREF(append_method);
4483 }
4484
4485 return 0;
4486 }
4487
4488
4489 static int
load_append(Unpicklerobject * self)4490 load_append(Unpicklerobject *self)
4491 {
4492 if (self->stack->length - 1 <= 0)
4493 return stackUnderflow();
4494 return do_append(self, self->stack->length - 1);
4495 }
4496
4497
4498 static int
load_appends(Unpicklerobject * self)4499 load_appends(Unpicklerobject *self)
4500 {
4501 Py_ssize_t i = marker(self);
4502 if (i < 0)
4503 return -1;
4504 return do_append(self, i);
4505 }
4506
4507
4508 static Py_ssize_t
do_setitems(Unpicklerobject * self,Py_ssize_t x)4509 do_setitems(Unpicklerobject *self, Py_ssize_t x)
4510 {
4511 PyObject *value = 0, *key = 0, *dict = 0;
4512 Py_ssize_t len, i, r=0;
4513
4514 if (!( (len=self->stack->length) >= x
4515 && x > 0 )) return stackUnderflow();
4516 if (len == x) /* nothing to do */
4517 return 0;
4518 if ((len - x) % 2 != 0) {
4519 /* Currupt or hostile pickle -- we never write one like this. */
4520 PyErr_SetString(UnpicklingError,
4521 "odd number of items for SETITEMS");
4522 return -1;
4523 }
4524
4525 dict=self->stack->data[x-1];
4526
4527 for (i = x+1; i < len; i += 2) {
4528 key =self->stack->data[i-1];
4529 value=self->stack->data[i ];
4530 if (PyObject_SetItem(dict, key, value) < 0) {
4531 r=-1;
4532 break;
4533 }
4534 }
4535
4536 Pdata_clear(self->stack, x);
4537
4538 return r;
4539 }
4540
4541
4542 static int
load_setitem(Unpicklerobject * self)4543 load_setitem(Unpicklerobject *self)
4544 {
4545 return do_setitems(self, self->stack->length - 2);
4546 }
4547
4548 static int
load_setitems(Unpicklerobject * self)4549 load_setitems(Unpicklerobject *self)
4550 {
4551 Py_ssize_t i = marker(self);
4552 if (i < 0)
4553 return -1;
4554 return do_setitems(self, i);
4555 }
4556
4557
4558 static int
load_build(Unpicklerobject * self)4559 load_build(Unpicklerobject *self)
4560 {
4561 PyObject *state, *inst, *slotstate;
4562 PyObject *__setstate__;
4563 PyObject *d_key, *d_value;
4564 int res = -1;
4565 Py_ssize_t i;
4566
4567 /* Stack is ... instance, state. We want to leave instance at
4568 * the stack top, possibly mutated via instance.__setstate__(state).
4569 */
4570 if (self->stack->length < 2)
4571 return stackUnderflow();
4572 PDATA_POP(self->stack, state);
4573 if (state == NULL)
4574 return -1;
4575 inst = self->stack->data[self->stack->length - 1];
4576
4577 __setstate__ = PyObject_GetAttr(inst, __setstate___str);
4578 if (__setstate__ != NULL) {
4579 PyObject *junk = NULL;
4580
4581 /* The explicit __setstate__ is responsible for everything. */
4582 ARG_TUP(self, state);
4583 if (self->arg) {
4584 junk = PyObject_Call(__setstate__, self->arg, NULL);
4585 FREE_ARG_TUP(self);
4586 }
4587 Py_DECREF(__setstate__);
4588 if (junk == NULL)
4589 return -1;
4590 Py_DECREF(junk);
4591 return 0;
4592 }
4593 if (!PyErr_ExceptionMatches(PyExc_AttributeError))
4594 return -1;
4595 PyErr_Clear();
4596
4597 /* A default __setstate__. First see whether state embeds a
4598 * slot state dict too (a proto 2 addition).
4599 */
4600 if (PyTuple_Check(state) && PyTuple_Size(state) == 2) {
4601 PyObject *temp = state;
4602 state = PyTuple_GET_ITEM(temp, 0);
4603 slotstate = PyTuple_GET_ITEM(temp, 1);
4604 Py_INCREF(state);
4605 Py_INCREF(slotstate);
4606 Py_DECREF(temp);
4607 }
4608 else
4609 slotstate = NULL;
4610
4611 /* Set inst.__dict__ from the state dict (if any). */
4612 if (state != Py_None) {
4613 PyObject *dict;
4614 if (! PyDict_Check(state)) {
4615 PyErr_SetString(UnpicklingError, "state is not a "
4616 "dictionary");
4617 goto finally;
4618 }
4619 dict = PyObject_GetAttr(inst, __dict___str);
4620 if (dict == NULL)
4621 goto finally;
4622
4623 i = 0;
4624 while (PyDict_Next(state, &i, &d_key, &d_value)) {
4625 /* normally the keys for instance attributes are
4626 interned. we should try to do that here. */
4627 Py_INCREF(d_key);
4628 if (PyString_CheckExact(d_key))
4629 PyString_InternInPlace(&d_key);
4630 if (PyObject_SetItem(dict, d_key, d_value) < 0) {
4631 Py_DECREF(d_key);
4632 goto finally;
4633 }
4634 Py_DECREF(d_key);
4635 }
4636 Py_DECREF(dict);
4637 }
4638
4639 /* Also set instance attributes from the slotstate dict (if any). */
4640 if (slotstate != NULL) {
4641 if (! PyDict_Check(slotstate)) {
4642 PyErr_SetString(UnpicklingError, "slot state is not "
4643 "a dictionary");
4644 goto finally;
4645 }
4646 i = 0;
4647 while (PyDict_Next(slotstate, &i, &d_key, &d_value)) {
4648 if (PyObject_SetAttr(inst, d_key, d_value) < 0)
4649 goto finally;
4650 }
4651 }
4652 res = 0;
4653
4654 finally:
4655 Py_DECREF(state);
4656 Py_XDECREF(slotstate);
4657 return res;
4658 }
4659
4660
4661 static int
load_mark(Unpicklerobject * self)4662 load_mark(Unpicklerobject *self)
4663 {
4664 Py_ssize_t s;
4665
4666 /* Note that we split the (pickle.py) stack into two stacks, an
4667 object stack and a mark stack. Here we push a mark onto the
4668 mark stack.
4669 */
4670
4671 if ((self->num_marks + 1) >= self->marks_size) {
4672 Py_ssize_t *marks;
4673 s=self->marks_size+20;
4674 if (s <= self->num_marks) s=self->num_marks + 1;
4675 if (self->marks == NULL)
4676 marks=(Py_ssize_t *)malloc(s * sizeof(Py_ssize_t));
4677 else
4678 marks=(Py_ssize_t *)realloc(self->marks,
4679 s * sizeof(Py_ssize_t));
4680 if (!marks) {
4681 PyErr_NoMemory();
4682 return -1;
4683 }
4684 self->marks = marks;
4685 self->marks_size = s;
4686 }
4687
4688 self->marks[self->num_marks++] = self->stack->length;
4689
4690 return 0;
4691 }
4692
4693 static int
load_reduce(Unpicklerobject * self)4694 load_reduce(Unpicklerobject *self)
4695 {
4696 PyObject *callable = 0, *arg_tup = 0, *ob = 0;
4697
4698 PDATA_POP(self->stack, arg_tup);
4699 if (! arg_tup) return -1;
4700 PDATA_POP(self->stack, callable);
4701 if (callable) {
4702 ob = Instance_New(callable, arg_tup);
4703 Py_DECREF(callable);
4704 }
4705 Py_DECREF(arg_tup);
4706
4707 if (! ob) return -1;
4708
4709 PDATA_PUSH(self->stack, ob, -1);
4710 return 0;
4711 }
4712
4713 /* Just raises an error if we don't know the protocol specified. PROTO
4714 * is the first opcode for protocols >= 2.
4715 */
4716 static int
load_proto(Unpicklerobject * self)4717 load_proto(Unpicklerobject *self)
4718 {
4719 int i;
4720 char *protobyte;
4721
4722 i = self->read_func(self, &protobyte, 1);
4723 if (i < 0)
4724 return -1;
4725
4726 i = calc_binint(protobyte, 1);
4727 /* No point checking for < 0, since calc_binint returns an unsigned
4728 * int when chewing on 1 byte.
4729 */
4730 assert(i >= 0);
4731 if (i <= HIGHEST_PROTOCOL)
4732 return 0;
4733
4734 PyErr_Format(PyExc_ValueError, "unsupported pickle protocol: %d", i);
4735 return -1;
4736 }
4737
4738 static PyObject *
load(Unpicklerobject * self)4739 load(Unpicklerobject *self)
4740 {
4741 PyObject *err = 0, *val = 0;
4742 char *s;
4743
4744 self->num_marks = 0;
4745 if (self->stack->length) Pdata_clear(self->stack, 0);
4746
4747 while (1) {
4748 if (self->read_func(self, &s, 1) < 0)
4749 break;
4750
4751 switch (s[0]) {
4752 case NONE:
4753 if (load_none(self) < 0)
4754 break;
4755 continue;
4756
4757 case BININT:
4758 if (load_binint(self) < 0)
4759 break;
4760 continue;
4761
4762 case BININT1:
4763 if (load_binint1(self) < 0)
4764 break;
4765 continue;
4766
4767 case BININT2:
4768 if (load_binint2(self) < 0)
4769 break;
4770 continue;
4771
4772 case INT:
4773 if (load_int(self) < 0)
4774 break;
4775 continue;
4776
4777 case LONG:
4778 if (load_long(self) < 0)
4779 break;
4780 continue;
4781
4782 case LONG1:
4783 if (load_counted_long(self, 1) < 0)
4784 break;
4785 continue;
4786
4787 case LONG4:
4788 if (load_counted_long(self, 4) < 0)
4789 break;
4790 continue;
4791
4792 case FLOAT:
4793 if (load_float(self) < 0)
4794 break;
4795 continue;
4796
4797 case BINFLOAT:
4798 if (load_binfloat(self) < 0)
4799 break;
4800 continue;
4801
4802 case BINSTRING:
4803 if (load_binstring(self) < 0)
4804 break;
4805 continue;
4806
4807 case SHORT_BINSTRING:
4808 if (load_short_binstring(self) < 0)
4809 break;
4810 continue;
4811
4812 case STRING:
4813 if (load_string(self) < 0)
4814 break;
4815 continue;
4816
4817 #ifdef Py_USING_UNICODE
4818 case UNICODE:
4819 if (load_unicode(self) < 0)
4820 break;
4821 continue;
4822
4823 case BINUNICODE:
4824 if (load_binunicode(self) < 0)
4825 break;
4826 continue;
4827 #endif
4828
4829 case EMPTY_TUPLE:
4830 if (load_counted_tuple(self, 0) < 0)
4831 break;
4832 continue;
4833
4834 case TUPLE1:
4835 if (load_counted_tuple(self, 1) < 0)
4836 break;
4837 continue;
4838
4839 case TUPLE2:
4840 if (load_counted_tuple(self, 2) < 0)
4841 break;
4842 continue;
4843
4844 case TUPLE3:
4845 if (load_counted_tuple(self, 3) < 0)
4846 break;
4847 continue;
4848
4849 case TUPLE:
4850 if (load_tuple(self) < 0)
4851 break;
4852 continue;
4853
4854 case EMPTY_LIST:
4855 if (load_empty_list(self) < 0)
4856 break;
4857 continue;
4858
4859 case LIST:
4860 if (load_list(self) < 0)
4861 break;
4862 continue;
4863
4864 case EMPTY_DICT:
4865 if (load_empty_dict(self) < 0)
4866 break;
4867 continue;
4868
4869 case DICT:
4870 if (load_dict(self) < 0)
4871 break;
4872 continue;
4873
4874 case OBJ:
4875 if (load_obj(self) < 0)
4876 break;
4877 continue;
4878
4879 case INST:
4880 if (load_inst(self) < 0)
4881 break;
4882 continue;
4883
4884 case NEWOBJ:
4885 if (load_newobj(self) < 0)
4886 break;
4887 continue;
4888
4889 case GLOBAL:
4890 if (load_global(self) < 0)
4891 break;
4892 continue;
4893
4894 case APPEND:
4895 if (load_append(self) < 0)
4896 break;
4897 continue;
4898
4899 case APPENDS:
4900 if (load_appends(self) < 0)
4901 break;
4902 continue;
4903
4904 case BUILD:
4905 if (load_build(self) < 0)
4906 break;
4907 continue;
4908
4909 case DUP:
4910 if (load_dup(self) < 0)
4911 break;
4912 continue;
4913
4914 case BINGET:
4915 if (load_binget(self) < 0)
4916 break;
4917 continue;
4918
4919 case LONG_BINGET:
4920 if (load_long_binget(self) < 0)
4921 break;
4922 continue;
4923
4924 case GET:
4925 if (load_get(self) < 0)
4926 break;
4927 continue;
4928
4929 case EXT1:
4930 if (load_extension(self, 1) < 0)
4931 break;
4932 continue;
4933
4934 case EXT2:
4935 if (load_extension(self, 2) < 0)
4936 break;
4937 continue;
4938
4939 case EXT4:
4940 if (load_extension(self, 4) < 0)
4941 break;
4942 continue;
4943 case MARK:
4944 if (load_mark(self) < 0)
4945 break;
4946 continue;
4947
4948 case BINPUT:
4949 if (load_binput(self) < 0)
4950 break;
4951 continue;
4952
4953 case LONG_BINPUT:
4954 if (load_long_binput(self) < 0)
4955 break;
4956 continue;
4957
4958 case PUT:
4959 if (load_put(self) < 0)
4960 break;
4961 continue;
4962
4963 case POP:
4964 if (load_pop(self) < 0)
4965 break;
4966 continue;
4967
4968 case POP_MARK:
4969 if (load_pop_mark(self) < 0)
4970 break;
4971 continue;
4972
4973 case SETITEM:
4974 if (load_setitem(self) < 0)
4975 break;
4976 continue;
4977
4978 case SETITEMS:
4979 if (load_setitems(self) < 0)
4980 break;
4981 continue;
4982
4983 case STOP:
4984 break;
4985
4986 case PERSID:
4987 if (load_persid(self) < 0)
4988 break;
4989 continue;
4990
4991 case BINPERSID:
4992 if (load_binpersid(self) < 0)
4993 break;
4994 continue;
4995
4996 case REDUCE:
4997 if (load_reduce(self) < 0)
4998 break;
4999 continue;
5000
5001 case PROTO:
5002 if (load_proto(self) < 0)
5003 break;
5004 continue;
5005
5006 case NEWTRUE:
5007 if (load_bool(self, Py_True) < 0)
5008 break;
5009 continue;
5010
5011 case NEWFALSE:
5012 if (load_bool(self, Py_False) < 0)
5013 break;
5014 continue;
5015
5016 case '\0':
5017 /* end of file */
5018 PyErr_SetNone(PyExc_EOFError);
5019 break;
5020
5021 default:
5022 cPickle_ErrFormat(UnpicklingError,
5023 "invalid load key, '%s'.",
5024 "c", s[0]);
5025 return NULL;
5026 }
5027
5028 break;
5029 }
5030
5031 if ((err = PyErr_Occurred())) {
5032 if (err == PyExc_EOFError) {
5033 PyErr_SetNone(PyExc_EOFError);
5034 }
5035 return NULL;
5036 }
5037
5038 PDATA_POP(self->stack, val);
5039 return val;
5040 }
5041
5042
5043 /* No-load functions to support noload, which is used to
5044 find persistent references. */
5045
5046 static int
noload_obj(Unpicklerobject * self)5047 noload_obj(Unpicklerobject *self)
5048 {
5049 Py_ssize_t i;
5050
5051 if ((i = marker(self)) < 0) return -1;
5052 return Pdata_clear(self->stack, i+1);
5053 }
5054
5055
5056 static int
noload_inst(Unpicklerobject * self)5057 noload_inst(Unpicklerobject *self)
5058 {
5059 Py_ssize_t i;
5060 char *s;
5061
5062 if ((i = marker(self)) < 0) return -1;
5063 Pdata_clear(self->stack, i);
5064 if (self->readline_func(self, &s) < 0) return -1;
5065 if (self->readline_func(self, &s) < 0) return -1;
5066 PDATA_APPEND(self->stack, Py_None, -1);
5067 return 0;
5068 }
5069
5070 static int
noload_newobj(Unpicklerobject * self)5071 noload_newobj(Unpicklerobject *self)
5072 {
5073 PyObject *obj;
5074
5075 PDATA_POP(self->stack, obj); /* pop argtuple */
5076 if (obj == NULL) return -1;
5077 Py_DECREF(obj);
5078
5079 PDATA_POP(self->stack, obj); /* pop cls */
5080 if (obj == NULL) return -1;
5081 Py_DECREF(obj);
5082
5083 PDATA_APPEND(self->stack, Py_None, -1);
5084 return 0;
5085 }
5086
5087 static int
noload_global(Unpicklerobject * self)5088 noload_global(Unpicklerobject *self)
5089 {
5090 char *s;
5091
5092 if (self->readline_func(self, &s) < 0) return -1;
5093 if (self->readline_func(self, &s) < 0) return -1;
5094 PDATA_APPEND(self->stack, Py_None,-1);
5095 return 0;
5096 }
5097
5098 static int
noload_reduce(Unpicklerobject * self)5099 noload_reduce(Unpicklerobject *self)
5100 {
5101
5102 if (self->stack->length < 2) return stackUnderflow();
5103 Pdata_clear(self->stack, self->stack->length-2);
5104 PDATA_APPEND(self->stack, Py_None,-1);
5105 return 0;
5106 }
5107
5108 static int
noload_build(Unpicklerobject * self)5109 noload_build(Unpicklerobject *self) {
5110
5111 if (self->stack->length < 1) return stackUnderflow();
5112 Pdata_clear(self->stack, self->stack->length-1);
5113 return 0;
5114 }
5115
5116 static int
noload_extension(Unpicklerobject * self,int nbytes)5117 noload_extension(Unpicklerobject *self, int nbytes)
5118 {
5119 char *codebytes;
5120
5121 assert(nbytes == 1 || nbytes == 2 || nbytes == 4);
5122 if (self->read_func(self, &codebytes, nbytes) < 0) return -1;
5123 PDATA_APPEND(self->stack, Py_None, -1);
5124 return 0;
5125 }
5126
5127 static int
noload_append(Unpicklerobject * self)5128 noload_append(Unpicklerobject *self)
5129 {
5130 return Pdata_clear(self->stack, self->stack->length - 1);
5131 }
5132
5133 static int
noload_appends(Unpicklerobject * self)5134 noload_appends(Unpicklerobject *self)
5135 {
5136 Py_ssize_t i;
5137 if ((i = marker(self)) < 0) return -1;
5138 return Pdata_clear(self->stack, i);
5139 }
5140
5141 static int
noload_setitem(Unpicklerobject * self)5142 noload_setitem(Unpicklerobject *self)
5143 {
5144 return Pdata_clear(self->stack, self->stack->length - 2);
5145 }
5146
5147 static int
noload_setitems(Unpicklerobject * self)5148 noload_setitems(Unpicklerobject *self)
5149 {
5150 Py_ssize_t i;
5151 if ((i = marker(self)) < 0) return -1;
5152 return Pdata_clear(self->stack, i);
5153 }
5154
5155 static PyObject *
noload(Unpicklerobject * self)5156 noload(Unpicklerobject *self)
5157 {
5158 PyObject *err = 0, *val = 0;
5159 char *s;
5160
5161 self->num_marks = 0;
5162 Pdata_clear(self->stack, 0);
5163
5164 while (1) {
5165 if (self->read_func(self, &s, 1) < 0)
5166 break;
5167
5168 switch (s[0]) {
5169 case NONE:
5170 if (load_none(self) < 0)
5171 break;
5172 continue;
5173
5174 case BININT:
5175 if (load_binint(self) < 0)
5176 break;
5177 continue;
5178
5179 case BININT1:
5180 if (load_binint1(self) < 0)
5181 break;
5182 continue;
5183
5184 case BININT2:
5185 if (load_binint2(self) < 0)
5186 break;
5187 continue;
5188
5189 case INT:
5190 if (load_int(self) < 0)
5191 break;
5192 continue;
5193
5194 case LONG:
5195 if (load_long(self) < 0)
5196 break;
5197 continue;
5198
5199 case LONG1:
5200 if (load_counted_long(self, 1) < 0)
5201 break;
5202 continue;
5203
5204 case LONG4:
5205 if (load_counted_long(self, 4) < 0)
5206 break;
5207 continue;
5208
5209 case FLOAT:
5210 if (load_float(self) < 0)
5211 break;
5212 continue;
5213
5214 case BINFLOAT:
5215 if (load_binfloat(self) < 0)
5216 break;
5217 continue;
5218
5219 case BINSTRING:
5220 if (load_binstring(self) < 0)
5221 break;
5222 continue;
5223
5224 case SHORT_BINSTRING:
5225 if (load_short_binstring(self) < 0)
5226 break;
5227 continue;
5228
5229 case STRING:
5230 if (load_string(self) < 0)
5231 break;
5232 continue;
5233
5234 #ifdef Py_USING_UNICODE
5235 case UNICODE:
5236 if (load_unicode(self) < 0)
5237 break;
5238 continue;
5239
5240 case BINUNICODE:
5241 if (load_binunicode(self) < 0)
5242 break;
5243 continue;
5244 #endif
5245
5246 case EMPTY_TUPLE:
5247 if (load_counted_tuple(self, 0) < 0)
5248 break;
5249 continue;
5250
5251 case TUPLE1:
5252 if (load_counted_tuple(self, 1) < 0)
5253 break;
5254 continue;
5255
5256 case TUPLE2:
5257 if (load_counted_tuple(self, 2) < 0)
5258 break;
5259 continue;
5260
5261 case TUPLE3:
5262 if (load_counted_tuple(self, 3) < 0)
5263 break;
5264 continue;
5265
5266 case TUPLE:
5267 if (load_tuple(self) < 0)
5268 break;
5269 continue;
5270
5271 case EMPTY_LIST:
5272 if (load_empty_list(self) < 0)
5273 break;
5274 continue;
5275
5276 case LIST:
5277 if (load_list(self) < 0)
5278 break;
5279 continue;
5280
5281 case EMPTY_DICT:
5282 if (load_empty_dict(self) < 0)
5283 break;
5284 continue;
5285
5286 case DICT:
5287 if (load_dict(self) < 0)
5288 break;
5289 continue;
5290
5291 case OBJ:
5292 if (noload_obj(self) < 0)
5293 break;
5294 continue;
5295
5296 case INST:
5297 if (noload_inst(self) < 0)
5298 break;
5299 continue;
5300
5301 case NEWOBJ:
5302 if (noload_newobj(self) < 0)
5303 break;
5304 continue;
5305
5306 case GLOBAL:
5307 if (noload_global(self) < 0)
5308 break;
5309 continue;
5310
5311 case APPEND:
5312 if (noload_append(self) < 0)
5313 break;
5314 continue;
5315
5316 case APPENDS:
5317 if (noload_appends(self) < 0)
5318 break;
5319 continue;
5320
5321 case BUILD:
5322 if (noload_build(self) < 0)
5323 break;
5324 continue;
5325
5326 case DUP:
5327 if (load_dup(self) < 0)
5328 break;
5329 continue;
5330
5331 case BINGET:
5332 if (load_binget(self) < 0)
5333 break;
5334 continue;
5335
5336 case LONG_BINGET:
5337 if (load_long_binget(self) < 0)
5338 break;
5339 continue;
5340
5341 case GET:
5342 if (load_get(self) < 0)
5343 break;
5344 continue;
5345
5346 case EXT1:
5347 if (noload_extension(self, 1) < 0)
5348 break;
5349 continue;
5350
5351 case EXT2:
5352 if (noload_extension(self, 2) < 0)
5353 break;
5354 continue;
5355
5356 case EXT4:
5357 if (noload_extension(self, 4) < 0)
5358 break;
5359 continue;
5360
5361 case MARK:
5362 if (load_mark(self) < 0)
5363 break;
5364 continue;
5365
5366 case BINPUT:
5367 if (load_binput(self) < 0)
5368 break;
5369 continue;
5370
5371 case LONG_BINPUT:
5372 if (load_long_binput(self) < 0)
5373 break;
5374 continue;
5375
5376 case PUT:
5377 if (load_put(self) < 0)
5378 break;
5379 continue;
5380
5381 case POP:
5382 if (load_pop(self) < 0)
5383 break;
5384 continue;
5385
5386 case POP_MARK:
5387 if (load_pop_mark(self) < 0)
5388 break;
5389 continue;
5390
5391 case SETITEM:
5392 if (noload_setitem(self) < 0)
5393 break;
5394 continue;
5395
5396 case SETITEMS:
5397 if (noload_setitems(self) < 0)
5398 break;
5399 continue;
5400
5401 case STOP:
5402 break;
5403
5404 case PERSID:
5405 if (load_persid(self) < 0)
5406 break;
5407 continue;
5408
5409 case BINPERSID:
5410 if (load_binpersid(self) < 0)
5411 break;
5412 continue;
5413
5414 case REDUCE:
5415 if (noload_reduce(self) < 0)
5416 break;
5417 continue;
5418
5419 case PROTO:
5420 if (load_proto(self) < 0)
5421 break;
5422 continue;
5423
5424 case NEWTRUE:
5425 if (load_bool(self, Py_True) < 0)
5426 break;
5427 continue;
5428
5429 case NEWFALSE:
5430 if (load_bool(self, Py_False) < 0)
5431 break;
5432 continue;
5433 default:
5434 cPickle_ErrFormat(UnpicklingError,
5435 "invalid load key, '%s'.",
5436 "c", s[0]);
5437 return NULL;
5438 }
5439
5440 break;
5441 }
5442
5443 if ((err = PyErr_Occurred())) {
5444 if (err == PyExc_EOFError) {
5445 PyErr_SetNone(PyExc_EOFError);
5446 }
5447 return NULL;
5448 }
5449
5450 PDATA_POP(self->stack, val);
5451 return val;
5452 }
5453
5454
5455 static PyObject *
Unpickler_load(Unpicklerobject * self,PyObject * unused)5456 Unpickler_load(Unpicklerobject *self, PyObject *unused)
5457 {
5458 return load(self);
5459 }
5460
5461 static PyObject *
Unpickler_noload(Unpicklerobject * self,PyObject * unused)5462 Unpickler_noload(Unpicklerobject *self, PyObject *unused)
5463 {
5464 return noload(self);
5465 }
5466
5467
5468 static struct PyMethodDef Unpickler_methods[] = {
5469 {"load", (PyCFunction)Unpickler_load, METH_NOARGS,
5470 PyDoc_STR("load() -- Load a pickle")
5471 },
5472 {"noload", (PyCFunction)Unpickler_noload, METH_NOARGS,
5473 PyDoc_STR(
5474 "noload() -- not load a pickle, but go through most of the motions\n"
5475 "\n"
5476 "This function can be used to read past a pickle without instantiating\n"
5477 "any objects or importing any modules. It can also be used to find all\n"
5478 "persistent references without instantiating any objects or importing\n"
5479 "any modules.\n")
5480 },
5481 {NULL, NULL} /* sentinel */
5482 };
5483
5484
5485 static Unpicklerobject *
newUnpicklerobject(PyObject * f)5486 newUnpicklerobject(PyObject *f)
5487 {
5488 Unpicklerobject *self;
5489
5490 if (!( self = PyObject_GC_New(Unpicklerobject, &Unpicklertype)))
5491 return NULL;
5492
5493 self->file = NULL;
5494 self->arg = NULL;
5495 self->stack = (Pdata*)Pdata_New();
5496 self->pers_func = NULL;
5497 self->last_string = NULL;
5498 self->marks = NULL;
5499 self->num_marks = 0;
5500 self->marks_size = 0;
5501 self->buf_size = 0;
5502 self->read = NULL;
5503 self->readline = NULL;
5504 self->find_class = NULL;
5505
5506 if (!( self->memo = PyDict_New()))
5507 goto err;
5508
5509 if (!self->stack)
5510 goto err;
5511
5512 Py_INCREF(f);
5513 self->file = f;
5514
5515 /* Set read, readline based on type of f */
5516 if (PyFile_Check(f)) {
5517 self->fp = PyFile_AsFile(f);
5518 if (self->fp == NULL) {
5519 PyErr_SetString(PyExc_ValueError,
5520 "I/O operation on closed file");
5521 goto err;
5522 }
5523 self->read_func = read_file;
5524 self->readline_func = readline_file;
5525 }
5526 else if (PycStringIO_InputCheck(f)) {
5527 self->fp = NULL;
5528 self->read_func = read_cStringIO;
5529 self->readline_func = readline_cStringIO;
5530 }
5531 else {
5532
5533 self->fp = NULL;
5534 self->read_func = read_other;
5535 self->readline_func = readline_other;
5536
5537 if (!( (self->readline = PyObject_GetAttr(f, readline_str)) &&
5538 (self->read = PyObject_GetAttr(f, read_str)))) {
5539 PyErr_Clear();
5540 PyErr_SetString( PyExc_TypeError,
5541 "argument must have 'read' and "
5542 "'readline' attributes" );
5543 goto err;
5544 }
5545 }
5546 PyObject_GC_Track(self);
5547
5548 return self;
5549
5550 err:
5551 Py_DECREF((PyObject *)self);
5552 return NULL;
5553 }
5554
5555
5556 static PyObject *
get_Unpickler(PyObject * self,PyObject * file)5557 get_Unpickler(PyObject *self, PyObject *file)
5558 {
5559 return (PyObject *)newUnpicklerobject(file);
5560 }
5561
5562
5563 static void
Unpickler_dealloc(Unpicklerobject * self)5564 Unpickler_dealloc(Unpicklerobject *self)
5565 {
5566 PyObject_GC_UnTrack((PyObject *)self);
5567 Py_XDECREF(self->readline);
5568 Py_XDECREF(self->read);
5569 Py_XDECREF(self->file);
5570 Py_XDECREF(self->memo);
5571 Py_XDECREF(self->stack);
5572 Py_XDECREF(self->pers_func);
5573 Py_XDECREF(self->arg);
5574 Py_XDECREF(self->last_string);
5575 Py_XDECREF(self->find_class);
5576
5577 if (self->marks) {
5578 free(self->marks);
5579 }
5580
5581 if (self->buf_size) {
5582 free(self->buf);
5583 }
5584
5585 Py_TYPE(self)->tp_free((PyObject *)self);
5586 }
5587
5588 static int
Unpickler_traverse(Unpicklerobject * self,visitproc visit,void * arg)5589 Unpickler_traverse(Unpicklerobject *self, visitproc visit, void *arg)
5590 {
5591 Py_VISIT(self->readline);
5592 Py_VISIT(self->read);
5593 Py_VISIT(self->file);
5594 Py_VISIT(self->memo);
5595 Py_VISIT(self->stack);
5596 Py_VISIT(self->pers_func);
5597 Py_VISIT(self->arg);
5598 Py_VISIT(self->last_string);
5599 Py_VISIT(self->find_class);
5600 return 0;
5601 }
5602
5603 static int
Unpickler_clear(Unpicklerobject * self)5604 Unpickler_clear(Unpicklerobject *self)
5605 {
5606 Py_CLEAR(self->readline);
5607 Py_CLEAR(self->read);
5608 Py_CLEAR(self->file);
5609 Py_CLEAR(self->memo);
5610 Py_CLEAR(self->stack);
5611 Py_CLEAR(self->pers_func);
5612 Py_CLEAR(self->arg);
5613 Py_CLEAR(self->last_string);
5614 Py_CLEAR(self->find_class);
5615 return 0;
5616 }
5617
5618 static PyObject *
Unpickler_getattr(Unpicklerobject * self,char * name)5619 Unpickler_getattr(Unpicklerobject *self, char *name)
5620 {
5621 if (!strcmp(name, "persistent_load")) {
5622 if (!self->pers_func) {
5623 PyErr_SetString(PyExc_AttributeError, name);
5624 return NULL;
5625 }
5626
5627 Py_INCREF(self->pers_func);
5628 return self->pers_func;
5629 }
5630
5631 if (!strcmp(name, "find_global")) {
5632 if (!self->find_class) {
5633 PyErr_SetString(PyExc_AttributeError, name);
5634 return NULL;
5635 }
5636
5637 Py_INCREF(self->find_class);
5638 return self->find_class;
5639 }
5640
5641 if (!strcmp(name, "memo")) {
5642 if (!self->memo) {
5643 PyErr_SetString(PyExc_AttributeError, name);
5644 return NULL;
5645 }
5646
5647 Py_INCREF(self->memo);
5648 return self->memo;
5649 }
5650
5651 if (!strcmp(name, "UnpicklingError")) {
5652 Py_INCREF(UnpicklingError);
5653 return UnpicklingError;
5654 }
5655
5656 return Py_FindMethod(Unpickler_methods, (PyObject *)self, name);
5657 }
5658
5659
5660 static int
Unpickler_setattr(Unpicklerobject * self,char * name,PyObject * value)5661 Unpickler_setattr(Unpicklerobject *self, char *name, PyObject *value)
5662 {
5663
5664 if (!strcmp(name, "persistent_load")) {
5665 Py_XINCREF(value);
5666 Py_XSETREF(self->pers_func, value);
5667 return 0;
5668 }
5669
5670 if (!strcmp(name, "find_global")) {
5671 Py_XINCREF(value);
5672 Py_XSETREF(self->find_class, value);
5673 return 0;
5674 }
5675
5676 if (! value) {
5677 PyErr_SetString(PyExc_TypeError,
5678 "attribute deletion is not supported");
5679 return -1;
5680 }
5681
5682 if (strcmp(name, "memo") == 0) {
5683 if (!PyDict_Check(value)) {
5684 PyErr_SetString(PyExc_TypeError,
5685 "memo must be a dictionary");
5686 return -1;
5687 }
5688 Py_INCREF(value);
5689 Py_XSETREF(self->memo, value);
5690 return 0;
5691 }
5692
5693 PyErr_SetString(PyExc_AttributeError, name);
5694 return -1;
5695 }
5696
5697 /* ---------------------------------------------------------------------------
5698 * Module-level functions.
5699 */
5700
5701 /* dump(obj, file, protocol=0). */
5702 static PyObject *
cpm_dump(PyObject * self,PyObject * args,PyObject * kwds)5703 cpm_dump(PyObject *self, PyObject *args, PyObject *kwds)
5704 {
5705 static char *kwlist[] = {"obj", "file", "protocol", NULL};
5706 PyObject *ob, *file, *res = NULL;
5707 Picklerobject *pickler = 0;
5708 int proto = 0;
5709
5710 if (!( PyArg_ParseTupleAndKeywords(args, kwds, "OO|i", kwlist,
5711 &ob, &file, &proto)))
5712 goto finally;
5713
5714 if (!( pickler = newPicklerobject(file, proto)))
5715 goto finally;
5716
5717 if (dump(pickler, ob) < 0)
5718 goto finally;
5719
5720 Py_INCREF(Py_None);
5721 res = Py_None;
5722
5723 finally:
5724 Py_XDECREF(pickler);
5725
5726 return res;
5727 }
5728
5729
5730 /* dumps(obj, protocol=0). */
5731 static PyObject *
cpm_dumps(PyObject * self,PyObject * args,PyObject * kwds)5732 cpm_dumps(PyObject *self, PyObject *args, PyObject *kwds)
5733 {
5734 static char *kwlist[] = {"obj", "protocol", NULL};
5735 PyObject *ob, *file = 0, *res = NULL;
5736 Picklerobject *pickler = 0;
5737 int proto = 0;
5738
5739 if (!( PyArg_ParseTupleAndKeywords(args, kwds, "O|i:dumps", kwlist,
5740 &ob, &proto)))
5741 goto finally;
5742
5743 if (!( file = PycStringIO->NewOutput(128)))
5744 goto finally;
5745
5746 if (!( pickler = newPicklerobject(file, proto)))
5747 goto finally;
5748
5749 if (dump(pickler, ob) < 0)
5750 goto finally;
5751
5752 res = PycStringIO->cgetvalue(file);
5753
5754 finally:
5755 Py_XDECREF(pickler);
5756 Py_XDECREF(file);
5757
5758 return res;
5759 }
5760
5761
5762 /* load(fileobj). */
5763 static PyObject *
cpm_load(PyObject * self,PyObject * ob)5764 cpm_load(PyObject *self, PyObject *ob)
5765 {
5766 Unpicklerobject *unpickler = 0;
5767 PyObject *res = NULL;
5768
5769 if (!( unpickler = newUnpicklerobject(ob)))
5770 goto finally;
5771
5772 res = load(unpickler);
5773
5774 finally:
5775 Py_XDECREF(unpickler);
5776
5777 return res;
5778 }
5779
5780
5781 /* loads(string) */
5782 static PyObject *
cpm_loads(PyObject * self,PyObject * args)5783 cpm_loads(PyObject *self, PyObject *args)
5784 {
5785 PyObject *ob, *file = 0, *res = NULL;
5786 Unpicklerobject *unpickler = 0;
5787
5788 if (!( PyArg_ParseTuple(args, "S:loads", &ob)))
5789 goto finally;
5790
5791 if (!( file = PycStringIO->NewInput(ob)))
5792 goto finally;
5793
5794 if (!( unpickler = newUnpicklerobject(file)))
5795 goto finally;
5796
5797 res = load(unpickler);
5798
5799 finally:
5800 Py_XDECREF(file);
5801 Py_XDECREF(unpickler);
5802
5803 return res;
5804 }
5805
5806
5807 PyDoc_STRVAR(Unpicklertype__doc__,
5808 "Objects that know how to unpickle");
5809
5810 static PyTypeObject Unpicklertype = {
5811 PyVarObject_HEAD_INIT(NULL, 0)
5812 "cPickle.Unpickler", /*tp_name*/
5813 sizeof(Unpicklerobject), /*tp_basicsize*/
5814 0,
5815 (destructor)Unpickler_dealloc, /* tp_dealloc */
5816 0, /* tp_print */
5817 (getattrfunc)Unpickler_getattr, /* tp_getattr */
5818 (setattrfunc)Unpickler_setattr, /* tp_setattr */
5819 0, /* tp_compare */
5820 0, /* tp_repr */
5821 0, /* tp_as_number */
5822 0, /* tp_as_sequence */
5823 0, /* tp_as_mapping */
5824 0, /* tp_hash */
5825 0, /* tp_call */
5826 0, /* tp_str */
5827 0, /* tp_getattro */
5828 0, /* tp_setattro */
5829 0, /* tp_as_buffer */
5830 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
5831 Unpicklertype__doc__, /* tp_doc */
5832 (traverseproc)Unpickler_traverse, /* tp_traverse */
5833 (inquiry)Unpickler_clear, /* tp_clear */
5834 };
5835
5836 static struct PyMethodDef cPickle_methods[] = {
5837 {"dump", (PyCFunction)cpm_dump, METH_VARARGS | METH_KEYWORDS,
5838 PyDoc_STR("dump(obj, file, protocol=0) -- "
5839 "Write an object in pickle format to the given file.\n"
5840 "\n"
5841 "See the Pickler docstring for the meaning of optional argument proto.")
5842 },
5843
5844 {"dumps", (PyCFunction)cpm_dumps, METH_VARARGS | METH_KEYWORDS,
5845 PyDoc_STR("dumps(obj, protocol=0) -- "
5846 "Return a string containing an object in pickle format.\n"
5847 "\n"
5848 "See the Pickler docstring for the meaning of optional argument proto.")
5849 },
5850
5851 {"load", (PyCFunction)cpm_load, METH_O,
5852 PyDoc_STR("load(file) -- Load a pickle from the given file")},
5853
5854 {"loads", (PyCFunction)cpm_loads, METH_VARARGS,
5855 PyDoc_STR("loads(string) -- Load a pickle from the given string")},
5856
5857 {"Pickler", (PyCFunction)get_Pickler, METH_VARARGS | METH_KEYWORDS,
5858 PyDoc_STR("Pickler(file, protocol=0) -- Create a pickler.\n"
5859 "\n"
5860 "This takes a file-like object for writing a pickle data stream.\n"
5861 "The optional proto argument tells the pickler to use the given\n"
5862 "protocol; supported protocols are 0, 1, 2. The default\n"
5863 "protocol is 0, to be backwards compatible. (Protocol 0 is the\n"
5864 "only protocol that can be written to a file opened in text\n"
5865 "mode and read back successfully. When using a protocol higher\n"
5866 "than 0, make sure the file is opened in binary mode, both when\n"
5867 "pickling and unpickling.)\n"
5868 "\n"
5869 "Protocol 1 is more efficient than protocol 0; protocol 2 is\n"
5870 "more efficient than protocol 1.\n"
5871 "\n"
5872 "Specifying a negative protocol version selects the highest\n"
5873 "protocol version supported. The higher the protocol used, the\n"
5874 "more recent the version of Python needed to read the pickle\n"
5875 "produced.\n"
5876 "\n"
5877 "The file parameter must have a write() method that accepts a single\n"
5878 "string argument. It can thus be an open file object, a StringIO\n"
5879 "object, or any other custom object that meets this interface.\n")
5880 },
5881
5882 {"Unpickler", (PyCFunction)get_Unpickler, METH_O,
5883 PyDoc_STR("Unpickler(file) -- Create an unpickler.")},
5884
5885 { NULL, NULL }
5886 };
5887
5888 static int
init_stuff(PyObject * module_dict)5889 init_stuff(PyObject *module_dict)
5890 {
5891 PyObject *copyreg, *t, *r;
5892
5893 #define INIT_STR(S) if (!( S ## _str=PyString_InternFromString(#S))) return -1;
5894
5895 if (PyType_Ready(&Unpicklertype) < 0)
5896 return -1;
5897 if (PyType_Ready(&Picklertype) < 0)
5898 return -1;
5899
5900 INIT_STR(__class__);
5901 INIT_STR(__getinitargs__);
5902 INIT_STR(__dict__);
5903 INIT_STR(__getstate__);
5904 INIT_STR(__setstate__);
5905 INIT_STR(__name__);
5906 INIT_STR(__main__);
5907 INIT_STR(__reduce__);
5908 INIT_STR(__reduce_ex__);
5909 INIT_STR(write);
5910 INIT_STR(append);
5911 INIT_STR(read);
5912 INIT_STR(readline);
5913 INIT_STR(dispatch_table);
5914
5915 if (!( copyreg = PyImport_ImportModule("copy_reg")))
5916 return -1;
5917
5918 /* This is special because we want to use a different
5919 one in restricted mode. */
5920 dispatch_table = PyObject_GetAttr(copyreg, dispatch_table_str);
5921 if (!dispatch_table) return -1;
5922
5923 extension_registry = PyObject_GetAttrString(copyreg,
5924 "_extension_registry");
5925 if (!extension_registry) return -1;
5926
5927 inverted_registry = PyObject_GetAttrString(copyreg,
5928 "_inverted_registry");
5929 if (!inverted_registry) return -1;
5930
5931 extension_cache = PyObject_GetAttrString(copyreg,
5932 "_extension_cache");
5933 if (!extension_cache) return -1;
5934
5935 Py_DECREF(copyreg);
5936
5937 if (!(empty_tuple = PyTuple_New(0)))
5938 return -1;
5939
5940 two_tuple = PyTuple_New(2);
5941 if (two_tuple == NULL)
5942 return -1;
5943 /* We use this temp container with no regard to refcounts, or to
5944 * keeping containees alive. Exempt from GC, because we don't
5945 * want anything looking at two_tuple() by magic.
5946 */
5947 PyObject_GC_UnTrack(two_tuple);
5948
5949 /* Ugh */
5950 if (!( t=PyImport_ImportModule("__builtin__"))) return -1;
5951 if (PyDict_SetItemString(module_dict, "__builtins__", t) < 0)
5952 return -1;
5953
5954 if (!( t=PyDict_New())) return -1;
5955 if (!( r=PyRun_String(
5956 "def __str__(self):\n"
5957 " return self.args and ('%s' % self.args[0]) or '(what)'\n",
5958 Py_file_input,
5959 module_dict, t) )) return -1;
5960 Py_DECREF(r);
5961
5962 PickleError = PyErr_NewException("cPickle.PickleError", NULL, t);
5963 if (!PickleError)
5964 return -1;
5965
5966 Py_DECREF(t);
5967
5968 PicklingError = PyErr_NewException("cPickle.PicklingError",
5969 PickleError, NULL);
5970 if (!PicklingError)
5971 return -1;
5972
5973 if (!( t=PyDict_New())) return -1;
5974 if (!( r=PyRun_String(
5975 "def __str__(self):\n"
5976 " a=self.args\n"
5977 " a=a and type(a[0]) or '(what)'\n"
5978 " return 'Cannot pickle %s objects' % a\n"
5979 , Py_file_input,
5980 module_dict, t) )) return -1;
5981 Py_DECREF(r);
5982
5983 if (!( UnpickleableError = PyErr_NewException(
5984 "cPickle.UnpickleableError", PicklingError, t)))
5985 return -1;
5986
5987 Py_DECREF(t);
5988
5989 if (!( UnpicklingError = PyErr_NewException("cPickle.UnpicklingError",
5990 PickleError, NULL)))
5991 return -1;
5992
5993 if (!( BadPickleGet = PyErr_NewException("cPickle.BadPickleGet",
5994 UnpicklingError, NULL)))
5995 return -1;
5996
5997 if (PyDict_SetItemString(module_dict, "PickleError",
5998 PickleError) < 0)
5999 return -1;
6000
6001 if (PyDict_SetItemString(module_dict, "PicklingError",
6002 PicklingError) < 0)
6003 return -1;
6004
6005 if (PyDict_SetItemString(module_dict, "UnpicklingError",
6006 UnpicklingError) < 0)
6007 return -1;
6008
6009 if (PyDict_SetItemString(module_dict, "UnpickleableError",
6010 UnpickleableError) < 0)
6011 return -1;
6012
6013 if (PyDict_SetItemString(module_dict, "BadPickleGet",
6014 BadPickleGet) < 0)
6015 return -1;
6016
6017 PycString_IMPORT;
6018
6019 return 0;
6020 }
6021
6022 #ifndef PyMODINIT_FUNC /* declarations for DLL import/export */
6023 #define PyMODINIT_FUNC void
6024 #endif
6025 PyMODINIT_FUNC
initcPickle(void)6026 initcPickle(void)
6027 {
6028 PyObject *m, *d, *di, *v, *k;
6029 Py_ssize_t i;
6030 char *rev = "1.71"; /* XXX when does this change? */
6031 PyObject *format_version;
6032 PyObject *compatible_formats;
6033
6034 Py_TYPE(&Picklertype) = &PyType_Type;
6035 Py_TYPE(&Unpicklertype) = &PyType_Type;
6036 Py_TYPE(&PdataType) = &PyType_Type;
6037
6038 /* Initialize some pieces. We need to do this before module creation,
6039 * so we're forced to use a temporary dictionary. :(
6040 */
6041 di = PyDict_New();
6042 if (!di) return;
6043 if (init_stuff(di) < 0) return;
6044
6045 /* Create the module and add the functions */
6046 m = Py_InitModule4("cPickle", cPickle_methods,
6047 cPickle_module_documentation,
6048 (PyObject*)NULL,PYTHON_API_VERSION);
6049 if (m == NULL)
6050 return;
6051
6052 /* Add some symbolic constants to the module */
6053 d = PyModule_GetDict(m);
6054 v = PyString_FromString(rev);
6055 PyDict_SetItemString(d, "__version__", v);
6056 Py_XDECREF(v);
6057
6058 /* Copy data from di. Waaa. */
6059 for (i=0; PyDict_Next(di, &i, &k, &v); ) {
6060 if (PyObject_SetItem(d, k, v) < 0) {
6061 Py_DECREF(di);
6062 return;
6063 }
6064 }
6065 Py_DECREF(di);
6066
6067 i = PyModule_AddIntConstant(m, "HIGHEST_PROTOCOL", HIGHEST_PROTOCOL);
6068 if (i < 0)
6069 return;
6070
6071 /* These are purely informational; no code uses them. */
6072 /* File format version we write. */
6073 format_version = PyString_FromString("2.0");
6074 /* Format versions we can read. */
6075 compatible_formats = Py_BuildValue("[sssss]",
6076 "1.0", /* Original protocol 0 */
6077 "1.1", /* Protocol 0 + INST */
6078 "1.2", /* Original protocol 1 */
6079 "1.3", /* Protocol 1 + BINFLOAT */
6080 "2.0"); /* Original protocol 2 */
6081 PyDict_SetItemString(d, "format_version", format_version);
6082 PyDict_SetItemString(d, "compatible_formats", compatible_formats);
6083 Py_XDECREF(format_version);
6084 Py_XDECREF(compatible_formats);
6085 }
6086