• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1import collections
2import copyreg
3import dbm
4import io
5import functools
6import os
7import math
8import pickle
9import pickletools
10import shutil
11import struct
12import sys
13import threading
14import unittest
15import weakref
16from textwrap import dedent
17from http.cookies import SimpleCookie
18
19try:
20    import _testbuffer
21except ImportError:
22    _testbuffer = None
23
24from test import support
25from test.support import (
26    TestFailed, TESTFN, run_with_locale, no_tracing,
27    _2G, _4G, bigmemtest, reap_threads, forget,
28    save_restore_warnings_filters
29    )
30
31from pickle import bytes_types
32
33
34# bpo-41003: Save/restore warnings filters to leave them unchanged.
35# Ignore filters installed by numpy.
36try:
37    with save_restore_warnings_filters():
38        import numpy as np
39except ImportError:
40    np = None
41
42
43requires_32b = unittest.skipUnless(sys.maxsize < 2**32,
44                                   "test is only meaningful on 32-bit builds")
45
46# Tests that try a number of pickle protocols should have a
47#     for proto in protocols:
48# kind of outer loop.
49protocols = range(pickle.HIGHEST_PROTOCOL + 1)
50
51
52# Return True if opcode code appears in the pickle, else False.
53def opcode_in_pickle(code, pickle):
54    for op, dummy, dummy in pickletools.genops(pickle):
55        if op.code == code.decode("latin-1"):
56            return True
57    return False
58
59# Return the number of times opcode code appears in pickle.
60def count_opcode(code, pickle):
61    n = 0
62    for op, dummy, dummy in pickletools.genops(pickle):
63        if op.code == code.decode("latin-1"):
64            n += 1
65    return n
66
67
68class UnseekableIO(io.BytesIO):
69    def peek(self, *args):
70        raise NotImplementedError
71
72    def seekable(self):
73        return False
74
75    def seek(self, *args):
76        raise io.UnsupportedOperation
77
78    def tell(self):
79        raise io.UnsupportedOperation
80
81
82class MinimalIO(object):
83    """
84    A file-like object that doesn't support readinto().
85    """
86    def __init__(self, *args):
87        self._bio = io.BytesIO(*args)
88        self.getvalue = self._bio.getvalue
89        self.read = self._bio.read
90        self.readline = self._bio.readline
91        self.write = self._bio.write
92
93
94# We can't very well test the extension registry without putting known stuff
95# in it, but we have to be careful to restore its original state.  Code
96# should do this:
97#
98#     e = ExtensionSaver(extension_code)
99#     try:
100#         fiddle w/ the extension registry's stuff for extension_code
101#     finally:
102#         e.restore()
103
104class ExtensionSaver:
105    # Remember current registration for code (if any), and remove it (if
106    # there is one).
107    def __init__(self, code):
108        self.code = code
109        if code in copyreg._inverted_registry:
110            self.pair = copyreg._inverted_registry[code]
111            copyreg.remove_extension(self.pair[0], self.pair[1], code)
112        else:
113            self.pair = None
114
115    # Restore previous registration for code.
116    def restore(self):
117        code = self.code
118        curpair = copyreg._inverted_registry.get(code)
119        if curpair is not None:
120            copyreg.remove_extension(curpair[0], curpair[1], code)
121        pair = self.pair
122        if pair is not None:
123            copyreg.add_extension(pair[0], pair[1], code)
124
125class C:
126    def __eq__(self, other):
127        return self.__dict__ == other.__dict__
128
129class D(C):
130    def __init__(self, arg):
131        pass
132
133class E(C):
134    def __getinitargs__(self):
135        return ()
136
137class H(object):
138    pass
139
140# Hashable mutable key
141class K(object):
142    def __init__(self, value):
143        self.value = value
144
145    def __reduce__(self):
146        # Shouldn't support the recursion itself
147        return K, (self.value,)
148
149import __main__
150__main__.C = C
151C.__module__ = "__main__"
152__main__.D = D
153D.__module__ = "__main__"
154__main__.E = E
155E.__module__ = "__main__"
156__main__.H = H
157H.__module__ = "__main__"
158__main__.K = K
159K.__module__ = "__main__"
160
161class myint(int):
162    def __init__(self, x):
163        self.str = str(x)
164
165class initarg(C):
166
167    def __init__(self, a, b):
168        self.a = a
169        self.b = b
170
171    def __getinitargs__(self):
172        return self.a, self.b
173
174class metaclass(type):
175    pass
176
177class use_metaclass(object, metaclass=metaclass):
178    pass
179
180class pickling_metaclass(type):
181    def __eq__(self, other):
182        return (type(self) == type(other) and
183                self.reduce_args == other.reduce_args)
184
185    def __reduce__(self):
186        return (create_dynamic_class, self.reduce_args)
187
188def create_dynamic_class(name, bases):
189    result = pickling_metaclass(name, bases, dict())
190    result.reduce_args = (name, bases)
191    return result
192
193
194class ZeroCopyBytes(bytes):
195    readonly = True
196    c_contiguous = True
197    f_contiguous = True
198    zero_copy_reconstruct = True
199
200    def __reduce_ex__(self, protocol):
201        if protocol >= 5:
202            return type(self)._reconstruct, (pickle.PickleBuffer(self),), None
203        else:
204            return type(self)._reconstruct, (bytes(self),)
205
206    def __repr__(self):
207        return "{}({!r})".format(self.__class__.__name__, bytes(self))
208
209    __str__ = __repr__
210
211    @classmethod
212    def _reconstruct(cls, obj):
213        with memoryview(obj) as m:
214            obj = m.obj
215            if type(obj) is cls:
216                # Zero-copy
217                return obj
218            else:
219                return cls(obj)
220
221
222class ZeroCopyBytearray(bytearray):
223    readonly = False
224    c_contiguous = True
225    f_contiguous = True
226    zero_copy_reconstruct = True
227
228    def __reduce_ex__(self, protocol):
229        if protocol >= 5:
230            return type(self)._reconstruct, (pickle.PickleBuffer(self),), None
231        else:
232            return type(self)._reconstruct, (bytes(self),)
233
234    def __repr__(self):
235        return "{}({!r})".format(self.__class__.__name__, bytes(self))
236
237    __str__ = __repr__
238
239    @classmethod
240    def _reconstruct(cls, obj):
241        with memoryview(obj) as m:
242            obj = m.obj
243            if type(obj) is cls:
244                # Zero-copy
245                return obj
246            else:
247                return cls(obj)
248
249
250if _testbuffer is not None:
251
252    class PicklableNDArray:
253        # A not-really-zero-copy picklable ndarray, as the ndarray()
254        # constructor doesn't allow for it
255
256        zero_copy_reconstruct = False
257
258        def __init__(self, *args, **kwargs):
259            self.array = _testbuffer.ndarray(*args, **kwargs)
260
261        def __getitem__(self, idx):
262            cls = type(self)
263            new = cls.__new__(cls)
264            new.array = self.array[idx]
265            return new
266
267        @property
268        def readonly(self):
269            return self.array.readonly
270
271        @property
272        def c_contiguous(self):
273            return self.array.c_contiguous
274
275        @property
276        def f_contiguous(self):
277            return self.array.f_contiguous
278
279        def __eq__(self, other):
280            if not isinstance(other, PicklableNDArray):
281                return NotImplemented
282            return (other.array.format == self.array.format and
283                    other.array.shape == self.array.shape and
284                    other.array.strides == self.array.strides and
285                    other.array.readonly == self.array.readonly and
286                    other.array.tobytes() == self.array.tobytes())
287
288        def __ne__(self, other):
289            if not isinstance(other, PicklableNDArray):
290                return NotImplemented
291            return not (self == other)
292
293        def __repr__(self):
294            return (f"{type(self)}(shape={self.array.shape},"
295                    f"strides={self.array.strides}, "
296                    f"bytes={self.array.tobytes()})")
297
298        def __reduce_ex__(self, protocol):
299            if not self.array.contiguous:
300                raise NotImplementedError("Reconstructing a non-contiguous "
301                                          "ndarray does not seem possible")
302            ndarray_kwargs = {"shape": self.array.shape,
303                              "strides": self.array.strides,
304                              "format": self.array.format,
305                              "flags": (0 if self.readonly
306                                        else _testbuffer.ND_WRITABLE)}
307            pb = pickle.PickleBuffer(self.array)
308            if protocol >= 5:
309                return (type(self)._reconstruct,
310                        (pb, ndarray_kwargs))
311            else:
312                # Need to serialize the bytes in physical order
313                with pb.raw() as m:
314                    return (type(self)._reconstruct,
315                            (m.tobytes(), ndarray_kwargs))
316
317        @classmethod
318        def _reconstruct(cls, obj, kwargs):
319            with memoryview(obj) as m:
320                # For some reason, ndarray() wants a list of integers...
321                # XXX This only works if format == 'B'
322                items = list(m.tobytes())
323            return cls(items, **kwargs)
324
325
326# DATA0 .. DATA4 are the pickles we expect under the various protocols, for
327# the object returned by create_data().
328
329DATA0 = (
330    b'(lp0\nL0L\naL1L\naF2.0\n'
331    b'ac__builtin__\ncomple'
332    b'x\np1\n(F3.0\nF0.0\ntp2\n'
333    b'Rp3\naL1L\naL-1L\naL255'
334    b'L\naL-255L\naL-256L\naL'
335    b'65535L\naL-65535L\naL-'
336    b'65536L\naL2147483647L'
337    b'\naL-2147483647L\naL-2'
338    b'147483648L\na(Vabc\np4'
339    b'\ng4\nccopy_reg\n_recon'
340    b'structor\np5\n(c__main'
341    b'__\nC\np6\nc__builtin__'
342    b'\nobject\np7\nNtp8\nRp9\n'
343    b'(dp10\nVfoo\np11\nL1L\ns'
344    b'Vbar\np12\nL2L\nsbg9\ntp'
345    b'13\nag13\naL5L\na.'
346)
347
348# Disassembly of DATA0
349DATA0_DIS = """\
350    0: (    MARK
351    1: l        LIST       (MARK at 0)
352    2: p    PUT        0
353    5: L    LONG       0
354    9: a    APPEND
355   10: L    LONG       1
356   14: a    APPEND
357   15: F    FLOAT      2.0
358   20: a    APPEND
359   21: c    GLOBAL     '__builtin__ complex'
360   42: p    PUT        1
361   45: (    MARK
362   46: F        FLOAT      3.0
363   51: F        FLOAT      0.0
364   56: t        TUPLE      (MARK at 45)
365   57: p    PUT        2
366   60: R    REDUCE
367   61: p    PUT        3
368   64: a    APPEND
369   65: L    LONG       1
370   69: a    APPEND
371   70: L    LONG       -1
372   75: a    APPEND
373   76: L    LONG       255
374   82: a    APPEND
375   83: L    LONG       -255
376   90: a    APPEND
377   91: L    LONG       -256
378   98: a    APPEND
379   99: L    LONG       65535
380  107: a    APPEND
381  108: L    LONG       -65535
382  117: a    APPEND
383  118: L    LONG       -65536
384  127: a    APPEND
385  128: L    LONG       2147483647
386  141: a    APPEND
387  142: L    LONG       -2147483647
388  156: a    APPEND
389  157: L    LONG       -2147483648
390  171: a    APPEND
391  172: (    MARK
392  173: V        UNICODE    'abc'
393  178: p        PUT        4
394  181: g        GET        4
395  184: c        GLOBAL     'copy_reg _reconstructor'
396  209: p        PUT        5
397  212: (        MARK
398  213: c            GLOBAL     '__main__ C'
399  225: p            PUT        6
400  228: c            GLOBAL     '__builtin__ object'
401  248: p            PUT        7
402  251: N            NONE
403  252: t            TUPLE      (MARK at 212)
404  253: p        PUT        8
405  256: R        REDUCE
406  257: p        PUT        9
407  260: (        MARK
408  261: d            DICT       (MARK at 260)
409  262: p        PUT        10
410  266: V        UNICODE    'foo'
411  271: p        PUT        11
412  275: L        LONG       1
413  279: s        SETITEM
414  280: V        UNICODE    'bar'
415  285: p        PUT        12
416  289: L        LONG       2
417  293: s        SETITEM
418  294: b        BUILD
419  295: g        GET        9
420  298: t        TUPLE      (MARK at 172)
421  299: p    PUT        13
422  303: a    APPEND
423  304: g    GET        13
424  308: a    APPEND
425  309: L    LONG       5
426  313: a    APPEND
427  314: .    STOP
428highest protocol among opcodes = 0
429"""
430
431DATA1 = (
432    b']q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c__'
433    b'builtin__\ncomplex\nq\x01'
434    b'(G@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00t'
435    b'q\x02Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xffJ'
436    b'\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff\xff'
437    b'\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00ab'
438    b'cq\x04h\x04ccopy_reg\n_reco'
439    b'nstructor\nq\x05(c__main'
440    b'__\nC\nq\x06c__builtin__\n'
441    b'object\nq\x07Ntq\x08Rq\t}q\n('
442    b'X\x03\x00\x00\x00fooq\x0bK\x01X\x03\x00\x00\x00bar'
443    b'q\x0cK\x02ubh\ttq\rh\rK\x05e.'
444)
445
446# Disassembly of DATA1
447DATA1_DIS = """\
448    0: ]    EMPTY_LIST
449    1: q    BINPUT     0
450    3: (    MARK
451    4: K        BININT1    0
452    6: K        BININT1    1
453    8: G        BINFLOAT   2.0
454   17: c        GLOBAL     '__builtin__ complex'
455   38: q        BINPUT     1
456   40: (        MARK
457   41: G            BINFLOAT   3.0
458   50: G            BINFLOAT   0.0
459   59: t            TUPLE      (MARK at 40)
460   60: q        BINPUT     2
461   62: R        REDUCE
462   63: q        BINPUT     3
463   65: K        BININT1    1
464   67: J        BININT     -1
465   72: K        BININT1    255
466   74: J        BININT     -255
467   79: J        BININT     -256
468   84: M        BININT2    65535
469   87: J        BININT     -65535
470   92: J        BININT     -65536
471   97: J        BININT     2147483647
472  102: J        BININT     -2147483647
473  107: J        BININT     -2147483648
474  112: (        MARK
475  113: X            BINUNICODE 'abc'
476  121: q            BINPUT     4
477  123: h            BINGET     4
478  125: c            GLOBAL     'copy_reg _reconstructor'
479  150: q            BINPUT     5
480  152: (            MARK
481  153: c                GLOBAL     '__main__ C'
482  165: q                BINPUT     6
483  167: c                GLOBAL     '__builtin__ object'
484  187: q                BINPUT     7
485  189: N                NONE
486  190: t                TUPLE      (MARK at 152)
487  191: q            BINPUT     8
488  193: R            REDUCE
489  194: q            BINPUT     9
490  196: }            EMPTY_DICT
491  197: q            BINPUT     10
492  199: (            MARK
493  200: X                BINUNICODE 'foo'
494  208: q                BINPUT     11
495  210: K                BININT1    1
496  212: X                BINUNICODE 'bar'
497  220: q                BINPUT     12
498  222: K                BININT1    2
499  224: u                SETITEMS   (MARK at 199)
500  225: b            BUILD
501  226: h            BINGET     9
502  228: t            TUPLE      (MARK at 112)
503  229: q        BINPUT     13
504  231: h        BINGET     13
505  233: K        BININT1    5
506  235: e        APPENDS    (MARK at 3)
507  236: .    STOP
508highest protocol among opcodes = 1
509"""
510
511DATA2 = (
512    b'\x80\x02]q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c'
513    b'__builtin__\ncomplex\n'
514    b'q\x01G@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00'
515    b'\x86q\x02Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xff'
516    b'J\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff'
517    b'\xff\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00a'
518    b'bcq\x04h\x04c__main__\nC\nq\x05'
519    b')\x81q\x06}q\x07(X\x03\x00\x00\x00fooq\x08K\x01'
520    b'X\x03\x00\x00\x00barq\tK\x02ubh\x06tq\nh'
521    b'\nK\x05e.'
522)
523
524# Disassembly of DATA2
525DATA2_DIS = """\
526    0: \x80 PROTO      2
527    2: ]    EMPTY_LIST
528    3: q    BINPUT     0
529    5: (    MARK
530    6: K        BININT1    0
531    8: K        BININT1    1
532   10: G        BINFLOAT   2.0
533   19: c        GLOBAL     '__builtin__ complex'
534   40: q        BINPUT     1
535   42: G        BINFLOAT   3.0
536   51: G        BINFLOAT   0.0
537   60: \x86     TUPLE2
538   61: q        BINPUT     2
539   63: R        REDUCE
540   64: q        BINPUT     3
541   66: K        BININT1    1
542   68: J        BININT     -1
543   73: K        BININT1    255
544   75: J        BININT     -255
545   80: J        BININT     -256
546   85: M        BININT2    65535
547   88: J        BININT     -65535
548   93: J        BININT     -65536
549   98: J        BININT     2147483647
550  103: J        BININT     -2147483647
551  108: J        BININT     -2147483648
552  113: (        MARK
553  114: X            BINUNICODE 'abc'
554  122: q            BINPUT     4
555  124: h            BINGET     4
556  126: c            GLOBAL     '__main__ C'
557  138: q            BINPUT     5
558  140: )            EMPTY_TUPLE
559  141: \x81         NEWOBJ
560  142: q            BINPUT     6
561  144: }            EMPTY_DICT
562  145: q            BINPUT     7
563  147: (            MARK
564  148: X                BINUNICODE 'foo'
565  156: q                BINPUT     8
566  158: K                BININT1    1
567  160: X                BINUNICODE 'bar'
568  168: q                BINPUT     9
569  170: K                BININT1    2
570  172: u                SETITEMS   (MARK at 147)
571  173: b            BUILD
572  174: h            BINGET     6
573  176: t            TUPLE      (MARK at 113)
574  177: q        BINPUT     10
575  179: h        BINGET     10
576  181: K        BININT1    5
577  183: e        APPENDS    (MARK at 5)
578  184: .    STOP
579highest protocol among opcodes = 2
580"""
581
582DATA3 = (
583    b'\x80\x03]q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c'
584    b'builtins\ncomplex\nq\x01G'
585    b'@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00\x86q\x02'
586    b'Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xffJ\x00\xff'
587    b'\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff\xff\xff\x7f'
588    b'J\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00abcq'
589    b'\x04h\x04c__main__\nC\nq\x05)\x81q'
590    b'\x06}q\x07(X\x03\x00\x00\x00barq\x08K\x02X\x03\x00'
591    b'\x00\x00fooq\tK\x01ubh\x06tq\nh\nK\x05'
592    b'e.'
593)
594
595# Disassembly of DATA3
596DATA3_DIS = """\
597    0: \x80 PROTO      3
598    2: ]    EMPTY_LIST
599    3: q    BINPUT     0
600    5: (    MARK
601    6: K        BININT1    0
602    8: K        BININT1    1
603   10: G        BINFLOAT   2.0
604   19: c        GLOBAL     'builtins complex'
605   37: q        BINPUT     1
606   39: G        BINFLOAT   3.0
607   48: G        BINFLOAT   0.0
608   57: \x86     TUPLE2
609   58: q        BINPUT     2
610   60: R        REDUCE
611   61: q        BINPUT     3
612   63: K        BININT1    1
613   65: J        BININT     -1
614   70: K        BININT1    255
615   72: J        BININT     -255
616   77: J        BININT     -256
617   82: M        BININT2    65535
618   85: J        BININT     -65535
619   90: J        BININT     -65536
620   95: J        BININT     2147483647
621  100: J        BININT     -2147483647
622  105: J        BININT     -2147483648
623  110: (        MARK
624  111: X            BINUNICODE 'abc'
625  119: q            BINPUT     4
626  121: h            BINGET     4
627  123: c            GLOBAL     '__main__ C'
628  135: q            BINPUT     5
629  137: )            EMPTY_TUPLE
630  138: \x81         NEWOBJ
631  139: q            BINPUT     6
632  141: }            EMPTY_DICT
633  142: q            BINPUT     7
634  144: (            MARK
635  145: X                BINUNICODE 'bar'
636  153: q                BINPUT     8
637  155: K                BININT1    2
638  157: X                BINUNICODE 'foo'
639  165: q                BINPUT     9
640  167: K                BININT1    1
641  169: u                SETITEMS   (MARK at 144)
642  170: b            BUILD
643  171: h            BINGET     6
644  173: t            TUPLE      (MARK at 110)
645  174: q        BINPUT     10
646  176: h        BINGET     10
647  178: K        BININT1    5
648  180: e        APPENDS    (MARK at 5)
649  181: .    STOP
650highest protocol among opcodes = 2
651"""
652
653DATA4 = (
654    b'\x80\x04\x95\xa8\x00\x00\x00\x00\x00\x00\x00]\x94(K\x00K\x01G@'
655    b'\x00\x00\x00\x00\x00\x00\x00\x8c\x08builtins\x94\x8c\x07'
656    b'complex\x94\x93\x94G@\x08\x00\x00\x00\x00\x00\x00G'
657    b'\x00\x00\x00\x00\x00\x00\x00\x00\x86\x94R\x94K\x01J\xff\xff\xff\xffK'
658    b'\xffJ\x01\xff\xff\xffJ\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ'
659    b'\x00\x00\xff\xffJ\xff\xff\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80('
660    b'\x8c\x03abc\x94h\x06\x8c\x08__main__\x94\x8c'
661    b'\x01C\x94\x93\x94)\x81\x94}\x94(\x8c\x03bar\x94K\x02\x8c'
662    b'\x03foo\x94K\x01ubh\nt\x94h\x0eK\x05e.'
663)
664
665# Disassembly of DATA4
666DATA4_DIS = """\
667    0: \x80 PROTO      4
668    2: \x95 FRAME      168
669   11: ]    EMPTY_LIST
670   12: \x94 MEMOIZE
671   13: (    MARK
672   14: K        BININT1    0
673   16: K        BININT1    1
674   18: G        BINFLOAT   2.0
675   27: \x8c     SHORT_BINUNICODE 'builtins'
676   37: \x94     MEMOIZE
677   38: \x8c     SHORT_BINUNICODE 'complex'
678   47: \x94     MEMOIZE
679   48: \x93     STACK_GLOBAL
680   49: \x94     MEMOIZE
681   50: G        BINFLOAT   3.0
682   59: G        BINFLOAT   0.0
683   68: \x86     TUPLE2
684   69: \x94     MEMOIZE
685   70: R        REDUCE
686   71: \x94     MEMOIZE
687   72: K        BININT1    1
688   74: J        BININT     -1
689   79: K        BININT1    255
690   81: J        BININT     -255
691   86: J        BININT     -256
692   91: M        BININT2    65535
693   94: J        BININT     -65535
694   99: J        BININT     -65536
695  104: J        BININT     2147483647
696  109: J        BININT     -2147483647
697  114: J        BININT     -2147483648
698  119: (        MARK
699  120: \x8c         SHORT_BINUNICODE 'abc'
700  125: \x94         MEMOIZE
701  126: h            BINGET     6
702  128: \x8c         SHORT_BINUNICODE '__main__'
703  138: \x94         MEMOIZE
704  139: \x8c         SHORT_BINUNICODE 'C'
705  142: \x94         MEMOIZE
706  143: \x93         STACK_GLOBAL
707  144: \x94         MEMOIZE
708  145: )            EMPTY_TUPLE
709  146: \x81         NEWOBJ
710  147: \x94         MEMOIZE
711  148: }            EMPTY_DICT
712  149: \x94         MEMOIZE
713  150: (            MARK
714  151: \x8c             SHORT_BINUNICODE 'bar'
715  156: \x94             MEMOIZE
716  157: K                BININT1    2
717  159: \x8c             SHORT_BINUNICODE 'foo'
718  164: \x94             MEMOIZE
719  165: K                BININT1    1
720  167: u                SETITEMS   (MARK at 150)
721  168: b            BUILD
722  169: h            BINGET     10
723  171: t            TUPLE      (MARK at 119)
724  172: \x94     MEMOIZE
725  173: h        BINGET     14
726  175: K        BININT1    5
727  177: e        APPENDS    (MARK at 13)
728  178: .    STOP
729highest protocol among opcodes = 4
730"""
731
732# set([1,2]) pickled from 2.x with protocol 2
733DATA_SET = b'\x80\x02c__builtin__\nset\nq\x00]q\x01(K\x01K\x02e\x85q\x02Rq\x03.'
734
735# xrange(5) pickled from 2.x with protocol 2
736DATA_XRANGE = b'\x80\x02c__builtin__\nxrange\nq\x00K\x00K\x05K\x01\x87q\x01Rq\x02.'
737
738# a SimpleCookie() object pickled from 2.x with protocol 2
739DATA_COOKIE = (b'\x80\x02cCookie\nSimpleCookie\nq\x00)\x81q\x01U\x03key'
740               b'q\x02cCookie\nMorsel\nq\x03)\x81q\x04(U\x07commentq\x05U'
741               b'\x00q\x06U\x06domainq\x07h\x06U\x06secureq\x08h\x06U\x07'
742               b'expiresq\th\x06U\x07max-ageq\nh\x06U\x07versionq\x0bh\x06U'
743               b'\x04pathq\x0ch\x06U\x08httponlyq\rh\x06u}q\x0e(U\x0b'
744               b'coded_valueq\x0fU\x05valueq\x10h\x10h\x10h\x02h\x02ubs}q\x11b.')
745
746# set([3]) pickled from 2.x with protocol 2
747DATA_SET2 = b'\x80\x02c__builtin__\nset\nq\x00]q\x01K\x03a\x85q\x02Rq\x03.'
748
749python2_exceptions_without_args = (
750    ArithmeticError,
751    AssertionError,
752    AttributeError,
753    BaseException,
754    BufferError,
755    BytesWarning,
756    DeprecationWarning,
757    EOFError,
758    EnvironmentError,
759    Exception,
760    FloatingPointError,
761    FutureWarning,
762    GeneratorExit,
763    IOError,
764    ImportError,
765    ImportWarning,
766    IndentationError,
767    IndexError,
768    KeyError,
769    KeyboardInterrupt,
770    LookupError,
771    MemoryError,
772    NameError,
773    NotImplementedError,
774    OSError,
775    OverflowError,
776    PendingDeprecationWarning,
777    ReferenceError,
778    RuntimeError,
779    RuntimeWarning,
780    # StandardError is gone in Python 3, we map it to Exception
781    StopIteration,
782    SyntaxError,
783    SyntaxWarning,
784    SystemError,
785    SystemExit,
786    TabError,
787    TypeError,
788    UnboundLocalError,
789    UnicodeError,
790    UnicodeWarning,
791    UserWarning,
792    ValueError,
793    Warning,
794    ZeroDivisionError,
795)
796
797exception_pickle = b'\x80\x02cexceptions\n?\nq\x00)Rq\x01.'
798
799# UnicodeEncodeError object pickled from 2.x with protocol 2
800DATA_UEERR = (b'\x80\x02cexceptions\nUnicodeEncodeError\n'
801              b'q\x00(U\x05asciiq\x01X\x03\x00\x00\x00fooq\x02K\x00K\x01'
802              b'U\x03badq\x03tq\x04Rq\x05.')
803
804
805def create_data():
806    c = C()
807    c.foo = 1
808    c.bar = 2
809    x = [0, 1, 2.0, 3.0+0j]
810    # Append some integer test cases at cPickle.c's internal size
811    # cutoffs.
812    uint1max = 0xff
813    uint2max = 0xffff
814    int4max = 0x7fffffff
815    x.extend([1, -1,
816              uint1max, -uint1max, -uint1max-1,
817              uint2max, -uint2max, -uint2max-1,
818               int4max,  -int4max,  -int4max-1])
819    y = ('abc', 'abc', c, c)
820    x.append(y)
821    x.append(y)
822    x.append(5)
823    return x
824
825
826class AbstractUnpickleTests(unittest.TestCase):
827    # Subclass must define self.loads.
828
829    _testdata = create_data()
830
831    def assert_is_copy(self, obj, objcopy, msg=None):
832        """Utility method to verify if two objects are copies of each others.
833        """
834        if msg is None:
835            msg = "{!r} is not a copy of {!r}".format(obj, objcopy)
836        self.assertEqual(obj, objcopy, msg=msg)
837        self.assertIs(type(obj), type(objcopy), msg=msg)
838        if hasattr(obj, '__dict__'):
839            self.assertDictEqual(obj.__dict__, objcopy.__dict__, msg=msg)
840            self.assertIsNot(obj.__dict__, objcopy.__dict__, msg=msg)
841        if hasattr(obj, '__slots__'):
842            self.assertListEqual(obj.__slots__, objcopy.__slots__, msg=msg)
843            for slot in obj.__slots__:
844                self.assertEqual(
845                    hasattr(obj, slot), hasattr(objcopy, slot), msg=msg)
846                self.assertEqual(getattr(obj, slot, None),
847                                 getattr(objcopy, slot, None), msg=msg)
848
849    def check_unpickling_error(self, errors, data):
850        with self.subTest(data=data), \
851             self.assertRaises(errors):
852            try:
853                self.loads(data)
854            except BaseException as exc:
855                if support.verbose > 1:
856                    print('%-32r - %s: %s' %
857                          (data, exc.__class__.__name__, exc))
858                raise
859
860    def test_load_from_data0(self):
861        self.assert_is_copy(self._testdata, self.loads(DATA0))
862
863    def test_load_from_data1(self):
864        self.assert_is_copy(self._testdata, self.loads(DATA1))
865
866    def test_load_from_data2(self):
867        self.assert_is_copy(self._testdata, self.loads(DATA2))
868
869    def test_load_from_data3(self):
870        self.assert_is_copy(self._testdata, self.loads(DATA3))
871
872    def test_load_from_data4(self):
873        self.assert_is_copy(self._testdata, self.loads(DATA4))
874
875    def test_load_classic_instance(self):
876        # See issue5180.  Test loading 2.x pickles that
877        # contain an instance of old style class.
878        for X, args in [(C, ()), (D, ('x',)), (E, ())]:
879            xname = X.__name__.encode('ascii')
880            # Protocol 0 (text mode pickle):
881            """
882             0: (    MARK
883             1: i        INST       '__main__ X' (MARK at 0)
884            13: p    PUT        0
885            16: (    MARK
886            17: d        DICT       (MARK at 16)
887            18: p    PUT        1
888            21: b    BUILD
889            22: .    STOP
890            """
891            pickle0 = (b"(i__main__\n"
892                       b"X\n"
893                       b"p0\n"
894                       b"(dp1\nb.").replace(b'X', xname)
895            self.assert_is_copy(X(*args), self.loads(pickle0))
896
897            # Protocol 1 (binary mode pickle)
898            """
899             0: (    MARK
900             1: c        GLOBAL     '__main__ X'
901            13: q        BINPUT     0
902            15: o        OBJ        (MARK at 0)
903            16: q    BINPUT     1
904            18: }    EMPTY_DICT
905            19: q    BINPUT     2
906            21: b    BUILD
907            22: .    STOP
908            """
909            pickle1 = (b'(c__main__\n'
910                       b'X\n'
911                       b'q\x00oq\x01}q\x02b.').replace(b'X', xname)
912            self.assert_is_copy(X(*args), self.loads(pickle1))
913
914            # Protocol 2 (pickle2 = b'\x80\x02' + pickle1)
915            """
916             0: \x80 PROTO      2
917             2: (    MARK
918             3: c        GLOBAL     '__main__ X'
919            15: q        BINPUT     0
920            17: o        OBJ        (MARK at 2)
921            18: q    BINPUT     1
922            20: }    EMPTY_DICT
923            21: q    BINPUT     2
924            23: b    BUILD
925            24: .    STOP
926            """
927            pickle2 = (b'\x80\x02(c__main__\n'
928                       b'X\n'
929                       b'q\x00oq\x01}q\x02b.').replace(b'X', xname)
930            self.assert_is_copy(X(*args), self.loads(pickle2))
931
932    def test_maxint64(self):
933        maxint64 = (1 << 63) - 1
934        data = b'I' + str(maxint64).encode("ascii") + b'\n.'
935        got = self.loads(data)
936        self.assert_is_copy(maxint64, got)
937
938        # Try too with a bogus literal.
939        data = b'I' + str(maxint64).encode("ascii") + b'JUNK\n.'
940        self.check_unpickling_error(ValueError, data)
941
942    def test_unpickle_from_2x(self):
943        # Unpickle non-trivial data from Python 2.x.
944        loaded = self.loads(DATA_SET)
945        self.assertEqual(loaded, set([1, 2]))
946        loaded = self.loads(DATA_XRANGE)
947        self.assertEqual(type(loaded), type(range(0)))
948        self.assertEqual(list(loaded), list(range(5)))
949        loaded = self.loads(DATA_COOKIE)
950        self.assertEqual(type(loaded), SimpleCookie)
951        self.assertEqual(list(loaded.keys()), ["key"])
952        self.assertEqual(loaded["key"].value, "value")
953
954        # Exception objects without arguments pickled from 2.x with protocol 2
955        for exc in python2_exceptions_without_args:
956            data = exception_pickle.replace(b'?', exc.__name__.encode("ascii"))
957            loaded = self.loads(data)
958            self.assertIs(type(loaded), exc)
959
960        # StandardError is mapped to Exception, test that separately
961        loaded = self.loads(exception_pickle.replace(b'?', b'StandardError'))
962        self.assertIs(type(loaded), Exception)
963
964        loaded = self.loads(DATA_UEERR)
965        self.assertIs(type(loaded), UnicodeEncodeError)
966        self.assertEqual(loaded.object, "foo")
967        self.assertEqual(loaded.encoding, "ascii")
968        self.assertEqual(loaded.start, 0)
969        self.assertEqual(loaded.end, 1)
970        self.assertEqual(loaded.reason, "bad")
971
972    def test_load_python2_str_as_bytes(self):
973        # From Python 2: pickle.dumps('a\x00\xa0', protocol=0)
974        self.assertEqual(self.loads(b"S'a\\x00\\xa0'\n.",
975                                    encoding="bytes"), b'a\x00\xa0')
976        # From Python 2: pickle.dumps('a\x00\xa0', protocol=1)
977        self.assertEqual(self.loads(b'U\x03a\x00\xa0.',
978                                    encoding="bytes"), b'a\x00\xa0')
979        # From Python 2: pickle.dumps('a\x00\xa0', protocol=2)
980        self.assertEqual(self.loads(b'\x80\x02U\x03a\x00\xa0.',
981                                    encoding="bytes"), b'a\x00\xa0')
982
983    def test_load_python2_unicode_as_str(self):
984        # From Python 2: pickle.dumps(u'π', protocol=0)
985        self.assertEqual(self.loads(b'V\\u03c0\n.',
986                                    encoding='bytes'), 'π')
987        # From Python 2: pickle.dumps(u'π', protocol=1)
988        self.assertEqual(self.loads(b'X\x02\x00\x00\x00\xcf\x80.',
989                                    encoding="bytes"), 'π')
990        # From Python 2: pickle.dumps(u'π', protocol=2)
991        self.assertEqual(self.loads(b'\x80\x02X\x02\x00\x00\x00\xcf\x80.',
992                                    encoding="bytes"), 'π')
993
994    def test_load_long_python2_str_as_bytes(self):
995        # From Python 2: pickle.dumps('x' * 300, protocol=1)
996        self.assertEqual(self.loads(pickle.BINSTRING +
997                                    struct.pack("<I", 300) +
998                                    b'x' * 300 + pickle.STOP,
999                                    encoding='bytes'), b'x' * 300)
1000
1001    def test_constants(self):
1002        self.assertIsNone(self.loads(b'N.'))
1003        self.assertIs(self.loads(b'\x88.'), True)
1004        self.assertIs(self.loads(b'\x89.'), False)
1005        self.assertIs(self.loads(b'I01\n.'), True)
1006        self.assertIs(self.loads(b'I00\n.'), False)
1007
1008    def test_empty_bytestring(self):
1009        # issue 11286
1010        empty = self.loads(b'\x80\x03U\x00q\x00.', encoding='koi8-r')
1011        self.assertEqual(empty, '')
1012
1013    def test_short_binbytes(self):
1014        dumped = b'\x80\x03C\x04\xe2\x82\xac\x00.'
1015        self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00')
1016
1017    def test_binbytes(self):
1018        dumped = b'\x80\x03B\x04\x00\x00\x00\xe2\x82\xac\x00.'
1019        self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00')
1020
1021    @requires_32b
1022    def test_negative_32b_binbytes(self):
1023        # On 32-bit builds, a BINBYTES of 2**31 or more is refused
1024        dumped = b'\x80\x03B\xff\xff\xff\xffxyzq\x00.'
1025        self.check_unpickling_error((pickle.UnpicklingError, OverflowError),
1026                                    dumped)
1027
1028    @requires_32b
1029    def test_negative_32b_binunicode(self):
1030        # On 32-bit builds, a BINUNICODE of 2**31 or more is refused
1031        dumped = b'\x80\x03X\xff\xff\xff\xffxyzq\x00.'
1032        self.check_unpickling_error((pickle.UnpicklingError, OverflowError),
1033                                    dumped)
1034
1035    def test_short_binunicode(self):
1036        dumped = b'\x80\x04\x8c\x04\xe2\x82\xac\x00.'
1037        self.assertEqual(self.loads(dumped), '\u20ac\x00')
1038
1039    def test_misc_get(self):
1040        self.check_unpickling_error(KeyError, b'g0\np0')
1041        self.assert_is_copy([(100,), (100,)],
1042                            self.loads(b'((Kdtp0\nh\x00l.))'))
1043
1044    def test_binbytes8(self):
1045        dumped = b'\x80\x04\x8e\4\0\0\0\0\0\0\0\xe2\x82\xac\x00.'
1046        self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00')
1047
1048    def test_binunicode8(self):
1049        dumped = b'\x80\x04\x8d\4\0\0\0\0\0\0\0\xe2\x82\xac\x00.'
1050        self.assertEqual(self.loads(dumped), '\u20ac\x00')
1051
1052    def test_bytearray8(self):
1053        dumped = b'\x80\x05\x96\x03\x00\x00\x00\x00\x00\x00\x00xxx.'
1054        self.assertEqual(self.loads(dumped), bytearray(b'xxx'))
1055
1056    @requires_32b
1057    def test_large_32b_binbytes8(self):
1058        dumped = b'\x80\x04\x8e\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.'
1059        self.check_unpickling_error((pickle.UnpicklingError, OverflowError),
1060                                    dumped)
1061
1062    @requires_32b
1063    def test_large_32b_bytearray8(self):
1064        dumped = b'\x80\x05\x96\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.'
1065        self.check_unpickling_error((pickle.UnpicklingError, OverflowError),
1066                                    dumped)
1067
1068    @requires_32b
1069    def test_large_32b_binunicode8(self):
1070        dumped = b'\x80\x04\x8d\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.'
1071        self.check_unpickling_error((pickle.UnpicklingError, OverflowError),
1072                                    dumped)
1073
1074    def test_get(self):
1075        pickled = b'((lp100000\ng100000\nt.'
1076        unpickled = self.loads(pickled)
1077        self.assertEqual(unpickled, ([],)*2)
1078        self.assertIs(unpickled[0], unpickled[1])
1079
1080    def test_binget(self):
1081        pickled = b'(]q\xffh\xfft.'
1082        unpickled = self.loads(pickled)
1083        self.assertEqual(unpickled, ([],)*2)
1084        self.assertIs(unpickled[0], unpickled[1])
1085
1086    def test_long_binget(self):
1087        pickled = b'(]r\x00\x00\x01\x00j\x00\x00\x01\x00t.'
1088        unpickled = self.loads(pickled)
1089        self.assertEqual(unpickled, ([],)*2)
1090        self.assertIs(unpickled[0], unpickled[1])
1091
1092    def test_dup(self):
1093        pickled = b'((l2t.'
1094        unpickled = self.loads(pickled)
1095        self.assertEqual(unpickled, ([],)*2)
1096        self.assertIs(unpickled[0], unpickled[1])
1097
1098    def test_negative_put(self):
1099        # Issue #12847
1100        dumped = b'Va\np-1\n.'
1101        self.check_unpickling_error(ValueError, dumped)
1102
1103    @requires_32b
1104    def test_negative_32b_binput(self):
1105        # Issue #12847
1106        dumped = b'\x80\x03X\x01\x00\x00\x00ar\xff\xff\xff\xff.'
1107        self.check_unpickling_error(ValueError, dumped)
1108
1109    def test_badly_escaped_string(self):
1110        self.check_unpickling_error(ValueError, b"S'\\'\n.")
1111
1112    def test_badly_quoted_string(self):
1113        # Issue #17710
1114        badpickles = [b"S'\n.",
1115                      b'S"\n.',
1116                      b'S\' \n.',
1117                      b'S" \n.',
1118                      b'S\'"\n.',
1119                      b'S"\'\n.',
1120                      b"S' ' \n.",
1121                      b'S" " \n.',
1122                      b"S ''\n.",
1123                      b'S ""\n.',
1124                      b'S \n.',
1125                      b'S\n.',
1126                      b'S.']
1127        for p in badpickles:
1128            self.check_unpickling_error(pickle.UnpicklingError, p)
1129
1130    def test_correctly_quoted_string(self):
1131        goodpickles = [(b"S''\n.", ''),
1132                       (b'S""\n.', ''),
1133                       (b'S"\\n"\n.', '\n'),
1134                       (b"S'\\n'\n.", '\n')]
1135        for p, expected in goodpickles:
1136            self.assertEqual(self.loads(p), expected)
1137
1138    def test_frame_readline(self):
1139        pickled = b'\x80\x04\x95\x05\x00\x00\x00\x00\x00\x00\x00I42\n.'
1140        #    0: \x80 PROTO      4
1141        #    2: \x95 FRAME      5
1142        #   11: I    INT        42
1143        #   15: .    STOP
1144        self.assertEqual(self.loads(pickled), 42)
1145
1146    def test_compat_unpickle(self):
1147        # xrange(1, 7)
1148        pickled = b'\x80\x02c__builtin__\nxrange\nK\x01K\x07K\x01\x87R.'
1149        unpickled = self.loads(pickled)
1150        self.assertIs(type(unpickled), range)
1151        self.assertEqual(unpickled, range(1, 7))
1152        self.assertEqual(list(unpickled), [1, 2, 3, 4, 5, 6])
1153        # reduce
1154        pickled = b'\x80\x02c__builtin__\nreduce\n.'
1155        self.assertIs(self.loads(pickled), functools.reduce)
1156        # whichdb.whichdb
1157        pickled = b'\x80\x02cwhichdb\nwhichdb\n.'
1158        self.assertIs(self.loads(pickled), dbm.whichdb)
1159        # Exception(), StandardError()
1160        for name in (b'Exception', b'StandardError'):
1161            pickled = (b'\x80\x02cexceptions\n' + name + b'\nU\x03ugh\x85R.')
1162            unpickled = self.loads(pickled)
1163            self.assertIs(type(unpickled), Exception)
1164            self.assertEqual(str(unpickled), 'ugh')
1165        # UserDict.UserDict({1: 2}), UserDict.IterableUserDict({1: 2})
1166        for name in (b'UserDict', b'IterableUserDict'):
1167            pickled = (b'\x80\x02(cUserDict\n' + name +
1168                       b'\no}U\x04data}K\x01K\x02ssb.')
1169            unpickled = self.loads(pickled)
1170            self.assertIs(type(unpickled), collections.UserDict)
1171            self.assertEqual(unpickled, collections.UserDict({1: 2}))
1172
1173    def test_bad_reduce(self):
1174        self.assertEqual(self.loads(b'cbuiltins\nint\n)R.'), 0)
1175        self.check_unpickling_error(TypeError, b'N)R.')
1176        self.check_unpickling_error(TypeError, b'cbuiltins\nint\nNR.')
1177
1178    def test_bad_newobj(self):
1179        error = (pickle.UnpicklingError, TypeError)
1180        self.assertEqual(self.loads(b'cbuiltins\nint\n)\x81.'), 0)
1181        self.check_unpickling_error(error, b'cbuiltins\nlen\n)\x81.')
1182        self.check_unpickling_error(error, b'cbuiltins\nint\nN\x81.')
1183
1184    def test_bad_newobj_ex(self):
1185        error = (pickle.UnpicklingError, TypeError)
1186        self.assertEqual(self.loads(b'cbuiltins\nint\n)}\x92.'), 0)
1187        self.check_unpickling_error(error, b'cbuiltins\nlen\n)}\x92.')
1188        self.check_unpickling_error(error, b'cbuiltins\nint\nN}\x92.')
1189        self.check_unpickling_error(error, b'cbuiltins\nint\n)N\x92.')
1190
1191    def test_bad_stack(self):
1192        badpickles = [
1193            b'.',                       # STOP
1194            b'0',                       # POP
1195            b'1',                       # POP_MARK
1196            b'2',                       # DUP
1197            b'(2',
1198            b'R',                       # REDUCE
1199            b')R',
1200            b'a',                       # APPEND
1201            b'Na',
1202            b'b',                       # BUILD
1203            b'Nb',
1204            b'd',                       # DICT
1205            b'e',                       # APPENDS
1206            b'(e',
1207            b'ibuiltins\nlist\n',       # INST
1208            b'l',                       # LIST
1209            b'o',                       # OBJ
1210            b'(o',
1211            b'p1\n',                    # PUT
1212            b'q\x00',                   # BINPUT
1213            b'r\x00\x00\x00\x00',       # LONG_BINPUT
1214            b's',                       # SETITEM
1215            b'Ns',
1216            b'NNs',
1217            b't',                       # TUPLE
1218            b'u',                       # SETITEMS
1219            b'(u',
1220            b'}(Nu',
1221            b'\x81',                    # NEWOBJ
1222            b')\x81',
1223            b'\x85',                    # TUPLE1
1224            b'\x86',                    # TUPLE2
1225            b'N\x86',
1226            b'\x87',                    # TUPLE3
1227            b'N\x87',
1228            b'NN\x87',
1229            b'\x90',                    # ADDITEMS
1230            b'(\x90',
1231            b'\x91',                    # FROZENSET
1232            b'\x92',                    # NEWOBJ_EX
1233            b')}\x92',
1234            b'\x93',                    # STACK_GLOBAL
1235            b'Vlist\n\x93',
1236            b'\x94',                    # MEMOIZE
1237        ]
1238        for p in badpickles:
1239            self.check_unpickling_error(self.bad_stack_errors, p)
1240
1241    def test_bad_mark(self):
1242        badpickles = [
1243            b'N(.',                     # STOP
1244            b'N(2',                     # DUP
1245            b'cbuiltins\nlist\n)(R',    # REDUCE
1246            b'cbuiltins\nlist\n()R',
1247            b']N(a',                    # APPEND
1248                                        # BUILD
1249            b'cbuiltins\nValueError\n)R}(b',
1250            b'cbuiltins\nValueError\n)R(}b',
1251            b'(Nd',                     # DICT
1252            b'N(p1\n',                  # PUT
1253            b'N(q\x00',                 # BINPUT
1254            b'N(r\x00\x00\x00\x00',     # LONG_BINPUT
1255            b'}NN(s',                   # SETITEM
1256            b'}N(Ns',
1257            b'}(NNs',
1258            b'}((u',                    # SETITEMS
1259            b'cbuiltins\nlist\n)(\x81', # NEWOBJ
1260            b'cbuiltins\nlist\n()\x81',
1261            b'N(\x85',                  # TUPLE1
1262            b'NN(\x86',                 # TUPLE2
1263            b'N(N\x86',
1264            b'NNN(\x87',                # TUPLE3
1265            b'NN(N\x87',
1266            b'N(NN\x87',
1267            b']((\x90',                 # ADDITEMS
1268                                        # NEWOBJ_EX
1269            b'cbuiltins\nlist\n)}(\x92',
1270            b'cbuiltins\nlist\n)(}\x92',
1271            b'cbuiltins\nlist\n()}\x92',
1272                                        # STACK_GLOBAL
1273            b'Vbuiltins\n(Vlist\n\x93',
1274            b'Vbuiltins\nVlist\n(\x93',
1275            b'N(\x94',                  # MEMOIZE
1276        ]
1277        for p in badpickles:
1278            self.check_unpickling_error(self.bad_stack_errors, p)
1279
1280    def test_truncated_data(self):
1281        self.check_unpickling_error(EOFError, b'')
1282        self.check_unpickling_error(EOFError, b'N')
1283        badpickles = [
1284            b'B',                       # BINBYTES
1285            b'B\x03\x00\x00',
1286            b'B\x03\x00\x00\x00',
1287            b'B\x03\x00\x00\x00ab',
1288            b'C',                       # SHORT_BINBYTES
1289            b'C\x03',
1290            b'C\x03ab',
1291            b'F',                       # FLOAT
1292            b'F0.0',
1293            b'F0.00',
1294            b'G',                       # BINFLOAT
1295            b'G\x00\x00\x00\x00\x00\x00\x00',
1296            b'I',                       # INT
1297            b'I0',
1298            b'J',                       # BININT
1299            b'J\x00\x00\x00',
1300            b'K',                       # BININT1
1301            b'L',                       # LONG
1302            b'L0',
1303            b'L10',
1304            b'L0L',
1305            b'L10L',
1306            b'M',                       # BININT2
1307            b'M\x00',
1308            # b'P',                       # PERSID
1309            # b'Pabc',
1310            b'S',                       # STRING
1311            b"S'abc'",
1312            b'T',                       # BINSTRING
1313            b'T\x03\x00\x00',
1314            b'T\x03\x00\x00\x00',
1315            b'T\x03\x00\x00\x00ab',
1316            b'U',                       # SHORT_BINSTRING
1317            b'U\x03',
1318            b'U\x03ab',
1319            b'V',                       # UNICODE
1320            b'Vabc',
1321            b'X',                       # BINUNICODE
1322            b'X\x03\x00\x00',
1323            b'X\x03\x00\x00\x00',
1324            b'X\x03\x00\x00\x00ab',
1325            b'(c',                      # GLOBAL
1326            b'(cbuiltins',
1327            b'(cbuiltins\n',
1328            b'(cbuiltins\nlist',
1329            b'Ng',                      # GET
1330            b'Ng0',
1331            b'(i',                      # INST
1332            b'(ibuiltins',
1333            b'(ibuiltins\n',
1334            b'(ibuiltins\nlist',
1335            b'Nh',                      # BINGET
1336            b'Nj',                      # LONG_BINGET
1337            b'Nj\x00\x00\x00',
1338            b'Np',                      # PUT
1339            b'Np0',
1340            b'Nq',                      # BINPUT
1341            b'Nr',                      # LONG_BINPUT
1342            b'Nr\x00\x00\x00',
1343            b'\x80',                    # PROTO
1344            b'\x82',                    # EXT1
1345            b'\x83',                    # EXT2
1346            b'\x84\x01',
1347            b'\x84',                    # EXT4
1348            b'\x84\x01\x00\x00',
1349            b'\x8a',                    # LONG1
1350            b'\x8b',                    # LONG4
1351            b'\x8b\x00\x00\x00',
1352            b'\x8c',                    # SHORT_BINUNICODE
1353            b'\x8c\x03',
1354            b'\x8c\x03ab',
1355            b'\x8d',                    # BINUNICODE8
1356            b'\x8d\x03\x00\x00\x00\x00\x00\x00',
1357            b'\x8d\x03\x00\x00\x00\x00\x00\x00\x00',
1358            b'\x8d\x03\x00\x00\x00\x00\x00\x00\x00ab',
1359            b'\x8e',                    # BINBYTES8
1360            b'\x8e\x03\x00\x00\x00\x00\x00\x00',
1361            b'\x8e\x03\x00\x00\x00\x00\x00\x00\x00',
1362            b'\x8e\x03\x00\x00\x00\x00\x00\x00\x00ab',
1363            b'\x96',                    # BYTEARRAY8
1364            b'\x96\x03\x00\x00\x00\x00\x00\x00',
1365            b'\x96\x03\x00\x00\x00\x00\x00\x00\x00',
1366            b'\x96\x03\x00\x00\x00\x00\x00\x00\x00ab',
1367            b'\x95',                    # FRAME
1368            b'\x95\x02\x00\x00\x00\x00\x00\x00',
1369            b'\x95\x02\x00\x00\x00\x00\x00\x00\x00',
1370            b'\x95\x02\x00\x00\x00\x00\x00\x00\x00N',
1371        ]
1372        for p in badpickles:
1373            self.check_unpickling_error(self.truncated_errors, p)
1374
1375    @reap_threads
1376    def test_unpickle_module_race(self):
1377        # https://bugs.python.org/issue34572
1378        locker_module = dedent("""
1379        import threading
1380        barrier = threading.Barrier(2)
1381        """)
1382        locking_import_module = dedent("""
1383        import locker
1384        locker.barrier.wait()
1385        class ToBeUnpickled(object):
1386            pass
1387        """)
1388
1389        os.mkdir(TESTFN)
1390        self.addCleanup(shutil.rmtree, TESTFN)
1391        sys.path.insert(0, TESTFN)
1392        self.addCleanup(sys.path.remove, TESTFN)
1393        with open(os.path.join(TESTFN, "locker.py"), "wb") as f:
1394            f.write(locker_module.encode('utf-8'))
1395        with open(os.path.join(TESTFN, "locking_import.py"), "wb") as f:
1396            f.write(locking_import_module.encode('utf-8'))
1397        self.addCleanup(forget, "locker")
1398        self.addCleanup(forget, "locking_import")
1399
1400        import locker
1401
1402        pickle_bytes = (
1403            b'\x80\x03clocking_import\nToBeUnpickled\nq\x00)\x81q\x01.')
1404
1405        # Then try to unpickle two of these simultaneously
1406        # One of them will cause the module import, and we want it to block
1407        # until the other one either:
1408        #   - fails (before the patch for this issue)
1409        #   - blocks on the import lock for the module, as it should
1410        results = []
1411        barrier = threading.Barrier(3)
1412        def t():
1413            # This ensures the threads have all started
1414            # presumably barrier release is faster than thread startup
1415            barrier.wait()
1416            results.append(pickle.loads(pickle_bytes))
1417
1418        t1 = threading.Thread(target=t)
1419        t2 = threading.Thread(target=t)
1420        t1.start()
1421        t2.start()
1422
1423        barrier.wait()
1424        # could have delay here
1425        locker.barrier.wait()
1426
1427        t1.join()
1428        t2.join()
1429
1430        from locking_import import ToBeUnpickled
1431        self.assertEqual(
1432            [type(x) for x in results],
1433            [ToBeUnpickled] * 2)
1434
1435
1436
1437class AbstractPickleTests(unittest.TestCase):
1438    # Subclass must define self.dumps, self.loads.
1439
1440    optimized = False
1441
1442    _testdata = AbstractUnpickleTests._testdata
1443
1444    def setUp(self):
1445        pass
1446
1447    assert_is_copy = AbstractUnpickleTests.assert_is_copy
1448
1449    def test_misc(self):
1450        # test various datatypes not tested by testdata
1451        for proto in protocols:
1452            x = myint(4)
1453            s = self.dumps(x, proto)
1454            y = self.loads(s)
1455            self.assert_is_copy(x, y)
1456
1457            x = (1, ())
1458            s = self.dumps(x, proto)
1459            y = self.loads(s)
1460            self.assert_is_copy(x, y)
1461
1462            x = initarg(1, x)
1463            s = self.dumps(x, proto)
1464            y = self.loads(s)
1465            self.assert_is_copy(x, y)
1466
1467        # XXX test __reduce__ protocol?
1468
1469    def test_roundtrip_equality(self):
1470        expected = self._testdata
1471        for proto in protocols:
1472            s = self.dumps(expected, proto)
1473            got = self.loads(s)
1474            self.assert_is_copy(expected, got)
1475
1476    # There are gratuitous differences between pickles produced by
1477    # pickle and cPickle, largely because cPickle starts PUT indices at
1478    # 1 and pickle starts them at 0.  See XXX comment in cPickle's put2() --
1479    # there's a comment with an exclamation point there whose meaning
1480    # is a mystery.  cPickle also suppresses PUT for objects with a refcount
1481    # of 1.
1482    def dont_test_disassembly(self):
1483        from io import StringIO
1484        from pickletools import dis
1485
1486        for proto, expected in (0, DATA0_DIS), (1, DATA1_DIS):
1487            s = self.dumps(self._testdata, proto)
1488            filelike = StringIO()
1489            dis(s, out=filelike)
1490            got = filelike.getvalue()
1491            self.assertEqual(expected, got)
1492
1493    def test_recursive_list(self):
1494        l = []
1495        l.append(l)
1496        for proto in protocols:
1497            s = self.dumps(l, proto)
1498            x = self.loads(s)
1499            self.assertIsInstance(x, list)
1500            self.assertEqual(len(x), 1)
1501            self.assertIs(x[0], x)
1502
1503    def test_recursive_tuple_and_list(self):
1504        t = ([],)
1505        t[0].append(t)
1506        for proto in protocols:
1507            s = self.dumps(t, proto)
1508            x = self.loads(s)
1509            self.assertIsInstance(x, tuple)
1510            self.assertEqual(len(x), 1)
1511            self.assertIsInstance(x[0], list)
1512            self.assertEqual(len(x[0]), 1)
1513            self.assertIs(x[0][0], x)
1514
1515    def test_recursive_dict(self):
1516        d = {}
1517        d[1] = d
1518        for proto in protocols:
1519            s = self.dumps(d, proto)
1520            x = self.loads(s)
1521            self.assertIsInstance(x, dict)
1522            self.assertEqual(list(x.keys()), [1])
1523            self.assertIs(x[1], x)
1524
1525    def test_recursive_dict_key(self):
1526        d = {}
1527        k = K(d)
1528        d[k] = 1
1529        for proto in protocols:
1530            s = self.dumps(d, proto)
1531            x = self.loads(s)
1532            self.assertIsInstance(x, dict)
1533            self.assertEqual(len(x.keys()), 1)
1534            self.assertIsInstance(list(x.keys())[0], K)
1535            self.assertIs(list(x.keys())[0].value, x)
1536
1537    def test_recursive_set(self):
1538        y = set()
1539        k = K(y)
1540        y.add(k)
1541        for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
1542            s = self.dumps(y, proto)
1543            x = self.loads(s)
1544            self.assertIsInstance(x, set)
1545            self.assertEqual(len(x), 1)
1546            self.assertIsInstance(list(x)[0], K)
1547            self.assertIs(list(x)[0].value, x)
1548
1549    def test_recursive_list_subclass(self):
1550        y = MyList()
1551        y.append(y)
1552        for proto in range(2, pickle.HIGHEST_PROTOCOL + 1):
1553            s = self.dumps(y, proto)
1554            x = self.loads(s)
1555            self.assertIsInstance(x, MyList)
1556            self.assertEqual(len(x), 1)
1557            self.assertIs(x[0], x)
1558
1559    def test_recursive_dict_subclass(self):
1560        d = MyDict()
1561        d[1] = d
1562        for proto in range(2, pickle.HIGHEST_PROTOCOL + 1):
1563            s = self.dumps(d, proto)
1564            x = self.loads(s)
1565            self.assertIsInstance(x, MyDict)
1566            self.assertEqual(list(x.keys()), [1])
1567            self.assertIs(x[1], x)
1568
1569    def test_recursive_dict_subclass_key(self):
1570        d = MyDict()
1571        k = K(d)
1572        d[k] = 1
1573        for proto in range(2, pickle.HIGHEST_PROTOCOL + 1):
1574            s = self.dumps(d, proto)
1575            x = self.loads(s)
1576            self.assertIsInstance(x, MyDict)
1577            self.assertEqual(len(list(x.keys())), 1)
1578            self.assertIsInstance(list(x.keys())[0], K)
1579            self.assertIs(list(x.keys())[0].value, x)
1580
1581    def test_recursive_inst(self):
1582        i = C()
1583        i.attr = i
1584        for proto in protocols:
1585            s = self.dumps(i, proto)
1586            x = self.loads(s)
1587            self.assertIsInstance(x, C)
1588            self.assertEqual(dir(x), dir(i))
1589            self.assertIs(x.attr, x)
1590
1591    def test_recursive_multi(self):
1592        l = []
1593        d = {1:l}
1594        i = C()
1595        i.attr = d
1596        l.append(i)
1597        for proto in protocols:
1598            s = self.dumps(l, proto)
1599            x = self.loads(s)
1600            self.assertIsInstance(x, list)
1601            self.assertEqual(len(x), 1)
1602            self.assertEqual(dir(x[0]), dir(i))
1603            self.assertEqual(list(x[0].attr.keys()), [1])
1604            self.assertTrue(x[0].attr[1] is x)
1605
1606    def check_recursive_collection_and_inst(self, factory):
1607        h = H()
1608        y = factory([h])
1609        h.attr = y
1610        for proto in protocols:
1611            s = self.dumps(y, proto)
1612            x = self.loads(s)
1613            self.assertIsInstance(x, type(y))
1614            self.assertEqual(len(x), 1)
1615            self.assertIsInstance(list(x)[0], H)
1616            self.assertIs(list(x)[0].attr, x)
1617
1618    def test_recursive_list_and_inst(self):
1619        self.check_recursive_collection_and_inst(list)
1620
1621    def test_recursive_tuple_and_inst(self):
1622        self.check_recursive_collection_and_inst(tuple)
1623
1624    def test_recursive_dict_and_inst(self):
1625        self.check_recursive_collection_and_inst(dict.fromkeys)
1626
1627    def test_recursive_set_and_inst(self):
1628        self.check_recursive_collection_and_inst(set)
1629
1630    def test_recursive_frozenset_and_inst(self):
1631        self.check_recursive_collection_and_inst(frozenset)
1632
1633    def test_recursive_list_subclass_and_inst(self):
1634        self.check_recursive_collection_and_inst(MyList)
1635
1636    def test_recursive_tuple_subclass_and_inst(self):
1637        self.check_recursive_collection_and_inst(MyTuple)
1638
1639    def test_recursive_dict_subclass_and_inst(self):
1640        self.check_recursive_collection_and_inst(MyDict.fromkeys)
1641
1642    def test_recursive_set_subclass_and_inst(self):
1643        self.check_recursive_collection_and_inst(MySet)
1644
1645    def test_recursive_frozenset_subclass_and_inst(self):
1646        self.check_recursive_collection_and_inst(MyFrozenSet)
1647
1648    def test_unicode(self):
1649        endcases = ['', '<\\u>', '<\\\u1234>', '<\n>',
1650                    '<\\>', '<\\\U00012345>',
1651                    # surrogates
1652                    '<\udc80>']
1653        for proto in protocols:
1654            for u in endcases:
1655                p = self.dumps(u, proto)
1656                u2 = self.loads(p)
1657                self.assert_is_copy(u, u2)
1658
1659    def test_unicode_high_plane(self):
1660        t = '\U00012345'
1661        for proto in protocols:
1662            p = self.dumps(t, proto)
1663            t2 = self.loads(p)
1664            self.assert_is_copy(t, t2)
1665
1666    def test_bytes(self):
1667        for proto in protocols:
1668            for s in b'', b'xyz', b'xyz'*100:
1669                p = self.dumps(s, proto)
1670                self.assert_is_copy(s, self.loads(p))
1671            for s in [bytes([i]) for i in range(256)]:
1672                p = self.dumps(s, proto)
1673                self.assert_is_copy(s, self.loads(p))
1674            for s in [bytes([i, i]) for i in range(256)]:
1675                p = self.dumps(s, proto)
1676                self.assert_is_copy(s, self.loads(p))
1677
1678    def test_bytearray(self):
1679        for proto in protocols:
1680            for s in b'', b'xyz', b'xyz'*100:
1681                b = bytearray(s)
1682                p = self.dumps(b, proto)
1683                bb = self.loads(p)
1684                self.assertIsNot(bb, b)
1685                self.assert_is_copy(b, bb)
1686                if proto <= 3:
1687                    # bytearray is serialized using a global reference
1688                    self.assertIn(b'bytearray', p)
1689                    self.assertTrue(opcode_in_pickle(pickle.GLOBAL, p))
1690                elif proto == 4:
1691                    self.assertIn(b'bytearray', p)
1692                    self.assertTrue(opcode_in_pickle(pickle.STACK_GLOBAL, p))
1693                elif proto == 5:
1694                    self.assertNotIn(b'bytearray', p)
1695                    self.assertTrue(opcode_in_pickle(pickle.BYTEARRAY8, p))
1696
1697    def test_ints(self):
1698        for proto in protocols:
1699            n = sys.maxsize
1700            while n:
1701                for expected in (-n, n):
1702                    s = self.dumps(expected, proto)
1703                    n2 = self.loads(s)
1704                    self.assert_is_copy(expected, n2)
1705                n = n >> 1
1706
1707    def test_long(self):
1708        for proto in protocols:
1709            # 256 bytes is where LONG4 begins.
1710            for nbits in 1, 8, 8*254, 8*255, 8*256, 8*257:
1711                nbase = 1 << nbits
1712                for npos in nbase-1, nbase, nbase+1:
1713                    for n in npos, -npos:
1714                        pickle = self.dumps(n, proto)
1715                        got = self.loads(pickle)
1716                        self.assert_is_copy(n, got)
1717        # Try a monster.  This is quadratic-time in protos 0 & 1, so don't
1718        # bother with those.
1719        nbase = int("deadbeeffeedface", 16)
1720        nbase += nbase << 1000000
1721        for n in nbase, -nbase:
1722            p = self.dumps(n, 2)
1723            got = self.loads(p)
1724            # assert_is_copy is very expensive here as it precomputes
1725            # a failure message by computing the repr() of n and got,
1726            # we just do the check ourselves.
1727            self.assertIs(type(got), int)
1728            self.assertEqual(n, got)
1729
1730    def test_float(self):
1731        test_values = [0.0, 4.94e-324, 1e-310, 7e-308, 6.626e-34, 0.1, 0.5,
1732                       3.14, 263.44582062374053, 6.022e23, 1e30]
1733        test_values = test_values + [-x for x in test_values]
1734        for proto in protocols:
1735            for value in test_values:
1736                pickle = self.dumps(value, proto)
1737                got = self.loads(pickle)
1738                self.assert_is_copy(value, got)
1739
1740    @run_with_locale('LC_ALL', 'de_DE', 'fr_FR')
1741    def test_float_format(self):
1742        # make sure that floats are formatted locale independent with proto 0
1743        self.assertEqual(self.dumps(1.2, 0)[0:3], b'F1.')
1744
1745    def test_reduce(self):
1746        for proto in protocols:
1747            inst = AAA()
1748            dumped = self.dumps(inst, proto)
1749            loaded = self.loads(dumped)
1750            self.assertEqual(loaded, REDUCE_A)
1751
1752    def test_getinitargs(self):
1753        for proto in protocols:
1754            inst = initarg(1, 2)
1755            dumped = self.dumps(inst, proto)
1756            loaded = self.loads(dumped)
1757            self.assert_is_copy(inst, loaded)
1758
1759    def test_metaclass(self):
1760        a = use_metaclass()
1761        for proto in protocols:
1762            s = self.dumps(a, proto)
1763            b = self.loads(s)
1764            self.assertEqual(a.__class__, b.__class__)
1765
1766    def test_dynamic_class(self):
1767        a = create_dynamic_class("my_dynamic_class", (object,))
1768        copyreg.pickle(pickling_metaclass, pickling_metaclass.__reduce__)
1769        for proto in protocols:
1770            s = self.dumps(a, proto)
1771            b = self.loads(s)
1772            self.assertEqual(a, b)
1773            self.assertIs(type(a), type(b))
1774
1775    def test_structseq(self):
1776        import time
1777        import os
1778
1779        t = time.localtime()
1780        for proto in protocols:
1781            s = self.dumps(t, proto)
1782            u = self.loads(s)
1783            self.assert_is_copy(t, u)
1784            t = os.stat(os.curdir)
1785            s = self.dumps(t, proto)
1786            u = self.loads(s)
1787            self.assert_is_copy(t, u)
1788            if hasattr(os, "statvfs"):
1789                t = os.statvfs(os.curdir)
1790                s = self.dumps(t, proto)
1791                u = self.loads(s)
1792                self.assert_is_copy(t, u)
1793
1794    def test_ellipsis(self):
1795        for proto in protocols:
1796            s = self.dumps(..., proto)
1797            u = self.loads(s)
1798            self.assertIs(..., u)
1799
1800    def test_notimplemented(self):
1801        for proto in protocols:
1802            s = self.dumps(NotImplemented, proto)
1803            u = self.loads(s)
1804            self.assertIs(NotImplemented, u)
1805
1806    def test_singleton_types(self):
1807        # Issue #6477: Test that types of built-in singletons can be pickled.
1808        singletons = [None, ..., NotImplemented]
1809        for singleton in singletons:
1810            for proto in protocols:
1811                s = self.dumps(type(singleton), proto)
1812                u = self.loads(s)
1813                self.assertIs(type(singleton), u)
1814
1815    # Tests for protocol 2
1816
1817    def test_proto(self):
1818        for proto in protocols:
1819            pickled = self.dumps(None, proto)
1820            if proto >= 2:
1821                proto_header = pickle.PROTO + bytes([proto])
1822                self.assertTrue(pickled.startswith(proto_header))
1823            else:
1824                self.assertEqual(count_opcode(pickle.PROTO, pickled), 0)
1825
1826        oob = protocols[-1] + 1     # a future protocol
1827        build_none = pickle.NONE + pickle.STOP
1828        badpickle = pickle.PROTO + bytes([oob]) + build_none
1829        try:
1830            self.loads(badpickle)
1831        except ValueError as err:
1832            self.assertIn("unsupported pickle protocol", str(err))
1833        else:
1834            self.fail("expected bad protocol number to raise ValueError")
1835
1836    def test_long1(self):
1837        x = 12345678910111213141516178920
1838        for proto in protocols:
1839            s = self.dumps(x, proto)
1840            y = self.loads(s)
1841            self.assert_is_copy(x, y)
1842            self.assertEqual(opcode_in_pickle(pickle.LONG1, s), proto >= 2)
1843
1844    def test_long4(self):
1845        x = 12345678910111213141516178920 << (256*8)
1846        for proto in protocols:
1847            s = self.dumps(x, proto)
1848            y = self.loads(s)
1849            self.assert_is_copy(x, y)
1850            self.assertEqual(opcode_in_pickle(pickle.LONG4, s), proto >= 2)
1851
1852    def test_short_tuples(self):
1853        # Map (proto, len(tuple)) to expected opcode.
1854        expected_opcode = {(0, 0): pickle.TUPLE,
1855                           (0, 1): pickle.TUPLE,
1856                           (0, 2): pickle.TUPLE,
1857                           (0, 3): pickle.TUPLE,
1858                           (0, 4): pickle.TUPLE,
1859
1860                           (1, 0): pickle.EMPTY_TUPLE,
1861                           (1, 1): pickle.TUPLE,
1862                           (1, 2): pickle.TUPLE,
1863                           (1, 3): pickle.TUPLE,
1864                           (1, 4): pickle.TUPLE,
1865
1866                           (2, 0): pickle.EMPTY_TUPLE,
1867                           (2, 1): pickle.TUPLE1,
1868                           (2, 2): pickle.TUPLE2,
1869                           (2, 3): pickle.TUPLE3,
1870                           (2, 4): pickle.TUPLE,
1871
1872                           (3, 0): pickle.EMPTY_TUPLE,
1873                           (3, 1): pickle.TUPLE1,
1874                           (3, 2): pickle.TUPLE2,
1875                           (3, 3): pickle.TUPLE3,
1876                           (3, 4): pickle.TUPLE,
1877                          }
1878        a = ()
1879        b = (1,)
1880        c = (1, 2)
1881        d = (1, 2, 3)
1882        e = (1, 2, 3, 4)
1883        for proto in protocols:
1884            for x in a, b, c, d, e:
1885                s = self.dumps(x, proto)
1886                y = self.loads(s)
1887                self.assert_is_copy(x, y)
1888                expected = expected_opcode[min(proto, 3), len(x)]
1889                self.assertTrue(opcode_in_pickle(expected, s))
1890
1891    def test_singletons(self):
1892        # Map (proto, singleton) to expected opcode.
1893        expected_opcode = {(0, None): pickle.NONE,
1894                           (1, None): pickle.NONE,
1895                           (2, None): pickle.NONE,
1896                           (3, None): pickle.NONE,
1897
1898                           (0, True): pickle.INT,
1899                           (1, True): pickle.INT,
1900                           (2, True): pickle.NEWTRUE,
1901                           (3, True): pickle.NEWTRUE,
1902
1903                           (0, False): pickle.INT,
1904                           (1, False): pickle.INT,
1905                           (2, False): pickle.NEWFALSE,
1906                           (3, False): pickle.NEWFALSE,
1907                          }
1908        for proto in protocols:
1909            for x in None, False, True:
1910                s = self.dumps(x, proto)
1911                y = self.loads(s)
1912                self.assertTrue(x is y, (proto, x, s, y))
1913                expected = expected_opcode[min(proto, 3), x]
1914                self.assertTrue(opcode_in_pickle(expected, s))
1915
1916    def test_newobj_tuple(self):
1917        x = MyTuple([1, 2, 3])
1918        x.foo = 42
1919        x.bar = "hello"
1920        for proto in protocols:
1921            s = self.dumps(x, proto)
1922            y = self.loads(s)
1923            self.assert_is_copy(x, y)
1924
1925    def test_newobj_list(self):
1926        x = MyList([1, 2, 3])
1927        x.foo = 42
1928        x.bar = "hello"
1929        for proto in protocols:
1930            s = self.dumps(x, proto)
1931            y = self.loads(s)
1932            self.assert_is_copy(x, y)
1933
1934    def test_newobj_generic(self):
1935        for proto in protocols:
1936            for C in myclasses:
1937                B = C.__base__
1938                x = C(C.sample)
1939                x.foo = 42
1940                s = self.dumps(x, proto)
1941                y = self.loads(s)
1942                detail = (proto, C, B, x, y, type(y))
1943                self.assert_is_copy(x, y) # XXX revisit
1944                self.assertEqual(B(x), B(y), detail)
1945                self.assertEqual(x.__dict__, y.__dict__, detail)
1946
1947    def test_newobj_proxies(self):
1948        # NEWOBJ should use the __class__ rather than the raw type
1949        classes = myclasses[:]
1950        # Cannot create weakproxies to these classes
1951        for c in (MyInt, MyTuple):
1952            classes.remove(c)
1953        for proto in protocols:
1954            for C in classes:
1955                B = C.__base__
1956                x = C(C.sample)
1957                x.foo = 42
1958                p = weakref.proxy(x)
1959                s = self.dumps(p, proto)
1960                y = self.loads(s)
1961                self.assertEqual(type(y), type(x))  # rather than type(p)
1962                detail = (proto, C, B, x, y, type(y))
1963                self.assertEqual(B(x), B(y), detail)
1964                self.assertEqual(x.__dict__, y.__dict__, detail)
1965
1966    def test_newobj_not_class(self):
1967        # Issue 24552
1968        global SimpleNewObj
1969        save = SimpleNewObj
1970        o = SimpleNewObj.__new__(SimpleNewObj)
1971        b = self.dumps(o, 4)
1972        try:
1973            SimpleNewObj = 42
1974            self.assertRaises((TypeError, pickle.UnpicklingError), self.loads, b)
1975        finally:
1976            SimpleNewObj = save
1977
1978    # Register a type with copyreg, with extension code extcode.  Pickle
1979    # an object of that type.  Check that the resulting pickle uses opcode
1980    # (EXT[124]) under proto 2, and not in proto 1.
1981
1982    def produce_global_ext(self, extcode, opcode):
1983        e = ExtensionSaver(extcode)
1984        try:
1985            copyreg.add_extension(__name__, "MyList", extcode)
1986            x = MyList([1, 2, 3])
1987            x.foo = 42
1988            x.bar = "hello"
1989
1990            # Dump using protocol 1 for comparison.
1991            s1 = self.dumps(x, 1)
1992            self.assertIn(__name__.encode("utf-8"), s1)
1993            self.assertIn(b"MyList", s1)
1994            self.assertFalse(opcode_in_pickle(opcode, s1))
1995
1996            y = self.loads(s1)
1997            self.assert_is_copy(x, y)
1998
1999            # Dump using protocol 2 for test.
2000            s2 = self.dumps(x, 2)
2001            self.assertNotIn(__name__.encode("utf-8"), s2)
2002            self.assertNotIn(b"MyList", s2)
2003            self.assertEqual(opcode_in_pickle(opcode, s2), True, repr(s2))
2004
2005            y = self.loads(s2)
2006            self.assert_is_copy(x, y)
2007        finally:
2008            e.restore()
2009
2010    def test_global_ext1(self):
2011        self.produce_global_ext(0x00000001, pickle.EXT1)  # smallest EXT1 code
2012        self.produce_global_ext(0x000000ff, pickle.EXT1)  # largest EXT1 code
2013
2014    def test_global_ext2(self):
2015        self.produce_global_ext(0x00000100, pickle.EXT2)  # smallest EXT2 code
2016        self.produce_global_ext(0x0000ffff, pickle.EXT2)  # largest EXT2 code
2017        self.produce_global_ext(0x0000abcd, pickle.EXT2)  # check endianness
2018
2019    def test_global_ext4(self):
2020        self.produce_global_ext(0x00010000, pickle.EXT4)  # smallest EXT4 code
2021        self.produce_global_ext(0x7fffffff, pickle.EXT4)  # largest EXT4 code
2022        self.produce_global_ext(0x12abcdef, pickle.EXT4)  # check endianness
2023
2024    def test_list_chunking(self):
2025        n = 10  # too small to chunk
2026        x = list(range(n))
2027        for proto in protocols:
2028            s = self.dumps(x, proto)
2029            y = self.loads(s)
2030            self.assert_is_copy(x, y)
2031            num_appends = count_opcode(pickle.APPENDS, s)
2032            self.assertEqual(num_appends, proto > 0)
2033
2034        n = 2500  # expect at least two chunks when proto > 0
2035        x = list(range(n))
2036        for proto in protocols:
2037            s = self.dumps(x, proto)
2038            y = self.loads(s)
2039            self.assert_is_copy(x, y)
2040            num_appends = count_opcode(pickle.APPENDS, s)
2041            if proto == 0:
2042                self.assertEqual(num_appends, 0)
2043            else:
2044                self.assertTrue(num_appends >= 2)
2045
2046    def test_dict_chunking(self):
2047        n = 10  # too small to chunk
2048        x = dict.fromkeys(range(n))
2049        for proto in protocols:
2050            s = self.dumps(x, proto)
2051            self.assertIsInstance(s, bytes_types)
2052            y = self.loads(s)
2053            self.assert_is_copy(x, y)
2054            num_setitems = count_opcode(pickle.SETITEMS, s)
2055            self.assertEqual(num_setitems, proto > 0)
2056
2057        n = 2500  # expect at least two chunks when proto > 0
2058        x = dict.fromkeys(range(n))
2059        for proto in protocols:
2060            s = self.dumps(x, proto)
2061            y = self.loads(s)
2062            self.assert_is_copy(x, y)
2063            num_setitems = count_opcode(pickle.SETITEMS, s)
2064            if proto == 0:
2065                self.assertEqual(num_setitems, 0)
2066            else:
2067                self.assertTrue(num_setitems >= 2)
2068
2069    def test_set_chunking(self):
2070        n = 10  # too small to chunk
2071        x = set(range(n))
2072        for proto in protocols:
2073            s = self.dumps(x, proto)
2074            y = self.loads(s)
2075            self.assert_is_copy(x, y)
2076            num_additems = count_opcode(pickle.ADDITEMS, s)
2077            if proto < 4:
2078                self.assertEqual(num_additems, 0)
2079            else:
2080                self.assertEqual(num_additems, 1)
2081
2082        n = 2500  # expect at least two chunks when proto >= 4
2083        x = set(range(n))
2084        for proto in protocols:
2085            s = self.dumps(x, proto)
2086            y = self.loads(s)
2087            self.assert_is_copy(x, y)
2088            num_additems = count_opcode(pickle.ADDITEMS, s)
2089            if proto < 4:
2090                self.assertEqual(num_additems, 0)
2091            else:
2092                self.assertGreaterEqual(num_additems, 2)
2093
2094    def test_simple_newobj(self):
2095        x = SimpleNewObj.__new__(SimpleNewObj, 0xface)  # avoid __init__
2096        x.abc = 666
2097        for proto in protocols:
2098            with self.subTest(proto=proto):
2099                s = self.dumps(x, proto)
2100                if proto < 1:
2101                    self.assertIn(b'\nI64206', s)  # INT
2102                else:
2103                    self.assertIn(b'M\xce\xfa', s)  # BININT2
2104                self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s),
2105                                 2 <= proto)
2106                self.assertFalse(opcode_in_pickle(pickle.NEWOBJ_EX, s))
2107                y = self.loads(s)   # will raise TypeError if __init__ called
2108                self.assert_is_copy(x, y)
2109
2110    def test_complex_newobj(self):
2111        x = ComplexNewObj.__new__(ComplexNewObj, 0xface)  # avoid __init__
2112        x.abc = 666
2113        for proto in protocols:
2114            with self.subTest(proto=proto):
2115                s = self.dumps(x, proto)
2116                if proto < 1:
2117                    self.assertIn(b'\nI64206', s)  # INT
2118                elif proto < 2:
2119                    self.assertIn(b'M\xce\xfa', s)  # BININT2
2120                elif proto < 4:
2121                    self.assertIn(b'X\x04\x00\x00\x00FACE', s)  # BINUNICODE
2122                else:
2123                    self.assertIn(b'\x8c\x04FACE', s)  # SHORT_BINUNICODE
2124                self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s),
2125                                 2 <= proto)
2126                self.assertFalse(opcode_in_pickle(pickle.NEWOBJ_EX, s))
2127                y = self.loads(s)   # will raise TypeError if __init__ called
2128                self.assert_is_copy(x, y)
2129
2130    def test_complex_newobj_ex(self):
2131        x = ComplexNewObjEx.__new__(ComplexNewObjEx, 0xface)  # avoid __init__
2132        x.abc = 666
2133        for proto in protocols:
2134            with self.subTest(proto=proto):
2135                s = self.dumps(x, proto)
2136                if proto < 1:
2137                    self.assertIn(b'\nI64206', s)  # INT
2138                elif proto < 2:
2139                    self.assertIn(b'M\xce\xfa', s)  # BININT2
2140                elif proto < 4:
2141                    self.assertIn(b'X\x04\x00\x00\x00FACE', s)  # BINUNICODE
2142                else:
2143                    self.assertIn(b'\x8c\x04FACE', s)  # SHORT_BINUNICODE
2144                self.assertFalse(opcode_in_pickle(pickle.NEWOBJ, s))
2145                self.assertEqual(opcode_in_pickle(pickle.NEWOBJ_EX, s),
2146                                 4 <= proto)
2147                y = self.loads(s)   # will raise TypeError if __init__ called
2148                self.assert_is_copy(x, y)
2149
2150    def test_newobj_list_slots(self):
2151        x = SlotList([1, 2, 3])
2152        x.foo = 42
2153        x.bar = "hello"
2154        s = self.dumps(x, 2)
2155        y = self.loads(s)
2156        self.assert_is_copy(x, y)
2157
2158    def test_reduce_overrides_default_reduce_ex(self):
2159        for proto in protocols:
2160            x = REX_one()
2161            self.assertEqual(x._reduce_called, 0)
2162            s = self.dumps(x, proto)
2163            self.assertEqual(x._reduce_called, 1)
2164            y = self.loads(s)
2165            self.assertEqual(y._reduce_called, 0)
2166
2167    def test_reduce_ex_called(self):
2168        for proto in protocols:
2169            x = REX_two()
2170            self.assertEqual(x._proto, None)
2171            s = self.dumps(x, proto)
2172            self.assertEqual(x._proto, proto)
2173            y = self.loads(s)
2174            self.assertEqual(y._proto, None)
2175
2176    def test_reduce_ex_overrides_reduce(self):
2177        for proto in protocols:
2178            x = REX_three()
2179            self.assertEqual(x._proto, None)
2180            s = self.dumps(x, proto)
2181            self.assertEqual(x._proto, proto)
2182            y = self.loads(s)
2183            self.assertEqual(y._proto, None)
2184
2185    def test_reduce_ex_calls_base(self):
2186        for proto in protocols:
2187            x = REX_four()
2188            self.assertEqual(x._proto, None)
2189            s = self.dumps(x, proto)
2190            self.assertEqual(x._proto, proto)
2191            y = self.loads(s)
2192            self.assertEqual(y._proto, proto)
2193
2194    def test_reduce_calls_base(self):
2195        for proto in protocols:
2196            x = REX_five()
2197            self.assertEqual(x._reduce_called, 0)
2198            s = self.dumps(x, proto)
2199            self.assertEqual(x._reduce_called, 1)
2200            y = self.loads(s)
2201            self.assertEqual(y._reduce_called, 1)
2202
2203    @no_tracing
2204    def test_bad_getattr(self):
2205        # Issue #3514: crash when there is an infinite loop in __getattr__
2206        x = BadGetattr()
2207        for proto in protocols:
2208            self.assertRaises(RuntimeError, self.dumps, x, proto)
2209
2210    def test_reduce_bad_iterator(self):
2211        # Issue4176: crash when 4th and 5th items of __reduce__()
2212        # are not iterators
2213        class C(object):
2214            def __reduce__(self):
2215                # 4th item is not an iterator
2216                return list, (), None, [], None
2217        class D(object):
2218            def __reduce__(self):
2219                # 5th item is not an iterator
2220                return dict, (), None, None, []
2221
2222        # Python implementation is less strict and also accepts iterables.
2223        for proto in protocols:
2224            try:
2225                self.dumps(C(), proto)
2226            except pickle.PicklingError:
2227                pass
2228            try:
2229                self.dumps(D(), proto)
2230            except pickle.PicklingError:
2231                pass
2232
2233    def test_many_puts_and_gets(self):
2234        # Test that internal data structures correctly deal with lots of
2235        # puts/gets.
2236        keys = ("aaa" + str(i) for i in range(100))
2237        large_dict = dict((k, [4, 5, 6]) for k in keys)
2238        obj = [dict(large_dict), dict(large_dict), dict(large_dict)]
2239
2240        for proto in protocols:
2241            with self.subTest(proto=proto):
2242                dumped = self.dumps(obj, proto)
2243                loaded = self.loads(dumped)
2244                self.assert_is_copy(obj, loaded)
2245
2246    def test_attribute_name_interning(self):
2247        # Test that attribute names of pickled objects are interned when
2248        # unpickling.
2249        for proto in protocols:
2250            x = C()
2251            x.foo = 42
2252            x.bar = "hello"
2253            s = self.dumps(x, proto)
2254            y = self.loads(s)
2255            x_keys = sorted(x.__dict__)
2256            y_keys = sorted(y.__dict__)
2257            for x_key, y_key in zip(x_keys, y_keys):
2258                self.assertIs(x_key, y_key)
2259
2260    def test_pickle_to_2x(self):
2261        # Pickle non-trivial data with protocol 2, expecting that it yields
2262        # the same result as Python 2.x did.
2263        # NOTE: this test is a bit too strong since we can produce different
2264        # bytecode that 2.x will still understand.
2265        dumped = self.dumps(range(5), 2)
2266        self.assertEqual(dumped, DATA_XRANGE)
2267        dumped = self.dumps(set([3]), 2)
2268        self.assertEqual(dumped, DATA_SET2)
2269
2270    def test_large_pickles(self):
2271        # Test the correctness of internal buffering routines when handling
2272        # large data.
2273        for proto in protocols:
2274            data = (1, min, b'xy' * (30 * 1024), len)
2275            dumped = self.dumps(data, proto)
2276            loaded = self.loads(dumped)
2277            self.assertEqual(len(loaded), len(data))
2278            self.assertEqual(loaded, data)
2279
2280    def test_int_pickling_efficiency(self):
2281        # Test compacity of int representation (see issue #12744)
2282        for proto in protocols:
2283            with self.subTest(proto=proto):
2284                pickles = [self.dumps(2**n, proto) for n in range(70)]
2285                sizes = list(map(len, pickles))
2286                # the size function is monotonic
2287                self.assertEqual(sorted(sizes), sizes)
2288                if proto >= 2:
2289                    for p in pickles:
2290                        self.assertFalse(opcode_in_pickle(pickle.LONG, p))
2291
2292    def _check_pickling_with_opcode(self, obj, opcode, proto):
2293        pickled = self.dumps(obj, proto)
2294        self.assertTrue(opcode_in_pickle(opcode, pickled))
2295        unpickled = self.loads(pickled)
2296        self.assertEqual(obj, unpickled)
2297
2298    def test_appends_on_non_lists(self):
2299        # Issue #17720
2300        obj = REX_six([1, 2, 3])
2301        for proto in protocols:
2302            if proto == 0:
2303                self._check_pickling_with_opcode(obj, pickle.APPEND, proto)
2304            else:
2305                self._check_pickling_with_opcode(obj, pickle.APPENDS, proto)
2306
2307    def test_setitems_on_non_dicts(self):
2308        obj = REX_seven({1: -1, 2: -2, 3: -3})
2309        for proto in protocols:
2310            if proto == 0:
2311                self._check_pickling_with_opcode(obj, pickle.SETITEM, proto)
2312            else:
2313                self._check_pickling_with_opcode(obj, pickle.SETITEMS, proto)
2314
2315    # Exercise framing (proto >= 4) for significant workloads
2316
2317    FRAME_SIZE_MIN = 4
2318    FRAME_SIZE_TARGET = 64 * 1024
2319
2320    def check_frame_opcodes(self, pickled):
2321        """
2322        Check the arguments of FRAME opcodes in a protocol 4+ pickle.
2323
2324        Note that binary objects that are larger than FRAME_SIZE_TARGET are not
2325        framed by default and are therefore considered a frame by themselves in
2326        the following consistency check.
2327        """
2328        frame_end = frameless_start = None
2329        frameless_opcodes = {'BINBYTES', 'BINUNICODE', 'BINBYTES8',
2330                             'BINUNICODE8', 'BYTEARRAY8'}
2331        for op, arg, pos in pickletools.genops(pickled):
2332            if frame_end is not None:
2333                self.assertLessEqual(pos, frame_end)
2334                if pos == frame_end:
2335                    frame_end = None
2336
2337            if frame_end is not None:  # framed
2338                self.assertNotEqual(op.name, 'FRAME')
2339                if op.name in frameless_opcodes:
2340                    # Only short bytes and str objects should be written
2341                    # in a frame
2342                    self.assertLessEqual(len(arg), self.FRAME_SIZE_TARGET)
2343
2344            else:  # not framed
2345                if (op.name == 'FRAME' or
2346                    (op.name in frameless_opcodes and
2347                     len(arg) > self.FRAME_SIZE_TARGET)):
2348                    # Frame or large bytes or str object
2349                    if frameless_start is not None:
2350                        # Only short data should be written outside of a frame
2351                        self.assertLess(pos - frameless_start,
2352                                        self.FRAME_SIZE_MIN)
2353                        frameless_start = None
2354                elif frameless_start is None and op.name != 'PROTO':
2355                    frameless_start = pos
2356
2357            if op.name == 'FRAME':
2358                self.assertGreaterEqual(arg, self.FRAME_SIZE_MIN)
2359                frame_end = pos + 9 + arg
2360
2361        pos = len(pickled)
2362        if frame_end is not None:
2363            self.assertEqual(frame_end, pos)
2364        elif frameless_start is not None:
2365            self.assertLess(pos - frameless_start, self.FRAME_SIZE_MIN)
2366
2367    @support.skip_if_pgo_task
2368    def test_framing_many_objects(self):
2369        obj = list(range(10**5))
2370        for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
2371            with self.subTest(proto=proto):
2372                pickled = self.dumps(obj, proto)
2373                unpickled = self.loads(pickled)
2374                self.assertEqual(obj, unpickled)
2375                bytes_per_frame = (len(pickled) /
2376                                   count_opcode(pickle.FRAME, pickled))
2377                self.assertGreater(bytes_per_frame,
2378                                   self.FRAME_SIZE_TARGET / 2)
2379                self.assertLessEqual(bytes_per_frame,
2380                                     self.FRAME_SIZE_TARGET * 1)
2381                self.check_frame_opcodes(pickled)
2382
2383    def test_framing_large_objects(self):
2384        N = 1024 * 1024
2385        small_items = [[i] for i in range(10)]
2386        obj = [b'x' * N, *small_items, b'y' * N, 'z' * N]
2387        for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
2388            for fast in [False, True]:
2389                with self.subTest(proto=proto, fast=fast):
2390                    if not fast:
2391                        # fast=False by default.
2392                        # This covers in-memory pickling with pickle.dumps().
2393                        pickled = self.dumps(obj, proto)
2394                    else:
2395                        # Pickler is required when fast=True.
2396                        if not hasattr(self, 'pickler'):
2397                            continue
2398                        buf = io.BytesIO()
2399                        pickler = self.pickler(buf, protocol=proto)
2400                        pickler.fast = fast
2401                        pickler.dump(obj)
2402                        pickled = buf.getvalue()
2403                    unpickled = self.loads(pickled)
2404                    # More informative error message in case of failure.
2405                    self.assertEqual([len(x) for x in obj],
2406                                     [len(x) for x in unpickled])
2407                    # Perform full equality check if the lengths match.
2408                    self.assertEqual(obj, unpickled)
2409                    n_frames = count_opcode(pickle.FRAME, pickled)
2410                    # A single frame for small objects between
2411                    # first two large objects.
2412                    self.assertEqual(n_frames, 1)
2413                    self.check_frame_opcodes(pickled)
2414
2415    def test_optional_frames(self):
2416        if pickle.HIGHEST_PROTOCOL < 4:
2417            return
2418
2419        def remove_frames(pickled, keep_frame=None):
2420            """Remove frame opcodes from the given pickle."""
2421            frame_starts = []
2422            # 1 byte for the opcode and 8 for the argument
2423            frame_opcode_size = 9
2424            for opcode, _, pos in pickletools.genops(pickled):
2425                if opcode.name == 'FRAME':
2426                    frame_starts.append(pos)
2427
2428            newpickle = bytearray()
2429            last_frame_end = 0
2430            for i, pos in enumerate(frame_starts):
2431                if keep_frame and keep_frame(i):
2432                    continue
2433                newpickle += pickled[last_frame_end:pos]
2434                last_frame_end = pos + frame_opcode_size
2435            newpickle += pickled[last_frame_end:]
2436            return newpickle
2437
2438        frame_size = self.FRAME_SIZE_TARGET
2439        num_frames = 20
2440        # Large byte objects (dict values) intermittent with small objects
2441        # (dict keys)
2442        for bytes_type in (bytes, bytearray):
2443            obj = {i: bytes_type([i]) * frame_size for i in range(num_frames)}
2444
2445            for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
2446                pickled = self.dumps(obj, proto)
2447
2448                frameless_pickle = remove_frames(pickled)
2449                self.assertEqual(count_opcode(pickle.FRAME, frameless_pickle), 0)
2450                self.assertEqual(obj, self.loads(frameless_pickle))
2451
2452                some_frames_pickle = remove_frames(pickled, lambda i: i % 2)
2453                self.assertLess(count_opcode(pickle.FRAME, some_frames_pickle),
2454                                count_opcode(pickle.FRAME, pickled))
2455                self.assertEqual(obj, self.loads(some_frames_pickle))
2456
2457    @support.skip_if_pgo_task
2458    def test_framed_write_sizes_with_delayed_writer(self):
2459        class ChunkAccumulator:
2460            """Accumulate pickler output in a list of raw chunks."""
2461            def __init__(self):
2462                self.chunks = []
2463            def write(self, chunk):
2464                self.chunks.append(chunk)
2465            def concatenate_chunks(self):
2466                return b"".join(self.chunks)
2467
2468        for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
2469            objects = [(str(i).encode('ascii'), i % 42, {'i': str(i)})
2470                       for i in range(int(1e4))]
2471            # Add a large unique ASCII string
2472            objects.append('0123456789abcdef' *
2473                           (self.FRAME_SIZE_TARGET // 16 + 1))
2474
2475            # Protocol 4 packs groups of small objects into frames and issues
2476            # calls to write only once or twice per frame:
2477            # The C pickler issues one call to write per-frame (header and
2478            # contents) while Python pickler issues two calls to write: one for
2479            # the frame header and one for the frame binary contents.
2480            writer = ChunkAccumulator()
2481            self.pickler(writer, proto).dump(objects)
2482
2483            # Actually read the binary content of the chunks after the end
2484            # of the call to dump: any memoryview passed to write should not
2485            # be released otherwise this delayed access would not be possible.
2486            pickled = writer.concatenate_chunks()
2487            reconstructed = self.loads(pickled)
2488            self.assertEqual(reconstructed, objects)
2489            self.assertGreater(len(writer.chunks), 1)
2490
2491            # memoryviews should own the memory.
2492            del objects
2493            support.gc_collect()
2494            self.assertEqual(writer.concatenate_chunks(), pickled)
2495
2496            n_frames = (len(pickled) - 1) // self.FRAME_SIZE_TARGET + 1
2497            # There should be at least one call to write per frame
2498            self.assertGreaterEqual(len(writer.chunks), n_frames)
2499
2500            # but not too many either: there can be one for the proto,
2501            # one per-frame header, one per frame for the actual contents,
2502            # and two for the header.
2503            self.assertLessEqual(len(writer.chunks), 2 * n_frames + 3)
2504
2505            chunk_sizes = [len(c) for c in writer.chunks]
2506            large_sizes = [s for s in chunk_sizes
2507                           if s >= self.FRAME_SIZE_TARGET]
2508            medium_sizes = [s for s in chunk_sizes
2509                           if 9 < s < self.FRAME_SIZE_TARGET]
2510            small_sizes = [s for s in chunk_sizes if s <= 9]
2511
2512            # Large chunks should not be too large:
2513            for chunk_size in large_sizes:
2514                self.assertLess(chunk_size, 2 * self.FRAME_SIZE_TARGET,
2515                                chunk_sizes)
2516            # There shouldn't bee too many small chunks: the protocol header,
2517            # the frame headers and the large string headers are written
2518            # in small chunks.
2519            self.assertLessEqual(len(small_sizes),
2520                                 len(large_sizes) + len(medium_sizes) + 3,
2521                                 chunk_sizes)
2522
2523    def test_nested_names(self):
2524        global Nested
2525        class Nested:
2526            class A:
2527                class B:
2528                    class C:
2529                        pass
2530        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
2531            for obj in [Nested.A, Nested.A.B, Nested.A.B.C]:
2532                with self.subTest(proto=proto, obj=obj):
2533                    unpickled = self.loads(self.dumps(obj, proto))
2534                    self.assertIs(obj, unpickled)
2535
2536    def test_recursive_nested_names(self):
2537        global Recursive
2538        class Recursive:
2539            pass
2540        Recursive.mod = sys.modules[Recursive.__module__]
2541        Recursive.__qualname__ = 'Recursive.mod.Recursive'
2542        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
2543            with self.subTest(proto=proto):
2544                unpickled = self.loads(self.dumps(Recursive, proto))
2545                self.assertIs(unpickled, Recursive)
2546        del Recursive.mod # break reference loop
2547
2548    def test_py_methods(self):
2549        global PyMethodsTest
2550        class PyMethodsTest:
2551            @staticmethod
2552            def cheese():
2553                return "cheese"
2554            @classmethod
2555            def wine(cls):
2556                assert cls is PyMethodsTest
2557                return "wine"
2558            def biscuits(self):
2559                assert isinstance(self, PyMethodsTest)
2560                return "biscuits"
2561            class Nested:
2562                "Nested class"
2563                @staticmethod
2564                def ketchup():
2565                    return "ketchup"
2566                @classmethod
2567                def maple(cls):
2568                    assert cls is PyMethodsTest.Nested
2569                    return "maple"
2570                def pie(self):
2571                    assert isinstance(self, PyMethodsTest.Nested)
2572                    return "pie"
2573
2574        py_methods = (
2575            PyMethodsTest.cheese,
2576            PyMethodsTest.wine,
2577            PyMethodsTest().biscuits,
2578            PyMethodsTest.Nested.ketchup,
2579            PyMethodsTest.Nested.maple,
2580            PyMethodsTest.Nested().pie
2581        )
2582        py_unbound_methods = (
2583            (PyMethodsTest.biscuits, PyMethodsTest),
2584            (PyMethodsTest.Nested.pie, PyMethodsTest.Nested)
2585        )
2586        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
2587            for method in py_methods:
2588                with self.subTest(proto=proto, method=method):
2589                    unpickled = self.loads(self.dumps(method, proto))
2590                    self.assertEqual(method(), unpickled())
2591            for method, cls in py_unbound_methods:
2592                obj = cls()
2593                with self.subTest(proto=proto, method=method):
2594                    unpickled = self.loads(self.dumps(method, proto))
2595                    self.assertEqual(method(obj), unpickled(obj))
2596
2597    def test_c_methods(self):
2598        global Subclass
2599        class Subclass(tuple):
2600            class Nested(str):
2601                pass
2602
2603        c_methods = (
2604            # bound built-in method
2605            ("abcd".index, ("c",)),
2606            # unbound built-in method
2607            (str.index, ("abcd", "c")),
2608            # bound "slot" method
2609            ([1, 2, 3].__len__, ()),
2610            # unbound "slot" method
2611            (list.__len__, ([1, 2, 3],)),
2612            # bound "coexist" method
2613            ({1, 2}.__contains__, (2,)),
2614            # unbound "coexist" method
2615            (set.__contains__, ({1, 2}, 2)),
2616            # built-in class method
2617            (dict.fromkeys, (("a", 1), ("b", 2))),
2618            # built-in static method
2619            (bytearray.maketrans, (b"abc", b"xyz")),
2620            # subclass methods
2621            (Subclass([1,2,2]).count, (2,)),
2622            (Subclass.count, (Subclass([1,2,2]), 2)),
2623            (Subclass.Nested("sweet").count, ("e",)),
2624            (Subclass.Nested.count, (Subclass.Nested("sweet"), "e")),
2625        )
2626        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
2627            for method, args in c_methods:
2628                with self.subTest(proto=proto, method=method):
2629                    unpickled = self.loads(self.dumps(method, proto))
2630                    self.assertEqual(method(*args), unpickled(*args))
2631
2632    def test_compat_pickle(self):
2633        tests = [
2634            (range(1, 7), '__builtin__', 'xrange'),
2635            (map(int, '123'), 'itertools', 'imap'),
2636            (functools.reduce, '__builtin__', 'reduce'),
2637            (dbm.whichdb, 'whichdb', 'whichdb'),
2638            (Exception(), 'exceptions', 'Exception'),
2639            (collections.UserDict(), 'UserDict', 'IterableUserDict'),
2640            (collections.UserList(), 'UserList', 'UserList'),
2641            (collections.defaultdict(), 'collections', 'defaultdict'),
2642        ]
2643        for val, mod, name in tests:
2644            for proto in range(3):
2645                with self.subTest(type=type(val), proto=proto):
2646                    pickled = self.dumps(val, proto)
2647                    self.assertIn(('c%s\n%s' % (mod, name)).encode(), pickled)
2648                    self.assertIs(type(self.loads(pickled)), type(val))
2649
2650    def test_local_lookup_error(self):
2651        # Test that whichmodule() errors out cleanly when looking up
2652        # an assumed globally-reachable object fails.
2653        def f():
2654            pass
2655        # Since the function is local, lookup will fail
2656        for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
2657            with self.assertRaises((AttributeError, pickle.PicklingError)):
2658                pickletools.dis(self.dumps(f, proto))
2659        # Same without a __module__ attribute (exercises a different path
2660        # in _pickle.c).
2661        del f.__module__
2662        for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
2663            with self.assertRaises((AttributeError, pickle.PicklingError)):
2664                pickletools.dis(self.dumps(f, proto))
2665        # Yet a different path.
2666        f.__name__ = f.__qualname__
2667        for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
2668            with self.assertRaises((AttributeError, pickle.PicklingError)):
2669                pickletools.dis(self.dumps(f, proto))
2670
2671    #
2672    # PEP 574 tests below
2673    #
2674
2675    def buffer_like_objects(self):
2676        # Yield buffer-like objects with the bytestring "abcdef" in them
2677        bytestring = b"abcdefgh"
2678        yield ZeroCopyBytes(bytestring)
2679        yield ZeroCopyBytearray(bytestring)
2680        if _testbuffer is not None:
2681            items = list(bytestring)
2682            value = int.from_bytes(bytestring, byteorder='little')
2683            for flags in (0, _testbuffer.ND_WRITABLE):
2684                # 1-D, contiguous
2685                yield PicklableNDArray(items, format='B', shape=(8,),
2686                                       flags=flags)
2687                # 2-D, C-contiguous
2688                yield PicklableNDArray(items, format='B', shape=(4, 2),
2689                                       strides=(2, 1), flags=flags)
2690                # 2-D, Fortran-contiguous
2691                yield PicklableNDArray(items, format='B',
2692                                       shape=(4, 2), strides=(1, 4),
2693                                       flags=flags)
2694
2695    def test_in_band_buffers(self):
2696        # Test in-band buffers (PEP 574)
2697        for obj in self.buffer_like_objects():
2698            for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
2699                data = self.dumps(obj, proto)
2700                if obj.c_contiguous and proto >= 5:
2701                    # The raw memory bytes are serialized in physical order
2702                    self.assertIn(b"abcdefgh", data)
2703                self.assertEqual(count_opcode(pickle.NEXT_BUFFER, data), 0)
2704                if proto >= 5:
2705                    self.assertEqual(count_opcode(pickle.SHORT_BINBYTES, data),
2706                                     1 if obj.readonly else 0)
2707                    self.assertEqual(count_opcode(pickle.BYTEARRAY8, data),
2708                                     0 if obj.readonly else 1)
2709                    # Return a true value from buffer_callback should have
2710                    # the same effect
2711                    def buffer_callback(obj):
2712                        return True
2713                    data2 = self.dumps(obj, proto,
2714                                       buffer_callback=buffer_callback)
2715                    self.assertEqual(data2, data)
2716
2717                new = self.loads(data)
2718                # It's a copy
2719                self.assertIsNot(new, obj)
2720                self.assertIs(type(new), type(obj))
2721                self.assertEqual(new, obj)
2722
2723    # XXX Unfortunately cannot test non-contiguous array
2724    # (see comment in PicklableNDArray.__reduce_ex__)
2725
2726    def test_oob_buffers(self):
2727        # Test out-of-band buffers (PEP 574)
2728        for obj in self.buffer_like_objects():
2729            for proto in range(0, 5):
2730                # Need protocol >= 5 for buffer_callback
2731                with self.assertRaises(ValueError):
2732                    self.dumps(obj, proto,
2733                               buffer_callback=[].append)
2734            for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
2735                buffers = []
2736                buffer_callback = lambda pb: buffers.append(pb.raw())
2737                data = self.dumps(obj, proto,
2738                                  buffer_callback=buffer_callback)
2739                self.assertNotIn(b"abcdefgh", data)
2740                self.assertEqual(count_opcode(pickle.SHORT_BINBYTES, data), 0)
2741                self.assertEqual(count_opcode(pickle.BYTEARRAY8, data), 0)
2742                self.assertEqual(count_opcode(pickle.NEXT_BUFFER, data), 1)
2743                self.assertEqual(count_opcode(pickle.READONLY_BUFFER, data),
2744                                 1 if obj.readonly else 0)
2745
2746                if obj.c_contiguous:
2747                    self.assertEqual(bytes(buffers[0]), b"abcdefgh")
2748                # Need buffers argument to unpickle properly
2749                with self.assertRaises(pickle.UnpicklingError):
2750                    self.loads(data)
2751
2752                new = self.loads(data, buffers=buffers)
2753                if obj.zero_copy_reconstruct:
2754                    # Zero-copy achieved
2755                    self.assertIs(new, obj)
2756                else:
2757                    self.assertIs(type(new), type(obj))
2758                    self.assertEqual(new, obj)
2759                # Non-sequence buffers accepted too
2760                new = self.loads(data, buffers=iter(buffers))
2761                if obj.zero_copy_reconstruct:
2762                    # Zero-copy achieved
2763                    self.assertIs(new, obj)
2764                else:
2765                    self.assertIs(type(new), type(obj))
2766                    self.assertEqual(new, obj)
2767
2768    def test_oob_buffers_writable_to_readonly(self):
2769        # Test reconstructing readonly object from writable buffer
2770        obj = ZeroCopyBytes(b"foobar")
2771        for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
2772            buffers = []
2773            buffer_callback = buffers.append
2774            data = self.dumps(obj, proto, buffer_callback=buffer_callback)
2775
2776            buffers = map(bytearray, buffers)
2777            new = self.loads(data, buffers=buffers)
2778            self.assertIs(type(new), type(obj))
2779            self.assertEqual(new, obj)
2780
2781    def test_picklebuffer_error(self):
2782        # PickleBuffer forbidden with protocol < 5
2783        pb = pickle.PickleBuffer(b"foobar")
2784        for proto in range(0, 5):
2785            with self.assertRaises(pickle.PickleError):
2786                self.dumps(pb, proto)
2787
2788    def test_buffer_callback_error(self):
2789        def buffer_callback(buffers):
2790            1/0
2791        pb = pickle.PickleBuffer(b"foobar")
2792        with self.assertRaises(ZeroDivisionError):
2793            self.dumps(pb, 5, buffer_callback=buffer_callback)
2794
2795    def test_buffers_error(self):
2796        pb = pickle.PickleBuffer(b"foobar")
2797        for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
2798            data = self.dumps(pb, proto, buffer_callback=[].append)
2799            # Non iterable buffers
2800            with self.assertRaises(TypeError):
2801                self.loads(data, buffers=object())
2802            # Buffer iterable exhausts too early
2803            with self.assertRaises(pickle.UnpicklingError):
2804                self.loads(data, buffers=[])
2805
2806    def test_inband_accept_default_buffers_argument(self):
2807        for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
2808            data_pickled = self.dumps(1, proto, buffer_callback=None)
2809            data = self.loads(data_pickled, buffers=None)
2810
2811    @unittest.skipIf(np is None, "Test needs Numpy")
2812    def test_buffers_numpy(self):
2813        def check_no_copy(x, y):
2814            np.testing.assert_equal(x, y)
2815            self.assertEqual(x.ctypes.data, y.ctypes.data)
2816
2817        def check_copy(x, y):
2818            np.testing.assert_equal(x, y)
2819            self.assertNotEqual(x.ctypes.data, y.ctypes.data)
2820
2821        def check_array(arr):
2822            # In-band
2823            for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
2824                data = self.dumps(arr, proto)
2825                new = self.loads(data)
2826                check_copy(arr, new)
2827            for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
2828                buffer_callback = lambda _: True
2829                data = self.dumps(arr, proto, buffer_callback=buffer_callback)
2830                new = self.loads(data)
2831                check_copy(arr, new)
2832            # Out-of-band
2833            for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
2834                buffers = []
2835                buffer_callback = buffers.append
2836                data = self.dumps(arr, proto, buffer_callback=buffer_callback)
2837                new = self.loads(data, buffers=buffers)
2838                if arr.flags.c_contiguous or arr.flags.f_contiguous:
2839                    check_no_copy(arr, new)
2840                else:
2841                    check_copy(arr, new)
2842
2843        # 1-D
2844        arr = np.arange(6)
2845        check_array(arr)
2846        # 1-D, non-contiguous
2847        check_array(arr[::2])
2848        # 2-D, C-contiguous
2849        arr = np.arange(12).reshape((3, 4))
2850        check_array(arr)
2851        # 2-D, F-contiguous
2852        check_array(arr.T)
2853        # 2-D, non-contiguous
2854        check_array(arr[::2])
2855
2856
2857class BigmemPickleTests(unittest.TestCase):
2858
2859    # Binary protocols can serialize longs of up to 2 GiB-1
2860
2861    @bigmemtest(size=_2G, memuse=3.6, dry_run=False)
2862    def test_huge_long_32b(self, size):
2863        data = 1 << (8 * size)
2864        try:
2865            for proto in protocols:
2866                if proto < 2:
2867                    continue
2868                with self.subTest(proto=proto):
2869                    with self.assertRaises((ValueError, OverflowError)):
2870                        self.dumps(data, protocol=proto)
2871        finally:
2872            data = None
2873
2874    # Protocol 3 can serialize up to 4 GiB-1 as a bytes object
2875    # (older protocols don't have a dedicated opcode for bytes and are
2876    # too inefficient)
2877
2878    @bigmemtest(size=_2G, memuse=2.5, dry_run=False)
2879    def test_huge_bytes_32b(self, size):
2880        data = b"abcd" * (size // 4)
2881        try:
2882            for proto in protocols:
2883                if proto < 3:
2884                    continue
2885                with self.subTest(proto=proto):
2886                    try:
2887                        pickled = self.dumps(data, protocol=proto)
2888                        header = (pickle.BINBYTES +
2889                                  struct.pack("<I", len(data)))
2890                        data_start = pickled.index(data)
2891                        self.assertEqual(
2892                            header,
2893                            pickled[data_start-len(header):data_start])
2894                    finally:
2895                        pickled = None
2896        finally:
2897            data = None
2898
2899    @bigmemtest(size=_4G, memuse=2.5, dry_run=False)
2900    def test_huge_bytes_64b(self, size):
2901        data = b"acbd" * (size // 4)
2902        try:
2903            for proto in protocols:
2904                if proto < 3:
2905                    continue
2906                with self.subTest(proto=proto):
2907                    if proto == 3:
2908                        # Protocol 3 does not support large bytes objects.
2909                        # Verify that we do not crash when processing one.
2910                        with self.assertRaises((ValueError, OverflowError)):
2911                            self.dumps(data, protocol=proto)
2912                        continue
2913                    try:
2914                        pickled = self.dumps(data, protocol=proto)
2915                        header = (pickle.BINBYTES8 +
2916                                  struct.pack("<Q", len(data)))
2917                        data_start = pickled.index(data)
2918                        self.assertEqual(
2919                            header,
2920                            pickled[data_start-len(header):data_start])
2921                    finally:
2922                        pickled = None
2923        finally:
2924            data = None
2925
2926    # All protocols use 1-byte per printable ASCII character; we add another
2927    # byte because the encoded form has to be copied into the internal buffer.
2928
2929    @bigmemtest(size=_2G, memuse=8, dry_run=False)
2930    def test_huge_str_32b(self, size):
2931        data = "abcd" * (size // 4)
2932        try:
2933            for proto in protocols:
2934                if proto == 0:
2935                    continue
2936                with self.subTest(proto=proto):
2937                    try:
2938                        pickled = self.dumps(data, protocol=proto)
2939                        header = (pickle.BINUNICODE +
2940                                  struct.pack("<I", len(data)))
2941                        data_start = pickled.index(b'abcd')
2942                        self.assertEqual(
2943                            header,
2944                            pickled[data_start-len(header):data_start])
2945                        self.assertEqual((pickled.rindex(b"abcd") + len(b"abcd") -
2946                                          pickled.index(b"abcd")), len(data))
2947                    finally:
2948                        pickled = None
2949        finally:
2950            data = None
2951
2952    # BINUNICODE (protocols 1, 2 and 3) cannot carry more than 2**32 - 1 bytes
2953    # of utf-8 encoded unicode. BINUNICODE8 (protocol 4) supports these huge
2954    # unicode strings however.
2955
2956    @bigmemtest(size=_4G, memuse=8, dry_run=False)
2957    def test_huge_str_64b(self, size):
2958        data = "abcd" * (size // 4)
2959        try:
2960            for proto in protocols:
2961                if proto == 0:
2962                    continue
2963                with self.subTest(proto=proto):
2964                    if proto < 4:
2965                        with self.assertRaises((ValueError, OverflowError)):
2966                            self.dumps(data, protocol=proto)
2967                        continue
2968                    try:
2969                        pickled = self.dumps(data, protocol=proto)
2970                        header = (pickle.BINUNICODE8 +
2971                                  struct.pack("<Q", len(data)))
2972                        data_start = pickled.index(b'abcd')
2973                        self.assertEqual(
2974                            header,
2975                            pickled[data_start-len(header):data_start])
2976                        self.assertEqual((pickled.rindex(b"abcd") + len(b"abcd") -
2977                                          pickled.index(b"abcd")), len(data))
2978                    finally:
2979                        pickled = None
2980        finally:
2981            data = None
2982
2983
2984# Test classes for reduce_ex
2985
2986class REX_one(object):
2987    """No __reduce_ex__ here, but inheriting it from object"""
2988    _reduce_called = 0
2989    def __reduce__(self):
2990        self._reduce_called = 1
2991        return REX_one, ()
2992
2993class REX_two(object):
2994    """No __reduce__ here, but inheriting it from object"""
2995    _proto = None
2996    def __reduce_ex__(self, proto):
2997        self._proto = proto
2998        return REX_two, ()
2999
3000class REX_three(object):
3001    _proto = None
3002    def __reduce_ex__(self, proto):
3003        self._proto = proto
3004        return REX_two, ()
3005    def __reduce__(self):
3006        raise TestFailed("This __reduce__ shouldn't be called")
3007
3008class REX_four(object):
3009    """Calling base class method should succeed"""
3010    _proto = None
3011    def __reduce_ex__(self, proto):
3012        self._proto = proto
3013        return object.__reduce_ex__(self, proto)
3014
3015class REX_five(object):
3016    """This one used to fail with infinite recursion"""
3017    _reduce_called = 0
3018    def __reduce__(self):
3019        self._reduce_called = 1
3020        return object.__reduce__(self)
3021
3022class REX_six(object):
3023    """This class is used to check the 4th argument (list iterator) of
3024    the reduce protocol.
3025    """
3026    def __init__(self, items=None):
3027        self.items = items if items is not None else []
3028    def __eq__(self, other):
3029        return type(self) is type(other) and self.items == other.items
3030    def append(self, item):
3031        self.items.append(item)
3032    def __reduce__(self):
3033        return type(self), (), None, iter(self.items), None
3034
3035class REX_seven(object):
3036    """This class is used to check the 5th argument (dict iterator) of
3037    the reduce protocol.
3038    """
3039    def __init__(self, table=None):
3040        self.table = table if table is not None else {}
3041    def __eq__(self, other):
3042        return type(self) is type(other) and self.table == other.table
3043    def __setitem__(self, key, value):
3044        self.table[key] = value
3045    def __reduce__(self):
3046        return type(self), (), None, None, iter(self.table.items())
3047
3048
3049# Test classes for newobj
3050
3051class MyInt(int):
3052    sample = 1
3053
3054class MyFloat(float):
3055    sample = 1.0
3056
3057class MyComplex(complex):
3058    sample = 1.0 + 0.0j
3059
3060class MyStr(str):
3061    sample = "hello"
3062
3063class MyUnicode(str):
3064    sample = "hello \u1234"
3065
3066class MyTuple(tuple):
3067    sample = (1, 2, 3)
3068
3069class MyList(list):
3070    sample = [1, 2, 3]
3071
3072class MyDict(dict):
3073    sample = {"a": 1, "b": 2}
3074
3075class MySet(set):
3076    sample = {"a", "b"}
3077
3078class MyFrozenSet(frozenset):
3079    sample = frozenset({"a", "b"})
3080
3081myclasses = [MyInt, MyFloat,
3082             MyComplex,
3083             MyStr, MyUnicode,
3084             MyTuple, MyList, MyDict, MySet, MyFrozenSet]
3085
3086
3087class SlotList(MyList):
3088    __slots__ = ["foo"]
3089
3090class SimpleNewObj(int):
3091    def __init__(self, *args, **kwargs):
3092        # raise an error, to make sure this isn't called
3093        raise TypeError("SimpleNewObj.__init__() didn't expect to get called")
3094    def __eq__(self, other):
3095        return int(self) == int(other) and self.__dict__ == other.__dict__
3096
3097class ComplexNewObj(SimpleNewObj):
3098    def __getnewargs__(self):
3099        return ('%X' % self, 16)
3100
3101class ComplexNewObjEx(SimpleNewObj):
3102    def __getnewargs_ex__(self):
3103        return ('%X' % self,), {'base': 16}
3104
3105class BadGetattr:
3106    def __getattr__(self, key):
3107        self.foo
3108
3109
3110class AbstractPickleModuleTests(unittest.TestCase):
3111
3112    def test_dump_closed_file(self):
3113        f = open(TESTFN, "wb")
3114        try:
3115            f.close()
3116            self.assertRaises(ValueError, self.dump, 123, f)
3117        finally:
3118            support.unlink(TESTFN)
3119
3120    def test_load_closed_file(self):
3121        f = open(TESTFN, "wb")
3122        try:
3123            f.close()
3124            self.assertRaises(ValueError, self.dump, 123, f)
3125        finally:
3126            support.unlink(TESTFN)
3127
3128    def test_load_from_and_dump_to_file(self):
3129        stream = io.BytesIO()
3130        data = [123, {}, 124]
3131        self.dump(data, stream)
3132        stream.seek(0)
3133        unpickled = self.load(stream)
3134        self.assertEqual(unpickled, data)
3135
3136    def test_highest_protocol(self):
3137        # Of course this needs to be changed when HIGHEST_PROTOCOL changes.
3138        self.assertEqual(pickle.HIGHEST_PROTOCOL, 5)
3139
3140    def test_callapi(self):
3141        f = io.BytesIO()
3142        # With and without keyword arguments
3143        self.dump(123, f, -1)
3144        self.dump(123, file=f, protocol=-1)
3145        self.dumps(123, -1)
3146        self.dumps(123, protocol=-1)
3147        self.Pickler(f, -1)
3148        self.Pickler(f, protocol=-1)
3149
3150    def test_dump_text_file(self):
3151        f = open(TESTFN, "w")
3152        try:
3153            for proto in protocols:
3154                self.assertRaises(TypeError, self.dump, 123, f, proto)
3155        finally:
3156            f.close()
3157            support.unlink(TESTFN)
3158
3159    def test_incomplete_input(self):
3160        s = io.BytesIO(b"X''.")
3161        self.assertRaises((EOFError, struct.error, pickle.UnpicklingError), self.load, s)
3162
3163    def test_bad_init(self):
3164        # Test issue3664 (pickle can segfault from a badly initialized Pickler).
3165        # Override initialization without calling __init__() of the superclass.
3166        class BadPickler(self.Pickler):
3167            def __init__(self): pass
3168
3169        class BadUnpickler(self.Unpickler):
3170            def __init__(self): pass
3171
3172        self.assertRaises(pickle.PicklingError, BadPickler().dump, 0)
3173        self.assertRaises(pickle.UnpicklingError, BadUnpickler().load)
3174
3175    def check_dumps_loads_oob_buffers(self, dumps, loads):
3176        # No need to do the full gamut of tests here, just enough to
3177        # check that dumps() and loads() redirect their arguments
3178        # to the underlying Pickler and Unpickler, respectively.
3179        obj = ZeroCopyBytes(b"foo")
3180
3181        for proto in range(0, 5):
3182            # Need protocol >= 5 for buffer_callback
3183            with self.assertRaises(ValueError):
3184                dumps(obj, protocol=proto,
3185                      buffer_callback=[].append)
3186        for proto in range(5, pickle.HIGHEST_PROTOCOL + 1):
3187            buffers = []
3188            buffer_callback = buffers.append
3189            data = dumps(obj, protocol=proto,
3190                         buffer_callback=buffer_callback)
3191            self.assertNotIn(b"foo", data)
3192            self.assertEqual(bytes(buffers[0]), b"foo")
3193            # Need buffers argument to unpickle properly
3194            with self.assertRaises(pickle.UnpicklingError):
3195                loads(data)
3196            new = loads(data, buffers=buffers)
3197            self.assertIs(new, obj)
3198
3199    def test_dumps_loads_oob_buffers(self):
3200        # Test out-of-band buffers (PEP 574) with top-level dumps() and loads()
3201        self.check_dumps_loads_oob_buffers(self.dumps, self.loads)
3202
3203    def test_dump_load_oob_buffers(self):
3204        # Test out-of-band buffers (PEP 574) with top-level dump() and load()
3205        def dumps(obj, **kwargs):
3206            f = io.BytesIO()
3207            self.dump(obj, f, **kwargs)
3208            return f.getvalue()
3209
3210        def loads(data, **kwargs):
3211            f = io.BytesIO(data)
3212            return self.load(f, **kwargs)
3213
3214        self.check_dumps_loads_oob_buffers(dumps, loads)
3215
3216
3217class AbstractPersistentPicklerTests(unittest.TestCase):
3218
3219    # This class defines persistent_id() and persistent_load()
3220    # functions that should be used by the pickler.  All even integers
3221    # are pickled using persistent ids.
3222
3223    def persistent_id(self, object):
3224        if isinstance(object, int) and object % 2 == 0:
3225            self.id_count += 1
3226            return str(object)
3227        elif object == "test_false_value":
3228            self.false_count += 1
3229            return ""
3230        else:
3231            return None
3232
3233    def persistent_load(self, oid):
3234        if not oid:
3235            self.load_false_count += 1
3236            return "test_false_value"
3237        else:
3238            self.load_count += 1
3239            object = int(oid)
3240            assert object % 2 == 0
3241            return object
3242
3243    def test_persistence(self):
3244        L = list(range(10)) + ["test_false_value"]
3245        for proto in protocols:
3246            self.id_count = 0
3247            self.false_count = 0
3248            self.load_false_count = 0
3249            self.load_count = 0
3250            self.assertEqual(self.loads(self.dumps(L, proto)), L)
3251            self.assertEqual(self.id_count, 5)
3252            self.assertEqual(self.false_count, 1)
3253            self.assertEqual(self.load_count, 5)
3254            self.assertEqual(self.load_false_count, 1)
3255
3256
3257class AbstractIdentityPersistentPicklerTests(unittest.TestCase):
3258
3259    def persistent_id(self, obj):
3260        return obj
3261
3262    def persistent_load(self, pid):
3263        return pid
3264
3265    def _check_return_correct_type(self, obj, proto):
3266        unpickled = self.loads(self.dumps(obj, proto))
3267        self.assertIsInstance(unpickled, type(obj))
3268        self.assertEqual(unpickled, obj)
3269
3270    def test_return_correct_type(self):
3271        for proto in protocols:
3272            # Protocol 0 supports only ASCII strings.
3273            if proto == 0:
3274                self._check_return_correct_type("abc", 0)
3275            else:
3276                for obj in [b"abc\n", "abc\n", -1, -1.1 * 0.1, str]:
3277                    self._check_return_correct_type(obj, proto)
3278
3279    def test_protocol0_is_ascii_only(self):
3280        non_ascii_str = "\N{EMPTY SET}"
3281        self.assertRaises(pickle.PicklingError, self.dumps, non_ascii_str, 0)
3282        pickled = pickle.PERSID + non_ascii_str.encode('utf-8') + b'\n.'
3283        self.assertRaises(pickle.UnpicklingError, self.loads, pickled)
3284
3285
3286class AbstractPicklerUnpicklerObjectTests(unittest.TestCase):
3287
3288    pickler_class = None
3289    unpickler_class = None
3290
3291    def setUp(self):
3292        assert self.pickler_class
3293        assert self.unpickler_class
3294
3295    def test_clear_pickler_memo(self):
3296        # To test whether clear_memo() has any effect, we pickle an object,
3297        # then pickle it again without clearing the memo; the two serialized
3298        # forms should be different. If we clear_memo() and then pickle the
3299        # object again, the third serialized form should be identical to the
3300        # first one we obtained.
3301        data = ["abcdefg", "abcdefg", 44]
3302        for proto in protocols:
3303            f = io.BytesIO()
3304            pickler = self.pickler_class(f, proto)
3305
3306            pickler.dump(data)
3307            first_pickled = f.getvalue()
3308
3309            # Reset BytesIO object.
3310            f.seek(0)
3311            f.truncate()
3312
3313            pickler.dump(data)
3314            second_pickled = f.getvalue()
3315
3316            # Reset the Pickler and BytesIO objects.
3317            pickler.clear_memo()
3318            f.seek(0)
3319            f.truncate()
3320
3321            pickler.dump(data)
3322            third_pickled = f.getvalue()
3323
3324            self.assertNotEqual(first_pickled, second_pickled)
3325            self.assertEqual(first_pickled, third_pickled)
3326
3327    def test_priming_pickler_memo(self):
3328        # Verify that we can set the Pickler's memo attribute.
3329        data = ["abcdefg", "abcdefg", 44]
3330        f = io.BytesIO()
3331        pickler = self.pickler_class(f)
3332
3333        pickler.dump(data)
3334        first_pickled = f.getvalue()
3335
3336        f = io.BytesIO()
3337        primed = self.pickler_class(f)
3338        primed.memo = pickler.memo
3339
3340        primed.dump(data)
3341        primed_pickled = f.getvalue()
3342
3343        self.assertNotEqual(first_pickled, primed_pickled)
3344
3345    def test_priming_unpickler_memo(self):
3346        # Verify that we can set the Unpickler's memo attribute.
3347        data = ["abcdefg", "abcdefg", 44]
3348        f = io.BytesIO()
3349        pickler = self.pickler_class(f)
3350
3351        pickler.dump(data)
3352        first_pickled = f.getvalue()
3353
3354        f = io.BytesIO()
3355        primed = self.pickler_class(f)
3356        primed.memo = pickler.memo
3357
3358        primed.dump(data)
3359        primed_pickled = f.getvalue()
3360
3361        unpickler = self.unpickler_class(io.BytesIO(first_pickled))
3362        unpickled_data1 = unpickler.load()
3363
3364        self.assertEqual(unpickled_data1, data)
3365
3366        primed = self.unpickler_class(io.BytesIO(primed_pickled))
3367        primed.memo = unpickler.memo
3368        unpickled_data2 = primed.load()
3369
3370        primed.memo.clear()
3371
3372        self.assertEqual(unpickled_data2, data)
3373        self.assertTrue(unpickled_data2 is unpickled_data1)
3374
3375    def test_reusing_unpickler_objects(self):
3376        data1 = ["abcdefg", "abcdefg", 44]
3377        f = io.BytesIO()
3378        pickler = self.pickler_class(f)
3379        pickler.dump(data1)
3380        pickled1 = f.getvalue()
3381
3382        data2 = ["abcdefg", 44, 44]
3383        f = io.BytesIO()
3384        pickler = self.pickler_class(f)
3385        pickler.dump(data2)
3386        pickled2 = f.getvalue()
3387
3388        f = io.BytesIO()
3389        f.write(pickled1)
3390        f.seek(0)
3391        unpickler = self.unpickler_class(f)
3392        self.assertEqual(unpickler.load(), data1)
3393
3394        f.seek(0)
3395        f.truncate()
3396        f.write(pickled2)
3397        f.seek(0)
3398        self.assertEqual(unpickler.load(), data2)
3399
3400    def _check_multiple_unpicklings(self, ioclass, *, seekable=True):
3401        for proto in protocols:
3402            with self.subTest(proto=proto):
3403                data1 = [(x, str(x)) for x in range(2000)] + [b"abcde", len]
3404                f = ioclass()
3405                pickler = self.pickler_class(f, protocol=proto)
3406                pickler.dump(data1)
3407                pickled = f.getvalue()
3408
3409                N = 5
3410                f = ioclass(pickled * N)
3411                unpickler = self.unpickler_class(f)
3412                for i in range(N):
3413                    if seekable:
3414                        pos = f.tell()
3415                    self.assertEqual(unpickler.load(), data1)
3416                    if seekable:
3417                        self.assertEqual(f.tell(), pos + len(pickled))
3418                self.assertRaises(EOFError, unpickler.load)
3419
3420    def test_multiple_unpicklings_seekable(self):
3421        self._check_multiple_unpicklings(io.BytesIO)
3422
3423    def test_multiple_unpicklings_unseekable(self):
3424        self._check_multiple_unpicklings(UnseekableIO, seekable=False)
3425
3426    def test_multiple_unpicklings_minimal(self):
3427        # File-like object that doesn't support peek() and readinto()
3428        # (bpo-39681)
3429        self._check_multiple_unpicklings(MinimalIO, seekable=False)
3430
3431    def test_unpickling_buffering_readline(self):
3432        # Issue #12687: the unpickler's buffering logic could fail with
3433        # text mode opcodes.
3434        data = list(range(10))
3435        for proto in protocols:
3436            for buf_size in range(1, 11):
3437                f = io.BufferedRandom(io.BytesIO(), buffer_size=buf_size)
3438                pickler = self.pickler_class(f, protocol=proto)
3439                pickler.dump(data)
3440                f.seek(0)
3441                unpickler = self.unpickler_class(f)
3442                self.assertEqual(unpickler.load(), data)
3443
3444
3445# Tests for dispatch_table attribute
3446
3447REDUCE_A = 'reduce_A'
3448
3449class AAA(object):
3450    def __reduce__(self):
3451        return str, (REDUCE_A,)
3452
3453class BBB(object):
3454    def __init__(self):
3455        # Add an instance attribute to enable state-saving routines at pickling
3456        # time.
3457        self.a = "some attribute"
3458
3459    def __setstate__(self, state):
3460        self.a = "BBB.__setstate__"
3461
3462
3463def setstate_bbb(obj, state):
3464    """Custom state setter for BBB objects
3465
3466    Such callable may be created by other persons than the ones who created the
3467    BBB class. If passed as the state_setter item of a custom reducer, this
3468    allows for custom state setting behavior of BBB objects. One can think of
3469    it as the analogous of list_setitems or dict_setitems but for foreign
3470    classes/functions.
3471    """
3472    obj.a = "custom state_setter"
3473
3474
3475
3476class AbstractCustomPicklerClass:
3477    """Pickler implementing a reducing hook using reducer_override."""
3478    def reducer_override(self, obj):
3479        obj_name = getattr(obj, "__name__", None)
3480
3481        if obj_name == 'f':
3482            # asking the pickler to save f as 5
3483            return int, (5, )
3484
3485        if obj_name == 'MyClass':
3486            return str, ('some str',)
3487
3488        elif obj_name == 'g':
3489            # in this case, the callback returns an invalid result (not a 2-5
3490            # tuple or a string), the pickler should raise a proper error.
3491            return False
3492
3493        elif obj_name == 'h':
3494            # Simulate a case when the reducer fails. The error should
3495            # be propagated to the original ``dump`` call.
3496            raise ValueError('The reducer just failed')
3497
3498        return NotImplemented
3499
3500class AbstractHookTests(unittest.TestCase):
3501    def test_pickler_hook(self):
3502        # test the ability of a custom, user-defined CPickler subclass to
3503        # override the default reducing routines of any type using the method
3504        # reducer_override
3505
3506        def f():
3507            pass
3508
3509        def g():
3510            pass
3511
3512        def h():
3513            pass
3514
3515        class MyClass:
3516            pass
3517
3518        for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
3519            with self.subTest(proto=proto):
3520                bio = io.BytesIO()
3521                p = self.pickler_class(bio, proto)
3522
3523                p.dump([f, MyClass, math.log])
3524                new_f, some_str, math_log = pickle.loads(bio.getvalue())
3525
3526                self.assertEqual(new_f, 5)
3527                self.assertEqual(some_str, 'some str')
3528                # math.log does not have its usual reducer overriden, so the
3529                # custom reduction callback should silently direct the pickler
3530                # to the default pickling by attribute, by returning
3531                # NotImplemented
3532                self.assertIs(math_log, math.log)
3533
3534                with self.assertRaises(pickle.PicklingError):
3535                    p.dump(g)
3536
3537                with self.assertRaisesRegex(
3538                        ValueError, 'The reducer just failed'):
3539                    p.dump(h)
3540
3541    @support.cpython_only
3542    def test_reducer_override_no_reference_cycle(self):
3543        # bpo-39492: reducer_override used to induce a spurious reference cycle
3544        # inside the Pickler object, that could prevent all serialized objects
3545        # from being garbage-collected without explicity invoking gc.collect.
3546
3547        for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
3548            with self.subTest(proto=proto):
3549                def f():
3550                    pass
3551
3552                wr = weakref.ref(f)
3553
3554                bio = io.BytesIO()
3555                p = self.pickler_class(bio, proto)
3556                p.dump(f)
3557                new_f = pickle.loads(bio.getvalue())
3558                assert new_f == 5
3559
3560                del p
3561                del f
3562
3563                self.assertIsNone(wr())
3564
3565
3566class AbstractDispatchTableTests(unittest.TestCase):
3567
3568    def test_default_dispatch_table(self):
3569        # No dispatch_table attribute by default
3570        f = io.BytesIO()
3571        p = self.pickler_class(f, 0)
3572        with self.assertRaises(AttributeError):
3573            p.dispatch_table
3574        self.assertFalse(hasattr(p, 'dispatch_table'))
3575
3576    def test_class_dispatch_table(self):
3577        # A dispatch_table attribute can be specified class-wide
3578        dt = self.get_dispatch_table()
3579
3580        class MyPickler(self.pickler_class):
3581            dispatch_table = dt
3582
3583        def dumps(obj, protocol=None):
3584            f = io.BytesIO()
3585            p = MyPickler(f, protocol)
3586            self.assertEqual(p.dispatch_table, dt)
3587            p.dump(obj)
3588            return f.getvalue()
3589
3590        self._test_dispatch_table(dumps, dt)
3591
3592    def test_instance_dispatch_table(self):
3593        # A dispatch_table attribute can also be specified instance-wide
3594        dt = self.get_dispatch_table()
3595
3596        def dumps(obj, protocol=None):
3597            f = io.BytesIO()
3598            p = self.pickler_class(f, protocol)
3599            p.dispatch_table = dt
3600            self.assertEqual(p.dispatch_table, dt)
3601            p.dump(obj)
3602            return f.getvalue()
3603
3604        self._test_dispatch_table(dumps, dt)
3605
3606    def _test_dispatch_table(self, dumps, dispatch_table):
3607        def custom_load_dump(obj):
3608            return pickle.loads(dumps(obj, 0))
3609
3610        def default_load_dump(obj):
3611            return pickle.loads(pickle.dumps(obj, 0))
3612
3613        # pickling complex numbers using protocol 0 relies on copyreg
3614        # so check pickling a complex number still works
3615        z = 1 + 2j
3616        self.assertEqual(custom_load_dump(z), z)
3617        self.assertEqual(default_load_dump(z), z)
3618
3619        # modify pickling of complex
3620        REDUCE_1 = 'reduce_1'
3621        def reduce_1(obj):
3622            return str, (REDUCE_1,)
3623        dispatch_table[complex] = reduce_1
3624        self.assertEqual(custom_load_dump(z), REDUCE_1)
3625        self.assertEqual(default_load_dump(z), z)
3626
3627        # check picklability of AAA and BBB
3628        a = AAA()
3629        b = BBB()
3630        self.assertEqual(custom_load_dump(a), REDUCE_A)
3631        self.assertIsInstance(custom_load_dump(b), BBB)
3632        self.assertEqual(default_load_dump(a), REDUCE_A)
3633        self.assertIsInstance(default_load_dump(b), BBB)
3634
3635        # modify pickling of BBB
3636        dispatch_table[BBB] = reduce_1
3637        self.assertEqual(custom_load_dump(a), REDUCE_A)
3638        self.assertEqual(custom_load_dump(b), REDUCE_1)
3639        self.assertEqual(default_load_dump(a), REDUCE_A)
3640        self.assertIsInstance(default_load_dump(b), BBB)
3641
3642        # revert pickling of BBB and modify pickling of AAA
3643        REDUCE_2 = 'reduce_2'
3644        def reduce_2(obj):
3645            return str, (REDUCE_2,)
3646        dispatch_table[AAA] = reduce_2
3647        del dispatch_table[BBB]
3648        self.assertEqual(custom_load_dump(a), REDUCE_2)
3649        self.assertIsInstance(custom_load_dump(b), BBB)
3650        self.assertEqual(default_load_dump(a), REDUCE_A)
3651        self.assertIsInstance(default_load_dump(b), BBB)
3652
3653        # End-to-end testing of save_reduce with the state_setter keyword
3654        # argument. This is a dispatch_table test as the primary goal of
3655        # state_setter is to tweak objects reduction behavior.
3656        # In particular, state_setter is useful when the default __setstate__
3657        # behavior is not flexible enough.
3658
3659        # No custom reducer for b has been registered for now, so
3660        # BBB.__setstate__ should be used at unpickling time
3661        self.assertEqual(default_load_dump(b).a, "BBB.__setstate__")
3662
3663        def reduce_bbb(obj):
3664            return BBB, (), obj.__dict__, None, None, setstate_bbb
3665
3666        dispatch_table[BBB] = reduce_bbb
3667
3668        # The custom reducer reduce_bbb includes a state setter, that should
3669        # have priority over BBB.__setstate__
3670        self.assertEqual(custom_load_dump(b).a, "custom state_setter")
3671
3672
3673if __name__ == "__main__":
3674    # Print some stuff that can be used to rewrite DATA{0,1,2}
3675    from pickletools import dis
3676    x = create_data()
3677    for i in range(pickle.HIGHEST_PROTOCOL+1):
3678        p = pickle.dumps(x, i)
3679        print("DATA{0} = (".format(i))
3680        for j in range(0, len(p), 20):
3681            b = bytes(p[j:j+20])
3682            print("    {0!r}".format(b))
3683        print(")")
3684        print()
3685        print("# Disassembly of DATA{0}".format(i))
3686        print("DATA{0}_DIS = \"\"\"\\".format(i))
3687        dis(p)
3688        print("\"\"\"")
3689        print()
3690