1import collections 2import copyreg 3import dbm 4import io 5import functools 6import os 7import math 8import pickle 9import pickletools 10import shutil 11import struct 12import sys 13import threading 14import unittest 15import weakref 16from textwrap import dedent 17from http.cookies import SimpleCookie 18 19try: 20 import _testbuffer 21except ImportError: 22 _testbuffer = None 23 24from test import support 25from test.support import ( 26 TestFailed, TESTFN, run_with_locale, no_tracing, 27 _2G, _4G, bigmemtest, reap_threads, forget, 28 save_restore_warnings_filters 29 ) 30 31from pickle import bytes_types 32 33 34# bpo-41003: Save/restore warnings filters to leave them unchanged. 35# Ignore filters installed by numpy. 36try: 37 with save_restore_warnings_filters(): 38 import numpy as np 39except ImportError: 40 np = None 41 42 43requires_32b = unittest.skipUnless(sys.maxsize < 2**32, 44 "test is only meaningful on 32-bit builds") 45 46# Tests that try a number of pickle protocols should have a 47# for proto in protocols: 48# kind of outer loop. 49protocols = range(pickle.HIGHEST_PROTOCOL + 1) 50 51 52# Return True if opcode code appears in the pickle, else False. 53def opcode_in_pickle(code, pickle): 54 for op, dummy, dummy in pickletools.genops(pickle): 55 if op.code == code.decode("latin-1"): 56 return True 57 return False 58 59# Return the number of times opcode code appears in pickle. 60def count_opcode(code, pickle): 61 n = 0 62 for op, dummy, dummy in pickletools.genops(pickle): 63 if op.code == code.decode("latin-1"): 64 n += 1 65 return n 66 67 68def identity(x): 69 return x 70 71 72class UnseekableIO(io.BytesIO): 73 def peek(self, *args): 74 raise NotImplementedError 75 76 def seekable(self): 77 return False 78 79 def seek(self, *args): 80 raise io.UnsupportedOperation 81 82 def tell(self): 83 raise io.UnsupportedOperation 84 85 86class MinimalIO(object): 87 """ 88 A file-like object that doesn't support readinto(). 89 """ 90 def __init__(self, *args): 91 self._bio = io.BytesIO(*args) 92 self.getvalue = self._bio.getvalue 93 self.read = self._bio.read 94 self.readline = self._bio.readline 95 self.write = self._bio.write 96 97 98# We can't very well test the extension registry without putting known stuff 99# in it, but we have to be careful to restore its original state. Code 100# should do this: 101# 102# e = ExtensionSaver(extension_code) 103# try: 104# fiddle w/ the extension registry's stuff for extension_code 105# finally: 106# e.restore() 107 108class ExtensionSaver: 109 # Remember current registration for code (if any), and remove it (if 110 # there is one). 111 def __init__(self, code): 112 self.code = code 113 if code in copyreg._inverted_registry: 114 self.pair = copyreg._inverted_registry[code] 115 copyreg.remove_extension(self.pair[0], self.pair[1], code) 116 else: 117 self.pair = None 118 119 # Restore previous registration for code. 120 def restore(self): 121 code = self.code 122 curpair = copyreg._inverted_registry.get(code) 123 if curpair is not None: 124 copyreg.remove_extension(curpair[0], curpair[1], code) 125 pair = self.pair 126 if pair is not None: 127 copyreg.add_extension(pair[0], pair[1], code) 128 129class C: 130 def __eq__(self, other): 131 return self.__dict__ == other.__dict__ 132 133class D(C): 134 def __init__(self, arg): 135 pass 136 137class E(C): 138 def __getinitargs__(self): 139 return () 140 141# Simple mutable object. 142class Object: 143 pass 144 145# Hashable immutable key object containing unheshable mutable data. 146class K: 147 def __init__(self, value): 148 self.value = value 149 150 def __reduce__(self): 151 # Shouldn't support the recursion itself 152 return K, (self.value,) 153 154import __main__ 155__main__.C = C 156C.__module__ = "__main__" 157__main__.D = D 158D.__module__ = "__main__" 159__main__.E = E 160E.__module__ = "__main__" 161 162class myint(int): 163 def __init__(self, x): 164 self.str = str(x) 165 166class initarg(C): 167 168 def __init__(self, a, b): 169 self.a = a 170 self.b = b 171 172 def __getinitargs__(self): 173 return self.a, self.b 174 175class metaclass(type): 176 pass 177 178class use_metaclass(object, metaclass=metaclass): 179 pass 180 181class pickling_metaclass(type): 182 def __eq__(self, other): 183 return (type(self) == type(other) and 184 self.reduce_args == other.reduce_args) 185 186 def __reduce__(self): 187 return (create_dynamic_class, self.reduce_args) 188 189def create_dynamic_class(name, bases): 190 result = pickling_metaclass(name, bases, dict()) 191 result.reduce_args = (name, bases) 192 return result 193 194 195class ZeroCopyBytes(bytes): 196 readonly = True 197 c_contiguous = True 198 f_contiguous = True 199 zero_copy_reconstruct = True 200 201 def __reduce_ex__(self, protocol): 202 if protocol >= 5: 203 return type(self)._reconstruct, (pickle.PickleBuffer(self),), None 204 else: 205 return type(self)._reconstruct, (bytes(self),) 206 207 def __repr__(self): 208 return "{}({!r})".format(self.__class__.__name__, bytes(self)) 209 210 __str__ = __repr__ 211 212 @classmethod 213 def _reconstruct(cls, obj): 214 with memoryview(obj) as m: 215 obj = m.obj 216 if type(obj) is cls: 217 # Zero-copy 218 return obj 219 else: 220 return cls(obj) 221 222 223class ZeroCopyBytearray(bytearray): 224 readonly = False 225 c_contiguous = True 226 f_contiguous = True 227 zero_copy_reconstruct = True 228 229 def __reduce_ex__(self, protocol): 230 if protocol >= 5: 231 return type(self)._reconstruct, (pickle.PickleBuffer(self),), None 232 else: 233 return type(self)._reconstruct, (bytes(self),) 234 235 def __repr__(self): 236 return "{}({!r})".format(self.__class__.__name__, bytes(self)) 237 238 __str__ = __repr__ 239 240 @classmethod 241 def _reconstruct(cls, obj): 242 with memoryview(obj) as m: 243 obj = m.obj 244 if type(obj) is cls: 245 # Zero-copy 246 return obj 247 else: 248 return cls(obj) 249 250 251if _testbuffer is not None: 252 253 class PicklableNDArray: 254 # A not-really-zero-copy picklable ndarray, as the ndarray() 255 # constructor doesn't allow for it 256 257 zero_copy_reconstruct = False 258 259 def __init__(self, *args, **kwargs): 260 self.array = _testbuffer.ndarray(*args, **kwargs) 261 262 def __getitem__(self, idx): 263 cls = type(self) 264 new = cls.__new__(cls) 265 new.array = self.array[idx] 266 return new 267 268 @property 269 def readonly(self): 270 return self.array.readonly 271 272 @property 273 def c_contiguous(self): 274 return self.array.c_contiguous 275 276 @property 277 def f_contiguous(self): 278 return self.array.f_contiguous 279 280 def __eq__(self, other): 281 if not isinstance(other, PicklableNDArray): 282 return NotImplemented 283 return (other.array.format == self.array.format and 284 other.array.shape == self.array.shape and 285 other.array.strides == self.array.strides and 286 other.array.readonly == self.array.readonly and 287 other.array.tobytes() == self.array.tobytes()) 288 289 def __ne__(self, other): 290 if not isinstance(other, PicklableNDArray): 291 return NotImplemented 292 return not (self == other) 293 294 def __repr__(self): 295 return (f"{type(self)}(shape={self.array.shape}," 296 f"strides={self.array.strides}, " 297 f"bytes={self.array.tobytes()})") 298 299 def __reduce_ex__(self, protocol): 300 if not self.array.contiguous: 301 raise NotImplementedError("Reconstructing a non-contiguous " 302 "ndarray does not seem possible") 303 ndarray_kwargs = {"shape": self.array.shape, 304 "strides": self.array.strides, 305 "format": self.array.format, 306 "flags": (0 if self.readonly 307 else _testbuffer.ND_WRITABLE)} 308 pb = pickle.PickleBuffer(self.array) 309 if protocol >= 5: 310 return (type(self)._reconstruct, 311 (pb, ndarray_kwargs)) 312 else: 313 # Need to serialize the bytes in physical order 314 with pb.raw() as m: 315 return (type(self)._reconstruct, 316 (m.tobytes(), ndarray_kwargs)) 317 318 @classmethod 319 def _reconstruct(cls, obj, kwargs): 320 with memoryview(obj) as m: 321 # For some reason, ndarray() wants a list of integers... 322 # XXX This only works if format == 'B' 323 items = list(m.tobytes()) 324 return cls(items, **kwargs) 325 326 327# DATA0 .. DATA4 are the pickles we expect under the various protocols, for 328# the object returned by create_data(). 329 330DATA0 = ( 331 b'(lp0\nL0L\naL1L\naF2.0\n' 332 b'ac__builtin__\ncomple' 333 b'x\np1\n(F3.0\nF0.0\ntp2\n' 334 b'Rp3\naL1L\naL-1L\naL255' 335 b'L\naL-255L\naL-256L\naL' 336 b'65535L\naL-65535L\naL-' 337 b'65536L\naL2147483647L' 338 b'\naL-2147483647L\naL-2' 339 b'147483648L\na(Vabc\np4' 340 b'\ng4\nccopy_reg\n_recon' 341 b'structor\np5\n(c__main' 342 b'__\nC\np6\nc__builtin__' 343 b'\nobject\np7\nNtp8\nRp9\n' 344 b'(dp10\nVfoo\np11\nL1L\ns' 345 b'Vbar\np12\nL2L\nsbg9\ntp' 346 b'13\nag13\naL5L\na.' 347) 348 349# Disassembly of DATA0 350DATA0_DIS = """\ 351 0: ( MARK 352 1: l LIST (MARK at 0) 353 2: p PUT 0 354 5: L LONG 0 355 9: a APPEND 356 10: L LONG 1 357 14: a APPEND 358 15: F FLOAT 2.0 359 20: a APPEND 360 21: c GLOBAL '__builtin__ complex' 361 42: p PUT 1 362 45: ( MARK 363 46: F FLOAT 3.0 364 51: F FLOAT 0.0 365 56: t TUPLE (MARK at 45) 366 57: p PUT 2 367 60: R REDUCE 368 61: p PUT 3 369 64: a APPEND 370 65: L LONG 1 371 69: a APPEND 372 70: L LONG -1 373 75: a APPEND 374 76: L LONG 255 375 82: a APPEND 376 83: L LONG -255 377 90: a APPEND 378 91: L LONG -256 379 98: a APPEND 380 99: L LONG 65535 381 107: a APPEND 382 108: L LONG -65535 383 117: a APPEND 384 118: L LONG -65536 385 127: a APPEND 386 128: L LONG 2147483647 387 141: a APPEND 388 142: L LONG -2147483647 389 156: a APPEND 390 157: L LONG -2147483648 391 171: a APPEND 392 172: ( MARK 393 173: V UNICODE 'abc' 394 178: p PUT 4 395 181: g GET 4 396 184: c GLOBAL 'copy_reg _reconstructor' 397 209: p PUT 5 398 212: ( MARK 399 213: c GLOBAL '__main__ C' 400 225: p PUT 6 401 228: c GLOBAL '__builtin__ object' 402 248: p PUT 7 403 251: N NONE 404 252: t TUPLE (MARK at 212) 405 253: p PUT 8 406 256: R REDUCE 407 257: p PUT 9 408 260: ( MARK 409 261: d DICT (MARK at 260) 410 262: p PUT 10 411 266: V UNICODE 'foo' 412 271: p PUT 11 413 275: L LONG 1 414 279: s SETITEM 415 280: V UNICODE 'bar' 416 285: p PUT 12 417 289: L LONG 2 418 293: s SETITEM 419 294: b BUILD 420 295: g GET 9 421 298: t TUPLE (MARK at 172) 422 299: p PUT 13 423 303: a APPEND 424 304: g GET 13 425 308: a APPEND 426 309: L LONG 5 427 313: a APPEND 428 314: . STOP 429highest protocol among opcodes = 0 430""" 431 432DATA1 = ( 433 b']q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c__' 434 b'builtin__\ncomplex\nq\x01' 435 b'(G@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00t' 436 b'q\x02Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xffJ' 437 b'\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff\xff' 438 b'\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00ab' 439 b'cq\x04h\x04ccopy_reg\n_reco' 440 b'nstructor\nq\x05(c__main' 441 b'__\nC\nq\x06c__builtin__\n' 442 b'object\nq\x07Ntq\x08Rq\t}q\n(' 443 b'X\x03\x00\x00\x00fooq\x0bK\x01X\x03\x00\x00\x00bar' 444 b'q\x0cK\x02ubh\ttq\rh\rK\x05e.' 445) 446 447# Disassembly of DATA1 448DATA1_DIS = """\ 449 0: ] EMPTY_LIST 450 1: q BINPUT 0 451 3: ( MARK 452 4: K BININT1 0 453 6: K BININT1 1 454 8: G BINFLOAT 2.0 455 17: c GLOBAL '__builtin__ complex' 456 38: q BINPUT 1 457 40: ( MARK 458 41: G BINFLOAT 3.0 459 50: G BINFLOAT 0.0 460 59: t TUPLE (MARK at 40) 461 60: q BINPUT 2 462 62: R REDUCE 463 63: q BINPUT 3 464 65: K BININT1 1 465 67: J BININT -1 466 72: K BININT1 255 467 74: J BININT -255 468 79: J BININT -256 469 84: M BININT2 65535 470 87: J BININT -65535 471 92: J BININT -65536 472 97: J BININT 2147483647 473 102: J BININT -2147483647 474 107: J BININT -2147483648 475 112: ( MARK 476 113: X BINUNICODE 'abc' 477 121: q BINPUT 4 478 123: h BINGET 4 479 125: c GLOBAL 'copy_reg _reconstructor' 480 150: q BINPUT 5 481 152: ( MARK 482 153: c GLOBAL '__main__ C' 483 165: q BINPUT 6 484 167: c GLOBAL '__builtin__ object' 485 187: q BINPUT 7 486 189: N NONE 487 190: t TUPLE (MARK at 152) 488 191: q BINPUT 8 489 193: R REDUCE 490 194: q BINPUT 9 491 196: } EMPTY_DICT 492 197: q BINPUT 10 493 199: ( MARK 494 200: X BINUNICODE 'foo' 495 208: q BINPUT 11 496 210: K BININT1 1 497 212: X BINUNICODE 'bar' 498 220: q BINPUT 12 499 222: K BININT1 2 500 224: u SETITEMS (MARK at 199) 501 225: b BUILD 502 226: h BINGET 9 503 228: t TUPLE (MARK at 112) 504 229: q BINPUT 13 505 231: h BINGET 13 506 233: K BININT1 5 507 235: e APPENDS (MARK at 3) 508 236: . STOP 509highest protocol among opcodes = 1 510""" 511 512DATA2 = ( 513 b'\x80\x02]q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c' 514 b'__builtin__\ncomplex\n' 515 b'q\x01G@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00' 516 b'\x86q\x02Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xff' 517 b'J\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff' 518 b'\xff\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00a' 519 b'bcq\x04h\x04c__main__\nC\nq\x05' 520 b')\x81q\x06}q\x07(X\x03\x00\x00\x00fooq\x08K\x01' 521 b'X\x03\x00\x00\x00barq\tK\x02ubh\x06tq\nh' 522 b'\nK\x05e.' 523) 524 525# Disassembly of DATA2 526DATA2_DIS = """\ 527 0: \x80 PROTO 2 528 2: ] EMPTY_LIST 529 3: q BINPUT 0 530 5: ( MARK 531 6: K BININT1 0 532 8: K BININT1 1 533 10: G BINFLOAT 2.0 534 19: c GLOBAL '__builtin__ complex' 535 40: q BINPUT 1 536 42: G BINFLOAT 3.0 537 51: G BINFLOAT 0.0 538 60: \x86 TUPLE2 539 61: q BINPUT 2 540 63: R REDUCE 541 64: q BINPUT 3 542 66: K BININT1 1 543 68: J BININT -1 544 73: K BININT1 255 545 75: J BININT -255 546 80: J BININT -256 547 85: M BININT2 65535 548 88: J BININT -65535 549 93: J BININT -65536 550 98: J BININT 2147483647 551 103: J BININT -2147483647 552 108: J BININT -2147483648 553 113: ( MARK 554 114: X BINUNICODE 'abc' 555 122: q BINPUT 4 556 124: h BINGET 4 557 126: c GLOBAL '__main__ C' 558 138: q BINPUT 5 559 140: ) EMPTY_TUPLE 560 141: \x81 NEWOBJ 561 142: q BINPUT 6 562 144: } EMPTY_DICT 563 145: q BINPUT 7 564 147: ( MARK 565 148: X BINUNICODE 'foo' 566 156: q BINPUT 8 567 158: K BININT1 1 568 160: X BINUNICODE 'bar' 569 168: q BINPUT 9 570 170: K BININT1 2 571 172: u SETITEMS (MARK at 147) 572 173: b BUILD 573 174: h BINGET 6 574 176: t TUPLE (MARK at 113) 575 177: q BINPUT 10 576 179: h BINGET 10 577 181: K BININT1 5 578 183: e APPENDS (MARK at 5) 579 184: . STOP 580highest protocol among opcodes = 2 581""" 582 583DATA3 = ( 584 b'\x80\x03]q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c' 585 b'builtins\ncomplex\nq\x01G' 586 b'@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00\x86q\x02' 587 b'Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xffJ\x00\xff' 588 b'\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff\xff\xff\x7f' 589 b'J\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00abcq' 590 b'\x04h\x04c__main__\nC\nq\x05)\x81q' 591 b'\x06}q\x07(X\x03\x00\x00\x00barq\x08K\x02X\x03\x00' 592 b'\x00\x00fooq\tK\x01ubh\x06tq\nh\nK\x05' 593 b'e.' 594) 595 596# Disassembly of DATA3 597DATA3_DIS = """\ 598 0: \x80 PROTO 3 599 2: ] EMPTY_LIST 600 3: q BINPUT 0 601 5: ( MARK 602 6: K BININT1 0 603 8: K BININT1 1 604 10: G BINFLOAT 2.0 605 19: c GLOBAL 'builtins complex' 606 37: q BINPUT 1 607 39: G BINFLOAT 3.0 608 48: G BINFLOAT 0.0 609 57: \x86 TUPLE2 610 58: q BINPUT 2 611 60: R REDUCE 612 61: q BINPUT 3 613 63: K BININT1 1 614 65: J BININT -1 615 70: K BININT1 255 616 72: J BININT -255 617 77: J BININT -256 618 82: M BININT2 65535 619 85: J BININT -65535 620 90: J BININT -65536 621 95: J BININT 2147483647 622 100: J BININT -2147483647 623 105: J BININT -2147483648 624 110: ( MARK 625 111: X BINUNICODE 'abc' 626 119: q BINPUT 4 627 121: h BINGET 4 628 123: c GLOBAL '__main__ C' 629 135: q BINPUT 5 630 137: ) EMPTY_TUPLE 631 138: \x81 NEWOBJ 632 139: q BINPUT 6 633 141: } EMPTY_DICT 634 142: q BINPUT 7 635 144: ( MARK 636 145: X BINUNICODE 'bar' 637 153: q BINPUT 8 638 155: K BININT1 2 639 157: X BINUNICODE 'foo' 640 165: q BINPUT 9 641 167: K BININT1 1 642 169: u SETITEMS (MARK at 144) 643 170: b BUILD 644 171: h BINGET 6 645 173: t TUPLE (MARK at 110) 646 174: q BINPUT 10 647 176: h BINGET 10 648 178: K BININT1 5 649 180: e APPENDS (MARK at 5) 650 181: . STOP 651highest protocol among opcodes = 2 652""" 653 654DATA4 = ( 655 b'\x80\x04\x95\xa8\x00\x00\x00\x00\x00\x00\x00]\x94(K\x00K\x01G@' 656 b'\x00\x00\x00\x00\x00\x00\x00\x8c\x08builtins\x94\x8c\x07' 657 b'complex\x94\x93\x94G@\x08\x00\x00\x00\x00\x00\x00G' 658 b'\x00\x00\x00\x00\x00\x00\x00\x00\x86\x94R\x94K\x01J\xff\xff\xff\xffK' 659 b'\xffJ\x01\xff\xff\xffJ\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ' 660 b'\x00\x00\xff\xffJ\xff\xff\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80(' 661 b'\x8c\x03abc\x94h\x06\x8c\x08__main__\x94\x8c' 662 b'\x01C\x94\x93\x94)\x81\x94}\x94(\x8c\x03bar\x94K\x02\x8c' 663 b'\x03foo\x94K\x01ubh\nt\x94h\x0eK\x05e.' 664) 665 666# Disassembly of DATA4 667DATA4_DIS = """\ 668 0: \x80 PROTO 4 669 2: \x95 FRAME 168 670 11: ] EMPTY_LIST 671 12: \x94 MEMOIZE 672 13: ( MARK 673 14: K BININT1 0 674 16: K BININT1 1 675 18: G BINFLOAT 2.0 676 27: \x8c SHORT_BINUNICODE 'builtins' 677 37: \x94 MEMOIZE 678 38: \x8c SHORT_BINUNICODE 'complex' 679 47: \x94 MEMOIZE 680 48: \x93 STACK_GLOBAL 681 49: \x94 MEMOIZE 682 50: G BINFLOAT 3.0 683 59: G BINFLOAT 0.0 684 68: \x86 TUPLE2 685 69: \x94 MEMOIZE 686 70: R REDUCE 687 71: \x94 MEMOIZE 688 72: K BININT1 1 689 74: J BININT -1 690 79: K BININT1 255 691 81: J BININT -255 692 86: J BININT -256 693 91: M BININT2 65535 694 94: J BININT -65535 695 99: J BININT -65536 696 104: J BININT 2147483647 697 109: J BININT -2147483647 698 114: J BININT -2147483648 699 119: ( MARK 700 120: \x8c SHORT_BINUNICODE 'abc' 701 125: \x94 MEMOIZE 702 126: h BINGET 6 703 128: \x8c SHORT_BINUNICODE '__main__' 704 138: \x94 MEMOIZE 705 139: \x8c SHORT_BINUNICODE 'C' 706 142: \x94 MEMOIZE 707 143: \x93 STACK_GLOBAL 708 144: \x94 MEMOIZE 709 145: ) EMPTY_TUPLE 710 146: \x81 NEWOBJ 711 147: \x94 MEMOIZE 712 148: } EMPTY_DICT 713 149: \x94 MEMOIZE 714 150: ( MARK 715 151: \x8c SHORT_BINUNICODE 'bar' 716 156: \x94 MEMOIZE 717 157: K BININT1 2 718 159: \x8c SHORT_BINUNICODE 'foo' 719 164: \x94 MEMOIZE 720 165: K BININT1 1 721 167: u SETITEMS (MARK at 150) 722 168: b BUILD 723 169: h BINGET 10 724 171: t TUPLE (MARK at 119) 725 172: \x94 MEMOIZE 726 173: h BINGET 14 727 175: K BININT1 5 728 177: e APPENDS (MARK at 13) 729 178: . STOP 730highest protocol among opcodes = 4 731""" 732 733# set([1,2]) pickled from 2.x with protocol 2 734DATA_SET = b'\x80\x02c__builtin__\nset\nq\x00]q\x01(K\x01K\x02e\x85q\x02Rq\x03.' 735 736# xrange(5) pickled from 2.x with protocol 2 737DATA_XRANGE = b'\x80\x02c__builtin__\nxrange\nq\x00K\x00K\x05K\x01\x87q\x01Rq\x02.' 738 739# a SimpleCookie() object pickled from 2.x with protocol 2 740DATA_COOKIE = (b'\x80\x02cCookie\nSimpleCookie\nq\x00)\x81q\x01U\x03key' 741 b'q\x02cCookie\nMorsel\nq\x03)\x81q\x04(U\x07commentq\x05U' 742 b'\x00q\x06U\x06domainq\x07h\x06U\x06secureq\x08h\x06U\x07' 743 b'expiresq\th\x06U\x07max-ageq\nh\x06U\x07versionq\x0bh\x06U' 744 b'\x04pathq\x0ch\x06U\x08httponlyq\rh\x06u}q\x0e(U\x0b' 745 b'coded_valueq\x0fU\x05valueq\x10h\x10h\x10h\x02h\x02ubs}q\x11b.') 746 747# set([3]) pickled from 2.x with protocol 2 748DATA_SET2 = b'\x80\x02c__builtin__\nset\nq\x00]q\x01K\x03a\x85q\x02Rq\x03.' 749 750python2_exceptions_without_args = ( 751 ArithmeticError, 752 AssertionError, 753 AttributeError, 754 BaseException, 755 BufferError, 756 BytesWarning, 757 DeprecationWarning, 758 EOFError, 759 EnvironmentError, 760 Exception, 761 FloatingPointError, 762 FutureWarning, 763 GeneratorExit, 764 IOError, 765 ImportError, 766 ImportWarning, 767 IndentationError, 768 IndexError, 769 KeyError, 770 KeyboardInterrupt, 771 LookupError, 772 MemoryError, 773 NameError, 774 NotImplementedError, 775 OSError, 776 OverflowError, 777 PendingDeprecationWarning, 778 ReferenceError, 779 RuntimeError, 780 RuntimeWarning, 781 # StandardError is gone in Python 3, we map it to Exception 782 StopIteration, 783 SyntaxError, 784 SyntaxWarning, 785 SystemError, 786 SystemExit, 787 TabError, 788 TypeError, 789 UnboundLocalError, 790 UnicodeError, 791 UnicodeWarning, 792 UserWarning, 793 ValueError, 794 Warning, 795 ZeroDivisionError, 796) 797 798exception_pickle = b'\x80\x02cexceptions\n?\nq\x00)Rq\x01.' 799 800# UnicodeEncodeError object pickled from 2.x with protocol 2 801DATA_UEERR = (b'\x80\x02cexceptions\nUnicodeEncodeError\n' 802 b'q\x00(U\x05asciiq\x01X\x03\x00\x00\x00fooq\x02K\x00K\x01' 803 b'U\x03badq\x03tq\x04Rq\x05.') 804 805 806def create_data(): 807 c = C() 808 c.foo = 1 809 c.bar = 2 810 x = [0, 1, 2.0, 3.0+0j] 811 # Append some integer test cases at cPickle.c's internal size 812 # cutoffs. 813 uint1max = 0xff 814 uint2max = 0xffff 815 int4max = 0x7fffffff 816 x.extend([1, -1, 817 uint1max, -uint1max, -uint1max-1, 818 uint2max, -uint2max, -uint2max-1, 819 int4max, -int4max, -int4max-1]) 820 y = ('abc', 'abc', c, c) 821 x.append(y) 822 x.append(y) 823 x.append(5) 824 return x 825 826 827class AbstractUnpickleTests(unittest.TestCase): 828 # Subclass must define self.loads. 829 830 _testdata = create_data() 831 832 def assert_is_copy(self, obj, objcopy, msg=None): 833 """Utility method to verify if two objects are copies of each others. 834 """ 835 if msg is None: 836 msg = "{!r} is not a copy of {!r}".format(obj, objcopy) 837 self.assertEqual(obj, objcopy, msg=msg) 838 self.assertIs(type(obj), type(objcopy), msg=msg) 839 if hasattr(obj, '__dict__'): 840 self.assertDictEqual(obj.__dict__, objcopy.__dict__, msg=msg) 841 self.assertIsNot(obj.__dict__, objcopy.__dict__, msg=msg) 842 if hasattr(obj, '__slots__'): 843 self.assertListEqual(obj.__slots__, objcopy.__slots__, msg=msg) 844 for slot in obj.__slots__: 845 self.assertEqual( 846 hasattr(obj, slot), hasattr(objcopy, slot), msg=msg) 847 self.assertEqual(getattr(obj, slot, None), 848 getattr(objcopy, slot, None), msg=msg) 849 850 def check_unpickling_error(self, errors, data): 851 with self.subTest(data=data), \ 852 self.assertRaises(errors): 853 try: 854 self.loads(data) 855 except BaseException as exc: 856 if support.verbose > 1: 857 print('%-32r - %s: %s' % 858 (data, exc.__class__.__name__, exc)) 859 raise 860 861 def test_load_from_data0(self): 862 self.assert_is_copy(self._testdata, self.loads(DATA0)) 863 864 def test_load_from_data1(self): 865 self.assert_is_copy(self._testdata, self.loads(DATA1)) 866 867 def test_load_from_data2(self): 868 self.assert_is_copy(self._testdata, self.loads(DATA2)) 869 870 def test_load_from_data3(self): 871 self.assert_is_copy(self._testdata, self.loads(DATA3)) 872 873 def test_load_from_data4(self): 874 self.assert_is_copy(self._testdata, self.loads(DATA4)) 875 876 def test_load_classic_instance(self): 877 # See issue5180. Test loading 2.x pickles that 878 # contain an instance of old style class. 879 for X, args in [(C, ()), (D, ('x',)), (E, ())]: 880 xname = X.__name__.encode('ascii') 881 # Protocol 0 (text mode pickle): 882 """ 883 0: ( MARK 884 1: i INST '__main__ X' (MARK at 0) 885 13: p PUT 0 886 16: ( MARK 887 17: d DICT (MARK at 16) 888 18: p PUT 1 889 21: b BUILD 890 22: . STOP 891 """ 892 pickle0 = (b"(i__main__\n" 893 b"X\n" 894 b"p0\n" 895 b"(dp1\nb.").replace(b'X', xname) 896 self.assert_is_copy(X(*args), self.loads(pickle0)) 897 898 # Protocol 1 (binary mode pickle) 899 """ 900 0: ( MARK 901 1: c GLOBAL '__main__ X' 902 13: q BINPUT 0 903 15: o OBJ (MARK at 0) 904 16: q BINPUT 1 905 18: } EMPTY_DICT 906 19: q BINPUT 2 907 21: b BUILD 908 22: . STOP 909 """ 910 pickle1 = (b'(c__main__\n' 911 b'X\n' 912 b'q\x00oq\x01}q\x02b.').replace(b'X', xname) 913 self.assert_is_copy(X(*args), self.loads(pickle1)) 914 915 # Protocol 2 (pickle2 = b'\x80\x02' + pickle1) 916 """ 917 0: \x80 PROTO 2 918 2: ( MARK 919 3: c GLOBAL '__main__ X' 920 15: q BINPUT 0 921 17: o OBJ (MARK at 2) 922 18: q BINPUT 1 923 20: } EMPTY_DICT 924 21: q BINPUT 2 925 23: b BUILD 926 24: . STOP 927 """ 928 pickle2 = (b'\x80\x02(c__main__\n' 929 b'X\n' 930 b'q\x00oq\x01}q\x02b.').replace(b'X', xname) 931 self.assert_is_copy(X(*args), self.loads(pickle2)) 932 933 def test_maxint64(self): 934 maxint64 = (1 << 63) - 1 935 data = b'I' + str(maxint64).encode("ascii") + b'\n.' 936 got = self.loads(data) 937 self.assert_is_copy(maxint64, got) 938 939 # Try too with a bogus literal. 940 data = b'I' + str(maxint64).encode("ascii") + b'JUNK\n.' 941 self.check_unpickling_error(ValueError, data) 942 943 def test_unpickle_from_2x(self): 944 # Unpickle non-trivial data from Python 2.x. 945 loaded = self.loads(DATA_SET) 946 self.assertEqual(loaded, set([1, 2])) 947 loaded = self.loads(DATA_XRANGE) 948 self.assertEqual(type(loaded), type(range(0))) 949 self.assertEqual(list(loaded), list(range(5))) 950 loaded = self.loads(DATA_COOKIE) 951 self.assertEqual(type(loaded), SimpleCookie) 952 self.assertEqual(list(loaded.keys()), ["key"]) 953 self.assertEqual(loaded["key"].value, "value") 954 955 # Exception objects without arguments pickled from 2.x with protocol 2 956 for exc in python2_exceptions_without_args: 957 data = exception_pickle.replace(b'?', exc.__name__.encode("ascii")) 958 loaded = self.loads(data) 959 self.assertIs(type(loaded), exc) 960 961 # StandardError is mapped to Exception, test that separately 962 loaded = self.loads(exception_pickle.replace(b'?', b'StandardError')) 963 self.assertIs(type(loaded), Exception) 964 965 loaded = self.loads(DATA_UEERR) 966 self.assertIs(type(loaded), UnicodeEncodeError) 967 self.assertEqual(loaded.object, "foo") 968 self.assertEqual(loaded.encoding, "ascii") 969 self.assertEqual(loaded.start, 0) 970 self.assertEqual(loaded.end, 1) 971 self.assertEqual(loaded.reason, "bad") 972 973 def test_load_python2_str_as_bytes(self): 974 # From Python 2: pickle.dumps('a\x00\xa0', protocol=0) 975 self.assertEqual(self.loads(b"S'a\\x00\\xa0'\n.", 976 encoding="bytes"), b'a\x00\xa0') 977 # From Python 2: pickle.dumps('a\x00\xa0', protocol=1) 978 self.assertEqual(self.loads(b'U\x03a\x00\xa0.', 979 encoding="bytes"), b'a\x00\xa0') 980 # From Python 2: pickle.dumps('a\x00\xa0', protocol=2) 981 self.assertEqual(self.loads(b'\x80\x02U\x03a\x00\xa0.', 982 encoding="bytes"), b'a\x00\xa0') 983 984 def test_load_python2_unicode_as_str(self): 985 # From Python 2: pickle.dumps(u'π', protocol=0) 986 self.assertEqual(self.loads(b'V\\u03c0\n.', 987 encoding='bytes'), 'π') 988 # From Python 2: pickle.dumps(u'π', protocol=1) 989 self.assertEqual(self.loads(b'X\x02\x00\x00\x00\xcf\x80.', 990 encoding="bytes"), 'π') 991 # From Python 2: pickle.dumps(u'π', protocol=2) 992 self.assertEqual(self.loads(b'\x80\x02X\x02\x00\x00\x00\xcf\x80.', 993 encoding="bytes"), 'π') 994 995 def test_load_long_python2_str_as_bytes(self): 996 # From Python 2: pickle.dumps('x' * 300, protocol=1) 997 self.assertEqual(self.loads(pickle.BINSTRING + 998 struct.pack("<I", 300) + 999 b'x' * 300 + pickle.STOP, 1000 encoding='bytes'), b'x' * 300) 1001 1002 def test_constants(self): 1003 self.assertIsNone(self.loads(b'N.')) 1004 self.assertIs(self.loads(b'\x88.'), True) 1005 self.assertIs(self.loads(b'\x89.'), False) 1006 self.assertIs(self.loads(b'I01\n.'), True) 1007 self.assertIs(self.loads(b'I00\n.'), False) 1008 1009 def test_empty_bytestring(self): 1010 # issue 11286 1011 empty = self.loads(b'\x80\x03U\x00q\x00.', encoding='koi8-r') 1012 self.assertEqual(empty, '') 1013 1014 def test_short_binbytes(self): 1015 dumped = b'\x80\x03C\x04\xe2\x82\xac\x00.' 1016 self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00') 1017 1018 def test_binbytes(self): 1019 dumped = b'\x80\x03B\x04\x00\x00\x00\xe2\x82\xac\x00.' 1020 self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00') 1021 1022 @requires_32b 1023 def test_negative_32b_binbytes(self): 1024 # On 32-bit builds, a BINBYTES of 2**31 or more is refused 1025 dumped = b'\x80\x03B\xff\xff\xff\xffxyzq\x00.' 1026 self.check_unpickling_error((pickle.UnpicklingError, OverflowError), 1027 dumped) 1028 1029 @requires_32b 1030 def test_negative_32b_binunicode(self): 1031 # On 32-bit builds, a BINUNICODE of 2**31 or more is refused 1032 dumped = b'\x80\x03X\xff\xff\xff\xffxyzq\x00.' 1033 self.check_unpickling_error((pickle.UnpicklingError, OverflowError), 1034 dumped) 1035 1036 def test_short_binunicode(self): 1037 dumped = b'\x80\x04\x8c\x04\xe2\x82\xac\x00.' 1038 self.assertEqual(self.loads(dumped), '\u20ac\x00') 1039 1040 def test_misc_get(self): 1041 self.check_unpickling_error(pickle.UnpicklingError, b'g0\np0') 1042 self.check_unpickling_error(pickle.UnpicklingError, b'jens:') 1043 self.check_unpickling_error(pickle.UnpicklingError, b'hens:') 1044 self.assert_is_copy([(100,), (100,)], 1045 self.loads(b'((Kdtp0\nh\x00l.))')) 1046 1047 def test_binbytes8(self): 1048 dumped = b'\x80\x04\x8e\4\0\0\0\0\0\0\0\xe2\x82\xac\x00.' 1049 self.assertEqual(self.loads(dumped), b'\xe2\x82\xac\x00') 1050 1051 def test_binunicode8(self): 1052 dumped = b'\x80\x04\x8d\4\0\0\0\0\0\0\0\xe2\x82\xac\x00.' 1053 self.assertEqual(self.loads(dumped), '\u20ac\x00') 1054 1055 def test_bytearray8(self): 1056 dumped = b'\x80\x05\x96\x03\x00\x00\x00\x00\x00\x00\x00xxx.' 1057 self.assertEqual(self.loads(dumped), bytearray(b'xxx')) 1058 1059 @requires_32b 1060 def test_large_32b_binbytes8(self): 1061 dumped = b'\x80\x04\x8e\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.' 1062 self.check_unpickling_error((pickle.UnpicklingError, OverflowError), 1063 dumped) 1064 1065 @requires_32b 1066 def test_large_32b_bytearray8(self): 1067 dumped = b'\x80\x05\x96\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.' 1068 self.check_unpickling_error((pickle.UnpicklingError, OverflowError), 1069 dumped) 1070 1071 @requires_32b 1072 def test_large_32b_binunicode8(self): 1073 dumped = b'\x80\x04\x8d\4\0\0\0\1\0\0\0\xe2\x82\xac\x00.' 1074 self.check_unpickling_error((pickle.UnpicklingError, OverflowError), 1075 dumped) 1076 1077 def test_get(self): 1078 pickled = b'((lp100000\ng100000\nt.' 1079 unpickled = self.loads(pickled) 1080 self.assertEqual(unpickled, ([],)*2) 1081 self.assertIs(unpickled[0], unpickled[1]) 1082 1083 def test_binget(self): 1084 pickled = b'(]q\xffh\xfft.' 1085 unpickled = self.loads(pickled) 1086 self.assertEqual(unpickled, ([],)*2) 1087 self.assertIs(unpickled[0], unpickled[1]) 1088 1089 def test_long_binget(self): 1090 pickled = b'(]r\x00\x00\x01\x00j\x00\x00\x01\x00t.' 1091 unpickled = self.loads(pickled) 1092 self.assertEqual(unpickled, ([],)*2) 1093 self.assertIs(unpickled[0], unpickled[1]) 1094 1095 def test_dup(self): 1096 pickled = b'((l2t.' 1097 unpickled = self.loads(pickled) 1098 self.assertEqual(unpickled, ([],)*2) 1099 self.assertIs(unpickled[0], unpickled[1]) 1100 1101 def test_negative_put(self): 1102 # Issue #12847 1103 dumped = b'Va\np-1\n.' 1104 self.check_unpickling_error(ValueError, dumped) 1105 1106 @requires_32b 1107 def test_negative_32b_binput(self): 1108 # Issue #12847 1109 dumped = b'\x80\x03X\x01\x00\x00\x00ar\xff\xff\xff\xff.' 1110 self.check_unpickling_error(ValueError, dumped) 1111 1112 def test_badly_escaped_string(self): 1113 self.check_unpickling_error(ValueError, b"S'\\'\n.") 1114 1115 def test_badly_quoted_string(self): 1116 # Issue #17710 1117 badpickles = [b"S'\n.", 1118 b'S"\n.', 1119 b'S\' \n.', 1120 b'S" \n.', 1121 b'S\'"\n.', 1122 b'S"\'\n.', 1123 b"S' ' \n.", 1124 b'S" " \n.', 1125 b"S ''\n.", 1126 b'S ""\n.', 1127 b'S \n.', 1128 b'S\n.', 1129 b'S.'] 1130 for p in badpickles: 1131 self.check_unpickling_error(pickle.UnpicklingError, p) 1132 1133 def test_correctly_quoted_string(self): 1134 goodpickles = [(b"S''\n.", ''), 1135 (b'S""\n.', ''), 1136 (b'S"\\n"\n.', '\n'), 1137 (b"S'\\n'\n.", '\n')] 1138 for p, expected in goodpickles: 1139 self.assertEqual(self.loads(p), expected) 1140 1141 def test_frame_readline(self): 1142 pickled = b'\x80\x04\x95\x05\x00\x00\x00\x00\x00\x00\x00I42\n.' 1143 # 0: \x80 PROTO 4 1144 # 2: \x95 FRAME 5 1145 # 11: I INT 42 1146 # 15: . STOP 1147 self.assertEqual(self.loads(pickled), 42) 1148 1149 def test_compat_unpickle(self): 1150 # xrange(1, 7) 1151 pickled = b'\x80\x02c__builtin__\nxrange\nK\x01K\x07K\x01\x87R.' 1152 unpickled = self.loads(pickled) 1153 self.assertIs(type(unpickled), range) 1154 self.assertEqual(unpickled, range(1, 7)) 1155 self.assertEqual(list(unpickled), [1, 2, 3, 4, 5, 6]) 1156 # reduce 1157 pickled = b'\x80\x02c__builtin__\nreduce\n.' 1158 self.assertIs(self.loads(pickled), functools.reduce) 1159 # whichdb.whichdb 1160 pickled = b'\x80\x02cwhichdb\nwhichdb\n.' 1161 self.assertIs(self.loads(pickled), dbm.whichdb) 1162 # Exception(), StandardError() 1163 for name in (b'Exception', b'StandardError'): 1164 pickled = (b'\x80\x02cexceptions\n' + name + b'\nU\x03ugh\x85R.') 1165 unpickled = self.loads(pickled) 1166 self.assertIs(type(unpickled), Exception) 1167 self.assertEqual(str(unpickled), 'ugh') 1168 # UserDict.UserDict({1: 2}), UserDict.IterableUserDict({1: 2}) 1169 for name in (b'UserDict', b'IterableUserDict'): 1170 pickled = (b'\x80\x02(cUserDict\n' + name + 1171 b'\no}U\x04data}K\x01K\x02ssb.') 1172 unpickled = self.loads(pickled) 1173 self.assertIs(type(unpickled), collections.UserDict) 1174 self.assertEqual(unpickled, collections.UserDict({1: 2})) 1175 1176 def test_bad_reduce(self): 1177 self.assertEqual(self.loads(b'cbuiltins\nint\n)R.'), 0) 1178 self.check_unpickling_error(TypeError, b'N)R.') 1179 self.check_unpickling_error(TypeError, b'cbuiltins\nint\nNR.') 1180 1181 def test_bad_newobj(self): 1182 error = (pickle.UnpicklingError, TypeError) 1183 self.assertEqual(self.loads(b'cbuiltins\nint\n)\x81.'), 0) 1184 self.check_unpickling_error(error, b'cbuiltins\nlen\n)\x81.') 1185 self.check_unpickling_error(error, b'cbuiltins\nint\nN\x81.') 1186 1187 def test_bad_newobj_ex(self): 1188 error = (pickle.UnpicklingError, TypeError) 1189 self.assertEqual(self.loads(b'cbuiltins\nint\n)}\x92.'), 0) 1190 self.check_unpickling_error(error, b'cbuiltins\nlen\n)}\x92.') 1191 self.check_unpickling_error(error, b'cbuiltins\nint\nN}\x92.') 1192 self.check_unpickling_error(error, b'cbuiltins\nint\n)N\x92.') 1193 1194 def test_bad_stack(self): 1195 badpickles = [ 1196 b'.', # STOP 1197 b'0', # POP 1198 b'1', # POP_MARK 1199 b'2', # DUP 1200 b'(2', 1201 b'R', # REDUCE 1202 b')R', 1203 b'a', # APPEND 1204 b'Na', 1205 b'b', # BUILD 1206 b'Nb', 1207 b'd', # DICT 1208 b'e', # APPENDS 1209 b'(e', 1210 b'ibuiltins\nlist\n', # INST 1211 b'l', # LIST 1212 b'o', # OBJ 1213 b'(o', 1214 b'p1\n', # PUT 1215 b'q\x00', # BINPUT 1216 b'r\x00\x00\x00\x00', # LONG_BINPUT 1217 b's', # SETITEM 1218 b'Ns', 1219 b'NNs', 1220 b't', # TUPLE 1221 b'u', # SETITEMS 1222 b'(u', 1223 b'}(Nu', 1224 b'\x81', # NEWOBJ 1225 b')\x81', 1226 b'\x85', # TUPLE1 1227 b'\x86', # TUPLE2 1228 b'N\x86', 1229 b'\x87', # TUPLE3 1230 b'N\x87', 1231 b'NN\x87', 1232 b'\x90', # ADDITEMS 1233 b'(\x90', 1234 b'\x91', # FROZENSET 1235 b'\x92', # NEWOBJ_EX 1236 b')}\x92', 1237 b'\x93', # STACK_GLOBAL 1238 b'Vlist\n\x93', 1239 b'\x94', # MEMOIZE 1240 ] 1241 for p in badpickles: 1242 self.check_unpickling_error(self.bad_stack_errors, p) 1243 1244 def test_bad_mark(self): 1245 badpickles = [ 1246 b'N(.', # STOP 1247 b'N(2', # DUP 1248 b'cbuiltins\nlist\n)(R', # REDUCE 1249 b'cbuiltins\nlist\n()R', 1250 b']N(a', # APPEND 1251 # BUILD 1252 b'cbuiltins\nValueError\n)R}(b', 1253 b'cbuiltins\nValueError\n)R(}b', 1254 b'(Nd', # DICT 1255 b'N(p1\n', # PUT 1256 b'N(q\x00', # BINPUT 1257 b'N(r\x00\x00\x00\x00', # LONG_BINPUT 1258 b'}NN(s', # SETITEM 1259 b'}N(Ns', 1260 b'}(NNs', 1261 b'}((u', # SETITEMS 1262 b'cbuiltins\nlist\n)(\x81', # NEWOBJ 1263 b'cbuiltins\nlist\n()\x81', 1264 b'N(\x85', # TUPLE1 1265 b'NN(\x86', # TUPLE2 1266 b'N(N\x86', 1267 b'NNN(\x87', # TUPLE3 1268 b'NN(N\x87', 1269 b'N(NN\x87', 1270 b']((\x90', # ADDITEMS 1271 # NEWOBJ_EX 1272 b'cbuiltins\nlist\n)}(\x92', 1273 b'cbuiltins\nlist\n)(}\x92', 1274 b'cbuiltins\nlist\n()}\x92', 1275 # STACK_GLOBAL 1276 b'Vbuiltins\n(Vlist\n\x93', 1277 b'Vbuiltins\nVlist\n(\x93', 1278 b'N(\x94', # MEMOIZE 1279 ] 1280 for p in badpickles: 1281 self.check_unpickling_error(self.bad_stack_errors, p) 1282 1283 def test_truncated_data(self): 1284 self.check_unpickling_error(EOFError, b'') 1285 self.check_unpickling_error(EOFError, b'N') 1286 badpickles = [ 1287 b'B', # BINBYTES 1288 b'B\x03\x00\x00', 1289 b'B\x03\x00\x00\x00', 1290 b'B\x03\x00\x00\x00ab', 1291 b'C', # SHORT_BINBYTES 1292 b'C\x03', 1293 b'C\x03ab', 1294 b'F', # FLOAT 1295 b'F0.0', 1296 b'F0.00', 1297 b'G', # BINFLOAT 1298 b'G\x00\x00\x00\x00\x00\x00\x00', 1299 b'I', # INT 1300 b'I0', 1301 b'J', # BININT 1302 b'J\x00\x00\x00', 1303 b'K', # BININT1 1304 b'L', # LONG 1305 b'L0', 1306 b'L10', 1307 b'L0L', 1308 b'L10L', 1309 b'M', # BININT2 1310 b'M\x00', 1311 # b'P', # PERSID 1312 # b'Pabc', 1313 b'S', # STRING 1314 b"S'abc'", 1315 b'T', # BINSTRING 1316 b'T\x03\x00\x00', 1317 b'T\x03\x00\x00\x00', 1318 b'T\x03\x00\x00\x00ab', 1319 b'U', # SHORT_BINSTRING 1320 b'U\x03', 1321 b'U\x03ab', 1322 b'V', # UNICODE 1323 b'Vabc', 1324 b'X', # BINUNICODE 1325 b'X\x03\x00\x00', 1326 b'X\x03\x00\x00\x00', 1327 b'X\x03\x00\x00\x00ab', 1328 b'(c', # GLOBAL 1329 b'(cbuiltins', 1330 b'(cbuiltins\n', 1331 b'(cbuiltins\nlist', 1332 b'Ng', # GET 1333 b'Ng0', 1334 b'(i', # INST 1335 b'(ibuiltins', 1336 b'(ibuiltins\n', 1337 b'(ibuiltins\nlist', 1338 b'Nh', # BINGET 1339 b'Nj', # LONG_BINGET 1340 b'Nj\x00\x00\x00', 1341 b'Np', # PUT 1342 b'Np0', 1343 b'Nq', # BINPUT 1344 b'Nr', # LONG_BINPUT 1345 b'Nr\x00\x00\x00', 1346 b'\x80', # PROTO 1347 b'\x82', # EXT1 1348 b'\x83', # EXT2 1349 b'\x84\x01', 1350 b'\x84', # EXT4 1351 b'\x84\x01\x00\x00', 1352 b'\x8a', # LONG1 1353 b'\x8b', # LONG4 1354 b'\x8b\x00\x00\x00', 1355 b'\x8c', # SHORT_BINUNICODE 1356 b'\x8c\x03', 1357 b'\x8c\x03ab', 1358 b'\x8d', # BINUNICODE8 1359 b'\x8d\x03\x00\x00\x00\x00\x00\x00', 1360 b'\x8d\x03\x00\x00\x00\x00\x00\x00\x00', 1361 b'\x8d\x03\x00\x00\x00\x00\x00\x00\x00ab', 1362 b'\x8e', # BINBYTES8 1363 b'\x8e\x03\x00\x00\x00\x00\x00\x00', 1364 b'\x8e\x03\x00\x00\x00\x00\x00\x00\x00', 1365 b'\x8e\x03\x00\x00\x00\x00\x00\x00\x00ab', 1366 b'\x96', # BYTEARRAY8 1367 b'\x96\x03\x00\x00\x00\x00\x00\x00', 1368 b'\x96\x03\x00\x00\x00\x00\x00\x00\x00', 1369 b'\x96\x03\x00\x00\x00\x00\x00\x00\x00ab', 1370 b'\x95', # FRAME 1371 b'\x95\x02\x00\x00\x00\x00\x00\x00', 1372 b'\x95\x02\x00\x00\x00\x00\x00\x00\x00', 1373 b'\x95\x02\x00\x00\x00\x00\x00\x00\x00N', 1374 ] 1375 for p in badpickles: 1376 self.check_unpickling_error(self.truncated_errors, p) 1377 1378 @reap_threads 1379 def test_unpickle_module_race(self): 1380 # https://bugs.python.org/issue34572 1381 locker_module = dedent(""" 1382 import threading 1383 barrier = threading.Barrier(2) 1384 """) 1385 locking_import_module = dedent(""" 1386 import locker 1387 locker.barrier.wait() 1388 class ToBeUnpickled(object): 1389 pass 1390 """) 1391 1392 os.mkdir(TESTFN) 1393 self.addCleanup(shutil.rmtree, TESTFN) 1394 sys.path.insert(0, TESTFN) 1395 self.addCleanup(sys.path.remove, TESTFN) 1396 with open(os.path.join(TESTFN, "locker.py"), "wb") as f: 1397 f.write(locker_module.encode('utf-8')) 1398 with open(os.path.join(TESTFN, "locking_import.py"), "wb") as f: 1399 f.write(locking_import_module.encode('utf-8')) 1400 self.addCleanup(forget, "locker") 1401 self.addCleanup(forget, "locking_import") 1402 1403 import locker 1404 1405 pickle_bytes = ( 1406 b'\x80\x03clocking_import\nToBeUnpickled\nq\x00)\x81q\x01.') 1407 1408 # Then try to unpickle two of these simultaneously 1409 # One of them will cause the module import, and we want it to block 1410 # until the other one either: 1411 # - fails (before the patch for this issue) 1412 # - blocks on the import lock for the module, as it should 1413 results = [] 1414 barrier = threading.Barrier(3) 1415 def t(): 1416 # This ensures the threads have all started 1417 # presumably barrier release is faster than thread startup 1418 barrier.wait() 1419 results.append(pickle.loads(pickle_bytes)) 1420 1421 t1 = threading.Thread(target=t) 1422 t2 = threading.Thread(target=t) 1423 t1.start() 1424 t2.start() 1425 1426 barrier.wait() 1427 # could have delay here 1428 locker.barrier.wait() 1429 1430 t1.join() 1431 t2.join() 1432 1433 from locking_import import ToBeUnpickled 1434 self.assertEqual( 1435 [type(x) for x in results], 1436 [ToBeUnpickled] * 2) 1437 1438 1439 1440class AbstractPickleTests(unittest.TestCase): 1441 # Subclass must define self.dumps, self.loads. 1442 1443 optimized = False 1444 1445 _testdata = AbstractUnpickleTests._testdata 1446 1447 def setUp(self): 1448 pass 1449 1450 assert_is_copy = AbstractUnpickleTests.assert_is_copy 1451 1452 def test_misc(self): 1453 # test various datatypes not tested by testdata 1454 for proto in protocols: 1455 x = myint(4) 1456 s = self.dumps(x, proto) 1457 y = self.loads(s) 1458 self.assert_is_copy(x, y) 1459 1460 x = (1, ()) 1461 s = self.dumps(x, proto) 1462 y = self.loads(s) 1463 self.assert_is_copy(x, y) 1464 1465 x = initarg(1, x) 1466 s = self.dumps(x, proto) 1467 y = self.loads(s) 1468 self.assert_is_copy(x, y) 1469 1470 # XXX test __reduce__ protocol? 1471 1472 def test_roundtrip_equality(self): 1473 expected = self._testdata 1474 for proto in protocols: 1475 s = self.dumps(expected, proto) 1476 got = self.loads(s) 1477 self.assert_is_copy(expected, got) 1478 1479 # There are gratuitous differences between pickles produced by 1480 # pickle and cPickle, largely because cPickle starts PUT indices at 1481 # 1 and pickle starts them at 0. See XXX comment in cPickle's put2() -- 1482 # there's a comment with an exclamation point there whose meaning 1483 # is a mystery. cPickle also suppresses PUT for objects with a refcount 1484 # of 1. 1485 def dont_test_disassembly(self): 1486 from io import StringIO 1487 from pickletools import dis 1488 1489 for proto, expected in (0, DATA0_DIS), (1, DATA1_DIS): 1490 s = self.dumps(self._testdata, proto) 1491 filelike = StringIO() 1492 dis(s, out=filelike) 1493 got = filelike.getvalue() 1494 self.assertEqual(expected, got) 1495 1496 def _test_recursive_list(self, cls, aslist=identity, minprotocol=0): 1497 # List containing itself. 1498 l = cls() 1499 l.append(l) 1500 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1): 1501 s = self.dumps(l, proto) 1502 x = self.loads(s) 1503 self.assertIsInstance(x, cls) 1504 y = aslist(x) 1505 self.assertEqual(len(y), 1) 1506 self.assertIs(y[0], x) 1507 1508 def test_recursive_list(self): 1509 self._test_recursive_list(list) 1510 1511 def test_recursive_list_subclass(self): 1512 self._test_recursive_list(MyList, minprotocol=2) 1513 1514 def test_recursive_list_like(self): 1515 self._test_recursive_list(REX_six, aslist=lambda x: x.items) 1516 1517 def _test_recursive_tuple_and_list(self, cls, aslist=identity, minprotocol=0): 1518 # Tuple containing a list containing the original tuple. 1519 t = (cls(),) 1520 t[0].append(t) 1521 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1): 1522 s = self.dumps(t, proto) 1523 x = self.loads(s) 1524 self.assertIsInstance(x, tuple) 1525 self.assertEqual(len(x), 1) 1526 self.assertIsInstance(x[0], cls) 1527 y = aslist(x[0]) 1528 self.assertEqual(len(y), 1) 1529 self.assertIs(y[0], x) 1530 1531 # List containing a tuple containing the original list. 1532 t, = t 1533 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1): 1534 s = self.dumps(t, proto) 1535 x = self.loads(s) 1536 self.assertIsInstance(x, cls) 1537 y = aslist(x) 1538 self.assertEqual(len(y), 1) 1539 self.assertIsInstance(y[0], tuple) 1540 self.assertEqual(len(y[0]), 1) 1541 self.assertIs(y[0][0], x) 1542 1543 def test_recursive_tuple_and_list(self): 1544 self._test_recursive_tuple_and_list(list) 1545 1546 def test_recursive_tuple_and_list_subclass(self): 1547 self._test_recursive_tuple_and_list(MyList, minprotocol=2) 1548 1549 def test_recursive_tuple_and_list_like(self): 1550 self._test_recursive_tuple_and_list(REX_six, aslist=lambda x: x.items) 1551 1552 def _test_recursive_dict(self, cls, asdict=identity, minprotocol=0): 1553 # Dict containing itself. 1554 d = cls() 1555 d[1] = d 1556 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1): 1557 s = self.dumps(d, proto) 1558 x = self.loads(s) 1559 self.assertIsInstance(x, cls) 1560 y = asdict(x) 1561 self.assertEqual(list(y.keys()), [1]) 1562 self.assertIs(y[1], x) 1563 1564 def test_recursive_dict(self): 1565 self._test_recursive_dict(dict) 1566 1567 def test_recursive_dict_subclass(self): 1568 self._test_recursive_dict(MyDict, minprotocol=2) 1569 1570 def test_recursive_dict_like(self): 1571 self._test_recursive_dict(REX_seven, asdict=lambda x: x.table) 1572 1573 def _test_recursive_tuple_and_dict(self, cls, asdict=identity, minprotocol=0): 1574 # Tuple containing a dict containing the original tuple. 1575 t = (cls(),) 1576 t[0][1] = t 1577 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1): 1578 s = self.dumps(t, proto) 1579 x = self.loads(s) 1580 self.assertIsInstance(x, tuple) 1581 self.assertEqual(len(x), 1) 1582 self.assertIsInstance(x[0], cls) 1583 y = asdict(x[0]) 1584 self.assertEqual(list(y), [1]) 1585 self.assertIs(y[1], x) 1586 1587 # Dict containing a tuple containing the original dict. 1588 t, = t 1589 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1): 1590 s = self.dumps(t, proto) 1591 x = self.loads(s) 1592 self.assertIsInstance(x, cls) 1593 y = asdict(x) 1594 self.assertEqual(list(y), [1]) 1595 self.assertIsInstance(y[1], tuple) 1596 self.assertEqual(len(y[1]), 1) 1597 self.assertIs(y[1][0], x) 1598 1599 def test_recursive_tuple_and_dict(self): 1600 self._test_recursive_tuple_and_dict(dict) 1601 1602 def test_recursive_tuple_and_dict_subclass(self): 1603 self._test_recursive_tuple_and_dict(MyDict, minprotocol=2) 1604 1605 def test_recursive_tuple_and_dict_like(self): 1606 self._test_recursive_tuple_and_dict(REX_seven, asdict=lambda x: x.table) 1607 1608 def _test_recursive_dict_key(self, cls, asdict=identity, minprotocol=0): 1609 # Dict containing an immutable object (as key) containing the original 1610 # dict. 1611 d = cls() 1612 d[K(d)] = 1 1613 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1): 1614 s = self.dumps(d, proto) 1615 x = self.loads(s) 1616 self.assertIsInstance(x, cls) 1617 y = asdict(x) 1618 self.assertEqual(len(y.keys()), 1) 1619 self.assertIsInstance(list(y.keys())[0], K) 1620 self.assertIs(list(y.keys())[0].value, x) 1621 1622 def test_recursive_dict_key(self): 1623 self._test_recursive_dict_key(dict) 1624 1625 def test_recursive_dict_subclass_key(self): 1626 self._test_recursive_dict_key(MyDict, minprotocol=2) 1627 1628 def test_recursive_dict_like_key(self): 1629 self._test_recursive_dict_key(REX_seven, asdict=lambda x: x.table) 1630 1631 def _test_recursive_tuple_and_dict_key(self, cls, asdict=identity, minprotocol=0): 1632 # Tuple containing a dict containing an immutable object (as key) 1633 # containing the original tuple. 1634 t = (cls(),) 1635 t[0][K(t)] = 1 1636 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1): 1637 s = self.dumps(t, proto) 1638 x = self.loads(s) 1639 self.assertIsInstance(x, tuple) 1640 self.assertEqual(len(x), 1) 1641 self.assertIsInstance(x[0], cls) 1642 y = asdict(x[0]) 1643 self.assertEqual(len(y), 1) 1644 self.assertIsInstance(list(y.keys())[0], K) 1645 self.assertIs(list(y.keys())[0].value, x) 1646 1647 # Dict containing an immutable object (as key) containing a tuple 1648 # containing the original dict. 1649 t, = t 1650 for proto in range(minprotocol, pickle.HIGHEST_PROTOCOL + 1): 1651 s = self.dumps(t, proto) 1652 x = self.loads(s) 1653 self.assertIsInstance(x, cls) 1654 y = asdict(x) 1655 self.assertEqual(len(y), 1) 1656 self.assertIsInstance(list(y.keys())[0], K) 1657 self.assertIs(list(y.keys())[0].value[0], x) 1658 1659 def test_recursive_tuple_and_dict_key(self): 1660 self._test_recursive_tuple_and_dict_key(dict) 1661 1662 def test_recursive_tuple_and_dict_subclass_key(self): 1663 self._test_recursive_tuple_and_dict_key(MyDict, minprotocol=2) 1664 1665 def test_recursive_tuple_and_dict_like_key(self): 1666 self._test_recursive_tuple_and_dict_key(REX_seven, asdict=lambda x: x.table) 1667 1668 def test_recursive_set(self): 1669 # Set containing an immutable object containing the original set. 1670 y = set() 1671 y.add(K(y)) 1672 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): 1673 s = self.dumps(y, proto) 1674 x = self.loads(s) 1675 self.assertIsInstance(x, set) 1676 self.assertEqual(len(x), 1) 1677 self.assertIsInstance(list(x)[0], K) 1678 self.assertIs(list(x)[0].value, x) 1679 1680 # Immutable object containing a set containing the original object. 1681 y, = y 1682 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): 1683 s = self.dumps(y, proto) 1684 x = self.loads(s) 1685 self.assertIsInstance(x, K) 1686 self.assertIsInstance(x.value, set) 1687 self.assertEqual(len(x.value), 1) 1688 self.assertIs(list(x.value)[0], x) 1689 1690 def test_recursive_inst(self): 1691 # Mutable object containing itself. 1692 i = Object() 1693 i.attr = i 1694 for proto in protocols: 1695 s = self.dumps(i, proto) 1696 x = self.loads(s) 1697 self.assertIsInstance(x, Object) 1698 self.assertEqual(dir(x), dir(i)) 1699 self.assertIs(x.attr, x) 1700 1701 def test_recursive_multi(self): 1702 l = [] 1703 d = {1:l} 1704 i = Object() 1705 i.attr = d 1706 l.append(i) 1707 for proto in protocols: 1708 s = self.dumps(l, proto) 1709 x = self.loads(s) 1710 self.assertIsInstance(x, list) 1711 self.assertEqual(len(x), 1) 1712 self.assertEqual(dir(x[0]), dir(i)) 1713 self.assertEqual(list(x[0].attr.keys()), [1]) 1714 self.assertIs(x[0].attr[1], x) 1715 1716 def _test_recursive_collection_and_inst(self, factory): 1717 # Mutable object containing a collection containing the original 1718 # object. 1719 o = Object() 1720 o.attr = factory([o]) 1721 t = type(o.attr) 1722 for proto in protocols: 1723 s = self.dumps(o, proto) 1724 x = self.loads(s) 1725 self.assertIsInstance(x.attr, t) 1726 self.assertEqual(len(x.attr), 1) 1727 self.assertIsInstance(list(x.attr)[0], Object) 1728 self.assertIs(list(x.attr)[0], x) 1729 1730 # Collection containing a mutable object containing the original 1731 # collection. 1732 o = o.attr 1733 for proto in protocols: 1734 s = self.dumps(o, proto) 1735 x = self.loads(s) 1736 self.assertIsInstance(x, t) 1737 self.assertEqual(len(x), 1) 1738 self.assertIsInstance(list(x)[0], Object) 1739 self.assertIs(list(x)[0].attr, x) 1740 1741 def test_recursive_list_and_inst(self): 1742 self._test_recursive_collection_and_inst(list) 1743 1744 def test_recursive_tuple_and_inst(self): 1745 self._test_recursive_collection_and_inst(tuple) 1746 1747 def test_recursive_dict_and_inst(self): 1748 self._test_recursive_collection_and_inst(dict.fromkeys) 1749 1750 def test_recursive_set_and_inst(self): 1751 self._test_recursive_collection_and_inst(set) 1752 1753 def test_recursive_frozenset_and_inst(self): 1754 self._test_recursive_collection_and_inst(frozenset) 1755 1756 def test_recursive_list_subclass_and_inst(self): 1757 self._test_recursive_collection_and_inst(MyList) 1758 1759 def test_recursive_tuple_subclass_and_inst(self): 1760 self._test_recursive_collection_and_inst(MyTuple) 1761 1762 def test_recursive_dict_subclass_and_inst(self): 1763 self._test_recursive_collection_and_inst(MyDict.fromkeys) 1764 1765 def test_recursive_set_subclass_and_inst(self): 1766 self._test_recursive_collection_and_inst(MySet) 1767 1768 def test_recursive_frozenset_subclass_and_inst(self): 1769 self._test_recursive_collection_and_inst(MyFrozenSet) 1770 1771 def test_recursive_inst_state(self): 1772 # Mutable object containing itself. 1773 y = REX_state() 1774 y.state = y 1775 for proto in protocols: 1776 s = self.dumps(y, proto) 1777 x = self.loads(s) 1778 self.assertIsInstance(x, REX_state) 1779 self.assertIs(x.state, x) 1780 1781 def test_recursive_tuple_and_inst_state(self): 1782 # Tuple containing a mutable object containing the original tuple. 1783 t = (REX_state(),) 1784 t[0].state = t 1785 for proto in protocols: 1786 s = self.dumps(t, proto) 1787 x = self.loads(s) 1788 self.assertIsInstance(x, tuple) 1789 self.assertEqual(len(x), 1) 1790 self.assertIsInstance(x[0], REX_state) 1791 self.assertIs(x[0].state, x) 1792 1793 # Mutable object containing a tuple containing the object. 1794 t, = t 1795 for proto in protocols: 1796 s = self.dumps(t, proto) 1797 x = self.loads(s) 1798 self.assertIsInstance(x, REX_state) 1799 self.assertIsInstance(x.state, tuple) 1800 self.assertEqual(len(x.state), 1) 1801 self.assertIs(x.state[0], x) 1802 1803 def test_unicode(self): 1804 endcases = ['', '<\\u>', '<\\\u1234>', '<\n>', 1805 '<\\>', '<\\\U00012345>', 1806 # surrogates 1807 '<\udc80>'] 1808 for proto in protocols: 1809 for u in endcases: 1810 p = self.dumps(u, proto) 1811 u2 = self.loads(p) 1812 self.assert_is_copy(u, u2) 1813 1814 def test_unicode_high_plane(self): 1815 t = '\U00012345' 1816 for proto in protocols: 1817 p = self.dumps(t, proto) 1818 t2 = self.loads(p) 1819 self.assert_is_copy(t, t2) 1820 1821 def test_bytes(self): 1822 for proto in protocols: 1823 for s in b'', b'xyz', b'xyz'*100: 1824 p = self.dumps(s, proto) 1825 self.assert_is_copy(s, self.loads(p)) 1826 for s in [bytes([i]) for i in range(256)]: 1827 p = self.dumps(s, proto) 1828 self.assert_is_copy(s, self.loads(p)) 1829 for s in [bytes([i, i]) for i in range(256)]: 1830 p = self.dumps(s, proto) 1831 self.assert_is_copy(s, self.loads(p)) 1832 1833 def test_bytearray(self): 1834 for proto in protocols: 1835 for s in b'', b'xyz', b'xyz'*100: 1836 b = bytearray(s) 1837 p = self.dumps(b, proto) 1838 bb = self.loads(p) 1839 self.assertIsNot(bb, b) 1840 self.assert_is_copy(b, bb) 1841 if proto <= 3: 1842 # bytearray is serialized using a global reference 1843 self.assertIn(b'bytearray', p) 1844 self.assertTrue(opcode_in_pickle(pickle.GLOBAL, p)) 1845 elif proto == 4: 1846 self.assertIn(b'bytearray', p) 1847 self.assertTrue(opcode_in_pickle(pickle.STACK_GLOBAL, p)) 1848 elif proto == 5: 1849 self.assertNotIn(b'bytearray', p) 1850 self.assertTrue(opcode_in_pickle(pickle.BYTEARRAY8, p)) 1851 1852 def test_ints(self): 1853 for proto in protocols: 1854 n = sys.maxsize 1855 while n: 1856 for expected in (-n, n): 1857 s = self.dumps(expected, proto) 1858 n2 = self.loads(s) 1859 self.assert_is_copy(expected, n2) 1860 n = n >> 1 1861 1862 def test_long(self): 1863 for proto in protocols: 1864 # 256 bytes is where LONG4 begins. 1865 for nbits in 1, 8, 8*254, 8*255, 8*256, 8*257: 1866 nbase = 1 << nbits 1867 for npos in nbase-1, nbase, nbase+1: 1868 for n in npos, -npos: 1869 pickle = self.dumps(n, proto) 1870 got = self.loads(pickle) 1871 self.assert_is_copy(n, got) 1872 # Try a monster. This is quadratic-time in protos 0 & 1, so don't 1873 # bother with those. 1874 nbase = int("deadbeeffeedface", 16) 1875 nbase += nbase << 1000000 1876 for n in nbase, -nbase: 1877 p = self.dumps(n, 2) 1878 got = self.loads(p) 1879 # assert_is_copy is very expensive here as it precomputes 1880 # a failure message by computing the repr() of n and got, 1881 # we just do the check ourselves. 1882 self.assertIs(type(got), int) 1883 self.assertEqual(n, got) 1884 1885 def test_float(self): 1886 test_values = [0.0, 4.94e-324, 1e-310, 7e-308, 6.626e-34, 0.1, 0.5, 1887 3.14, 263.44582062374053, 6.022e23, 1e30] 1888 test_values = test_values + [-x for x in test_values] 1889 for proto in protocols: 1890 for value in test_values: 1891 pickle = self.dumps(value, proto) 1892 got = self.loads(pickle) 1893 self.assert_is_copy(value, got) 1894 1895 @run_with_locale('LC_ALL', 'de_DE', 'fr_FR') 1896 def test_float_format(self): 1897 # make sure that floats are formatted locale independent with proto 0 1898 self.assertEqual(self.dumps(1.2, 0)[0:3], b'F1.') 1899 1900 def test_reduce(self): 1901 for proto in protocols: 1902 inst = AAA() 1903 dumped = self.dumps(inst, proto) 1904 loaded = self.loads(dumped) 1905 self.assertEqual(loaded, REDUCE_A) 1906 1907 def test_getinitargs(self): 1908 for proto in protocols: 1909 inst = initarg(1, 2) 1910 dumped = self.dumps(inst, proto) 1911 loaded = self.loads(dumped) 1912 self.assert_is_copy(inst, loaded) 1913 1914 def test_metaclass(self): 1915 a = use_metaclass() 1916 for proto in protocols: 1917 s = self.dumps(a, proto) 1918 b = self.loads(s) 1919 self.assertEqual(a.__class__, b.__class__) 1920 1921 def test_dynamic_class(self): 1922 a = create_dynamic_class("my_dynamic_class", (object,)) 1923 copyreg.pickle(pickling_metaclass, pickling_metaclass.__reduce__) 1924 for proto in protocols: 1925 s = self.dumps(a, proto) 1926 b = self.loads(s) 1927 self.assertEqual(a, b) 1928 self.assertIs(type(a), type(b)) 1929 1930 def test_structseq(self): 1931 import time 1932 import os 1933 1934 t = time.localtime() 1935 for proto in protocols: 1936 s = self.dumps(t, proto) 1937 u = self.loads(s) 1938 self.assert_is_copy(t, u) 1939 t = os.stat(os.curdir) 1940 s = self.dumps(t, proto) 1941 u = self.loads(s) 1942 self.assert_is_copy(t, u) 1943 if hasattr(os, "statvfs"): 1944 t = os.statvfs(os.curdir) 1945 s = self.dumps(t, proto) 1946 u = self.loads(s) 1947 self.assert_is_copy(t, u) 1948 1949 def test_ellipsis(self): 1950 for proto in protocols: 1951 s = self.dumps(..., proto) 1952 u = self.loads(s) 1953 self.assertIs(..., u) 1954 1955 def test_notimplemented(self): 1956 for proto in protocols: 1957 s = self.dumps(NotImplemented, proto) 1958 u = self.loads(s) 1959 self.assertIs(NotImplemented, u) 1960 1961 def test_singleton_types(self): 1962 # Issue #6477: Test that types of built-in singletons can be pickled. 1963 singletons = [None, ..., NotImplemented] 1964 for singleton in singletons: 1965 for proto in protocols: 1966 s = self.dumps(type(singleton), proto) 1967 u = self.loads(s) 1968 self.assertIs(type(singleton), u) 1969 1970 # Tests for protocol 2 1971 1972 def test_proto(self): 1973 for proto in protocols: 1974 pickled = self.dumps(None, proto) 1975 if proto >= 2: 1976 proto_header = pickle.PROTO + bytes([proto]) 1977 self.assertTrue(pickled.startswith(proto_header)) 1978 else: 1979 self.assertEqual(count_opcode(pickle.PROTO, pickled), 0) 1980 1981 oob = protocols[-1] + 1 # a future protocol 1982 build_none = pickle.NONE + pickle.STOP 1983 badpickle = pickle.PROTO + bytes([oob]) + build_none 1984 try: 1985 self.loads(badpickle) 1986 except ValueError as err: 1987 self.assertIn("unsupported pickle protocol", str(err)) 1988 else: 1989 self.fail("expected bad protocol number to raise ValueError") 1990 1991 def test_long1(self): 1992 x = 12345678910111213141516178920 1993 for proto in protocols: 1994 s = self.dumps(x, proto) 1995 y = self.loads(s) 1996 self.assert_is_copy(x, y) 1997 self.assertEqual(opcode_in_pickle(pickle.LONG1, s), proto >= 2) 1998 1999 def test_long4(self): 2000 x = 12345678910111213141516178920 << (256*8) 2001 for proto in protocols: 2002 s = self.dumps(x, proto) 2003 y = self.loads(s) 2004 self.assert_is_copy(x, y) 2005 self.assertEqual(opcode_in_pickle(pickle.LONG4, s), proto >= 2) 2006 2007 def test_short_tuples(self): 2008 # Map (proto, len(tuple)) to expected opcode. 2009 expected_opcode = {(0, 0): pickle.TUPLE, 2010 (0, 1): pickle.TUPLE, 2011 (0, 2): pickle.TUPLE, 2012 (0, 3): pickle.TUPLE, 2013 (0, 4): pickle.TUPLE, 2014 2015 (1, 0): pickle.EMPTY_TUPLE, 2016 (1, 1): pickle.TUPLE, 2017 (1, 2): pickle.TUPLE, 2018 (1, 3): pickle.TUPLE, 2019 (1, 4): pickle.TUPLE, 2020 2021 (2, 0): pickle.EMPTY_TUPLE, 2022 (2, 1): pickle.TUPLE1, 2023 (2, 2): pickle.TUPLE2, 2024 (2, 3): pickle.TUPLE3, 2025 (2, 4): pickle.TUPLE, 2026 2027 (3, 0): pickle.EMPTY_TUPLE, 2028 (3, 1): pickle.TUPLE1, 2029 (3, 2): pickle.TUPLE2, 2030 (3, 3): pickle.TUPLE3, 2031 (3, 4): pickle.TUPLE, 2032 } 2033 a = () 2034 b = (1,) 2035 c = (1, 2) 2036 d = (1, 2, 3) 2037 e = (1, 2, 3, 4) 2038 for proto in protocols: 2039 for x in a, b, c, d, e: 2040 s = self.dumps(x, proto) 2041 y = self.loads(s) 2042 self.assert_is_copy(x, y) 2043 expected = expected_opcode[min(proto, 3), len(x)] 2044 self.assertTrue(opcode_in_pickle(expected, s)) 2045 2046 def test_singletons(self): 2047 # Map (proto, singleton) to expected opcode. 2048 expected_opcode = {(0, None): pickle.NONE, 2049 (1, None): pickle.NONE, 2050 (2, None): pickle.NONE, 2051 (3, None): pickle.NONE, 2052 2053 (0, True): pickle.INT, 2054 (1, True): pickle.INT, 2055 (2, True): pickle.NEWTRUE, 2056 (3, True): pickle.NEWTRUE, 2057 2058 (0, False): pickle.INT, 2059 (1, False): pickle.INT, 2060 (2, False): pickle.NEWFALSE, 2061 (3, False): pickle.NEWFALSE, 2062 } 2063 for proto in protocols: 2064 for x in None, False, True: 2065 s = self.dumps(x, proto) 2066 y = self.loads(s) 2067 self.assertTrue(x is y, (proto, x, s, y)) 2068 expected = expected_opcode[min(proto, 3), x] 2069 self.assertTrue(opcode_in_pickle(expected, s)) 2070 2071 def test_newobj_tuple(self): 2072 x = MyTuple([1, 2, 3]) 2073 x.foo = 42 2074 x.bar = "hello" 2075 for proto in protocols: 2076 s = self.dumps(x, proto) 2077 y = self.loads(s) 2078 self.assert_is_copy(x, y) 2079 2080 def test_newobj_list(self): 2081 x = MyList([1, 2, 3]) 2082 x.foo = 42 2083 x.bar = "hello" 2084 for proto in protocols: 2085 s = self.dumps(x, proto) 2086 y = self.loads(s) 2087 self.assert_is_copy(x, y) 2088 2089 def test_newobj_generic(self): 2090 for proto in protocols: 2091 for C in myclasses: 2092 B = C.__base__ 2093 x = C(C.sample) 2094 x.foo = 42 2095 s = self.dumps(x, proto) 2096 y = self.loads(s) 2097 detail = (proto, C, B, x, y, type(y)) 2098 self.assert_is_copy(x, y) # XXX revisit 2099 self.assertEqual(B(x), B(y), detail) 2100 self.assertEqual(x.__dict__, y.__dict__, detail) 2101 2102 def test_newobj_proxies(self): 2103 # NEWOBJ should use the __class__ rather than the raw type 2104 classes = myclasses[:] 2105 # Cannot create weakproxies to these classes 2106 for c in (MyInt, MyTuple): 2107 classes.remove(c) 2108 for proto in protocols: 2109 for C in classes: 2110 B = C.__base__ 2111 x = C(C.sample) 2112 x.foo = 42 2113 p = weakref.proxy(x) 2114 s = self.dumps(p, proto) 2115 y = self.loads(s) 2116 self.assertEqual(type(y), type(x)) # rather than type(p) 2117 detail = (proto, C, B, x, y, type(y)) 2118 self.assertEqual(B(x), B(y), detail) 2119 self.assertEqual(x.__dict__, y.__dict__, detail) 2120 2121 def test_newobj_overridden_new(self): 2122 # Test that Python class with C implemented __new__ is pickleable 2123 for proto in protocols: 2124 x = MyIntWithNew2(1) 2125 x.foo = 42 2126 s = self.dumps(x, proto) 2127 y = self.loads(s) 2128 self.assertIs(type(y), MyIntWithNew2) 2129 self.assertEqual(int(y), 1) 2130 self.assertEqual(y.foo, 42) 2131 2132 def test_newobj_not_class(self): 2133 # Issue 24552 2134 global SimpleNewObj 2135 save = SimpleNewObj 2136 o = SimpleNewObj.__new__(SimpleNewObj) 2137 b = self.dumps(o, 4) 2138 try: 2139 SimpleNewObj = 42 2140 self.assertRaises((TypeError, pickle.UnpicklingError), self.loads, b) 2141 finally: 2142 SimpleNewObj = save 2143 2144 # Register a type with copyreg, with extension code extcode. Pickle 2145 # an object of that type. Check that the resulting pickle uses opcode 2146 # (EXT[124]) under proto 2, and not in proto 1. 2147 2148 def produce_global_ext(self, extcode, opcode): 2149 e = ExtensionSaver(extcode) 2150 try: 2151 copyreg.add_extension(__name__, "MyList", extcode) 2152 x = MyList([1, 2, 3]) 2153 x.foo = 42 2154 x.bar = "hello" 2155 2156 # Dump using protocol 1 for comparison. 2157 s1 = self.dumps(x, 1) 2158 self.assertIn(__name__.encode("utf-8"), s1) 2159 self.assertIn(b"MyList", s1) 2160 self.assertFalse(opcode_in_pickle(opcode, s1)) 2161 2162 y = self.loads(s1) 2163 self.assert_is_copy(x, y) 2164 2165 # Dump using protocol 2 for test. 2166 s2 = self.dumps(x, 2) 2167 self.assertNotIn(__name__.encode("utf-8"), s2) 2168 self.assertNotIn(b"MyList", s2) 2169 self.assertEqual(opcode_in_pickle(opcode, s2), True, repr(s2)) 2170 2171 y = self.loads(s2) 2172 self.assert_is_copy(x, y) 2173 finally: 2174 e.restore() 2175 2176 def test_global_ext1(self): 2177 self.produce_global_ext(0x00000001, pickle.EXT1) # smallest EXT1 code 2178 self.produce_global_ext(0x000000ff, pickle.EXT1) # largest EXT1 code 2179 2180 def test_global_ext2(self): 2181 self.produce_global_ext(0x00000100, pickle.EXT2) # smallest EXT2 code 2182 self.produce_global_ext(0x0000ffff, pickle.EXT2) # largest EXT2 code 2183 self.produce_global_ext(0x0000abcd, pickle.EXT2) # check endianness 2184 2185 def test_global_ext4(self): 2186 self.produce_global_ext(0x00010000, pickle.EXT4) # smallest EXT4 code 2187 self.produce_global_ext(0x7fffffff, pickle.EXT4) # largest EXT4 code 2188 self.produce_global_ext(0x12abcdef, pickle.EXT4) # check endianness 2189 2190 def test_list_chunking(self): 2191 n = 10 # too small to chunk 2192 x = list(range(n)) 2193 for proto in protocols: 2194 s = self.dumps(x, proto) 2195 y = self.loads(s) 2196 self.assert_is_copy(x, y) 2197 num_appends = count_opcode(pickle.APPENDS, s) 2198 self.assertEqual(num_appends, proto > 0) 2199 2200 n = 2500 # expect at least two chunks when proto > 0 2201 x = list(range(n)) 2202 for proto in protocols: 2203 s = self.dumps(x, proto) 2204 y = self.loads(s) 2205 self.assert_is_copy(x, y) 2206 num_appends = count_opcode(pickle.APPENDS, s) 2207 if proto == 0: 2208 self.assertEqual(num_appends, 0) 2209 else: 2210 self.assertTrue(num_appends >= 2) 2211 2212 def test_dict_chunking(self): 2213 n = 10 # too small to chunk 2214 x = dict.fromkeys(range(n)) 2215 for proto in protocols: 2216 s = self.dumps(x, proto) 2217 self.assertIsInstance(s, bytes_types) 2218 y = self.loads(s) 2219 self.assert_is_copy(x, y) 2220 num_setitems = count_opcode(pickle.SETITEMS, s) 2221 self.assertEqual(num_setitems, proto > 0) 2222 2223 n = 2500 # expect at least two chunks when proto > 0 2224 x = dict.fromkeys(range(n)) 2225 for proto in protocols: 2226 s = self.dumps(x, proto) 2227 y = self.loads(s) 2228 self.assert_is_copy(x, y) 2229 num_setitems = count_opcode(pickle.SETITEMS, s) 2230 if proto == 0: 2231 self.assertEqual(num_setitems, 0) 2232 else: 2233 self.assertTrue(num_setitems >= 2) 2234 2235 def test_set_chunking(self): 2236 n = 10 # too small to chunk 2237 x = set(range(n)) 2238 for proto in protocols: 2239 s = self.dumps(x, proto) 2240 y = self.loads(s) 2241 self.assert_is_copy(x, y) 2242 num_additems = count_opcode(pickle.ADDITEMS, s) 2243 if proto < 4: 2244 self.assertEqual(num_additems, 0) 2245 else: 2246 self.assertEqual(num_additems, 1) 2247 2248 n = 2500 # expect at least two chunks when proto >= 4 2249 x = set(range(n)) 2250 for proto in protocols: 2251 s = self.dumps(x, proto) 2252 y = self.loads(s) 2253 self.assert_is_copy(x, y) 2254 num_additems = count_opcode(pickle.ADDITEMS, s) 2255 if proto < 4: 2256 self.assertEqual(num_additems, 0) 2257 else: 2258 self.assertGreaterEqual(num_additems, 2) 2259 2260 def test_simple_newobj(self): 2261 x = SimpleNewObj.__new__(SimpleNewObj, 0xface) # avoid __init__ 2262 x.abc = 666 2263 for proto in protocols: 2264 with self.subTest(proto=proto): 2265 s = self.dumps(x, proto) 2266 if proto < 1: 2267 self.assertIn(b'\nI64206', s) # INT 2268 else: 2269 self.assertIn(b'M\xce\xfa', s) # BININT2 2270 self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s), 2271 2 <= proto) 2272 self.assertFalse(opcode_in_pickle(pickle.NEWOBJ_EX, s)) 2273 y = self.loads(s) # will raise TypeError if __init__ called 2274 self.assert_is_copy(x, y) 2275 2276 def test_complex_newobj(self): 2277 x = ComplexNewObj.__new__(ComplexNewObj, 0xface) # avoid __init__ 2278 x.abc = 666 2279 for proto in protocols: 2280 with self.subTest(proto=proto): 2281 s = self.dumps(x, proto) 2282 if proto < 1: 2283 self.assertIn(b'\nI64206', s) # INT 2284 elif proto < 2: 2285 self.assertIn(b'M\xce\xfa', s) # BININT2 2286 elif proto < 4: 2287 self.assertIn(b'X\x04\x00\x00\x00FACE', s) # BINUNICODE 2288 else: 2289 self.assertIn(b'\x8c\x04FACE', s) # SHORT_BINUNICODE 2290 self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s), 2291 2 <= proto) 2292 self.assertFalse(opcode_in_pickle(pickle.NEWOBJ_EX, s)) 2293 y = self.loads(s) # will raise TypeError if __init__ called 2294 self.assert_is_copy(x, y) 2295 2296 def test_complex_newobj_ex(self): 2297 x = ComplexNewObjEx.__new__(ComplexNewObjEx, 0xface) # avoid __init__ 2298 x.abc = 666 2299 for proto in protocols: 2300 with self.subTest(proto=proto): 2301 s = self.dumps(x, proto) 2302 if proto < 1: 2303 self.assertIn(b'\nI64206', s) # INT 2304 elif proto < 2: 2305 self.assertIn(b'M\xce\xfa', s) # BININT2 2306 elif proto < 4: 2307 self.assertIn(b'X\x04\x00\x00\x00FACE', s) # BINUNICODE 2308 else: 2309 self.assertIn(b'\x8c\x04FACE', s) # SHORT_BINUNICODE 2310 self.assertFalse(opcode_in_pickle(pickle.NEWOBJ, s)) 2311 self.assertEqual(opcode_in_pickle(pickle.NEWOBJ_EX, s), 2312 4 <= proto) 2313 y = self.loads(s) # will raise TypeError if __init__ called 2314 self.assert_is_copy(x, y) 2315 2316 def test_newobj_list_slots(self): 2317 x = SlotList([1, 2, 3]) 2318 x.foo = 42 2319 x.bar = "hello" 2320 s = self.dumps(x, 2) 2321 y = self.loads(s) 2322 self.assert_is_copy(x, y) 2323 2324 def test_reduce_overrides_default_reduce_ex(self): 2325 for proto in protocols: 2326 x = REX_one() 2327 self.assertEqual(x._reduce_called, 0) 2328 s = self.dumps(x, proto) 2329 self.assertEqual(x._reduce_called, 1) 2330 y = self.loads(s) 2331 self.assertEqual(y._reduce_called, 0) 2332 2333 def test_reduce_ex_called(self): 2334 for proto in protocols: 2335 x = REX_two() 2336 self.assertEqual(x._proto, None) 2337 s = self.dumps(x, proto) 2338 self.assertEqual(x._proto, proto) 2339 y = self.loads(s) 2340 self.assertEqual(y._proto, None) 2341 2342 def test_reduce_ex_overrides_reduce(self): 2343 for proto in protocols: 2344 x = REX_three() 2345 self.assertEqual(x._proto, None) 2346 s = self.dumps(x, proto) 2347 self.assertEqual(x._proto, proto) 2348 y = self.loads(s) 2349 self.assertEqual(y._proto, None) 2350 2351 def test_reduce_ex_calls_base(self): 2352 for proto in protocols: 2353 x = REX_four() 2354 self.assertEqual(x._proto, None) 2355 s = self.dumps(x, proto) 2356 self.assertEqual(x._proto, proto) 2357 y = self.loads(s) 2358 self.assertEqual(y._proto, proto) 2359 2360 def test_reduce_calls_base(self): 2361 for proto in protocols: 2362 x = REX_five() 2363 self.assertEqual(x._reduce_called, 0) 2364 s = self.dumps(x, proto) 2365 self.assertEqual(x._reduce_called, 1) 2366 y = self.loads(s) 2367 self.assertEqual(y._reduce_called, 1) 2368 2369 @no_tracing 2370 def test_bad_getattr(self): 2371 # Issue #3514: crash when there is an infinite loop in __getattr__ 2372 x = BadGetattr() 2373 for proto in protocols: 2374 self.assertRaises(RuntimeError, self.dumps, x, proto) 2375 2376 def test_reduce_bad_iterator(self): 2377 # Issue4176: crash when 4th and 5th items of __reduce__() 2378 # are not iterators 2379 class C(object): 2380 def __reduce__(self): 2381 # 4th item is not an iterator 2382 return list, (), None, [], None 2383 class D(object): 2384 def __reduce__(self): 2385 # 5th item is not an iterator 2386 return dict, (), None, None, [] 2387 2388 # Python implementation is less strict and also accepts iterables. 2389 for proto in protocols: 2390 try: 2391 self.dumps(C(), proto) 2392 except pickle.PicklingError: 2393 pass 2394 try: 2395 self.dumps(D(), proto) 2396 except pickle.PicklingError: 2397 pass 2398 2399 def test_many_puts_and_gets(self): 2400 # Test that internal data structures correctly deal with lots of 2401 # puts/gets. 2402 keys = ("aaa" + str(i) for i in range(100)) 2403 large_dict = dict((k, [4, 5, 6]) for k in keys) 2404 obj = [dict(large_dict), dict(large_dict), dict(large_dict)] 2405 2406 for proto in protocols: 2407 with self.subTest(proto=proto): 2408 dumped = self.dumps(obj, proto) 2409 loaded = self.loads(dumped) 2410 self.assert_is_copy(obj, loaded) 2411 2412 def test_attribute_name_interning(self): 2413 # Test that attribute names of pickled objects are interned when 2414 # unpickling. 2415 for proto in protocols: 2416 x = C() 2417 x.foo = 42 2418 x.bar = "hello" 2419 s = self.dumps(x, proto) 2420 y = self.loads(s) 2421 x_keys = sorted(x.__dict__) 2422 y_keys = sorted(y.__dict__) 2423 for x_key, y_key in zip(x_keys, y_keys): 2424 self.assertIs(x_key, y_key) 2425 2426 def test_pickle_to_2x(self): 2427 # Pickle non-trivial data with protocol 2, expecting that it yields 2428 # the same result as Python 2.x did. 2429 # NOTE: this test is a bit too strong since we can produce different 2430 # bytecode that 2.x will still understand. 2431 dumped = self.dumps(range(5), 2) 2432 self.assertEqual(dumped, DATA_XRANGE) 2433 dumped = self.dumps(set([3]), 2) 2434 self.assertEqual(dumped, DATA_SET2) 2435 2436 def test_large_pickles(self): 2437 # Test the correctness of internal buffering routines when handling 2438 # large data. 2439 for proto in protocols: 2440 data = (1, min, b'xy' * (30 * 1024), len) 2441 dumped = self.dumps(data, proto) 2442 loaded = self.loads(dumped) 2443 self.assertEqual(len(loaded), len(data)) 2444 self.assertEqual(loaded, data) 2445 2446 def test_int_pickling_efficiency(self): 2447 # Test compacity of int representation (see issue #12744) 2448 for proto in protocols: 2449 with self.subTest(proto=proto): 2450 pickles = [self.dumps(2**n, proto) for n in range(70)] 2451 sizes = list(map(len, pickles)) 2452 # the size function is monotonic 2453 self.assertEqual(sorted(sizes), sizes) 2454 if proto >= 2: 2455 for p in pickles: 2456 self.assertFalse(opcode_in_pickle(pickle.LONG, p)) 2457 2458 def _check_pickling_with_opcode(self, obj, opcode, proto): 2459 pickled = self.dumps(obj, proto) 2460 self.assertTrue(opcode_in_pickle(opcode, pickled)) 2461 unpickled = self.loads(pickled) 2462 self.assertEqual(obj, unpickled) 2463 2464 def test_appends_on_non_lists(self): 2465 # Issue #17720 2466 obj = REX_six([1, 2, 3]) 2467 for proto in protocols: 2468 if proto == 0: 2469 self._check_pickling_with_opcode(obj, pickle.APPEND, proto) 2470 else: 2471 self._check_pickling_with_opcode(obj, pickle.APPENDS, proto) 2472 2473 def test_setitems_on_non_dicts(self): 2474 obj = REX_seven({1: -1, 2: -2, 3: -3}) 2475 for proto in protocols: 2476 if proto == 0: 2477 self._check_pickling_with_opcode(obj, pickle.SETITEM, proto) 2478 else: 2479 self._check_pickling_with_opcode(obj, pickle.SETITEMS, proto) 2480 2481 # Exercise framing (proto >= 4) for significant workloads 2482 2483 FRAME_SIZE_MIN = 4 2484 FRAME_SIZE_TARGET = 64 * 1024 2485 2486 def check_frame_opcodes(self, pickled): 2487 """ 2488 Check the arguments of FRAME opcodes in a protocol 4+ pickle. 2489 2490 Note that binary objects that are larger than FRAME_SIZE_TARGET are not 2491 framed by default and are therefore considered a frame by themselves in 2492 the following consistency check. 2493 """ 2494 frame_end = frameless_start = None 2495 frameless_opcodes = {'BINBYTES', 'BINUNICODE', 'BINBYTES8', 2496 'BINUNICODE8', 'BYTEARRAY8'} 2497 for op, arg, pos in pickletools.genops(pickled): 2498 if frame_end is not None: 2499 self.assertLessEqual(pos, frame_end) 2500 if pos == frame_end: 2501 frame_end = None 2502 2503 if frame_end is not None: # framed 2504 self.assertNotEqual(op.name, 'FRAME') 2505 if op.name in frameless_opcodes: 2506 # Only short bytes and str objects should be written 2507 # in a frame 2508 self.assertLessEqual(len(arg), self.FRAME_SIZE_TARGET) 2509 2510 else: # not framed 2511 if (op.name == 'FRAME' or 2512 (op.name in frameless_opcodes and 2513 len(arg) > self.FRAME_SIZE_TARGET)): 2514 # Frame or large bytes or str object 2515 if frameless_start is not None: 2516 # Only short data should be written outside of a frame 2517 self.assertLess(pos - frameless_start, 2518 self.FRAME_SIZE_MIN) 2519 frameless_start = None 2520 elif frameless_start is None and op.name != 'PROTO': 2521 frameless_start = pos 2522 2523 if op.name == 'FRAME': 2524 self.assertGreaterEqual(arg, self.FRAME_SIZE_MIN) 2525 frame_end = pos + 9 + arg 2526 2527 pos = len(pickled) 2528 if frame_end is not None: 2529 self.assertEqual(frame_end, pos) 2530 elif frameless_start is not None: 2531 self.assertLess(pos - frameless_start, self.FRAME_SIZE_MIN) 2532 2533 @support.skip_if_pgo_task 2534 def test_framing_many_objects(self): 2535 obj = list(range(10**5)) 2536 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): 2537 with self.subTest(proto=proto): 2538 pickled = self.dumps(obj, proto) 2539 unpickled = self.loads(pickled) 2540 self.assertEqual(obj, unpickled) 2541 bytes_per_frame = (len(pickled) / 2542 count_opcode(pickle.FRAME, pickled)) 2543 self.assertGreater(bytes_per_frame, 2544 self.FRAME_SIZE_TARGET / 2) 2545 self.assertLessEqual(bytes_per_frame, 2546 self.FRAME_SIZE_TARGET * 1) 2547 self.check_frame_opcodes(pickled) 2548 2549 def test_framing_large_objects(self): 2550 N = 1024 * 1024 2551 small_items = [[i] for i in range(10)] 2552 obj = [b'x' * N, *small_items, b'y' * N, 'z' * N] 2553 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): 2554 for fast in [False, True]: 2555 with self.subTest(proto=proto, fast=fast): 2556 if not fast: 2557 # fast=False by default. 2558 # This covers in-memory pickling with pickle.dumps(). 2559 pickled = self.dumps(obj, proto) 2560 else: 2561 # Pickler is required when fast=True. 2562 if not hasattr(self, 'pickler'): 2563 continue 2564 buf = io.BytesIO() 2565 pickler = self.pickler(buf, protocol=proto) 2566 pickler.fast = fast 2567 pickler.dump(obj) 2568 pickled = buf.getvalue() 2569 unpickled = self.loads(pickled) 2570 # More informative error message in case of failure. 2571 self.assertEqual([len(x) for x in obj], 2572 [len(x) for x in unpickled]) 2573 # Perform full equality check if the lengths match. 2574 self.assertEqual(obj, unpickled) 2575 n_frames = count_opcode(pickle.FRAME, pickled) 2576 # A single frame for small objects between 2577 # first two large objects. 2578 self.assertEqual(n_frames, 1) 2579 self.check_frame_opcodes(pickled) 2580 2581 def test_optional_frames(self): 2582 if pickle.HIGHEST_PROTOCOL < 4: 2583 return 2584 2585 def remove_frames(pickled, keep_frame=None): 2586 """Remove frame opcodes from the given pickle.""" 2587 frame_starts = [] 2588 # 1 byte for the opcode and 8 for the argument 2589 frame_opcode_size = 9 2590 for opcode, _, pos in pickletools.genops(pickled): 2591 if opcode.name == 'FRAME': 2592 frame_starts.append(pos) 2593 2594 newpickle = bytearray() 2595 last_frame_end = 0 2596 for i, pos in enumerate(frame_starts): 2597 if keep_frame and keep_frame(i): 2598 continue 2599 newpickle += pickled[last_frame_end:pos] 2600 last_frame_end = pos + frame_opcode_size 2601 newpickle += pickled[last_frame_end:] 2602 return newpickle 2603 2604 frame_size = self.FRAME_SIZE_TARGET 2605 num_frames = 20 2606 # Large byte objects (dict values) intermittent with small objects 2607 # (dict keys) 2608 for bytes_type in (bytes, bytearray): 2609 obj = {i: bytes_type([i]) * frame_size for i in range(num_frames)} 2610 2611 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): 2612 pickled = self.dumps(obj, proto) 2613 2614 frameless_pickle = remove_frames(pickled) 2615 self.assertEqual(count_opcode(pickle.FRAME, frameless_pickle), 0) 2616 self.assertEqual(obj, self.loads(frameless_pickle)) 2617 2618 some_frames_pickle = remove_frames(pickled, lambda i: i % 2) 2619 self.assertLess(count_opcode(pickle.FRAME, some_frames_pickle), 2620 count_opcode(pickle.FRAME, pickled)) 2621 self.assertEqual(obj, self.loads(some_frames_pickle)) 2622 2623 @support.skip_if_pgo_task 2624 def test_framed_write_sizes_with_delayed_writer(self): 2625 class ChunkAccumulator: 2626 """Accumulate pickler output in a list of raw chunks.""" 2627 def __init__(self): 2628 self.chunks = [] 2629 def write(self, chunk): 2630 self.chunks.append(chunk) 2631 def concatenate_chunks(self): 2632 return b"".join(self.chunks) 2633 2634 for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): 2635 objects = [(str(i).encode('ascii'), i % 42, {'i': str(i)}) 2636 for i in range(int(1e4))] 2637 # Add a large unique ASCII string 2638 objects.append('0123456789abcdef' * 2639 (self.FRAME_SIZE_TARGET // 16 + 1)) 2640 2641 # Protocol 4 packs groups of small objects into frames and issues 2642 # calls to write only once or twice per frame: 2643 # The C pickler issues one call to write per-frame (header and 2644 # contents) while Python pickler issues two calls to write: one for 2645 # the frame header and one for the frame binary contents. 2646 writer = ChunkAccumulator() 2647 self.pickler(writer, proto).dump(objects) 2648 2649 # Actually read the binary content of the chunks after the end 2650 # of the call to dump: any memoryview passed to write should not 2651 # be released otherwise this delayed access would not be possible. 2652 pickled = writer.concatenate_chunks() 2653 reconstructed = self.loads(pickled) 2654 self.assertEqual(reconstructed, objects) 2655 self.assertGreater(len(writer.chunks), 1) 2656 2657 # memoryviews should own the memory. 2658 del objects 2659 support.gc_collect() 2660 self.assertEqual(writer.concatenate_chunks(), pickled) 2661 2662 n_frames = (len(pickled) - 1) // self.FRAME_SIZE_TARGET + 1 2663 # There should be at least one call to write per frame 2664 self.assertGreaterEqual(len(writer.chunks), n_frames) 2665 2666 # but not too many either: there can be one for the proto, 2667 # one per-frame header, one per frame for the actual contents, 2668 # and two for the header. 2669 self.assertLessEqual(len(writer.chunks), 2 * n_frames + 3) 2670 2671 chunk_sizes = [len(c) for c in writer.chunks] 2672 large_sizes = [s for s in chunk_sizes 2673 if s >= self.FRAME_SIZE_TARGET] 2674 medium_sizes = [s for s in chunk_sizes 2675 if 9 < s < self.FRAME_SIZE_TARGET] 2676 small_sizes = [s for s in chunk_sizes if s <= 9] 2677 2678 # Large chunks should not be too large: 2679 for chunk_size in large_sizes: 2680 self.assertLess(chunk_size, 2 * self.FRAME_SIZE_TARGET, 2681 chunk_sizes) 2682 # There shouldn't bee too many small chunks: the protocol header, 2683 # the frame headers and the large string headers are written 2684 # in small chunks. 2685 self.assertLessEqual(len(small_sizes), 2686 len(large_sizes) + len(medium_sizes) + 3, 2687 chunk_sizes) 2688 2689 def test_nested_names(self): 2690 global Nested 2691 class Nested: 2692 class A: 2693 class B: 2694 class C: 2695 pass 2696 for proto in range(pickle.HIGHEST_PROTOCOL + 1): 2697 for obj in [Nested.A, Nested.A.B, Nested.A.B.C]: 2698 with self.subTest(proto=proto, obj=obj): 2699 unpickled = self.loads(self.dumps(obj, proto)) 2700 self.assertIs(obj, unpickled) 2701 2702 def test_recursive_nested_names(self): 2703 global Recursive 2704 class Recursive: 2705 pass 2706 Recursive.mod = sys.modules[Recursive.__module__] 2707 Recursive.__qualname__ = 'Recursive.mod.Recursive' 2708 for proto in range(pickle.HIGHEST_PROTOCOL + 1): 2709 with self.subTest(proto=proto): 2710 unpickled = self.loads(self.dumps(Recursive, proto)) 2711 self.assertIs(unpickled, Recursive) 2712 del Recursive.mod # break reference loop 2713 2714 def test_py_methods(self): 2715 global PyMethodsTest 2716 class PyMethodsTest: 2717 @staticmethod 2718 def cheese(): 2719 return "cheese" 2720 @classmethod 2721 def wine(cls): 2722 assert cls is PyMethodsTest 2723 return "wine" 2724 def biscuits(self): 2725 assert isinstance(self, PyMethodsTest) 2726 return "biscuits" 2727 class Nested: 2728 "Nested class" 2729 @staticmethod 2730 def ketchup(): 2731 return "ketchup" 2732 @classmethod 2733 def maple(cls): 2734 assert cls is PyMethodsTest.Nested 2735 return "maple" 2736 def pie(self): 2737 assert isinstance(self, PyMethodsTest.Nested) 2738 return "pie" 2739 2740 py_methods = ( 2741 PyMethodsTest.cheese, 2742 PyMethodsTest.wine, 2743 PyMethodsTest().biscuits, 2744 PyMethodsTest.Nested.ketchup, 2745 PyMethodsTest.Nested.maple, 2746 PyMethodsTest.Nested().pie 2747 ) 2748 py_unbound_methods = ( 2749 (PyMethodsTest.biscuits, PyMethodsTest), 2750 (PyMethodsTest.Nested.pie, PyMethodsTest.Nested) 2751 ) 2752 for proto in range(pickle.HIGHEST_PROTOCOL + 1): 2753 for method in py_methods: 2754 with self.subTest(proto=proto, method=method): 2755 unpickled = self.loads(self.dumps(method, proto)) 2756 self.assertEqual(method(), unpickled()) 2757 for method, cls in py_unbound_methods: 2758 obj = cls() 2759 with self.subTest(proto=proto, method=method): 2760 unpickled = self.loads(self.dumps(method, proto)) 2761 self.assertEqual(method(obj), unpickled(obj)) 2762 2763 def test_c_methods(self): 2764 global Subclass 2765 class Subclass(tuple): 2766 class Nested(str): 2767 pass 2768 2769 c_methods = ( 2770 # bound built-in method 2771 ("abcd".index, ("c",)), 2772 # unbound built-in method 2773 (str.index, ("abcd", "c")), 2774 # bound "slot" method 2775 ([1, 2, 3].__len__, ()), 2776 # unbound "slot" method 2777 (list.__len__, ([1, 2, 3],)), 2778 # bound "coexist" method 2779 ({1, 2}.__contains__, (2,)), 2780 # unbound "coexist" method 2781 (set.__contains__, ({1, 2}, 2)), 2782 # built-in class method 2783 (dict.fromkeys, (("a", 1), ("b", 2))), 2784 # built-in static method 2785 (bytearray.maketrans, (b"abc", b"xyz")), 2786 # subclass methods 2787 (Subclass([1,2,2]).count, (2,)), 2788 (Subclass.count, (Subclass([1,2,2]), 2)), 2789 (Subclass.Nested("sweet").count, ("e",)), 2790 (Subclass.Nested.count, (Subclass.Nested("sweet"), "e")), 2791 ) 2792 for proto in range(pickle.HIGHEST_PROTOCOL + 1): 2793 for method, args in c_methods: 2794 with self.subTest(proto=proto, method=method): 2795 unpickled = self.loads(self.dumps(method, proto)) 2796 self.assertEqual(method(*args), unpickled(*args)) 2797 2798 def test_compat_pickle(self): 2799 tests = [ 2800 (range(1, 7), '__builtin__', 'xrange'), 2801 (map(int, '123'), 'itertools', 'imap'), 2802 (functools.reduce, '__builtin__', 'reduce'), 2803 (dbm.whichdb, 'whichdb', 'whichdb'), 2804 (Exception(), 'exceptions', 'Exception'), 2805 (collections.UserDict(), 'UserDict', 'IterableUserDict'), 2806 (collections.UserList(), 'UserList', 'UserList'), 2807 (collections.defaultdict(), 'collections', 'defaultdict'), 2808 ] 2809 for val, mod, name in tests: 2810 for proto in range(3): 2811 with self.subTest(type=type(val), proto=proto): 2812 pickled = self.dumps(val, proto) 2813 self.assertIn(('c%s\n%s' % (mod, name)).encode(), pickled) 2814 self.assertIs(type(self.loads(pickled)), type(val)) 2815 2816 def test_local_lookup_error(self): 2817 # Test that whichmodule() errors out cleanly when looking up 2818 # an assumed globally-reachable object fails. 2819 def f(): 2820 pass 2821 # Since the function is local, lookup will fail 2822 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): 2823 with self.assertRaises((AttributeError, pickle.PicklingError)): 2824 pickletools.dis(self.dumps(f, proto)) 2825 # Same without a __module__ attribute (exercises a different path 2826 # in _pickle.c). 2827 del f.__module__ 2828 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): 2829 with self.assertRaises((AttributeError, pickle.PicklingError)): 2830 pickletools.dis(self.dumps(f, proto)) 2831 # Yet a different path. 2832 f.__name__ = f.__qualname__ 2833 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): 2834 with self.assertRaises((AttributeError, pickle.PicklingError)): 2835 pickletools.dis(self.dumps(f, proto)) 2836 2837 # 2838 # PEP 574 tests below 2839 # 2840 2841 def buffer_like_objects(self): 2842 # Yield buffer-like objects with the bytestring "abcdef" in them 2843 bytestring = b"abcdefgh" 2844 yield ZeroCopyBytes(bytestring) 2845 yield ZeroCopyBytearray(bytestring) 2846 if _testbuffer is not None: 2847 items = list(bytestring) 2848 value = int.from_bytes(bytestring, byteorder='little') 2849 for flags in (0, _testbuffer.ND_WRITABLE): 2850 # 1-D, contiguous 2851 yield PicklableNDArray(items, format='B', shape=(8,), 2852 flags=flags) 2853 # 2-D, C-contiguous 2854 yield PicklableNDArray(items, format='B', shape=(4, 2), 2855 strides=(2, 1), flags=flags) 2856 # 2-D, Fortran-contiguous 2857 yield PicklableNDArray(items, format='B', 2858 shape=(4, 2), strides=(1, 4), 2859 flags=flags) 2860 2861 def test_in_band_buffers(self): 2862 # Test in-band buffers (PEP 574) 2863 for obj in self.buffer_like_objects(): 2864 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): 2865 data = self.dumps(obj, proto) 2866 if obj.c_contiguous and proto >= 5: 2867 # The raw memory bytes are serialized in physical order 2868 self.assertIn(b"abcdefgh", data) 2869 self.assertEqual(count_opcode(pickle.NEXT_BUFFER, data), 0) 2870 if proto >= 5: 2871 self.assertEqual(count_opcode(pickle.SHORT_BINBYTES, data), 2872 1 if obj.readonly else 0) 2873 self.assertEqual(count_opcode(pickle.BYTEARRAY8, data), 2874 0 if obj.readonly else 1) 2875 # Return a true value from buffer_callback should have 2876 # the same effect 2877 def buffer_callback(obj): 2878 return True 2879 data2 = self.dumps(obj, proto, 2880 buffer_callback=buffer_callback) 2881 self.assertEqual(data2, data) 2882 2883 new = self.loads(data) 2884 # It's a copy 2885 self.assertIsNot(new, obj) 2886 self.assertIs(type(new), type(obj)) 2887 self.assertEqual(new, obj) 2888 2889 # XXX Unfortunately cannot test non-contiguous array 2890 # (see comment in PicklableNDArray.__reduce_ex__) 2891 2892 def test_oob_buffers(self): 2893 # Test out-of-band buffers (PEP 574) 2894 for obj in self.buffer_like_objects(): 2895 for proto in range(0, 5): 2896 # Need protocol >= 5 for buffer_callback 2897 with self.assertRaises(ValueError): 2898 self.dumps(obj, proto, 2899 buffer_callback=[].append) 2900 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): 2901 buffers = [] 2902 buffer_callback = lambda pb: buffers.append(pb.raw()) 2903 data = self.dumps(obj, proto, 2904 buffer_callback=buffer_callback) 2905 self.assertNotIn(b"abcdefgh", data) 2906 self.assertEqual(count_opcode(pickle.SHORT_BINBYTES, data), 0) 2907 self.assertEqual(count_opcode(pickle.BYTEARRAY8, data), 0) 2908 self.assertEqual(count_opcode(pickle.NEXT_BUFFER, data), 1) 2909 self.assertEqual(count_opcode(pickle.READONLY_BUFFER, data), 2910 1 if obj.readonly else 0) 2911 2912 if obj.c_contiguous: 2913 self.assertEqual(bytes(buffers[0]), b"abcdefgh") 2914 # Need buffers argument to unpickle properly 2915 with self.assertRaises(pickle.UnpicklingError): 2916 self.loads(data) 2917 2918 new = self.loads(data, buffers=buffers) 2919 if obj.zero_copy_reconstruct: 2920 # Zero-copy achieved 2921 self.assertIs(new, obj) 2922 else: 2923 self.assertIs(type(new), type(obj)) 2924 self.assertEqual(new, obj) 2925 # Non-sequence buffers accepted too 2926 new = self.loads(data, buffers=iter(buffers)) 2927 if obj.zero_copy_reconstruct: 2928 # Zero-copy achieved 2929 self.assertIs(new, obj) 2930 else: 2931 self.assertIs(type(new), type(obj)) 2932 self.assertEqual(new, obj) 2933 2934 def test_oob_buffers_writable_to_readonly(self): 2935 # Test reconstructing readonly object from writable buffer 2936 obj = ZeroCopyBytes(b"foobar") 2937 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): 2938 buffers = [] 2939 buffer_callback = buffers.append 2940 data = self.dumps(obj, proto, buffer_callback=buffer_callback) 2941 2942 buffers = map(bytearray, buffers) 2943 new = self.loads(data, buffers=buffers) 2944 self.assertIs(type(new), type(obj)) 2945 self.assertEqual(new, obj) 2946 2947 def test_picklebuffer_error(self): 2948 # PickleBuffer forbidden with protocol < 5 2949 pb = pickle.PickleBuffer(b"foobar") 2950 for proto in range(0, 5): 2951 with self.assertRaises(pickle.PickleError): 2952 self.dumps(pb, proto) 2953 2954 def test_buffer_callback_error(self): 2955 def buffer_callback(buffers): 2956 1/0 2957 pb = pickle.PickleBuffer(b"foobar") 2958 with self.assertRaises(ZeroDivisionError): 2959 self.dumps(pb, 5, buffer_callback=buffer_callback) 2960 2961 def test_buffers_error(self): 2962 pb = pickle.PickleBuffer(b"foobar") 2963 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): 2964 data = self.dumps(pb, proto, buffer_callback=[].append) 2965 # Non iterable buffers 2966 with self.assertRaises(TypeError): 2967 self.loads(data, buffers=object()) 2968 # Buffer iterable exhausts too early 2969 with self.assertRaises(pickle.UnpicklingError): 2970 self.loads(data, buffers=[]) 2971 2972 def test_inband_accept_default_buffers_argument(self): 2973 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): 2974 data_pickled = self.dumps(1, proto, buffer_callback=None) 2975 data = self.loads(data_pickled, buffers=None) 2976 2977 @unittest.skipIf(np is None, "Test needs Numpy") 2978 def test_buffers_numpy(self): 2979 def check_no_copy(x, y): 2980 np.testing.assert_equal(x, y) 2981 self.assertEqual(x.ctypes.data, y.ctypes.data) 2982 2983 def check_copy(x, y): 2984 np.testing.assert_equal(x, y) 2985 self.assertNotEqual(x.ctypes.data, y.ctypes.data) 2986 2987 def check_array(arr): 2988 # In-band 2989 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): 2990 data = self.dumps(arr, proto) 2991 new = self.loads(data) 2992 check_copy(arr, new) 2993 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): 2994 buffer_callback = lambda _: True 2995 data = self.dumps(arr, proto, buffer_callback=buffer_callback) 2996 new = self.loads(data) 2997 check_copy(arr, new) 2998 # Out-of-band 2999 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): 3000 buffers = [] 3001 buffer_callback = buffers.append 3002 data = self.dumps(arr, proto, buffer_callback=buffer_callback) 3003 new = self.loads(data, buffers=buffers) 3004 if arr.flags.c_contiguous or arr.flags.f_contiguous: 3005 check_no_copy(arr, new) 3006 else: 3007 check_copy(arr, new) 3008 3009 # 1-D 3010 arr = np.arange(6) 3011 check_array(arr) 3012 # 1-D, non-contiguous 3013 check_array(arr[::2]) 3014 # 2-D, C-contiguous 3015 arr = np.arange(12).reshape((3, 4)) 3016 check_array(arr) 3017 # 2-D, F-contiguous 3018 check_array(arr.T) 3019 # 2-D, non-contiguous 3020 check_array(arr[::2]) 3021 3022 3023class BigmemPickleTests(unittest.TestCase): 3024 3025 # Binary protocols can serialize longs of up to 2 GiB-1 3026 3027 @bigmemtest(size=_2G, memuse=3.6, dry_run=False) 3028 def test_huge_long_32b(self, size): 3029 data = 1 << (8 * size) 3030 try: 3031 for proto in protocols: 3032 if proto < 2: 3033 continue 3034 with self.subTest(proto=proto): 3035 with self.assertRaises((ValueError, OverflowError)): 3036 self.dumps(data, protocol=proto) 3037 finally: 3038 data = None 3039 3040 # Protocol 3 can serialize up to 4 GiB-1 as a bytes object 3041 # (older protocols don't have a dedicated opcode for bytes and are 3042 # too inefficient) 3043 3044 @bigmemtest(size=_2G, memuse=2.5, dry_run=False) 3045 def test_huge_bytes_32b(self, size): 3046 data = b"abcd" * (size // 4) 3047 try: 3048 for proto in protocols: 3049 if proto < 3: 3050 continue 3051 with self.subTest(proto=proto): 3052 try: 3053 pickled = self.dumps(data, protocol=proto) 3054 header = (pickle.BINBYTES + 3055 struct.pack("<I", len(data))) 3056 data_start = pickled.index(data) 3057 self.assertEqual( 3058 header, 3059 pickled[data_start-len(header):data_start]) 3060 finally: 3061 pickled = None 3062 finally: 3063 data = None 3064 3065 @bigmemtest(size=_4G, memuse=2.5, dry_run=False) 3066 def test_huge_bytes_64b(self, size): 3067 data = b"acbd" * (size // 4) 3068 try: 3069 for proto in protocols: 3070 if proto < 3: 3071 continue 3072 with self.subTest(proto=proto): 3073 if proto == 3: 3074 # Protocol 3 does not support large bytes objects. 3075 # Verify that we do not crash when processing one. 3076 with self.assertRaises((ValueError, OverflowError)): 3077 self.dumps(data, protocol=proto) 3078 continue 3079 try: 3080 pickled = self.dumps(data, protocol=proto) 3081 header = (pickle.BINBYTES8 + 3082 struct.pack("<Q", len(data))) 3083 data_start = pickled.index(data) 3084 self.assertEqual( 3085 header, 3086 pickled[data_start-len(header):data_start]) 3087 finally: 3088 pickled = None 3089 finally: 3090 data = None 3091 3092 # All protocols use 1-byte per printable ASCII character; we add another 3093 # byte because the encoded form has to be copied into the internal buffer. 3094 3095 @bigmemtest(size=_2G, memuse=8, dry_run=False) 3096 def test_huge_str_32b(self, size): 3097 data = "abcd" * (size // 4) 3098 try: 3099 for proto in protocols: 3100 if proto == 0: 3101 continue 3102 with self.subTest(proto=proto): 3103 try: 3104 pickled = self.dumps(data, protocol=proto) 3105 header = (pickle.BINUNICODE + 3106 struct.pack("<I", len(data))) 3107 data_start = pickled.index(b'abcd') 3108 self.assertEqual( 3109 header, 3110 pickled[data_start-len(header):data_start]) 3111 self.assertEqual((pickled.rindex(b"abcd") + len(b"abcd") - 3112 pickled.index(b"abcd")), len(data)) 3113 finally: 3114 pickled = None 3115 finally: 3116 data = None 3117 3118 # BINUNICODE (protocols 1, 2 and 3) cannot carry more than 2**32 - 1 bytes 3119 # of utf-8 encoded unicode. BINUNICODE8 (protocol 4) supports these huge 3120 # unicode strings however. 3121 3122 @bigmemtest(size=_4G, memuse=8, dry_run=False) 3123 def test_huge_str_64b(self, size): 3124 data = "abcd" * (size // 4) 3125 try: 3126 for proto in protocols: 3127 if proto == 0: 3128 continue 3129 with self.subTest(proto=proto): 3130 if proto < 4: 3131 with self.assertRaises((ValueError, OverflowError)): 3132 self.dumps(data, protocol=proto) 3133 continue 3134 try: 3135 pickled = self.dumps(data, protocol=proto) 3136 header = (pickle.BINUNICODE8 + 3137 struct.pack("<Q", len(data))) 3138 data_start = pickled.index(b'abcd') 3139 self.assertEqual( 3140 header, 3141 pickled[data_start-len(header):data_start]) 3142 self.assertEqual((pickled.rindex(b"abcd") + len(b"abcd") - 3143 pickled.index(b"abcd")), len(data)) 3144 finally: 3145 pickled = None 3146 finally: 3147 data = None 3148 3149 3150# Test classes for reduce_ex 3151 3152class REX_one(object): 3153 """No __reduce_ex__ here, but inheriting it from object""" 3154 _reduce_called = 0 3155 def __reduce__(self): 3156 self._reduce_called = 1 3157 return REX_one, () 3158 3159class REX_two(object): 3160 """No __reduce__ here, but inheriting it from object""" 3161 _proto = None 3162 def __reduce_ex__(self, proto): 3163 self._proto = proto 3164 return REX_two, () 3165 3166class REX_three(object): 3167 _proto = None 3168 def __reduce_ex__(self, proto): 3169 self._proto = proto 3170 return REX_two, () 3171 def __reduce__(self): 3172 raise TestFailed("This __reduce__ shouldn't be called") 3173 3174class REX_four(object): 3175 """Calling base class method should succeed""" 3176 _proto = None 3177 def __reduce_ex__(self, proto): 3178 self._proto = proto 3179 return object.__reduce_ex__(self, proto) 3180 3181class REX_five(object): 3182 """This one used to fail with infinite recursion""" 3183 _reduce_called = 0 3184 def __reduce__(self): 3185 self._reduce_called = 1 3186 return object.__reduce__(self) 3187 3188class REX_six(object): 3189 """This class is used to check the 4th argument (list iterator) of 3190 the reduce protocol. 3191 """ 3192 def __init__(self, items=None): 3193 self.items = items if items is not None else [] 3194 def __eq__(self, other): 3195 return type(self) is type(other) and self.items == other.items 3196 def append(self, item): 3197 self.items.append(item) 3198 def __reduce__(self): 3199 return type(self), (), None, iter(self.items), None 3200 3201class REX_seven(object): 3202 """This class is used to check the 5th argument (dict iterator) of 3203 the reduce protocol. 3204 """ 3205 def __init__(self, table=None): 3206 self.table = table if table is not None else {} 3207 def __eq__(self, other): 3208 return type(self) is type(other) and self.table == other.table 3209 def __setitem__(self, key, value): 3210 self.table[key] = value 3211 def __reduce__(self): 3212 return type(self), (), None, None, iter(self.table.items()) 3213 3214class REX_state(object): 3215 """This class is used to check the 3th argument (state) of 3216 the reduce protocol. 3217 """ 3218 def __init__(self, state=None): 3219 self.state = state 3220 def __eq__(self, other): 3221 return type(self) is type(other) and self.state == other.state 3222 def __setstate__(self, state): 3223 self.state = state 3224 def __reduce__(self): 3225 return type(self), (), self.state 3226 3227 3228# Test classes for newobj 3229 3230class MyInt(int): 3231 sample = 1 3232 3233class MyFloat(float): 3234 sample = 1.0 3235 3236class MyComplex(complex): 3237 sample = 1.0 + 0.0j 3238 3239class MyStr(str): 3240 sample = "hello" 3241 3242class MyUnicode(str): 3243 sample = "hello \u1234" 3244 3245class MyTuple(tuple): 3246 sample = (1, 2, 3) 3247 3248class MyList(list): 3249 sample = [1, 2, 3] 3250 3251class MyDict(dict): 3252 sample = {"a": 1, "b": 2} 3253 3254class MySet(set): 3255 sample = {"a", "b"} 3256 3257class MyFrozenSet(frozenset): 3258 sample = frozenset({"a", "b"}) 3259 3260myclasses = [MyInt, MyFloat, 3261 MyComplex, 3262 MyStr, MyUnicode, 3263 MyTuple, MyList, MyDict, MySet, MyFrozenSet] 3264 3265class MyIntWithNew(int): 3266 def __new__(cls, value): 3267 raise AssertionError 3268 3269class MyIntWithNew2(MyIntWithNew): 3270 __new__ = int.__new__ 3271 3272 3273class SlotList(MyList): 3274 __slots__ = ["foo"] 3275 3276class SimpleNewObj(int): 3277 def __init__(self, *args, **kwargs): 3278 # raise an error, to make sure this isn't called 3279 raise TypeError("SimpleNewObj.__init__() didn't expect to get called") 3280 def __eq__(self, other): 3281 return int(self) == int(other) and self.__dict__ == other.__dict__ 3282 3283class ComplexNewObj(SimpleNewObj): 3284 def __getnewargs__(self): 3285 return ('%X' % self, 16) 3286 3287class ComplexNewObjEx(SimpleNewObj): 3288 def __getnewargs_ex__(self): 3289 return ('%X' % self,), {'base': 16} 3290 3291class BadGetattr: 3292 def __getattr__(self, key): 3293 self.foo 3294 3295 3296class AbstractPickleModuleTests(unittest.TestCase): 3297 3298 def test_dump_closed_file(self): 3299 f = open(TESTFN, "wb") 3300 try: 3301 f.close() 3302 self.assertRaises(ValueError, self.dump, 123, f) 3303 finally: 3304 support.unlink(TESTFN) 3305 3306 def test_load_closed_file(self): 3307 f = open(TESTFN, "wb") 3308 try: 3309 f.close() 3310 self.assertRaises(ValueError, self.dump, 123, f) 3311 finally: 3312 support.unlink(TESTFN) 3313 3314 def test_load_from_and_dump_to_file(self): 3315 stream = io.BytesIO() 3316 data = [123, {}, 124] 3317 self.dump(data, stream) 3318 stream.seek(0) 3319 unpickled = self.load(stream) 3320 self.assertEqual(unpickled, data) 3321 3322 def test_highest_protocol(self): 3323 # Of course this needs to be changed when HIGHEST_PROTOCOL changes. 3324 self.assertEqual(pickle.HIGHEST_PROTOCOL, 5) 3325 3326 def test_callapi(self): 3327 f = io.BytesIO() 3328 # With and without keyword arguments 3329 self.dump(123, f, -1) 3330 self.dump(123, file=f, protocol=-1) 3331 self.dumps(123, -1) 3332 self.dumps(123, protocol=-1) 3333 self.Pickler(f, -1) 3334 self.Pickler(f, protocol=-1) 3335 3336 def test_dump_text_file(self): 3337 f = open(TESTFN, "w") 3338 try: 3339 for proto in protocols: 3340 self.assertRaises(TypeError, self.dump, 123, f, proto) 3341 finally: 3342 f.close() 3343 support.unlink(TESTFN) 3344 3345 def test_incomplete_input(self): 3346 s = io.BytesIO(b"X''.") 3347 self.assertRaises((EOFError, struct.error, pickle.UnpicklingError), self.load, s) 3348 3349 def test_bad_init(self): 3350 # Test issue3664 (pickle can segfault from a badly initialized Pickler). 3351 # Override initialization without calling __init__() of the superclass. 3352 class BadPickler(self.Pickler): 3353 def __init__(self): pass 3354 3355 class BadUnpickler(self.Unpickler): 3356 def __init__(self): pass 3357 3358 self.assertRaises(pickle.PicklingError, BadPickler().dump, 0) 3359 self.assertRaises(pickle.UnpicklingError, BadUnpickler().load) 3360 3361 def check_dumps_loads_oob_buffers(self, dumps, loads): 3362 # No need to do the full gamut of tests here, just enough to 3363 # check that dumps() and loads() redirect their arguments 3364 # to the underlying Pickler and Unpickler, respectively. 3365 obj = ZeroCopyBytes(b"foo") 3366 3367 for proto in range(0, 5): 3368 # Need protocol >= 5 for buffer_callback 3369 with self.assertRaises(ValueError): 3370 dumps(obj, protocol=proto, 3371 buffer_callback=[].append) 3372 for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): 3373 buffers = [] 3374 buffer_callback = buffers.append 3375 data = dumps(obj, protocol=proto, 3376 buffer_callback=buffer_callback) 3377 self.assertNotIn(b"foo", data) 3378 self.assertEqual(bytes(buffers[0]), b"foo") 3379 # Need buffers argument to unpickle properly 3380 with self.assertRaises(pickle.UnpicklingError): 3381 loads(data) 3382 new = loads(data, buffers=buffers) 3383 self.assertIs(new, obj) 3384 3385 def test_dumps_loads_oob_buffers(self): 3386 # Test out-of-band buffers (PEP 574) with top-level dumps() and loads() 3387 self.check_dumps_loads_oob_buffers(self.dumps, self.loads) 3388 3389 def test_dump_load_oob_buffers(self): 3390 # Test out-of-band buffers (PEP 574) with top-level dump() and load() 3391 def dumps(obj, **kwargs): 3392 f = io.BytesIO() 3393 self.dump(obj, f, **kwargs) 3394 return f.getvalue() 3395 3396 def loads(data, **kwargs): 3397 f = io.BytesIO(data) 3398 return self.load(f, **kwargs) 3399 3400 self.check_dumps_loads_oob_buffers(dumps, loads) 3401 3402 3403class AbstractPersistentPicklerTests(unittest.TestCase): 3404 3405 # This class defines persistent_id() and persistent_load() 3406 # functions that should be used by the pickler. All even integers 3407 # are pickled using persistent ids. 3408 3409 def persistent_id(self, object): 3410 if isinstance(object, int) and object % 2 == 0: 3411 self.id_count += 1 3412 return str(object) 3413 elif object == "test_false_value": 3414 self.false_count += 1 3415 return "" 3416 else: 3417 return None 3418 3419 def persistent_load(self, oid): 3420 if not oid: 3421 self.load_false_count += 1 3422 return "test_false_value" 3423 else: 3424 self.load_count += 1 3425 object = int(oid) 3426 assert object % 2 == 0 3427 return object 3428 3429 def test_persistence(self): 3430 L = list(range(10)) + ["test_false_value"] 3431 for proto in protocols: 3432 self.id_count = 0 3433 self.false_count = 0 3434 self.load_false_count = 0 3435 self.load_count = 0 3436 self.assertEqual(self.loads(self.dumps(L, proto)), L) 3437 self.assertEqual(self.id_count, 5) 3438 self.assertEqual(self.false_count, 1) 3439 self.assertEqual(self.load_count, 5) 3440 self.assertEqual(self.load_false_count, 1) 3441 3442 3443class AbstractIdentityPersistentPicklerTests(unittest.TestCase): 3444 3445 def persistent_id(self, obj): 3446 return obj 3447 3448 def persistent_load(self, pid): 3449 return pid 3450 3451 def _check_return_correct_type(self, obj, proto): 3452 unpickled = self.loads(self.dumps(obj, proto)) 3453 self.assertIsInstance(unpickled, type(obj)) 3454 self.assertEqual(unpickled, obj) 3455 3456 def test_return_correct_type(self): 3457 for proto in protocols: 3458 # Protocol 0 supports only ASCII strings. 3459 if proto == 0: 3460 self._check_return_correct_type("abc", 0) 3461 else: 3462 for obj in [b"abc\n", "abc\n", -1, -1.1 * 0.1, str]: 3463 self._check_return_correct_type(obj, proto) 3464 3465 def test_protocol0_is_ascii_only(self): 3466 non_ascii_str = "\N{EMPTY SET}" 3467 self.assertRaises(pickle.PicklingError, self.dumps, non_ascii_str, 0) 3468 pickled = pickle.PERSID + non_ascii_str.encode('utf-8') + b'\n.' 3469 self.assertRaises(pickle.UnpicklingError, self.loads, pickled) 3470 3471 3472class AbstractPicklerUnpicklerObjectTests(unittest.TestCase): 3473 3474 pickler_class = None 3475 unpickler_class = None 3476 3477 def setUp(self): 3478 assert self.pickler_class 3479 assert self.unpickler_class 3480 3481 def test_clear_pickler_memo(self): 3482 # To test whether clear_memo() has any effect, we pickle an object, 3483 # then pickle it again without clearing the memo; the two serialized 3484 # forms should be different. If we clear_memo() and then pickle the 3485 # object again, the third serialized form should be identical to the 3486 # first one we obtained. 3487 data = ["abcdefg", "abcdefg", 44] 3488 for proto in protocols: 3489 f = io.BytesIO() 3490 pickler = self.pickler_class(f, proto) 3491 3492 pickler.dump(data) 3493 first_pickled = f.getvalue() 3494 3495 # Reset BytesIO object. 3496 f.seek(0) 3497 f.truncate() 3498 3499 pickler.dump(data) 3500 second_pickled = f.getvalue() 3501 3502 # Reset the Pickler and BytesIO objects. 3503 pickler.clear_memo() 3504 f.seek(0) 3505 f.truncate() 3506 3507 pickler.dump(data) 3508 third_pickled = f.getvalue() 3509 3510 self.assertNotEqual(first_pickled, second_pickled) 3511 self.assertEqual(first_pickled, third_pickled) 3512 3513 def test_priming_pickler_memo(self): 3514 # Verify that we can set the Pickler's memo attribute. 3515 data = ["abcdefg", "abcdefg", 44] 3516 f = io.BytesIO() 3517 pickler = self.pickler_class(f) 3518 3519 pickler.dump(data) 3520 first_pickled = f.getvalue() 3521 3522 f = io.BytesIO() 3523 primed = self.pickler_class(f) 3524 primed.memo = pickler.memo 3525 3526 primed.dump(data) 3527 primed_pickled = f.getvalue() 3528 3529 self.assertNotEqual(first_pickled, primed_pickled) 3530 3531 def test_priming_unpickler_memo(self): 3532 # Verify that we can set the Unpickler's memo attribute. 3533 data = ["abcdefg", "abcdefg", 44] 3534 f = io.BytesIO() 3535 pickler = self.pickler_class(f) 3536 3537 pickler.dump(data) 3538 first_pickled = f.getvalue() 3539 3540 f = io.BytesIO() 3541 primed = self.pickler_class(f) 3542 primed.memo = pickler.memo 3543 3544 primed.dump(data) 3545 primed_pickled = f.getvalue() 3546 3547 unpickler = self.unpickler_class(io.BytesIO(first_pickled)) 3548 unpickled_data1 = unpickler.load() 3549 3550 self.assertEqual(unpickled_data1, data) 3551 3552 primed = self.unpickler_class(io.BytesIO(primed_pickled)) 3553 primed.memo = unpickler.memo 3554 unpickled_data2 = primed.load() 3555 3556 primed.memo.clear() 3557 3558 self.assertEqual(unpickled_data2, data) 3559 self.assertTrue(unpickled_data2 is unpickled_data1) 3560 3561 def test_reusing_unpickler_objects(self): 3562 data1 = ["abcdefg", "abcdefg", 44] 3563 f = io.BytesIO() 3564 pickler = self.pickler_class(f) 3565 pickler.dump(data1) 3566 pickled1 = f.getvalue() 3567 3568 data2 = ["abcdefg", 44, 44] 3569 f = io.BytesIO() 3570 pickler = self.pickler_class(f) 3571 pickler.dump(data2) 3572 pickled2 = f.getvalue() 3573 3574 f = io.BytesIO() 3575 f.write(pickled1) 3576 f.seek(0) 3577 unpickler = self.unpickler_class(f) 3578 self.assertEqual(unpickler.load(), data1) 3579 3580 f.seek(0) 3581 f.truncate() 3582 f.write(pickled2) 3583 f.seek(0) 3584 self.assertEqual(unpickler.load(), data2) 3585 3586 def _check_multiple_unpicklings(self, ioclass, *, seekable=True): 3587 for proto in protocols: 3588 with self.subTest(proto=proto): 3589 data1 = [(x, str(x)) for x in range(2000)] + [b"abcde", len] 3590 f = ioclass() 3591 pickler = self.pickler_class(f, protocol=proto) 3592 pickler.dump(data1) 3593 pickled = f.getvalue() 3594 3595 N = 5 3596 f = ioclass(pickled * N) 3597 unpickler = self.unpickler_class(f) 3598 for i in range(N): 3599 if seekable: 3600 pos = f.tell() 3601 self.assertEqual(unpickler.load(), data1) 3602 if seekable: 3603 self.assertEqual(f.tell(), pos + len(pickled)) 3604 self.assertRaises(EOFError, unpickler.load) 3605 3606 def test_multiple_unpicklings_seekable(self): 3607 self._check_multiple_unpicklings(io.BytesIO) 3608 3609 def test_multiple_unpicklings_unseekable(self): 3610 self._check_multiple_unpicklings(UnseekableIO, seekable=False) 3611 3612 def test_multiple_unpicklings_minimal(self): 3613 # File-like object that doesn't support peek() and readinto() 3614 # (bpo-39681) 3615 self._check_multiple_unpicklings(MinimalIO, seekable=False) 3616 3617 def test_unpickling_buffering_readline(self): 3618 # Issue #12687: the unpickler's buffering logic could fail with 3619 # text mode opcodes. 3620 data = list(range(10)) 3621 for proto in protocols: 3622 for buf_size in range(1, 11): 3623 f = io.BufferedRandom(io.BytesIO(), buffer_size=buf_size) 3624 pickler = self.pickler_class(f, protocol=proto) 3625 pickler.dump(data) 3626 f.seek(0) 3627 unpickler = self.unpickler_class(f) 3628 self.assertEqual(unpickler.load(), data) 3629 3630 3631# Tests for dispatch_table attribute 3632 3633REDUCE_A = 'reduce_A' 3634 3635class AAA(object): 3636 def __reduce__(self): 3637 return str, (REDUCE_A,) 3638 3639class BBB(object): 3640 def __init__(self): 3641 # Add an instance attribute to enable state-saving routines at pickling 3642 # time. 3643 self.a = "some attribute" 3644 3645 def __setstate__(self, state): 3646 self.a = "BBB.__setstate__" 3647 3648 3649def setstate_bbb(obj, state): 3650 """Custom state setter for BBB objects 3651 3652 Such callable may be created by other persons than the ones who created the 3653 BBB class. If passed as the state_setter item of a custom reducer, this 3654 allows for custom state setting behavior of BBB objects. One can think of 3655 it as the analogous of list_setitems or dict_setitems but for foreign 3656 classes/functions. 3657 """ 3658 obj.a = "custom state_setter" 3659 3660 3661 3662class AbstractCustomPicklerClass: 3663 """Pickler implementing a reducing hook using reducer_override.""" 3664 def reducer_override(self, obj): 3665 obj_name = getattr(obj, "__name__", None) 3666 3667 if obj_name == 'f': 3668 # asking the pickler to save f as 5 3669 return int, (5, ) 3670 3671 if obj_name == 'MyClass': 3672 return str, ('some str',) 3673 3674 elif obj_name == 'g': 3675 # in this case, the callback returns an invalid result (not a 2-5 3676 # tuple or a string), the pickler should raise a proper error. 3677 return False 3678 3679 elif obj_name == 'h': 3680 # Simulate a case when the reducer fails. The error should 3681 # be propagated to the original ``dump`` call. 3682 raise ValueError('The reducer just failed') 3683 3684 return NotImplemented 3685 3686class AbstractHookTests(unittest.TestCase): 3687 def test_pickler_hook(self): 3688 # test the ability of a custom, user-defined CPickler subclass to 3689 # override the default reducing routines of any type using the method 3690 # reducer_override 3691 3692 def f(): 3693 pass 3694 3695 def g(): 3696 pass 3697 3698 def h(): 3699 pass 3700 3701 class MyClass: 3702 pass 3703 3704 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): 3705 with self.subTest(proto=proto): 3706 bio = io.BytesIO() 3707 p = self.pickler_class(bio, proto) 3708 3709 p.dump([f, MyClass, math.log]) 3710 new_f, some_str, math_log = pickle.loads(bio.getvalue()) 3711 3712 self.assertEqual(new_f, 5) 3713 self.assertEqual(some_str, 'some str') 3714 # math.log does not have its usual reducer overriden, so the 3715 # custom reduction callback should silently direct the pickler 3716 # to the default pickling by attribute, by returning 3717 # NotImplemented 3718 self.assertIs(math_log, math.log) 3719 3720 with self.assertRaises(pickle.PicklingError): 3721 p.dump(g) 3722 3723 with self.assertRaisesRegex( 3724 ValueError, 'The reducer just failed'): 3725 p.dump(h) 3726 3727 @support.cpython_only 3728 def test_reducer_override_no_reference_cycle(self): 3729 # bpo-39492: reducer_override used to induce a spurious reference cycle 3730 # inside the Pickler object, that could prevent all serialized objects 3731 # from being garbage-collected without explicity invoking gc.collect. 3732 3733 for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): 3734 with self.subTest(proto=proto): 3735 def f(): 3736 pass 3737 3738 wr = weakref.ref(f) 3739 3740 bio = io.BytesIO() 3741 p = self.pickler_class(bio, proto) 3742 p.dump(f) 3743 new_f = pickle.loads(bio.getvalue()) 3744 assert new_f == 5 3745 3746 del p 3747 del f 3748 3749 self.assertIsNone(wr()) 3750 3751 3752class AbstractDispatchTableTests(unittest.TestCase): 3753 3754 def test_default_dispatch_table(self): 3755 # No dispatch_table attribute by default 3756 f = io.BytesIO() 3757 p = self.pickler_class(f, 0) 3758 with self.assertRaises(AttributeError): 3759 p.dispatch_table 3760 self.assertFalse(hasattr(p, 'dispatch_table')) 3761 3762 def test_class_dispatch_table(self): 3763 # A dispatch_table attribute can be specified class-wide 3764 dt = self.get_dispatch_table() 3765 3766 class MyPickler(self.pickler_class): 3767 dispatch_table = dt 3768 3769 def dumps(obj, protocol=None): 3770 f = io.BytesIO() 3771 p = MyPickler(f, protocol) 3772 self.assertEqual(p.dispatch_table, dt) 3773 p.dump(obj) 3774 return f.getvalue() 3775 3776 self._test_dispatch_table(dumps, dt) 3777 3778 def test_instance_dispatch_table(self): 3779 # A dispatch_table attribute can also be specified instance-wide 3780 dt = self.get_dispatch_table() 3781 3782 def dumps(obj, protocol=None): 3783 f = io.BytesIO() 3784 p = self.pickler_class(f, protocol) 3785 p.dispatch_table = dt 3786 self.assertEqual(p.dispatch_table, dt) 3787 p.dump(obj) 3788 return f.getvalue() 3789 3790 self._test_dispatch_table(dumps, dt) 3791 3792 def _test_dispatch_table(self, dumps, dispatch_table): 3793 def custom_load_dump(obj): 3794 return pickle.loads(dumps(obj, 0)) 3795 3796 def default_load_dump(obj): 3797 return pickle.loads(pickle.dumps(obj, 0)) 3798 3799 # pickling complex numbers using protocol 0 relies on copyreg 3800 # so check pickling a complex number still works 3801 z = 1 + 2j 3802 self.assertEqual(custom_load_dump(z), z) 3803 self.assertEqual(default_load_dump(z), z) 3804 3805 # modify pickling of complex 3806 REDUCE_1 = 'reduce_1' 3807 def reduce_1(obj): 3808 return str, (REDUCE_1,) 3809 dispatch_table[complex] = reduce_1 3810 self.assertEqual(custom_load_dump(z), REDUCE_1) 3811 self.assertEqual(default_load_dump(z), z) 3812 3813 # check picklability of AAA and BBB 3814 a = AAA() 3815 b = BBB() 3816 self.assertEqual(custom_load_dump(a), REDUCE_A) 3817 self.assertIsInstance(custom_load_dump(b), BBB) 3818 self.assertEqual(default_load_dump(a), REDUCE_A) 3819 self.assertIsInstance(default_load_dump(b), BBB) 3820 3821 # modify pickling of BBB 3822 dispatch_table[BBB] = reduce_1 3823 self.assertEqual(custom_load_dump(a), REDUCE_A) 3824 self.assertEqual(custom_load_dump(b), REDUCE_1) 3825 self.assertEqual(default_load_dump(a), REDUCE_A) 3826 self.assertIsInstance(default_load_dump(b), BBB) 3827 3828 # revert pickling of BBB and modify pickling of AAA 3829 REDUCE_2 = 'reduce_2' 3830 def reduce_2(obj): 3831 return str, (REDUCE_2,) 3832 dispatch_table[AAA] = reduce_2 3833 del dispatch_table[BBB] 3834 self.assertEqual(custom_load_dump(a), REDUCE_2) 3835 self.assertIsInstance(custom_load_dump(b), BBB) 3836 self.assertEqual(default_load_dump(a), REDUCE_A) 3837 self.assertIsInstance(default_load_dump(b), BBB) 3838 3839 # End-to-end testing of save_reduce with the state_setter keyword 3840 # argument. This is a dispatch_table test as the primary goal of 3841 # state_setter is to tweak objects reduction behavior. 3842 # In particular, state_setter is useful when the default __setstate__ 3843 # behavior is not flexible enough. 3844 3845 # No custom reducer for b has been registered for now, so 3846 # BBB.__setstate__ should be used at unpickling time 3847 self.assertEqual(default_load_dump(b).a, "BBB.__setstate__") 3848 3849 def reduce_bbb(obj): 3850 return BBB, (), obj.__dict__, None, None, setstate_bbb 3851 3852 dispatch_table[BBB] = reduce_bbb 3853 3854 # The custom reducer reduce_bbb includes a state setter, that should 3855 # have priority over BBB.__setstate__ 3856 self.assertEqual(custom_load_dump(b).a, "custom state_setter") 3857 3858 3859if __name__ == "__main__": 3860 # Print some stuff that can be used to rewrite DATA{0,1,2} 3861 from pickletools import dis 3862 x = create_data() 3863 for i in range(pickle.HIGHEST_PROTOCOL+1): 3864 p = pickle.dumps(x, i) 3865 print("DATA{0} = (".format(i)) 3866 for j in range(0, len(p), 20): 3867 b = bytes(p[j:j+20]) 3868 print(" {0!r}".format(b)) 3869 print(")") 3870 print() 3871 print("# Disassembly of DATA{0}".format(i)) 3872 print("DATA{0}_DIS = \"\"\"\\".format(i)) 3873 dis(p) 3874 print("\"\"\"") 3875 print() 3876