1from test import support 2from test.support import os_helper 3import array 4import io 5import marshal 6import sys 7import unittest 8import os 9import types 10 11try: 12 import _testcapi 13except ImportError: 14 _testcapi = None 15 16class HelperMixin: 17 def helper(self, sample, *extra): 18 new = marshal.loads(marshal.dumps(sample, *extra)) 19 self.assertEqual(sample, new) 20 try: 21 with open(os_helper.TESTFN, "wb") as f: 22 marshal.dump(sample, f, *extra) 23 with open(os_helper.TESTFN, "rb") as f: 24 new = marshal.load(f) 25 self.assertEqual(sample, new) 26 finally: 27 os_helper.unlink(os_helper.TESTFN) 28 29class IntTestCase(unittest.TestCase, HelperMixin): 30 def test_ints(self): 31 # Test a range of Python ints larger than the machine word size. 32 n = sys.maxsize ** 2 33 while n: 34 for expected in (-n, n): 35 self.helper(expected) 36 n = n >> 1 37 38 def test_int64(self): 39 # Simulate int marshaling with TYPE_INT64. 40 maxint64 = (1 << 63) - 1 41 minint64 = -maxint64-1 42 for base in maxint64, minint64, -maxint64, -(minint64 >> 1): 43 while base: 44 s = b'I' + int.to_bytes(base, 8, 'little', signed=True) 45 got = marshal.loads(s) 46 self.assertEqual(base, got) 47 if base == -1: # a fixed-point for shifting right 1 48 base = 0 49 else: 50 base >>= 1 51 52 got = marshal.loads(b'I\xfe\xdc\xba\x98\x76\x54\x32\x10') 53 self.assertEqual(got, 0x1032547698badcfe) 54 got = marshal.loads(b'I\x01\x23\x45\x67\x89\xab\xcd\xef') 55 self.assertEqual(got, -0x1032547698badcff) 56 got = marshal.loads(b'I\x08\x19\x2a\x3b\x4c\x5d\x6e\x7f') 57 self.assertEqual(got, 0x7f6e5d4c3b2a1908) 58 got = marshal.loads(b'I\xf7\xe6\xd5\xc4\xb3\xa2\x91\x80') 59 self.assertEqual(got, -0x7f6e5d4c3b2a1909) 60 61 def test_bool(self): 62 for b in (True, False): 63 self.helper(b) 64 65class FloatTestCase(unittest.TestCase, HelperMixin): 66 def test_floats(self): 67 # Test a few floats 68 small = 1e-25 69 n = sys.maxsize * 3.7e250 70 while n > small: 71 for expected in (-n, n): 72 self.helper(float(expected)) 73 n /= 123.4567 74 75 f = 0.0 76 s = marshal.dumps(f, 2) 77 got = marshal.loads(s) 78 self.assertEqual(f, got) 79 # and with version <= 1 (floats marshalled differently then) 80 s = marshal.dumps(f, 1) 81 got = marshal.loads(s) 82 self.assertEqual(f, got) 83 84 n = sys.maxsize * 3.7e-250 85 while n < small: 86 for expected in (-n, n): 87 f = float(expected) 88 self.helper(f) 89 self.helper(f, 1) 90 n *= 123.4567 91 92class StringTestCase(unittest.TestCase, HelperMixin): 93 def test_unicode(self): 94 for s in ["", "Andr\xe8 Previn", "abc", " "*10000]: 95 self.helper(marshal.loads(marshal.dumps(s))) 96 97 def test_string(self): 98 for s in ["", "Andr\xe8 Previn", "abc", " "*10000]: 99 self.helper(s) 100 101 def test_bytes(self): 102 for s in [b"", b"Andr\xe8 Previn", b"abc", b" "*10000]: 103 self.helper(s) 104 105class ExceptionTestCase(unittest.TestCase): 106 def test_exceptions(self): 107 new = marshal.loads(marshal.dumps(StopIteration)) 108 self.assertEqual(StopIteration, new) 109 110class CodeTestCase(unittest.TestCase): 111 def test_code(self): 112 co = ExceptionTestCase.test_exceptions.__code__ 113 new = marshal.loads(marshal.dumps(co)) 114 self.assertEqual(co, new) 115 116 def test_many_codeobjects(self): 117 # Issue2957: bad recursion count on code objects 118 count = 5000 # more than MAX_MARSHAL_STACK_DEPTH 119 codes = (ExceptionTestCase.test_exceptions.__code__,) * count 120 marshal.loads(marshal.dumps(codes)) 121 122 def test_different_filenames(self): 123 co1 = compile("x", "f1", "exec") 124 co2 = compile("y", "f2", "exec") 125 co1, co2 = marshal.loads(marshal.dumps((co1, co2))) 126 self.assertEqual(co1.co_filename, "f1") 127 self.assertEqual(co2.co_filename, "f2") 128 129 @support.cpython_only 130 def test_same_filename_used(self): 131 s = """def f(): pass\ndef g(): pass""" 132 co = compile(s, "myfile", "exec") 133 co = marshal.loads(marshal.dumps(co)) 134 for obj in co.co_consts: 135 if isinstance(obj, types.CodeType): 136 self.assertIs(co.co_filename, obj.co_filename) 137 138class ContainerTestCase(unittest.TestCase, HelperMixin): 139 d = {'astring': 'foo@bar.baz.spam', 140 'afloat': 7283.43, 141 'anint': 2**20, 142 'ashortlong': 2, 143 'alist': ['.zyx.41'], 144 'atuple': ('.zyx.41',)*10, 145 'aboolean': False, 146 'aunicode': "Andr\xe8 Previn" 147 } 148 149 def test_dict(self): 150 self.helper(self.d) 151 152 def test_list(self): 153 self.helper(list(self.d.items())) 154 155 def test_tuple(self): 156 self.helper(tuple(self.d.keys())) 157 158 def test_sets(self): 159 for constructor in (set, frozenset): 160 self.helper(constructor(self.d.keys())) 161 162 163class BufferTestCase(unittest.TestCase, HelperMixin): 164 165 def test_bytearray(self): 166 b = bytearray(b"abc") 167 self.helper(b) 168 new = marshal.loads(marshal.dumps(b)) 169 self.assertEqual(type(new), bytes) 170 171 def test_memoryview(self): 172 b = memoryview(b"abc") 173 self.helper(b) 174 new = marshal.loads(marshal.dumps(b)) 175 self.assertEqual(type(new), bytes) 176 177 def test_array(self): 178 a = array.array('B', b"abc") 179 new = marshal.loads(marshal.dumps(a)) 180 self.assertEqual(new, b"abc") 181 182 183class BugsTestCase(unittest.TestCase): 184 def test_bug_5888452(self): 185 # Simple-minded check for SF 588452: Debug build crashes 186 marshal.dumps([128] * 1000) 187 188 def test_patch_873224(self): 189 self.assertRaises(Exception, marshal.loads, b'0') 190 self.assertRaises(Exception, marshal.loads, b'f') 191 self.assertRaises(Exception, marshal.loads, marshal.dumps(2**65)[:-1]) 192 193 def test_version_argument(self): 194 # Python 2.4.0 crashes for any call to marshal.dumps(x, y) 195 self.assertEqual(marshal.loads(marshal.dumps(5, 0)), 5) 196 self.assertEqual(marshal.loads(marshal.dumps(5, 1)), 5) 197 198 def test_fuzz(self): 199 # simple test that it's at least not *totally* trivial to 200 # crash from bad marshal data 201 for i in range(256): 202 c = bytes([i]) 203 try: 204 marshal.loads(c) 205 except Exception: 206 pass 207 208 def test_loads_recursion(self): 209 def run_tests(N, check): 210 # (((...None...),),) 211 check(b')\x01' * N + b'N') 212 check(b'(\x01\x00\x00\x00' * N + b'N') 213 # [[[...None...]]] 214 check(b'[\x01\x00\x00\x00' * N + b'N') 215 # {None: {None: {None: ...None...}}} 216 check(b'{N' * N + b'N' + b'0' * N) 217 # frozenset([frozenset([frozenset([...None...])])]) 218 check(b'>\x01\x00\x00\x00' * N + b'N') 219 # Check that the generated marshal data is valid and marshal.loads() 220 # works for moderately deep nesting 221 run_tests(100, marshal.loads) 222 # Very deeply nested structure shouldn't blow the stack 223 def check(s): 224 self.assertRaises(ValueError, marshal.loads, s) 225 run_tests(2**20, check) 226 227 def test_recursion_limit(self): 228 # Create a deeply nested structure. 229 head = last = [] 230 # The max stack depth should match the value in Python/marshal.c. 231 # BUG: https://bugs.python.org/issue33720 232 # Windows always limits the maximum depth on release and debug builds 233 #if os.name == 'nt' and hasattr(sys, 'gettotalrefcount'): 234 if os.name == 'nt': 235 MAX_MARSHAL_STACK_DEPTH = 1000 236 else: 237 MAX_MARSHAL_STACK_DEPTH = 2000 238 for i in range(MAX_MARSHAL_STACK_DEPTH - 2): 239 last.append([0]) 240 last = last[-1] 241 242 # Verify we don't blow out the stack with dumps/load. 243 data = marshal.dumps(head) 244 new_head = marshal.loads(data) 245 # Don't use == to compare objects, it can exceed the recursion limit. 246 self.assertEqual(len(new_head), len(head)) 247 self.assertEqual(len(new_head[0]), len(head[0])) 248 self.assertEqual(len(new_head[-1]), len(head[-1])) 249 250 last.append([0]) 251 self.assertRaises(ValueError, marshal.dumps, head) 252 253 def test_exact_type_match(self): 254 # Former bug: 255 # >>> class Int(int): pass 256 # >>> type(loads(dumps(Int()))) 257 # <type 'int'> 258 for typ in (int, float, complex, tuple, list, dict, set, frozenset): 259 # Note: str subclasses are not tested because they get handled 260 # by marshal's routines for objects supporting the buffer API. 261 subtyp = type('subtyp', (typ,), {}) 262 self.assertRaises(ValueError, marshal.dumps, subtyp()) 263 264 # Issue #1792 introduced a change in how marshal increases the size of its 265 # internal buffer; this test ensures that the new code is exercised. 266 def test_large_marshal(self): 267 size = int(1e6) 268 testString = 'abc' * size 269 marshal.dumps(testString) 270 271 def test_invalid_longs(self): 272 # Issue #7019: marshal.loads shouldn't produce unnormalized PyLongs 273 invalid_string = b'l\x02\x00\x00\x00\x00\x00\x00\x00' 274 self.assertRaises(ValueError, marshal.loads, invalid_string) 275 276 def test_multiple_dumps_and_loads(self): 277 # Issue 12291: marshal.load() should be callable multiple times 278 # with interleaved data written by non-marshal code 279 # Adapted from a patch by Engelbert Gruber. 280 data = (1, 'abc', b'def', 1.0, (2, 'a', ['b', b'c'])) 281 for interleaved in (b'', b'0123'): 282 ilen = len(interleaved) 283 positions = [] 284 try: 285 with open(os_helper.TESTFN, 'wb') as f: 286 for d in data: 287 marshal.dump(d, f) 288 if ilen: 289 f.write(interleaved) 290 positions.append(f.tell()) 291 with open(os_helper.TESTFN, 'rb') as f: 292 for i, d in enumerate(data): 293 self.assertEqual(d, marshal.load(f)) 294 if ilen: 295 f.read(ilen) 296 self.assertEqual(positions[i], f.tell()) 297 finally: 298 os_helper.unlink(os_helper.TESTFN) 299 300 def test_loads_reject_unicode_strings(self): 301 # Issue #14177: marshal.loads() should not accept unicode strings 302 unicode_string = 'T' 303 self.assertRaises(TypeError, marshal.loads, unicode_string) 304 305 def test_bad_reader(self): 306 class BadReader(io.BytesIO): 307 def readinto(self, buf): 308 n = super().readinto(buf) 309 if n is not None and n > 4: 310 n += 10**6 311 return n 312 for value in (1.0, 1j, b'0123456789', '0123456789'): 313 self.assertRaises(ValueError, marshal.load, 314 BadReader(marshal.dumps(value))) 315 316 def test_eof(self): 317 data = marshal.dumps(("hello", "dolly", None)) 318 for i in range(len(data)): 319 self.assertRaises(EOFError, marshal.loads, data[0: i]) 320 321LARGE_SIZE = 2**31 322pointer_size = 8 if sys.maxsize > 0xFFFFFFFF else 4 323 324class NullWriter: 325 def write(self, s): 326 pass 327 328@unittest.skipIf(LARGE_SIZE > sys.maxsize, "test cannot run on 32-bit systems") 329class LargeValuesTestCase(unittest.TestCase): 330 def check_unmarshallable(self, data): 331 self.assertRaises(ValueError, marshal.dump, data, NullWriter()) 332 333 @support.bigmemtest(size=LARGE_SIZE, memuse=2, dry_run=False) 334 def test_bytes(self, size): 335 self.check_unmarshallable(b'x' * size) 336 337 @support.bigmemtest(size=LARGE_SIZE, memuse=2, dry_run=False) 338 def test_str(self, size): 339 self.check_unmarshallable('x' * size) 340 341 @support.bigmemtest(size=LARGE_SIZE, memuse=pointer_size + 1, dry_run=False) 342 def test_tuple(self, size): 343 self.check_unmarshallable((None,) * size) 344 345 @support.bigmemtest(size=LARGE_SIZE, memuse=pointer_size + 1, dry_run=False) 346 def test_list(self, size): 347 self.check_unmarshallable([None] * size) 348 349 @support.bigmemtest(size=LARGE_SIZE, 350 memuse=pointer_size*12 + sys.getsizeof(LARGE_SIZE-1), 351 dry_run=False) 352 def test_set(self, size): 353 self.check_unmarshallable(set(range(size))) 354 355 @support.bigmemtest(size=LARGE_SIZE, 356 memuse=pointer_size*12 + sys.getsizeof(LARGE_SIZE-1), 357 dry_run=False) 358 def test_frozenset(self, size): 359 self.check_unmarshallable(frozenset(range(size))) 360 361 @support.bigmemtest(size=LARGE_SIZE, memuse=2, dry_run=False) 362 def test_bytearray(self, size): 363 self.check_unmarshallable(bytearray(size)) 364 365def CollectObjectIDs(ids, obj): 366 """Collect object ids seen in a structure""" 367 if id(obj) in ids: 368 return 369 ids.add(id(obj)) 370 if isinstance(obj, (list, tuple, set, frozenset)): 371 for e in obj: 372 CollectObjectIDs(ids, e) 373 elif isinstance(obj, dict): 374 for k, v in obj.items(): 375 CollectObjectIDs(ids, k) 376 CollectObjectIDs(ids, v) 377 return len(ids) 378 379class InstancingTestCase(unittest.TestCase, HelperMixin): 380 keys = (123, 1.2345, 'abc', (123, 'abc'), frozenset({123, 'abc'})) 381 382 def helper3(self, rsample, recursive=False, simple=False): 383 #we have two instances 384 sample = (rsample, rsample) 385 386 n0 = CollectObjectIDs(set(), sample) 387 388 for v in range(3, marshal.version + 1): 389 s3 = marshal.dumps(sample, v) 390 n3 = CollectObjectIDs(set(), marshal.loads(s3)) 391 392 #same number of instances generated 393 self.assertEqual(n3, n0) 394 395 if not recursive: 396 #can compare with version 2 397 s2 = marshal.dumps(sample, 2) 398 n2 = CollectObjectIDs(set(), marshal.loads(s2)) 399 #old format generated more instances 400 self.assertGreater(n2, n0) 401 402 #if complex objects are in there, old format is larger 403 if not simple: 404 self.assertGreater(len(s2), len(s3)) 405 else: 406 self.assertGreaterEqual(len(s2), len(s3)) 407 408 def testInt(self): 409 intobj = 123321 410 self.helper(intobj) 411 self.helper3(intobj, simple=True) 412 413 def testFloat(self): 414 floatobj = 1.2345 415 self.helper(floatobj) 416 self.helper3(floatobj) 417 418 def testStr(self): 419 strobj = "abcde"*3 420 self.helper(strobj) 421 self.helper3(strobj) 422 423 def testBytes(self): 424 bytesobj = b"abcde"*3 425 self.helper(bytesobj) 426 self.helper3(bytesobj) 427 428 def testList(self): 429 for obj in self.keys: 430 listobj = [obj, obj] 431 self.helper(listobj) 432 self.helper3(listobj) 433 434 def testTuple(self): 435 for obj in self.keys: 436 tupleobj = (obj, obj) 437 self.helper(tupleobj) 438 self.helper3(tupleobj) 439 440 def testSet(self): 441 for obj in self.keys: 442 setobj = {(obj, 1), (obj, 2)} 443 self.helper(setobj) 444 self.helper3(setobj) 445 446 def testFrozenSet(self): 447 for obj in self.keys: 448 frozensetobj = frozenset({(obj, 1), (obj, 2)}) 449 self.helper(frozensetobj) 450 self.helper3(frozensetobj) 451 452 def testDict(self): 453 for obj in self.keys: 454 dictobj = {"hello": obj, "goodbye": obj, obj: "hello"} 455 self.helper(dictobj) 456 self.helper3(dictobj) 457 458 def testModule(self): 459 with open(__file__, "rb") as f: 460 code = f.read() 461 if __file__.endswith(".py"): 462 code = compile(code, __file__, "exec") 463 self.helper(code) 464 self.helper3(code) 465 466 def testRecursion(self): 467 obj = 1.2345 468 d = {"hello": obj, "goodbye": obj, obj: "hello"} 469 d["self"] = d 470 self.helper3(d, recursive=True) 471 l = [obj, obj] 472 l.append(l) 473 self.helper3(l, recursive=True) 474 475class CompatibilityTestCase(unittest.TestCase): 476 def _test(self, version): 477 with open(__file__, "rb") as f: 478 code = f.read() 479 if __file__.endswith(".py"): 480 code = compile(code, __file__, "exec") 481 data = marshal.dumps(code, version) 482 marshal.loads(data) 483 484 def test0To3(self): 485 self._test(0) 486 487 def test1To3(self): 488 self._test(1) 489 490 def test2To3(self): 491 self._test(2) 492 493 def test3To3(self): 494 self._test(3) 495 496class InterningTestCase(unittest.TestCase, HelperMixin): 497 strobj = "this is an interned string" 498 strobj = sys.intern(strobj) 499 500 def testIntern(self): 501 s = marshal.loads(marshal.dumps(self.strobj)) 502 self.assertEqual(s, self.strobj) 503 self.assertEqual(id(s), id(self.strobj)) 504 s2 = sys.intern(s) 505 self.assertEqual(id(s2), id(s)) 506 507 def testNoIntern(self): 508 s = marshal.loads(marshal.dumps(self.strobj, 2)) 509 self.assertEqual(s, self.strobj) 510 self.assertNotEqual(id(s), id(self.strobj)) 511 s2 = sys.intern(s) 512 self.assertNotEqual(id(s2), id(s)) 513 514@support.cpython_only 515@unittest.skipUnless(_testcapi, 'requires _testcapi') 516class CAPI_TestCase(unittest.TestCase, HelperMixin): 517 518 def test_write_long_to_file(self): 519 for v in range(marshal.version + 1): 520 _testcapi.pymarshal_write_long_to_file(0x12345678, os_helper.TESTFN, v) 521 with open(os_helper.TESTFN, 'rb') as f: 522 data = f.read() 523 os_helper.unlink(os_helper.TESTFN) 524 self.assertEqual(data, b'\x78\x56\x34\x12') 525 526 def test_write_object_to_file(self): 527 obj = ('\u20ac', b'abc', 123, 45.6, 7+8j, 'long line '*1000) 528 for v in range(marshal.version + 1): 529 _testcapi.pymarshal_write_object_to_file(obj, os_helper.TESTFN, v) 530 with open(os_helper.TESTFN, 'rb') as f: 531 data = f.read() 532 os_helper.unlink(os_helper.TESTFN) 533 self.assertEqual(marshal.loads(data), obj) 534 535 def test_read_short_from_file(self): 536 with open(os_helper.TESTFN, 'wb') as f: 537 f.write(b'\x34\x12xxxx') 538 r, p = _testcapi.pymarshal_read_short_from_file(os_helper.TESTFN) 539 os_helper.unlink(os_helper.TESTFN) 540 self.assertEqual(r, 0x1234) 541 self.assertEqual(p, 2) 542 543 with open(os_helper.TESTFN, 'wb') as f: 544 f.write(b'\x12') 545 with self.assertRaises(EOFError): 546 _testcapi.pymarshal_read_short_from_file(os_helper.TESTFN) 547 os_helper.unlink(os_helper.TESTFN) 548 549 def test_read_long_from_file(self): 550 with open(os_helper.TESTFN, 'wb') as f: 551 f.write(b'\x78\x56\x34\x12xxxx') 552 r, p = _testcapi.pymarshal_read_long_from_file(os_helper.TESTFN) 553 os_helper.unlink(os_helper.TESTFN) 554 self.assertEqual(r, 0x12345678) 555 self.assertEqual(p, 4) 556 557 with open(os_helper.TESTFN, 'wb') as f: 558 f.write(b'\x56\x34\x12') 559 with self.assertRaises(EOFError): 560 _testcapi.pymarshal_read_long_from_file(os_helper.TESTFN) 561 os_helper.unlink(os_helper.TESTFN) 562 563 def test_read_last_object_from_file(self): 564 obj = ('\u20ac', b'abc', 123, 45.6, 7+8j) 565 for v in range(marshal.version + 1): 566 data = marshal.dumps(obj, v) 567 with open(os_helper.TESTFN, 'wb') as f: 568 f.write(data + b'xxxx') 569 r, p = _testcapi.pymarshal_read_last_object_from_file(os_helper.TESTFN) 570 os_helper.unlink(os_helper.TESTFN) 571 self.assertEqual(r, obj) 572 573 with open(os_helper.TESTFN, 'wb') as f: 574 f.write(data[:1]) 575 with self.assertRaises(EOFError): 576 _testcapi.pymarshal_read_last_object_from_file(os_helper.TESTFN) 577 os_helper.unlink(os_helper.TESTFN) 578 579 def test_read_object_from_file(self): 580 obj = ('\u20ac', b'abc', 123, 45.6, 7+8j) 581 for v in range(marshal.version + 1): 582 data = marshal.dumps(obj, v) 583 with open(os_helper.TESTFN, 'wb') as f: 584 f.write(data + b'xxxx') 585 r, p = _testcapi.pymarshal_read_object_from_file(os_helper.TESTFN) 586 os_helper.unlink(os_helper.TESTFN) 587 self.assertEqual(r, obj) 588 self.assertEqual(p, len(data)) 589 590 with open(os_helper.TESTFN, 'wb') as f: 591 f.write(data[:1]) 592 with self.assertRaises(EOFError): 593 _testcapi.pymarshal_read_object_from_file(os_helper.TESTFN) 594 os_helper.unlink(os_helper.TESTFN) 595 596 597if __name__ == "__main__": 598 unittest.main() 599