1import os 2import array 3import unittest 4import struct 5import inspect 6from test import test_support as support 7from test.test_support import (check_warnings, check_py3k_warnings) 8 9import sys 10ISBIGENDIAN = sys.byteorder == "big" 11IS32BIT = sys.maxsize == 0x7fffffff 12 13integer_codes = 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q' 14 15testmod_filename = os.path.splitext(__file__)[0] + '.py' 16# Native 'q' packing isn't available on systems that don't have the C 17# long long type. 18try: 19 struct.pack('q', 5) 20except struct.error: 21 HAVE_LONG_LONG = False 22else: 23 HAVE_LONG_LONG = True 24 25def string_reverse(s): 26 return "".join(reversed(s)) 27 28def bigendian_to_native(value): 29 if ISBIGENDIAN: 30 return value 31 else: 32 return string_reverse(value) 33 34class StructTest(unittest.TestCase): 35 36 def check_float_coerce(self, format, number): 37 # SF bug 1530559. struct.pack raises TypeError where it used 38 # to convert. 39 with check_warnings((".*integer argument expected, got float", 40 DeprecationWarning)) as w: 41 got = struct.pack(format, number) 42 lineno = inspect.currentframe().f_lineno - 1 43 self.assertEqual(w.filename, testmod_filename) 44 self.assertEqual(w.lineno, lineno) 45 self.assertEqual(len(w.warnings), 1) 46 expected = struct.pack(format, int(number)) 47 self.assertEqual(got, expected) 48 49 def test_isbigendian(self): 50 self.assertEqual((struct.pack('=i', 1)[0] == chr(0)), ISBIGENDIAN) 51 52 def test_consistence(self): 53 self.assertRaises(struct.error, struct.calcsize, 'Z') 54 55 sz = struct.calcsize('i') 56 self.assertEqual(sz * 3, struct.calcsize('iii')) 57 58 fmt = 'cbxxxxxxhhhhiillffd?' 59 fmt3 = '3c3b18x12h6i6l6f3d3?' 60 sz = struct.calcsize(fmt) 61 sz3 = struct.calcsize(fmt3) 62 self.assertEqual(sz * 3, sz3) 63 64 self.assertRaises(struct.error, struct.pack, 'iii', 3) 65 self.assertRaises(struct.error, struct.pack, 'i', 3, 3, 3) 66 self.assertRaises((TypeError, struct.error), struct.pack, 'i', 'foo') 67 self.assertRaises((TypeError, struct.error), struct.pack, 'P', 'foo') 68 self.assertRaises(struct.error, struct.unpack, 'd', 'flap') 69 s = struct.pack('ii', 1, 2) 70 self.assertRaises(struct.error, struct.unpack, 'iii', s) 71 self.assertRaises(struct.error, struct.unpack, 'i', s) 72 73 def test_transitiveness(self): 74 c = 'a' 75 b = 1 76 h = 255 77 i = 65535 78 l = 65536 79 f = 3.1415 80 d = 3.1415 81 t = True 82 83 for prefix in ('', '@', '<', '>', '=', '!'): 84 for format in ('xcbhilfd?', 'xcBHILfd?'): 85 format = prefix + format 86 s = struct.pack(format, c, b, h, i, l, f, d, t) 87 cp, bp, hp, ip, lp, fp, dp, tp = struct.unpack(format, s) 88 self.assertEqual(cp, c) 89 self.assertEqual(bp, b) 90 self.assertEqual(hp, h) 91 self.assertEqual(ip, i) 92 self.assertEqual(lp, l) 93 self.assertEqual(int(100 * fp), int(100 * f)) 94 self.assertEqual(int(100 * dp), int(100 * d)) 95 self.assertEqual(tp, t) 96 97 def test_new_features(self): 98 # Test some of the new features in detail 99 # (format, argument, big-endian result, little-endian result, asymmetric) 100 tests = [ 101 ('c', 'a', 'a', 'a', 0), 102 ('xc', 'a', '\0a', '\0a', 0), 103 ('cx', 'a', 'a\0', 'a\0', 0), 104 ('s', 'a', 'a', 'a', 0), 105 ('0s', 'helloworld', '', '', 1), 106 ('1s', 'helloworld', 'h', 'h', 1), 107 ('9s', 'helloworld', 'helloworl', 'helloworl', 1), 108 ('10s', 'helloworld', 'helloworld', 'helloworld', 0), 109 ('11s', 'helloworld', 'helloworld\0', 'helloworld\0', 1), 110 ('20s', 'helloworld', 'helloworld'+10*'\0', 'helloworld'+10*'\0', 1), 111 ('b', 7, '\7', '\7', 0), 112 ('b', -7, '\371', '\371', 0), 113 ('B', 7, '\7', '\7', 0), 114 ('B', 249, '\371', '\371', 0), 115 ('h', 700, '\002\274', '\274\002', 0), 116 ('h', -700, '\375D', 'D\375', 0), 117 ('H', 700, '\002\274', '\274\002', 0), 118 ('H', 0x10000-700, '\375D', 'D\375', 0), 119 ('i', 70000000, '\004,\035\200', '\200\035,\004', 0), 120 ('i', -70000000, '\373\323\342\200', '\200\342\323\373', 0), 121 ('I', 70000000L, '\004,\035\200', '\200\035,\004', 0), 122 ('I', 0x100000000L-70000000, '\373\323\342\200', '\200\342\323\373', 0), 123 ('l', 70000000, '\004,\035\200', '\200\035,\004', 0), 124 ('l', -70000000, '\373\323\342\200', '\200\342\323\373', 0), 125 ('L', 70000000L, '\004,\035\200', '\200\035,\004', 0), 126 ('L', 0x100000000L-70000000, '\373\323\342\200', '\200\342\323\373', 0), 127 ('f', 2.0, '@\000\000\000', '\000\000\000@', 0), 128 ('d', 2.0, '@\000\000\000\000\000\000\000', 129 '\000\000\000\000\000\000\000@', 0), 130 ('f', -2.0, '\300\000\000\000', '\000\000\000\300', 0), 131 ('d', -2.0, '\300\000\000\000\000\000\000\000', 132 '\000\000\000\000\000\000\000\300', 0), 133 ('?', 0, '\0', '\0', 0), 134 ('?', 3, '\1', '\1', 1), 135 ('?', True, '\1', '\1', 0), 136 ('?', [], '\0', '\0', 1), 137 ('?', (1,), '\1', '\1', 1), 138 ] 139 140 for fmt, arg, big, lil, asy in tests: 141 for (xfmt, exp) in [('>'+fmt, big), ('!'+fmt, big), ('<'+fmt, lil), 142 ('='+fmt, ISBIGENDIAN and big or lil)]: 143 res = struct.pack(xfmt, arg) 144 self.assertEqual(res, exp) 145 self.assertEqual(struct.calcsize(xfmt), len(res)) 146 rev = struct.unpack(xfmt, res)[0] 147 if rev != arg: 148 self.assertTrue(asy) 149 150 def test_calcsize(self): 151 expected_size = { 152 'b': 1, 'B': 1, 153 'h': 2, 'H': 2, 154 'i': 4, 'I': 4, 155 'l': 4, 'L': 4, 156 'q': 8, 'Q': 8, 157 } 158 159 # standard integer sizes 160 for code in integer_codes: 161 for byteorder in ('=', '<', '>', '!'): 162 format = byteorder+code 163 size = struct.calcsize(format) 164 self.assertEqual(size, expected_size[code]) 165 166 # native integer sizes, except 'q' and 'Q' 167 for format_pair in ('bB', 'hH', 'iI', 'lL'): 168 for byteorder in ['', '@']: 169 signed_size = struct.calcsize(byteorder + format_pair[0]) 170 unsigned_size = struct.calcsize(byteorder + format_pair[1]) 171 self.assertEqual(signed_size, unsigned_size) 172 173 # bounds for native integer sizes 174 self.assertEqual(struct.calcsize('b'), 1) 175 self.assertLessEqual(2, struct.calcsize('h')) 176 self.assertLessEqual(4, struct.calcsize('l')) 177 self.assertLessEqual(struct.calcsize('h'), struct.calcsize('i')) 178 self.assertLessEqual(struct.calcsize('i'), struct.calcsize('l')) 179 180 # tests for native 'q' and 'Q' when applicable 181 if HAVE_LONG_LONG: 182 self.assertEqual(struct.calcsize('q'), struct.calcsize('Q')) 183 self.assertLessEqual(8, struct.calcsize('q')) 184 self.assertLessEqual(struct.calcsize('l'), struct.calcsize('q')) 185 186 def test_integers(self): 187 # Integer tests (bBhHiIlLqQ). 188 import binascii 189 190 class IntTester(unittest.TestCase): 191 def __init__(self, format): 192 super(IntTester, self).__init__(methodName='test_one') 193 self.format = format 194 self.code = format[-1] 195 self.direction = format[:-1] 196 if not self.direction in ('', '@', '=', '<', '>', '!'): 197 raise ValueError("unrecognized packing direction: %s" % 198 self.direction) 199 self.bytesize = struct.calcsize(format) 200 self.bitsize = self.bytesize * 8 201 if self.code in tuple('bhilq'): 202 self.signed = True 203 self.min_value = -(2L**(self.bitsize-1)) 204 self.max_value = 2L**(self.bitsize-1) - 1 205 elif self.code in tuple('BHILQ'): 206 self.signed = False 207 self.min_value = 0 208 self.max_value = 2L**self.bitsize - 1 209 else: 210 raise ValueError("unrecognized format code: %s" % 211 self.code) 212 213 def test_one(self, x, pack=struct.pack, 214 unpack=struct.unpack, 215 unhexlify=binascii.unhexlify): 216 217 format = self.format 218 if self.min_value <= x <= self.max_value: 219 expected = long(x) 220 if self.signed and x < 0: 221 expected += 1L << self.bitsize 222 self.assertGreaterEqual(expected, 0) 223 expected = '%x' % expected 224 if len(expected) & 1: 225 expected = "0" + expected 226 expected = unhexlify(expected) 227 expected = ("\x00" * (self.bytesize - len(expected)) + 228 expected) 229 if (self.direction == '<' or 230 self.direction in ('', '@', '=') and not ISBIGENDIAN): 231 expected = string_reverse(expected) 232 self.assertEqual(len(expected), self.bytesize) 233 234 # Pack work? 235 got = pack(format, x) 236 self.assertEqual(got, expected) 237 238 # Unpack work? 239 retrieved = unpack(format, got)[0] 240 self.assertEqual(x, retrieved) 241 242 # Adding any byte should cause a "too big" error. 243 self.assertRaises((struct.error, TypeError), unpack, format, 244 '\x01' + got) 245 else: 246 # x is out of range -- verify pack realizes that. 247 self.assertRaises((OverflowError, ValueError, struct.error), 248 pack, format, x) 249 250 def run(self): 251 from random import randrange 252 253 # Create all interesting powers of 2. 254 values = [] 255 for exp in range(self.bitsize + 3): 256 values.append(1L << exp) 257 258 # Add some random values. 259 for i in range(self.bitsize): 260 val = 0L 261 for j in range(self.bytesize): 262 val = (val << 8) | randrange(256) 263 values.append(val) 264 265 # Values absorbed from other tests 266 values.extend([300, 700000, sys.maxint*4]) 267 268 # Try all those, and their negations, and +-1 from 269 # them. Note that this tests all power-of-2 270 # boundaries in range, and a few out of range, plus 271 # +-(2**n +- 1). 272 for base in values: 273 for val in -base, base: 274 for incr in -1, 0, 1: 275 x = val + incr 276 self.test_one(int(x)) 277 self.test_one(long(x)) 278 279 # Some error cases. 280 class NotAnIntNS(object): 281 def __int__(self): 282 return 42 283 284 def __long__(self): 285 return 1729L 286 287 class NotAnIntOS: 288 def __int__(self): 289 return 85 290 291 def __long__(self): 292 return -163L 293 294 # Objects with an '__index__' method should be allowed 295 # to pack as integers. That is assuming the implemented 296 # '__index__' method returns and 'int' or 'long'. 297 class Indexable(object): 298 def __init__(self, value): 299 self._value = value 300 301 def __index__(self): 302 return self._value 303 304 # If the '__index__' method raises a type error, then 305 # '__int__' should be used with a deprecation warning. 306 class BadIndex(object): 307 def __index__(self): 308 raise TypeError 309 310 def __int__(self): 311 return 42 312 313 self.assertRaises((TypeError, struct.error), 314 struct.pack, self.format, 315 "a string") 316 self.assertRaises((TypeError, struct.error), 317 struct.pack, self.format, 318 randrange) 319 with check_warnings(("integer argument expected, " 320 "got non-integer", DeprecationWarning)): 321 with self.assertRaises((TypeError, struct.error)): 322 struct.pack(self.format, 3+42j) 323 324 # an attempt to convert a non-integer (with an 325 # implicit conversion via __int__) should succeed, 326 # with a DeprecationWarning 327 for nonint in NotAnIntNS(), NotAnIntOS(), BadIndex(): 328 with check_warnings((".*integer argument expected, got non" 329 "-integer", DeprecationWarning)) as w: 330 got = struct.pack(self.format, nonint) 331 lineno = inspect.currentframe().f_lineno - 1 332 self.assertEqual(w.filename, testmod_filename) 333 self.assertEqual(w.lineno, lineno) 334 self.assertEqual(len(w.warnings), 1) 335 expected = struct.pack(self.format, int(nonint)) 336 self.assertEqual(got, expected) 337 338 # Check for legitimate values from '__index__'. 339 for obj in (Indexable(0), Indexable(10), Indexable(17), 340 Indexable(42), Indexable(100), Indexable(127)): 341 try: 342 struct.pack(format, obj) 343 except: 344 self.fail("integer code pack failed on object " 345 "with '__index__' method") 346 347 # Check for bogus values from '__index__'. 348 for obj in (Indexable('a'), Indexable(u'b'), Indexable(None), 349 Indexable({'a': 1}), Indexable([1, 2, 3])): 350 self.assertRaises((TypeError, struct.error), 351 struct.pack, self.format, 352 obj) 353 354 byteorders = '', '@', '=', '<', '>', '!' 355 for code in integer_codes: 356 for byteorder in byteorders: 357 if (byteorder in ('', '@') and code in ('q', 'Q') and 358 not HAVE_LONG_LONG): 359 continue 360 format = byteorder+code 361 t = IntTester(format) 362 t.run() 363 364 def test_p_code(self): 365 # Test p ("Pascal string") code. 366 for code, input, expected, expectedback in [ 367 ('p','abc', '\x00', ''), 368 ('1p', 'abc', '\x00', ''), 369 ('2p', 'abc', '\x01a', 'a'), 370 ('3p', 'abc', '\x02ab', 'ab'), 371 ('4p', 'abc', '\x03abc', 'abc'), 372 ('5p', 'abc', '\x03abc\x00', 'abc'), 373 ('6p', 'abc', '\x03abc\x00\x00', 'abc'), 374 ('1000p', 'x'*1000, '\xff' + 'x'*999, 'x'*255)]: 375 got = struct.pack(code, input) 376 self.assertEqual(got, expected) 377 (got,) = struct.unpack(code, got) 378 self.assertEqual(got, expectedback) 379 380 def test_705836(self): 381 # SF bug 705836. "<f" and ">f" had a severe rounding bug, where a carry 382 # from the low-order discarded bits could propagate into the exponent 383 # field, causing the result to be wrong by a factor of 2. 384 import math 385 386 for base in range(1, 33): 387 # smaller <- largest representable float less than base. 388 delta = 0.5 389 while base - delta / 2.0 != base: 390 delta /= 2.0 391 smaller = base - delta 392 # Packing this rounds away a solid string of trailing 1 bits. 393 packed = struct.pack("<f", smaller) 394 unpacked = struct.unpack("<f", packed)[0] 395 # This failed at base = 2, 4, and 32, with unpacked = 1, 2, and 396 # 16, respectively. 397 self.assertEqual(base, unpacked) 398 bigpacked = struct.pack(">f", smaller) 399 self.assertEqual(bigpacked, string_reverse(packed)) 400 unpacked = struct.unpack(">f", bigpacked)[0] 401 self.assertEqual(base, unpacked) 402 403 # Largest finite IEEE single. 404 big = (1 << 24) - 1 405 big = math.ldexp(big, 127 - 23) 406 packed = struct.pack(">f", big) 407 unpacked = struct.unpack(">f", packed)[0] 408 self.assertEqual(big, unpacked) 409 410 # The same, but tack on a 1 bit so it rounds up to infinity. 411 big = (1 << 25) - 1 412 big = math.ldexp(big, 127 - 24) 413 self.assertRaises(OverflowError, struct.pack, ">f", big) 414 415 def test_1530559(self): 416 # SF bug 1530559. struct.pack raises TypeError where it used to convert. 417 for endian in ('', '>', '<'): 418 for fmt in integer_codes: 419 self.check_float_coerce(endian + fmt, 1.0) 420 self.check_float_coerce(endian + fmt, 1.5) 421 422 def test_unpack_from(self, cls=str): 423 data = cls('abcd01234') 424 fmt = '4s' 425 s = struct.Struct(fmt) 426 427 self.assertEqual(s.unpack_from(data), ('abcd',)) 428 self.assertEqual(struct.unpack_from(fmt, data), ('abcd',)) 429 for i in xrange(6): 430 self.assertEqual(s.unpack_from(data, i), (data[i:i+4],)) 431 self.assertEqual(struct.unpack_from(fmt, data, i), (data[i:i+4],)) 432 for i in xrange(6, len(data) + 1): 433 self.assertRaises(struct.error, s.unpack_from, data, i) 434 self.assertRaises(struct.error, struct.unpack_from, fmt, data, i) 435 436 def test_pack_into(self, cls=bytearray, tobytes=str): 437 test_string = 'Reykjavik rocks, eow!' 438 writable_buf = cls(' '*100) 439 fmt = '21s' 440 s = struct.Struct(fmt) 441 442 # Test without offset 443 s.pack_into(writable_buf, 0, test_string) 444 from_buf = tobytes(writable_buf)[:len(test_string)] 445 self.assertEqual(from_buf, test_string) 446 447 # Test with offset. 448 s.pack_into(writable_buf, 10, test_string) 449 from_buf = tobytes(writable_buf)[:len(test_string)+10] 450 self.assertEqual(from_buf, test_string[:10] + test_string) 451 452 # Go beyond boundaries. 453 small_buf = cls(' '*10) 454 self.assertRaises((ValueError, struct.error), s.pack_into, small_buf, 0, 455 test_string) 456 self.assertRaises((ValueError, struct.error), s.pack_into, small_buf, 2, 457 test_string) 458 459 # Test bogus offset (issue 3694) 460 sb = small_buf 461 self.assertRaises((TypeError, struct.error), struct.pack_into, b'', sb, 462 None) 463 464 def test_pack_into_array(self): 465 self.test_pack_into(cls=lambda b: array.array('c', b), 466 tobytes=array.array.tostring) 467 468 def test_pack_into_memoryview(self): 469 # Issue #22113 470 self.test_pack_into(cls=lambda b: memoryview(bytearray(b)), 471 tobytes=memoryview.tobytes) 472 473 def test_pack_into_fn(self): 474 test_string = 'Reykjavik rocks, eow!' 475 writable_buf = array.array('c', ' '*100) 476 fmt = '21s' 477 pack_into = lambda *args: struct.pack_into(fmt, *args) 478 479 # Test without offset. 480 pack_into(writable_buf, 0, test_string) 481 from_buf = writable_buf.tostring()[:len(test_string)] 482 self.assertEqual(from_buf, test_string) 483 484 # Test with offset. 485 pack_into(writable_buf, 10, test_string) 486 from_buf = writable_buf.tostring()[:len(test_string)+10] 487 self.assertEqual(from_buf, test_string[:10] + test_string) 488 489 # Go beyond boundaries. 490 small_buf = array.array('c', ' '*10) 491 self.assertRaises((ValueError, struct.error), pack_into, small_buf, 0, 492 test_string) 493 self.assertRaises((ValueError, struct.error), pack_into, small_buf, 2, 494 test_string) 495 496 def test_unpack_with_buffer(self): 497 with check_py3k_warnings(("buffer.. not supported in 3.x", 498 DeprecationWarning)): 499 # SF bug 1563759: struct.unpack doesn't support buffer protocol objects 500 data1 = array.array('B', '\x12\x34\x56\x78') 501 data2 = buffer('......\x12\x34\x56\x78......', 6, 4) 502 for data in [data1, data2]: 503 value, = struct.unpack('>I', data) 504 self.assertEqual(value, 0x12345678) 505 506 self.test_unpack_from(cls=buffer) 507 508 def test_unpack_with_memoryview(self): 509 # Bug 10212: struct.unpack doesn't support new buffer protocol objects 510 data1 = memoryview('\x12\x34\x56\x78') 511 for data in [data1,]: 512 value, = struct.unpack('>I', data) 513 self.assertEqual(value, 0x12345678) 514 self.test_unpack_from(cls=memoryview) 515 516 def test_bool(self): 517 class ExplodingBool(object): 518 def __nonzero__(self): 519 raise IOError 520 for prefix in tuple("<>!=")+('',): 521 false = (), [], [], '', 0 522 true = [1], 'test', 5, -1, 0xffffffffL+1, 0xffffffff//2 523 524 falseFormat = prefix + '?' * len(false) 525 packedFalse = struct.pack(falseFormat, *false) 526 unpackedFalse = struct.unpack(falseFormat, packedFalse) 527 528 trueFormat = prefix + '?' * len(true) 529 packedTrue = struct.pack(trueFormat, *true) 530 unpackedTrue = struct.unpack(trueFormat, packedTrue) 531 532 self.assertEqual(len(true), len(unpackedTrue)) 533 self.assertEqual(len(false), len(unpackedFalse)) 534 535 for t in unpackedFalse: 536 self.assertFalse(t) 537 for t in unpackedTrue: 538 self.assertTrue(t) 539 540 packed = struct.pack(prefix+'?', 1) 541 542 self.assertEqual(len(packed), struct.calcsize(prefix+'?')) 543 544 if len(packed) != 1: 545 self.assertFalse(prefix, msg='encoded bool is not one byte: %r' 546 %packed) 547 548 self.assertRaises(IOError, struct.pack, prefix + '?', 549 ExplodingBool()) 550 551 for c in [b'\x01', b'\x7f', b'\xff', b'\x0f', b'\xf0']: 552 self.assertTrue(struct.unpack('>?', c)[0]) 553 554 @unittest.skipUnless(IS32BIT, "Specific to 32bit machines") 555 def test_crasher(self): 556 self.assertRaises(MemoryError, struct.pack, "357913941c", "a") 557 558 def test_count_overflow(self): 559 hugecount = '{}b'.format(sys.maxsize+1) 560 self.assertRaises(struct.error, struct.calcsize, hugecount) 561 562 hugecount2 = '{}b{}H'.format(sys.maxsize//2, sys.maxsize//2) 563 self.assertRaises(struct.error, struct.calcsize, hugecount2) 564 565 def check_sizeof(self, format_str, number_of_codes): 566 # The size of 'PyStructObject' 567 totalsize = support.calcobjsize('5P') 568 # The size taken up by the 'formatcode' dynamic array 569 totalsize += struct.calcsize('3P') * (number_of_codes + 1) 570 support.check_sizeof(self, struct.Struct(format_str), totalsize) 571 572 @support.cpython_only 573 def test__sizeof__(self): 574 for code in integer_codes: 575 self.check_sizeof(code, 1) 576 self.check_sizeof('BHILfdspP', 9) 577 self.check_sizeof('B' * 1234, 1234) 578 self.check_sizeof('fd', 2) 579 self.check_sizeof('xxxxxxxxxxxxxx', 0) 580 self.check_sizeof('100H', 100) 581 self.check_sizeof('187s', 1) 582 self.check_sizeof('20p', 1) 583 self.check_sizeof('0s', 1) 584 self.check_sizeof('0c', 0) 585 586 def test_unicode_format(self): 587 try: 588 unicode 589 except NameError: 590 self.skipTest('no unicode support') 591 # Issue #19099 592 s = struct.Struct(unichr(ord('I'))) 593 self.assertEqual(s.format, 'I') 594 self.assertIs(type(s.format), str) 595 self.assertRaises(ValueError, struct.Struct, unichr(0x80)) 596 597 598def test_main(): 599 support.run_unittest(StructTest) 600 601if __name__ == '__main__': 602 test_main() 603