• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1import os
2import array
3import unittest
4import struct
5import inspect
6from test import test_support as support
7from test.test_support import (check_warnings, check_py3k_warnings)
8
9import sys
10ISBIGENDIAN = sys.byteorder == "big"
11IS32BIT = sys.maxsize == 0x7fffffff
12
13integer_codes = 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q'
14
15testmod_filename = os.path.splitext(__file__)[0] + '.py'
16# Native 'q' packing isn't available on systems that don't have the C
17# long long type.
18try:
19    struct.pack('q', 5)
20except struct.error:
21    HAVE_LONG_LONG = False
22else:
23    HAVE_LONG_LONG = True
24
25def string_reverse(s):
26    return "".join(reversed(s))
27
28def bigendian_to_native(value):
29    if ISBIGENDIAN:
30        return value
31    else:
32        return string_reverse(value)
33
34class StructTest(unittest.TestCase):
35
36    def check_float_coerce(self, format, number):
37        # SF bug 1530559. struct.pack raises TypeError where it used
38        # to convert.
39        with check_warnings((".*integer argument expected, got float",
40                             DeprecationWarning)) as w:
41            got = struct.pack(format, number)
42        lineno = inspect.currentframe().f_lineno - 1
43        self.assertEqual(w.filename, testmod_filename)
44        self.assertEqual(w.lineno, lineno)
45        self.assertEqual(len(w.warnings), 1)
46        expected = struct.pack(format, int(number))
47        self.assertEqual(got, expected)
48
49    def test_isbigendian(self):
50        self.assertEqual((struct.pack('=i', 1)[0] == chr(0)), ISBIGENDIAN)
51
52    def test_consistence(self):
53        self.assertRaises(struct.error, struct.calcsize, 'Z')
54
55        sz = struct.calcsize('i')
56        self.assertEqual(sz * 3, struct.calcsize('iii'))
57
58        fmt = 'cbxxxxxxhhhhiillffd?'
59        fmt3 = '3c3b18x12h6i6l6f3d3?'
60        sz = struct.calcsize(fmt)
61        sz3 = struct.calcsize(fmt3)
62        self.assertEqual(sz * 3, sz3)
63
64        self.assertRaises(struct.error, struct.pack, 'iii', 3)
65        self.assertRaises(struct.error, struct.pack, 'i', 3, 3, 3)
66        self.assertRaises((TypeError, struct.error), struct.pack, 'i', 'foo')
67        self.assertRaises((TypeError, struct.error), struct.pack, 'P', 'foo')
68        self.assertRaises(struct.error, struct.unpack, 'd', 'flap')
69        s = struct.pack('ii', 1, 2)
70        self.assertRaises(struct.error, struct.unpack, 'iii', s)
71        self.assertRaises(struct.error, struct.unpack, 'i', s)
72
73    def test_transitiveness(self):
74        c = 'a'
75        b = 1
76        h = 255
77        i = 65535
78        l = 65536
79        f = 3.1415
80        d = 3.1415
81        t = True
82
83        for prefix in ('', '@', '<', '>', '=', '!'):
84            for format in ('xcbhilfd?', 'xcBHILfd?'):
85                format = prefix + format
86                s = struct.pack(format, c, b, h, i, l, f, d, t)
87                cp, bp, hp, ip, lp, fp, dp, tp = struct.unpack(format, s)
88                self.assertEqual(cp, c)
89                self.assertEqual(bp, b)
90                self.assertEqual(hp, h)
91                self.assertEqual(ip, i)
92                self.assertEqual(lp, l)
93                self.assertEqual(int(100 * fp), int(100 * f))
94                self.assertEqual(int(100 * dp), int(100 * d))
95                self.assertEqual(tp, t)
96
97    def test_new_features(self):
98        # Test some of the new features in detail
99        # (format, argument, big-endian result, little-endian result, asymmetric)
100        tests = [
101            ('c', 'a', 'a', 'a', 0),
102            ('xc', 'a', '\0a', '\0a', 0),
103            ('cx', 'a', 'a\0', 'a\0', 0),
104            ('s', 'a', 'a', 'a', 0),
105            ('0s', 'helloworld', '', '', 1),
106            ('1s', 'helloworld', 'h', 'h', 1),
107            ('9s', 'helloworld', 'helloworl', 'helloworl', 1),
108            ('10s', 'helloworld', 'helloworld', 'helloworld', 0),
109            ('11s', 'helloworld', 'helloworld\0', 'helloworld\0', 1),
110            ('20s', 'helloworld', 'helloworld'+10*'\0', 'helloworld'+10*'\0', 1),
111            ('b', 7, '\7', '\7', 0),
112            ('b', -7, '\371', '\371', 0),
113            ('B', 7, '\7', '\7', 0),
114            ('B', 249, '\371', '\371', 0),
115            ('h', 700, '\002\274', '\274\002', 0),
116            ('h', -700, '\375D', 'D\375', 0),
117            ('H', 700, '\002\274', '\274\002', 0),
118            ('H', 0x10000-700, '\375D', 'D\375', 0),
119            ('i', 70000000, '\004,\035\200', '\200\035,\004', 0),
120            ('i', -70000000, '\373\323\342\200', '\200\342\323\373', 0),
121            ('I', 70000000L, '\004,\035\200', '\200\035,\004', 0),
122            ('I', 0x100000000L-70000000, '\373\323\342\200', '\200\342\323\373', 0),
123            ('l', 70000000, '\004,\035\200', '\200\035,\004', 0),
124            ('l', -70000000, '\373\323\342\200', '\200\342\323\373', 0),
125            ('L', 70000000L, '\004,\035\200', '\200\035,\004', 0),
126            ('L', 0x100000000L-70000000, '\373\323\342\200', '\200\342\323\373', 0),
127            ('f', 2.0, '@\000\000\000', '\000\000\000@', 0),
128            ('d', 2.0, '@\000\000\000\000\000\000\000',
129                       '\000\000\000\000\000\000\000@', 0),
130            ('f', -2.0, '\300\000\000\000', '\000\000\000\300', 0),
131            ('d', -2.0, '\300\000\000\000\000\000\000\000',
132                       '\000\000\000\000\000\000\000\300', 0),
133                ('?', 0, '\0', '\0', 0),
134                ('?', 3, '\1', '\1', 1),
135                ('?', True, '\1', '\1', 0),
136                ('?', [], '\0', '\0', 1),
137                ('?', (1,), '\1', '\1', 1),
138        ]
139
140        for fmt, arg, big, lil, asy in tests:
141            for (xfmt, exp) in [('>'+fmt, big), ('!'+fmt, big), ('<'+fmt, lil),
142                                ('='+fmt, ISBIGENDIAN and big or lil)]:
143                res = struct.pack(xfmt, arg)
144                self.assertEqual(res, exp)
145                self.assertEqual(struct.calcsize(xfmt), len(res))
146                rev = struct.unpack(xfmt, res)[0]
147                if rev != arg:
148                    self.assertTrue(asy)
149
150    def test_calcsize(self):
151        expected_size = {
152            'b': 1, 'B': 1,
153            'h': 2, 'H': 2,
154            'i': 4, 'I': 4,
155            'l': 4, 'L': 4,
156            'q': 8, 'Q': 8,
157            }
158
159        # standard integer sizes
160        for code in integer_codes:
161            for byteorder in ('=', '<', '>', '!'):
162                format = byteorder+code
163                size = struct.calcsize(format)
164                self.assertEqual(size, expected_size[code])
165
166        # native integer sizes, except 'q' and 'Q'
167        for format_pair in ('bB', 'hH', 'iI', 'lL'):
168            for byteorder in ['', '@']:
169                signed_size = struct.calcsize(byteorder + format_pair[0])
170                unsigned_size = struct.calcsize(byteorder + format_pair[1])
171                self.assertEqual(signed_size, unsigned_size)
172
173        # bounds for native integer sizes
174        self.assertEqual(struct.calcsize('b'), 1)
175        self.assertLessEqual(2, struct.calcsize('h'))
176        self.assertLessEqual(4, struct.calcsize('l'))
177        self.assertLessEqual(struct.calcsize('h'), struct.calcsize('i'))
178        self.assertLessEqual(struct.calcsize('i'), struct.calcsize('l'))
179
180        # tests for native 'q' and 'Q' when applicable
181        if HAVE_LONG_LONG:
182            self.assertEqual(struct.calcsize('q'), struct.calcsize('Q'))
183            self.assertLessEqual(8, struct.calcsize('q'))
184            self.assertLessEqual(struct.calcsize('l'), struct.calcsize('q'))
185
186    def test_integers(self):
187        # Integer tests (bBhHiIlLqQ).
188        import binascii
189
190        class IntTester(unittest.TestCase):
191            def __init__(self, format):
192                super(IntTester, self).__init__(methodName='test_one')
193                self.format = format
194                self.code = format[-1]
195                self.direction = format[:-1]
196                if not self.direction in ('', '@', '=', '<', '>', '!'):
197                    raise ValueError("unrecognized packing direction: %s" %
198                                     self.direction)
199                self.bytesize = struct.calcsize(format)
200                self.bitsize = self.bytesize * 8
201                if self.code in tuple('bhilq'):
202                    self.signed = True
203                    self.min_value = -(2L**(self.bitsize-1))
204                    self.max_value = 2L**(self.bitsize-1) - 1
205                elif self.code in tuple('BHILQ'):
206                    self.signed = False
207                    self.min_value = 0
208                    self.max_value = 2L**self.bitsize - 1
209                else:
210                    raise ValueError("unrecognized format code: %s" %
211                                     self.code)
212
213            def test_one(self, x, pack=struct.pack,
214                                  unpack=struct.unpack,
215                                  unhexlify=binascii.unhexlify):
216
217                format = self.format
218                if self.min_value <= x <= self.max_value:
219                    expected = long(x)
220                    if self.signed and x < 0:
221                        expected += 1L << self.bitsize
222                    self.assertGreaterEqual(expected, 0)
223                    expected = '%x' % expected
224                    if len(expected) & 1:
225                        expected = "0" + expected
226                    expected = unhexlify(expected)
227                    expected = ("\x00" * (self.bytesize - len(expected)) +
228                                expected)
229                    if (self.direction == '<' or
230                        self.direction in ('', '@', '=') and not ISBIGENDIAN):
231                        expected = string_reverse(expected)
232                    self.assertEqual(len(expected), self.bytesize)
233
234                    # Pack work?
235                    got = pack(format, x)
236                    self.assertEqual(got, expected)
237
238                    # Unpack work?
239                    retrieved = unpack(format, got)[0]
240                    self.assertEqual(x, retrieved)
241
242                    # Adding any byte should cause a "too big" error.
243                    self.assertRaises((struct.error, TypeError), unpack, format,
244                                                                 '\x01' + got)
245                else:
246                    # x is out of range -- verify pack realizes that.
247                    self.assertRaises((OverflowError, ValueError, struct.error),
248                                      pack, format, x)
249
250            def run(self):
251                from random import randrange
252
253                # Create all interesting powers of 2.
254                values = []
255                for exp in range(self.bitsize + 3):
256                    values.append(1L << exp)
257
258                # Add some random values.
259                for i in range(self.bitsize):
260                    val = 0L
261                    for j in range(self.bytesize):
262                        val = (val << 8) | randrange(256)
263                    values.append(val)
264
265                # Values absorbed from other tests
266                values.extend([300, 700000, sys.maxint*4])
267
268                # Try all those, and their negations, and +-1 from
269                # them.  Note that this tests all power-of-2
270                # boundaries in range, and a few out of range, plus
271                # +-(2**n +- 1).
272                for base in values:
273                    for val in -base, base:
274                        for incr in -1, 0, 1:
275                            x = val + incr
276                            self.test_one(int(x))
277                            self.test_one(long(x))
278
279                # Some error cases.
280                class NotAnIntNS(object):
281                    def __int__(self):
282                        return 42
283
284                    def __long__(self):
285                        return 1729L
286
287                class NotAnIntOS:
288                    def __int__(self):
289                        return 85
290
291                    def __long__(self):
292                        return -163L
293
294                # Objects with an '__index__' method should be allowed
295                # to pack as integers.  That is assuming the implemented
296                # '__index__' method returns and 'int' or 'long'.
297                class Indexable(object):
298                    def __init__(self, value):
299                        self._value = value
300
301                    def __index__(self):
302                        return self._value
303
304                # If the '__index__' method raises a type error, then
305                # '__int__' should be used with a deprecation warning.
306                class BadIndex(object):
307                    def __index__(self):
308                        raise TypeError
309
310                    def __int__(self):
311                        return 42
312
313                self.assertRaises((TypeError, struct.error),
314                                  struct.pack, self.format,
315                                  "a string")
316                self.assertRaises((TypeError, struct.error),
317                                  struct.pack, self.format,
318                                  randrange)
319                with check_warnings(("integer argument expected, "
320                                     "got non-integer", DeprecationWarning)):
321                    with self.assertRaises((TypeError, struct.error)):
322                        struct.pack(self.format, 3+42j)
323
324                # an attempt to convert a non-integer (with an
325                # implicit conversion via __int__) should succeed,
326                # with a DeprecationWarning
327                for nonint in NotAnIntNS(), NotAnIntOS(), BadIndex():
328                    with check_warnings((".*integer argument expected, got non"
329                                         "-integer", DeprecationWarning)) as w:
330                        got = struct.pack(self.format, nonint)
331                    lineno = inspect.currentframe().f_lineno - 1
332                    self.assertEqual(w.filename, testmod_filename)
333                    self.assertEqual(w.lineno, lineno)
334                    self.assertEqual(len(w.warnings), 1)
335                    expected = struct.pack(self.format, int(nonint))
336                    self.assertEqual(got, expected)
337
338                # Check for legitimate values from '__index__'.
339                for obj in (Indexable(0), Indexable(10), Indexable(17),
340                            Indexable(42), Indexable(100), Indexable(127)):
341                    try:
342                        struct.pack(format, obj)
343                    except:
344                        self.fail("integer code pack failed on object "
345                                  "with '__index__' method")
346
347                # Check for bogus values from '__index__'.
348                for obj in (Indexable('a'), Indexable(u'b'), Indexable(None),
349                            Indexable({'a': 1}), Indexable([1, 2, 3])):
350                    self.assertRaises((TypeError, struct.error),
351                                      struct.pack, self.format,
352                                      obj)
353
354        byteorders = '', '@', '=', '<', '>', '!'
355        for code in integer_codes:
356            for byteorder in byteorders:
357                if (byteorder in ('', '@') and code in ('q', 'Q') and
358                    not HAVE_LONG_LONG):
359                    continue
360                format = byteorder+code
361                t = IntTester(format)
362                t.run()
363
364    def test_p_code(self):
365        # Test p ("Pascal string") code.
366        for code, input, expected, expectedback in [
367                ('p','abc', '\x00', ''),
368                ('1p', 'abc', '\x00', ''),
369                ('2p', 'abc', '\x01a', 'a'),
370                ('3p', 'abc', '\x02ab', 'ab'),
371                ('4p', 'abc', '\x03abc', 'abc'),
372                ('5p', 'abc', '\x03abc\x00', 'abc'),
373                ('6p', 'abc', '\x03abc\x00\x00', 'abc'),
374                ('1000p', 'x'*1000, '\xff' + 'x'*999, 'x'*255)]:
375            got = struct.pack(code, input)
376            self.assertEqual(got, expected)
377            (got,) = struct.unpack(code, got)
378            self.assertEqual(got, expectedback)
379
380    def test_705836(self):
381        # SF bug 705836.  "<f" and ">f" had a severe rounding bug, where a carry
382        # from the low-order discarded bits could propagate into the exponent
383        # field, causing the result to be wrong by a factor of 2.
384        import math
385
386        for base in range(1, 33):
387            # smaller <- largest representable float less than base.
388            delta = 0.5
389            while base - delta / 2.0 != base:
390                delta /= 2.0
391            smaller = base - delta
392            # Packing this rounds away a solid string of trailing 1 bits.
393            packed = struct.pack("<f", smaller)
394            unpacked = struct.unpack("<f", packed)[0]
395            # This failed at base = 2, 4, and 32, with unpacked = 1, 2, and
396            # 16, respectively.
397            self.assertEqual(base, unpacked)
398            bigpacked = struct.pack(">f", smaller)
399            self.assertEqual(bigpacked, string_reverse(packed))
400            unpacked = struct.unpack(">f", bigpacked)[0]
401            self.assertEqual(base, unpacked)
402
403        # Largest finite IEEE single.
404        big = (1 << 24) - 1
405        big = math.ldexp(big, 127 - 23)
406        packed = struct.pack(">f", big)
407        unpacked = struct.unpack(">f", packed)[0]
408        self.assertEqual(big, unpacked)
409
410        # The same, but tack on a 1 bit so it rounds up to infinity.
411        big = (1 << 25) - 1
412        big = math.ldexp(big, 127 - 24)
413        self.assertRaises(OverflowError, struct.pack, ">f", big)
414
415    def test_1530559(self):
416        # SF bug 1530559. struct.pack raises TypeError where it used to convert.
417        for endian in ('', '>', '<'):
418            for fmt in integer_codes:
419                self.check_float_coerce(endian + fmt, 1.0)
420                self.check_float_coerce(endian + fmt, 1.5)
421
422    def test_unpack_from(self, cls=str):
423        data = cls('abcd01234')
424        fmt = '4s'
425        s = struct.Struct(fmt)
426
427        self.assertEqual(s.unpack_from(data), ('abcd',))
428        self.assertEqual(struct.unpack_from(fmt, data), ('abcd',))
429        for i in xrange(6):
430            self.assertEqual(s.unpack_from(data, i), (data[i:i+4],))
431            self.assertEqual(struct.unpack_from(fmt, data, i), (data[i:i+4],))
432        for i in xrange(6, len(data) + 1):
433            self.assertRaises(struct.error, s.unpack_from, data, i)
434            self.assertRaises(struct.error, struct.unpack_from, fmt, data, i)
435
436    def test_pack_into(self, cls=bytearray, tobytes=str):
437        test_string = 'Reykjavik rocks, eow!'
438        writable_buf = cls(' '*100)
439        fmt = '21s'
440        s = struct.Struct(fmt)
441
442        # Test without offset
443        s.pack_into(writable_buf, 0, test_string)
444        from_buf = tobytes(writable_buf)[:len(test_string)]
445        self.assertEqual(from_buf, test_string)
446
447        # Test with offset.
448        s.pack_into(writable_buf, 10, test_string)
449        from_buf = tobytes(writable_buf)[:len(test_string)+10]
450        self.assertEqual(from_buf, test_string[:10] + test_string)
451
452        # Go beyond boundaries.
453        small_buf = cls(' '*10)
454        self.assertRaises((ValueError, struct.error), s.pack_into, small_buf, 0,
455                          test_string)
456        self.assertRaises((ValueError, struct.error), s.pack_into, small_buf, 2,
457                          test_string)
458
459        # Test bogus offset (issue 3694)
460        sb = small_buf
461        self.assertRaises((TypeError, struct.error), struct.pack_into, b'', sb,
462                          None)
463
464    def test_pack_into_array(self):
465        self.test_pack_into(cls=lambda b: array.array('c', b),
466                            tobytes=array.array.tostring)
467
468    def test_pack_into_memoryview(self):
469        # Issue #22113
470        self.test_pack_into(cls=lambda b: memoryview(bytearray(b)),
471                            tobytes=memoryview.tobytes)
472
473    def test_pack_into_fn(self):
474        test_string = 'Reykjavik rocks, eow!'
475        writable_buf = array.array('c', ' '*100)
476        fmt = '21s'
477        pack_into = lambda *args: struct.pack_into(fmt, *args)
478
479        # Test without offset.
480        pack_into(writable_buf, 0, test_string)
481        from_buf = writable_buf.tostring()[:len(test_string)]
482        self.assertEqual(from_buf, test_string)
483
484        # Test with offset.
485        pack_into(writable_buf, 10, test_string)
486        from_buf = writable_buf.tostring()[:len(test_string)+10]
487        self.assertEqual(from_buf, test_string[:10] + test_string)
488
489        # Go beyond boundaries.
490        small_buf = array.array('c', ' '*10)
491        self.assertRaises((ValueError, struct.error), pack_into, small_buf, 0,
492                          test_string)
493        self.assertRaises((ValueError, struct.error), pack_into, small_buf, 2,
494                          test_string)
495
496    def test_unpack_with_buffer(self):
497        with check_py3k_warnings(("buffer.. not supported in 3.x",
498                                  DeprecationWarning)):
499            # SF bug 1563759: struct.unpack doesn't support buffer protocol objects
500            data1 = array.array('B', '\x12\x34\x56\x78')
501            data2 = buffer('......\x12\x34\x56\x78......', 6, 4)
502            for data in [data1, data2]:
503                value, = struct.unpack('>I', data)
504                self.assertEqual(value, 0x12345678)
505
506            self.test_unpack_from(cls=buffer)
507
508    def test_unpack_with_memoryview(self):
509        # Bug 10212: struct.unpack doesn't support new buffer protocol objects
510        data1 = memoryview('\x12\x34\x56\x78')
511        for data in [data1,]:
512            value, = struct.unpack('>I', data)
513            self.assertEqual(value, 0x12345678)
514        self.test_unpack_from(cls=memoryview)
515
516    def test_bool(self):
517        class ExplodingBool(object):
518            def __nonzero__(self):
519                raise IOError
520        for prefix in tuple("<>!=")+('',):
521            false = (), [], [], '', 0
522            true = [1], 'test', 5, -1, 0xffffffffL+1, 0xffffffff//2
523
524            falseFormat = prefix + '?' * len(false)
525            packedFalse = struct.pack(falseFormat, *false)
526            unpackedFalse = struct.unpack(falseFormat, packedFalse)
527
528            trueFormat = prefix + '?' * len(true)
529            packedTrue = struct.pack(trueFormat, *true)
530            unpackedTrue = struct.unpack(trueFormat, packedTrue)
531
532            self.assertEqual(len(true), len(unpackedTrue))
533            self.assertEqual(len(false), len(unpackedFalse))
534
535            for t in unpackedFalse:
536                self.assertFalse(t)
537            for t in unpackedTrue:
538                self.assertTrue(t)
539
540            packed = struct.pack(prefix+'?', 1)
541
542            self.assertEqual(len(packed), struct.calcsize(prefix+'?'))
543
544            if len(packed) != 1:
545                self.assertFalse(prefix, msg='encoded bool is not one byte: %r'
546                                             %packed)
547
548            self.assertRaises(IOError, struct.pack, prefix + '?',
549                              ExplodingBool())
550
551        for c in [b'\x01', b'\x7f', b'\xff', b'\x0f', b'\xf0']:
552            self.assertTrue(struct.unpack('>?', c)[0])
553
554    @unittest.skipUnless(IS32BIT, "Specific to 32bit machines")
555    def test_crasher(self):
556        self.assertRaises(MemoryError, struct.pack, "357913941c", "a")
557
558    def test_count_overflow(self):
559        hugecount = '{}b'.format(sys.maxsize+1)
560        self.assertRaises(struct.error, struct.calcsize, hugecount)
561
562        hugecount2 = '{}b{}H'.format(sys.maxsize//2, sys.maxsize//2)
563        self.assertRaises(struct.error, struct.calcsize, hugecount2)
564
565    def check_sizeof(self, format_str, number_of_codes):
566        # The size of 'PyStructObject'
567        totalsize = support.calcobjsize('5P')
568        # The size taken up by the 'formatcode' dynamic array
569        totalsize += struct.calcsize('3P') * (number_of_codes + 1)
570        support.check_sizeof(self, struct.Struct(format_str), totalsize)
571
572    @support.cpython_only
573    def test__sizeof__(self):
574        for code in integer_codes:
575            self.check_sizeof(code, 1)
576        self.check_sizeof('BHILfdspP', 9)
577        self.check_sizeof('B' * 1234, 1234)
578        self.check_sizeof('fd', 2)
579        self.check_sizeof('xxxxxxxxxxxxxx', 0)
580        self.check_sizeof('100H', 100)
581        self.check_sizeof('187s', 1)
582        self.check_sizeof('20p', 1)
583        self.check_sizeof('0s', 1)
584        self.check_sizeof('0c', 0)
585
586    def test_unicode_format(self):
587        try:
588            unicode
589        except NameError:
590            self.skipTest('no unicode support')
591        # Issue #19099
592        s = struct.Struct(unichr(ord('I')))
593        self.assertEqual(s.format, 'I')
594        self.assertIs(type(s.format), str)
595        self.assertRaises(ValueError, struct.Struct, unichr(0x80))
596
597
598def test_main():
599    support.run_unittest(StructTest)
600
601if __name__ == '__main__':
602    test_main()
603