• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1"""Tests for the unparse.py script in the Tools/parser directory."""
2
3import unittest
4import test.support
5import pathlib
6import random
7import tokenize
8import ast
9
10
11def read_pyfile(filename):
12    """Read and return the contents of a Python source file (as a
13    string), taking into account the file encoding."""
14    with tokenize.open(filename) as stream:
15        return stream.read()
16
17
18for_else = """\
19def f():
20    for x in range(10):
21        break
22    else:
23        y = 2
24    z = 3
25"""
26
27while_else = """\
28def g():
29    while True:
30        break
31    else:
32        y = 2
33    z = 3
34"""
35
36relative_import = """\
37from . import fred
38from .. import barney
39from .australia import shrimp as prawns
40"""
41
42nonlocal_ex = """\
43def f():
44    x = 1
45    def g():
46        nonlocal x
47        x = 2
48        y = 7
49        def h():
50            nonlocal x, y
51"""
52
53# also acts as test for 'except ... as ...'
54raise_from = """\
55try:
56    1 / 0
57except ZeroDivisionError as e:
58    raise ArithmeticError from e
59"""
60
61class_decorator = """\
62@f1(arg)
63@f2
64class Foo: pass
65"""
66
67elif1 = """\
68if cond1:
69    suite1
70elif cond2:
71    suite2
72else:
73    suite3
74"""
75
76elif2 = """\
77if cond1:
78    suite1
79elif cond2:
80    suite2
81"""
82
83try_except_finally = """\
84try:
85    suite1
86except ex1:
87    suite2
88except ex2:
89    suite3
90else:
91    suite4
92finally:
93    suite5
94"""
95
96with_simple = """\
97with f():
98    suite1
99"""
100
101with_as = """\
102with f() as x:
103    suite1
104"""
105
106with_two_items = """\
107with f() as x, g() as y:
108    suite1
109"""
110
111docstring_prefixes = (
112    "",
113    "class foo:\n    ",
114    "def foo():\n    ",
115    "async def foo():\n    ",
116)
117
118class ASTTestCase(unittest.TestCase):
119    def assertASTEqual(self, ast1, ast2):
120        self.assertEqual(ast.dump(ast1), ast.dump(ast2))
121
122    def check_ast_roundtrip(self, code1, **kwargs):
123        with self.subTest(code1=code1, ast_parse_kwargs=kwargs):
124            ast1 = ast.parse(code1, **kwargs)
125            code2 = ast.unparse(ast1)
126            ast2 = ast.parse(code2, **kwargs)
127            self.assertASTEqual(ast1, ast2)
128
129    def check_invalid(self, node, raises=ValueError):
130        with self.subTest(node=node):
131            self.assertRaises(raises, ast.unparse, node)
132
133    def get_source(self, code1, code2=None):
134        code2 = code2 or code1
135        code1 = ast.unparse(ast.parse(code1))
136        return code1, code2
137
138    def check_src_roundtrip(self, code1, code2=None):
139        code1, code2 = self.get_source(code1, code2)
140        with self.subTest(code1=code1, code2=code2):
141            self.assertEqual(code2, code1)
142
143    def check_src_dont_roundtrip(self, code1, code2=None):
144        code1, code2 = self.get_source(code1, code2)
145        with self.subTest(code1=code1, code2=code2):
146            self.assertNotEqual(code2, code1)
147
148class UnparseTestCase(ASTTestCase):
149    # Tests for specific bugs found in earlier versions of unparse
150
151    def test_fstrings(self):
152        # See issue 25180
153        self.check_ast_roundtrip(r"""f'{f"{0}"*3}'""")
154        self.check_ast_roundtrip(r"""f'{f"{y}"*3}'""")
155        self.check_ast_roundtrip("""f''""")
156        self.check_ast_roundtrip('''f"""'end' "quote\\""""''')
157
158    def test_fstrings_complicated(self):
159        # See issue 28002
160        self.check_ast_roundtrip("""f'''{"'"}'''""")
161        self.check_ast_roundtrip('''f\'\'\'-{f"""*{f"+{f'.{x}.'}+"}*"""}-\'\'\'''')
162        self.check_ast_roundtrip('''f\'\'\'-{f"""*{f"+{f'.{x}.'}+"}*"""}-'single quote\\'\'\'\'''')
163        self.check_ast_roundtrip('f"""{\'\'\'\n\'\'\'}"""')
164        self.check_ast_roundtrip('f"""{g(\'\'\'\n\'\'\')}"""')
165        self.check_ast_roundtrip('''f"a\\r\\nb"''')
166        self.check_ast_roundtrip('''f"\\u2028{'x'}"''')
167
168    def test_strings(self):
169        self.check_ast_roundtrip("u'foo'")
170        self.check_ast_roundtrip("r'foo'")
171        self.check_ast_roundtrip("b'foo'")
172
173    def test_del_statement(self):
174        self.check_ast_roundtrip("del x, y, z")
175
176    def test_shifts(self):
177        self.check_ast_roundtrip("45 << 2")
178        self.check_ast_roundtrip("13 >> 7")
179
180    def test_for_else(self):
181        self.check_ast_roundtrip(for_else)
182
183    def test_while_else(self):
184        self.check_ast_roundtrip(while_else)
185
186    def test_unary_parens(self):
187        self.check_ast_roundtrip("(-1)**7")
188        self.check_ast_roundtrip("(-1.)**8")
189        self.check_ast_roundtrip("(-1j)**6")
190        self.check_ast_roundtrip("not True or False")
191        self.check_ast_roundtrip("True or not False")
192
193    def test_integer_parens(self):
194        self.check_ast_roundtrip("3 .__abs__()")
195
196    def test_huge_float(self):
197        self.check_ast_roundtrip("1e1000")
198        self.check_ast_roundtrip("-1e1000")
199        self.check_ast_roundtrip("1e1000j")
200        self.check_ast_roundtrip("-1e1000j")
201
202    def test_nan(self):
203        self.assertASTEqual(
204            ast.parse(ast.unparse(ast.Constant(value=float('nan')))),
205            ast.parse('1e1000 - 1e1000')
206        )
207
208    def test_min_int(self):
209        self.check_ast_roundtrip(str(-(2 ** 31)))
210        self.check_ast_roundtrip(str(-(2 ** 63)))
211
212    def test_imaginary_literals(self):
213        self.check_ast_roundtrip("7j")
214        self.check_ast_roundtrip("-7j")
215        self.check_ast_roundtrip("0j")
216        self.check_ast_roundtrip("-0j")
217
218    def test_lambda_parentheses(self):
219        self.check_ast_roundtrip("(lambda: int)()")
220
221    def test_chained_comparisons(self):
222        self.check_ast_roundtrip("1 < 4 <= 5")
223        self.check_ast_roundtrip("a is b is c is not d")
224
225    def test_function_arguments(self):
226        self.check_ast_roundtrip("def f(): pass")
227        self.check_ast_roundtrip("def f(a): pass")
228        self.check_ast_roundtrip("def f(b = 2): pass")
229        self.check_ast_roundtrip("def f(a, b): pass")
230        self.check_ast_roundtrip("def f(a, b = 2): pass")
231        self.check_ast_roundtrip("def f(a = 5, b = 2): pass")
232        self.check_ast_roundtrip("def f(*, a = 1, b = 2): pass")
233        self.check_ast_roundtrip("def f(*, a = 1, b): pass")
234        self.check_ast_roundtrip("def f(*, a, b = 2): pass")
235        self.check_ast_roundtrip("def f(a, b = None, *, c, **kwds): pass")
236        self.check_ast_roundtrip("def f(a=2, *args, c=5, d, **kwds): pass")
237        self.check_ast_roundtrip("def f(*args, **kwargs): pass")
238
239    def test_relative_import(self):
240        self.check_ast_roundtrip(relative_import)
241
242    def test_nonlocal(self):
243        self.check_ast_roundtrip(nonlocal_ex)
244
245    def test_raise_from(self):
246        self.check_ast_roundtrip(raise_from)
247
248    def test_bytes(self):
249        self.check_ast_roundtrip("b'123'")
250
251    def test_annotations(self):
252        self.check_ast_roundtrip("def f(a : int): pass")
253        self.check_ast_roundtrip("def f(a: int = 5): pass")
254        self.check_ast_roundtrip("def f(*args: [int]): pass")
255        self.check_ast_roundtrip("def f(**kwargs: dict): pass")
256        self.check_ast_roundtrip("def f() -> None: pass")
257
258    def test_set_literal(self):
259        self.check_ast_roundtrip("{'a', 'b', 'c'}")
260
261    def test_empty_set(self):
262        self.assertASTEqual(
263            ast.parse(ast.unparse(ast.Set(elts=[]))),
264            ast.parse('{*()}')
265        )
266
267    def test_set_comprehension(self):
268        self.check_ast_roundtrip("{x for x in range(5)}")
269
270    def test_dict_comprehension(self):
271        self.check_ast_roundtrip("{x: x*x for x in range(10)}")
272
273    def test_class_decorators(self):
274        self.check_ast_roundtrip(class_decorator)
275
276    def test_class_definition(self):
277        self.check_ast_roundtrip("class A(metaclass=type, *[], **{}): pass")
278
279    def test_elifs(self):
280        self.check_ast_roundtrip(elif1)
281        self.check_ast_roundtrip(elif2)
282
283    def test_try_except_finally(self):
284        self.check_ast_roundtrip(try_except_finally)
285
286    def test_starred_assignment(self):
287        self.check_ast_roundtrip("a, *b, c = seq")
288        self.check_ast_roundtrip("a, (*b, c) = seq")
289        self.check_ast_roundtrip("a, *b[0], c = seq")
290        self.check_ast_roundtrip("a, *(b, c) = seq")
291
292    def test_with_simple(self):
293        self.check_ast_roundtrip(with_simple)
294
295    def test_with_as(self):
296        self.check_ast_roundtrip(with_as)
297
298    def test_with_two_items(self):
299        self.check_ast_roundtrip(with_two_items)
300
301    def test_dict_unpacking_in_dict(self):
302        # See issue 26489
303        self.check_ast_roundtrip(r"""{**{'y': 2}, 'x': 1}""")
304        self.check_ast_roundtrip(r"""{**{'y': 2}, **{'x': 1}}""")
305
306    def test_slices(self):
307        self.check_ast_roundtrip("a[i]")
308        self.check_ast_roundtrip("a[i,]")
309        self.check_ast_roundtrip("a[i, j]")
310        self.check_ast_roundtrip("a[(*a,)]")
311        self.check_ast_roundtrip("a[(a:=b)]")
312        self.check_ast_roundtrip("a[(a:=b,c)]")
313        self.check_ast_roundtrip("a[()]")
314        self.check_ast_roundtrip("a[i:j]")
315        self.check_ast_roundtrip("a[:j]")
316        self.check_ast_roundtrip("a[i:]")
317        self.check_ast_roundtrip("a[i:j:k]")
318        self.check_ast_roundtrip("a[:j:k]")
319        self.check_ast_roundtrip("a[i::k]")
320        self.check_ast_roundtrip("a[i:j,]")
321        self.check_ast_roundtrip("a[i:j, k]")
322
323    def test_invalid_raise(self):
324        self.check_invalid(ast.Raise(exc=None, cause=ast.Name(id="X")))
325
326    def test_invalid_fstring_constant(self):
327        self.check_invalid(ast.JoinedStr(values=[ast.Constant(value=100)]))
328
329    def test_invalid_fstring_conversion(self):
330        self.check_invalid(
331            ast.FormattedValue(
332                value=ast.Constant(value="a", kind=None),
333                conversion=ord("Y"),  # random character
334                format_spec=None,
335            )
336        )
337
338    def test_invalid_fstring_backslash(self):
339        self.check_invalid(ast.FormattedValue(value=ast.Constant(value="\\\\")))
340
341    def test_invalid_yield_from(self):
342        self.check_invalid(ast.YieldFrom(value=None))
343
344    def test_docstrings(self):
345        docstrings = (
346            'this ends with double quote"',
347            'this includes a """triple quote"""',
348            '\r',
349            '\\r',
350            '\t',
351            '\\t',
352            '\n',
353            '\\n',
354            '\r\\r\t\\t\n\\n',
355            '""">>> content = \"\"\"blabla\"\"\" <<<"""',
356            r'foo\n\x00',
357            "' \\'\\'\\'\"\"\" \"\"\\'\\' \\'",
358            '��⛎��üéş^\\\\X\\\\BB\N{LONG RIGHTWARDS SQUIGGLE ARROW}'
359        )
360        for docstring in docstrings:
361            # check as Module docstrings for easy testing
362            self.check_ast_roundtrip(f"'''{docstring}'''")
363
364    def test_constant_tuples(self):
365        self.check_src_roundtrip(ast.Constant(value=(1,), kind=None), "(1,)")
366        self.check_src_roundtrip(
367            ast.Constant(value=(1, 2, 3), kind=None), "(1, 2, 3)"
368        )
369
370    def test_function_type(self):
371        for function_type in (
372            "() -> int",
373            "(int, int) -> int",
374            "(Callable[complex], More[Complex(call.to_typevar())]) -> None"
375        ):
376            self.check_ast_roundtrip(function_type, mode="func_type")
377
378    def test_type_comments(self):
379        for statement in (
380            "a = 5 # type:",
381            "a = 5 # type: int",
382            "a = 5 # type: int and more",
383            "def x(): # type: () -> None\n\tpass",
384            "def x(y): # type: (int) -> None and more\n\tpass",
385            "async def x(): # type: () -> None\n\tpass",
386            "async def x(y): # type: (int) -> None and more\n\tpass",
387            "for x in y: # type: int\n\tpass",
388            "async for x in y: # type: int\n\tpass",
389            "with x(): # type: int\n\tpass",
390            "async with x(): # type: int\n\tpass"
391        ):
392            self.check_ast_roundtrip(statement, type_comments=True)
393
394    def test_type_ignore(self):
395        for statement in (
396            "a = 5 # type: ignore",
397            "a = 5 # type: ignore and more",
398            "def x(): # type: ignore\n\tpass",
399            "def x(y): # type: ignore and more\n\tpass",
400            "async def x(): # type: ignore\n\tpass",
401            "async def x(y): # type: ignore and more\n\tpass",
402            "for x in y: # type: ignore\n\tpass",
403            "async for x in y: # type: ignore\n\tpass",
404            "with x(): # type: ignore\n\tpass",
405            "async with x(): # type: ignore\n\tpass"
406        ):
407            self.check_ast_roundtrip(statement, type_comments=True)
408
409
410class CosmeticTestCase(ASTTestCase):
411    """Test if there are cosmetic issues caused by unnecessary additions"""
412
413    def test_simple_expressions_parens(self):
414        self.check_src_roundtrip("(a := b)")
415        self.check_src_roundtrip("await x")
416        self.check_src_roundtrip("x if x else y")
417        self.check_src_roundtrip("lambda x: x")
418        self.check_src_roundtrip("1 + 1")
419        self.check_src_roundtrip("1 + 2 / 3")
420        self.check_src_roundtrip("(1 + 2) / 3")
421        self.check_src_roundtrip("(1 + 2) * 3 + 4 * (5 + 2)")
422        self.check_src_roundtrip("(1 + 2) * 3 + 4 * (5 + 2) ** 2")
423        self.check_src_roundtrip("~x")
424        self.check_src_roundtrip("x and y")
425        self.check_src_roundtrip("x and y and z")
426        self.check_src_roundtrip("x and (y and x)")
427        self.check_src_roundtrip("(x and y) and z")
428        self.check_src_roundtrip("(x ** y) ** z ** q")
429        self.check_src_roundtrip("x >> y")
430        self.check_src_roundtrip("x << y")
431        self.check_src_roundtrip("x >> y and x >> z")
432        self.check_src_roundtrip("x + y - z * q ^ t ** k")
433        self.check_src_roundtrip("P * V if P and V else n * R * T")
434        self.check_src_roundtrip("lambda P, V, n: P * V == n * R * T")
435        self.check_src_roundtrip("flag & (other | foo)")
436        self.check_src_roundtrip("not x == y")
437        self.check_src_roundtrip("x == (not y)")
438        self.check_src_roundtrip("yield x")
439        self.check_src_roundtrip("yield from x")
440        self.check_src_roundtrip("call((yield x))")
441        self.check_src_roundtrip("return x + (yield x)")
442
443    def test_class_bases_and_keywords(self):
444        self.check_src_roundtrip("class X:\n    pass")
445        self.check_src_roundtrip("class X(A):\n    pass")
446        self.check_src_roundtrip("class X(A, B, C, D):\n    pass")
447        self.check_src_roundtrip("class X(x=y):\n    pass")
448        self.check_src_roundtrip("class X(metaclass=z):\n    pass")
449        self.check_src_roundtrip("class X(x=y, z=d):\n    pass")
450        self.check_src_roundtrip("class X(A, x=y):\n    pass")
451        self.check_src_roundtrip("class X(A, **kw):\n    pass")
452        self.check_src_roundtrip("class X(*args):\n    pass")
453        self.check_src_roundtrip("class X(*args, **kwargs):\n    pass")
454
455    def test_fstrings(self):
456        self.check_src_roundtrip('''f\'\'\'-{f"""*{f"+{f'.{x}.'}+"}*"""}-\'\'\'''')
457        self.check_src_roundtrip('''f"\\u2028{'x'}"''')
458        self.check_src_roundtrip(r"f'{x}\n'")
459        self.check_src_roundtrip('''f''\'{"""\n"""}\\n''\'''')
460        self.check_src_roundtrip('''f''\'{f"""{x}\n"""}\\n''\'''')
461
462    def test_docstrings(self):
463        docstrings = (
464            '"""simple doc string"""',
465            '''"""A more complex one
466            with some newlines"""''',
467            '''"""Foo bar baz
468
469            empty newline"""''',
470            '"""With some \t"""',
471            '"""Foo "bar" baz """',
472            '"""\\r"""',
473            '""""""',
474            '"""\'\'\'"""',
475            '"""\'\'\'\'\'\'"""',
476            '"""��⛎��üéş^\\\\X\\\\BB⟿"""',
477            '"""end in single \'quote\'"""',
478            "'''end in double \"quote\"'''",
479            '"""almost end in double "quote"."""',
480        )
481
482        for prefix in docstring_prefixes:
483            for docstring in docstrings:
484                self.check_src_roundtrip(f"{prefix}{docstring}")
485
486    def test_docstrings_negative_cases(self):
487        # Test some cases that involve strings in the children of the
488        # first node but aren't docstrings to make sure we don't have
489        # False positives.
490        docstrings_negative = (
491            'a = """false"""',
492            '"""false""" + """unless its optimized"""',
493            '1 + 1\n"""false"""',
494            'f"""no, top level but f-fstring"""'
495        )
496        for prefix in docstring_prefixes:
497            for negative in docstrings_negative:
498                # this cases should be result with single quote
499                # rather then triple quoted docstring
500                src = f"{prefix}{negative}"
501                self.check_ast_roundtrip(src)
502                self.check_src_dont_roundtrip(src)
503
504    def test_unary_op_factor(self):
505        for prefix in ("+", "-", "~"):
506            self.check_src_roundtrip(f"{prefix}1")
507        for prefix in ("not",):
508            self.check_src_roundtrip(f"{prefix} 1")
509
510    def test_slices(self):
511        self.check_src_roundtrip("a[1]")
512        self.check_src_roundtrip("a[1, 2]")
513        self.check_src_roundtrip("a[(1, *a)]")
514
515class DirectoryTestCase(ASTTestCase):
516    """Test roundtrip behaviour on all files in Lib and Lib/test."""
517
518    lib_dir = pathlib.Path(__file__).parent / ".."
519    test_directories = (lib_dir, lib_dir / "test")
520    run_always_files = {"test_grammar.py", "test_syntax.py", "test_compile.py",
521                        "test_ast.py", "test_asdl_parser.py", "test_fstring.py",
522                        "test_patma.py"}
523
524    _files_to_test = None
525
526    @classmethod
527    def files_to_test(cls):
528
529        if cls._files_to_test is not None:
530            return cls._files_to_test
531
532        items = [
533            item.resolve()
534            for directory in cls.test_directories
535            for item in directory.glob("*.py")
536            if not item.name.startswith("bad")
537        ]
538
539        # Test limited subset of files unless the 'cpu' resource is specified.
540        if not test.support.is_resource_enabled("cpu"):
541
542            tests_to_run_always = {item for item in items if
543                                   item.name in cls.run_always_files}
544
545            items = set(random.sample(items, 10))
546
547            # Make sure that at least tests that heavily use grammar features are
548            # always considered in order to reduce the chance of missing something.
549            items = list(items | tests_to_run_always)
550
551        # bpo-31174: Store the names sample to always test the same files.
552        # It prevents false alarms when hunting reference leaks.
553        cls._files_to_test = items
554
555        return items
556
557    def test_files(self):
558        for item in self.files_to_test():
559            if test.support.verbose:
560                print(f"Testing {item.absolute()}")
561
562            with self.subTest(filename=item):
563                source = read_pyfile(item)
564                self.check_ast_roundtrip(source)
565
566
567if __name__ == "__main__":
568    unittest.main()
569