• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Test the Unicode versions of normal file functions
2# open, os.open, os.stat. os.listdir, os.rename, os.remove, os.mkdir, os.chdir, os.rmdir
3import os
4import sys
5import unittest
6import warnings
7from unicodedata import normalize
8from test.support import os_helper
9
10
11filenames = [
12    '1_abc',
13    '2_ascii',
14    '3_Gr\xfc\xdf-Gott',
15    '4_\u0393\u03b5\u03b9\u03ac-\u03c3\u03b1\u03c2',
16    '5_\u0417\u0434\u0440\u0430\u0432\u0441\u0442\u0432\u0443\u0439\u0442\u0435',
17    '6_\u306b\u307d\u3093',
18    '7_\u05d4\u05e9\u05e7\u05e6\u05e5\u05e1',
19    '8_\u66e8\u66e9\u66eb',
20    '9_\u66e8\u05e9\u3093\u0434\u0393\xdf',
21    # Specific code points: fn, NFC(fn) and NFKC(fn) all different
22    '10_\u1fee\u1ffd',
23    ]
24
25# Mac OS X decomposes Unicode names, using Normal Form D.
26# http://developer.apple.com/mac/library/qa/qa2001/qa1173.html
27# "However, most volume formats do not follow the exact specification for
28# these normal forms.  For example, HFS Plus uses a variant of Normal Form D
29# in which U+2000 through U+2FFF, U+F900 through U+FAFF, and U+2F800 through
30# U+2FAFF are not decomposed."
31if sys.platform != 'darwin':
32    filenames.extend([
33        # Specific code points: NFC(fn), NFD(fn), NFKC(fn) and NFKD(fn) all different
34        '11_\u0385\u03d3\u03d4',
35        '12_\u00a8\u0301\u03d2\u0301\u03d2\u0308', # == NFD('\u0385\u03d3\u03d4')
36        '13_\u0020\u0308\u0301\u038e\u03ab',       # == NFKC('\u0385\u03d3\u03d4')
37        '14_\u1e9b\u1fc1\u1fcd\u1fce\u1fcf\u1fdd\u1fde\u1fdf\u1fed',
38
39        # Specific code points: fn, NFC(fn) and NFKC(fn) all different
40        '15_\u1fee\u1ffd\ufad1',
41        '16_\u2000\u2000\u2000A',
42        '17_\u2001\u2001\u2001A',
43        '18_\u2003\u2003\u2003A',  # == NFC('\u2001\u2001\u2001A')
44        '19_\u0020\u0020\u0020A',  # '\u0020' == ' ' == NFKC('\u2000') ==
45                                   #  NFKC('\u2001') == NFKC('\u2003')
46    ])
47
48
49# Is it Unicode-friendly?
50if not os.path.supports_unicode_filenames:
51    fsencoding = sys.getfilesystemencoding()
52    try:
53        for name in filenames:
54            name.encode(fsencoding)
55    except UnicodeEncodeError:
56        raise unittest.SkipTest("only NT+ and systems with "
57                                "Unicode-friendly filesystem encoding")
58
59
60class UnicodeFileTests(unittest.TestCase):
61    files = set(filenames)
62    normal_form = None
63
64    def setUp(self):
65        try:
66            os.mkdir(os_helper.TESTFN)
67        except FileExistsError:
68            pass
69        self.addCleanup(os_helper.rmtree, os_helper.TESTFN)
70
71        files = set()
72        for name in self.files:
73            name = os.path.join(os_helper.TESTFN, self.norm(name))
74            with open(name, 'wb') as f:
75                f.write((name+'\n').encode("utf-8"))
76            os.stat(name)
77            files.add(name)
78        self.files = files
79
80    def norm(self, s):
81        if self.normal_form:
82            return normalize(self.normal_form, s)
83        return s
84
85    def _apply_failure(self, fn, filename,
86                       expected_exception=FileNotFoundError,
87                       check_filename=True):
88        with self.assertRaises(expected_exception) as c:
89            fn(filename)
90        exc_filename = c.exception.filename
91        if check_filename:
92            self.assertEqual(exc_filename, filename, "Function '%s(%a) failed "
93                             "with bad filename in the exception: %a" %
94                             (fn.__name__, filename, exc_filename))
95
96    def test_failures(self):
97        # Pass non-existing Unicode filenames all over the place.
98        for name in self.files:
99            name = "not_" + name
100            self._apply_failure(open, name)
101            self._apply_failure(os.stat, name)
102            self._apply_failure(os.chdir, name)
103            self._apply_failure(os.rmdir, name)
104            self._apply_failure(os.remove, name)
105            self._apply_failure(os.listdir, name)
106
107    if sys.platform == 'win32':
108        # Windows is lunatic. Issue #13366.
109        _listdir_failure = NotADirectoryError, FileNotFoundError
110    else:
111        _listdir_failure = NotADirectoryError
112
113    def test_open(self):
114        for name in self.files:
115            f = open(name, 'wb')
116            f.write((name+'\n').encode("utf-8"))
117            f.close()
118            os.stat(name)
119            self._apply_failure(os.listdir, name, self._listdir_failure)
120
121    # Skip the test on darwin, because darwin does normalize the filename to
122    # NFD (a variant of Unicode NFD form). Normalize the filename to NFC, NFKC,
123    # NFKD in Python is useless, because darwin will normalize it later and so
124    # open(), os.stat(), etc. don't raise any exception.
125    @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X')
126    def test_normalize(self):
127        files = set(self.files)
128        others = set()
129        for nf in set(['NFC', 'NFD', 'NFKC', 'NFKD']):
130            others |= set(normalize(nf, file) for file in files)
131        others -= files
132        for name in others:
133            self._apply_failure(open, name)
134            self._apply_failure(os.stat, name)
135            self._apply_failure(os.chdir, name)
136            self._apply_failure(os.rmdir, name)
137            self._apply_failure(os.remove, name)
138            self._apply_failure(os.listdir, name)
139
140    # Skip the test on darwin, because darwin uses a normalization different
141    # than Python NFD normalization: filenames are different even if we use
142    # Python NFD normalization.
143    @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X')
144    def test_listdir(self):
145        sf0 = set(self.files)
146        with warnings.catch_warnings():
147            warnings.simplefilter("ignore", DeprecationWarning)
148            f1 = os.listdir(os_helper.TESTFN.encode(
149                            sys.getfilesystemencoding()))
150        f2 = os.listdir(os_helper.TESTFN)
151        sf2 = set(os.path.join(os_helper.TESTFN, f) for f in f2)
152        self.assertEqual(sf0, sf2, "%a != %a" % (sf0, sf2))
153        self.assertEqual(len(f1), len(f2))
154
155    def test_rename(self):
156        for name in self.files:
157            os.rename(name, "tmp")
158            os.rename("tmp", name)
159
160    def test_directory(self):
161        dirname = os.path.join(os_helper.TESTFN,
162                               'Gr\xfc\xdf-\u66e8\u66e9\u66eb')
163        filename = '\xdf-\u66e8\u66e9\u66eb'
164        with os_helper.temp_cwd(dirname):
165            with open(filename, 'wb') as f:
166                f.write((filename + '\n').encode("utf-8"))
167            os.access(filename,os.R_OK)
168            os.remove(filename)
169
170
171class UnicodeNFCFileTests(UnicodeFileTests):
172    normal_form = 'NFC'
173
174
175class UnicodeNFDFileTests(UnicodeFileTests):
176    normal_form = 'NFD'
177
178
179class UnicodeNFKCFileTests(UnicodeFileTests):
180    normal_form = 'NFKC'
181
182
183class UnicodeNFKDFileTests(UnicodeFileTests):
184    normal_form = 'NFKD'
185
186
187if __name__ == "__main__":
188    unittest.main()
189