1# Test the Unicode versions of normal file functions 2# open, os.open, os.stat. os.listdir, os.rename, os.remove, os.mkdir, os.chdir, os.rmdir 3import os 4import sys 5import unittest 6import warnings 7from unicodedata import normalize 8from test import support 9 10filenames = [ 11 '1_abc', 12 '2_ascii', 13 '3_Gr\xfc\xdf-Gott', 14 '4_\u0393\u03b5\u03b9\u03ac-\u03c3\u03b1\u03c2', 15 '5_\u0417\u0434\u0440\u0430\u0432\u0441\u0442\u0432\u0443\u0439\u0442\u0435', 16 '6_\u306b\u307d\u3093', 17 '7_\u05d4\u05e9\u05e7\u05e6\u05e5\u05e1', 18 '8_\u66e8\u66e9\u66eb', 19 '9_\u66e8\u05e9\u3093\u0434\u0393\xdf', 20 # Specific code points: fn, NFC(fn) and NFKC(fn) all different 21 '10_\u1fee\u1ffd', 22 ] 23 24# Mac OS X decomposes Unicode names, using Normal Form D. 25# http://developer.apple.com/mac/library/qa/qa2001/qa1173.html 26# "However, most volume formats do not follow the exact specification for 27# these normal forms. For example, HFS Plus uses a variant of Normal Form D 28# in which U+2000 through U+2FFF, U+F900 through U+FAFF, and U+2F800 through 29# U+2FAFF are not decomposed." 30if sys.platform != 'darwin': 31 filenames.extend([ 32 # Specific code points: NFC(fn), NFD(fn), NFKC(fn) and NFKD(fn) all different 33 '11_\u0385\u03d3\u03d4', 34 '12_\u00a8\u0301\u03d2\u0301\u03d2\u0308', # == NFD('\u0385\u03d3\u03d4') 35 '13_\u0020\u0308\u0301\u038e\u03ab', # == NFKC('\u0385\u03d3\u03d4') 36 '14_\u1e9b\u1fc1\u1fcd\u1fce\u1fcf\u1fdd\u1fde\u1fdf\u1fed', 37 38 # Specific code points: fn, NFC(fn) and NFKC(fn) all different 39 '15_\u1fee\u1ffd\ufad1', 40 '16_\u2000\u2000\u2000A', 41 '17_\u2001\u2001\u2001A', 42 '18_\u2003\u2003\u2003A', # == NFC('\u2001\u2001\u2001A') 43 '19_\u0020\u0020\u0020A', # '\u0020' == ' ' == NFKC('\u2000') == 44 # NFKC('\u2001') == NFKC('\u2003') 45 ]) 46 47 48# Is it Unicode-friendly? 49if not os.path.supports_unicode_filenames: 50 fsencoding = sys.getfilesystemencoding() 51 try: 52 for name in filenames: 53 name.encode(fsencoding) 54 except UnicodeEncodeError: 55 raise unittest.SkipTest("only NT+ and systems with " 56 "Unicode-friendly filesystem encoding") 57 58 59class UnicodeFileTests(unittest.TestCase): 60 files = set(filenames) 61 normal_form = None 62 63 def setUp(self): 64 try: 65 os.mkdir(support.TESTFN) 66 except FileExistsError: 67 pass 68 self.addCleanup(support.rmtree, support.TESTFN) 69 70 files = set() 71 for name in self.files: 72 name = os.path.join(support.TESTFN, self.norm(name)) 73 with open(name, 'wb') as f: 74 f.write((name+'\n').encode("utf-8")) 75 os.stat(name) 76 files.add(name) 77 self.files = files 78 79 def norm(self, s): 80 if self.normal_form: 81 return normalize(self.normal_form, s) 82 return s 83 84 def _apply_failure(self, fn, filename, 85 expected_exception=FileNotFoundError, 86 check_filename=True): 87 with self.assertRaises(expected_exception) as c: 88 fn(filename) 89 exc_filename = c.exception.filename 90 if check_filename: 91 self.assertEqual(exc_filename, filename, "Function '%s(%a) failed " 92 "with bad filename in the exception: %a" % 93 (fn.__name__, filename, exc_filename)) 94 95 def test_failures(self): 96 # Pass non-existing Unicode filenames all over the place. 97 for name in self.files: 98 name = "not_" + name 99 self._apply_failure(open, name) 100 self._apply_failure(os.stat, name) 101 self._apply_failure(os.chdir, name) 102 self._apply_failure(os.rmdir, name) 103 self._apply_failure(os.remove, name) 104 self._apply_failure(os.listdir, name) 105 106 if sys.platform == 'win32': 107 # Windows is lunatic. Issue #13366. 108 _listdir_failure = NotADirectoryError, FileNotFoundError 109 else: 110 _listdir_failure = NotADirectoryError 111 112 def test_open(self): 113 for name in self.files: 114 f = open(name, 'wb') 115 f.write((name+'\n').encode("utf-8")) 116 f.close() 117 os.stat(name) 118 self._apply_failure(os.listdir, name, self._listdir_failure) 119 120 # Skip the test on darwin, because darwin does normalize the filename to 121 # NFD (a variant of Unicode NFD form). Normalize the filename to NFC, NFKC, 122 # NFKD in Python is useless, because darwin will normalize it later and so 123 # open(), os.stat(), etc. don't raise any exception. 124 @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X') 125 def test_normalize(self): 126 files = set(self.files) 127 others = set() 128 for nf in set(['NFC', 'NFD', 'NFKC', 'NFKD']): 129 others |= set(normalize(nf, file) for file in files) 130 others -= files 131 for name in others: 132 self._apply_failure(open, name) 133 self._apply_failure(os.stat, name) 134 self._apply_failure(os.chdir, name) 135 self._apply_failure(os.rmdir, name) 136 self._apply_failure(os.remove, name) 137 self._apply_failure(os.listdir, name) 138 139 # Skip the test on darwin, because darwin uses a normalization different 140 # than Python NFD normalization: filenames are different even if we use 141 # Python NFD normalization. 142 @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X') 143 def test_listdir(self): 144 sf0 = set(self.files) 145 with warnings.catch_warnings(): 146 warnings.simplefilter("ignore", DeprecationWarning) 147 f1 = os.listdir(support.TESTFN.encode(sys.getfilesystemencoding())) 148 f2 = os.listdir(support.TESTFN) 149 sf2 = set(os.path.join(support.TESTFN, f) for f in f2) 150 self.assertEqual(sf0, sf2, "%a != %a" % (sf0, sf2)) 151 self.assertEqual(len(f1), len(f2)) 152 153 def test_rename(self): 154 for name in self.files: 155 os.rename(name, "tmp") 156 os.rename("tmp", name) 157 158 def test_directory(self): 159 dirname = os.path.join(support.TESTFN, 'Gr\xfc\xdf-\u66e8\u66e9\u66eb') 160 filename = '\xdf-\u66e8\u66e9\u66eb' 161 with support.temp_cwd(dirname): 162 with open(filename, 'wb') as f: 163 f.write((filename + '\n').encode("utf-8")) 164 os.access(filename,os.R_OK) 165 os.remove(filename) 166 167 168class UnicodeNFCFileTests(UnicodeFileTests): 169 normal_form = 'NFC' 170 171 172class UnicodeNFDFileTests(UnicodeFileTests): 173 normal_form = 'NFD' 174 175 176class UnicodeNFKCFileTests(UnicodeFileTests): 177 normal_form = 'NFKC' 178 179 180class UnicodeNFKDFileTests(UnicodeFileTests): 181 normal_form = 'NFKD' 182 183 184def test_main(): 185 support.run_unittest( 186 UnicodeFileTests, 187 UnicodeNFCFileTests, 188 UnicodeNFDFileTests, 189 UnicodeNFKCFileTests, 190 UnicodeNFKDFileTests, 191 ) 192 193 194if __name__ == "__main__": 195 test_main() 196