1 //
2 // Copyright (c) 2009-2011 Artyom Beilis (Tonkikh)
3 //
4 // Distributed under the Boost Software License, Version 1.0. (See
5 // accompanying file LICENSE_1_0.txt or copy at
6 // http://www.boost.org/LICENSE_1_0.txt)
7 //
8 #include "test_locale.hpp"
9 #include "test_locale_tools.hpp"
10 #include <boost/locale/utf.hpp>
11
12 #include <string.h>
13
14 using namespace boost::locale::utf;
15
make2(unsigned v)16 char *make2(unsigned v)
17 {
18 static unsigned char buf[3] = {0};
19 buf[0] = 0xC0 | (v >> 6);
20 buf[1] = 0x80 | (v & 0x3F );
21 return reinterpret_cast<char*>(buf);
22 }
23
make3(unsigned v)24 char *make3(unsigned v)
25 {
26 static unsigned char buf[4] = {0};
27 buf[0] = 0xE0 | ((v >> 12) ) ;
28 buf[1] = 0x80 | ((v >> 6) & 0x3F );
29 buf[2] = 0x80 | ((v >> 0) & 0x3F );
30 return reinterpret_cast<char*>(buf);
31 }
32
make4(unsigned v)33 char *make4(unsigned v)
34 {
35 static unsigned char buf[5] = {0};
36 buf[0] = 0xF0 | ((v >> 18) ) ;
37 buf[1] = 0x80 | ((v >> 12) & 0x3F );
38 buf[2] = 0x80 | ((v >> 6) & 0x3F );
39 buf[3] = 0x80 | ((v >> 0) & 0x3F );
40 return reinterpret_cast<char*>(buf);
41 }
42
u32_seq(boost::uint32_t a)43 boost::uint32_t const *u32_seq(boost::uint32_t a)
44 {
45 static boost::uint32_t buf[2];
46 buf[0]=a;
47 buf[1]=0;
48 return buf;
49 }
50
u16_seq(boost::uint16_t a)51 boost::uint16_t const *u16_seq(boost::uint16_t a)
52 {
53 static boost::uint16_t buf[2];
54 buf[0]=a;
55 buf[1]=0;
56 return buf;
57 }
58
u16_seq(boost::uint16_t a,boost::uint16_t b)59 boost::uint16_t const *u16_seq(boost::uint16_t a,boost::uint16_t b)
60 {
61 static boost::uint16_t buf[3];
62 buf[0]=a;
63 buf[1]=b;
64 buf[2]=0;
65 return buf;
66 }
67
68 template<typename CharType>
test_to(CharType const * s,unsigned codepoint)69 void test_to(CharType const *s,unsigned codepoint)
70 {
71 CharType const *begin = s;
72 CharType const *end = begin;
73
74 while(*end)
75 end++;
76
77 typedef utf_traits<CharType> tr;
78
79 TEST(tr::max_width == 4 / sizeof(CharType));
80
81 TEST(tr::template decode<CharType const *>(begin,end) == codepoint);
82
83 if(codepoint == incomplete || codepoint != illegal)
84 TEST(end == begin);
85
86 if(codepoint == incomplete) {
87 TEST(*s== 0 || 0 < tr::trail_length(*s));
88 TEST(tr::trail_length(*s) + 1 > end - s);
89 }
90
91 if(codepoint != incomplete && codepoint != illegal) {
92 begin=s;
93 TEST(tr::is_lead(*begin));
94 TEST(!tr::is_trail(*begin));
95 begin++;
96 while(begin!=end) {
97 TEST(tr::is_trail(*begin));
98 TEST(!tr::is_lead(*begin));
99 begin++;
100 }
101 TEST(tr::width(codepoint)==end - s);
102 TEST(tr::trail_length(*s) == tr::width(codepoint) - 1);
103 begin = s;
104 TEST(tr::decode_valid(begin) == codepoint);
105 TEST(begin == end);
106 }
107 }
108
109 template<typename CharType>
test_from(CharType const * str,unsigned codepoint)110 void test_from(CharType const *str,unsigned codepoint)
111 {
112 CharType buf[5] = {1,1,1,1,1};
113 CharType *p=buf;
114 p = utf_traits<CharType>::template encode<CharType *>(codepoint,p);
115 CharType const *end = str;
116 while(*end)
117 end++;
118 TEST(end - str == p-buf );
119 TEST(*p);
120 *p=0;
121 TEST(memcmp(str,buf,sizeof(CharType) * (end-str))==0);
122 }
123
124
main()125 int main()
126 {
127 try {
128
129 std::cout << "Test UTF-8" << std::endl;
130 std::cout << "- From UTF-8" << std::endl;
131
132
133 std::cout << "-- Correct" << std::endl;
134
135 test_to("\x7f",0x7f);
136 test_to("\xc2\x80",0x80);
137 test_to("\xdf\xbf",0x7ff);
138 test_to("\xe0\xa0\x80",0x800);
139 test_to("\xef\xbf\xbf",0xffff);
140 test_to("\xf0\x90\x80\x80",0x10000);
141 test_to("\xf4\x8f\xbf\xbf",0x10ffff);
142
143 std::cout << "-- Too big" << std::endl;
144 test_to("\xf4\x9f\x80\x80",illegal); // 11 0000
145 test_to("\xfb\xbf\xbf\xbf",illegal); // 3ff ffff
146 test_to("\xf8\x90\x80\x80\x80",illegal); // 400 0000
147 test_to("\xfd\xbf\xbf\xbf\xbf\xbf",illegal); // 7fff ffff
148
149 std::cout << "-- Invalid length" << std::endl;
150
151 /// test that this actually works
152 test_to(make2(0x80),0x80);
153 test_to(make2(0x7ff),0x7ff);
154
155 test_to(make3(0x800),0x800);
156 test_to(make3(0xffff),0xffff);
157
158 test_to(make4(0x10000),0x10000);
159 test_to(make4(0x10ffff),0x10ffff);
160
161 test_to(make4(0x110000),illegal);
162 test_to(make4(0x1fffff),illegal);
163
164 test_to(make2(0),illegal);
165 test_to(make3(0),illegal);
166 test_to(make4(0),illegal);
167 test_to(make2(0x7f),illegal);
168 test_to(make3(0x7f),illegal);
169 test_to(make4(0x7f),illegal);
170
171 test_to(make3(0x80),illegal);
172 test_to(make4(0x80),illegal);
173 test_to(make3(0x7ff),illegal);
174 test_to(make4(0x7ff),illegal);
175
176 test_to(make4(0x8000),illegal);
177 test_to(make4(0xffff),illegal);
178
179 std::cout << "-- Invalid surrogate" << std::endl;
180
181 test_to(make3(0xd800),illegal);
182 test_to(make3(0xdbff),illegal);
183 test_to(make3(0xdc00),illegal);
184 test_to(make3(0xdfff),illegal);
185
186 test_to(make4(0xd800),illegal);
187 test_to(make4(0xdbff),illegal);
188 test_to(make4(0xdc00),illegal);
189 test_to(make4(0xdfff),illegal);
190
191 std::cout <<"-- Incomplete" << std::endl;
192
193 test_to("",incomplete);
194
195 test_to("\x80",illegal);
196 test_to("\xc2",incomplete);
197
198 test_to("\xdf",incomplete);
199
200 test_to("\xe0",incomplete);
201 test_to("\xe0\xa0",incomplete);
202
203 test_to("\xef\xbf",incomplete);
204 test_to("\xef",incomplete);
205
206 test_to("\xf0\x90\x80",incomplete);
207 test_to("\xf0\x90",incomplete);
208 test_to("\xf0",incomplete);
209
210 test_to("\xf4\x8f\xbf",incomplete);
211 test_to("\xf4\x8f",incomplete);
212 test_to("\xf4",incomplete);
213
214 std::cout << "- To UTF-8" << std::endl;
215
216 std::cout << "-- Test correct" << std::endl;
217
218 test_from("\x7f",0x7f);
219 test_from("\xc2\x80",0x80);
220 test_from("\xdf\xbf",0x7ff);
221 test_from("\xe0\xa0\x80",0x800);
222 test_from("\xef\xbf\xbf",0xffff);
223 test_from("\xf0\x90\x80\x80",0x10000);
224 test_from("\xf4\x8f\xbf\xbf",0x10ffff);
225
226 std::cout << "Test UTF-16" << std::endl;
227 std::cout << "- From UTF-16" << std::endl;
228
229
230 std::cout << "-- Correct" << std::endl;
231
232 test_to(u16_seq(0x10),0x10);
233 test_to(u16_seq(0xffff),0xffff);
234 test_to(u16_seq(0xD800,0xDC00),0x10000);
235 test_to(u16_seq(0xDBFF,0xDFFF),0x10FFFF);
236
237
238 std::cout << "-- Invalid surrogate" << std::endl;
239
240 test_to(u16_seq(0xDFFF),illegal);
241 test_to(u16_seq(0xDC00),illegal);
242
243 std::cout <<"-- Incomplete" << std::endl;
244
245 test_to(u16_seq(0),incomplete);
246 test_to(u16_seq(0xD800),incomplete);
247 test_to(u16_seq(0xDBFF),incomplete);
248
249 std::cout << "- To UTF-16" << std::endl;
250
251 std::cout << "-- Test correct" << std::endl;
252
253 test_to(u16_seq(0x10),0x10);
254 test_to(u16_seq(0xffff),0xffff);
255 test_to(u16_seq(0xD800,0xDC00),0x10000);
256 test_to(u16_seq(0xDBFF,0xDFFF),0x10FFFF);
257
258
259 std::cout << "Test UTF-32" << std::endl;
260 std::cout << "- From UTF-32" << std::endl;
261
262
263 std::cout << "-- Correct" << std::endl;
264
265 test_to(u32_seq(0x10),0x10);
266 test_to(u32_seq(0xffff),0xffff);
267 test_to(u32_seq(0x10000),0x10000);
268 test_to(u32_seq(0x10ffff),0x10ffff);
269
270
271
272 std::cout << "-- Invalid surrogate" << std::endl;
273
274 test_to(u32_seq(0xD800),illegal);
275 test_to(u32_seq(0xDBFF),illegal);
276 test_to(u32_seq(0xDFFF),illegal);
277 test_to(u32_seq(0xDC00),illegal);
278 test_to(u32_seq(0x110000),illegal);
279
280 std::cout <<"-- Incomplete" << std::endl;
281
282 test_to(u32_seq(0),incomplete);
283
284 std::cout << "- To UTF-32" << std::endl;
285
286 std::cout << "-- Test correct" << std::endl;
287
288 test_to(u32_seq(0x10),0x10);
289 test_to(u32_seq(0xffff),0xffff);
290 test_to(u32_seq(0x10ffff),0x10ffff);
291
292
293
294 }
295 catch(std::exception const &e) {
296 std::cerr << "Failed " << e.what() << std::endl;
297 return EXIT_FAILURE;
298 }
299 FINALIZE();
300 }
301
302 // vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
303 // boostinspect:noascii
304