1 // Copyright (C) 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
3 /********************************************************************
4 * COPYRIGHT:
5 * Copyright (c) 1997-2016, International Business Machines Corporation and
6 * others. All Rights Reserved.
7 ********************************************************************/
8 /*****************************************************************************
9 *
10 * File ncnvtst.c
11 *
12 * Modification History:
13 * Name Description
14 * Madhu Katragadda 7/7/2000 Converter Tests for extended code coverage
15 ******************************************************************************
16 */
17 #include <stdio.h>
18 #include <stdlib.h>
19 #include <string.h>
20 #include "unicode/uloc.h"
21 #include "unicode/ucnv.h"
22 #include "unicode/utypes.h"
23 #include "unicode/ustring.h"
24 #include "unicode/uset.h"
25 #include "cintltst.h"
26 #include "cmemory.h"
27
28 #define MAX_LENGTH 999
29
30 #define UNICODE_LIMIT 0x10FFFF
31 #define SURROGATE_HIGH_START 0xD800
32 #define SURROGATE_LOW_END 0xDFFF
33
34 static int32_t gInBufferSize = 0;
35 static int32_t gOutBufferSize = 0;
36 static char gNuConvTestName[1024];
37
38 #define nct_min(x,y) ((x<y) ? x : y)
39
40 static void printSeq(const unsigned char* a, int len);
41 static void printSeqErr(const unsigned char* a, int len);
42 static void printUSeq(const UChar* a, int len);
43 static void printUSeqErr(const UChar* a, int len);
44 static UBool convertFromU( const UChar *source, int sourceLen, const uint8_t *expect, int expectLen,
45 const char *codepage, const int32_t *expectOffsets, UBool doFlush, UErrorCode expectedStatus);
46 static UBool convertToU( const uint8_t *source, int sourceLen, const UChar *expect, int expectLen,
47 const char *codepage, const int32_t *expectOffsets, UBool doFlush, UErrorCode expectedStatus);
48
49 static UBool testConvertFromU( const UChar *source, int sourceLen, const uint8_t *expect, int expectLen,
50 const char *codepage, UConverterFromUCallback callback, const int32_t *expectOffsets, UBool testReset);
51 static UBool testConvertToU( const uint8_t *source, int sourcelen, const UChar *expect, int expectlen,
52 const char *codepage, UConverterToUCallback callback, const int32_t *expectOffsets, UBool testReset);
53
setNuConvTestName(const char * codepage,const char * direction)54 static void setNuConvTestName(const char *codepage, const char *direction)
55 {
56 sprintf(gNuConvTestName, "[Testing %s %s Unicode, InputBufSiz=%d, OutputBufSiz=%d]",
57 codepage,
58 direction,
59 (int)gInBufferSize,
60 (int)gOutBufferSize);
61 }
62
63
64 static void TestSurrogateBehaviour(void);
65 static void TestErrorBehaviour(void);
66
67 #if !UCONFIG_NO_LEGACY_CONVERSION
68 static void TestToUnicodeErrorBehaviour(void);
69 static void TestGetNextErrorBehaviour(void);
70 #endif
71
72 static void TestRegressionUTF8(void);
73 static void TestRegressionUTF32(void);
74 static void TestAvailableConverters(void);
75 static void TestFlushInternalBuffer(void); /*for improved code coverage in ucnv_cnv.c*/
76 static void TestResetBehaviour(void);
77 static void TestTruncated(void);
78 static void TestUnicodeSet(void);
79
80 static void TestWithBufferSize(int32_t osize, int32_t isize);
81
82
printSeq(const unsigned char * a,int len)83 static void printSeq(const unsigned char* a, int len)
84 {
85 int i=0;
86 log_verbose("\n{");
87 while (i<len)
88 log_verbose("0x%02X ", a[i++]);
89 log_verbose("}\n");
90 }
91
printUSeq(const UChar * a,int len)92 static void printUSeq(const UChar* a, int len)
93 {
94 int i=0;
95 log_verbose("\n{");
96 while (i<len)
97 log_verbose("%0x04X ", a[i++]);
98 log_verbose("}\n");
99 }
100
printSeqErr(const unsigned char * a,int len)101 static void printSeqErr(const unsigned char* a, int len)
102 {
103 int i=0;
104 fprintf(stderr, "\n{");
105 while (i<len) fprintf(stderr, "0x%02X ", a[i++]);
106 fprintf(stderr, "}\n");
107 }
108
printUSeqErr(const UChar * a,int len)109 static void printUSeqErr(const UChar* a, int len)
110 {
111 int i=0;
112 fprintf(stderr, "\n{");
113 while (i<len)
114 fprintf(stderr, "0x%04X ", a[i++]);
115 fprintf(stderr,"}\n");
116 }
117
118 void addExtraTests(TestNode** root);
119
addExtraTests(TestNode ** root)120 void addExtraTests(TestNode** root)
121 {
122 addTest(root, &TestSurrogateBehaviour, "tsconv/ncnvtst/TestSurrogateBehaviour");
123 addTest(root, &TestErrorBehaviour, "tsconv/ncnvtst/TestErrorBehaviour");
124
125 #if !UCONFIG_NO_LEGACY_CONVERSION
126 addTest(root, &TestToUnicodeErrorBehaviour, "tsconv/ncnvtst/ToUnicodeErrorBehaviour");
127 addTest(root, &TestGetNextErrorBehaviour, "tsconv/ncnvtst/TestGetNextErrorBehaviour");
128 #endif
129
130 addTest(root, &TestAvailableConverters, "tsconv/ncnvtst/TestAvailableConverters");
131 addTest(root, &TestFlushInternalBuffer, "tsconv/ncnvtst/TestFlushInternalBuffer");
132 addTest(root, &TestResetBehaviour, "tsconv/ncnvtst/TestResetBehaviour");
133 addTest(root, &TestRegressionUTF8, "tsconv/ncnvtst/TestRegressionUTF8");
134 addTest(root, &TestRegressionUTF32, "tsconv/ncnvtst/TestRegressionUTF32");
135 addTest(root, &TestTruncated, "tsconv/ncnvtst/TestTruncated");
136 addTest(root, &TestUnicodeSet, "tsconv/ncnvtst/TestUnicodeSet");
137 }
138
139 /*test surrogate behaviour*/
TestSurrogateBehaviour()140 static void TestSurrogateBehaviour(){
141 log_verbose("Testing for SBCS and LATIN_1\n");
142 {
143 UChar sampleText[] = {0x0031, 0xd801, 0xdc01, 0x0032};
144 const uint8_t expected[] = {0x31, 0x1a, 0x32};
145
146 #if !UCONFIG_NO_LEGACY_CONVERSION
147 /*SBCS*/
148 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
149 expected, sizeof(expected), "ibm-920", 0 , TRUE, U_ZERO_ERROR))
150 log_err("u-> ibm-920 [UCNV_SBCS] not match.\n");
151 #endif
152
153 /*LATIN_1*/
154 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
155 expected, sizeof(expected), "LATIN_1", 0, TRUE, U_ZERO_ERROR ))
156 log_err("u-> LATIN_1 not match.\n");
157
158 }
159
160 #if !UCONFIG_NO_LEGACY_CONVERSION
161 log_verbose("Testing for DBCS and MBCS\n");
162 {
163 UChar sampleText[] = {0x00a1, 0xd801, 0xdc01, 0x00a4};
164 const uint8_t expected[] = {0xa2, 0xae, 0xa1, 0xe0, 0xa2, 0xb4};
165 int32_t offsets[] = {0x00, 0x00, 0x01, 0x01, 0x03, 0x03 };
166
167 /*DBCS*/
168 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
169 expected, sizeof(expected), "ibm-1363", 0 , TRUE, U_ZERO_ERROR))
170 log_err("u-> ibm-1363 [UCNV_DBCS portion] not match.\n");
171 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
172 expected, sizeof(expected), "ibm-1363", offsets , TRUE, U_ZERO_ERROR))
173 log_err("u-> ibm-1363 [UCNV_DBCS portion] not match.\n");
174 /*MBCS*/
175 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
176 expected, sizeof(expected), "ibm-1363", 0 , TRUE, U_ZERO_ERROR))
177 log_err("u-> ibm-1363 [UCNV_MBCS] not match.\n");
178 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
179 expected, sizeof(expected), "ibm-1363", offsets, TRUE, U_ZERO_ERROR))
180 log_err("u-> ibm-1363 [UCNV_MBCS] not match.\n");
181 }
182
183 log_verbose("Testing for ISO-2022-jp\n");
184 {
185 UChar sampleText[] = { 0x4e00, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
186
187 const uint8_t expected[] = {0x1b, 0x24, 0x42,0x30,0x6c,0x43,0x7a,0x1b,0x28,0x42,
188 0x31,0x1A, 0x32};
189
190
191 int32_t offsets[] = {0,0,0,0,0,1,1,2,2,2,2,3,5 };
192
193 /*iso-2022-jp*/
194 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
195 expected, sizeof(expected), "iso-2022-jp", 0 , TRUE, U_ZERO_ERROR))
196 log_err("u-> not match.\n");
197 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
198 expected, sizeof(expected), "iso-2022-jp", offsets , TRUE, U_ZERO_ERROR))
199 log_err("u-> not match.\n");
200 }
201
202 /* BEGIN android-removed */
203 /* To save space, Android does not build full ISO-2022-CN tables.
204 We skip the tests for ISO-2022-CN. */
205 /*
206 log_verbose("Testing for ISO-2022-cn\n");
207 {
208 static const UChar sampleText[] = { 0x4e00, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
209
210 static const uint8_t expected[] = {
211 0x1B, 0x24, 0x29, 0x41, 0x0E, 0x52, 0x3B,
212 0x36, 0x21,
213 0x0F, 0x31,
214 0x1A,
215 0x32
216 };
217
218
219
220 static const int32_t offsets[] = {
221 0, 0, 0, 0, 0, 0, 0,
222 1, 1,
223 2, 2,
224 3,
225 5, };
226
227 // iso-2022-CN android-change
228 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
229 expected, sizeof(expected), "iso-2022-cn", 0 , TRUE, U_ZERO_ERROR))
230 log_err("u-> not match.\n");
231 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
232 expected, sizeof(expected), "iso-2022-cn", offsets , TRUE, U_ZERO_ERROR))
233 log_err("u-> not match.\n");
234 }
235 */
236 /* END android-removed */
237
238 log_verbose("Testing for ISO-2022-kr\n");
239 {
240 static const UChar sampleText[] = { 0x4e00,0xd801, 0xdc01, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
241
242 static const uint8_t expected[] = {0x1B, 0x24, 0x29, 0x43,
243 0x0E, 0x6C, 0x69,
244 0x0f, 0x1A,
245 0x0e, 0x6F, 0x4B,
246 0x0F, 0x31,
247 0x1A,
248 0x32 };
249
250 static const int32_t offsets[] = {-1, -1, -1, -1,
251 0, 0, 0,
252 1, 1,
253 3, 3, 3,
254 4, 4,
255 5,
256 7,
257 };
258
259 /*iso-2022-kr*/
260 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
261 expected, sizeof(expected), "iso-2022-kr", 0 , TRUE, U_ZERO_ERROR))
262 log_err("u-> iso-2022-kr [UCNV_DBCS] not match.\n");
263 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
264 expected, sizeof(expected), "iso-2022-kr", offsets , TRUE, U_ZERO_ERROR))
265 log_err("u-> iso-2022-kr [UCNV_DBCS] not match.\n");
266 }
267
268 log_verbose("Testing for HZ\n");
269 {
270 static const UChar sampleText[] = { 0x4e00, 0xd801, 0xdc01, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
271
272 static const uint8_t expected[] = {0x7E, 0x7B, 0x52, 0x3B,
273 0x7E, 0x7D, 0x1A,
274 0x7E, 0x7B, 0x36, 0x21,
275 0x7E, 0x7D, 0x31,
276 0x1A,
277 0x32 };
278
279
280 static const int32_t offsets[] = {0,0,0,0,
281 1,1,1,
282 3,3,3,3,
283 4,4,4,
284 5,
285 7,};
286
287 /*hz*/
288 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
289 expected, sizeof(expected), "HZ", 0 , TRUE, U_ZERO_ERROR))
290 log_err("u-> HZ not match.\n");
291 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
292 expected, sizeof(expected), "HZ", offsets , TRUE, U_ZERO_ERROR))
293 log_err("u-> HZ not match.\n");
294 }
295 #endif
296
297 /*UTF-8*/
298 log_verbose("Testing for UTF8\n");
299 {
300 static const UChar sampleText[] = { 0x4e00, 0x0701, 0x0031, 0xbfc1, 0xd801, 0xdc01, 0x0032};
301 static const int32_t offsets[]={0x00, 0x00, 0x00, 0x01, 0x01, 0x02,
302 0x03, 0x03, 0x03, 0x04, 0x04, 0x04,
303 0x04, 0x06 };
304 static const uint8_t expected[] = {0xe4, 0xb8, 0x80, 0xdc, 0x81, 0x31,
305 0xeb, 0xbf, 0x81, 0xF0, 0x90, 0x90, 0x81, 0x32};
306
307
308 static const int32_t fromOffsets[] = { 0x0000, 0x0003, 0x0005, 0x0006, 0x0009, 0x0009, 0x000D };
309 /*UTF-8*/
310 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
311 expected, sizeof(expected), "UTF8", offsets, TRUE, U_ZERO_ERROR ))
312 log_err("u-> UTF8 with offsets and flush true did not match.\n");
313 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
314 expected, sizeof(expected), "UTF8", 0, TRUE, U_ZERO_ERROR ))
315 log_err("u-> UTF8 with offsets and flush true did not match.\n");
316 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
317 expected, sizeof(expected), "UTF8", offsets, FALSE, U_ZERO_ERROR ))
318 log_err("u-> UTF8 with offsets and flush true did not match.\n");
319 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
320 expected, sizeof(expected), "UTF8", 0, FALSE, U_ZERO_ERROR ))
321 log_err("u-> UTF8 with offsets and flush true did not match.\n");
322
323 if(!convertToU(expected, sizeof(expected),
324 sampleText, UPRV_LENGTHOF(sampleText), "UTF8", 0, TRUE, U_ZERO_ERROR ))
325 log_err("UTF8 -> u did not match.\n");
326 if(!convertToU(expected, sizeof(expected),
327 sampleText, UPRV_LENGTHOF(sampleText), "UTF8", 0, FALSE, U_ZERO_ERROR ))
328 log_err("UTF8 -> u did not match.\n");
329 if(!convertToU(expected, sizeof(expected),
330 sampleText, UPRV_LENGTHOF(sampleText), "UTF8", fromOffsets, TRUE, U_ZERO_ERROR ))
331 log_err("UTF8 ->u did not match.\n");
332 if(!convertToU(expected, sizeof(expected),
333 sampleText, UPRV_LENGTHOF(sampleText), "UTF8", fromOffsets, FALSE, U_ZERO_ERROR ))
334 log_err("UTF8 -> u did not match.\n");
335
336 }
337 }
338
339 /*test various error behaviours*/
TestErrorBehaviour()340 static void TestErrorBehaviour(){
341 log_verbose("Testing for SBCS and LATIN_1\n");
342 {
343 static const UChar sampleText[] = { 0x0031, 0xd801};
344 static const UChar sampleText2[] = { 0x0031, 0xd801, 0x0032};
345 static const uint8_t expected0[] = { 0x31};
346 static const uint8_t expected[] = { 0x31, 0x1a};
347 static const uint8_t expected2[] = { 0x31, 0x1a, 0x32};
348
349 #if !UCONFIG_NO_LEGACY_CONVERSION
350 /*SBCS*/
351 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
352 expected, sizeof(expected), "ibm-920", 0, TRUE, U_ZERO_ERROR))
353 log_err("u-> ibm-920 [UCNV_SBCS] \n");
354 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
355 expected0, sizeof(expected0), "ibm-920", 0, FALSE, U_ZERO_ERROR))
356 log_err("u-> ibm-920 [UCNV_SBCS] \n");
357 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
358 expected2, sizeof(expected2), "ibm-920", 0, TRUE, U_ZERO_ERROR))
359 log_err("u-> ibm-920 [UCNV_SBCS] did not match\n");
360 #endif
361
362 /*LATIN_1*/
363 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
364 expected, sizeof(expected), "LATIN_1", 0, TRUE, U_ZERO_ERROR))
365 log_err("u-> LATIN_1 is supposed to fail\n");
366 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
367 expected0, sizeof(expected0), "LATIN_1", 0, FALSE, U_ZERO_ERROR))
368 log_err("u-> LATIN_1 is supposed to fail\n");
369
370 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
371 expected2, sizeof(expected2), "LATIN_1", 0, TRUE, U_ZERO_ERROR))
372 log_err("u-> LATIN_1 did not match\n");
373 }
374
375 #if !UCONFIG_NO_LEGACY_CONVERSION
376 log_verbose("Testing for DBCS and MBCS\n");
377 {
378 static const UChar sampleText[] = { 0x00a1, 0xd801};
379 static const uint8_t expected[] = { 0xa2, 0xae};
380 static const int32_t offsets[] = { 0x00, 0x00};
381 static const uint8_t expectedSUB[] = { 0xa2, 0xae, 0xa1, 0xe0};
382 static const int32_t offsetsSUB[] = { 0x00, 0x00, 0x01, 0x01};
383
384 static const UChar sampleText2[] = { 0x00a1, 0xd801, 0x00a4};
385 static const uint8_t expected2[] = { 0xa2, 0xae, 0xa1, 0xe0, 0xa2, 0xb4};
386 static const int32_t offsets2[] = { 0x00, 0x00, 0x01, 0x01, 0x02, 0x02};
387
388 static const UChar sampleText3MBCS[] = { 0x0001, 0x00a4, 0xdc01};
389 static const uint8_t expected3MBCS[] = { 0x01, 0xa2, 0xb4, 0xa1, 0xe0};
390 static const int32_t offsets3MBCS[] = { 0x00, 0x01, 0x01, 0x02, 0x02};
391
392 static const UChar sampleText4MBCS[] = { 0x0061, 0xFFE4, 0xdc01};
393 static const uint8_t expected4MBCS[] = { 0x61, 0x8f, 0xa2, 0xc3, 0xf4, 0xfe};
394 static const int32_t offsets4MBCS[] = { 0x00, 0x01, 0x01, 0x01, 0x02, 0x02 };
395
396 /*DBCS*/
397 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
398 expectedSUB, sizeof(expectedSUB), "ibm-1363", 0, TRUE, U_ZERO_ERROR))
399 log_err("u-> ibm-1363 [UCNV_DBCS portion] is supposed to fail\n");
400 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
401 expected, sizeof(expected), "ibm-1363", 0, FALSE, U_AMBIGUOUS_ALIAS_WARNING))
402 log_err("u-> ibm-1363 [UCNV_DBCS portion] is supposed to fail\n");
403
404 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
405 expectedSUB, sizeof(expectedSUB), "ibm-1363", offsetsSUB, TRUE, U_ZERO_ERROR))
406 log_err("u-> ibm-1363 [UCNV_DBCS portion] is supposed to fail\n");
407 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
408 expected, sizeof(expected), "ibm-1363", offsets, FALSE, U_AMBIGUOUS_ALIAS_WARNING))
409 log_err("u-> ibm-1363 [UCNV_DBCS portion] is supposed to fail\n");
410
411
412 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
413 expected2, sizeof(expected2), "ibm-1363", 0, TRUE, U_ZERO_ERROR))
414 log_err("u-> ibm-1363 [UCNV_DBCS portion] did not match \n");
415 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
416 expected2, sizeof(expected2), "ibm-1363", offsets2, TRUE, U_ZERO_ERROR))
417 log_err("u-> ibm-1363 [UCNV_DBCS portion] did not match \n");
418
419 /*MBCS*/
420 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
421 expectedSUB, sizeof(expectedSUB), "ibm-1363", 0, TRUE, U_ZERO_ERROR))
422 log_err("u-> ibm-1363 [UCNV_MBCS] \n");
423 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
424 expected, sizeof(expected), "ibm-1363", 0, FALSE, U_AMBIGUOUS_ALIAS_WARNING))
425 log_err("u-> ibm-1363 [UCNV_MBCS] \n");
426
427 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
428 expected2, sizeof(expected2), "ibm-1363", 0, TRUE, U_ZERO_ERROR))
429 log_err("u-> ibm-1363 [UCNV_DBCS] did not match\n");
430 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
431 expected2, sizeof(expected2), "ibm-1363", 0, FALSE, U_ZERO_ERROR))
432 log_err("u-> ibm-1363 [UCNV_DBCS] did not match\n");
433 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
434 expected2, sizeof(expected2), "ibm-1363", offsets2, FALSE, U_ZERO_ERROR))
435 log_err("u-> ibm-1363 [UCNV_DBCS] did not match\n");
436
437 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
438 expected3MBCS, sizeof(expected3MBCS), "ibm-1363", offsets3MBCS, TRUE, U_ZERO_ERROR))
439 log_err("u-> ibm-1363 [UCNV_MBCS] \n");
440 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
441 expected3MBCS, sizeof(expected3MBCS), "ibm-1363", offsets3MBCS, FALSE, U_ZERO_ERROR))
442 log_err("u-> ibm-1363 [UCNV_MBCS] \n");
443
444 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
445 expected4MBCS, sizeof(expected4MBCS), "IBM-eucJP", offsets4MBCS, TRUE, U_ZERO_ERROR))
446 log_err("u-> euc-jp [UCNV_MBCS] \n");
447 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
448 expected4MBCS, sizeof(expected4MBCS), "IBM-eucJP", offsets4MBCS, FALSE, U_ZERO_ERROR))
449 log_err("u-> euc-jp [UCNV_MBCS] \n");
450 }
451
452 /*iso-2022-jp*/
453 log_verbose("Testing for iso-2022-jp\n");
454 {
455 static const UChar sampleText[] = { 0x0031, 0xd801};
456 static const uint8_t expected[] = { 0x31};
457 static const uint8_t expectedSUB[] = { 0x31, 0x1a};
458 static const int32_t offsets[] = { 0x00, 1};
459
460 static const UChar sampleText2[] = { 0x0031, 0xd801, 0x0032};
461 static const uint8_t expected2[] = { 0x31,0x1A,0x32};
462 static const int32_t offsets2[] = { 0x00,0x01,0x02};
463
464 static const UChar sampleText4MBCS[] = { 0x0061, 0x4e00, 0xdc01};
465 static const uint8_t expected4MBCS[] = { 0x61, 0x1b, 0x24, 0x42, 0x30, 0x6c,0x1b,0x28,0x42,0x1a};
466 static const int32_t offsets4MBCS[] = { 0x00, 0x01, 0x01 ,0x01, 0x01, 0x01,0x02,0x02,0x02,0x02 };
467 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
468 expectedSUB, sizeof(expectedSUB), "iso-2022-jp", offsets, TRUE, U_ZERO_ERROR))
469 log_err("u-> iso-2022-jp [UCNV_MBCS] \n");
470 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
471 expected, sizeof(expected), "iso-2022-jp", offsets, FALSE, U_AMBIGUOUS_ALIAS_WARNING))
472 log_err("u-> iso-2022-jp [UCNV_MBCS] \n");
473
474 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
475 expected2, sizeof(expected2), "iso-2022-jp", offsets2, TRUE, U_ZERO_ERROR))
476 log_err("u->iso-2022-jp[UCNV_DBCS] did not match\n");
477 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
478 expected2, sizeof(expected2), "iso-2022-jp", offsets2, FALSE, U_ZERO_ERROR))
479 log_err("u-> iso-2022-jp [UCNV_DBCS] did not match\n");
480 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
481 expected2, sizeof(expected2), "iso-2022-jp", offsets2, FALSE, U_ZERO_ERROR))
482 log_err("u-> iso-2022-jp [UCNV_DBCS] did not match\n");
483
484 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
485 expected4MBCS, sizeof(expected4MBCS), "iso-2022-jp", offsets4MBCS, TRUE, U_ZERO_ERROR))
486 log_err("u-> iso-2022-jp [UCNV_MBCS] \n");
487 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
488 expected4MBCS, sizeof(expected4MBCS), "iso-2022-jp", offsets4MBCS, FALSE, U_ZERO_ERROR))
489 log_err("u-> iso-2022-jp [UCNV_MBCS] \n");
490 }
491
492 /* BEGIN android-removed */
493 /* To save space, Android does not build full ISO-2022-CN tables.
494 We skip the tests for ISO-2022-CN. */
495 /*iso-2022-cn*/
496 /*
497 log_verbose("Testing for iso-2022-cn\n");
498 {
499 static const UChar sampleText[] = { 0x0031, 0xd801};
500 static const uint8_t expected[] = { 0x31};
501 static const uint8_t expectedSUB[] = { 0x31, 0x1A};
502 static const int32_t offsets[] = { 0x00, 1};
503
504 static const UChar sampleText2[] = { 0x0031, 0xd801, 0x0032};
505 static const uint8_t expected2[] = { 0x31, 0x1A,0x32};
506 static const int32_t offsets2[] = { 0x00, 0x01,0x02};
507
508 static const UChar sampleText3MBCS[] = { 0x0051, 0x0050, 0xdc01};
509 static const uint8_t expected3MBCS[] = {0x51, 0x50, 0x1A};
510 static const int32_t offsets3MBCS[] = { 0x00, 0x01, 0x02 };
511
512 static const UChar sampleText4MBCS[] = { 0x0061, 0x4e00, 0xdc01};
513 static const uint8_t expected4MBCS[] = { 0x61, 0x1b, 0x24, 0x29, 0x41, 0x0e, 0x52, 0x3b, 0x0f, 0x1a };
514 static const int32_t offsets4MBCS[] = { 0x00, 0x01, 0x01 ,0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x02 };
515 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
516 expectedSUB, sizeof(expectedSUB), "iso-2022-cn", offsets, TRUE, U_ZERO_ERROR))
517 log_err("u-> iso-2022-cn [UCNV_MBCS] \n");
518 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
519 expected, sizeof(expected), "iso-2022-cn", offsets, FALSE, U_ZERO_ERROR))
520 log_err("u-> ibm-1363 [UCNV_MBCS] \n");
521
522 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
523 expected2, sizeof(expected2), "iso-2022-cn", offsets2, TRUE, U_ZERO_ERROR))
524 log_err("u->iso-2022-cn[UCNV_DBCS] did not match\n");
525 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
526 expected2, sizeof(expected2), "iso-2022-cn", offsets2, FALSE, U_ZERO_ERROR))
527 log_err("u-> iso-2022-cn [UCNV_DBCS] did not match\n");
528 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
529 expected2, sizeof(expected2), "iso-2022-cn", offsets2, FALSE, U_ZERO_ERROR))
530 log_err("u-> iso-2022-cn [UCNV_DBCS] did not match\n");
531
532 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
533 expected3MBCS, sizeof(expected3MBCS), "iso-2022-cn", offsets3MBCS, TRUE, U_ZERO_ERROR))
534 log_err("u->iso-2022-cn [UCNV_MBCS] \n");
535 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
536 expected3MBCS, sizeof(expected3MBCS), "iso-2022-cn", offsets3MBCS, FALSE, U_ZERO_ERROR))
537 log_err("u-> iso-2022-cn[UCNV_MBCS] \n");
538
539 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
540 expected4MBCS, sizeof(expected4MBCS), "iso-2022-cn", offsets4MBCS, TRUE, U_ZERO_ERROR))
541 log_err("u-> iso-2022-cn [UCNV_MBCS] \n");
542 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
543 expected4MBCS, sizeof(expected4MBCS), "iso-2022-cn", offsets4MBCS, FALSE, U_ZERO_ERROR))
544 log_err("u-> iso-2022-cn [UCNV_MBCS] \n");
545 }
546 */
547 /* END android-removed */
548
549 /*iso-2022-kr*/
550 log_verbose("Testing for iso-2022-kr\n");
551 {
552 static const UChar sampleText[] = { 0x0031, 0xd801};
553 static const uint8_t expected[] = { 0x1b, 0x24, 0x29, 0x43, 0x31};
554 static const uint8_t expectedSUB[] = { 0x1b, 0x24, 0x29, 0x43, 0x31, 0x1A};
555 static const int32_t offsets[] = { -1, -1, -1, -1, 0x00, 1};
556
557 static const UChar sampleText2[] = { 0x0031, 0xd801, 0x0032};
558 static const uint8_t expected2[] = { 0x1b, 0x24, 0x29, 0x43, 0x31, 0x1A, 0x32};
559 static const int32_t offsets2[] = { -1, -1, -1, -1, 0x00, 0x01, 0x02};
560
561 static const UChar sampleText3MBCS[] = { 0x0051, 0x0050, 0xdc01};
562 static const uint8_t expected3MBCS[] = { 0x1b, 0x24, 0x29, 0x43, 0x51, 0x50, 0x1A };
563 static const int32_t offsets3MBCS[] = { -1, -1, -1, -1, 0x00, 0x01, 0x02, 0x02 };
564
565 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
566 expectedSUB, sizeof(expectedSUB), "iso-2022-kr", offsets, TRUE, U_ZERO_ERROR))
567 log_err("u-> iso-2022-kr [UCNV_MBCS] \n");
568 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
569 expected, sizeof(expected), "iso-2022-kr", offsets, FALSE, U_ZERO_ERROR))
570 log_err("u-> ibm-1363 [UCNV_MBCS] \n");
571
572 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
573 expected2, sizeof(expected2), "iso-2022-kr", offsets2, TRUE, U_ZERO_ERROR))
574 log_err("u->iso-2022-kr[UCNV_DBCS] did not match\n");
575 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
576 expected2, sizeof(expected2), "iso-2022-kr", offsets2, FALSE, U_ZERO_ERROR))
577 log_err("u-> iso-2022-kr [UCNV_DBCS] did not match\n");
578 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
579 expected2, sizeof(expected2), "iso-2022-kr", offsets2, FALSE, U_ZERO_ERROR))
580 log_err("u-> iso-2022-kr [UCNV_DBCS] did not match\n");
581
582 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
583 expected3MBCS, sizeof(expected3MBCS), "iso-2022-kr", offsets3MBCS, TRUE, U_ZERO_ERROR))
584 log_err("u->iso-2022-kr [UCNV_MBCS] \n");
585 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
586 expected3MBCS, sizeof(expected3MBCS), "iso-2022-kr", offsets3MBCS, FALSE, U_ZERO_ERROR))
587 log_err("u-> iso-2022-kr[UCNV_MBCS] \n");
588 }
589
590 /*HZ*/
591 log_verbose("Testing for HZ\n");
592 {
593 static const UChar sampleText[] = { 0x0031, 0xd801};
594 static const uint8_t expected[] = { 0x7e, 0x7d, 0x31};
595 static const uint8_t expectedSUB[] = { 0x7e, 0x7d, 0x31, 0x1A};
596 static const int32_t offsets[] = { 0x00, 0x00, 0x00, 1};
597
598 static const UChar sampleText2[] = { 0x0031, 0xd801, 0x0032};
599 static const uint8_t expected2[] = { 0x7e, 0x7d, 0x31, 0x1A, 0x32 };
600 static const int32_t offsets2[] = { 0x00, 0x00, 0x00, 0x01, 0x02 };
601
602 static const UChar sampleText3MBCS[] = { 0x0051, 0x0050, 0xdc01};
603 static const uint8_t expected3MBCS[] = { 0x7e, 0x7d, 0x51, 0x50, 0x1A };
604 static const int32_t offsets3MBCS[] = { 0x00, 0x00, 0x00, 0x01, 0x02};
605
606 static const UChar sampleText4MBCS[] = { 0x0061, 0x4e00, 0xdc01};
607 static const uint8_t expected4MBCS[] = { 0x7e, 0x7d, 0x61, 0x7e, 0x7b, 0x52, 0x3b, 0x7e, 0x7d, 0x1a };
608 static const int32_t offsets4MBCS[] = { 0x00, 0x00, 0x00, 0x01, 0x01, 0x01 ,0x01, 0x02, 0x02, 0x02 };
609 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
610 expectedSUB, sizeof(expectedSUB), "HZ", offsets, TRUE, U_ZERO_ERROR))
611 log_err("u-> HZ [UCNV_MBCS] \n");
612 if(!convertFromU(sampleText, UPRV_LENGTHOF(sampleText),
613 expected, sizeof(expected), "HZ", offsets, FALSE, U_ZERO_ERROR))
614 log_err("u-> ibm-1363 [UCNV_MBCS] \n");
615
616 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
617 expected2, sizeof(expected2), "HZ", offsets2, TRUE, U_ZERO_ERROR))
618 log_err("u->HZ[UCNV_DBCS] did not match\n");
619 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
620 expected2, sizeof(expected2), "HZ", offsets2, FALSE, U_ZERO_ERROR))
621 log_err("u-> HZ [UCNV_DBCS] did not match\n");
622 if(!convertFromU(sampleText2, UPRV_LENGTHOF(sampleText2),
623 expected2, sizeof(expected2), "HZ", offsets2, FALSE, U_ZERO_ERROR))
624 log_err("u-> HZ [UCNV_DBCS] did not match\n");
625
626 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
627 expected3MBCS, sizeof(expected3MBCS), "HZ", offsets3MBCS, TRUE, U_ZERO_ERROR))
628 log_err("u->HZ [UCNV_MBCS] \n");
629 if(!convertFromU(sampleText3MBCS, UPRV_LENGTHOF(sampleText3MBCS),
630 expected3MBCS, sizeof(expected3MBCS), "HZ", offsets3MBCS, FALSE, U_ZERO_ERROR))
631 log_err("u-> HZ[UCNV_MBCS] \n");
632
633 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
634 expected4MBCS, sizeof(expected4MBCS), "HZ", offsets4MBCS, TRUE, U_ZERO_ERROR))
635 log_err("u-> HZ [UCNV_MBCS] \n");
636 if(!convertFromU(sampleText4MBCS, UPRV_LENGTHOF(sampleText4MBCS),
637 expected4MBCS, sizeof(expected4MBCS), "HZ", offsets4MBCS, FALSE, U_ZERO_ERROR))
638 log_err("u-> HZ [UCNV_MBCS] \n");
639 }
640 #endif
641 }
642
643 #if !UCONFIG_NO_LEGACY_CONVERSION
644 /*test different convertToUnicode error behaviours*/
TestToUnicodeErrorBehaviour()645 static void TestToUnicodeErrorBehaviour()
646 {
647 log_verbose("Testing error conditions for DBCS\n");
648 {
649 uint8_t sampleText[] = { 0xa2, 0xae, 0x03, 0x04};
650 const UChar expected[] = { 0x00a1 };
651
652 if(!convertToU(sampleText, sizeof(sampleText),
653 expected, UPRV_LENGTHOF(expected), "ibm-1363", 0, TRUE, U_AMBIGUOUS_ALIAS_WARNING ))
654 log_err("DBCS (ibm-1363)->Unicode did not match.\n");
655 if(!convertToU(sampleText, sizeof(sampleText),
656 expected, UPRV_LENGTHOF(expected), "ibm-1363", 0, FALSE, U_AMBIGUOUS_ALIAS_WARNING ))
657 log_err("DBCS (ibm-1363)->Unicode with flush = false did not match.\n");
658 }
659 log_verbose("Testing error conditions for SBCS\n");
660 {
661 uint8_t sampleText[] = { 0xa2, 0xFF};
662 const UChar expected[] = { 0x00c2 };
663
664 /* uint8_t sampleText2[] = { 0xa2, 0x70 };
665 const UChar expected2[] = { 0x0073 };*/
666
667 if(!convertToU(sampleText, sizeof(sampleText),
668 expected, UPRV_LENGTHOF(expected), "ibm-1051", 0, TRUE, U_ZERO_ERROR ))
669 log_err("SBCS (ibm-1051)->Unicode did not match.\n");
670 if(!convertToU(sampleText, sizeof(sampleText),
671 expected, UPRV_LENGTHOF(expected), "ibm-1051", 0, FALSE, U_ZERO_ERROR ))
672 log_err("SBCS (ibm-1051)->Unicode with flush = false did not match.\n");
673
674 }
675 }
676
TestGetNextErrorBehaviour()677 static void TestGetNextErrorBehaviour(){
678 /*Test for unassigned character*/
679 #define INPUT_SIZE 1
680 static const char input1[INPUT_SIZE]={ 0x70 };
681 const char* source=(const char*)input1;
682 UErrorCode err=U_ZERO_ERROR;
683 UChar32 c=0;
684 UConverter *cnv=ucnv_open("ibm-424", &err);
685 if(U_FAILURE(err)) {
686 log_data_err("Unable to open a SBCS(ibm-424) converter: %s\n", u_errorName(err));
687 return;
688 }
689 c=ucnv_getNextUChar(cnv, &source, source + INPUT_SIZE, &err);
690 if(err != U_INVALID_CHAR_FOUND && c!=0xfffd){
691 log_err("FAIL in TestGetNextErrorBehaviour(unassigned): Expected: U_INVALID_CHAR_ERROR or 0xfffd ----Got:%s and 0x%lx\n", myErrorName(err), c);
692 }
693 ucnv_close(cnv);
694 }
695 #endif
696
697 #define MAX_UTF16_LEN 2
698 #define MAX_UTF8_LEN 4
699
700 /*Regression test for utf8 converter*/
TestRegressionUTF8()701 static void TestRegressionUTF8(){
702 UChar32 currCh = 0;
703 int32_t offset8;
704 int32_t offset16;
705 UChar *standardForm = (UChar*)malloc(MAX_LENGTH*sizeof(UChar));
706 uint8_t *utf8 = (uint8_t*)malloc(MAX_LENGTH);
707
708 while (currCh <= UNICODE_LIMIT) {
709 offset16 = 0;
710 offset8 = 0;
711 while(currCh <= UNICODE_LIMIT
712 && offset16 < (MAX_LENGTH/sizeof(UChar) - MAX_UTF16_LEN)
713 && offset8 < (MAX_LENGTH - MAX_UTF8_LEN))
714 {
715 if (currCh == SURROGATE_HIGH_START) {
716 currCh = SURROGATE_LOW_END + 1; /* Skip surrogate range */
717 }
718 UTF16_APPEND_CHAR_SAFE(standardForm, offset16, MAX_LENGTH, currCh);
719 UTF8_APPEND_CHAR_SAFE(utf8, offset8, MAX_LENGTH, currCh);
720 currCh++;
721 }
722 if(!convertFromU(standardForm, offset16,
723 utf8, offset8, "UTF8", 0, TRUE, U_ZERO_ERROR )) {
724 log_err("Unicode->UTF8 did not match.\n");
725 }
726 if(!convertToU(utf8, offset8,
727 standardForm, offset16, "UTF8", 0, TRUE, U_ZERO_ERROR )) {
728 log_err("UTF8->Unicode did not match.\n");
729 }
730 }
731
732 free(standardForm);
733 free(utf8);
734
735 {
736 static const char src8[] = { (char)0xCC, (char)0x81, (char)0xCC, (char)0x80 };
737 static const UChar expected[] = { 0x0301, 0x0300 };
738 UConverter *conv8;
739 UErrorCode err = U_ZERO_ERROR;
740 UChar pivotBuffer[100];
741 const UChar* const pivEnd = pivotBuffer + 100;
742 const char* srcBeg;
743 const char* srcEnd;
744 UChar* pivBeg;
745
746 conv8 = ucnv_open("UTF-8", &err);
747
748 srcBeg = src8;
749 pivBeg = pivotBuffer;
750 srcEnd = src8 + 3;
751 ucnv_toUnicode(conv8, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, FALSE, &err);
752 if (srcBeg != srcEnd) {
753 log_err("Did not consume whole buffer on first call.\n");
754 }
755
756 srcEnd = src8 + 4;
757 ucnv_toUnicode(conv8, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, TRUE, &err);
758 if (srcBeg != srcEnd) {
759 log_err("Did not consume whole buffer on second call.\n");
760 }
761
762 if (U_FAILURE(err) || (int32_t)(pivBeg - pivotBuffer) != 2 || u_strncmp(pivotBuffer, expected, 2) != 0) {
763 log_err("Did not get expected results for UTF-8.\n");
764 }
765 ucnv_close(conv8);
766 }
767 }
768
769 #define MAX_UTF32_LEN 1
770
TestRegressionUTF32()771 static void TestRegressionUTF32(){
772 #if !UCONFIG_ONLY_HTML_CONVERSION
773 UChar32 currCh = 0;
774 int32_t offset32;
775 int32_t offset16;
776 UChar *standardForm = (UChar*)malloc(MAX_LENGTH*sizeof(UChar));
777 UChar32 *utf32 = (UChar32*)malloc(MAX_LENGTH*sizeof(UChar32));
778
779 while (currCh <= UNICODE_LIMIT) {
780 offset16 = 0;
781 offset32 = 0;
782 while(currCh <= UNICODE_LIMIT
783 && offset16 < (MAX_LENGTH/sizeof(UChar) - MAX_UTF16_LEN)
784 && offset32 < (MAX_LENGTH/sizeof(UChar32) - MAX_UTF32_LEN))
785 {
786 if (currCh == SURROGATE_HIGH_START) {
787 currCh = SURROGATE_LOW_END + 1; /* Skip surrogate range */
788 }
789 UTF16_APPEND_CHAR_SAFE(standardForm, offset16, MAX_LENGTH, currCh);
790 UTF32_APPEND_CHAR_SAFE(utf32, offset32, MAX_LENGTH, currCh);
791 currCh++;
792 }
793 if(!convertFromU(standardForm, offset16,
794 (const uint8_t *)utf32, offset32*sizeof(UChar32), "UTF32_PlatformEndian", 0, TRUE, U_ZERO_ERROR )) {
795 log_err("Unicode->UTF32 did not match.\n");
796 }
797 if(!convertToU((const uint8_t *)utf32, offset32*sizeof(UChar32),
798 standardForm, offset16, "UTF32_PlatformEndian", 0, TRUE, U_ZERO_ERROR )) {
799 log_err("UTF32->Unicode did not match.\n");
800 }
801 }
802 free(standardForm);
803 free(utf32);
804
805 {
806 /* Check for lone surrogate error handling. */
807 static const UChar sampleBadStartSurrogate[] = { 0x0031, 0xD800, 0x0032 };
808 static const UChar sampleBadEndSurrogate[] = { 0x0031, 0xDC00, 0x0032 };
809 static const uint8_t expectedUTF32BE[] = {
810 0x00, 0x00, 0x00, 0x31,
811 0x00, 0x00, 0xff, 0xfd,
812 0x00, 0x00, 0x00, 0x32
813 };
814 static const uint8_t expectedUTF32LE[] = {
815 0x31, 0x00, 0x00, 0x00,
816 0xfd, 0xff, 0x00, 0x00,
817 0x32, 0x00, 0x00, 0x00
818 };
819 static const int32_t offsetsUTF32[] = {
820 0x00, 0x00, 0x00, 0x00,
821 0x01, 0x01, 0x01, 0x01,
822 0x02, 0x02, 0x02, 0x02
823 };
824
825 if(!convertFromU(sampleBadStartSurrogate, UPRV_LENGTHOF(sampleBadStartSurrogate),
826 expectedUTF32BE, sizeof(expectedUTF32BE), "UTF-32BE", offsetsUTF32, TRUE, U_ZERO_ERROR))
827 log_err("u->UTF-32BE\n");
828 if(!convertFromU(sampleBadEndSurrogate, UPRV_LENGTHOF(sampleBadEndSurrogate),
829 expectedUTF32BE, sizeof(expectedUTF32BE), "UTF-32BE", offsetsUTF32, TRUE, U_ZERO_ERROR))
830 log_err("u->UTF-32BE\n");
831
832 if(!convertFromU(sampleBadStartSurrogate, UPRV_LENGTHOF(sampleBadStartSurrogate),
833 expectedUTF32LE, sizeof(expectedUTF32LE), "UTF-32LE", offsetsUTF32, TRUE, U_ZERO_ERROR))
834 log_err("u->UTF-32LE\n");
835 if(!convertFromU(sampleBadEndSurrogate, UPRV_LENGTHOF(sampleBadEndSurrogate),
836 expectedUTF32LE, sizeof(expectedUTF32LE), "UTF-32LE", offsetsUTF32, TRUE, U_ZERO_ERROR))
837 log_err("u->UTF-32LE\n");
838 }
839
840 {
841 static const char srcBE[] = { 0, 0, 0, 0x31, 0, 0, 0, 0x30 };
842 static const UChar expected[] = { 0x0031, 0x0030 };
843 UConverter *convBE;
844 UErrorCode err = U_ZERO_ERROR;
845 UChar pivotBuffer[100];
846 const UChar* const pivEnd = pivotBuffer + 100;
847 const char* srcBeg;
848 const char* srcEnd;
849 UChar* pivBeg;
850
851 convBE = ucnv_open("UTF-32BE", &err);
852
853 srcBeg = srcBE;
854 pivBeg = pivotBuffer;
855 srcEnd = srcBE + 5;
856 ucnv_toUnicode(convBE, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, FALSE, &err);
857 if (srcBeg != srcEnd) {
858 log_err("Did not consume whole buffer on first call.\n");
859 }
860
861 srcEnd = srcBE + 8;
862 ucnv_toUnicode(convBE, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, TRUE, &err);
863 if (srcBeg != srcEnd) {
864 log_err("Did not consume whole buffer on second call.\n");
865 }
866
867 if (U_FAILURE(err) || (int32_t)(pivBeg - pivotBuffer) != 2 || u_strncmp(pivotBuffer, expected, 2) != 0) {
868 log_err("Did not get expected results for UTF-32BE.\n");
869 }
870 ucnv_close(convBE);
871 }
872 {
873 static const char srcLE[] = { 0x31, 0, 0, 0, 0x30, 0, 0, 0 };
874 static const UChar expected[] = { 0x0031, 0x0030 };
875 UConverter *convLE;
876 UErrorCode err = U_ZERO_ERROR;
877 UChar pivotBuffer[100];
878 const UChar* const pivEnd = pivotBuffer + 100;
879 const char* srcBeg;
880 const char* srcEnd;
881 UChar* pivBeg;
882
883 convLE = ucnv_open("UTF-32LE", &err);
884
885 srcBeg = srcLE;
886 pivBeg = pivotBuffer;
887 srcEnd = srcLE + 5;
888 ucnv_toUnicode(convLE, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, FALSE, &err);
889 if (srcBeg != srcEnd) {
890 log_err("Did not consume whole buffer on first call.\n");
891 }
892
893 srcEnd = srcLE + 8;
894 ucnv_toUnicode(convLE, &pivBeg, pivEnd, &srcBeg, srcEnd, 0, TRUE, &err);
895 if (srcBeg != srcEnd) {
896 log_err("Did not consume whole buffer on second call.\n");
897 }
898
899 if (U_FAILURE(err) || (int32_t)(pivBeg - pivotBuffer) != 2 || u_strncmp(pivotBuffer, expected, 2) != 0) {
900 log_err("Did not get expected results for UTF-32LE.\n");
901 }
902 ucnv_close(convLE);
903 }
904 #endif
905 }
906
907 /*Walk through the available converters*/
TestAvailableConverters()908 static void TestAvailableConverters(){
909 UErrorCode status=U_ZERO_ERROR;
910 UConverter *conv=NULL;
911 int32_t i=0;
912 for(i=0; i < ucnv_countAvailable(); i++){
913 status=U_ZERO_ERROR;
914 conv=ucnv_open(ucnv_getAvailableName(i), &status);
915 if(U_FAILURE(status)){
916 log_err("ERROR: converter creation failed. Failure in alias table or the data table for \n converter=%s. Error=%s\n",
917 ucnv_getAvailableName(i), myErrorName(status));
918 continue;
919 }
920 ucnv_close(conv);
921 }
922
923 }
924
TestFlushInternalBuffer()925 static void TestFlushInternalBuffer(){
926 TestWithBufferSize(MAX_LENGTH, 1);
927 TestWithBufferSize(1, 1);
928 TestWithBufferSize(1, MAX_LENGTH);
929 TestWithBufferSize(MAX_LENGTH, MAX_LENGTH);
930 }
931
TestWithBufferSize(int32_t insize,int32_t outsize)932 static void TestWithBufferSize(int32_t insize, int32_t outsize){
933
934 gInBufferSize =insize;
935 gOutBufferSize = outsize;
936
937 log_verbose("Testing fromUnicode for UTF-8 with UCNV_TO_U_CALLBACK_SUBSTITUTE \n");
938 {
939 UChar sampleText[] =
940 { 0x0031, 0x0032, 0x0033, 0x0000, 0x4e00, 0x4e8c, 0x4e09, 0x002E };
941 const uint8_t expectedUTF8[] =
942 { 0x31, 0x32, 0x33, 0x00, 0xe4, 0xb8, 0x80, 0xe4, 0xba, 0x8c, 0xe4, 0xb8, 0x89, 0x2E };
943 int32_t toUTF8Offs[] =
944 { 0x00, 0x01, 0x02, 0x03, 0x04, 0x04, 0x04, 0x05, 0x05, 0x05, 0x06, 0x06, 0x06, 0x07};
945 /* int32_t fmUTF8Offs[] =
946 { 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0007, 0x000a, 0x000d };*/
947
948 /*UTF-8*/
949 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
950 expectedUTF8, sizeof(expectedUTF8), "UTF8", UCNV_FROM_U_CALLBACK_SUBSTITUTE, toUTF8Offs ,FALSE))
951 log_err("u-> UTF8 did not match.\n");
952 }
953
954 #if !UCONFIG_NO_LEGACY_CONVERSION
955 log_verbose("Testing fromUnicode with UCNV_FROM_U_CALLBACK_ESCAPE \n");
956 {
957 UChar inputTest[] = { 0x0061, 0xd801, 0xdc01, 0xd801, 0x0061 };
958 const uint8_t toIBM943[]= { 0x61,
959 0x25, 0x55, 0x44, 0x38, 0x30, 0x31,
960 0x25, 0x55, 0x44, 0x43, 0x30, 0x31,
961 0x25, 0x55, 0x44, 0x38, 0x30, 0x31,
962 0x61 };
963 int32_t offset[]= {0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 3, 3, 3, 3, 4};
964
965 if(!testConvertFromU(inputTest, UPRV_LENGTHOF(inputTest),
966 toIBM943, sizeof(toIBM943), "ibm-943",
967 (UConverterFromUCallback)UCNV_FROM_U_CALLBACK_ESCAPE, offset,FALSE))
968 log_err("u-> ibm-943 with subst with value did not match.\n");
969 }
970 #endif
971
972 log_verbose("Testing fromUnicode for UTF-8 with UCNV_TO_U_CALLBACK_SUBSTITUTE \n");
973 {
974 const uint8_t sampleText1[] = { 0x31, 0xe4, 0xba, 0x8c,
975 0xe0, 0x80, 0x61};
976 UChar expected1[] = { 0x0031, 0x4e8c, 0xfffd, 0x0061};
977 int32_t offsets1[] = { 0x0000, 0x0001, 0x0004, 0x0006};
978
979 if(!testConvertToU(sampleText1, sizeof(sampleText1),
980 expected1, UPRV_LENGTHOF(expected1),"utf8", UCNV_TO_U_CALLBACK_SUBSTITUTE, offsets1,FALSE))
981 log_err("utf8->u with substitute did not match.\n");;
982 }
983
984 #if !UCONFIG_NO_LEGACY_CONVERSION
985 log_verbose("Testing toUnicode with UCNV_TO_U_CALLBACK_ESCAPE \n");
986 /*to Unicode*/
987 {
988 const uint8_t sampleTxtToU[]= { 0x00, 0x9f, 0xaf,
989 0x81, 0xad, /*unassigned*/
990 0x89, 0xd3 };
991 UChar IBM_943toUnicode[] = { 0x0000, 0x6D63,
992 0x25, 0x58, 0x38, 0x31, 0x25, 0x58, 0x41, 0x44,
993 0x7B87};
994 int32_t fromIBM943Offs [] = { 0, 1, 3, 3, 3, 3, 3, 3, 3, 3, 5};
995
996 if(!testConvertToU(sampleTxtToU, sizeof(sampleTxtToU),
997 IBM_943toUnicode, UPRV_LENGTHOF(IBM_943toUnicode),"ibm-943",
998 (UConverterToUCallback)UCNV_TO_U_CALLBACK_ESCAPE, fromIBM943Offs,FALSE))
999 log_err("ibm-943->u with substitute with value did not match.\n");
1000
1001 }
1002 #endif
1003 }
1004
convertFromU(const UChar * source,int sourceLen,const uint8_t * expect,int expectLen,const char * codepage,const int32_t * expectOffsets,UBool doFlush,UErrorCode expectedStatus)1005 static UBool convertFromU( const UChar *source, int sourceLen, const uint8_t *expect, int expectLen,
1006 const char *codepage, const int32_t *expectOffsets, UBool doFlush, UErrorCode expectedStatus)
1007 {
1008
1009 int32_t i=0;
1010 char *p=0;
1011 const UChar *src;
1012 char buffer[MAX_LENGTH];
1013 int32_t offsetBuffer[MAX_LENGTH];
1014 int32_t *offs=0;
1015 char *targ;
1016 char *targetLimit;
1017 UChar *sourceLimit=0;
1018 UErrorCode status = U_ZERO_ERROR;
1019 UConverter *conv = 0;
1020 conv = ucnv_open(codepage, &status);
1021 if(U_FAILURE(status))
1022 {
1023 log_data_err("Couldn't open converter %s\n",codepage);
1024 return TRUE;
1025 }
1026 log_verbose("Converter %s opened..\n", ucnv_getName(conv, &status));
1027
1028 for(i=0; i<MAX_LENGTH; i++){
1029 buffer[i]=(char)0xF0;
1030 offsetBuffer[i]=0xFF;
1031 }
1032
1033 src=source;
1034 sourceLimit=(UChar*)src+(sourceLen);
1035 targ=buffer;
1036 targetLimit=targ+MAX_LENGTH;
1037 offs=offsetBuffer;
1038 ucnv_fromUnicode (conv,
1039 (char **)&targ,
1040 (const char *)targetLimit,
1041 &src,
1042 sourceLimit,
1043 expectOffsets ? offs : NULL,
1044 doFlush,
1045 &status);
1046 ucnv_close(conv);
1047 if(status != expectedStatus){
1048 log_err("ucnv_fromUnicode() failed for codepage=%s. Error =%s Expected=%s\n", codepage, myErrorName(status), myErrorName(expectedStatus));
1049 return FALSE;
1050 }
1051
1052 log_verbose("\nConversion done [%d uchars in -> %d chars out]. \nResult :",
1053 sourceLen, targ-buffer);
1054
1055 if(expectLen != targ-buffer)
1056 {
1057 log_err("Expected %d chars out, got %d FROM Unicode to %s\n", expectLen, targ-buffer, codepage);
1058 log_verbose("Expected %d chars out, got %d FROM Unicode to %s\n", expectLen, targ-buffer, codepage);
1059 printSeqErr((const unsigned char *)buffer, (int32_t)(targ-buffer));
1060 printSeqErr((const unsigned char*)expect, expectLen);
1061 return FALSE;
1062 }
1063
1064 if(memcmp(buffer, expect, expectLen)){
1065 log_err("String does not match. FROM Unicode to codePage%s\n", codepage);
1066 log_info("\nGot:");
1067 printSeqErr((const unsigned char *)buffer, expectLen);
1068 log_info("\nExpected:");
1069 printSeqErr((const unsigned char *)expect, expectLen);
1070 return FALSE;
1071 }
1072 else {
1073 log_verbose("Matches!\n");
1074 }
1075
1076 if (expectOffsets != 0){
1077 log_verbose("comparing %d offsets..\n", targ-buffer);
1078 if(memcmp(offsetBuffer,expectOffsets,(targ-buffer) * sizeof(int32_t) )){
1079 log_err("did not get the expected offsets. for FROM Unicode to %s\n", codepage);
1080 log_info("\nGot : ");
1081 printSeqErr((const unsigned char*)buffer, (int32_t)(targ-buffer));
1082 for(p=buffer;p<targ;p++)
1083 log_info("%d, ", offsetBuffer[p-buffer]);
1084 log_info("\nExpected: ");
1085 for(i=0; i< (targ-buffer); i++)
1086 log_info("%d,", expectOffsets[i]);
1087 }
1088 }
1089
1090 return TRUE;
1091 }
1092
1093
convertToU(const uint8_t * source,int sourceLen,const UChar * expect,int expectLen,const char * codepage,const int32_t * expectOffsets,UBool doFlush,UErrorCode expectedStatus)1094 static UBool convertToU( const uint8_t *source, int sourceLen, const UChar *expect, int expectLen,
1095 const char *codepage, const int32_t *expectOffsets, UBool doFlush, UErrorCode expectedStatus)
1096 {
1097 UErrorCode status = U_ZERO_ERROR;
1098 UConverter *conv = 0;
1099 int32_t i=0;
1100 UChar *p=0;
1101 const char* src;
1102 UChar buffer[MAX_LENGTH];
1103 int32_t offsetBuffer[MAX_LENGTH];
1104 int32_t *offs=0;
1105 UChar *targ;
1106 UChar *targetLimit;
1107 uint8_t *sourceLimit=0;
1108
1109
1110
1111 conv = ucnv_open(codepage, &status);
1112 if(U_FAILURE(status))
1113 {
1114 log_data_err("Couldn't open converter %s\n",codepage);
1115 return TRUE;
1116 }
1117 log_verbose("Converter %s opened..\n", ucnv_getName(conv, &status));
1118
1119
1120
1121 for(i=0; i<MAX_LENGTH; i++){
1122 buffer[i]=0xFFFE;
1123 offsetBuffer[i]=-1;
1124 }
1125
1126 src=(const char *)source;
1127 sourceLimit=(uint8_t*)(src+(sourceLen));
1128 targ=buffer;
1129 targetLimit=targ+MAX_LENGTH;
1130 offs=offsetBuffer;
1131
1132
1133
1134 ucnv_toUnicode (conv,
1135 &targ,
1136 targetLimit,
1137 (const char **)&src,
1138 (const char *)sourceLimit,
1139 expectOffsets ? offs : NULL,
1140 doFlush,
1141 &status);
1142
1143 ucnv_close(conv);
1144 if(status != expectedStatus){
1145 log_err("ucnv_fromUnicode() failed for codepage=%s. Error =%s Expected=%s\n", codepage, myErrorName(status), myErrorName(expectedStatus));
1146 return FALSE;
1147 }
1148 log_verbose("\nConversion done [%d uchars in -> %d chars out]. \nResult :",
1149 sourceLen, targ-buffer);
1150
1151
1152
1153
1154 log_verbose("comparing %d uchars (%d bytes)..\n",expectLen,expectLen*2);
1155
1156 if (expectOffsets != 0) {
1157 if(memcmp(offsetBuffer, expectOffsets, (targ-buffer) * sizeof(int32_t))){
1158
1159 log_err("did not get the expected offsets from %s To UNICODE\n", codepage);
1160 log_info("\nGot : ");
1161 for(p=buffer;p<targ;p++)
1162 log_info("%d, ", offsetBuffer[p-buffer]);
1163 log_info("\nExpected: ");
1164 for(i=0; i<(targ-buffer); i++)
1165 log_info("%d, ", expectOffsets[i]);
1166 log_info("\nGot result:");
1167 for(i=0; i<(targ-buffer); i++)
1168 log_info("0x%04X,", buffer[i]);
1169 log_info("\nFrom Input:");
1170 for(i=0; i<(src-(const char *)source); i++)
1171 log_info("0x%02X,", (unsigned char)source[i]);
1172 log_info("\n");
1173 }
1174 }
1175 if(memcmp(buffer, expect, expectLen*2)){
1176 log_err("String does not match. from codePage %s TO Unicode\n", codepage);
1177 log_info("\nGot:");
1178 printUSeqErr(buffer, expectLen);
1179 log_info("\nExpected:");
1180 printUSeqErr(expect, expectLen);
1181 return FALSE;
1182 }
1183 else {
1184 log_verbose("Matches!\n");
1185 }
1186
1187 return TRUE;
1188 }
1189
1190
testConvertFromU(const UChar * source,int sourceLen,const uint8_t * expect,int expectLen,const char * codepage,UConverterFromUCallback callback,const int32_t * expectOffsets,UBool testReset)1191 static UBool testConvertFromU( const UChar *source, int sourceLen, const uint8_t *expect, int expectLen,
1192 const char *codepage, UConverterFromUCallback callback , const int32_t *expectOffsets, UBool testReset)
1193 {
1194 UErrorCode status = U_ZERO_ERROR;
1195 UConverter *conv = 0;
1196 char junkout[MAX_LENGTH]; /* FIX */
1197 int32_t junokout[MAX_LENGTH]; /* FIX */
1198 char *p;
1199 const UChar *src;
1200 char *end;
1201 char *targ;
1202 int32_t *offs;
1203 int i;
1204 int32_t realBufferSize;
1205 char *realBufferEnd;
1206 const UChar *realSourceEnd;
1207 const UChar *sourceLimit;
1208 UBool checkOffsets = TRUE;
1209 UBool doFlush;
1210
1211 UConverterFromUCallback oldAction = NULL;
1212 const void* oldContext = NULL;
1213
1214 for(i=0;i<MAX_LENGTH;i++)
1215 junkout[i] = (char)0xF0;
1216 for(i=0;i<MAX_LENGTH;i++)
1217 junokout[i] = 0xFF;
1218
1219 setNuConvTestName(codepage, "FROM");
1220
1221 log_verbose("\n========= %s\n", gNuConvTestName);
1222
1223 conv = ucnv_open(codepage, &status);
1224 if(U_FAILURE(status))
1225 {
1226 log_data_err("Couldn't open converter %s\n",codepage);
1227 return TRUE;
1228 }
1229
1230 log_verbose("Converter opened..\n");
1231 /*----setting the callback routine----*/
1232 ucnv_setFromUCallBack (conv, callback, NULL, &oldAction, &oldContext, &status);
1233 if (U_FAILURE(status)) {
1234 log_err("FAILURE in setting the callback Function! %s\n", myErrorName(status));
1235 }
1236 /*------------------------*/
1237
1238 src = source;
1239 targ = junkout;
1240 offs = junokout;
1241
1242 realBufferSize = UPRV_LENGTHOF(junkout);
1243 realBufferEnd = junkout + realBufferSize;
1244 realSourceEnd = source + sourceLen;
1245
1246 if ( gOutBufferSize != realBufferSize )
1247 checkOffsets = FALSE;
1248
1249 if( gInBufferSize != MAX_LENGTH )
1250 checkOffsets = FALSE;
1251
1252 do
1253 {
1254 end = nct_min(targ + gOutBufferSize, realBufferEnd);
1255 sourceLimit = nct_min(src + gInBufferSize, realSourceEnd);
1256
1257 doFlush = (UBool)(sourceLimit == realSourceEnd);
1258
1259 if(targ == realBufferEnd)
1260 {
1261 log_err("Error, overflowed the real buffer while about to call fromUnicode! targ=%08lx %s", targ, gNuConvTestName);
1262 return FALSE;
1263 }
1264 log_verbose("calling fromUnicode @ SOURCE:%08lx to %08lx TARGET: %08lx to %08lx, flush=%s\n", src,sourceLimit, targ,end, doFlush?"TRUE":"FALSE");
1265
1266
1267 status = U_ZERO_ERROR;
1268 if(gInBufferSize ==999 && gOutBufferSize==999)
1269 doFlush = FALSE;
1270 ucnv_fromUnicode (conv,
1271 (char **)&targ,
1272 (const char *)end,
1273 &src,
1274 sourceLimit,
1275 offs,
1276 doFlush, /* flush if we're at the end of the input data */
1277 &status);
1278 if(testReset)
1279 ucnv_resetToUnicode(conv);
1280 if(gInBufferSize ==999 && gOutBufferSize==999)
1281 ucnv_resetToUnicode(conv);
1282
1283 } while ( (status == U_BUFFER_OVERFLOW_ERROR) || (U_SUCCESS(status) && sourceLimit < realSourceEnd) );
1284
1285 if(U_FAILURE(status)) {
1286 log_err("Problem doing fromUnicode to %s, errcode %s %s\n", codepage, myErrorName(status), gNuConvTestName);
1287 return FALSE;
1288 }
1289
1290 log_verbose("\nConversion done [%d uchars in -> %d chars out]. \nResult :",
1291 sourceLen, targ-junkout);
1292 if(getTestOption(VERBOSITY_OPTION))
1293 {
1294 char junk[999];
1295 char offset_str[999];
1296 char *ptr;
1297
1298 junk[0] = 0;
1299 offset_str[0] = 0;
1300 for(ptr = junkout;ptr<targ;ptr++)
1301 {
1302 sprintf(junk + strlen(junk), "0x%02x, ", (0xFF) & (unsigned int)*ptr);
1303 sprintf(offset_str + strlen(offset_str), "0x%02x, ", (0xFF) & (unsigned int)junokout[ptr-junkout]);
1304 }
1305
1306 log_verbose(junk);
1307 printSeq((const unsigned char *)expect, expectLen);
1308 if ( checkOffsets )
1309 {
1310 log_verbose("\nOffsets:");
1311 log_verbose(offset_str);
1312 }
1313 log_verbose("\n");
1314 }
1315 ucnv_close(conv);
1316
1317
1318 if(expectLen != targ-junkout)
1319 {
1320 log_err("Expected %d chars out, got %d %s\n", expectLen, targ-junkout, gNuConvTestName);
1321 log_verbose("Expected %d chars out, got %d %s\n", expectLen, targ-junkout, gNuConvTestName);
1322 log_info("\nGot:");
1323 printSeqErr((const unsigned char*)junkout, (int32_t)(targ-junkout));
1324 log_info("\nExpected:");
1325 printSeqErr((const unsigned char*)expect, expectLen);
1326 return FALSE;
1327 }
1328
1329 if (checkOffsets && (expectOffsets != 0) )
1330 {
1331 log_verbose("comparing %d offsets..\n", targ-junkout);
1332 if(memcmp(junokout,expectOffsets,(targ-junkout) * sizeof(int32_t) )){
1333 log_err("did not get the expected offsets. %s", gNuConvTestName);
1334 log_err("Got : ");
1335 printSeqErr((const unsigned char*)junkout, (int32_t)(targ-junkout));
1336 for(p=junkout;p<targ;p++)
1337 log_err("%d, ", junokout[p-junkout]);
1338 log_err("\nExpected: ");
1339 for(i=0; i<(targ-junkout); i++)
1340 log_err("%d,", expectOffsets[i]);
1341 }
1342 }
1343
1344 log_verbose("comparing..\n");
1345 if(!memcmp(junkout, expect, expectLen))
1346 {
1347 log_verbose("Matches!\n");
1348 return TRUE;
1349 }
1350 else
1351 {
1352 log_err("String does not match. %s\n", gNuConvTestName);
1353 printUSeqErr(source, sourceLen);
1354 log_info("\nGot:");
1355 printSeqErr((const unsigned char *)junkout, expectLen);
1356 log_info("\nExpected:");
1357 printSeqErr((const unsigned char *)expect, expectLen);
1358
1359 return FALSE;
1360 }
1361 }
1362
testConvertToU(const uint8_t * source,int sourcelen,const UChar * expect,int expectlen,const char * codepage,UConverterToUCallback callback,const int32_t * expectOffsets,UBool testReset)1363 static UBool testConvertToU( const uint8_t *source, int sourcelen, const UChar *expect, int expectlen,
1364 const char *codepage, UConverterToUCallback callback, const int32_t *expectOffsets, UBool testReset)
1365 {
1366 UErrorCode status = U_ZERO_ERROR;
1367 UConverter *conv = 0;
1368 UChar junkout[MAX_LENGTH]; /* FIX */
1369 int32_t junokout[MAX_LENGTH]; /* FIX */
1370 const char *src;
1371 const char *realSourceEnd;
1372 const char *srcLimit;
1373 UChar *p;
1374 UChar *targ;
1375 UChar *end;
1376 int32_t *offs;
1377 int i;
1378 UBool checkOffsets = TRUE;
1379 int32_t realBufferSize;
1380 UChar *realBufferEnd;
1381 UBool doFlush;
1382
1383 UConverterToUCallback oldAction = NULL;
1384 const void* oldContext = NULL;
1385
1386
1387 for(i=0;i<MAX_LENGTH;i++)
1388 junkout[i] = 0xFFFE;
1389
1390 for(i=0;i<MAX_LENGTH;i++)
1391 junokout[i] = -1;
1392
1393 setNuConvTestName(codepage, "TO");
1394
1395 log_verbose("\n========= %s\n", gNuConvTestName);
1396
1397 conv = ucnv_open(codepage, &status);
1398 if(U_FAILURE(status))
1399 {
1400 log_data_err("Couldn't open converter %s\n",gNuConvTestName);
1401 return TRUE;
1402 }
1403
1404 log_verbose("Converter opened..\n");
1405 /*----setting the callback routine----*/
1406 ucnv_setToUCallBack (conv, callback, NULL, &oldAction, &oldContext, &status);
1407 if (U_FAILURE(status)) {
1408 log_err("FAILURE in setting the callback Function! %s\n", myErrorName(status));
1409 }
1410 /*-------------------------------------*/
1411
1412 src = (const char *)source;
1413 targ = junkout;
1414 offs = junokout;
1415
1416 realBufferSize = UPRV_LENGTHOF(junkout);
1417 realBufferEnd = junkout + realBufferSize;
1418 realSourceEnd = src + sourcelen;
1419
1420 if ( gOutBufferSize != realBufferSize )
1421 checkOffsets = FALSE;
1422
1423 if( gInBufferSize != MAX_LENGTH )
1424 checkOffsets = FALSE;
1425
1426 do
1427 {
1428 end = nct_min( targ + gOutBufferSize, realBufferEnd);
1429 srcLimit = nct_min(realSourceEnd, src + gInBufferSize);
1430
1431 if(targ == realBufferEnd)
1432 {
1433 log_err("Error, the end would overflow the real output buffer while about to call toUnicode! tarjey=%08lx %s",targ,gNuConvTestName);
1434 return FALSE;
1435 }
1436 log_verbose("calling toUnicode @ %08lx to %08lx\n", targ,end);
1437
1438 /* oldTarg = targ; */
1439
1440 status = U_ZERO_ERROR;
1441 doFlush=(UBool)((gInBufferSize ==999 && gOutBufferSize==999)?(srcLimit == realSourceEnd) : FALSE);
1442
1443 ucnv_toUnicode (conv,
1444 &targ,
1445 end,
1446 (const char **)&src,
1447 (const char *)srcLimit,
1448 offs,
1449 doFlush, /* flush if we're at the end of hte source data */
1450 &status);
1451 if(testReset)
1452 ucnv_resetFromUnicode(conv);
1453 if(gInBufferSize ==999 && gOutBufferSize==999)
1454 ucnv_resetToUnicode(conv);
1455 /* offs += (targ-oldTarg); */
1456
1457 } while ( (status == U_BUFFER_OVERFLOW_ERROR) || (U_SUCCESS(status) && (srcLimit < realSourceEnd)) ); /* while we just need another buffer */
1458
1459 if(U_FAILURE(status))
1460 {
1461 log_err("Problem doing %s toUnicode, errcode %s %s\n", codepage, myErrorName(status), gNuConvTestName);
1462 return FALSE;
1463 }
1464
1465 log_verbose("\nConversion done. %d bytes -> %d chars.\nResult :",
1466 sourcelen, targ-junkout);
1467 if(getTestOption(VERBOSITY_OPTION))
1468 {
1469 char junk[999];
1470 char offset_str[999];
1471
1472 UChar *ptr;
1473
1474 junk[0] = 0;
1475 offset_str[0] = 0;
1476
1477 for(ptr = junkout;ptr<targ;ptr++)
1478 {
1479 sprintf(junk + strlen(junk), "0x%04x, ", (0xFFFF) & (unsigned int)*ptr);
1480 sprintf(offset_str + strlen(offset_str), "0x%04x, ", (0xFFFF) & (unsigned int)junokout[ptr-junkout]);
1481 }
1482
1483 log_verbose(junk);
1484
1485 if ( checkOffsets )
1486 {
1487 log_verbose("\nOffsets:");
1488 log_verbose(offset_str);
1489 }
1490 log_verbose("\n");
1491 }
1492 ucnv_close(conv);
1493
1494 log_verbose("comparing %d uchars (%d bytes)..\n",expectlen,expectlen*2);
1495
1496 if (checkOffsets && (expectOffsets != 0))
1497 {
1498 if(memcmp(junokout,expectOffsets,(targ-junkout) * sizeof(int32_t))){
1499
1500 log_err("did not get the expected offsets. %s",gNuConvTestName);
1501 for(p=junkout;p<targ;p++)
1502 log_err("%d, ", junokout[p-junkout]);
1503 log_err("\nExpected: ");
1504 for(i=0; i<(targ-junkout); i++)
1505 log_err("%d,", expectOffsets[i]);
1506 log_err("");
1507 for(i=0; i<(targ-junkout); i++)
1508 log_err("%X,", junkout[i]);
1509 log_err("");
1510 for(i=0; i<(src-(const char *)source); i++)
1511 log_err("%X,", (unsigned char)source[i]);
1512 }
1513 }
1514
1515 if(!memcmp(junkout, expect, expectlen*2))
1516 {
1517 log_verbose("Matches!\n");
1518 return TRUE;
1519 }
1520 else
1521 {
1522 log_err("String does not match. %s\n", gNuConvTestName);
1523 log_verbose("String does not match. %s\n", gNuConvTestName);
1524 log_info("\nGot:");
1525 printUSeq(junkout, expectlen);
1526 log_info("\nExpected:");
1527 printUSeq(expect, expectlen);
1528 return FALSE;
1529 }
1530 }
1531
1532
TestResetBehaviour(void)1533 static void TestResetBehaviour(void){
1534 #if !UCONFIG_NO_LEGACY_CONVERSION
1535 log_verbose("Testing Reset for DBCS and MBCS\n");
1536 {
1537 static const UChar sampleText[] = {0x00a1, 0xd801, 0xdc01, 0x00a4};
1538 static const uint8_t expected[] = {0xa2, 0xae, 0xa1, 0xe0, 0xa2, 0xb4};
1539 static const int32_t offsets[] = {0x00, 0x00, 0x01, 0x01, 0x03, 0x03 };
1540
1541
1542 static const UChar sampleText1[] = {0x00a1, 0x00a4, 0x00a7, 0x00a8};
1543 static const uint8_t expected1[] = {0xa2, 0xae,0xA2,0xB4,0xA1,0xD7,0xA1,0xA7};
1544 static const int32_t offsets1[] = { 0,2,4,6};
1545
1546 /*DBCS*/
1547 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1548 expected, sizeof(expected), "ibm-1363", UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1549 log_err("u-> ibm-1363 [UCNV_DBCS portion] not match.\n");
1550 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1551 expected, sizeof(expected), "ibm-1363", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1552 log_err("u-> ibm-1363 [UCNV_DBCS portion] not match.\n");
1553
1554 if(!testConvertToU(expected1, sizeof(expected1),
1555 sampleText1, UPRV_LENGTHOF(sampleText1), "ibm-1363",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
1556 offsets1, TRUE))
1557 log_err("ibm-1363 -> did not match.\n");
1558 /*MBCS*/
1559 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1560 expected, sizeof(expected), "ibm-1363", UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1561 log_err("u-> ibm-1363 [UCNV_MBCS] not match.\n");
1562 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1563 expected, sizeof(expected), "ibm-1363", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1564 log_err("u-> ibm-1363 [UCNV_MBCS] not match.\n");
1565
1566 if(!testConvertToU(expected1, sizeof(expected1),
1567 sampleText1, UPRV_LENGTHOF(sampleText1), "ibm-1363",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
1568 offsets1, TRUE))
1569 log_err("ibm-1363 -> did not match.\n");
1570
1571 }
1572
1573 log_verbose("Testing Reset for ISO-2022-jp\n");
1574 {
1575 static const UChar sampleText[] = { 0x4e00, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
1576
1577 static const uint8_t expected[] = {0x1b, 0x24, 0x42,0x30,0x6c,0x43,0x7a,0x1b,0x28,0x42,
1578 0x31,0x1A, 0x32};
1579
1580
1581 static const int32_t offsets[] = {0,0,0,0,0,1,1,2,2,2,2,3,5 };
1582
1583
1584 static const UChar sampleText1[] = {0x4e00, 0x04e01, 0x0031,0x001A, 0x0032};
1585 static const uint8_t expected1[] = {0x1b, 0x24, 0x42,0x30,0x6c,0x43,0x7a,0x1b,0x28,0x42,
1586 0x31,0x1A, 0x32};
1587 static const int32_t offsets1[] = { 3,5,10,11,12};
1588
1589 /*iso-2022-jp*/
1590 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1591 expected, sizeof(expected), "iso-2022-jp", UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1592 log_err("u-> not match.\n");
1593 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1594 expected, sizeof(expected), "iso-2022-jp", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1595 log_err("u-> not match.\n");
1596
1597 if(!testConvertToU(expected1, sizeof(expected1),
1598 sampleText1, UPRV_LENGTHOF(sampleText1), "iso-2022-jp",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
1599 offsets1, TRUE))
1600 log_err("iso-2022-jp -> did not match.\n");
1601
1602 }
1603
1604 /* BEGIN android-removed */
1605 /* To save space, Android does not build full ISO-2022-CN tables.
1606 We skip the tests for ISO-2022-CN. */
1607 /*
1608 log_verbose("Testing Reset for ISO-2022-cn\n");
1609 {
1610 static const UChar sampleText[] = { 0x4e00, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
1611
1612 static const uint8_t expected[] = {
1613 0x1B, 0x24, 0x29, 0x41, 0x0E, 0x52, 0x3B,
1614 0x36, 0x21,
1615 0x0f, 0x31,
1616 0x1A,
1617 0x32
1618 };
1619
1620
1621 static const int32_t offsets[] = {
1622 0, 0, 0, 0, 0, 0, 0,
1623 1, 1,
1624 2, 2,
1625 3,
1626 5, };
1627
1628 UChar sampleText1[] = {0x4e00, 0x04e01, 0x0031,0x001A, 0x0032};
1629 static const uint8_t expected1[] = {
1630 0x1B, 0x24, 0x29, 0x41, 0x0E, 0x52, 0x3B,
1631 0x36, 0x21,
1632 0x1B, 0x24, 0x29, 0x47, 0x24, 0x22,
1633 0x0f, 0x1A,
1634 0x32
1635 };
1636 static const int32_t offsets1[] = { 5,7,13,16,17};
1637
1638 // iso-2022-CN android-change
1639 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1640 expected, sizeof(expected), "iso-2022-cn", UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1641 log_err("u-> not match.\n");
1642 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1643 expected, sizeof(expected), "iso-2022-cn", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1644 log_err("u-> not match.\n");
1645
1646 if(!testConvertToU(expected1, sizeof(expected1),
1647 sampleText1, UPRV_LENGTHOF(sampleText1), "iso-2022-cn",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
1648 offsets1, TRUE))
1649 log_err("iso-2022-cn -> did not match.\n");
1650 }
1651 */
1652 /* END android-removed */
1653
1654 log_verbose("Testing Reset for ISO-2022-kr\n");
1655 {
1656 UChar sampleText[] = { 0x4e00,0xd801, 0xdc01, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
1657
1658 static const uint8_t expected[] = {0x1B, 0x24, 0x29, 0x43,
1659 0x0E, 0x6C, 0x69,
1660 0x0f, 0x1A,
1661 0x0e, 0x6F, 0x4B,
1662 0x0F, 0x31,
1663 0x1A,
1664 0x32 };
1665
1666 static const int32_t offsets[] = {-1, -1, -1, -1,
1667 0, 0, 0,
1668 1, 1,
1669 3, 3, 3,
1670 4, 4,
1671 5,
1672 7,
1673 };
1674 static const UChar sampleText1[] = { 0x4e00,0x0041, 0x04e01, 0x0031, 0x0042, 0x0032};
1675
1676 static const uint8_t expected1[] = {0x1B, 0x24, 0x29, 0x43,
1677 0x0E, 0x6C, 0x69,
1678 0x0f, 0x41,
1679 0x0e, 0x6F, 0x4B,
1680 0x0F, 0x31,
1681 0x42,
1682 0x32 };
1683
1684 static const int32_t offsets1[] = {
1685 5, 8, 10,
1686 13, 14, 15
1687
1688 };
1689 /*iso-2022-kr*/
1690 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1691 expected, sizeof(expected), "iso-2022-kr", UCNV_FROM_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1692 log_err("u-> iso-2022-kr [UCNV_DBCS] not match.\n");
1693 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1694 expected, sizeof(expected), "iso-2022-kr", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1695 log_err("u-> iso-2022-kr [UCNV_DBCS] not match.\n");
1696 if(!testConvertToU(expected1, sizeof(expected1),
1697 sampleText1, UPRV_LENGTHOF(sampleText1), "iso-2022-kr",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
1698 offsets1, TRUE))
1699 log_err("iso-2022-kr -> did not match.\n");
1700 }
1701
1702 log_verbose("Testing Reset for HZ\n");
1703 {
1704 static const UChar sampleText[] = { 0x4e00, 0xd801, 0xdc01, 0x04e01, 0x0031, 0xd801, 0xdc01, 0x0032};
1705
1706 static const uint8_t expected[] = {0x7E, 0x7B, 0x52, 0x3B,
1707 0x7E, 0x7D, 0x1A,
1708 0x7E, 0x7B, 0x36, 0x21,
1709 0x7E, 0x7D, 0x31,
1710 0x1A,
1711 0x32 };
1712
1713
1714 static const int32_t offsets[] = {0,0,0,0,
1715 1,1,1,
1716 3,3,3,3,
1717 4,4,4,
1718 5,
1719 7,};
1720 static const UChar sampleText1[] = { 0x4e00, 0x0035, 0x04e01, 0x0031, 0x0041, 0x0032};
1721
1722 static const uint8_t expected1[] = {0x7E, 0x7B, 0x52, 0x3B,
1723 0x7E, 0x7D, 0x35,
1724 0x7E, 0x7B, 0x36, 0x21,
1725 0x7E, 0x7D, 0x31,
1726 0x41,
1727 0x32 };
1728
1729
1730 static const int32_t offsets1[] = {2,6,9,13,14,15
1731 };
1732
1733 /*hz*/
1734 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1735 expected, sizeof(expected), "HZ", UCNV_FROM_U_CALLBACK_SUBSTITUTE,NULL , TRUE))
1736 log_err("u-> not match.\n");
1737 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1738 expected, sizeof(expected), "HZ", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1739 log_err("u-> not match.\n");
1740 if(!testConvertToU(expected1, sizeof(expected1),
1741 sampleText1, UPRV_LENGTHOF(sampleText1), "hz",UCNV_TO_U_CALLBACK_SUBSTITUTE ,
1742 offsets1, TRUE))
1743 log_err("hz -> did not match.\n");
1744 }
1745 #endif
1746
1747 /*UTF-8*/
1748 log_verbose("Testing for UTF8\n");
1749 {
1750 static const UChar sampleText[] = { 0x4e00, 0x0701, 0x0031, 0xbfc1, 0xd801, 0xdc01, 0x0032};
1751 int32_t offsets[]={0x00, 0x00, 0x00, 0x01, 0x01, 0x02,
1752 0x03, 0x03, 0x03, 0x04, 0x04, 0x04,
1753 0x04, 0x06 };
1754 static const uint8_t expected[] = {0xe4, 0xb8, 0x80, 0xdc, 0x81, 0x31,
1755 0xeb, 0xbf, 0x81, 0xF0, 0x90, 0x90, 0x81, 0x32};
1756
1757
1758 static const int32_t fromOffsets[] = { 0x0000, 0x0003, 0x0005, 0x0006, 0x0009, 0x0009, 0x000D };
1759 /*UTF-8*/
1760 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1761 expected, sizeof(expected), "UTF8", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1762 log_err("u-> UTF8 with offsets and flush true did not match.\n");
1763 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1764 expected, sizeof(expected), "UTF8", UCNV_FROM_U_CALLBACK_SUBSTITUTE,NULL , TRUE))
1765 log_err("u-> UTF8 with offsets and flush true did not match.\n");
1766 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1767 expected, sizeof(expected), "UTF8", UCNV_FROM_U_CALLBACK_SUBSTITUTE,offsets , TRUE))
1768 log_err("u-> UTF8 with offsets and flush true did not match.\n");
1769 if(!testConvertFromU(sampleText, UPRV_LENGTHOF(sampleText),
1770 expected, sizeof(expected), "UTF8", UCNV_FROM_U_CALLBACK_SUBSTITUTE,NULL , TRUE))
1771 log_err("u-> UTF8 with offsets and flush true did not match.\n");
1772 if(!testConvertToU(expected, sizeof(expected),
1773 sampleText, UPRV_LENGTHOF(sampleText), "UTF8",UCNV_TO_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1774 log_err("UTF8 -> did not match.\n");
1775 if(!testConvertToU(expected, sizeof(expected),
1776 sampleText, UPRV_LENGTHOF(sampleText), "UTF8", UCNV_TO_U_CALLBACK_SUBSTITUTE , NULL, TRUE))
1777 log_err("UTF8 -> did not match.\n");
1778 if(!testConvertToU(expected, sizeof(expected),
1779 sampleText, UPRV_LENGTHOF(sampleText), "UTF8",UCNV_TO_U_CALLBACK_SUBSTITUTE , fromOffsets, TRUE))
1780 log_err("UTF8 -> did not match.\n");
1781 if(!testConvertToU(expected, sizeof(expected),
1782 sampleText, UPRV_LENGTHOF(sampleText), "UTF8", UCNV_TO_U_CALLBACK_SUBSTITUTE , fromOffsets, TRUE))
1783 log_err("UTF8 -> did not match.\n");
1784
1785 }
1786
1787 }
1788
1789 /* Test that U_TRUNCATED_CHAR_FOUND is set. */
1790 static void
doTestTruncated(const char * cnvName,const uint8_t * bytes,int32_t length)1791 doTestTruncated(const char *cnvName, const uint8_t *bytes, int32_t length) {
1792 UConverter *cnv;
1793
1794 UChar buffer[2];
1795 UChar *target, *targetLimit;
1796 const char *source, *sourceLimit;
1797
1798 UErrorCode errorCode;
1799
1800 errorCode=U_ZERO_ERROR;
1801 cnv=ucnv_open(cnvName, &errorCode);
1802 if(U_FAILURE(errorCode)) {
1803 log_data_err("error TestTruncated: unable to open \"%s\" - %s\n", cnvName, u_errorName(errorCode));
1804 return;
1805 }
1806 ucnv_setToUCallBack(cnv, UCNV_TO_U_CALLBACK_STOP, NULL, NULL, NULL, &errorCode);
1807 if(U_FAILURE(errorCode)) {
1808 log_data_err("error TestTruncated: unable to set the stop callback on \"%s\" - %s\n",
1809 cnvName, u_errorName(errorCode));
1810 ucnv_close(cnv);
1811 return;
1812 }
1813
1814 source=(const char *)bytes;
1815 sourceLimit=source+length;
1816 target=buffer;
1817 targetLimit=buffer+UPRV_LENGTHOF(buffer);
1818
1819 /* 1. input bytes with flush=FALSE, then input nothing with flush=TRUE */
1820 ucnv_toUnicode(cnv, &target, targetLimit, &source, sourceLimit, NULL, FALSE, &errorCode);
1821 if(U_FAILURE(errorCode) || source!=sourceLimit || target!=buffer) {
1822 log_err("error TestTruncated(%s, 1a): input bytes[%d], flush=FALSE: %s, input left %d, output %d\n",
1823 cnvName, length, u_errorName(errorCode), (int)(sourceLimit-source), (int)(target-buffer));
1824 }
1825
1826 errorCode=U_ZERO_ERROR;
1827 source=sourceLimit;
1828 target=buffer;
1829 ucnv_toUnicode(cnv, &target, targetLimit, &source, sourceLimit, NULL, TRUE, &errorCode);
1830 if(errorCode!=U_TRUNCATED_CHAR_FOUND || target!=buffer) {
1831 log_err("error TestTruncated(%s, 1b): no input (previously %d), flush=TRUE: %s (should be U_TRUNCATED_CHAR_FOUND), output %d\n",
1832 cnvName, (int)length, u_errorName(errorCode), (int)(target-buffer));
1833 }
1834
1835 /* 2. input bytes with flush=TRUE */
1836 ucnv_resetToUnicode(cnv);
1837
1838 errorCode=U_ZERO_ERROR;
1839 source=(const char *)bytes;
1840 target=buffer;
1841 ucnv_toUnicode(cnv, &target, targetLimit, &source, sourceLimit, NULL, TRUE, &errorCode);
1842 if(errorCode!=U_TRUNCATED_CHAR_FOUND || source!=sourceLimit || target!=buffer) {
1843 log_err("error TestTruncated(%s, 2): input bytes[%d], flush=TRUE: %s (should be U_TRUNCATED_CHAR_FOUND), input left %d, output %d\n",
1844 cnvName, length, u_errorName(errorCode), (int)(sourceLimit-source), (int)(target-buffer));
1845 }
1846
1847
1848 ucnv_close(cnv);
1849 }
1850
1851 static void
TestTruncated()1852 TestTruncated() {
1853 static const struct {
1854 const char *cnvName;
1855 uint8_t bytes[8]; /* partial input bytes resulting in no output */
1856 int32_t length;
1857 } testCases[]={
1858 { "IMAP-mailbox-name", { 0x26 }, 1 }, /* & */
1859 { "IMAP-mailbox-name", { 0x26, 0x42 }, 2 }, /* &B */
1860 { "IMAP-mailbox-name", { 0x26, 0x42, 0x42 }, 3 }, /* &BB */
1861 { "IMAP-mailbox-name", { 0x26, 0x41, 0x41 }, 3 }, /* &AA */
1862
1863 { "UTF-7", { 0x2b, 0x42 }, 2 }, /* +B */
1864 { "UTF-8", { 0xd1 }, 1 },
1865
1866 { "UTF-16BE", { 0x4e }, 1 },
1867 { "UTF-16LE", { 0x4e }, 1 },
1868 { "UTF-16", { 0x4e }, 1 },
1869 { "UTF-16", { 0xff }, 1 },
1870 { "UTF-16", { 0xfe, 0xff, 0x4e }, 3 },
1871
1872 { "UTF-32BE", { 0, 0, 0x4e }, 3 },
1873 { "UTF-32LE", { 0x4e }, 1 },
1874 { "UTF-32", { 0, 0, 0x4e }, 3 },
1875 { "UTF-32", { 0xff }, 1 },
1876 { "UTF-32", { 0, 0, 0xfe, 0xff, 0 }, 5 },
1877 { "SCSU", { 0x0e, 0x4e }, 2 }, /* SQU 0x4e */
1878
1879 #if !UCONFIG_NO_LEGACY_CONVERSION
1880 { "BOCU-1", { 0xd5 }, 1 },
1881
1882 { "Shift-JIS", { 0xe0 }, 1 },
1883
1884 { "ibm-939", { 0x0e, 0x41 }, 2 } /* SO 0x41 */
1885 #else
1886 { "BOCU-1", { 0xd5 }, 1 ,}
1887 #endif
1888 };
1889 int32_t i;
1890
1891 for(i=0; i<UPRV_LENGTHOF(testCases); ++i) {
1892 doTestTruncated(testCases[i].cnvName, testCases[i].bytes, testCases[i].length);
1893 }
1894 }
1895
1896 typedef struct NameRange {
1897 const char *name;
1898 UChar32 start, end, start2, end2, notStart, notEnd;
1899 } NameRange;
1900
1901 static void
TestUnicodeSet()1902 TestUnicodeSet() {
1903 UErrorCode errorCode;
1904 UConverter *cnv;
1905 USet *set;
1906 const char *name;
1907 int32_t i, count;
1908
1909 static const char *const completeSetNames[]={
1910 "UTF-7",
1911 "UTF-8",
1912 "UTF-16",
1913 "UTF-16BE",
1914 "UTF-16LE",
1915 "UTF-32",
1916 "UTF-32BE",
1917 "UTF-32LE",
1918 "SCSU",
1919 "BOCU-1",
1920 "CESU-8",
1921 #if !UCONFIG_NO_LEGACY_CONVERSION
1922 "gb18030",
1923 #endif
1924 "IMAP-mailbox-name"
1925 };
1926 #if !UCONFIG_NO_LEGACY_CONVERSION
1927 static const char *const lmbcsNames[]={
1928 "LMBCS-1",
1929 "LMBCS-2",
1930 "LMBCS-3",
1931 "LMBCS-4",
1932 "LMBCS-5",
1933 "LMBCS-6",
1934 "LMBCS-8",
1935 "LMBCS-11",
1936 "LMBCS-16",
1937 "LMBCS-17",
1938 "LMBCS-18",
1939 "LMBCS-19"
1940 };
1941 #endif
1942
1943 static const NameRange nameRanges[]={
1944 { "US-ASCII", 0, 0x7f, -1, -1, 0x80, 0x10ffff },
1945 #if !UCONFIG_NO_LEGACY_CONVERSION
1946 { "ibm-367", 0, 0x7f, -1, -1, 0x80, 0x10ffff },
1947 #endif
1948 { "ISO-8859-1", 0, 0x7f, -1, -1, 0x100, 0x10ffff },
1949 #if !UCONFIG_NO_LEGACY_CONVERSION
1950 { "UTF-8", 0, 0xd7ff, 0xe000, 0x10ffff, 0xd800, 0xdfff },
1951 { "windows-1251", 0, 0x7f, 0x410, 0x44f, 0x3000, 0xd7ff },
1952 /* HZ test case fixed and moved to intltest's conversion.txt, ticket #6002 */
1953 { "shift-jis", 0x3041, 0x3093, 0x30a1, 0x30f3, 0x900, 0x1cff }
1954 #else
1955 { "UTF-8", 0, 0xd7ff, 0xe000, 0x10ffff, 0xd800, 0xdfff }
1956 #endif
1957 };
1958
1959 /* open an empty set */
1960 set=uset_open(1, 0);
1961
1962 count=ucnv_countAvailable();
1963 for(i=0; i<count; ++i) {
1964 errorCode=U_ZERO_ERROR;
1965 name=ucnv_getAvailableName(i);
1966 cnv=ucnv_open(name, &errorCode);
1967 if(U_FAILURE(errorCode)) {
1968 log_data_err("error: unable to open converter %s - %s\n",
1969 name, u_errorName(errorCode));
1970 continue;
1971 }
1972
1973 uset_clear(set);
1974 ucnv_getUnicodeSet(cnv, set, UCNV_ROUNDTRIP_SET, &errorCode);
1975 if(U_FAILURE(errorCode)) {
1976 log_err("error: ucnv_getUnicodeSet(%s) failed - %s\n",
1977 name, u_errorName(errorCode));
1978 } else if(uset_size(set)==0) {
1979 log_err("error: ucnv_getUnicodeSet(%s) returns an empty set\n", name);
1980 }
1981
1982 ucnv_close(cnv);
1983 }
1984
1985 /* test converters that are known to convert all of Unicode (except maybe for surrogates) */
1986 for(i=0; i<UPRV_LENGTHOF(completeSetNames); ++i) {
1987 errorCode=U_ZERO_ERROR;
1988 name=completeSetNames[i];
1989 cnv=ucnv_open(name, &errorCode);
1990 if(U_FAILURE(errorCode)) {
1991 log_data_err("error: unable to open converter %s - %s\n",
1992 name, u_errorName(errorCode));
1993 continue;
1994 }
1995
1996 uset_clear(set);
1997 ucnv_getUnicodeSet(cnv, set, UCNV_ROUNDTRIP_SET, &errorCode);
1998 if(U_FAILURE(errorCode)) {
1999 log_err("error: ucnv_getUnicodeSet(%s) failed - %s\n",
2000 name, u_errorName(errorCode));
2001 } else if(!uset_containsRange(set, 0, 0xd7ff) || !uset_containsRange(set, 0xe000, 0x10ffff)) {
2002 log_err("error: ucnv_getUnicodeSet(%s) does not return an all-Unicode set\n", name);
2003 }
2004
2005 ucnv_close(cnv);
2006 }
2007
2008 #if !UCONFIG_NO_LEGACY_CONVERSION
2009 /* test LMBCS variants which convert all of Unicode except for U+F6xx */
2010 for(i=0; i<UPRV_LENGTHOF(lmbcsNames); ++i) {
2011 errorCode=U_ZERO_ERROR;
2012 name=lmbcsNames[i];
2013 cnv=ucnv_open(name, &errorCode);
2014 if(U_FAILURE(errorCode)) {
2015 log_data_err("error: unable to open converter %s - %s\n",
2016 name, u_errorName(errorCode));
2017 continue;
2018 }
2019
2020 uset_clear(set);
2021 ucnv_getUnicodeSet(cnv, set, UCNV_ROUNDTRIP_SET, &errorCode);
2022 if(U_FAILURE(errorCode)) {
2023 log_err("error: ucnv_getUnicodeSet(%s) failed - %s\n",
2024 name, u_errorName(errorCode));
2025 } else if(!uset_containsRange(set, 0, 0xf5ff) || !uset_containsRange(set, 0xf700, 0x10ffff)) {
2026 log_err("error: ucnv_getUnicodeSet(%s) does not return an all-Unicode set (minus U+F6xx)\n", name);
2027 }
2028
2029 ucnv_close(cnv);
2030 }
2031 #endif
2032
2033 /* test specific sets */
2034 for(i=0; i<UPRV_LENGTHOF(nameRanges); ++i) {
2035 errorCode=U_ZERO_ERROR;
2036 name=nameRanges[i].name;
2037 cnv=ucnv_open(name, &errorCode);
2038 if(U_FAILURE(errorCode)) {
2039 log_data_err("error: unable to open converter %s - %s\n",
2040 name, u_errorName(errorCode));
2041 continue;
2042 }
2043
2044 uset_clear(set);
2045 ucnv_getUnicodeSet(cnv, set, UCNV_ROUNDTRIP_SET, &errorCode);
2046 if(U_FAILURE(errorCode)) {
2047 log_err("error: ucnv_getUnicodeSet(%s) failed - %s\n",
2048 name, u_errorName(errorCode));
2049 } else if(
2050 !uset_containsRange(set, nameRanges[i].start, nameRanges[i].end) ||
2051 (nameRanges[i].start2>=0 && !uset_containsRange(set, nameRanges[i].start2, nameRanges[i].end2))
2052 ) {
2053 log_err("error: ucnv_getUnicodeSet(%s) does not contain the expected ranges\n", name);
2054 } else if(nameRanges[i].notStart>=0) {
2055 /* simulate containsAny() with the C API */
2056 uset_complement(set);
2057 if(!uset_containsRange(set, nameRanges[i].notStart, nameRanges[i].notEnd)) {
2058 log_err("error: ucnv_getUnicodeSet(%s) contains part of the unexpected range\n", name);
2059 }
2060 }
2061
2062 ucnv_close(cnv);
2063 }
2064
2065 errorCode = U_ZERO_ERROR;
2066 ucnv_getUnicodeSet(NULL, set, UCNV_ROUNDTRIP_SET, &errorCode);
2067 if (errorCode != U_ILLEGAL_ARGUMENT_ERROR) {
2068 log_err("error: ucnv_getUnicodeSet(NULL) returned wrong status code %s\n", u_errorName(errorCode));
2069 }
2070 errorCode = U_PARSE_ERROR;
2071 /* Make sure that it does nothing if an error is passed in. Difficult to proper test for. */
2072 ucnv_getUnicodeSet(NULL, NULL, UCNV_ROUNDTRIP_SET, &errorCode);
2073 if (errorCode != U_PARSE_ERROR) {
2074 log_err("error: ucnv_getUnicodeSet(NULL) returned wrong status code %s\n", u_errorName(errorCode));
2075 }
2076
2077 uset_close(set);
2078 }
2079