1 /********************************************************************
2 * COPYRIGHT:
3 * Copyright (c) 1997-2007, International Business Machines Corporation and
4 * others. All Rights Reserved.
5 ********************************************************************/
6 /*****************************************************************************
7 *
8 * File CU_CAPITST.C
9 *
10 * Modification History:
11 * Name Description
12 * Madhu Katragadda Ported for C API
13 ******************************************************************************
14 */
15 #include <stdio.h>
16 #include <stdlib.h>
17 #include <string.h>
18 #include <ctype.h>
19 #include "unicode/uloc.h"
20 #include "unicode/ucnv.h"
21 #include "unicode/ucnv_err.h"
22 #include "unicode/putil.h"
23 #include "unicode/uset.h"
24 #include "unicode/ustring.h"
25 #include "ucnv_bld.h" /* for sizeof(UConverter) */
26 #include "cmemory.h" /* for UAlignedMemory */
27 #include "cintltst.h"
28 #include "ccapitst.h"
29
30 /* for not including "cstring.h" -begin*/
31 #ifdef U_WINDOWS
32 # define ctest_stricmp(str1, str2) U_STANDARD_CPP_NAMESPACE _stricmp(str1, str2)
33 #elif defined(POSIX)
34 # define ctest_stricmp(str1, str2) U_STANDARD_CPP_NAMESPACE strcasecmp(str1, str2)
35 #else
36 # define ctest_stricmp(str1, str2) T_CString_stricmp(str1, str2)
37 #endif
38
39 static int U_EXPORT2
T_CString_stricmp(const char * str1,const char * str2)40 T_CString_stricmp(const char *str1, const char *str2) {
41 if(str1==NULL) {
42 if(str2==NULL) {
43 return 0;
44 } else {
45 return -1;
46 }
47 } else if(str2==NULL) {
48 return 1;
49 } else {
50 /* compare non-NULL strings lexically with lowercase */
51 int rc;
52 unsigned char c1, c2;
53 for(;;) {
54 c1=(unsigned char)*str1;
55 c2=(unsigned char)*str2;
56 if(c1==0) {
57 if(c2==0) {
58 return 0;
59 } else {
60 return -1;
61 }
62 } else if(c2==0) {
63 return 1;
64 } else {
65 /* compare non-zero characters with lowercase */
66 rc=(int)(unsigned char)tolower(c1)-(int)(unsigned char)tolower(c2);
67 if(rc!=0) {
68 return rc;
69 }
70 }
71 ++str1;
72 ++str2;
73 }
74 }
75 }
76 /* for not including "cstring.h" -end*/
77
78 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
79
80 #define NUM_CODEPAGE 1
81 #define MAX_FILE_LEN 1024*20
82 #define UCS_FILE_NAME_SIZE 512
83
84 /*returns an action other than the one provided*/
85 static UConverterFromUCallback otherUnicodeAction(UConverterFromUCallback MIA);
86 static UConverterToUCallback otherCharAction(UConverterToUCallback MIA);
87
88 static UConverter *
cnv_open(const char * name,UErrorCode * pErrorCode)89 cnv_open(const char *name, UErrorCode *pErrorCode) {
90 if(name!=NULL && name[0]=='*') {
91 return ucnv_openPackage(loadTestData(pErrorCode), name+1, pErrorCode);
92 } else {
93 return ucnv_open(name, pErrorCode);
94 }
95 }
96
97
98 static void ListNames(void);
99 static void TestFlushCache(void);
100 static void TestDuplicateAlias(void);
101 static void TestCCSID(void);
102 static void TestJ932(void);
103 static void TestJ1968(void);
104 static void TestLMBCSMaxChar(void);
105
106 #if !UCONFIG_NO_LEGACY_CONVERSION
107 static void TestConvertSafeCloneCallback(void);
108 #endif
109
110 static void TestEBCDICSwapLFNL(void);
111 static void TestConvertEx(void);
112 static void TestConvertExFromUTF8(void);
113 static void TestConvertAlgorithmic(void);
114 void TestDefaultConverterError(void); /* defined in cctest.c */
115 static void TestToUCountPending(void);
116 static void TestFromUCountPending(void);
117 static void TestDefaultName(void);
118 static void TestCompareNames(void);
119 static void TestSubstString(void);
120 static void InvalidArguments(void);
121
122 void addTestConvert(TestNode** root);
123
addTestConvert(TestNode ** root)124 void addTestConvert(TestNode** root)
125 {
126 addTest(root, &ListNames, "tsconv/ccapitst/ListNames");
127 addTest(root, &TestConvert, "tsconv/ccapitst/TestConvert");
128 addTest(root, &TestFlushCache, "tsconv/ccapitst/TestFlushCache");
129 addTest(root, &TestAlias, "tsconv/ccapitst/TestAlias");
130 addTest(root, &TestDuplicateAlias, "tsconv/ccapitst/TestDuplicateAlias");
131 addTest(root, &TestConvertSafeClone, "tsconv/ccapitst/TestConvertSafeClone");
132 #if !UCONFIG_NO_LEGACY_CONVERSION
133 addTest(root, &TestConvertSafeCloneCallback,"tsconv/ccapitst/TestConvertSafeCloneCallback");
134 #endif
135 addTest(root, &TestCCSID, "tsconv/ccapitst/TestCCSID");
136 addTest(root, &TestJ932, "tsconv/ccapitst/TestJ932");
137 addTest(root, &TestJ1968, "tsconv/ccapitst/TestJ1968");
138 addTest(root, &TestLMBCSMaxChar, "tsconv/ccapitst/TestLMBCSMaxChar");
139 addTest(root, &TestEBCDICSwapLFNL, "tsconv/ccapitst/TestEBCDICSwapLFNL");
140 addTest(root, &TestConvertEx, "tsconv/ccapitst/TestConvertEx");
141 addTest(root, &TestConvertExFromUTF8, "tsconv/ccapitst/TestConvertExFromUTF8");
142 addTest(root, &TestConvertAlgorithmic, "tsconv/ccapitst/TestConvertAlgorithmic");
143 addTest(root, &TestDefaultConverterError, "tsconv/ccapitst/TestDefaultConverterError");
144 addTest(root, &TestToUCountPending, "tsconv/ccapitst/TestToUCountPending");
145 addTest(root, &TestFromUCountPending, "tsconv/ccapitst/TestFromUCountPending");
146 addTest(root, &TestDefaultName, "tsconv/ccapitst/TestDefaultName");
147 addTest(root, &TestCompareNames, "tsconv/ccapitst/TestCompareNames");
148 addTest(root, &TestSubstString, "tsconv/ccapitst/TestSubstString");
149 addTest(root, &InvalidArguments, "tsconv/ccapitst/InvalidArguments");
150 }
151
ListNames(void)152 static void ListNames(void) {
153 UErrorCode err = U_ZERO_ERROR;
154 int32_t testLong1 = 0;
155 const char* available_conv;
156 UEnumeration *allNamesEnum = NULL;
157 int32_t allNamesCount = 0;
158 uint16_t count;
159
160 log_verbose("Testing ucnv_openAllNames()...");
161 allNamesEnum = ucnv_openAllNames(&err);
162 if(U_FAILURE(err)) {
163 log_err("FAILURE! ucnv_openAllNames() -> %s\n", myErrorName(err));
164 }
165 else {
166 const char *string = NULL;
167 int32_t len = 0;
168 int32_t count1 = 0;
169 int32_t count2 = 0;
170 allNamesCount = uenum_count(allNamesEnum, &err);
171 while ((string = uenum_next(allNamesEnum, &len, &err))) {
172 count1++;
173 log_verbose("read \"%s\", length %i\n", string, len);
174 }
175 if (U_FAILURE(err)) {
176 log_err("FAILURE! uenum_next(allNamesEnum...) set an error: %s\n", u_errorName(err));
177 err = U_ZERO_ERROR;
178 }
179 uenum_reset(allNamesEnum, &err);
180 while ((string = uenum_next(allNamesEnum, &len, &err))) {
181 count2++;
182 ucnv_close(ucnv_open(string, &err));
183 log_verbose("read \"%s\", length %i (%s)\n", string, len, U_SUCCESS(err) ? "available" : "unavailable");
184 err = U_ZERO_ERROR;
185 }
186 if (count1 != count2) {
187 log_err("FAILURE! uenum_reset(allNamesEnum, &err); doesn't work\n");
188 }
189 }
190 uenum_close(allNamesEnum);
191 err = U_ZERO_ERROR;
192
193 /*Tests ucnv_getAvailableName(), getAvialableCount()*/
194
195 log_verbose("Testing ucnv_countAvailable()...");
196
197 testLong1=ucnv_countAvailable();
198 log_info("Number of available codepages: %d/%d\n", testLong1, allNamesCount);
199
200 log_verbose("\n---Testing ucnv_getAvailableName.."); /*need to check this out */
201
202 available_conv = ucnv_getAvailableName(testLong1);
203 /*test ucnv_getAvailableName with err condition*/
204 log_verbose("\n---Testing ucnv_getAvailableName..with index < 0 ");
205 available_conv = ucnv_getAvailableName(-1);
206 if(available_conv != NULL){
207 log_err("ucnv_getAvailableName() with index < 0) should return NULL\n");
208 }
209
210 /* Test ucnv_countAliases() etc. */
211 count = ucnv_countAliases("utf-8", &err);
212 if(U_FAILURE(err)) {
213 log_err("FAILURE! ucnv_countAliases(\"utf-8\") -> %s\n", myErrorName(err));
214 } else if(count <= 0) {
215 log_err("FAILURE! ucnv_countAliases(\"utf-8\") -> %d aliases\n", count);
216 } else {
217 /* try to get the aliases individually */
218 const char *alias;
219 alias = ucnv_getAlias("utf-8", 0, &err);
220 if(U_FAILURE(err)) {
221 log_err("FAILURE! ucnv_getAlias(\"utf-8\", 0) -> %s\n", myErrorName(err));
222 } else if(strcmp("UTF-8", alias) != 0) {
223 log_err("FAILURE! ucnv_getAlias(\"utf-8\", 0) -> %s instead of UTF-8\n", alias);
224 } else {
225 uint16_t aliasNum;
226 for(aliasNum = 0; aliasNum < count; ++aliasNum) {
227 alias = ucnv_getAlias("utf-8", aliasNum, &err);
228 if(U_FAILURE(err)) {
229 log_err("FAILURE! ucnv_getAlias(\"utf-8\", %d) -> %s\n", aliasNum, myErrorName(err));
230 } else if(strlen(alias) > 20) {
231 /* sanity check */
232 log_err("FAILURE! ucnv_getAlias(\"utf-8\", %d) -> alias %s insanely long, corrupt?!\n", aliasNum, alias);
233 } else {
234 log_verbose("alias %d for utf-8: %s\n", aliasNum, alias);
235 }
236 }
237 if(U_SUCCESS(err)) {
238 /* try to fill an array with all aliases */
239 const char **aliases;
240 aliases=(const char **)malloc(count * sizeof(const char *));
241 if(aliases != 0) {
242 ucnv_getAliases("utf-8", aliases, &err);
243 if(U_FAILURE(err)) {
244 log_err("FAILURE! ucnv_getAliases(\"utf-8\") -> %s\n", myErrorName(err));
245 } else {
246 for(aliasNum = 0; aliasNum < count; ++aliasNum) {
247 /* compare the pointers with the ones returned individually */
248 alias = ucnv_getAlias("utf-8", aliasNum, &err);
249 if(U_FAILURE(err)) {
250 log_err("FAILURE! ucnv_getAlias(\"utf-8\", %d) -> %s\n", aliasNum, myErrorName(err));
251 } else if(aliases[aliasNum] != alias) {
252 log_err("FAILURE! ucnv_getAliases(\"utf-8\")[%d] != ucnv_getAlias(\"utf-8\", %d)\n", aliasNum, aliasNum);
253 }
254 }
255 }
256 free((char **)aliases);
257 }
258 }
259 }
260 }
261 }
262
263
TestConvert()264 static void TestConvert()
265 {
266 #if !UCONFIG_NO_LEGACY_CONVERSION
267 char myptr[4];
268 char save[4];
269 int32_t testLong1 = 0;
270 uint16_t rest = 0;
271 int32_t len = 0;
272 int32_t x = 0;
273 FILE* ucs_file_in = NULL;
274 UChar BOM = 0x0000;
275 UChar myUChar = 0x0000;
276 char* mytarget; /* [MAX_FILE_LEN] */
277 char* mytarget_1;
278 char* mytarget_use;
279 UChar* consumedUni = NULL;
280 char* consumed = NULL;
281 char* output_cp_buffer; /* [MAX_FILE_LEN] */
282 UChar* ucs_file_buffer; /* [MAX_FILE_LEN] */
283 UChar* ucs_file_buffer_use;
284 UChar* my_ucs_file_buffer; /* [MAX_FILE_LEN] */
285 UChar* my_ucs_file_buffer_1;
286 int8_t ii = 0;
287 int32_t j = 0;
288 uint16_t codepage_index = 0;
289 int32_t cp = 0;
290 UErrorCode err = U_ZERO_ERROR;
291 char ucs_file_name[UCS_FILE_NAME_SIZE];
292 UConverterFromUCallback MIA1, MIA1_2;
293 UConverterToUCallback MIA2, MIA2_2;
294 const void *MIA1Context, *MIA1Context2, *MIA2Context, *MIA2Context2;
295 UConverter* someConverters[5];
296 UConverter* myConverter = 0;
297 UChar* displayname = 0;
298
299 const char* locale;
300
301 UChar* uchar1 = 0;
302 UChar* uchar2 = 0;
303 UChar* uchar3 = 0;
304 int32_t targetcapacity2;
305 int32_t targetcapacity;
306 int32_t targetsize;
307 int32_t disnamelen;
308
309 const UChar* tmp_ucs_buf;
310 const UChar* tmp_consumedUni=NULL;
311 const char* tmp_mytarget_use;
312 const char* tmp_consumed;
313
314 /******************************************************************
315 Checking Unicode -> ksc
316 ******************************************************************/
317
318 const char* CodePagesToTest[NUM_CODEPAGE] =
319 {
320 "ibm-949_P110-1999"
321
322
323 };
324 const uint16_t CodePageNumberToTest[NUM_CODEPAGE] =
325 {
326 949
327 };
328
329
330 const int8_t CodePagesMinChars[NUM_CODEPAGE] =
331 {
332 1
333
334 };
335
336 const int8_t CodePagesMaxChars[NUM_CODEPAGE] =
337 {
338 2
339
340 };
341
342 const uint16_t CodePagesSubstitutionChars[NUM_CODEPAGE] =
343 {
344 0xAFFE
345 };
346
347 const char* CodePagesTestFiles[NUM_CODEPAGE] =
348 {
349 "uni-text.bin"
350 };
351
352
353 const UConverterPlatform CodePagesPlatform[NUM_CODEPAGE] =
354 {
355 UCNV_IBM
356
357 };
358
359 const char* CodePagesLocale[NUM_CODEPAGE] =
360 {
361 "ko_KR"
362 };
363
364 UConverterFromUCallback oldFromUAction = NULL;
365 UConverterToUCallback oldToUAction = NULL;
366 const void* oldFromUContext = NULL;
367 const void* oldToUContext = NULL;
368
369 /* Allocate memory */
370 mytarget = (char*) malloc(MAX_FILE_LEN * sizeof(mytarget[0]));
371 output_cp_buffer = (char*) malloc(MAX_FILE_LEN * sizeof(output_cp_buffer[0]));
372 ucs_file_buffer = (UChar*) malloc(MAX_FILE_LEN * sizeof(ucs_file_buffer[0]));
373 my_ucs_file_buffer = (UChar*) malloc(MAX_FILE_LEN * sizeof(my_ucs_file_buffer[0]));
374
375 ucs_file_buffer_use = ucs_file_buffer;
376 mytarget_1=mytarget;
377 mytarget_use = mytarget;
378 my_ucs_file_buffer_1=my_ucs_file_buffer;
379
380 /* flush the converter cache to get a consistent state before the flushing is tested */
381 ucnv_flushCache();
382
383 /*Testing ucnv_openU()*/
384 {
385 UChar converterName[]={ 0x0069, 0x0062, 0x006d, 0x002d, 0x0039, 0x0034, 0x0033, 0x0000}; /*ibm-943*/
386 UChar firstSortedName[]={ 0x0021, 0x0000}; /* ! */
387 UChar lastSortedName[]={ 0x007E, 0x0000}; /* ~ */
388 const char *illegalNameChars={ "ibm-943 ibm-943 ibm-943 ibm-943 ibm-943 ibm-943 ibm-943 ibm-943 ibm-943 ibm-943"};
389 UChar illegalName[100];
390 UConverter *converter=NULL;
391 err=U_ZERO_ERROR;
392 converter=ucnv_openU(converterName, &err);
393 if(U_FAILURE(err)){
394 log_data_err("FAILURE! ucnv_openU(ibm-943, err) failed. %s\n", myErrorName(err));
395 }
396 ucnv_close(converter);
397 err=U_ZERO_ERROR;
398 converter=ucnv_openU(NULL, &err);
399 if(U_FAILURE(err)){
400 log_err("FAILURE! ucnv_openU(NULL, err) failed. %s\n", myErrorName(err));
401 }
402 ucnv_close(converter);
403 /*testing with error value*/
404 err=U_ILLEGAL_ARGUMENT_ERROR;
405 converter=ucnv_openU(converterName, &err);
406 if(!(converter == NULL)){
407 log_data_err("FAILURE! ucnv_openU(ibm-943, U_ILLEGAL_ARGUMENT_ERROR) is expected to fail\n");
408 }
409 ucnv_close(converter);
410 err=U_ZERO_ERROR;
411 u_uastrcpy(illegalName, "");
412 u_uastrcpy(illegalName, illegalNameChars);
413 ucnv_openU(illegalName, &err);
414 if(!(err==U_ILLEGAL_ARGUMENT_ERROR)){
415 log_err("FAILURE! ucnv_openU(illegalName, err) is expected to fail\n");
416 }
417
418 err=U_ZERO_ERROR;
419 ucnv_openU(firstSortedName, &err);
420 if(err!=U_FILE_ACCESS_ERROR){
421 log_err("FAILURE! ucnv_openU(firstSortedName, err) is expected to fail\n");
422 }
423
424 err=U_ZERO_ERROR;
425 ucnv_openU(lastSortedName, &err);
426 if(err!=U_FILE_ACCESS_ERROR){
427 log_err("FAILURE! ucnv_openU(lastSortedName, err) is expected to fail\n");
428 }
429
430 err=U_ZERO_ERROR;
431 }
432 log_verbose("Testing ucnv_open() with converter name greater than 7 characters\n");
433 {
434 UConverter *cnv=NULL;
435 err=U_ZERO_ERROR;
436 cnv=ucnv_open("ibm-949,Madhu", &err);
437 if(U_FAILURE(err)){
438 log_data_err("FAILURE! ucnv_open(\"ibm-949,Madhu\", err) failed. %s\n", myErrorName(err));
439 }
440 ucnv_close(cnv);
441
442 }
443 /*Testing ucnv_convert()*/
444 {
445 int32_t targetLimit=0, sourceLimit=0, i=0, targetCapacity=0;
446 const uint8_t source[]={ 0x00, 0x04, 0x05, 0x06, 0xa2, 0xb4, 0x00};
447 const uint8_t expectedTarget[]={ 0x00, 0x37, 0x2d, 0x2e, 0x0e, 0x49, 0x62, 0x0f, 0x00};
448 char *target=0;
449 sourceLimit=sizeof(source)/sizeof(source[0]);
450 err=U_ZERO_ERROR;
451 targetLimit=0;
452
453 targetCapacity=ucnv_convert("ibm-1364", "ibm-1363", NULL, targetLimit , (const char*)source, sourceLimit, &err);
454 if(err == U_BUFFER_OVERFLOW_ERROR){
455 err=U_ZERO_ERROR;
456 targetLimit=targetCapacity+1;
457 target=(char*)malloc(sizeof(char) * targetLimit);
458 targetCapacity=ucnv_convert("ibm-1364", "ibm-1363", target, targetLimit , (const char*)source, sourceLimit, &err);
459 }
460 if(U_FAILURE(err)){
461 log_data_err("FAILURE! ucnv_convert(ibm-1363->ibm-1364) failed. %s\n", myErrorName(err));
462 }
463 else {
464 for(i=0; i<targetCapacity; i++){
465 if(target[i] != expectedTarget[i]){
466 log_err("FAIL: ucnv_convert(ibm-1363->ibm-1364) failed.at index \n i=%d, Expected: %lx Got: %lx\n", i, (UChar)expectedTarget[i], (uint8_t)target[i]);
467 }
468 }
469
470 i=ucnv_convert("ibm-1364", "ibm-1363", target, targetLimit , (const char*)source+1, -1, &err);
471 if(U_FAILURE(err) || i!=7){
472 log_err("FAILURE! ucnv_convert() with sourceLimit=-1 failed: %s, returned %d instead of 7\n",
473 u_errorName(err), i);
474 }
475
476 /*Test error conditions*/
477 err=U_ZERO_ERROR;
478 i=ucnv_convert("ibm-1364", "ibm-1363", target, targetLimit , (const char*)source, 0, &err);
479 if(i !=0){
480 log_err("FAILURE! ucnv_convert() with sourceLimit=0 is expected to return 0\n");
481 }
482
483 err=U_ILLEGAL_ARGUMENT_ERROR;
484 sourceLimit=sizeof(source)/sizeof(source[0]);
485 i=ucnv_convert("ibm-1364", "ibm-1363", target, targetLimit , (const char*)source, sourceLimit, &err);
486 if(i !=0 ){
487 log_err("FAILURE! ucnv_convert() with err=U_ILLEGAL_ARGUMENT_ERROR is expected to return 0\n");
488 }
489
490 err=U_ZERO_ERROR;
491 sourceLimit=sizeof(source)/sizeof(source[0]);
492 targetLimit=0;
493 i=ucnv_convert("ibm-1364", "ibm-1363", target, targetLimit , (const char*)source, sourceLimit, &err);
494 if(!(U_FAILURE(err) && err==U_BUFFER_OVERFLOW_ERROR)){
495 log_err("FAILURE! ucnv_convert() with targetLimit=0 is expected to throw U_BUFFER_OVERFLOW_ERROR\n");
496 }
497 err=U_ZERO_ERROR;
498 free(target);
499 }
500 }
501
502 /*Testing ucnv_openCCSID and ucnv_open with error conditions*/
503 log_verbose("\n---Testing ucnv_open with err ! = U_ZERO_ERROR...\n");
504 err=U_ILLEGAL_ARGUMENT_ERROR;
505 if(ucnv_open(NULL, &err) != NULL){
506 log_err("ucnv_open with err != U_ZERO_ERROR is supposed to fail\n");
507 }
508 if(ucnv_openCCSID(1051, UCNV_IBM, &err) != NULL){
509 log_err("ucnv_open with err != U_ZERO_ERROR is supposed to fail\n");
510 }
511 err=U_ZERO_ERROR;
512
513 /* Testing ucnv_openCCSID(), ucnv_open(), ucnv_getName() */
514 log_verbose("\n---Testing ucnv_open default...\n");
515 someConverters[0] = ucnv_open(NULL,&err);
516 someConverters[1] = ucnv_open(NULL,&err);
517 someConverters[2] = ucnv_open("utf8", &err);
518 someConverters[3] = ucnv_openCCSID(949,UCNV_IBM,&err);
519 ucnv_close(ucnv_openCCSID(1051, UCNV_IBM, &err)); /* test for j350; ucnv_close(NULL) is safe */
520 if (U_FAILURE(err)){ log_data_err("FAILURE! %s\n", myErrorName(err));}
521
522 /* Testing ucnv_getName()*/
523 /*default code page */
524 ucnv_getName(someConverters[0], &err);
525 if(U_FAILURE(err)) {
526 log_data_err("getName[0] failed\n");
527 } else {
528 log_verbose("getName(someConverters[0]) returned %s\n", ucnv_getName(someConverters[0], &err));
529 }
530 ucnv_getName(someConverters[1], &err);
531 if(U_FAILURE(err)) {
532 log_data_err("getName[1] failed\n");
533 } else {
534 log_verbose("getName(someConverters[1]) returned %s\n", ucnv_getName(someConverters[1], &err));
535 }
536
537 ucnv_close(someConverters[0]);
538 ucnv_close(someConverters[1]);
539 ucnv_close(someConverters[2]);
540 ucnv_close(someConverters[3]);
541
542
543 for (codepage_index=0; codepage_index < NUM_CODEPAGE; ++codepage_index)
544 {
545 int32_t i = 0;
546
547 err = U_ZERO_ERROR;
548 #ifdef U_TOPSRCDIR
549 strcpy(ucs_file_name, U_TOPSRCDIR U_FILE_SEP_STRING"test"U_FILE_SEP_STRING"testdata"U_FILE_SEP_STRING);
550 #else
551 strcpy(ucs_file_name, loadTestData(&err));
552
553 if(U_FAILURE(err)){
554 log_err("\nCouldn't get the test data directory... Exiting...Error:%s\n", u_errorName(err));
555 return;
556 }
557
558 {
559 char* index = strrchr(ucs_file_name,(char)U_FILE_SEP_CHAR);
560
561 if((unsigned int)(index-ucs_file_name) != (strlen(ucs_file_name)-1)){
562 *(index+1)=0;
563 }
564 }
565
566 strcat(ucs_file_name,".."U_FILE_SEP_STRING);
567 #endif
568 strcat(ucs_file_name, CodePagesTestFiles[codepage_index]);
569
570 ucs_file_in = fopen(ucs_file_name,"rb");
571 if (!ucs_file_in)
572 {
573 log_err("Couldn't open the Unicode file [%s]... Exiting...\n", ucs_file_name);
574 return;
575 }
576
577 /*Creates a converter and testing ucnv_openCCSID(u_int code_page, platform, errstatus*/
578
579 /* myConverter =ucnv_openCCSID(CodePageNumberToTest[codepage_index],UCNV_IBM, &err); */
580 /* ucnv_flushCache(); */
581 myConverter =ucnv_open( "ibm-949", &err);
582 if (!myConverter || U_FAILURE(err))
583 {
584 log_data_err("Error creating the ibm-949 converter - %s \n", u_errorName(err));
585
586 return;
587 }
588
589 /*testing for ucnv_getName() */
590 log_verbose("Testing ucnv_getName()...\n");
591 ucnv_getName(myConverter, &err);
592 if(U_FAILURE(err))
593 log_err("Error in getName\n");
594 else
595 {
596 log_verbose("getName o.k. %s\n", ucnv_getName(myConverter, &err));
597 }
598 if (ctest_stricmp(ucnv_getName(myConverter, &err), CodePagesToTest[codepage_index]))
599 log_err("getName failed\n");
600 else
601 log_verbose("getName ok\n");
602 /*Test getName with error condition*/
603 {
604 const char* name=0;
605 err=U_ILLEGAL_ARGUMENT_ERROR;
606 log_verbose("Testing ucnv_getName with err != U_ZERO_ERROR");
607 name=ucnv_getName(myConverter, &err);
608 if(name != NULL){
609 log_err("ucnv_getName() with err != U_ZERO_ERROR is expected to fail");
610 }
611 err=U_ZERO_ERROR;
612 }
613
614
615 /*Tests ucnv_getMaxCharSize() and ucnv_getMinCharSize()*/
616
617 log_verbose("Testing ucnv_getMaxCharSize()...\n");
618 if (ucnv_getMaxCharSize(myConverter)==CodePagesMaxChars[codepage_index])
619 log_verbose("Max byte per character OK\n");
620 else
621 log_err("Max byte per character failed\n");
622
623 log_verbose("\n---Testing ucnv_getMinCharSize()...\n");
624 if (ucnv_getMinCharSize(myConverter)==CodePagesMinChars[codepage_index])
625 log_verbose("Min byte per character OK\n");
626 else
627 log_err("Min byte per character failed\n");
628
629
630 /*Testing for ucnv_getSubstChars() and ucnv_setSubstChars()*/
631 log_verbose("\n---Testing ucnv_getSubstChars...\n");
632 ii=4;
633 ucnv_getSubstChars(myConverter, myptr, &ii, &err);
634 if (ii <= 0) {
635 log_err("ucnv_getSubstChars returned a negative number %d\n", ii);
636 }
637
638 for(x=0;x<ii;x++)
639 rest = (uint16_t)(((unsigned char)rest << 8) + (unsigned char)myptr[x]);
640 if (rest==CodePagesSubstitutionChars[codepage_index])
641 log_verbose("Substitution character ok\n");
642 else
643 log_err("Substitution character failed.\n");
644
645 log_verbose("\n---Testing ucnv_setSubstChars RoundTrip Test ...\n");
646 ucnv_setSubstChars(myConverter, myptr, ii, &err);
647 if (U_FAILURE(err))
648 {
649 log_err("FAILURE! %s\n", myErrorName(err));
650 }
651 ucnv_getSubstChars(myConverter,save, &ii, &err);
652 if (U_FAILURE(err))
653 {
654 log_err("FAILURE! %s\n", myErrorName(err));
655 }
656
657 if (strncmp(save, myptr, ii))
658 log_err("Saved substitution character failed\n");
659 else
660 log_verbose("Saved substitution character ok\n");
661
662 /*Testing for ucnv_getSubstChars() and ucnv_setSubstChars() with error conditions*/
663 log_verbose("\n---Testing ucnv_getSubstChars.. with len < minBytesPerChar\n");
664 ii=1;
665 ucnv_getSubstChars(myConverter, myptr, &ii, &err);
666 if(err != U_INDEX_OUTOFBOUNDS_ERROR){
667 log_err("ucnv_getSubstChars() with len < minBytesPerChar should throw U_INDEX_OUTOFBOUNDS_ERROR Got %s\n", myErrorName(err));
668 }
669 err=U_ZERO_ERROR;
670 ii=4;
671 ucnv_getSubstChars(myConverter, myptr, &ii, &err);
672 log_verbose("\n---Testing ucnv_setSubstChars.. with len < minBytesPerChar\n");
673 ucnv_setSubstChars(myConverter, myptr, 0, &err);
674 if(err != U_ILLEGAL_ARGUMENT_ERROR){
675 log_err("ucnv_setSubstChars() with len < minBytesPerChar should throw U_ILLEGAL_ARGUMENT_ERROR Got %s\n", myErrorName(err));
676 }
677 log_verbose("\n---Testing ucnv_setSubstChars.. with err != U_ZERO_ERROR \n");
678 strcpy(myptr, "abc");
679 ucnv_setSubstChars(myConverter, myptr, ii, &err);
680 err=U_ZERO_ERROR;
681 ucnv_getSubstChars(myConverter, save, &ii, &err);
682 if(strncmp(save, myptr, ii) == 0){
683 log_err("uncv_setSubstChars() with err != U_ZERO_ERROR shouldn't set the SubstChars and just return\n");
684 }
685 log_verbose("\n---Testing ucnv_getSubstChars.. with err != U_ZERO_ERROR \n");
686 err=U_ZERO_ERROR;
687 strcpy(myptr, "abc");
688 ucnv_setSubstChars(myConverter, myptr, ii, &err);
689 err=U_ILLEGAL_ARGUMENT_ERROR;
690 ucnv_getSubstChars(myConverter, save, &ii, &err);
691 if(strncmp(save, myptr, ii) == 0){
692 log_err("uncv_setSubstChars() with err != U_ZERO_ERROR shouldn't fill the SubstChars in the buffer, it just returns\n");
693 }
694 err=U_ZERO_ERROR;
695 /*------*/
696
697 #ifdef U_ENABLE_GENERIC_ISO_2022
698 /*resetState ucnv_reset()*/
699 log_verbose("\n---Testing ucnv_reset()..\n");
700 ucnv_reset(myConverter);
701 {
702 UChar32 c;
703 const uint8_t in[]={ 0x1b, 0x25, 0x42, 0x31, 0x32, 0x61, 0xc0, 0x80, 0xe0, 0x80, 0x80, 0xf0, 0x80, 0x80, 0x80};
704 const char *source=(const char *)in, *limit=(const char *)in+sizeof(in);
705 UConverter *cnv=ucnv_open("ISO_2022", &err);
706 if(U_FAILURE(err)) {
707 log_err("Unable to open a iso-2022 converter: %s\n", u_errorName(err));
708 }
709 c=ucnv_getNextUChar(cnv, &source, limit, &err);
710 if((U_FAILURE(err) || c != (UChar32)0x0031)) {
711 log_err("ucnv_getNextUChar() failed: %s\n", u_errorName(err));
712 }
713 ucnv_reset(cnv);
714 ucnv_close(cnv);
715
716 }
717 #endif
718
719 /*getDisplayName*/
720 log_verbose("\n---Testing ucnv_getDisplayName()...\n");
721 locale=CodePagesLocale[codepage_index];
722 len=0;
723 displayname=NULL;
724 disnamelen = ucnv_getDisplayName(myConverter, locale, displayname, len, &err);
725 if(err==U_BUFFER_OVERFLOW_ERROR) {
726 err=U_ZERO_ERROR;
727 displayname=(UChar*)malloc((disnamelen+1) * sizeof(UChar));
728 ucnv_getDisplayName(myConverter,locale,displayname,disnamelen+1, &err);
729 if(U_FAILURE(err)) {
730 log_err("getDisplayName failed. The error is %s\n", myErrorName(err));
731 }
732 else {
733 log_verbose(" getDisplayName o.k.\n");
734 }
735 free(displayname);
736 displayname=NULL;
737 }
738 else {
739 log_err("getDisplayName preflight doesn't work. Error is %s\n", myErrorName(err));
740 }
741 /*test ucnv_getDiaplayName with error condition*/
742 err= U_ILLEGAL_ARGUMENT_ERROR;
743 len=ucnv_getDisplayName(myConverter,locale,NULL,0, &err);
744 if( len !=0 ){
745 log_err("ucnv_getDisplayName() with err != U_ZERO_ERROR is supposed to return 0\n");
746 }
747 /*test ucnv_getDiaplayName with error condition*/
748 err=U_ZERO_ERROR;
749 len=ucnv_getDisplayName(NULL,locale,NULL,0, &err);
750 if( len !=0 || U_SUCCESS(err)){
751 log_err("ucnv_getDisplayName(NULL) with cnv == NULL is supposed to return 0\n");
752 }
753 err=U_ZERO_ERROR;
754
755 /* testing ucnv_setFromUCallBack() and ucnv_getFromUCallBack()*/
756 ucnv_getFromUCallBack(myConverter, &MIA1, &MIA1Context);
757
758 log_verbose("\n---Testing ucnv_setFromUCallBack...\n");
759 ucnv_setFromUCallBack(myConverter, otherUnicodeAction(MIA1), &BOM, &oldFromUAction, &oldFromUContext, &err);
760 if (U_FAILURE(err) || oldFromUAction != MIA1 || oldFromUContext != MIA1Context)
761 {
762 log_err("FAILURE! %s\n", myErrorName(err));
763 }
764
765 ucnv_getFromUCallBack(myConverter, &MIA1_2, &MIA1Context2);
766 if (MIA1_2 != otherUnicodeAction(MIA1) || MIA1Context2 != &BOM)
767 log_err("get From UCallBack failed\n");
768 else
769 log_verbose("get From UCallBack ok\n");
770
771 log_verbose("\n---Testing getFromUCallBack Roundtrip...\n");
772 ucnv_setFromUCallBack(myConverter,MIA1, MIA1Context, &oldFromUAction, &oldFromUContext, &err);
773 if (U_FAILURE(err) || oldFromUAction != otherUnicodeAction(MIA1) || oldFromUContext != &BOM)
774 {
775 log_err("FAILURE! %s\n", myErrorName(err));
776 }
777
778 ucnv_getFromUCallBack(myConverter, &MIA1_2, &MIA1Context2);
779 if (MIA1_2 != MIA1 || MIA1Context2 != MIA1Context)
780 log_err("get From UCallBack action failed\n");
781 else
782 log_verbose("get From UCallBack action ok\n");
783
784 /*testing ucnv_setToUCallBack with error conditions*/
785 err=U_ILLEGAL_ARGUMENT_ERROR;
786 log_verbose("\n---Testing setFromUCallBack. with err != U_ZERO_ERROR..\n");
787 ucnv_setFromUCallBack(myConverter, otherUnicodeAction(MIA1), &BOM, &oldFromUAction, &oldFromUContext, &err);
788 ucnv_getFromUCallBack(myConverter, &MIA1_2, &MIA1Context2);
789 if(MIA1_2 == otherUnicodeAction(MIA1) || MIA1Context2 == &BOM){
790 log_err("To setFromUCallBack with err != U_ZERO_ERROR is supposed to fail\n");
791 }
792 err=U_ZERO_ERROR;
793
794
795 /*testing ucnv_setToUCallBack() and ucnv_getToUCallBack()*/
796 ucnv_getToUCallBack(myConverter, &MIA2, &MIA2Context);
797
798 log_verbose("\n---Testing setTo UCallBack...\n");
799 ucnv_setToUCallBack(myConverter,otherCharAction(MIA2), &BOM, &oldToUAction, &oldToUContext, &err);
800 if (U_FAILURE(err) || oldToUAction != MIA2 || oldToUContext != MIA2Context)
801 {
802 log_err("FAILURE! %s\n", myErrorName(err));
803 }
804
805 ucnv_getToUCallBack(myConverter, &MIA2_2, &MIA2Context2);
806 if (MIA2_2 != otherCharAction(MIA2) || MIA2Context2 != &BOM)
807 log_err("To UCallBack failed\n");
808 else
809 log_verbose("To UCallBack ok\n");
810
811 log_verbose("\n---Testing setTo UCallBack Roundtrip...\n");
812 ucnv_setToUCallBack(myConverter,MIA2, MIA2Context, &oldToUAction, &oldToUContext, &err);
813 if (U_FAILURE(err) || oldToUAction != otherCharAction(MIA2) || oldToUContext != &BOM)
814 { log_err("FAILURE! %s\n", myErrorName(err)); }
815
816 ucnv_getToUCallBack(myConverter, &MIA2_2, &MIA2Context2);
817 if (MIA2_2 != MIA2 || MIA2Context2 != MIA2Context)
818 log_err("To UCallBack failed\n");
819 else
820 log_verbose("To UCallBack ok\n");
821
822 /*testing ucnv_setToUCallBack with error conditions*/
823 err=U_ILLEGAL_ARGUMENT_ERROR;
824 log_verbose("\n---Testing setToUCallBack. with err != U_ZERO_ERROR..\n");
825 ucnv_setToUCallBack(myConverter,otherCharAction(MIA2), NULL, &oldToUAction, &oldToUContext, &err);
826 ucnv_getToUCallBack(myConverter, &MIA2_2, &MIA2Context2);
827 if (MIA2_2 == otherCharAction(MIA2) || MIA2Context2 == &BOM){
828 log_err("To setToUCallBack with err != U_ZERO_ERROR is supposed to fail\n");
829 }
830 err=U_ZERO_ERROR;
831
832
833 /*getcodepageid testing ucnv_getCCSID() */
834 log_verbose("\n----Testing getCCSID....\n");
835 cp = ucnv_getCCSID(myConverter,&err);
836 if (U_FAILURE(err))
837 {
838 log_err("FAILURE!..... %s\n", myErrorName(err));
839 }
840 if (cp != CodePageNumberToTest[codepage_index])
841 log_err("Codepage number test failed\n");
842 else
843 log_verbose("Codepage number test OK\n");
844
845 /*testing ucnv_getCCSID() with err != U_ZERO_ERROR*/
846 err=U_ILLEGAL_ARGUMENT_ERROR;
847 if( ucnv_getCCSID(myConverter,&err) != -1){
848 log_err("ucnv_getCCSID() with err != U_ZERO_ERROR is supposed to fail\n");
849 }
850 err=U_ZERO_ERROR;
851
852 /*getCodepagePlatform testing ucnv_getPlatform()*/
853 log_verbose("\n---Testing getCodepagePlatform ..\n");
854 if (CodePagesPlatform[codepage_index]!=ucnv_getPlatform(myConverter, &err))
855 log_err("Platform codepage test failed\n");
856 else
857 log_verbose("Platform codepage test ok\n");
858
859 if (U_FAILURE(err))
860 {
861 log_err("FAILURE! %s\n", myErrorName(err));
862 }
863 /*testing ucnv_getPlatform() with err != U_ZERO_ERROR*/
864 err= U_ILLEGAL_ARGUMENT_ERROR;
865 if(ucnv_getPlatform(myConverter, &err) != UCNV_UNKNOWN){
866 log_err("ucnv)getPlatform with err != U_ZERO_ERROR is supposed to fail\n");
867 }
868 err=U_ZERO_ERROR;
869
870
871 /*Reads the BOM*/
872 fread(&BOM, sizeof(UChar), 1, ucs_file_in);
873 if (BOM!=0xFEFF && BOM!=0xFFFE)
874 {
875 log_err("File Missing BOM...Bailing!\n");
876 return;
877 }
878
879
880 /*Reads in the file*/
881 while(!feof(ucs_file_in)&&(i+=fread(ucs_file_buffer+i, sizeof(UChar), 1, ucs_file_in)))
882 {
883 myUChar = ucs_file_buffer[i-1];
884
885 ucs_file_buffer[i-1] = (UChar)((BOM==0xFEFF)?myUChar:((myUChar >> 8) | (myUChar << 8))); /*adjust if BIG_ENDIAN*/
886 }
887
888 myUChar = ucs_file_buffer[i-1];
889 ucs_file_buffer[i-1] = (UChar)((BOM==0xFEFF)?myUChar:((myUChar >> 8) | (myUChar << 8))); /*adjust if BIG_ENDIAN Corner Case*/
890
891
892 /*testing ucnv_fromUChars() and ucnv_toUChars() */
893 /*uchar1---fromUChar--->output_cp_buffer --toUChar--->uchar2*/
894
895 uchar1=(UChar*)malloc(sizeof(UChar) * (i+1));
896 u_uastrcpy(uchar1,"");
897 u_strncpy(uchar1,ucs_file_buffer,i);
898 uchar1[i] = 0;
899
900 uchar3=(UChar*)malloc(sizeof(UChar)*(i+1));
901 u_uastrcpy(uchar3,"");
902 u_strncpy(uchar3,ucs_file_buffer,i);
903 uchar3[i] = 0;
904
905 /*Calls the Conversion Routine */
906 testLong1 = MAX_FILE_LEN;
907 log_verbose("\n---Testing ucnv_fromUChars()\n");
908 targetcapacity = ucnv_fromUChars(myConverter, output_cp_buffer, testLong1, uchar1, -1, &err);
909 if (U_FAILURE(err))
910 {
911 log_err("\nFAILURE...%s\n", myErrorName(err));
912 }
913 else
914 log_verbose(" ucnv_fromUChars() o.k.\n");
915
916 /*test the conversion routine */
917 log_verbose("\n---Testing ucnv_toUChars()\n");
918 /*call it first time for trapping the targetcapacity and size needed to allocate memory for the buffer uchar2 */
919 targetcapacity2=0;
920 targetsize = ucnv_toUChars(myConverter,
921 NULL,
922 targetcapacity2,
923 output_cp_buffer,
924 strlen(output_cp_buffer),
925 &err);
926 /*if there is an buffer overflow then trap the values and pass them and make the actual call*/
927
928 if(err==U_BUFFER_OVERFLOW_ERROR)
929 {
930 err=U_ZERO_ERROR;
931 uchar2=(UChar*)malloc((targetsize+1) * sizeof(UChar));
932 targetsize = ucnv_toUChars(myConverter,
933 uchar2,
934 targetsize+1,
935 output_cp_buffer,
936 strlen(output_cp_buffer),
937 &err);
938
939 if(U_FAILURE(err))
940 log_err("ucnv_toUChars() FAILED %s\n", myErrorName(err));
941 else
942 log_verbose(" ucnv_toUChars() o.k.\n");
943
944 if(u_strcmp(uchar1,uchar2)!=0)
945 log_err("equality test failed with conversion routine\n");
946 }
947 else
948 {
949 log_err("ERR: calling toUChars: Didn't get U_BUFFER_OVERFLOW .. expected it.\n");
950 }
951 /*Testing ucnv_fromUChars and ucnv_toUChars with error conditions*/
952 err=U_ILLEGAL_ARGUMENT_ERROR;
953 log_verbose("\n---Testing ucnv_fromUChars() with err != U_ZERO_ERROR\n");
954 targetcapacity = ucnv_fromUChars(myConverter, output_cp_buffer, testLong1, uchar1, -1, &err);
955 if (targetcapacity !=0) {
956 log_err("\nFAILURE: ucnv_fromUChars with err != U_ZERO_ERROR is expected to fail and return 0\n");
957 }
958 err=U_ZERO_ERROR;
959 log_verbose("\n---Testing ucnv_fromUChars() with converter=NULL\n");
960 targetcapacity = ucnv_fromUChars(NULL, output_cp_buffer, testLong1, uchar1, -1, &err);
961 if (targetcapacity !=0 || err != U_ILLEGAL_ARGUMENT_ERROR) {
962 log_err("\nFAILURE: ucnv_fromUChars with converter=NULL is expected to fail\n");
963 }
964 err=U_ZERO_ERROR;
965 log_verbose("\n---Testing ucnv_fromUChars() with sourceLength = 0\n");
966 targetcapacity = ucnv_fromUChars(myConverter, output_cp_buffer, testLong1, uchar1, 0, &err);
967 if (targetcapacity !=0) {
968 log_err("\nFAILURE: ucnv_fromUChars with sourceLength 0 is expected to return 0\n");
969 }
970 log_verbose("\n---Testing ucnv_fromUChars() with targetLength = 0\n");
971 targetcapacity = ucnv_fromUChars(myConverter, output_cp_buffer, 0, uchar1, -1, &err);
972 if (err != U_BUFFER_OVERFLOW_ERROR) {
973 log_err("\nFAILURE: ucnv_fromUChars with targetLength 0 is expected to fail and throw U_BUFFER_OVERFLOW_ERROR\n");
974 }
975 /*toUChars with error conditions*/
976 targetsize = ucnv_toUChars(myConverter, uchar2, targetsize, output_cp_buffer, strlen(output_cp_buffer), &err);
977 if(targetsize != 0){
978 log_err("\nFAILURE: ucnv_toUChars with err != U_ZERO_ERROR is expected to fail and return 0\n");
979 }
980 err=U_ZERO_ERROR;
981 targetsize = ucnv_toUChars(myConverter, uchar2, -1, output_cp_buffer, strlen(output_cp_buffer), &err);
982 if(targetsize != 0 || err != U_ILLEGAL_ARGUMENT_ERROR){
983 log_err("\nFAILURE: ucnv_toUChars with targetsize < 0 is expected to throw U_ILLEGAL_ARGUMENT_ERROR and return 0\n");
984 }
985 err=U_ZERO_ERROR;
986 targetsize = ucnv_toUChars(myConverter, uchar2, 0, output_cp_buffer, 0, &err);
987 if (targetsize !=0) {
988 log_err("\nFAILURE: ucnv_toUChars with sourceLength 0 is expected to return 0\n");
989 }
990 targetcapacity2=0;
991 targetsize = ucnv_toUChars(myConverter, NULL, targetcapacity2, output_cp_buffer, strlen(output_cp_buffer), &err);
992 if (err != U_STRING_NOT_TERMINATED_WARNING) {
993 log_err("\nFAILURE: ucnv_toUChars(targetLength)->%s instead of U_STRING_NOT_TERMINATED_WARNING\n",
994 u_errorName(err));
995 }
996 err=U_ZERO_ERROR;
997 /*-----*/
998
999
1000 /*testing for ucnv_fromUnicode() and ucnv_toUnicode() */
1001 /*Clean up re-usable vars*/
1002 j=0;
1003 log_verbose("Testing ucnv_fromUnicode().....\n");
1004 tmp_ucs_buf=ucs_file_buffer_use;
1005 ucnv_fromUnicode(myConverter, &mytarget_1,
1006 mytarget + MAX_FILE_LEN,
1007 &tmp_ucs_buf,
1008 ucs_file_buffer_use+i,
1009 NULL,
1010 TRUE,
1011 &err);
1012 consumedUni = (UChar*)tmp_consumedUni;
1013
1014 if (U_FAILURE(err))
1015 {
1016 log_err("FAILURE! %s\n", myErrorName(err));
1017 }
1018 else
1019 log_verbose("ucnv_fromUnicode() o.k.\n");
1020
1021 /*Uni1 ----ToUnicode----> Cp2 ----FromUnicode---->Uni3 */
1022 log_verbose("Testing ucnv_toUnicode().....\n");
1023 tmp_mytarget_use=mytarget_use;
1024 tmp_consumed = consumed;
1025 ucnv_toUnicode(myConverter, &my_ucs_file_buffer_1,
1026 my_ucs_file_buffer + MAX_FILE_LEN,
1027 &tmp_mytarget_use,
1028 mytarget_use + (mytarget_1 - mytarget),
1029 NULL,
1030 FALSE,
1031 &err);
1032 consumed = (char*)tmp_consumed;
1033 if (U_FAILURE(err))
1034 {
1035 log_err("FAILURE! %s\n", myErrorName(err));
1036 }
1037 else
1038 log_verbose("ucnv_toUnicode() o.k.\n");
1039
1040
1041 log_verbose("\n---Testing RoundTrip ...\n");
1042
1043
1044 u_strncpy(uchar3, my_ucs_file_buffer,i);
1045 uchar3[i] = 0;
1046
1047 if(u_strcmp(uchar1,uchar3)==0)
1048 log_verbose("Equality test o.k.\n");
1049 else
1050 log_err("Equality test failed\n");
1051
1052 /*sanity compare */
1053 if(uchar2 == NULL)
1054 {
1055 log_err("uchar2 was NULL (ccapitst.c line %d), couldn't do sanity check\n", __LINE__);
1056 }
1057 else
1058 {
1059 if(u_strcmp(uchar2, uchar3)==0)
1060 log_verbose("Equality test o.k.\n");
1061 else
1062 log_err("Equality test failed\n");
1063 }
1064
1065 fclose(ucs_file_in);
1066 ucnv_close(myConverter);
1067 if (uchar1 != 0) free(uchar1);
1068 if (uchar2 != 0) free(uchar2);
1069 if (uchar3 != 0) free(uchar3);
1070 }
1071
1072 free((void*)mytarget);
1073 free((void*)output_cp_buffer);
1074 free((void*)ucs_file_buffer);
1075 free((void*)my_ucs_file_buffer);
1076 #endif
1077 }
1078
otherUnicodeAction(UConverterFromUCallback MIA)1079 static UConverterFromUCallback otherUnicodeAction(UConverterFromUCallback MIA)
1080 {
1081 return (MIA==(UConverterFromUCallback)UCNV_FROM_U_CALLBACK_STOP)?(UConverterFromUCallback)UCNV_FROM_U_CALLBACK_SUBSTITUTE:(UConverterFromUCallback)UCNV_FROM_U_CALLBACK_STOP;
1082 }
1083
1084
otherCharAction(UConverterToUCallback MIA)1085 static UConverterToUCallback otherCharAction(UConverterToUCallback MIA)
1086 {
1087 return (MIA==(UConverterToUCallback)UCNV_TO_U_CALLBACK_STOP)?(UConverterToUCallback)UCNV_TO_U_CALLBACK_SUBSTITUTE:(UConverterToUCallback)UCNV_TO_U_CALLBACK_STOP;
1088 }
1089
TestFlushCache(void)1090 static void TestFlushCache(void) {
1091 #if !UCONFIG_NO_LEGACY_CONVERSION
1092 UErrorCode err = U_ZERO_ERROR;
1093 UConverter* someConverters[5];
1094 int flushCount = 0;
1095
1096 /* flush the converter cache to get a consistent state before the flushing is tested */
1097 ucnv_flushCache();
1098
1099 /*Testing ucnv_open()*/
1100 /* Note: These converters have been chosen because they do NOT
1101 encode the Latin characters (U+0041, ...), and therefore are
1102 highly unlikely to be chosen as system default codepages */
1103
1104 someConverters[0] = ucnv_open("ibm-1047", &err);
1105 if (U_FAILURE(err)) {
1106 log_data_err("FAILURE! %s\n", myErrorName(err));
1107 }
1108
1109 someConverters[1] = ucnv_open("ibm-1047", &err);
1110 if (U_FAILURE(err)) {
1111 log_data_err("FAILURE! %s\n", myErrorName(err));
1112 }
1113
1114 someConverters[2] = ucnv_open("ibm-1047", &err);
1115 if (U_FAILURE(err)) {
1116 log_data_err("FAILURE! %s\n", myErrorName(err));
1117 }
1118
1119 someConverters[3] = ucnv_open("gb18030", &err);
1120 if (U_FAILURE(err)) {
1121 log_data_err("FAILURE! %s\n", myErrorName(err));
1122 }
1123
1124 someConverters[4] = ucnv_open("ibm-954", &err);
1125 if (U_FAILURE(err)) {
1126 log_data_err("FAILURE! %s\n", myErrorName(err));
1127 }
1128
1129
1130 /* Testing ucnv_flushCache() */
1131 log_verbose("\n---Testing ucnv_flushCache...\n");
1132 if ((flushCount=ucnv_flushCache())==0)
1133 log_verbose("Flush cache ok\n");
1134 else
1135 log_data_err("Flush Cache failed [line %d], expect 0 got %d \n", __LINE__, flushCount);
1136
1137 /*testing ucnv_close() and ucnv_flushCache() */
1138 ucnv_close(someConverters[0]);
1139 ucnv_close(someConverters[1]);
1140
1141 if ((flushCount=ucnv_flushCache())==0)
1142 log_verbose("Flush cache ok\n");
1143 else
1144 log_data_err("Flush Cache failed [line %d], expect 0 got %d \n", __LINE__, flushCount);
1145
1146 ucnv_close(someConverters[2]);
1147 ucnv_close(someConverters[3]);
1148
1149 if ((flushCount=ucnv_flushCache())==2)
1150 log_verbose("Flush cache ok\n"); /*because first, second and third are same */
1151 else
1152 log_data_err("Flush Cache failed line %d, got %d expected 2 or there is an error in ucnv_close()\n",
1153 __LINE__,
1154 flushCount);
1155
1156 ucnv_close(someConverters[4]);
1157 if ( (flushCount=ucnv_flushCache())==1)
1158 log_verbose("Flush cache ok\n");
1159 else
1160 log_data_err("Flush Cache failed line %d, expected 1 got %d \n", __LINE__, flushCount);
1161 #endif
1162 }
1163
1164 /**
1165 * Test the converter alias API, specifically the fuzzy matching of
1166 * alias names and the alias table integrity. Make sure each
1167 * converter has at least one alias (itself), and that its listed
1168 * aliases map back to itself. Check some hard-coded UTF-8 and
1169 * ISO_2022 aliases to make sure they work.
1170 */
TestAlias()1171 static void TestAlias() {
1172 int32_t i, ncnv;
1173 UErrorCode status = U_ZERO_ERROR;
1174
1175 /* Predetermined aliases that we expect to map back to ISO_2022
1176 * and UTF-8. UPDATE THIS DATA AS NECESSARY. */
1177 const char* ISO_2022_NAMES[] =
1178 {"ISO_2022,locale=ja,version=2", "ISO-2022-JP-2", "csISO2022JP2",
1179 "Iso-2022jP2", "isO-2022_Jp_2", "iSo--2022,locale=ja,version=2"};
1180 int32_t ISO_2022_NAMES_LENGTH =
1181 sizeof(ISO_2022_NAMES) / sizeof(ISO_2022_NAMES[0]);
1182 const char *UTF8_NAMES[] =
1183 { "UTF-8", "utf-8", "utf8", "ibm-1208",
1184 "utf_8", "ibm1208", "cp1208" };
1185 int32_t UTF8_NAMES_LENGTH =
1186 sizeof(UTF8_NAMES) / sizeof(UTF8_NAMES[0]);
1187
1188 struct {
1189 const char *name;
1190 const char *alias;
1191 } CONVERTERS_NAMES[] = {
1192 { "UTF-32BE", "UTF32_BigEndian" },
1193 { "UTF-32LE", "UTF32_LittleEndian" },
1194 { "UTF-32", "ISO-10646-UCS-4" },
1195 { "UTF32_PlatformEndian", "UTF32_PlatformEndian" },
1196 { "UTF-32", "ucs-4" }
1197 };
1198 int32_t CONVERTERS_NAMES_LENGTH = sizeof(CONVERTERS_NAMES) / sizeof(*CONVERTERS_NAMES);
1199
1200 /* When there are bugs in gencnval or in ucnv_io, converters can
1201 appear to have no aliases. */
1202 ncnv = ucnv_countAvailable();
1203 log_verbose("%d converters\n", ncnv);
1204 for (i=0; i<ncnv; ++i) {
1205 const char *name = ucnv_getAvailableName(i);
1206 const char *alias0;
1207 uint16_t na = ucnv_countAliases(name, &status);
1208 uint16_t j;
1209 UConverter *cnv;
1210
1211 if (na == 0) {
1212 log_err("FAIL: Converter \"%s\" (i=%d)"
1213 " has no aliases; expect at least one\n",
1214 name, i);
1215 continue;
1216 }
1217 cnv = ucnv_open(name, &status);
1218 if (U_FAILURE(status)) {
1219 log_data_err("FAIL: Converter \"%s\" (i=%d)"
1220 " can't be opened.\n",
1221 name, i);
1222 }
1223 else {
1224 if (strcmp(ucnv_getName(cnv, &status), name) != 0
1225 && (strstr(name, "PlatformEndian") == 0 && strstr(name, "OppositeEndian") == 0)) {
1226 log_err("FAIL: Converter \"%s\" returned \"%s\" for getName. "
1227 "The should be the same\n",
1228 name, ucnv_getName(cnv, &status));
1229 }
1230 }
1231 ucnv_close(cnv);
1232
1233 status = U_ZERO_ERROR;
1234 alias0 = ucnv_getAlias(name, 0, &status);
1235 for (j=1; j<na; ++j) {
1236 const char *alias;
1237 /* Make sure each alias maps back to the the same list of
1238 aliases. Assume that if alias 0 is the same, the whole
1239 list is the same (this should always be true). */
1240 const char *mapBack;
1241
1242 status = U_ZERO_ERROR;
1243 alias = ucnv_getAlias(name, j, &status);
1244 if (status == U_AMBIGUOUS_ALIAS_WARNING) {
1245 log_err("FAIL: Converter \"%s\"is ambiguous\n", name);
1246 }
1247
1248 if (alias == NULL) {
1249 log_err("FAIL: Converter \"%s\" -> "
1250 "alias[%d]=NULL\n",
1251 name, j);
1252 continue;
1253 }
1254
1255 mapBack = ucnv_getAlias(alias, 0, &status);
1256
1257 if (mapBack == NULL) {
1258 log_err("FAIL: Converter \"%s\" -> "
1259 "alias[%d]=\"%s\" -> "
1260 "alias[0]=NULL, exp. \"%s\"\n",
1261 name, j, alias, alias0);
1262 continue;
1263 }
1264
1265 if (0 != strcmp(alias0, mapBack)) {
1266 int32_t idx;
1267 UBool foundAlias = FALSE;
1268 if (status == U_AMBIGUOUS_ALIAS_WARNING) {
1269 /* Make sure that we only get this mismapping when there is
1270 an ambiguous alias, and the other converter has this alias too. */
1271 for (idx = 0; idx < ucnv_countAliases(mapBack, &status); idx++) {
1272 if (strcmp(ucnv_getAlias(mapBack, (uint16_t)idx, &status), alias) == 0) {
1273 foundAlias = TRUE;
1274 break;
1275 }
1276 }
1277 }
1278 /* else not ambiguous, and this is a real problem. foundAlias = FALSE */
1279
1280 if (!foundAlias) {
1281 log_err("FAIL: Converter \"%s\" -> "
1282 "alias[%d]=\"%s\" -> "
1283 "alias[0]=\"%s\", exp. \"%s\"\n",
1284 name, j, alias, mapBack, alias0);
1285 }
1286 }
1287 }
1288 }
1289
1290
1291 /* Check a list of predetermined aliases that we expect to map
1292 * back to ISO_2022 and UTF-8. */
1293 for (i=1; i<ISO_2022_NAMES_LENGTH; ++i) {
1294 const char* mapBack = ucnv_getAlias(ISO_2022_NAMES[i], 0, &status);
1295 if(!mapBack) {
1296 log_data_err("Couldn't get alias for %s. You probably have no data\n", ISO_2022_NAMES[i]);
1297 continue;
1298 }
1299 if (0 != strcmp(mapBack, ISO_2022_NAMES[0])) {
1300 log_err("FAIL: \"%s\" -> \"%s\", expect \"ISO_2022,locale=ja,version=2\"\n",
1301 ISO_2022_NAMES[i], mapBack);
1302 }
1303 }
1304
1305
1306 for (i=1; i<UTF8_NAMES_LENGTH; ++i) {
1307 const char* mapBack = ucnv_getAlias(UTF8_NAMES[i], 0, &status);
1308 if(!mapBack) {
1309 log_data_err("Couldn't get alias for %s. You probably have no data\n", UTF8_NAMES[i]);
1310 continue;
1311 }
1312 if (mapBack && 0 != strcmp(mapBack, UTF8_NAMES[0])) {
1313 log_err("FAIL: \"%s\" -> \"%s\", expect UTF-8\n",
1314 UTF8_NAMES[i], mapBack);
1315 }
1316 }
1317
1318 /*
1319 * Check a list of predetermined aliases that we expect to map
1320 * back to predermined converter names.
1321 */
1322
1323 for (i = 0; i < CONVERTERS_NAMES_LENGTH; ++i) {
1324 const char* mapBack = ucnv_getAlias(CONVERTERS_NAMES[i].alias, 0, &status);
1325 if(!mapBack) {
1326 log_data_err("Couldn't get alias for %s. You probably have no data\n", CONVERTERS_NAMES[i].name);
1327 continue;
1328 }
1329 if (0 != strcmp(mapBack, CONVERTERS_NAMES[i].name)) {
1330 log_err("FAIL: \"%s\" -> \"%s\", expect %s\n",
1331 CONVERTERS_NAMES[i].alias, mapBack, CONVERTERS_NAMES[i].name);
1332 }
1333 }
1334
1335 }
1336
TestDuplicateAlias(void)1337 static void TestDuplicateAlias(void) {
1338 const char *alias;
1339 UErrorCode status = U_ZERO_ERROR;
1340
1341 status = U_ZERO_ERROR;
1342 alias = ucnv_getStandardName("Shift_JIS", "IBM", &status);
1343 if (alias == NULL || strcmp(alias, "ibm-943") != 0 || status != U_AMBIGUOUS_ALIAS_WARNING) {
1344 log_err("FAIL: Didn't get ibm-943 for Shift_JIS {IBM}. Got %s\n", alias);
1345 }
1346 status = U_ZERO_ERROR;
1347 alias = ucnv_getStandardName("ibm-943", "IANA", &status);
1348 if (alias == NULL || strcmp(alias, "Shift_JIS") != 0 || status != U_AMBIGUOUS_ALIAS_WARNING) {
1349 log_data_err("FAIL: Didn't get Shift_JIS for ibm-943 {IANA}. Got %s\n", alias);
1350 }
1351 status = U_ZERO_ERROR;
1352 alias = ucnv_getStandardName("ibm-943_P130-2000", "IANA", &status);
1353 if (alias != NULL || status == U_AMBIGUOUS_ALIAS_WARNING) {
1354 log_data_err("FAIL: Didn't get NULL for ibm-943 {IANA}. Got %s\n", alias);
1355 }
1356 }
1357
1358
1359 /* Test safe clone callback */
1360
TSCC_nextSerial()1361 static uint32_t TSCC_nextSerial()
1362 {
1363 static uint32_t n = 1;
1364
1365 return (n++);
1366 }
1367
1368 typedef struct
1369 {
1370 uint32_t magic; /* 0xC0FFEE to identify that the object is OK */
1371 uint32_t serial; /* minted from nextSerial, above */
1372 UBool wasClosed; /* close happened on the object */
1373 } TSCCContext;
1374
TSCC_clone(TSCCContext * ctx)1375 static TSCCContext *TSCC_clone(TSCCContext *ctx)
1376 {
1377 TSCCContext *newCtx = (TSCCContext *)malloc(sizeof(TSCCContext));
1378
1379 newCtx->serial = TSCC_nextSerial();
1380 newCtx->wasClosed = 0;
1381 newCtx->magic = 0xC0FFEE;
1382
1383 log_verbose("TSCC_clone: %p:%d -> new context %p:%d\n", ctx, ctx->serial, newCtx, newCtx->serial);
1384
1385 return newCtx;
1386 }
1387
TSCC_fromU(const void * context,UConverterFromUnicodeArgs * fromUArgs,const UChar * codeUnits,int32_t length,UChar32 codePoint,UConverterCallbackReason reason,UErrorCode * err)1388 static void TSCC_fromU(const void *context,
1389 UConverterFromUnicodeArgs *fromUArgs,
1390 const UChar* codeUnits,
1391 int32_t length,
1392 UChar32 codePoint,
1393 UConverterCallbackReason reason,
1394 UErrorCode * err)
1395 {
1396 TSCCContext *ctx = (TSCCContext*)context;
1397 UConverterFromUCallback junkFrom;
1398
1399 log_verbose("TSCC_fromU: Context %p:%d called, reason %d on cnv %p\n", ctx, ctx->serial, reason, fromUArgs->converter);
1400
1401 if(ctx->magic != 0xC0FFEE) {
1402 log_err("TSCC_fromU: Context %p:%d magic is 0x%x should be 0xC0FFEE.\n", ctx,ctx->serial, ctx->magic);
1403 return;
1404 }
1405
1406 if(reason == UCNV_CLONE) {
1407 UErrorCode subErr = U_ZERO_ERROR;
1408 TSCCContext *newCtx;
1409 TSCCContext *junkCtx;
1410 TSCCContext **pjunkCtx = &junkCtx;
1411
1412 /* "recreate" it */
1413 log_verbose("TSCC_fromU: cloning..\n");
1414 newCtx = TSCC_clone(ctx);
1415
1416 if(newCtx == NULL) {
1417 log_err("TSCC_fromU: internal clone failed on %p\n", ctx);
1418 }
1419
1420 /* now, SET it */
1421 ucnv_getFromUCallBack(fromUArgs->converter, &junkFrom, (const void**)pjunkCtx);
1422 ucnv_setFromUCallBack(fromUArgs->converter, junkFrom, newCtx, NULL, NULL, &subErr);
1423
1424 if(U_FAILURE(subErr)) {
1425 *err = subErr;
1426 }
1427 }
1428
1429 if(reason == UCNV_CLOSE) {
1430 log_verbose("TSCC_fromU: Context %p:%d closing\n", ctx, ctx->serial);
1431 ctx->wasClosed = TRUE;
1432 }
1433 }
1434
1435
TSCC_toU(const void * context,UConverterToUnicodeArgs * toUArgs,const char * codeUnits,int32_t length,UConverterCallbackReason reason,UErrorCode * err)1436 static void TSCC_toU(const void *context,
1437 UConverterToUnicodeArgs *toUArgs,
1438 const char* codeUnits,
1439 int32_t length,
1440 UConverterCallbackReason reason,
1441 UErrorCode * err)
1442 {
1443 TSCCContext *ctx = (TSCCContext*)context;
1444 UConverterToUCallback junkFrom;
1445
1446 log_verbose("TSCC_toU: Context %p:%d called, reason %d on cnv %p\n", ctx, ctx->serial, reason, toUArgs->converter);
1447
1448 if(ctx->magic != 0xC0FFEE) {
1449 log_err("TSCC_toU: Context %p:%d magic is 0x%x should be 0xC0FFEE.\n", ctx,ctx->serial, ctx->magic);
1450 return;
1451 }
1452
1453 if(reason == UCNV_CLONE) {
1454 UErrorCode subErr = U_ZERO_ERROR;
1455 TSCCContext *newCtx;
1456 TSCCContext *junkCtx;
1457 TSCCContext **pjunkCtx = &junkCtx;
1458
1459 /* "recreate" it */
1460 log_verbose("TSCC_toU: cloning..\n");
1461 newCtx = TSCC_clone(ctx);
1462
1463 if(newCtx == NULL) {
1464 log_err("TSCC_toU: internal clone failed on %p\n", ctx);
1465 }
1466
1467 /* now, SET it */
1468 ucnv_getToUCallBack(toUArgs->converter, &junkFrom, (const void**)pjunkCtx);
1469 ucnv_setToUCallBack(toUArgs->converter, junkFrom, newCtx, NULL, NULL, &subErr);
1470
1471 if(U_FAILURE(subErr)) {
1472 *err = subErr;
1473 }
1474 }
1475
1476 if(reason == UCNV_CLOSE) {
1477 log_verbose("TSCC_toU: Context %p:%d closing\n", ctx, ctx->serial);
1478 ctx->wasClosed = TRUE;
1479 }
1480 }
1481
TSCC_init(TSCCContext * q)1482 static void TSCC_init(TSCCContext *q)
1483 {
1484 q->magic = 0xC0FFEE;
1485 q->serial = TSCC_nextSerial();
1486 q->wasClosed = 0;
1487 }
1488
TSCC_print_log(TSCCContext * q,const char * name)1489 static void TSCC_print_log(TSCCContext *q, const char *name)
1490 {
1491 if(q==NULL) {
1492 log_verbose("TSCContext: %s is NULL!!\n", name);
1493 } else {
1494 if(q->magic != 0xC0FFEE) {
1495 log_err("TSCCContext: %p:%d's magic is %x, supposed to be 0xC0FFEE\n",
1496 q,q->serial, q->magic);
1497 }
1498 log_verbose("TSCCContext %p:%d=%s - magic %x, %s\n",
1499 q, q->serial, name, q->magic, q->wasClosed?"CLOSED":"open");
1500 }
1501 }
1502
1503 #if !UCONFIG_NO_LEGACY_CONVERSION
TestConvertSafeCloneCallback()1504 static void TestConvertSafeCloneCallback()
1505 {
1506 UErrorCode err = U_ZERO_ERROR;
1507 TSCCContext from1, to1;
1508 TSCCContext *from2, *from3, *to2, *to3;
1509 TSCCContext **pfrom2 = &from2, **pfrom3 = &from3, **pto2 = &to2, **pto3 = &to3;
1510 char hunk[8192];
1511 int32_t hunkSize = 8192;
1512 UConverterFromUCallback junkFrom;
1513 UConverterToUCallback junkTo;
1514 UConverter *conv1, *conv2 = NULL;
1515
1516 conv1 = ucnv_open("iso-8859-3", &err);
1517
1518 if(U_FAILURE(err)) {
1519 log_data_err("Err opening iso-8859-3, %s", u_errorName(err));
1520 return;
1521 }
1522
1523 log_verbose("Opened conv1=%p\n", conv1);
1524
1525 TSCC_init(&from1);
1526 TSCC_init(&to1);
1527
1528 TSCC_print_log(&from1, "from1");
1529 TSCC_print_log(&to1, "to1");
1530
1531 ucnv_setFromUCallBack(conv1, TSCC_fromU, &from1, NULL, NULL, &err);
1532 log_verbose("Set from1 on conv1\n");
1533 TSCC_print_log(&from1, "from1");
1534
1535 ucnv_setToUCallBack(conv1, TSCC_toU, &to1, NULL, NULL, &err);
1536 log_verbose("Set to1 on conv1\n");
1537 TSCC_print_log(&to1, "to1");
1538
1539 conv2 = ucnv_safeClone(conv1, hunk, &hunkSize, &err);
1540 if(U_FAILURE(err)) {
1541 log_err("safeClone failed: %s\n", u_errorName(err));
1542 return;
1543 }
1544 log_verbose("Cloned to conv2=%p.\n", conv2);
1545
1546 /********** from *********************/
1547 ucnv_getFromUCallBack(conv2, &junkFrom, (const void**)pfrom2);
1548 ucnv_getFromUCallBack(conv1, &junkFrom, (const void**)pfrom3);
1549
1550 TSCC_print_log(from2, "from2");
1551 TSCC_print_log(from3, "from3(==from1)");
1552
1553 if(from2 == NULL) {
1554 log_err("FAIL! from2 is null \n");
1555 return;
1556 }
1557
1558 if(from3 == NULL) {
1559 log_err("FAIL! from3 is null \n");
1560 return;
1561 }
1562
1563 if(from3 != (&from1) ) {
1564 log_err("FAIL! conv1's FROM context changed!\n");
1565 }
1566
1567 if(from2 == (&from1) ) {
1568 log_err("FAIL! conv1's FROM context is the same as conv2's!\n");
1569 }
1570
1571 if(from1.wasClosed) {
1572 log_err("FAIL! from1 is closed \n");
1573 }
1574
1575 if(from2->wasClosed) {
1576 log_err("FAIL! from2 was closed\n");
1577 }
1578
1579 /********** to *********************/
1580 ucnv_getToUCallBack(conv2, &junkTo, (const void**)pto2);
1581 ucnv_getToUCallBack(conv1, &junkTo, (const void**)pto3);
1582
1583 TSCC_print_log(to2, "to2");
1584 TSCC_print_log(to3, "to3(==to1)");
1585
1586 if(to2 == NULL) {
1587 log_err("FAIL! to2 is null \n");
1588 return;
1589 }
1590
1591 if(to3 == NULL) {
1592 log_err("FAIL! to3 is null \n");
1593 return;
1594 }
1595
1596 if(to3 != (&to1) ) {
1597 log_err("FAIL! conv1's TO context changed!\n");
1598 }
1599
1600 if(to2 == (&to1) ) {
1601 log_err("FAIL! conv1's TO context is the same as conv2's!\n");
1602 }
1603
1604 if(to1.wasClosed) {
1605 log_err("FAIL! to1 is closed \n");
1606 }
1607
1608 if(to2->wasClosed) {
1609 log_err("FAIL! to2 was closed\n");
1610 }
1611
1612 /*************************************/
1613
1614 ucnv_close(conv1);
1615 log_verbose("ucnv_closed (conv1)\n");
1616 TSCC_print_log(&from1, "from1");
1617 TSCC_print_log(from2, "from2");
1618 TSCC_print_log(&to1, "to1");
1619 TSCC_print_log(to2, "to2");
1620
1621 if(from1.wasClosed == FALSE) {
1622 log_err("FAIL! from1 is NOT closed \n");
1623 }
1624
1625 if(from2->wasClosed) {
1626 log_err("FAIL! from2 was closed\n");
1627 }
1628
1629 if(to1.wasClosed == FALSE) {
1630 log_err("FAIL! to1 is NOT closed \n");
1631 }
1632
1633 if(to2->wasClosed) {
1634 log_err("FAIL! to2 was closed\n");
1635 }
1636
1637 ucnv_close(conv2);
1638 log_verbose("ucnv_closed (conv2)\n");
1639
1640 TSCC_print_log(&from1, "from1");
1641 TSCC_print_log(from2, "from2");
1642
1643 if(from1.wasClosed == FALSE) {
1644 log_err("FAIL! from1 is NOT closed \n");
1645 }
1646
1647 if(from2->wasClosed == FALSE) {
1648 log_err("FAIL! from2 was NOT closed\n");
1649 }
1650
1651 TSCC_print_log(&to1, "to1");
1652 TSCC_print_log(to2, "to2");
1653
1654 if(to1.wasClosed == FALSE) {
1655 log_err("FAIL! to1 is NOT closed \n");
1656 }
1657
1658 if(to2->wasClosed == FALSE) {
1659 log_err("FAIL! to2 was NOT closed\n");
1660 }
1661
1662 if(to2 != (&to1)) {
1663 free(to2); /* to1 is stack based */
1664 }
1665 if(from2 != (&from1)) {
1666 free(from2); /* from1 is stack based */
1667 }
1668 }
1669 #endif
1670
1671 static UBool
containsAnyOtherByte(uint8_t * p,int32_t length,uint8_t b)1672 containsAnyOtherByte(uint8_t *p, int32_t length, uint8_t b) {
1673 while(length>0) {
1674 if(*p!=b) {
1675 return TRUE;
1676 }
1677 ++p;
1678 --length;
1679 }
1680 return FALSE;
1681 }
1682
TestConvertSafeClone()1683 static void TestConvertSafeClone()
1684 {
1685 /* one 'regular' & all the 'private stateful' converters */
1686 static const char *const names[] = {
1687 #if !UCONFIG_NO_LEGACY_CONVERSION
1688 "ibm-1047",
1689 "ISO_2022,locale=zh,version=1",
1690 #endif
1691 "SCSU",
1692 #if !UCONFIG_NO_LEGACY_CONVERSION
1693 "HZ",
1694 "lmbcs",
1695 "ISCII,version=0",
1696 "ISO_2022,locale=kr,version=1",
1697 "ISO_2022,locale=jp,version=2",
1698 #endif
1699 "BOCU-1",
1700 "UTF-7",
1701 #if !UCONFIG_NO_LEGACY_CONVERSION
1702 "IMAP-mailbox-name",
1703 "ibm-1047-s390"
1704 #else
1705 "IMAP=mailbox-name"
1706 #endif
1707 };
1708
1709 /* store the actual sizes of each converter */
1710 int32_t actualSizes[LENGTHOF(names)];
1711
1712 static const int32_t bufferSizes[] = {
1713 U_CNV_SAFECLONE_BUFFERSIZE,
1714 (int32_t)(3*sizeof(UConverter))/2, /* 1.5*sizeof(UConverter) */
1715 (int32_t)sizeof(UConverter)/2 /* 0.5*sizeof(UConverter) */
1716 };
1717
1718 char charBuffer[21]; /* Leave at an odd number for alignment testing */
1719 uint8_t buffer[3] [U_CNV_SAFECLONE_BUFFERSIZE];
1720 int32_t bufferSize, maxBufferSize;
1721 const char *maxName;
1722 UConverter * cnv, *cnv2;
1723 UErrorCode err;
1724
1725 char *pCharBuffer;
1726 const char *pConstCharBuffer;
1727 const char *charBufferLimit = charBuffer + sizeof(charBuffer)/sizeof(*charBuffer);
1728 UChar uniBuffer[] = {0x0058, 0x0059, 0x005A}; /* "XYZ" */
1729 UChar uniCharBuffer[20];
1730 char charSourceBuffer[] = { 0x1b, 0x24, 0x42 };
1731 const char *pCharSource = charSourceBuffer;
1732 const char *pCharSourceLimit = charSourceBuffer + sizeof(charSourceBuffer);
1733 UChar *pUCharTarget = uniCharBuffer;
1734 UChar *pUCharTargetLimit = uniCharBuffer + sizeof(uniCharBuffer)/sizeof(*uniCharBuffer);
1735 const UChar * pUniBuffer;
1736 const UChar *uniBufferLimit = uniBuffer + sizeof(uniBuffer)/sizeof(*uniBuffer);
1737 int32_t index, j;
1738
1739 err = U_ZERO_ERROR;
1740 cnv = ucnv_open(names[0], &err);
1741 if(U_SUCCESS(err)) {
1742 /* Check the various error & informational states: */
1743
1744 /* Null status - just returns NULL */
1745 bufferSize = U_CNV_SAFECLONE_BUFFERSIZE;
1746 if (0 != ucnv_safeClone(cnv, buffer[0], &bufferSize, 0))
1747 {
1748 log_err("FAIL: Cloned converter failed to deal correctly with null status\n");
1749 }
1750 /* error status - should return 0 & keep error the same */
1751 err = U_MEMORY_ALLOCATION_ERROR;
1752 if (0 != ucnv_safeClone(cnv, buffer[0], &bufferSize, &err) || err != U_MEMORY_ALLOCATION_ERROR)
1753 {
1754 log_err("FAIL: Cloned converter failed to deal correctly with incoming error status\n");
1755 }
1756 err = U_ZERO_ERROR;
1757
1758 /* Null buffer size pointer - just returns NULL & set error to U_ILLEGAL_ARGUMENT_ERROR*/
1759 if (0 != ucnv_safeClone(cnv, buffer[0], 0, &err) || err != U_ILLEGAL_ARGUMENT_ERROR)
1760 {
1761 log_err("FAIL: Cloned converter failed to deal correctly with null bufferSize pointer\n");
1762 }
1763 err = U_ZERO_ERROR;
1764
1765 /* buffer size pointer is 0 - fill in pbufferSize with a size */
1766 bufferSize = 0;
1767 if (0 != ucnv_safeClone(cnv, buffer[0], &bufferSize, &err) || U_FAILURE(err) || bufferSize <= 0)
1768 {
1769 log_err("FAIL: Cloned converter failed a sizing request ('preflighting')\n");
1770 }
1771 /* Verify our define is large enough */
1772 if (U_CNV_SAFECLONE_BUFFERSIZE < bufferSize)
1773 {
1774 log_err("FAIL: Pre-calculated buffer size is too small\n");
1775 }
1776 /* Verify we can use this run-time calculated size */
1777 if (0 == (cnv2 = ucnv_safeClone(cnv, buffer[0], &bufferSize, &err)) || U_FAILURE(err))
1778 {
1779 log_err("FAIL: Converter can't be cloned with run-time size\n");
1780 }
1781 if (cnv2) {
1782 ucnv_close(cnv2);
1783 }
1784
1785 /* size one byte too small - should allocate & let us know */
1786 --bufferSize;
1787 if (0 == (cnv2 = ucnv_safeClone(cnv, 0, &bufferSize, &err)) || err != U_SAFECLONE_ALLOCATED_WARNING)
1788 {
1789 log_err("FAIL: Cloned converter failed to deal correctly with too-small buffer size\n");
1790 }
1791 if (cnv2) {
1792 ucnv_close(cnv2);
1793 }
1794
1795 err = U_ZERO_ERROR;
1796 bufferSize = U_CNV_SAFECLONE_BUFFERSIZE;
1797
1798 /* Null buffer pointer - return converter & set error to U_SAFECLONE_ALLOCATED_ERROR */
1799 if (0 == (cnv2 = ucnv_safeClone(cnv, 0, &bufferSize, &err)) || err != U_SAFECLONE_ALLOCATED_WARNING)
1800 {
1801 log_err("FAIL: Cloned converter failed to deal correctly with null buffer pointer\n");
1802 }
1803 if (cnv2) {
1804 ucnv_close(cnv2);
1805 }
1806
1807 err = U_ZERO_ERROR;
1808
1809 /* Null converter - return NULL & set U_ILLEGAL_ARGUMENT_ERROR */
1810 if (0 != ucnv_safeClone(0, buffer[0], &bufferSize, &err) || err != U_ILLEGAL_ARGUMENT_ERROR)
1811 {
1812 log_err("FAIL: Cloned converter failed to deal correctly with null converter pointer\n");
1813 }
1814
1815 ucnv_close(cnv);
1816 }
1817
1818 maxBufferSize = 0;
1819 maxName = "";
1820
1821 /* Do these cloned converters work at all - shuffle UChars to chars & back again..*/
1822
1823 for(j = 0; j < LENGTHOF(bufferSizes); ++j) {
1824 for (index = 0; index < LENGTHOF(names); index++)
1825 {
1826 err = U_ZERO_ERROR;
1827 cnv = ucnv_open(names[index], &err);
1828 if(U_FAILURE(err)) {
1829 log_data_err("ucnv_open(\"%s\") failed - %s\n", names[index], u_errorName(err));
1830 continue;
1831 }
1832
1833 if(j == 0) {
1834 /* preflight to get maxBufferSize */
1835 actualSizes[index] = 0;
1836 ucnv_safeClone(cnv, NULL, &actualSizes[index], &err);
1837 if(actualSizes[index] > maxBufferSize) {
1838 maxBufferSize = actualSizes[index];
1839 maxName = names[index];
1840 }
1841 }
1842
1843 memset(buffer, 0xaa, sizeof(buffer));
1844
1845 bufferSize = bufferSizes[j];
1846 cnv2 = ucnv_safeClone(cnv, buffer[1], &bufferSize, &err);
1847
1848 /* close the original immediately to make sure that the clone works by itself */
1849 ucnv_close(cnv);
1850
1851 if( actualSizes[index] <= (bufferSizes[j] - (int32_t)sizeof(UAlignedMemory)) &&
1852 err == U_SAFECLONE_ALLOCATED_WARNING
1853 ) {
1854 log_err("ucnv_safeClone(%s) did a heap clone although the buffer was large enough\n", names[index]);
1855 }
1856
1857 /* check if the clone function overwrote any bytes that it is not supposed to touch */
1858 if(bufferSize <= bufferSizes[j]) {
1859 /* used the stack buffer */
1860 if( containsAnyOtherByte(buffer[0], (int32_t)sizeof(buffer[0]), 0xaa) ||
1861 containsAnyOtherByte(buffer[1]+bufferSize, (int32_t)(sizeof(buffer)-(sizeof(buffer[0])+bufferSize)), 0xaa)
1862 ) {
1863 log_err("cloning %s in a stack buffer overwrote bytes outside the bufferSize %d (requested %d)\n",
1864 names[index], bufferSize, bufferSizes[j]);
1865 }
1866 } else {
1867 /* heap-allocated the clone */
1868 if(containsAnyOtherByte(buffer[0], (int32_t)sizeof(buffer), 0xaa)) {
1869 log_err("cloning %s used the heap (bufferSize %d, requested %d) but overwrote stack buffer bytes\n",
1870 names[index], bufferSize, bufferSizes[j]);
1871 }
1872 }
1873
1874 pCharBuffer = charBuffer;
1875 pUniBuffer = uniBuffer;
1876
1877 ucnv_fromUnicode(cnv2,
1878 &pCharBuffer,
1879 charBufferLimit,
1880 &pUniBuffer,
1881 uniBufferLimit,
1882 NULL,
1883 TRUE,
1884 &err);
1885 if(U_FAILURE(err)){
1886 log_err("FAIL: cloned converter failed to do fromU conversion. Error: %s\n",u_errorName(err));
1887 }
1888 ucnv_toUnicode(cnv2,
1889 &pUCharTarget,
1890 pUCharTargetLimit,
1891 &pCharSource,
1892 pCharSourceLimit,
1893 NULL,
1894 TRUE,
1895 &err
1896 );
1897
1898 if(U_FAILURE(err)){
1899 log_err("FAIL: cloned converter failed to do toU conversion. Error: %s\n",u_errorName(err));
1900 }
1901
1902 pConstCharBuffer = charBuffer;
1903 if (uniBuffer [0] != ucnv_getNextUChar(cnv2, &pConstCharBuffer, pCharBuffer, &err))
1904 {
1905 log_err("FAIL: Cloned converter failed to do conversion. Error: %s\n",u_errorName(err));
1906 }
1907 ucnv_close(cnv2);
1908 }
1909 }
1910
1911 log_verbose("ucnv_safeClone(): sizeof(UConverter)=%lu max preflighted clone size=%d (%s) U_CNV_SAFECLONE_BUFFERSIZE=%d\n",
1912 sizeof(UConverter), maxBufferSize, maxName, (int)U_CNV_SAFECLONE_BUFFERSIZE);
1913 if(maxBufferSize > U_CNV_SAFECLONE_BUFFERSIZE) {
1914 log_err("ucnv_safeClone(): max preflighted clone size=%d (%s) is larger than U_CNV_SAFECLONE_BUFFERSIZE=%d\n",
1915 maxBufferSize, maxName, (int)U_CNV_SAFECLONE_BUFFERSIZE);
1916 }
1917 }
1918
TestCCSID()1919 static void TestCCSID() {
1920 #if !UCONFIG_NO_LEGACY_CONVERSION
1921 UConverter *cnv;
1922 UErrorCode errorCode;
1923 int32_t ccsids[]={ 37, 850, 943, 949, 950, 1047, 1252, 1392, 33722 };
1924 int32_t i, ccsid;
1925
1926 for(i=0; i<(int32_t)(sizeof(ccsids)/sizeof(int32_t)); ++i) {
1927 ccsid=ccsids[i];
1928
1929 errorCode=U_ZERO_ERROR;
1930 cnv=ucnv_openCCSID(ccsid, UCNV_IBM, &errorCode);
1931 if(U_FAILURE(errorCode)) {
1932 log_data_err("error: ucnv_openCCSID(%ld) failed (%s)\n", ccsid, u_errorName(errorCode));
1933 continue;
1934 }
1935
1936 if(ccsid!=ucnv_getCCSID(cnv, &errorCode)) {
1937 log_err("error: ucnv_getCCSID(ucnv_openCCSID(%ld))=%ld\n", ccsid, ucnv_getCCSID(cnv, &errorCode));
1938 }
1939
1940 /* skip gb18030(ccsid 1392) */
1941 if(ccsid != 1392 && UCNV_IBM!=ucnv_getPlatform(cnv, &errorCode)) {
1942 log_err("error: ucnv_getPlatform(ucnv_openCCSID(%ld))=%ld!=UCNV_IBM\n", ccsid, ucnv_getPlatform(cnv, &errorCode));
1943 }
1944
1945 ucnv_close(cnv);
1946 }
1947 #endif
1948 }
1949
1950 /* jitterbug 932: ucnv_convert() bugs --------------------------------------- */
1951
1952 /* CHUNK_SIZE defined in common\ucnv.c: */
1953 #define CHUNK_SIZE 1024
1954
1955 static void bug1(void);
1956 static void bug2(void);
1957 static void bug3(void);
1958
1959 static void
TestJ932(void)1960 TestJ932(void)
1961 {
1962 bug1(); /* Unicode intermediate buffer straddle bug */
1963 bug2(); /* pre-flighting size incorrect caused by simple overflow */
1964 bug3(); /* pre-flighting size incorrect caused by expansion overflow */
1965 }
1966
1967 /*
1968 * jitterbug 932: test chunking boundary conditions in
1969
1970 int32_t ucnv_convert(const char *toConverterName,
1971 const char *fromConverterName,
1972 char *target,
1973 int32_t targetSize,
1974 const char *source,
1975 int32_t sourceSize,
1976 UErrorCode * err)
1977
1978 * See discussions on the icu mailing list in
1979 * 2001-April with the subject "converter 'flush' question".
1980 *
1981 * Bug report and test code provided by Edward J. Batutis.
1982 */
bug1()1983 static void bug1()
1984 {
1985 #if !UCONFIG_NO_LEGACY_CONVERSION
1986 char char_in[CHUNK_SIZE+32];
1987 char char_out[CHUNK_SIZE*2];
1988
1989 /* GB 18030 equivalent of U+10000 is 90308130 */
1990 static const char test_seq[]={ (char)0x90u, 0x30, (char)0x81u, 0x30 };
1991
1992 UErrorCode err = U_ZERO_ERROR;
1993 int32_t i, test_seq_len = sizeof(test_seq);
1994
1995 /*
1996 * causes straddle bug in Unicode intermediate buffer by sliding the test sequence forward
1997 * until the straddle bug appears. I didn't want to hard-code everything so this test could
1998 * be expanded - however this is the only type of straddle bug I can think of at the moment -
1999 * a high surrogate in the last position of the Unicode intermediate buffer. Apparently no
2000 * other Unicode sequences cause a bug since combining sequences are not supported by the
2001 * converters.
2002 */
2003
2004 for (i = test_seq_len; i >= 0; i--) {
2005 /* put character sequence into input buffer */
2006 memset(char_in, 0x61, sizeof(char_in)); /* GB 18030 'a' */
2007 memcpy(char_in + (CHUNK_SIZE - i), test_seq, test_seq_len);
2008
2009 /* do the conversion */
2010 ucnv_convert("us-ascii", /* out */
2011 "gb18030", /* in */
2012 char_out,
2013 sizeof(char_out),
2014 char_in,
2015 sizeof(char_in),
2016 &err);
2017
2018 /* bug1: */
2019 if (err == U_TRUNCATED_CHAR_FOUND) {
2020 /* this happens when surrogate pair straddles the intermediate buffer in
2021 * T_UConverter_fromCodepageToCodepage */
2022 log_err("error j932 bug 1: expected success, got U_TRUNCATED_CHAR_FOUND\n");
2023 }
2024 }
2025 #endif
2026 }
2027
2028 /* bug2: pre-flighting loop bug: simple overflow causes bug */
bug2()2029 static void bug2()
2030 {
2031 /* US-ASCII "1234567890" */
2032 static const char source[]={ 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39 };
2033 static const char sourceUTF8[]={ 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, (char)0xef, (char)0x80, (char)0x80 };
2034 static const char sourceUTF32[]={ 0x00, 0x00, 0x00, 0x30,
2035 0x00, 0x00, 0x00, 0x31,
2036 0x00, 0x00, 0x00, 0x32,
2037 0x00, 0x00, 0x00, 0x33,
2038 0x00, 0x00, 0x00, 0x34,
2039 0x00, 0x00, 0x00, 0x35,
2040 0x00, 0x00, 0x00, 0x36,
2041 0x00, 0x00, 0x00, 0x37,
2042 0x00, 0x00, 0x00, 0x38,
2043 0x00, 0x00, (char)0xf0, 0x00};
2044 static char target[5];
2045
2046 UErrorCode err = U_ZERO_ERROR;
2047 int32_t size;
2048
2049 /* do the conversion */
2050 size = ucnv_convert("iso-8859-1", /* out */
2051 "us-ascii", /* in */
2052 target,
2053 sizeof(target),
2054 source,
2055 sizeof(source),
2056 &err);
2057
2058 if ( size != 10 ) {
2059 /* bug2: size is 5, should be 10 */
2060 log_data_err("error j932 bug 2 us-ascii->iso-8859-1: got preflighting size %d instead of 10\n", size);
2061 }
2062
2063 err = U_ZERO_ERROR;
2064 /* do the conversion */
2065 size = ucnv_convert("UTF-32BE", /* out */
2066 "UTF-8", /* in */
2067 target,
2068 sizeof(target),
2069 sourceUTF8,
2070 sizeof(sourceUTF8),
2071 &err);
2072
2073 if ( size != 32 ) {
2074 /* bug2: size is 5, should be 32 */
2075 log_err("error j932 bug 2 UTF-8->UTF-32BE: got preflighting size %d instead of 32\n", size);
2076 }
2077
2078 err = U_ZERO_ERROR;
2079 /* do the conversion */
2080 size = ucnv_convert("UTF-8", /* out */
2081 "UTF-32BE", /* in */
2082 target,
2083 sizeof(target),
2084 sourceUTF32,
2085 sizeof(sourceUTF32),
2086 &err);
2087
2088 if ( size != 12 ) {
2089 /* bug2: size is 5, should be 12 */
2090 log_err("error j932 bug 2 UTF-32BE->UTF-8: got preflighting size %d instead of 12\n", size);
2091 }
2092 }
2093
2094 /*
2095 * bug3: when the characters expand going from source to target codepage
2096 * you get bug3 in addition to bug2
2097 */
bug3()2098 static void bug3()
2099 {
2100 #if !UCONFIG_NO_LEGACY_CONVERSION
2101 char char_in[CHUNK_SIZE*4];
2102 char target[5];
2103 UErrorCode err = U_ZERO_ERROR;
2104 int32_t size;
2105
2106 /*
2107 * first get the buggy size from bug2 then
2108 * compare it to buggy size with an expansion
2109 */
2110 memset(char_in, 0x61, sizeof(char_in)); /* US-ASCII 'a' */
2111
2112 /* do the conversion */
2113 size = ucnv_convert("lmbcs", /* out */
2114 "us-ascii", /* in */
2115 target,
2116 sizeof(target),
2117 char_in,
2118 sizeof(char_in),
2119 &err);
2120
2121 if ( size != sizeof(char_in) ) {
2122 /*
2123 * bug2: size is 0x2805 (CHUNK_SIZE*2+5 - maybe 5 is the size of the overflow buffer
2124 * in the converter?), should be CHUNK_SIZE*4
2125 *
2126 * Markus 2001-05-18: 5 is the size of our target[] here, ucnv_convert() did not reset targetSize...
2127 */
2128 log_data_err("error j932 bug 2/3a: expected preflighting size 0x%04x, got 0x%04x\n", sizeof(char_in), size);
2129 }
2130
2131 /*
2132 * now do the conversion with expansion
2133 * ascii 0x08 expands to 0x0F 0x28 in lmbcs
2134 */
2135 memset(char_in, 8, sizeof(char_in));
2136 err = U_ZERO_ERROR;
2137
2138 /* do the conversion */
2139 size = ucnv_convert("lmbcs", /* out */
2140 "us-ascii", /* in */
2141 target,
2142 sizeof(target),
2143 char_in,
2144 sizeof(char_in),
2145 &err);
2146
2147 /* expect 2X expansion */
2148 if ( size != sizeof(char_in) * 2 ) {
2149 /*
2150 * bug3:
2151 * bug2 would lead us to expect 0x2805, but it isn't that either, it is 0x3c05:
2152 */
2153 log_data_err("error j932 bug 3b: expected 0x%04x, got 0x%04x\n", sizeof(char_in) * 2, size);
2154 }
2155 #endif
2156 }
2157
2158 static void
convertExStreaming(UConverter * srcCnv,UConverter * targetCnv,const char * src,int32_t srcLength,const char * expectTarget,int32_t expectTargetLength,int32_t chunkSize,const char * testName,UErrorCode expectCode)2159 convertExStreaming(UConverter *srcCnv, UConverter *targetCnv,
2160 const char *src, int32_t srcLength,
2161 const char *expectTarget, int32_t expectTargetLength,
2162 int32_t chunkSize,
2163 const char *testName,
2164 UErrorCode expectCode) {
2165 UChar pivotBuffer[CHUNK_SIZE];
2166 UChar *pivotSource, *pivotTarget;
2167 const UChar *pivotLimit;
2168
2169 char targetBuffer[CHUNK_SIZE];
2170 char *target;
2171 const char *srcLimit, *finalSrcLimit, *targetLimit;
2172
2173 int32_t targetLength;
2174
2175 UBool flush;
2176
2177 UErrorCode errorCode;
2178
2179 /* setup */
2180 if(chunkSize>CHUNK_SIZE) {
2181 chunkSize=CHUNK_SIZE;
2182 }
2183
2184 pivotSource=pivotTarget=pivotBuffer;
2185 pivotLimit=pivotBuffer+chunkSize;
2186
2187 finalSrcLimit=src+srcLength;
2188 target=targetBuffer;
2189 targetLimit=targetBuffer+chunkSize;
2190
2191 ucnv_resetToUnicode(srcCnv);
2192 ucnv_resetFromUnicode(targetCnv);
2193
2194 errorCode=U_ZERO_ERROR;
2195 flush=FALSE;
2196
2197 /* convert, streaming-style (both converters and pivot keep state) */
2198 for(;;) {
2199 /* for testing, give ucnv_convertEx() at most <chunkSize> input/pivot/output units at a time */
2200 if(src+chunkSize<=finalSrcLimit) {
2201 srcLimit=src+chunkSize;
2202 } else {
2203 srcLimit=finalSrcLimit;
2204 }
2205 ucnv_convertEx(targetCnv, srcCnv,
2206 &target, targetLimit,
2207 &src, srcLimit,
2208 pivotBuffer, &pivotSource, &pivotTarget, pivotLimit,
2209 FALSE, flush, &errorCode);
2210 targetLength=(int32_t)(target-targetBuffer);
2211 if(target>targetLimit) {
2212 log_err("ucnv_convertEx(%s) chunk[%d] target %p exceeds targetLimit %p\n",
2213 testName, chunkSize, target, targetLimit);
2214 break; /* TODO: major problem! */
2215 }
2216 if(errorCode==U_BUFFER_OVERFLOW_ERROR) {
2217 /* continue converting another chunk */
2218 errorCode=U_ZERO_ERROR;
2219 if(targetLength+chunkSize<=sizeof(targetBuffer)) {
2220 targetLimit=target+chunkSize;
2221 } else {
2222 targetLimit=targetBuffer+sizeof(targetBuffer);
2223 }
2224 } else if(U_FAILURE(errorCode)) {
2225 /* failure */
2226 break;
2227 } else if(flush) {
2228 /* all done */
2229 break;
2230 } else if(src==finalSrcLimit && pivotSource==pivotTarget) {
2231 /* all consumed, now flush without input (separate from conversion for testing) */
2232 flush=TRUE;
2233 }
2234 }
2235
2236 if(!(errorCode==expectCode || (expectCode==U_ZERO_ERROR && errorCode==U_STRING_NOT_TERMINATED_WARNING))) {
2237 log_err("ucnv_convertEx(%s) chunk[%d] results in %s instead of %s\n",
2238 testName, chunkSize, u_errorName(errorCode), u_errorName(expectCode));
2239 } else if(targetLength!=expectTargetLength) {
2240 log_err("ucnv_convertEx(%s) chunk[%d] writes %d bytes instead of %d\n",
2241 testName, chunkSize, targetLength, expectTargetLength);
2242 } else if(memcmp(targetBuffer, expectTarget, targetLength)!=0) {
2243 log_err("ucnv_convertEx(%s) chunk[%d] writes different bytes than expected\n",
2244 testName, chunkSize);
2245 }
2246 }
2247
2248 static void
convertExMultiStreaming(UConverter * srcCnv,UConverter * targetCnv,const char * src,int32_t srcLength,const char * expectTarget,int32_t expectTargetLength,const char * testName,UErrorCode expectCode)2249 convertExMultiStreaming(UConverter *srcCnv, UConverter *targetCnv,
2250 const char *src, int32_t srcLength,
2251 const char *expectTarget, int32_t expectTargetLength,
2252 const char *testName,
2253 UErrorCode expectCode) {
2254 convertExStreaming(srcCnv, targetCnv,
2255 src, srcLength,
2256 expectTarget, expectTargetLength,
2257 1, testName, expectCode);
2258 convertExStreaming(srcCnv, targetCnv,
2259 src, srcLength,
2260 expectTarget, expectTargetLength,
2261 3, testName, expectCode);
2262 convertExStreaming(srcCnv, targetCnv,
2263 src, srcLength,
2264 expectTarget, expectTargetLength,
2265 7, testName, expectCode);
2266 }
2267
TestConvertEx()2268 static void TestConvertEx() {
2269 #if !UCONFIG_NO_LEGACY_CONVERSION
2270 static const uint8_t
2271 utf8[]={
2272 /* 4e00 30a1 ff61 0410 */
2273 0xe4, 0xb8, 0x80, 0xe3, 0x82, 0xa1, 0xef, 0xbd, 0xa1, 0xd0, 0x90
2274 },
2275 shiftJIS[]={
2276 0x88, 0xea, 0x83, 0x40, 0xa1, 0x84, 0x40
2277 },
2278 errorTarget[]={
2279 /*
2280 * expected output when converting shiftJIS[] from UTF-8 to Shift-JIS:
2281 * SUB, SUB, 0x40, SUB, SUB, 0x40
2282 */
2283 0xfc, 0xfc, 0xfc, 0xfc, 0x40, 0xfc, 0xfc, 0xfc, 0xfc, 0x40
2284 };
2285
2286 char srcBuffer[100], targetBuffer[100];
2287
2288 const char *src;
2289 char *target;
2290
2291 UChar pivotBuffer[100];
2292 UChar *pivotSource, *pivotTarget;
2293
2294 UConverter *cnv1, *cnv2;
2295 UErrorCode errorCode;
2296
2297 errorCode=U_ZERO_ERROR;
2298 cnv1=ucnv_open("UTF-8", &errorCode);
2299 if(U_FAILURE(errorCode)) {
2300 log_err("unable to open a UTF-8 converter - %s\n", u_errorName(errorCode));
2301 return;
2302 }
2303
2304 cnv2=ucnv_open("Shift-JIS", &errorCode);
2305 if(U_FAILURE(errorCode)) {
2306 log_data_err("unable to open a Shift-JIS converter - %s\n", u_errorName(errorCode));
2307 ucnv_close(cnv1);
2308 return;
2309 }
2310
2311 /* test ucnv_convertEx() with streaming conversion style */
2312 convertExMultiStreaming(cnv1, cnv2,
2313 (const char *)utf8, sizeof(utf8), (const char *)shiftJIS, sizeof(shiftJIS),
2314 "UTF-8 -> Shift-JIS", U_ZERO_ERROR);
2315
2316 convertExMultiStreaming(cnv2, cnv1,
2317 (const char *)shiftJIS, sizeof(shiftJIS), (const char *)utf8, sizeof(utf8),
2318 "Shift-JIS -> UTF-8", U_ZERO_ERROR);
2319
2320 /* U_ZERO_ERROR because by default the SUB callbacks are set */
2321 convertExMultiStreaming(cnv1, cnv2,
2322 (const char *)shiftJIS, sizeof(shiftJIS), (const char *)errorTarget, sizeof(errorTarget),
2323 "shiftJIS[] UTF-8 -> Shift-JIS", U_ZERO_ERROR);
2324
2325 /* test some simple conversions */
2326
2327 /* NUL-terminated source and target */
2328 errorCode=U_STRING_NOT_TERMINATED_WARNING;
2329 memcpy(srcBuffer, utf8, sizeof(utf8));
2330 srcBuffer[sizeof(utf8)]=0;
2331 src=srcBuffer;
2332 target=targetBuffer;
2333 ucnv_convertEx(cnv2, cnv1, &target, targetBuffer+sizeof(targetBuffer), &src, NULL,
2334 NULL, NULL, NULL, NULL, TRUE, TRUE, &errorCode);
2335 if( errorCode!=U_ZERO_ERROR ||
2336 target-targetBuffer!=sizeof(shiftJIS) ||
2337 *target!=0 ||
2338 memcmp(targetBuffer, shiftJIS, sizeof(shiftJIS))!=0
2339 ) {
2340 log_err("ucnv_convertEx(simple UTF-8 -> Shift_JIS) fails: %s - writes %d bytes, expect %d\n",
2341 u_errorName(errorCode), target-targetBuffer, sizeof(shiftJIS));
2342 }
2343
2344 /* NUL-terminated source and U_STRING_NOT_TERMINATED_WARNING */
2345 errorCode=U_AMBIGUOUS_ALIAS_WARNING;
2346 memset(targetBuffer, 0xff, sizeof(targetBuffer));
2347 src=srcBuffer;
2348 target=targetBuffer;
2349 ucnv_convertEx(cnv2, cnv1, &target, targetBuffer+sizeof(shiftJIS), &src, NULL,
2350 NULL, NULL, NULL, NULL, TRUE, TRUE, &errorCode);
2351 if( errorCode!=U_STRING_NOT_TERMINATED_WARNING ||
2352 target-targetBuffer!=sizeof(shiftJIS) ||
2353 *target!=(char)0xff ||
2354 memcmp(targetBuffer, shiftJIS, sizeof(shiftJIS))!=0
2355 ) {
2356 log_err("ucnv_convertEx(simple UTF-8 -> Shift_JIS) fails: %s, expect U_STRING_NOT_TERMINATED_WARNING - writes %d bytes, expect %d\n",
2357 u_errorName(errorCode), target-targetBuffer, sizeof(shiftJIS));
2358 }
2359
2360 /* bad arguments */
2361 errorCode=U_MESSAGE_PARSE_ERROR;
2362 src=srcBuffer;
2363 target=targetBuffer;
2364 ucnv_convertEx(cnv2, cnv1, &target, targetBuffer+sizeof(targetBuffer), &src, NULL,
2365 NULL, NULL, NULL, NULL, TRUE, TRUE, &errorCode);
2366 if(errorCode!=U_MESSAGE_PARSE_ERROR) {
2367 log_err("ucnv_convertEx(U_MESSAGE_PARSE_ERROR) sets %s\n", u_errorName(errorCode));
2368 }
2369
2370 /* pivotLimit==pivotStart */
2371 errorCode=U_ZERO_ERROR;
2372 pivotSource=pivotTarget=pivotBuffer;
2373 ucnv_convertEx(cnv2, cnv1, &target, targetBuffer+sizeof(targetBuffer), &src, NULL,
2374 pivotBuffer, &pivotSource, &pivotTarget, pivotBuffer, TRUE, TRUE, &errorCode);
2375 if(errorCode!=U_ILLEGAL_ARGUMENT_ERROR) {
2376 log_err("ucnv_convertEx(pivotLimit==pivotStart) sets %s\n", u_errorName(errorCode));
2377 }
2378
2379 /* *pivotSource==NULL */
2380 errorCode=U_ZERO_ERROR;
2381 pivotSource=NULL;
2382 ucnv_convertEx(cnv2, cnv1, &target, targetBuffer+sizeof(targetBuffer), &src, NULL,
2383 pivotBuffer, &pivotSource, &pivotTarget, pivotBuffer+1, TRUE, TRUE, &errorCode);
2384 if(errorCode!=U_ILLEGAL_ARGUMENT_ERROR) {
2385 log_err("ucnv_convertEx(*pivotSource==NULL) sets %s\n", u_errorName(errorCode));
2386 }
2387
2388 /* *source==NULL */
2389 errorCode=U_ZERO_ERROR;
2390 src=NULL;
2391 pivotSource=pivotBuffer;
2392 ucnv_convertEx(cnv2, cnv1, &target, targetBuffer+sizeof(targetBuffer), &src, NULL,
2393 pivotBuffer, &pivotSource, &pivotTarget, pivotBuffer+1, TRUE, TRUE, &errorCode);
2394 if(errorCode!=U_ILLEGAL_ARGUMENT_ERROR) {
2395 log_err("ucnv_convertEx(*source==NULL) sets %s\n", u_errorName(errorCode));
2396 }
2397
2398 /* streaming conversion without a pivot buffer */
2399 errorCode=U_ZERO_ERROR;
2400 src=srcBuffer;
2401 pivotSource=pivotBuffer;
2402 ucnv_convertEx(cnv2, cnv1, &target, targetBuffer+sizeof(targetBuffer), &src, NULL,
2403 NULL, &pivotSource, &pivotTarget, pivotBuffer+1, TRUE, FALSE, &errorCode);
2404 if(errorCode!=U_ILLEGAL_ARGUMENT_ERROR) {
2405 log_err("ucnv_convertEx(pivotStart==NULL) sets %s\n", u_errorName(errorCode));
2406 }
2407
2408 ucnv_close(cnv1);
2409 ucnv_close(cnv2);
2410 #endif
2411 }
2412
2413 /* Test illegal UTF-8 input: Data and functions for TestConvertExFromUTF8(). */
2414 static const char *const badUTF8[]={
2415 /* trail byte */
2416 "\x80",
2417
2418 /* truncated multi-byte sequences */
2419 "\xd0",
2420 "\xe0",
2421 "\xe1",
2422 "\xed",
2423 "\xee",
2424 "\xf0",
2425 "\xf1",
2426 "\xf4",
2427 "\xf8",
2428 "\xfc",
2429
2430 "\xe0\x80",
2431 "\xe0\xa0",
2432 "\xe1\x80",
2433 "\xed\x80",
2434 "\xed\xa0",
2435 "\xee\x80",
2436 "\xf0\x80",
2437 "\xf0\x90",
2438 "\xf1\x80",
2439 "\xf4\x80",
2440 "\xf4\x90",
2441 "\xf8\x80",
2442 "\xfc\x80",
2443
2444 "\xf0\x80\x80",
2445 "\xf0\x90\x80",
2446 "\xf1\x80\x80",
2447 "\xf4\x80\x80",
2448 "\xf4\x90\x80",
2449 "\xf8\x80\x80",
2450 "\xfc\x80\x80",
2451
2452 "\xf8\x80\x80\x80",
2453 "\xfc\x80\x80\x80",
2454
2455 "\xfc\x80\x80\x80\x80",
2456
2457 /* complete sequences but non-shortest forms or out of range etc. */
2458 "\xc0\x80",
2459 "\xe0\x80\x80",
2460 "\xed\xa0\x80",
2461 "\xf0\x80\x80\x80",
2462 "\xf4\x90\x80\x80",
2463 "\xf8\x80\x80\x80\x80",
2464 "\xfc\x80\x80\x80\x80\x80",
2465 "\xfe",
2466 "\xff"
2467 };
2468
2469 /* get some character that can be converted and convert it */
getTestChar(UConverter * cnv,const char * converterName,char charUTF8[4],int32_t * pCharUTF8Length,char char0[8],int32_t * pChar0Length,char char1[8],int32_t * pChar1Length)2470 static UBool getTestChar(UConverter *cnv, const char *converterName,
2471 char charUTF8[4], int32_t *pCharUTF8Length,
2472 char char0[8], int32_t *pChar0Length,
2473 char char1[8], int32_t *pChar1Length) {
2474 UChar utf16[U16_MAX_LENGTH];
2475 int32_t utf16Length;
2476
2477 const UChar *utf16Source;
2478 char *target;
2479
2480 USet *set;
2481 UChar32 c;
2482 UErrorCode errorCode;
2483
2484 errorCode=U_ZERO_ERROR;
2485 set=uset_open(1, 0);
2486 ucnv_getUnicodeSet(cnv, set, UCNV_ROUNDTRIP_SET, &errorCode);
2487 c=uset_charAt(set, uset_size(set)/2);
2488 uset_close(set);
2489
2490 utf16Length=0;
2491 U16_APPEND_UNSAFE(utf16, utf16Length, c);
2492 *pCharUTF8Length=0;
2493 U8_APPEND_UNSAFE(charUTF8, *pCharUTF8Length, c);
2494
2495 utf16Source=utf16;
2496 target=char0;
2497 ucnv_fromUnicode(cnv,
2498 &target, char0+sizeof(char0),
2499 &utf16Source, utf16+utf16Length,
2500 NULL, FALSE, &errorCode);
2501 *pChar0Length=(int32_t)(target-char0);
2502
2503 utf16Source=utf16;
2504 target=char1;
2505 ucnv_fromUnicode(cnv,
2506 &target, char1+sizeof(char1),
2507 &utf16Source, utf16+utf16Length,
2508 NULL, FALSE, &errorCode);
2509 *pChar1Length=(int32_t)(target-char1);
2510
2511 if(U_FAILURE(errorCode)) {
2512 log_err("unable to get test character for %s - %s\n", converterName, u_errorName(errorCode));
2513 return FALSE;
2514 }
2515 return TRUE;
2516 }
2517
testFromTruncatedUTF8(UConverter * utf8Cnv,UConverter * cnv,const char * converterName,char charUTF8[4],int32_t charUTF8Length,char char0[8],int32_t char0Length,char char1[8],int32_t char1Length)2518 static void testFromTruncatedUTF8(UConverter *utf8Cnv, UConverter *cnv, const char *converterName,
2519 char charUTF8[4], int32_t charUTF8Length,
2520 char char0[8], int32_t char0Length,
2521 char char1[8], int32_t char1Length) {
2522 char utf8[16];
2523 int32_t utf8Length;
2524
2525 char output[16];
2526 int32_t outputLength;
2527
2528 char invalidChars[8];
2529 int8_t invalidLength;
2530
2531 const char *source;
2532 char *target;
2533
2534 UChar pivotBuffer[8];
2535 UChar *pivotSource, *pivotTarget;
2536
2537 UErrorCode errorCode;
2538 int32_t i;
2539
2540 /* test truncated sequences */
2541 errorCode=U_ZERO_ERROR;
2542 ucnv_setToUCallBack(utf8Cnv, UCNV_TO_U_CALLBACK_STOP, NULL, NULL, NULL, &errorCode);
2543
2544 memcpy(utf8, charUTF8, charUTF8Length);
2545
2546 for(i=0; i<LENGTHOF(badUTF8); ++i) {
2547 /* truncated sequence? */
2548 int32_t length=strlen(badUTF8[i]);
2549 if(length>=(1+U8_COUNT_TRAIL_BYTES(badUTF8[i][0]))) {
2550 continue;
2551 }
2552
2553 /* assemble a string with the test character and the truncated sequence */
2554 memcpy(utf8+charUTF8Length, badUTF8[i], length);
2555 utf8Length=charUTF8Length+length;
2556
2557 /* convert and check the invalidChars */
2558 source=utf8;
2559 target=output;
2560 pivotSource=pivotTarget=pivotBuffer;
2561 errorCode=U_ZERO_ERROR;
2562 ucnv_convertEx(cnv, utf8Cnv,
2563 &target, output+sizeof(output),
2564 &source, utf8+utf8Length,
2565 pivotBuffer, &pivotSource, &pivotTarget, pivotBuffer+LENGTHOF(pivotBuffer),
2566 TRUE, TRUE, /* reset & flush */
2567 &errorCode);
2568 outputLength=(int32_t)(target-output);
2569 if(errorCode!=U_TRUNCATED_CHAR_FOUND || pivotSource!=pivotBuffer) {
2570 log_err("unexpected error %s from %s badUTF8[%ld]\n", u_errorName(errorCode), converterName, (long)i);
2571 continue;
2572 }
2573
2574 errorCode=U_ZERO_ERROR;
2575 invalidLength=(int8_t)sizeof(invalidChars);
2576 ucnv_getInvalidChars(utf8Cnv, invalidChars, &invalidLength, &errorCode);
2577 if(invalidLength!=length || 0!=memcmp(invalidChars, badUTF8[i], length)) {
2578 log_err("wrong invalidChars from %s badUTF8[%ld]\n", converterName, (long)i);
2579 }
2580 }
2581 }
2582
testFromBadUTF8(UConverter * utf8Cnv,UConverter * cnv,const char * converterName,char charUTF8[4],int32_t charUTF8Length,char char0[8],int32_t char0Length,char char1[8],int32_t char1Length)2583 static void testFromBadUTF8(UConverter *utf8Cnv, UConverter *cnv, const char *converterName,
2584 char charUTF8[4], int32_t charUTF8Length,
2585 char char0[8], int32_t char0Length,
2586 char char1[8], int32_t char1Length) {
2587 char utf8[600], expect[600];
2588 int32_t utf8Length, expectLength;
2589
2590 char testName[32];
2591
2592 UErrorCode errorCode;
2593 int32_t i;
2594
2595 errorCode=U_ZERO_ERROR;
2596 ucnv_setToUCallBack(utf8Cnv, UCNV_TO_U_CALLBACK_SKIP, NULL, NULL, NULL, &errorCode);
2597
2598 /*
2599 * assemble an input string with the test character between each
2600 * bad sequence,
2601 * and an expected string with repeated test character output
2602 */
2603 memcpy(utf8, charUTF8, charUTF8Length);
2604 utf8Length=charUTF8Length;
2605
2606 memcpy(expect, char0, char0Length);
2607 expectLength=char0Length;
2608
2609 for(i=0; i<LENGTHOF(badUTF8); ++i) {
2610 int32_t length=strlen(badUTF8[i]);
2611 memcpy(utf8+utf8Length, badUTF8[i], length);
2612 utf8Length+=length;
2613
2614 memcpy(utf8+utf8Length, charUTF8, charUTF8Length);
2615 utf8Length+=charUTF8Length;
2616
2617 memcpy(expect+expectLength, char1, char1Length);
2618 expectLength+=char1Length;
2619 }
2620
2621 /* expect that each bad UTF-8 sequence is detected and skipped */
2622 strcpy(testName, "from bad UTF-8 to ");
2623 strcat(testName, converterName);
2624
2625 convertExMultiStreaming(utf8Cnv, cnv,
2626 utf8, utf8Length,
2627 expect, expectLength,
2628 testName,
2629 U_ZERO_ERROR);
2630 }
2631
2632 /* Test illegal UTF-8 input. */
TestConvertExFromUTF8()2633 static void TestConvertExFromUTF8() {
2634 static const char *const converterNames[]={
2635 #if !UCONFIG_NO_LEGACY_CONVERSION
2636 "windows-1252",
2637 "shift-jis",
2638 #endif
2639 "us-ascii",
2640 "iso-8859-1",
2641 "utf-8"
2642 };
2643
2644 UConverter *utf8Cnv, *cnv;
2645 UErrorCode errorCode;
2646 int32_t i;
2647
2648 /* fromUnicode versions of some character, from initial state and later */
2649 char charUTF8[4], char0[8], char1[8];
2650 int32_t charUTF8Length, char0Length, char1Length;
2651
2652 errorCode=U_ZERO_ERROR;
2653 utf8Cnv=ucnv_open("UTF-8", &errorCode);
2654 if(U_FAILURE(errorCode)) {
2655 log_err("unable to open UTF-8 converter - %s\n", u_errorName(errorCode));
2656 return;
2657 }
2658
2659 for(i=0; i<LENGTHOF(converterNames); ++i) {
2660 errorCode=U_ZERO_ERROR;
2661 cnv=ucnv_open(converterNames[i], &errorCode);
2662 if(U_FAILURE(errorCode)) {
2663 log_err("unable to open %s converter - %s\n", converterNames[i], u_errorName(errorCode));
2664 continue;
2665 }
2666 if(!getTestChar(cnv, converterNames[i], charUTF8, &charUTF8Length, char0, &char0Length, char1, &char1Length)) {
2667 continue;
2668 }
2669 testFromTruncatedUTF8(utf8Cnv, cnv, converterNames[i], charUTF8, charUTF8Length, char0, char0Length, char1, char1Length);
2670 testFromBadUTF8(utf8Cnv, cnv, converterNames[i], charUTF8, charUTF8Length, char0, char0Length, char1, char1Length);
2671 ucnv_close(cnv);
2672 }
2673 ucnv_close(utf8Cnv);
2674 }
2675
2676 static void
TestConvertAlgorithmic()2677 TestConvertAlgorithmic() {
2678 #if !UCONFIG_NO_LEGACY_CONVERSION
2679 static const uint8_t
2680 utf8[]={
2681 /* 4e00 30a1 ff61 0410 */
2682 0xe4, 0xb8, 0x80, 0xe3, 0x82, 0xa1, 0xef, 0xbd, 0xa1, 0xd0, 0x90
2683 },
2684 shiftJIS[]={
2685 0x88, 0xea, 0x83, 0x40, 0xa1, 0x84, 0x40
2686 },
2687 /*errorTarget[]={*/
2688 /*
2689 * expected output when converting shiftJIS[] from UTF-8 to Shift-JIS:
2690 * SUB, SUB, 0x40, SUB, SUB, 0x40
2691 */
2692 /* 0x81, 0xa1, 0x81, 0xa1, 0x40, 0x81, 0xa1, 0x81, 0xa1, 0x40*/
2693 /*},*/
2694 utf16[]={
2695 0xfe, 0xff /* BOM only, no text */
2696 },
2697 utf32[]={
2698 0xff, 0xfe, 0, 0 /* BOM only, no text */
2699 };
2700
2701 char target[100], utf8NUL[100], shiftJISNUL[100];
2702
2703 UConverter *cnv;
2704 UErrorCode errorCode;
2705
2706 int32_t length;
2707
2708 errorCode=U_ZERO_ERROR;
2709 cnv=ucnv_open("Shift-JIS", &errorCode);
2710 if(U_FAILURE(errorCode)) {
2711 log_data_err("unable to open a Shift-JIS converter - %s\n", u_errorName(errorCode));
2712 ucnv_close(cnv);
2713 return;
2714 }
2715
2716 memcpy(utf8NUL, utf8, sizeof(utf8));
2717 utf8NUL[sizeof(utf8)]=0;
2718 memcpy(shiftJISNUL, shiftJIS, sizeof(shiftJIS));
2719 shiftJISNUL[sizeof(shiftJIS)]=0;
2720
2721 /*
2722 * The to/from algorithmic convenience functions share a common implementation,
2723 * so we need not test all permutations of them.
2724 */
2725
2726 /* length in, not terminated out */
2727 errorCode=U_ZERO_ERROR;
2728 length=ucnv_fromAlgorithmic(cnv, UCNV_UTF8, target, sizeof(shiftJIS), (const char *)utf8, sizeof(utf8), &errorCode);
2729 if( errorCode!=U_STRING_NOT_TERMINATED_WARNING ||
2730 length!=sizeof(shiftJIS) ||
2731 memcmp(target, shiftJIS, length)!=0
2732 ) {
2733 log_err("ucnv_fromAlgorithmic(UTF-8 -> Shift-JIS) fails (%s expect U_STRING_NOT_TERMINATED_WARNING), returns %d expect %d\n",
2734 u_errorName(errorCode), length, sizeof(shiftJIS));
2735 }
2736
2737 /* terminated in and out */
2738 memset(target, 0x55, sizeof(target));
2739 errorCode=U_STRING_NOT_TERMINATED_WARNING;
2740 length=ucnv_toAlgorithmic(UCNV_UTF8, cnv, target, sizeof(target), shiftJISNUL, -1, &errorCode);
2741 if( errorCode!=U_ZERO_ERROR ||
2742 length!=sizeof(utf8) ||
2743 memcmp(target, utf8, length)!=0
2744 ) {
2745 log_err("ucnv_toAlgorithmic(Shift-JIS -> UTF-8) fails (%s expect U_ZERO_ERROR), returns %d expect %d\n",
2746 u_errorName(errorCode), length, sizeof(shiftJIS));
2747 }
2748
2749 /* empty string, some target buffer */
2750 errorCode=U_STRING_NOT_TERMINATED_WARNING;
2751 length=ucnv_toAlgorithmic(UCNV_UTF8, cnv, target, sizeof(target), shiftJISNUL, 0, &errorCode);
2752 if( errorCode!=U_ZERO_ERROR ||
2753 length!=0
2754 ) {
2755 log_err("ucnv_toAlgorithmic(empty string -> UTF-8) fails (%s expect U_ZERO_ERROR), returns %d expect 0\n",
2756 u_errorName(errorCode), length);
2757 }
2758
2759 /* pseudo-empty string, no target buffer */
2760 errorCode=U_ZERO_ERROR;
2761 length=ucnv_fromAlgorithmic(cnv, UCNV_UTF16, target, 0, (const char *)utf16, 2, &errorCode);
2762 if( errorCode!=U_STRING_NOT_TERMINATED_WARNING ||
2763 length!=0
2764 ) {
2765 log_err("ucnv_fromAlgorithmic(UTF-16 only BOM -> Shift-JIS) fails (%s expect U_STRING_NOT_TERMINATED_WARNING), returns %d expect 0\n",
2766 u_errorName(errorCode), length);
2767 }
2768
2769 errorCode=U_ZERO_ERROR;
2770 length=ucnv_fromAlgorithmic(cnv, UCNV_UTF32, target, 0, (const char *)utf32, 4, &errorCode);
2771 if( errorCode!=U_STRING_NOT_TERMINATED_WARNING ||
2772 length!=0
2773 ) {
2774 log_err("ucnv_fromAlgorithmic(UTF-32 only BOM -> Shift-JIS) fails (%s expect U_STRING_NOT_TERMINATED_WARNING), returns %d expect 0\n",
2775 u_errorName(errorCode), length);
2776 }
2777
2778 /* bad arguments */
2779 errorCode=U_MESSAGE_PARSE_ERROR;
2780 length=ucnv_fromAlgorithmic(cnv, UCNV_UTF16, target, 0, (const char *)utf16, 2, &errorCode);
2781 if(errorCode!=U_MESSAGE_PARSE_ERROR) {
2782 log_err("ucnv_fromAlgorithmic(U_MESSAGE_PARSE_ERROR) sets %s\n", u_errorName(errorCode));
2783 }
2784
2785 /* source==NULL */
2786 errorCode=U_ZERO_ERROR;
2787 length=ucnv_fromAlgorithmic(cnv, UCNV_UTF16, target, 0, NULL, 2, &errorCode);
2788 if(errorCode!=U_ILLEGAL_ARGUMENT_ERROR) {
2789 log_err("ucnv_fromAlgorithmic(source==NULL) sets %s\n", u_errorName(errorCode));
2790 }
2791
2792 /* illegal alg. type */
2793 errorCode=U_ZERO_ERROR;
2794 length=ucnv_fromAlgorithmic(cnv, (UConverterType)99, target, 0, (const char *)utf16, 2, &errorCode);
2795 if(errorCode!=U_ILLEGAL_ARGUMENT_ERROR) {
2796 log_err("ucnv_fromAlgorithmic(illegal alg. type) sets %s\n", u_errorName(errorCode));
2797 }
2798 ucnv_close(cnv);
2799 #endif
2800 }
2801
TestLMBCSMaxChar(void)2802 static void TestLMBCSMaxChar(void) {
2803 static const struct {
2804 int8_t maxSize;
2805 const char *name;
2806 } converter[] = {
2807 /* some non-LMBCS converters - perfect test setup here */
2808 { 1, "US-ASCII"},
2809 { 1, "ISO-8859-1"},
2810
2811 { 2, "UTF-16"},
2812 { 2, "UTF-16BE"},
2813 { 3, "UTF-8"},
2814 { 3, "CESU-8"},
2815 { 3, "SCSU"},
2816 { 4, "UTF-32"},
2817 { 4, "UTF-7"},
2818 { 4, "IMAP-mailbox-name"},
2819 { 4, "BOCU-1"},
2820
2821 { 1, "windows-1256"},
2822 { 2, "Shift-JIS"},
2823 { 2, "ibm-16684"},
2824 { 3, "ibm-930"},
2825 { 3, "ibm-1390"},
2826 { 4, "*test3"},
2827 { 16,"*test4"},
2828
2829 { 4, "ISCII"},
2830 { 4, "HZ"},
2831
2832 { 3, "ISO-2022"},
2833 { 3, "ISO-2022-KR"},
2834 { 6, "ISO-2022-JP"},
2835 { 8, "ISO-2022-CN"},
2836
2837 /* LMBCS */
2838 { 3, "LMBCS-1"},
2839 { 3, "LMBCS-2"},
2840 { 3, "LMBCS-3"},
2841 { 3, "LMBCS-4"},
2842 { 3, "LMBCS-5"},
2843 { 3, "LMBCS-6"},
2844 { 3, "LMBCS-8"},
2845 { 3, "LMBCS-11"},
2846 { 3, "LMBCS-16"},
2847 { 3, "LMBCS-17"},
2848 { 3, "LMBCS-18"},
2849 { 3, "LMBCS-19"}
2850 };
2851 int32_t idx;
2852
2853 for (idx = 0; idx < LENGTHOF(converter); idx++) {
2854 UErrorCode status = U_ZERO_ERROR;
2855 UConverter *cnv = cnv_open(converter[idx].name, &status);
2856 if (U_FAILURE(status)) {
2857 continue;
2858 }
2859 if (converter[idx].maxSize != ucnv_getMaxCharSize(cnv)) {
2860 log_err("error: ucnv_getMaxCharSize(%s) expected %d, got %d\n",
2861 converter[idx].name, converter[idx].maxSize, ucnv_getMaxCharSize(cnv));
2862 }
2863 ucnv_close(cnv);
2864 }
2865
2866 /* mostly test that the macro compiles */
2867 if(UCNV_GET_MAX_BYTES_FOR_STRING(1, 2)<10) {
2868 log_err("error UCNV_GET_MAX_BYTES_FOR_STRING(1, 2)<10\n");
2869 }
2870 }
2871
2872
TestJ1968(void)2873 static void TestJ1968(void) {
2874 UErrorCode err = U_ZERO_ERROR;
2875 UConverter *cnv;
2876 char myConvName[] = "My really really really really really really really really really really really"
2877 " really really really really really really really really really really really"
2878 " really really really really really really really really long converter name";
2879 UChar myConvNameU[sizeof(myConvName)];
2880
2881 u_charsToUChars(myConvName, myConvNameU, sizeof(myConvName));
2882
2883 err = U_ZERO_ERROR;
2884 myConvNameU[UCNV_MAX_CONVERTER_NAME_LENGTH+1] = 0;
2885 cnv = ucnv_openU(myConvNameU, &err);
2886 if (cnv || err != U_ILLEGAL_ARGUMENT_ERROR) {
2887 log_err("1U) Didn't get U_ILLEGAL_ARGUMENT_ERROR as expected %s\n", u_errorName(err));
2888 }
2889
2890 err = U_ZERO_ERROR;
2891 myConvNameU[UCNV_MAX_CONVERTER_NAME_LENGTH] = 0;
2892 cnv = ucnv_openU(myConvNameU, &err);
2893 if (cnv || err != U_ILLEGAL_ARGUMENT_ERROR) {
2894 log_err("2U) Didn't get U_ILLEGAL_ARGUMENT_ERROR as expected %s\n", u_errorName(err));
2895 }
2896
2897 err = U_ZERO_ERROR;
2898 myConvNameU[UCNV_MAX_CONVERTER_NAME_LENGTH-1] = 0;
2899 cnv = ucnv_openU(myConvNameU, &err);
2900 if (cnv || err != U_FILE_ACCESS_ERROR) {
2901 log_err("3U) Didn't get U_FILE_ACCESS_ERROR as expected %s\n", u_errorName(err));
2902 }
2903
2904
2905
2906
2907 err = U_ZERO_ERROR;
2908 cnv = ucnv_open(myConvName, &err);
2909 if (cnv || err != U_ILLEGAL_ARGUMENT_ERROR) {
2910 log_err("1) Didn't get U_ILLEGAL_ARGUMENT_ERROR as expected %s\n", u_errorName(err));
2911 }
2912
2913 err = U_ZERO_ERROR;
2914 myConvName[UCNV_MAX_CONVERTER_NAME_LENGTH] = ',';
2915 cnv = ucnv_open(myConvName, &err);
2916 if (cnv || err != U_ILLEGAL_ARGUMENT_ERROR) {
2917 log_err("2) Didn't get U_ILLEGAL_ARGUMENT_ERROR as expected %s\n", u_errorName(err));
2918 }
2919
2920 err = U_ZERO_ERROR;
2921 myConvName[UCNV_MAX_CONVERTER_NAME_LENGTH-1] = ',';
2922 cnv = ucnv_open(myConvName, &err);
2923 if (cnv || err != U_FILE_ACCESS_ERROR) {
2924 log_err("3) Didn't get U_FILE_ACCESS_ERROR as expected %s\n", u_errorName(err));
2925 }
2926
2927 err = U_ZERO_ERROR;
2928 myConvName[UCNV_MAX_CONVERTER_NAME_LENGTH-1] = ',';
2929 strncpy(myConvName + UCNV_MAX_CONVERTER_NAME_LENGTH, "locale=", 7);
2930 cnv = ucnv_open(myConvName, &err);
2931 if (cnv || err != U_ILLEGAL_ARGUMENT_ERROR) {
2932 log_err("4) Didn't get U_ILLEGAL_ARGUMENT_ERROR as expected %s\n", u_errorName(err));
2933 }
2934
2935 /* The comma isn't really a part of the converter name. */
2936 err = U_ZERO_ERROR;
2937 myConvName[UCNV_MAX_CONVERTER_NAME_LENGTH] = 0;
2938 cnv = ucnv_open(myConvName, &err);
2939 if (cnv || err != U_FILE_ACCESS_ERROR) {
2940 log_err("5) Didn't get U_FILE_ACCESS_ERROR as expected %s\n", u_errorName(err));
2941 }
2942
2943 err = U_ZERO_ERROR;
2944 myConvName[UCNV_MAX_CONVERTER_NAME_LENGTH-1] = ' ';
2945 cnv = ucnv_open(myConvName, &err);
2946 if (cnv || err != U_ILLEGAL_ARGUMENT_ERROR) {
2947 log_err("6) Didn't get U_ILLEGAL_ARGUMENT_ERROR as expected %s\n", u_errorName(err));
2948 }
2949
2950 err = U_ZERO_ERROR;
2951 myConvName[UCNV_MAX_CONVERTER_NAME_LENGTH-1] = 0;
2952 cnv = ucnv_open(myConvName, &err);
2953 if (cnv || err != U_FILE_ACCESS_ERROR) {
2954 log_err("7) Didn't get U_FILE_ACCESS_ERROR as expected %s\n", u_errorName(err));
2955 }
2956
2957 }
2958
2959 #if !UCONFIG_NO_LEGACY_CONVERSION
2960 static void
testSwap(const char * name,UBool swap)2961 testSwap(const char *name, UBool swap) {
2962 /*
2963 * Test Unicode text.
2964 * Contains characters that are the highest for some of the
2965 * tested conversions, to make sure that the ucnvmbcs.c code that modifies the
2966 * tables copies the entire tables.
2967 */
2968 static const UChar text[]={
2969 0x61, 0xd, 0x62, 0xa, 0x4e00, 0x3000, 0xfffd, 0xa, 0x20, 0x85, 0xff5e, 0x7a
2970 };
2971
2972 UChar uNormal[32], uSwapped[32];
2973 char normal[32], swapped[32];
2974 const UChar *pcu;
2975 UChar *pu;
2976 char *pc;
2977 int32_t i, normalLength, swappedLength;
2978 UChar u;
2979 char c;
2980
2981 const char *swappedName;
2982 UConverter *cnv, *swapCnv;
2983 UErrorCode errorCode;
2984
2985 /* if the swap flag is FALSE, then the test encoding is not EBCDIC and must not swap */
2986
2987 /* open both the normal and the LF/NL-swapping converters */
2988 strcpy(swapped, name);
2989 strcat(swapped, UCNV_SWAP_LFNL_OPTION_STRING);
2990
2991 errorCode=U_ZERO_ERROR;
2992 swapCnv=ucnv_open(swapped, &errorCode);
2993 cnv=ucnv_open(name, &errorCode);
2994 if(U_FAILURE(errorCode)) {
2995 log_data_err("TestEBCDICSwapLFNL error: unable to open %s or %s (%s)\n", name, swapped, u_errorName(errorCode));
2996 goto cleanup;
2997 }
2998
2999 /* the name must contain the swap option if and only if we expect the converter to swap */
3000 swappedName=ucnv_getName(swapCnv, &errorCode);
3001 if(U_FAILURE(errorCode)) {
3002 log_err("TestEBCDICSwapLFNL error: ucnv_getName(%s,swaplfnl) failed (%s)\n", name, u_errorName(errorCode));
3003 goto cleanup;
3004 }
3005
3006 pc=strstr(swappedName, UCNV_SWAP_LFNL_OPTION_STRING);
3007 if(swap != (pc!=NULL)) {
3008 log_err("TestEBCDICSwapLFNL error: ucnv_getName(%s,swaplfnl)=%s should (%d) contain 'swaplfnl'\n", name, swappedName, swap);
3009 goto cleanup;
3010 }
3011
3012 /* convert to EBCDIC */
3013 pcu=text;
3014 pc=normal;
3015 ucnv_fromUnicode(cnv, &pc, normal+LENGTHOF(normal), &pcu, text+LENGTHOF(text), NULL, TRUE, &errorCode);
3016 normalLength=(int32_t)(pc-normal);
3017
3018 pcu=text;
3019 pc=swapped;
3020 ucnv_fromUnicode(swapCnv, &pc, swapped+LENGTHOF(swapped), &pcu, text+LENGTHOF(text), NULL, TRUE, &errorCode);
3021 swappedLength=(int32_t)(pc-swapped);
3022
3023 if(U_FAILURE(errorCode)) {
3024 log_err("TestEBCDICSwapLFNL error converting to %s - (%s)\n", name, u_errorName(errorCode));
3025 goto cleanup;
3026 }
3027
3028 /* compare EBCDIC output */
3029 if(normalLength!=swappedLength) {
3030 log_err("TestEBCDICSwapLFNL error converting to %s - output lengths %d vs. %d\n", name, normalLength, swappedLength);
3031 goto cleanup;
3032 }
3033 for(i=0; i<normalLength; ++i) {
3034 /* swap EBCDIC LF/NL for comparison */
3035 c=normal[i];
3036 if(swap) {
3037 if(c==0x15) {
3038 c=0x25;
3039 } else if(c==0x25) {
3040 c=0x15;
3041 }
3042 }
3043
3044 if(c!=swapped[i]) {
3045 log_err("TestEBCDICSwapLFNL error converting to %s - did not swap properly, output[%d]=0x%02x\n", name, i, (uint8_t)swapped[i]);
3046 goto cleanup;
3047 }
3048 }
3049
3050 /* convert back to Unicode (may not roundtrip) */
3051 pc=normal;
3052 pu=uNormal;
3053 ucnv_toUnicode(cnv, &pu, uNormal+LENGTHOF(uNormal), (const char **)&pc, normal+normalLength, NULL, TRUE, &errorCode);
3054 normalLength=(int32_t)(pu-uNormal);
3055
3056 pc=normal;
3057 pu=uSwapped;
3058 ucnv_toUnicode(swapCnv, &pu, uSwapped+LENGTHOF(uSwapped), (const char **)&pc, normal+swappedLength, NULL, TRUE, &errorCode);
3059 swappedLength=(int32_t)(pu-uSwapped);
3060
3061 if(U_FAILURE(errorCode)) {
3062 log_err("TestEBCDICSwapLFNL error converting from %s - (%s)\n", name, u_errorName(errorCode));
3063 goto cleanup;
3064 }
3065
3066 /* compare EBCDIC output */
3067 if(normalLength!=swappedLength) {
3068 log_err("TestEBCDICSwapLFNL error converting from %s - output lengths %d vs. %d\n", name, normalLength, swappedLength);
3069 goto cleanup;
3070 }
3071 for(i=0; i<normalLength; ++i) {
3072 /* swap EBCDIC LF/NL for comparison */
3073 u=uNormal[i];
3074 if(swap) {
3075 if(u==0xa) {
3076 u=0x85;
3077 } else if(u==0x85) {
3078 u=0xa;
3079 }
3080 }
3081
3082 if(u!=uSwapped[i]) {
3083 log_err("TestEBCDICSwapLFNL error converting from %s - did not swap properly, output[%d]=U+%04x\n", name, i, uSwapped[i]);
3084 goto cleanup;
3085 }
3086 }
3087
3088 /* clean up */
3089 cleanup:
3090 ucnv_close(cnv);
3091 ucnv_close(swapCnv);
3092 }
3093
3094 static void
TestEBCDICSwapLFNL()3095 TestEBCDICSwapLFNL() {
3096 static const struct {
3097 const char *name;
3098 UBool swap;
3099 } tests[]={
3100 { "ibm-37", TRUE },
3101 { "ibm-1047", TRUE },
3102 { "ibm-1140", TRUE },
3103 { "ibm-930", TRUE },
3104 { "iso-8859-3", FALSE }
3105 };
3106
3107 int i;
3108
3109 for(i=0; i<LENGTHOF(tests); ++i) {
3110 testSwap(tests[i].name, tests[i].swap);
3111 }
3112 }
3113 #else
3114 static void
TestEBCDICSwapLFNL()3115 TestEBCDICSwapLFNL() {
3116 /* test nothing... */
3117 }
3118 #endif
3119
3120 static const UVersionInfo ICU_34 = {3,4,0,0};
3121
TestFromUCountPending()3122 static void TestFromUCountPending(){
3123 #if !UCONFIG_NO_LEGACY_CONVERSION
3124 UErrorCode status = U_ZERO_ERROR;
3125 /* const UChar expectedUnicode[] = { 0x20ac, 0x0005, 0x0006, 0x000b, 0xdbc4, 0xde34, 0xd84d, 0xdc56, 0xfffd}; */
3126 static const struct {
3127 UChar input[6];
3128 int32_t len;
3129 int32_t exp;
3130 }fromUnicodeTests[] = {
3131 /*m:n conversion*/
3132 {{0xdbc4},1,1},
3133 {{ 0xdbc4, 0xde34, 0xd84d},3,1},
3134 {{ 0xdbc4, 0xde34, 0xd900},3,3},
3135 };
3136 int i;
3137 UConverter* cnv = ucnv_openPackage(loadTestData(&status), "test3", &status);
3138 if(U_FAILURE(status)){
3139 log_err("Could not create converter for test3. Error: %s\n", u_errorName(status));
3140 return;
3141 }
3142 for(i=0; i<LENGTHOF(fromUnicodeTests); ++i) {
3143 char tgt[10];
3144 char* target = tgt;
3145 char* targetLimit = target + 10;
3146 const UChar* source = fromUnicodeTests[i].input;
3147 const UChar* sourceLimit = source + fromUnicodeTests[i].len;
3148 int32_t len = 0;
3149 ucnv_reset(cnv);
3150 ucnv_fromUnicode(cnv,&target, targetLimit, &source, sourceLimit, NULL, FALSE, &status);
3151 len = ucnv_fromUCountPending(cnv, &status);
3152 if(U_FAILURE(status)){
3153 log_err("ucnv_fromUnicode call did not succeed. Error: %s\n", u_errorName(status));
3154 status = U_ZERO_ERROR;
3155 continue;
3156 }
3157 if(len != fromUnicodeTests[i].exp){
3158 log_err("Did not get the expeced output for ucnv_fromUInputConsumed.\n");
3159 }
3160 }
3161 status = U_ZERO_ERROR;
3162 {
3163 /*
3164 * The converter has to read the tail before it knows that
3165 * only head alone matches.
3166 * At the end, the output for head will overflow the target,
3167 * middle will be pending, and tail will not have been consumed.
3168 */
3169 /*
3170 \U00101234 -> x (<U101234> \x07 |0)
3171 \U00101234\U00050005 -> y (<U101234>+<U50005> \x07+\x00+\x01\x02\x0e+\x05 |0)
3172 \U00101234\U00050005\U00060006 -> z (<U101234>+<U50005>+<U60006> \x07+\x00+\x01\x02\x0f+\x09 |0)
3173 \U00060007 -> unassigned
3174 */
3175 static const UChar head[] = {0xDBC4,0xDE34,0xD900,0xDC05,0x0000};/* \U00101234\U00050005 */
3176 static const UChar middle[] = {0xD940,0x0000}; /* first half of \U00060006 or \U00060007 */
3177 static const UChar tail[] = {0xDC07,0x0000};/* second half of \U00060007 */
3178 char tgt[10];
3179 char* target = tgt;
3180 char* targetLimit = target + 2; /* expect overflow from converting \U00101234\U00050005 */
3181 const UChar* source = head;
3182 const UChar* sourceLimit = source + u_strlen(head);
3183 int32_t len = 0;
3184 ucnv_reset(cnv);
3185 ucnv_fromUnicode(cnv,&target, targetLimit, &source, sourceLimit, NULL, FALSE, &status);
3186 len = ucnv_fromUCountPending(cnv, &status);
3187 if(U_FAILURE(status)){
3188 log_err("ucnv_fromUnicode call did not succeed. Error: %s\n", u_errorName(status));
3189 status = U_ZERO_ERROR;
3190 }
3191 if(len!=4){
3192 log_err("ucnv_fromUInputHeld did not return correct length for head\n");
3193 }
3194 source = middle;
3195 sourceLimit = source + u_strlen(middle);
3196 ucnv_fromUnicode(cnv,&target, targetLimit, &source, sourceLimit, NULL, FALSE, &status);
3197 len = ucnv_fromUCountPending(cnv, &status);
3198 if(U_FAILURE(status)){
3199 log_err("ucnv_fromUnicode call did not succeed. Error: %s\n", u_errorName(status));
3200 status = U_ZERO_ERROR;
3201 }
3202 if(len!=5){
3203 log_err("ucnv_fromUInputHeld did not return correct length for middle\n");
3204 }
3205 source = tail;
3206 sourceLimit = source + u_strlen(tail);
3207 ucnv_fromUnicode(cnv,&target, targetLimit, &source, sourceLimit, NULL, FALSE, &status);
3208 if(status != U_BUFFER_OVERFLOW_ERROR){
3209 log_err("ucnv_fromUnicode call did not succeed. Error: %s\n", u_errorName(status));
3210 }
3211 status = U_ZERO_ERROR;
3212 len = ucnv_fromUCountPending(cnv, &status);
3213 /* middle[1] is pending, tail has not been consumed */
3214 if(U_FAILURE(status)){
3215 log_err("ucnv_fromUInputHeld call did not succeed. Error: %s\n", u_errorName(status));
3216 }
3217 if(len!=1){
3218 log_err("ucnv_fromUInputHeld did not return correct length for tail\n");
3219 }
3220 }
3221 ucnv_close(cnv);
3222 #endif
3223 }
3224
3225 static void
TestToUCountPending()3226 TestToUCountPending(){
3227 #if !UCONFIG_NO_LEGACY_CONVERSION
3228 UErrorCode status = U_ZERO_ERROR;
3229 static const struct {
3230 char input[6];
3231 int32_t len;
3232 int32_t exp;
3233 }toUnicodeTests[] = {
3234 /*m:n conversion*/
3235 {{0x05, 0x01, 0x02},3,3},
3236 {{0x01, 0x02},2,2},
3237 {{0x07, 0x00, 0x01, 0x02},4,4},
3238 };
3239
3240 int i;
3241 UConverterToUCallback *oldToUAction= NULL;
3242 UConverter* cnv = ucnv_openPackage(loadTestData(&status), "test3", &status);
3243 if(U_FAILURE(status)){
3244 log_err("Could not create converter for test3. Error: %s\n", u_errorName(status));
3245 return;
3246 }
3247 ucnv_setToUCallBack(cnv, UCNV_TO_U_CALLBACK_STOP, NULL, oldToUAction, NULL, &status);
3248 for(i=0; i<LENGTHOF(toUnicodeTests); ++i) {
3249 UChar tgt[10];
3250 UChar* target = tgt;
3251 UChar* targetLimit = target + 20;
3252 const char* source = toUnicodeTests[i].input;
3253 const char* sourceLimit = source + toUnicodeTests[i].len;
3254 int32_t len = 0;
3255 ucnv_reset(cnv);
3256 ucnv_toUnicode(cnv,&target, targetLimit, &source, sourceLimit, NULL, FALSE, &status);
3257 len = ucnv_toUCountPending(cnv,&status);
3258 if(U_FAILURE(status)){
3259 log_err("ucnv_toUnicode call did not succeed. Error: %s\n", u_errorName(status));
3260 status = U_ZERO_ERROR;
3261 continue;
3262 }
3263 if(len != toUnicodeTests[i].exp){
3264 log_err("Did not get the expeced output for ucnv_toUInputConsumed.\n");
3265 }
3266 }
3267 status = U_ZERO_ERROR;
3268 ucnv_close(cnv);
3269
3270 {
3271 /*
3272 * The converter has to read the tail before it knows that
3273 * only head alone matches.
3274 * At the end, the output for head will overflow the target,
3275 * mid will be pending, and tail will not have been consumed.
3276 */
3277 char head[] = { 0x01, 0x02, 0x03, 0x0a , 0x00};
3278 char mid[] = { 0x01, 0x02, 0x03, 0x0b, 0x00 };
3279 char tail[] = { 0x01, 0x02, 0x03, 0x0d, 0x00 };
3280 /*
3281 0x01, 0x02, 0x03, 0x0a -> x (<U23456> \x01\x02\x03\x0a |0)
3282 0x01, 0x02, 0x03, 0x0b -> y (<U000b> \x01\x02\x03\x0b |0)
3283 0x01, 0x02, 0x03, 0x0d -> z (<U34567> \x01\x02\x03\x0d |3)
3284 0x01, 0x02, 0x03, 0x0a + 0x01, 0x02, 0x03, 0x0b + 0x01 + many more -> z (see test4 "many bytes, and bytes per UChar")
3285 */
3286 UChar tgt[10];
3287 UChar* target = tgt;
3288 UChar* targetLimit = target + 1; /* expect overflow from converting */
3289 const char* source = head;
3290 const char* sourceLimit = source + strlen(head);
3291 int32_t len = 0;
3292 cnv = ucnv_openPackage(loadTestData(&status), "test4", &status);
3293 if(U_FAILURE(status)){
3294 log_err("Could not create converter for test3. Error: %s\n", u_errorName(status));
3295 return;
3296 }
3297 ucnv_setToUCallBack(cnv, UCNV_TO_U_CALLBACK_STOP, NULL, oldToUAction, NULL, &status);
3298 ucnv_toUnicode(cnv,&target, targetLimit, &source, sourceLimit, NULL, FALSE, &status);
3299 len = ucnv_toUCountPending(cnv,&status);
3300 if(U_FAILURE(status)){
3301 log_err("ucnv_toUnicode call did not succeed. Error: %s\n", u_errorName(status));
3302 }
3303 if(len != 4){
3304 log_err("Did not get the expected len for head.\n");
3305 }
3306 source=mid;
3307 sourceLimit = source+strlen(mid);
3308 ucnv_toUnicode(cnv,&target, targetLimit, &source, sourceLimit, NULL, FALSE, &status);
3309 len = ucnv_toUCountPending(cnv,&status);
3310 if(U_FAILURE(status)){
3311 log_err("ucnv_toUnicode call did not succeed. Error: %s\n", u_errorName(status));
3312 }
3313 if(len != 8){
3314 log_err("Did not get the expected len for mid.\n");
3315 }
3316
3317 source=tail;
3318 sourceLimit = source+strlen(tail);
3319 targetLimit = target;
3320 ucnv_toUnicode(cnv,&target, targetLimit, &source, sourceLimit, NULL, FALSE, &status);
3321 if(status != U_BUFFER_OVERFLOW_ERROR){
3322 log_err("ucnv_toUnicode call did not succeed. Error: %s\n", u_errorName(status));
3323 }
3324 status = U_ZERO_ERROR;
3325 len = ucnv_toUCountPending(cnv,&status);
3326 /* mid[4] is pending, tail has not been consumed */
3327 if(U_FAILURE(status)){
3328 log_err("ucnv_toUCountPending call did not succeed. Error: %s\n", u_errorName(status));
3329 }
3330 if(len != 4){
3331 log_err("Did not get the expected len for tail.\n");
3332 }
3333 ucnv_close(cnv);
3334 }
3335 #endif
3336 }
3337
TestOneDefaultNameChange(const char * name)3338 static void TestOneDefaultNameChange(const char *name) {
3339 UErrorCode status = U_ZERO_ERROR;
3340 UConverter *cnv;
3341 ucnv_setDefaultName(name);
3342 if(strcmp(ucnv_getDefaultName(), name)==0)
3343 log_verbose("setDefaultName of %s works.\n", name);
3344 else
3345 log_err("setDefaultName of %s failed\n", name);
3346 cnv=ucnv_open(NULL, &status);
3347 if (U_FAILURE(status) || cnv == NULL) {
3348 log_err("opening the default converter of %s failed\n", name);
3349 return;
3350 }
3351 if(strcmp(ucnv_getName(cnv, &status), name)==0)
3352 log_verbose("ucnv_getName of %s works.\n", name);
3353 else
3354 log_err("ucnv_getName of %s failed\n", name);
3355 ucnv_close(cnv);
3356 }
3357
TestDefaultName(void)3358 static void TestDefaultName(void) {
3359 /*Testing ucnv_getDefaultName() and ucnv_setDefaultNAme()*/
3360 static char defaultName[UCNV_MAX_CONVERTER_NAME_LENGTH + 1];
3361 strcpy(defaultName, ucnv_getDefaultName());
3362
3363 log_verbose("getDefaultName returned %s\n", defaultName);
3364
3365 /*change the default name by setting it */
3366 TestOneDefaultNameChange("UTF-8");
3367 #if !UCONFIG_NO_LEGACY_CONVERSION
3368 TestOneDefaultNameChange("ISCII,version=1");
3369 TestOneDefaultNameChange("ISCII,version=2");
3370 #endif
3371 TestOneDefaultNameChange("ISO-8859-1");
3372
3373 /*set the default name back*/
3374 ucnv_setDefaultName(defaultName);
3375 }
3376
3377 /* Test that ucnv_compareNames() matches names according to spec. ----------- */
3378
3379 static U_INLINE int
sign(int n)3380 sign(int n) {
3381 if(n==0) {
3382 return 0;
3383 } else if(n<0) {
3384 return -1;
3385 } else /* n>0 */ {
3386 return 1;
3387 }
3388 }
3389
3390 static void
compareNames(const char ** names)3391 compareNames(const char **names) {
3392 const char *relation, *name1, *name2;
3393 int rel, result;
3394
3395 relation=*names++;
3396 if(*relation=='=') {
3397 rel = 0;
3398 } else if(*relation=='<') {
3399 rel = -1;
3400 } else {
3401 rel = 1;
3402 }
3403
3404 name1=*names++;
3405 if(name1==NULL) {
3406 return;
3407 }
3408 while((name2=*names++)!=NULL) {
3409 result=ucnv_compareNames(name1, name2);
3410 if(sign(result)!=rel) {
3411 log_err("ucnv_compareNames(\"%s\", \"%s\")=%d, sign!=%d\n", name1, name2, result, rel);
3412 }
3413 name1=name2;
3414 }
3415 }
3416
3417 static void
TestCompareNames()3418 TestCompareNames() {
3419 static const char *equalUTF8[]={ "=", "UTF-8", "utf_8", "u*T@f08", "Utf 8", NULL };
3420 static const char *equalIBM[]={ "=", "ibm-37", "IBM037", "i-B-m 00037", "ibm-0037", "IBM00037", NULL };
3421 static const char *lessMac[]={ "<", "macos-0_1-10.2", "macos-1-10.0.2", "macos-1-10.2", NULL };
3422 static const char *lessUTF080[]={ "<", "UTF-0008", "utf$080", "u*T@f0800", "Utf 0000000009", NULL };
3423
3424 compareNames(equalUTF8);
3425 compareNames(equalIBM);
3426 compareNames(lessMac);
3427 compareNames(lessUTF080);
3428 }
3429
3430 static void
TestSubstString()3431 TestSubstString() {
3432 static const UChar surrogate[1]={ 0xd900 };
3433 char buffer[16];
3434
3435 static const UChar sub[5]={ 0x61, 0x62, 0x63, 0x64, 0x65 };
3436 static const char subChars[5]={ 0x61, 0x62, 0x63, 0x64, 0x65 };
3437 UConverter *cnv;
3438 UErrorCode errorCode;
3439 int32_t length;
3440 int8_t len8;
3441
3442 /* UTF-16/32: test that the BOM is output before the sub character */
3443 errorCode=U_ZERO_ERROR;
3444 cnv=ucnv_open("UTF-16", &errorCode);
3445 if(U_FAILURE(errorCode)) {
3446 log_err("ucnv_open(UTF-16) failed - %s\n", u_errorName(errorCode));
3447 return;
3448 }
3449 length=ucnv_fromUChars(cnv, buffer, (int32_t)sizeof(buffer), surrogate, 1, &errorCode);
3450 ucnv_close(cnv);
3451 if(U_FAILURE(errorCode) ||
3452 length!=4 ||
3453 NULL == ucnv_detectUnicodeSignature(buffer, length, NULL, &errorCode)
3454 ) {
3455 log_err("ucnv_fromUChars(UTF-16, U+D900) did not write a BOM\n");
3456 }
3457
3458 errorCode=U_ZERO_ERROR;
3459 cnv=ucnv_open("UTF-32", &errorCode);
3460 if(U_FAILURE(errorCode)) {
3461 log_err("ucnv_open(UTF-32) failed - %s\n", u_errorName(errorCode));
3462 return;
3463 }
3464 length=ucnv_fromUChars(cnv, buffer, (int32_t)sizeof(buffer), surrogate, 1, &errorCode);
3465 ucnv_close(cnv);
3466 if(U_FAILURE(errorCode) ||
3467 length!=8 ||
3468 NULL == ucnv_detectUnicodeSignature(buffer, length, NULL, &errorCode)
3469 ) {
3470 log_err("ucnv_fromUChars(UTF-32, U+D900) did not write a BOM\n");
3471 }
3472
3473 /* Simple API test of ucnv_setSubstString() + ucnv_getSubstChars(). */
3474 errorCode=U_ZERO_ERROR;
3475 cnv=ucnv_open("ISO-8859-1", &errorCode);
3476 if(U_FAILURE(errorCode)) {
3477 log_err("ucnv_open(ISO-8859-1) failed - %s\n", u_errorName(errorCode));
3478 return;
3479 }
3480 ucnv_setSubstString(cnv, sub, LENGTHOF(sub), &errorCode);
3481 if(U_FAILURE(errorCode)) {
3482 log_err("ucnv_setSubstString(ISO-8859-1, sub[5]) failed - %s\n", u_errorName(errorCode));
3483 } else {
3484 len8 = sizeof(buffer);
3485 ucnv_getSubstChars(cnv, buffer, &len8, &errorCode);
3486 /* Stateless converter, we expect the string converted to charset bytes. */
3487 if(U_FAILURE(errorCode) || len8!=sizeof(subChars) || 0!=uprv_memcmp(buffer, subChars, len8)) {
3488 log_err("ucnv_getSubstChars(ucnv_setSubstString(ISO-8859-1, sub[5])) failed - %s\n", u_errorName(errorCode));
3489 }
3490 }
3491 ucnv_close(cnv);
3492
3493 #if !UCONFIG_NO_LEGACY_CONVERSION
3494 errorCode=U_ZERO_ERROR;
3495 cnv=ucnv_open("HZ", &errorCode);
3496 if(U_FAILURE(errorCode)) {
3497 log_err("ucnv_open(HZ) failed - %s\n", u_errorName(errorCode));
3498 return;
3499 }
3500 ucnv_setSubstString(cnv, sub, LENGTHOF(sub), &errorCode);
3501 if(U_FAILURE(errorCode)) {
3502 log_err("ucnv_setSubstString(HZ, sub[5]) failed - %s\n", u_errorName(errorCode));
3503 } else {
3504 len8 = sizeof(buffer);
3505 ucnv_getSubstChars(cnv, buffer, &len8, &errorCode);
3506 /* Stateful converter, we expect that the Unicode string was set and that we get an empty char * string now. */
3507 if(U_FAILURE(errorCode) || len8!=0) {
3508 log_err("ucnv_getSubstChars(ucnv_setSubstString(HZ, sub[5])) failed - %s\n", u_errorName(errorCode));
3509 }
3510 }
3511 ucnv_close(cnv);
3512 #endif
3513 /*
3514 * Further testing of ucnv_setSubstString() is done via intltest convert.
3515 * We do not test edge cases of illegal arguments and similar because the
3516 * function implementation uses all of its parameters in calls to other
3517 * functions with UErrorCode parameters.
3518 */
3519 }
3520
3521 static void
InvalidArguments()3522 InvalidArguments() {
3523 UConverter *cnv;
3524 UErrorCode errorCode;
3525 char charBuffer[2] = {1, 1};
3526 char ucharAsCharBuffer[2] = {2, 2};
3527 char *charsPtr = charBuffer;
3528 UChar *ucharsPtr = (UChar *)ucharAsCharBuffer;
3529 UChar *ucharsBadPtr = (UChar *)(ucharAsCharBuffer + 1);
3530
3531 errorCode=U_ZERO_ERROR;
3532 cnv=ucnv_open("UTF-8", &errorCode);
3533 if(U_FAILURE(errorCode)) {
3534 log_err("ucnv_open() failed - %s\n", u_errorName(errorCode));
3535 return;
3536 }
3537
3538 errorCode=U_ZERO_ERROR;
3539 /* This one should fail because an incomplete UChar is being passed in */
3540 ucnv_fromUnicode(cnv, &charsPtr, charsPtr, (const UChar **)&ucharsPtr, ucharsBadPtr, NULL, TRUE, &errorCode);
3541 if(errorCode != U_ILLEGAL_ARGUMENT_ERROR) {
3542 log_err("ucnv_fromUnicode() failed to return U_ILLEGAL_ARGUMENT_ERROR for incomplete UChar * buffer - %s\n", u_errorName(errorCode));
3543 }
3544
3545 errorCode=U_ZERO_ERROR;
3546 /* This one should fail because ucharsBadPtr is > than ucharsPtr */
3547 ucnv_fromUnicode(cnv, &charsPtr, charsPtr, (const UChar **)&ucharsBadPtr, ucharsPtr, NULL, TRUE, &errorCode);
3548 if(errorCode != U_ILLEGAL_ARGUMENT_ERROR) {
3549 log_err("ucnv_fromUnicode() failed to return U_ILLEGAL_ARGUMENT_ERROR for bad limit pointer - %s\n", u_errorName(errorCode));
3550 }
3551
3552 errorCode=U_ZERO_ERROR;
3553 /* This one should fail because an incomplete UChar is being passed in */
3554 ucnv_toUnicode(cnv, &ucharsPtr, ucharsBadPtr, (const char **)&charsPtr, charsPtr, NULL, TRUE, &errorCode);
3555 if(errorCode != U_ILLEGAL_ARGUMENT_ERROR) {
3556 log_err("ucnv_toUnicode() failed to return U_ILLEGAL_ARGUMENT_ERROR for incomplete UChar * buffer - %s\n", u_errorName(errorCode));
3557 }
3558
3559 errorCode=U_ZERO_ERROR;
3560 /* This one should fail because ucharsBadPtr is > than ucharsPtr */
3561 ucnv_toUnicode(cnv, &ucharsBadPtr, ucharsPtr, (const char **)&charsPtr, charsPtr, NULL, TRUE, &errorCode);
3562 if(errorCode != U_ILLEGAL_ARGUMENT_ERROR) {
3563 log_err("ucnv_toUnicode() failed to return U_ILLEGAL_ARGUMENT_ERROR for bad limit pointer - %s\n", u_errorName(errorCode));
3564 }
3565
3566 if (charBuffer[0] != 1 || charBuffer[1] != 1
3567 || ucharAsCharBuffer[0] != 2 || ucharAsCharBuffer[1] != 2)
3568 {
3569 log_err("Data was incorrectly written to buffers\n");
3570 }
3571
3572 ucnv_close(cnv);
3573 }
3574
3575
3576