1 // © 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
3 /*
4 *******************************************************************************
5 *
6 * Copyright (C) 2003-2014, International Business Machines
7 * Corporation and others. All Rights Reserved.
8 *
9 *******************************************************************************
10 * file name: convtest.cpp
11 * encoding: UTF-8
12 * tab size: 8 (not used)
13 * indentation:4
14 *
15 * created on: 2003jul15
16 * created by: Markus W. Scherer
17 *
18 * Test file for data-driven conversion tests.
19 */
20
21 #include "unicode/utypes.h"
22
23 #if !UCONFIG_NO_LEGACY_CONVERSION
24 /*
25 * Note: Turning off all of convtest.cpp if !UCONFIG_NO_LEGACY_CONVERSION
26 * is slightly unnecessary - it removes tests for Unicode charsets
27 * like UTF-8 that should work.
28 * However, there is no easy way for the test to detect whether a test case
29 * is for a Unicode charset, so it would be difficult to only exclude those.
30 * Also, regular testing of ICU is done with all modules on, therefore
31 * not testing conversion for a custom configuration like this should be ok.
32 */
33
34 #include "unicode/ucnv.h"
35 #include "unicode/unistr.h"
36 #include "unicode/parsepos.h"
37 #include "unicode/uniset.h"
38 #include "unicode/ustring.h"
39 #include "unicode/ures.h"
40 #include "unicode/utf16.h"
41 #include "convtest.h"
42 #include "cmemory.h"
43 #include "unicode/tstdtmod.h"
44 #include <string.h>
45 #include <stdlib.h>
46
47 enum {
48 // characters used in test data for callbacks
49 SUB_CB='?',
50 SKIP_CB='0',
51 STOP_CB='.',
52 ESC_CB='&'
53 };
54
ConversionTest()55 ConversionTest::ConversionTest() {
56 UErrorCode errorCode=U_ZERO_ERROR;
57 utf8Cnv=ucnv_open("UTF-8", &errorCode);
58 ucnv_setToUCallBack(utf8Cnv, UCNV_TO_U_CALLBACK_STOP, NULL, NULL, NULL, &errorCode);
59 if(U_FAILURE(errorCode)) {
60 errln("unable to open UTF-8 converter");
61 }
62 }
63
~ConversionTest()64 ConversionTest::~ConversionTest() {
65 ucnv_close(utf8Cnv);
66 }
67
68 void
runIndexedTest(int32_t index,UBool exec,const char * & name,char *)69 ConversionTest::runIndexedTest(int32_t index, UBool exec, const char *&name, char * /*par*/) {
70 if (exec) logln("TestSuite ConversionTest: ");
71 TESTCASE_AUTO_BEGIN;
72 #if !UCONFIG_NO_FILE_IO
73 TESTCASE_AUTO(TestToUnicode);
74 TESTCASE_AUTO(TestFromUnicode);
75 TESTCASE_AUTO(TestGetUnicodeSet);
76 #endif
77 TESTCASE_AUTO(TestGetUnicodeSet2);
78 TESTCASE_AUTO(TestDefaultIgnorableCallback);
79 TESTCASE_AUTO(TestUTF8ToUTF8Overflow);
80 TESTCASE_AUTO(TestUTF8ToUTF8Streaming);
81 TESTCASE_AUTO_END;
82 }
83
84 // test data interface ----------------------------------------------------- ***
85
86 void
TestToUnicode()87 ConversionTest::TestToUnicode() {
88 ConversionCase cc;
89 char charset[100], cbopt[4];
90 const char *option;
91 UnicodeString s, unicode;
92 int32_t offsetsLength;
93 UConverterToUCallback callback;
94
95 TestDataModule *dataModule;
96 TestData *testData;
97 const DataMap *testCase;
98 UErrorCode errorCode;
99 int32_t i;
100
101 errorCode=U_ZERO_ERROR;
102 dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
103 if(U_SUCCESS(errorCode)) {
104 testData=dataModule->createTestData("toUnicode", errorCode);
105 if(U_SUCCESS(errorCode)) {
106 for(i=0; testData->nextCase(testCase, errorCode); ++i) {
107 if(U_FAILURE(errorCode)) {
108 errln("error retrieving conversion/toUnicode test case %d - %s",
109 i, u_errorName(errorCode));
110 errorCode=U_ZERO_ERROR;
111 continue;
112 }
113
114 cc.caseNr=i;
115
116 s=testCase->getString("charset", errorCode);
117 s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
118 cc.charset=charset;
119
120 cc.bytes=testCase->getBinary(cc.bytesLength, "bytes", errorCode);
121 unicode=testCase->getString("unicode", errorCode);
122 cc.unicode=unicode.getBuffer();
123 cc.unicodeLength=unicode.length();
124
125 offsetsLength=0;
126 cc.offsets=testCase->getIntVector(offsetsLength, "offsets", errorCode);
127 if(offsetsLength==0) {
128 cc.offsets=NULL;
129 } else if(offsetsLength!=unicode.length()) {
130 errln("toUnicode[%d] unicode[%d] and offsets[%d] must have the same length",
131 i, unicode.length(), offsetsLength);
132 errorCode=U_ILLEGAL_ARGUMENT_ERROR;
133 }
134
135 cc.finalFlush= 0!=testCase->getInt28("flush", errorCode);
136 cc.fallbacks= 0!=testCase->getInt28("fallbacks", errorCode);
137
138 s=testCase->getString("errorCode", errorCode);
139 if(s==UNICODE_STRING("invalid", 7)) {
140 cc.outErrorCode=U_INVALID_CHAR_FOUND;
141 } else if(s==UNICODE_STRING("illegal", 7)) {
142 cc.outErrorCode=U_ILLEGAL_CHAR_FOUND;
143 } else if(s==UNICODE_STRING("truncated", 9)) {
144 cc.outErrorCode=U_TRUNCATED_CHAR_FOUND;
145 } else if(s==UNICODE_STRING("illesc", 6)) {
146 cc.outErrorCode=U_ILLEGAL_ESCAPE_SEQUENCE;
147 } else if(s==UNICODE_STRING("unsuppesc", 9)) {
148 cc.outErrorCode=U_UNSUPPORTED_ESCAPE_SEQUENCE;
149 } else {
150 cc.outErrorCode=U_ZERO_ERROR;
151 }
152
153 s=testCase->getString("callback", errorCode);
154 s.extract(0, 0x7fffffff, cbopt, sizeof(cbopt), "");
155 cc.cbopt=cbopt;
156 switch(cbopt[0]) {
157 case SUB_CB:
158 callback=UCNV_TO_U_CALLBACK_SUBSTITUTE;
159 break;
160 case SKIP_CB:
161 callback=UCNV_TO_U_CALLBACK_SKIP;
162 break;
163 case STOP_CB:
164 callback=UCNV_TO_U_CALLBACK_STOP;
165 break;
166 case ESC_CB:
167 callback=UCNV_TO_U_CALLBACK_ESCAPE;
168 break;
169 default:
170 callback=NULL;
171 break;
172 }
173 option=callback==NULL ? cbopt : cbopt+1;
174 if(*option==0) {
175 option=NULL;
176 }
177
178 cc.invalidChars=testCase->getBinary(cc.invalidLength, "invalidChars", errorCode);
179
180 if(U_FAILURE(errorCode)) {
181 errln("error parsing conversion/toUnicode test case %d - %s",
182 i, u_errorName(errorCode));
183 errorCode=U_ZERO_ERROR;
184 } else {
185 logln("TestToUnicode[%d] %s", i, charset);
186 ToUnicodeCase(cc, callback, option);
187 }
188 }
189 delete testData;
190 }
191 delete dataModule;
192 }
193 else {
194 dataerrln("Could not load test conversion data");
195 }
196 }
197
198 void
TestFromUnicode()199 ConversionTest::TestFromUnicode() {
200 ConversionCase cc;
201 char charset[100], cbopt[4];
202 const char *option;
203 UnicodeString s, unicode, invalidUChars;
204 int32_t offsetsLength, index;
205 UConverterFromUCallback callback;
206
207 TestDataModule *dataModule;
208 TestData *testData;
209 const DataMap *testCase;
210 const UChar *p;
211 UErrorCode errorCode;
212 int32_t i, length;
213
214 errorCode=U_ZERO_ERROR;
215 dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
216 if(U_SUCCESS(errorCode)) {
217 testData=dataModule->createTestData("fromUnicode", errorCode);
218 if(U_SUCCESS(errorCode)) {
219 for(i=0; testData->nextCase(testCase, errorCode); ++i) {
220 if(U_FAILURE(errorCode)) {
221 errln("error retrieving conversion/fromUnicode test case %d - %s",
222 i, u_errorName(errorCode));
223 errorCode=U_ZERO_ERROR;
224 continue;
225 }
226
227 cc.caseNr=i;
228
229 s=testCase->getString("charset", errorCode);
230 s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
231 cc.charset=charset;
232
233 unicode=testCase->getString("unicode", errorCode);
234 cc.unicode=unicode.getBuffer();
235 cc.unicodeLength=unicode.length();
236 cc.bytes=testCase->getBinary(cc.bytesLength, "bytes", errorCode);
237
238 offsetsLength=0;
239 cc.offsets=testCase->getIntVector(offsetsLength, "offsets", errorCode);
240 if(offsetsLength==0) {
241 cc.offsets=NULL;
242 } else if(offsetsLength!=cc.bytesLength) {
243 errln("fromUnicode[%d] bytes[%d] and offsets[%d] must have the same length",
244 i, cc.bytesLength, offsetsLength);
245 errorCode=U_ILLEGAL_ARGUMENT_ERROR;
246 }
247
248 cc.finalFlush= 0!=testCase->getInt28("flush", errorCode);
249 cc.fallbacks= 0!=testCase->getInt28("fallbacks", errorCode);
250
251 s=testCase->getString("errorCode", errorCode);
252 if(s==UNICODE_STRING("invalid", 7)) {
253 cc.outErrorCode=U_INVALID_CHAR_FOUND;
254 } else if(s==UNICODE_STRING("illegal", 7)) {
255 cc.outErrorCode=U_ILLEGAL_CHAR_FOUND;
256 } else if(s==UNICODE_STRING("truncated", 9)) {
257 cc.outErrorCode=U_TRUNCATED_CHAR_FOUND;
258 } else {
259 cc.outErrorCode=U_ZERO_ERROR;
260 }
261
262 s=testCase->getString("callback", errorCode);
263 cc.setSub=0; // default: no subchar
264
265 if((index=s.indexOf((UChar)0))>0) {
266 // read NUL-separated subchar first, if any
267 // copy the subchar from Latin-1 characters
268 // start after the NUL
269 p=s.getTerminatedBuffer();
270 length=index+1;
271 p+=length;
272 length=s.length()-length;
273 if(length<=0 || length>=(int32_t)sizeof(cc.subchar)) {
274 errorCode=U_ILLEGAL_ARGUMENT_ERROR;
275 } else {
276 int32_t j;
277
278 for(j=0; j<length; ++j) {
279 cc.subchar[j]=(char)p[j];
280 }
281 // NUL-terminate the subchar
282 cc.subchar[j]=0;
283 cc.setSub=1;
284 }
285
286 // remove the NUL and subchar from s
287 s.truncate(index);
288 } else if((index=s.indexOf((UChar)0x3d))>0) /* '=' */ {
289 // read a substitution string, separated by an equal sign
290 p=s.getBuffer()+index+1;
291 length=s.length()-(index+1);
292 if(length<0 || length>=UPRV_LENGTHOF(cc.subString)) {
293 errorCode=U_ILLEGAL_ARGUMENT_ERROR;
294 } else {
295 u_memcpy(cc.subString, p, length);
296 // NUL-terminate the subString
297 cc.subString[length]=0;
298 cc.setSub=-1;
299 }
300
301 // remove the equal sign and subString from s
302 s.truncate(index);
303 }
304
305 s.extract(0, 0x7fffffff, cbopt, sizeof(cbopt), "");
306 cc.cbopt=cbopt;
307 switch(cbopt[0]) {
308 case SUB_CB:
309 callback=UCNV_FROM_U_CALLBACK_SUBSTITUTE;
310 break;
311 case SKIP_CB:
312 callback=UCNV_FROM_U_CALLBACK_SKIP;
313 break;
314 case STOP_CB:
315 callback=UCNV_FROM_U_CALLBACK_STOP;
316 break;
317 case ESC_CB:
318 callback=UCNV_FROM_U_CALLBACK_ESCAPE;
319 break;
320 default:
321 callback=NULL;
322 break;
323 }
324 option=callback==NULL ? cbopt : cbopt+1;
325 if(*option==0) {
326 option=NULL;
327 }
328
329 invalidUChars=testCase->getString("invalidUChars", errorCode);
330 cc.invalidUChars=invalidUChars.getBuffer();
331 cc.invalidLength=invalidUChars.length();
332
333 if(U_FAILURE(errorCode)) {
334 errln("error parsing conversion/fromUnicode test case %d - %s",
335 i, u_errorName(errorCode));
336 errorCode=U_ZERO_ERROR;
337 } else {
338 logln("TestFromUnicode[%d] %s", i, charset);
339 FromUnicodeCase(cc, callback, option);
340 }
341 }
342 delete testData;
343 }
344 delete dataModule;
345 }
346 else {
347 dataerrln("Could not load test conversion data");
348 }
349 }
350
351 static const UChar ellipsis[]={ 0x2e, 0x2e, 0x2e };
352
353 void
TestGetUnicodeSet()354 ConversionTest::TestGetUnicodeSet() {
355 char charset[100];
356 UnicodeString s, map, mapnot;
357 int32_t which;
358
359 ParsePosition pos;
360 UnicodeSet cnvSet, mapSet, mapnotSet, diffSet;
361 UnicodeSet *cnvSetPtr = &cnvSet;
362 LocalUConverterPointer cnv;
363
364 TestDataModule *dataModule;
365 TestData *testData;
366 const DataMap *testCase;
367 UErrorCode errorCode;
368 int32_t i;
369
370 errorCode=U_ZERO_ERROR;
371 dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
372 if(U_SUCCESS(errorCode)) {
373 testData=dataModule->createTestData("getUnicodeSet", errorCode);
374 if(U_SUCCESS(errorCode)) {
375 for(i=0; testData->nextCase(testCase, errorCode); ++i) {
376 if(U_FAILURE(errorCode)) {
377 errln("error retrieving conversion/getUnicodeSet test case %d - %s",
378 i, u_errorName(errorCode));
379 errorCode=U_ZERO_ERROR;
380 continue;
381 }
382
383 s=testCase->getString("charset", errorCode);
384 s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
385
386 map=testCase->getString("map", errorCode);
387 mapnot=testCase->getString("mapnot", errorCode);
388
389 which=testCase->getInt28("which", errorCode);
390
391 if(U_FAILURE(errorCode)) {
392 errln("error parsing conversion/getUnicodeSet test case %d - %s",
393 i, u_errorName(errorCode));
394 errorCode=U_ZERO_ERROR;
395 continue;
396 }
397
398 // test this test case
399 mapSet.clear();
400 mapnotSet.clear();
401
402 pos.setIndex(0);
403 mapSet.applyPattern(map, pos, 0, NULL, errorCode);
404 if(U_FAILURE(errorCode) || pos.getIndex()!=map.length()) {
405 errln("error creating the map set for conversion/getUnicodeSet test case %d - %s\n"
406 " error index %d index %d U+%04x",
407 i, u_errorName(errorCode), pos.getErrorIndex(), pos.getIndex(), map.char32At(pos.getIndex()));
408 errorCode=U_ZERO_ERROR;
409 continue;
410 }
411
412 pos.setIndex(0);
413 mapnotSet.applyPattern(mapnot, pos, 0, NULL, errorCode);
414 if(U_FAILURE(errorCode) || pos.getIndex()!=mapnot.length()) {
415 errln("error creating the mapnot set for conversion/getUnicodeSet test case %d - %s\n"
416 " error index %d index %d U+%04x",
417 i, u_errorName(errorCode), pos.getErrorIndex(), pos.getIndex(), mapnot.char32At(pos.getIndex()));
418 errorCode=U_ZERO_ERROR;
419 continue;
420 }
421
422 logln("TestGetUnicodeSet[%d] %s", i, charset);
423
424 cnv.adoptInstead(cnv_open(charset, errorCode));
425 if(U_FAILURE(errorCode)) {
426 errcheckln(errorCode, "error opening \"%s\" for conversion/getUnicodeSet test case %d - %s",
427 charset, i, u_errorName(errorCode));
428 errorCode=U_ZERO_ERROR;
429 continue;
430 }
431
432 ucnv_getUnicodeSet(cnv.getAlias(), cnvSetPtr->toUSet(), (UConverterUnicodeSet)which, &errorCode);
433
434 if(U_FAILURE(errorCode)) {
435 errln("error in ucnv_getUnicodeSet(\"%s\") for conversion/getUnicodeSet test case %d - %s",
436 charset, i, u_errorName(errorCode));
437 errorCode=U_ZERO_ERROR;
438 continue;
439 }
440
441 // are there items that must be in cnvSet but are not?
442 (diffSet=mapSet).removeAll(cnvSet);
443 if(!diffSet.isEmpty()) {
444 diffSet.toPattern(s, TRUE);
445 if(s.length()>100) {
446 s.replace(100, 0x7fffffff, ellipsis, UPRV_LENGTHOF(ellipsis));
447 }
448 errln("error: ucnv_getUnicodeSet(\"%s\") is missing items - conversion/getUnicodeSet test case %d",
449 charset, i);
450 errln(s);
451 }
452
453 // are there items that must not be in cnvSet but are?
454 (diffSet=mapnotSet).retainAll(cnvSet);
455 if(!diffSet.isEmpty()) {
456 diffSet.toPattern(s, TRUE);
457 if(s.length()>100) {
458 s.replace(100, 0x7fffffff, ellipsis, UPRV_LENGTHOF(ellipsis));
459 }
460 errln("error: ucnv_getUnicodeSet(\"%s\") contains unexpected items - conversion/getUnicodeSet test case %d",
461 charset, i);
462 errln(s);
463 }
464 }
465 delete testData;
466 }
467 delete dataModule;
468 }
469 else {
470 dataerrln("Could not load test conversion data");
471 }
472 }
473
474 U_CDECL_BEGIN
475 static void U_CALLCONV
getUnicodeSetCallback(const void * context,UConverterFromUnicodeArgs *,const UChar *,int32_t,UChar32 codePoint,UConverterCallbackReason reason,UErrorCode * pErrorCode)476 getUnicodeSetCallback(const void *context,
477 UConverterFromUnicodeArgs * /*fromUArgs*/,
478 const UChar* /*codeUnits*/,
479 int32_t /*length*/,
480 UChar32 codePoint,
481 UConverterCallbackReason reason,
482 UErrorCode *pErrorCode) {
483 if(reason<=UCNV_IRREGULAR) {
484 ((UnicodeSet *)context)->remove(codePoint); // the converter cannot convert this code point
485 *pErrorCode=U_ZERO_ERROR; // skip
486 } // else ignore the reset, close and clone calls.
487 }
488 U_CDECL_END
489
490 // Compare ucnv_getUnicodeSet() with the set of characters that can be converted.
491 void
TestGetUnicodeSet2()492 ConversionTest::TestGetUnicodeSet2() {
493 // Build a string with all code points.
494 UChar32 cpLimit;
495 int32_t s0Length;
496 if(quick) {
497 cpLimit=s0Length=0x10000; // BMP only
498 } else {
499 cpLimit=0x110000;
500 s0Length=0x10000+0x200000; // BMP + surrogate pairs
501 }
502 UChar *s0=new UChar[s0Length];
503 if(s0==NULL) {
504 return;
505 }
506 UChar *s=s0;
507 UChar32 c;
508 UChar c2;
509 // low BMP
510 for(c=0; c<=0xd7ff; ++c) {
511 *s++=(UChar)c;
512 }
513 // trail surrogates
514 for(c=0xdc00; c<=0xdfff; ++c) {
515 *s++=(UChar)c;
516 }
517 // lead surrogates
518 // (after trails so that there is not even one surrogate pair in between)
519 for(c=0xd800; c<=0xdbff; ++c) {
520 *s++=(UChar)c;
521 }
522 // high BMP
523 for(c=0xe000; c<=0xffff; ++c) {
524 *s++=(UChar)c;
525 }
526 // supplementary code points = surrogate pairs
527 if(cpLimit==0x110000) {
528 for(c=0xd800; c<=0xdbff; ++c) {
529 for(c2=0xdc00; c2<=0xdfff; ++c2) {
530 *s++=(UChar)c;
531 *s++=c2;
532 }
533 }
534 }
535
536 static const char *const cnvNames[]={
537 "UTF-8",
538 "UTF-7",
539 "UTF-16",
540 "US-ASCII",
541 "ISO-8859-1",
542 "windows-1252",
543 "Shift-JIS",
544 "ibm-1390", // EBCDIC_STATEFUL table
545 "ibm-16684", // DBCS-only extension table based on EBCDIC_STATEFUL table
546 "HZ",
547 "ISO-2022-JP",
548 "JIS7",
549 "ISO-2022-CN",
550 "ISO-2022-CN-EXT",
551 "LMBCS"
552 };
553 LocalUConverterPointer cnv;
554 char buffer[1024];
555 int32_t i;
556 for(i=0; i<UPRV_LENGTHOF(cnvNames); ++i) {
557 UErrorCode errorCode=U_ZERO_ERROR;
558 cnv.adoptInstead(cnv_open(cnvNames[i], errorCode));
559 if(U_FAILURE(errorCode)) {
560 errcheckln(errorCode, "failed to open converter %s - %s", cnvNames[i], u_errorName(errorCode));
561 continue;
562 }
563 UnicodeSet expected;
564 ucnv_setFromUCallBack(cnv.getAlias(), getUnicodeSetCallback, &expected, NULL, NULL, &errorCode);
565 if(U_FAILURE(errorCode)) {
566 errln("failed to set the callback on converter %s - %s", cnvNames[i], u_errorName(errorCode));
567 continue;
568 }
569 UConverterUnicodeSet which;
570 for(which=UCNV_ROUNDTRIP_SET; which<UCNV_SET_COUNT; which=(UConverterUnicodeSet)((int)which+1)) {
571 if(which==UCNV_ROUNDTRIP_AND_FALLBACK_SET) {
572 ucnv_setFallback(cnv.getAlias(), TRUE);
573 }
574 expected.add(0, cpLimit-1);
575 s=s0;
576 UBool flush;
577 do {
578 char *t=buffer;
579 flush=(UBool)(s==s0+s0Length);
580 ucnv_fromUnicode(cnv.getAlias(), &t, buffer+sizeof(buffer), (const UChar **)&s, s0+s0Length, NULL, flush, &errorCode);
581 if(U_FAILURE(errorCode)) {
582 if(errorCode==U_BUFFER_OVERFLOW_ERROR) {
583 errorCode=U_ZERO_ERROR;
584 continue;
585 } else {
586 break; // unexpected error, should not occur
587 }
588 }
589 } while(!flush);
590 UnicodeSet set;
591 ucnv_getUnicodeSet(cnv.getAlias(), set.toUSet(), which, &errorCode);
592 if(cpLimit<0x110000) {
593 set.remove(cpLimit, 0x10ffff);
594 }
595 if(which==UCNV_ROUNDTRIP_SET) {
596 // ignore PUA code points because they will be converted even if they
597 // are fallbacks and when other fallbacks are turned off,
598 // but ucnv_getUnicodeSet(UCNV_ROUNDTRIP_SET) delivers true roundtrips
599 expected.remove(0xe000, 0xf8ff);
600 expected.remove(0xf0000, 0xffffd);
601 expected.remove(0x100000, 0x10fffd);
602 set.remove(0xe000, 0xf8ff);
603 set.remove(0xf0000, 0xffffd);
604 set.remove(0x100000, 0x10fffd);
605 }
606 if(set!=expected) {
607 // First try to see if we have different sets because ucnv_getUnicodeSet()
608 // added strings: The above conversion method does not tell us what strings might be convertible.
609 // Remove strings from the set and compare again.
610 set.removeAllStrings();
611 }
612 if(set!=expected) {
613 UnicodeSet diffSet;
614 UnicodeString out;
615
616 // are there items that must be in the set but are not?
617 (diffSet=expected).removeAll(set);
618 if(!diffSet.isEmpty()) {
619 diffSet.toPattern(out, TRUE);
620 if(out.length()>100) {
621 out.replace(100, 0x7fffffff, ellipsis, UPRV_LENGTHOF(ellipsis));
622 }
623 errln("error: ucnv_getUnicodeSet(\"%s\") is missing items - which set: %d",
624 cnvNames[i], which);
625 errln(out);
626 }
627
628 // are there items that must not be in the set but are?
629 (diffSet=set).removeAll(expected);
630 if(!diffSet.isEmpty()) {
631 diffSet.toPattern(out, TRUE);
632 if(out.length()>100) {
633 out.replace(100, 0x7fffffff, ellipsis, UPRV_LENGTHOF(ellipsis));
634 }
635 errln("error: ucnv_getUnicodeSet(\"%s\") contains unexpected items - which set: %d",
636 cnvNames[i], which);
637 errln(out);
638 }
639 }
640 }
641 }
642
643 delete [] s0;
644 }
645
646 // Test all codepoints which has the default ignorable Unicode property are ignored if they have no mapping
647 // If there are any failures, the hard coded list (IS_DEFAULT_IGNORABLE_CODE_POINT) in ucnv_err.c should be updated
648 void
TestDefaultIgnorableCallback()649 ConversionTest::TestDefaultIgnorableCallback() {
650 UErrorCode status = U_ZERO_ERROR;
651 const char *cnv_name = "euc-jp-2007";
652 const char *pattern_ignorable = "[:Default_Ignorable_Code_Point:]";
653 const char *pattern_not_ignorable = "[:^Default_Ignorable_Code_Point:]";
654
655 LocalPointer<UnicodeSet> set_ignorable(new UnicodeSet(pattern_ignorable, status));
656 if (U_FAILURE(status)) {
657 dataerrln("Unable to create Unicodeset: %s - %s\n", pattern_ignorable, u_errorName(status));
658 return;
659 }
660
661 LocalPointer<UnicodeSet> set_not_ignorable(new UnicodeSet(pattern_not_ignorable, status));
662 if (U_FAILURE(status)) {
663 dataerrln("Unable to create Unicodeset: %s - %s\n", pattern_not_ignorable, u_errorName(status));
664 return;
665 }
666
667 LocalUConverterPointer cnv(cnv_open(cnv_name, status));
668 if (U_FAILURE(status)) {
669 dataerrln("Unable to open converter: %s - %s\n", cnv_name, u_errorName(status));
670 return;
671 }
672
673 // set callback for the converter
674 ucnv_setFromUCallBack(cnv.getAlias(), UCNV_FROM_U_CALLBACK_SUBSTITUTE, NULL, NULL, NULL, &status);
675
676 UChar32 input[1];
677 char output[10];
678 int32_t outputLength;
679
680 // test default ignorables are ignored
681 int size = set_ignorable->size();
682 for (int i = 0; i < size; i++) {
683 status = U_ZERO_ERROR;
684 outputLength= 0;
685
686 input[0] = set_ignorable->charAt(i);
687
688 outputLength = ucnv_fromUChars(cnv.getAlias(), output, 10, UnicodeString::fromUTF32(input, 1).getTerminatedBuffer(), -1, &status);
689 if (U_FAILURE(status) || outputLength != 0) {
690 errln("Ignorable code point: U+%04X not skipped as expected - %s", input[0], u_errorName(status));
691 }
692 }
693
694 // test non-ignorables are not ignored
695 size = set_not_ignorable->size();
696 for (int i = 0; i < size; i++) {
697 status = U_ZERO_ERROR;
698 outputLength= 0;
699
700 input[0] = set_not_ignorable->charAt(i);
701
702 if (input[0] == 0) {
703 continue;
704 }
705
706 outputLength = ucnv_fromUChars(cnv.getAlias(), output, 10, UnicodeString::fromUTF32(input, 1).getTerminatedBuffer(), -1, &status);
707 if (U_FAILURE(status) || outputLength <= 0) {
708 errln("Non-ignorable code point: U+%04X skipped unexpectedly - %s", input[0], u_errorName(status));
709 }
710 }
711 }
712
713 void
TestUTF8ToUTF8Overflow()714 ConversionTest::TestUTF8ToUTF8Overflow() {
715 IcuTestErrorCode errorCode(*this, "TestUTF8ToUTF8Overflow");
716 LocalUConverterPointer cnv1(ucnv_open("UTF-8", errorCode));
717 LocalUConverterPointer cnv2(ucnv_open("UTF-8", errorCode));
718 static const char *text = "aä"; // ä: 2 bytes
719 const char *source = text;
720 const char *sourceLimit = text + strlen(text);
721 char result[20];
722 char *target = result;
723 const char *targetLimit = result + sizeof(result);
724 UChar buffer16[20];
725 UChar *pivotSource = buffer16;
726 UChar *pivotTarget = buffer16;
727 const UChar *pivotLimit = buffer16 + UPRV_LENGTHOF(buffer16);
728 int32_t length;
729
730 // Convert with insufficient target capacity.
731 result[2] = 5;
732 ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
733 &target, result + 2, &source, sourceLimit,
734 buffer16, &pivotSource, &pivotTarget, pivotLimit,
735 FALSE, FALSE, errorCode);
736 assertEquals("overflow", U_BUFFER_OVERFLOW_ERROR, errorCode.reset());
737 length = (int32_t)(target - result);
738 assertEquals("number of bytes written", 2, length);
739 assertEquals("next byte not clobbered", 5, result[2]);
740
741 // Convert the rest and flush.
742 ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
743 &target, targetLimit, &source, sourceLimit,
744 buffer16, &pivotSource, &pivotTarget, pivotLimit,
745 FALSE, TRUE, errorCode);
746
747 assertSuccess("UTF-8->UTF-8", errorCode);
748 length = (int32_t)(target - result);
749 assertEquals("3 bytes", 3, length);
750 if (length == 3) {
751 assertTrue("result same as input", memcmp(text, result, length) == 0);
752 }
753
754 ucnv_reset(cnv1.getAlias());
755 ucnv_reset(cnv2.getAlias());
756 memset(result, 0, sizeof(result));
757 static const char *text2 = "a��"; // U+1F6B2 bicycle: 4 bytes
758 source = text2;
759 sourceLimit = text2 + strlen(text2);
760 target = result;
761 pivotSource = pivotTarget = buffer16;
762
763 // Convert with insufficient target capacity.
764 result[3] = 5;
765 ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
766 &target, result + 3, &source, sourceLimit,
767 buffer16, &pivotSource, &pivotTarget, pivotLimit,
768 FALSE, FALSE, errorCode);
769 assertEquals("text2 overflow", U_BUFFER_OVERFLOW_ERROR, errorCode.reset());
770 length = (int32_t)(target - result);
771 assertEquals("text2 number of bytes written", 3, length);
772 assertEquals("text2 next byte not clobbered", 5, result[3]);
773
774 // Convert the rest and flush.
775 ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
776 &target, targetLimit, &source, sourceLimit,
777 buffer16, &pivotSource, &pivotTarget, pivotLimit,
778 FALSE, TRUE, errorCode);
779
780 assertSuccess("text2 UTF-8->UTF-8", errorCode);
781 length = (int32_t)(target - result);
782 assertEquals("text2 5 bytes", 5, length);
783 if (length == 5) {
784 assertTrue("text2 result same as input", memcmp(text2, result, length) == 0);
785 }
786
787 ucnv_reset(cnv1.getAlias());
788 ucnv_reset(cnv2.getAlias());
789 memset(result, 0, sizeof(result));
790 static const char *illFormed = "\xf1\x91\x93\x96\x91\x94"; // U+514D6 + two more trail bytes
791 source = illFormed;
792 sourceLimit = illFormed + strlen(illFormed);
793 target = result;
794 pivotSource = pivotTarget = buffer16;
795
796 ucnv_setToUCallBack(cnv1.getAlias(), UCNV_TO_U_CALLBACK_STOP, nullptr, nullptr, nullptr, errorCode);
797
798 // Convert only two bytes and flush (but expect failure).
799 char errorBytes[10];
800 int8_t errorLength;
801 result[0] = 5;
802 ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
803 &target, targetLimit, &source, source + 2,
804 buffer16, &pivotSource, &pivotTarget, pivotLimit,
805 FALSE, TRUE, errorCode);
806 assertEquals("illFormed truncated", U_TRUNCATED_CHAR_FOUND, errorCode.reset());
807 length = (int32_t)(target - result);
808 assertEquals("illFormed number of bytes written", 0, length);
809 errorLength = UPRV_LENGTHOF(errorBytes);
810 ucnv_getInvalidChars(cnv1.getAlias(), errorBytes, &errorLength, errorCode);
811 assertEquals("illFormed truncated errorLength", 2, (int32_t)errorLength);
812 if (errorLength == 2) {
813 assertEquals("illFormed truncated errorBytes", 0xf191,
814 ((int32_t)(uint8_t)errorBytes[0] << 8) | (uint8_t)errorBytes[1]);
815 }
816
817 // Continue conversion starting with a trail byte.
818 ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
819 &target, targetLimit, &source, sourceLimit,
820 buffer16, &pivotSource, &pivotTarget, pivotLimit,
821 FALSE, TRUE, errorCode);
822
823 assertEquals("illFormed trail byte", U_ILLEGAL_CHAR_FOUND, errorCode.reset());
824 length = (int32_t)(target - result);
825 assertEquals("illFormed trail byte number of bytes written", 0, length);
826 errorLength = UPRV_LENGTHOF(errorBytes);
827 ucnv_getInvalidChars(cnv1.getAlias(), errorBytes, &errorLength, errorCode);
828 assertEquals("illFormed trail byte errorLength", 1, (int32_t)errorLength);
829 if (errorLength == 1) {
830 assertEquals("illFormed trail byte errorBytes", 0x93, (int32_t)(uint8_t)errorBytes[0]);
831 }
832 }
833
834 void
TestUTF8ToUTF8Streaming()835 ConversionTest::TestUTF8ToUTF8Streaming() {
836 IcuTestErrorCode errorCode(*this, "TestUTF8ToUTF8Streaming");
837 LocalUConverterPointer cnv1(ucnv_open("UTF-8", errorCode));
838 LocalUConverterPointer cnv2(ucnv_open("UTF-8", errorCode));
839
840 // UTF8 encoded cyrillic part of 'Lorem ipsum'
841 static const char* text =
842 "\xd0\xb5\xd1\x82\x20\xd1\x81\xd1\x86\xd0\xb0\xd0\xb5\xd0\xb2\xd0"
843 "\xbe\xd0\xbb\xd0\xb0\x20\xd1\x81\xd0\xb0\xd0\xb4\xd0\xb8\xd0\xbf"
844 "\xd1\x81\xd1\x86\xd0\xb8\xd0\xbd\xd0\xb3\x20\xd0\xb0\xd1\x86\xd1"
845 "\x86\xd0\xbe\xd0\xbc\xd0\xbc\xd0\xbe\xd0\xb4\xd0\xb0\xd1\x80\xd0"
846 "\xb5\x20\xd1\x85\xd0\xb0\xd1\x81";
847
848 int32_t chunk1 = 25; // partial lead at the end: 0xd0
849 int32_t chunk2 = 47; // partial tail at the beginning: 0xb0
850
851 char result[128];
852
853 int32_t sourceLen = (int32_t)strlen(text);
854 const char* source = text;
855 const char* sourceLimit = text + chunk1;
856
857 int32_t targetLen = sizeof(result);
858 char* target = result;
859 const char* targetLimit = result + targetLen;
860
861 UChar buffer16[20];
862 UChar* pivotSource = buffer16;
863 UChar* pivotTarget = buffer16;
864 const UChar* pivotLimit = buffer16 + UPRV_LENGTHOF(buffer16);
865
866 int32_t length;
867 ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
868 &target, result + targetLen, &source, sourceLimit,
869 buffer16, &pivotSource, &pivotTarget, pivotLimit,
870 FALSE, FALSE, errorCode);
871
872 length = (int32_t)(target - result);
873 targetLen -= length;
874 assertEquals("First chunk -1 doesn't match converted length", chunk1 - 1, length);
875
876 source = text + chunk1;
877 sourceLimit = source + chunk2;
878
879 // Convert the rest and flush.
880 ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
881 &target, targetLimit, &source, sourceLimit,
882 buffer16, &pivotSource, &pivotTarget, pivotLimit,
883 FALSE, TRUE, errorCode);
884
885 length = (int32_t)(target - result - length);
886 targetLen -= length;
887 assertEquals("Second chunk + 2 doesn't match converted length", chunk2 + 1, length);
888
889 assertEquals("Full text length match", sourceLen, sizeof(result) - targetLen);
890 assertSuccess("UTF-8->UTF-8", errorCode);
891 }
892
893 // open testdata or ICU data converter ------------------------------------- ***
894
895 UConverter *
cnv_open(const char * name,UErrorCode & errorCode)896 ConversionTest::cnv_open(const char *name, UErrorCode &errorCode) {
897 if(name!=NULL && *name=='+') {
898 // Converter names that start with '+' are ignored in ICU4J tests.
899 ++name;
900 }
901 if(name!=NULL && *name=='*') {
902 /* loadTestData(): set the data directory */
903 return ucnv_openPackage(loadTestData(errorCode), name+1, &errorCode);
904 } else {
905 return ucnv_open(name, &errorCode);
906 }
907 }
908
909 // output helpers ---------------------------------------------------------- ***
910
911 static inline char
hexDigit(uint8_t digit)912 hexDigit(uint8_t digit) {
913 return digit<=9 ? (char)('0'+digit) : (char)('a'-10+digit);
914 }
915
916 static char *
printBytes(const uint8_t * bytes,int32_t length,char * out)917 printBytes(const uint8_t *bytes, int32_t length, char *out) {
918 uint8_t b;
919
920 if(length>0) {
921 b=*bytes++;
922 --length;
923 *out++=hexDigit((uint8_t)(b>>4));
924 *out++=hexDigit((uint8_t)(b&0xf));
925 }
926
927 while(length>0) {
928 b=*bytes++;
929 --length;
930 *out++=' ';
931 *out++=hexDigit((uint8_t)(b>>4));
932 *out++=hexDigit((uint8_t)(b&0xf));
933 }
934 *out++=0;
935 return out;
936 }
937
938 static char *
printUnicode(const UChar * unicode,int32_t length,char * out)939 printUnicode(const UChar *unicode, int32_t length, char *out) {
940 UChar32 c;
941 int32_t i;
942
943 for(i=0; i<length;) {
944 if(i>0) {
945 *out++=' ';
946 }
947 U16_NEXT(unicode, i, length, c);
948 // write 4..6 digits
949 if(c>=0x100000) {
950 *out++='1';
951 }
952 if(c>=0x10000) {
953 *out++=hexDigit((uint8_t)((c>>16)&0xf));
954 }
955 *out++=hexDigit((uint8_t)((c>>12)&0xf));
956 *out++=hexDigit((uint8_t)((c>>8)&0xf));
957 *out++=hexDigit((uint8_t)((c>>4)&0xf));
958 *out++=hexDigit((uint8_t)(c&0xf));
959 }
960 *out++=0;
961 return out;
962 }
963
964 static char *
printOffsets(const int32_t * offsets,int32_t length,char * out)965 printOffsets(const int32_t *offsets, int32_t length, char *out) {
966 int32_t i, o, d;
967
968 if(offsets==NULL) {
969 length=0;
970 }
971
972 for(i=0; i<length; ++i) {
973 if(i>0) {
974 *out++=' ';
975 }
976 o=offsets[i];
977
978 // print all offsets with 2 characters each (-x, -9..99, xx)
979 if(o<-9) {
980 *out++='-';
981 *out++='x';
982 } else if(o<0) {
983 *out++='-';
984 *out++=(char)('0'-o);
985 } else if(o<=99) {
986 *out++=(d=o/10)==0 ? ' ' : (char)('0'+d);
987 *out++=(char)('0'+o%10);
988 } else /* o>99 */ {
989 *out++='x';
990 *out++='x';
991 }
992 }
993 *out++=0;
994 return out;
995 }
996
997 // toUnicode test worker functions ----------------------------------------- ***
998
999 static int32_t
stepToUnicode(ConversionCase & cc,UConverter * cnv,UChar * result,int32_t resultCapacity,int32_t * resultOffsets,int32_t step,UErrorCode * pErrorCode)1000 stepToUnicode(ConversionCase &cc, UConverter *cnv,
1001 UChar *result, int32_t resultCapacity,
1002 int32_t *resultOffsets, /* also resultCapacity */
1003 int32_t step,
1004 UErrorCode *pErrorCode) {
1005 const char *source, *sourceLimit, *bytesLimit;
1006 UChar *target, *targetLimit, *resultLimit;
1007 UBool flush;
1008
1009 source=(const char *)cc.bytes;
1010 target=result;
1011 bytesLimit=source+cc.bytesLength;
1012 resultLimit=result+resultCapacity;
1013
1014 if(step>=0) {
1015 // call ucnv_toUnicode() with in/out buffers no larger than (step) at a time
1016 // move only one buffer (in vs. out) at a time to be extra mean
1017 // step==0 performs bulk conversion and generates offsets
1018
1019 // initialize the partial limits for the loop
1020 if(step==0) {
1021 // use the entire buffers
1022 sourceLimit=bytesLimit;
1023 targetLimit=resultLimit;
1024 flush=cc.finalFlush;
1025 } else {
1026 // start with empty partial buffers
1027 sourceLimit=source;
1028 targetLimit=target;
1029 flush=FALSE;
1030
1031 // output offsets only for bulk conversion
1032 resultOffsets=NULL;
1033 }
1034
1035 for(;;) {
1036 // resetting the opposite conversion direction must not affect this one
1037 ucnv_resetFromUnicode(cnv);
1038
1039 // convert
1040 ucnv_toUnicode(cnv,
1041 &target, targetLimit,
1042 &source, sourceLimit,
1043 resultOffsets,
1044 flush, pErrorCode);
1045
1046 // check pointers and errors
1047 if(source>sourceLimit || target>targetLimit) {
1048 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1049 break;
1050 } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
1051 if(target!=targetLimit) {
1052 // buffer overflow must only be set when the target is filled
1053 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1054 break;
1055 } else if(targetLimit==resultLimit) {
1056 // not just a partial overflow
1057 break;
1058 }
1059
1060 // the partial target is filled, set a new limit, reset the error and continue
1061 targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
1062 *pErrorCode=U_ZERO_ERROR;
1063 } else if(U_FAILURE(*pErrorCode)) {
1064 // some other error occurred, done
1065 break;
1066 } else {
1067 if(source!=sourceLimit) {
1068 // when no error occurs, then the input must be consumed
1069 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1070 break;
1071 }
1072
1073 if(sourceLimit==bytesLimit) {
1074 // we are done
1075 break;
1076 }
1077
1078 // the partial conversion succeeded, set a new limit and continue
1079 sourceLimit=(bytesLimit-source)>=step ? source+step : bytesLimit;
1080 flush=(UBool)(cc.finalFlush && sourceLimit==bytesLimit);
1081 }
1082 }
1083 } else /* step<0 */ {
1084 /*
1085 * step==-1: call only ucnv_getNextUChar()
1086 * otherwise alternate between ucnv_toUnicode() and ucnv_getNextUChar()
1087 * if step==-2 or -3, then give ucnv_toUnicode() the whole remaining input,
1088 * else give it at most (-step-2)/2 bytes
1089 */
1090 UChar32 c;
1091
1092 // end the loop by getting an index out of bounds error
1093 for(;;) {
1094 // resetting the opposite conversion direction must not affect this one
1095 ucnv_resetFromUnicode(cnv);
1096
1097 // convert
1098 if((step&1)!=0 /* odd: -1, -3, -5, ... */) {
1099 sourceLimit=source; // use sourceLimit not as a real limit
1100 // but to remember the pre-getNextUChar source pointer
1101 c=ucnv_getNextUChar(cnv, &source, bytesLimit, pErrorCode);
1102
1103 // check pointers and errors
1104 if(*pErrorCode==U_INDEX_OUTOFBOUNDS_ERROR) {
1105 if(source!=bytesLimit) {
1106 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1107 } else {
1108 *pErrorCode=U_ZERO_ERROR;
1109 }
1110 break;
1111 } else if(U_FAILURE(*pErrorCode)) {
1112 break;
1113 }
1114 // source may not move if c is from previous overflow
1115
1116 if(target==resultLimit) {
1117 *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
1118 break;
1119 }
1120 if(c<=0xffff) {
1121 *target++=(UChar)c;
1122 } else {
1123 *target++=U16_LEAD(c);
1124 if(target==resultLimit) {
1125 *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
1126 break;
1127 }
1128 *target++=U16_TRAIL(c);
1129 }
1130
1131 // alternate between -n-1 and -n but leave -1 alone
1132 if(step<-1) {
1133 ++step;
1134 }
1135 } else /* step is even */ {
1136 // allow only one UChar output
1137 targetLimit=target<resultLimit ? target+1 : resultLimit;
1138
1139 // as with ucnv_getNextUChar(), we always flush (if we go to bytesLimit)
1140 // and never output offsets
1141 if(step==-2) {
1142 sourceLimit=bytesLimit;
1143 } else {
1144 sourceLimit=source+(-step-2)/2;
1145 if(sourceLimit>bytesLimit) {
1146 sourceLimit=bytesLimit;
1147 }
1148 }
1149
1150 ucnv_toUnicode(cnv,
1151 &target, targetLimit,
1152 &source, sourceLimit,
1153 NULL, (UBool)(sourceLimit==bytesLimit), pErrorCode);
1154
1155 // check pointers and errors
1156 if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
1157 if(target!=targetLimit) {
1158 // buffer overflow must only be set when the target is filled
1159 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1160 break;
1161 } else if(targetLimit==resultLimit) {
1162 // not just a partial overflow
1163 break;
1164 }
1165
1166 // the partial target is filled, set a new limit and continue
1167 *pErrorCode=U_ZERO_ERROR;
1168 } else if(U_FAILURE(*pErrorCode)) {
1169 // some other error occurred, done
1170 break;
1171 } else {
1172 if(source!=sourceLimit) {
1173 // when no error occurs, then the input must be consumed
1174 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1175 break;
1176 }
1177
1178 // we are done (flush==TRUE) but we continue, to get the index out of bounds error above
1179 }
1180
1181 --step;
1182 }
1183 }
1184 }
1185
1186 return (int32_t)(target-result);
1187 }
1188
1189 UBool
ToUnicodeCase(ConversionCase & cc,UConverterToUCallback callback,const char * option)1190 ConversionTest::ToUnicodeCase(ConversionCase &cc, UConverterToUCallback callback, const char *option) {
1191 // open the converter
1192 IcuTestErrorCode errorCode(*this, "ToUnicodeCase");
1193 LocalUConverterPointer cnv(cnv_open(cc.charset, errorCode));
1194 // with no data, the above crashes with "pointer being freed was not allocated" for charset "x11-compound-text", see #13078
1195 if(errorCode.isFailure()) {
1196 errcheckln(errorCode, "toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_open() failed - %s",
1197 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, errorCode.errorName());
1198 errorCode.reset();
1199 return FALSE;
1200 }
1201
1202 // set the callback
1203 if(callback!=NULL) {
1204 ucnv_setToUCallBack(cnv.getAlias(), callback, option, NULL, NULL, errorCode);
1205 if(U_FAILURE(errorCode)) {
1206 errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setToUCallBack() failed - %s",
1207 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1208 return FALSE;
1209 }
1210 }
1211
1212 int32_t resultOffsets[256];
1213 UChar result[256];
1214 int32_t resultLength;
1215 UBool ok;
1216
1217 static const struct {
1218 int32_t step;
1219 const char *name;
1220 } steps[]={
1221 { 0, "bulk" }, // must be first for offsets to be checked
1222 { 1, "step=1" },
1223 { 3, "step=3" },
1224 { 7, "step=7" },
1225 { -1, "getNext" },
1226 { -2, "toU(bulk)+getNext" },
1227 { -3, "getNext+toU(bulk)" },
1228 { -4, "toU(1)+getNext" },
1229 { -5, "getNext+toU(1)" },
1230 { -12, "toU(5)+getNext" },
1231 { -13, "getNext+toU(5)" },
1232 };
1233 int32_t i, step;
1234
1235 ok=TRUE;
1236 for(i=0; i<UPRV_LENGTHOF(steps) && ok; ++i) {
1237 step=steps[i].step;
1238 if(step<0 && !cc.finalFlush) {
1239 // skip ucnv_getNextUChar() if !finalFlush because
1240 // ucnv_getNextUChar() always implies flush
1241 continue;
1242 }
1243 if(step!=0) {
1244 // bulk test is first, then offsets are not checked any more
1245 cc.offsets=NULL;
1246 }
1247 else {
1248 for (int32_t i = 0; i < UPRV_LENGTHOF(resultOffsets); i++) {
1249 resultOffsets[i] = -1;
1250 }
1251 }
1252 for (int32_t i = 0; i < UPRV_LENGTHOF(result); i++) {
1253 result[i] = -1;
1254 }
1255 errorCode.reset();
1256 resultLength=stepToUnicode(cc, cnv.getAlias(),
1257 result, UPRV_LENGTHOF(result),
1258 step==0 ? resultOffsets : NULL,
1259 step, errorCode);
1260 ok=checkToUnicode(
1261 cc, cnv.getAlias(), steps[i].name,
1262 result, resultLength,
1263 cc.offsets!=NULL ? resultOffsets : NULL,
1264 errorCode);
1265 if(errorCode.isFailure() || !cc.finalFlush) {
1266 // reset if an error occurred or we did not flush
1267 // otherwise do nothing to make sure that flushing resets
1268 ucnv_resetToUnicode(cnv.getAlias());
1269 }
1270 if (cc.offsets != NULL && resultOffsets[resultLength] != -1) {
1271 errln("toUnicode[%d](%s) Conversion wrote too much to offsets at index %d",
1272 cc.caseNr, cc.charset, resultLength);
1273 }
1274 if (result[resultLength] != (UChar)-1) {
1275 errln("toUnicode[%d](%s) Conversion wrote too much to result at index %d",
1276 cc.caseNr, cc.charset, resultLength);
1277 }
1278 }
1279
1280 // not a real loop, just a convenience for breaking out of the block
1281 while(ok && cc.finalFlush) {
1282 // test ucnv_toUChars()
1283 memset(result, 0, sizeof(result));
1284
1285 errorCode.reset();
1286 resultLength=ucnv_toUChars(cnv.getAlias(),
1287 result, UPRV_LENGTHOF(result),
1288 (const char *)cc.bytes, cc.bytesLength,
1289 errorCode);
1290 ok=checkToUnicode(
1291 cc, cnv.getAlias(), "toUChars",
1292 result, resultLength,
1293 NULL,
1294 errorCode);
1295 if(!ok) {
1296 break;
1297 }
1298
1299 // test preflighting
1300 // keep the correct result for simple checking
1301 errorCode.reset();
1302 resultLength=ucnv_toUChars(cnv.getAlias(),
1303 NULL, 0,
1304 (const char *)cc.bytes, cc.bytesLength,
1305 errorCode);
1306 if(errorCode.get()==U_STRING_NOT_TERMINATED_WARNING || errorCode.get()==U_BUFFER_OVERFLOW_ERROR) {
1307 errorCode.reset();
1308 }
1309 ok=checkToUnicode(
1310 cc, cnv.getAlias(), "preflight toUChars",
1311 result, resultLength,
1312 NULL,
1313 errorCode);
1314 break;
1315 }
1316
1317 errorCode.reset(); // all errors have already been reported
1318 return ok;
1319 }
1320
1321 UBool
checkToUnicode(ConversionCase & cc,UConverter * cnv,const char * name,const UChar * result,int32_t resultLength,const int32_t * resultOffsets,UErrorCode resultErrorCode)1322 ConversionTest::checkToUnicode(ConversionCase &cc, UConverter *cnv, const char *name,
1323 const UChar *result, int32_t resultLength,
1324 const int32_t *resultOffsets,
1325 UErrorCode resultErrorCode) {
1326 char resultInvalidChars[8];
1327 int8_t resultInvalidLength;
1328 UErrorCode errorCode;
1329
1330 const char *msg;
1331
1332 // reset the message; NULL will mean "ok"
1333 msg=NULL;
1334
1335 errorCode=U_ZERO_ERROR;
1336 resultInvalidLength=sizeof(resultInvalidChars);
1337 ucnv_getInvalidChars(cnv, resultInvalidChars, &resultInvalidLength, &errorCode);
1338 if(U_FAILURE(errorCode)) {
1339 errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) ucnv_getInvalidChars() failed - %s",
1340 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, u_errorName(errorCode));
1341 return FALSE;
1342 }
1343
1344 // check everything that might have gone wrong
1345 if(cc.unicodeLength!=resultLength) {
1346 msg="wrong result length";
1347 } else if(0!=u_memcmp(cc.unicode, result, cc.unicodeLength)) {
1348 msg="wrong result string";
1349 } else if(cc.offsets!=NULL && 0!=memcmp(cc.offsets, resultOffsets, cc.unicodeLength*sizeof(*cc.offsets))) {
1350 msg="wrong offsets";
1351 } else if(cc.outErrorCode!=resultErrorCode) {
1352 msg="wrong error code";
1353 } else if(cc.invalidLength!=resultInvalidLength) {
1354 msg="wrong length of last invalid input";
1355 } else if(0!=memcmp(cc.invalidChars, resultInvalidChars, cc.invalidLength)) {
1356 msg="wrong last invalid input";
1357 }
1358
1359 if(msg==NULL) {
1360 return TRUE;
1361 } else {
1362 char buffer[2000]; // one buffer for all strings
1363 char *s, *bytesString, *unicodeString, *resultString,
1364 *offsetsString, *resultOffsetsString,
1365 *invalidCharsString, *resultInvalidCharsString;
1366
1367 bytesString=s=buffer;
1368 s=printBytes(cc.bytes, cc.bytesLength, bytesString);
1369 s=printUnicode(cc.unicode, cc.unicodeLength, unicodeString=s);
1370 s=printUnicode(result, resultLength, resultString=s);
1371 s=printOffsets(cc.offsets, cc.unicodeLength, offsetsString=s);
1372 s=printOffsets(resultOffsets, resultLength, resultOffsetsString=s);
1373 s=printBytes(cc.invalidChars, cc.invalidLength, invalidCharsString=s);
1374 s=printBytes((uint8_t *)resultInvalidChars, resultInvalidLength, resultInvalidCharsString=s);
1375
1376 if((s-buffer)>(int32_t)sizeof(buffer)) {
1377 errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) fatal error: checkToUnicode() test output buffer overflow writing %d chars\n",
1378 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, (int)(s-buffer));
1379 exit(1);
1380 }
1381
1382 errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) failed: %s\n"
1383 " bytes <%s>[%d]\n"
1384 " expected <%s>[%d]\n"
1385 " result <%s>[%d]\n"
1386 " offsets <%s>\n"
1387 " result offsets <%s>\n"
1388 " error code expected %s got %s\n"
1389 " invalidChars expected <%s> got <%s>\n",
1390 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, msg,
1391 bytesString, cc.bytesLength,
1392 unicodeString, cc.unicodeLength,
1393 resultString, resultLength,
1394 offsetsString,
1395 resultOffsetsString,
1396 u_errorName(cc.outErrorCode), u_errorName(resultErrorCode),
1397 invalidCharsString, resultInvalidCharsString);
1398
1399 return FALSE;
1400 }
1401 }
1402
1403 // fromUnicode test worker functions --------------------------------------- ***
1404
1405 static int32_t
stepFromUTF8(ConversionCase & cc,UConverter * utf8Cnv,UConverter * cnv,char * result,int32_t resultCapacity,int32_t step,UErrorCode * pErrorCode)1406 stepFromUTF8(ConversionCase &cc,
1407 UConverter *utf8Cnv, UConverter *cnv,
1408 char *result, int32_t resultCapacity,
1409 int32_t step,
1410 UErrorCode *pErrorCode) {
1411 const char *source, *sourceLimit, *utf8Limit;
1412 UChar pivotBuffer[32];
1413 UChar *pivotSource, *pivotTarget, *pivotLimit;
1414 char *target, *targetLimit, *resultLimit;
1415 UBool flush;
1416
1417 source=cc.utf8;
1418 pivotSource=pivotTarget=pivotBuffer;
1419 target=result;
1420 utf8Limit=source+cc.utf8Length;
1421 resultLimit=result+resultCapacity;
1422
1423 // call ucnv_convertEx() with in/out buffers no larger than (step) at a time
1424 // move only one buffer (in vs. out) at a time to be extra mean
1425 // step==0 performs bulk conversion
1426
1427 // initialize the partial limits for the loop
1428 if(step==0) {
1429 // use the entire buffers
1430 sourceLimit=utf8Limit;
1431 targetLimit=resultLimit;
1432 flush=cc.finalFlush;
1433
1434 pivotLimit=pivotBuffer+UPRV_LENGTHOF(pivotBuffer);
1435 } else {
1436 // start with empty partial buffers
1437 sourceLimit=source;
1438 targetLimit=target;
1439 flush=FALSE;
1440
1441 // empty pivot is not allowed, make it of length step
1442 pivotLimit=pivotBuffer+step;
1443 }
1444
1445 for(;;) {
1446 // resetting the opposite conversion direction must not affect this one
1447 ucnv_resetFromUnicode(utf8Cnv);
1448 ucnv_resetToUnicode(cnv);
1449
1450 // convert
1451 ucnv_convertEx(cnv, utf8Cnv,
1452 &target, targetLimit,
1453 &source, sourceLimit,
1454 pivotBuffer, &pivotSource, &pivotTarget, pivotLimit,
1455 FALSE, flush, pErrorCode);
1456
1457 // check pointers and errors
1458 if(source>sourceLimit || target>targetLimit) {
1459 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1460 break;
1461 } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
1462 if(target!=targetLimit) {
1463 // buffer overflow must only be set when the target is filled
1464 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1465 break;
1466 } else if(targetLimit==resultLimit) {
1467 // not just a partial overflow
1468 break;
1469 }
1470
1471 // the partial target is filled, set a new limit, reset the error and continue
1472 targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
1473 *pErrorCode=U_ZERO_ERROR;
1474 } else if(U_FAILURE(*pErrorCode)) {
1475 if(pivotSource==pivotBuffer) {
1476 // toUnicode error, should not occur
1477 // toUnicode errors are tested in cintltst TestConvertExFromUTF8()
1478 break;
1479 } else {
1480 // fromUnicode error
1481 // some other error occurred, done
1482 break;
1483 }
1484 } else {
1485 if(source!=sourceLimit) {
1486 // when no error occurs, then the input must be consumed
1487 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1488 break;
1489 }
1490
1491 if(sourceLimit==utf8Limit) {
1492 // we are done
1493 if(*pErrorCode==U_STRING_NOT_TERMINATED_WARNING) {
1494 // ucnv_convertEx() warns about not terminating the output
1495 // but ucnv_fromUnicode() does not and so
1496 // checkFromUnicode() does not expect it
1497 *pErrorCode=U_ZERO_ERROR;
1498 }
1499 break;
1500 }
1501
1502 // the partial conversion succeeded, set a new limit and continue
1503 sourceLimit=(utf8Limit-source)>=step ? source+step : utf8Limit;
1504 flush=(UBool)(cc.finalFlush && sourceLimit==utf8Limit);
1505 }
1506 }
1507
1508 return (int32_t)(target-result);
1509 }
1510
1511 static int32_t
stepFromUnicode(ConversionCase & cc,UConverter * cnv,char * result,int32_t resultCapacity,int32_t * resultOffsets,int32_t step,UErrorCode * pErrorCode)1512 stepFromUnicode(ConversionCase &cc, UConverter *cnv,
1513 char *result, int32_t resultCapacity,
1514 int32_t *resultOffsets, /* also resultCapacity */
1515 int32_t step,
1516 UErrorCode *pErrorCode) {
1517 const UChar *source, *sourceLimit, *unicodeLimit;
1518 char *target, *targetLimit, *resultLimit;
1519 UBool flush;
1520
1521 source=cc.unicode;
1522 target=result;
1523 unicodeLimit=source+cc.unicodeLength;
1524 resultLimit=result+resultCapacity;
1525
1526 // call ucnv_fromUnicode() with in/out buffers no larger than (step) at a time
1527 // move only one buffer (in vs. out) at a time to be extra mean
1528 // step==0 performs bulk conversion and generates offsets
1529
1530 // initialize the partial limits for the loop
1531 if(step==0) {
1532 // use the entire buffers
1533 sourceLimit=unicodeLimit;
1534 targetLimit=resultLimit;
1535 flush=cc.finalFlush;
1536 } else {
1537 // start with empty partial buffers
1538 sourceLimit=source;
1539 targetLimit=target;
1540 flush=FALSE;
1541
1542 // output offsets only for bulk conversion
1543 resultOffsets=NULL;
1544 }
1545
1546 for(;;) {
1547 // resetting the opposite conversion direction must not affect this one
1548 ucnv_resetToUnicode(cnv);
1549
1550 // convert
1551 ucnv_fromUnicode(cnv,
1552 &target, targetLimit,
1553 &source, sourceLimit,
1554 resultOffsets,
1555 flush, pErrorCode);
1556
1557 // check pointers and errors
1558 if(source>sourceLimit || target>targetLimit) {
1559 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1560 break;
1561 } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
1562 if(target!=targetLimit) {
1563 // buffer overflow must only be set when the target is filled
1564 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1565 break;
1566 } else if(targetLimit==resultLimit) {
1567 // not just a partial overflow
1568 break;
1569 }
1570
1571 // the partial target is filled, set a new limit, reset the error and continue
1572 targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
1573 *pErrorCode=U_ZERO_ERROR;
1574 } else if(U_FAILURE(*pErrorCode)) {
1575 // some other error occurred, done
1576 break;
1577 } else {
1578 if(source!=sourceLimit) {
1579 // when no error occurs, then the input must be consumed
1580 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1581 break;
1582 }
1583
1584 if(sourceLimit==unicodeLimit) {
1585 // we are done
1586 break;
1587 }
1588
1589 // the partial conversion succeeded, set a new limit and continue
1590 sourceLimit=(unicodeLimit-source)>=step ? source+step : unicodeLimit;
1591 flush=(UBool)(cc.finalFlush && sourceLimit==unicodeLimit);
1592 }
1593 }
1594
1595 return (int32_t)(target-result);
1596 }
1597
1598 UBool
FromUnicodeCase(ConversionCase & cc,UConverterFromUCallback callback,const char * option)1599 ConversionTest::FromUnicodeCase(ConversionCase &cc, UConverterFromUCallback callback, const char *option) {
1600 UConverter *cnv;
1601 UErrorCode errorCode;
1602
1603 // open the converter
1604 errorCode=U_ZERO_ERROR;
1605 cnv=cnv_open(cc.charset, errorCode);
1606 if(U_FAILURE(errorCode)) {
1607 errcheckln(errorCode, "fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_open() failed - %s",
1608 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1609 return FALSE;
1610 }
1611 ucnv_resetToUnicode(utf8Cnv);
1612
1613 // set the callback
1614 if(callback!=NULL) {
1615 ucnv_setFromUCallBack(cnv, callback, option, NULL, NULL, &errorCode);
1616 if(U_FAILURE(errorCode)) {
1617 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setFromUCallBack() failed - %s",
1618 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1619 ucnv_close(cnv);
1620 return FALSE;
1621 }
1622 }
1623
1624 // set the fallbacks flag
1625 // TODO change with Jitterbug 2401, then add a similar call for toUnicode too
1626 ucnv_setFallback(cnv, cc.fallbacks);
1627
1628 // set the subchar
1629 int32_t length;
1630
1631 if(cc.setSub>0) {
1632 length=(int32_t)strlen(cc.subchar);
1633 ucnv_setSubstChars(cnv, cc.subchar, (int8_t)length, &errorCode);
1634 if(U_FAILURE(errorCode)) {
1635 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setSubstChars() failed - %s",
1636 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1637 ucnv_close(cnv);
1638 return FALSE;
1639 }
1640 } else if(cc.setSub<0) {
1641 ucnv_setSubstString(cnv, cc.subString, -1, &errorCode);
1642 if(U_FAILURE(errorCode)) {
1643 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setSubstString() failed - %s",
1644 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1645 ucnv_close(cnv);
1646 return FALSE;
1647 }
1648 }
1649
1650 // convert unicode to utf8
1651 char utf8[256];
1652 cc.utf8=utf8;
1653 u_strToUTF8(utf8, UPRV_LENGTHOF(utf8), &cc.utf8Length,
1654 cc.unicode, cc.unicodeLength,
1655 &errorCode);
1656 if(U_FAILURE(errorCode)) {
1657 // skip UTF-8 testing of a string with an unpaired surrogate,
1658 // or of one that's too long
1659 // toUnicode errors are tested in cintltst TestConvertExFromUTF8()
1660 cc.utf8Length=-1;
1661 }
1662
1663 int32_t resultOffsets[256];
1664 char result[256];
1665 int32_t resultLength;
1666 UBool ok;
1667
1668 static const struct {
1669 int32_t step;
1670 const char *name, *utf8Name;
1671 } steps[]={
1672 { 0, "bulk", "utf8" }, // must be first for offsets to be checked
1673 { 1, "step=1", "utf8 step=1" },
1674 { 3, "step=3", "utf8 step=3" },
1675 { 7, "step=7", "utf8 step=7" }
1676 };
1677 int32_t i, step;
1678
1679 ok=TRUE;
1680 for(i=0; i<UPRV_LENGTHOF(steps) && ok; ++i) {
1681 step=steps[i].step;
1682 for (int32_t i = 0; i < UPRV_LENGTHOF(resultOffsets); i++) {
1683 resultOffsets[i] = -1;
1684 }
1685 for (int32_t i = 0; i < UPRV_LENGTHOF(result); i++) {
1686 result[i] = -1;
1687 }
1688 errorCode=U_ZERO_ERROR;
1689 resultLength=stepFromUnicode(cc, cnv,
1690 result, UPRV_LENGTHOF(result),
1691 step==0 ? resultOffsets : NULL,
1692 step, &errorCode);
1693 ok=checkFromUnicode(
1694 cc, cnv, steps[i].name,
1695 (uint8_t *)result, resultLength,
1696 cc.offsets!=NULL ? resultOffsets : NULL,
1697 errorCode);
1698 if(U_FAILURE(errorCode) || !cc.finalFlush) {
1699 // reset if an error occurred or we did not flush
1700 // otherwise do nothing to make sure that flushing resets
1701 ucnv_resetFromUnicode(cnv);
1702 }
1703 if (resultOffsets[resultLength] != -1) {
1704 errln("fromUnicode[%d](%s) Conversion wrote too much to offsets at index %d",
1705 cc.caseNr, cc.charset, resultLength);
1706 }
1707 if (result[resultLength] != (char)-1) {
1708 errln("fromUnicode[%d](%s) Conversion wrote too much to result at index %d",
1709 cc.caseNr, cc.charset, resultLength);
1710 }
1711
1712 // bulk test is first, then offsets are not checked any more
1713 cc.offsets=NULL;
1714
1715 // test direct conversion from UTF-8
1716 if(cc.utf8Length>=0) {
1717 errorCode=U_ZERO_ERROR;
1718 resultLength=stepFromUTF8(cc, utf8Cnv, cnv,
1719 result, UPRV_LENGTHOF(result),
1720 step, &errorCode);
1721 ok=checkFromUnicode(
1722 cc, cnv, steps[i].utf8Name,
1723 (uint8_t *)result, resultLength,
1724 NULL,
1725 errorCode);
1726 if(U_FAILURE(errorCode) || !cc.finalFlush) {
1727 // reset if an error occurred or we did not flush
1728 // otherwise do nothing to make sure that flushing resets
1729 ucnv_resetToUnicode(utf8Cnv);
1730 ucnv_resetFromUnicode(cnv);
1731 }
1732 }
1733 }
1734
1735 // not a real loop, just a convenience for breaking out of the block
1736 while(ok && cc.finalFlush) {
1737 // test ucnv_fromUChars()
1738 memset(result, 0, sizeof(result));
1739
1740 errorCode=U_ZERO_ERROR;
1741 resultLength=ucnv_fromUChars(cnv,
1742 result, UPRV_LENGTHOF(result),
1743 cc.unicode, cc.unicodeLength,
1744 &errorCode);
1745 ok=checkFromUnicode(
1746 cc, cnv, "fromUChars",
1747 (uint8_t *)result, resultLength,
1748 NULL,
1749 errorCode);
1750 if(!ok) {
1751 break;
1752 }
1753
1754 // test preflighting
1755 // keep the correct result for simple checking
1756 errorCode=U_ZERO_ERROR;
1757 resultLength=ucnv_fromUChars(cnv,
1758 NULL, 0,
1759 cc.unicode, cc.unicodeLength,
1760 &errorCode);
1761 if(errorCode==U_STRING_NOT_TERMINATED_WARNING || errorCode==U_BUFFER_OVERFLOW_ERROR) {
1762 errorCode=U_ZERO_ERROR;
1763 }
1764 ok=checkFromUnicode(
1765 cc, cnv, "preflight fromUChars",
1766 (uint8_t *)result, resultLength,
1767 NULL,
1768 errorCode);
1769 break;
1770 }
1771
1772 ucnv_close(cnv);
1773 return ok;
1774 }
1775
1776 UBool
checkFromUnicode(ConversionCase & cc,UConverter * cnv,const char * name,const uint8_t * result,int32_t resultLength,const int32_t * resultOffsets,UErrorCode resultErrorCode)1777 ConversionTest::checkFromUnicode(ConversionCase &cc, UConverter *cnv, const char *name,
1778 const uint8_t *result, int32_t resultLength,
1779 const int32_t *resultOffsets,
1780 UErrorCode resultErrorCode) {
1781 UChar resultInvalidUChars[8];
1782 int8_t resultInvalidLength;
1783 UErrorCode errorCode;
1784
1785 const char *msg;
1786
1787 // reset the message; NULL will mean "ok"
1788 msg=NULL;
1789
1790 errorCode=U_ZERO_ERROR;
1791 resultInvalidLength=UPRV_LENGTHOF(resultInvalidUChars);
1792 ucnv_getInvalidUChars(cnv, resultInvalidUChars, &resultInvalidLength, &errorCode);
1793 if(U_FAILURE(errorCode)) {
1794 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) ucnv_getInvalidUChars() failed - %s",
1795 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, u_errorName(errorCode));
1796 return FALSE;
1797 }
1798
1799 // check everything that might have gone wrong
1800 if(cc.bytesLength!=resultLength) {
1801 msg="wrong result length";
1802 } else if(0!=memcmp(cc.bytes, result, cc.bytesLength)) {
1803 msg="wrong result string";
1804 } else if(cc.offsets!=NULL && 0!=memcmp(cc.offsets, resultOffsets, cc.bytesLength*sizeof(*cc.offsets))) {
1805 msg="wrong offsets";
1806 } else if(cc.outErrorCode!=resultErrorCode) {
1807 msg="wrong error code";
1808 } else if(cc.invalidLength!=resultInvalidLength) {
1809 msg="wrong length of last invalid input";
1810 } else if(0!=u_memcmp(cc.invalidUChars, resultInvalidUChars, cc.invalidLength)) {
1811 msg="wrong last invalid input";
1812 }
1813
1814 if(msg==NULL) {
1815 return TRUE;
1816 } else {
1817 char buffer[2000]; // one buffer for all strings
1818 char *s, *unicodeString, *bytesString, *resultString,
1819 *offsetsString, *resultOffsetsString,
1820 *invalidCharsString, *resultInvalidUCharsString;
1821
1822 unicodeString=s=buffer;
1823 s=printUnicode(cc.unicode, cc.unicodeLength, unicodeString);
1824 s=printBytes(cc.bytes, cc.bytesLength, bytesString=s);
1825 s=printBytes(result, resultLength, resultString=s);
1826 s=printOffsets(cc.offsets, cc.bytesLength, offsetsString=s);
1827 s=printOffsets(resultOffsets, resultLength, resultOffsetsString=s);
1828 s=printUnicode(cc.invalidUChars, cc.invalidLength, invalidCharsString=s);
1829 s=printUnicode(resultInvalidUChars, resultInvalidLength, resultInvalidUCharsString=s);
1830
1831 if((s-buffer)>(int32_t)sizeof(buffer)) {
1832 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) fatal error: checkFromUnicode() test output buffer overflow writing %d chars\n",
1833 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, (int)(s-buffer));
1834 exit(1);
1835 }
1836
1837 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) failed: %s\n"
1838 " unicode <%s>[%d]\n"
1839 " expected <%s>[%d]\n"
1840 " result <%s>[%d]\n"
1841 " offsets <%s>\n"
1842 " result offsets <%s>\n"
1843 " error code expected %s got %s\n"
1844 " invalidChars expected <%s> got <%s>\n",
1845 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, msg,
1846 unicodeString, cc.unicodeLength,
1847 bytesString, cc.bytesLength,
1848 resultString, resultLength,
1849 offsetsString,
1850 resultOffsetsString,
1851 u_errorName(cc.outErrorCode), u_errorName(resultErrorCode),
1852 invalidCharsString, resultInvalidUCharsString);
1853
1854 return FALSE;
1855 }
1856 }
1857
1858 #endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */
1859