1 // © 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
3 /*
4 *******************************************************************************
5 *
6 * Copyright (C) 2003-2014, International Business Machines
7 * Corporation and others. All Rights Reserved.
8 *
9 *******************************************************************************
10 * file name: convtest.cpp
11 * encoding: UTF-8
12 * tab size: 8 (not used)
13 * indentation:4
14 *
15 * created on: 2003jul15
16 * created by: Markus W. Scherer
17 *
18 * Test file for data-driven conversion tests.
19 */
20
21 #include "unicode/utypes.h"
22
23 #if !UCONFIG_NO_LEGACY_CONVERSION
24 /*
25 * Note: Turning off all of convtest.cpp if !UCONFIG_NO_LEGACY_CONVERSION
26 * is slightly unnecessary - it removes tests for Unicode charsets
27 * like UTF-8 that should work.
28 * However, there is no easy way for the test to detect whether a test case
29 * is for a Unicode charset, so it would be difficult to only exclude those.
30 * Also, regular testing of ICU is done with all modules on, therefore
31 * not testing conversion for a custom configuration like this should be ok.
32 */
33
34 #include "unicode/ucnv.h"
35 #include "unicode/unistr.h"
36 #include "unicode/parsepos.h"
37 #include "unicode/uniset.h"
38 #include "unicode/ustring.h"
39 #include "unicode/ures.h"
40 #include "unicode/utf16.h"
41 #include "convtest.h"
42 #include "cmemory.h"
43 #include "unicode/tstdtmod.h"
44 #include <string.h>
45 #include <stdlib.h>
46
47 enum {
48 // characters used in test data for callbacks
49 SUB_CB='?',
50 SKIP_CB='0',
51 STOP_CB='.',
52 ESC_CB='&'
53 };
54
ConversionTest()55 ConversionTest::ConversionTest() {
56 UErrorCode errorCode=U_ZERO_ERROR;
57 utf8Cnv=ucnv_open("UTF-8", &errorCode);
58 ucnv_setToUCallBack(utf8Cnv, UCNV_TO_U_CALLBACK_STOP, NULL, NULL, NULL, &errorCode);
59 if(U_FAILURE(errorCode)) {
60 errln("unable to open UTF-8 converter");
61 }
62 }
63
~ConversionTest()64 ConversionTest::~ConversionTest() {
65 ucnv_close(utf8Cnv);
66 }
67
68 void
runIndexedTest(int32_t index,UBool exec,const char * & name,char *)69 ConversionTest::runIndexedTest(int32_t index, UBool exec, const char *&name, char * /*par*/) {
70 if (exec) logln("TestSuite ConversionTest: ");
71 TESTCASE_AUTO_BEGIN;
72 #if !UCONFIG_NO_FILE_IO
73 TESTCASE_AUTO(TestToUnicode);
74 TESTCASE_AUTO(TestFromUnicode);
75 TESTCASE_AUTO(TestGetUnicodeSet);
76 #endif
77 TESTCASE_AUTO(TestGetUnicodeSet2);
78 TESTCASE_AUTO(TestDefaultIgnorableCallback);
79 TESTCASE_AUTO(TestUTF8ToUTF8Overflow);
80 TESTCASE_AUTO_END;
81 }
82
83 // test data interface ----------------------------------------------------- ***
84
85 void
TestToUnicode()86 ConversionTest::TestToUnicode() {
87 ConversionCase cc;
88 char charset[100], cbopt[4];
89 const char *option;
90 UnicodeString s, unicode;
91 int32_t offsetsLength;
92 UConverterToUCallback callback;
93
94 TestDataModule *dataModule;
95 TestData *testData;
96 const DataMap *testCase;
97 UErrorCode errorCode;
98 int32_t i;
99
100 errorCode=U_ZERO_ERROR;
101 dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
102 if(U_SUCCESS(errorCode)) {
103 testData=dataModule->createTestData("toUnicode", errorCode);
104 if(U_SUCCESS(errorCode)) {
105 for(i=0; testData->nextCase(testCase, errorCode); ++i) {
106 if(U_FAILURE(errorCode)) {
107 errln("error retrieving conversion/toUnicode test case %d - %s",
108 i, u_errorName(errorCode));
109 errorCode=U_ZERO_ERROR;
110 continue;
111 }
112
113 cc.caseNr=i;
114
115 s=testCase->getString("charset", errorCode);
116 s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
117 cc.charset=charset;
118
119 // BEGIN android-added
120 // To save space, Android does not build full ISO-2022-CN tables.
121 // We skip the TestGetKeywordValuesForLocale for counting available collations.
122 if (strlen(charset) >= 8 &&
123 strncmp(charset+4, "2022-CN", 4) == 0) {
124 continue;
125 }
126 // END android-added
127
128 cc.bytes=testCase->getBinary(cc.bytesLength, "bytes", errorCode);
129 unicode=testCase->getString("unicode", errorCode);
130 cc.unicode=unicode.getBuffer();
131 cc.unicodeLength=unicode.length();
132
133 offsetsLength=0;
134 cc.offsets=testCase->getIntVector(offsetsLength, "offsets", errorCode);
135 if(offsetsLength==0) {
136 cc.offsets=NULL;
137 } else if(offsetsLength!=unicode.length()) {
138 errln("toUnicode[%d] unicode[%d] and offsets[%d] must have the same length",
139 i, unicode.length(), offsetsLength);
140 errorCode=U_ILLEGAL_ARGUMENT_ERROR;
141 }
142
143 cc.finalFlush= 0!=testCase->getInt28("flush", errorCode);
144 cc.fallbacks= 0!=testCase->getInt28("fallbacks", errorCode);
145
146 s=testCase->getString("errorCode", errorCode);
147 if(s==UNICODE_STRING("invalid", 7)) {
148 cc.outErrorCode=U_INVALID_CHAR_FOUND;
149 } else if(s==UNICODE_STRING("illegal", 7)) {
150 cc.outErrorCode=U_ILLEGAL_CHAR_FOUND;
151 } else if(s==UNICODE_STRING("truncated", 9)) {
152 cc.outErrorCode=U_TRUNCATED_CHAR_FOUND;
153 } else if(s==UNICODE_STRING("illesc", 6)) {
154 cc.outErrorCode=U_ILLEGAL_ESCAPE_SEQUENCE;
155 } else if(s==UNICODE_STRING("unsuppesc", 9)) {
156 cc.outErrorCode=U_UNSUPPORTED_ESCAPE_SEQUENCE;
157 } else {
158 cc.outErrorCode=U_ZERO_ERROR;
159 }
160
161 s=testCase->getString("callback", errorCode);
162 s.extract(0, 0x7fffffff, cbopt, sizeof(cbopt), "");
163 cc.cbopt=cbopt;
164 switch(cbopt[0]) {
165 case SUB_CB:
166 callback=UCNV_TO_U_CALLBACK_SUBSTITUTE;
167 break;
168 case SKIP_CB:
169 callback=UCNV_TO_U_CALLBACK_SKIP;
170 break;
171 case STOP_CB:
172 callback=UCNV_TO_U_CALLBACK_STOP;
173 break;
174 case ESC_CB:
175 callback=UCNV_TO_U_CALLBACK_ESCAPE;
176 break;
177 default:
178 callback=NULL;
179 break;
180 }
181 option=callback==NULL ? cbopt : cbopt+1;
182 if(*option==0) {
183 option=NULL;
184 }
185
186 cc.invalidChars=testCase->getBinary(cc.invalidLength, "invalidChars", errorCode);
187
188 if(U_FAILURE(errorCode)) {
189 errln("error parsing conversion/toUnicode test case %d - %s",
190 i, u_errorName(errorCode));
191 errorCode=U_ZERO_ERROR;
192 } else {
193 logln("TestToUnicode[%d] %s", i, charset);
194 ToUnicodeCase(cc, callback, option);
195 }
196 }
197 delete testData;
198 }
199 delete dataModule;
200 }
201 else {
202 dataerrln("Could not load test conversion data");
203 }
204 }
205
206 void
TestFromUnicode()207 ConversionTest::TestFromUnicode() {
208 ConversionCase cc;
209 char charset[100], cbopt[4];
210 const char *option;
211 UnicodeString s, unicode, invalidUChars;
212 int32_t offsetsLength, index;
213 UConverterFromUCallback callback;
214
215 TestDataModule *dataModule;
216 TestData *testData;
217 const DataMap *testCase;
218 const UChar *p;
219 UErrorCode errorCode;
220 int32_t i, length;
221
222 errorCode=U_ZERO_ERROR;
223 dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
224 if(U_SUCCESS(errorCode)) {
225 testData=dataModule->createTestData("fromUnicode", errorCode);
226 if(U_SUCCESS(errorCode)) {
227 for(i=0; testData->nextCase(testCase, errorCode); ++i) {
228 if(U_FAILURE(errorCode)) {
229 errln("error retrieving conversion/fromUnicode test case %d - %s",
230 i, u_errorName(errorCode));
231 errorCode=U_ZERO_ERROR;
232 continue;
233 }
234
235 cc.caseNr=i;
236
237 s=testCase->getString("charset", errorCode);
238 s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
239 cc.charset=charset;
240
241 // BEGIN android-added
242 // To save space, Android does not build full ISO-2022-CN tables.
243 // We skip the TestGetKeywordValuesForLocale for counting available collations.
244 if (strlen(charset) >= 8 &&
245 strncmp(charset+4, "2022-CN", 4) == 0) {
246 continue;
247 }
248 // END android-added
249
250 unicode=testCase->getString("unicode", errorCode);
251 cc.unicode=unicode.getBuffer();
252 cc.unicodeLength=unicode.length();
253 cc.bytes=testCase->getBinary(cc.bytesLength, "bytes", errorCode);
254
255 offsetsLength=0;
256 cc.offsets=testCase->getIntVector(offsetsLength, "offsets", errorCode);
257 if(offsetsLength==0) {
258 cc.offsets=NULL;
259 } else if(offsetsLength!=cc.bytesLength) {
260 errln("fromUnicode[%d] bytes[%d] and offsets[%d] must have the same length",
261 i, cc.bytesLength, offsetsLength);
262 errorCode=U_ILLEGAL_ARGUMENT_ERROR;
263 }
264
265 cc.finalFlush= 0!=testCase->getInt28("flush", errorCode);
266 cc.fallbacks= 0!=testCase->getInt28("fallbacks", errorCode);
267
268 s=testCase->getString("errorCode", errorCode);
269 if(s==UNICODE_STRING("invalid", 7)) {
270 cc.outErrorCode=U_INVALID_CHAR_FOUND;
271 } else if(s==UNICODE_STRING("illegal", 7)) {
272 cc.outErrorCode=U_ILLEGAL_CHAR_FOUND;
273 } else if(s==UNICODE_STRING("truncated", 9)) {
274 cc.outErrorCode=U_TRUNCATED_CHAR_FOUND;
275 } else {
276 cc.outErrorCode=U_ZERO_ERROR;
277 }
278
279 s=testCase->getString("callback", errorCode);
280 cc.setSub=0; // default: no subchar
281
282 if((index=s.indexOf((UChar)0))>0) {
283 // read NUL-separated subchar first, if any
284 // copy the subchar from Latin-1 characters
285 // start after the NUL
286 p=s.getTerminatedBuffer();
287 length=index+1;
288 p+=length;
289 length=s.length()-length;
290 if(length<=0 || length>=(int32_t)sizeof(cc.subchar)) {
291 errorCode=U_ILLEGAL_ARGUMENT_ERROR;
292 } else {
293 int32_t j;
294
295 for(j=0; j<length; ++j) {
296 cc.subchar[j]=(char)p[j];
297 }
298 // NUL-terminate the subchar
299 cc.subchar[j]=0;
300 cc.setSub=1;
301 }
302
303 // remove the NUL and subchar from s
304 s.truncate(index);
305 } else if((index=s.indexOf((UChar)0x3d))>0) /* '=' */ {
306 // read a substitution string, separated by an equal sign
307 p=s.getBuffer()+index+1;
308 length=s.length()-(index+1);
309 if(length<0 || length>=UPRV_LENGTHOF(cc.subString)) {
310 errorCode=U_ILLEGAL_ARGUMENT_ERROR;
311 } else {
312 u_memcpy(cc.subString, p, length);
313 // NUL-terminate the subString
314 cc.subString[length]=0;
315 cc.setSub=-1;
316 }
317
318 // remove the equal sign and subString from s
319 s.truncate(index);
320 }
321
322 s.extract(0, 0x7fffffff, cbopt, sizeof(cbopt), "");
323 cc.cbopt=cbopt;
324 switch(cbopt[0]) {
325 case SUB_CB:
326 callback=UCNV_FROM_U_CALLBACK_SUBSTITUTE;
327 break;
328 case SKIP_CB:
329 callback=UCNV_FROM_U_CALLBACK_SKIP;
330 break;
331 case STOP_CB:
332 callback=UCNV_FROM_U_CALLBACK_STOP;
333 break;
334 case ESC_CB:
335 callback=UCNV_FROM_U_CALLBACK_ESCAPE;
336 break;
337 default:
338 callback=NULL;
339 break;
340 }
341 option=callback==NULL ? cbopt : cbopt+1;
342 if(*option==0) {
343 option=NULL;
344 }
345
346 invalidUChars=testCase->getString("invalidUChars", errorCode);
347 cc.invalidUChars=invalidUChars.getBuffer();
348 cc.invalidLength=invalidUChars.length();
349
350 if(U_FAILURE(errorCode)) {
351 errln("error parsing conversion/fromUnicode test case %d - %s",
352 i, u_errorName(errorCode));
353 errorCode=U_ZERO_ERROR;
354 } else {
355 logln("TestFromUnicode[%d] %s", i, charset);
356 FromUnicodeCase(cc, callback, option);
357 }
358 }
359 delete testData;
360 }
361 delete dataModule;
362 }
363 else {
364 dataerrln("Could not load test conversion data");
365 }
366 }
367
368 static const UChar ellipsis[]={ 0x2e, 0x2e, 0x2e };
369
370 void
TestGetUnicodeSet()371 ConversionTest::TestGetUnicodeSet() {
372 char charset[100];
373 UnicodeString s, map, mapnot;
374 int32_t which;
375
376 ParsePosition pos;
377 UnicodeSet cnvSet, mapSet, mapnotSet, diffSet;
378 UnicodeSet *cnvSetPtr = &cnvSet;
379 LocalUConverterPointer cnv;
380
381 TestDataModule *dataModule;
382 TestData *testData;
383 const DataMap *testCase;
384 UErrorCode errorCode;
385 int32_t i;
386
387 errorCode=U_ZERO_ERROR;
388 dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode);
389 if(U_SUCCESS(errorCode)) {
390 testData=dataModule->createTestData("getUnicodeSet", errorCode);
391 if(U_SUCCESS(errorCode)) {
392 for(i=0; testData->nextCase(testCase, errorCode); ++i) {
393 if(U_FAILURE(errorCode)) {
394 errln("error retrieving conversion/getUnicodeSet test case %d - %s",
395 i, u_errorName(errorCode));
396 errorCode=U_ZERO_ERROR;
397 continue;
398 }
399
400 s=testCase->getString("charset", errorCode);
401 s.extract(0, 0x7fffffff, charset, sizeof(charset), "");
402
403 // BEGIN android-added
404 // To save space, Android does not build full ISO-2022-CN tables.
405 // We skip the TestGetKeywordValuesForLocale for counting available collations.
406 if (strlen(charset) >= 8 &&
407 strncmp(charset+4, "2022-CN", 4) == 0) {
408 continue;
409 }
410 // END android-added
411
412 map=testCase->getString("map", errorCode);
413 mapnot=testCase->getString("mapnot", errorCode);
414
415 which=testCase->getInt28("which", errorCode);
416
417 if(U_FAILURE(errorCode)) {
418 errln("error parsing conversion/getUnicodeSet test case %d - %s",
419 i, u_errorName(errorCode));
420 errorCode=U_ZERO_ERROR;
421 continue;
422 }
423
424 // test this test case
425 mapSet.clear();
426 mapnotSet.clear();
427
428 pos.setIndex(0);
429 mapSet.applyPattern(map, pos, 0, NULL, errorCode);
430 if(U_FAILURE(errorCode) || pos.getIndex()!=map.length()) {
431 errln("error creating the map set for conversion/getUnicodeSet test case %d - %s\n"
432 " error index %d index %d U+%04x",
433 i, u_errorName(errorCode), pos.getErrorIndex(), pos.getIndex(), map.char32At(pos.getIndex()));
434 errorCode=U_ZERO_ERROR;
435 continue;
436 }
437
438 pos.setIndex(0);
439 mapnotSet.applyPattern(mapnot, pos, 0, NULL, errorCode);
440 if(U_FAILURE(errorCode) || pos.getIndex()!=mapnot.length()) {
441 errln("error creating the mapnot set for conversion/getUnicodeSet test case %d - %s\n"
442 " error index %d index %d U+%04x",
443 i, u_errorName(errorCode), pos.getErrorIndex(), pos.getIndex(), mapnot.char32At(pos.getIndex()));
444 errorCode=U_ZERO_ERROR;
445 continue;
446 }
447
448 logln("TestGetUnicodeSet[%d] %s", i, charset);
449
450 cnv.adoptInstead(cnv_open(charset, errorCode));
451 if(U_FAILURE(errorCode)) {
452 errcheckln(errorCode, "error opening \"%s\" for conversion/getUnicodeSet test case %d - %s",
453 charset, i, u_errorName(errorCode));
454 errorCode=U_ZERO_ERROR;
455 continue;
456 }
457
458 ucnv_getUnicodeSet(cnv.getAlias(), cnvSetPtr->toUSet(), (UConverterUnicodeSet)which, &errorCode);
459
460 if(U_FAILURE(errorCode)) {
461 errln("error in ucnv_getUnicodeSet(\"%s\") for conversion/getUnicodeSet test case %d - %s",
462 charset, i, u_errorName(errorCode));
463 errorCode=U_ZERO_ERROR;
464 continue;
465 }
466
467 // are there items that must be in cnvSet but are not?
468 (diffSet=mapSet).removeAll(cnvSet);
469 if(!diffSet.isEmpty()) {
470 diffSet.toPattern(s, TRUE);
471 if(s.length()>100) {
472 s.replace(100, 0x7fffffff, ellipsis, UPRV_LENGTHOF(ellipsis));
473 }
474 errln("error: ucnv_getUnicodeSet(\"%s\") is missing items - conversion/getUnicodeSet test case %d",
475 charset, i);
476 errln(s);
477 }
478
479 // are there items that must not be in cnvSet but are?
480 (diffSet=mapnotSet).retainAll(cnvSet);
481 if(!diffSet.isEmpty()) {
482 diffSet.toPattern(s, TRUE);
483 if(s.length()>100) {
484 s.replace(100, 0x7fffffff, ellipsis, UPRV_LENGTHOF(ellipsis));
485 }
486 errln("error: ucnv_getUnicodeSet(\"%s\") contains unexpected items - conversion/getUnicodeSet test case %d",
487 charset, i);
488 errln(s);
489 }
490 }
491 delete testData;
492 }
493 delete dataModule;
494 }
495 else {
496 dataerrln("Could not load test conversion data");
497 }
498 }
499
500 U_CDECL_BEGIN
501 static void U_CALLCONV
getUnicodeSetCallback(const void * context,UConverterFromUnicodeArgs *,const UChar *,int32_t,UChar32 codePoint,UConverterCallbackReason reason,UErrorCode * pErrorCode)502 getUnicodeSetCallback(const void *context,
503 UConverterFromUnicodeArgs * /*fromUArgs*/,
504 const UChar* /*codeUnits*/,
505 int32_t /*length*/,
506 UChar32 codePoint,
507 UConverterCallbackReason reason,
508 UErrorCode *pErrorCode) {
509 if(reason<=UCNV_IRREGULAR) {
510 ((UnicodeSet *)context)->remove(codePoint); // the converter cannot convert this code point
511 *pErrorCode=U_ZERO_ERROR; // skip
512 } // else ignore the reset, close and clone calls.
513 }
514 U_CDECL_END
515
516 // Compare ucnv_getUnicodeSet() with the set of characters that can be converted.
517 void
TestGetUnicodeSet2()518 ConversionTest::TestGetUnicodeSet2() {
519 // Build a string with all code points.
520 UChar32 cpLimit;
521 int32_t s0Length;
522 if(quick) {
523 cpLimit=s0Length=0x10000; // BMP only
524 } else {
525 cpLimit=0x110000;
526 s0Length=0x10000+0x200000; // BMP + surrogate pairs
527 }
528 UChar *s0=new UChar[s0Length];
529 if(s0==NULL) {
530 return;
531 }
532 UChar *s=s0;
533 UChar32 c;
534 UChar c2;
535 // low BMP
536 for(c=0; c<=0xd7ff; ++c) {
537 *s++=(UChar)c;
538 }
539 // trail surrogates
540 for(c=0xdc00; c<=0xdfff; ++c) {
541 *s++=(UChar)c;
542 }
543 // lead surrogates
544 // (after trails so that there is not even one surrogate pair in between)
545 for(c=0xd800; c<=0xdbff; ++c) {
546 *s++=(UChar)c;
547 }
548 // high BMP
549 for(c=0xe000; c<=0xffff; ++c) {
550 *s++=(UChar)c;
551 }
552 // supplementary code points = surrogate pairs
553 if(cpLimit==0x110000) {
554 for(c=0xd800; c<=0xdbff; ++c) {
555 for(c2=0xdc00; c2<=0xdfff; ++c2) {
556 *s++=(UChar)c;
557 *s++=c2;
558 }
559 }
560 }
561
562 static const char *const cnvNames[]={
563 "UTF-8",
564 "UTF-7",
565 "UTF-16",
566 "US-ASCII",
567 "ISO-8859-1",
568 "windows-1252",
569 "Shift-JIS",
570 "ibm-1390", // EBCDIC_STATEFUL table
571 "ibm-16684", // DBCS-only extension table based on EBCDIC_STATEFUL table
572 "HZ",
573 "ISO-2022-JP",
574 "JIS7",
575 "ISO-2022-CN",
576 "ISO-2022-CN-EXT",
577 "LMBCS"
578 };
579 LocalUConverterPointer cnv;
580 char buffer[1024];
581 int32_t i;
582 for(i=0; i<UPRV_LENGTHOF(cnvNames); ++i) {
583 UErrorCode errorCode=U_ZERO_ERROR;
584 cnv.adoptInstead(cnv_open(cnvNames[i], errorCode));
585 if(U_FAILURE(errorCode)) {
586 errcheckln(errorCode, "failed to open converter %s - %s", cnvNames[i], u_errorName(errorCode));
587 continue;
588 }
589 UnicodeSet expected;
590 ucnv_setFromUCallBack(cnv.getAlias(), getUnicodeSetCallback, &expected, NULL, NULL, &errorCode);
591 if(U_FAILURE(errorCode)) {
592 errln("failed to set the callback on converter %s - %s", cnvNames[i], u_errorName(errorCode));
593 continue;
594 }
595 UConverterUnicodeSet which;
596 for(which=UCNV_ROUNDTRIP_SET; which<UCNV_SET_COUNT; which=(UConverterUnicodeSet)((int)which+1)) {
597 if(which==UCNV_ROUNDTRIP_AND_FALLBACK_SET) {
598 ucnv_setFallback(cnv.getAlias(), TRUE);
599 }
600 expected.add(0, cpLimit-1);
601 s=s0;
602 UBool flush;
603 do {
604 char *t=buffer;
605 flush=(UBool)(s==s0+s0Length);
606 ucnv_fromUnicode(cnv.getAlias(), &t, buffer+sizeof(buffer), (const UChar **)&s, s0+s0Length, NULL, flush, &errorCode);
607 if(U_FAILURE(errorCode)) {
608 if(errorCode==U_BUFFER_OVERFLOW_ERROR) {
609 errorCode=U_ZERO_ERROR;
610 continue;
611 } else {
612 break; // unexpected error, should not occur
613 }
614 }
615 } while(!flush);
616 UnicodeSet set;
617 ucnv_getUnicodeSet(cnv.getAlias(), set.toUSet(), which, &errorCode);
618 if(cpLimit<0x110000) {
619 set.remove(cpLimit, 0x10ffff);
620 }
621 if(which==UCNV_ROUNDTRIP_SET) {
622 // ignore PUA code points because they will be converted even if they
623 // are fallbacks and when other fallbacks are turned off,
624 // but ucnv_getUnicodeSet(UCNV_ROUNDTRIP_SET) delivers true roundtrips
625 expected.remove(0xe000, 0xf8ff);
626 expected.remove(0xf0000, 0xffffd);
627 expected.remove(0x100000, 0x10fffd);
628 set.remove(0xe000, 0xf8ff);
629 set.remove(0xf0000, 0xffffd);
630 set.remove(0x100000, 0x10fffd);
631 }
632 if(set!=expected) {
633 // First try to see if we have different sets because ucnv_getUnicodeSet()
634 // added strings: The above conversion method does not tell us what strings might be convertible.
635 // Remove strings from the set and compare again.
636 set.removeAllStrings();
637 }
638 if(set!=expected) {
639 UnicodeSet diffSet;
640 UnicodeString out;
641
642 // are there items that must be in the set but are not?
643 (diffSet=expected).removeAll(set);
644 if(!diffSet.isEmpty()) {
645 diffSet.toPattern(out, TRUE);
646 if(out.length()>100) {
647 out.replace(100, 0x7fffffff, ellipsis, UPRV_LENGTHOF(ellipsis));
648 }
649 errln("error: ucnv_getUnicodeSet(\"%s\") is missing items - which set: %d",
650 cnvNames[i], which);
651 errln(out);
652 }
653
654 // are there items that must not be in the set but are?
655 (diffSet=set).removeAll(expected);
656 if(!diffSet.isEmpty()) {
657 diffSet.toPattern(out, TRUE);
658 if(out.length()>100) {
659 out.replace(100, 0x7fffffff, ellipsis, UPRV_LENGTHOF(ellipsis));
660 }
661 errln("error: ucnv_getUnicodeSet(\"%s\") contains unexpected items - which set: %d",
662 cnvNames[i], which);
663 errln(out);
664 }
665 }
666 }
667 }
668
669 delete [] s0;
670 }
671
672 // Test all codepoints which has the default ignorable Unicode property are ignored if they have no mapping
673 // If there are any failures, the hard coded list (IS_DEFAULT_IGNORABLE_CODE_POINT) in ucnv_err.c should be updated
674 void
TestDefaultIgnorableCallback()675 ConversionTest::TestDefaultIgnorableCallback() {
676 UErrorCode status = U_ZERO_ERROR;
677 const char *cnv_name = "euc-jp-2007";
678 const char *pattern_ignorable = "[:Default_Ignorable_Code_Point:]";
679 const char *pattern_not_ignorable = "[:^Default_Ignorable_Code_Point:]";
680
681 UnicodeSet *set_ignorable = new UnicodeSet(pattern_ignorable, status);
682 if (U_FAILURE(status)) {
683 dataerrln("Unable to create Unicodeset: %s - %s\n", pattern_ignorable, u_errorName(status));
684 return;
685 }
686
687 UnicodeSet *set_not_ignorable = new UnicodeSet(pattern_not_ignorable, status);
688 if (U_FAILURE(status)) {
689 dataerrln("Unable to create Unicodeset: %s - %s\n", pattern_not_ignorable, u_errorName(status));
690 return;
691 }
692
693 UConverter *cnv = cnv_open(cnv_name, status);
694 if (U_FAILURE(status)) {
695 dataerrln("Unable to open converter: %s - %s\n", cnv_name, u_errorName(status));
696 return;
697 }
698
699 // set callback for the converter
700 ucnv_setFromUCallBack(cnv, UCNV_FROM_U_CALLBACK_SUBSTITUTE, NULL, NULL, NULL, &status);
701
702 UChar32 input[1];
703 char output[10];
704 int32_t outputLength;
705
706 // test default ignorables are ignored
707 int size = set_ignorable->size();
708 for (int i = 0; i < size; i++) {
709 status = U_ZERO_ERROR;
710 outputLength= 0;
711
712 input[0] = set_ignorable->charAt(i);
713
714 outputLength = ucnv_fromUChars(cnv, output, 10, UnicodeString::fromUTF32(input, 1).getTerminatedBuffer(), -1, &status);
715 if (U_FAILURE(status) || outputLength != 0) {
716 errln("Ignorable code point: U+%04X not skipped as expected - %s", input[0], u_errorName(status));
717 }
718 }
719
720 // test non-ignorables are not ignored
721 size = set_not_ignorable->size();
722 for (int i = 0; i < size; i++) {
723 status = U_ZERO_ERROR;
724 outputLength= 0;
725
726 input[0] = set_not_ignorable->charAt(i);
727
728 if (input[0] == 0) {
729 continue;
730 }
731
732 outputLength = ucnv_fromUChars(cnv, output, 10, UnicodeString::fromUTF32(input, 1).getTerminatedBuffer(), -1, &status);
733 if (U_FAILURE(status) || outputLength <= 0) {
734 errln("Non-ignorable code point: U+%04X skipped unexpectedly - %s", input[0], u_errorName(status));
735 }
736 }
737
738 ucnv_close(cnv);
739 delete set_not_ignorable;
740 delete set_ignorable;
741 }
742
743 void
TestUTF8ToUTF8Overflow()744 ConversionTest::TestUTF8ToUTF8Overflow() {
745 IcuTestErrorCode errorCode(*this, "TestUTF8ToUTF8Overflow");
746 LocalUConverterPointer cnv1(ucnv_open("UTF-8", errorCode));
747 LocalUConverterPointer cnv2(ucnv_open("UTF-8", errorCode));
748 static const char *text = "aä"; // ä: 2 bytes
749 const char *source = text;
750 const char *sourceLimit = text + strlen(text);
751 char result[20];
752 char *target = result;
753 const char *targetLimit = result + sizeof(result);
754 UChar buffer16[20];
755 UChar *pivotSource = buffer16;
756 UChar *pivotTarget = buffer16;
757 const UChar *pivotLimit = buffer16 + UPRV_LENGTHOF(buffer16);
758 int32_t length;
759
760 // Convert with insufficient target capacity.
761 result[2] = 5;
762 ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
763 &target, result + 2, &source, sourceLimit,
764 buffer16, &pivotSource, &pivotTarget, pivotLimit,
765 FALSE, FALSE, errorCode);
766 assertEquals("overflow", U_BUFFER_OVERFLOW_ERROR, errorCode.reset());
767 length = (int32_t)(target - result);
768 assertEquals("number of bytes written", 2, length);
769 assertEquals("next byte not clobbered", 5, result[2]);
770
771 // Convert the rest and flush.
772 ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
773 &target, targetLimit, &source, sourceLimit,
774 buffer16, &pivotSource, &pivotTarget, pivotLimit,
775 FALSE, TRUE, errorCode);
776
777 assertSuccess("UTF-8->UTF-8", errorCode);
778 length = (int32_t)(target - result);
779 assertEquals("3 bytes", 3, length);
780 if (length == 3) {
781 assertTrue("result same as input", memcmp(text, result, length) == 0);
782 }
783
784 ucnv_reset(cnv1.getAlias());
785 ucnv_reset(cnv2.getAlias());
786 memset(result, 0, sizeof(result));
787 static const char *text2 = "a��"; // U+1F6B2 bicycle: 4 bytes
788 source = text2;
789 sourceLimit = text2 + strlen(text2);
790 target = result;
791 pivotSource = pivotTarget = buffer16;
792
793 // Convert with insufficient target capacity.
794 result[3] = 5;
795 ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
796 &target, result + 3, &source, sourceLimit,
797 buffer16, &pivotSource, &pivotTarget, pivotLimit,
798 FALSE, FALSE, errorCode);
799 assertEquals("text2 overflow", U_BUFFER_OVERFLOW_ERROR, errorCode.reset());
800 length = (int32_t)(target - result);
801 assertEquals("text2 number of bytes written", 3, length);
802 assertEquals("text2 next byte not clobbered", 5, result[3]);
803
804 // Convert the rest and flush.
805 ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
806 &target, targetLimit, &source, sourceLimit,
807 buffer16, &pivotSource, &pivotTarget, pivotLimit,
808 FALSE, TRUE, errorCode);
809
810 assertSuccess("text2 UTF-8->UTF-8", errorCode);
811 length = (int32_t)(target - result);
812 assertEquals("text2 5 bytes", 5, length);
813 if (length == 5) {
814 assertTrue("text2 result same as input", memcmp(text2, result, length) == 0);
815 }
816
817 ucnv_reset(cnv1.getAlias());
818 ucnv_reset(cnv2.getAlias());
819 memset(result, 0, sizeof(result));
820 static const char *illFormed = "\xf1\x91\x93\x96\x91\x94"; // U+514D6 + two more trail bytes
821 source = illFormed;
822 sourceLimit = illFormed + strlen(illFormed);
823 target = result;
824 pivotSource = pivotTarget = buffer16;
825
826 ucnv_setToUCallBack(cnv1.getAlias(), UCNV_TO_U_CALLBACK_STOP, nullptr, nullptr, nullptr, errorCode);
827
828 // Convert only two bytes and flush (but expect failure).
829 char errorBytes[10];
830 int8_t errorLength;
831 result[0] = 5;
832 ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
833 &target, targetLimit, &source, source + 2,
834 buffer16, &pivotSource, &pivotTarget, pivotLimit,
835 FALSE, TRUE, errorCode);
836 assertEquals("illFormed truncated", U_TRUNCATED_CHAR_FOUND, errorCode.reset());
837 length = (int32_t)(target - result);
838 assertEquals("illFormed number of bytes written", 0, length);
839 errorLength = UPRV_LENGTHOF(errorBytes);
840 ucnv_getInvalidChars(cnv1.getAlias(), errorBytes, &errorLength, errorCode);
841 assertEquals("illFormed truncated errorLength", 2, (int32_t)errorLength);
842 if (errorLength == 2) {
843 assertEquals("illFormed truncated errorBytes", 0xf191,
844 ((int32_t)(uint8_t)errorBytes[0] << 8) | (uint8_t)errorBytes[1]);
845 }
846
847 // Continue conversion starting with a trail byte.
848 ucnv_convertEx(cnv2.getAlias(), cnv1.getAlias(),
849 &target, targetLimit, &source, sourceLimit,
850 buffer16, &pivotSource, &pivotTarget, pivotLimit,
851 FALSE, TRUE, errorCode);
852
853 assertEquals("illFormed trail byte", U_ILLEGAL_CHAR_FOUND, errorCode.reset());
854 length = (int32_t)(target - result);
855 assertEquals("illFormed trail byte number of bytes written", 0, length);
856 errorLength = UPRV_LENGTHOF(errorBytes);
857 ucnv_getInvalidChars(cnv1.getAlias(), errorBytes, &errorLength, errorCode);
858 assertEquals("illFormed trail byte errorLength", 1, (int32_t)errorLength);
859 if (errorLength == 1) {
860 assertEquals("illFormed trail byte errorBytes", 0x93, (int32_t)(uint8_t)errorBytes[0]);
861 }
862 }
863
864 // open testdata or ICU data converter ------------------------------------- ***
865
866 UConverter *
cnv_open(const char * name,UErrorCode & errorCode)867 ConversionTest::cnv_open(const char *name, UErrorCode &errorCode) {
868 if(name!=NULL && *name=='+') {
869 // Converter names that start with '+' are ignored in ICU4J tests.
870 ++name;
871 }
872 if(name!=NULL && *name=='*') {
873 /* loadTestData(): set the data directory */
874 return ucnv_openPackage(loadTestData(errorCode), name+1, &errorCode);
875 } else {
876 return ucnv_open(name, &errorCode);
877 }
878 }
879
880 // output helpers ---------------------------------------------------------- ***
881
882 static inline char
hexDigit(uint8_t digit)883 hexDigit(uint8_t digit) {
884 return digit<=9 ? (char)('0'+digit) : (char)('a'-10+digit);
885 }
886
887 static char *
printBytes(const uint8_t * bytes,int32_t length,char * out)888 printBytes(const uint8_t *bytes, int32_t length, char *out) {
889 uint8_t b;
890
891 if(length>0) {
892 b=*bytes++;
893 --length;
894 *out++=hexDigit((uint8_t)(b>>4));
895 *out++=hexDigit((uint8_t)(b&0xf));
896 }
897
898 while(length>0) {
899 b=*bytes++;
900 --length;
901 *out++=' ';
902 *out++=hexDigit((uint8_t)(b>>4));
903 *out++=hexDigit((uint8_t)(b&0xf));
904 }
905 *out++=0;
906 return out;
907 }
908
909 static char *
printUnicode(const UChar * unicode,int32_t length,char * out)910 printUnicode(const UChar *unicode, int32_t length, char *out) {
911 UChar32 c;
912 int32_t i;
913
914 for(i=0; i<length;) {
915 if(i>0) {
916 *out++=' ';
917 }
918 U16_NEXT(unicode, i, length, c);
919 // write 4..6 digits
920 if(c>=0x100000) {
921 *out++='1';
922 }
923 if(c>=0x10000) {
924 *out++=hexDigit((uint8_t)((c>>16)&0xf));
925 }
926 *out++=hexDigit((uint8_t)((c>>12)&0xf));
927 *out++=hexDigit((uint8_t)((c>>8)&0xf));
928 *out++=hexDigit((uint8_t)((c>>4)&0xf));
929 *out++=hexDigit((uint8_t)(c&0xf));
930 }
931 *out++=0;
932 return out;
933 }
934
935 static char *
printOffsets(const int32_t * offsets,int32_t length,char * out)936 printOffsets(const int32_t *offsets, int32_t length, char *out) {
937 int32_t i, o, d;
938
939 if(offsets==NULL) {
940 length=0;
941 }
942
943 for(i=0; i<length; ++i) {
944 if(i>0) {
945 *out++=' ';
946 }
947 o=offsets[i];
948
949 // print all offsets with 2 characters each (-x, -9..99, xx)
950 if(o<-9) {
951 *out++='-';
952 *out++='x';
953 } else if(o<0) {
954 *out++='-';
955 *out++=(char)('0'-o);
956 } else if(o<=99) {
957 *out++=(d=o/10)==0 ? ' ' : (char)('0'+d);
958 *out++=(char)('0'+o%10);
959 } else /* o>99 */ {
960 *out++='x';
961 *out++='x';
962 }
963 }
964 *out++=0;
965 return out;
966 }
967
968 // toUnicode test worker functions ----------------------------------------- ***
969
970 static int32_t
stepToUnicode(ConversionCase & cc,UConverter * cnv,UChar * result,int32_t resultCapacity,int32_t * resultOffsets,int32_t step,UErrorCode * pErrorCode)971 stepToUnicode(ConversionCase &cc, UConverter *cnv,
972 UChar *result, int32_t resultCapacity,
973 int32_t *resultOffsets, /* also resultCapacity */
974 int32_t step,
975 UErrorCode *pErrorCode) {
976 const char *source, *sourceLimit, *bytesLimit;
977 UChar *target, *targetLimit, *resultLimit;
978 UBool flush;
979
980 source=(const char *)cc.bytes;
981 target=result;
982 bytesLimit=source+cc.bytesLength;
983 resultLimit=result+resultCapacity;
984
985 if(step>=0) {
986 // call ucnv_toUnicode() with in/out buffers no larger than (step) at a time
987 // move only one buffer (in vs. out) at a time to be extra mean
988 // step==0 performs bulk conversion and generates offsets
989
990 // initialize the partial limits for the loop
991 if(step==0) {
992 // use the entire buffers
993 sourceLimit=bytesLimit;
994 targetLimit=resultLimit;
995 flush=cc.finalFlush;
996 } else {
997 // start with empty partial buffers
998 sourceLimit=source;
999 targetLimit=target;
1000 flush=FALSE;
1001
1002 // output offsets only for bulk conversion
1003 resultOffsets=NULL;
1004 }
1005
1006 for(;;) {
1007 // resetting the opposite conversion direction must not affect this one
1008 ucnv_resetFromUnicode(cnv);
1009
1010 // convert
1011 ucnv_toUnicode(cnv,
1012 &target, targetLimit,
1013 &source, sourceLimit,
1014 resultOffsets,
1015 flush, pErrorCode);
1016
1017 // check pointers and errors
1018 if(source>sourceLimit || target>targetLimit) {
1019 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1020 break;
1021 } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
1022 if(target!=targetLimit) {
1023 // buffer overflow must only be set when the target is filled
1024 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1025 break;
1026 } else if(targetLimit==resultLimit) {
1027 // not just a partial overflow
1028 break;
1029 }
1030
1031 // the partial target is filled, set a new limit, reset the error and continue
1032 targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
1033 *pErrorCode=U_ZERO_ERROR;
1034 } else if(U_FAILURE(*pErrorCode)) {
1035 // some other error occurred, done
1036 break;
1037 } else {
1038 if(source!=sourceLimit) {
1039 // when no error occurs, then the input must be consumed
1040 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1041 break;
1042 }
1043
1044 if(sourceLimit==bytesLimit) {
1045 // we are done
1046 break;
1047 }
1048
1049 // the partial conversion succeeded, set a new limit and continue
1050 sourceLimit=(bytesLimit-source)>=step ? source+step : bytesLimit;
1051 flush=(UBool)(cc.finalFlush && sourceLimit==bytesLimit);
1052 }
1053 }
1054 } else /* step<0 */ {
1055 /*
1056 * step==-1: call only ucnv_getNextUChar()
1057 * otherwise alternate between ucnv_toUnicode() and ucnv_getNextUChar()
1058 * if step==-2 or -3, then give ucnv_toUnicode() the whole remaining input,
1059 * else give it at most (-step-2)/2 bytes
1060 */
1061 UChar32 c;
1062
1063 // end the loop by getting an index out of bounds error
1064 for(;;) {
1065 // resetting the opposite conversion direction must not affect this one
1066 ucnv_resetFromUnicode(cnv);
1067
1068 // convert
1069 if((step&1)!=0 /* odd: -1, -3, -5, ... */) {
1070 sourceLimit=source; // use sourceLimit not as a real limit
1071 // but to remember the pre-getNextUChar source pointer
1072 c=ucnv_getNextUChar(cnv, &source, bytesLimit, pErrorCode);
1073
1074 // check pointers and errors
1075 if(*pErrorCode==U_INDEX_OUTOFBOUNDS_ERROR) {
1076 if(source!=bytesLimit) {
1077 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1078 } else {
1079 *pErrorCode=U_ZERO_ERROR;
1080 }
1081 break;
1082 } else if(U_FAILURE(*pErrorCode)) {
1083 break;
1084 }
1085 // source may not move if c is from previous overflow
1086
1087 if(target==resultLimit) {
1088 *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
1089 break;
1090 }
1091 if(c<=0xffff) {
1092 *target++=(UChar)c;
1093 } else {
1094 *target++=U16_LEAD(c);
1095 if(target==resultLimit) {
1096 *pErrorCode=U_BUFFER_OVERFLOW_ERROR;
1097 break;
1098 }
1099 *target++=U16_TRAIL(c);
1100 }
1101
1102 // alternate between -n-1 and -n but leave -1 alone
1103 if(step<-1) {
1104 ++step;
1105 }
1106 } else /* step is even */ {
1107 // allow only one UChar output
1108 targetLimit=target<resultLimit ? target+1 : resultLimit;
1109
1110 // as with ucnv_getNextUChar(), we always flush (if we go to bytesLimit)
1111 // and never output offsets
1112 if(step==-2) {
1113 sourceLimit=bytesLimit;
1114 } else {
1115 sourceLimit=source+(-step-2)/2;
1116 if(sourceLimit>bytesLimit) {
1117 sourceLimit=bytesLimit;
1118 }
1119 }
1120
1121 ucnv_toUnicode(cnv,
1122 &target, targetLimit,
1123 &source, sourceLimit,
1124 NULL, (UBool)(sourceLimit==bytesLimit), pErrorCode);
1125
1126 // check pointers and errors
1127 if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
1128 if(target!=targetLimit) {
1129 // buffer overflow must only be set when the target is filled
1130 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1131 break;
1132 } else if(targetLimit==resultLimit) {
1133 // not just a partial overflow
1134 break;
1135 }
1136
1137 // the partial target is filled, set a new limit and continue
1138 *pErrorCode=U_ZERO_ERROR;
1139 } else if(U_FAILURE(*pErrorCode)) {
1140 // some other error occurred, done
1141 break;
1142 } else {
1143 if(source!=sourceLimit) {
1144 // when no error occurs, then the input must be consumed
1145 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1146 break;
1147 }
1148
1149 // we are done (flush==TRUE) but we continue, to get the index out of bounds error above
1150 }
1151
1152 --step;
1153 }
1154 }
1155 }
1156
1157 return (int32_t)(target-result);
1158 }
1159
1160 UBool
ToUnicodeCase(ConversionCase & cc,UConverterToUCallback callback,const char * option)1161 ConversionTest::ToUnicodeCase(ConversionCase &cc, UConverterToUCallback callback, const char *option) {
1162 // open the converter
1163 IcuTestErrorCode errorCode(*this, "ToUnicodeCase");
1164 LocalUConverterPointer cnv(cnv_open(cc.charset, errorCode));
1165 // with no data, the above crashes with "pointer being freed was not allocated" for charset "x11-compound-text", see #13078
1166 if(errorCode.isFailure()) {
1167 errcheckln(errorCode, "toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_open() failed - %s",
1168 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, errorCode.errorName());
1169 errorCode.reset();
1170 return FALSE;
1171 }
1172
1173 // set the callback
1174 if(callback!=NULL) {
1175 ucnv_setToUCallBack(cnv.getAlias(), callback, option, NULL, NULL, errorCode);
1176 if(U_FAILURE(errorCode)) {
1177 errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setToUCallBack() failed - %s",
1178 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1179 return FALSE;
1180 }
1181 }
1182
1183 int32_t resultOffsets[256];
1184 UChar result[256];
1185 int32_t resultLength;
1186 UBool ok;
1187
1188 static const struct {
1189 int32_t step;
1190 const char *name;
1191 } steps[]={
1192 { 0, "bulk" }, // must be first for offsets to be checked
1193 { 1, "step=1" },
1194 { 3, "step=3" },
1195 { 7, "step=7" },
1196 { -1, "getNext" },
1197 { -2, "toU(bulk)+getNext" },
1198 { -3, "getNext+toU(bulk)" },
1199 { -4, "toU(1)+getNext" },
1200 { -5, "getNext+toU(1)" },
1201 { -12, "toU(5)+getNext" },
1202 { -13, "getNext+toU(5)" },
1203 };
1204 int32_t i, step;
1205
1206 ok=TRUE;
1207 for(i=0; i<UPRV_LENGTHOF(steps) && ok; ++i) {
1208 step=steps[i].step;
1209 if(step<0 && !cc.finalFlush) {
1210 // skip ucnv_getNextUChar() if !finalFlush because
1211 // ucnv_getNextUChar() always implies flush
1212 continue;
1213 }
1214 if(step!=0) {
1215 // bulk test is first, then offsets are not checked any more
1216 cc.offsets=NULL;
1217 }
1218 else {
1219 memset(resultOffsets, -1, UPRV_LENGTHOF(resultOffsets));
1220 }
1221 memset(result, -1, UPRV_LENGTHOF(result));
1222 errorCode.reset();
1223 resultLength=stepToUnicode(cc, cnv.getAlias(),
1224 result, UPRV_LENGTHOF(result),
1225 step==0 ? resultOffsets : NULL,
1226 step, errorCode);
1227 ok=checkToUnicode(
1228 cc, cnv.getAlias(), steps[i].name,
1229 result, resultLength,
1230 cc.offsets!=NULL ? resultOffsets : NULL,
1231 errorCode);
1232 if(errorCode.isFailure() || !cc.finalFlush) {
1233 // reset if an error occurred or we did not flush
1234 // otherwise do nothing to make sure that flushing resets
1235 ucnv_resetToUnicode(cnv.getAlias());
1236 }
1237 if (cc.offsets != NULL && resultOffsets[resultLength] != -1) {
1238 errln("toUnicode[%d](%s) Conversion wrote too much to offsets at index %d",
1239 cc.caseNr, cc.charset, resultLength);
1240 }
1241 if (result[resultLength] != (UChar)-1) {
1242 errln("toUnicode[%d](%s) Conversion wrote too much to result at index %d",
1243 cc.caseNr, cc.charset, resultLength);
1244 }
1245 }
1246
1247 // not a real loop, just a convenience for breaking out of the block
1248 while(ok && cc.finalFlush) {
1249 // test ucnv_toUChars()
1250 memset(result, 0, sizeof(result));
1251
1252 errorCode.reset();
1253 resultLength=ucnv_toUChars(cnv.getAlias(),
1254 result, UPRV_LENGTHOF(result),
1255 (const char *)cc.bytes, cc.bytesLength,
1256 errorCode);
1257 ok=checkToUnicode(
1258 cc, cnv.getAlias(), "toUChars",
1259 result, resultLength,
1260 NULL,
1261 errorCode);
1262 if(!ok) {
1263 break;
1264 }
1265
1266 // test preflighting
1267 // keep the correct result for simple checking
1268 errorCode.reset();
1269 resultLength=ucnv_toUChars(cnv.getAlias(),
1270 NULL, 0,
1271 (const char *)cc.bytes, cc.bytesLength,
1272 errorCode);
1273 if(errorCode.get()==U_STRING_NOT_TERMINATED_WARNING || errorCode.get()==U_BUFFER_OVERFLOW_ERROR) {
1274 errorCode.reset();
1275 }
1276 ok=checkToUnicode(
1277 cc, cnv.getAlias(), "preflight toUChars",
1278 result, resultLength,
1279 NULL,
1280 errorCode);
1281 break;
1282 }
1283
1284 errorCode.reset(); // all errors have already been reported
1285 return ok;
1286 }
1287
1288 UBool
checkToUnicode(ConversionCase & cc,UConverter * cnv,const char * name,const UChar * result,int32_t resultLength,const int32_t * resultOffsets,UErrorCode resultErrorCode)1289 ConversionTest::checkToUnicode(ConversionCase &cc, UConverter *cnv, const char *name,
1290 const UChar *result, int32_t resultLength,
1291 const int32_t *resultOffsets,
1292 UErrorCode resultErrorCode) {
1293 char resultInvalidChars[8];
1294 int8_t resultInvalidLength;
1295 UErrorCode errorCode;
1296
1297 const char *msg;
1298
1299 // reset the message; NULL will mean "ok"
1300 msg=NULL;
1301
1302 errorCode=U_ZERO_ERROR;
1303 resultInvalidLength=sizeof(resultInvalidChars);
1304 ucnv_getInvalidChars(cnv, resultInvalidChars, &resultInvalidLength, &errorCode);
1305 if(U_FAILURE(errorCode)) {
1306 errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) ucnv_getInvalidChars() failed - %s",
1307 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, u_errorName(errorCode));
1308 return FALSE;
1309 }
1310
1311 // check everything that might have gone wrong
1312 if(cc.unicodeLength!=resultLength) {
1313 msg="wrong result length";
1314 } else if(0!=u_memcmp(cc.unicode, result, cc.unicodeLength)) {
1315 msg="wrong result string";
1316 } else if(cc.offsets!=NULL && 0!=memcmp(cc.offsets, resultOffsets, cc.unicodeLength*sizeof(*cc.offsets))) {
1317 msg="wrong offsets";
1318 } else if(cc.outErrorCode!=resultErrorCode) {
1319 msg="wrong error code";
1320 } else if(cc.invalidLength!=resultInvalidLength) {
1321 msg="wrong length of last invalid input";
1322 } else if(0!=memcmp(cc.invalidChars, resultInvalidChars, cc.invalidLength)) {
1323 msg="wrong last invalid input";
1324 }
1325
1326 if(msg==NULL) {
1327 return TRUE;
1328 } else {
1329 char buffer[2000]; // one buffer for all strings
1330 char *s, *bytesString, *unicodeString, *resultString,
1331 *offsetsString, *resultOffsetsString,
1332 *invalidCharsString, *resultInvalidCharsString;
1333
1334 bytesString=s=buffer;
1335 s=printBytes(cc.bytes, cc.bytesLength, bytesString);
1336 s=printUnicode(cc.unicode, cc.unicodeLength, unicodeString=s);
1337 s=printUnicode(result, resultLength, resultString=s);
1338 s=printOffsets(cc.offsets, cc.unicodeLength, offsetsString=s);
1339 s=printOffsets(resultOffsets, resultLength, resultOffsetsString=s);
1340 s=printBytes(cc.invalidChars, cc.invalidLength, invalidCharsString=s);
1341 s=printBytes((uint8_t *)resultInvalidChars, resultInvalidLength, resultInvalidCharsString=s);
1342
1343 if((s-buffer)>(int32_t)sizeof(buffer)) {
1344 errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) fatal error: checkToUnicode() test output buffer overflow writing %d chars\n",
1345 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, (int)(s-buffer));
1346 exit(1);
1347 }
1348
1349 errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) failed: %s\n"
1350 " bytes <%s>[%d]\n"
1351 " expected <%s>[%d]\n"
1352 " result <%s>[%d]\n"
1353 " offsets <%s>\n"
1354 " result offsets <%s>\n"
1355 " error code expected %s got %s\n"
1356 " invalidChars expected <%s> got <%s>\n",
1357 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, msg,
1358 bytesString, cc.bytesLength,
1359 unicodeString, cc.unicodeLength,
1360 resultString, resultLength,
1361 offsetsString,
1362 resultOffsetsString,
1363 u_errorName(cc.outErrorCode), u_errorName(resultErrorCode),
1364 invalidCharsString, resultInvalidCharsString);
1365
1366 return FALSE;
1367 }
1368 }
1369
1370 // fromUnicode test worker functions --------------------------------------- ***
1371
1372 static int32_t
stepFromUTF8(ConversionCase & cc,UConverter * utf8Cnv,UConverter * cnv,char * result,int32_t resultCapacity,int32_t step,UErrorCode * pErrorCode)1373 stepFromUTF8(ConversionCase &cc,
1374 UConverter *utf8Cnv, UConverter *cnv,
1375 char *result, int32_t resultCapacity,
1376 int32_t step,
1377 UErrorCode *pErrorCode) {
1378 const char *source, *sourceLimit, *utf8Limit;
1379 UChar pivotBuffer[32];
1380 UChar *pivotSource, *pivotTarget, *pivotLimit;
1381 char *target, *targetLimit, *resultLimit;
1382 UBool flush;
1383
1384 source=cc.utf8;
1385 pivotSource=pivotTarget=pivotBuffer;
1386 target=result;
1387 utf8Limit=source+cc.utf8Length;
1388 resultLimit=result+resultCapacity;
1389
1390 // call ucnv_convertEx() with in/out buffers no larger than (step) at a time
1391 // move only one buffer (in vs. out) at a time to be extra mean
1392 // step==0 performs bulk conversion
1393
1394 // initialize the partial limits for the loop
1395 if(step==0) {
1396 // use the entire buffers
1397 sourceLimit=utf8Limit;
1398 targetLimit=resultLimit;
1399 flush=cc.finalFlush;
1400
1401 pivotLimit=pivotBuffer+UPRV_LENGTHOF(pivotBuffer);
1402 } else {
1403 // start with empty partial buffers
1404 sourceLimit=source;
1405 targetLimit=target;
1406 flush=FALSE;
1407
1408 // empty pivot is not allowed, make it of length step
1409 pivotLimit=pivotBuffer+step;
1410 }
1411
1412 for(;;) {
1413 // resetting the opposite conversion direction must not affect this one
1414 ucnv_resetFromUnicode(utf8Cnv);
1415 ucnv_resetToUnicode(cnv);
1416
1417 // convert
1418 ucnv_convertEx(cnv, utf8Cnv,
1419 &target, targetLimit,
1420 &source, sourceLimit,
1421 pivotBuffer, &pivotSource, &pivotTarget, pivotLimit,
1422 FALSE, flush, pErrorCode);
1423
1424 // check pointers and errors
1425 if(source>sourceLimit || target>targetLimit) {
1426 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1427 break;
1428 } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
1429 if(target!=targetLimit) {
1430 // buffer overflow must only be set when the target is filled
1431 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1432 break;
1433 } else if(targetLimit==resultLimit) {
1434 // not just a partial overflow
1435 break;
1436 }
1437
1438 // the partial target is filled, set a new limit, reset the error and continue
1439 targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
1440 *pErrorCode=U_ZERO_ERROR;
1441 } else if(U_FAILURE(*pErrorCode)) {
1442 if(pivotSource==pivotBuffer) {
1443 // toUnicode error, should not occur
1444 // toUnicode errors are tested in cintltst TestConvertExFromUTF8()
1445 break;
1446 } else {
1447 // fromUnicode error
1448 // some other error occurred, done
1449 break;
1450 }
1451 } else {
1452 if(source!=sourceLimit) {
1453 // when no error occurs, then the input must be consumed
1454 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1455 break;
1456 }
1457
1458 if(sourceLimit==utf8Limit) {
1459 // we are done
1460 if(*pErrorCode==U_STRING_NOT_TERMINATED_WARNING) {
1461 // ucnv_convertEx() warns about not terminating the output
1462 // but ucnv_fromUnicode() does not and so
1463 // checkFromUnicode() does not expect it
1464 *pErrorCode=U_ZERO_ERROR;
1465 }
1466 break;
1467 }
1468
1469 // the partial conversion succeeded, set a new limit and continue
1470 sourceLimit=(utf8Limit-source)>=step ? source+step : utf8Limit;
1471 flush=(UBool)(cc.finalFlush && sourceLimit==utf8Limit);
1472 }
1473 }
1474
1475 return (int32_t)(target-result);
1476 }
1477
1478 static int32_t
stepFromUnicode(ConversionCase & cc,UConverter * cnv,char * result,int32_t resultCapacity,int32_t * resultOffsets,int32_t step,UErrorCode * pErrorCode)1479 stepFromUnicode(ConversionCase &cc, UConverter *cnv,
1480 char *result, int32_t resultCapacity,
1481 int32_t *resultOffsets, /* also resultCapacity */
1482 int32_t step,
1483 UErrorCode *pErrorCode) {
1484 const UChar *source, *sourceLimit, *unicodeLimit;
1485 char *target, *targetLimit, *resultLimit;
1486 UBool flush;
1487
1488 source=cc.unicode;
1489 target=result;
1490 unicodeLimit=source+cc.unicodeLength;
1491 resultLimit=result+resultCapacity;
1492
1493 // call ucnv_fromUnicode() with in/out buffers no larger than (step) at a time
1494 // move only one buffer (in vs. out) at a time to be extra mean
1495 // step==0 performs bulk conversion and generates offsets
1496
1497 // initialize the partial limits for the loop
1498 if(step==0) {
1499 // use the entire buffers
1500 sourceLimit=unicodeLimit;
1501 targetLimit=resultLimit;
1502 flush=cc.finalFlush;
1503 } else {
1504 // start with empty partial buffers
1505 sourceLimit=source;
1506 targetLimit=target;
1507 flush=FALSE;
1508
1509 // output offsets only for bulk conversion
1510 resultOffsets=NULL;
1511 }
1512
1513 for(;;) {
1514 // resetting the opposite conversion direction must not affect this one
1515 ucnv_resetToUnicode(cnv);
1516
1517 // convert
1518 ucnv_fromUnicode(cnv,
1519 &target, targetLimit,
1520 &source, sourceLimit,
1521 resultOffsets,
1522 flush, pErrorCode);
1523
1524 // check pointers and errors
1525 if(source>sourceLimit || target>targetLimit) {
1526 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1527 break;
1528 } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) {
1529 if(target!=targetLimit) {
1530 // buffer overflow must only be set when the target is filled
1531 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1532 break;
1533 } else if(targetLimit==resultLimit) {
1534 // not just a partial overflow
1535 break;
1536 }
1537
1538 // the partial target is filled, set a new limit, reset the error and continue
1539 targetLimit=(resultLimit-target)>=step ? target+step : resultLimit;
1540 *pErrorCode=U_ZERO_ERROR;
1541 } else if(U_FAILURE(*pErrorCode)) {
1542 // some other error occurred, done
1543 break;
1544 } else {
1545 if(source!=sourceLimit) {
1546 // when no error occurs, then the input must be consumed
1547 *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
1548 break;
1549 }
1550
1551 if(sourceLimit==unicodeLimit) {
1552 // we are done
1553 break;
1554 }
1555
1556 // the partial conversion succeeded, set a new limit and continue
1557 sourceLimit=(unicodeLimit-source)>=step ? source+step : unicodeLimit;
1558 flush=(UBool)(cc.finalFlush && sourceLimit==unicodeLimit);
1559 }
1560 }
1561
1562 return (int32_t)(target-result);
1563 }
1564
1565 UBool
FromUnicodeCase(ConversionCase & cc,UConverterFromUCallback callback,const char * option)1566 ConversionTest::FromUnicodeCase(ConversionCase &cc, UConverterFromUCallback callback, const char *option) {
1567 UConverter *cnv;
1568 UErrorCode errorCode;
1569
1570 // open the converter
1571 errorCode=U_ZERO_ERROR;
1572 cnv=cnv_open(cc.charset, errorCode);
1573 if(U_FAILURE(errorCode)) {
1574 errcheckln(errorCode, "fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_open() failed - %s",
1575 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1576 return FALSE;
1577 }
1578 ucnv_resetToUnicode(utf8Cnv);
1579
1580 // set the callback
1581 if(callback!=NULL) {
1582 ucnv_setFromUCallBack(cnv, callback, option, NULL, NULL, &errorCode);
1583 if(U_FAILURE(errorCode)) {
1584 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setFromUCallBack() failed - %s",
1585 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1586 ucnv_close(cnv);
1587 return FALSE;
1588 }
1589 }
1590
1591 // set the fallbacks flag
1592 // TODO change with Jitterbug 2401, then add a similar call for toUnicode too
1593 ucnv_setFallback(cnv, cc.fallbacks);
1594
1595 // set the subchar
1596 int32_t length;
1597
1598 if(cc.setSub>0) {
1599 length=(int32_t)strlen(cc.subchar);
1600 ucnv_setSubstChars(cnv, cc.subchar, (int8_t)length, &errorCode);
1601 if(U_FAILURE(errorCode)) {
1602 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setSubstChars() failed - %s",
1603 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1604 ucnv_close(cnv);
1605 return FALSE;
1606 }
1607 } else if(cc.setSub<0) {
1608 ucnv_setSubstString(cnv, cc.subString, -1, &errorCode);
1609 if(U_FAILURE(errorCode)) {
1610 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setSubstString() failed - %s",
1611 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
1612 ucnv_close(cnv);
1613 return FALSE;
1614 }
1615 }
1616
1617 // convert unicode to utf8
1618 char utf8[256];
1619 cc.utf8=utf8;
1620 u_strToUTF8(utf8, UPRV_LENGTHOF(utf8), &cc.utf8Length,
1621 cc.unicode, cc.unicodeLength,
1622 &errorCode);
1623 if(U_FAILURE(errorCode)) {
1624 // skip UTF-8 testing of a string with an unpaired surrogate,
1625 // or of one that's too long
1626 // toUnicode errors are tested in cintltst TestConvertExFromUTF8()
1627 cc.utf8Length=-1;
1628 }
1629
1630 int32_t resultOffsets[256];
1631 char result[256];
1632 int32_t resultLength;
1633 UBool ok;
1634
1635 static const struct {
1636 int32_t step;
1637 const char *name, *utf8Name;
1638 } steps[]={
1639 { 0, "bulk", "utf8" }, // must be first for offsets to be checked
1640 { 1, "step=1", "utf8 step=1" },
1641 { 3, "step=3", "utf8 step=3" },
1642 { 7, "step=7", "utf8 step=7" }
1643 };
1644 int32_t i, step;
1645
1646 ok=TRUE;
1647 for(i=0; i<UPRV_LENGTHOF(steps) && ok; ++i) {
1648 step=steps[i].step;
1649 memset(resultOffsets, -1, UPRV_LENGTHOF(resultOffsets));
1650 memset(result, -1, UPRV_LENGTHOF(result));
1651 errorCode=U_ZERO_ERROR;
1652 resultLength=stepFromUnicode(cc, cnv,
1653 result, UPRV_LENGTHOF(result),
1654 step==0 ? resultOffsets : NULL,
1655 step, &errorCode);
1656 ok=checkFromUnicode(
1657 cc, cnv, steps[i].name,
1658 (uint8_t *)result, resultLength,
1659 cc.offsets!=NULL ? resultOffsets : NULL,
1660 errorCode);
1661 if(U_FAILURE(errorCode) || !cc.finalFlush) {
1662 // reset if an error occurred or we did not flush
1663 // otherwise do nothing to make sure that flushing resets
1664 ucnv_resetFromUnicode(cnv);
1665 }
1666 if (resultOffsets[resultLength] != -1) {
1667 errln("fromUnicode[%d](%s) Conversion wrote too much to offsets at index %d",
1668 cc.caseNr, cc.charset, resultLength);
1669 }
1670 if (result[resultLength] != (char)-1) {
1671 errln("fromUnicode[%d](%s) Conversion wrote too much to result at index %d",
1672 cc.caseNr, cc.charset, resultLength);
1673 }
1674
1675 // bulk test is first, then offsets are not checked any more
1676 cc.offsets=NULL;
1677
1678 // test direct conversion from UTF-8
1679 if(cc.utf8Length>=0) {
1680 errorCode=U_ZERO_ERROR;
1681 resultLength=stepFromUTF8(cc, utf8Cnv, cnv,
1682 result, UPRV_LENGTHOF(result),
1683 step, &errorCode);
1684 ok=checkFromUnicode(
1685 cc, cnv, steps[i].utf8Name,
1686 (uint8_t *)result, resultLength,
1687 NULL,
1688 errorCode);
1689 if(U_FAILURE(errorCode) || !cc.finalFlush) {
1690 // reset if an error occurred or we did not flush
1691 // otherwise do nothing to make sure that flushing resets
1692 ucnv_resetToUnicode(utf8Cnv);
1693 ucnv_resetFromUnicode(cnv);
1694 }
1695 }
1696 }
1697
1698 // not a real loop, just a convenience for breaking out of the block
1699 while(ok && cc.finalFlush) {
1700 // test ucnv_fromUChars()
1701 memset(result, 0, sizeof(result));
1702
1703 errorCode=U_ZERO_ERROR;
1704 resultLength=ucnv_fromUChars(cnv,
1705 result, UPRV_LENGTHOF(result),
1706 cc.unicode, cc.unicodeLength,
1707 &errorCode);
1708 ok=checkFromUnicode(
1709 cc, cnv, "fromUChars",
1710 (uint8_t *)result, resultLength,
1711 NULL,
1712 errorCode);
1713 if(!ok) {
1714 break;
1715 }
1716
1717 // test preflighting
1718 // keep the correct result for simple checking
1719 errorCode=U_ZERO_ERROR;
1720 resultLength=ucnv_fromUChars(cnv,
1721 NULL, 0,
1722 cc.unicode, cc.unicodeLength,
1723 &errorCode);
1724 if(errorCode==U_STRING_NOT_TERMINATED_WARNING || errorCode==U_BUFFER_OVERFLOW_ERROR) {
1725 errorCode=U_ZERO_ERROR;
1726 }
1727 ok=checkFromUnicode(
1728 cc, cnv, "preflight fromUChars",
1729 (uint8_t *)result, resultLength,
1730 NULL,
1731 errorCode);
1732 break;
1733 }
1734
1735 ucnv_close(cnv);
1736 return ok;
1737 }
1738
1739 UBool
checkFromUnicode(ConversionCase & cc,UConverter * cnv,const char * name,const uint8_t * result,int32_t resultLength,const int32_t * resultOffsets,UErrorCode resultErrorCode)1740 ConversionTest::checkFromUnicode(ConversionCase &cc, UConverter *cnv, const char *name,
1741 const uint8_t *result, int32_t resultLength,
1742 const int32_t *resultOffsets,
1743 UErrorCode resultErrorCode) {
1744 UChar resultInvalidUChars[8];
1745 int8_t resultInvalidLength;
1746 UErrorCode errorCode;
1747
1748 const char *msg;
1749
1750 // reset the message; NULL will mean "ok"
1751 msg=NULL;
1752
1753 errorCode=U_ZERO_ERROR;
1754 resultInvalidLength=UPRV_LENGTHOF(resultInvalidUChars);
1755 ucnv_getInvalidUChars(cnv, resultInvalidUChars, &resultInvalidLength, &errorCode);
1756 if(U_FAILURE(errorCode)) {
1757 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) ucnv_getInvalidUChars() failed - %s",
1758 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, u_errorName(errorCode));
1759 return FALSE;
1760 }
1761
1762 // check everything that might have gone wrong
1763 if(cc.bytesLength!=resultLength) {
1764 msg="wrong result length";
1765 } else if(0!=memcmp(cc.bytes, result, cc.bytesLength)) {
1766 msg="wrong result string";
1767 } else if(cc.offsets!=NULL && 0!=memcmp(cc.offsets, resultOffsets, cc.bytesLength*sizeof(*cc.offsets))) {
1768 msg="wrong offsets";
1769 } else if(cc.outErrorCode!=resultErrorCode) {
1770 msg="wrong error code";
1771 } else if(cc.invalidLength!=resultInvalidLength) {
1772 msg="wrong length of last invalid input";
1773 } else if(0!=u_memcmp(cc.invalidUChars, resultInvalidUChars, cc.invalidLength)) {
1774 msg="wrong last invalid input";
1775 }
1776
1777 if(msg==NULL) {
1778 return TRUE;
1779 } else {
1780 char buffer[2000]; // one buffer for all strings
1781 char *s, *unicodeString, *bytesString, *resultString,
1782 *offsetsString, *resultOffsetsString,
1783 *invalidCharsString, *resultInvalidUCharsString;
1784
1785 unicodeString=s=buffer;
1786 s=printUnicode(cc.unicode, cc.unicodeLength, unicodeString);
1787 s=printBytes(cc.bytes, cc.bytesLength, bytesString=s);
1788 s=printBytes(result, resultLength, resultString=s);
1789 s=printOffsets(cc.offsets, cc.bytesLength, offsetsString=s);
1790 s=printOffsets(resultOffsets, resultLength, resultOffsetsString=s);
1791 s=printUnicode(cc.invalidUChars, cc.invalidLength, invalidCharsString=s);
1792 s=printUnicode(resultInvalidUChars, resultInvalidLength, resultInvalidUCharsString=s);
1793
1794 if((s-buffer)>(int32_t)sizeof(buffer)) {
1795 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) fatal error: checkFromUnicode() test output buffer overflow writing %d chars\n",
1796 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, (int)(s-buffer));
1797 exit(1);
1798 }
1799
1800 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) failed: %s\n"
1801 " unicode <%s>[%d]\n"
1802 " expected <%s>[%d]\n"
1803 " result <%s>[%d]\n"
1804 " offsets <%s>\n"
1805 " result offsets <%s>\n"
1806 " error code expected %s got %s\n"
1807 " invalidChars expected <%s> got <%s>\n",
1808 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, msg,
1809 unicodeString, cc.unicodeLength,
1810 bytesString, cc.bytesLength,
1811 resultString, resultLength,
1812 offsetsString,
1813 resultOffsetsString,
1814 u_errorName(cc.outErrorCode), u_errorName(resultErrorCode),
1815 invalidCharsString, resultInvalidUCharsString);
1816
1817 return FALSE;
1818 }
1819 }
1820
1821 #endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */
1822