1 // Copyright (C) 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
3 /********************************************************************
4 * Copyright (c) 2016, International Business Machines Corporation and
5 * others. All Rights Reserved.
6 ********************************************************************/
7
8
9 #include "unicode/utypes.h"
10
11 #if !UCONFIG_NO_BREAK_ITERATION && !UCONFIG_NO_REGULAR_EXPRESSIONS && !UCONFIG_NO_FORMATTING
12
13 #include "rbbimonkeytest.h"
14 #include "unicode/utypes.h"
15 #include "unicode/brkiter.h"
16 #include "unicode/utf16.h"
17 #include "unicode/uniset.h"
18 #include "unicode/unistr.h"
19
20 #include "charstr.h"
21 #include "cmemory.h"
22 #include "cstr.h"
23 #include "uelement.h"
24 #include "uhash.h"
25
26 #include <iostream>
27 #include <stdio.h>
28 #include <stdlib.h>
29 #include <string>
30
31 using namespace icu;
32
33
runIndexedTest(int32_t index,UBool exec,const char * & name,char * params)34 void RBBIMonkeyTest::runIndexedTest(int32_t index, UBool exec, const char* &name, char* params) {
35 fParams = params; // Work around TESTCASE_AUTO not being able to pass params to test function.
36
37 TESTCASE_AUTO_BEGIN;
38 TESTCASE_AUTO(testMonkey);
39 TESTCASE_AUTO_END;
40 }
41
42 //---------------------------------------------------------------------------------------
43 //
44 // class BreakRule implementation.
45 //
46 //---------------------------------------------------------------------------------------
47
BreakRule()48 BreakRule::BreakRule() // : all field default initialized.
49 {
50 }
51
~BreakRule()52 BreakRule::~BreakRule() {}
53
54
55 //---------------------------------------------------------------------------------------
56 //
57 // class BreakRules implementation.
58 //
59 //---------------------------------------------------------------------------------------
BreakRules(RBBIMonkeyImpl * monkeyImpl,UErrorCode & status)60 BreakRules::BreakRules(RBBIMonkeyImpl *monkeyImpl, UErrorCode &status) :
61 fMonkeyImpl(monkeyImpl), fBreakRules(status), fType(UBRK_COUNT) {
62 fCharClasses.adoptInstead(uhash_open(uhash_hashUnicodeString,
63 uhash_compareUnicodeString,
64 NULL, // value comparator.
65 &status));
66 if (U_FAILURE(status)) {
67 return;
68 }
69 uhash_setKeyDeleter(fCharClasses.getAlias(), uprv_deleteUObject);
70 uhash_setValueDeleter(fCharClasses.getAlias(), uprv_deleteUObject);
71 fBreakRules.setDeleter(uprv_deleteUObject);
72
73 fCharClassList.adoptInstead(new UVector(status));
74
75 fSetRefsMatcher.adoptInstead(new RegexMatcher(UnicodeString(
76 "(?!(?:\\{|=|\\[:)[ \\t]{0,4})" // Negative lookbehind for '{' or '=' or '[:'
77 // (the identifier is a unicode property name or value)
78 "(?<ClassName>[A-Za-z_][A-Za-z0-9_]*)"), // The char class name
79 0, status));
80
81 // Match comments and blank lines. Matches will be replaced with "", stripping the comments from the rules.
82 fCommentsMatcher.adoptInstead(new RegexMatcher(UnicodeString(
83 "(^|(?<=;))" // Start either at start of line, or just after a ';' (look-behind for ';')
84 "[ \\t]*+" // Match white space.
85 "(#.*)?+" // Optional # plus whatever follows
86 "\\R$" // new-line at end of line.
87 ), 0, status));
88
89 // Match (initial parse) of a character class defintion line.
90 fClassDefMatcher.adoptInstead(new RegexMatcher(UnicodeString(
91 "[ \\t]*" // leading white space
92 "(?<ClassName>[A-Za-z_][A-Za-z0-9_]*)" // The char class name
93 "[ \\t]*=[ \\t]*" // =
94 "(?<ClassDef>.*?)" // The char class UnicodeSet expression
95 "[ \\t]*;$"), // ; <end of line>
96 0, status));
97
98 // Match (initial parse) of a break rule line.
99 fRuleDefMatcher.adoptInstead(new RegexMatcher(UnicodeString(
100 "[ \\t]*" // leading white space
101 "(?<RuleName>[A-Za-z_][A-Za-z0-9_.]*)" // The rule name
102 "[ \\t]*:[ \\t]*" // :
103 "(?<RuleDef>.*?)" // The rule definition
104 "[ \\t]*;$"), // ; <end of line>
105 0, status));
106
107 }
108
109
~BreakRules()110 BreakRules::~BreakRules() {}
111
112
addCharClass(const UnicodeString & name,const UnicodeString & definition,UErrorCode & status)113 CharClass *BreakRules::addCharClass(const UnicodeString &name, const UnicodeString &definition, UErrorCode &status) {
114
115 // Create the expanded definition for this char class,
116 // replacing any set references with the corresponding definition.
117
118 UnicodeString expandedDef;
119 UnicodeString emptyString;
120 fSetRefsMatcher->reset(definition);
121 while (fSetRefsMatcher->find() && U_SUCCESS(status)) {
122 const UnicodeString name =
123 fSetRefsMatcher->group(fSetRefsMatcher->pattern().groupNumberFromName("ClassName", status), status);
124 CharClass *nameClass = static_cast<CharClass *>(uhash_get(fCharClasses.getAlias(), &name));
125 const UnicodeString &expansionForName = nameClass ? nameClass->fExpandedDef : name;
126
127 fSetRefsMatcher->appendReplacement(expandedDef, emptyString, status);
128 expandedDef.append(expansionForName);
129 }
130 fSetRefsMatcher->appendTail(expandedDef);
131
132 // Verify that the expanded set defintion is valid.
133
134 if (fMonkeyImpl->fDumpExpansions) {
135 printf("epandedDef: %s\n", CStr(expandedDef)());
136 }
137
138 UnicodeSet *s = new UnicodeSet(expandedDef, USET_IGNORE_SPACE, NULL, status);
139 if (U_FAILURE(status)) {
140 IntlTest::gTest->errln("%s:%d: error %s creating UnicodeSet %s", __FILE__, __LINE__,
141 u_errorName(status), CStr(name)());
142 return NULL;
143 }
144 CharClass *cclass = new CharClass(name, definition, expandedDef, s);
145 CharClass *previousClass = static_cast<CharClass *>(uhash_put(fCharClasses.getAlias(),
146 new UnicodeString(name), // Key, owned by hash table.
147 cclass, // Value, owned by hash table.
148 &status));
149
150 if (previousClass != NULL) {
151 // Duplicate class def.
152 // These are legitimate, they are adustments of an existing class.
153 // TODO: will need to keep the old around when we handle tailorings.
154 IntlTest::gTest->logln("Redefinition of character class %s\n", CStr(cclass->fName)());
155 delete previousClass;
156 }
157 return cclass;
158 }
159
160
addRule(const UnicodeString & name,const UnicodeString & definition,UErrorCode & status)161 void BreakRules::addRule(const UnicodeString &name, const UnicodeString &definition, UErrorCode &status) {
162 LocalPointer<BreakRule> thisRule(new BreakRule);
163 thisRule->fName = name;
164 thisRule->fRule = definition;
165
166 // If the rule name contains embedded digits, pad the first numeric field to a fixed length with leading zeroes,
167 // This gives a numeric sort order that matches Unicode UAX rule numbering conventions.
168 UnicodeString emptyString;
169
170 // Expand the char class definitions within the rule.
171 fSetRefsMatcher->reset(definition);
172 while (fSetRefsMatcher->find() && U_SUCCESS(status)) {
173 const UnicodeString name =
174 fSetRefsMatcher->group(fSetRefsMatcher->pattern().groupNumberFromName("ClassName", status), status);
175 CharClass *nameClass = static_cast<CharClass *>(uhash_get(fCharClasses.getAlias(), &name));
176 if (!nameClass) {
177 IntlTest::gTest->errln("%s:%d char class \"%s\" unrecognized in rule \"%s\"",
178 __FILE__, __LINE__, CStr(name)(), CStr(definition)());
179 }
180 const UnicodeString &expansionForName = nameClass ? nameClass->fExpandedDef : name;
181
182 fSetRefsMatcher->appendReplacement(thisRule->fExpandedRule, emptyString, status);
183 thisRule->fExpandedRule.append(expansionForName);
184 }
185 fSetRefsMatcher->appendTail(thisRule->fExpandedRule);
186
187 // Replace the divide sign (\u00f7) with a regular expression named capture.
188 // When running the rules, a match that includes this group means we found a break position.
189
190 int32_t dividePos = thisRule->fExpandedRule.indexOf((UChar)0x00f7);
191 if (dividePos >= 0) {
192 thisRule->fExpandedRule.replace(dividePos, 1, UnicodeString("(?<BreakPosition>)"));
193 }
194 if (thisRule->fExpandedRule.indexOf((UChar)0x00f7) != -1) {
195 status = U_ILLEGAL_ARGUMENT_ERROR; // TODO: produce a good error message.
196 }
197
198 // UAX break rule set definitions can be empty, just [].
199 // Regular expression set expressions don't accept this. Substitute with [^\u0000-\U0010ffff], which
200 // also matches nothing.
201
202 static const UChar emptySet[] = {(UChar)0x5b, (UChar)0x5d, 0};
203 int32_t where = 0;
204 while ((where = thisRule->fExpandedRule.indexOf(emptySet, 2, 0)) >= 0) {
205 thisRule->fExpandedRule.replace(where, 2, UnicodeString("[^\\u0000-\\U0010ffff]"));
206 }
207 if (fMonkeyImpl->fDumpExpansions) {
208 printf("fExpandedRule: %s\n", CStr(thisRule->fExpandedRule)());
209 }
210
211 // Compile a regular expression for this rule.
212 thisRule->fRuleMatcher.adoptInstead(new RegexMatcher(thisRule->fExpandedRule, UREGEX_COMMENTS | UREGEX_DOTALL, status));
213 if (U_FAILURE(status)) {
214 IntlTest::gTest->errln("%s:%d Error creating regular expression for %s",
215 __FILE__, __LINE__, CStr(thisRule->fExpandedRule)());
216 return;
217 }
218
219 // Put this new rule into the vector of all Rules.
220 fBreakRules.addElement(thisRule.orphan(), status);
221 }
222
223
setKeywordParameter(const UnicodeString & keyword,const UnicodeString & value,UErrorCode & status)224 bool BreakRules::setKeywordParameter(const UnicodeString &keyword, const UnicodeString &value, UErrorCode &status) {
225 if (keyword == UnicodeString("locale")) {
226 CharString localeName;
227 localeName.append(CStr(value)(), -1, status);
228 fLocale = Locale::createFromName(localeName.data());
229 return true;
230 }
231 if (keyword == UnicodeString("type")) {
232 if (value == UnicodeString("grapheme")) {
233 fType = UBRK_CHARACTER;
234 } else if (value == UnicodeString("word")) {
235 fType = UBRK_WORD;
236 } else if (value == UnicodeString("line")) {
237 fType = UBRK_LINE;
238 } else if (value == UnicodeString("sentence")) {
239 fType = UBRK_SENTENCE;
240 } else {
241 IntlTest::gTest->errln("%s:%d Unrecognized break type %s", __FILE__, __LINE__, CStr(value)());
242 }
243 return true;
244 }
245 // TODO: add tailoring base setting here.
246 return false;
247 }
248
createICUBreakIterator(UErrorCode & status)249 RuleBasedBreakIterator *BreakRules::createICUBreakIterator(UErrorCode &status) {
250 if (U_FAILURE(status)) {
251 return NULL;
252 }
253 RuleBasedBreakIterator *bi = NULL;
254 switch(fType) {
255 case UBRK_CHARACTER:
256 bi = dynamic_cast<RuleBasedBreakIterator *>(BreakIterator::createCharacterInstance(fLocale, status));
257 break;
258 case UBRK_WORD:
259 bi = dynamic_cast<RuleBasedBreakIterator *>(BreakIterator::createWordInstance(fLocale, status));
260 break;
261 case UBRK_LINE:
262 bi = dynamic_cast<RuleBasedBreakIterator *>(BreakIterator::createLineInstance(fLocale, status));
263 break;
264 case UBRK_SENTENCE:
265 bi = dynamic_cast<RuleBasedBreakIterator *>(BreakIterator::createSentenceInstance(fLocale, status));
266 break;
267 default:
268 IntlTest::gTest->errln("%s:%d Bad break iterator type of %d", __FILE__, __LINE__, fType);
269 status = U_ILLEGAL_ARGUMENT_ERROR;
270 }
271 return bi;
272 }
273
274
compileRules(UCHARBUF * rules,UErrorCode & status)275 void BreakRules::compileRules(UCHARBUF *rules, UErrorCode &status) {
276 if (U_FAILURE(status)) {
277 return;
278 }
279
280 UnicodeString emptyString;
281 for (int32_t lineNumber=0; ;lineNumber++) { // Loop once per input line.
282 if (U_FAILURE(status)) {
283 return;
284 }
285 int32_t lineLength = 0;
286 const UChar *lineBuf = ucbuf_readline(rules, &lineLength, &status);
287 if (lineBuf == NULL) {
288 break;
289 }
290 UnicodeString line(lineBuf, lineLength);
291
292 // Strip comment lines.
293 fCommentsMatcher->reset(line);
294 line = fCommentsMatcher->replaceFirst(emptyString, status);
295 if (line.isEmpty()) {
296 continue;
297 }
298
299 // Recognize character class definition and keyword lines
300 fClassDefMatcher->reset(line);
301 if (fClassDefMatcher->matches(status)) {
302 UnicodeString className = fClassDefMatcher->group(fClassDefMatcher->pattern().groupNumberFromName("ClassName", status), status);
303 UnicodeString classDef = fClassDefMatcher->group(fClassDefMatcher->pattern().groupNumberFromName("ClassDef", status), status);
304 if (fMonkeyImpl->fDumpExpansions) {
305 printf("scanned class: %s = %s\n", CStr(className)(), CStr(classDef)());
306 }
307 if (setKeywordParameter(className, classDef, status)) {
308 // The scanned item was "type = ..." or "locale = ...", etc.
309 // which are not actual character classes.
310 continue;
311 }
312 addCharClass(className, classDef, status);
313 continue;
314 }
315
316 // Recognize rule lines.
317 fRuleDefMatcher->reset(line);
318 if (fRuleDefMatcher->matches(status)) {
319 UnicodeString ruleName = fRuleDefMatcher->group(fRuleDefMatcher->pattern().groupNumberFromName("RuleName", status), status);
320 UnicodeString ruleDef = fRuleDefMatcher->group(fRuleDefMatcher->pattern().groupNumberFromName("RuleDef", status), status);
321 if (fMonkeyImpl->fDumpExpansions) {
322 printf("scanned rule: %s : %s\n", CStr(ruleName)(), CStr(ruleDef)());
323 }
324 addRule(ruleName, ruleDef, status);
325 continue;
326 }
327
328 IntlTest::gTest->errln("%s:%d: Unrecognized line in rule file %s: \"%s\"\n",
329 __FILE__, __LINE__, fMonkeyImpl->fRuleFileName, CStr(line)());
330 }
331
332 // Build the vector of char classes, omitting the dictionary class if there is one.
333 // This will be used when constructing the random text to be tested.
334
335 // Also compute the "other" set, consisting of any characters not included in
336 // one or more of the user defined sets.
337
338 UnicodeSet otherSet((UChar32)0, 0x10ffff);
339 int32_t pos = UHASH_FIRST;
340 const UHashElement *el = NULL;
341 while ((el = uhash_nextElement(fCharClasses.getAlias(), &pos)) != NULL) {
342 const UnicodeString *ccName = static_cast<const UnicodeString *>(el->key.pointer);
343 CharClass *cclass = static_cast<CharClass *>(el->value.pointer);
344 // printf(" Adding %s\n", CStr(*ccName)());
345 if (*ccName != cclass->fName) {
346 IntlTest::gTest->errln("%s:%d: internal error, set names (%s, %s) inconsistent.\n",
347 __FILE__, __LINE__, CStr(*ccName)(), CStr(cclass->fName)());
348 }
349 const UnicodeSet *set = cclass->fSet.getAlias();
350 otherSet.removeAll(*set);
351 if (*ccName == UnicodeString("dictionary")) {
352 fDictionarySet = *set;
353 } else {
354 fCharClassList->addElement(cclass, status);
355 }
356 }
357
358 if (!otherSet.isEmpty()) {
359 // fprintf(stderr, "have an other set.\n");
360 UnicodeString pattern;
361 CharClass *cclass = addCharClass(UnicodeString("__Others"), otherSet.toPattern(pattern), status);
362 fCharClassList->addElement(cclass, status);
363 }
364 }
365
366
getClassForChar(UChar32 c,int32_t * iter) const367 const CharClass *BreakRules::getClassForChar(UChar32 c, int32_t *iter) const {
368 int32_t localIter = 0;
369 int32_t &it = iter? *iter : localIter;
370
371 while (it < fCharClassList->size()) {
372 const CharClass *cc = static_cast<const CharClass *>(fCharClassList->elementAt(it));
373 ++it;
374 if (cc->fSet->contains(c)) {
375 return cc;
376 }
377 }
378 return NULL;
379 }
380
381 //---------------------------------------------------------------------------------------
382 //
383 // class MonkeyTestData implementation.
384 //
385 //---------------------------------------------------------------------------------------
386
set(BreakRules * rules,IntlTest::icu_rand & rand,UErrorCode & status)387 void MonkeyTestData::set(BreakRules *rules, IntlTest::icu_rand &rand, UErrorCode &status) {
388 const int32_t dataLength = 1000;
389
390 // Fill the test string with random characters.
391 // First randomly pick a char class, then randomly pick a character from that class.
392 // Exclude any characters from the dictionary set.
393
394 // std::cout << "Populating Test Data" << std::endl;
395 fRandomSeed = rand.getSeed(); // Save initial seed for use in error messages,
396 // allowing recreation of failing data.
397 fBkRules = rules;
398 fString.remove();
399 for (int32_t n=0; n<dataLength;) {
400 int charClassIndex = rand() % rules->fCharClassList->size();
401 const CharClass *cclass = static_cast<CharClass *>(rules->fCharClassList->elementAt(charClassIndex));
402 if (cclass->fSet->size() == 0) {
403 // Some rules or tailorings do end up with empty char classes.
404 continue;
405 }
406 int32_t charIndex = rand() % cclass->fSet->size();
407 UChar32 c = cclass->fSet->charAt(charIndex);
408 if (U16_IS_TRAIL(c) && fString.length() > 0 && U16_IS_LEAD(fString.charAt(fString.length()-1))) {
409 // Character classes may contain unpaired surrogates, e.g. Grapheme_Cluster_Break = Control.
410 // Don't let random unpaired surrogates combine in the test data because they might
411 // produce an unwanted dictionary character.
412 continue;
413 }
414
415 if (!rules->fDictionarySet.contains(c)) {
416 fString.append(c);
417 ++n;
418 }
419 }
420
421 // Reset each rule matcher regex with this new string.
422 // (Although we are always using the same string object, ICU regular expressions
423 // don't like the underlying string data changing without doing a reset).
424
425 for (int32_t ruleNum=0; ruleNum<rules->fBreakRules.size(); ruleNum++) {
426 BreakRule *rule = static_cast<BreakRule *>(rules->fBreakRules.elementAt(ruleNum));
427 rule->fRuleMatcher->reset(fString);
428 }
429
430 // Init the expectedBreaks, actualBreaks and ruleForPosition strings (used as arrays).
431 // Expected and Actual breaks are one longer than the input string; a non-zero value
432 // will indicate a boundary preceding that position.
433
434 clearActualBreaks();
435 fExpectedBreaks = fActualBreaks;
436 fRuleForPosition = fActualBreaks;
437 f2ndRuleForPos = fActualBreaks;
438
439 // Apply reference rules to find the expected breaks.
440
441 fExpectedBreaks.setCharAt(0, (UChar)1); // Force an expected break before the start of the text.
442 // ICU always reports a break there.
443 // The reference rules do not have a means to do so.
444 int32_t strIdx = 0;
445 while (strIdx < fString.length()) {
446 BreakRule *matchingRule = NULL;
447 UBool hasBreak = FALSE;
448 int32_t ruleNum = 0;
449 int32_t matchStart = 0;
450 int32_t matchEnd = 0;
451 int32_t breakGroup = 0;
452 for (ruleNum=0; ruleNum<rules->fBreakRules.size(); ruleNum++) {
453 BreakRule *rule = static_cast<BreakRule *>(rules->fBreakRules.elementAt(ruleNum));
454 rule->fRuleMatcher->reset();
455 if (rule->fRuleMatcher->lookingAt(strIdx, status)) {
456 // A candidate rule match, check further to see if we take it or continue to check other rules.
457 // Matches of zero or one codepoint count only if they also specify a break.
458 matchStart = rule->fRuleMatcher->start(status);
459 matchEnd = rule->fRuleMatcher->end(status);
460 breakGroup = rule->fRuleMatcher->pattern().groupNumberFromName("BreakPosition", status);
461 hasBreak = U_SUCCESS(status);
462 if (status == U_REGEX_INVALID_CAPTURE_GROUP_NAME) {
463 status = U_ZERO_ERROR;
464 }
465 if (hasBreak || fString.moveIndex32(matchStart, 1) < matchEnd) {
466 matchingRule = rule;
467 break;
468 }
469 }
470 }
471 if (matchingRule == NULL) {
472 // No reference rule matched. This is an error in the rules that should never happen.
473 IntlTest::gTest->errln("%s:%d Trouble with monkey test reference rules at position %d. ",
474 __FILE__, __LINE__, strIdx);
475 dump(strIdx);
476 status = U_INVALID_FORMAT_ERROR;
477 return;
478 }
479 if (matchingRule->fRuleMatcher->group(status).length() == 0) {
480 // Zero length rule match. This is also an error in the rule expressions.
481 IntlTest::gTest->errln("%s:%d Zero length rule match.",
482 __FILE__, __LINE__);
483 status = U_INVALID_FORMAT_ERROR;
484 return;
485 }
486
487 // Record which rule matched over the length of the match.
488 for (int i = matchStart; i < matchEnd; i++) {
489 if (fRuleForPosition.charAt(i) == 0) {
490 fRuleForPosition.setCharAt(i, (UChar)ruleNum);
491 } else {
492 f2ndRuleForPos.setCharAt(i, (UChar)ruleNum);
493 }
494 }
495
496 // Break positions appear in rules as a matching named capture of zero length at the break position,
497 // the adjusted pattern contains (?<BreakPosition>)
498 if (hasBreak) {
499 int32_t breakPos = matchingRule->fRuleMatcher->start(breakGroup, status);
500 if (U_FAILURE(status) || breakPos < 0) {
501 // Rule specified a break, but that break wasn't part of the match, even
502 // though the rule as a whole matched.
503 // Can't happen with regular expressions derived from (equivalent to) ICU break rules.
504 // Shouldn't get here.
505 IntlTest::gTest->errln("%s:%d Internal Rule Error.", __FILE__, __LINE__);
506 status = U_INVALID_FORMAT_ERROR;
507 break;
508 }
509 fExpectedBreaks.setCharAt(breakPos, (UChar)1);
510 // printf("recording break at %d\n", breakPos);
511 // For the next iteration, pick up applying rules immediately after the break,
512 // which may differ from end of the match. The matching rule may have included
513 // context following the boundary that needs to be looked at again.
514 strIdx = matchingRule->fRuleMatcher->end(breakGroup, status);
515 } else {
516 // Original rule didn't specify a break.
517 // Continue applying rules starting on the last code point of this match.
518 strIdx = fString.moveIndex32(matchEnd, -1);
519 if (strIdx == matchStart) {
520 // Match was only one code point, no progress if we continue.
521 // Shouldn't get here, case is filtered out at top of loop.
522 CharString ruleName;
523 ruleName.appendInvariantChars(matchingRule->fName, status);
524 IntlTest::gTest->errln("%s:%d Rule %s internal error",
525 __FILE__, __LINE__, ruleName.data());
526 status = U_INVALID_FORMAT_ERROR;
527 break;
528 }
529 }
530 if (U_FAILURE(status)) {
531 IntlTest::gTest->errln("%s:%d status = %s. Unexpected failure, perhaps problem internal to test.",
532 __FILE__, __LINE__, u_errorName(status));
533 break;
534 }
535 }
536 }
537
clearActualBreaks()538 void MonkeyTestData::clearActualBreaks() {
539 fActualBreaks.remove();
540 // Actual Breaks length is one longer than the data string length, allowing
541 // for breaks before the first and after the last character in the data.
542 for (int32_t i=0; i<=fString.length(); i++) {
543 fActualBreaks.append((UChar)0);
544 }
545 }
546
dump(int32_t around) const547 void MonkeyTestData::dump(int32_t around) const {
548 printf("\n"
549 " char break Rule Character\n"
550 " pos code class R I name name\n"
551 "---------------------------------------------------------------------------------------------\n");
552
553 int32_t start;
554 int32_t end;
555
556 if (around == -1) {
557 start = 0;
558 end = fString.length();
559 } else {
560 // Display context around a failure.
561 start = fString.moveIndex32(around, -30);
562 end = fString.moveIndex32(around, +30);
563 }
564
565 for (int charIdx = start; charIdx < end; charIdx=fString.moveIndex32(charIdx, 1)) {
566 UErrorCode status = U_ZERO_ERROR;
567 UChar32 c = fString.char32At(charIdx);
568 const CharClass *cc = fBkRules->getClassForChar(c);
569 CharString ccName;
570 ccName.appendInvariantChars(cc->fName, status);
571 CharString ruleName, secondRuleName;
572 const BreakRule *rule = static_cast<BreakRule *>(fBkRules->fBreakRules.elementAt(fRuleForPosition.charAt(charIdx)));
573 ruleName.appendInvariantChars(rule->fName, status);
574 if (f2ndRuleForPos.charAt(charIdx) > 0) {
575 const BreakRule *secondRule = static_cast<BreakRule *>(fBkRules->fBreakRules.elementAt(f2ndRuleForPos.charAt(charIdx)));
576 secondRuleName.appendInvariantChars(secondRule->fName, status);
577 }
578 char cName[200];
579 u_charName(c, U_EXTENDED_CHAR_NAME, cName, sizeof(cName), &status);
580
581 printf(" %4.1d %6.4x %-20s %c %c %-10s %-10s %s\n",
582 charIdx, c, ccName.data(),
583 fExpectedBreaks.charAt(charIdx) ? '*' : '.',
584 fActualBreaks.charAt(charIdx) ? '*' : '.',
585 ruleName.data(), secondRuleName.data(), cName
586 );
587 }
588 }
589
590
591 //---------------------------------------------------------------------------------------
592 //
593 // class RBBIMonkeyImpl
594 //
595 //---------------------------------------------------------------------------------------
596
RBBIMonkeyImpl(UErrorCode & status)597 RBBIMonkeyImpl::RBBIMonkeyImpl(UErrorCode &status) : fDumpExpansions(FALSE), fThread(this) {
598 (void)status; // suppress unused parameter compiler warning.
599 }
600
601
602 // RBBIMonkeyImpl setup does all of the setup for a single rule set - compiling the
603 // reference rules and creating the icu breakiterator to test,
604 // with its type and locale coming from the reference rules.
605
setup(const char * ruleFile,UErrorCode & status)606 void RBBIMonkeyImpl::setup(const char *ruleFile, UErrorCode &status) {
607 fRuleFileName = ruleFile;
608 openBreakRules(ruleFile, status);
609 if (U_FAILURE(status)) {
610 IntlTest::gTest->errln("%s:%d Error %s opening file %s.", __FILE__, __LINE__, u_errorName(status), ruleFile);
611 return;
612 }
613 fRuleSet.adoptInstead(new BreakRules(this, status));
614 fRuleSet->compileRules(fRuleCharBuffer.getAlias(), status);
615 if (U_FAILURE(status)) {
616 IntlTest::gTest->errln("%s:%d Error %s processing file %s.", __FILE__, __LINE__, u_errorName(status), ruleFile);
617 return;
618 }
619 fBI.adoptInstead(fRuleSet->createICUBreakIterator(status));
620 fTestData.adoptInstead(new MonkeyTestData());
621 }
622
623
~RBBIMonkeyImpl()624 RBBIMonkeyImpl::~RBBIMonkeyImpl() {
625 }
626
627
openBreakRules(const char * fileName,UErrorCode & status)628 void RBBIMonkeyImpl::openBreakRules(const char *fileName, UErrorCode &status) {
629 CharString path;
630 path.append(IntlTest::getSourceTestData(status), status);
631 path.append("break_rules" U_FILE_SEP_STRING, status);
632 path.appendPathPart(fileName, status);
633 const char *codePage = "UTF-8";
634 fRuleCharBuffer.adoptInstead(ucbuf_open(path.data(), &codePage, TRUE, FALSE, &status));
635 }
636
637
startTest()638 void RBBIMonkeyImpl::startTest() {
639 fThread.start(); // invokes runTest() in a separate thread.
640 }
641
join()642 void RBBIMonkeyImpl::join() {
643 fThread.join();
644 }
645
646
647 #define MONKEY_ERROR(msg, index) { \
648 IntlTest::gTest->errln("%s:%d %s at index %d. Parameters to reproduce: @rules=%s,seed=%u,loop=1,verbose ", \
649 __FILE__, __LINE__, msg, index, fRuleFileName, fTestData->fRandomSeed); \
650 if (fVerbose) { fTestData->dump(index); } \
651 status = U_INVALID_STATE_ERROR; \
652 }
653
runTest()654 void RBBIMonkeyImpl::runTest() {
655 UErrorCode status = U_ZERO_ERROR;
656 int32_t errorCount = 0;
657 for (int64_t loopCount = 0; fLoopCount < 0 || loopCount < fLoopCount; loopCount++) {
658 status = U_ZERO_ERROR;
659 fTestData->set(fRuleSet.getAlias(), fRandomGenerator, status);
660 if (fBI.isNull()) {
661 IntlTest::gTest->dataerrln("Unable to run test because fBI is null.");
662 return;
663 }
664 // fTestData->dump();
665 testForwards(status);
666 testPrevious(status);
667 testFollowing(status);
668 testPreceding(status);
669 testIsBoundary(status);
670
671 if (fLoopCount < 0 && loopCount % 100 == 0) {
672 fprintf(stderr, ".");
673 }
674 if (U_FAILURE(status)) {
675 if (++errorCount > 10) {
676 return;
677 }
678 }
679 }
680 }
681
testForwards(UErrorCode & status)682 void RBBIMonkeyImpl::testForwards(UErrorCode &status) {
683 if (U_FAILURE(status)) {
684 return;
685 }
686 fTestData->clearActualBreaks();
687 fBI->setText(fTestData->fString);
688 int32_t previousBreak = -2;
689 for (int32_t bk=fBI->first(); bk != BreakIterator::DONE; bk=fBI->next()) {
690 if (bk <= previousBreak) {
691 MONKEY_ERROR("Break Iterator Stall", bk);
692 return;
693 }
694 if (bk < 0 || bk > fTestData->fString.length()) {
695 MONKEY_ERROR("Boundary out of bounds", bk);
696 return;
697 }
698 fTestData->fActualBreaks.setCharAt(bk, 1);
699 }
700 checkResults("testForwards", FORWARD, status);
701 }
702
testFollowing(UErrorCode & status)703 void RBBIMonkeyImpl::testFollowing(UErrorCode &status) {
704 if (U_FAILURE(status)) {
705 return;
706 }
707 fTestData->clearActualBreaks();
708 fBI->setText(fTestData->fString);
709 int32_t nextBreak = -1;
710 for (int32_t i=-1 ; i<fTestData->fString.length(); ++i) {
711 int32_t bk = fBI->following(i);
712 if (bk == BreakIterator::DONE && i == fTestData->fString.length()) {
713 continue;
714 }
715 if (bk == nextBreak && bk > i) {
716 // i is in the gap between two breaks.
717 continue;
718 }
719 if (i == nextBreak && bk > nextBreak) {
720 fTestData->fActualBreaks.setCharAt(bk, 1);
721 nextBreak = bk;
722 continue;
723 }
724 MONKEY_ERROR("following(i)", i);
725 return;
726 }
727 checkResults("testFollowing", FORWARD, status);
728 }
729
730
731
testPrevious(UErrorCode & status)732 void RBBIMonkeyImpl::testPrevious(UErrorCode &status) {
733 if (U_FAILURE(status)) {return;}
734
735 fTestData->clearActualBreaks();
736 fBI->setText(fTestData->fString);
737 int32_t previousBreak = INT32_MAX;
738 for (int32_t bk=fBI->last(); bk != BreakIterator::DONE; bk=fBI->previous()) {
739 if (bk >= previousBreak) {
740 MONKEY_ERROR("Break Iterator Stall", bk);
741 return;
742 }
743 if (bk < 0 || bk > fTestData->fString.length()) {
744 MONKEY_ERROR("Boundary out of bounds", bk);
745 return;
746 }
747 fTestData->fActualBreaks.setCharAt(bk, 1);
748 }
749 checkResults("testPrevius", REVERSE, status);
750 }
751
752
testPreceding(UErrorCode & status)753 void RBBIMonkeyImpl::testPreceding(UErrorCode &status) {
754 if (U_FAILURE(status)) {
755 return;
756 }
757 fTestData->clearActualBreaks();
758 fBI->setText(fTestData->fString);
759 int32_t nextBreak = fTestData->fString.length()+1;
760 for (int32_t i=fTestData->fString.length()+1 ; i>=0; --i) {
761 int32_t bk = fBI->preceding(i);
762 // printf("i:%d bk:%d nextBreak:%d\n", i, bk, nextBreak);
763 if (bk == BreakIterator::DONE && i == 0) {
764 continue;
765 }
766 if (bk == nextBreak && bk < i) {
767 // i is in the gap between two breaks.
768 continue;
769 }
770 if (i<fTestData->fString.length() && fTestData->fString.getChar32Start(i) < i) {
771 // i indexes to a trailing surrogate.
772 // Break Iterators treat an index to either half as referring to the supplemental code point,
773 // with preceding going to some preceding code point.
774 if (fBI->preceding(i) != fBI->preceding(fTestData->fString.getChar32Start(i))) {
775 MONKEY_ERROR("preceding of trailing surrogate error", i);
776 }
777 continue;
778 }
779 if (i == nextBreak && bk < nextBreak) {
780 fTestData->fActualBreaks.setCharAt(bk, 1);
781 nextBreak = bk;
782 continue;
783 }
784 MONKEY_ERROR("preceding(i)", i);
785 return;
786 }
787 checkResults("testPreceding", REVERSE, status);
788 }
789
790
testIsBoundary(UErrorCode & status)791 void RBBIMonkeyImpl::testIsBoundary(UErrorCode &status) {
792 if (U_FAILURE(status)) {
793 return;
794 }
795 fTestData->clearActualBreaks();
796 fBI->setText(fTestData->fString);
797 for (int i=fTestData->fString.length(); i>=0; --i) {
798 if (fBI->isBoundary(i)) {
799 fTestData->fActualBreaks.setCharAt(i, 1);
800 }
801 }
802 checkResults("testForwards", FORWARD, status);
803 }
804
checkResults(const char * msg,CheckDirection direction,UErrorCode & status)805 void RBBIMonkeyImpl::checkResults(const char *msg, CheckDirection direction, UErrorCode &status) {
806 if (U_FAILURE(status)) {
807 return;
808 }
809 if (direction == FORWARD) {
810 for (int i=0; i<=fTestData->fString.length(); ++i) {
811 if (fTestData->fExpectedBreaks.charAt(i) != fTestData->fActualBreaks.charAt(i)) {
812 IntlTest::gTest->errln("%s:%d %s failure at index %d. Parameters to reproduce: @rules=%s,seed=%u,loop=1,verbose ",
813 __FILE__, __LINE__, msg, i, fRuleFileName, fTestData->fRandomSeed);
814 if (fVerbose) {
815 fTestData->dump(i);
816 }
817 status = U_INVALID_STATE_ERROR; // Prevent the test from continuing, which would likely
818 break; // produce many redundant errors.
819 }
820 }
821 } else {
822 for (int i=fTestData->fString.length(); i>=0; i--) {
823 if (fTestData->fExpectedBreaks.charAt(i) != fTestData->fActualBreaks.charAt(i)) {
824 IntlTest::gTest->errln("%s:%d %s failure at index %d. Parameters to reproduce: @rules=%s,seed=%u,loop=1,verbose ",
825 __FILE__, __LINE__, msg, i, fRuleFileName, fTestData->fRandomSeed);
826 if (fVerbose) {
827 fTestData->dump(i);
828 }
829 status = U_INVALID_STATE_ERROR;
830 break;
831 }
832 }
833 }
834 }
835
836
837
838 //---------------------------------------------------------------------------------------
839 //
840 // class RBBIMonkeyTest implementation.
841 //
842 //---------------------------------------------------------------------------------------
RBBIMonkeyTest()843 RBBIMonkeyTest::RBBIMonkeyTest() {
844 }
845
~RBBIMonkeyTest()846 RBBIMonkeyTest::~RBBIMonkeyTest() {
847 }
848
849
850 // params, taken from this->fParams.
851 // rules=file_name Name of file containing the reference rules.
852 // seed=nnnnn Random number starting seed.
853 // Setting the seed allows errors to be reproduced.
854 // loop=nnn Looping count. Controls running time.
855 // -1: run forever.
856 // 0 or greater: run length.
857 // expansions debug option, show expansions of rules and sets.
858 // verbose Display details of the failure.
859 //
860 // Parameters on the intltest command line follow the test name, and are preceded by '@'.
861 // For example,
862 // intltest rbbi/RBBIMonkeyTest/testMonkey@rules=line.txt,loop=-1
863 //
testMonkey()864 void RBBIMonkeyTest::testMonkey() {
865 // printf("Test parameters: %s\n", fParams);
866 UnicodeString params(fParams);
867 UErrorCode status = U_ZERO_ERROR;
868
869 const char *tests[] = {"grapheme.txt", "word.txt", "line.txt", "sentence.txt", "line_normal.txt",
870 "line_normal_cj.txt", "line_loose.txt", "line_loose_cj.txt", "word_POSIX.txt",
871 NULL };
872 CharString testNameFromParams;
873 if (getStringParam("rules", params, testNameFromParams, status)) {
874 tests[0] = testNameFromParams.data();
875 tests[1] = NULL;
876 }
877
878 int64_t loopCount = quick? 100 : 5000;
879 getIntParam("loop", params, loopCount, status);
880
881 UBool dumpExpansions = FALSE;
882 getBoolParam("expansions", params, dumpExpansions, status);
883
884 UBool verbose = FALSE;
885 getBoolParam("verbose", params, verbose, status);
886
887 int64_t seed = 0;
888 getIntParam("seed", params, seed, status);
889
890 if (params.length() != 0) {
891 // Options processing did not consume all of the parameters. Something unrecognized was present.
892 CharString unrecognizedParameters;
893 unrecognizedParameters.append(CStr(params)(), -1, status);
894 errln("%s:%d unrecognized test parameter(s) \"%s\"", __FILE__, __LINE__, unrecognizedParameters.data());
895 return;
896 }
897
898 UVector startedTests(status);
899 if (U_FAILURE(status)) {
900 errln("%s:%d: error %s while setting up test.", __FILE__, __LINE__, u_errorName(status));
901 return;
902 }
903
904 // Monkey testing is multi-threaded.
905 // Each set of break rules to be tested is run in a separate thread.
906 // Each thread/set of rules gets a separate RBBIMonkeyImpl object.
907 int32_t i;
908 for (i=0; tests[i] != NULL; ++i) {
909 logln("beginning testing of %s", tests[i]);
910 RBBIMonkeyImpl *test = new RBBIMonkeyImpl(status);
911 if (U_FAILURE(status)) {
912 errln("%s:%d: error %s while starting test %s.", __FILE__, __LINE__, u_errorName(status), tests[i]);
913 break;
914 }
915 test->fDumpExpansions = dumpExpansions;
916 test->fVerbose = verbose;
917 test->fRandomGenerator.seed((uint32_t)seed);
918 test->fLoopCount = loopCount;
919 test->setup(tests[i], status);
920 if (U_FAILURE(status)) {
921 errln("%s:%d: error %s while starting test %s.", __FILE__, __LINE__, u_errorName(status), tests[i]);
922 break;
923 }
924 test->startTest();
925 startedTests.addElement(test, status);
926 if (U_FAILURE(status)) {
927 errln("%s:%d: error %s while starting test %s.", __FILE__, __LINE__, u_errorName(status), tests[i]);
928 break;
929 }
930 }
931
932 for (i=0; i<startedTests.size(); ++i) {
933 RBBIMonkeyImpl *test = static_cast<RBBIMonkeyImpl *>(startedTests.elementAt(i));
934 test->join();
935 delete test;
936 }
937 }
938
939
getIntParam(UnicodeString name,UnicodeString & params,int64_t & val,UErrorCode & status)940 UBool RBBIMonkeyTest::getIntParam(UnicodeString name, UnicodeString ¶ms, int64_t &val, UErrorCode &status) {
941 name.append(" *= *(-?\\d+) *,? *");
942 RegexMatcher m(name, params, 0, status);
943 if (m.find()) {
944 // The param exists. Convert the string to an int.
945 CharString str;
946 str.append(CStr(m.group(1, status))(), -1, status);
947 val = strtol(str.data(), NULL, 10);
948
949 // Delete this parameter from the params string.
950 m.reset();
951 params = m.replaceFirst(UnicodeString(), status);
952 return TRUE;
953 }
954 return FALSE;
955 }
956
getStringParam(UnicodeString name,UnicodeString & params,CharString & dest,UErrorCode & status)957 UBool RBBIMonkeyTest::getStringParam(UnicodeString name, UnicodeString ¶ms, CharString &dest, UErrorCode &status) {
958 name.append(" *= *([^ ,]*) *,? *");
959 RegexMatcher m(name, params, 0, status);
960 if (m.find()) {
961 // The param exists.
962 dest.append(CStr(m.group(1, status))(), -1, status);
963
964 // Delete this parameter from the params string.
965 m.reset();
966 params = m.replaceFirst(UnicodeString(), status);
967 return TRUE;
968 }
969 return FALSE;
970 }
971
getBoolParam(UnicodeString name,UnicodeString & params,UBool & dest,UErrorCode & status)972 UBool RBBIMonkeyTest::getBoolParam(UnicodeString name, UnicodeString ¶ms, UBool &dest, UErrorCode &status) {
973 name.append("(?: *= *(true|false))? *,? *");
974 RegexMatcher m(name, params, UREGEX_CASE_INSENSITIVE, status);
975 if (m.find()) {
976 if (m.start(1, status) > 0) {
977 // user option included a value.
978 dest = m.group(1, status).caseCompare(UnicodeString("true"), U_FOLD_CASE_DEFAULT) == 0;
979 } else {
980 // No explicit user value, implies true.
981 dest = TRUE;
982 }
983
984 // Delete this parameter from the params string.
985 m.reset();
986 params = m.replaceFirst(UnicodeString(), status);
987 return TRUE;
988 }
989 return FALSE;
990 }
991
992 #endif /* !UCONFIG_NO_BREAK_ITERATION && !UCONFIG_NO_REGULAR_EXPRESSIONS && !UCONFIG_NO_FORMATTING */
993