• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // © 2018 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
3 
4 #include "unicode/utypes.h"
5 
6 #if !UCONFIG_NO_FORMATTING
7 
8 // Allow implicit conversion from char16_t* to UnicodeString for this file:
9 // Helpful in toString methods and elsewhere.
10 #define UNISTR_FROM_STRING_EXPLICIT
11 
12 #include "numparse_types.h"
13 #include "numparse_affixes.h"
14 #include "numparse_utils.h"
15 #include "number_utils.h"
16 
17 using namespace icu;
18 using namespace icu::numparse;
19 using namespace icu::numparse::impl;
20 using namespace icu::number;
21 using namespace icu::number::impl;
22 
23 
24 namespace {
25 
26 /**
27  * Helper method to return whether the given AffixPatternMatcher equals the given pattern string.
28  * Either both arguments must be null or the pattern string inside the AffixPatternMatcher must equal
29  * the given pattern string.
30  */
matched(const AffixPatternMatcher * affix,const UnicodeString & patternString)31 static bool matched(const AffixPatternMatcher* affix, const UnicodeString& patternString) {
32     return (affix == nullptr && patternString.isBogus()) ||
33            (affix != nullptr && affix->getPattern() == patternString);
34 }
35 
36 /**
37  * Helper method to return the length of the given AffixPatternMatcher. Returns 0 for null.
38  */
length(const AffixPatternMatcher * matcher)39 static int32_t length(const AffixPatternMatcher* matcher) {
40     return matcher == nullptr ? 0 : matcher->getPattern().length();
41 }
42 
43 /**
44  * Helper method to return whether (1) both lhs and rhs are null/invalid, or (2) if they are both
45  * valid, whether they are equal according to operator==.  Similar to Java Objects.equals()
46  */
equals(const AffixPatternMatcher * lhs,const AffixPatternMatcher * rhs)47 static bool equals(const AffixPatternMatcher* lhs, const AffixPatternMatcher* rhs) {
48     if (lhs == nullptr && rhs == nullptr) {
49         return true;
50     }
51     if (lhs == nullptr || rhs == nullptr) {
52         return false;
53     }
54     return *lhs == *rhs;
55 }
56 
57 }
58 
59 
AffixPatternMatcherBuilder(const UnicodeString & pattern,AffixTokenMatcherWarehouse & warehouse,IgnorablesMatcher * ignorables)60 AffixPatternMatcherBuilder::AffixPatternMatcherBuilder(const UnicodeString& pattern,
61                                                        AffixTokenMatcherWarehouse& warehouse,
62                                                        IgnorablesMatcher* ignorables)
63         : fMatchersLen(0),
64           fLastTypeOrCp(0),
65           fPattern(pattern),
66           fWarehouse(warehouse),
67           fIgnorables(ignorables) {}
68 
consumeToken(AffixPatternType type,UChar32 cp,UErrorCode & status)69 void AffixPatternMatcherBuilder::consumeToken(AffixPatternType type, UChar32 cp, UErrorCode& status) {
70     // This is called by AffixUtils.iterateWithConsumer() for each token.
71 
72     // Add an ignorables matcher between tokens except between two literals, and don't put two
73     // ignorables matchers in a row.
74     if (fIgnorables != nullptr && fMatchersLen > 0 &&
75         (fLastTypeOrCp < 0 || !fIgnorables->getSet()->contains(fLastTypeOrCp))) {
76         addMatcher(*fIgnorables);
77     }
78 
79     if (type != TYPE_CODEPOINT) {
80         // Case 1: the token is a symbol.
81         switch (type) {
82             case TYPE_MINUS_SIGN:
83                 addMatcher(fWarehouse.minusSign());
84                 break;
85             case TYPE_PLUS_SIGN:
86                 addMatcher(fWarehouse.plusSign());
87                 break;
88             case TYPE_PERCENT:
89                 addMatcher(fWarehouse.percent());
90                 break;
91             case TYPE_PERMILLE:
92                 addMatcher(fWarehouse.permille());
93                 break;
94             case TYPE_CURRENCY_SINGLE:
95             case TYPE_CURRENCY_DOUBLE:
96             case TYPE_CURRENCY_TRIPLE:
97             case TYPE_CURRENCY_QUAD:
98             case TYPE_CURRENCY_QUINT:
99                 // All currency symbols use the same matcher
100                 addMatcher(fWarehouse.currency(status));
101                 break;
102             default:
103                 U_ASSERT(FALSE);
104         }
105 
106     } else if (fIgnorables != nullptr && fIgnorables->getSet()->contains(cp)) {
107         // Case 2: the token is an ignorable literal.
108         // No action necessary: the ignorables matcher has already been added.
109 
110     } else {
111         // Case 3: the token is a non-ignorable literal.
112         addMatcher(fWarehouse.nextCodePointMatcher(cp));
113     }
114     fLastTypeOrCp = type != TYPE_CODEPOINT ? type : cp;
115 }
116 
addMatcher(NumberParseMatcher & matcher)117 void AffixPatternMatcherBuilder::addMatcher(NumberParseMatcher& matcher) {
118     if (fMatchersLen >= fMatchers.getCapacity()) {
119         fMatchers.resize(fMatchersLen * 2, fMatchersLen);
120     }
121     fMatchers[fMatchersLen++] = &matcher;
122 }
123 
build()124 AffixPatternMatcher AffixPatternMatcherBuilder::build() {
125     return AffixPatternMatcher(fMatchers, fMatchersLen, fPattern);
126 }
127 
128 
CodePointMatcherWarehouse()129 CodePointMatcherWarehouse::CodePointMatcherWarehouse()
130         : codePointCount(0), codePointNumBatches(0) {}
131 
~CodePointMatcherWarehouse()132 CodePointMatcherWarehouse::~CodePointMatcherWarehouse() {
133     // Delete the variable number of batches of code point matchers
134     for (int32_t i = 0; i < codePointNumBatches; i++) {
135         delete[] codePointsOverflow[i];
136     }
137 }
138 
CodePointMatcherWarehouse(CodePointMatcherWarehouse && src)139 CodePointMatcherWarehouse::CodePointMatcherWarehouse(CodePointMatcherWarehouse&& src) U_NOEXCEPT
140         : codePoints(std::move(src.codePoints)),
141           codePointsOverflow(std::move(src.codePointsOverflow)),
142           codePointCount(src.codePointCount),
143           codePointNumBatches(src.codePointNumBatches) {}
144 
145 CodePointMatcherWarehouse&
operator =(CodePointMatcherWarehouse && src)146 CodePointMatcherWarehouse::operator=(CodePointMatcherWarehouse&& src) U_NOEXCEPT {
147     codePoints = std::move(src.codePoints);
148     codePointsOverflow = std::move(src.codePointsOverflow);
149     codePointCount = src.codePointCount;
150     codePointNumBatches = src.codePointNumBatches;
151     return *this;
152 }
153 
nextCodePointMatcher(UChar32 cp)154 NumberParseMatcher& CodePointMatcherWarehouse::nextCodePointMatcher(UChar32 cp) {
155     if (codePointCount < CODE_POINT_STACK_CAPACITY) {
156         return codePoints[codePointCount++] = {cp};
157     }
158     int32_t totalCapacity = CODE_POINT_STACK_CAPACITY + codePointNumBatches * CODE_POINT_BATCH_SIZE;
159     if (codePointCount >= totalCapacity) {
160         // Need a new batch
161         auto* nextBatch = new CodePointMatcher[CODE_POINT_BATCH_SIZE];
162         if (codePointNumBatches >= codePointsOverflow.getCapacity()) {
163             // Need more room for storing pointers to batches
164             codePointsOverflow.resize(codePointNumBatches * 2, codePointNumBatches);
165         }
166         codePointsOverflow[codePointNumBatches++] = nextBatch;
167     }
168     return codePointsOverflow[codePointNumBatches - 1][(codePointCount++ - CODE_POINT_STACK_CAPACITY) %
169                                                        CODE_POINT_BATCH_SIZE] = {cp};
170 }
171 
172 
AffixTokenMatcherWarehouse(const AffixTokenMatcherSetupData * setupData)173 AffixTokenMatcherWarehouse::AffixTokenMatcherWarehouse(const AffixTokenMatcherSetupData* setupData)
174         : fSetupData(setupData) {}
175 
minusSign()176 NumberParseMatcher& AffixTokenMatcherWarehouse::minusSign() {
177     return fMinusSign = {fSetupData->dfs, true};
178 }
179 
plusSign()180 NumberParseMatcher& AffixTokenMatcherWarehouse::plusSign() {
181     return fPlusSign = {fSetupData->dfs, true};
182 }
183 
percent()184 NumberParseMatcher& AffixTokenMatcherWarehouse::percent() {
185     return fPercent = {fSetupData->dfs};
186 }
187 
permille()188 NumberParseMatcher& AffixTokenMatcherWarehouse::permille() {
189     return fPermille = {fSetupData->dfs};
190 }
191 
currency(UErrorCode & status)192 NumberParseMatcher& AffixTokenMatcherWarehouse::currency(UErrorCode& status) {
193     return fCurrency = {fSetupData->currencySymbols, fSetupData->dfs, fSetupData->parseFlags, status};
194 }
195 
ignorables()196 IgnorablesMatcher& AffixTokenMatcherWarehouse::ignorables() {
197     return fSetupData->ignorables;
198 }
199 
nextCodePointMatcher(UChar32 cp)200 NumberParseMatcher& AffixTokenMatcherWarehouse::nextCodePointMatcher(UChar32 cp) {
201     return fCodePoints.nextCodePointMatcher(cp);
202 }
203 
204 
CodePointMatcher(UChar32 cp)205 CodePointMatcher::CodePointMatcher(UChar32 cp)
206         : fCp(cp) {}
207 
match(StringSegment & segment,ParsedNumber & result,UErrorCode &) const208 bool CodePointMatcher::match(StringSegment& segment, ParsedNumber& result, UErrorCode&) const {
209     if (segment.startsWith(fCp)) {
210         segment.adjustOffsetByCodePoint();
211         result.setCharsConsumed(segment);
212     }
213     return false;
214 }
215 
smokeTest(const StringSegment & segment) const216 bool CodePointMatcher::smokeTest(const StringSegment& segment) const {
217     return segment.startsWith(fCp);
218 }
219 
toString() const220 UnicodeString CodePointMatcher::toString() const {
221     return u"<CodePoint>";
222 }
223 
224 
fromAffixPattern(const UnicodeString & affixPattern,AffixTokenMatcherWarehouse & tokenWarehouse,parse_flags_t parseFlags,bool * success,UErrorCode & status)225 AffixPatternMatcher AffixPatternMatcher::fromAffixPattern(const UnicodeString& affixPattern,
226                                                           AffixTokenMatcherWarehouse& tokenWarehouse,
227                                                           parse_flags_t parseFlags, bool* success,
228                                                           UErrorCode& status) {
229     if (affixPattern.isEmpty()) {
230         *success = false;
231         return {};
232     }
233     *success = true;
234 
235     IgnorablesMatcher* ignorables;
236     if (0 != (parseFlags & PARSE_FLAG_EXACT_AFFIX)) {
237         ignorables = nullptr;
238     } else {
239         ignorables = &tokenWarehouse.ignorables();
240     }
241 
242     AffixPatternMatcherBuilder builder(affixPattern, tokenWarehouse, ignorables);
243     AffixUtils::iterateWithConsumer(affixPattern, builder, status);
244     return builder.build();
245 }
246 
AffixPatternMatcher(MatcherArray & matchers,int32_t matchersLen,const UnicodeString & pattern)247 AffixPatternMatcher::AffixPatternMatcher(MatcherArray& matchers, int32_t matchersLen,
248                                          const UnicodeString& pattern)
249         : ArraySeriesMatcher(matchers, matchersLen), fPattern(pattern) {}
250 
getPattern() const251 UnicodeString AffixPatternMatcher::getPattern() const {
252     return fPattern.toAliasedUnicodeString();
253 }
254 
operator ==(const AffixPatternMatcher & other) const255 bool AffixPatternMatcher::operator==(const AffixPatternMatcher& other) const {
256     return fPattern == other.fPattern;
257 }
258 
259 
AffixMatcherWarehouse(AffixTokenMatcherWarehouse * tokenWarehouse)260 AffixMatcherWarehouse::AffixMatcherWarehouse(AffixTokenMatcherWarehouse* tokenWarehouse)
261         : fTokenWarehouse(tokenWarehouse) {
262 }
263 
isInteresting(const AffixPatternProvider & patternInfo,const IgnorablesMatcher & ignorables,parse_flags_t parseFlags,UErrorCode & status)264 bool AffixMatcherWarehouse::isInteresting(const AffixPatternProvider& patternInfo,
265                                           const IgnorablesMatcher& ignorables, parse_flags_t parseFlags,
266                                           UErrorCode& status) {
267     UnicodeString posPrefixString = patternInfo.getString(AffixPatternProvider::AFFIX_POS_PREFIX);
268     UnicodeString posSuffixString = patternInfo.getString(AffixPatternProvider::AFFIX_POS_SUFFIX);
269     UnicodeString negPrefixString;
270     UnicodeString negSuffixString;
271     if (patternInfo.hasNegativeSubpattern()) {
272         negPrefixString = patternInfo.getString(AffixPatternProvider::AFFIX_NEG_PREFIX);
273         negSuffixString = patternInfo.getString(AffixPatternProvider::AFFIX_NEG_SUFFIX);
274     }
275 
276     if (0 == (parseFlags & PARSE_FLAG_USE_FULL_AFFIXES) &&
277         AffixUtils::containsOnlySymbolsAndIgnorables(posPrefixString, *ignorables.getSet(), status) &&
278         AffixUtils::containsOnlySymbolsAndIgnorables(posSuffixString, *ignorables.getSet(), status) &&
279         AffixUtils::containsOnlySymbolsAndIgnorables(negPrefixString, *ignorables.getSet(), status) &&
280         AffixUtils::containsOnlySymbolsAndIgnorables(negSuffixString, *ignorables.getSet(), status)
281         // HACK: Plus and minus sign are a special case: we accept them trailing only if they are
282         // trailing in the pattern string.
283         && !AffixUtils::containsType(posSuffixString, TYPE_PLUS_SIGN, status) &&
284         !AffixUtils::containsType(posSuffixString, TYPE_MINUS_SIGN, status) &&
285         !AffixUtils::containsType(negSuffixString, TYPE_PLUS_SIGN, status) &&
286         !AffixUtils::containsType(negSuffixString, TYPE_MINUS_SIGN, status)) {
287         // The affixes contain only symbols and ignorables.
288         // No need to generate affix matchers.
289         return false;
290     }
291     return true;
292 }
293 
createAffixMatchers(const AffixPatternProvider & patternInfo,MutableMatcherCollection & output,const IgnorablesMatcher & ignorables,parse_flags_t parseFlags,UErrorCode & status)294 void AffixMatcherWarehouse::createAffixMatchers(const AffixPatternProvider& patternInfo,
295                                                 MutableMatcherCollection& output,
296                                                 const IgnorablesMatcher& ignorables,
297                                                 parse_flags_t parseFlags, UErrorCode& status) {
298     if (!isInteresting(patternInfo, ignorables, parseFlags, status)) {
299         return;
300     }
301 
302     // The affixes have interesting characters, or we are in strict mode.
303     // Use initial capacity of 6, the highest possible number of AffixMatchers.
304     UnicodeString sb;
305     bool includeUnpaired = 0 != (parseFlags & PARSE_FLAG_INCLUDE_UNPAIRED_AFFIXES);
306     UNumberSignDisplay signDisplay = (0 != (parseFlags & PARSE_FLAG_PLUS_SIGN_ALLOWED)) ? UNUM_SIGN_ALWAYS
307                                                                                         : UNUM_SIGN_AUTO;
308 
309     int32_t numAffixMatchers = 0;
310     int32_t numAffixPatternMatchers = 0;
311 
312     AffixPatternMatcher* posPrefix = nullptr;
313     AffixPatternMatcher* posSuffix = nullptr;
314 
315     // Pre-process the affix strings to resolve LDML rules like sign display.
316     for (int8_t signum = 1; signum >= -1; signum--) {
317         // Generate Prefix
318         bool hasPrefix = false;
319         PatternStringUtils::patternInfoToStringBuilder(
320                 patternInfo, true, signum, signDisplay, StandardPlural::OTHER, false, sb);
321         fAffixPatternMatchers[numAffixPatternMatchers] = AffixPatternMatcher::fromAffixPattern(
322                 sb, *fTokenWarehouse, parseFlags, &hasPrefix, status);
323         AffixPatternMatcher* prefix = hasPrefix ? &fAffixPatternMatchers[numAffixPatternMatchers++]
324                                                 : nullptr;
325 
326         // Generate Suffix
327         bool hasSuffix = false;
328         PatternStringUtils::patternInfoToStringBuilder(
329                 patternInfo, false, signum, signDisplay, StandardPlural::OTHER, false, sb);
330         fAffixPatternMatchers[numAffixPatternMatchers] = AffixPatternMatcher::fromAffixPattern(
331                 sb, *fTokenWarehouse, parseFlags, &hasSuffix, status);
332         AffixPatternMatcher* suffix = hasSuffix ? &fAffixPatternMatchers[numAffixPatternMatchers++]
333                                                 : nullptr;
334 
335         if (signum == 1) {
336             posPrefix = prefix;
337             posSuffix = suffix;
338         } else if (equals(prefix, posPrefix) && equals(suffix, posSuffix)) {
339             // Skip adding these matchers (we already have equivalents)
340             continue;
341         }
342 
343         // Flags for setting in the ParsedNumber; the token matchers may add more.
344         int flags = (signum == -1) ? FLAG_NEGATIVE : 0;
345 
346         // Note: it is indeed possible for posPrefix and posSuffix to both be null.
347         // We still need to add that matcher for strict mode to work.
348         fAffixMatchers[numAffixMatchers++] = {prefix, suffix, flags};
349         if (includeUnpaired && prefix != nullptr && suffix != nullptr) {
350             // The following if statements are designed to prevent adding two identical matchers.
351             if (signum == 1 || !equals(prefix, posPrefix)) {
352                 fAffixMatchers[numAffixMatchers++] = {prefix, nullptr, flags};
353             }
354             if (signum == 1 || !equals(suffix, posSuffix)) {
355                 fAffixMatchers[numAffixMatchers++] = {nullptr, suffix, flags};
356             }
357         }
358     }
359 
360     // Put the AffixMatchers in order, and then add them to the output.
361     // Since there are at most 9 elements, do a simple-to-implement bubble sort.
362     bool madeChanges;
363     do {
364         madeChanges = false;
365         for (int32_t i = 1; i < numAffixMatchers; i++) {
366             if (fAffixMatchers[i - 1].compareTo(fAffixMatchers[i]) > 0) {
367                 madeChanges = true;
368                 AffixMatcher temp = std::move(fAffixMatchers[i - 1]);
369                 fAffixMatchers[i - 1] = std::move(fAffixMatchers[i]);
370                 fAffixMatchers[i] = std::move(temp);
371             }
372         }
373     } while (madeChanges);
374 
375     for (int32_t i = 0; i < numAffixMatchers; i++) {
376         // Enable the following line to debug affixes
377         //std::cout << "Adding affix matcher: " << CStr(fAffixMatchers[i].toString())() << std::endl;
378         output.addMatcher(fAffixMatchers[i]);
379     }
380 }
381 
382 
AffixMatcher(AffixPatternMatcher * prefix,AffixPatternMatcher * suffix,result_flags_t flags)383 AffixMatcher::AffixMatcher(AffixPatternMatcher* prefix, AffixPatternMatcher* suffix, result_flags_t flags)
384         : fPrefix(prefix), fSuffix(suffix), fFlags(flags) {}
385 
match(StringSegment & segment,ParsedNumber & result,UErrorCode & status) const386 bool AffixMatcher::match(StringSegment& segment, ParsedNumber& result, UErrorCode& status) const {
387     if (!result.seenNumber()) {
388         // Prefix
389         // Do not match if:
390         // 1. We have already seen a prefix (result.prefix != null)
391         // 2. The prefix in this AffixMatcher is empty (prefix == null)
392         if (!result.prefix.isBogus() || fPrefix == nullptr) {
393             return false;
394         }
395 
396         // Attempt to match the prefix.
397         int initialOffset = segment.getOffset();
398         bool maybeMore = fPrefix->match(segment, result, status);
399         if (initialOffset != segment.getOffset()) {
400             result.prefix = fPrefix->getPattern();
401         }
402         return maybeMore;
403 
404     } else {
405         // Suffix
406         // Do not match if:
407         // 1. We have already seen a suffix (result.suffix != null)
408         // 2. The suffix in this AffixMatcher is empty (suffix == null)
409         // 3. The matched prefix does not equal this AffixMatcher's prefix
410         if (!result.suffix.isBogus() || fSuffix == nullptr || !matched(fPrefix, result.prefix)) {
411             return false;
412         }
413 
414         // Attempt to match the suffix.
415         int initialOffset = segment.getOffset();
416         bool maybeMore = fSuffix->match(segment, result, status);
417         if (initialOffset != segment.getOffset()) {
418             result.suffix = fSuffix->getPattern();
419         }
420         return maybeMore;
421     }
422 }
423 
smokeTest(const StringSegment & segment) const424 bool AffixMatcher::smokeTest(const StringSegment& segment) const {
425     return (fPrefix != nullptr && fPrefix->smokeTest(segment)) ||
426            (fSuffix != nullptr && fSuffix->smokeTest(segment));
427 }
428 
postProcess(ParsedNumber & result) const429 void AffixMatcher::postProcess(ParsedNumber& result) const {
430     // Check to see if our affix is the one that was matched. If so, set the flags in the result.
431     if (matched(fPrefix, result.prefix) && matched(fSuffix, result.suffix)) {
432         // Fill in the result prefix and suffix with non-null values (empty string).
433         // Used by strict mode to determine whether an entire affix pair was matched.
434         if (result.prefix.isBogus()) {
435             result.prefix = UnicodeString();
436         }
437         if (result.suffix.isBogus()) {
438             result.suffix = UnicodeString();
439         }
440         result.flags |= fFlags;
441         if (fPrefix != nullptr) {
442             fPrefix->postProcess(result);
443         }
444         if (fSuffix != nullptr) {
445             fSuffix->postProcess(result);
446         }
447     }
448 }
449 
compareTo(const AffixMatcher & rhs) const450 int8_t AffixMatcher::compareTo(const AffixMatcher& rhs) const {
451     const AffixMatcher& lhs = *this;
452     if (length(lhs.fPrefix) != length(rhs.fPrefix)) {
453         return length(lhs.fPrefix) > length(rhs.fPrefix) ? -1 : 1;
454     } else if (length(lhs.fSuffix) != length(rhs.fSuffix)) {
455         return length(lhs.fSuffix) > length(rhs.fSuffix) ? -1 : 1;
456     } else {
457         return 0;
458     }
459 }
460 
toString() const461 UnicodeString AffixMatcher::toString() const {
462     bool isNegative = 0 != (fFlags & FLAG_NEGATIVE);
463     return UnicodeString(u"<Affix") + (isNegative ? u":negative " : u" ") +
464            (fPrefix ? fPrefix->getPattern() : u"null") + u"#" +
465            (fSuffix ? fSuffix->getPattern() : u"null") + u">";
466 
467 }
468 
469 
470 #endif /* #if !UCONFIG_NO_FORMATTING */
471 
472 
473 
474 
475 
476 
477 
478 
479 
480 
481 
482 
483 
484 
485 
486 
487 
488 
489 
490 
491 
492 
493 
494 
495 
496