1 /*
2 **********************************************************************
3 * Copyright (C) 1999-2011, International Business Machines
4 * Corporation and others. All Rights Reserved.
5 **********************************************************************
6 * Date Name Description
7 * 11/17/99 aliu Creation.
8 **********************************************************************
9 */
10
11 #include "unicode/utypes.h"
12
13 #if !UCONFIG_NO_TRANSLITERATION
14
15 #include "unicode/uobject.h"
16 #include "unicode/parseerr.h"
17 #include "unicode/parsepos.h"
18 #include "unicode/putil.h"
19 #include "unicode/uchar.h"
20 #include "unicode/ustring.h"
21 #include "unicode/uniset.h"
22 #include "cstring.h"
23 #include "funcrepl.h"
24 #include "hash.h"
25 #include "quant.h"
26 #include "rbt.h"
27 #include "rbt_data.h"
28 #include "rbt_pars.h"
29 #include "rbt_rule.h"
30 #include "strmatch.h"
31 #include "strrepl.h"
32 #include "unicode/symtable.h"
33 #include "tridpars.h"
34 #include "uvector.h"
35 #include "hash.h"
36 #include "patternprops.h"
37 #include "util.h"
38 #include "cmemory.h"
39 #include "uprops.h"
40 #include "putilimp.h"
41
42 // Operators
43 #define VARIABLE_DEF_OP ((UChar)0x003D) /*=*/
44 #define FORWARD_RULE_OP ((UChar)0x003E) /*>*/
45 #define REVERSE_RULE_OP ((UChar)0x003C) /*<*/
46 #define FWDREV_RULE_OP ((UChar)0x007E) /*~*/ // internal rep of <> op
47
48 // Other special characters
49 #define QUOTE ((UChar)0x0027) /*'*/
50 #define ESCAPE ((UChar)0x005C) /*\*/
51 #define END_OF_RULE ((UChar)0x003B) /*;*/
52 #define RULE_COMMENT_CHAR ((UChar)0x0023) /*#*/
53
54 #define SEGMENT_OPEN ((UChar)0x0028) /*(*/
55 #define SEGMENT_CLOSE ((UChar)0x0029) /*)*/
56 #define CONTEXT_ANTE ((UChar)0x007B) /*{*/
57 #define CONTEXT_POST ((UChar)0x007D) /*}*/
58 #define CURSOR_POS ((UChar)0x007C) /*|*/
59 #define CURSOR_OFFSET ((UChar)0x0040) /*@*/
60 #define ANCHOR_START ((UChar)0x005E) /*^*/
61 #define KLEENE_STAR ((UChar)0x002A) /***/
62 #define ONE_OR_MORE ((UChar)0x002B) /*+*/
63 #define ZERO_OR_ONE ((UChar)0x003F) /*?*/
64
65 #define DOT ((UChar)46) /*.*/
66
67 static const UChar DOT_SET[] = { // "[^[:Zp:][:Zl:]\r\n$]";
68 91, 94, 91, 58, 90, 112, 58, 93, 91, 58, 90,
69 108, 58, 93, 92, 114, 92, 110, 36, 93, 0
70 };
71
72 // A function is denoted &Source-Target/Variant(text)
73 #define FUNCTION ((UChar)38) /*&*/
74
75 // Aliases for some of the syntax characters. These are provided so
76 // transliteration rules can be expressed in XML without clashing with
77 // XML syntax characters '<', '>', and '&'.
78 #define ALT_REVERSE_RULE_OP ((UChar)0x2190) // Left Arrow
79 #define ALT_FORWARD_RULE_OP ((UChar)0x2192) // Right Arrow
80 #define ALT_FWDREV_RULE_OP ((UChar)0x2194) // Left Right Arrow
81 #define ALT_FUNCTION ((UChar)0x2206) // Increment (~Greek Capital Delta)
82
83 // Special characters disallowed at the top level
84 static const UChar ILLEGAL_TOP[] = {41,0}; // ")"
85
86 // Special characters disallowed within a segment
87 static const UChar ILLEGAL_SEG[] = {123,125,124,64,0}; // "{}|@"
88
89 // Special characters disallowed within a function argument
90 static const UChar ILLEGAL_FUNC[] = {94,40,46,42,43,63,123,125,124,64,0}; // "^(.*+?{}|@"
91
92 // By definition, the ANCHOR_END special character is a
93 // trailing SymbolTable.SYMBOL_REF character.
94 // private static final char ANCHOR_END = '$';
95
96 static const UChar gOPERATORS[] = { // "=><"
97 VARIABLE_DEF_OP, FORWARD_RULE_OP, REVERSE_RULE_OP,
98 ALT_FORWARD_RULE_OP, ALT_REVERSE_RULE_OP, ALT_FWDREV_RULE_OP,
99 0
100 };
101
102 static const UChar HALF_ENDERS[] = { // "=><;"
103 VARIABLE_DEF_OP, FORWARD_RULE_OP, REVERSE_RULE_OP,
104 ALT_FORWARD_RULE_OP, ALT_REVERSE_RULE_OP, ALT_FWDREV_RULE_OP,
105 END_OF_RULE,
106 0
107 };
108
109 // These are also used in Transliterator::toRules()
110 static const int32_t ID_TOKEN_LEN = 2;
111 static const UChar ID_TOKEN[] = { 0x3A, 0x3A }; // ':', ':'
112
113 /*
114 commented out until we do real ::BEGIN/::END functionality
115 static const int32_t BEGIN_TOKEN_LEN = 5;
116 static const UChar BEGIN_TOKEN[] = { 0x42, 0x45, 0x47, 0x49, 0x4e }; // 'BEGIN'
117
118 static const int32_t END_TOKEN_LEN = 3;
119 static const UChar END_TOKEN[] = { 0x45, 0x4e, 0x44 }; // 'END'
120 */
121
122 U_NAMESPACE_BEGIN
123
124 //----------------------------------------------------------------------
125 // BEGIN ParseData
126 //----------------------------------------------------------------------
127
128 /**
129 * This class implements the SymbolTable interface. It is used
130 * during parsing to give UnicodeSet access to variables that
131 * have been defined so far. Note that it uses variablesVector,
132 * _not_ data.setVariables.
133 */
134 class ParseData : public UMemory, public SymbolTable {
135 public:
136 const TransliterationRuleData* data; // alias
137
138 const UVector* variablesVector; // alias
139
140 const Hashtable* variableNames; // alias
141
142 ParseData(const TransliterationRuleData* data = 0,
143 const UVector* variablesVector = 0,
144 const Hashtable* variableNames = 0);
145
146 virtual const UnicodeString* lookup(const UnicodeString& s) const;
147
148 virtual const UnicodeFunctor* lookupMatcher(UChar32 ch) const;
149
150 virtual UnicodeString parseReference(const UnicodeString& text,
151 ParsePosition& pos, int32_t limit) const;
152 /**
153 * Return true if the given character is a matcher standin or a plain
154 * character (non standin).
155 */
156 UBool isMatcher(UChar32 ch);
157
158 /**
159 * Return true if the given character is a replacer standin or a plain
160 * character (non standin).
161 */
162 UBool isReplacer(UChar32 ch);
163
164 private:
165 ParseData(const ParseData &other); // forbid copying of this class
166 ParseData &operator=(const ParseData &other); // forbid copying of this class
167 };
168
ParseData(const TransliterationRuleData * d,const UVector * sets,const Hashtable * vNames)169 ParseData::ParseData(const TransliterationRuleData* d,
170 const UVector* sets,
171 const Hashtable* vNames) :
172 data(d), variablesVector(sets), variableNames(vNames) {}
173
174 /**
175 * Implement SymbolTable API.
176 */
lookup(const UnicodeString & name) const177 const UnicodeString* ParseData::lookup(const UnicodeString& name) const {
178 return (const UnicodeString*) variableNames->get(name);
179 }
180
181 /**
182 * Implement SymbolTable API.
183 */
lookupMatcher(UChar32 ch) const184 const UnicodeFunctor* ParseData::lookupMatcher(UChar32 ch) const {
185 // Note that we cannot use data.lookupSet() because the
186 // set array has not been constructed yet.
187 const UnicodeFunctor* set = NULL;
188 int32_t i = ch - data->variablesBase;
189 if (i >= 0 && i < variablesVector->size()) {
190 int32_t i = ch - data->variablesBase;
191 set = (i < variablesVector->size()) ?
192 (UnicodeFunctor*) variablesVector->elementAt(i) : 0;
193 }
194 return set;
195 }
196
197 /**
198 * Implement SymbolTable API. Parse out a symbol reference
199 * name.
200 */
parseReference(const UnicodeString & text,ParsePosition & pos,int32_t limit) const201 UnicodeString ParseData::parseReference(const UnicodeString& text,
202 ParsePosition& pos, int32_t limit) const {
203 int32_t start = pos.getIndex();
204 int32_t i = start;
205 UnicodeString result;
206 while (i < limit) {
207 UChar c = text.charAt(i);
208 if ((i==start && !u_isIDStart(c)) || !u_isIDPart(c)) {
209 break;
210 }
211 ++i;
212 }
213 if (i == start) { // No valid name chars
214 return result; // Indicate failure with empty string
215 }
216 pos.setIndex(i);
217 text.extractBetween(start, i, result);
218 return result;
219 }
220
isMatcher(UChar32 ch)221 UBool ParseData::isMatcher(UChar32 ch) {
222 // Note that we cannot use data.lookup() because the
223 // set array has not been constructed yet.
224 int32_t i = ch - data->variablesBase;
225 if (i >= 0 && i < variablesVector->size()) {
226 UnicodeFunctor *f = (UnicodeFunctor*) variablesVector->elementAt(i);
227 return f != NULL && f->toMatcher() != NULL;
228 }
229 return TRUE;
230 }
231
232 /**
233 * Return true if the given character is a replacer standin or a plain
234 * character (non standin).
235 */
isReplacer(UChar32 ch)236 UBool ParseData::isReplacer(UChar32 ch) {
237 // Note that we cannot use data.lookup() because the
238 // set array has not been constructed yet.
239 int i = ch - data->variablesBase;
240 if (i >= 0 && i < variablesVector->size()) {
241 UnicodeFunctor *f = (UnicodeFunctor*) variablesVector->elementAt(i);
242 return f != NULL && f->toReplacer() != NULL;
243 }
244 return TRUE;
245 }
246
247 //----------------------------------------------------------------------
248 // BEGIN RuleHalf
249 //----------------------------------------------------------------------
250
251 /**
252 * A class representing one side of a rule. This class knows how to
253 * parse half of a rule. It is tightly coupled to the method
254 * RuleBasedTransliterator.Parser.parseRule().
255 */
256 class RuleHalf : public UMemory {
257
258 public:
259
260 UnicodeString text;
261
262 int32_t cursor; // position of cursor in text
263 int32_t ante; // position of ante context marker '{' in text
264 int32_t post; // position of post context marker '}' in text
265
266 // Record the offset to the cursor either to the left or to the
267 // right of the key. This is indicated by characters on the output
268 // side that allow the cursor to be positioned arbitrarily within
269 // the matching text. For example, abc{def} > | @@@ xyz; changes
270 // def to xyz and moves the cursor to before abc. Offset characters
271 // must be at the start or end, and they cannot move the cursor past
272 // the ante- or postcontext text. Placeholders are only valid in
273 // output text. The length of the ante and post context is
274 // determined at runtime, because of supplementals and quantifiers.
275 int32_t cursorOffset; // only nonzero on output side
276
277 // Position of first CURSOR_OFFSET on _right_. This will be -1
278 // for |@, -2 for |@@, etc., and 1 for @|, 2 for @@|, etc.
279 int32_t cursorOffsetPos;
280
281 UBool anchorStart;
282 UBool anchorEnd;
283
284 /**
285 * The segment number from 1..n of the next '(' we see
286 * during parsing; 1-based.
287 */
288 int32_t nextSegmentNumber;
289
290 TransliteratorParser& parser;
291
292 //--------------------------------------------------
293 // Methods
294
295 RuleHalf(TransliteratorParser& parser);
296 ~RuleHalf();
297
298 int32_t parse(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status);
299
300 int32_t parseSection(const UnicodeString& rule, int32_t pos, int32_t limit,
301 UnicodeString& buf,
302 const UnicodeString& illegal,
303 UBool isSegment,
304 UErrorCode& status);
305
306 /**
307 * Remove context.
308 */
309 void removeContext();
310
311 /**
312 * Return true if this half looks like valid output, that is, does not
313 * contain quantifiers or other special input-only elements.
314 */
315 UBool isValidOutput(TransliteratorParser& parser);
316
317 /**
318 * Return true if this half looks like valid input, that is, does not
319 * contain functions or other special output-only elements.
320 */
321 UBool isValidInput(TransliteratorParser& parser);
322
syntaxError(UErrorCode code,const UnicodeString & rule,int32_t start,UErrorCode & status)323 int syntaxError(UErrorCode code,
324 const UnicodeString& rule,
325 int32_t start,
326 UErrorCode& status) {
327 return parser.syntaxError(code, rule, start, status);
328 }
329
330 private:
331 // Disallowed methods; no impl.
332 RuleHalf(const RuleHalf&);
333 RuleHalf& operator=(const RuleHalf&);
334 };
335
RuleHalf(TransliteratorParser & p)336 RuleHalf::RuleHalf(TransliteratorParser& p) :
337 parser(p)
338 {
339 cursor = -1;
340 ante = -1;
341 post = -1;
342 cursorOffset = 0;
343 cursorOffsetPos = 0;
344 anchorStart = anchorEnd = FALSE;
345 nextSegmentNumber = 1;
346 }
347
~RuleHalf()348 RuleHalf::~RuleHalf() {
349 }
350
351 /**
352 * Parse one side of a rule, stopping at either the limit,
353 * the END_OF_RULE character, or an operator.
354 * @return the index after the terminating character, or
355 * if limit was reached, limit
356 */
parse(const UnicodeString & rule,int32_t pos,int32_t limit,UErrorCode & status)357 int32_t RuleHalf::parse(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status) {
358 int32_t start = pos;
359 text.truncate(0);
360 pos = parseSection(rule, pos, limit, text, ILLEGAL_TOP, FALSE, status);
361
362 if (cursorOffset > 0 && cursor != cursorOffsetPos) {
363 return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status);
364 }
365
366 return pos;
367 }
368
369 /**
370 * Parse a section of one side of a rule, stopping at either
371 * the limit, the END_OF_RULE character, an operator, or a
372 * segment close character. This method parses both a
373 * top-level rule half and a segment within such a rule half.
374 * It calls itself recursively to parse segments and nested
375 * segments.
376 * @param buf buffer into which to accumulate the rule pattern
377 * characters, either literal characters from the rule or
378 * standins for UnicodeMatcher objects including segments.
379 * @param illegal the set of special characters that is illegal during
380 * this parse.
381 * @param isSegment if true, then we've already seen a '(' and
382 * pos on entry points right after it. Accumulate everything
383 * up to the closing ')', put it in a segment matcher object,
384 * generate a standin for it, and add the standin to buf. As
385 * a side effect, update the segments vector with a reference
386 * to the segment matcher. This works recursively for nested
387 * segments. If isSegment is false, just accumulate
388 * characters into buf.
389 * @return the index after the terminating character, or
390 * if limit was reached, limit
391 */
parseSection(const UnicodeString & rule,int32_t pos,int32_t limit,UnicodeString & buf,const UnicodeString & illegal,UBool isSegment,UErrorCode & status)392 int32_t RuleHalf::parseSection(const UnicodeString& rule, int32_t pos, int32_t limit,
393 UnicodeString& buf,
394 const UnicodeString& illegal,
395 UBool isSegment, UErrorCode& status) {
396 int32_t start = pos;
397 ParsePosition pp;
398 UnicodeString scratch;
399 UBool done = FALSE;
400 int32_t quoteStart = -1; // Most recent 'single quoted string'
401 int32_t quoteLimit = -1;
402 int32_t varStart = -1; // Most recent $variableReference
403 int32_t varLimit = -1;
404 int32_t bufStart = buf.length();
405
406 while (pos < limit && !done) {
407 // Since all syntax characters are in the BMP, fetching
408 // 16-bit code units suffices here.
409 UChar c = rule.charAt(pos++);
410 if (PatternProps::isWhiteSpace(c)) {
411 // Ignore whitespace. Note that this is not Unicode
412 // spaces, but Java spaces -- a subset, representing
413 // whitespace likely to be seen in code.
414 continue;
415 }
416 if (u_strchr(HALF_ENDERS, c) != NULL) {
417 if (isSegment) {
418 // Unclosed segment
419 return syntaxError(U_UNCLOSED_SEGMENT, rule, start, status);
420 }
421 break;
422 }
423 if (anchorEnd) {
424 // Text after a presumed end anchor is a syntax err
425 return syntaxError(U_MALFORMED_VARIABLE_REFERENCE, rule, start, status);
426 }
427 if (UnicodeSet::resemblesPattern(rule, pos-1)) {
428 pp.setIndex(pos-1); // Backup to opening '['
429 buf.append(parser.parseSet(rule, pp, status));
430 if (U_FAILURE(status)) {
431 return syntaxError(U_MALFORMED_SET, rule, start, status);
432 }
433 pos = pp.getIndex();
434 continue;
435 }
436 // Handle escapes
437 if (c == ESCAPE) {
438 if (pos == limit) {
439 return syntaxError(U_TRAILING_BACKSLASH, rule, start, status);
440 }
441 UChar32 escaped = rule.unescapeAt(pos); // pos is already past '\\'
442 if (escaped == (UChar32) -1) {
443 return syntaxError(U_MALFORMED_UNICODE_ESCAPE, rule, start, status);
444 }
445 if (!parser.checkVariableRange(escaped)) {
446 return syntaxError(U_VARIABLE_RANGE_OVERLAP, rule, start, status);
447 }
448 buf.append(escaped);
449 continue;
450 }
451 // Handle quoted matter
452 if (c == QUOTE) {
453 int32_t iq = rule.indexOf(QUOTE, pos);
454 if (iq == pos) {
455 buf.append(c); // Parse [''] outside quotes as [']
456 ++pos;
457 } else {
458 /* This loop picks up a run of quoted text of the
459 * form 'aaaa' each time through. If this run
460 * hasn't really ended ('aaaa''bbbb') then it keeps
461 * looping, each time adding on a new run. When it
462 * reaches the final quote it breaks.
463 */
464 quoteStart = buf.length();
465 for (;;) {
466 if (iq < 0) {
467 return syntaxError(U_UNTERMINATED_QUOTE, rule, start, status);
468 }
469 scratch.truncate(0);
470 rule.extractBetween(pos, iq, scratch);
471 buf.append(scratch);
472 pos = iq+1;
473 if (pos < limit && rule.charAt(pos) == QUOTE) {
474 // Parse [''] inside quotes as [']
475 iq = rule.indexOf(QUOTE, pos+1);
476 // Continue looping
477 } else {
478 break;
479 }
480 }
481 quoteLimit = buf.length();
482
483 for (iq=quoteStart; iq<quoteLimit; ++iq) {
484 if (!parser.checkVariableRange(buf.charAt(iq))) {
485 return syntaxError(U_VARIABLE_RANGE_OVERLAP, rule, start, status);
486 }
487 }
488 }
489 continue;
490 }
491
492 if (!parser.checkVariableRange(c)) {
493 return syntaxError(U_VARIABLE_RANGE_OVERLAP, rule, start, status);
494 }
495
496 if (illegal.indexOf(c) >= 0) {
497 syntaxError(U_ILLEGAL_CHARACTER, rule, start, status);
498 }
499
500 switch (c) {
501
502 //------------------------------------------------------
503 // Elements allowed within and out of segments
504 //------------------------------------------------------
505 case ANCHOR_START:
506 if (buf.length() == 0 && !anchorStart) {
507 anchorStart = TRUE;
508 } else {
509 return syntaxError(U_MISPLACED_ANCHOR_START,
510 rule, start, status);
511 }
512 break;
513 case SEGMENT_OPEN:
514 {
515 // bufSegStart is the offset in buf to the first
516 // character of the segment we are parsing.
517 int32_t bufSegStart = buf.length();
518
519 // Record segment number now, since nextSegmentNumber
520 // will be incremented during the call to parseSection
521 // if there are nested segments.
522 int32_t segmentNumber = nextSegmentNumber++; // 1-based
523
524 // Parse the segment
525 pos = parseSection(rule, pos, limit, buf, ILLEGAL_SEG, TRUE, status);
526
527 // After parsing a segment, the relevant characters are
528 // in buf, starting at offset bufSegStart. Extract them
529 // into a string matcher, and replace them with a
530 // standin for that matcher.
531 StringMatcher* m =
532 new StringMatcher(buf, bufSegStart, buf.length(),
533 segmentNumber, *parser.curData);
534 if (m == NULL) {
535 return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status);
536 }
537
538 // Record and associate object and segment number
539 parser.setSegmentObject(segmentNumber, m, status);
540 buf.truncate(bufSegStart);
541 buf.append(parser.getSegmentStandin(segmentNumber, status));
542 }
543 break;
544 case FUNCTION:
545 case ALT_FUNCTION:
546 {
547 int32_t iref = pos;
548 TransliteratorIDParser::SingleID* single =
549 TransliteratorIDParser::parseFilterID(rule, iref);
550 // The next character MUST be a segment open
551 if (single == NULL ||
552 !ICU_Utility::parseChar(rule, iref, SEGMENT_OPEN)) {
553 return syntaxError(U_INVALID_FUNCTION, rule, start, status);
554 }
555
556 Transliterator *t = single->createInstance();
557 delete single;
558 if (t == NULL) {
559 return syntaxError(U_INVALID_FUNCTION, rule, start, status);
560 }
561
562 // bufSegStart is the offset in buf to the first
563 // character of the segment we are parsing.
564 int32_t bufSegStart = buf.length();
565
566 // Parse the segment
567 pos = parseSection(rule, iref, limit, buf, ILLEGAL_FUNC, TRUE, status);
568
569 // After parsing a segment, the relevant characters are
570 // in buf, starting at offset bufSegStart.
571 UnicodeString output;
572 buf.extractBetween(bufSegStart, buf.length(), output);
573 FunctionReplacer *r =
574 new FunctionReplacer(t, new StringReplacer(output, parser.curData));
575 if (r == NULL) {
576 return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status);
577 }
578
579 // Replace the buffer contents with a stand-in
580 buf.truncate(bufSegStart);
581 buf.append(parser.generateStandInFor(r, status));
582 }
583 break;
584 case SymbolTable::SYMBOL_REF:
585 // Handle variable references and segment references "$1" .. "$9"
586 {
587 // A variable reference must be followed immediately
588 // by a Unicode identifier start and zero or more
589 // Unicode identifier part characters, or by a digit
590 // 1..9 if it is a segment reference.
591 if (pos == limit) {
592 // A variable ref character at the end acts as
593 // an anchor to the context limit, as in perl.
594 anchorEnd = TRUE;
595 break;
596 }
597 // Parse "$1" "$2" .. "$9" .. (no upper limit)
598 c = rule.charAt(pos);
599 int32_t r = u_digit(c, 10);
600 if (r >= 1 && r <= 9) {
601 r = ICU_Utility::parseNumber(rule, pos, 10);
602 if (r < 0) {
603 return syntaxError(U_UNDEFINED_SEGMENT_REFERENCE,
604 rule, start, status);
605 }
606 buf.append(parser.getSegmentStandin(r, status));
607 } else {
608 pp.setIndex(pos);
609 UnicodeString name = parser.parseData->
610 parseReference(rule, pp, limit);
611 if (name.length() == 0) {
612 // This means the '$' was not followed by a
613 // valid name. Try to interpret it as an
614 // end anchor then. If this also doesn't work
615 // (if we see a following character) then signal
616 // an error.
617 anchorEnd = TRUE;
618 break;
619 }
620 pos = pp.getIndex();
621 // If this is a variable definition statement,
622 // then the LHS variable will be undefined. In
623 // that case appendVariableDef() will append the
624 // special placeholder char variableLimit-1.
625 varStart = buf.length();
626 parser.appendVariableDef(name, buf, status);
627 varLimit = buf.length();
628 }
629 }
630 break;
631 case DOT:
632 buf.append(parser.getDotStandIn(status));
633 break;
634 case KLEENE_STAR:
635 case ONE_OR_MORE:
636 case ZERO_OR_ONE:
637 // Quantifiers. We handle single characters, quoted strings,
638 // variable references, and segments.
639 // a+ matches aaa
640 // 'foo'+ matches foofoofoo
641 // $v+ matches xyxyxy if $v == xy
642 // (seg)+ matches segsegseg
643 {
644 if (isSegment && buf.length() == bufStart) {
645 // The */+ immediately follows '('
646 return syntaxError(U_MISPLACED_QUANTIFIER, rule, start, status);
647 }
648
649 int32_t qstart, qlimit;
650 // The */+ follows an isolated character or quote
651 // or variable reference
652 if (buf.length() == quoteLimit) {
653 // The */+ follows a 'quoted string'
654 qstart = quoteStart;
655 qlimit = quoteLimit;
656 } else if (buf.length() == varLimit) {
657 // The */+ follows a $variableReference
658 qstart = varStart;
659 qlimit = varLimit;
660 } else {
661 // The */+ follows a single character, possibly
662 // a segment standin
663 qstart = buf.length() - 1;
664 qlimit = qstart + 1;
665 }
666
667 UnicodeFunctor *m =
668 new StringMatcher(buf, qstart, qlimit, 0, *parser.curData);
669 if (m == NULL) {
670 return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status);
671 }
672 int32_t min = 0;
673 int32_t max = Quantifier::MAX;
674 switch (c) {
675 case ONE_OR_MORE:
676 min = 1;
677 break;
678 case ZERO_OR_ONE:
679 min = 0;
680 max = 1;
681 break;
682 // case KLEENE_STAR:
683 // do nothing -- min, max already set
684 }
685 m = new Quantifier(m, min, max);
686 if (m == NULL) {
687 return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status);
688 }
689 buf.truncate(qstart);
690 buf.append(parser.generateStandInFor(m, status));
691 }
692 break;
693
694 //------------------------------------------------------
695 // Elements allowed ONLY WITHIN segments
696 //------------------------------------------------------
697 case SEGMENT_CLOSE:
698 // assert(isSegment);
699 // We're done parsing a segment.
700 done = TRUE;
701 break;
702
703 //------------------------------------------------------
704 // Elements allowed ONLY OUTSIDE segments
705 //------------------------------------------------------
706 case CONTEXT_ANTE:
707 if (ante >= 0) {
708 return syntaxError(U_MULTIPLE_ANTE_CONTEXTS, rule, start, status);
709 }
710 ante = buf.length();
711 break;
712 case CONTEXT_POST:
713 if (post >= 0) {
714 return syntaxError(U_MULTIPLE_POST_CONTEXTS, rule, start, status);
715 }
716 post = buf.length();
717 break;
718 case CURSOR_POS:
719 if (cursor >= 0) {
720 return syntaxError(U_MULTIPLE_CURSORS, rule, start, status);
721 }
722 cursor = buf.length();
723 break;
724 case CURSOR_OFFSET:
725 if (cursorOffset < 0) {
726 if (buf.length() > 0) {
727 return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status);
728 }
729 --cursorOffset;
730 } else if (cursorOffset > 0) {
731 if (buf.length() != cursorOffsetPos || cursor >= 0) {
732 return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status);
733 }
734 ++cursorOffset;
735 } else {
736 if (cursor == 0 && buf.length() == 0) {
737 cursorOffset = -1;
738 } else if (cursor < 0) {
739 cursorOffsetPos = buf.length();
740 cursorOffset = 1;
741 } else {
742 return syntaxError(U_MISPLACED_CURSOR_OFFSET, rule, start, status);
743 }
744 }
745 break;
746
747
748 //------------------------------------------------------
749 // Non-special characters
750 //------------------------------------------------------
751 default:
752 // Disallow unquoted characters other than [0-9A-Za-z]
753 // in the printable ASCII range. These characters are
754 // reserved for possible future use.
755 if (c >= 0x0021 && c <= 0x007E &&
756 !((c >= 0x0030/*'0'*/ && c <= 0x0039/*'9'*/) ||
757 (c >= 0x0041/*'A'*/ && c <= 0x005A/*'Z'*/) ||
758 (c >= 0x0061/*'a'*/ && c <= 0x007A/*'z'*/))) {
759 return syntaxError(U_UNQUOTED_SPECIAL, rule, start, status);
760 }
761 buf.append(c);
762 break;
763 }
764 }
765
766 return pos;
767 }
768
769 /**
770 * Remove context.
771 */
removeContext()772 void RuleHalf::removeContext() {
773 //text = text.substring(ante < 0 ? 0 : ante,
774 // post < 0 ? text.length() : post);
775 if (post >= 0) {
776 text.remove(post);
777 }
778 if (ante >= 0) {
779 text.removeBetween(0, ante);
780 }
781 ante = post = -1;
782 anchorStart = anchorEnd = FALSE;
783 }
784
785 /**
786 * Return true if this half looks like valid output, that is, does not
787 * contain quantifiers or other special input-only elements.
788 */
isValidOutput(TransliteratorParser & transParser)789 UBool RuleHalf::isValidOutput(TransliteratorParser& transParser) {
790 for (int32_t i=0; i<text.length(); ) {
791 UChar32 c = text.char32At(i);
792 i += UTF_CHAR_LENGTH(c);
793 if (!transParser.parseData->isReplacer(c)) {
794 return FALSE;
795 }
796 }
797 return TRUE;
798 }
799
800 /**
801 * Return true if this half looks like valid input, that is, does not
802 * contain functions or other special output-only elements.
803 */
isValidInput(TransliteratorParser & transParser)804 UBool RuleHalf::isValidInput(TransliteratorParser& transParser) {
805 for (int32_t i=0; i<text.length(); ) {
806 UChar32 c = text.char32At(i);
807 i += UTF_CHAR_LENGTH(c);
808 if (!transParser.parseData->isMatcher(c)) {
809 return FALSE;
810 }
811 }
812 return TRUE;
813 }
814
815 //----------------------------------------------------------------------
816 // PUBLIC API
817 //----------------------------------------------------------------------
818
819 /**
820 * Constructor.
821 */
TransliteratorParser(UErrorCode & statusReturn)822 TransliteratorParser::TransliteratorParser(UErrorCode &statusReturn) :
823 dataVector(statusReturn),
824 idBlockVector(statusReturn),
825 variablesVector(statusReturn),
826 segmentObjects(statusReturn)
827 {
828 idBlockVector.setDeleter(uhash_deleteUnicodeString);
829 curData = NULL;
830 compoundFilter = NULL;
831 parseData = NULL;
832 variableNames.setValueDeleter(uhash_deleteUnicodeString);
833 }
834
835 /**
836 * Destructor.
837 */
~TransliteratorParser()838 TransliteratorParser::~TransliteratorParser() {
839 while (!dataVector.isEmpty())
840 delete (TransliterationRuleData*)(dataVector.orphanElementAt(0));
841 delete compoundFilter;
842 delete parseData;
843 while (!variablesVector.isEmpty())
844 delete (UnicodeFunctor*)variablesVector.orphanElementAt(0);
845 }
846
847 void
parse(const UnicodeString & rules,UTransDirection transDirection,UParseError & pe,UErrorCode & ec)848 TransliteratorParser::parse(const UnicodeString& rules,
849 UTransDirection transDirection,
850 UParseError& pe,
851 UErrorCode& ec) {
852 if (U_SUCCESS(ec)) {
853 parseRules(rules, transDirection, ec);
854 pe = parseError;
855 }
856 }
857
858 /**
859 * Return the compound filter parsed by parse(). Caller owns result.
860 */
orphanCompoundFilter()861 UnicodeSet* TransliteratorParser::orphanCompoundFilter() {
862 UnicodeSet* f = compoundFilter;
863 compoundFilter = NULL;
864 return f;
865 }
866
867 //----------------------------------------------------------------------
868 // Private implementation
869 //----------------------------------------------------------------------
870
871 /**
872 * Parse the given string as a sequence of rules, separated by newline
873 * characters ('\n'), and cause this object to implement those rules. Any
874 * previous rules are discarded. Typically this method is called exactly
875 * once, during construction.
876 * @exception IllegalArgumentException if there is a syntax error in the
877 * rules
878 */
parseRules(const UnicodeString & rule,UTransDirection theDirection,UErrorCode & status)879 void TransliteratorParser::parseRules(const UnicodeString& rule,
880 UTransDirection theDirection,
881 UErrorCode& status)
882 {
883 // Clear error struct
884 uprv_memset(&parseError, 0, sizeof(parseError));
885 parseError.line = parseError.offset = -1;
886
887 UBool parsingIDs = TRUE;
888 int32_t ruleCount = 0;
889
890 while (!dataVector.isEmpty()) {
891 delete (TransliterationRuleData*)(dataVector.orphanElementAt(0));
892 }
893 if (U_FAILURE(status)) {
894 return;
895 }
896
897 idBlockVector.removeAllElements();
898 curData = NULL;
899 direction = theDirection;
900 ruleCount = 0;
901
902 delete compoundFilter;
903 compoundFilter = NULL;
904
905 while (!variablesVector.isEmpty()) {
906 delete (UnicodeFunctor*)variablesVector.orphanElementAt(0);
907 }
908 variableNames.removeAll();
909 parseData = new ParseData(0, &variablesVector, &variableNames);
910 if (parseData == NULL) {
911 status = U_MEMORY_ALLOCATION_ERROR;
912 return;
913 }
914
915 dotStandIn = (UChar) -1;
916
917 UnicodeString *tempstr = NULL; // used for memory allocation error checking
918 UnicodeString str; // scratch
919 UnicodeString idBlockResult;
920 int32_t pos = 0;
921 int32_t limit = rule.length();
922
923 // The compound filter offset is an index into idBlockResult.
924 // If it is 0, then the compound filter occurred at the start,
925 // and it is the offset to the _start_ of the compound filter
926 // pattern. Otherwise it is the offset to the _limit_ of the
927 // compound filter pattern within idBlockResult.
928 compoundFilter = NULL;
929 int32_t compoundFilterOffset = -1;
930
931 while (pos < limit && U_SUCCESS(status)) {
932 UChar c = rule.charAt(pos++);
933 if (PatternProps::isWhiteSpace(c)) {
934 // Ignore leading whitespace.
935 continue;
936 }
937 // Skip lines starting with the comment character
938 if (c == RULE_COMMENT_CHAR) {
939 pos = rule.indexOf((UChar)0x000A /*\n*/, pos) + 1;
940 if (pos == 0) {
941 break; // No "\n" found; rest of rule is a commnet
942 }
943 continue; // Either fall out or restart with next line
944 }
945
946 // skip empty rules
947 if (c == END_OF_RULE)
948 continue;
949
950 // keep track of how many rules we've seen
951 ++ruleCount;
952
953 // We've found the start of a rule or ID. c is its first
954 // character, and pos points past c.
955 --pos;
956 // Look for an ID token. Must have at least ID_TOKEN_LEN + 1
957 // chars left.
958 if ((pos + ID_TOKEN_LEN + 1) <= limit &&
959 rule.compare(pos, ID_TOKEN_LEN, ID_TOKEN) == 0) {
960 pos += ID_TOKEN_LEN;
961 c = rule.charAt(pos);
962 while (PatternProps::isWhiteSpace(c) && pos < limit) {
963 ++pos;
964 c = rule.charAt(pos);
965 }
966
967 int32_t p = pos;
968
969 if (!parsingIDs) {
970 if (curData != NULL) {
971 if (direction == UTRANS_FORWARD)
972 dataVector.addElement(curData, status);
973 else
974 dataVector.insertElementAt(curData, 0, status);
975 curData = NULL;
976 }
977 parsingIDs = TRUE;
978 }
979
980 TransliteratorIDParser::SingleID* id =
981 TransliteratorIDParser::parseSingleID(rule, p, direction, status);
982 if (p != pos && ICU_Utility::parseChar(rule, p, END_OF_RULE)) {
983 // Successful ::ID parse.
984
985 if (direction == UTRANS_FORWARD) {
986 idBlockResult.append(id->canonID).append(END_OF_RULE);
987 } else {
988 idBlockResult.insert(0, END_OF_RULE);
989 idBlockResult.insert(0, id->canonID);
990 }
991
992 } else {
993 // Couldn't parse an ID. Try to parse a global filter
994 int32_t withParens = -1;
995 UnicodeSet* f = TransliteratorIDParser::parseGlobalFilter(rule, p, direction, withParens, NULL);
996 if (f != NULL) {
997 if (ICU_Utility::parseChar(rule, p, END_OF_RULE)
998 && (direction == UTRANS_FORWARD) == (withParens == 0))
999 {
1000 if (compoundFilter != NULL) {
1001 // Multiple compound filters
1002 syntaxError(U_MULTIPLE_COMPOUND_FILTERS, rule, pos, status);
1003 delete f;
1004 } else {
1005 compoundFilter = f;
1006 compoundFilterOffset = ruleCount;
1007 }
1008 } else {
1009 delete f;
1010 }
1011 } else {
1012 // Invalid ::id
1013 // Can be parsed as neither an ID nor a global filter
1014 syntaxError(U_INVALID_ID, rule, pos, status);
1015 }
1016 }
1017 delete id;
1018 pos = p;
1019 } else {
1020 if (parsingIDs) {
1021 tempstr = new UnicodeString(idBlockResult);
1022 // NULL pointer check
1023 if (tempstr == NULL) {
1024 status = U_MEMORY_ALLOCATION_ERROR;
1025 return;
1026 }
1027 if (direction == UTRANS_FORWARD)
1028 idBlockVector.addElement(tempstr, status);
1029 else
1030 idBlockVector.insertElementAt(tempstr, 0, status);
1031 idBlockResult.remove();
1032 parsingIDs = FALSE;
1033 curData = new TransliterationRuleData(status);
1034 // NULL pointer check
1035 if (curData == NULL) {
1036 status = U_MEMORY_ALLOCATION_ERROR;
1037 return;
1038 }
1039 parseData->data = curData;
1040
1041 // By default, rules use part of the private use area
1042 // E000..F8FF for variables and other stand-ins. Currently
1043 // the range F000..F8FF is typically sufficient. The 'use
1044 // variable range' pragma allows rule sets to modify this.
1045 setVariableRange(0xF000, 0xF8FF, status);
1046 }
1047
1048 if (resemblesPragma(rule, pos, limit)) {
1049 int32_t ppp = parsePragma(rule, pos, limit, status);
1050 if (ppp < 0) {
1051 syntaxError(U_MALFORMED_PRAGMA, rule, pos, status);
1052 }
1053 pos = ppp;
1054 // Parse a rule
1055 } else {
1056 pos = parseRule(rule, pos, limit, status);
1057 }
1058 }
1059 }
1060
1061 if (parsingIDs && idBlockResult.length() > 0) {
1062 tempstr = new UnicodeString(idBlockResult);
1063 // NULL pointer check
1064 if (tempstr == NULL) {
1065 status = U_MEMORY_ALLOCATION_ERROR;
1066 return;
1067 }
1068 if (direction == UTRANS_FORWARD)
1069 idBlockVector.addElement(tempstr, status);
1070 else
1071 idBlockVector.insertElementAt(tempstr, 0, status);
1072 }
1073 else if (!parsingIDs && curData != NULL) {
1074 if (direction == UTRANS_FORWARD)
1075 dataVector.addElement(curData, status);
1076 else
1077 dataVector.insertElementAt(curData, 0, status);
1078 }
1079
1080 if (U_SUCCESS(status)) {
1081 // Convert the set vector to an array
1082 int32_t i, dataVectorSize = dataVector.size();
1083 for (i = 0; i < dataVectorSize; i++) {
1084 TransliterationRuleData* data = (TransliterationRuleData*)dataVector.elementAt(i);
1085 data->variablesLength = variablesVector.size();
1086 if (data->variablesLength == 0) {
1087 data->variables = 0;
1088 } else {
1089 data->variables = (UnicodeFunctor**)uprv_malloc(data->variablesLength * sizeof(UnicodeFunctor*));
1090 // NULL pointer check
1091 if (data->variables == NULL) {
1092 status = U_MEMORY_ALLOCATION_ERROR;
1093 return;
1094 }
1095 data->variablesAreOwned = (i == 0);
1096 }
1097
1098 for (int32_t j = 0; j < data->variablesLength; j++) {
1099 data->variables[j] =
1100 ((UnicodeSet*)variablesVector.elementAt(j));
1101 }
1102
1103 data->variableNames.removeAll();
1104 int32_t pos = -1;
1105 const UHashElement* he = variableNames.nextElement(pos);
1106 while (he != NULL) {
1107 UnicodeString* tempus = (UnicodeString*)(((UnicodeString*)(he->value.pointer))->clone());
1108 if (tempus == NULL) {
1109 status = U_MEMORY_ALLOCATION_ERROR;
1110 return;
1111 }
1112 data->variableNames.put(*((UnicodeString*)(he->key.pointer)),
1113 tempus, status);
1114 he = variableNames.nextElement(pos);
1115 }
1116 }
1117 variablesVector.removeAllElements(); // keeps them from getting deleted when we succeed
1118
1119 // Index the rules
1120 if (compoundFilter != NULL) {
1121 if ((direction == UTRANS_FORWARD && compoundFilterOffset != 1) ||
1122 (direction == UTRANS_REVERSE && compoundFilterOffset != ruleCount)) {
1123 status = U_MISPLACED_COMPOUND_FILTER;
1124 }
1125 }
1126
1127 for (i = 0; i < dataVectorSize; i++) {
1128 TransliterationRuleData* data = (TransliterationRuleData*)dataVector.elementAt(i);
1129 data->ruleSet.freeze(parseError, status);
1130 }
1131 if (idBlockVector.size() == 1 && ((UnicodeString*)idBlockVector.elementAt(0))->isEmpty()) {
1132 idBlockVector.removeElementAt(0);
1133 }
1134 }
1135 }
1136
1137 /**
1138 * Set the variable range to [start, end] (inclusive).
1139 */
setVariableRange(int32_t start,int32_t end,UErrorCode & status)1140 void TransliteratorParser::setVariableRange(int32_t start, int32_t end, UErrorCode& status) {
1141 if (start > end || start < 0 || end > 0xFFFF) {
1142 status = U_MALFORMED_PRAGMA;
1143 return;
1144 }
1145
1146 curData->variablesBase = (UChar) start;
1147 if (dataVector.size() == 0) {
1148 variableNext = (UChar) start;
1149 variableLimit = (UChar) (end + 1);
1150 }
1151 }
1152
1153 /**
1154 * Assert that the given character is NOT within the variable range.
1155 * If it is, return FALSE. This is neccesary to ensure that the
1156 * variable range does not overlap characters used in a rule.
1157 */
checkVariableRange(UChar32 ch) const1158 UBool TransliteratorParser::checkVariableRange(UChar32 ch) const {
1159 return !(ch >= curData->variablesBase && ch < variableLimit);
1160 }
1161
1162 /**
1163 * Set the maximum backup to 'backup', in response to a pragma
1164 * statement.
1165 */
pragmaMaximumBackup(int32_t)1166 void TransliteratorParser::pragmaMaximumBackup(int32_t /*backup*/) {
1167 //TODO Finish
1168 }
1169
1170 /**
1171 * Begin normalizing all rules using the given mode, in response
1172 * to a pragma statement.
1173 */
pragmaNormalizeRules(UNormalizationMode)1174 void TransliteratorParser::pragmaNormalizeRules(UNormalizationMode /*mode*/) {
1175 //TODO Finish
1176 }
1177
1178 static const UChar PRAGMA_USE[] = {0x75,0x73,0x65,0x20,0}; // "use "
1179
1180 static const UChar PRAGMA_VARIABLE_RANGE[] = {0x7E,0x76,0x61,0x72,0x69,0x61,0x62,0x6C,0x65,0x20,0x72,0x61,0x6E,0x67,0x65,0x20,0x23,0x20,0x23,0x7E,0x3B,0}; // "~variable range # #~;"
1181
1182 static const UChar PRAGMA_MAXIMUM_BACKUP[] = {0x7E,0x6D,0x61,0x78,0x69,0x6D,0x75,0x6D,0x20,0x62,0x61,0x63,0x6B,0x75,0x70,0x20,0x23,0x7E,0x3B,0}; // "~maximum backup #~;"
1183
1184 static const UChar PRAGMA_NFD_RULES[] = {0x7E,0x6E,0x66,0x64,0x20,0x72,0x75,0x6C,0x65,0x73,0x7E,0x3B,0}; // "~nfd rules~;"
1185
1186 static const UChar PRAGMA_NFC_RULES[] = {0x7E,0x6E,0x66,0x63,0x20,0x72,0x75,0x6C,0x65,0x73,0x7E,0x3B,0}; // "~nfc rules~;"
1187
1188 /**
1189 * Return true if the given rule looks like a pragma.
1190 * @param pos offset to the first non-whitespace character
1191 * of the rule.
1192 * @param limit pointer past the last character of the rule.
1193 */
resemblesPragma(const UnicodeString & rule,int32_t pos,int32_t limit)1194 UBool TransliteratorParser::resemblesPragma(const UnicodeString& rule, int32_t pos, int32_t limit) {
1195 // Must start with /use\s/i
1196 return ICU_Utility::parsePattern(rule, pos, limit, PRAGMA_USE, NULL) >= 0;
1197 }
1198
1199 /**
1200 * Parse a pragma. This method assumes resemblesPragma() has
1201 * already returned true.
1202 * @param pos offset to the first non-whitespace character
1203 * of the rule.
1204 * @param limit pointer past the last character of the rule.
1205 * @return the position index after the final ';' of the pragma,
1206 * or -1 on failure.
1207 */
parsePragma(const UnicodeString & rule,int32_t pos,int32_t limit,UErrorCode & status)1208 int32_t TransliteratorParser::parsePragma(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status) {
1209 int32_t array[2];
1210
1211 // resemblesPragma() has already returned true, so we
1212 // know that pos points to /use\s/i; we can skip 4 characters
1213 // immediately
1214 pos += 4;
1215
1216 // Here are the pragmas we recognize:
1217 // use variable range 0xE000 0xEFFF;
1218 // use maximum backup 16;
1219 // use nfd rules;
1220 // use nfc rules;
1221 int p = ICU_Utility::parsePattern(rule, pos, limit, PRAGMA_VARIABLE_RANGE, array);
1222 if (p >= 0) {
1223 setVariableRange(array[0], array[1], status);
1224 return p;
1225 }
1226
1227 p = ICU_Utility::parsePattern(rule, pos, limit, PRAGMA_MAXIMUM_BACKUP, array);
1228 if (p >= 0) {
1229 pragmaMaximumBackup(array[0]);
1230 return p;
1231 }
1232
1233 p = ICU_Utility::parsePattern(rule, pos, limit, PRAGMA_NFD_RULES, NULL);
1234 if (p >= 0) {
1235 pragmaNormalizeRules(UNORM_NFD);
1236 return p;
1237 }
1238
1239 p = ICU_Utility::parsePattern(rule, pos, limit, PRAGMA_NFC_RULES, NULL);
1240 if (p >= 0) {
1241 pragmaNormalizeRules(UNORM_NFC);
1242 return p;
1243 }
1244
1245 // Syntax error: unable to parse pragma
1246 return -1;
1247 }
1248
1249 /**
1250 * MAIN PARSER. Parse the next rule in the given rule string, starting
1251 * at pos. Return the index after the last character parsed. Do not
1252 * parse characters at or after limit.
1253 *
1254 * Important: The character at pos must be a non-whitespace character
1255 * that is not the comment character.
1256 *
1257 * This method handles quoting, escaping, and whitespace removal. It
1258 * parses the end-of-rule character. It recognizes context and cursor
1259 * indicators. Once it does a lexical breakdown of the rule at pos, it
1260 * creates a rule object and adds it to our rule list.
1261 */
parseRule(const UnicodeString & rule,int32_t pos,int32_t limit,UErrorCode & status)1262 int32_t TransliteratorParser::parseRule(const UnicodeString& rule, int32_t pos, int32_t limit, UErrorCode& status) {
1263 // Locate the left side, operator, and right side
1264 int32_t start = pos;
1265 UChar op = 0;
1266 int32_t i;
1267
1268 // Set up segments data
1269 segmentStandins.truncate(0);
1270 segmentObjects.removeAllElements();
1271
1272 // Use pointers to automatics to make swapping possible.
1273 RuleHalf _left(*this), _right(*this);
1274 RuleHalf* left = &_left;
1275 RuleHalf* right = &_right;
1276
1277 undefinedVariableName.remove();
1278 pos = left->parse(rule, pos, limit, status);
1279 if (U_FAILURE(status)) {
1280 return start;
1281 }
1282
1283 if (pos == limit || u_strchr(gOPERATORS, (op = rule.charAt(--pos))) == NULL) {
1284 return syntaxError(U_MISSING_OPERATOR, rule, start, status);
1285 }
1286 ++pos;
1287
1288 // Found an operator char. Check for forward-reverse operator.
1289 if (op == REVERSE_RULE_OP &&
1290 (pos < limit && rule.charAt(pos) == FORWARD_RULE_OP)) {
1291 ++pos;
1292 op = FWDREV_RULE_OP;
1293 }
1294
1295 // Translate alternate op characters.
1296 switch (op) {
1297 case ALT_FORWARD_RULE_OP:
1298 op = FORWARD_RULE_OP;
1299 break;
1300 case ALT_REVERSE_RULE_OP:
1301 op = REVERSE_RULE_OP;
1302 break;
1303 case ALT_FWDREV_RULE_OP:
1304 op = FWDREV_RULE_OP;
1305 break;
1306 }
1307
1308 pos = right->parse(rule, pos, limit, status);
1309 if (U_FAILURE(status)) {
1310 return start;
1311 }
1312
1313 if (pos < limit) {
1314 if (rule.charAt(--pos) == END_OF_RULE) {
1315 ++pos;
1316 } else {
1317 // RuleHalf parser must have terminated at an operator
1318 return syntaxError(U_UNQUOTED_SPECIAL, rule, start, status);
1319 }
1320 }
1321
1322 if (op == VARIABLE_DEF_OP) {
1323 // LHS is the name. RHS is a single character, either a literal
1324 // or a set (already parsed). If RHS is longer than one
1325 // character, it is either a multi-character string, or multiple
1326 // sets, or a mixture of chars and sets -- syntax error.
1327
1328 // We expect to see a single undefined variable (the one being
1329 // defined).
1330 if (undefinedVariableName.length() == 0) {
1331 // "Missing '$' or duplicate definition"
1332 return syntaxError(U_BAD_VARIABLE_DEFINITION, rule, start, status);
1333 }
1334 if (left->text.length() != 1 || left->text.charAt(0) != variableLimit) {
1335 // "Malformed LHS"
1336 return syntaxError(U_MALFORMED_VARIABLE_DEFINITION, rule, start, status);
1337 }
1338 if (left->anchorStart || left->anchorEnd ||
1339 right->anchorStart || right->anchorEnd) {
1340 return syntaxError(U_MALFORMED_VARIABLE_DEFINITION, rule, start, status);
1341 }
1342 // We allow anything on the right, including an empty string.
1343 UnicodeString* value = new UnicodeString(right->text);
1344 // NULL pointer check
1345 if (value == NULL) {
1346 return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status);
1347 }
1348 variableNames.put(undefinedVariableName, value, status);
1349 ++variableLimit;
1350 return pos;
1351 }
1352
1353 // If this is not a variable definition rule, we shouldn't have
1354 // any undefined variable names.
1355 if (undefinedVariableName.length() != 0) {
1356 return syntaxError(// "Undefined variable $" + undefinedVariableName,
1357 U_UNDEFINED_VARIABLE,
1358 rule, start, status);
1359 }
1360
1361 // Verify segments
1362 if (segmentStandins.length() > segmentObjects.size()) {
1363 syntaxError(U_UNDEFINED_SEGMENT_REFERENCE, rule, start, status);
1364 }
1365 for (i=0; i<segmentStandins.length(); ++i) {
1366 if (segmentStandins.charAt(i) == 0) {
1367 syntaxError(U_INTERNAL_TRANSLITERATOR_ERROR, rule, start, status); // will never happen
1368 }
1369 }
1370 for (i=0; i<segmentObjects.size(); ++i) {
1371 if (segmentObjects.elementAt(i) == NULL) {
1372 syntaxError(U_INTERNAL_TRANSLITERATOR_ERROR, rule, start, status); // will never happen
1373 }
1374 }
1375
1376 // If the direction we want doesn't match the rule
1377 // direction, do nothing.
1378 if (op != FWDREV_RULE_OP &&
1379 ((direction == UTRANS_FORWARD) != (op == FORWARD_RULE_OP))) {
1380 return pos;
1381 }
1382
1383 // Transform the rule into a forward rule by swapping the
1384 // sides if necessary.
1385 if (direction == UTRANS_REVERSE) {
1386 left = &_right;
1387 right = &_left;
1388 }
1389
1390 // Remove non-applicable elements in forward-reverse
1391 // rules. Bidirectional rules ignore elements that do not
1392 // apply.
1393 if (op == FWDREV_RULE_OP) {
1394 right->removeContext();
1395 left->cursor = -1;
1396 left->cursorOffset = 0;
1397 }
1398
1399 // Normalize context
1400 if (left->ante < 0) {
1401 left->ante = 0;
1402 }
1403 if (left->post < 0) {
1404 left->post = left->text.length();
1405 }
1406
1407 // Context is only allowed on the input side. Cursors are only
1408 // allowed on the output side. Segment delimiters can only appear
1409 // on the left, and references on the right. Cursor offset
1410 // cannot appear without an explicit cursor. Cursor offset
1411 // cannot place the cursor outside the limits of the context.
1412 // Anchors are only allowed on the input side.
1413 if (right->ante >= 0 || right->post >= 0 || left->cursor >= 0 ||
1414 (right->cursorOffset != 0 && right->cursor < 0) ||
1415 // - The following two checks were used to ensure that the
1416 // - the cursor offset stayed within the ante- or postcontext.
1417 // - However, with the addition of quantifiers, we have to
1418 // - allow arbitrary cursor offsets and do runtime checking.
1419 //(right->cursorOffset > (left->text.length() - left->post)) ||
1420 //(-right->cursorOffset > left->ante) ||
1421 right->anchorStart || right->anchorEnd ||
1422 !left->isValidInput(*this) || !right->isValidOutput(*this) ||
1423 left->ante > left->post) {
1424
1425 return syntaxError(U_MALFORMED_RULE, rule, start, status);
1426 }
1427
1428 // Flatten segment objects vector to an array
1429 UnicodeFunctor** segmentsArray = NULL;
1430 if (segmentObjects.size() > 0) {
1431 segmentsArray = (UnicodeFunctor **)uprv_malloc(segmentObjects.size() * sizeof(UnicodeFunctor *));
1432 // Null pointer check
1433 if (segmentsArray == NULL) {
1434 return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status);
1435 }
1436 segmentObjects.toArray((void**) segmentsArray);
1437 }
1438 TransliterationRule* temptr = new TransliterationRule(
1439 left->text, left->ante, left->post,
1440 right->text, right->cursor, right->cursorOffset,
1441 segmentsArray,
1442 segmentObjects.size(),
1443 left->anchorStart, left->anchorEnd,
1444 curData,
1445 status);
1446 //Null pointer check
1447 if (temptr == NULL) {
1448 uprv_free(segmentsArray);
1449 return syntaxError(U_MEMORY_ALLOCATION_ERROR, rule, start, status);
1450 }
1451
1452 curData->ruleSet.addRule(temptr, status);
1453
1454 return pos;
1455 }
1456
1457 /**
1458 * Called by main parser upon syntax error. Search the rule string
1459 * for the probable end of the rule. Of course, if the error is that
1460 * the end of rule marker is missing, then the rule end will not be found.
1461 * In any case the rule start will be correctly reported.
1462 * @param msg error description
1463 * @param rule pattern string
1464 * @param start position of first character of current rule
1465 */
syntaxError(UErrorCode parseErrorCode,const UnicodeString & rule,int32_t pos,UErrorCode & status)1466 int32_t TransliteratorParser::syntaxError(UErrorCode parseErrorCode,
1467 const UnicodeString& rule,
1468 int32_t pos,
1469 UErrorCode& status)
1470 {
1471 parseError.offset = pos;
1472 parseError.line = 0 ; /* we are not using line numbers */
1473
1474 // for pre-context
1475 const int32_t LEN = U_PARSE_CONTEXT_LEN - 1;
1476 int32_t start = uprv_max(pos - LEN, 0);
1477 int32_t stop = pos;
1478
1479 rule.extract(start,stop-start,parseError.preContext);
1480 //null terminate the buffer
1481 parseError.preContext[stop-start] = 0;
1482
1483 //for post-context
1484 start = pos;
1485 stop = uprv_min(pos + LEN, rule.length());
1486
1487 rule.extract(start,stop-start,parseError.postContext);
1488 //null terminate the buffer
1489 parseError.postContext[stop-start]= 0;
1490
1491 status = (UErrorCode)parseErrorCode;
1492 return pos;
1493
1494 }
1495
1496 /**
1497 * Parse a UnicodeSet out, store it, and return the stand-in character
1498 * used to represent it.
1499 */
parseSet(const UnicodeString & rule,ParsePosition & pos,UErrorCode & status)1500 UChar TransliteratorParser::parseSet(const UnicodeString& rule,
1501 ParsePosition& pos,
1502 UErrorCode& status) {
1503 UnicodeSet* set = new UnicodeSet(rule, pos, USET_IGNORE_SPACE, parseData, status);
1504 // Null pointer check
1505 if (set == NULL) {
1506 status = U_MEMORY_ALLOCATION_ERROR;
1507 return (UChar)0x0000; // Return empty character with error.
1508 }
1509 set->compact();
1510 return generateStandInFor(set, status);
1511 }
1512
1513 /**
1514 * Generate and return a stand-in for a new UnicodeFunctor. Store
1515 * the matcher (adopt it).
1516 */
generateStandInFor(UnicodeFunctor * adopted,UErrorCode & status)1517 UChar TransliteratorParser::generateStandInFor(UnicodeFunctor* adopted, UErrorCode& status) {
1518 // assert(obj != null);
1519
1520 // Look up previous stand-in, if any. This is a short list
1521 // (typical n is 0, 1, or 2); linear search is optimal.
1522 for (int32_t i=0; i<variablesVector.size(); ++i) {
1523 if (variablesVector.elementAt(i) == adopted) { // [sic] pointer comparison
1524 return (UChar) (curData->variablesBase + i);
1525 }
1526 }
1527
1528 if (variableNext >= variableLimit) {
1529 delete adopted;
1530 status = U_VARIABLE_RANGE_EXHAUSTED;
1531 return 0;
1532 }
1533 variablesVector.addElement(adopted, status);
1534 return variableNext++;
1535 }
1536
1537 /**
1538 * Return the standin for segment seg (1-based).
1539 */
getSegmentStandin(int32_t seg,UErrorCode & status)1540 UChar TransliteratorParser::getSegmentStandin(int32_t seg, UErrorCode& status) {
1541 // Special character used to indicate an empty spot
1542 UChar empty = curData->variablesBase - 1;
1543 while (segmentStandins.length() < seg) {
1544 segmentStandins.append(empty);
1545 }
1546 UChar c = segmentStandins.charAt(seg-1);
1547 if (c == empty) {
1548 if (variableNext >= variableLimit) {
1549 status = U_VARIABLE_RANGE_EXHAUSTED;
1550 return 0;
1551 }
1552 c = variableNext++;
1553 // Set a placeholder in the master variables vector that will be
1554 // filled in later by setSegmentObject(). We know that we will get
1555 // called first because setSegmentObject() will call us.
1556 variablesVector.addElement((void*) NULL, status);
1557 segmentStandins.setCharAt(seg-1, c);
1558 }
1559 return c;
1560 }
1561
1562 /**
1563 * Set the object for segment seg (1-based).
1564 */
setSegmentObject(int32_t seg,StringMatcher * adopted,UErrorCode & status)1565 void TransliteratorParser::setSegmentObject(int32_t seg, StringMatcher* adopted, UErrorCode& status) {
1566 // Since we call parseSection() recursively, nested
1567 // segments will result in segment i+1 getting parsed
1568 // and stored before segment i; be careful with the
1569 // vector handling here.
1570 if (segmentObjects.size() < seg) {
1571 segmentObjects.setSize(seg, status);
1572 }
1573 int32_t index = getSegmentStandin(seg, status) - curData->variablesBase;
1574 if (segmentObjects.elementAt(seg-1) != NULL ||
1575 variablesVector.elementAt(index) != NULL) {
1576 // should never happen
1577 status = U_INTERNAL_TRANSLITERATOR_ERROR;
1578 return;
1579 }
1580 segmentObjects.setElementAt(adopted, seg-1);
1581 variablesVector.setElementAt(adopted, index);
1582 }
1583
1584 /**
1585 * Return the stand-in for the dot set. It is allocated the first
1586 * time and reused thereafter.
1587 */
getDotStandIn(UErrorCode & status)1588 UChar TransliteratorParser::getDotStandIn(UErrorCode& status) {
1589 if (dotStandIn == (UChar) -1) {
1590 UnicodeSet* tempus = new UnicodeSet(DOT_SET, status);
1591 // Null pointer check.
1592 if (tempus == NULL) {
1593 status = U_MEMORY_ALLOCATION_ERROR;
1594 return (UChar)0x0000;
1595 }
1596 dotStandIn = generateStandInFor(tempus, status);
1597 }
1598 return dotStandIn;
1599 }
1600
1601 /**
1602 * Append the value of the given variable name to the given
1603 * UnicodeString.
1604 */
appendVariableDef(const UnicodeString & name,UnicodeString & buf,UErrorCode & status)1605 void TransliteratorParser::appendVariableDef(const UnicodeString& name,
1606 UnicodeString& buf,
1607 UErrorCode& status) {
1608 const UnicodeString* s = (const UnicodeString*) variableNames.get(name);
1609 if (s == NULL) {
1610 // We allow one undefined variable so that variable definition
1611 // statements work. For the first undefined variable we return
1612 // the special placeholder variableLimit-1, and save the variable
1613 // name.
1614 if (undefinedVariableName.length() == 0) {
1615 undefinedVariableName = name;
1616 if (variableNext >= variableLimit) {
1617 // throw new RuntimeException("Private use variables exhausted");
1618 status = U_ILLEGAL_ARGUMENT_ERROR;
1619 return;
1620 }
1621 buf.append((UChar) --variableLimit);
1622 } else {
1623 //throw new IllegalArgumentException("Undefined variable $"
1624 // + name);
1625 status = U_ILLEGAL_ARGUMENT_ERROR;
1626 return;
1627 }
1628 } else {
1629 buf.append(*s);
1630 }
1631 }
1632
1633 /**
1634 * Glue method to get around access restrictions in C++.
1635 */
1636 /*Transliterator* TransliteratorParser::createBasicInstance(const UnicodeString& id, const UnicodeString* canonID) {
1637 return Transliterator::createBasicInstance(id, canonID);
1638 }*/
1639
1640 U_NAMESPACE_END
1641
1642 U_CAPI int32_t
utrans_stripRules(const UChar * source,int32_t sourceLen,UChar * target,UErrorCode * status)1643 utrans_stripRules(const UChar *source, int32_t sourceLen, UChar *target, UErrorCode *status) {
1644 U_NAMESPACE_USE
1645
1646 //const UChar *sourceStart = source;
1647 const UChar *targetStart = target;
1648 const UChar *sourceLimit = source+sourceLen;
1649 UChar *targetLimit = target+sourceLen;
1650 UChar32 c = 0;
1651 UBool quoted = FALSE;
1652 int32_t index;
1653
1654 uprv_memset(target, 0, sourceLen*U_SIZEOF_UCHAR);
1655
1656 /* read the rules into the buffer */
1657 while (source < sourceLimit)
1658 {
1659 index=0;
1660 U16_NEXT_UNSAFE(source, index, c);
1661 source+=index;
1662 if(c == QUOTE) {
1663 quoted = (UBool)!quoted;
1664 }
1665 else if (!quoted) {
1666 if (c == RULE_COMMENT_CHAR) {
1667 /* skip comments and all preceding spaces */
1668 while (targetStart < target && *(target - 1) == 0x0020) {
1669 target--;
1670 }
1671 do {
1672 c = *(source++);
1673 }
1674 while (c != CR && c != LF);
1675 }
1676 else if (c == ESCAPE) {
1677 UChar32 c2 = *source;
1678 if (c2 == CR || c2 == LF) {
1679 /* A backslash at the end of a line. */
1680 /* Since we're stripping lines, ignore the backslash. */
1681 source++;
1682 continue;
1683 }
1684 if (c2 == 0x0075 && source+5 < sourceLimit) { /* \u seen. \U isn't unescaped. */
1685 int32_t escapeOffset = 0;
1686 UnicodeString escapedStr(source, 5);
1687 c2 = escapedStr.unescapeAt(escapeOffset);
1688
1689 if (c2 == (UChar32)0xFFFFFFFF || escapeOffset == 0)
1690 {
1691 *status = U_PARSE_ERROR;
1692 return 0;
1693 }
1694 if (!PatternProps::isWhiteSpace(c2) && !u_iscntrl(c2) && !u_ispunct(c2)) {
1695 /* It was escaped for a reason. Write what it was suppose to be. */
1696 source+=5;
1697 c = c2;
1698 }
1699 }
1700 else if (c2 == QUOTE) {
1701 /* \' seen. Make sure we don't do anything when we see it again. */
1702 quoted = (UBool)!quoted;
1703 }
1704 }
1705 }
1706 if (c == CR || c == LF)
1707 {
1708 /* ignore spaces carriage returns, and all leading spaces on the next line.
1709 * and line feed unless in the form \uXXXX
1710 */
1711 quoted = FALSE;
1712 while (source < sourceLimit) {
1713 c = *(source);
1714 if (c != CR && c != LF && c != 0x0020) {
1715 break;
1716 }
1717 source++;
1718 }
1719 continue;
1720 }
1721
1722 /* Append UChar * after dissembling if c > 0xffff*/
1723 index=0;
1724 U16_APPEND_UNSAFE(target, index, c);
1725 target+=index;
1726 }
1727 if (target < targetLimit) {
1728 *target = 0;
1729 }
1730 return (int32_t)(target-targetStart);
1731 }
1732
1733 #endif /* #if !UCONFIG_NO_TRANSLITERATION */
1734