• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // © 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html#License
3 /*
4  *******************************************************************************
5  * Copyright (C) 2006-2009, Google, International Business Machines Corporation *
6  * and others. All Rights Reserved.                                            *
7  *******************************************************************************
8  */
9 package com.ibm.icu.impl;
10 
11 import com.ibm.icu.text.UTF16;
12 import com.ibm.icu.text.UnicodeSet;
13 
14 /**
15  * A simple parsing class for patterns and rules. Handles '...' quotations, \\uxxxx and \\Uxxxxxxxx, and symple syntax.
16  * The '' (two quotes) is treated as a single quote, inside or outside a quote
17  * <ul>
18  * <li>Any ignorable characters are ignored in parsing.</li>
19  * <li>Any syntax characters are broken into separate tokens</li>
20  * <li>Quote characters can be specified: '...', "...", and \x </li>
21  * <li>Other characters are treated as literals</li>
22  * </ul>
23  */
24 public class PatternTokenizer {
25     // settings used in the interpretation of the pattern
26     private UnicodeSet ignorableCharacters = new UnicodeSet();
27     private UnicodeSet syntaxCharacters = new UnicodeSet();
28     private UnicodeSet extraQuotingCharacters = new UnicodeSet();
29     private UnicodeSet escapeCharacters = new UnicodeSet();
30     private boolean usingSlash = false;
31     private boolean usingQuote = false;
32 
33     // transient data, set when needed. Null it out for any changes in the above fields.
34     private transient UnicodeSet needingQuoteCharacters = null;
35 
36     // data about the current pattern being parsed. start gets moved as we go along.
37     private int start;
38     private int limit;
39     private String pattern;
40 
getIgnorableCharacters()41     public UnicodeSet getIgnorableCharacters() {
42         return (UnicodeSet) ignorableCharacters.clone();
43     }
44     /**
45      * Sets the characters to be ignored in parsing, eg new UnicodeSet("[:pattern_whitespace:]");
46      * @param ignorableCharacters Characters to be ignored.
47      * @return A PatternTokenizer object in which characters are specified as ignored characters.
48      */
setIgnorableCharacters(UnicodeSet ignorableCharacters)49     public PatternTokenizer setIgnorableCharacters(UnicodeSet ignorableCharacters) {
50         this.ignorableCharacters = (UnicodeSet) ignorableCharacters.clone();
51         needingQuoteCharacters = null;
52         return this;
53     }
getSyntaxCharacters()54     public UnicodeSet getSyntaxCharacters() {
55         return (UnicodeSet) syntaxCharacters.clone();
56     }
getExtraQuotingCharacters()57     public UnicodeSet getExtraQuotingCharacters() {
58         return (UnicodeSet) extraQuotingCharacters.clone();
59     }
60     /**
61      *  Sets the characters to be interpreted as syntax characters in parsing, eg new UnicodeSet("[:pattern_syntax:]")
62      * @param syntaxCharacters Characters to be set as syntax characters.
63      * @return A PatternTokenizer object in which characters are specified as syntax characters.
64      */
setSyntaxCharacters(UnicodeSet syntaxCharacters)65     public PatternTokenizer setSyntaxCharacters(UnicodeSet syntaxCharacters) {
66         this.syntaxCharacters = (UnicodeSet) syntaxCharacters.clone();
67         needingQuoteCharacters = null;
68         return this;
69     }
70     /**
71      *  Sets the extra characters to be quoted in literals
72      * @param syntaxCharacters Characters to be set as extra quoting characters.
73      * @return A PatternTokenizer object in which characters are specified as extra quoting characters.
74      */
setExtraQuotingCharacters(UnicodeSet syntaxCharacters)75     public PatternTokenizer setExtraQuotingCharacters(UnicodeSet syntaxCharacters) {
76         this.extraQuotingCharacters = (UnicodeSet) syntaxCharacters.clone();
77         needingQuoteCharacters = null;
78         return this;
79     }
80 
getEscapeCharacters()81     public UnicodeSet getEscapeCharacters() {
82         return (UnicodeSet) escapeCharacters.clone();
83     }
84     /**
85      * Set characters to be escaped in literals, in quoteLiteral and normalize, eg new UnicodeSet("[^\\u0020-\\u007E]");
86      * @param escapeCharacters Characters to be set as escape characters.
87      * @return A PatternTokenizer object in which characters are specified as escape characters.
88      */
setEscapeCharacters(UnicodeSet escapeCharacters)89     public PatternTokenizer setEscapeCharacters(UnicodeSet escapeCharacters) {
90         this.escapeCharacters = (UnicodeSet) escapeCharacters.clone();
91         return this;
92     }
isUsingQuote()93     public boolean isUsingQuote() {
94         return usingQuote;
95     }
setUsingQuote(boolean usingQuote)96     public PatternTokenizer setUsingQuote(boolean usingQuote) {
97         this.usingQuote = usingQuote;
98         needingQuoteCharacters = null;
99         return this;
100     }
isUsingSlash()101     public boolean isUsingSlash() {
102         return usingSlash;
103     }
setUsingSlash(boolean usingSlash)104     public PatternTokenizer setUsingSlash(boolean usingSlash) {
105         this.usingSlash = usingSlash;
106         needingQuoteCharacters = null;
107         return this;
108     }
109     //    public UnicodeSet getQuoteCharacters() {
110 //  return (UnicodeSet) quoteCharacters.clone();
111 //  }
112 //  public PatternTokenizer setQuoteCharacters(UnicodeSet quoteCharacters) {
113 //  this.quoteCharacters = (UnicodeSet) quoteCharacters.clone();
114 //  needingQuoteCharacters = null;
115 //  return this;
116 //  }
getLimit()117     public int getLimit() {
118         return limit;
119     }
setLimit(int limit)120     public PatternTokenizer setLimit(int limit) {
121         this.limit = limit;
122         return this;
123     }
getStart()124     public int getStart() {
125         return start;
126     }
setStart(int start)127     public PatternTokenizer setStart(int start) {
128         this.start = start;
129         return this;
130     }
131 
setPattern(CharSequence pattern)132     public PatternTokenizer setPattern(CharSequence pattern) {
133         return setPattern(pattern.toString());
134     }
135 
setPattern(String pattern)136     public PatternTokenizer setPattern(String pattern) {
137         if (pattern == null) {
138             throw new IllegalArgumentException("Inconsistent arguments");
139         }
140         this.start = 0;
141         this.limit = pattern.length();
142         this.pattern = pattern;
143         return this;
144     }
145 
146     public static final char SINGLE_QUOTE = '\'';
147     public static final char BACK_SLASH = '\\';
148     private static int NO_QUOTE = -1, IN_QUOTE = -2;
149 
quoteLiteral(CharSequence string)150     public String quoteLiteral(CharSequence string) {
151         return quoteLiteral(string.toString());
152     }
153 
154     /**
155      * Quote a literal string, using the available settings. Thus syntax characters, quote characters, and ignorable characters will be put into quotes.
156      * @param string String passed to quote a literal string.
157      * @return A string using the available settings will place syntax, quote, or ignorable characters into quotes.
158      */
quoteLiteral(String string)159     public String quoteLiteral(String string) {
160         if (needingQuoteCharacters == null) {
161             needingQuoteCharacters = new UnicodeSet().addAll(syntaxCharacters).addAll(ignorableCharacters).addAll(extraQuotingCharacters); // .addAll(quoteCharacters)
162             if (usingSlash) needingQuoteCharacters.add(BACK_SLASH);
163             if (usingQuote) needingQuoteCharacters.add(SINGLE_QUOTE);
164         }
165         StringBuffer result = new StringBuffer();
166         int quotedChar = NO_QUOTE;
167         int cp;
168         for (int i = 0; i < string.length(); i += UTF16.getCharCount(cp)) {
169             cp = UTF16.charAt(string, i);
170             if (escapeCharacters.contains(cp)) {
171                 // we may have to fix up previous characters
172                 if (quotedChar == IN_QUOTE) {
173                     result.append(SINGLE_QUOTE);
174                     quotedChar = NO_QUOTE;
175                 }
176                 appendEscaped(result, cp);
177                 continue;
178             }
179 
180             if (needingQuoteCharacters.contains(cp)) {
181                 // if we have already started a quote
182                 if (quotedChar == IN_QUOTE) {
183                     UTF16.append(result, cp);
184                     if (usingQuote && cp == SINGLE_QUOTE) { // double it
185                         result.append(SINGLE_QUOTE);
186                     }
187                     continue;
188                 }
189                 // otherwise not already in quote
190                 if (usingSlash) {
191                     result.append(BACK_SLASH);
192                     UTF16.append(result, cp);
193                     continue;
194                 }
195                 if (usingQuote) {
196                     if (cp == SINGLE_QUOTE) { // double it and continue
197                         result.append(SINGLE_QUOTE);
198                         result.append(SINGLE_QUOTE);
199                         continue;
200                     }
201                     result.append(SINGLE_QUOTE);
202                     UTF16.append(result, cp);
203                     quotedChar = IN_QUOTE;
204                     continue;
205                 }
206                 // we have no choice but to use \\u or \\U
207                 appendEscaped(result, cp);
208                 continue;
209             }
210             // otherwise cp doesn't need quoting
211             // we may have to fix up previous characters
212             if (quotedChar == IN_QUOTE) {
213                 result.append(SINGLE_QUOTE);
214                 quotedChar = NO_QUOTE;
215             }
216             UTF16.append(result, cp);
217         }
218         // all done.
219         // we may have to fix up previous characters
220         if (quotedChar == IN_QUOTE) {
221             result.append(SINGLE_QUOTE);
222         }
223         return result.toString();
224     }
225 
appendEscaped(StringBuffer result, int cp)226     private void appendEscaped(StringBuffer result, int cp) {
227         if (cp <= 0xFFFF) {
228             result.append("\\u").append(Utility.hex(cp,4));
229         } else {
230             result.append("\\U").append(Utility.hex(cp,8));
231         }
232     }
233 
normalize()234     public String normalize() {
235         int oldStart = start;
236         StringBuffer result = new StringBuffer();
237         StringBuffer buffer = new StringBuffer();
238         while (true) {
239             buffer.setLength(0);
240             int status = next(buffer);
241             if (status == DONE) {
242                 start = oldStart;
243                 return result.toString();
244             }
245             if (status != SYNTAX) {
246                 result.append(quoteLiteral(buffer));
247             } else {
248                 result.append(buffer);
249             }
250         }
251     }
252 
253     public static final int DONE = 0, SYNTAX = 1, LITERAL = 2, BROKEN_QUOTE = 3, BROKEN_ESCAPE = 4, UNKNOWN = 5;
254 
255     private static final int AFTER_QUOTE = -1, NONE = 0, START_QUOTE = 1, NORMAL_QUOTE = 2, SLASH_START = 3, HEX = 4;
256 
next(StringBuffer buffer)257     public int next(StringBuffer buffer) {
258         if (start >= limit) return DONE;
259         int status = UNKNOWN;
260         int lastQuote = UNKNOWN;
261         int quoteStatus = NONE;
262         int hexCount = 0;
263         int hexValue = 0;
264         int cp;
265         main:
266             for (int i = start; i < limit; i += UTF16.getCharCount(cp)) {
267                 cp = UTF16.charAt(pattern, i);
268                 // if we are in a quote, then handle it.
269                 switch (quoteStatus) {
270                 case SLASH_START:
271                     switch (cp) {
272                     case 'u':
273                         quoteStatus = HEX;
274                         hexCount = 4;
275                         hexValue = 0;
276                         continue main;
277                     case 'U':
278                         quoteStatus = HEX;
279                         hexCount = 8;
280                         hexValue = 0;
281                         continue main;
282                     default:
283                         if (usingSlash) {
284                             UTF16.append(buffer, cp);
285                             quoteStatus = NONE;
286                             continue main;
287                         } else {
288                             buffer.append(BACK_SLASH);
289                             quoteStatus = NONE;
290                         }
291                     }
292                     break; // fall through to NONE
293                 case HEX:
294                     hexValue <<= 4;
295                     hexValue += cp;
296                     switch (cp) {
297                     case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9':
298                         hexValue -= '0'; break;
299                     case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
300                         hexValue -= 'a' - 10; break;
301                     case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
302                         hexValue -= 'A' - 10; break;
303                     default:
304                         start = i;
305                     return BROKEN_ESCAPE;
306                     }
307                     --hexCount;
308                     if (hexCount == 0) {
309                         quoteStatus = NONE;
310                         UTF16.append(buffer, hexValue);
311                     }
312                     continue main;
313                 case AFTER_QUOTE:
314                     // see if we get another quote character
315                     // if we just ended a quote BUT the following character is the lastQuote character, then we have a situation like '...''...', so we restart the quote
316                     if (cp == lastQuote) {
317                         UTF16.append(buffer, cp);
318                         quoteStatus = NORMAL_QUOTE;
319                         continue main;
320                     }
321                     quoteStatus = NONE;
322                     break; // fall through to NONE
323                 case START_QUOTE:
324                     // if we are at the very start of a quote, and we hit another quote mark then we emit a literal quote character and end the quote
325                     if (cp == lastQuote) {
326                         UTF16.append(buffer, cp);
327                         quoteStatus = NONE; // get out of quote, with no trace remaining
328                         continue;
329                     }
330                     // otherwise get into quote
331                     UTF16.append(buffer, cp);
332                     quoteStatus = NORMAL_QUOTE;
333                     continue main;
334                 case NORMAL_QUOTE:
335                     if (cp == lastQuote) {
336                         quoteStatus = AFTER_QUOTE; // get out of quote
337                         continue main;
338                     }
339                     UTF16.append(buffer, cp);
340                     continue main;
341                 }
342 
343                 if (ignorableCharacters.contains(cp)) {
344                     continue;
345                 }
346                 // do syntax characters
347                 if (syntaxCharacters.contains(cp)) {
348                     if (status == UNKNOWN) {
349                         UTF16.append(buffer, cp);
350                         start = i + UTF16.getCharCount(cp);
351                         return SYNTAX;
352                     } else { // LITERAL, so back up and break
353                         start = i;
354                         return status;
355                     }
356                 }
357                 // otherwise it is a literal; keep on going
358                 status = LITERAL;
359                 if (cp == BACK_SLASH) {
360                     quoteStatus = SLASH_START;
361                     continue;
362                 } else if (usingQuote && cp == SINGLE_QUOTE) {
363                     lastQuote = cp;
364                     quoteStatus = START_QUOTE;
365                     continue;
366                 }
367                 // normal literals
368                 UTF16.append(buffer, cp);
369             }
370         // handle final cleanup
371         start = limit;
372         switch (quoteStatus) {
373         case HEX:
374             status = BROKEN_ESCAPE;
375             break;
376         case SLASH_START:
377             if (usingSlash) {
378                 status = BROKEN_ESCAPE;
379             } else {
380                 buffer.append(BACK_SLASH);
381             }
382             break;
383         case START_QUOTE: case NORMAL_QUOTE:
384             status = BROKEN_QUOTE;
385             break;
386         }
387         return status;
388     }
389 
390 
391 }
392 //eof
393