• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2 ******************************************************************************
3 *
4 *   Copyright (C) 2008-2011, International Business Machines
5 *   Corporation and others.  All Rights Reserved.
6 *
7 ******************************************************************************
8 *   file name:  uspoof_wsconf.cpp
9 *   encoding:   US-ASCII
10 *   tab size:   8 (not used)
11 *   indentation:4
12 *
13 *   created on: 2009Jan05  (refactoring earlier files)
14 *   created by: Andy Heninger
15 *
16 *   Internal functions for compililing Whole Script confusable source data
17 *   into its binary (runtime) form.  The binary data format is described
18 *   in uspoof_impl.h
19 */
20 
21 #include "unicode/utypes.h"
22 #include "unicode/uspoof.h"
23 
24 #if !UCONFIG_NO_NORMALIZATION
25 
26 #if !UCONFIG_NO_REGULAR_EXPRESSIONS
27 
28 #include "unicode/unorm.h"
29 #include "unicode/uregex.h"
30 #include "unicode/ustring.h"
31 #include "cmemory.h"
32 #include "uspoof_impl.h"
33 #include "uhash.h"
34 #include "uvector.h"
35 #include "uassert.h"
36 #include "uspoof_wsconf.h"
37 
38 U_NAMESPACE_USE
39 
40 
41 // Regular expression for parsing a line from the Unicode file confusablesWholeScript.txt
42 // Example Lines:
43 //   006F          ; Latn; Deva; A #      (o)  LATIN SMALL LETTER O
44 //   0048..0049    ; Latn; Grek; A #  [2] (H..I)  LATIN CAPITAL LETTER H..LATIN CAPITAL LETTER I
45 //    |               |     |    |
46 //    |               |     |    |---- Which table, Any Case or Lower Case (A or L)
47 //    |               |     |----------Target script.   We need this.
48 //    |               |----------------Src script.  Should match the script of the source
49 //    |                                code points.  Beyond checking that, we don't keep it.
50 //    |--------------------------------Source code points or range.
51 //
52 // The expression will match _all_ lines, including erroneous lines.
53 // The result of the parse is returned via the contents of the (match) groups.
54 static const char *parseExp =
55         "(?m)"                                         // Multi-line mode
56         "^([ \\t]*(?:#.*?)?)$"                         // A blank or comment line.  Matches Group 1.
57         "|^(?:"                                        //   OR
58         "\\s*([0-9A-F]{4,})(?:..([0-9A-F]{4,}))?\\s*;" // Code point range.  Groups 2 and 3.
59         "\\s*([A-Za-z]+)\\s*;"                         // The source script.  Group 4.
60         "\\s*([A-Za-z]+)\\s*;"                         // The target script.  Group 5.
61         "\\s*(?:(A)|(L))"                              // The table A or L.   Group 6 or 7
62         "[ \\t]*(?:#.*?)?"                             // Trailing commment
63         ")$|"                                          //   OR
64         "^(.*?)$";                                     // An error line.      Group 8.
65                                                        //    Any line not matching the preceding
66                                                        //    parts of the expression.will match
67                                                        //    this, and thus be flagged as an error
68 
69 
70 // Extract a regular expression match group into a char * string.
71 //    The group must contain only invariant characters.
72 //    Used for script names
73 //
extractGroup(URegularExpression * e,int32_t group,char * destBuf,int32_t destCapacity,UErrorCode & status)74 static void extractGroup(
75     URegularExpression *e, int32_t group, char *destBuf, int32_t destCapacity, UErrorCode &status) {
76 
77     UChar ubuf[50];
78     ubuf[0] = 0;
79     destBuf[0] = 0;
80     int32_t len = uregex_group(e, group, ubuf, 50, &status);
81     if (U_FAILURE(status) || len == -1 || len >= destCapacity) {
82         return;
83     }
84     UnicodeString s(FALSE, ubuf, len);   // Aliasing constructor
85     s.extract(0, len, destBuf, destCapacity, US_INV);
86 }
87 
88 
89 
90 //  Build the Whole Script Confusable data
91 //
92 //     TODO:  Reorganize.  Either get rid of the WSConfusableDataBuilder class,
93 //                         because everything is local to this one build function anyhow,
94 //                           OR
95 //                         break this function into more reasonably sized pieces, with
96 //                         state in WSConfusableDataBuilder.
97 //
buildWSConfusableData(SpoofImpl * spImpl,const char * confusablesWS,int32_t confusablesWSLen,UParseError * pe,UErrorCode & status)98 void buildWSConfusableData(SpoofImpl *spImpl, const char * confusablesWS,
99           int32_t confusablesWSLen, UParseError *pe, UErrorCode &status)
100 {
101     if (U_FAILURE(status)) {
102         return;
103     }
104     URegularExpression *parseRegexp = NULL;
105     int32_t             inputLen    = 0;
106     UChar              *input       = NULL;
107     int32_t             lineNum     = 0;
108 
109     UVector            *scriptSets        = NULL;
110     uint32_t            rtScriptSetsCount = 2;
111 
112     UTrie2             *anyCaseTrie   = NULL;
113     UTrie2             *lowerCaseTrie = NULL;
114 
115     anyCaseTrie = utrie2_open(0, 0, &status);
116     lowerCaseTrie = utrie2_open(0, 0, &status);
117 
118     UnicodeString pattern(parseExp, -1, US_INV);
119 
120     // The scriptSets vector provides a mapping from TRIE values to the set of scripts.
121     //
122     // Reserved TRIE values:
123     //   0:  Code point has no whole script confusables.
124     //   1:  Code point is of script Common or Inherited.
125     //       These code points do not participate in whole script confusable detection.
126     //       (This is logically equivalent to saying that they contain confusables in
127     //        all scripts)
128     //
129     // Because Trie values are indexes into the ScriptSets vector, pre-fill
130     // vector positions 0 and 1 to avoid conflicts with the reserved values.
131 
132     scriptSets = new UVector(status);
133     if (scriptSets == NULL) {
134         status = U_MEMORY_ALLOCATION_ERROR;
135         goto cleanup;
136     }
137     scriptSets->addElement((void *)NULL, status);
138     scriptSets->addElement((void *)NULL, status);
139 
140     // Convert the user input data from UTF-8 to UChar (UTF-16)
141     u_strFromUTF8(NULL, 0, &inputLen, confusablesWS, confusablesWSLen, &status);
142     if (status != U_BUFFER_OVERFLOW_ERROR) {
143         goto cleanup;
144     }
145     status = U_ZERO_ERROR;
146     input = static_cast<UChar *>(uprv_malloc((inputLen+1) * sizeof(UChar)));
147     if (input == NULL) {
148         status = U_MEMORY_ALLOCATION_ERROR;
149         goto cleanup;
150     }
151     u_strFromUTF8(input, inputLen+1, NULL, confusablesWS, confusablesWSLen, &status);
152 
153     parseRegexp = uregex_open(pattern.getBuffer(), pattern.length(), 0, NULL, &status);
154 
155     // Zap any Byte Order Mark at the start of input.  Changing it to a space is benign
156     //   given the syntax of the input.
157     if (*input == 0xfeff) {
158         *input = 0x20;
159     }
160 
161     // Parse the input, one line per iteration of this loop.
162     uregex_setText(parseRegexp, input, inputLen, &status);
163     while (uregex_findNext(parseRegexp, &status)) {
164         lineNum++;
165         if (uregex_start(parseRegexp, 1, &status) >= 0) {
166             // this was a blank or comment line.
167             continue;
168         }
169         if (uregex_start(parseRegexp, 8, &status) >= 0) {
170             // input file syntax error.
171             status = U_PARSE_ERROR;
172             goto cleanup;
173         }
174         if (U_FAILURE(status)) {
175             goto cleanup;
176         }
177 
178         // Pick up the start and optional range end code points from the parsed line.
179         UChar32  startCodePoint = SpoofImpl::ScanHex(
180             input, uregex_start(parseRegexp, 2, &status), uregex_end(parseRegexp, 2, &status), status);
181         UChar32  endCodePoint = startCodePoint;
182         if (uregex_start(parseRegexp, 3, &status) >=0) {
183             endCodePoint = SpoofImpl::ScanHex(
184                 input, uregex_start(parseRegexp, 3, &status), uregex_end(parseRegexp, 3, &status), status);
185         }
186 
187         // Extract the two script names from the source line.  We need these in an 8 bit
188         //   default encoding (will be EBCDIC on IBM mainframes) in order to pass them on
189         //   to the ICU u_getPropertyValueEnum() function.  Ugh.
190         char  srcScriptName[20];
191         char  targScriptName[20];
192         extractGroup(parseRegexp, 4, srcScriptName, sizeof(srcScriptName), status);
193         extractGroup(parseRegexp, 5, targScriptName, sizeof(targScriptName), status);
194         UScriptCode srcScript  =
195             static_cast<UScriptCode>(u_getPropertyValueEnum(UCHAR_SCRIPT, srcScriptName));
196         UScriptCode targScript =
197             static_cast<UScriptCode>(u_getPropertyValueEnum(UCHAR_SCRIPT, targScriptName));
198         if (U_FAILURE(status)) {
199             goto cleanup;
200         }
201         if (srcScript == USCRIPT_INVALID_CODE || targScript == USCRIPT_INVALID_CODE) {
202             status = U_INVALID_FORMAT_ERROR;
203             goto cleanup;
204         }
205 
206         // select the table - (A) any case or (L) lower case only
207         UTrie2 *table = anyCaseTrie;
208         if (uregex_start(parseRegexp, 7, &status) >= 0) {
209             table = lowerCaseTrie;
210         }
211 
212         // Build the set of scripts containing confusable characters for
213         //   the code point(s) specified in this input line.
214         // Sanity check that the script of the source code point is the same
215         //   as the source script indicated in the input file.  Failure of this check is
216         //   an error in the input file.
217         // Include the source script in the set (needed for Mixed Script Confusable detection).
218         //
219         UChar32 cp;
220         for (cp=startCodePoint; cp<=endCodePoint; cp++) {
221             int32_t setIndex = utrie2_get32(table, cp);
222             BuilderScriptSet *bsset = NULL;
223             if (setIndex > 0) {
224                 U_ASSERT(setIndex < scriptSets->size());
225                 bsset = static_cast<BuilderScriptSet *>(scriptSets->elementAt(setIndex));
226             } else {
227                 bsset = new BuilderScriptSet();
228                 if (bsset == NULL) {
229                     status = U_MEMORY_ALLOCATION_ERROR;
230                     goto cleanup;
231                 }
232                 bsset->codePoint = cp;
233                 bsset->trie = table;
234                 bsset->sset = new ScriptSet();
235                 setIndex = scriptSets->size();
236                 bsset->index = setIndex;
237                 bsset->rindex = 0;
238                 if (bsset->sset == NULL) {
239                     status = U_MEMORY_ALLOCATION_ERROR;
240                     goto cleanup;
241                 }
242                 scriptSets->addElement(bsset, status);
243                 utrie2_set32(table, cp, setIndex, &status);
244             }
245             bsset->sset->Union(targScript);
246             bsset->sset->Union(srcScript);
247 
248             if (U_FAILURE(status)) {
249                 goto cleanup;
250             }
251             UScriptCode cpScript = uscript_getScript(cp, &status);
252             if (cpScript != srcScript) {
253                 status = U_INVALID_FORMAT_ERROR;
254                 goto cleanup;
255             }
256         }
257     }
258 
259     // Eliminate duplicate script sets.  At this point we have a separate
260     // script set for every code point that had data in the input file.
261     //
262     // We eliminate underlying ScriptSet objects, not the BuildScriptSets that wrap them
263     //
264     // printf("Number of scriptSets: %d\n", scriptSets->size());
265     {
266         int32_t duplicateCount = 0;
267         rtScriptSetsCount = 2;
268         for (int32_t outeri=2; outeri<scriptSets->size(); outeri++) {
269             BuilderScriptSet *outerSet = static_cast<BuilderScriptSet *>(scriptSets->elementAt(outeri));
270             if (outerSet->index != static_cast<uint32_t>(outeri)) {
271                 // This set was already identified as a duplicate.
272                 //   It will not be allocated a position in the runtime array of ScriptSets.
273                 continue;
274             }
275             outerSet->rindex = rtScriptSetsCount++;
276             for (int32_t inneri=outeri+1; inneri<scriptSets->size(); inneri++) {
277                 BuilderScriptSet *innerSet = static_cast<BuilderScriptSet *>(scriptSets->elementAt(inneri));
278                 if (*(outerSet->sset) == *(innerSet->sset) && outerSet->sset != innerSet->sset) {
279                     delete innerSet->sset;
280                     innerSet->scriptSetOwned = FALSE;
281                     innerSet->sset = outerSet->sset;
282                     innerSet->index = outeri;
283                     innerSet->rindex = outerSet->rindex;
284                     duplicateCount++;
285                 }
286                 // But this doesn't get all.  We need to fix the TRIE.
287             }
288         }
289         // printf("Number of distinct script sets: %d\n", rtScriptSetsCount);
290     }
291 
292 
293 
294     // Update the Trie values to be reflect the run time script indexes (after duplicate merging).
295     //    (Trie Values 0 and 1 are reserved, and the corresponding slots in scriptSets
296     //     are unused, which is why the loop index starts at 2.)
297     {
298         for (int32_t i=2; i<scriptSets->size(); i++) {
299             BuilderScriptSet *bSet = static_cast<BuilderScriptSet *>(scriptSets->elementAt(i));
300             if (bSet->rindex != (uint32_t)i) {
301                 utrie2_set32(bSet->trie, bSet->codePoint, bSet->rindex, &status);
302             }
303         }
304     }
305 
306     // For code points with script==Common or script==Inherited,
307     //   Set the reserved value of 1 into both Tries.  These characters do not participate
308     //   in Whole Script Confusable detection; this reserved value is the means
309     //   by which they are detected.
310     {
311         UnicodeSet ignoreSet;
312         ignoreSet.applyIntPropertyValue(UCHAR_SCRIPT, USCRIPT_COMMON, status);
313         UnicodeSet inheritedSet;
314         inheritedSet.applyIntPropertyValue(UCHAR_SCRIPT, USCRIPT_INHERITED, status);
315         ignoreSet.addAll(inheritedSet);
316         for (int32_t rn=0; rn<ignoreSet.getRangeCount(); rn++) {
317             UChar32 rangeStart = ignoreSet.getRangeStart(rn);
318             UChar32 rangeEnd   = ignoreSet.getRangeEnd(rn);
319             utrie2_setRange32(anyCaseTrie,   rangeStart, rangeEnd, 1, TRUE, &status);
320             utrie2_setRange32(lowerCaseTrie, rangeStart, rangeEnd, 1, TRUE, &status);
321         }
322     }
323 
324     // Serialize the data to the Spoof Detector
325     {
326         utrie2_freeze(anyCaseTrie,   UTRIE2_16_VALUE_BITS, &status);
327         int32_t size = utrie2_serialize(anyCaseTrie, NULL, 0, &status);
328         // printf("Any case Trie size: %d\n", size);
329         if (status != U_BUFFER_OVERFLOW_ERROR) {
330             goto cleanup;
331         }
332         status = U_ZERO_ERROR;
333         spImpl->fSpoofData->fRawData->fAnyCaseTrie = spImpl->fSpoofData->fMemLimit;
334         spImpl->fSpoofData->fRawData->fAnyCaseTrieLength = size;
335         spImpl->fSpoofData->fAnyCaseTrie = anyCaseTrie;
336         void *where = spImpl->fSpoofData->reserveSpace(size, status);
337         utrie2_serialize(anyCaseTrie, where, size, &status);
338 
339         utrie2_freeze(lowerCaseTrie, UTRIE2_16_VALUE_BITS, &status);
340         size = utrie2_serialize(lowerCaseTrie, NULL, 0, &status);
341         // printf("Lower case Trie size: %d\n", size);
342         if (status != U_BUFFER_OVERFLOW_ERROR) {
343             goto cleanup;
344         }
345         status = U_ZERO_ERROR;
346         spImpl->fSpoofData->fRawData->fLowerCaseTrie = spImpl->fSpoofData->fMemLimit;
347         spImpl->fSpoofData->fRawData->fLowerCaseTrieLength = size;
348         spImpl->fSpoofData->fLowerCaseTrie = lowerCaseTrie;
349         where = spImpl->fSpoofData->reserveSpace(size, status);
350         utrie2_serialize(lowerCaseTrie, where, size, &status);
351 
352         spImpl->fSpoofData->fRawData->fScriptSets = spImpl->fSpoofData->fMemLimit;
353         spImpl->fSpoofData->fRawData->fScriptSetsLength = rtScriptSetsCount;
354         ScriptSet *rtScriptSets =  static_cast<ScriptSet *>
355             (spImpl->fSpoofData->reserveSpace(rtScriptSetsCount * sizeof(ScriptSet), status));
356         uint32_t rindex = 2;
357         for (int32_t i=2; i<scriptSets->size(); i++) {
358             BuilderScriptSet *bSet = static_cast<BuilderScriptSet *>(scriptSets->elementAt(i));
359             if (bSet->rindex < rindex) {
360                 // We have already copied this script set to the serialized data.
361                 continue;
362             }
363             U_ASSERT(rindex == bSet->rindex);
364             rtScriptSets[rindex] = *bSet->sset;   // Assignment of a ScriptSet just copies the bits.
365             rindex++;
366         }
367     }
368 
369     // Open new utrie2s from the serialized data.  We don't want to keep the ones
370     //   we just built because we would then have two copies of the data, one internal to
371     //   the utries that we have already constructed, and one in the serialized data area.
372     //   An alternative would be to not pre-serialize the Trie data, but that makes the
373     //   spoof detector data different, depending on how the detector was constructed.
374     //   It's simpler to keep the data always the same.
375 
376     spImpl->fSpoofData->fAnyCaseTrie = utrie2_openFromSerialized(
377             UTRIE2_16_VALUE_BITS,
378             (const char *)spImpl->fSpoofData->fRawData + spImpl->fSpoofData->fRawData->fAnyCaseTrie,
379             spImpl->fSpoofData->fRawData->fAnyCaseTrieLength,
380             NULL,
381             &status);
382 
383     spImpl->fSpoofData->fLowerCaseTrie = utrie2_openFromSerialized(
384             UTRIE2_16_VALUE_BITS,
385             (const char *)spImpl->fSpoofData->fRawData + spImpl->fSpoofData->fRawData->fLowerCaseTrie,
386             spImpl->fSpoofData->fRawData->fAnyCaseTrieLength,
387             NULL,
388             &status);
389 
390 
391 
392 cleanup:
393     if (U_FAILURE(status)) {
394         pe->line = lineNum;
395     }
396     uregex_close(parseRegexp);
397     uprv_free(input);
398 
399     int32_t i;
400     for (i=0; i<scriptSets->size(); i++) {
401         BuilderScriptSet *bsset = static_cast<BuilderScriptSet *>(scriptSets->elementAt(i));
402         delete bsset;
403     }
404     delete scriptSets;
405     utrie2_close(anyCaseTrie);
406     utrie2_close(lowerCaseTrie);
407     return;
408 }
409 
410 
411 
412 
413 
BuilderScriptSet()414 BuilderScriptSet::BuilderScriptSet() {
415     codePoint = -1;
416     trie = NULL;
417     sset = NULL;
418     index = 0;
419     rindex = 0;
420     scriptSetOwned = TRUE;
421 }
422 
~BuilderScriptSet()423 BuilderScriptSet::~BuilderScriptSet() {
424     if (scriptSetOwned) {
425         delete sset;
426     }
427 }
428 
429 #endif
430 #endif //  !UCONFIG_NO_REGULAR_EXPRESSIONS
431 
432