1 /* 2 *************************************************************************** 3 * Copyright (C) 2008-2010, International Business Machines Corporation 4 * and others. All Rights Reserved. 5 *************************************************************************** 6 * 7 * uspoof_impl.h 8 * 9 * Implemenation header for spoof detection 10 * 11 */ 12 13 #ifndef USPOOFIM_H 14 #define USPOOFIM_H 15 16 #include "unicode/utypes.h" 17 #include "unicode/uspoof.h" 18 #include "utrie2.h" 19 #include "unicode/uscript.h" 20 #include "unicode/udata.h" 21 22 23 #if !UCONFIG_NO_NORMALIZATION 24 25 #ifdef XP_CPLUSPLUS 26 27 U_NAMESPACE_BEGIN 28 29 // The maximium length (in UTF-16 UChars) of the skeleton replacement string resulting from 30 // a single input code point. This is function of the unicode.org data. 31 #define USPOOF_MAX_SKELETON_EXPANSION 20 32 33 // The default stack buffer size for copies or conversions or normalizations 34 // of input strings being checked. (Used in multiple places.) 35 #define USPOOF_STACK_BUFFER_SIZE 100 36 37 // Magic number for sanity checking spoof data. 38 #define USPOOF_MAGIC 0x3845fdef 39 40 class SpoofData; 41 struct SpoofDataHeader; 42 struct SpoofStringLengthsElement; 43 class ScriptSet; 44 45 /** 46 * Class SpoofImpl corresponds directly to the plain C API opaque type 47 * USpoofChecker. One can be cast to the other. 48 */ 49 class SpoofImpl : public UObject { 50 public: 51 SpoofImpl(SpoofData *data, UErrorCode &status); 52 SpoofImpl(); 53 virtual ~SpoofImpl(); 54 55 /** Copy constructor, used by the user level uspoof_clone() function. 56 */ 57 SpoofImpl(const SpoofImpl &src, UErrorCode &status); 58 59 static SpoofImpl *validateThis(USpoofChecker *sc, UErrorCode &status); 60 static const SpoofImpl *validateThis(const USpoofChecker *sc, UErrorCode &status); 61 62 /** Get the confusable skeleton transform for a single code point. 63 * The result is a string with a length between 1 and 18. 64 * @param tableMask bit flag specifying which confusable table to use. 65 * One of USPOOF_SL_TABLE_FLAG, USPOOF_MA_TABLE_FLAG, etc. 66 * @return The length in UTF-16 code units of the substition string. 67 */ 68 int32_t confusableLookup(UChar32 inChar, int32_t tableMask, UChar *destBuf) const; 69 70 /** Set and Get AllowedLocales, implementations of the corresponding API */ 71 void setAllowedLocales(const char *localesList, UErrorCode &status); 72 const char * getAllowedLocales(UErrorCode &status); 73 74 // Add (union) to the UnicodeSet all of the characters for the scripts used for 75 // the specified locale. Part of the implementation of setAllowedLocales. 76 void addScriptChars(const char *locale, UnicodeSet *allowedChars, UErrorCode &status); 77 78 79 /** parse a hex number. Untility used by the builders. */ 80 static UChar32 ScanHex(const UChar *s, int32_t start, int32_t limit, UErrorCode &status); 81 82 // Implementation for Whole Script tests. 83 // Return the test bit flag to be ORed into the eventual user return value 84 // if a Spoof opportunity is detected. 85 void wholeScriptCheck( 86 const UChar *text, int32_t length, ScriptSet *result, UErrorCode &status) const; 87 88 /** Scan a string to determine how many scripts it includes. 89 * Ignore characters with script=Common and scirpt=Inherited. 90 * @param text The UChar text to be scanned 91 * @param length The length of the input text, -1 for nul termintated. 92 * @param pos An out parameter, set to the first input postion at which 93 * a second script was encountered, ignoring Common and Inherited. 94 * @param status For errors. 95 * @return the number of (non-common,inherited) scripts encountered, 96 * clipped to a max of two. 97 */ 98 int32_t scriptScan(const UChar *text, int32_t length, int32_t &pos, UErrorCode &status) const; 99 100 101 // WholeScript and MixedScript check implementation. 102 // 103 ScriptSet *WholeScriptCheck(const UChar *text, int32_t length, UErrorCode &status) const; 104 105 static UClassID U_EXPORT2 getStaticClassID(void); 106 virtual UClassID getDynamicClassID(void) const; 107 108 // 109 // Data Members 110 // 111 112 int32_t fMagic; // Internal sanity check. 113 int32_t fChecks; // Bit vector of checks to perform. 114 115 SpoofData *fSpoofData; 116 117 int32_t fCheckMask; // Spoof table selector. f(Check Type) 118 119 const UnicodeSet *fAllowedCharsSet; // The UnicodeSet of allowed characters. 120 // for this Spoof Checker. Defaults to all chars. 121 122 const char *fAllowedLocales; // The list of allowed locales. 123 }; 124 125 126 127 // 128 // Confusable Mappings Data Structures 129 // 130 // For the confusable data, we are essentially implementing a map, 131 // key: a code point 132 // value: a string. Most commonly one char in length, but can be more. 133 // 134 // The keys are stored as a sorted array of 32 bit ints. 135 // bits 0-23 a code point value 136 // bits 24-31 flags 137 // 24: 1 if entry applies to SL table 138 // 25: 1 if entry applies to SA table 139 // 26: 1 if entry applies to ML table 140 // 27: 1 if entry applies to MA table 141 // 28: 1 if there are multiple entries for this code point. 142 // 29-30: length of value string, in UChars. 143 // values are (1, 2, 3, other) 144 // The key table is sorted in ascending code point order. (not on the 145 // 32 bit int value, the flag bits do not participate in the sorting.) 146 // 147 // Lookup is done by means of a binary search in the key table. 148 // 149 // The corresponding values are kept in a parallel array of 16 bit ints. 150 // If the value string is of length 1, it is literally in the value array. 151 // For longer strings, the value array contains an index into the strings table. 152 // 153 // String Table: 154 // The strings table contains all of the value strings (those of length two or greater) 155 // concatentated together into one long UChar (UTF-16) array. 156 // 157 // The array is arranged by length of the strings - all strings of the same length 158 // are stored together. The sections are ordered by length of the strings - 159 // all two char strings first, followed by all of the three Char strings, etc. 160 // 161 // There is no nul character or other mark between adjacent strings. 162 // 163 // String Lengths table 164 // The length of strings from 1 to 3 is flagged in the key table. 165 // For strings of length 4 or longer, the string length table provides a 166 // mapping between an index into the string table and the corresponding length. 167 // Strings of these lengths are rare, so lookup time is not an issue. 168 // Each entry consists of 169 // uint16_t index of the _last_ string with this length 170 // uint16_t the length 171 // 172 173 // Flag bits in the Key entries 174 #define USPOOF_SL_TABLE_FLAG (1<<24) 175 #define USPOOF_SA_TABLE_FLAG (1<<25) 176 #define USPOOF_ML_TABLE_FLAG (1<<26) 177 #define USPOOF_MA_TABLE_FLAG (1<<27) 178 #define USPOOF_KEY_MULTIPLE_VALUES (1<<28) 179 #define USPOOF_KEY_LENGTH_SHIFT 29 180 #define USPOOF_KEY_LENGTH_FIELD(x) (((x)>>29) & 3) 181 182 183 struct SpoofStringLengthsElement { 184 uint16_t fLastString; // index in string table of last string with this length 185 uint16_t fStrLength; // Length of strings 186 }; 187 188 189 //------------------------------------------------------------------------------- 190 // 191 // ScriptSet - Wrapper class for the Script code bit sets that are part of the 192 // whole script confusable data. 193 // 194 // This class is used both at data build and at run time. 195 // The constructor is only used at build time. 196 // At run time, just point at the prebuilt data and go. 197 // 198 //------------------------------------------------------------------------------- 199 class ScriptSet: public UMemory { 200 public: 201 ScriptSet(); 202 ~ScriptSet(); 203 204 UBool operator == (const ScriptSet &other); 205 ScriptSet & operator = (const ScriptSet &other); 206 207 void Union(const ScriptSet &other); 208 void Union(UScriptCode script); 209 void intersect(const ScriptSet &other); 210 void intersect(UScriptCode script); 211 void setAll(); 212 void resetAll(); 213 int32_t countMembers(); 214 215 private: 216 uint32_t bits[6]; 217 }; 218 219 220 221 222 //------------------------------------------------------------------------------- 223 // 224 // NFKDBuffer A little class to handle the NFKD normalization that is 225 // needed on incoming identifiers to be checked. 226 // Takes care of buffer handling and normalization 227 // 228 // Instances of this class are intended to be stack-allocated. 229 // 230 // TODO: how to map position offsets back to user values? 231 // 232 //-------------------------------------------------------------------------------- 233 class NFKDBuffer: public UMemory { 234 public: 235 NFKDBuffer(const UChar *text, int32_t length, UErrorCode &status); 236 ~NFKDBuffer(); 237 const UChar *getBuffer(); 238 int32_t getLength(); 239 240 private: 241 const UChar *fOriginalText; 242 UChar *fNormalizedText; 243 int32_t fNormalizedTextLength; 244 UChar fSmallBuf[USPOOF_STACK_BUFFER_SIZE]; 245 }; 246 247 248 249 250 251 //------------------------------------------------------------------------------------- 252 // 253 // SpoofData 254 // 255 // A small class that wraps the raw (usually memory mapped) spoof data. 256 // Serves two primary functions: 257 // 1. Convenience. Contains real pointers to the data, to avoid dealing with 258 // the offsets in the raw data. 259 // 2. Reference counting. When a spoof checker is cloned, the raw data is shared 260 // and must be retained until all checkers using the data are closed. 261 // Nothing in this struct includes state that is specific to any particular 262 // USpoofDetector object. 263 // 264 //--------------------------------------------------------------------------------------- 265 class SpoofData: public UMemory { 266 public: 267 static SpoofData *getDefault(UErrorCode &status); // Load standard ICU spoof data. 268 SpoofData(UErrorCode &status); // Create new spoof data wrapper. 269 // Only used when building new data from rules. 270 271 // Constructor for use when creating from prebuilt default data. 272 // A UDataMemory is what the ICU internal data loading functions provide. 273 // The udm is adopted by the SpoofData. 274 SpoofData(UDataMemory *udm, UErrorCode &status); 275 276 // Constructor for use when creating from serialized data. 277 // 278 SpoofData(const void *serializedData, int32_t length, UErrorCode &status); 279 280 // Check raw Spoof Data Version compatibility. 281 // Return TRUE it looks good. 282 static UBool validateDataVersion(const SpoofDataHeader *rawData, UErrorCode &status); 283 ~SpoofData(); // Destructor not normally used. 284 // Use removeReference() instead. 285 // Reference Counting functions. 286 // Clone of a user-level spoof detector increments the ref count on the data. 287 // Close of a user-level spoof detector decrements the ref count. 288 // If the data is owned by us, it will be deleted when count goes to zero. 289 SpoofData *addReference(); 290 void removeReference(); 291 292 // Reserve space in the raw data. For use by builder when putting together a 293 // new set of data. Init the new storage to zero, to prevent inconsistent 294 // results if it is not all otherwise set by the requester. 295 // Return: 296 // pointer to the new space that was added by this function. 297 void *reserveSpace(int32_t numBytes, UErrorCode &status); 298 299 // initialize the pointers from this object to the raw data. 300 void initPtrs(UErrorCode &status); 301 302 // Reset all fields to an initial state. 303 // Called from the top of all constructors. 304 void reset(); 305 306 SpoofDataHeader *fRawData; // Ptr to the raw memory-mapped data 307 UBool fDataOwned; // True if the raw data is owned, and needs 308 // to be deleted when refcount goes to zero. 309 UDataMemory *fUDM; // If not NULL, our data came from a 310 // UDataMemory, which we must close when 311 // we're done. 312 313 uint32_t fMemLimit; // Limit of available raw data space 314 int32_t fRefCount; 315 316 // Confusable data 317 int32_t *fCFUKeys; 318 uint16_t *fCFUValues; 319 SpoofStringLengthsElement *fCFUStringLengths; 320 UChar *fCFUStrings; 321 322 // Whole Script Confusable Data 323 UTrie2 *fAnyCaseTrie; 324 UTrie2 *fLowerCaseTrie; 325 ScriptSet *fScriptSets; 326 }; 327 328 329 //--------------------------------------------------------------------------------------- 330 // 331 // Raw Binary Data Formats, as loaded from the ICU data file, 332 // or as built by the builder. 333 // 334 //--------------------------------------------------------------------------------------- 335 struct SpoofDataHeader { 336 int32_t fMagic; // (0x3845fdef) 337 uint8_t fFormatVersion[4]; // Data Format. Same as the value in struct UDataInfo 338 // if there is one associated with this data. 339 int32_t fLength; // Total lenght in bytes of this spoof data, 340 // including all sections, not just the header. 341 342 // The following four sections refer to data representing the confusable data 343 // from the Unicode.org data from "confusables.txt" 344 345 int32_t fCFUKeys; // byte offset to Keys table (from SpoofDataHeader *) 346 int32_t fCFUKeysSize; // number of entries in keys table (32 bits each) 347 348 // TODO: change name to fCFUValues, for consistency. 349 int32_t fCFUStringIndex; // byte offset to String Indexes table 350 int32_t fCFUStringIndexSize; // number of entries in String Indexes table (16 bits each) 351 // (number of entries must be same as in Keys table 352 353 int32_t fCFUStringTable; // byte offset of String table 354 int32_t fCFUStringTableLen; // length of string table (in 16 bit UChars) 355 356 int32_t fCFUStringLengths; // byte offset to String Lengths table 357 int32_t fCFUStringLengthsSize; // number of entries in lengths table. (2 x 16 bits each) 358 359 360 // The following sections are for data from confusablesWholeScript.txt 361 362 int32_t fAnyCaseTrie; // byte offset to the serialized Any Case Trie 363 int32_t fAnyCaseTrieLength; // Length (bytes) of the serialized Any Case Trie 364 365 int32_t fLowerCaseTrie; // byte offset to the serialized Lower Case Trie 366 int32_t fLowerCaseTrieLength; // Length (bytes) of the serialized Lower Case Trie 367 368 int32_t fScriptSets; // byte offset to array of ScriptSets 369 int32_t fScriptSetsLength; // Number of ScriptSets (24 bytes each) 370 371 372 // The following sections are for data from xidmodifications.txt 373 374 375 int32_t unused[15]; // Padding, Room for Expansion 376 377 }; 378 379 380 381 382 // 383 // Structure for the Whole Script Confusable Data 384 // See Unicode UAX-39, Unicode Security Mechanisms, for a description of the 385 // Whole Script confusable data 386 // 387 // The data provides mappings from code points to a set of scripts 388 // that contain characters that might be confused with the code point. 389 // There are two mappings, one for lower case only, and one for characters 390 // of any case. 391 // 392 // The actual data consists of a utrie2 to map from a code point to an offset, 393 // and an array of UScriptSets (essentially bit maps) that is indexed 394 // by the offsets obtained from the Trie. 395 // 396 // 397 398 399 U_NAMESPACE_END 400 #endif /* XP_CPLUSPLUS */ 401 402 /** 403 * Endianness swap function for binary spoof data. 404 * @internal 405 */ 406 U_CAPI int32_t U_EXPORT2 407 uspoof_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData, 408 UErrorCode *status); 409 410 411 #endif 412 413 #endif /* USPOOFIM_H */ 414 415