1 /* 2 *************************************************************************** 3 * Copyright (C) 2008-2013, International Business Machines Corporation 4 * and others. All Rights Reserved. 5 *************************************************************************** 6 * 7 * uspoof_impl.h 8 * 9 * Implemenation header for spoof detection 10 * 11 */ 12 13 #ifndef USPOOFIM_H 14 #define USPOOFIM_H 15 16 #include "unicode/utypes.h" 17 #include "unicode/uspoof.h" 18 #include "unicode/uscript.h" 19 #include "unicode/udata.h" 20 21 #include "utrie2.h" 22 23 #if !UCONFIG_NO_NORMALIZATION 24 25 #ifdef __cplusplus 26 27 U_NAMESPACE_BEGIN 28 29 // The maximium length (in UTF-16 UChars) of the skeleton replacement string resulting from 30 // a single input code point. This is function of the unicode.org data. 31 #define USPOOF_MAX_SKELETON_EXPANSION 20 32 33 // The default stack buffer size for copies or conversions or normalizations 34 // of input strings being checked. (Used in multiple places.) 35 #define USPOOF_STACK_BUFFER_SIZE 100 36 37 // Magic number for sanity checking spoof data. 38 #define USPOOF_MAGIC 0x3845fdef 39 40 class IdentifierInfo; 41 class ScriptSet; 42 class SpoofData; 43 struct SpoofDataHeader; 44 struct SpoofStringLengthsElement; 45 46 /** 47 * Class SpoofImpl corresponds directly to the plain C API opaque type 48 * USpoofChecker. One can be cast to the other. 49 */ 50 class SpoofImpl : public UObject { 51 public: 52 SpoofImpl(SpoofData *data, UErrorCode &status); 53 SpoofImpl(); 54 virtual ~SpoofImpl(); 55 56 /** Copy constructor, used by the user level uspoof_clone() function. 57 */ 58 SpoofImpl(const SpoofImpl &src, UErrorCode &status); 59 60 static SpoofImpl *validateThis(USpoofChecker *sc, UErrorCode &status); 61 static const SpoofImpl *validateThis(const USpoofChecker *sc, UErrorCode &status); 62 63 /** Get the confusable skeleton transform for a single code point. 64 * The result is a string with a length between 1 and 18. 65 * @param tableMask bit flag specifying which confusable table to use. 66 * One of USPOOF_SL_TABLE_FLAG, USPOOF_MA_TABLE_FLAG, etc. 67 * @return The length in UTF-16 code units of the substition string. 68 */ 69 int32_t confusableLookup(UChar32 inChar, int32_t tableMask, UnicodeString &destBuf) const; 70 71 /** Set and Get AllowedLocales, implementations of the corresponding API */ 72 void setAllowedLocales(const char *localesList, UErrorCode &status); 73 const char * getAllowedLocales(UErrorCode &status); 74 75 // Add (union) to the UnicodeSet all of the characters for the scripts used for 76 // the specified locale. Part of the implementation of setAllowedLocales. 77 void addScriptChars(const char *locale, UnicodeSet *allowedChars, UErrorCode &status); 78 79 80 /** parse a hex number. Untility used by the builders. */ 81 static UChar32 ScanHex(const UChar *s, int32_t start, int32_t limit, UErrorCode &status); 82 83 // Implementation for Whole Script tests. 84 // Return the test bit flag to be ORed into the eventual user return value 85 // if a Spoof opportunity is detected. 86 void wholeScriptCheck( 87 const UnicodeString &text, ScriptSet *result, UErrorCode &status) const; 88 89 static UClassID U_EXPORT2 getStaticClassID(void); 90 virtual UClassID getDynamicClassID(void) const; 91 92 // IdentifierInfo Cache. IdentifierInfo objects are somewhat expensive to create. 93 // Maintain a one-element cache, which is sufficient to avoid repeatedly 94 // creating new ones unless we get multi-thread concurrency in spoof 95 // check operations, which should be statistically uncommon. 96 IdentifierInfo *getIdentifierInfo(UErrorCode &status) const; 97 void releaseIdentifierInfo(IdentifierInfo *idInfo) const; 98 99 // 100 // Data Members 101 // 102 103 int32_t fMagic; // Internal sanity check. 104 int32_t fChecks; // Bit vector of checks to perform. 105 106 SpoofData *fSpoofData; 107 108 const UnicodeSet *fAllowedCharsSet; // The UnicodeSet of allowed characters. 109 // for this Spoof Checker. Defaults to all chars. 110 111 const char *fAllowedLocales; // The list of allowed locales. 112 URestrictionLevel fRestrictionLevel; // The maximum restriction level for an acceptable identifier. 113 114 IdentifierInfo *fCachedIdentifierInfo; // Do not use directly. See getIdentifierInfo().:w 115 }; 116 117 118 119 // 120 // Confusable Mappings Data Structures 121 // 122 // For the confusable data, we are essentially implementing a map, 123 // key: a code point 124 // value: a string. Most commonly one char in length, but can be more. 125 // 126 // The keys are stored as a sorted array of 32 bit ints. 127 // bits 0-23 a code point value 128 // bits 24-31 flags 129 // 24: 1 if entry applies to SL table 130 // 25: 1 if entry applies to SA table 131 // 26: 1 if entry applies to ML table 132 // 27: 1 if entry applies to MA table 133 // 28: 1 if there are multiple entries for this code point. 134 // 29-30: length of value string, in UChars. 135 // values are (1, 2, 3, other) 136 // The key table is sorted in ascending code point order. (not on the 137 // 32 bit int value, the flag bits do not participate in the sorting.) 138 // 139 // Lookup is done by means of a binary search in the key table. 140 // 141 // The corresponding values are kept in a parallel array of 16 bit ints. 142 // If the value string is of length 1, it is literally in the value array. 143 // For longer strings, the value array contains an index into the strings table. 144 // 145 // String Table: 146 // The strings table contains all of the value strings (those of length two or greater) 147 // concatentated together into one long UChar (UTF-16) array. 148 // 149 // The array is arranged by length of the strings - all strings of the same length 150 // are stored together. The sections are ordered by length of the strings - 151 // all two char strings first, followed by all of the three Char strings, etc. 152 // 153 // There is no nul character or other mark between adjacent strings. 154 // 155 // String Lengths table 156 // The length of strings from 1 to 3 is flagged in the key table. 157 // For strings of length 4 or longer, the string length table provides a 158 // mapping between an index into the string table and the corresponding length. 159 // Strings of these lengths are rare, so lookup time is not an issue. 160 // Each entry consists of 161 // uint16_t index of the _last_ string with this length 162 // uint16_t the length 163 // 164 165 // Flag bits in the Key entries 166 #define USPOOF_SL_TABLE_FLAG (1<<24) 167 #define USPOOF_SA_TABLE_FLAG (1<<25) 168 #define USPOOF_ML_TABLE_FLAG (1<<26) 169 #define USPOOF_MA_TABLE_FLAG (1<<27) 170 #define USPOOF_KEY_MULTIPLE_VALUES (1<<28) 171 #define USPOOF_KEY_LENGTH_SHIFT 29 172 #define USPOOF_KEY_LENGTH_FIELD(x) (((x)>>29) & 3) 173 174 175 struct SpoofStringLengthsElement { 176 uint16_t fLastString; // index in string table of last string with this length 177 uint16_t fStrLength; // Length of strings 178 }; 179 180 181 182 //------------------------------------------------------------------------------------- 183 // 184 // SpoofData 185 // 186 // A small class that wraps the raw (usually memory mapped) spoof data. 187 // Serves two primary functions: 188 // 1. Convenience. Contains real pointers to the data, to avoid dealing with 189 // the offsets in the raw data. 190 // 2. Reference counting. When a spoof checker is cloned, the raw data is shared 191 // and must be retained until all checkers using the data are closed. 192 // Nothing in this struct includes state that is specific to any particular 193 // USpoofDetector object. 194 // 195 //--------------------------------------------------------------------------------------- 196 class SpoofData: public UMemory { 197 public: 198 static SpoofData *getDefault(UErrorCode &status); // Load standard ICU spoof data. 199 SpoofData(UErrorCode &status); // Create new spoof data wrapper. 200 // Only used when building new data from rules. 201 202 // Constructor for use when creating from prebuilt default data. 203 // A UDataMemory is what the ICU internal data loading functions provide. 204 // The udm is adopted by the SpoofData. 205 SpoofData(UDataMemory *udm, UErrorCode &status); 206 207 // Constructor for use when creating from serialized data. 208 // 209 SpoofData(const void *serializedData, int32_t length, UErrorCode &status); 210 211 // Check raw Spoof Data Version compatibility. 212 // Return TRUE it looks good. 213 static UBool validateDataVersion(const SpoofDataHeader *rawData, UErrorCode &status); 214 ~SpoofData(); // Destructor not normally used. 215 // Use removeReference() instead. 216 // Reference Counting functions. 217 // Clone of a user-level spoof detector increments the ref count on the data. 218 // Close of a user-level spoof detector decrements the ref count. 219 // If the data is owned by us, it will be deleted when count goes to zero. 220 SpoofData *addReference(); 221 void removeReference(); 222 223 // Reserve space in the raw data. For use by builder when putting together a 224 // new set of data. Init the new storage to zero, to prevent inconsistent 225 // results if it is not all otherwise set by the requester. 226 // Return: 227 // pointer to the new space that was added by this function. 228 void *reserveSpace(int32_t numBytes, UErrorCode &status); 229 230 // initialize the pointers from this object to the raw data. 231 void initPtrs(UErrorCode &status); 232 233 // Reset all fields to an initial state. 234 // Called from the top of all constructors. 235 void reset(); 236 237 SpoofDataHeader *fRawData; // Ptr to the raw memory-mapped data 238 UBool fDataOwned; // True if the raw data is owned, and needs 239 // to be deleted when refcount goes to zero. 240 UDataMemory *fUDM; // If not NULL, our data came from a 241 // UDataMemory, which we must close when 242 // we are done. 243 244 uint32_t fMemLimit; // Limit of available raw data space 245 u_atomic_int32_t fRefCount; 246 247 // Confusable data 248 int32_t *fCFUKeys; 249 uint16_t *fCFUValues; 250 SpoofStringLengthsElement *fCFUStringLengths; 251 UChar *fCFUStrings; 252 253 // Whole Script Confusable Data 254 UTrie2 *fAnyCaseTrie; 255 UTrie2 *fLowerCaseTrie; 256 ScriptSet *fScriptSets; 257 }; 258 259 260 //--------------------------------------------------------------------------------------- 261 // 262 // Raw Binary Data Formats, as loaded from the ICU data file, 263 // or as built by the builder. 264 // 265 //--------------------------------------------------------------------------------------- 266 struct SpoofDataHeader { 267 int32_t fMagic; // (0x3845fdef) 268 uint8_t fFormatVersion[4]; // Data Format. Same as the value in struct UDataInfo 269 // if there is one associated with this data. 270 int32_t fLength; // Total lenght in bytes of this spoof data, 271 // including all sections, not just the header. 272 273 // The following four sections refer to data representing the confusable data 274 // from the Unicode.org data from "confusables.txt" 275 276 int32_t fCFUKeys; // byte offset to Keys table (from SpoofDataHeader *) 277 int32_t fCFUKeysSize; // number of entries in keys table (32 bits each) 278 279 // TODO: change name to fCFUValues, for consistency. 280 int32_t fCFUStringIndex; // byte offset to String Indexes table 281 int32_t fCFUStringIndexSize; // number of entries in String Indexes table (16 bits each) 282 // (number of entries must be same as in Keys table 283 284 int32_t fCFUStringTable; // byte offset of String table 285 int32_t fCFUStringTableLen; // length of string table (in 16 bit UChars) 286 287 int32_t fCFUStringLengths; // byte offset to String Lengths table 288 int32_t fCFUStringLengthsSize; // number of entries in lengths table. (2 x 16 bits each) 289 290 291 // The following sections are for data from confusablesWholeScript.txt 292 293 int32_t fAnyCaseTrie; // byte offset to the serialized Any Case Trie 294 int32_t fAnyCaseTrieLength; // Length (bytes) of the serialized Any Case Trie 295 296 int32_t fLowerCaseTrie; // byte offset to the serialized Lower Case Trie 297 int32_t fLowerCaseTrieLength; // Length (bytes) of the serialized Lower Case Trie 298 299 int32_t fScriptSets; // byte offset to array of ScriptSets 300 int32_t fScriptSetsLength; // Number of ScriptSets (24 bytes each) 301 302 303 // The following sections are for data from xidmodifications.txt 304 305 306 int32_t unused[15]; // Padding, Room for Expansion 307 308 }; 309 310 311 312 313 // 314 // Structure for the Whole Script Confusable Data 315 // See Unicode UAX-39, Unicode Security Mechanisms, for a description of the 316 // Whole Script confusable data 317 // 318 // The data provides mappings from code points to a set of scripts 319 // that contain characters that might be confused with the code point. 320 // There are two mappings, one for lower case only, and one for characters 321 // of any case. 322 // 323 // The actual data consists of a utrie2 to map from a code point to an offset, 324 // and an array of UScriptSets (essentially bit maps) that is indexed 325 // by the offsets obtained from the Trie. 326 // 327 // 328 329 330 U_NAMESPACE_END 331 #endif /* __cplusplus */ 332 333 /** 334 * Endianness swap function for binary spoof data. 335 * @internal 336 */ 337 U_CAPI int32_t U_EXPORT2 338 uspoof_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData, 339 UErrorCode *status); 340 341 342 #endif 343 344 #endif /* USPOOFIM_H */ 345 346