1 // © 2016 and later: Unicode, Inc. and others. 2 // License & terms of use: http://www.unicode.org/copyright.html 3 /* 4 *************************************************************************** 5 * Copyright (C) 2008-2013, International Business Machines Corporation 6 * and others. All Rights Reserved. 7 *************************************************************************** 8 * 9 * uspoof_impl.h 10 * 11 * Implemenation header for spoof detection 12 * 13 */ 14 15 #ifndef USPOOFIM_H 16 #define USPOOFIM_H 17 18 #include "uassert.h" 19 #include "unicode/utypes.h" 20 #include "unicode/uspoof.h" 21 #include "unicode/uscript.h" 22 #include "unicode/udata.h" 23 #include "udataswp.h" 24 #include "utrie2.h" 25 26 #if !UCONFIG_NO_NORMALIZATION 27 28 #ifdef __cplusplus 29 30 #include "capi_helper.h" 31 32 U_NAMESPACE_BEGIN 33 34 // The maximium length (in UTF-16 UChars) of the skeleton replacement string resulting from 35 // a single input code point. This is function of the unicode.org data. 36 #define USPOOF_MAX_SKELETON_EXPANSION 20 37 38 // The default stack buffer size for copies or conversions or normalizations 39 // of input strings being checked. (Used in multiple places.) 40 #define USPOOF_STACK_BUFFER_SIZE 100 41 42 // Magic number for sanity checking spoof data. 43 #define USPOOF_MAGIC 0x3845fdef 44 45 // Magic number for sanity checking spoof checkers. 46 #define USPOOF_CHECK_MAGIC 0x2734ecde 47 48 class ScriptSet; 49 class SpoofData; 50 struct SpoofDataHeader; 51 class ConfusableDataUtils; 52 53 /** 54 * Class SpoofImpl corresponds directly to the plain C API opaque type 55 * USpoofChecker. One can be cast to the other. 56 */ 57 class SpoofImpl : public UObject, 58 public IcuCApiHelper<USpoofChecker, SpoofImpl, USPOOF_MAGIC> { 59 public: 60 SpoofImpl(SpoofData *data, UErrorCode& status); 61 SpoofImpl(UErrorCode& status); 62 SpoofImpl(); 63 void construct(UErrorCode& status); 64 virtual ~SpoofImpl(); 65 66 /** Copy constructor, used by the user level uspoof_clone() function. 67 */ 68 SpoofImpl(const SpoofImpl &src, UErrorCode &status); 69 70 USpoofChecker *asUSpoofChecker(); 71 static SpoofImpl *validateThis(USpoofChecker *sc, UErrorCode &status); 72 static const SpoofImpl *validateThis(const USpoofChecker *sc, UErrorCode &status); 73 74 /** Set and Get AllowedLocales, implementations of the corresponding API */ 75 void setAllowedLocales(const char *localesList, UErrorCode &status); 76 const char * getAllowedLocales(UErrorCode &status); 77 78 // Add (union) to the UnicodeSet all of the characters for the scripts used for 79 // the specified locale. Part of the implementation of setAllowedLocales. 80 void addScriptChars(const char *locale, UnicodeSet *allowedChars, UErrorCode &status); 81 82 // Functions implementing the features of UTS 39 section 5. 83 static void getAugmentedScriptSet(UChar32 codePoint, ScriptSet& result, UErrorCode& status); 84 void getResolvedScriptSet(const UnicodeString& input, ScriptSet& result, UErrorCode& status) const; 85 void getResolvedScriptSetWithout(const UnicodeString& input, UScriptCode script, ScriptSet& result, UErrorCode& status) const; 86 void getNumerics(const UnicodeString& input, UnicodeSet& result, UErrorCode& status) const; 87 URestrictionLevel getRestrictionLevel(const UnicodeString& input, UErrorCode& status) const; 88 89 int32_t findHiddenOverlay(const UnicodeString& input, UErrorCode& status) const; 90 bool isIllegalCombiningDotLeadCharacter(UChar32 cp) const; 91 92 /** parse a hex number. Untility used by the builders. */ 93 static UChar32 ScanHex(const UChar *s, int32_t start, int32_t limit, UErrorCode &status); 94 95 static UClassID U_EXPORT2 getStaticClassID(void); 96 virtual UClassID getDynamicClassID(void) const; 97 98 // 99 // Data Members 100 // 101 102 int32_t fChecks; // Bit vector of checks to perform. 103 104 SpoofData *fSpoofData; 105 106 const UnicodeSet *fAllowedCharsSet; // The UnicodeSet of allowed characters. 107 // for this Spoof Checker. Defaults to all chars. 108 109 const char *fAllowedLocales; // The list of allowed locales. 110 URestrictionLevel fRestrictionLevel; // The maximum restriction level for an acceptable identifier. 111 }; 112 113 /** 114 * Class CheckResult corresponds directly to the plain C API opaque type 115 * USpoofCheckResult. One can be cast to the other. 116 */ 117 class CheckResult : public UObject, 118 public IcuCApiHelper<USpoofCheckResult, CheckResult, USPOOF_CHECK_MAGIC> { 119 public: 120 CheckResult(); 121 virtual ~CheckResult(); 122 123 USpoofCheckResult *asUSpoofCheckResult(); 124 static CheckResult *validateThis(USpoofCheckResult *ptr, UErrorCode &status); 125 static const CheckResult *validateThis(const USpoofCheckResult *ptr, UErrorCode &status); 126 127 void clear(); 128 129 // Used to convert this CheckResult to the older int32_t return value API 130 int32_t toCombinedBitmask(int32_t expectedChecks); 131 132 // Data Members 133 int32_t fChecks; // Bit vector of checks that were failed. 134 UnicodeSet fNumerics; // Set of numerics found in the string. 135 URestrictionLevel fRestrictionLevel; // The restriction level of the string. 136 }; 137 138 139 // 140 // Confusable Mappings Data Structures, version 2.0 141 // 142 // For the confusable data, we are essentially implementing a map, 143 // key: a code point 144 // value: a string. Most commonly one char in length, but can be more. 145 // 146 // The keys are stored as a sorted array of 32 bit ints. 147 // bits 0-23 a code point value 148 // bits 24-31 length of value string, in UChars (between 1 and 256 UChars). 149 // The key table is sorted in ascending code point order. (not on the 150 // 32 bit int value, the flag bits do not participate in the sorting.) 151 // 152 // Lookup is done by means of a binary search in the key table. 153 // 154 // The corresponding values are kept in a parallel array of 16 bit ints. 155 // If the value string is of length 1, it is literally in the value array. 156 // For longer strings, the value array contains an index into the strings table. 157 // 158 // String Table: 159 // The strings table contains all of the value strings (those of length two or greater) 160 // concatentated together into one long UChar (UTF-16) array. 161 // 162 // There is no nul character or other mark between adjacent strings. 163 // 164 //---------------------------------------------------------------------------- 165 // 166 // Changes from format version 1 to format version 2: 167 // 1) Removal of the whole-script confusable data tables. 168 // 2) Removal of the SL/SA/ML/MA and multi-table flags in the key bitmask. 169 // 3) Expansion of string length value in the key bitmask from 2 bits to 8 bits. 170 // 4) Removal of the string lengths table since 8 bits is sufficient for the 171 // lengths of all entries in confusables.txt. 172 173 174 175 // Internal functions for manipulating confusable data table keys 176 #define USPOOF_CONFUSABLE_DATA_FORMAT_VERSION 2 // version for ICU 58 177 class ConfusableDataUtils { 178 public: keyToCodePoint(int32_t key)179 inline static UChar32 keyToCodePoint(int32_t key) { 180 return key & 0x00ffffff; 181 } keyToLength(int32_t key)182 inline static int32_t keyToLength(int32_t key) { 183 return ((key & 0xff000000) >> 24) + 1; 184 } codePointAndLengthToKey(UChar32 codePoint,int32_t length)185 inline static int32_t codePointAndLengthToKey(UChar32 codePoint, int32_t length) { 186 U_ASSERT((codePoint & 0x00ffffff) == codePoint); 187 U_ASSERT(length <= 256); 188 return codePoint | ((length - 1) << 24); 189 } 190 }; 191 192 193 //------------------------------------------------------------------------------------- 194 // 195 // SpoofData 196 // 197 // A small class that wraps the raw (usually memory mapped) spoof data. 198 // Serves two primary functions: 199 // 1. Convenience. Contains real pointers to the data, to avoid dealing with 200 // the offsets in the raw data. 201 // 2. Reference counting. When a spoof checker is cloned, the raw data is shared 202 // and must be retained until all checkers using the data are closed. 203 // Nothing in this struct includes state that is specific to any particular 204 // USpoofDetector object. 205 // 206 //--------------------------------------------------------------------------------------- 207 class SpoofData: public UMemory { 208 public: 209 static SpoofData* getDefault(UErrorCode &status); // Get standard ICU spoof data. 210 static void releaseDefault(); // Cleanup reference to default spoof data. 211 212 SpoofData(UErrorCode &status); // Create new spoof data wrapper. 213 // Only used when building new data from rules. 214 215 // Constructor for use when creating from prebuilt default data. 216 // A UDataMemory is what the ICU internal data loading functions provide. 217 // The udm is adopted by the SpoofData. 218 SpoofData(UDataMemory *udm, UErrorCode &status); 219 220 // Constructor for use when creating from serialized data. 221 // 222 SpoofData(const void *serializedData, int32_t length, UErrorCode &status); 223 224 // Check raw Spoof Data Version compatibility. 225 // Return TRUE it looks good. 226 UBool validateDataVersion(UErrorCode &status) const; 227 228 ~SpoofData(); // Destructor not normally used. 229 // Use removeReference() instead. 230 // Reference Counting functions. 231 // Clone of a user-level spoof detector increments the ref count on the data. 232 // Close of a user-level spoof detector decrements the ref count. 233 // If the data is owned by us, it will be deleted when count goes to zero. 234 SpoofData *addReference(); 235 void removeReference(); 236 237 // Reset all fields to an initial state. 238 // Called from the top of all constructors. 239 void reset(); 240 241 // Copy this instance's raw data buffer to the specified address. 242 int32_t serialize(void *buf, int32_t capacity, UErrorCode &status) const; 243 244 // Get the total number of bytes of data backed by this SpoofData. 245 // Not to be confused with length, which returns the number of confusable entries. 246 int32_t size() const; 247 248 // Get the confusable skeleton transform for a single code point. 249 // The result is a string with a length between 1 and 18 as of Unicode 9. 250 // This is the main public endpoint for this class. 251 // @return The length in UTF-16 code units of the substition string. 252 int32_t confusableLookup(UChar32 inChar, UnicodeString &dest) const; 253 254 // Get the number of confusable entries in this SpoofData. 255 int32_t length() const; 256 257 // Get the code point (key) at the specified index. 258 UChar32 codePointAt(int32_t index) const; 259 260 // Get the confusable skeleton (value) at the specified index. 261 // Append it to the specified UnicodeString&. 262 // @return The length in UTF-16 code units of the skeleton string. 263 int32_t appendValueTo(int32_t index, UnicodeString& dest) const; 264 265 private: 266 // Reserve space in the raw data. For use by builder when putting together a 267 // new set of data. Init the new storage to zero, to prevent inconsistent 268 // results if it is not all otherwise set by the requester. 269 // Return: 270 // pointer to the new space that was added by this function. 271 void *reserveSpace(int32_t numBytes, UErrorCode &status); 272 273 // initialize the pointers from this object to the raw data. 274 void initPtrs(UErrorCode &status); 275 276 SpoofDataHeader *fRawData; // Ptr to the raw memory-mapped data 277 UBool fDataOwned; // True if the raw data is owned, and needs 278 // to be deleted when refcount goes to zero. 279 UDataMemory *fUDM; // If not NULL, our data came from a 280 // UDataMemory, which we must close when 281 // we are done. 282 283 uint32_t fMemLimit; // Limit of available raw data space 284 u_atomic_int32_t fRefCount; 285 286 // Confusable data 287 int32_t *fCFUKeys; 288 uint16_t *fCFUValues; 289 UChar *fCFUStrings; 290 291 friend class ConfusabledataBuilder; 292 }; 293 294 //--------------------------------------------------------------------------------------- 295 // 296 // Raw Binary Data Formats, as loaded from the ICU data file, 297 // or as built by the builder. 298 // 299 //--------------------------------------------------------------------------------------- 300 struct SpoofDataHeader { 301 int32_t fMagic; // (0x3845fdef) 302 uint8_t fFormatVersion[4]; // Data Format. Same as the value in struct UDataInfo 303 // if there is one associated with this data. 304 int32_t fLength; // Total lenght in bytes of this spoof data, 305 // including all sections, not just the header. 306 307 // The following four sections refer to data representing the confusable data 308 // from the Unicode.org data from "confusables.txt" 309 310 int32_t fCFUKeys; // byte offset to Keys table (from SpoofDataHeader *) 311 int32_t fCFUKeysSize; // number of entries in keys table (32 bits each) 312 313 // TODO: change name to fCFUValues, for consistency. 314 int32_t fCFUStringIndex; // byte offset to String Indexes table 315 int32_t fCFUStringIndexSize; // number of entries in String Indexes table (16 bits each) 316 // (number of entries must be same as in Keys table 317 318 int32_t fCFUStringTable; // byte offset of String table 319 int32_t fCFUStringTableLen; // length of string table (in 16 bit UChars) 320 321 // The following sections are for data from xidmodifications.txt 322 323 int32_t unused[15]; // Padding, Room for Expansion 324 }; 325 326 327 328 U_NAMESPACE_END 329 #endif /* __cplusplus */ 330 331 /** 332 * Endianness swap function for binary spoof data. 333 * @internal 334 */ 335 U_CAPI int32_t U_EXPORT2 336 uspoof_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData, 337 UErrorCode *status); 338 339 340 #endif 341 342 #endif /* USPOOFIM_H */ 343 344