1 /*
2 * Copyright (C) 2006, 2007, 2011 Apple Inc. All rights reserved.
3 * Copyright (C) 2007-2009 Torch Mobile, Inc.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY
15 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE COMPUTER, INC. OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
21 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
22 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
24 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 #include "config.h"
28 #include "wtf/text/TextEncodingRegistry.h"
29
30 #include "wtf/ASCIICType.h"
31 #include "wtf/CurrentTime.h"
32 #include "wtf/HashMap.h"
33 #include "wtf/HashSet.h"
34 #include "wtf/MainThread.h"
35 #include "wtf/StdLibExtras.h"
36 #include "wtf/StringExtras.h"
37 #include "wtf/ThreadingPrimitives.h"
38 #include "wtf/text/CString.h"
39 #include "wtf/text/TextCodecICU.h"
40 #include "wtf/text/TextCodecLatin1.h"
41 #include "wtf/text/TextCodecUTF16.h"
42 #include "wtf/text/TextCodecUTF8.h"
43 #include "wtf/text/TextCodecUserDefined.h"
44 #include "wtf/text/TextEncoding.h"
45
46 namespace WTF {
47
48 const size_t maxEncodingNameLength = 63;
49
50 // Hash for all-ASCII strings that does case folding.
51 struct TextEncodingNameHash {
equalWTF::TextEncodingNameHash52 static bool equal(const char* s1, const char* s2)
53 {
54 char c1;
55 char c2;
56 do {
57 #if defined(_MSC_FULL_VER) && _MSC_FULL_VER == 170051106
58 // Workaround for a bug in the VS2012 Update 1 optimizer, remove once the fix is released.
59 // https://connect.microsoft.com/VisualStudio/feedback/details/777533/vs2012-c-optimizing-bug-when-using-inline-and-char-return-type-x86-target-only
60 c1 = toASCIILower(*s1++);
61 c2 = toASCIILower(*s2++);
62 if (c1 != c2)
63 return false;
64 #else
65 c1 = *s1++;
66 c2 = *s2++;
67 if (toASCIILower(c1) != toASCIILower(c2))
68 return false;
69 #endif
70 } while (c1 && c2);
71 return !c1 && !c2;
72 }
73
74 // This algorithm is the one-at-a-time hash from:
75 // http://burtleburtle.net/bob/hash/hashfaq.html
76 // http://burtleburtle.net/bob/hash/doobs.html
hashWTF::TextEncodingNameHash77 static unsigned hash(const char* s)
78 {
79 unsigned h = WTF::stringHashingStartValue;
80 for (;;) {
81 char c = *s++;
82 if (!c) {
83 h += (h << 3);
84 h ^= (h >> 11);
85 h += (h << 15);
86 return h;
87 }
88 h += toASCIILower(c);
89 h += (h << 10);
90 h ^= (h >> 6);
91 }
92 }
93
94 static const bool safeToCompareToEmptyOrDeleted = false;
95 };
96
97 struct TextCodecFactory {
98 NewTextCodecFunction function;
99 const void* additionalData;
TextCodecFactoryWTF::TextCodecFactory100 TextCodecFactory(NewTextCodecFunction f = 0, const void* d = 0) : function(f), additionalData(d) { }
101 };
102
103 typedef HashMap<const char*, const char*, TextEncodingNameHash> TextEncodingNameMap;
104 typedef HashMap<const char*, TextCodecFactory> TextCodecMap;
105
encodingRegistryMutex()106 static Mutex& encodingRegistryMutex()
107 {
108 // We don't have to use AtomicallyInitializedStatic here because
109 // this function is called on the main thread for any page before
110 // it is used in worker threads.
111 DEFINE_STATIC_LOCAL(Mutex, mutex, ());
112 return mutex;
113 }
114
115 static TextEncodingNameMap* textEncodingNameMap;
116 static TextCodecMap* textCodecMap;
117 static bool didExtendTextCodecMaps;
118 static HashSet<const char*>* japaneseEncodings;
119 static HashSet<const char*>* nonBackslashEncodings;
120
121 static const char textEncodingNameBlacklist[][6] = { "UTF-7" };
122
123 #if ERROR_DISABLED
124
checkExistingName(const char *,const char *)125 static inline void checkExistingName(const char*, const char*) { }
126
127 #else
128
checkExistingName(const char * alias,const char * atomicName)129 static void checkExistingName(const char* alias, const char* atomicName)
130 {
131 const char* oldAtomicName = textEncodingNameMap->get(alias);
132 if (!oldAtomicName)
133 return;
134 if (oldAtomicName == atomicName)
135 return;
136 // Keep the warning silent about one case where we know this will happen.
137 if (strcmp(alias, "ISO-8859-8-I") == 0
138 && strcmp(oldAtomicName, "ISO-8859-8-I") == 0
139 && strcasecmp(atomicName, "iso-8859-8") == 0)
140 return;
141 WTF_LOG_ERROR("alias %s maps to %s already, but someone is trying to make it map to %s", alias, oldAtomicName, atomicName);
142 }
143
144 #endif
145
isUndesiredAlias(const char * alias)146 static bool isUndesiredAlias(const char* alias)
147 {
148 // Reject aliases with version numbers that are supported by some back-ends (such as "ISO_2022,locale=ja,version=0" in ICU).
149 for (const char* p = alias; *p; ++p) {
150 if (*p == ',')
151 return true;
152 }
153 // 8859_1 is known to (at least) ICU, but other browsers don't support this name - and having it caused a compatibility
154 // problem, see bug 43554.
155 if (0 == strcmp(alias, "8859_1"))
156 return true;
157 return false;
158 }
159
addToTextEncodingNameMap(const char * alias,const char * name)160 static void addToTextEncodingNameMap(const char* alias, const char* name)
161 {
162 ASSERT(strlen(alias) <= maxEncodingNameLength);
163 if (isUndesiredAlias(alias))
164 return;
165 const char* atomicName = textEncodingNameMap->get(name);
166 ASSERT(strcmp(alias, name) == 0 || atomicName);
167 if (!atomicName)
168 atomicName = name;
169 checkExistingName(alias, atomicName);
170 textEncodingNameMap->add(alias, atomicName);
171 }
172
addToTextCodecMap(const char * name,NewTextCodecFunction function,const void * additionalData)173 static void addToTextCodecMap(const char* name, NewTextCodecFunction function, const void* additionalData)
174 {
175 const char* atomicName = textEncodingNameMap->get(name);
176 ASSERT(atomicName);
177 textCodecMap->add(atomicName, TextCodecFactory(function, additionalData));
178 }
179
pruneBlacklistedCodecs()180 static void pruneBlacklistedCodecs()
181 {
182 for (size_t i = 0; i < WTF_ARRAY_LENGTH(textEncodingNameBlacklist); ++i) {
183 const char* atomicName = textEncodingNameMap->get(textEncodingNameBlacklist[i]);
184 if (!atomicName)
185 continue;
186
187 Vector<const char*> names;
188 TextEncodingNameMap::const_iterator it = textEncodingNameMap->begin();
189 TextEncodingNameMap::const_iterator end = textEncodingNameMap->end();
190 for (; it != end; ++it) {
191 if (it->value == atomicName)
192 names.append(it->key);
193 }
194
195 size_t length = names.size();
196 for (size_t j = 0; j < length; ++j)
197 textEncodingNameMap->remove(names[j]);
198
199 textCodecMap->remove(atomicName);
200 }
201 }
202
buildBaseTextCodecMaps()203 static void buildBaseTextCodecMaps()
204 {
205 ASSERT(isMainThread());
206 ASSERT(!textCodecMap);
207 ASSERT(!textEncodingNameMap);
208
209 textCodecMap = new TextCodecMap;
210 textEncodingNameMap = new TextEncodingNameMap;
211
212 TextCodecLatin1::registerEncodingNames(addToTextEncodingNameMap);
213 TextCodecLatin1::registerCodecs(addToTextCodecMap);
214
215 TextCodecUTF8::registerEncodingNames(addToTextEncodingNameMap);
216 TextCodecUTF8::registerCodecs(addToTextCodecMap);
217
218 TextCodecUTF16::registerEncodingNames(addToTextEncodingNameMap);
219 TextCodecUTF16::registerCodecs(addToTextCodecMap);
220
221 TextCodecUserDefined::registerEncodingNames(addToTextEncodingNameMap);
222 TextCodecUserDefined::registerCodecs(addToTextCodecMap);
223 }
224
addEncodingName(HashSet<const char * > * set,const char * name)225 static void addEncodingName(HashSet<const char*>* set, const char* name)
226 {
227 // We must not use atomicCanonicalTextEncodingName() because this function is called in it.
228 const char* atomicName = textEncodingNameMap->get(name);
229 if (atomicName)
230 set->add(atomicName);
231 }
232
buildQuirksSets()233 static void buildQuirksSets()
234 {
235 // FIXME: Having isJapaneseEncoding() and shouldShowBackslashAsCurrencySymbolIn()
236 // and initializing the sets for them in TextEncodingRegistry.cpp look strange.
237
238 ASSERT(!japaneseEncodings);
239 ASSERT(!nonBackslashEncodings);
240
241 japaneseEncodings = new HashSet<const char*>;
242 addEncodingName(japaneseEncodings, "EUC-JP");
243 addEncodingName(japaneseEncodings, "ISO-2022-JP");
244 addEncodingName(japaneseEncodings, "ISO-2022-JP-1");
245 addEncodingName(japaneseEncodings, "ISO-2022-JP-2");
246 addEncodingName(japaneseEncodings, "ISO-2022-JP-3");
247 addEncodingName(japaneseEncodings, "JIS_C6226-1978");
248 addEncodingName(japaneseEncodings, "JIS_X0201");
249 addEncodingName(japaneseEncodings, "JIS_X0208-1983");
250 addEncodingName(japaneseEncodings, "JIS_X0208-1990");
251 addEncodingName(japaneseEncodings, "JIS_X0212-1990");
252 addEncodingName(japaneseEncodings, "Shift_JIS");
253 addEncodingName(japaneseEncodings, "Shift_JIS_X0213-2000");
254 addEncodingName(japaneseEncodings, "cp932");
255 addEncodingName(japaneseEncodings, "x-mac-japanese");
256
257 nonBackslashEncodings = new HashSet<const char*>;
258 // The text encodings below treat backslash as a currency symbol for IE compatibility.
259 // See http://blogs.msdn.com/michkap/archive/2005/09/17/469941.aspx for more information.
260 addEncodingName(nonBackslashEncodings, "x-mac-japanese");
261 addEncodingName(nonBackslashEncodings, "ISO-2022-JP");
262 addEncodingName(nonBackslashEncodings, "EUC-JP");
263 // Shift_JIS_X0213-2000 is not the same encoding as Shift_JIS on Mac. We need to register both of them.
264 addEncodingName(nonBackslashEncodings, "Shift_JIS");
265 addEncodingName(nonBackslashEncodings, "Shift_JIS_X0213-2000");
266 }
267
isJapaneseEncoding(const char * canonicalEncodingName)268 bool isJapaneseEncoding(const char* canonicalEncodingName)
269 {
270 return canonicalEncodingName && japaneseEncodings && japaneseEncodings->contains(canonicalEncodingName);
271 }
272
shouldShowBackslashAsCurrencySymbolIn(const char * canonicalEncodingName)273 bool shouldShowBackslashAsCurrencySymbolIn(const char* canonicalEncodingName)
274 {
275 return canonicalEncodingName && nonBackslashEncodings && nonBackslashEncodings->contains(canonicalEncodingName);
276 }
277
extendTextCodecMaps()278 static void extendTextCodecMaps()
279 {
280 TextCodecICU::registerEncodingNames(addToTextEncodingNameMap);
281 TextCodecICU::registerCodecs(addToTextCodecMap);
282
283 pruneBlacklistedCodecs();
284 buildQuirksSets();
285 }
286
newTextCodec(const TextEncoding & encoding)287 PassOwnPtr<TextCodec> newTextCodec(const TextEncoding& encoding)
288 {
289 MutexLocker lock(encodingRegistryMutex());
290
291 ASSERT(textCodecMap);
292 TextCodecFactory factory = textCodecMap->get(encoding.name());
293 ASSERT(factory.function);
294 return factory.function(encoding, factory.additionalData);
295 }
296
atomicCanonicalTextEncodingName(const char * name)297 const char* atomicCanonicalTextEncodingName(const char* name)
298 {
299 if (!name || !name[0])
300 return 0;
301 if (!textEncodingNameMap)
302 buildBaseTextCodecMaps();
303
304 MutexLocker lock(encodingRegistryMutex());
305
306 if (const char* atomicName = textEncodingNameMap->get(name))
307 return atomicName;
308 if (didExtendTextCodecMaps)
309 return 0;
310 extendTextCodecMaps();
311 didExtendTextCodecMaps = true;
312 return textEncodingNameMap->get(name);
313 }
314
315 template <typename CharacterType>
atomicCanonicalTextEncodingName(const CharacterType * characters,size_t length)316 const char* atomicCanonicalTextEncodingName(const CharacterType* characters, size_t length)
317 {
318 char buffer[maxEncodingNameLength + 1];
319 size_t j = 0;
320 for (size_t i = 0; i < length; ++i) {
321 CharacterType c = characters[i];
322 if (j == maxEncodingNameLength)
323 return 0;
324 buffer[j++] = c;
325 }
326 buffer[j] = 0;
327 return atomicCanonicalTextEncodingName(buffer);
328 }
329
atomicCanonicalTextEncodingName(const String & alias)330 const char* atomicCanonicalTextEncodingName(const String& alias)
331 {
332 if (!alias.length())
333 return 0;
334
335 if (alias.is8Bit())
336 return atomicCanonicalTextEncodingName<LChar>(alias.characters8(), alias.length());
337
338 return atomicCanonicalTextEncodingName<UChar>(alias.characters16(), alias.length());
339 }
340
noExtendedTextEncodingNameUsed()341 bool noExtendedTextEncodingNameUsed()
342 {
343 // If the calling thread did not use extended encoding names, it is fine for it to use a stale false value.
344 return !didExtendTextCodecMaps;
345 }
346
347 #ifndef NDEBUG
dumpTextEncodingNameMap()348 void dumpTextEncodingNameMap()
349 {
350 unsigned size = textEncodingNameMap->size();
351 fprintf(stderr, "Dumping %u entries in WTF::TextEncodingNameMap...\n", size);
352
353 MutexLocker lock(encodingRegistryMutex());
354
355 TextEncodingNameMap::const_iterator it = textEncodingNameMap->begin();
356 TextEncodingNameMap::const_iterator end = textEncodingNameMap->end();
357 for (; it != end; ++it)
358 fprintf(stderr, "'%s' => '%s'\n", it->key, it->value);
359 }
360 #endif
361
362 } // namespace WTF
363