1 /*
2 * Copyright (C) 2008, 2009 Google Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions are
6 * met:
7 *
8 * * Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * * Redistributions in binary form must reproduce the above
11 * copyright notice, this list of conditions and the following disclaimer
12 * in the documentation and/or other materials provided with the
13 * distribution.
14 * * Neither the name of Google Inc. nor the names of its
15 * contributors may be used to endorse or promote products derived from
16 * this software without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30
31 #include "config.h"
32 #include "TextEncodingDetector.h"
33
34 #include "TextEncoding.h"
35 #include <wtf/UnusedParam.h>
36
37 #ifndef BUILDING_ON_TIGER
38 #include "unicode/ucnv.h"
39 #include "unicode/ucsdet.h"
40 #endif
41
42 namespace WebCore {
43
detectTextEncoding(const char * data,size_t len,const char * hintEncodingName,TextEncoding * detectedEncoding)44 bool detectTextEncoding(const char* data, size_t len,
45 const char* hintEncodingName,
46 TextEncoding* detectedEncoding)
47 {
48 *detectedEncoding = TextEncoding();
49 #ifdef BUILDING_ON_TIGER
50 // Tiger came with ICU 3.2 and does not have the encoding detector.
51 UNUSED_PARAM(data);
52 UNUSED_PARAM(len);
53 UNUSED_PARAM(hintEncodingName);
54 return false;
55 #else
56 int matchesCount = 0;
57 UErrorCode status = U_ZERO_ERROR;
58 UCharsetDetector* detector = ucsdet_open(&status);
59 if (U_FAILURE(status))
60 return false;
61 ucsdet_enableInputFilter(detector, true);
62 ucsdet_setText(detector, data, static_cast<int32_t>(len), &status);
63 if (U_FAILURE(status))
64 return false;
65
66 // FIXME: A few things we can do other than improving
67 // the ICU detector itself.
68 // 1. Use ucsdet_detectAll and pick the most likely one given
69 // "the context" (parent-encoding, referrer encoding, etc).
70 // 2. 'Emulate' Firefox/IE's non-Universal detectors (e.g.
71 // Chinese, Japanese, Russian, Korean and Hebrew) by picking the
72 // encoding with a highest confidence among the detector-specific
73 // limited set of candidate encodings.
74 // Below is a partial implementation of the first part of what's outlined
75 // above.
76 const UCharsetMatch** matches = ucsdet_detectAll(detector, &matchesCount, &status);
77 if (U_FAILURE(status)) {
78 ucsdet_close(detector);
79 return false;
80 }
81
82 const char* encoding = 0;
83 if (hintEncodingName) {
84 TextEncoding hintEncoding(hintEncodingName);
85 // 10 is the minimum confidence value consistent with the codepoint
86 // allocation in a given encoding. The size of a chunk passed to
87 // us varies even for the same html file (apparently depending on
88 // the network load). When we're given a rather short chunk, we
89 // don't have a sufficiently reliable signal other than the fact that
90 // the chunk is consistent with a set of encodings. So, instead of
91 // setting an arbitrary threshold, we have to scan all the encodings
92 // consistent with the data.
93 const int32_t kThresold = 10;
94 for (int i = 0; i < matchesCount; ++i) {
95 int32_t confidence = ucsdet_getConfidence(matches[i], &status);
96 if (U_FAILURE(status)) {
97 status = U_ZERO_ERROR;
98 continue;
99 }
100 if (confidence < kThresold)
101 break;
102 const char* matchEncoding = ucsdet_getName(matches[i], &status);
103 if (U_FAILURE(status)) {
104 status = U_ZERO_ERROR;
105 continue;
106 }
107 if (TextEncoding(matchEncoding) == hintEncoding) {
108 encoding = hintEncodingName;
109 break;
110 }
111 }
112 }
113 // If no match is found so far, just pick the top match.
114 // This can happen, say, when a parent frame in EUC-JP refers to
115 // a child frame in Shift_JIS and both frames do NOT specify the encoding
116 // making us resort to auto-detection (when it IS turned on).
117 if (!encoding && matchesCount > 0)
118 encoding = ucsdet_getName(matches[0], &status);
119 if (U_SUCCESS(status)) {
120 *detectedEncoding = TextEncoding(encoding);
121 ucsdet_close(detector);
122 return true;
123 }
124 ucsdet_close(detector);
125 return false;
126 #endif
127 }
128
129 }
130