1 // © 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
3 /*
4 **********************************************************************
5 * Copyright (C) 2005-2013, International Business Machines
6 * Corporation and others. All Rights Reserved.
7 **********************************************************************
8 */
9
10 #include "unicode/utypes.h"
11
12 #if !UCONFIG_NO_CONVERSION
13
14 #include "csrucode.h"
15 #include "csmatch.h"
16
17 U_NAMESPACE_BEGIN
18
~CharsetRecog_Unicode()19 CharsetRecog_Unicode::~CharsetRecog_Unicode()
20 {
21 // nothing to do
22 }
23
~CharsetRecog_UTF_16_BE()24 CharsetRecog_UTF_16_BE::~CharsetRecog_UTF_16_BE()
25 {
26 // nothing to do
27 }
28
getName() const29 const char *CharsetRecog_UTF_16_BE::getName() const
30 {
31 return "UTF-16BE";
32 }
33
34 // UTF-16 confidence calculation. Very simple minded, but better than nothing.
35 // Any 8 bit non-control characters bump the confidence up. These have a zero high byte,
36 // and are very likely to be UTF-16, although they could also be part of a UTF-32 code.
37 // NULs are a contra-indication, they will appear commonly if the actual encoding is UTF-32.
38 // NULs should be rare in actual text.
39
adjustConfidence(UChar codeUnit,int32_t confidence)40 static int32_t adjustConfidence(UChar codeUnit, int32_t confidence) {
41 if (codeUnit == 0) {
42 confidence -= 10;
43 } else if ((codeUnit >= 0x20 && codeUnit <= 0xff) || codeUnit == 0x0a) {
44 confidence += 10;
45 }
46 if (confidence < 0) {
47 confidence = 0;
48 } else if (confidence > 100) {
49 confidence = 100;
50 }
51 return confidence;
52 }
53
54
match(InputText * textIn,CharsetMatch * results) const55 UBool CharsetRecog_UTF_16_BE::match(InputText* textIn, CharsetMatch *results) const
56 {
57 const uint8_t *input = textIn->fRawInput;
58 int32_t confidence = 10;
59 int32_t length = textIn->fRawLength;
60
61 int32_t bytesToCheck = (length > 30) ? 30 : length;
62 for (int32_t charIndex=0; charIndex<bytesToCheck-1; charIndex+=2) {
63 UChar codeUnit = (input[charIndex] << 8) | input[charIndex + 1];
64 if (charIndex == 0 && codeUnit == 0xFEFF) {
65 confidence = 100;
66 break;
67 }
68 confidence = adjustConfidence(codeUnit, confidence);
69 if (confidence == 0 || confidence == 100) {
70 break;
71 }
72 }
73 if (bytesToCheck < 4 && confidence < 100) {
74 confidence = 0;
75 }
76 results->set(textIn, this, confidence);
77 return (confidence > 0);
78 }
79
~CharsetRecog_UTF_16_LE()80 CharsetRecog_UTF_16_LE::~CharsetRecog_UTF_16_LE()
81 {
82 // nothing to do
83 }
84
getName() const85 const char *CharsetRecog_UTF_16_LE::getName() const
86 {
87 return "UTF-16LE";
88 }
89
match(InputText * textIn,CharsetMatch * results) const90 UBool CharsetRecog_UTF_16_LE::match(InputText* textIn, CharsetMatch *results) const
91 {
92 const uint8_t *input = textIn->fRawInput;
93 int32_t confidence = 10;
94 int32_t length = textIn->fRawLength;
95
96 int32_t bytesToCheck = (length > 30) ? 30 : length;
97 for (int32_t charIndex=0; charIndex<bytesToCheck-1; charIndex+=2) {
98 UChar codeUnit = input[charIndex] | (input[charIndex + 1] << 8);
99 if (charIndex == 0 && codeUnit == 0xFEFF) {
100 confidence = 100; // UTF-16 BOM
101 if (length >= 4 && input[2] == 0 && input[3] == 0) {
102 confidence = 0; // UTF-32 BOM
103 }
104 break;
105 }
106 confidence = adjustConfidence(codeUnit, confidence);
107 if (confidence == 0 || confidence == 100) {
108 break;
109 }
110 }
111 if (bytesToCheck < 4 && confidence < 100) {
112 confidence = 0;
113 }
114 results->set(textIn, this, confidence);
115 return (confidence > 0);
116 }
117
~CharsetRecog_UTF_32()118 CharsetRecog_UTF_32::~CharsetRecog_UTF_32()
119 {
120 // nothing to do
121 }
122
match(InputText * textIn,CharsetMatch * results) const123 UBool CharsetRecog_UTF_32::match(InputText* textIn, CharsetMatch *results) const
124 {
125 const uint8_t *input = textIn->fRawInput;
126 int32_t limit = (textIn->fRawLength / 4) * 4;
127 int32_t numValid = 0;
128 int32_t numInvalid = 0;
129 bool hasBOM = false;
130 int32_t confidence = 0;
131
132 if (limit > 0 && getChar(input, 0) == 0x0000FEFFUL) {
133 hasBOM = true;
134 }
135
136 for(int32_t i = 0; i < limit; i += 4) {
137 int32_t ch = getChar(input, i);
138
139 if (ch < 0 || ch >= 0x10FFFF || (ch >= 0xD800 && ch <= 0xDFFF)) {
140 numInvalid += 1;
141 } else {
142 numValid += 1;
143 }
144 }
145
146
147 // Cook up some sort of confidence score, based on presence of a BOM
148 // and the existence of valid and/or invalid multi-byte sequences.
149 if (hasBOM && numInvalid==0) {
150 confidence = 100;
151 } else if (hasBOM && numValid > numInvalid*10) {
152 confidence = 80;
153 } else if (numValid > 3 && numInvalid == 0) {
154 confidence = 100;
155 } else if (numValid > 0 && numInvalid == 0) {
156 confidence = 80;
157 } else if (numValid > numInvalid*10) {
158 // Probably corrupt UTF-32BE data. Valid sequences aren't likely by chance.
159 confidence = 25;
160 }
161
162 results->set(textIn, this, confidence);
163 return (confidence > 0);
164 }
165
~CharsetRecog_UTF_32_BE()166 CharsetRecog_UTF_32_BE::~CharsetRecog_UTF_32_BE()
167 {
168 // nothing to do
169 }
170
getName() const171 const char *CharsetRecog_UTF_32_BE::getName() const
172 {
173 return "UTF-32BE";
174 }
175
getChar(const uint8_t * input,int32_t index) const176 int32_t CharsetRecog_UTF_32_BE::getChar(const uint8_t *input, int32_t index) const
177 {
178 return input[index + 0] << 24 | input[index + 1] << 16 |
179 input[index + 2] << 8 | input[index + 3];
180 }
181
~CharsetRecog_UTF_32_LE()182 CharsetRecog_UTF_32_LE::~CharsetRecog_UTF_32_LE()
183 {
184 // nothing to do
185 }
186
getName() const187 const char *CharsetRecog_UTF_32_LE::getName() const
188 {
189 return "UTF-32LE";
190 }
191
getChar(const uint8_t * input,int32_t index) const192 int32_t CharsetRecog_UTF_32_LE::getChar(const uint8_t *input, int32_t index) const
193 {
194 return input[index + 3] << 24 | input[index + 2] << 16 |
195 input[index + 1] << 8 | input[index + 0];
196 }
197
198 U_NAMESPACE_END
199 #endif
200
201