1 // Copyright 2011 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #ifdef UNSAFE_BUFFERS_BUILD
6 // TODO(crbug.com/40284755): Remove this and spanify to fix the errors.
7 #pragma allow_unsafe_buffers
8 #endif
9
10 #include "base/i18n/icu_string_conversions.h"
11
12 #include <math.h>
13 #include <stdarg.h>
14 #include <stddef.h>
15
16 #include <limits>
17 #include <sstream>
18
19 #include "base/check_op.h"
20 #include "base/format_macros.h"
21 #include "base/strings/stringprintf.h"
22 #include "base/strings/utf_string_conversions.h"
23 #include "build/build_config.h"
24 #include "testing/gtest/include/gtest/gtest.h"
25
26 namespace base {
27
28 namespace {
29
30 // Given a null-terminated string of wchar_t with each wchar_t representing
31 // a UTF-16 code unit, returns a std::u16string made up of wchar_t's in the
32 // input. Each wchar_t should be <= 0xFFFF and a non-BMP character (> U+FFFF)
33 // should be represented as a surrogate pair (two UTF-16 units)
34 // *even* where wchar_t is 32-bit (Linux and Mac).
35 //
36 // This is to help write tests for functions with std::u16string params until
37 // the C++ 0x UTF-16 literal is well-supported by compilers.
BuildString16(const wchar_t * s)38 std::u16string BuildString16(const wchar_t* s) {
39 #if defined(WCHAR_T_IS_16_BIT)
40 return WideToUTF16(s);
41 #elif defined(WCHAR_T_IS_32_BIT)
42 std::u16string u16;
43 while (*s != 0) {
44 DCHECK_LE(static_cast<unsigned int>(*s), 0xFFFFu);
45 u16.push_back(*s++);
46 }
47 return u16;
48 #endif
49 }
50
51 } // namespace
52
53 // kConverterCodepageCases is not comprehensive. There are a number of cases
54 // to add if we really want to have a comprehensive coverage of various
55 // codepages and their 'idiosyncrasies'. Currently, the only implementation
56 // for CodepageTo* and *ToCodepage uses ICU, which has a very extensive
57 // set of tests for the charset conversion. So, we can get away with a
58 // relatively small number of cases listed below.
59 //
60 // Note about |u16_wide| in the following struct.
61 // On Windows, the field is always identical to |wide|. On Mac and Linux,
62 // it's identical as long as there's no character outside the
63 // BMP (<= U+FFFF). When there is, it is different from |wide| and
64 // is not a real wide string (UTF-32 string) in that each wchar_t in
65 // the string is a UTF-16 code unit zero-extended to be 32-bit
66 // even when the code unit belongs to a surrogate pair.
67 // For instance, a Unicode string (U+0041 U+010000) is represented as
68 // L"\x0041\xD800\xDC00" instead of L"\x0041\x10000".
69 // To avoid the clutter, |u16_wide| will be set to NULL
70 // if it's identical to |wide| on *all* platforms.
71
72 static const struct {
73 const char* codepage_name;
74 const char* encoded;
75 OnStringConversionError::Type on_error;
76 bool success;
77 const wchar_t* wide;
78 const wchar_t* u16_wide;
79 } kConvertCodepageCases[] = {
80 // Test a case where the input cannot be decoded, using SKIP, FAIL
81 // and SUBSTITUTE error handling rules. "A7 41" is valid, but "A6" isn't.
82 {"big5", "\xA7\x41\xA6", OnStringConversionError::FAIL, false, L"",
83 nullptr},
84 {"big5", "\xA7\x41\xA6", OnStringConversionError::SKIP, true, L"\x4F60",
85 nullptr},
86 {"big5", "\xA7\x41\xA6", OnStringConversionError::SUBSTITUTE, true,
87 L"\x4F60\xFFFD", nullptr},
88 // Arabic (ISO-8859)
89 {"iso-8859-6",
90 "\xC7\xEE\xE4\xD3\xF1\xEE\xE4\xC7\xE5\xEF"
91 " "
92 "\xD9\xEE\xE4\xEE\xEA\xF2\xE3\xEF\xE5\xF2",
93 OnStringConversionError::FAIL, true,
94 L"\x0627\x064E\x0644\x0633\x0651\x064E\x0644\x0627\x0645\x064F"
95 L" "
96 L"\x0639\x064E\x0644\x064E\x064A\x0652\x0643\x064F\x0645\x0652",
97 nullptr},
98 // Chinese Simplified (GB2312)
99 {"gb2312", "\xC4\xE3\xBA\xC3", OnStringConversionError::FAIL, true,
100 L"\x4F60\x597D", nullptr},
101 // Chinese (GB18030) : 4 byte sequences mapped to BMP characters
102 {"gb18030", "\x81\x30\x84\x36\xA1\xA7", OnStringConversionError::FAIL, true,
103 L"\x00A5\x00A8", nullptr},
104 // Chinese (GB18030) : A 4 byte sequence mapped to plane 2 (U+20000)
105 {"gb18030", "\x95\x32\x82\x36\xD2\xBB", OnStringConversionError::FAIL, true,
106 #if defined(WCHAR_T_IS_16_BIT)
107 L"\xD840\xDC00\x4E00",
108 #elif defined(WCHAR_T_IS_32_BIT)
109 L"\x20000\x4E00",
110 #endif
111 L"\xD840\xDC00\x4E00"},
112 {"big5", "\xA7\x41\xA6\x6E", OnStringConversionError::FAIL, true,
113 L"\x4F60\x597D", nullptr},
114 // Greek (ISO-8859)
115 {"iso-8859-7",
116 "\xE3\xE5\xE9\xDC"
117 " "
118 "\xF3\xEF\xF5",
119 OnStringConversionError::FAIL, true,
120 L"\x03B3\x03B5\x03B9\x03AC"
121 L" "
122 L"\x03C3\x03BF\x03C5",
123 nullptr},
124 // Hebrew (Windows)
125 {"windows-1255", "\xF9\xD1\xC8\xEC\xE5\xC9\xED",
126 OnStringConversionError::FAIL, true,
127 L"\x05E9\x05C1\x05B8\x05DC\x05D5\x05B9\x05DD", nullptr},
128 // Korean (EUC)
129 {"euc-kr", "\xBE\xC8\xB3\xE7\xC7\xCF\xBC\xBC\xBF\xE4",
130 OnStringConversionError::FAIL, true, L"\xC548\xB155\xD558\xC138\xC694",
131 nullptr},
132 // Japanese (EUC)
133 {"euc-jp", "\xA4\xB3\xA4\xF3\xA4\xCB\xA4\xC1\xA4\xCF\xB0\xEC\x8E\xA6",
134 OnStringConversionError::FAIL, true,
135 L"\x3053\x3093\x306B\x3061\x306F\x4E00\xFF66", nullptr},
136 // Japanese (ISO-2022)
137 {"iso-2022-jp",
138 "\x1B$B"
139 "\x24\x33\x24\x73\x24\x4B\x24\x41\x24\x4F\x30\x6C"
140 "\x1B(B"
141 "ab"
142 "\x1B(J"
143 "\x5C\x7E#$"
144 "\x1B(B",
145 OnStringConversionError::FAIL, true,
146 L"\x3053\x3093\x306B\x3061\x306F\x4E00"
147 L"ab\x00A5\x203E#$",
148 nullptr},
149 // Japanese (Shift-JIS)
150 {"sjis", "\x82\xB1\x82\xF1\x82\xC9\x82\xBF\x82\xCD\x88\xEA\xA6",
151 OnStringConversionError::FAIL, true,
152 L"\x3053\x3093\x306B\x3061\x306F\x4E00\xFF66", nullptr},
153 // Russian (KOI8)
154 {"koi8-r", "\xDA\xC4\xD2\xC1\xD7\xD3\xD4\xD7\xD5\xCA\xD4\xC5",
155 OnStringConversionError::FAIL, true,
156 L"\x0437\x0434\x0440\x0430\x0432\x0441\x0442\x0432"
157 L"\x0443\x0439\x0442\x0435",
158 nullptr},
159 // Thai (windows-874)
160 {"windows-874",
161 "\xCA\xC7\xD1\xCA\xB4\xD5"
162 "\xA4\xC3\xD1\xBA",
163 OnStringConversionError::FAIL, true,
164 L"\x0E2A\x0E27\x0E31\x0E2A\x0E14\x0E35"
165 L"\x0E04\x0E23\x0e31\x0E1A",
166 nullptr},
167 };
168
TEST(ICUStringConversionsTest,ConvertBetweenCodepageAndUTF16)169 TEST(ICUStringConversionsTest, ConvertBetweenCodepageAndUTF16) {
170 for (size_t i = 0; i < std::size(kConvertCodepageCases); ++i) {
171 SCOPED_TRACE(base::StringPrintf(
172 "Test[%" PRIuS "]: <encoded: %s> <codepage: %s>", i,
173 kConvertCodepageCases[i].encoded,
174 kConvertCodepageCases[i].codepage_name));
175
176 std::u16string utf16;
177 bool success = CodepageToUTF16(kConvertCodepageCases[i].encoded,
178 kConvertCodepageCases[i].codepage_name,
179 kConvertCodepageCases[i].on_error,
180 &utf16);
181 std::u16string utf16_expected;
182 if (kConvertCodepageCases[i].u16_wide == nullptr)
183 utf16_expected = BuildString16(kConvertCodepageCases[i].wide);
184 else
185 utf16_expected = BuildString16(kConvertCodepageCases[i].u16_wide);
186 EXPECT_EQ(kConvertCodepageCases[i].success, success);
187 EXPECT_EQ(utf16_expected, utf16);
188
189 // When decoding was successful and nothing was skipped, we also check the
190 // reverse conversion. See also the corresponding comment in
191 // ConvertBetweenCodepageAndWide.
192 if (success &&
193 kConvertCodepageCases[i].on_error == OnStringConversionError::FAIL) {
194 std::string encoded;
195 success = UTF16ToCodepage(utf16, kConvertCodepageCases[i].codepage_name,
196 kConvertCodepageCases[i].on_error, &encoded);
197 EXPECT_EQ(kConvertCodepageCases[i].success, success);
198 EXPECT_EQ(kConvertCodepageCases[i].encoded, encoded);
199 }
200 }
201 }
202
203 static const struct {
204 const char* encoded;
205 const char* codepage_name;
206 bool expected_success;
207 const char* expected_value;
208 } kConvertAndNormalizeCases[] = {
209 {"foo-\xe4.html", "iso-8859-1", true, "foo-\xc3\xa4.html"},
210 {"foo-\xe4.html", "iso-8859-7", true, "foo-\xce\xb4.html"},
211 {"foo-\xe4.html", "foo-bar", false, ""},
212 // HTML Encoding spec treats US-ASCII as synonymous with windows-1252
213 {"foo-\xff.html", "ascii", true, "foo-\xc3\xbf.html"},
214 {"foo.html", "ascii", true, "foo.html"},
215 {"foo-a\xcc\x88.html", "utf-8", true, "foo-\xc3\xa4.html"},
216 {"\x95\x32\x82\x36\xD2\xBB", "gb18030", true, "\xF0\xA0\x80\x80\xE4\xB8\x80"},
217 {"\xA7\x41\xA6\x6E", "big5", true, "\xE4\xBD\xA0\xE5\xA5\xBD"},
218 // Windows-1258 does have a combining character at xD2 (which is U+0309).
219 // The sequence of (U+00E2, U+0309) is also encoded as U+1EA9.
220 {"foo\xE2\xD2", "windows-1258", true, "foo\xE1\xBA\xA9"},
221 {"", "iso-8859-1", true, ""},
222 };
TEST(ICUStringConversionsTest,ConvertToUtf8AndNormalize)223 TEST(ICUStringConversionsTest, ConvertToUtf8AndNormalize) {
224 std::string result;
225 for (size_t i = 0; i < std::size(kConvertAndNormalizeCases); ++i) {
226 SCOPED_TRACE(base::StringPrintf(
227 "Test[%" PRIuS "]: <encoded: %s> <codepage: %s>", i,
228 kConvertAndNormalizeCases[i].encoded,
229 kConvertAndNormalizeCases[i].codepage_name));
230
231 bool success = ConvertToUtf8AndNormalize(
232 kConvertAndNormalizeCases[i].encoded,
233 kConvertAndNormalizeCases[i].codepage_name, &result);
234 EXPECT_EQ(kConvertAndNormalizeCases[i].expected_success, success);
235 EXPECT_EQ(kConvertAndNormalizeCases[i].expected_value, result);
236 }
237 }
238
239 } // namespace base
240