1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "base/i18n/icu_string_conversions.h"
6
7 #include <stddef.h>
8 #include <stdint.h>
9
10 #include <memory>
11 #include <vector>
12
13 #include "base/logging.h"
14 #include "base/strings/string_util.h"
15 #include "base/strings/utf_string_conversions.h"
16 #include "third_party/icu/source/common/unicode/normalizer2.h"
17 #include "third_party/icu/source/common/unicode/ucnv.h"
18 #include "third_party/icu/source/common/unicode/ucnv_cb.h"
19 #include "third_party/icu/source/common/unicode/ucnv_err.h"
20 #include "third_party/icu/source/common/unicode/ustring.h"
21
22 namespace base {
23
24 namespace {
25 // ToUnicodeCallbackSubstitute() is based on UCNV_TO_U_CALLBACK_SUBSTITUTE
26 // in source/common/ucnv_err.c.
27
28 // Copyright (c) 1995-2006 International Business Machines Corporation
29 // and others
30 //
31 // All rights reserved.
32 //
33
34 // Permission is hereby granted, free of charge, to any person obtaining a
35 // copy of this software and associated documentation files (the "Software"),
36 // to deal in the Software without restriction, including without limitation
37 // the rights to use, copy, modify, merge, publish, distribute, and/or
38 // sell copies of the Software, and to permit persons to whom the Software
39 // is furnished to do so, provided that the above copyright notice(s) and
40 // this permission notice appear in all copies of the Software and that
41 // both the above copyright notice(s) and this permission notice appear in
42 // supporting documentation.
43 //
44 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
45 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
46 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
47 // OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS
48 // INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT
49 // OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
50 // OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
51 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE
52 // OR PERFORMANCE OF THIS SOFTWARE.
53 //
54 // Except as contained in this notice, the name of a copyright holder
55 // shall not be used in advertising or otherwise to promote the sale, use
56 // or other dealings in this Software without prior written authorization
57 // of the copyright holder.
58
59 // ___________________________________________________________________________
60 //
61 // All trademarks and registered trademarks mentioned herein are the property
62 // of their respective owners.
63
ToUnicodeCallbackSubstitute(const void * context,UConverterToUnicodeArgs * to_args,const char * code_units,int32_t length,UConverterCallbackReason reason,UErrorCode * err)64 void ToUnicodeCallbackSubstitute(const void* context,
65 UConverterToUnicodeArgs *to_args,
66 const char* code_units,
67 int32_t length,
68 UConverterCallbackReason reason,
69 UErrorCode * err) {
70 static const UChar kReplacementChar = 0xFFFD;
71 if (reason <= UCNV_IRREGULAR) {
72 if (context == nullptr ||
73 (*(reinterpret_cast<const char*>(context)) == 'i' &&
74 reason == UCNV_UNASSIGNED)) {
75 *err = U_ZERO_ERROR;
76 ucnv_cbToUWriteUChars(to_args, &kReplacementChar, 1, 0, err);
77 }
78 // else the caller must have set the error code accordingly.
79 }
80 // else ignore the reset, close and clone calls.
81 }
82
ConvertFromUTF16(UConverter * converter,const UChar * uchar_src,int uchar_len,OnStringConversionError::Type on_error,std::string * encoded)83 bool ConvertFromUTF16(UConverter* converter, const UChar* uchar_src,
84 int uchar_len, OnStringConversionError::Type on_error,
85 std::string* encoded) {
86 int encoded_max_length = UCNV_GET_MAX_BYTES_FOR_STRING(uchar_len,
87 ucnv_getMaxCharSize(converter));
88 encoded->resize(encoded_max_length);
89
90 UErrorCode status = U_ZERO_ERROR;
91
92 // Setup our error handler.
93 switch (on_error) {
94 case OnStringConversionError::FAIL:
95 ucnv_setFromUCallBack(converter, UCNV_FROM_U_CALLBACK_STOP, nullptr,
96 nullptr, nullptr, &status);
97 break;
98 case OnStringConversionError::SKIP:
99 case OnStringConversionError::SUBSTITUTE:
100 ucnv_setFromUCallBack(converter, UCNV_FROM_U_CALLBACK_SKIP, nullptr,
101 nullptr, nullptr, &status);
102 break;
103 default:
104 NOTREACHED();
105 }
106
107 // ucnv_fromUChars returns size not including terminating null
108 int actual_size = ucnv_fromUChars(converter, &(*encoded)[0],
109 encoded_max_length, uchar_src, uchar_len, &status);
110 encoded->resize(actual_size);
111 ucnv_close(converter);
112 if (U_SUCCESS(status))
113 return true;
114 encoded->clear(); // Make sure the output is empty on error.
115 return false;
116 }
117
118 // Set up our error handler for ToUTF-16 converters
SetUpErrorHandlerForToUChars(OnStringConversionError::Type on_error,UConverter * converter,UErrorCode * status)119 void SetUpErrorHandlerForToUChars(OnStringConversionError::Type on_error,
120 UConverter* converter, UErrorCode* status) {
121 switch (on_error) {
122 case OnStringConversionError::FAIL:
123 ucnv_setToUCallBack(converter, UCNV_TO_U_CALLBACK_STOP, nullptr, nullptr,
124 nullptr, status);
125 break;
126 case OnStringConversionError::SKIP:
127 ucnv_setToUCallBack(converter, UCNV_TO_U_CALLBACK_SKIP, nullptr, nullptr,
128 nullptr, status);
129 break;
130 case OnStringConversionError::SUBSTITUTE:
131 ucnv_setToUCallBack(converter, ToUnicodeCallbackSubstitute, nullptr,
132 nullptr, nullptr, status);
133 break;
134 default:
135 NOTREACHED();
136 }
137 }
138
139 } // namespace
140
141 // Codepage <-> Wide/UTF-16 ---------------------------------------------------
142
UTF16ToCodepage(const string16 & utf16,const char * codepage_name,OnStringConversionError::Type on_error,std::string * encoded)143 bool UTF16ToCodepage(const string16& utf16,
144 const char* codepage_name,
145 OnStringConversionError::Type on_error,
146 std::string* encoded) {
147 encoded->clear();
148
149 UErrorCode status = U_ZERO_ERROR;
150 UConverter* converter = ucnv_open(codepage_name, &status);
151 if (!U_SUCCESS(status))
152 return false;
153
154 return ConvertFromUTF16(converter, utf16.c_str(),
155 static_cast<int>(utf16.length()), on_error, encoded);
156 }
157
CodepageToUTF16(const std::string & encoded,const char * codepage_name,OnStringConversionError::Type on_error,string16 * utf16)158 bool CodepageToUTF16(const std::string& encoded,
159 const char* codepage_name,
160 OnStringConversionError::Type on_error,
161 string16* utf16) {
162 utf16->clear();
163
164 UErrorCode status = U_ZERO_ERROR;
165 UConverter* converter = ucnv_open(codepage_name, &status);
166 if (!U_SUCCESS(status))
167 return false;
168
169 // Even in the worst case, the maximum length in 2-byte units of UTF-16
170 // output would be at most the same as the number of bytes in input. There
171 // is no single-byte encoding in which a character is mapped to a
172 // non-BMP character requiring two 2-byte units.
173 //
174 // Moreover, non-BMP characters in legacy multibyte encodings
175 // (e.g. EUC-JP, GB18030) take at least 2 bytes. The only exceptions are
176 // BOCU and SCSU, but we don't care about them.
177 size_t uchar_max_length = encoded.length() + 1;
178
179 SetUpErrorHandlerForToUChars(on_error, converter, &status);
180 std::unique_ptr<char16[]> buffer(new char16[uchar_max_length]);
181 int actual_size = ucnv_toUChars(converter, buffer.get(),
182 static_cast<int>(uchar_max_length), encoded.data(),
183 static_cast<int>(encoded.length()), &status);
184 ucnv_close(converter);
185 if (!U_SUCCESS(status)) {
186 utf16->clear(); // Make sure the output is empty on error.
187 return false;
188 }
189
190 utf16->assign(buffer.get(), actual_size);
191 return true;
192 }
193
ConvertToUtf8AndNormalize(const std::string & text,const std::string & charset,std::string * result)194 bool ConvertToUtf8AndNormalize(const std::string& text,
195 const std::string& charset,
196 std::string* result) {
197 result->clear();
198 string16 utf16;
199 if (!CodepageToUTF16(
200 text, charset.c_str(), OnStringConversionError::FAIL, &utf16))
201 return false;
202
203 UErrorCode status = U_ZERO_ERROR;
204 const icu::Normalizer2* normalizer = icu::Normalizer2::getNFCInstance(status);
205 DCHECK(U_SUCCESS(status));
206 if (U_FAILURE(status))
207 return false;
208 int32_t utf16_length = static_cast<int32_t>(utf16.length());
209 icu::UnicodeString normalized(utf16.data(), utf16_length);
210 int32_t normalized_prefix_length =
211 normalizer->spanQuickCheckYes(normalized, status);
212 if (normalized_prefix_length < utf16_length) {
213 icu::UnicodeString un_normalized(normalized, normalized_prefix_length);
214 normalized.truncate(normalized_prefix_length);
215 normalizer->normalizeSecondAndAppend(normalized, un_normalized, status);
216 }
217 if (U_FAILURE(status))
218 return false;
219 normalized.toUTF8String(*result);
220 return true;
221 }
222
223 } // namespace base
224