1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "components/translate/core/browser/translate_language_list.h"
6
7 #include <set>
8
9 #include "base/bind.h"
10 #include "base/json/json_reader.h"
11 #include "base/lazy_instance.h"
12 #include "base/logging.h"
13 #include "base/strings/string_util.h"
14 #include "base/strings/stringprintf.h"
15 #include "base/values.h"
16 #include "components/translate/core/browser/translate_browser_metrics.h"
17 #include "components/translate/core/browser/translate_download_manager.h"
18 #include "components/translate/core/browser/translate_event_details.h"
19 #include "components/translate/core/browser/translate_url_fetcher.h"
20 #include "components/translate/core/browser/translate_url_util.h"
21 #include "components/translate/core/common/translate_util.h"
22 #include "net/base/url_util.h"
23 #include "ui/base/l10n/l10n_util.h"
24 #include "url/gurl.h"
25
26 namespace {
27
28 // The default list of languages the Google translation server supports.
29 // We use this list until we receive the list that the server exposes.
30 // For information, here is the list of languages that Chrome can be run in
31 // but that the translation server does not support:
32 // am Amharic
33 // bn Bengali
34 // gu Gujarati
35 // kn Kannada
36 // ml Malayalam
37 // mr Marathi
38 // ta Tamil
39 // te Telugu
40 const char* const kDefaultSupportedLanguages[] = {
41 "af", // Afrikaans
42 "sq", // Albanian
43 "ar", // Arabic
44 "be", // Belarusian
45 "bg", // Bulgarian
46 "ca", // Catalan
47 "zh-CN", // Chinese (Simplified)
48 "zh-TW", // Chinese (Traditional)
49 "hr", // Croatian
50 "cs", // Czech
51 "da", // Danish
52 "nl", // Dutch
53 "en", // English
54 "eo", // Esperanto
55 "et", // Estonian
56 "tl", // Filipino
57 "fi", // Finnish
58 "fr", // French
59 "gl", // Galician
60 "de", // German
61 "el", // Greek
62 "ht", // Haitian Creole
63 "iw", // Hebrew
64 "hi", // Hindi
65 "hu", // Hungarian
66 "is", // Icelandic
67 "id", // Indonesian
68 "ga", // Irish
69 "it", // Italian
70 "ja", // Japanese
71 "ko", // Korean
72 "lv", // Latvian
73 "lt", // Lithuanian
74 "mk", // Macedonian
75 "ms", // Malay
76 "mt", // Maltese
77 "no", // Norwegian
78 "fa", // Persian
79 "pl", // Polish
80 "pt", // Portuguese
81 "ro", // Romanian
82 "ru", // Russian
83 "sr", // Serbian
84 "sk", // Slovak
85 "sl", // Slovenian
86 "es", // Spanish
87 "sw", // Swahili
88 "sv", // Swedish
89 "th", // Thai
90 "tr", // Turkish
91 "uk", // Ukrainian
92 "vi", // Vietnamese
93 "cy", // Welsh
94 "yi", // Yiddish
95 };
96
97 // Constant URL string to fetch server supporting language list.
98 const char kLanguageListFetchPath[] = "translate_a/l?client=chrome&cb=sl";
99
100 // Used in kTranslateScriptURL to request supporting languages list including
101 // "alpha languages".
102 const char kAlphaLanguageQueryName[] = "alpha";
103 const char kAlphaLanguageQueryValue[] = "1";
104
105 // Represent if the language list updater is disabled.
106 bool update_is_disabled = false;
107
108 // Retry parameter for fetching.
109 const int kMaxRetryOn5xx = 5;
110
111 } // namespace
112
113 // This must be kept in sync with the &cb= value in the kLanguageListFetchURL.
114 const char TranslateLanguageList::kLanguageListCallbackName[] = "sl(";
115 const char TranslateLanguageList::kTargetLanguagesKey[] = "tl";
116 const char TranslateLanguageList::kAlphaLanguagesKey[] = "al";
117
TranslateLanguageList()118 TranslateLanguageList::TranslateLanguageList()
119 : resource_requests_allowed_(false), request_pending_(false) {
120 // We default to our hard coded list of languages in
121 // |kDefaultSupportedLanguages|. This list will be overriden by a server
122 // providing supported langauges list.
123 for (size_t i = 0; i < arraysize(kDefaultSupportedLanguages); ++i)
124 all_supported_languages_.insert(kDefaultSupportedLanguages[i]);
125
126 if (update_is_disabled)
127 return;
128
129 language_list_fetcher_.reset(new TranslateURLFetcher(kFetcherId));
130 language_list_fetcher_->set_max_retry_on_5xx(kMaxRetryOn5xx);
131 }
132
~TranslateLanguageList()133 TranslateLanguageList::~TranslateLanguageList() {}
134
GetSupportedLanguages(std::vector<std::string> * languages)135 void TranslateLanguageList::GetSupportedLanguages(
136 std::vector<std::string>* languages) {
137 DCHECK(languages && languages->empty());
138 std::set<std::string>::const_iterator iter = all_supported_languages_.begin();
139 for (; iter != all_supported_languages_.end(); ++iter)
140 languages->push_back(*iter);
141
142 // Update language lists if they are not updated after Chrome was launched
143 // for later requests.
144 if (!update_is_disabled && language_list_fetcher_.get())
145 RequestLanguageList();
146 }
147
GetLanguageCode(const std::string & language)148 std::string TranslateLanguageList::GetLanguageCode(
149 const std::string& language) {
150 // Only remove the country code for country specific languages we don't
151 // support specifically yet.
152 if (IsSupportedLanguage(language))
153 return language;
154
155 size_t hypen_index = language.find('-');
156 if (hypen_index == std::string::npos)
157 return language;
158 return language.substr(0, hypen_index);
159 }
160
IsSupportedLanguage(const std::string & language)161 bool TranslateLanguageList::IsSupportedLanguage(const std::string& language) {
162 return all_supported_languages_.count(language) != 0;
163 }
164
IsAlphaLanguage(const std::string & language)165 bool TranslateLanguageList::IsAlphaLanguage(const std::string& language) {
166 return alpha_languages_.count(language) != 0;
167 }
168
TranslateLanguageUrl()169 GURL TranslateLanguageList::TranslateLanguageUrl() {
170 std::string url = translate::GetTranslateSecurityOrigin().spec() +
171 kLanguageListFetchPath;
172 return GURL(url);
173 }
174
RequestLanguageList()175 void TranslateLanguageList::RequestLanguageList() {
176 // If resource requests are not allowed, we'll get a callback when they are.
177 if (!resource_requests_allowed_) {
178 request_pending_ = true;
179 return;
180 }
181
182 request_pending_ = false;
183
184 if (language_list_fetcher_.get() &&
185 (language_list_fetcher_->state() == TranslateURLFetcher::IDLE ||
186 language_list_fetcher_->state() == TranslateURLFetcher::FAILED)) {
187 GURL url = TranslateLanguageUrl();
188 url = TranslateURLUtil::AddHostLocaleToUrl(url);
189 url = TranslateURLUtil::AddApiKeyToUrl(url);
190 url = net::AppendQueryParameter(
191 url, kAlphaLanguageQueryName, kAlphaLanguageQueryValue);
192
193 std::string message = base::StringPrintf(
194 "Language list including alpha languages fetch starts (URL: %s)",
195 url.spec().c_str());
196 NotifyEvent(__LINE__, message);
197
198 bool result = language_list_fetcher_->Request(
199 url,
200 base::Bind(&TranslateLanguageList::OnLanguageListFetchComplete,
201 base::Unretained(this)));
202 if (!result)
203 NotifyEvent(__LINE__, "Request is omitted due to retry limit");
204 }
205 }
206
SetResourceRequestsAllowed(bool allowed)207 void TranslateLanguageList::SetResourceRequestsAllowed(bool allowed) {
208 resource_requests_allowed_ = allowed;
209 if (resource_requests_allowed_ && request_pending_) {
210 RequestLanguageList();
211 DCHECK(!request_pending_);
212 }
213 }
214
215 scoped_ptr<TranslateLanguageList::EventCallbackList::Subscription>
RegisterEventCallback(const EventCallback & callback)216 TranslateLanguageList::RegisterEventCallback(const EventCallback& callback) {
217 return callback_list_.Add(callback);
218 }
219
220 // static
DisableUpdate()221 void TranslateLanguageList::DisableUpdate() {
222 update_is_disabled = true;
223 }
224
OnLanguageListFetchComplete(int id,bool success,const std::string & data)225 void TranslateLanguageList::OnLanguageListFetchComplete(
226 int id,
227 bool success,
228 const std::string& data) {
229 if (!success) {
230 // Since it fails just now, omit to schedule resource requests if
231 // ResourceRequestAllowedNotifier think it's ready. Otherwise, a callback
232 // will be invoked later to request resources again.
233 // The TranslateURLFetcher has a limit for retried requests and aborts
234 // re-try not to invoke OnLanguageListFetchComplete anymore if it's asked to
235 // re-try too many times.
236 NotifyEvent(__LINE__, "Failed to fetch languages");
237 return;
238 }
239
240 NotifyEvent(__LINE__, "Language list is updated");
241
242 DCHECK_EQ(kFetcherId, id);
243
244 SetSupportedLanguages(data);
245 language_list_fetcher_.reset();
246
247 last_updated_ = base::Time::Now();
248 }
249
NotifyEvent(int line,const std::string & message)250 void TranslateLanguageList::NotifyEvent(int line, const std::string& message) {
251 TranslateEventDetails details(__FILE__, line, message);
252 callback_list_.Notify(details);
253 }
254
SetSupportedLanguages(const std::string & language_list)255 void TranslateLanguageList::SetSupportedLanguages(
256 const std::string& language_list) {
257 // The format is:
258 // sl({
259 // "sl": {"XX": "LanguageName", ...},
260 // "tl": {"XX": "LanguageName", ...},
261 // "al": {"XX": 1, ...}
262 // })
263 // Where "sl(" is set in kLanguageListCallbackName, "tl" is
264 // kTargetLanguagesKey and "al" kAlphaLanguagesKey.
265 if (!StartsWithASCII(language_list,
266 TranslateLanguageList::kLanguageListCallbackName,
267 false) ||
268 !EndsWith(language_list, ")", false)) {
269 // We don't have a NOTREACHED here since this can happen in ui_tests, even
270 // though the the BrowserMain function won't call us with parameters.ui_task
271 // is NULL some tests don't set it, so we must bail here.
272 return;
273 }
274 static const size_t kLanguageListCallbackNameLength =
275 strlen(TranslateLanguageList::kLanguageListCallbackName);
276 std::string languages_json = language_list.substr(
277 kLanguageListCallbackNameLength,
278 language_list.size() - kLanguageListCallbackNameLength - 1);
279 scoped_ptr<base::Value> json_value(
280 base::JSONReader::Read(languages_json, base::JSON_ALLOW_TRAILING_COMMAS));
281 if (json_value == NULL || !json_value->IsType(base::Value::TYPE_DICTIONARY)) {
282 NOTREACHED();
283 return;
284 }
285 // The first level dictionary contains three sub-dict, first for source
286 // languages and second for target languages, we want to use the target
287 // languages. The last is for alpha languages.
288 base::DictionaryValue* language_dict =
289 static_cast<base::DictionaryValue*>(json_value.get());
290 base::DictionaryValue* target_languages = NULL;
291 if (!language_dict->GetDictionary(TranslateLanguageList::kTargetLanguagesKey,
292 &target_languages) ||
293 target_languages == NULL) {
294 NOTREACHED();
295 return;
296 }
297
298 const std::string& locale =
299 TranslateDownloadManager::GetInstance()->application_locale();
300
301 // Now we can clear language list.
302 all_supported_languages_.clear();
303 std::string message;
304 // ... and replace it with the values we just fetched from the server.
305 for (base::DictionaryValue::Iterator iter(*target_languages);
306 !iter.IsAtEnd();
307 iter.Advance()) {
308 const std::string& lang = iter.key();
309 if (!l10n_util::IsLocaleNameTranslated(lang.c_str(), locale)) {
310 TranslateBrowserMetrics::ReportUndisplayableLanguage(lang);
311 continue;
312 }
313 all_supported_languages_.insert(lang);
314 if (message.empty())
315 message += lang;
316 else
317 message += ", " + lang;
318 }
319 NotifyEvent(__LINE__, message);
320
321 // Get the alpha languages. The "al" parameter could be abandoned.
322 base::DictionaryValue* alpha_languages = NULL;
323 if (!language_dict->GetDictionary(TranslateLanguageList::kAlphaLanguagesKey,
324 &alpha_languages) ||
325 alpha_languages == NULL) {
326 return;
327 }
328
329 // We assume that the alpha languages are included in the above target
330 // languages, and don't use UMA or NotifyEvent.
331 alpha_languages_.clear();
332 for (base::DictionaryValue::Iterator iter(*alpha_languages);
333 !iter.IsAtEnd(); iter.Advance()) {
334 const std::string& lang = iter.key();
335 if (!l10n_util::IsLocaleNameTranslated(lang.c_str(), locale))
336 continue;
337 alpha_languages_.insert(lang);
338 }
339 }
340