1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "chrome/browser/autocomplete/base_search_provider.h"
6
7 #include "base/i18n/case_conversion.h"
8 #include "base/i18n/icu_string_conversions.h"
9 #include "base/json/json_string_value_serializer.h"
10 #include "base/json/json_writer.h"
11 #include "base/prefs/pref_registry_simple.h"
12 #include "base/prefs/pref_service.h"
13 #include "base/strings/string_util.h"
14 #include "base/strings/utf_string_conversions.h"
15 #include "chrome/browser/autocomplete/autocomplete_provider_listener.h"
16 #include "chrome/browser/bitmap_fetcher/bitmap_fetcher_service.h"
17 #include "chrome/browser/bitmap_fetcher/bitmap_fetcher_service_factory.h"
18 #include "chrome/browser/history/history_service.h"
19 #include "chrome/browser/history/history_service_factory.h"
20 #include "chrome/browser/omnibox/omnibox_field_trial.h"
21 #include "chrome/browser/profiles/profile.h"
22 #include "chrome/browser/search/instant_service.h"
23 #include "chrome/browser/search/instant_service_factory.h"
24 #include "chrome/browser/search/search.h"
25 #include "chrome/browser/search_engines/template_url.h"
26 #include "chrome/browser/search_engines/template_url_prepopulate_data.h"
27 #include "chrome/browser/search_engines/template_url_service.h"
28 #include "chrome/browser/search_engines/template_url_service_factory.h"
29 #include "chrome/browser/search_engines/ui_thread_search_terms_data.h"
30 #include "chrome/browser/sync/profile_sync_service.h"
31 #include "chrome/browser/sync/profile_sync_service_factory.h"
32 #include "chrome/common/pref_names.h"
33 #include "components/autocomplete/url_prefix.h"
34 #include "components/metrics/proto/omnibox_event.pb.h"
35 #include "components/metrics/proto/omnibox_input_type.pb.h"
36 #include "components/sync_driver/sync_prefs.h"
37 #include "components/url_fixer/url_fixer.h"
38 #include "content/public/common/url_constants.h"
39 #include "net/base/escape.h"
40 #include "net/base/net_util.h"
41 #include "net/base/registry_controlled_domains/registry_controlled_domain.h"
42 #include "net/http/http_response_headers.h"
43 #include "net/url_request/url_fetcher.h"
44 #include "net/url_request/url_fetcher_delegate.h"
45 #include "url/gurl.h"
46
47 using metrics::OmniboxEventProto;
48
49 namespace {
50
GetAutocompleteMatchType(const std::string & type)51 AutocompleteMatchType::Type GetAutocompleteMatchType(const std::string& type) {
52 if (type == "ENTITY")
53 return AutocompleteMatchType::SEARCH_SUGGEST_ENTITY;
54 if (type == "INFINITE")
55 return AutocompleteMatchType::SEARCH_SUGGEST_INFINITE;
56 if (type == "PERSONALIZED_QUERY")
57 return AutocompleteMatchType::SEARCH_SUGGEST_PERSONALIZED;
58 if (type == "PROFILE")
59 return AutocompleteMatchType::SEARCH_SUGGEST_PROFILE;
60 if (type == "NAVIGATION")
61 return AutocompleteMatchType::NAVSUGGEST;
62 if (type == "PERSONALIZED_NAVIGATION")
63 return AutocompleteMatchType::NAVSUGGEST_PERSONALIZED;
64 return AutocompleteMatchType::SEARCH_SUGGEST;
65 }
66
67 } // namespace
68
69 // SuggestionDeletionHandler -------------------------------------------------
70
71 // This class handles making requests to the server in order to delete
72 // personalized suggestions.
73 class SuggestionDeletionHandler : public net::URLFetcherDelegate {
74 public:
75 typedef base::Callback<void(bool, SuggestionDeletionHandler*)>
76 DeletionCompletedCallback;
77
78 SuggestionDeletionHandler(
79 const std::string& deletion_url,
80 Profile* profile,
81 const DeletionCompletedCallback& callback);
82
83 virtual ~SuggestionDeletionHandler();
84
85 private:
86 // net::URLFetcherDelegate:
87 virtual void OnURLFetchComplete(const net::URLFetcher* source) OVERRIDE;
88
89 scoped_ptr<net::URLFetcher> deletion_fetcher_;
90 DeletionCompletedCallback callback_;
91
92 DISALLOW_COPY_AND_ASSIGN(SuggestionDeletionHandler);
93 };
94
SuggestionDeletionHandler(const std::string & deletion_url,Profile * profile,const DeletionCompletedCallback & callback)95 SuggestionDeletionHandler::SuggestionDeletionHandler(
96 const std::string& deletion_url,
97 Profile* profile,
98 const DeletionCompletedCallback& callback) : callback_(callback) {
99 GURL url(deletion_url);
100 DCHECK(url.is_valid());
101
102 deletion_fetcher_.reset(net::URLFetcher::Create(
103 BaseSearchProvider::kDeletionURLFetcherID,
104 url,
105 net::URLFetcher::GET,
106 this));
107 deletion_fetcher_->SetRequestContext(profile->GetRequestContext());
108 deletion_fetcher_->Start();
109 }
110
~SuggestionDeletionHandler()111 SuggestionDeletionHandler::~SuggestionDeletionHandler() {
112 }
113
OnURLFetchComplete(const net::URLFetcher * source)114 void SuggestionDeletionHandler::OnURLFetchComplete(
115 const net::URLFetcher* source) {
116 DCHECK(source == deletion_fetcher_.get());
117 callback_.Run(
118 source->GetStatus().is_success() && (source->GetResponseCode() == 200),
119 this);
120 }
121
122 // BaseSearchProvider ---------------------------------------------------------
123
124 // static
125 const int BaseSearchProvider::kDefaultProviderURLFetcherID = 1;
126 const int BaseSearchProvider::kKeywordProviderURLFetcherID = 2;
127 const int BaseSearchProvider::kDeletionURLFetcherID = 3;
128
BaseSearchProvider(AutocompleteProviderListener * listener,Profile * profile,AutocompleteProvider::Type type)129 BaseSearchProvider::BaseSearchProvider(AutocompleteProviderListener* listener,
130 Profile* profile,
131 AutocompleteProvider::Type type)
132 : AutocompleteProvider(listener, profile, type),
133 field_trial_triggered_(false),
134 field_trial_triggered_in_session_(false),
135 suggest_results_pending_(0),
136 in_app_list_(false) {
137 }
138
139 // static
ShouldPrefetch(const AutocompleteMatch & match)140 bool BaseSearchProvider::ShouldPrefetch(const AutocompleteMatch& match) {
141 return match.GetAdditionalInfo(kShouldPrefetchKey) == kTrue;
142 }
143
144 // static
CreateSearchSuggestion(const base::string16 & suggestion,AutocompleteMatchType::Type type,bool from_keyword_provider,const TemplateURL * template_url,const SearchTermsData & search_terms_data)145 AutocompleteMatch BaseSearchProvider::CreateSearchSuggestion(
146 const base::string16& suggestion,
147 AutocompleteMatchType::Type type,
148 bool from_keyword_provider,
149 const TemplateURL* template_url,
150 const SearchTermsData& search_terms_data) {
151 return CreateSearchSuggestion(
152 NULL, AutocompleteInput(), BaseSearchProvider::SuggestResult(
153 suggestion, type, suggestion, base::string16(), base::string16(),
154 base::string16(), base::string16(), std::string(), std::string(),
155 from_keyword_provider, 0, false, false, base::string16()),
156 template_url, search_terms_data, 0, 0, false, false);
157 }
158
Stop(bool clear_cached_results)159 void BaseSearchProvider::Stop(bool clear_cached_results) {
160 StopSuggest();
161 done_ = true;
162
163 if (clear_cached_results)
164 ClearAllResults();
165 }
166
DeleteMatch(const AutocompleteMatch & match)167 void BaseSearchProvider::DeleteMatch(const AutocompleteMatch& match) {
168 DCHECK(match.deletable);
169 if (!match.GetAdditionalInfo(BaseSearchProvider::kDeletionUrlKey).empty()) {
170 deletion_handlers_.push_back(new SuggestionDeletionHandler(
171 match.GetAdditionalInfo(BaseSearchProvider::kDeletionUrlKey),
172 profile_,
173 base::Bind(&BaseSearchProvider::OnDeletionComplete,
174 base::Unretained(this))));
175 }
176
177 HistoryService* const history_service =
178 HistoryServiceFactory::GetForProfile(profile_, Profile::EXPLICIT_ACCESS);
179 TemplateURL* template_url = match.GetTemplateURL(profile_, false);
180 // This may be NULL if the template corresponding to the keyword has been
181 // deleted or there is no keyword set.
182 if (template_url != NULL) {
183 history_service->DeleteMatchingURLsForKeyword(template_url->id(),
184 match.contents);
185 }
186
187 // Immediately update the list of matches to show the match was deleted,
188 // regardless of whether the server request actually succeeds.
189 DeleteMatchFromMatches(match);
190 }
191
AddProviderInfo(ProvidersInfo * provider_info) const192 void BaseSearchProvider::AddProviderInfo(ProvidersInfo* provider_info) const {
193 provider_info->push_back(metrics::OmniboxEventProto_ProviderInfo());
194 metrics::OmniboxEventProto_ProviderInfo& new_entry = provider_info->back();
195 new_entry.set_provider(AsOmniboxEventProviderType());
196 new_entry.set_provider_done(done_);
197 std::vector<uint32> field_trial_hashes;
198 OmniboxFieldTrial::GetActiveSuggestFieldTrialHashes(&field_trial_hashes);
199 for (size_t i = 0; i < field_trial_hashes.size(); ++i) {
200 if (field_trial_triggered_)
201 new_entry.mutable_field_trial_triggered()->Add(field_trial_hashes[i]);
202 if (field_trial_triggered_in_session_) {
203 new_entry.mutable_field_trial_triggered_in_session()->Add(
204 field_trial_hashes[i]);
205 }
206 }
207 ModifyProviderInfo(&new_entry);
208 }
209
210 // static
211 const char BaseSearchProvider::kRelevanceFromServerKey[] =
212 "relevance_from_server";
213 const char BaseSearchProvider::kShouldPrefetchKey[] = "should_prefetch";
214 const char BaseSearchProvider::kSuggestMetadataKey[] = "suggest_metadata";
215 const char BaseSearchProvider::kDeletionUrlKey[] = "deletion_url";
216 const char BaseSearchProvider::kTrue[] = "true";
217 const char BaseSearchProvider::kFalse[] = "false";
218
~BaseSearchProvider()219 BaseSearchProvider::~BaseSearchProvider() {}
220
221 // BaseSearchProvider::Result --------------------------------------------------
222
Result(bool from_keyword_provider,int relevance,bool relevance_from_server,AutocompleteMatchType::Type type,const std::string & deletion_url)223 BaseSearchProvider::Result::Result(bool from_keyword_provider,
224 int relevance,
225 bool relevance_from_server,
226 AutocompleteMatchType::Type type,
227 const std::string& deletion_url)
228 : from_keyword_provider_(from_keyword_provider),
229 type_(type),
230 relevance_(relevance),
231 relevance_from_server_(relevance_from_server),
232 deletion_url_(deletion_url) {}
233
~Result()234 BaseSearchProvider::Result::~Result() {}
235
236 // BaseSearchProvider::SuggestResult -------------------------------------------
237
SuggestResult(const base::string16 & suggestion,AutocompleteMatchType::Type type,const base::string16 & match_contents,const base::string16 & match_contents_prefix,const base::string16 & annotation,const base::string16 & answer_contents,const base::string16 & answer_type,const std::string & suggest_query_params,const std::string & deletion_url,bool from_keyword_provider,int relevance,bool relevance_from_server,bool should_prefetch,const base::string16 & input_text)238 BaseSearchProvider::SuggestResult::SuggestResult(
239 const base::string16& suggestion,
240 AutocompleteMatchType::Type type,
241 const base::string16& match_contents,
242 const base::string16& match_contents_prefix,
243 const base::string16& annotation,
244 const base::string16& answer_contents,
245 const base::string16& answer_type,
246 const std::string& suggest_query_params,
247 const std::string& deletion_url,
248 bool from_keyword_provider,
249 int relevance,
250 bool relevance_from_server,
251 bool should_prefetch,
252 const base::string16& input_text)
253 : Result(from_keyword_provider,
254 relevance,
255 relevance_from_server,
256 type,
257 deletion_url),
258 suggestion_(suggestion),
259 match_contents_prefix_(match_contents_prefix),
260 annotation_(annotation),
261 suggest_query_params_(suggest_query_params),
262 answer_contents_(answer_contents),
263 answer_type_(answer_type),
264 should_prefetch_(should_prefetch) {
265 match_contents_ = match_contents;
266 DCHECK(!match_contents_.empty());
267 ClassifyMatchContents(true, input_text);
268 }
269
~SuggestResult()270 BaseSearchProvider::SuggestResult::~SuggestResult() {}
271
ClassifyMatchContents(const bool allow_bolding_all,const base::string16 & input_text)272 void BaseSearchProvider::SuggestResult::ClassifyMatchContents(
273 const bool allow_bolding_all,
274 const base::string16& input_text) {
275 if (input_text.empty()) {
276 // In case of zero-suggest results, do not highlight matches.
277 match_contents_class_.push_back(
278 ACMatchClassification(0, ACMatchClassification::NONE));
279 return;
280 }
281
282 base::string16 lookup_text = input_text;
283 if (type_ == AutocompleteMatchType::SEARCH_SUGGEST_INFINITE) {
284 const size_t contents_index =
285 suggestion_.length() - match_contents_.length();
286 // Ensure the query starts with the input text, and ends with the match
287 // contents, and the input text has an overlap with contents.
288 if (StartsWith(suggestion_, input_text, true) &&
289 EndsWith(suggestion_, match_contents_, true) &&
290 (input_text.length() > contents_index)) {
291 lookup_text = input_text.substr(contents_index);
292 }
293 }
294 size_t lookup_position = match_contents_.find(lookup_text);
295 if (!allow_bolding_all && (lookup_position == base::string16::npos)) {
296 // Bail if the code below to update the bolding would bold the whole
297 // string. Note that the string may already be entirely bolded; if
298 // so, leave it as is.
299 return;
300 }
301 match_contents_class_.clear();
302 // We do intra-string highlighting for suggestions - the suggested segment
303 // will be highlighted, e.g. for input_text = "you" the suggestion may be
304 // "youtube", so we'll bold the "tube" section: you*tube*.
305 if (input_text != match_contents_) {
306 if (lookup_position == base::string16::npos) {
307 // The input text is not a substring of the query string, e.g. input
308 // text is "slasdot" and the query string is "slashdot", so we bold the
309 // whole thing.
310 match_contents_class_.push_back(
311 ACMatchClassification(0, ACMatchClassification::MATCH));
312 } else {
313 // We don't iterate over the string here annotating all matches because
314 // it looks odd to have every occurrence of a substring that may be as
315 // short as a single character highlighted in a query suggestion result,
316 // e.g. for input text "s" and query string "southwest airlines", it
317 // looks odd if both the first and last s are highlighted.
318 if (lookup_position != 0) {
319 match_contents_class_.push_back(
320 ACMatchClassification(0, ACMatchClassification::MATCH));
321 }
322 match_contents_class_.push_back(
323 ACMatchClassification(lookup_position, ACMatchClassification::NONE));
324 size_t next_fragment_position = lookup_position + lookup_text.length();
325 if (next_fragment_position < match_contents_.length()) {
326 match_contents_class_.push_back(ACMatchClassification(
327 next_fragment_position, ACMatchClassification::MATCH));
328 }
329 }
330 } else {
331 // Otherwise, match_contents_ is a verbatim (what-you-typed) match, either
332 // for the default provider or a keyword search provider.
333 match_contents_class_.push_back(
334 ACMatchClassification(0, ACMatchClassification::NONE));
335 }
336 }
337
IsInlineable(const base::string16 & input) const338 bool BaseSearchProvider::SuggestResult::IsInlineable(
339 const base::string16& input) const {
340 return StartsWith(suggestion_, input, false);
341 }
342
CalculateRelevance(const AutocompleteInput & input,bool keyword_provider_requested) const343 int BaseSearchProvider::SuggestResult::CalculateRelevance(
344 const AutocompleteInput& input,
345 bool keyword_provider_requested) const {
346 if (!from_keyword_provider_ && keyword_provider_requested)
347 return 100;
348 return ((input.type() == metrics::OmniboxInputType::URL) ? 300 : 600);
349 }
350
351 // BaseSearchProvider::NavigationResult ----------------------------------------
352
NavigationResult(const AutocompleteProvider & provider,const GURL & url,AutocompleteMatchType::Type type,const base::string16 & description,const std::string & deletion_url,bool from_keyword_provider,int relevance,bool relevance_from_server,const base::string16 & input_text,const std::string & languages)353 BaseSearchProvider::NavigationResult::NavigationResult(
354 const AutocompleteProvider& provider,
355 const GURL& url,
356 AutocompleteMatchType::Type type,
357 const base::string16& description,
358 const std::string& deletion_url,
359 bool from_keyword_provider,
360 int relevance,
361 bool relevance_from_server,
362 const base::string16& input_text,
363 const std::string& languages)
364 : Result(from_keyword_provider,
365 relevance,
366 relevance_from_server,
367 type,
368 deletion_url),
369 url_(url),
370 formatted_url_(AutocompleteInput::FormattedStringWithEquivalentMeaning(
371 url,
372 provider.StringForURLDisplay(url, true, false))),
373 description_(description) {
374 DCHECK(url_.is_valid());
375 CalculateAndClassifyMatchContents(true, input_text, languages);
376 }
377
~NavigationResult()378 BaseSearchProvider::NavigationResult::~NavigationResult() {}
379
CalculateAndClassifyMatchContents(const bool allow_bolding_nothing,const base::string16 & input_text,const std::string & languages)380 void BaseSearchProvider::NavigationResult::CalculateAndClassifyMatchContents(
381 const bool allow_bolding_nothing,
382 const base::string16& input_text,
383 const std::string& languages) {
384 if (input_text.empty()) {
385 // In case of zero-suggest results, do not highlight matches.
386 match_contents_class_.push_back(
387 ACMatchClassification(0, ACMatchClassification::NONE));
388 return;
389 }
390
391 // First look for the user's input inside the formatted url as it would be
392 // without trimming the scheme, so we can find matches at the beginning of the
393 // scheme.
394 const URLPrefix* prefix =
395 URLPrefix::BestURLPrefix(formatted_url_, input_text);
396 size_t match_start = (prefix == NULL) ?
397 formatted_url_.find(input_text) : prefix->prefix.length();
398 bool trim_http = !AutocompleteInput::HasHTTPScheme(input_text) &&
399 (!prefix || (match_start != 0));
400 const net::FormatUrlTypes format_types =
401 net::kFormatUrlOmitAll & ~(trim_http ? 0 : net::kFormatUrlOmitHTTP);
402
403 base::string16 match_contents = net::FormatUrl(url_, languages, format_types,
404 net::UnescapeRule::SPACES, NULL, NULL, &match_start);
405 // If the first match in the untrimmed string was inside a scheme that we
406 // trimmed, look for a subsequent match.
407 if (match_start == base::string16::npos)
408 match_start = match_contents.find(input_text);
409 // Update |match_contents_| and |match_contents_class_| if it's allowed.
410 if (allow_bolding_nothing || (match_start != base::string16::npos)) {
411 match_contents_ = match_contents;
412 // Safe if |match_start| is npos; also safe if the input is longer than the
413 // remaining contents after |match_start|.
414 AutocompleteMatch::ClassifyLocationInString(match_start,
415 input_text.length(), match_contents_.length(),
416 ACMatchClassification::URL, &match_contents_class_);
417 }
418 }
419
IsInlineable(const base::string16 & input) const420 bool BaseSearchProvider::NavigationResult::IsInlineable(
421 const base::string16& input) const {
422 return
423 URLPrefix::BestURLPrefix(base::UTF8ToUTF16(url_.spec()), input) != NULL;
424 }
425
CalculateRelevance(const AutocompleteInput & input,bool keyword_provider_requested) const426 int BaseSearchProvider::NavigationResult::CalculateRelevance(
427 const AutocompleteInput& input,
428 bool keyword_provider_requested) const {
429 return (from_keyword_provider_ || !keyword_provider_requested) ? 800 : 150;
430 }
431
432 // BaseSearchProvider::Results -------------------------------------------------
433
Results()434 BaseSearchProvider::Results::Results() : verbatim_relevance(-1) {}
435
~Results()436 BaseSearchProvider::Results::~Results() {}
437
Clear()438 void BaseSearchProvider::Results::Clear() {
439 suggest_results.clear();
440 navigation_results.clear();
441 verbatim_relevance = -1;
442 metadata.clear();
443 }
444
HasServerProvidedScores() const445 bool BaseSearchProvider::Results::HasServerProvidedScores() const {
446 if (verbatim_relevance >= 0)
447 return true;
448
449 // Right now either all results of one type will be server-scored or they will
450 // all be locally scored, but in case we change this later, we'll just check
451 // them all.
452 for (SuggestResults::const_iterator i(suggest_results.begin());
453 i != suggest_results.end(); ++i) {
454 if (i->relevance_from_server())
455 return true;
456 }
457 for (NavigationResults::const_iterator i(navigation_results.begin());
458 i != navigation_results.end(); ++i) {
459 if (i->relevance_from_server())
460 return true;
461 }
462
463 return false;
464 }
465
SetDeletionURL(const std::string & deletion_url,AutocompleteMatch * match)466 void BaseSearchProvider::SetDeletionURL(const std::string& deletion_url,
467 AutocompleteMatch* match) {
468 if (deletion_url.empty())
469 return;
470 TemplateURLService* template_service =
471 TemplateURLServiceFactory::GetForProfile(profile_);
472 if (!template_service)
473 return;
474 GURL url = template_service->GetDefaultSearchProvider()->GenerateSearchURL(
475 template_service->search_terms_data());
476 url = url.GetOrigin().Resolve(deletion_url);
477 if (url.is_valid()) {
478 match->RecordAdditionalInfo(BaseSearchProvider::kDeletionUrlKey,
479 url.spec());
480 match->deletable = true;
481 }
482 }
483
484 // BaseSearchProvider ---------------------------------------------------------
485
486 // static
CreateSearchSuggestion(AutocompleteProvider * autocomplete_provider,const AutocompleteInput & input,const SuggestResult & suggestion,const TemplateURL * template_url,const SearchTermsData & search_terms_data,int accepted_suggestion,int omnibox_start_margin,bool append_extra_query_params,bool from_app_list)487 AutocompleteMatch BaseSearchProvider::CreateSearchSuggestion(
488 AutocompleteProvider* autocomplete_provider,
489 const AutocompleteInput& input,
490 const SuggestResult& suggestion,
491 const TemplateURL* template_url,
492 const SearchTermsData& search_terms_data,
493 int accepted_suggestion,
494 int omnibox_start_margin,
495 bool append_extra_query_params,
496 bool from_app_list) {
497 AutocompleteMatch match(autocomplete_provider, suggestion.relevance(), false,
498 suggestion.type());
499
500 if (!template_url)
501 return match;
502 match.keyword = template_url->keyword();
503 match.contents = suggestion.match_contents();
504 match.contents_class = suggestion.match_contents_class();
505 match.answer_contents = suggestion.answer_contents();
506 match.answer_type = suggestion.answer_type();
507 if (suggestion.type() == AutocompleteMatchType::SEARCH_SUGGEST_INFINITE) {
508 match.RecordAdditionalInfo(
509 kACMatchPropertyInputText, base::UTF16ToUTF8(input.text()));
510 match.RecordAdditionalInfo(
511 kACMatchPropertyContentsPrefix,
512 base::UTF16ToUTF8(suggestion.match_contents_prefix()));
513 match.RecordAdditionalInfo(
514 kACMatchPropertyContentsStartIndex,
515 static_cast<int>(
516 suggestion.suggestion().length() - match.contents.length()));
517 }
518
519 if (!suggestion.annotation().empty())
520 match.description = suggestion.annotation();
521
522 // suggestion.match_contents() should have already been collapsed.
523 match.allowed_to_be_default_match =
524 (base::CollapseWhitespace(input.text(), false) ==
525 suggestion.match_contents());
526
527 // When the user forced a query, we need to make sure all the fill_into_edit
528 // values preserve that property. Otherwise, if the user starts editing a
529 // suggestion, non-Search results will suddenly appear.
530 if (input.type() == metrics::OmniboxInputType::FORCED_QUERY)
531 match.fill_into_edit.assign(base::ASCIIToUTF16("?"));
532 if (suggestion.from_keyword_provider())
533 match.fill_into_edit.append(match.keyword + base::char16(' '));
534 if (!input.prevent_inline_autocomplete() &&
535 StartsWith(suggestion.suggestion(), input.text(), false)) {
536 match.inline_autocompletion =
537 suggestion.suggestion().substr(input.text().length());
538 match.allowed_to_be_default_match = true;
539 }
540 match.fill_into_edit.append(suggestion.suggestion());
541
542 const TemplateURLRef& search_url = template_url->url_ref();
543 DCHECK(search_url.SupportsReplacement(search_terms_data));
544 match.search_terms_args.reset(
545 new TemplateURLRef::SearchTermsArgs(suggestion.suggestion()));
546 match.search_terms_args->original_query = input.text();
547 match.search_terms_args->accepted_suggestion = accepted_suggestion;
548 match.search_terms_args->omnibox_start_margin = omnibox_start_margin;
549 match.search_terms_args->suggest_query_params =
550 suggestion.suggest_query_params();
551 match.search_terms_args->append_extra_query_params =
552 append_extra_query_params;
553 match.search_terms_args->from_app_list = from_app_list;
554 // This is the destination URL sans assisted query stats. This must be set
555 // so the AutocompleteController can properly de-dupe; the controller will
556 // eventually overwrite it before it reaches the user.
557 match.destination_url =
558 GURL(search_url.ReplaceSearchTerms(*match.search_terms_args.get(),
559 search_terms_data));
560
561 // Search results don't look like URLs.
562 match.transition = suggestion.from_keyword_provider() ?
563 content::PAGE_TRANSITION_KEYWORD : content::PAGE_TRANSITION_GENERATED;
564
565 return match;
566 }
567
568 // static
DeserializeJsonData(std::string json_data)569 scoped_ptr<base::Value> BaseSearchProvider::DeserializeJsonData(
570 std::string json_data) {
571 // The JSON response should be an array.
572 for (size_t response_start_index = json_data.find("["), i = 0;
573 response_start_index != std::string::npos && i < 5;
574 response_start_index = json_data.find("[", 1), i++) {
575 // Remove any XSSI guards to allow for JSON parsing.
576 if (response_start_index > 0)
577 json_data.erase(0, response_start_index);
578
579 JSONStringValueSerializer deserializer(json_data);
580 deserializer.set_allow_trailing_comma(true);
581 int error_code = 0;
582 scoped_ptr<base::Value> data(deserializer.Deserialize(&error_code, NULL));
583 if (error_code == 0)
584 return data.Pass();
585 }
586 return scoped_ptr<base::Value>();
587 }
588
589 // static
ZeroSuggestEnabled(const GURL & suggest_url,const TemplateURL * template_url,OmniboxEventProto::PageClassification page_classification,Profile * profile)590 bool BaseSearchProvider::ZeroSuggestEnabled(
591 const GURL& suggest_url,
592 const TemplateURL* template_url,
593 OmniboxEventProto::PageClassification page_classification,
594 Profile* profile) {
595 if (!OmniboxFieldTrial::InZeroSuggestFieldTrial())
596 return false;
597
598 // Make sure we are sending the suggest request through HTTPS to prevent
599 // exposing the current page URL or personalized results without encryption.
600 if (!suggest_url.SchemeIs(url::kHttpsScheme))
601 return false;
602
603 // Don't show zero suggest on the NTP.
604 // TODO(hfung): Experiment with showing MostVisited zero suggest on NTP
605 // under the conditions described in crbug.com/305366.
606 if ((page_classification ==
607 OmniboxEventProto::INSTANT_NTP_WITH_FAKEBOX_AS_STARTING_FOCUS) ||
608 (page_classification ==
609 OmniboxEventProto::INSTANT_NTP_WITH_OMNIBOX_AS_STARTING_FOCUS))
610 return false;
611
612 // Don't run if there's no profile or in incognito mode.
613 if (profile == NULL || profile->IsOffTheRecord())
614 return false;
615
616 // Don't run if we can't get preferences or search suggest is not enabled.
617 PrefService* prefs = profile->GetPrefs();
618 if (!prefs->GetBoolean(prefs::kSearchSuggestEnabled))
619 return false;
620
621 // Only make the request if we know that the provider supports zero suggest
622 // (currently only the prepopulated Google provider).
623 UIThreadSearchTermsData search_terms_data(profile);
624 if (template_url == NULL ||
625 !template_url->SupportsReplacement(search_terms_data) ||
626 TemplateURLPrepopulateData::GetEngineType(
627 *template_url, search_terms_data) != SEARCH_ENGINE_GOOGLE)
628 return false;
629
630 return true;
631 }
632
633 // static
CanSendURL(const GURL & current_page_url,const GURL & suggest_url,const TemplateURL * template_url,OmniboxEventProto::PageClassification page_classification,Profile * profile)634 bool BaseSearchProvider::CanSendURL(
635 const GURL& current_page_url,
636 const GURL& suggest_url,
637 const TemplateURL* template_url,
638 OmniboxEventProto::PageClassification page_classification,
639 Profile* profile) {
640 if (!ZeroSuggestEnabled(suggest_url, template_url, page_classification,
641 profile))
642 return false;
643
644 if (!current_page_url.is_valid())
645 return false;
646
647 // Only allow HTTP URLs or HTTPS URLs for the same domain as the search
648 // provider.
649 if ((current_page_url.scheme() != url::kHttpScheme) &&
650 ((current_page_url.scheme() != url::kHttpsScheme) ||
651 !net::registry_controlled_domains::SameDomainOrHost(
652 current_page_url, suggest_url,
653 net::registry_controlled_domains::EXCLUDE_PRIVATE_REGISTRIES)))
654 return false;
655
656 // Check field trials and settings allow sending the URL on suggest requests.
657 ProfileSyncService* service =
658 ProfileSyncServiceFactory::GetInstance()->GetForProfile(profile);
659 sync_driver::SyncPrefs sync_prefs(profile->GetPrefs());
660 if (service == NULL ||
661 !service->IsSyncEnabledAndLoggedIn() ||
662 !sync_prefs.GetPreferredDataTypes(syncer::UserTypes()).Has(
663 syncer::PROXY_TABS) ||
664 service->GetEncryptedDataTypes().Has(syncer::SESSIONS))
665 return false;
666
667 return true;
668 }
669
OnURLFetchComplete(const net::URLFetcher * source)670 void BaseSearchProvider::OnURLFetchComplete(const net::URLFetcher* source) {
671 DCHECK(!done_);
672 suggest_results_pending_--;
673 DCHECK_GE(suggest_results_pending_, 0); // Should never go negative.
674
675 const bool is_keyword = IsKeywordFetcher(source);
676
677 // Ensure the request succeeded and that the provider used is still available.
678 // A verbatim match cannot be generated without this provider, causing errors.
679 const bool request_succeeded =
680 source->GetStatus().is_success() && (source->GetResponseCode() == 200) &&
681 GetTemplateURL(is_keyword);
682
683 LogFetchComplete(request_succeeded, is_keyword);
684
685 bool results_updated = false;
686 if (request_succeeded) {
687 const net::HttpResponseHeaders* const response_headers =
688 source->GetResponseHeaders();
689 std::string json_data;
690 source->GetResponseAsString(&json_data);
691
692 // JSON is supposed to be UTF-8, but some suggest service providers send
693 // JSON files in non-UTF-8 encodings. The actual encoding is usually
694 // specified in the Content-Type header field.
695 if (response_headers) {
696 std::string charset;
697 if (response_headers->GetCharset(&charset)) {
698 base::string16 data_16;
699 // TODO(jungshik): Switch to CodePageToUTF8 after it's added.
700 if (base::CodepageToUTF16(json_data, charset.c_str(),
701 base::OnStringConversionError::FAIL,
702 &data_16))
703 json_data = base::UTF16ToUTF8(data_16);
704 }
705 }
706
707 scoped_ptr<base::Value> data(DeserializeJsonData(json_data));
708 if (data && StoreSuggestionResponse(json_data, *data.get()))
709 return;
710
711 results_updated = data.get() && ParseSuggestResults(
712 *data.get(), is_keyword, GetResultsToFill(is_keyword));
713 }
714
715 UpdateMatches();
716 if (done_ || results_updated)
717 listener_->OnProviderUpdate(results_updated);
718 }
719
AddMatchToMap(const SuggestResult & result,const std::string & metadata,int accepted_suggestion,bool mark_as_deletable,MatchMap * map)720 void BaseSearchProvider::AddMatchToMap(const SuggestResult& result,
721 const std::string& metadata,
722 int accepted_suggestion,
723 bool mark_as_deletable,
724 MatchMap* map) {
725 InstantService* instant_service =
726 InstantServiceFactory::GetForProfile(profile_);
727 // Android and iOS have no InstantService.
728 const int omnibox_start_margin = instant_service ?
729 instant_service->omnibox_start_margin() : chrome::kDisableStartMargin;
730
731 AutocompleteMatch match = CreateSearchSuggestion(
732 this, GetInput(result.from_keyword_provider()), result,
733 GetTemplateURL(result.from_keyword_provider()),
734 UIThreadSearchTermsData(profile_), accepted_suggestion,
735 omnibox_start_margin, ShouldAppendExtraParams(result),
736 in_app_list_);
737 if (!match.destination_url.is_valid())
738 return;
739 match.search_terms_args->bookmark_bar_pinned =
740 profile_->GetPrefs()->GetBoolean(prefs::kShowBookmarkBar);
741 match.RecordAdditionalInfo(kRelevanceFromServerKey,
742 result.relevance_from_server() ? kTrue : kFalse);
743 match.RecordAdditionalInfo(kShouldPrefetchKey,
744 result.should_prefetch() ? kTrue : kFalse);
745 SetDeletionURL(result.deletion_url(), &match);
746 if (mark_as_deletable)
747 match.deletable = true;
748 // Metadata is needed only for prefetching queries.
749 if (result.should_prefetch())
750 match.RecordAdditionalInfo(kSuggestMetadataKey, metadata);
751
752 // Try to add |match| to |map|. If a match for this suggestion is
753 // already in |map|, replace it if |match| is more relevant.
754 // NOTE: Keep this ToLower() call in sync with url_database.cc.
755 MatchKey match_key(
756 std::make_pair(base::i18n::ToLower(result.suggestion()),
757 match.search_terms_args->suggest_query_params));
758 const std::pair<MatchMap::iterator, bool> i(
759 map->insert(std::make_pair(match_key, match)));
760
761 bool should_prefetch = result.should_prefetch();
762 if (!i.second) {
763 // NOTE: We purposefully do a direct relevance comparison here instead of
764 // using AutocompleteMatch::MoreRelevant(), so that we'll prefer "items
765 // added first" rather than "items alphabetically first" when the scores
766 // are equal. The only case this matters is when a user has results with
767 // the same score that differ only by capitalization; because the history
768 // system returns results sorted by recency, this means we'll pick the most
769 // recent such result even if the precision of our relevance score is too
770 // low to distinguish the two.
771 if (match.relevance > i.first->second.relevance) {
772 match.duplicate_matches.insert(match.duplicate_matches.end(),
773 i.first->second.duplicate_matches.begin(),
774 i.first->second.duplicate_matches.end());
775 i.first->second.duplicate_matches.clear();
776 match.duplicate_matches.push_back(i.first->second);
777 i.first->second = match;
778 } else {
779 i.first->second.duplicate_matches.push_back(match);
780 if (match.keyword == i.first->second.keyword) {
781 // Old and new matches are from the same search provider. It is okay to
782 // record one match's prefetch data onto a different match (for the same
783 // query string) for the following reasons:
784 // 1. Because the suggest server only sends down a query string from
785 // which we construct a URL, rather than sending a full URL, and because
786 // we construct URLs from query strings in the same way every time, the
787 // URLs for the two matches will be the same. Therefore, we won't end up
788 // prefetching something the server didn't intend.
789 // 2. Presumably the server sets the prefetch bit on a match it things
790 // is sufficiently relevant that the user is likely to choose it.
791 // Surely setting the prefetch bit on a match of even higher relevance
792 // won't violate this assumption.
793 should_prefetch |= ShouldPrefetch(i.first->second);
794 i.first->second.RecordAdditionalInfo(kShouldPrefetchKey,
795 should_prefetch ? kTrue : kFalse);
796 if (should_prefetch)
797 i.first->second.RecordAdditionalInfo(kSuggestMetadataKey, metadata);
798 }
799 }
800 }
801 }
802
ParseSuggestResults(const base::Value & root_val,bool is_keyword_result,Results * results)803 bool BaseSearchProvider::ParseSuggestResults(const base::Value& root_val,
804 bool is_keyword_result,
805 Results* results) {
806 base::string16 query;
807 const base::ListValue* root_list = NULL;
808 const base::ListValue* results_list = NULL;
809 const AutocompleteInput& input = GetInput(is_keyword_result);
810
811 if (!root_val.GetAsList(&root_list) || !root_list->GetString(0, &query) ||
812 query != input.text() || !root_list->GetList(1, &results_list))
813 return false;
814
815 // 3rd element: Description list.
816 const base::ListValue* descriptions = NULL;
817 root_list->GetList(2, &descriptions);
818
819 // 4th element: Disregard the query URL list for now.
820
821 // Reset suggested relevance information.
822 results->verbatim_relevance = -1;
823
824 // 5th element: Optional key-value pairs from the Suggest server.
825 const base::ListValue* types = NULL;
826 const base::ListValue* relevances = NULL;
827 const base::ListValue* suggestion_details = NULL;
828 const base::DictionaryValue* extras = NULL;
829 int prefetch_index = -1;
830 if (root_list->GetDictionary(4, &extras)) {
831 extras->GetList("google:suggesttype", &types);
832
833 // Discard this list if its size does not match that of the suggestions.
834 if (extras->GetList("google:suggestrelevance", &relevances) &&
835 (relevances->GetSize() != results_list->GetSize()))
836 relevances = NULL;
837 extras->GetInteger("google:verbatimrelevance",
838 &results->verbatim_relevance);
839
840 // Check if the active suggest field trial (if any) has triggered either
841 // for the default provider or keyword provider.
842 bool triggered = false;
843 extras->GetBoolean("google:fieldtrialtriggered", &triggered);
844 field_trial_triggered_ |= triggered;
845 field_trial_triggered_in_session_ |= triggered;
846
847 const base::DictionaryValue* client_data = NULL;
848 if (extras->GetDictionary("google:clientdata", &client_data) && client_data)
849 client_data->GetInteger("phi", &prefetch_index);
850
851 if (extras->GetList("google:suggestdetail", &suggestion_details) &&
852 suggestion_details->GetSize() != results_list->GetSize())
853 suggestion_details = NULL;
854
855 // Store the metadata that came with the response in case we need to pass it
856 // along with the prefetch query to Instant.
857 JSONStringValueSerializer json_serializer(&results->metadata);
858 json_serializer.Serialize(*extras);
859 }
860
861 // Clear the previous results now that new results are available.
862 results->suggest_results.clear();
863 results->navigation_results.clear();
864
865 base::string16 suggestion;
866 std::string type;
867 int relevance = GetDefaultResultRelevance();
868 // Prohibit navsuggest in FORCED_QUERY mode. Users wants queries, not URLs.
869 const bool allow_navsuggest =
870 input.type() != metrics::OmniboxInputType::FORCED_QUERY;
871 const std::string languages(
872 profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
873 const base::string16& trimmed_input =
874 base::CollapseWhitespace(input.text(), false);
875 for (size_t index = 0; results_list->GetString(index, &suggestion); ++index) {
876 // Google search may return empty suggestions for weird input characters,
877 // they make no sense at all and can cause problems in our code.
878 if (suggestion.empty())
879 continue;
880
881 // Apply valid suggested relevance scores; discard invalid lists.
882 if (relevances != NULL && !relevances->GetInteger(index, &relevance))
883 relevances = NULL;
884 AutocompleteMatchType::Type match_type =
885 AutocompleteMatchType::SEARCH_SUGGEST;
886 if (types && types->GetString(index, &type))
887 match_type = GetAutocompleteMatchType(type);
888 const base::DictionaryValue* suggestion_detail = NULL;
889 std::string deletion_url;
890
891 if (suggestion_details &&
892 suggestion_details->GetDictionary(index, &suggestion_detail))
893 suggestion_detail->GetString("du", &deletion_url);
894
895 if ((match_type == AutocompleteMatchType::NAVSUGGEST) ||
896 (match_type == AutocompleteMatchType::NAVSUGGEST_PERSONALIZED)) {
897 // Do not blindly trust the URL coming from the server to be valid.
898 GURL url(
899 url_fixer::FixupURL(base::UTF16ToUTF8(suggestion), std::string()));
900 if (url.is_valid() && allow_navsuggest) {
901 base::string16 title;
902 if (descriptions != NULL)
903 descriptions->GetString(index, &title);
904 results->navigation_results.push_back(NavigationResult(
905 *this, url, match_type, title, deletion_url, is_keyword_result,
906 relevance, relevances != NULL, input.text(), languages));
907 }
908 } else {
909 base::string16 match_contents = suggestion;
910 base::string16 match_contents_prefix;
911 base::string16 annotation;
912 base::string16 answer_contents;
913 base::string16 answer_type;
914 std::string suggest_query_params;
915
916 if (suggestion_details) {
917 suggestion_details->GetDictionary(index, &suggestion_detail);
918 if (suggestion_detail) {
919 suggestion_detail->GetString("t", &match_contents);
920 suggestion_detail->GetString("mp", &match_contents_prefix);
921 // Error correction for bad data from server.
922 if (match_contents.empty())
923 match_contents = suggestion;
924 suggestion_detail->GetString("a", &annotation);
925 suggestion_detail->GetString("q", &suggest_query_params);
926
927 // Extract Answers, if provided.
928 const base::DictionaryValue* answer_json = NULL;
929 if (suggestion_detail->GetDictionary("ansa", &answer_json)) {
930 match_type = AutocompleteMatchType::SEARCH_SUGGEST_ANSWER;
931 PrefetchAnswersImages(answer_json);
932 std::string contents;
933 base::JSONWriter::Write(answer_json, &contents);
934 answer_contents = base::UTF8ToUTF16(contents);
935 suggestion_detail->GetString("ansb", &answer_type);
936 }
937 }
938 }
939
940 bool should_prefetch = static_cast<int>(index) == prefetch_index;
941 // TODO(kochi): Improve calculator suggestion presentation.
942 results->suggest_results.push_back(SuggestResult(
943 base::CollapseWhitespace(suggestion, false), match_type,
944 base::CollapseWhitespace(match_contents, false),
945 match_contents_prefix, annotation, answer_contents, answer_type,
946 suggest_query_params, deletion_url, is_keyword_result, relevance,
947 relevances != NULL, should_prefetch, trimmed_input));
948 }
949 }
950 SortResults(is_keyword_result, relevances, results);
951 return true;
952 }
953
PrefetchAnswersImages(const base::DictionaryValue * answer_json)954 void BaseSearchProvider::PrefetchAnswersImages(
955 const base::DictionaryValue* answer_json) {
956 DCHECK(answer_json);
957 const base::ListValue* lines = NULL;
958 answer_json->GetList("l", &lines);
959 if (!lines || lines->GetSize() == 0)
960 return;
961
962 BitmapFetcherService* image_service =
963 BitmapFetcherServiceFactory::GetForBrowserContext(profile_);
964 DCHECK(image_service);
965
966 for (size_t line = 0; line < lines->GetSize(); ++line) {
967 const base::DictionaryValue* imageLine = NULL;
968 lines->GetDictionary(line, &imageLine);
969 if (!imageLine)
970 continue;
971 const base::DictionaryValue* imageData = NULL;
972 imageLine->GetDictionary("i", &imageData);
973 if (!imageData)
974 continue;
975 std::string imageUrl;
976 imageData->GetString("d", &imageUrl);
977 image_service->Prefetch(GURL(imageUrl));
978 }
979 }
980
SortResults(bool is_keyword,const base::ListValue * relevances,Results * results)981 void BaseSearchProvider::SortResults(bool is_keyword,
982 const base::ListValue* relevances,
983 Results* results) {
984 }
985
StoreSuggestionResponse(const std::string & json_data,const base::Value & parsed_data)986 bool BaseSearchProvider::StoreSuggestionResponse(
987 const std::string& json_data,
988 const base::Value& parsed_data) {
989 return false;
990 }
991
ModifyProviderInfo(metrics::OmniboxEventProto_ProviderInfo * provider_info) const992 void BaseSearchProvider::ModifyProviderInfo(
993 metrics::OmniboxEventProto_ProviderInfo* provider_info) const {
994 }
995
DeleteMatchFromMatches(const AutocompleteMatch & match)996 void BaseSearchProvider::DeleteMatchFromMatches(
997 const AutocompleteMatch& match) {
998 for (ACMatches::iterator i(matches_.begin()); i != matches_.end(); ++i) {
999 // Find the desired match to delete by checking the type and contents.
1000 // We can't check the destination URL, because the autocomplete controller
1001 // may have reformulated that. Not that while checking for matching
1002 // contents works for personalized suggestions, if more match types gain
1003 // deletion support, this algorithm may need to be re-examined.
1004 if (i->contents == match.contents && i->type == match.type) {
1005 matches_.erase(i);
1006 break;
1007 }
1008 }
1009 }
1010
OnDeletionComplete(bool success,SuggestionDeletionHandler * handler)1011 void BaseSearchProvider::OnDeletionComplete(
1012 bool success, SuggestionDeletionHandler* handler) {
1013 RecordDeletionResult(success);
1014 SuggestionDeletionHandlers::iterator it = std::find(
1015 deletion_handlers_.begin(), deletion_handlers_.end(), handler);
1016 DCHECK(it != deletion_handlers_.end());
1017 deletion_handlers_.erase(it);
1018 }
1019