1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "chrome/browser/autocomplete/zero_suggest_provider.h"
7 #include "base/callback.h"
8 #include "base/i18n/case_conversion.h"
9 #include "base/json/json_string_value_serializer.h"
10 #include "base/metrics/histogram.h"
11 #include "base/metrics/user_metrics.h"
12 #include "base/prefs/pref_service.h"
13 #include "base/strings/string16.h"
14 #include "base/strings/string_util.h"
15 #include "base/strings/utf_string_conversions.h"
16 #include "base/time/time.h"
17 #include "chrome/browser/autocomplete/autocomplete_classifier.h"
18 #include "chrome/browser/autocomplete/autocomplete_classifier_factory.h"
19 #include "chrome/browser/autocomplete/chrome_autocomplete_provider_client.h"
20 #include "chrome/browser/autocomplete/chrome_autocomplete_scheme_classifier.h"
21 #include "chrome/browser/autocomplete/history_url_provider.h"
22 #include "chrome/browser/history/top_sites.h"
23 #include "chrome/browser/profiles/profile.h"
24 #include "chrome/browser/search_engines/template_url_service_factory.h"
25 #include "chrome/common/pref_names.h"
26 #include "components/history/core/browser/history_types.h"
27 #include "components/metrics/proto/omnibox_input_type.pb.h"
28 #include "components/omnibox/autocomplete_input.h"
29 #include "components/omnibox/autocomplete_match.h"
30 #include "components/omnibox/autocomplete_provider_listener.h"
31 #include "components/omnibox/omnibox_field_trial.h"
32 #include "components/omnibox/search_provider.h"
33 #include "components/pref_registry/pref_registry_syncable.h"
34 #include "components/search_engines/template_url_service.h"
35 #include "components/variations/net/variations_http_header_provider.h"
36 #include "net/base/escape.h"
37 #include "net/base/load_flags.h"
38 #include "net/base/net_util.h"
39 #include "net/http/http_request_headers.h"
40 #include "net/url_request/url_fetcher.h"
41 #include "net/url_request/url_request_status.h"
46 // TODO(hfung): The histogram code was copied and modified from
47 // search_provider.cc. Refactor and consolidate the code.
48 // We keep track in a histogram how many suggest requests we send, how
49 // many suggest requests we invalidate (e.g., due to a user typing
50 // another character), and how many replies we receive.
51 // *** ADD NEW ENUMS AFTER ALL PREVIOUSLY DEFINED ONES! ***
52 // (excluding the end-of-list enum value)
53 // We do not want values of existing enums to change or else it screws
55 enum ZeroSuggestRequestsHistogramValue {
56 ZERO_SUGGEST_REQUEST_SENT = 1,
57 ZERO_SUGGEST_REQUEST_INVALIDATED,
58 ZERO_SUGGEST_REPLY_RECEIVED,
59 ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE
62 void LogOmniboxZeroSuggestRequest(
63 ZeroSuggestRequestsHistogramValue request_value) {
64 UMA_HISTOGRAM_ENUMERATION("Omnibox.ZeroSuggestRequests", request_value,
65 ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE);
68 // The maximum relevance of the top match from this provider.
69 const int kDefaultVerbatimZeroSuggestRelevance = 1300;
71 // Relevance value to use if it was not set explicitly by the server.
72 const int kDefaultZeroSuggestRelevance = 100;
77 ZeroSuggestProvider* ZeroSuggestProvider::Create(
78 AutocompleteProviderListener* listener,
79 TemplateURLService* template_url_service,
81 return new ZeroSuggestProvider(listener, template_url_service, profile);
85 void ZeroSuggestProvider::RegisterProfilePrefs(
86 user_prefs::PrefRegistrySyncable* registry) {
87 registry->RegisterStringPref(
88 prefs::kZeroSuggestCachedResults,
90 user_prefs::PrefRegistrySyncable::UNSYNCABLE_PREF);
93 void ZeroSuggestProvider::Start(const AutocompleteInput& input,
94 bool minimal_changes) {
96 if (input.type() == metrics::OmniboxInputType::INVALID)
100 field_trial_triggered_ = false;
101 field_trial_triggered_in_session_ = false;
102 results_from_cache_ = false;
103 permanent_text_ = input.text();
104 current_query_ = input.current_url().spec();
105 current_page_classification_ = input.current_page_classification();
106 current_url_match_ = MatchForCurrentURL();
108 const TemplateURL* default_provider =
109 template_url_service_->GetDefaultSearchProvider();
110 if (default_provider == NULL)
113 base::string16 prefix;
114 TemplateURLRef::SearchTermsArgs search_term_args(prefix);
115 GURL suggest_url(default_provider->suggestions_url_ref().ReplaceSearchTerms(
116 search_term_args, template_url_service_->search_terms_data()));
117 if (!suggest_url.is_valid())
120 // No need to send the current page URL in personalized suggest or
121 // most visited field trials.
122 if (CanSendURL(input.current_url(), suggest_url, default_provider,
123 current_page_classification_,
124 template_url_service_->search_terms_data(), client_.get()) &&
125 !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial() &&
126 !OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
127 // Update suggest_url to include the current_page_url.
128 search_term_args.current_page_url = current_query_;
129 suggest_url = GURL(default_provider->suggestions_url_ref().
132 template_url_service_->search_terms_data()));
133 } else if (!ShouldShowNonContextualZeroSuggest(suggest_url,
134 input.current_url())) {
139 // TODO(jered): Consider adding locally-sourced zero-suggestions here too.
140 // These may be useful on the NTP or more relevant to the user than server
141 // suggestions, if based on local browsing history.
142 MaybeUseCachedSuggestions();
146 void ZeroSuggestProvider::Stop(bool clear_cached_results) {
148 LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_INVALIDATED);
150 waiting_for_most_visited_urls_request_ = false;
153 if (clear_cached_results) {
154 // We do not call Clear() on |results_| to retain |verbatim_relevance|
155 // value in the |results_| object. |verbatim_relevance| is used at the
156 // beginning of the next StartZeroSuggest() call to determine the current
157 // url match relevance.
158 results_.suggest_results.clear();
159 results_.navigation_results.clear();
160 current_query_.clear();
164 void ZeroSuggestProvider::DeleteMatch(const AutocompleteMatch& match) {
165 if (OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial()) {
166 // Remove the deleted match from the cache, so it is not shown to the user
167 // again. Since we cannot remove just one result, blow away the cache.
168 profile_->GetPrefs()->SetString(prefs::kZeroSuggestCachedResults,
171 BaseSearchProvider::DeleteMatch(match);
174 void ZeroSuggestProvider::AddProviderInfo(ProvidersInfo* provider_info) const {
175 BaseSearchProvider::AddProviderInfo(provider_info);
176 if (!results_.suggest_results.empty() || !results_.navigation_results.empty())
177 provider_info->back().set_times_returned_results_in_session(1);
180 void ZeroSuggestProvider::ResetSession() {
181 // The user has started editing in the omnibox, so leave
182 // |field_trial_triggered_in_session_| unchanged and set
183 // |field_trial_triggered_| to false since zero suggest is inactive now.
184 field_trial_triggered_ = false;
187 ZeroSuggestProvider::ZeroSuggestProvider(
188 AutocompleteProviderListener* listener,
189 TemplateURLService* template_url_service,
191 : BaseSearchProvider(template_url_service,
192 scoped_ptr<AutocompleteProviderClient>(
193 new ChromeAutocompleteProviderClient(profile)),
194 AutocompleteProvider::TYPE_ZERO_SUGGEST),
197 results_from_cache_(false),
198 waiting_for_most_visited_urls_request_(false),
199 weak_ptr_factory_(this) {
202 ZeroSuggestProvider::~ZeroSuggestProvider() {
205 const TemplateURL* ZeroSuggestProvider::GetTemplateURL(bool is_keyword) const {
206 // Zero suggest provider should not receive keyword results.
208 return template_url_service_->GetDefaultSearchProvider();
211 const AutocompleteInput ZeroSuggestProvider::GetInput(bool is_keyword) const {
212 return AutocompleteInput(
213 base::string16(), base::string16::npos, std::string(),
214 GURL(current_query_), current_page_classification_, true, false, false,
215 true, ChromeAutocompleteSchemeClassifier(profile_));
218 bool ZeroSuggestProvider::ShouldAppendExtraParams(
219 const SearchSuggestionParser::SuggestResult& result) const {
220 // We always use the default provider for search, so append the params.
224 void ZeroSuggestProvider::RecordDeletionResult(bool success) {
227 base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Success"));
230 base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Failure"));
234 void ZeroSuggestProvider::OnURLFetchComplete(const net::URLFetcher* source) {
236 DCHECK_EQ(fetcher_.get(), source);
238 LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REPLY_RECEIVED);
240 bool results_updated = false;
241 if (source->GetStatus().is_success() && source->GetResponseCode() == 200) {
242 std::string json_data = SearchSuggestionParser::ExtractJsonData(source);
243 scoped_ptr<base::Value> data(
244 SearchSuggestionParser::DeserializeJsonData(json_data));
246 if (StoreSuggestionResponse(json_data, *data))
248 results_updated = ParseSuggestResults(
249 *data, kDefaultZeroSuggestRelevance, false, &results_);
254 ConvertResultsToAutocompleteMatches();
255 listener_->OnProviderUpdate(results_updated);
258 bool ZeroSuggestProvider::StoreSuggestionResponse(
259 const std::string& json_data,
260 const base::Value& parsed_data) {
261 if (!OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial() ||
264 profile_->GetPrefs()->SetString(prefs::kZeroSuggestCachedResults, json_data);
266 // If we received an empty result list, we should update the display, as it
267 // may be showing cached results that should not be shown.
268 const base::ListValue* root_list = NULL;
269 const base::ListValue* results_list = NULL;
270 if (parsed_data.GetAsList(&root_list) &&
271 root_list->GetList(1, &results_list) &&
272 results_list->empty())
275 // We are finished with the request and want to bail early.
276 if (results_from_cache_)
279 return results_from_cache_;
282 void ZeroSuggestProvider::AddSuggestResultsToMap(
283 const SearchSuggestionParser::SuggestResults& results,
285 for (size_t i = 0; i < results.size(); ++i)
286 AddMatchToMap(results[i], std::string(), i, false, false, map);
289 AutocompleteMatch ZeroSuggestProvider::NavigationToMatch(
290 const SearchSuggestionParser::NavigationResult& navigation) {
291 AutocompleteMatch match(this, navigation.relevance(), false,
293 match.destination_url = navigation.url();
295 // Zero suggest results should always omit protocols and never appear bold.
296 const std::string languages(
297 profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
298 match.contents = net::FormatUrl(navigation.url(), languages,
299 net::kFormatUrlOmitAll, net::UnescapeRule::SPACES, NULL, NULL, NULL);
300 match.fill_into_edit +=
301 AutocompleteInput::FormattedStringWithEquivalentMeaning(navigation.url(),
302 match.contents, ChromeAutocompleteSchemeClassifier(profile_));
304 AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
305 match.contents.length(), ACMatchClassification::URL,
306 &match.contents_class);
309 AutocompleteMatch::SanitizeString(navigation.description());
310 AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
311 match.description.length(), ACMatchClassification::NONE,
312 &match.description_class);
316 void ZeroSuggestProvider::Run(const GURL& suggest_url) {
317 if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
318 most_visited_urls_.clear();
319 history::TopSites* ts = profile_->GetTopSites();
321 waiting_for_most_visited_urls_request_ = true;
322 ts->GetMostVisitedURLs(
323 base::Bind(&ZeroSuggestProvider::OnMostVisitedUrlsAvailable,
324 weak_ptr_factory_.GetWeakPtr()), false);
327 const int kFetcherID = 1;
329 net::URLFetcher::Create(kFetcherID,
331 net::URLFetcher::GET, this));
332 fetcher_->SetRequestContext(profile_->GetRequestContext());
333 fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES);
334 // Add Chrome experiment state to the request headers.
335 net::HttpRequestHeaders headers;
336 variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
337 fetcher_->GetOriginalURL(), profile_->IsOffTheRecord(), false,
339 fetcher_->SetExtraRequestHeaders(headers.ToString());
341 LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_SENT);
345 void ZeroSuggestProvider::OnMostVisitedUrlsAvailable(
346 const history::MostVisitedURLList& urls) {
347 if (!waiting_for_most_visited_urls_request_) return;
348 most_visited_urls_ = urls;
349 waiting_for_most_visited_urls_request_ = false;
351 ConvertResultsToAutocompleteMatches();
352 listener_->OnProviderUpdate(true);
355 void ZeroSuggestProvider::ConvertResultsToAutocompleteMatches() {
358 const TemplateURL* default_provider =
359 template_url_service_->GetDefaultSearchProvider();
360 // Fail if we can't set the clickthrough URL for query suggestions.
361 if (default_provider == NULL || !default_provider->SupportsReplacement(
362 template_url_service_->search_terms_data()))
366 AddSuggestResultsToMap(results_.suggest_results, &map);
368 const int num_query_results = map.size();
369 const int num_nav_results = results_.navigation_results.size();
370 const int num_results = num_query_results + num_nav_results;
371 UMA_HISTOGRAM_COUNTS("ZeroSuggest.QueryResults", num_query_results);
372 UMA_HISTOGRAM_COUNTS("ZeroSuggest.URLResults", num_nav_results);
373 UMA_HISTOGRAM_COUNTS("ZeroSuggest.AllResults", num_results);
375 // Show Most Visited results after ZeroSuggest response is received.
376 if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
377 if (!current_url_match_.destination_url.is_valid())
379 matches_.push_back(current_url_match_);
381 if (num_results > 0) {
382 UMA_HISTOGRAM_COUNTS(
383 "Omnibox.ZeroSuggest.MostVisitedResultsCounterfactual",
384 most_visited_urls_.size());
386 const base::string16 current_query_string16(
387 base::ASCIIToUTF16(current_query_));
388 const std::string languages(
389 profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
390 for (size_t i = 0; i < most_visited_urls_.size(); i++) {
391 const history::MostVisitedURL& url = most_visited_urls_[i];
392 SearchSuggestionParser::NavigationResult nav(
393 ChromeAutocompleteSchemeClassifier(profile_), url.url,
394 AutocompleteMatchType::NAVSUGGEST, url.title, std::string(), false,
395 relevance, true, current_query_string16, languages);
396 matches_.push_back(NavigationToMatch(nav));
402 if (num_results == 0)
405 // TODO(jered): Rip this out once the first match is decoupled from the
406 // current typing in the omnibox.
407 matches_.push_back(current_url_match_);
409 for (MatchMap::const_iterator it(map.begin()); it != map.end(); ++it)
410 matches_.push_back(it->second);
412 const SearchSuggestionParser::NavigationResults& nav_results(
413 results_.navigation_results);
414 for (SearchSuggestionParser::NavigationResults::const_iterator it(
415 nav_results.begin()); it != nav_results.end(); ++it)
416 matches_.push_back(NavigationToMatch(*it));
419 AutocompleteMatch ZeroSuggestProvider::MatchForCurrentURL() {
420 AutocompleteMatch match;
421 AutocompleteClassifierFactory::GetForProfile(profile_)->Classify(
422 permanent_text_, false, true, current_page_classification_, &match, NULL);
423 match.is_history_what_you_typed_match = false;
424 match.allowed_to_be_default_match = true;
426 // The placeholder suggestion for the current URL has high relevance so
427 // that it is in the first suggestion slot and inline autocompleted. It
428 // gets dropped as soon as the user types something.
429 match.relevance = GetVerbatimRelevance();
434 int ZeroSuggestProvider::GetVerbatimRelevance() const {
435 return results_.verbatim_relevance >= 0 ?
436 results_.verbatim_relevance : kDefaultVerbatimZeroSuggestRelevance;
439 bool ZeroSuggestProvider::ShouldShowNonContextualZeroSuggest(
440 const GURL& suggest_url,
441 const GURL& current_page_url) const {
442 if (!ZeroSuggestEnabled(suggest_url,
443 template_url_service_->GetDefaultSearchProvider(),
444 current_page_classification_,
445 template_url_service_->search_terms_data(),
449 // If we cannot send URLs, then only the MostVisited and Personalized
450 // variations can be shown.
451 if (!OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial() &&
452 !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial())
455 // Only show zero suggest for HTTP[S] pages.
456 // TODO(mariakhomenko): We may be able to expand this set to include pages
457 // with other schemes (e.g. chrome://). That may require improvements to
458 // the formatting of the verbatim result returned by MatchForCurrentURL().
459 if (!current_page_url.is_valid() ||
460 ((current_page_url.scheme() != url::kHttpScheme) &&
461 (current_page_url.scheme() != url::kHttpsScheme)))
464 if (OmniboxFieldTrial::InZeroSuggestMostVisitedWithoutSerpFieldTrial() &&
465 template_url_service_->
466 IsSearchResultsPageFromDefaultSearchProvider(current_page_url))
472 void ZeroSuggestProvider::MaybeUseCachedSuggestions() {
473 if (!OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial())
476 std::string json_data = profile_->GetPrefs()->GetString(
477 prefs::kZeroSuggestCachedResults);
478 if (!json_data.empty()) {
479 scoped_ptr<base::Value> data(
480 SearchSuggestionParser::DeserializeJsonData(json_data));
481 if (data && ParseSuggestResults(
482 *data, kDefaultZeroSuggestRelevance, false, &results_)) {
483 ConvertResultsToAutocompleteMatches();
484 results_from_cache_ = !matches_.empty();