src/chrome/browser/autocomplete/zero_suggest_provider.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/autocomplete/zero_suggest_provider.h"
   6
   7 #include "base/callback.h"
   8 #include "base/i18n/case_conversion.h"
   9 #include "base/json/json_string_value_serializer.h"
  10 #include "base/metrics/histogram.h"
  11 #include "base/metrics/user_metrics.h"
  12 #include "base/prefs/pref_service.h"
  13 #include "base/strings/string16.h"
  14 #include "base/strings/string_util.h"
  15 #include "base/strings/utf_string_conversions.h"
  16 #include "base/time/time.h"
  17 #include "chrome/browser/autocomplete/autocomplete_classifier.h"
  18 #include "chrome/browser/autocomplete/autocomplete_classifier_factory.h"
  19 #include "chrome/browser/autocomplete/chrome_autocomplete_provider_client.h"
  20 #include "chrome/browser/autocomplete/chrome_autocomplete_scheme_classifier.h"
  21 #include "chrome/browser/autocomplete/history_url_provider.h"
  22 #include "chrome/browser/history/top_sites.h"
  23 #include "chrome/browser/profiles/profile.h"
  24 #include "chrome/common/pref_names.h"
  25 #include "components/history/core/browser/history_types.h"
  26 #include "components/metrics/proto/omnibox_input_type.pb.h"
  27 #include "components/omnibox/autocomplete_input.h"
  28 #include "components/omnibox/autocomplete_match.h"
  29 #include "components/omnibox/autocomplete_provider_listener.h"
  30 #include "components/omnibox/omnibox_field_trial.h"
  31 #include "components/omnibox/search_provider.h"
  32 #include "components/pref_registry/pref_registry_syncable.h"
  33 #include "components/search_engines/template_url_service.h"
  34 #include "components/variations/variations_http_header_provider.h"
  35 #include "net/base/escape.h"
  36 #include "net/base/load_flags.h"
  37 #include "net/base/net_util.h"
  38 #include "net/http/http_request_headers.h"
  39 #include "net/url_request/url_fetcher.h"
  40 #include "net/url_request/url_request_status.h"
  41 #include "url/gurl.h"
  42
  43 namespace {
  44
  45 // TODO(hfung): The histogram code was copied and modified from
  46 // search_provider.cc.  Refactor and consolidate the code.
  47 // We keep track in a histogram how many suggest requests we send, how
  48 // many suggest requests we invalidate (e.g., due to a user typing
  49 // another character), and how many replies we receive.
  50 // *** ADD NEW ENUMS AFTER ALL PREVIOUSLY DEFINED ONES! ***
  51 //     (excluding the end-of-list enum value)
  52 // We do not want values of existing enums to change or else it screws
  53 // up the statistics.
  54 enum ZeroSuggestRequestsHistogramValue {
  55   ZERO_SUGGEST_REQUEST_SENT = 1,
  56   ZERO_SUGGEST_REQUEST_INVALIDATED,
  57   ZERO_SUGGEST_REPLY_RECEIVED,
  58   ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE
  59 };
  60
  61 void LogOmniboxZeroSuggestRequest(
  62     ZeroSuggestRequestsHistogramValue request_value) {
  63   UMA_HISTOGRAM_ENUMERATION("Omnibox.ZeroSuggestRequests", request_value,
  64                             ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE);
  65 }
  66
  67 // The maximum relevance of the top match from this provider.
  68 const int kDefaultVerbatimZeroSuggestRelevance = 1300;
  69
  70 // Relevance value to use if it was not set explicitly by the server.
  71 const int kDefaultZeroSuggestRelevance = 100;
  72
  73 }  // namespace
  74
  75 // static
  76 ZeroSuggestProvider* ZeroSuggestProvider::Create(
  77     AutocompleteProviderListener* listener,
  78     TemplateURLService* template_url_service,
  79     Profile* profile) {
  80   return new ZeroSuggestProvider(listener, template_url_service, profile);
  81 }
  82
  83 // static
  84 void ZeroSuggestProvider::RegisterProfilePrefs(
  85     user_prefs::PrefRegistrySyncable* registry) {
  86   registry->RegisterStringPref(
  87       prefs::kZeroSuggestCachedResults,
  88       std::string(),
  89       user_prefs::PrefRegistrySyncable::UNSYNCABLE_PREF);
  90 }
  91
  92 void ZeroSuggestProvider::Start(const AutocompleteInput& input,
  93                                 bool minimal_changes) {
  94   matches_.clear();
  95   if (input.type() == metrics::OmniboxInputType::INVALID)
  96     return;
  97
  98   Stop(true);
  99   field_trial_triggered_ = false;
 100   field_trial_triggered_in_session_ = false;
 101   results_from_cache_ = false;
 102   permanent_text_ = input.text();
 103   current_query_ = input.current_url().spec();
 104   current_page_classification_ = input.current_page_classification();
 105   current_url_match_ = MatchForCurrentURL();
 106
 107   const TemplateURL* default_provider =
 108      template_url_service_->GetDefaultSearchProvider();
 109   if (default_provider == NULL)
 110     return;
 111
 112   base::string16 prefix;
 113   TemplateURLRef::SearchTermsArgs search_term_args(prefix);
 114   GURL suggest_url(default_provider->suggestions_url_ref().ReplaceSearchTerms(
 115       search_term_args, template_url_service_->search_terms_data()));
 116   if (!suggest_url.is_valid())
 117     return;
 118
 119   // No need to send the current page URL in personalized suggest field trial.
 120   if (CanSendURL(input.current_url(), suggest_url, default_provider,
 121                  current_page_classification_,
 122                  template_url_service_->search_terms_data(), client_.get()) &&
 123       !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial()) {
 124     // Update suggest_url to include the current_page_url.
 125     search_term_args.current_page_url = current_query_;
 126     suggest_url = GURL(default_provider->suggestions_url_ref().
 127                        ReplaceSearchTerms(
 128                            search_term_args,
 129                            template_url_service_->search_terms_data()));
 130   } else if (!CanShowZeroSuggestWithoutSendingURL(suggest_url,
 131                                                   input.current_url())) {
 132     return;
 133   }
 134
 135   done_ = false;
 136   // TODO(jered): Consider adding locally-sourced zero-suggestions here too.
 137   // These may be useful on the NTP or more relevant to the user than server
 138   // suggestions, if based on local browsing history.
 139   MaybeUseCachedSuggestions();
 140   Run(suggest_url);
 141 }
 142
 143 void ZeroSuggestProvider::Stop(bool clear_cached_results) {
 144   if (fetcher_)
 145     LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_INVALIDATED);
 146   fetcher_.reset();
 147   done_ = true;
 148
 149   if (clear_cached_results) {
 150     // We do not call Clear() on |results_| to retain |verbatim_relevance|
 151     // value in the |results_| object. |verbatim_relevance| is used at the
 152     // beginning of the next StartZeroSuggest() call to determine the current
 153     // url match relevance.
 154     results_.suggest_results.clear();
 155     results_.navigation_results.clear();
 156     current_query_.clear();
 157   }
 158 }
 159
 160 void ZeroSuggestProvider::DeleteMatch(const AutocompleteMatch& match) {
 161   if (OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial()) {
 162     // Remove the deleted match from the cache, so it is not shown to the user
 163     // again. Since we cannot remove just one result, blow away the cache.
 164     profile_->GetPrefs()->SetString(prefs::kZeroSuggestCachedResults,
 165                                     std::string());
 166   }
 167   BaseSearchProvider::DeleteMatch(match);
 168 }
 169
 170 void ZeroSuggestProvider::AddProviderInfo(ProvidersInfo* provider_info) const {
 171   BaseSearchProvider::AddProviderInfo(provider_info);
 172   if (!results_.suggest_results.empty() || !results_.navigation_results.empty())
 173     provider_info->back().set_times_returned_results_in_session(1);
 174 }
 175
 176 void ZeroSuggestProvider::ResetSession() {
 177   // The user has started editing in the omnibox, so leave
 178   // |field_trial_triggered_in_session_| unchanged and set
 179   // |field_trial_triggered_| to false since zero suggest is inactive now.
 180   field_trial_triggered_ = false;
 181 }
 182
 183 ZeroSuggestProvider::ZeroSuggestProvider(
 184   AutocompleteProviderListener* listener,
 185   TemplateURLService* template_url_service,
 186   Profile* profile)
 187     : BaseSearchProvider(template_url_service,
 188                          scoped_ptr<AutocompleteProviderClient>(
 189                              new ChromeAutocompleteProviderClient(profile)),
 190                          AutocompleteProvider::TYPE_ZERO_SUGGEST),
 191       listener_(listener),
 192       profile_(profile),
 193       results_from_cache_(false),
 194       weak_ptr_factory_(this) {
 195 }
 196
 197 ZeroSuggestProvider::~ZeroSuggestProvider() {
 198 }
 199
 200 const TemplateURL* ZeroSuggestProvider::GetTemplateURL(bool is_keyword) const {
 201   // Zero suggest provider should not receive keyword results.
 202   DCHECK(!is_keyword);
 203   return template_url_service_->GetDefaultSearchProvider();
 204 }
 205
 206 const AutocompleteInput ZeroSuggestProvider::GetInput(bool is_keyword) const {
 207   return AutocompleteInput(
 208       base::string16(), base::string16::npos, base::string16(),
 209       GURL(current_query_), current_page_classification_, true, false, false,
 210       true, ChromeAutocompleteSchemeClassifier(profile_));
 211 }
 212
 213 bool ZeroSuggestProvider::ShouldAppendExtraParams(
 214       const SearchSuggestionParser::SuggestResult& result) const {
 215   // We always use the default provider for search, so append the params.
 216   return true;
 217 }
 218
 219 void ZeroSuggestProvider::RecordDeletionResult(bool success) {
 220   if (success) {
 221     base::RecordAction(
 222         base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Success"));
 223   } else {
 224     base::RecordAction(
 225         base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Failure"));
 226   }
 227 }
 228
 229 void ZeroSuggestProvider::OnURLFetchComplete(const net::URLFetcher* source) {
 230   DCHECK(!done_);
 231   DCHECK_EQ(fetcher_.get(), source);
 232
 233   LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REPLY_RECEIVED);
 234
 235   bool results_updated = false;
 236   if (source->GetStatus().is_success() && source->GetResponseCode() == 200) {
 237     std::string json_data = SearchSuggestionParser::ExtractJsonData(source);
 238     scoped_ptr<base::Value> data(
 239         SearchSuggestionParser::DeserializeJsonData(json_data));
 240     if (data) {
 241       if (StoreSuggestionResponse(json_data, *data))
 242         return;
 243       results_updated = ParseSuggestResults(
 244           *data, kDefaultZeroSuggestRelevance, false, &results_);
 245     }
 246   }
 247   fetcher_.reset();
 248   done_ = true;
 249   ConvertResultsToAutocompleteMatches();
 250   listener_->OnProviderUpdate(results_updated);
 251 }
 252
 253 bool ZeroSuggestProvider::StoreSuggestionResponse(
 254     const std::string& json_data,
 255     const base::Value& parsed_data) {
 256   if (!OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial() ||
 257       json_data.empty())
 258     return false;
 259   profile_->GetPrefs()->SetString(prefs::kZeroSuggestCachedResults, json_data);
 260
 261   // If we received an empty result list, we should update the display, as it
 262   // may be showing cached results that should not be shown.
 263   const base::ListValue* root_list = NULL;
 264   const base::ListValue* results_list = NULL;
 265   if (parsed_data.GetAsList(&root_list) &&
 266       root_list->GetList(1, &results_list) &&
 267       results_list->empty())
 268     return false;
 269
 270   // We are finished with the request and want to bail early.
 271   if (results_from_cache_)
 272     done_ = true;
 273
 274   return results_from_cache_;
 275 }
 276
 277 void ZeroSuggestProvider::AddSuggestResultsToMap(
 278     const SearchSuggestionParser::SuggestResults& results,
 279     MatchMap* map) {
 280   for (size_t i = 0; i < results.size(); ++i)
 281     AddMatchToMap(results[i], std::string(), i, false, false, map);
 282 }
 283
 284 AutocompleteMatch ZeroSuggestProvider::NavigationToMatch(
 285     const SearchSuggestionParser::NavigationResult& navigation) {
 286   AutocompleteMatch match(this, navigation.relevance(), false,
 287                           navigation.type());
 288   match.destination_url = navigation.url();
 289
 290   // Zero suggest results should always omit protocols and never appear bold.
 291   const std::string languages(
 292       profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
 293   match.contents = net::FormatUrl(navigation.url(), languages,
 294       net::kFormatUrlOmitAll, net::UnescapeRule::SPACES, NULL, NULL, NULL);
 295   match.fill_into_edit +=
 296       AutocompleteInput::FormattedStringWithEquivalentMeaning(navigation.url(),
 297           match.contents, ChromeAutocompleteSchemeClassifier(profile_));
 298
 299   AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
 300       match.contents.length(), ACMatchClassification::URL,
 301       &match.contents_class);
 302
 303   match.description =
 304       AutocompleteMatch::SanitizeString(navigation.description());
 305   AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
 306       match.description.length(), ACMatchClassification::NONE,
 307       &match.description_class);
 308   return match;
 309 }
 310
 311 void ZeroSuggestProvider::Run(const GURL& suggest_url) {
 312   const int kFetcherID = 1;
 313   fetcher_.reset(
 314       net::URLFetcher::Create(kFetcherID,
 315           suggest_url,
 316           net::URLFetcher::GET, this));
 317   fetcher_->SetRequestContext(profile_->GetRequestContext());
 318   fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES);
 319   // Add Chrome experiment state to the request headers.
 320   net::HttpRequestHeaders headers;
 321   variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
 322       fetcher_->GetOriginalURL(), profile_->IsOffTheRecord(), false, &headers);
 323   fetcher_->SetExtraRequestHeaders(headers.ToString());
 324   fetcher_->Start();
 325
 326   if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
 327     most_visited_urls_.clear();
 328     history::TopSites* ts = profile_->GetTopSites();
 329     if (ts) {
 330       ts->GetMostVisitedURLs(
 331           base::Bind(&ZeroSuggestProvider::OnMostVisitedUrlsAvailable,
 332                      weak_ptr_factory_.GetWeakPtr()), false);
 333     }
 334   }
 335   LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_SENT);
 336 }
 337
 338 void ZeroSuggestProvider::OnMostVisitedUrlsAvailable(
 339     const history::MostVisitedURLList& urls) {
 340   most_visited_urls_ = urls;
 341 }
 342
 343 void ZeroSuggestProvider::ConvertResultsToAutocompleteMatches() {
 344   matches_.clear();
 345
 346   const TemplateURL* default_provider =
 347       template_url_service_->GetDefaultSearchProvider();
 348   // Fail if we can't set the clickthrough URL for query suggestions.
 349   if (default_provider == NULL || !default_provider->SupportsReplacement(
 350           template_url_service_->search_terms_data()))
 351     return;
 352
 353   MatchMap map;
 354   AddSuggestResultsToMap(results_.suggest_results, &map);
 355
 356   const int num_query_results = map.size();
 357   const int num_nav_results = results_.navigation_results.size();
 358   const int num_results = num_query_results + num_nav_results;
 359   UMA_HISTOGRAM_COUNTS("ZeroSuggest.QueryResults", num_query_results);
 360   UMA_HISTOGRAM_COUNTS("ZeroSuggest.URLResults", num_nav_results);
 361   UMA_HISTOGRAM_COUNTS("ZeroSuggest.AllResults", num_results);
 362
 363   // Show Most Visited results after ZeroSuggest response is received.
 364   if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
 365     if (!current_url_match_.destination_url.is_valid())
 366       return;
 367     matches_.push_back(current_url_match_);
 368     int relevance = 600;
 369     if (num_results > 0) {
 370       UMA_HISTOGRAM_COUNTS(
 371           "Omnibox.ZeroSuggest.MostVisitedResultsCounterfactual",
 372           most_visited_urls_.size());
 373     }
 374     const base::string16 current_query_string16(
 375         base::ASCIIToUTF16(current_query_));
 376     const std::string languages(
 377         profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
 378     for (size_t i = 0; i < most_visited_urls_.size(); i++) {
 379       const history::MostVisitedURL& url = most_visited_urls_[i];
 380       SearchSuggestionParser::NavigationResult nav(
 381           ChromeAutocompleteSchemeClassifier(profile_), url.url,
 382           AutocompleteMatchType::NAVSUGGEST, url.title, std::string(), false,
 383           relevance, true, current_query_string16, languages);
 384       matches_.push_back(NavigationToMatch(nav));
 385       --relevance;
 386     }
 387     return;
 388   }
 389
 390   if (num_results == 0)
 391     return;
 392
 393   // TODO(jered): Rip this out once the first match is decoupled from the
 394   // current typing in the omnibox.
 395   matches_.push_back(current_url_match_);
 396
 397   for (MatchMap::const_iterator it(map.begin()); it != map.end(); ++it)
 398     matches_.push_back(it->second);
 399
 400   const SearchSuggestionParser::NavigationResults& nav_results(
 401       results_.navigation_results);
 402   for (SearchSuggestionParser::NavigationResults::const_iterator it(
 403            nav_results.begin()); it != nav_results.end(); ++it)
 404     matches_.push_back(NavigationToMatch(*it));
 405 }
 406
 407 AutocompleteMatch ZeroSuggestProvider::MatchForCurrentURL() {
 408   AutocompleteMatch match;
 409   AutocompleteClassifierFactory::GetForProfile(profile_)->Classify(
 410       permanent_text_, false, true, current_page_classification_, &match, NULL);
 411   match.is_history_what_you_typed_match = false;
 412   match.allowed_to_be_default_match = true;
 413
 414   // The placeholder suggestion for the current URL has high relevance so
 415   // that it is in the first suggestion slot and inline autocompleted. It
 416   // gets dropped as soon as the user types something.
 417   match.relevance = GetVerbatimRelevance();
 418
 419   return match;
 420 }
 421
 422 int ZeroSuggestProvider::GetVerbatimRelevance() const {
 423   return results_.verbatim_relevance >= 0 ?
 424       results_.verbatim_relevance : kDefaultVerbatimZeroSuggestRelevance;
 425 }
 426
 427 bool ZeroSuggestProvider::CanShowZeroSuggestWithoutSendingURL(
 428     const GURL& suggest_url,
 429     const GURL& current_page_url) const {
 430   if (!ZeroSuggestEnabled(suggest_url,
 431                           template_url_service_->GetDefaultSearchProvider(),
 432                           current_page_classification_,
 433                           template_url_service_->search_terms_data(),
 434                           client_.get()))
 435     return false;
 436
 437   // If we cannot send URLs, then only the MostVisited and Personalized
 438   // variations can be shown.
 439   if (!OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial() &&
 440       !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial())
 441     return false;
 442
 443   // Only show zero suggest for HTTP[S] pages.
 444   // TODO(mariakhomenko): We may be able to expand this set to include pages
 445   // with other schemes (e.g. chrome://). That may require improvements to
 446   // the formatting of the verbatim result returned by MatchForCurrentURL().
 447   if (!current_page_url.is_valid() ||
 448       ((current_page_url.scheme() != url::kHttpScheme) &&
 449       (current_page_url.scheme() != url::kHttpsScheme)))
 450     return false;
 451
 452   return true;
 453 }
 454
 455 void ZeroSuggestProvider::MaybeUseCachedSuggestions() {
 456   if (!OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial())
 457     return;
 458
 459   std::string json_data = profile_->GetPrefs()->GetString(
 460       prefs::kZeroSuggestCachedResults);
 461   if (!json_data.empty()) {
 462     scoped_ptr<base::Value> data(
 463         SearchSuggestionParser::DeserializeJsonData(json_data));
 464     if (data && ParseSuggestResults(
 465             *data, kDefaultZeroSuggestRelevance, false, &results_)) {
 466       ConvertResultsToAutocompleteMatches();
 467       results_from_cache_ = !matches_.empty();
 468     }
 469   }
 470 }