src/chrome/browser/autocomplete/zero_suggest_provider.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/autocomplete/zero_suggest_provider.h"
   6
   7 #include "base/callback.h"
   8 #include "base/i18n/case_conversion.h"
   9 #include "base/json/json_string_value_serializer.h"
  10 #include "base/metrics/histogram.h"
  11 #include "base/prefs/pref_service.h"
  12 #include "base/strings/string16.h"
  13 #include "base/strings/string_util.h"
  14 #include "base/strings/utf_string_conversions.h"
  15 #include "base/time/time.h"
  16 #include "chrome/browser/autocomplete/autocomplete_classifier.h"
  17 #include "chrome/browser/autocomplete/autocomplete_classifier_factory.h"
  18 #include "chrome/browser/autocomplete/autocomplete_input.h"
  19 #include "chrome/browser/autocomplete/autocomplete_match.h"
  20 #include "chrome/browser/autocomplete/autocomplete_provider_listener.h"
  21 #include "chrome/browser/autocomplete/history_url_provider.h"
  22 #include "chrome/browser/autocomplete/search_provider.h"
  23 #include "chrome/browser/autocomplete/url_prefix.h"
  24 #include "chrome/browser/history/history_types.h"
  25 #include "chrome/browser/history/top_sites.h"
  26 #include "chrome/browser/metrics/variations/variations_http_header_provider.h"
  27 #include "chrome/browser/omnibox/omnibox_field_trial.h"
  28 #include "chrome/browser/profiles/profile.h"
  29 #include "chrome/browser/search/search.h"
  30 #include "chrome/browser/search_engines/template_url_service.h"
  31 #include "chrome/browser/search_engines/template_url_service_factory.h"
  32 #include "chrome/common/net/url_fixer_upper.h"
  33 #include "chrome/common/pref_names.h"
  34 #include "chrome/common/url_constants.h"
  35 #include "components/user_prefs/pref_registry_syncable.h"
  36 #include "content/public/browser/user_metrics.h"
  37 #include "net/base/escape.h"
  38 #include "net/base/load_flags.h"
  39 #include "net/base/net_util.h"
  40 #include "net/http/http_request_headers.h"
  41 #include "net/url_request/url_fetcher.h"
  42 #include "net/url_request/url_request_status.h"
  43 #include "url/gurl.h"
  44
  45 namespace {
  46
  47 // TODO(hfung): The histogram code was copied and modified from
  48 // search_provider.cc.  Refactor and consolidate the code.
  49 // We keep track in a histogram how many suggest requests we send, how
  50 // many suggest requests we invalidate (e.g., due to a user typing
  51 // another character), and how many replies we receive.
  52 // *** ADD NEW ENUMS AFTER ALL PREVIOUSLY DEFINED ONES! ***
  53 //     (excluding the end-of-list enum value)
  54 // We do not want values of existing enums to change or else it screws
  55 // up the statistics.
  56 enum ZeroSuggestRequestsHistogramValue {
  57   ZERO_SUGGEST_REQUEST_SENT = 1,
  58   ZERO_SUGGEST_REQUEST_INVALIDATED,
  59   ZERO_SUGGEST_REPLY_RECEIVED,
  60   ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE
  61 };
  62
  63 void LogOmniboxZeroSuggestRequest(
  64     ZeroSuggestRequestsHistogramValue request_value) {
  65   UMA_HISTOGRAM_ENUMERATION("Omnibox.ZeroSuggestRequests", request_value,
  66                             ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE);
  67 }
  68
  69 // The maximum relevance of the top match from this provider.
  70 const int kDefaultVerbatimZeroSuggestRelevance = 1300;
  71
  72 // Relevance value to use if it was not set explicitly by the server.
  73 const int kDefaultZeroSuggestRelevance = 100;
  74
  75 }  // namespace
  76
  77 // static
  78 ZeroSuggestProvider* ZeroSuggestProvider::Create(
  79     AutocompleteProviderListener* listener,
  80     Profile* profile) {
  81   return new ZeroSuggestProvider(listener, profile);
  82 }
  83
  84 // static
  85 void ZeroSuggestProvider::RegisterProfilePrefs(
  86     user_prefs::PrefRegistrySyncable* registry) {
  87   registry->RegisterStringPref(
  88       prefs::kZeroSuggestCachedResults,
  89       std::string(),
  90       user_prefs::PrefRegistrySyncable::UNSYNCABLE_PREF);
  91 }
  92
  93 void ZeroSuggestProvider::Start(const AutocompleteInput& input,
  94                                 bool minimal_changes) {
  95   matches_.clear();
  96   if (input.type() == AutocompleteInput::INVALID)
  97     return;
  98
  99   Stop(true);
 100   field_trial_triggered_ = false;
 101   field_trial_triggered_in_session_ = false;
 102   results_from_cache_ = false;
 103   permanent_text_ = input.text();
 104   current_query_ = input.current_url().spec();
 105   current_page_classification_ = input.current_page_classification();
 106   current_url_match_ = MatchForCurrentURL();
 107
 108   const TemplateURL* default_provider =
 109      template_url_service_->GetDefaultSearchProvider();
 110   if (default_provider == NULL)
 111     return;
 112
 113   base::string16 prefix;
 114   TemplateURLRef::SearchTermsArgs search_term_args(prefix);
 115   GURL suggest_url(default_provider->suggestions_url_ref().ReplaceSearchTerms(
 116       search_term_args));
 117   if (!suggest_url.is_valid())
 118     return;
 119
 120   // No need to send the current page URL in personalized suggest field trial.
 121   if (CanSendURL(input.current_url(), suggest_url, default_provider,
 122                  current_page_classification_, profile_) &&
 123       !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial()) {
 124     // Update suggest_url to include the current_page_url.
 125     search_term_args.current_page_url = current_query_;
 126     suggest_url = GURL(default_provider->suggestions_url_ref().
 127                        ReplaceSearchTerms(search_term_args));
 128   } else if (!CanShowZeroSuggestWithoutSendingURL(suggest_url,
 129                                                   input.current_url())) {
 130     return;
 131   }
 132
 133   done_ = false;
 134   // TODO(jered): Consider adding locally-sourced zero-suggestions here too.
 135   // These may be useful on the NTP or more relevant to the user than server
 136   // suggestions, if based on local browsing history.
 137   MaybeUseCachedSuggestions();
 138   Run(suggest_url);
 139 }
 140
 141 void ZeroSuggestProvider::DeleteMatch(const AutocompleteMatch& match) {
 142   if (OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial()) {
 143     // Remove the deleted match from the cache, so it is not shown to the user
 144     // again. Since we cannot remove just one result, blow away the cache.
 145     profile_->GetPrefs()->SetString(prefs::kZeroSuggestCachedResults,
 146                                     std::string());
 147   }
 148   BaseSearchProvider::DeleteMatch(match);
 149 }
 150
 151 void ZeroSuggestProvider::ResetSession() {
 152   // The user has started editing in the omnibox, so leave
 153   // |field_trial_triggered_in_session_| unchanged and set
 154   // |field_trial_triggered_| to false since zero suggest is inactive now.
 155   field_trial_triggered_ = false;
 156 }
 157
 158 void ZeroSuggestProvider::ModifyProviderInfo(
 159     metrics::OmniboxEventProto_ProviderInfo* provider_info) const {
 160   if (!results_.suggest_results.empty() || !results_.navigation_results.empty())
 161     provider_info->set_times_returned_results_in_session(1);
 162 }
 163
 164 ZeroSuggestProvider::ZeroSuggestProvider(
 165   AutocompleteProviderListener* listener,
 166   Profile* profile)
 167     : BaseSearchProvider(listener, profile,
 168                          AutocompleteProvider::TYPE_ZERO_SUGGEST),
 169       template_url_service_(TemplateURLServiceFactory::GetForProfile(profile)),
 170       results_from_cache_(false),
 171       weak_ptr_factory_(this) {
 172 }
 173
 174 ZeroSuggestProvider::~ZeroSuggestProvider() {
 175 }
 176
 177 bool ZeroSuggestProvider::StoreSuggestionResponse(
 178     const std::string& json_data,
 179     const base::Value& parsed_data) {
 180   if (!OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial() ||
 181       json_data.empty())
 182     return false;
 183   profile_->GetPrefs()->SetString(prefs::kZeroSuggestCachedResults, json_data);
 184
 185   // If we received an empty result list, we should update the display, as it
 186   // may be showing cached results that should not be shown.
 187   const base::ListValue* root_list = NULL;
 188   const base::ListValue* results_list = NULL;
 189   if (parsed_data.GetAsList(&root_list) &&
 190       root_list->GetList(1, &results_list) &&
 191       results_list->empty())
 192     return false;
 193
 194   // We are finished with the request and want to bail early.
 195   if (results_from_cache_)
 196     done_ = true;
 197
 198   return results_from_cache_;
 199 }
 200
 201 const TemplateURL* ZeroSuggestProvider::GetTemplateURL(bool is_keyword) const {
 202   // Zero suggest provider should not receive keyword results.
 203   DCHECK(!is_keyword);
 204   return template_url_service_->GetDefaultSearchProvider();
 205 }
 206
 207 const AutocompleteInput ZeroSuggestProvider::GetInput(bool is_keyword) const {
 208   return AutocompleteInput(
 209       base::string16(), base::string16::npos, base::string16(),
 210       GURL(current_query_), current_page_classification_, true, false, false,
 211       true);
 212 }
 213
 214 BaseSearchProvider::Results* ZeroSuggestProvider::GetResultsToFill(
 215     bool is_keyword) {
 216   DCHECK(!is_keyword);
 217   return &results_;
 218 }
 219
 220 bool ZeroSuggestProvider::ShouldAppendExtraParams(
 221       const SuggestResult& result) const {
 222   // We always use the default provider for search, so append the params.
 223   return true;
 224 }
 225
 226 void ZeroSuggestProvider::StopSuggest() {
 227   if (suggest_results_pending_ > 0)
 228     LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_INVALIDATED);
 229   suggest_results_pending_ = 0;
 230   fetcher_.reset();
 231 }
 232
 233 void ZeroSuggestProvider::ClearAllResults() {
 234   // We do not call Clear() on |results_| to retain |verbatim_relevance|
 235   // value in the |results_| object. |verbatim_relevance| is used at the
 236   // beginning of the next StartZeroSuggest() call to determine the current url
 237   // match relevance.
 238   results_.suggest_results.clear();
 239   results_.navigation_results.clear();
 240   current_query_.clear();
 241 }
 242
 243 int ZeroSuggestProvider::GetDefaultResultRelevance() const {
 244   return kDefaultZeroSuggestRelevance;
 245 }
 246
 247 void ZeroSuggestProvider::RecordDeletionResult(bool success) {
 248   if (success) {
 249     content::RecordAction(
 250         base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Success"));
 251   } else {
 252     content::RecordAction(
 253         base::UserMetricsAction("Omnibox.ZeroSuggestDelete.Failure"));
 254   }
 255 }
 256
 257 void ZeroSuggestProvider::LogFetchComplete(bool success, bool is_keyword) {
 258   LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REPLY_RECEIVED);
 259 }
 260
 261 bool ZeroSuggestProvider::IsKeywordFetcher(
 262     const net::URLFetcher* fetcher) const {
 263   // ZeroSuggestProvider does not have a keyword provider.
 264   DCHECK_EQ(fetcher, fetcher_.get());
 265   return false;
 266 }
 267
 268 void ZeroSuggestProvider::UpdateMatches() {
 269   done_ = true;
 270   ConvertResultsToAutocompleteMatches();
 271 }
 272
 273 void ZeroSuggestProvider::AddSuggestResultsToMap(
 274     const SuggestResults& results,
 275     MatchMap* map) {
 276   for (size_t i = 0; i < results.size(); ++i)
 277     AddMatchToMap(results[i], std::string(), i, false, map);
 278 }
 279
 280 AutocompleteMatch ZeroSuggestProvider::NavigationToMatch(
 281     const NavigationResult& navigation) {
 282   AutocompleteMatch match(this, navigation.relevance(), false,
 283                           navigation.type());
 284   match.destination_url = navigation.url();
 285
 286   // Zero suggest results should always omit protocols and never appear bold.
 287   const std::string languages(
 288       profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
 289   match.contents = net::FormatUrl(navigation.url(), languages,
 290       net::kFormatUrlOmitAll, net::UnescapeRule::SPACES, NULL, NULL, NULL);
 291   match.fill_into_edit +=
 292       AutocompleteInput::FormattedStringWithEquivalentMeaning(navigation.url(),
 293           match.contents);
 294
 295   AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
 296       match.contents.length(), ACMatchClassification::URL,
 297       &match.contents_class);
 298
 299   match.description =
 300       AutocompleteMatch::SanitizeString(navigation.description());
 301   AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
 302       match.description.length(), ACMatchClassification::NONE,
 303       &match.description_class);
 304   return match;
 305 }
 306
 307 void ZeroSuggestProvider::Run(const GURL& suggest_url) {
 308   suggest_results_pending_ = 0;
 309   const int kFetcherID = 1;
 310   fetcher_.reset(
 311       net::URLFetcher::Create(kFetcherID,
 312           suggest_url,
 313           net::URLFetcher::GET, this));
 314   fetcher_->SetRequestContext(profile_->GetRequestContext());
 315   fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES);
 316   // Add Chrome experiment state to the request headers.
 317   net::HttpRequestHeaders headers;
 318   chrome_variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
 319       fetcher_->GetOriginalURL(), profile_->IsOffTheRecord(), false, &headers);
 320   fetcher_->SetExtraRequestHeaders(headers.ToString());
 321   fetcher_->Start();
 322
 323   if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
 324     most_visited_urls_.clear();
 325     history::TopSites* ts = profile_->GetTopSites();
 326     if (ts) {
 327       ts->GetMostVisitedURLs(
 328           base::Bind(&ZeroSuggestProvider::OnMostVisitedUrlsAvailable,
 329                      weak_ptr_factory_.GetWeakPtr()), false);
 330     }
 331   }
 332   suggest_results_pending_ = 1;
 333   LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_SENT);
 334 }
 335
 336 void ZeroSuggestProvider::OnMostVisitedUrlsAvailable(
 337     const history::MostVisitedURLList& urls) {
 338   most_visited_urls_ = urls;
 339 }
 340
 341 void ZeroSuggestProvider::ConvertResultsToAutocompleteMatches() {
 342   matches_.clear();
 343
 344   const TemplateURL* default_provider =
 345       template_url_service_->GetDefaultSearchProvider();
 346   // Fail if we can't set the clickthrough URL for query suggestions.
 347   if (default_provider == NULL || !default_provider->SupportsReplacement())
 348     return;
 349
 350   MatchMap map;
 351   AddSuggestResultsToMap(results_.suggest_results, &map);
 352
 353   const int num_query_results = map.size();
 354   const int num_nav_results = results_.navigation_results.size();
 355   const int num_results = num_query_results + num_nav_results;
 356   UMA_HISTOGRAM_COUNTS("ZeroSuggest.QueryResults", num_query_results);
 357   UMA_HISTOGRAM_COUNTS("ZeroSuggest.URLResults", num_nav_results);
 358   UMA_HISTOGRAM_COUNTS("ZeroSuggest.AllResults", num_results);
 359
 360   // Show Most Visited results after ZeroSuggest response is received.
 361   if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
 362     if (!current_url_match_.destination_url.is_valid())
 363       return;
 364     matches_.push_back(current_url_match_);
 365     int relevance = 600;
 366     if (num_results > 0) {
 367       UMA_HISTOGRAM_COUNTS(
 368           "Omnibox.ZeroSuggest.MostVisitedResultsCounterfactual",
 369           most_visited_urls_.size());
 370     }
 371     const base::string16 current_query_string16(
 372         base::ASCIIToUTF16(current_query_));
 373     const std::string languages(
 374         profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
 375     for (size_t i = 0; i < most_visited_urls_.size(); i++) {
 376       const history::MostVisitedURL& url = most_visited_urls_[i];
 377       NavigationResult nav(*this, url.url, AutocompleteMatchType::NAVSUGGEST,
 378           url.title, std::string(), false, relevance, true,
 379           current_query_string16, languages);
 380       matches_.push_back(NavigationToMatch(nav));
 381       --relevance;
 382     }
 383     return;
 384   }
 385
 386   if (num_results == 0)
 387     return;
 388
 389   // TODO(jered): Rip this out once the first match is decoupled from the
 390   // current typing in the omnibox.
 391   matches_.push_back(current_url_match_);
 392
 393   for (MatchMap::const_iterator it(map.begin()); it != map.end(); ++it)
 394     matches_.push_back(it->second);
 395
 396   const NavigationResults& nav_results(results_.navigation_results);
 397   for (NavigationResults::const_iterator it(nav_results.begin());
 398        it != nav_results.end(); ++it)
 399     matches_.push_back(NavigationToMatch(*it));
 400 }
 401
 402 AutocompleteMatch ZeroSuggestProvider::MatchForCurrentURL() {
 403   AutocompleteMatch match;
 404   AutocompleteClassifierFactory::GetForProfile(profile_)->Classify(
 405       permanent_text_, false, true, current_page_classification_, &match, NULL);
 406   match.is_history_what_you_typed_match = false;
 407   match.allowed_to_be_default_match = true;
 408
 409   // The placeholder suggestion for the current URL has high relevance so
 410   // that it is in the first suggestion slot and inline autocompleted. It
 411   // gets dropped as soon as the user types something.
 412   match.relevance = GetVerbatimRelevance();
 413
 414   return match;
 415 }
 416
 417 int ZeroSuggestProvider::GetVerbatimRelevance() const {
 418   return results_.verbatim_relevance >= 0 ?
 419       results_.verbatim_relevance : kDefaultVerbatimZeroSuggestRelevance;
 420 }
 421
 422 bool ZeroSuggestProvider::CanShowZeroSuggestWithoutSendingURL(
 423     const GURL& suggest_url,
 424     const GURL& current_page_url) const {
 425   if (!ZeroSuggestEnabled(suggest_url,
 426                           template_url_service_->GetDefaultSearchProvider(),
 427                           current_page_classification_, profile_))
 428     return false;
 429
 430   // If we cannot send URLs, then only the MostVisited and Personalized
 431   // variations can be shown.
 432   if (!OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial() &&
 433       !OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial())
 434     return false;
 435
 436   // Only show zero suggest for HTTP[S] pages.
 437   // TODO(mariakhomenko): We may be able to expand this set to include pages
 438   // with other schemes (e.g. chrome://). That may require improvements to
 439   // the formatting of the verbatim result returned by MatchForCurrentURL().
 440   if (!current_page_url.is_valid() ||
 441       ((current_page_url.scheme() != url::kHttpScheme) &&
 442       (current_page_url.scheme() != url::kHttpsScheme)))
 443     return false;
 444
 445   return true;
 446 }
 447
 448 void ZeroSuggestProvider::MaybeUseCachedSuggestions() {
 449   if (!OmniboxFieldTrial::InZeroSuggestPersonalizedFieldTrial())
 450     return;
 451
 452   std::string json_data = profile_->GetPrefs()->GetString(
 453       prefs::kZeroSuggestCachedResults);
 454   if (!json_data.empty()) {
 455     scoped_ptr<base::Value> data(DeserializeJsonData(json_data));
 456     if (data && ParseSuggestResults(*data.get(), false, &results_)) {
 457       ConvertResultsToAutocompleteMatches();
 458       results_from_cache_ = !matches_.empty();
 459     }
 460   }
 461 }