src/chrome/browser/autocomplete/zero_suggest_provider.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/autocomplete/zero_suggest_provider.h"
   6
   7 #include "base/callback.h"
   8 #include "base/i18n/case_conversion.h"
   9 #include "base/json/json_string_value_serializer.h"
  10 #include "base/metrics/histogram.h"
  11 #include "base/prefs/pref_service.h"
  12 #include "base/strings/string16.h"
  13 #include "base/strings/string_util.h"
  14 #include "base/strings/utf_string_conversions.h"
  15 #include "base/time/time.h"
  16 #include "chrome/browser/autocomplete/autocomplete_classifier.h"
  17 #include "chrome/browser/autocomplete/autocomplete_classifier_factory.h"
  18 #include "chrome/browser/autocomplete/autocomplete_input.h"
  19 #include "chrome/browser/autocomplete/autocomplete_match.h"
  20 #include "chrome/browser/autocomplete/autocomplete_provider_listener.h"
  21 #include "chrome/browser/autocomplete/history_url_provider.h"
  22 #include "chrome/browser/autocomplete/search_provider.h"
  23 #include "chrome/browser/autocomplete/url_prefix.h"
  24 #include "chrome/browser/history/history_types.h"
  25 #include "chrome/browser/history/top_sites.h"
  26 #include "chrome/browser/metrics/variations/variations_http_header_provider.h"
  27 #include "chrome/browser/omnibox/omnibox_field_trial.h"
  28 #include "chrome/browser/profiles/profile.h"
  29 #include "chrome/browser/search/search.h"
  30 #include "chrome/browser/search_engines/template_url_service.h"
  31 #include "chrome/browser/search_engines/template_url_service_factory.h"
  32 #include "chrome/common/net/url_fixer_upper.h"
  33 #include "chrome/common/pref_names.h"
  34 #include "chrome/common/url_constants.h"
  35 #include "net/base/escape.h"
  36 #include "net/base/load_flags.h"
  37 #include "net/base/net_util.h"
  38 #include "net/http/http_request_headers.h"
  39 #include "net/http/http_response_headers.h"
  40 #include "net/url_request/url_fetcher.h"
  41 #include "net/url_request/url_request_status.h"
  42 #include "url/gurl.h"
  43
  44 namespace {
  45
  46 // TODO(hfung): The histogram code was copied and modified from
  47 // search_provider.cc.  Refactor and consolidate the code.
  48 // We keep track in a histogram how many suggest requests we send, how
  49 // many suggest requests we invalidate (e.g., due to a user typing
  50 // another character), and how many replies we receive.
  51 // *** ADD NEW ENUMS AFTER ALL PREVIOUSLY DEFINED ONES! ***
  52 //     (excluding the end-of-list enum value)
  53 // We do not want values of existing enums to change or else it screws
  54 // up the statistics.
  55 enum ZeroSuggestRequestsHistogramValue {
  56   ZERO_SUGGEST_REQUEST_SENT = 1,
  57   ZERO_SUGGEST_REQUEST_INVALIDATED,
  58   ZERO_SUGGEST_REPLY_RECEIVED,
  59   ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE
  60 };
  61
  62 void LogOmniboxZeroSuggestRequest(
  63     ZeroSuggestRequestsHistogramValue request_value) {
  64   UMA_HISTOGRAM_ENUMERATION("Omnibox.ZeroSuggestRequests", request_value,
  65                             ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE);
  66 }
  67
  68 // The maximum relevance of the top match from this provider.
  69 const int kDefaultVerbatimZeroSuggestRelevance = 1300;
  70
  71 // Relevance value to use if it was not set explicitly by the server.
  72 const int kDefaultZeroSuggestRelevance = 100;
  73
  74 }  // namespace
  75
  76 // static
  77 ZeroSuggestProvider* ZeroSuggestProvider::Create(
  78     AutocompleteProviderListener* listener,
  79     Profile* profile) {
  80   return new ZeroSuggestProvider(listener, profile);
  81 }
  82
  83 void ZeroSuggestProvider::Start(const AutocompleteInput& input,
  84                                 bool /*minimal_changes*/) {
  85 }
  86
  87 void ZeroSuggestProvider::ResetSession() {
  88   // The user has started editing in the omnibox, so leave
  89   // |field_trial_triggered_in_session_| unchanged and set
  90   // |field_trial_triggered_| to false since zero suggest is inactive now.
  91   field_trial_triggered_ = false;
  92   Stop(true);
  93 }
  94
  95 void ZeroSuggestProvider::OnURLFetchComplete(const net::URLFetcher* source) {
  96   have_pending_request_ = false;
  97   LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REPLY_RECEIVED);
  98
  99   std::string json_data;
 100   source->GetResponseAsString(&json_data);
 101   const bool request_succeeded =
 102       source->GetStatus().is_success() && source->GetResponseCode() == 200;
 103
 104   if (request_succeeded) {
 105     scoped_ptr<base::Value> data(DeserializeJsonData(json_data));
 106     if (data.get())
 107       ParseSuggestResults(*data.get());
 108   }
 109   done_ = true;
 110
 111   ConvertResultsToAutocompleteMatches();
 112   if (!matches_.empty())
 113     listener_->OnProviderUpdate(true);
 114 }
 115
 116 void ZeroSuggestProvider::StartZeroSuggest(
 117     const GURL& current_page_url,
 118     AutocompleteInput::PageClassification page_classification,
 119     const base::string16& permanent_text) {
 120   Stop(true);
 121   field_trial_triggered_ = false;
 122   field_trial_triggered_in_session_ = false;
 123   permanent_text_ = permanent_text;
 124   current_query_ = current_page_url.spec();
 125   current_page_classification_ = page_classification;
 126   current_url_match_ = MatchForCurrentURL();
 127
 128   const TemplateURL* default_provider =
 129      template_url_service_->GetDefaultSearchProvider();
 130   if (default_provider == NULL)
 131     return;
 132   base::string16 prefix;
 133   TemplateURLRef::SearchTermsArgs search_term_args(prefix);
 134   search_term_args.current_page_url = current_query_;
 135   GURL suggest_url(default_provider->suggestions_url_ref().
 136                    ReplaceSearchTerms(search_term_args));
 137   if (!CanSendURL(current_page_url, suggest_url,
 138           template_url_service_->GetDefaultSearchProvider(),
 139           page_classification, profile_) ||
 140       !OmniboxFieldTrial::InZeroSuggestFieldTrial())
 141     return;
 142   verbatim_relevance_ = kDefaultVerbatimZeroSuggestRelevance;
 143   done_ = false;
 144   // TODO(jered): Consider adding locally-sourced zero-suggestions here too.
 145   // These may be useful on the NTP or more relevant to the user than server
 146   // suggestions, if based on local browsing history.
 147   Run(suggest_url);
 148 }
 149
 150 ZeroSuggestProvider::ZeroSuggestProvider(
 151   AutocompleteProviderListener* listener,
 152   Profile* profile)
 153     : BaseSearchProvider(listener, profile,
 154                          AutocompleteProvider::TYPE_ZERO_SUGGEST),
 155       template_url_service_(TemplateURLServiceFactory::GetForProfile(profile)),
 156       have_pending_request_(false),
 157       verbatim_relevance_(kDefaultVerbatimZeroSuggestRelevance),
 158       weak_ptr_factory_(this) {
 159 }
 160
 161 ZeroSuggestProvider::~ZeroSuggestProvider() {
 162 }
 163
 164 const TemplateURL* ZeroSuggestProvider::GetTemplateURL(
 165     const SuggestResult& result) const {
 166   // Zero suggest provider should not receive keyword results.
 167   DCHECK(!result.from_keyword_provider());
 168   return template_url_service_->GetDefaultSearchProvider();
 169 }
 170
 171 const AutocompleteInput ZeroSuggestProvider::GetInput(
 172     const SuggestResult& result) const {
 173   AutocompleteInput input;
 174   // Set |input|'s text to be |query_string| to avoid bolding.
 175   input.UpdateText(result.suggestion(), base::string16::npos, input.parts());
 176   return input;
 177 }
 178
 179 bool ZeroSuggestProvider::ShouldAppendExtraParams(
 180       const SuggestResult& result) const {
 181   // We always use the default provider for search, so append the params.
 182   return true;
 183 }
 184
 185 void ZeroSuggestProvider::StopSuggest() {
 186   if (have_pending_request_)
 187     LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_INVALIDATED);
 188   have_pending_request_ = false;
 189   fetcher_.reset();
 190 }
 191
 192 void ZeroSuggestProvider::ClearAllResults() {
 193   query_matches_map_.clear();
 194   navigation_results_.clear();
 195   current_query_.clear();
 196   matches_.clear();
 197 }
 198
 199 void ZeroSuggestProvider::FillResults(const base::Value& root_val,
 200                                       int* verbatim_relevance,
 201                                       SuggestResults* suggest_results,
 202                                       NavigationResults* navigation_results) {
 203   base::string16 query;
 204   const base::ListValue* root_list = NULL;
 205   const base::ListValue* results = NULL;
 206   const base::ListValue* relevances = NULL;
 207   // The response includes the query, which should be empty for ZeroSuggest
 208   // responses.
 209   if (!root_val.GetAsList(&root_list) || !root_list->GetString(0, &query) ||
 210       (!query.empty()) || !root_list->GetList(1, &results))
 211     return;
 212
 213   // 3rd element: Description list.
 214   const base::ListValue* descriptions = NULL;
 215   root_list->GetList(2, &descriptions);
 216
 217   // 4th element: Disregard the query URL list for now.
 218
 219   // Reset suggested relevance information from the provider.
 220   *verbatim_relevance = kDefaultVerbatimZeroSuggestRelevance;
 221
 222   // 5th element: Optional key-value pairs from the Suggest server.
 223   const base::ListValue* types = NULL;
 224   const base::DictionaryValue* extras = NULL;
 225   if (root_list->GetDictionary(4, &extras)) {
 226     extras->GetList("google:suggesttype", &types);
 227
 228     // Discard this list if its size does not match that of the suggestions.
 229     if (extras->GetList("google:suggestrelevance", &relevances) &&
 230         relevances->GetSize() != results->GetSize())
 231       relevances = NULL;
 232     extras->GetInteger("google:verbatimrelevance", verbatim_relevance);
 233
 234     // Check if the active suggest field trial (if any) has triggered.
 235     bool triggered = false;
 236     extras->GetBoolean("google:fieldtrialtriggered", &triggered);
 237     field_trial_triggered_ |= triggered;
 238     field_trial_triggered_in_session_ |= triggered;
 239   }
 240
 241   // Clear the previous results now that new results are available.
 242   suggest_results->clear();
 243   navigation_results->clear();
 244
 245   base::string16 result, title;
 246   std::string type;
 247   const base::string16 current_query_string16 =
 248       base::ASCIIToUTF16(current_query_);
 249   const std::string languages(
 250       profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
 251   for (size_t index = 0; results->GetString(index, &result); ++index) {
 252     // Google search may return empty suggestions for weird input characters,
 253     // they make no sense at all and can cause problems in our code.
 254     if (result.empty())
 255       continue;
 256
 257     int relevance = kDefaultZeroSuggestRelevance;
 258
 259     // Apply valid suggested relevance scores; discard invalid lists.
 260     if (relevances != NULL && !relevances->GetInteger(index, &relevance))
 261       relevances = NULL;
 262     if (types && types->GetString(index, &type) && (type == "NAVIGATION")) {
 263       // Do not blindly trust the URL coming from the server to be valid.
 264       GURL url(URLFixerUpper::FixupURL(
 265           base::UTF16ToUTF8(result), std::string()));
 266       if (url.is_valid()) {
 267         if (descriptions != NULL)
 268           descriptions->GetString(index, &title);
 269         navigation_results->push_back(NavigationResult(
 270             *this, url, title, false, relevance, relevances != NULL,
 271             current_query_string16, languages));
 272       }
 273     } else {
 274       suggest_results->push_back(SuggestResult(
 275           result, AutocompleteMatchType::SEARCH_SUGGEST, result,
 276           base::string16(), std::string(), std::string(), false, relevance,
 277           relevances != NULL, false, current_query_string16));
 278     }
 279   }
 280 }
 281
 282 void ZeroSuggestProvider::AddSuggestResultsToMap(
 283     const SuggestResults& results,
 284     MatchMap* map) {
 285   for (size_t i = 0; i < results.size(); ++i) {
 286     const base::string16& query_string(results[i].suggestion());
 287     // TODO(mariakhomenko): Do not reconstruct SuggestResult objects with
 288     // a different query -- create correct objects to begin with.
 289     const SuggestResult suggestion(
 290         query_string, AutocompleteMatchType::SEARCH_SUGGEST, query_string,
 291         base::string16(), std::string(), std::string(), false,
 292         results[i].relevance(), true, false, query_string);
 293     AddMatchToMap(suggestion, std::string(), i, map);
 294   }
 295 }
 296
 297 AutocompleteMatch ZeroSuggestProvider::NavigationToMatch(
 298     const NavigationResult& navigation) {
 299   AutocompleteMatch match(this, navigation.relevance(), false,
 300                           AutocompleteMatchType::NAVSUGGEST);
 301   match.destination_url = navigation.url();
 302
 303   // Zero suggest results should always omit protocols and never appear bold.
 304   const std::string languages(
 305       profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
 306   match.contents = net::FormatUrl(navigation.url(), languages,
 307       net::kFormatUrlOmitAll, net::UnescapeRule::SPACES, NULL, NULL, NULL);
 308   match.fill_into_edit +=
 309       AutocompleteInput::FormattedStringWithEquivalentMeaning(navigation.url(),
 310           match.contents);
 311
 312   AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
 313       match.contents.length(), ACMatchClassification::URL,
 314       &match.contents_class);
 315
 316   match.description =
 317       AutocompleteMatch::SanitizeString(navigation.description());
 318   AutocompleteMatch::ClassifyLocationInString(base::string16::npos, 0,
 319       match.description.length(), ACMatchClassification::NONE,
 320       &match.description_class);
 321   return match;
 322 }
 323
 324 void ZeroSuggestProvider::Run(const GURL& suggest_url) {
 325   have_pending_request_ = false;
 326   const int kFetcherID = 1;
 327   fetcher_.reset(
 328       net::URLFetcher::Create(kFetcherID,
 329           suggest_url,
 330           net::URLFetcher::GET, this));
 331   fetcher_->SetRequestContext(profile_->GetRequestContext());
 332   fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES);
 333   // Add Chrome experiment state to the request headers.
 334   net::HttpRequestHeaders headers;
 335   chrome_variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
 336       fetcher_->GetOriginalURL(), profile_->IsOffTheRecord(), false, &headers);
 337   fetcher_->SetExtraRequestHeaders(headers.ToString());
 338
 339   fetcher_->Start();
 340
 341   if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
 342     most_visited_urls_.clear();
 343     history::TopSites* ts = profile_->GetTopSites();
 344     if (ts) {
 345       ts->GetMostVisitedURLs(
 346           base::Bind(&ZeroSuggestProvider::OnMostVisitedUrlsAvailable,
 347                      weak_ptr_factory_.GetWeakPtr()), false);
 348     }
 349   }
 350   have_pending_request_ = true;
 351   LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_SENT);
 352 }
 353
 354 void ZeroSuggestProvider::ParseSuggestResults(const base::Value& root_val) {
 355   SuggestResults suggest_results;
 356   FillResults(root_val, &verbatim_relevance_,
 357               &suggest_results, &navigation_results_);
 358
 359   query_matches_map_.clear();
 360   AddSuggestResultsToMap(suggest_results, &query_matches_map_);
 361 }
 362
 363 void ZeroSuggestProvider::OnMostVisitedUrlsAvailable(
 364     const history::MostVisitedURLList& urls) {
 365   most_visited_urls_ = urls;
 366 }
 367
 368 void ZeroSuggestProvider::ConvertResultsToAutocompleteMatches() {
 369   matches_.clear();
 370
 371   const TemplateURL* default_provider =
 372       template_url_service_->GetDefaultSearchProvider();
 373   // Fail if we can't set the clickthrough URL for query suggestions.
 374   if (default_provider == NULL || !default_provider->SupportsReplacement())
 375     return;
 376
 377   const int num_query_results = query_matches_map_.size();
 378   const int num_nav_results = navigation_results_.size();
 379   const int num_results = num_query_results + num_nav_results;
 380   UMA_HISTOGRAM_COUNTS("ZeroSuggest.QueryResults", num_query_results);
 381   UMA_HISTOGRAM_COUNTS("ZeroSuggest.URLResults",  num_nav_results);
 382   UMA_HISTOGRAM_COUNTS("ZeroSuggest.AllResults", num_results);
 383
 384   // Show Most Visited results after ZeroSuggest response is received.
 385   if (OmniboxFieldTrial::InZeroSuggestMostVisitedFieldTrial()) {
 386     if (!current_url_match_.destination_url.is_valid())
 387       return;
 388     matches_.push_back(current_url_match_);
 389     int relevance = 600;
 390     if (num_results > 0) {
 391       UMA_HISTOGRAM_COUNTS(
 392           "Omnibox.ZeroSuggest.MostVisitedResultsCounterfactual",
 393           most_visited_urls_.size());
 394     }
 395     const base::string16 current_query_string16(
 396         base::ASCIIToUTF16(current_query_));
 397     const std::string languages(
 398         profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
 399     for (size_t i = 0; i < most_visited_urls_.size(); i++) {
 400       const history::MostVisitedURL& url = most_visited_urls_[i];
 401       NavigationResult nav(*this, url.url, url.title, false, relevance, true,
 402           current_query_string16, languages);
 403       matches_.push_back(NavigationToMatch(nav));
 404       --relevance;
 405     }
 406     return;
 407   }
 408
 409   if (num_results == 0)
 410     return;
 411
 412   // TODO(jered): Rip this out once the first match is decoupled from the
 413   // current typing in the omnibox.
 414   matches_.push_back(current_url_match_);
 415
 416   for (MatchMap::const_iterator it(query_matches_map_.begin());
 417        it != query_matches_map_.end(); ++it)
 418     matches_.push_back(it->second);
 419
 420   for (NavigationResults::const_iterator it(navigation_results_.begin());
 421        it != navigation_results_.end(); ++it)
 422     matches_.push_back(NavigationToMatch(*it));
 423 }
 424
 425 AutocompleteMatch ZeroSuggestProvider::MatchForCurrentURL() {
 426   AutocompleteInput input(permanent_text_, base::string16::npos, base::string16(),
 427                           GURL(current_query_), current_page_classification_,
 428                           false, false, true, AutocompleteInput::ALL_MATCHES);
 429
 430   AutocompleteMatch match;
 431   AutocompleteClassifierFactory::GetForProfile(profile_)->Classify(
 432       permanent_text_, false, true, current_page_classification_, &match, NULL);
 433   match.is_history_what_you_typed_match = false;
 434   match.allowed_to_be_default_match = true;
 435
 436   // The placeholder suggestion for the current URL has high relevance so
 437   // that it is in the first suggestion slot and inline autocompleted. It
 438   // gets dropped as soon as the user types something.
 439   match.relevance = verbatim_relevance_;
 440
 441   return match;
 442 }