src/chrome/browser/autocomplete/base_search_provider.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/autocomplete/base_search_provider.h"
   6
   7 #include "base/i18n/case_conversion.h"
   8 #include "base/prefs/pref_registry_simple.h"
   9 #include "base/prefs/pref_service.h"
  10 #include "base/strings/string_util.h"
  11 #include "base/strings/utf_string_conversions.h"
  12 #include "chrome/browser/autocomplete/chrome_autocomplete_scheme_classifier.h"
  13 #include "chrome/browser/bitmap_fetcher/bitmap_fetcher_service.h"
  14 #include "chrome/browser/bitmap_fetcher/bitmap_fetcher_service_factory.h"
  15 #include "chrome/browser/history/history_service.h"
  16 #include "chrome/browser/history/history_service_factory.h"
  17 #include "chrome/browser/profiles/profile.h"
  18 #include "chrome/browser/sync/profile_sync_service.h"
  19 #include "chrome/browser/sync/profile_sync_service_factory.h"
  20 #include "chrome/common/pref_names.h"
  21 #include "components/metrics/proto/omnibox_event.pb.h"
  22 #include "components/metrics/proto/omnibox_input_type.pb.h"
  23 #include "components/omnibox/autocomplete_provider_listener.h"
  24 #include "components/omnibox/omnibox_field_trial.h"
  25 #include "components/search_engines/template_url.h"
  26 #include "components/search_engines/template_url_prepopulate_data.h"
  27 #include "components/search_engines/template_url_service.h"
  28 #include "components/sync_driver/sync_prefs.h"
  29 #include "net/base/escape.h"
  30 #include "net/base/registry_controlled_domains/registry_controlled_domain.h"
  31 #include "net/url_request/url_fetcher.h"
  32 #include "net/url_request/url_fetcher_delegate.h"
  33 #include "url/gurl.h"
  34
  35 using metrics::OmniboxEventProto;
  36
  37 // SuggestionDeletionHandler -------------------------------------------------
  38
  39 // This class handles making requests to the server in order to delete
  40 // personalized suggestions.
  41 class SuggestionDeletionHandler : public net::URLFetcherDelegate {
  42  public:
  43   typedef base::Callback<void(bool, SuggestionDeletionHandler*)>
  44       DeletionCompletedCallback;
  45
  46   SuggestionDeletionHandler(
  47       const std::string& deletion_url,
  48       Profile* profile,
  49       const DeletionCompletedCallback& callback);
  50
  51   virtual ~SuggestionDeletionHandler();
  52
  53  private:
  54   // net::URLFetcherDelegate:
  55   virtual void OnURLFetchComplete(const net::URLFetcher* source) OVERRIDE;
  56
  57   scoped_ptr<net::URLFetcher> deletion_fetcher_;
  58   DeletionCompletedCallback callback_;
  59
  60   DISALLOW_COPY_AND_ASSIGN(SuggestionDeletionHandler);
  61 };
  62
  63 SuggestionDeletionHandler::SuggestionDeletionHandler(
  64     const std::string& deletion_url,
  65     Profile* profile,
  66     const DeletionCompletedCallback& callback) : callback_(callback) {
  67   GURL url(deletion_url);
  68   DCHECK(url.is_valid());
  69
  70   deletion_fetcher_.reset(net::URLFetcher::Create(
  71       BaseSearchProvider::kDeletionURLFetcherID,
  72       url,
  73       net::URLFetcher::GET,
  74       this));
  75   deletion_fetcher_->SetRequestContext(profile->GetRequestContext());
  76   deletion_fetcher_->Start();
  77 }
  78
  79 SuggestionDeletionHandler::~SuggestionDeletionHandler() {
  80 }
  81
  82 void SuggestionDeletionHandler::OnURLFetchComplete(
  83     const net::URLFetcher* source) {
  84   DCHECK(source == deletion_fetcher_.get());
  85   callback_.Run(
  86       source->GetStatus().is_success() && (source->GetResponseCode() == 200),
  87       this);
  88 }
  89
  90 // BaseSearchProvider ---------------------------------------------------------
  91
  92 // static
  93 const int BaseSearchProvider::kDefaultProviderURLFetcherID = 1;
  94 const int BaseSearchProvider::kKeywordProviderURLFetcherID = 2;
  95 const int BaseSearchProvider::kDeletionURLFetcherID = 3;
  96
  97 BaseSearchProvider::BaseSearchProvider(TemplateURLService* template_url_service,
  98                                        Profile* profile,
  99                                        AutocompleteProvider::Type type)
 100     : AutocompleteProvider(type),
 101       template_url_service_(template_url_service),
 102       profile_(profile),
 103       field_trial_triggered_(false),
 104       field_trial_triggered_in_session_(false) {
 105 }
 106
 107 // static
 108 bool BaseSearchProvider::ShouldPrefetch(const AutocompleteMatch& match) {
 109   return match.GetAdditionalInfo(kShouldPrefetchKey) == kTrue;
 110 }
 111
 112 // static
 113 AutocompleteMatch BaseSearchProvider::CreateSearchSuggestion(
 114     const base::string16& suggestion,
 115     AutocompleteMatchType::Type type,
 116     bool from_keyword_provider,
 117     const TemplateURL* template_url,
 118     const SearchTermsData& search_terms_data) {
 119   // This call uses a number of default values.  For instance, it assumes that
 120   // if this match is from a keyword provider than the user is in keyword mode.
 121   return CreateSearchSuggestion(
 122       NULL, AutocompleteInput(), from_keyword_provider,
 123       SearchSuggestionParser::SuggestResult(
 124           suggestion, type, suggestion, base::string16(), base::string16(),
 125           base::string16(), base::string16(), std::string(), std::string(),
 126           from_keyword_provider, 0, false, false, base::string16()),
 127       template_url, search_terms_data, 0, false);
 128 }
 129
 130 void BaseSearchProvider::Stop(bool clear_cached_results) {
 131   StopSuggest();
 132   done_ = true;
 133
 134   if (clear_cached_results)
 135     ClearAllResults();
 136 }
 137
 138 void BaseSearchProvider::DeleteMatch(const AutocompleteMatch& match) {
 139   DCHECK(match.deletable);
 140   if (!match.GetAdditionalInfo(BaseSearchProvider::kDeletionUrlKey).empty()) {
 141     deletion_handlers_.push_back(new SuggestionDeletionHandler(
 142         match.GetAdditionalInfo(BaseSearchProvider::kDeletionUrlKey),
 143         profile_,
 144         base::Bind(&BaseSearchProvider::OnDeletionComplete,
 145                    base::Unretained(this))));
 146   }
 147
 148   HistoryService* const history_service =
 149       HistoryServiceFactory::GetForProfile(profile_, Profile::EXPLICIT_ACCESS);
 150   TemplateURL* template_url =
 151       match.GetTemplateURL(template_url_service_, false);
 152   // This may be NULL if the template corresponding to the keyword has been
 153   // deleted or there is no keyword set.
 154   if (template_url != NULL) {
 155     history_service->DeleteMatchingURLsForKeyword(template_url->id(),
 156                                                   match.contents);
 157   }
 158
 159   // Immediately update the list of matches to show the match was deleted,
 160   // regardless of whether the server request actually succeeds.
 161   DeleteMatchFromMatches(match);
 162 }
 163
 164 void BaseSearchProvider::AddProviderInfo(ProvidersInfo* provider_info) const {
 165   provider_info->push_back(metrics::OmniboxEventProto_ProviderInfo());
 166   metrics::OmniboxEventProto_ProviderInfo& new_entry = provider_info->back();
 167   new_entry.set_provider(AsOmniboxEventProviderType());
 168   new_entry.set_provider_done(done_);
 169   std::vector<uint32> field_trial_hashes;
 170   OmniboxFieldTrial::GetActiveSuggestFieldTrialHashes(&field_trial_hashes);
 171   for (size_t i = 0; i < field_trial_hashes.size(); ++i) {
 172     if (field_trial_triggered_)
 173       new_entry.mutable_field_trial_triggered()->Add(field_trial_hashes[i]);
 174     if (field_trial_triggered_in_session_) {
 175       new_entry.mutable_field_trial_triggered_in_session()->Add(
 176           field_trial_hashes[i]);
 177     }
 178   }
 179   ModifyProviderInfo(&new_entry);
 180 }
 181
 182 // static
 183 const char BaseSearchProvider::kRelevanceFromServerKey[] =
 184     "relevance_from_server";
 185 const char BaseSearchProvider::kShouldPrefetchKey[] = "should_prefetch";
 186 const char BaseSearchProvider::kSuggestMetadataKey[] = "suggest_metadata";
 187 const char BaseSearchProvider::kDeletionUrlKey[] = "deletion_url";
 188 const char BaseSearchProvider::kTrue[] = "true";
 189 const char BaseSearchProvider::kFalse[] = "false";
 190
 191 BaseSearchProvider::~BaseSearchProvider() {}
 192
 193 void BaseSearchProvider::SetDeletionURL(const std::string& deletion_url,
 194                                         AutocompleteMatch* match) {
 195   if (deletion_url.empty())
 196     return;
 197   if (!template_url_service_)
 198     return;
 199   GURL url =
 200       template_url_service_->GetDefaultSearchProvider()->GenerateSearchURL(
 201           template_url_service_->search_terms_data());
 202   url = url.GetOrigin().Resolve(deletion_url);
 203   if (url.is_valid()) {
 204     match->RecordAdditionalInfo(BaseSearchProvider::kDeletionUrlKey,
 205         url.spec());
 206     match->deletable = true;
 207   }
 208 }
 209
 210 // static
 211 AutocompleteMatch BaseSearchProvider::CreateSearchSuggestion(
 212     AutocompleteProvider* autocomplete_provider,
 213     const AutocompleteInput& input,
 214     const bool in_keyword_mode,
 215     const SearchSuggestionParser::SuggestResult& suggestion,
 216     const TemplateURL* template_url,
 217     const SearchTermsData& search_terms_data,
 218     int accepted_suggestion,
 219     bool append_extra_query_params) {
 220   AutocompleteMatch match(autocomplete_provider, suggestion.relevance(), false,
 221                           suggestion.type());
 222
 223   if (!template_url)
 224     return match;
 225   match.keyword = template_url->keyword();
 226   match.contents = suggestion.match_contents();
 227   match.contents_class = suggestion.match_contents_class();
 228   match.answer_contents = suggestion.answer_contents();
 229   match.answer_type = suggestion.answer_type();
 230   if (suggestion.type() == AutocompleteMatchType::SEARCH_SUGGEST_INFINITE) {
 231     match.RecordAdditionalInfo(
 232         kACMatchPropertyInputText, base::UTF16ToUTF8(input.text()));
 233     match.RecordAdditionalInfo(
 234         kACMatchPropertyContentsPrefix,
 235         base::UTF16ToUTF8(suggestion.match_contents_prefix()));
 236     match.RecordAdditionalInfo(
 237         kACMatchPropertyContentsStartIndex,
 238         static_cast<int>(
 239             suggestion.suggestion().length() - match.contents.length()));
 240   }
 241
 242   if (!suggestion.annotation().empty())
 243     match.description = suggestion.annotation();
 244
 245   // suggestion.match_contents() should have already been collapsed.
 246   match.allowed_to_be_default_match =
 247       (!in_keyword_mode || suggestion.from_keyword_provider()) &&
 248       (base::CollapseWhitespace(input.text(), false) ==
 249        suggestion.match_contents());
 250
 251   // When the user forced a query, we need to make sure all the fill_into_edit
 252   // values preserve that property.  Otherwise, if the user starts editing a
 253   // suggestion, non-Search results will suddenly appear.
 254   if (input.type() == metrics::OmniboxInputType::FORCED_QUERY)
 255     match.fill_into_edit.assign(base::ASCIIToUTF16("?"));
 256   if (suggestion.from_keyword_provider())
 257     match.fill_into_edit.append(match.keyword + base::char16(' '));
 258   if (!input.prevent_inline_autocomplete() &&
 259       (!in_keyword_mode || suggestion.from_keyword_provider()) &&
 260       StartsWith(suggestion.suggestion(), input.text(), false)) {
 261     match.inline_autocompletion =
 262         suggestion.suggestion().substr(input.text().length());
 263     match.allowed_to_be_default_match = true;
 264   }
 265   match.fill_into_edit.append(suggestion.suggestion());
 266
 267   const TemplateURLRef& search_url = template_url->url_ref();
 268   DCHECK(search_url.SupportsReplacement(search_terms_data));
 269   match.search_terms_args.reset(
 270       new TemplateURLRef::SearchTermsArgs(suggestion.suggestion()));
 271   match.search_terms_args->original_query = input.text();
 272   match.search_terms_args->accepted_suggestion = accepted_suggestion;
 273   match.search_terms_args->enable_omnibox_start_margin = true;
 274   match.search_terms_args->suggest_query_params =
 275       suggestion.suggest_query_params();
 276   match.search_terms_args->append_extra_query_params =
 277       append_extra_query_params;
 278   // This is the destination URL sans assisted query stats.  This must be set
 279   // so the AutocompleteController can properly de-dupe; the controller will
 280   // eventually overwrite it before it reaches the user.
 281   match.destination_url =
 282       GURL(search_url.ReplaceSearchTerms(*match.search_terms_args.get(),
 283                                          search_terms_data));
 284
 285   // Search results don't look like URLs.
 286   match.transition = suggestion.from_keyword_provider() ?
 287       content::PAGE_TRANSITION_KEYWORD : content::PAGE_TRANSITION_GENERATED;
 288
 289   return match;
 290 }
 291
 292 // static
 293 bool BaseSearchProvider::ZeroSuggestEnabled(
 294     const GURL& suggest_url,
 295     const TemplateURL* template_url,
 296     OmniboxEventProto::PageClassification page_classification,
 297     const SearchTermsData& search_terms_data,
 298     Profile* profile) {
 299   if (!OmniboxFieldTrial::InZeroSuggestFieldTrial())
 300     return false;
 301
 302   // Make sure we are sending the suggest request through HTTPS to prevent
 303   // exposing the current page URL or personalized results without encryption.
 304   if (!suggest_url.SchemeIs(url::kHttpsScheme))
 305     return false;
 306
 307   // Don't show zero suggest on the NTP.
 308   // TODO(hfung): Experiment with showing MostVisited zero suggest on NTP
 309   // under the conditions described in crbug.com/305366.
 310   if ((page_classification ==
 311        OmniboxEventProto::INSTANT_NTP_WITH_FAKEBOX_AS_STARTING_FOCUS) ||
 312       (page_classification ==
 313        OmniboxEventProto::INSTANT_NTP_WITH_OMNIBOX_AS_STARTING_FOCUS))
 314     return false;
 315
 316   // Don't run if there's no profile or in incognito mode.
 317   if (profile == NULL || profile->IsOffTheRecord())
 318     return false;
 319
 320   // Don't run if we can't get preferences or search suggest is not enabled.
 321   PrefService* prefs = profile->GetPrefs();
 322   if (!prefs->GetBoolean(prefs::kSearchSuggestEnabled))
 323     return false;
 324
 325   // Only make the request if we know that the provider supports zero suggest
 326   // (currently only the prepopulated Google provider).
 327   if (template_url == NULL ||
 328       !template_url->SupportsReplacement(search_terms_data) ||
 329       TemplateURLPrepopulateData::GetEngineType(
 330           *template_url, search_terms_data) != SEARCH_ENGINE_GOOGLE)
 331     return false;
 332
 333   return true;
 334 }
 335
 336 // static
 337 bool BaseSearchProvider::CanSendURL(
 338     const GURL& current_page_url,
 339     const GURL& suggest_url,
 340     const TemplateURL* template_url,
 341     OmniboxEventProto::PageClassification page_classification,
 342     const SearchTermsData& search_terms_data,
 343     Profile* profile) {
 344   if (!ZeroSuggestEnabled(suggest_url, template_url, page_classification,
 345                           search_terms_data, profile))
 346     return false;
 347
 348   if (!current_page_url.is_valid())
 349     return false;
 350
 351   // Only allow HTTP URLs or HTTPS URLs for the same domain as the search
 352   // provider.
 353   if ((current_page_url.scheme() != url::kHttpScheme) &&
 354       ((current_page_url.scheme() != url::kHttpsScheme) ||
 355        !net::registry_controlled_domains::SameDomainOrHost(
 356            current_page_url, suggest_url,
 357            net::registry_controlled_domains::EXCLUDE_PRIVATE_REGISTRIES)))
 358     return false;
 359
 360   // Check field trials and settings allow sending the URL on suggest requests.
 361   ProfileSyncService* service =
 362       ProfileSyncServiceFactory::GetInstance()->GetForProfile(profile);
 363   sync_driver::SyncPrefs sync_prefs(profile->GetPrefs());
 364   if (service == NULL ||
 365       !service->IsSyncEnabledAndLoggedIn() ||
 366       !sync_prefs.GetPreferredDataTypes(syncer::UserTypes()).Has(
 367           syncer::PROXY_TABS) ||
 368       service->GetEncryptedDataTypes().Has(syncer::SESSIONS))
 369     return false;
 370
 371   return true;
 372 }
 373
 374 void BaseSearchProvider::AddMatchToMap(
 375     const SearchSuggestionParser::SuggestResult& result,
 376     const std::string& metadata,
 377     int accepted_suggestion,
 378     bool mark_as_deletable,
 379     bool in_keyword_mode,
 380     MatchMap* map) {
 381   AutocompleteMatch match = CreateSearchSuggestion(
 382       this, GetInput(result.from_keyword_provider()), in_keyword_mode, result,
 383       GetTemplateURL(result.from_keyword_provider()),
 384       template_url_service_->search_terms_data(), accepted_suggestion,
 385       ShouldAppendExtraParams(result));
 386   if (!match.destination_url.is_valid())
 387     return;
 388   match.search_terms_args->bookmark_bar_pinned =
 389       profile_->GetPrefs()->GetBoolean(prefs::kShowBookmarkBar);
 390   match.RecordAdditionalInfo(kRelevanceFromServerKey,
 391                              result.relevance_from_server() ? kTrue : kFalse);
 392   match.RecordAdditionalInfo(kShouldPrefetchKey,
 393                              result.should_prefetch() ? kTrue : kFalse);
 394   SetDeletionURL(result.deletion_url(), &match);
 395   if (mark_as_deletable)
 396     match.deletable = true;
 397   // Metadata is needed only for prefetching queries.
 398   if (result.should_prefetch())
 399     match.RecordAdditionalInfo(kSuggestMetadataKey, metadata);
 400
 401   // Try to add |match| to |map|.  If a match for this suggestion is
 402   // already in |map|, replace it if |match| is more relevant.
 403   // NOTE: Keep this ToLower() call in sync with url_database.cc.
 404   MatchKey match_key(
 405       std::make_pair(base::i18n::ToLower(result.suggestion()),
 406                      match.search_terms_args->suggest_query_params));
 407   const std::pair<MatchMap::iterator, bool> i(
 408        map->insert(std::make_pair(match_key, match)));
 409
 410   bool should_prefetch = result.should_prefetch();
 411   if (!i.second) {
 412     // NOTE: We purposefully do a direct relevance comparison here instead of
 413     // using AutocompleteMatch::MoreRelevant(), so that we'll prefer "items
 414     // added first" rather than "items alphabetically first" when the scores
 415     // are equal. The only case this matters is when a user has results with
 416     // the same score that differ only by capitalization; because the history
 417     // system returns results sorted by recency, this means we'll pick the most
 418     // recent such result even if the precision of our relevance score is too
 419     // low to distinguish the two.
 420     if (match.relevance > i.first->second.relevance) {
 421       match.duplicate_matches.insert(match.duplicate_matches.end(),
 422                                      i.first->second.duplicate_matches.begin(),
 423                                      i.first->second.duplicate_matches.end());
 424       i.first->second.duplicate_matches.clear();
 425       match.duplicate_matches.push_back(i.first->second);
 426       i.first->second = match;
 427     } else {
 428       i.first->second.duplicate_matches.push_back(match);
 429       if (match.keyword == i.first->second.keyword) {
 430         // Old and new matches are from the same search provider. It is okay to
 431         // record one match's prefetch data onto a different match (for the same
 432         // query string) for the following reasons:
 433         // 1. Because the suggest server only sends down a query string from
 434         // which we construct a URL, rather than sending a full URL, and because
 435         // we construct URLs from query strings in the same way every time, the
 436         // URLs for the two matches will be the same. Therefore, we won't end up
 437         // prefetching something the server didn't intend.
 438         // 2. Presumably the server sets the prefetch bit on a match it things
 439         // is sufficiently relevant that the user is likely to choose it.
 440         // Surely setting the prefetch bit on a match of even higher relevance
 441         // won't violate this assumption.
 442         should_prefetch |= ShouldPrefetch(i.first->second);
 443         i.first->second.RecordAdditionalInfo(kShouldPrefetchKey,
 444                                              should_prefetch ? kTrue : kFalse);
 445         if (should_prefetch)
 446           i.first->second.RecordAdditionalInfo(kSuggestMetadataKey, metadata);
 447       }
 448     }
 449   }
 450 }
 451
 452 bool BaseSearchProvider::ParseSuggestResults(
 453     const base::Value& root_val,
 454     int default_result_relevance,
 455     bool is_keyword_result,
 456     SearchSuggestionParser::Results* results) {
 457   if (!SearchSuggestionParser::ParseSuggestResults(
 458       root_val, GetInput(is_keyword_result),
 459       ChromeAutocompleteSchemeClassifier(profile_), default_result_relevance,
 460       profile_->GetPrefs()->GetString(prefs::kAcceptLanguages),
 461       is_keyword_result, results))
 462     return false;
 463
 464   BitmapFetcherService* image_service =
 465       BitmapFetcherServiceFactory::GetForBrowserContext(profile_);
 466   DCHECK(image_service);
 467   for (std::vector<GURL>::const_iterator it =
 468            results->answers_image_urls.begin();
 469        it != results->answers_image_urls.end(); ++it)
 470     image_service->Prefetch(*it);
 471
 472   field_trial_triggered_ |= results->field_trial_triggered;
 473   field_trial_triggered_in_session_ |= results->field_trial_triggered;
 474   return true;
 475 }
 476
 477 void BaseSearchProvider::ModifyProviderInfo(
 478     metrics::OmniboxEventProto_ProviderInfo* provider_info) const {
 479 }
 480
 481 void BaseSearchProvider::DeleteMatchFromMatches(
 482     const AutocompleteMatch& match) {
 483   for (ACMatches::iterator i(matches_.begin()); i != matches_.end(); ++i) {
 484     // Find the desired match to delete by checking the type and contents.
 485     // We can't check the destination URL, because the autocomplete controller
 486     // may have reformulated that. Not that while checking for matching
 487     // contents works for personalized suggestions, if more match types gain
 488     // deletion support, this algorithm may need to be re-examined.
 489     if (i->contents == match.contents && i->type == match.type) {
 490       matches_.erase(i);
 491       break;
 492     }
 493   }
 494 }
 495
 496 void BaseSearchProvider::OnDeletionComplete(
 497     bool success, SuggestionDeletionHandler* handler) {
 498   RecordDeletionResult(success);
 499   SuggestionDeletionHandlers::iterator it = std::find(
 500       deletion_handlers_.begin(), deletion_handlers_.end(), handler);
 501   DCHECK(it != deletion_handlers_.end());
 502   deletion_handlers_.erase(it);
 503 }