src/chrome/browser/autocomplete/base_search_provider.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/autocomplete/base_search_provider.h"
   6
   7 #include "base/i18n/case_conversion.h"
   8 #include "base/json/json_string_value_serializer.h"
   9 #include "base/prefs/pref_service.h"
  10 #include "base/strings/string_util.h"
  11 #include "base/strings/utf_string_conversions.h"
  12 #include "chrome/browser/autocomplete/autocomplete_provider_listener.h"
  13 #include "chrome/browser/autocomplete/url_prefix.h"
  14 #include "chrome/browser/omnibox/omnibox_field_trial.h"
  15 #include "chrome/browser/profiles/profile.h"
  16 #include "chrome/browser/search/instant_service.h"
  17 #include "chrome/browser/search/instant_service_factory.h"
  18 #include "chrome/browser/search/search.h"
  19 #include "chrome/browser/search_engines/template_url.h"
  20 #include "chrome/browser/search_engines/template_url_prepopulate_data.h"
  21 #include "chrome/browser/sync/profile_sync_service.h"
  22 #include "chrome/browser/sync/profile_sync_service_factory.h"
  23 #include "chrome/common/pref_names.h"
  24 #include "content/public/common/url_constants.h"
  25 #include "net/base/escape.h"
  26 #include "net/base/net_util.h"
  27 #include "net/base/registry_controlled_domains/registry_controlled_domain.h"
  28 #include "net/url_request/url_fetcher_delegate.h"
  29 #include "url/gurl.h"
  30
  31 // BaseSearchProvider ---------------------------------------------------------
  32
  33 BaseSearchProvider::BaseSearchProvider(AutocompleteProviderListener* listener,
  34                                        Profile* profile,
  35                                        AutocompleteProvider::Type type)
  36     : AutocompleteProvider(listener, profile, type),
  37       field_trial_triggered_(false),
  38       field_trial_triggered_in_session_(false) {}
  39
  40 // static
  41 bool BaseSearchProvider::ShouldPrefetch(const AutocompleteMatch& match) {
  42   return match.GetAdditionalInfo(kShouldPrefetchKey) == kTrue;
  43 }
  44
  45 void BaseSearchProvider::Stop(bool clear_cached_results) {
  46   StopSuggest();
  47   done_ = true;
  48
  49   if (clear_cached_results)
  50     ClearAllResults();
  51 }
  52
  53 void BaseSearchProvider::AddProviderInfo(ProvidersInfo* provider_info) const {
  54   provider_info->push_back(metrics::OmniboxEventProto_ProviderInfo());
  55   metrics::OmniboxEventProto_ProviderInfo& new_entry = provider_info->back();
  56   new_entry.set_provider(AsOmniboxEventProviderType());
  57   new_entry.set_provider_done(done_);
  58   std::vector<uint32> field_trial_hashes;
  59   OmniboxFieldTrial::GetActiveSuggestFieldTrialHashes(&field_trial_hashes);
  60   for (size_t i = 0; i < field_trial_hashes.size(); ++i) {
  61     if (field_trial_triggered_)
  62       new_entry.mutable_field_trial_triggered()->Add(field_trial_hashes[i]);
  63     if (field_trial_triggered_in_session_) {
  64       new_entry.mutable_field_trial_triggered_in_session()->Add(
  65           field_trial_hashes[i]);
  66      }
  67   }
  68 }
  69
  70 // static
  71 const char BaseSearchProvider::kRelevanceFromServerKey[] =
  72     "relevance_from_server";
  73 const char BaseSearchProvider::kShouldPrefetchKey[] = "should_prefetch";
  74 const char BaseSearchProvider::kSuggestMetadataKey[] = "suggest_metadata";
  75 const char BaseSearchProvider::kDeletionUrlKey[] = "deletion_url";
  76 const char BaseSearchProvider::kTrue[] = "true";
  77 const char BaseSearchProvider::kFalse[] = "false";
  78
  79 BaseSearchProvider::~BaseSearchProvider() {}
  80
  81 // BaseSearchProvider::Result --------------------------------------------------
  82
  83 BaseSearchProvider::Result::Result(bool from_keyword_provider,
  84                                    int relevance,
  85                                    bool relevance_from_server)
  86     : from_keyword_provider_(from_keyword_provider),
  87       relevance_(relevance),
  88       relevance_from_server_(relevance_from_server) {}
  89
  90 BaseSearchProvider::Result::~Result() {}
  91
  92 // BaseSearchProvider::SuggestResult -------------------------------------------
  93
  94 BaseSearchProvider::SuggestResult::SuggestResult(
  95     const base::string16& suggestion,
  96     AutocompleteMatchType::Type type,
  97     const base::string16& match_contents,
  98     const base::string16& annotation,
  99     const std::string& suggest_query_params,
 100     const std::string& deletion_url,
 101     bool from_keyword_provider,
 102     int relevance,
 103     bool relevance_from_server,
 104     bool should_prefetch,
 105     const base::string16& input_text)
 106     : Result(from_keyword_provider, relevance, relevance_from_server),
 107       suggestion_(suggestion),
 108       type_(type),
 109       annotation_(annotation),
 110       suggest_query_params_(suggest_query_params),
 111       deletion_url_(deletion_url),
 112       should_prefetch_(should_prefetch) {
 113   match_contents_ = match_contents;
 114   DCHECK(!match_contents_.empty());
 115   ClassifyMatchContents(true, input_text);
 116 }
 117
 118 BaseSearchProvider::SuggestResult::~SuggestResult() {}
 119
 120 void BaseSearchProvider::SuggestResult::ClassifyMatchContents(
 121     const bool allow_bolding_all,
 122     const base::string16& input_text) {
 123   size_t input_position = match_contents_.find(input_text);
 124   if (!allow_bolding_all && (input_position == base::string16::npos)) {
 125     // Bail if the code below to update the bolding would bold the whole
 126     // string.  Note that the string may already be entirely bolded; if
 127     // so, leave it as is.
 128     return;
 129   }
 130   match_contents_class_.clear();
 131   // We do intra-string highlighting for suggestions - the suggested segment
 132   // will be highlighted, e.g. for input_text = "you" the suggestion may be
 133   // "youtube", so we'll bold the "tube" section: you*tube*.
 134   if (input_text != match_contents_) {
 135     if (input_position == base::string16::npos) {
 136       // The input text is not a substring of the query string, e.g. input
 137       // text is "slasdot" and the query string is "slashdot", so we bold the
 138       // whole thing.
 139       match_contents_class_.push_back(
 140           ACMatchClassification(0, ACMatchClassification::MATCH));
 141     } else {
 142       // We don't iterate over the string here annotating all matches because
 143       // it looks odd to have every occurrence of a substring that may be as
 144       // short as a single character highlighted in a query suggestion result,
 145       // e.g. for input text "s" and query string "southwest airlines", it
 146       // looks odd if both the first and last s are highlighted.
 147       if (input_position != 0) {
 148         match_contents_class_.push_back(
 149             ACMatchClassification(0, ACMatchClassification::MATCH));
 150       }
 151       match_contents_class_.push_back(
 152           ACMatchClassification(input_position, ACMatchClassification::NONE));
 153       size_t next_fragment_position = input_position + input_text.length();
 154       if (next_fragment_position < match_contents_.length()) {
 155         match_contents_class_.push_back(ACMatchClassification(
 156             next_fragment_position, ACMatchClassification::MATCH));
 157       }
 158     }
 159   } else {
 160     // Otherwise, match_contents_ is a verbatim (what-you-typed) match, either
 161     // for the default provider or a keyword search provider.
 162     match_contents_class_.push_back(
 163         ACMatchClassification(0, ACMatchClassification::NONE));
 164   }
 165 }
 166
 167 bool BaseSearchProvider::SuggestResult::IsInlineable(
 168     const base::string16& input) const {
 169   return StartsWith(suggestion_, input, false);
 170 }
 171
 172 int BaseSearchProvider::SuggestResult::CalculateRelevance(
 173     const AutocompleteInput& input,
 174     bool keyword_provider_requested) const {
 175   if (!from_keyword_provider_ && keyword_provider_requested)
 176     return 100;
 177   return ((input.type() == AutocompleteInput::URL) ? 300 : 600);
 178 }
 179
 180 // BaseSearchProvider::NavigationResult ----------------------------------------
 181
 182 BaseSearchProvider::NavigationResult::NavigationResult(
 183     const AutocompleteProvider& provider,
 184     const GURL& url,
 185     const base::string16& description,
 186     bool from_keyword_provider,
 187     int relevance,
 188     bool relevance_from_server,
 189     const base::string16& input_text,
 190     const std::string& languages)
 191     : Result(from_keyword_provider, relevance, relevance_from_server),
 192       url_(url),
 193       formatted_url_(AutocompleteInput::FormattedStringWithEquivalentMeaning(
 194           url,
 195           provider.StringForURLDisplay(url, true, false))),
 196       description_(description) {
 197   DCHECK(url_.is_valid());
 198   CalculateAndClassifyMatchContents(true, input_text, languages);
 199 }
 200
 201 BaseSearchProvider::NavigationResult::~NavigationResult() {}
 202
 203 void BaseSearchProvider::NavigationResult::CalculateAndClassifyMatchContents(
 204     const bool allow_bolding_nothing,
 205     const base::string16& input_text,
 206     const std::string& languages) {
 207   // First look for the user's input inside the formatted url as it would be
 208   // without trimming the scheme, so we can find matches at the beginning of the
 209   // scheme.
 210   const URLPrefix* prefix =
 211       URLPrefix::BestURLPrefix(formatted_url_, input_text);
 212   size_t match_start = (prefix == NULL) ?
 213       formatted_url_.find(input_text) : prefix->prefix.length();
 214   bool trim_http = !AutocompleteInput::HasHTTPScheme(input_text) &&
 215                    (!prefix || (match_start != 0));
 216   const net::FormatUrlTypes format_types =
 217       net::kFormatUrlOmitAll & ~(trim_http ? 0 : net::kFormatUrlOmitHTTP);
 218
 219   base::string16 match_contents = net::FormatUrl(url_, languages, format_types,
 220       net::UnescapeRule::SPACES, NULL, NULL, &match_start);
 221   // If the first match in the untrimmed string was inside a scheme that we
 222   // trimmed, look for a subsequent match.
 223   if (match_start == base::string16::npos)
 224     match_start = match_contents.find(input_text);
 225   // Update |match_contents_| and |match_contents_class_| if it's allowed.
 226   if (allow_bolding_nothing || (match_start != base::string16::npos)) {
 227     match_contents_ = match_contents;
 228     // Safe if |match_start| is npos; also safe if the input is longer than the
 229     // remaining contents after |match_start|.
 230     AutocompleteMatch::ClassifyLocationInString(match_start,
 231         input_text.length(), match_contents_.length(),
 232         ACMatchClassification::URL, &match_contents_class_);
 233   }
 234 }
 235
 236 bool BaseSearchProvider::NavigationResult::IsInlineable(
 237     const base::string16& input) const {
 238   return
 239       URLPrefix::BestURLPrefix(base::UTF8ToUTF16(url_.spec()), input) != NULL;
 240 }
 241
 242 int BaseSearchProvider::NavigationResult::CalculateRelevance(
 243     const AutocompleteInput& input,
 244     bool keyword_provider_requested) const {
 245   return (from_keyword_provider_ || !keyword_provider_requested) ? 800 : 150;
 246 }
 247
 248 // BaseSearchProvider::Results -------------------------------------------------
 249
 250 BaseSearchProvider::Results::Results() : verbatim_relevance(-1) {}
 251
 252 BaseSearchProvider::Results::~Results() {}
 253
 254 void BaseSearchProvider::Results::Clear() {
 255   suggest_results.clear();
 256   navigation_results.clear();
 257   verbatim_relevance = -1;
 258   metadata.clear();
 259 }
 260
 261 bool BaseSearchProvider::Results::HasServerProvidedScores() const {
 262   if (verbatim_relevance >= 0)
 263     return true;
 264
 265   // Right now either all results of one type will be server-scored or they will
 266   // all be locally scored, but in case we change this later, we'll just check
 267   // them all.
 268   for (SuggestResults::const_iterator i(suggest_results.begin());
 269        i != suggest_results.end(); ++i) {
 270     if (i->relevance_from_server())
 271       return true;
 272   }
 273   for (NavigationResults::const_iterator i(navigation_results.begin());
 274        i != navigation_results.end(); ++i) {
 275     if (i->relevance_from_server())
 276       return true;
 277   }
 278
 279   return false;
 280 }
 281
 282 // BaseSearchProvider ---------------------------------------------------------
 283
 284 // static
 285 AutocompleteMatch BaseSearchProvider::CreateSearchSuggestion(
 286     AutocompleteProvider* autocomplete_provider,
 287     const AutocompleteInput& input,
 288     const SuggestResult& suggestion,
 289     const TemplateURL* template_url,
 290     int accepted_suggestion,
 291     int omnibox_start_margin,
 292     bool append_extra_query_params) {
 293   AutocompleteMatch match(autocomplete_provider, suggestion.relevance(), false,
 294                           suggestion.type());
 295
 296   if (!template_url)
 297     return match;
 298   match.keyword = template_url->keyword();
 299   match.contents = suggestion.match_contents();
 300   match.contents_class = suggestion.match_contents_class();
 301
 302   if (!suggestion.annotation().empty())
 303     match.description = suggestion.annotation();
 304
 305   match.allowed_to_be_default_match =
 306       (input.text() == suggestion.match_contents());
 307
 308   // When the user forced a query, we need to make sure all the fill_into_edit
 309   // values preserve that property.  Otherwise, if the user starts editing a
 310   // suggestion, non-Search results will suddenly appear.
 311   if (input.type() == AutocompleteInput::FORCED_QUERY)
 312     match.fill_into_edit.assign(base::ASCIIToUTF16("?"));
 313   if (suggestion.from_keyword_provider())
 314     match.fill_into_edit.append(match.keyword + base::char16(' '));
 315   if (!input.prevent_inline_autocomplete() &&
 316       StartsWith(suggestion.suggestion(), input.text(), false)) {
 317     match.inline_autocompletion =
 318         suggestion.suggestion().substr(input.text().length());
 319     match.allowed_to_be_default_match = true;
 320   }
 321   match.fill_into_edit.append(suggestion.suggestion());
 322
 323   const TemplateURLRef& search_url = template_url->url_ref();
 324   DCHECK(search_url.SupportsReplacement());
 325   match.search_terms_args.reset(
 326       new TemplateURLRef::SearchTermsArgs(suggestion.suggestion()));
 327   match.search_terms_args->original_query = input.text();
 328   match.search_terms_args->accepted_suggestion = accepted_suggestion;
 329   match.search_terms_args->omnibox_start_margin = omnibox_start_margin;
 330   match.search_terms_args->suggest_query_params =
 331       suggestion.suggest_query_params();
 332   match.search_terms_args->append_extra_query_params =
 333       append_extra_query_params;
 334   // This is the destination URL sans assisted query stats.  This must be set
 335   // so the AutocompleteController can properly de-dupe; the controller will
 336   // eventually overwrite it before it reaches the user.
 337   match.destination_url =
 338       GURL(search_url.ReplaceSearchTerms(*match.search_terms_args.get()));
 339
 340   // Search results don't look like URLs.
 341   match.transition = suggestion.from_keyword_provider() ?
 342       content::PAGE_TRANSITION_KEYWORD : content::PAGE_TRANSITION_GENERATED;
 343
 344   return match;
 345 }
 346
 347 // static
 348 scoped_ptr<base::Value> BaseSearchProvider::DeserializeJsonData(
 349     std::string json_data) {
 350   // The JSON response should be an array.
 351   for (size_t response_start_index = json_data.find("["), i = 0;
 352        response_start_index != std::string::npos && i < 5;
 353        response_start_index = json_data.find("[", 1), i++) {
 354     // Remove any XSSI guards to allow for JSON parsing.
 355     if (response_start_index > 0)
 356       json_data.erase(0, response_start_index);
 357
 358     JSONStringValueSerializer deserializer(json_data);
 359     deserializer.set_allow_trailing_comma(true);
 360     int error_code = 0;
 361     scoped_ptr<base::Value> data(deserializer.Deserialize(&error_code, NULL));
 362     if (error_code == 0)
 363       return data.Pass();
 364   }
 365   return scoped_ptr<base::Value>();
 366 }
 367
 368 // static
 369 bool BaseSearchProvider::CanSendURL(
 370     const GURL& current_page_url,
 371     const GURL& suggest_url,
 372     const TemplateURL* template_url,
 373     AutocompleteInput::PageClassification page_classification,
 374     Profile* profile) {
 375   if (!current_page_url.is_valid())
 376     return false;
 377
 378   // TODO(hfung): Show Most Visited on NTP with appropriate verbatim
 379   // description when the user actively focuses on the omnibox as discussed in
 380   // crbug/305366 if Most Visited (or something similar) will launch.
 381   if ((page_classification ==
 382        AutocompleteInput::INSTANT_NTP_WITH_FAKEBOX_AS_STARTING_FOCUS) ||
 383       (page_classification ==
 384        AutocompleteInput::INSTANT_NTP_WITH_OMNIBOX_AS_STARTING_FOCUS))
 385     return false;
 386
 387   // Only allow HTTP URLs or HTTPS URLs for the same domain as the search
 388   // provider.
 389   if ((current_page_url.scheme() != content::kHttpScheme) &&
 390       ((current_page_url.scheme() != content::kHttpsScheme) ||
 391        !net::registry_controlled_domains::SameDomainOrHost(
 392            current_page_url, suggest_url,
 393            net::registry_controlled_domains::EXCLUDE_PRIVATE_REGISTRIES)))
 394     return false;
 395
 396   // Make sure we are sending the suggest request through HTTPS to prevent
 397   // exposing the current page URL to networks before the search provider.
 398   if (!suggest_url.SchemeIs(content::kHttpsScheme))
 399     return false;
 400
 401   // Don't run if there's no profile or in incognito mode.
 402   if (profile == NULL || profile->IsOffTheRecord())
 403     return false;
 404
 405   // Don't run if we can't get preferences or search suggest is not enabled.
 406   PrefService* prefs = profile->GetPrefs();
 407   if (!prefs->GetBoolean(prefs::kSearchSuggestEnabled))
 408     return false;
 409
 410   // Only make the request if we know that the provider supports zero suggest
 411   // (currently only the prepopulated Google provider).
 412   if (template_url == NULL || !template_url->SupportsReplacement() ||
 413       TemplateURLPrepopulateData::GetEngineType(*template_url) !=
 414       SEARCH_ENGINE_GOOGLE)
 415     return false;
 416
 417   // Check field trials and settings allow sending the URL on suggest requests.
 418   ProfileSyncService* service =
 419       ProfileSyncServiceFactory::GetInstance()->GetForProfile(profile);
 420   browser_sync::SyncPrefs sync_prefs(prefs);
 421   if (!OmniboxFieldTrial::InZeroSuggestFieldTrial() ||
 422       service == NULL ||
 423       !service->IsSyncEnabledAndLoggedIn() ||
 424       !sync_prefs.GetPreferredDataTypes(syncer::UserTypes()).Has(
 425           syncer::PROXY_TABS) ||
 426       service->GetEncryptedDataTypes().Has(syncer::SESSIONS))
 427     return false;
 428
 429   return true;
 430 }
 431
 432 void BaseSearchProvider::AddMatchToMap(const SuggestResult& result,
 433                                        const std::string& metadata,
 434                                        int accepted_suggestion,
 435                                        MatchMap* map) {
 436   InstantService* instant_service =
 437       InstantServiceFactory::GetForProfile(profile_);
 438   // Android and iOS have no InstantService.
 439   const int omnibox_start_margin = instant_service ?
 440       instant_service->omnibox_start_margin() : chrome::kDisableStartMargin;
 441
 442   AutocompleteMatch match = CreateSearchSuggestion(
 443       this, GetInput(result), result, GetTemplateURL(result),
 444       accepted_suggestion, omnibox_start_margin,
 445       ShouldAppendExtraParams(result));
 446   if (!match.destination_url.is_valid())
 447     return;
 448   match.search_terms_args->bookmark_bar_pinned =
 449       profile_->GetPrefs()->GetBoolean(prefs::kShowBookmarkBar);
 450   match.RecordAdditionalInfo(kRelevanceFromServerKey,
 451                              result.relevance_from_server() ? kTrue : kFalse);
 452   match.RecordAdditionalInfo(kShouldPrefetchKey,
 453                              result.should_prefetch() ? kTrue : kFalse);
 454
 455   if (!result.deletion_url().empty()) {
 456     GURL url(match.destination_url.GetOrigin().Resolve(result.deletion_url()));
 457     if (url.is_valid()) {
 458       match.RecordAdditionalInfo(kDeletionUrlKey, url.spec());
 459       match.deletable = true;
 460     }
 461   }
 462
 463   // Metadata is needed only for prefetching queries.
 464   if (result.should_prefetch())
 465     match.RecordAdditionalInfo(kSuggestMetadataKey, metadata);
 466
 467   // Try to add |match| to |map|.  If a match for this suggestion is
 468   // already in |map|, replace it if |match| is more relevant.
 469   // NOTE: Keep this ToLower() call in sync with url_database.cc.
 470   MatchKey match_key(
 471       std::make_pair(base::i18n::ToLower(result.suggestion()),
 472                      match.search_terms_args->suggest_query_params));
 473   const std::pair<MatchMap::iterator, bool> i(
 474        map->insert(std::make_pair(match_key, match)));
 475
 476   bool should_prefetch = result.should_prefetch();
 477   if (!i.second) {
 478     // NOTE: We purposefully do a direct relevance comparison here instead of
 479     // using AutocompleteMatch::MoreRelevant(), so that we'll prefer "items
 480     // added first" rather than "items alphabetically first" when the scores
 481     // are equal. The only case this matters is when a user has results with
 482     // the same score that differ only by capitalization; because the history
 483     // system returns results sorted by recency, this means we'll pick the most
 484     // recent such result even if the precision of our relevance score is too
 485     // low to distinguish the two.
 486     if (match.relevance > i.first->second.relevance) {
 487       i.first->second = match;
 488     } else if (match.keyword == i.first->second.keyword) {
 489       // Old and new matches are from the same search provider. It is okay to
 490       // record one match's prefetch data onto a different match (for the same
 491       // query string) for the following reasons:
 492       // 1. Because the suggest server only sends down a query string from
 493       // which we construct a URL, rather than sending a full URL, and because
 494       // we construct URLs from query strings in the same way every time, the
 495       // URLs for the two matches will be the same. Therefore, we won't end up
 496       // prefetching something the server didn't intend.
 497       // 2. Presumably the server sets the prefetch bit on a match it things is
 498       // sufficiently relevant that the user is likely to choose it. Surely
 499       // setting the prefetch bit on a match of even higher relevance won't
 500       // violate this assumption.
 501       should_prefetch |= ShouldPrefetch(i.first->second);
 502       i.first->second.RecordAdditionalInfo(kShouldPrefetchKey,
 503                                            should_prefetch ? kTrue : kFalse);
 504       if (should_prefetch)
 505         i.first->second.RecordAdditionalInfo(kSuggestMetadataKey, metadata);
 506     }
 507   }
 508 }