1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 // This file contains the Search autocomplete provider. This provider is
6 // responsible for all autocomplete entries that start with "Search <engine>
7 // for ...", including searching for the current input string, search
8 // history, and search suggestions. An instance of it gets created and
9 // managed by the autocomplete controller.
11 #ifndef CHROME_BROWSER_AUTOCOMPLETE_SEARCH_PROVIDER_H_
12 #define CHROME_BROWSER_AUTOCOMPLETE_SEARCH_PROVIDER_H_
18 #include "base/basictypes.h"
19 #include "base/compiler_specific.h"
20 #include "base/memory/scoped_ptr.h"
21 #include "base/time/time.h"
22 #include "base/timer/timer.h"
23 #include "chrome/browser/autocomplete/autocomplete_input.h"
24 #include "chrome/browser/autocomplete/autocomplete_match.h"
25 #include "chrome/browser/autocomplete/autocomplete_provider.h"
26 #include "chrome/browser/history/history_types.h"
27 #include "chrome/browser/search_engines/template_url.h"
28 #include "net/url_request/url_fetcher_delegate.h"
31 class SearchProviderTest;
32 class TemplateURLService;
42 // Autocomplete provider for searches and suggestions from a search engine.
44 // After construction, the autocomplete controller repeatedly calls Start()
45 // with some user input, each time expecting to receive a small set of the best
46 // matches (either synchronously or asynchronously).
48 // Initially the provider creates a match that searches for the current input
49 // text. It also starts a task to query the Suggest servers. When that data
50 // comes back, the provider creates and returns matches for the best
52 class SearchProvider : public AutocompleteProvider,
53 public net::URLFetcherDelegate {
55 // ID used in creating URLFetcher for default provider's suggest results.
56 static const int kDefaultProviderURLFetcherID;
58 // ID used in creating URLFetcher for keyword provider's suggest results.
59 static const int kKeywordProviderURLFetcherID;
61 SearchProvider(AutocompleteProviderListener* listener, Profile* profile);
63 // Returns an AutocompleteMatch with the given |autocomplete_provider|,
64 // |relevance|, and |type|, which represents a search via |template_url| for
65 // |query_string|. If |template_url| is NULL, returns a match with an invalid
68 // |input_text| is the original user input, which may differ from
69 // |query_string|; e.g. the user typed "foo" and got a search suggestion of
70 // "food", which we're now marking up. This is used to highlight portions of
71 // the match contents to distinguish locally-typed text from suggested text.
73 // |input| and |is_keyword| are necessary for various other details, like
74 // whether we should allow inline autocompletion and what the transition type
75 // should be. |accepted_suggestion| and |omnibox_start_margin| are used along
76 // with |input_text| to generate Assisted Query Stats.
77 // |append_extra_query_params| should be set if |template_url| is the default
78 // search engine, so the destination URL will contain any
79 // command-line-specified query params.
80 static AutocompleteMatch CreateSearchSuggestion(
81 AutocompleteProvider* autocomplete_provider,
82 const AutocompleteInput& input,
83 const string16& input_text,
85 AutocompleteMatch::Type type,
87 const string16& match_contents,
88 const string16& annotation,
89 const TemplateURL* template_url,
90 const string16& query_string,
91 const std::string& suggest_query_params,
92 int accepted_suggestion,
93 int omnibox_start_margin,
94 bool append_extra_query_params);
96 // Returns whether the SearchProvider previously flagged |match| as a query
97 // that should be prefetched.
98 static bool ShouldPrefetch(const AutocompleteMatch& match);
100 // Extracts the suggest response metadata which SearchProvider previously
101 // stored for |match|.
102 static std::string GetSuggestMetadata(const AutocompleteMatch& match);
104 // AutocompleteProvider:
105 virtual void AddProviderInfo(ProvidersInfo* provider_info) const OVERRIDE;
106 virtual void ResetSession() OVERRIDE;
108 bool field_trial_triggered_in_session() const {
109 return field_trial_triggered_in_session_;
113 // TODO(hfung): Remove ZeroSuggestProvider as a friend class after
114 // refactoring common code to a new base class.
115 friend class SearchProviderTest;
116 friend class ZeroSuggestProvider;
117 FRIEND_TEST_ALL_PREFIXES(SearchProviderTest, NavigationInline);
118 FRIEND_TEST_ALL_PREFIXES(SearchProviderTest, NavigationInlineDomainClassify);
119 FRIEND_TEST_ALL_PREFIXES(SearchProviderTest, NavigationInlineSchemeSubstring);
120 FRIEND_TEST_ALL_PREFIXES(SearchProviderTest, RemoveStaleResultsTest);
121 FRIEND_TEST_ALL_PREFIXES(SearchProviderTest, SuggestRelevanceExperiment);
122 FRIEND_TEST_ALL_PREFIXES(AutocompleteProviderTest, GetDestinationURL);
123 FRIEND_TEST_ALL_PREFIXES(InstantExtendedPrefetchTest, ClearPrefetchedResults);
124 FRIEND_TEST_ALL_PREFIXES(InstantExtendedPrefetchTest, SetPrefetchQuery);
126 // Manages the providers (TemplateURLs) used by SearchProvider. Two providers
128 // . The default provider. This corresponds to the user's default search
129 // engine. This is always used, except for the rare case of no default
131 // . The keyword provider. This is used if the user has typed in a keyword.
134 explicit Providers(TemplateURLService* template_url_service);
136 // Returns true if the specified providers match the two providers cached
138 bool equal(const string16& default_provider,
139 const string16& keyword_provider) const {
140 return (default_provider == default_provider_) &&
141 (keyword_provider == keyword_provider_);
144 // Resets the cached providers.
145 void set(const string16& default_provider,
146 const string16& keyword_provider) {
147 default_provider_ = default_provider;
148 keyword_provider_ = keyword_provider;
151 TemplateURLService* template_url_service() { return template_url_service_; }
152 const string16& default_provider() const { return default_provider_; }
153 const string16& keyword_provider() const { return keyword_provider_; }
155 // NOTE: These may return NULL even if the provider members are nonempty!
156 const TemplateURL* GetDefaultProviderURL() const;
157 const TemplateURL* GetKeywordProviderURL() const;
159 // Returns true if there is a valid keyword provider.
160 bool has_keyword_provider() const { return !keyword_provider_.empty(); }
163 TemplateURLService* template_url_service_;
165 // Cached across the life of a query so we behave consistently even if the
166 // user changes their default while the query is running.
167 string16 default_provider_;
168 string16 keyword_provider_;
170 DISALLOW_COPY_AND_ASSIGN(Providers);
173 // The Result classes are intermediate representations of AutocompleteMatches,
174 // simply containing relevance-ranked search and navigation suggestions.
175 // They may be cached to provide some synchronous matches while requests for
176 // new suggestions from updated input are in flight.
177 // TODO(msw) Extend these classes to generate their corresponding matches and
178 // other requisite data, in order to consolidate and simplify the
179 // highly fragmented SearchProvider logic for each Result type.
182 Result(bool from_keyword_provider,
184 bool relevance_from_server);
187 bool from_keyword_provider() const { return from_keyword_provider_; }
189 int relevance() const { return relevance_; }
190 void set_relevance(int relevance) { relevance_ = relevance; }
192 bool relevance_from_server() const { return relevance_from_server_; }
193 void set_relevance_from_server(bool relevance_from_server) {
194 relevance_from_server_ = relevance_from_server;
197 // Returns if this result is inlineable against the current input |input|.
198 // Non-inlineable results are stale.
199 virtual bool IsInlineable(const string16& input) const = 0;
201 // Returns the default relevance value for this result (which may
202 // be left over from a previous omnibox input) given the current
203 // input and whether the current input caused a keyword provider
205 virtual int CalculateRelevance(const AutocompleteInput& input,
206 bool keyword_provider_requested) const = 0;
209 // True if the result came from the keyword provider.
210 bool from_keyword_provider_;
212 // The relevance score.
216 // Whether this result's relevance score was fully or partly calculated
217 // based on server information, and thus is assumed to be more accurate.
218 // This is ultimately used in
219 // SearchProvider::ConvertResultsToAutocompleteMatches(), see comments
221 bool relevance_from_server_;
224 class SuggestResult : public Result {
226 SuggestResult(const string16& suggestion,
227 const string16& match_contents,
228 const string16& annotation,
229 const std::string& suggest_query_params,
230 bool from_keyword_provider,
232 bool relevance_from_server,
233 bool should_prefetch);
234 virtual ~SuggestResult();
236 const string16& suggestion() const { return suggestion_; }
237 const string16& match_contents() const { return match_contents_; }
238 const string16& annotation() const { return annotation_; }
239 const std::string& suggest_query_params() const {
240 return suggest_query_params_;
242 bool should_prefetch() const { return should_prefetch_; }
245 virtual bool IsInlineable(const string16& input) const OVERRIDE;
246 virtual int CalculateRelevance(
247 const AutocompleteInput& input,
248 bool keyword_provider_requested) const OVERRIDE;
251 // The search terms to be used for this suggestion.
252 string16 suggestion_;
254 // The contents to be displayed in the autocomplete match.
255 string16 match_contents_;
257 // Optional annotation for the |match_contents_| for disambiguation.
258 // This may be displayed in the autocomplete match contents, but is defined
259 // separately to facilitate different formatting.
260 string16 annotation_;
262 // Optional additional parameters to be added to the search URL.
263 std::string suggest_query_params_;
265 // Should this result be prefetched?
266 bool should_prefetch_;
269 class NavigationResult : public Result {
271 // |provider| is necessary to use StringForURLDisplay() in order to
272 // compute |formatted_url_|.
273 NavigationResult(const AutocompleteProvider& provider,
275 const string16& description,
276 bool from_keyword_provider,
278 bool relevance_from_server);
279 virtual ~NavigationResult();
281 const GURL& url() const { return url_; }
282 const string16& description() const { return description_; }
283 const string16& formatted_url() const { return formatted_url_; }
286 virtual bool IsInlineable(const string16& input) const OVERRIDE;
287 virtual int CalculateRelevance(
288 const AutocompleteInput& input,
289 bool keyword_provider_requested) const OVERRIDE;
292 // The suggested url for navigation.
295 // The properly formatted ("fixed up") URL string with equivalent meaning
296 // to the one in |url_|.
297 string16 formatted_url_;
299 // The suggested navigational result description; generally the site name.
300 string16 description_;
303 class CompareScoredResults;
305 typedef std::vector<SuggestResult> SuggestResults;
306 typedef std::vector<NavigationResult> NavigationResults;
307 typedef std::vector<history::KeywordSearchTermVisit> HistoryResults;
308 typedef std::pair<string16, std::string> MatchKey;
309 typedef std::map<MatchKey, AutocompleteMatch> MatchMap;
311 // A simple structure bundling most of the information (including
312 // both SuggestResults and NavigationResults) returned by a call to
313 // the suggest server.
315 // This has to be declared after the typedefs since it relies on some of them.
320 // Clears |suggest_results| and |navigation_results| and resets
321 // |verbatim_relevance| to -1 (implies unset).
324 // Returns whether any of the results (including verbatim) have
325 // server-provided scores.
326 bool HasServerProvidedScores() const;
328 // Query suggestions sorted by relevance score.
329 SuggestResults suggest_results;
331 // Navigational suggestions sorted by relevance score.
332 NavigationResults navigation_results;
334 // The server supplied verbatim relevance scores. Negative values
335 // indicate that there is no suggested score; a value of 0
336 // suppresses the verbatim result.
337 int verbatim_relevance;
339 // The JSON metadata associated with this server response.
340 std::string metadata;
343 DISALLOW_COPY_AND_ASSIGN(Results);
346 virtual ~SearchProvider();
348 // Removes non-inlineable results until either the top result can inline
349 // autocomplete the current input or verbatim outscores the top result.
350 static void RemoveStaleResults(const string16& input,
351 int verbatim_relevance,
352 SuggestResults* suggest_results,
353 NavigationResults* navigation_results);
355 // Calculates the relevance score for the keyword verbatim result (if the
356 // input matches one of the profile's keyword).
357 static int CalculateRelevanceForKeywordVerbatim(AutocompleteInput::Type type,
358 bool prefer_keyword);
360 // AutocompleteProvider:
361 virtual void Start(const AutocompleteInput& input,
362 bool minimal_changes) OVERRIDE;
363 virtual void Stop(bool clear_cached_results) OVERRIDE;
365 // net::URLFetcherDelegate:
366 virtual void OnURLFetchComplete(const net::URLFetcher* source) OVERRIDE;
368 // Called when timer_ expires.
371 // Runs the history query, if necessary. The history query is synchronous.
372 // This does not update |done_|.
373 void DoHistoryQuery(bool minimal_changes);
375 // Determines whether an asynchronous subcomponent query should run for the
376 // current input. If so, starts it if necessary; otherwise stops it.
377 // NOTE: This function does not update |done_|. Callers must do so.
378 void StartOrStopSuggestQuery(bool minimal_changes);
380 // Returns true when the current query can be sent to the Suggest service.
381 // This will be false e.g. when Suggest is disabled, the query contains
382 // potentially private data, etc.
383 bool IsQuerySuitableForSuggest() const;
385 // Stops the suggest query.
386 // NOTE: This does not update |done_|. Callers must do so.
389 // Clears the current results.
390 void ClearAllResults();
392 // Removes stale results for both default and keyword providers. See comments
393 // on RemoveStaleResults().
394 void RemoveAllStaleResults();
396 // Apply calculated relevance scores to the current results.
397 void ApplyCalculatedRelevance();
398 void ApplyCalculatedSuggestRelevance(SuggestResults* list);
399 void ApplyCalculatedNavigationRelevance(NavigationResults* list);
401 // Starts a new URLFetcher requesting suggest results from |template_url|;
402 // callers own the returned URLFetcher, which is NULL for invalid providers.
403 net::URLFetcher* CreateSuggestFetcher(int id,
404 const TemplateURL* template_url,
405 const AutocompleteInput& input);
407 // Parses results from the suggest server and updates the appropriate suggest
408 // and navigation result lists, depending on whether |is_keyword| is true.
409 // Returns whether the appropriate result list members were updated.
410 bool ParseSuggestResults(base::Value* root_val, bool is_keyword);
412 // Converts the parsed results to a set of AutocompleteMatches, |matches_|.
413 void ConvertResultsToAutocompleteMatches();
415 // Checks if suggested relevances violate certain expected constraints.
416 // See UpdateMatches() for the use and explanation of these constraints.
417 bool IsTopMatchNavigationInKeywordMode() const;
418 bool IsTopMatchScoreTooLow() const;
419 bool IsTopMatchSearchWithURLInput() const;
420 bool HasValidDefaultMatch(
421 bool autocomplete_result_will_reorder_for_default_match) const;
423 // Updates |matches_| from the latest results; applies calculated relevances
424 // if suggested relevances cause undesriable behavior. Updates |done_|.
425 void UpdateMatches();
427 // Converts an appropriate number of navigation results in
428 // |navigation_results| to matches and adds them to |matches|.
429 void AddNavigationResultsToMatches(
430 const NavigationResults& navigation_results,
433 // Adds a match for each result in |results| to |map|. |is_keyword| indicates
434 // whether the results correspond to the keyword provider or default provider.
435 void AddHistoryResultsToMap(const HistoryResults& results,
437 int did_not_accept_suggestion,
440 // Calculates relevance scores for all |results|.
441 SuggestResults ScoreHistoryResults(const HistoryResults& results,
442 bool base_prevent_inline_autocomplete,
443 bool input_multiple_words,
444 const string16& input_text,
447 // Adds matches for |results| to |map|.
448 void AddSuggestResultsToMap(const SuggestResults& results,
449 const std::string& metadata,
452 // Gets the relevance score for the verbatim result. This value may be
453 // provided by the suggest server or calculated locally; if
454 // |relevance_from_server| is non-NULL, it will be set to indicate which of
456 int GetVerbatimRelevance(bool* relevance_from_server) const;
458 // Calculates the relevance score for the verbatim result from the
459 // default search engine. This version takes into account context:
460 // i.e., whether the user has entered a keyword-based search or not.
461 int CalculateRelevanceForVerbatim() const;
463 // Calculates the relevance score for the verbatim result from the default
464 // search engine *ignoring* whether the input is a keyword-based search
465 // or not. This function should only be used to determine the minimum
466 // relevance score that the best result from this provider should have.
467 // For normal use, prefer the above function.
468 int CalculateRelevanceForVerbatimIgnoringKeywordModeState() const;
470 // Gets the relevance score for the keyword verbatim result.
471 // |relevance_from_server| is handled as in GetVerbatimRelevance().
472 // TODO(mpearson): Refactor so this duplication isn't necesary or
473 // restructure so one static function takes all the parameters it needs
474 // (rather than looking at internal state).
475 int GetKeywordVerbatimRelevance(bool* relevance_from_server) const;
477 // |time| is the time at which this query was last seen. |is_keyword|
478 // indicates whether the results correspond to the keyword provider or default
479 // provider. |use_aggressive_method| says whether this function can use a
480 // method that gives high scores (1200+) rather than one that gives lower
481 // scores. When using the aggressive method, scores may exceed 1300
482 // unless |prevent_search_history_inlining| is set.
483 int CalculateRelevanceForHistory(const base::Time& time,
485 bool use_aggressive_method,
486 bool prevent_search_history_inlining) const;
488 // Creates an AutocompleteMatch for "Search <engine> for |query_string|" with
489 // the supplied relevance. Adds this match to |map|; if such a match already
490 // exists, whichever one has lower relevance is eliminated.
491 void AddMatchToMap(const string16& input_text,
493 bool relevance_from_server,
494 bool should_prefetch,
495 const std::string& metadata,
496 AutocompleteMatch::Type type,
498 const string16& match_contents,
499 const string16& annotation,
500 const string16& query_string,
501 int accepted_suggestion,
502 const std::string& suggest_query_params,
505 // Returns an AutocompleteMatch for a navigational suggestion.
506 AutocompleteMatch NavigationToMatch(const NavigationResult& navigation);
508 // Resets the scores of all |keyword_navigation_results_| matches to
509 // be below that of the top keyword query match (the verbatim match
510 // as expressed by |keyword_verbatim_relevance_| or keyword query
511 // suggestions stored in |keyword_suggest_results_|). If there
512 // are no keyword suggestions and keyword verbatim is suppressed,
513 // then drops the suggested relevance scores for the navsuggestions
514 // and drops the request to suppress verbatim, thereby introducing the
515 // keyword verbatim match which will naturally outscore the navsuggestions.
516 void DemoteKeywordNavigationMatchesPastTopQuery();
518 // Updates the value of |done_| from the internal state.
521 // The amount of time to wait before sending a new suggest request after the
522 // previous one. Non-const because some unittests modify this value.
523 static int kMinimumTimeBetweenSuggestQueriesMs;
525 // The following keys are used to record additional information on matches.
527 // We annotate our AutocompleteMatches with whether their relevance scores
528 // were server-provided using this key in the |additional_info| field.
529 static const char kRelevanceFromServerKey[];
531 // Indicates whether the server said a match should be prefetched.
532 static const char kShouldPrefetchKey[];
534 // Used to store metadata from the server response, which is needed for
536 static const char kSuggestMetadataKey[];
538 // These are the values for the above keys.
539 static const char kTrue[];
540 static const char kFalse[];
542 // Maintains the TemplateURLs used.
543 Providers providers_;
546 AutocompleteInput input_;
548 // Input when searching against the keyword provider.
549 AutocompleteInput keyword_input_;
551 // Searches in the user's history that begin with the input text.
552 HistoryResults keyword_history_results_;
553 HistoryResults default_history_results_;
555 // Number of suggest results that haven't yet arrived. If greater than 0 it
556 // indicates one of the URLFetchers is still running.
557 int suggest_results_pending_;
559 // A timer to start a query to the suggest server after the user has stopped
560 // typing for long enough.
561 base::OneShotTimer<SearchProvider> timer_;
563 // The time at which we sent a query to the suggest server.
564 base::TimeTicks time_suggest_request_sent_;
566 // Fetchers used to retrieve results for the keyword and default providers.
567 scoped_ptr<net::URLFetcher> keyword_fetcher_;
568 scoped_ptr<net::URLFetcher> default_fetcher_;
570 // Results from the default and keyword search providers.
571 Results default_results_;
572 Results keyword_results_;
574 // Whether a field trial, if any, has triggered in the most recent
575 // autocomplete query. This field is set to false in Start() and may be set
576 // to true if either the default provider or keyword provider has completed
577 // and their corresponding suggest response contained
578 // '"google:fieldtrialtriggered":true'.
579 // If the autocomplete query has not returned, this field is set to false.
580 bool field_trial_triggered_;
582 // Same as above except that it is maintained across the current Omnibox
584 bool field_trial_triggered_in_session_;
586 // If true, search history query suggestions will score low enough that
587 // they will not be inlined.
588 bool prevent_search_history_inlining_;
590 DISALLOW_COPY_AND_ASSIGN(SearchProvider);
593 #endif // CHROME_BROWSER_AUTOCOMPLETE_SEARCH_PROVIDER_H_