1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "chrome/browser/autocomplete/autocomplete_input.h"
7 #include "base/strings/string_util.h"
8 #include "base/strings/utf_string_conversions.h"
9 #include "chrome/browser/external_protocol/external_protocol_handler.h"
10 #include "chrome/browser/profiles/profile_io_data.h"
11 #include "components/metrics/proto/omnibox_event.pb.h"
12 #include "components/url_fixer/url_fixer.h"
13 #include "content/public/common/url_constants.h"
14 #include "net/base/net_util.h"
15 #include "net/base/registry_controlled_domains/registry_controlled_domain.h"
16 #include "url/url_canon_ip.h"
17 #include "url/url_util.h"
21 void AdjustCursorPositionIfNecessary(size_t num_leading_chars_removed,
22 size_t* cursor_position) {
23 if (*cursor_position == base::string16::npos)
25 if (num_leading_chars_removed < *cursor_position)
26 *cursor_position -= num_leading_chars_removed;
33 AutocompleteInput::AutocompleteInput()
34 : cursor_position_(base::string16::npos),
35 current_page_classification_(metrics::OmniboxEventProto::INVALID_SPEC),
36 type_(metrics::OmniboxInputType::INVALID),
37 prevent_inline_autocomplete_(false),
38 prefer_keyword_(false),
39 allow_exact_keyword_match_(true),
40 want_asynchronous_matches_(true) {
43 AutocompleteInput::AutocompleteInput(
44 const base::string16& text,
45 size_t cursor_position,
46 const base::string16& desired_tld,
47 const GURL& current_url,
48 metrics::OmniboxEventProto::PageClassification current_page_classification,
49 bool prevent_inline_autocomplete,
51 bool allow_exact_keyword_match,
52 bool want_asynchronous_matches)
53 : cursor_position_(cursor_position),
54 current_url_(current_url),
55 current_page_classification_(current_page_classification),
56 prevent_inline_autocomplete_(prevent_inline_autocomplete),
57 prefer_keyword_(prefer_keyword),
58 allow_exact_keyword_match_(allow_exact_keyword_match),
59 want_asynchronous_matches_(want_asynchronous_matches) {
60 DCHECK(cursor_position <= text.length() ||
61 cursor_position == base::string16::npos)
62 << "Text: '" << text << "', cp: " << cursor_position;
63 // None of the providers care about leading white space so we always trim it.
64 // Providers that care about trailing white space handle trimming themselves.
65 if ((base::TrimWhitespace(text, base::TRIM_LEADING, &text_) &
66 base::TRIM_LEADING) != 0)
67 AdjustCursorPositionIfNecessary(text.length() - text_.length(),
70 GURL canonicalized_url;
71 type_ = Parse(text_, desired_tld, &parts_, &scheme_, &canonicalized_url);
73 if (type_ == metrics::OmniboxInputType::INVALID)
76 if (((type_ == metrics::OmniboxInputType::UNKNOWN) ||
77 (type_ == metrics::OmniboxInputType::URL)) &&
78 canonicalized_url.is_valid() &&
79 (!canonicalized_url.IsStandard() || canonicalized_url.SchemeIsFile() ||
80 canonicalized_url.SchemeIsFileSystem() ||
81 !canonicalized_url.host().empty()))
82 canonicalized_url_ = canonicalized_url;
84 size_t chars_removed = RemoveForcedQueryStringIfNecessary(type_, &text_);
85 AdjustCursorPositionIfNecessary(chars_removed, &cursor_position_);
87 // Remove spaces between opening question mark and first actual character.
88 base::string16 trimmed_text;
89 if ((base::TrimWhitespace(text_, base::TRIM_LEADING, &trimmed_text) &
90 base::TRIM_LEADING) != 0) {
91 AdjustCursorPositionIfNecessary(text_.length() - trimmed_text.length(),
98 AutocompleteInput::~AutocompleteInput() {
102 size_t AutocompleteInput::RemoveForcedQueryStringIfNecessary(
103 metrics::OmniboxInputType::Type type,
104 base::string16* text) {
105 if ((type != metrics::OmniboxInputType::FORCED_QUERY) || text->empty() ||
108 // Drop the leading '?'.
114 std::string AutocompleteInput::TypeToString(
115 metrics::OmniboxInputType::Type type) {
117 case metrics::OmniboxInputType::INVALID: return "invalid";
118 case metrics::OmniboxInputType::UNKNOWN: return "unknown";
119 case metrics::OmniboxInputType::DEPRECATED_REQUESTED_URL:
120 return "deprecated-requested-url";
121 case metrics::OmniboxInputType::URL: return "url";
122 case metrics::OmniboxInputType::QUERY: return "query";
123 case metrics::OmniboxInputType::FORCED_QUERY: return "forced-query";
125 return std::string();
129 metrics::OmniboxInputType::Type AutocompleteInput::Parse(
130 const base::string16& text,
131 const base::string16& desired_tld,
133 base::string16* scheme,
134 GURL* canonicalized_url) {
135 size_t first_non_white = text.find_first_not_of(base::kWhitespaceUTF16, 0);
136 if (first_non_white == base::string16::npos)
137 return metrics::OmniboxInputType::INVALID; // All whitespace.
139 if (text[first_non_white] == L'?') {
140 // If the first non-whitespace character is a '?', we magically treat this
142 return metrics::OmniboxInputType::FORCED_QUERY;
145 // Ask our parsing back-end to help us understand what the user typed. We
146 // use the URLFixerUpper here because we want to be smart about what we
147 // consider a scheme. For example, we shouldn't consider www.google.com:80
149 url::Parsed local_parts;
151 parts = &local_parts;
152 const base::string16 parsed_scheme(url_fixer::SegmentURL(text, parts));
154 *scheme = parsed_scheme;
156 // If we can't canonicalize the user's input, the rest of the autocomplete
157 // system isn't going to be able to produce a navigable URL match for it.
158 // So we just return QUERY immediately in these cases.
159 GURL placeholder_canonicalized_url;
160 if (!canonicalized_url)
161 canonicalized_url = &placeholder_canonicalized_url;
162 *canonicalized_url = url_fixer::FixupURL(base::UTF16ToUTF8(text),
163 base::UTF16ToUTF8(desired_tld));
164 if (!canonicalized_url->is_valid())
165 return metrics::OmniboxInputType::QUERY;
167 if (LowerCaseEqualsASCII(parsed_scheme, url::kFileScheme)) {
168 // A user might or might not type a scheme when entering a file URL. In
169 // either case, |parsed_scheme| will tell us that this is a file URL, but
170 // |parts->scheme| might be empty, e.g. if the user typed "C:\foo".
171 return metrics::OmniboxInputType::URL;
174 // If the user typed a scheme, and it's HTTP or HTTPS, we know how to parse it
175 // well enough that we can fall through to the heuristics below. If it's
176 // something else, we can just determine our action based on what we do with
177 // any input of this scheme. In theory we could do better with some schemes
178 // (e.g. "ftp" or "view-source") but I'll wait to spend the effort on that
179 // until I run into some cases that really need it.
180 if (parts->scheme.is_nonempty() &&
181 !LowerCaseEqualsASCII(parsed_scheme, url::kHttpScheme) &&
182 !LowerCaseEqualsASCII(parsed_scheme, url::kHttpsScheme)) {
183 // See if we know how to handle the URL internally. There are some schemes
184 // that we convert to other things before they reach the renderer or else
185 // the renderer handles internally without reaching the net::URLRequest
186 // logic. They thus won't be listed as "handled protocols", but we should
187 // still claim to handle them.
188 if (ProfileIOData::IsHandledProtocol(base::UTF16ToASCII(parsed_scheme)) ||
189 LowerCaseEqualsASCII(parsed_scheme, content::kViewSourceScheme) ||
190 LowerCaseEqualsASCII(parsed_scheme, url::kJavaScriptScheme) ||
191 LowerCaseEqualsASCII(parsed_scheme, url::kDataScheme))
192 return metrics::OmniboxInputType::URL;
194 // Not an internal protocol. Check and see if the user has explicitly
195 // opened this scheme as a URL before, or if the "scheme" is actually a
196 // username. We need to do this after the check above because some
197 // handlable schemes (e.g. "javascript") may be treated as "blocked" by the
198 // external protocol handler because we don't want pages to open them, but
200 ExternalProtocolHandler::BlockState block_state =
201 ExternalProtocolHandler::GetBlockState(
202 base::UTF16ToUTF8(parsed_scheme));
203 switch (block_state) {
204 case ExternalProtocolHandler::DONT_BLOCK:
205 return metrics::OmniboxInputType::URL;
207 case ExternalProtocolHandler::BLOCK:
208 // If we don't want the user to open the URL, don't let it be navigated
210 return metrics::OmniboxInputType::QUERY;
213 // We don't know about this scheme. It might be that the user typed a
214 // URL of the form "username:password@foo.com".
215 const base::string16 http_scheme_prefix =
216 base::ASCIIToUTF16(std::string(url::kHttpScheme) +
217 url::kStandardSchemeSeparator);
218 url::Parsed http_parts;
219 base::string16 http_scheme;
220 GURL http_canonicalized_url;
221 metrics::OmniboxInputType::Type http_type =
222 Parse(http_scheme_prefix + text, desired_tld, &http_parts,
223 &http_scheme, &http_canonicalized_url);
224 DCHECK_EQ(std::string(url::kHttpScheme),
225 base::UTF16ToUTF8(http_scheme));
227 if ((http_type == metrics::OmniboxInputType::URL) &&
228 http_parts.username.is_nonempty() &&
229 http_parts.password.is_nonempty()) {
230 // Manually re-jigger the parsed parts to match |text| (without the
231 // http scheme added).
232 http_parts.scheme.reset();
233 url::Component* components[] = {
234 &http_parts.username,
235 &http_parts.password,
242 for (size_t i = 0; i < arraysize(components); ++i) {
243 url_fixer::OffsetComponent(
244 -static_cast<int>(http_scheme_prefix.length()), components[i]);
250 *canonicalized_url = http_canonicalized_url;
252 return metrics::OmniboxInputType::URL;
255 // We don't know about this scheme and it doesn't look like the user
256 // typed a username and password. It's likely to be a search operator
257 // like "site:" or "link:". We classify it as UNKNOWN so the user has
258 // the option of treating it as a URL if we're wrong.
259 // Note that SegmentURL() is smart so we aren't tricked by "c:\foo" or
260 // "www.example.com:81" in this case.
261 return metrics::OmniboxInputType::UNKNOWN;
266 // Either the user didn't type a scheme, in which case we need to distinguish
267 // between an HTTP URL and a query, or the scheme is HTTP or HTTPS, in which
268 // case we should reject invalid formulations.
270 // If we have an empty host it can't be a valid HTTP[S] URL. (This should
271 // only trigger for input that begins with a colon, which GURL will parse as a
272 // valid, non-standard URL; for standard URLs, an empty host would have
273 // resulted in an invalid |canonicalized_url| above.)
274 if (!parts->host.is_nonempty())
275 return metrics::OmniboxInputType::QUERY;
277 // Sanity-check: GURL should have failed to canonicalize this URL if it had an
279 DCHECK_NE(url::PORT_INVALID, url::ParsePort(text.c_str(), parts->port));
281 // Likewise, the RCDS can reject certain obviously-invalid hosts. (We also
282 // use the registry length later below.)
283 const base::string16 host(text.substr(parts->host.begin, parts->host.len));
284 const size_t registry_length =
285 net::registry_controlled_domains::GetRegistryLength(
286 base::UTF16ToUTF8(host),
287 net::registry_controlled_domains::EXCLUDE_UNKNOWN_REGISTRIES,
288 net::registry_controlled_domains::EXCLUDE_PRIVATE_REGISTRIES);
289 if (registry_length == std::string::npos) {
290 // Try to append the desired_tld.
291 if (!desired_tld.empty()) {
292 base::string16 host_with_tld(host);
293 if (host[host.length() - 1] != '.')
294 host_with_tld += '.';
295 host_with_tld += desired_tld;
296 const size_t tld_length =
297 net::registry_controlled_domains::GetRegistryLength(
298 base::UTF16ToUTF8(host_with_tld),
299 net::registry_controlled_domains::EXCLUDE_UNKNOWN_REGISTRIES,
300 net::registry_controlled_domains::EXCLUDE_PRIVATE_REGISTRIES);
301 if (tld_length != std::string::npos) {
302 // Something like "99999999999" that looks like a bad IP
303 // address, but becomes valid on attaching a TLD.
304 return metrics::OmniboxInputType::URL;
307 // Could be a broken IP address, etc.
308 return metrics::OmniboxInputType::QUERY;
312 // See if the hostname is valid. While IE and GURL allow hostnames to contain
313 // many other characters (perhaps for weird intranet machines), it's extremely
314 // unlikely that a user would be trying to type those in for anything other
315 // than a search query.
316 url::CanonHostInfo host_info;
317 const std::string canonicalized_host(net::CanonicalizeHost(
318 base::UTF16ToUTF8(host), &host_info));
319 if ((host_info.family == url::CanonHostInfo::NEUTRAL) &&
320 !net::IsCanonicalizedHostCompliant(canonicalized_host,
321 base::UTF16ToUTF8(desired_tld))) {
322 // Invalid hostname. There are several possible cases:
323 // * Our checker is too strict and the user pasted in a real-world URL
324 // that's "invalid" but resolves. To catch these, we return UNKNOWN when
325 // the user explicitly typed a scheme, so we'll still search by default
326 // but we'll show the accidental search infobar if necessary.
327 // * The user is typing a multi-word query. If we see a space anywhere in
328 // the hostname we assume this is a search and return QUERY.
329 // * Our checker is too strict and the user is typing a real-world hostname
330 // that's "invalid" but resolves. We return UNKNOWN if the TLD is known.
331 // Note that we explicitly excluded hosts with spaces above so that
332 // "toys at amazon.com" will be treated as a search.
333 // * The user is typing some garbage string. Return QUERY.
335 // Thus we fall down in the following cases:
336 // * Trying to navigate to a hostname with spaces
337 // * Trying to navigate to a hostname with invalid characters and an unknown
339 // These are rare, though probably possible in intranets.
340 return (parts->scheme.is_nonempty() ||
341 ((registry_length != 0) &&
342 (host.find(' ') == base::string16::npos))) ?
343 metrics::OmniboxInputType::UNKNOWN : metrics::OmniboxInputType::QUERY;
346 // Now that we've ruled out all schemes other than http or https and done a
347 // little more sanity checking, the presence of a scheme means this is likely
349 if (parts->scheme.is_nonempty())
350 return metrics::OmniboxInputType::URL;
352 // See if the host is an IP address.
353 if (host_info.family == url::CanonHostInfo::IPV6)
354 return metrics::OmniboxInputType::URL;
355 // If the user originally typed a host that looks like an IP address (a
356 // dotted quad), they probably want to open it. If the original input was
357 // something else (like a single number), they probably wanted to search for
358 // it, unless they explicitly typed a scheme. This is true even if the URL
359 // appears to have a path: "1.2/45" is more likely a search (for the answer
360 // to a math problem) than a URL. However, if there are more non-host
361 // components, then maybe this really was intended to be a navigation. For
362 // this reason we only check the dotted-quad case here, and save the "other
363 // IP addresses" case for after we check the number of non-host components
365 if ((host_info.family == url::CanonHostInfo::IPV4) &&
366 (host_info.num_ipv4_components == 4))
367 return metrics::OmniboxInputType::URL;
369 // Presence of a password means this is likely a URL. Note that unless the
370 // user has typed an explicit "http://" or similar, we'll probably think that
371 // the username is some unknown scheme, and bail out in the scheme-handling
373 if (parts->password.is_nonempty())
374 return metrics::OmniboxInputType::URL;
376 // Trailing slashes force the input to be treated as a URL.
377 if (parts->path.is_nonempty()) {
378 char c = text[parts->path.end() - 1];
379 if ((c == '\\') || (c == '/'))
380 return metrics::OmniboxInputType::URL;
383 // If there is more than one recognized non-host component, this is likely to
384 // be a URL, even if the TLD is unknown (in which case this is likely an
386 if (NumNonHostComponents(*parts) > 1)
387 return metrics::OmniboxInputType::URL;
389 // If the host has a known TLD or a port, it's probably a URL, with the
390 // following exceptions:
391 // * Any "IP addresses" that make it here are more likely searches
393 // * If we reach here with a username, our input looks like "user@host[.tld]".
394 // Because there is no scheme explicitly specified, we think this is more
395 // likely an email address than an HTTP auth attempt. Hence, we search by
396 // default and let users correct us on a case-by-case basis.
397 // Note that we special-case "localhost" as a known hostname.
398 if ((host_info.family != url::CanonHostInfo::IPV4) &&
399 ((registry_length != 0) || (host == base::ASCIIToUTF16("localhost") ||
400 parts->port.is_nonempty()))) {
401 return parts->username.is_nonempty() ? metrics::OmniboxInputType::UNKNOWN :
402 metrics::OmniboxInputType::URL;
405 // If we reach this point, we know there's no known TLD on the input, so if
406 // the user wishes to add a desired_tld, the fixup code will oblige; thus this
408 if (!desired_tld.empty())
409 return metrics::OmniboxInputType::URL;
411 // No scheme, password, port, path, and no known TLD on the host.
413 // * An "incomplete IP address"; likely a search (see above).
414 // * An email-like input like "user@host", where "host" has no known TLD.
415 // It's not clear what the user means here and searching seems reasonable.
416 // * A single word "foo"; possibly an intranet site, but more likely a search.
417 // This is ideally an UNKNOWN, and we can let the Alternate Nav URL code
418 // catch our mistakes.
419 // * A URL with a valid TLD we don't know about yet. If e.g. a registrar adds
420 // "xxx" as a TLD, then until we add it to our data file, Chrome won't know
421 // "foo.xxx" is a real URL. So ideally this is a URL, but we can't really
422 // distinguish this case from:
423 // * A "URL-like" string that's not really a URL (like
424 // "browser.tabs.closeButtons" or "java.awt.event.*"). This is ideally a
425 // QUERY. Since this is indistinguishable from the case above, and this
426 // case is much more likely, claim these are UNKNOWN, which should default
427 // to the right thing and let users correct us on a case-by-case basis.
428 return metrics::OmniboxInputType::UNKNOWN;
432 void AutocompleteInput::ParseForEmphasizeComponents(const base::string16& text,
433 url::Component* scheme,
434 url::Component* host) {
436 base::string16 scheme_str;
437 Parse(text, base::string16(), &parts, &scheme_str, NULL);
439 *scheme = parts.scheme;
442 int after_scheme_and_colon = parts.scheme.end() + 1;
443 // For the view-source scheme, we should emphasize the scheme and host of the
444 // URL qualified by the view-source prefix.
445 if (LowerCaseEqualsASCII(scheme_str, content::kViewSourceScheme) &&
446 (static_cast<int>(text.length()) > after_scheme_and_colon)) {
447 // Obtain the URL prefixed by view-source and parse it.
448 base::string16 real_url(text.substr(after_scheme_and_colon));
449 url::Parsed real_parts;
450 AutocompleteInput::Parse(real_url, base::string16(), &real_parts, NULL, NULL);
451 if (real_parts.scheme.is_nonempty() || real_parts.host.is_nonempty()) {
452 if (real_parts.scheme.is_nonempty()) {
453 *scheme = url::Component(
454 after_scheme_and_colon + real_parts.scheme.begin,
455 real_parts.scheme.len);
459 if (real_parts.host.is_nonempty()) {
460 *host = url::Component(after_scheme_and_colon + real_parts.host.begin,
461 real_parts.host.len);
466 } else if (LowerCaseEqualsASCII(scheme_str, url::kFileSystemScheme) &&
467 parts.inner_parsed() && parts.inner_parsed()->scheme.is_valid()) {
468 *host = parts.inner_parsed()->host;
473 base::string16 AutocompleteInput::FormattedStringWithEquivalentMeaning(
475 const base::string16& formatted_url) {
476 if (!net::CanStripTrailingSlash(url))
477 return formatted_url;
478 const base::string16 url_with_path(formatted_url + base::char16('/'));
479 return (AutocompleteInput::Parse(formatted_url, base::string16(), NULL, NULL,
481 AutocompleteInput::Parse(url_with_path, base::string16(), NULL, NULL,
483 formatted_url : url_with_path;
487 int AutocompleteInput::NumNonHostComponents(const url::Parsed& parts) {
488 int num_nonhost_components = 0;
489 if (parts.scheme.is_nonempty())
490 ++num_nonhost_components;
491 if (parts.username.is_nonempty())
492 ++num_nonhost_components;
493 if (parts.password.is_nonempty())
494 ++num_nonhost_components;
495 if (parts.port.is_nonempty())
496 ++num_nonhost_components;
497 if (parts.path.is_nonempty())
498 ++num_nonhost_components;
499 if (parts.query.is_nonempty())
500 ++num_nonhost_components;
501 if (parts.ref.is_nonempty())
502 ++num_nonhost_components;
503 return num_nonhost_components;
507 bool AutocompleteInput::HasHTTPScheme(const base::string16& input) {
508 std::string utf8_input(base::UTF16ToUTF8(input));
509 url::Component scheme;
510 if (url::FindAndCompareScheme(utf8_input, content::kViewSourceScheme,
512 utf8_input.erase(0, scheme.end() + 1);
514 return url::FindAndCompareScheme(utf8_input, url::kHttpScheme, NULL);
517 void AutocompleteInput::UpdateText(const base::string16& text,
518 size_t cursor_position,
519 const url::Parsed& parts) {
520 DCHECK(cursor_position <= text.length() ||
521 cursor_position == base::string16::npos)
522 << "Text: '" << text << "', cp: " << cursor_position;
524 cursor_position_ = cursor_position;
528 void AutocompleteInput::Clear() {
530 cursor_position_ = base::string16::npos;
531 current_url_ = GURL();
532 current_page_classification_ = metrics::OmniboxEventProto::INVALID_SPEC;
533 type_ = metrics::OmniboxInputType::INVALID;
534 parts_ = url::Parsed();
536 canonicalized_url_ = GURL();
537 prevent_inline_autocomplete_ = false;
538 prefer_keyword_ = false;
539 allow_exact_keyword_match_ = false;
540 want_asynchronous_matches_ = true;