1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "chrome/common/net/url_fixer_upper.h"
10 #include "base/environment.h"
12 #include "base/file_util.h"
13 #include "base/logging.h"
14 #include "base/strings/string_util.h"
15 #include "base/strings/utf_string_conversions.h"
16 #include "chrome/common/url_constants.h"
17 #include "net/base/escape.h"
18 #include "net/base/net_util.h"
19 #include "net/base/registry_controlled_domains/registry_controlled_domain.h"
20 #include "url/url_file.h"
21 #include "url/url_parse.h"
22 #include "url/url_util.h"
24 const char* URLFixerUpper::home_directory_override = NULL;
28 // TODO(estade): Remove these ugly, ugly functions. They are only used in
29 // SegmentURL. A url_parse::Parsed object keeps track of a bunch of indices into
30 // a url string, and these need to be updated when the URL is converted from
31 // UTF8 to UTF16. Instead of this after-the-fact adjustment, we should parse it
32 // in the correct string format to begin with.
33 url_parse::Component UTF8ComponentToUTF16Component(
34 const std::string& text_utf8,
35 const url_parse::Component& component_utf8) {
36 if (component_utf8.len == -1)
37 return url_parse::Component();
39 std::string before_component_string =
40 text_utf8.substr(0, component_utf8.begin);
41 std::string component_string = text_utf8.substr(component_utf8.begin,
43 base::string16 before_component_string_16 =
44 base::UTF8ToUTF16(before_component_string);
45 base::string16 component_string_16 = base::UTF8ToUTF16(component_string);
46 url_parse::Component component_16(before_component_string_16.length(),
47 component_string_16.length());
51 void UTF8PartsToUTF16Parts(const std::string& text_utf8,
52 const url_parse::Parsed& parts_utf8,
53 url_parse::Parsed* parts) {
54 if (IsStringASCII(text_utf8)) {
60 UTF8ComponentToUTF16Component(text_utf8, parts_utf8.scheme);
62 UTF8ComponentToUTF16Component(text_utf8, parts_utf8.username);
64 UTF8ComponentToUTF16Component(text_utf8, parts_utf8.password);
66 UTF8ComponentToUTF16Component(text_utf8, parts_utf8.host);
68 UTF8ComponentToUTF16Component(text_utf8, parts_utf8.port);
70 UTF8ComponentToUTF16Component(text_utf8, parts_utf8.path);
72 UTF8ComponentToUTF16Component(text_utf8, parts_utf8.query);
74 UTF8ComponentToUTF16Component(text_utf8, parts_utf8.ref);
77 TrimPositions TrimWhitespaceUTF8(const std::string& input,
78 TrimPositions positions,
79 std::string* output) {
80 // This implementation is not so fast since it converts the text encoding
81 // twice. Please feel free to file a bug if this function hurts the
82 // performance of Chrome.
83 DCHECK(IsStringUTF8(input));
84 base::string16 input16 = base::UTF8ToUTF16(input);
85 base::string16 output16;
86 TrimPositions result = TrimWhitespace(input16, positions, &output16);
87 *output = base::UTF16ToUTF8(output16);
91 // does some basic fixes for input that we want to test for file-ness
92 void PrepareStringForFileOps(const base::FilePath& text,
93 base::FilePath::StringType* output) {
95 TrimWhitespace(text.value(), TRIM_ALL, output);
96 replace(output->begin(), output->end(), '/', '\\');
98 TrimWhitespaceUTF8(text.value(), TRIM_ALL, output);
102 // Tries to create a full path from |text|. If the result is valid and the
103 // file exists, returns true and sets |full_path| to the result. Otherwise,
104 // returns false and leaves |full_path| unchanged.
105 bool ValidPathForFile(const base::FilePath::StringType& text,
106 base::FilePath* full_path) {
107 base::FilePath file_path = base::MakeAbsoluteFilePath(base::FilePath(text));
108 if (file_path.empty())
111 if (!base::PathExists(file_path))
114 *full_path = file_path;
118 #if defined(OS_POSIX)
119 // Given a path that starts with ~, return a path that starts with an
120 // expanded-out /user/foobar directory.
121 std::string FixupHomedir(const std::string& text) {
122 DCHECK(text.length() > 0 && text[0] == '~');
124 if (text.length() == 1 || text[1] == '/') {
125 const char* home = getenv(base::env_vars::kHome);
126 if (URLFixerUpper::home_directory_override)
127 home = URLFixerUpper::home_directory_override;
128 // We'll probably break elsewhere if $HOME is undefined, but check here
132 return home + text.substr(1);
135 // Otherwise, this is a path like ~foobar/baz, where we must expand to
136 // user foobar's home directory. Officially, we should use getpwent(),
137 // but that is a nasty blocking call.
139 #if defined(OS_MACOSX)
140 static const char kHome[] = "/Users/";
142 static const char kHome[] = "/home/";
144 return kHome + text.substr(1);
148 // Tries to create a file: URL from |text| if it looks like a filename, even if
149 // it doesn't resolve as a valid path or to an existing file. Returns a
150 // (possibly invalid) file: URL in |fixed_up_url| for input beginning
151 // with a drive specifier or "\\". Returns the unchanged input in other cases
152 // (including file: URLs: these don't look like filenames).
153 std::string FixupPath(const std::string& text) {
154 DCHECK(!text.empty());
156 base::FilePath::StringType filename;
158 base::FilePath input_path(base::UTF8ToWide(text));
159 PrepareStringForFileOps(input_path, &filename);
161 // Fixup Windows-style drive letters, where "C:" gets rewritten to "C|".
162 if (filename.length() > 1 && filename[1] == '|')
164 #elif defined(OS_POSIX)
165 base::FilePath input_path(text);
166 PrepareStringForFileOps(input_path, &filename);
167 if (filename.length() > 0 && filename[0] == '~')
168 filename = FixupHomedir(filename);
171 // Here, we know the input looks like a file.
172 GURL file_url = net::FilePathToFileURL(base::FilePath(filename));
173 if (file_url.is_valid()) {
174 return base::UTF16ToUTF8(net::FormatUrl(file_url, std::string(),
175 net::kFormatUrlOmitUsernamePassword, net::UnescapeRule::NORMAL, NULL,
179 // Invalid file URL, just return the input.
183 // Checks |domain| to see if a valid TLD is already present. If not, appends
184 // |desired_tld| to the domain, and prepends "www." unless it's already present.
185 void AddDesiredTLD(const std::string& desired_tld, std::string* domain) {
186 if (desired_tld.empty() || domain->empty())
189 // Check the TLD. If the return value is positive, we already have a TLD, so
190 // abort. If the return value is std::string::npos, there's no valid host,
191 // but we can try to append a TLD anyway, since the host may become valid once
192 // the TLD is attached -- for example, "999999999999" is detected as a broken
193 // IP address and marked invalid, but attaching ".com" makes it legal. When
194 // the return value is 0, there's a valid host with no known TLD, so we can
195 // definitely append the user's TLD. We disallow unknown registries here so
196 // users can input "mail.yahoo" and hit ctrl-enter to get
197 // "www.mail.yahoo.com".
198 const size_t registry_length =
199 net::registry_controlled_domains::GetRegistryLength(
201 net::registry_controlled_domains::EXCLUDE_UNKNOWN_REGISTRIES,
202 net::registry_controlled_domains::EXCLUDE_PRIVATE_REGISTRIES);
203 if ((registry_length != 0) && (registry_length != std::string::npos))
206 // Add the suffix at the end of the domain.
207 const size_t domain_length(domain->length());
208 DCHECK_GT(domain_length, 0U);
209 DCHECK_NE(desired_tld[0], '.');
210 if ((*domain)[domain_length - 1] != '.')
211 domain->push_back('.');
212 domain->append(desired_tld);
214 // Now, if the domain begins with "www.", stop.
215 const std::string prefix("www.");
216 if (domain->compare(0, prefix.length(), prefix) != 0) {
217 // Otherwise, add www. to the beginning of the URL.
218 domain->insert(0, prefix);
222 inline void FixupUsername(const std::string& text,
223 const url_parse::Component& part,
225 if (!part.is_valid())
228 // We don't fix up the username at the moment.
229 url->append(text, part.begin, part.len);
230 // Do not append the trailing '@' because we might need to include the user's
231 // password. FixupURL itself will append the '@' for us.
234 inline void FixupPassword(const std::string& text,
235 const url_parse::Component& part,
237 if (!part.is_valid())
240 // We don't fix up the password at the moment.
242 url->append(text, part.begin, part.len);
245 void FixupHost(const std::string& text,
246 const url_parse::Component& part,
248 const std::string& desired_tld,
250 if (!part.is_valid())
253 // Make domain valid.
254 // Strip all leading dots and all but one trailing dot, unless the user only
255 // typed dots, in which case their input is totally invalid and we should just
256 // leave it unchanged.
257 std::string domain(text, part.begin, part.len);
258 const size_t first_nondot(domain.find_first_not_of('.'));
259 if (first_nondot != std::string::npos) {
260 domain.erase(0, first_nondot);
261 size_t last_nondot(domain.find_last_not_of('.'));
262 DCHECK(last_nondot != std::string::npos);
263 last_nondot += 2; // Point at second period in ending string
264 if (last_nondot < domain.length())
265 domain.erase(last_nondot);
268 // Add any user-specified TLD, if applicable.
269 AddDesiredTLD(desired_tld, &domain);
274 void FixupPort(const std::string& text,
275 const url_parse::Component& part,
277 if (!part.is_valid())
280 // We don't fix up the port at the moment.
282 url->append(text, part.begin, part.len);
285 inline void FixupPath(const std::string& text,
286 const url_parse::Component& part,
288 if (!part.is_valid() || part.len == 0) {
289 // We should always have a path.
294 // Append the path as is.
295 url->append(text, part.begin, part.len);
298 inline void FixupQuery(const std::string& text,
299 const url_parse::Component& part,
301 if (!part.is_valid())
304 // We don't fix up the query at the moment.
306 url->append(text, part.begin, part.len);
309 inline void FixupRef(const std::string& text,
310 const url_parse::Component& part,
312 if (!part.is_valid())
315 // We don't fix up the ref at the moment.
317 url->append(text, part.begin, part.len);
320 bool HasPort(const std::string& original_text,
321 const url_parse::Component& scheme_component) {
322 // Find the range between the ":" and the "/".
323 size_t port_start = scheme_component.end() + 1;
324 size_t port_end = port_start;
325 while ((port_end < original_text.length()) &&
326 !url_parse::IsAuthorityTerminator(original_text[port_end]))
328 if (port_end == port_start)
331 // Scan the range to see if it is entirely digits.
332 for (size_t i = port_start; i < port_end; ++i) {
333 if (!IsAsciiDigit(original_text[i]))
340 // Try to extract a valid scheme from the beginning of |text|.
341 // If successful, set |scheme_component| to the text range where the scheme
342 // was located, and fill |canon_scheme| with its canonicalized form.
343 // Otherwise, return false and leave the outputs in an indeterminate state.
344 bool GetValidScheme(const std::string& text,
345 url_parse::Component* scheme_component,
346 std::string* canon_scheme) {
347 canon_scheme->clear();
349 // Locate everything up to (but not including) the first ':'
350 if (!url_parse::ExtractScheme(text.data(), static_cast<int>(text.length()),
355 // Make sure the scheme contains only valid characters, and convert
356 // to lowercase. This also catches IPv6 literals like [::1], because
357 // brackets are not in the whitelist.
358 url_canon::StdStringCanonOutput canon_scheme_output(canon_scheme);
359 url_parse::Component canon_scheme_component;
360 if (!url_canon::CanonicalizeScheme(text.data(), *scheme_component,
361 &canon_scheme_output,
362 &canon_scheme_component))
365 // Strip the ':', and any trailing buffer space.
366 DCHECK_EQ(0, canon_scheme_component.begin);
367 canon_scheme->erase(canon_scheme_component.len);
369 // We need to fix up the segmentation for "www.example.com:/". For this
370 // case, we guess that schemes with a "." are not actually schemes.
371 if (canon_scheme->find('.') != std::string::npos)
374 // We need to fix up the segmentation for "www:123/". For this case, we
375 // will add an HTTP scheme later and make the URL parser happy.
376 // TODO(pkasting): Maybe we should try to use GURL's parser for this?
377 if (HasPort(text, *scheme_component))
380 // Everything checks out.
384 // Performs the work for URLFixerUpper::SegmentURL. |text| may be modified on
385 // output on success: a semicolon following a valid scheme is replaced with a
387 std::string SegmentURLInternal(std::string* text, url_parse::Parsed* parts) {
388 // Initialize the result.
389 *parts = url_parse::Parsed();
392 TrimWhitespaceUTF8(*text, TRIM_ALL, &trimmed);
394 return std::string(); // Nothing to segment.
397 int trimmed_length = static_cast<int>(trimmed.length());
398 if (url_parse::DoesBeginWindowsDriveSpec(trimmed.data(), 0, trimmed_length) ||
399 url_parse::DoesBeginUNCPath(trimmed.data(), 0, trimmed_length, true))
401 #elif defined(OS_POSIX)
402 if (base::FilePath::IsSeparator(trimmed.data()[0]) ||
403 trimmed.data()[0] == '~')
407 // Otherwise, we need to look at things carefully.
409 if (!GetValidScheme(*text, &parts->scheme, &scheme)) {
410 // Try again if there is a ';' in the text. If changing it to a ':' results
411 // in a scheme being found, continue processing with the modified text.
412 bool found_scheme = false;
413 size_t semicolon = text->find(';');
414 if (semicolon != 0 && semicolon != std::string::npos) {
415 (*text)[semicolon] = ':';
416 if (GetValidScheme(*text, &parts->scheme, &scheme))
419 (*text)[semicolon] = ';';
422 // Couldn't determine the scheme, so just pick one.
423 parts->scheme.reset();
424 scheme = StartsWithASCII(*text, "ftp.", false) ?
425 content::kFtpScheme : content::kHttpScheme;
429 // Proceed with about and chrome schemes, but not file or nonstandard schemes.
430 if ((scheme != chrome::kAboutScheme) &&
431 (scheme != content::kChromeUIScheme) &&
432 ((scheme == content::kFileScheme) ||
433 !url_util::IsStandard(
435 url_parse::Component(0, static_cast<int>(scheme.length())))))
438 if (scheme == content::kFileSystemScheme) {
439 // Have the GURL parser do the heavy lifting for us.
440 url_parse::ParseFileSystemURL(text->data(),
441 static_cast<int>(text->length()), parts);
445 if (parts->scheme.is_valid()) {
446 // Have the GURL parser do the heavy lifting for us.
447 url_parse::ParseStandardURL(text->data(), static_cast<int>(text->length()),
452 // We need to add a scheme in order for ParseStandardURL to be happy.
453 // Find the first non-whitespace character.
454 std::string::iterator first_nonwhite = text->begin();
455 while ((first_nonwhite != text->end()) && IsWhitespace(*first_nonwhite))
458 // Construct the text to parse by inserting the scheme.
459 std::string inserted_text(scheme);
460 inserted_text.append(content::kStandardSchemeSeparator);
461 std::string text_to_parse(text->begin(), first_nonwhite);
462 text_to_parse.append(inserted_text);
463 text_to_parse.append(first_nonwhite, text->end());
465 // Have the GURL parser do the heavy lifting for us.
466 url_parse::ParseStandardURL(text_to_parse.data(),
467 static_cast<int>(text_to_parse.length()),
470 // Offset the results of the parse to match the original text.
471 const int offset = -static_cast<int>(inserted_text.length());
472 URLFixerUpper::OffsetComponent(offset, &parts->scheme);
473 URLFixerUpper::OffsetComponent(offset, &parts->username);
474 URLFixerUpper::OffsetComponent(offset, &parts->password);
475 URLFixerUpper::OffsetComponent(offset, &parts->host);
476 URLFixerUpper::OffsetComponent(offset, &parts->port);
477 URLFixerUpper::OffsetComponent(offset, &parts->path);
478 URLFixerUpper::OffsetComponent(offset, &parts->query);
479 URLFixerUpper::OffsetComponent(offset, &parts->ref);
486 std::string URLFixerUpper::SegmentURL(const std::string& text,
487 url_parse::Parsed* parts) {
488 std::string mutable_text(text);
489 return SegmentURLInternal(&mutable_text, parts);
492 base::string16 URLFixerUpper::SegmentURL(const base::string16& text,
493 url_parse::Parsed* parts) {
494 std::string text_utf8 = base::UTF16ToUTF8(text);
495 url_parse::Parsed parts_utf8;
496 std::string scheme_utf8 = SegmentURL(text_utf8, &parts_utf8);
497 UTF8PartsToUTF16Parts(text_utf8, parts_utf8, parts);
498 return base::UTF8ToUTF16(scheme_utf8);
501 GURL URLFixerUpper::FixupURL(const std::string& text,
502 const std::string& desired_tld) {
504 TrimWhitespaceUTF8(text, TRIM_ALL, &trimmed);
506 return GURL(); // Nothing here.
509 url_parse::Parsed parts;
510 std::string scheme(SegmentURLInternal(&trimmed, &parts));
512 // For view-source: URLs, we strip "view-source:", do fixup, and stick it back
513 // on. This allows us to handle things like "view-source:google.com".
514 if (scheme == content::kViewSourceScheme) {
515 // Reject "view-source:view-source:..." to avoid deep recursion.
516 std::string view_source(content::kViewSourceScheme + std::string(":"));
517 if (!StartsWithASCII(text, view_source + view_source, false)) {
518 return GURL(content::kViewSourceScheme + std::string(":") +
519 FixupURL(trimmed.substr(scheme.length() + 1),
520 desired_tld).possibly_invalid_spec());
524 // We handle the file scheme separately.
525 if (scheme == content::kFileScheme)
526 return GURL(parts.scheme.is_valid() ? text : FixupPath(text));
528 // We handle the filesystem scheme separately.
529 if (scheme == content::kFileSystemScheme) {
530 if (parts.inner_parsed() && parts.inner_parsed()->scheme.is_valid())
535 // Parse and rebuild about: and chrome: URLs, except about:blank.
536 bool chrome_url = !LowerCaseEqualsASCII(trimmed, content::kAboutBlankURL) &&
537 ((scheme == chrome::kAboutScheme) ||
538 (scheme == content::kChromeUIScheme));
540 // For some schemes whose layouts we understand, we rebuild it.
541 if (chrome_url || url_util::IsStandard(scheme.c_str(),
542 url_parse::Component(0, static_cast<int>(scheme.length())))) {
543 // Replace the about: scheme with the chrome: scheme.
544 std::string url(chrome_url ? content::kChromeUIScheme : scheme);
545 url.append(content::kStandardSchemeSeparator);
547 // We need to check whether the |username| is valid because it is our
548 // responsibility to append the '@' to delineate the user information from
549 // the host portion of the URL.
550 if (parts.username.is_valid()) {
551 FixupUsername(trimmed, parts.username, &url);
552 FixupPassword(trimmed, parts.password, &url);
556 FixupHost(trimmed, parts.host, parts.scheme.is_valid(), desired_tld, &url);
557 if (chrome_url && !parts.host.is_valid())
558 url.append(chrome::kChromeUIDefaultHost);
559 FixupPort(trimmed, parts.port, &url);
560 FixupPath(trimmed, parts.path, &url);
561 FixupQuery(trimmed, parts.query, &url);
562 FixupRef(trimmed, parts.ref, &url);
567 // In the worst-case, we insert a scheme if the URL lacks one.
568 if (!parts.scheme.is_valid()) {
569 std::string fixed_scheme(scheme);
570 fixed_scheme.append(content::kStandardSchemeSeparator);
571 trimmed.insert(0, fixed_scheme);
574 return GURL(trimmed);
577 // The rules are different here than for regular fixup, since we need to handle
578 // input like "hello.html" and know to look in the current directory. Regular
579 // fixup will look for cues that it is actually a file path before trying to
580 // figure out what file it is. If our logic doesn't work, we will fall back on
582 GURL URLFixerUpper::FixupRelativeFile(const base::FilePath& base_dir,
583 const base::FilePath& text) {
584 base::FilePath old_cur_directory;
585 if (!base_dir.empty()) {
586 // Save the old current directory before we move to the new one.
587 file_util::GetCurrentDirectory(&old_cur_directory);
588 file_util::SetCurrentDirectory(base_dir);
591 // Allow funny input with extra whitespace and the wrong kind of slashes.
592 base::FilePath::StringType trimmed;
593 PrepareStringForFileOps(text, &trimmed);
596 // Avoid recognizing definite non-file URLs as file paths.
598 if (gurl.is_valid() && gurl.IsStandard())
600 base::FilePath full_path;
601 if (is_file && !ValidPathForFile(trimmed, &full_path)) {
602 // Not a path as entered, try unescaping it in case the user has
603 // escaped things. We need to go through 8-bit since the escaped values
604 // only represent 8-bit values.
606 std::wstring unescaped = base::UTF8ToWide(net::UnescapeURLComponent(
607 base::WideToUTF8(trimmed),
608 net::UnescapeRule::SPACES | net::UnescapeRule::URL_SPECIAL_CHARS));
609 #elif defined(OS_POSIX)
610 std::string unescaped = net::UnescapeURLComponent(
612 net::UnescapeRule::SPACES | net::UnescapeRule::URL_SPECIAL_CHARS);
615 if (!ValidPathForFile(unescaped, &full_path))
619 // Put back the current directory if we saved it.
620 if (!base_dir.empty())
621 file_util::SetCurrentDirectory(old_cur_directory);
624 GURL file_url = net::FilePathToFileURL(full_path);
625 if (file_url.is_valid())
626 return GURL(base::UTF16ToUTF8(net::FormatUrl(file_url, std::string(),
627 net::kFormatUrlOmitUsernamePassword, net::UnescapeRule::NORMAL, NULL,
629 // Invalid files fall through to regular processing.
632 // Fall back on regular fixup for this input.
634 std::string text_utf8 = base::WideToUTF8(text.value());
635 #elif defined(OS_POSIX)
636 std::string text_utf8 = text.value();
638 return FixupURL(text_utf8, std::string());
641 void URLFixerUpper::OffsetComponent(int offset, url_parse::Component* part) {
644 if (part->is_valid()) {
645 // Offset the location of this component.
646 part->begin += offset;
648 // This part might not have existed in the original text.