url/url_canon_unittest.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include <errno.h>
   6 #include <stddef.h>
   7
   8 #include "base/macros.h"
   9 #include "base/strings/utf_string_conversions.h"
  10 #include "testing/gtest/include/gtest/gtest.h"
  11 #include "url/third_party/mozilla/url_parse.h"
  12 #include "url/url_canon.h"
  13 #include "url/url_canon_internal.h"
  14 #include "url/url_canon_stdstring.h"
  15 #include "url/url_test_utils.h"
  16
  17 namespace url {
  18
  19 namespace {
  20
  21 struct ComponentCase {
  22   const char* input;
  23   const char* expected;
  24   Component expected_component;
  25   bool expected_success;
  26 };
  27
  28 // ComponentCase but with dual 8-bit/16-bit input. Generally, the unit tests
  29 // treat each input as optional, and will only try processing if non-NULL.
  30 // The output is always 8-bit.
  31 struct DualComponentCase {
  32   const char* input8;
  33   const wchar_t* input16;
  34   const char* expected;
  35   Component expected_component;
  36   bool expected_success;
  37 };
  38
  39 // Test cases for CanonicalizeIPAddress(). The inputs are identical to
  40 // DualComponentCase, but the output has extra CanonHostInfo fields.
  41 struct IPAddressCase {
  42   const char* input8;
  43   const wchar_t* input16;
  44   const char* expected;
  45   Component expected_component;
  46
  47   // CanonHostInfo fields, for verbose output.
  48   CanonHostInfo::Family expected_family;
  49   int expected_num_ipv4_components;
  50   const char* expected_address_hex;  // Two hex chars per IP address byte.
  51 };
  52
  53 std::string BytesToHexString(unsigned char bytes[16], int length) {
  54   EXPECT_TRUE(length == 0 || length == 4 || length == 16)
  55       << "Bad IP address length: " << length;
  56   std::string result;
  57   for (int i = 0; i < length; ++i) {
  58     result.push_back(kHexCharLookup[(bytes[i] >> 4) & 0xf]);
  59     result.push_back(kHexCharLookup[bytes[i] & 0xf]);
  60   }
  61   return result;
  62 }
  63
  64 struct ReplaceCase {
  65   const char* base;
  66   const char* scheme;
  67   const char* username;
  68   const char* password;
  69   const char* host;
  70   const char* port;
  71   const char* path;
  72   const char* query;
  73   const char* ref;
  74   const char* expected;
  75 };
  76
  77 // Magic string used in the replacements code that tells SetupReplComp to
  78 // call the clear function.
  79 const char kDeleteComp[] = "|";
  80
  81 // Sets up a replacement for a single component. This is given pointers to
  82 // the set and clear function for the component being replaced, and will
  83 // either set the component (if it exists) or clear it (if the replacement
  84 // string matches kDeleteComp).
  85 //
  86 // This template is currently used only for the 8-bit case, and the strlen
  87 // causes it to fail in other cases. It is left a template in case we have
  88 // tests for wide replacements.
  89 template<typename CHAR>
  90 void SetupReplComp(
  91     void (Replacements<CHAR>::*set)(const CHAR*, const Component&),
  92     void (Replacements<CHAR>::*clear)(),
  93     Replacements<CHAR>* rep,
  94     const CHAR* str) {
  95   if (str && str[0] == kDeleteComp[0]) {
  96     (rep->*clear)();
  97   } else if (str) {
  98     (rep->*set)(str, Component(0, static_cast<int>(strlen(str))));
  99   }
 100 }
 101
 102 }  // namespace
 103
 104 TEST(URLCanonTest, DoAppendUTF8) {
 105   struct UTF8Case {
 106     unsigned input;
 107     const char* output;
 108   } utf_cases[] = {
 109     // Valid code points.
 110     {0x24, "\x24"},
 111     {0xA2, "\xC2\xA2"},
 112     {0x20AC, "\xE2\x82\xAC"},
 113     {0x24B62, "\xF0\xA4\xAD\xA2"},
 114     {0x10FFFF, "\xF4\x8F\xBF\xBF"},
 115   };
 116   std::string out_str;
 117   for (size_t i = 0; i < arraysize(utf_cases); i++) {
 118     out_str.clear();
 119     StdStringCanonOutput output(&out_str);
 120     AppendUTF8Value(utf_cases[i].input, &output);
 121     output.Complete();
 122     EXPECT_EQ(utf_cases[i].output, out_str);
 123   }
 124 }
 125
 126 #if defined(GTEST_HAS_DEATH_TEST)
 127 // TODO(mattm): Can't run this in debug mode for now, since the DCHECK will
 128 // cause the Chromium stack trace dialog to appear and hang the test.
 129 // See http://crbug.com/49580.
 130 #if defined(NDEBUG) && !defined(DCHECK_ALWAYS_ON)
 131 #define MAYBE_DoAppendUTF8Invalid DoAppendUTF8Invalid
 132 #else
 133 #define MAYBE_DoAppendUTF8Invalid DISABLED_DoAppendUTF8Invalid
 134 #endif
 135 TEST(URLCanonTest, MAYBE_DoAppendUTF8Invalid) {
 136   std::string out_str;
 137   StdStringCanonOutput output(&out_str);
 138   // Invalid code point (too large).
 139   ASSERT_DEBUG_DEATH({
 140     AppendUTF8Value(0x110000, &output);
 141     output.Complete();
 142     EXPECT_EQ("", out_str);
 143   }, "");
 144 }
 145 #endif  // defined(GTEST_HAS_DEATH_TEST)
 146
 147 TEST(URLCanonTest, UTF) {
 148   // Low-level test that we handle reading, canonicalization, and writing
 149   // UTF-8/UTF-16 strings properly.
 150   struct UTFCase {
 151     const char* input8;
 152     const wchar_t* input16;
 153     bool expected_success;
 154     const char* output;
 155   } utf_cases[] = {
 156       // Valid canonical input should get passed through & escaped.
 157     {"\xe4\xbd\xa0\xe5\xa5\xbd", L"\x4f60\x597d", true, "%E4%BD%A0%E5%A5%BD"},
 158       // Test a character that takes > 16 bits (U+10300 = old italic letter A)
 159     {"\xF0\x90\x8C\x80", L"\xd800\xdf00", true, "%F0%90%8C%80"},
 160       // Non-shortest-form UTF-8 characters are invalid. The bad bytes should
 161       // each be replaced with the invalid character (EF BF DB in UTF-8).
 162     {"\xf0\x84\xbd\xa0\xe5\xa5\xbd", NULL, false,
 163      "%EF%BF%BD%EF%BF%BD%EF%BF%BD%EF%BF%BD%E5%A5%BD"},
 164       // Invalid UTF-8 sequences should be marked as invalid (the first
 165       // sequence is truncated).
 166     {"\xe4\xa0\xe5\xa5\xbd", L"\xd800\x597d", false, "%EF%BF%BD%E5%A5%BD"},
 167       // Character going off the end.
 168     {"\xe4\xbd\xa0\xe5\xa5", L"\x4f60\xd800", false, "%E4%BD%A0%EF%BF%BD"},
 169       // ...same with low surrogates with no high surrogate.
 170     {nullptr, L"\xdc00", false, "%EF%BF%BD"},
 171       // Test a UTF-8 encoded surrogate value is marked as invalid.
 172       // ED A0 80 = U+D800
 173     {"\xed\xa0\x80", NULL, false, "%EF%BF%BD%EF%BF%BD%EF%BF%BD"},
 174       // ...even when paired.
 175     {"\xed\xa0\x80\xed\xb0\x80", nullptr, false,
 176      "%EF%BF%BD%EF%BF%BD%EF%BF%BD%EF%BF%BD%EF%BF%BD%EF%BF%BD"},
 177   };
 178
 179   std::string out_str;
 180   for (size_t i = 0; i < arraysize(utf_cases); i++) {
 181     if (utf_cases[i].input8) {
 182       out_str.clear();
 183       StdStringCanonOutput output(&out_str);
 184
 185       int input_len = static_cast<int>(strlen(utf_cases[i].input8));
 186       bool success = true;
 187       for (int ch = 0; ch < input_len; ch++) {
 188         success &= AppendUTF8EscapedChar(utf_cases[i].input8, &ch, input_len,
 189                                          &output);
 190       }
 191       output.Complete();
 192       EXPECT_EQ(utf_cases[i].expected_success, success);
 193       EXPECT_EQ(std::string(utf_cases[i].output), out_str);
 194     }
 195     if (utf_cases[i].input16) {
 196       out_str.clear();
 197       StdStringCanonOutput output(&out_str);
 198
 199       base::string16 input_str(
 200           test_utils::TruncateWStringToUTF16(utf_cases[i].input16));
 201       int input_len = static_cast<int>(input_str.length());
 202       bool success = true;
 203       for (int ch = 0; ch < input_len; ch++) {
 204         success &= AppendUTF8EscapedChar(input_str.c_str(), &ch, input_len,
 205                                          &output);
 206       }
 207       output.Complete();
 208       EXPECT_EQ(utf_cases[i].expected_success, success);
 209       EXPECT_EQ(std::string(utf_cases[i].output), out_str);
 210     }
 211
 212     if (utf_cases[i].input8 && utf_cases[i].input16 &&
 213         utf_cases[i].expected_success) {
 214       // Check that the UTF-8 and UTF-16 inputs are equivalent.
 215
 216       // UTF-16 -> UTF-8
 217       std::string input8_str(utf_cases[i].input8);
 218       base::string16 input16_str(
 219           test_utils::TruncateWStringToUTF16(utf_cases[i].input16));
 220       EXPECT_EQ(input8_str, base::UTF16ToUTF8(input16_str));
 221
 222       // UTF-8 -> UTF-16
 223       EXPECT_EQ(input16_str, base::UTF8ToUTF16(input8_str));
 224     }
 225   }
 226 }
 227
 228 TEST(URLCanonTest, Scheme) {
 229   // Here, we're mostly testing that unusual characters are handled properly.
 230   // The canonicalizer doesn't do any parsing or whitespace detection. It will
 231   // also do its best on error, and will escape funny sequences (these won't be
 232   // valid schemes and it will return error).
 233   //
 234   // Note that the canonicalizer will append a colon to the output to separate
 235   // out the rest of the URL, which is not present in the input. We check,
 236   // however, that the output range includes everything but the colon.
 237   ComponentCase scheme_cases[] = {
 238     {"http", "http:", Component(0, 4), true},
 239     {"HTTP", "http:", Component(0, 4), true},
 240     {" HTTP ", "%20http%20:", Component(0, 10), false},
 241     {"htt: ", "htt%3A%20:", Component(0, 9), false},
 242     {"\xe4\xbd\xa0\xe5\xa5\xbdhttp", "%E4%BD%A0%E5%A5%BDhttp:", Component(0, 22), false},
 243       // Don't re-escape something already escaped. Note that it will
 244       // "canonicalize" the 'A' to 'a', but that's OK.
 245     {"ht%3Atp", "ht%3atp:", Component(0, 7), false},
 246     {"", ":", Component(0, 0), false},
 247   };
 248
 249   std::string out_str;
 250
 251   for (size_t i = 0; i < arraysize(scheme_cases); i++) {
 252     int url_len = static_cast<int>(strlen(scheme_cases[i].input));
 253     Component in_comp(0, url_len);
 254     Component out_comp;
 255
 256     out_str.clear();
 257     StdStringCanonOutput output1(&out_str);
 258     bool success = CanonicalizeScheme(scheme_cases[i].input, in_comp, &output1,
 259                                       &out_comp);
 260     output1.Complete();
 261
 262     EXPECT_EQ(scheme_cases[i].expected_success, success);
 263     EXPECT_EQ(std::string(scheme_cases[i].expected), out_str);
 264     EXPECT_EQ(scheme_cases[i].expected_component.begin, out_comp.begin);
 265     EXPECT_EQ(scheme_cases[i].expected_component.len, out_comp.len);
 266
 267     // Now try the wide version.
 268     out_str.clear();
 269     StdStringCanonOutput output2(&out_str);
 270
 271     base::string16 wide_input(base::UTF8ToUTF16(scheme_cases[i].input));
 272     in_comp.len = static_cast<int>(wide_input.length());
 273     success = CanonicalizeScheme(wide_input.c_str(), in_comp, &output2,
 274                                  &out_comp);
 275     output2.Complete();
 276
 277     EXPECT_EQ(scheme_cases[i].expected_success, success);
 278     EXPECT_EQ(std::string(scheme_cases[i].expected), out_str);
 279     EXPECT_EQ(scheme_cases[i].expected_component.begin, out_comp.begin);
 280     EXPECT_EQ(scheme_cases[i].expected_component.len, out_comp.len);
 281   }
 282
 283   // Test the case where the scheme is declared nonexistent, it should be
 284   // converted into an empty scheme.
 285   Component out_comp;
 286   out_str.clear();
 287   StdStringCanonOutput output(&out_str);
 288
 289   EXPECT_FALSE(CanonicalizeScheme("", Component(0, -1), &output, &out_comp));
 290   output.Complete();
 291
 292   EXPECT_EQ(std::string(":"), out_str);
 293   EXPECT_EQ(0, out_comp.begin);
 294   EXPECT_EQ(0, out_comp.len);
 295 }
 296
 297 TEST(URLCanonTest, Host) {
 298   IPAddressCase host_cases[] = {
 299        // Basic canonicalization, uppercase should be converted to lowercase.
 300     {"GoOgLe.CoM", L"GoOgLe.CoM", "google.com", Component(0, 10), CanonHostInfo::NEUTRAL, -1, ""},
 301       // Spaces and some other characters should be escaped.
 302     {"Goo%20 goo%7C|.com", L"Goo%20 goo%7C|.com", "goo%20%20goo%7C%7C.com", Component(0, 22), CanonHostInfo::NEUTRAL, -1, ""},
 303       // Exciting different types of spaces!
 304     {NULL, L"GOO\x00a0\x3000goo.com", "goo%20%20goo.com", Component(0, 16), CanonHostInfo::NEUTRAL, -1, ""},
 305       // Other types of space (no-break, zero-width, zero-width-no-break) are
 306       // name-prepped away to nothing.
 307     {NULL, L"GOO\x200b\x2060\xfeffgoo.com", "googoo.com", Component(0, 10), CanonHostInfo::NEUTRAL, -1, ""},
 308       // Ideographic full stop (full-width period for Chinese, etc.) should be
 309       // treated as a dot.
 310     {NULL, L"www.foo\x3002" L"bar.com", "www.foo.bar.com", Component(0, 15), CanonHostInfo::NEUTRAL, -1, ""},
 311       // Invalid unicode characters should fail...
 312       // ...In wide input, ICU will barf and we'll end up with the input as
 313       //    escaped UTF-8 (the invalid character should be replaced with the
 314       //    replacement character).
 315     {"\xef\xb7\x90zyx.com", L"\xfdd0zyx.com", "%EF%BF%BDzyx.com", Component(0, 16), CanonHostInfo::BROKEN, -1, ""},
 316       // ...This is the same as previous but with with escaped.
 317     {"%ef%b7%90zyx.com", L"%ef%b7%90zyx.com", "%EF%BF%BDzyx.com", Component(0, 16), CanonHostInfo::BROKEN, -1, ""},
 318       // Test name prepping, fullwidth input should be converted to ASCII and NOT
 319       // IDN-ized. This is "Go" in fullwidth UTF-8/UTF-16.
 320     {"\xef\xbc\xa7\xef\xbd\x8f.com", L"\xff27\xff4f.com", "go.com", Component(0, 6), CanonHostInfo::NEUTRAL, -1, ""},
 321       // Test that fullwidth escaped values are properly name-prepped,
 322       // then converted or rejected.
 323       // ...%41 in fullwidth = 'A' (also as escaped UTF-8 input)
 324     {"\xef\xbc\x85\xef\xbc\x94\xef\xbc\x91.com", L"\xff05\xff14\xff11.com", "a.com", Component(0, 5), CanonHostInfo::NEUTRAL, -1, ""},
 325     {"%ef%bc%85%ef%bc%94%ef%bc%91.com", L"%ef%bc%85%ef%bc%94%ef%bc%91.com", "a.com", Component(0, 5), CanonHostInfo::NEUTRAL, -1, ""},
 326       // ...%00 in fullwidth should fail (also as escaped UTF-8 input)
 327     {"\xef\xbc\x85\xef\xbc\x90\xef\xbc\x90.com", L"\xff05\xff10\xff10.com", "%00.com", Component(0, 7), CanonHostInfo::BROKEN, -1, ""},
 328     {"%ef%bc%85%ef%bc%90%ef%bc%90.com", L"%ef%bc%85%ef%bc%90%ef%bc%90.com", "%00.com", Component(0, 7), CanonHostInfo::BROKEN, -1, ""},
 329       // ICU will convert weird percents into ASCII percents, but not unescape
 330       // further. A weird percent is U+FE6A (EF B9 AA in UTF-8) which is a
 331       // "small percent". At this point we should be within our rights to mark
 332       // anything as invalid since the URL is corrupt or malicious. The code
 333       // happens to allow ASCII characters (%41 = "A" -> 'a') to be unescaped
 334       // and kept as valid, so we validate that behavior here, but this level
 335       // of fixing the input shouldn't be seen as required. "%81" is invalid.
 336     {"\xef\xb9\xaa" "41.com", L"\xfe6a" L"41.com", "a.com", Component(0, 5), CanonHostInfo::NEUTRAL, -1, ""},
 337     {"%ef%b9%aa" "41.com", L"\xfe6a" L"41.com", "a.com", Component(0, 5), CanonHostInfo::NEUTRAL, -1, ""},
 338     {"\xef\xb9\xaa" "81.com", L"\xfe6a" L"81.com", "%81.com", Component(0, 7), CanonHostInfo::BROKEN, -1, ""},
 339     {"%ef%b9%aa" "81.com", L"\xfe6a" L"81.com", "%81.com", Component(0, 7), CanonHostInfo::BROKEN, -1, ""},
 340       // Basic IDN support, UTF-8 and UTF-16 input should be converted to IDN
 341     {"\xe4\xbd\xa0\xe5\xa5\xbd\xe4\xbd\xa0\xe5\xa5\xbd", L"\x4f60\x597d\x4f60\x597d", "xn--6qqa088eba", Component(0, 14), CanonHostInfo::NEUTRAL, -1, ""},
 342       // See http://unicode.org/cldr/utility/idna.jsp for other
 343       // examples/experiments and http://goo.gl/7yG11o
 344       // for the full list of characters handled differently by
 345       // IDNA 2003, UTS 46 (http://unicode.org/reports/tr46/ ) and IDNA 2008.
 346
 347       // 4 Deviation characters are mapped/ignored in UTS 46 transitional
 348       // mechansm. UTS 46, table 4 row (g).
 349       // Sharp-s is mapped to 'ss' in UTS 46 and IDNA 2003.
 350       // Otherwise, it'd be "xn--fuball-cta.de".
 351     {"fu\xc3\x9f" "ball.de", L"fu\x00df" L"ball.de", "fussball.de",
 352       Component(0, 11), CanonHostInfo::NEUTRAL, -1, ""},
 353       // Final-sigma (U+03C3) is mapped to regular sigma (U+03C2).
 354       // Otherwise, it'd be "xn--wxaijb9b".
 355     {"\xcf\x83\xcf\x8c\xce\xbb\xce\xbf\xcf\x82", L"\x3c3\x3cc\x3bb\x3bf\x3c2",
 356       "xn--wxaikc6b", Component(0, 12),
 357       CanonHostInfo::NEUTRAL, -1, ""},
 358       // ZWNJ (U+200C) and ZWJ (U+200D) are mapped away in UTS 46 transitional
 359       // handling as well as in IDNA 2003.
 360     {"a\xe2\x80\x8c" "b\xe2\x80\x8d" "c", L"a\x200c" L"b\x200d" L"c", "abc",
 361       Component(0, 3), CanonHostInfo::NEUTRAL, -1, ""},
 362       // ZWJ between Devanagari characters is still mapped away in UTS 46
 363       // transitional handling. IDNA 2008 would give xn--11bo0mv54g.
 364     {"\xe0\xa4\x95\xe0\xa5\x8d\xe2\x80\x8d\xe0\xa4\x9c",
 365      L"\x915\x94d\x200d\x91c", "xn--11bo0m",
 366      Component(0, 10), CanonHostInfo::NEUTRAL, -1, ""},
 367       // Fullwidth exclamation mark is disallowed. UTS 46, table 4, row (b)
 368       // However, we do allow this at the moment because we don't use
 369       // STD3 rules and canonicalize full-width ASCII to ASCII.
 370     {"wow\xef\xbc\x81", L"wow\xff01", "wow%21",
 371       Component(0, 6), CanonHostInfo::NEUTRAL, -1, ""},
 372       // U+2132 (turned capital F) is disallowed. UTS 46, table 4, row (c)
 373       // Allowed in IDNA 2003, but the mapping changed after Unicode 3.2
 374     {"\xe2\x84\xb2oo", L"\x2132oo", "%E2%84%B2oo",
 375       Component(0, 11), CanonHostInfo::BROKEN, -1, ""},
 376       // U+2F868 (CJK Comp) is disallowed. UTS 46, table 4, row (d)
 377       // Allowed in IDNA 2003, but the mapping changed after Unicode 3.2
 378     {"\xf0\xaf\xa1\xa8\xe5\xa7\xbb.cn", L"\xd87e\xdc68\x59fb.cn",
 379       "%F0%AF%A1%A8%E5%A7%BB.cn",
 380       Component(0, 24), CanonHostInfo::BROKEN, -1, ""},
 381       // Maps uppercase letters to lower case letters. UTS 46 table 4 row (e)
 382     {"M\xc3\x9cNCHEN", L"M\xdcNCHEN", "xn--mnchen-3ya",
 383       Component(0, 14), CanonHostInfo::NEUTRAL, -1, ""},
 384       // An already-IDNA host is not modified.
 385     {"xn--mnchen-3ya", L"xn--mnchen-3ya", "xn--mnchen-3ya",
 386       Component(0, 14), CanonHostInfo::NEUTRAL, -1, ""},
 387       // Symbol/punctuations are allowed in IDNA 2003/UTS46.
 388       // Not allowed in IDNA 2008. UTS 46 table 4 row (f).
 389     {"\xe2\x99\xa5ny.us", L"\x2665ny.us", "xn--ny-s0x.us",
 390       Component(0, 13), CanonHostInfo::NEUTRAL, -1, ""},
 391       // U+11013 is new in Unicode 6.0 and is allowed. UTS 46 table 4, row (h)
 392       // We used to allow it because we passed through unassigned code points.
 393     {"\xf0\x91\x80\x93.com", L"\xd804\xdc13.com", "xn--n00d.com",
 394       Component(0, 12), CanonHostInfo::NEUTRAL, -1, ""},
 395       // U+0602 is disallowed in UTS46/IDNA 2008. UTS 46 table 4, row(i)
 396       // Used to be allowed in INDA 2003.
 397     {"\xd8\x82.eg", L"\x602.eg", "%D8%82.eg",
 398       Component(0, 9), CanonHostInfo::BROKEN, -1, ""},
 399       // U+20B7 is new in Unicode 5.2 (not a part of IDNA 2003 based
 400       // on Unicode 3.2). We did allow it in the past because we let unassigned
 401       // code point pass. We continue to allow it even though it's a
 402       // "punctuation and symbol" blocked in IDNA 2008.
 403       // UTS 46 table 4, row (j)
 404     {"\xe2\x82\xb7.com", L"\x20b7.com", "xn--wzg.com",
 405       Component(0, 11), CanonHostInfo::NEUTRAL, -1, ""},
 406       // Maps uppercase letters to lower case letters.
 407       // In IDNA 2003, it's allowed without case-folding
 408       // ( xn--bc-7cb.com ) because it's not defined in Unicode 3.2
 409       // (added in Unicode 4.1). UTS 46 table 4 row (k)
 410     {"bc\xc8\xba.com", L"bc\x23a.com", "xn--bc-is1a.com",
 411       Component(0, 15), CanonHostInfo::NEUTRAL, -1, ""},
 412       // Maps U+FF43 (Full Width Small Letter C) to 'c'.
 413     {"ab\xef\xbd\x83.xyz", L"ab\xff43.xyz", "abc.xyz",
 414       Component(0, 7), CanonHostInfo::NEUTRAL, -1, ""},
 415       // Maps U+1D68C (Math Monospace Small C) to 'c'.
 416       // U+1D68C = \xD835\xDE8C in UTF-16
 417     {"ab\xf0\x9d\x9a\x8c.xyz", L"ab\xd835\xde8c.xyz", "abc.xyz",
 418       Component(0, 7), CanonHostInfo::NEUTRAL, -1, ""},
 419       // BiDi check test
 420       // "Divehi" in Divehi (Thaana script) ends with BidiClass=NSM.
 421       // Disallowed in IDNA 2003 but now allowed in UTS 46/IDNA 2008.
 422     {"\xde\x8b\xde\xa8\xde\x88\xde\xac\xde\x80\xde\xa8",
 423      L"\x78b\x7a8\x788\x7ac\x780\x7a8", "xn--hqbpi0jcw",
 424      Component(0, 13), CanonHostInfo::NEUTRAL, -1, ""},
 425       // Disallowed in both IDNA 2003 and 2008 with BiDi check.
 426       // Labels starting with a RTL character cannot end with a LTR character.
 427     {"\xd8\xac\xd8\xa7\xd8\xb1xyz", L"\x62c\x627\x631xyz",
 428      "%D8%AC%D8%A7%D8%B1xyz", Component(0, 21),
 429      CanonHostInfo::BROKEN, -1, ""},
 430       // Labels starting with a RTL character can end with BC=EN (European
 431       // number). Disallowed in IDNA 2003 but now allowed.
 432     {"\xd8\xac\xd8\xa7\xd8\xb1" "2", L"\x62c\x627\x631" L"2",
 433      "xn--2-ymcov", Component(0, 11),
 434      CanonHostInfo::NEUTRAL, -1, ""},
 435       // Labels starting with a RTL character cannot have "L" characters
 436       // even if it ends with an BC=EN. Disallowed in both IDNA 2003/2008.
 437     {"\xd8\xac\xd8\xa7\xd8\xb1xy2", L"\x62c\x627\x631xy2",
 438      "%D8%AC%D8%A7%D8%B1xy2", Component(0, 21),
 439      CanonHostInfo::BROKEN, -1, ""},
 440       // Labels starting with a RTL character can end with BC=AN (Arabic number)
 441       // Disallowed in IDNA 2003, but now allowed.
 442     {"\xd8\xac\xd8\xa7\xd8\xb1\xd9\xa2", L"\x62c\x627\x631\x662",
 443      "xn--mgbjq0r", Component(0, 11),
 444      CanonHostInfo::NEUTRAL, -1, ""},
 445       // Labels starting with a RTL character cannot have "L" characters
 446       // even if it ends with an BC=AN (Arabic number).
 447       // Disallowed in both IDNA 2003/2008.
 448     {"\xd8\xac\xd8\xa7\xd8\xb1xy\xd9\xa2", L"\x62c\x627\x631xy\x662",
 449      "%D8%AC%D8%A7%D8%B1xy%D9%A2", Component(0, 26),
 450      CanonHostInfo::BROKEN, -1, ""},
 451       // Labels starting with a RTL character cannot mix BC=EN and BC=AN
 452     {"\xd8\xac\xd8\xa7\xd8\xb1xy2\xd9\xa2", L"\x62c\x627\x631xy2\x662",
 453      "%D8%AC%D8%A7%D8%B1xy2%D9%A2", Component(0, 27),
 454      CanonHostInfo::BROKEN, -1, ""},
 455       // As of Unicode 6.2, U+20CF is not assigned. We do not allow it.
 456     {"\xe2\x83\x8f.com", L"\x20cf.com", "%E2%83%8F.com",
 457       Component(0, 13), CanonHostInfo::BROKEN, -1, ""},
 458       // U+0080 is not allowed.
 459     {"\xc2\x80.com", L"\x80.com", "%C2%80.com",
 460       Component(0, 10), CanonHostInfo::BROKEN, -1, ""},
 461       // Mixed UTF-8 and escaped UTF-8 (narrow case) and UTF-16 and escaped
 462       // Mixed UTF-8 and escaped UTF-8 (narrow case) and UTF-16 and escaped
 463       // UTF-8 (wide case). The output should be equivalent to the true wide
 464       // character input above).
 465     {"%E4%BD%A0%E5%A5%BD\xe4\xbd\xa0\xe5\xa5\xbd",
 466       L"%E4%BD%A0%E5%A5%BD\x4f60\x597d", "xn--6qqa088eba",
 467       Component(0, 14), CanonHostInfo::NEUTRAL, -1, ""},
 468       // Invalid escaped characters should fail and the percents should be
 469       // escaped.
 470     {"%zz%66%a", L"%zz%66%a", "%25zzf%25a", Component(0, 10),
 471       CanonHostInfo::BROKEN, -1, ""},
 472       // If we get an invalid character that has been escaped.
 473     {"%25", L"%25", "%25", Component(0, 3),
 474       CanonHostInfo::BROKEN, -1, ""},
 475     {"hello%00", L"hello%00", "hello%00", Component(0, 8),
 476       CanonHostInfo::BROKEN, -1, ""},
 477       // Escaped numbers should be treated like IP addresses if they are.
 478     {"%30%78%63%30%2e%30%32%35%30.01", L"%30%78%63%30%2e%30%32%35%30.01",
 479       "192.168.0.1", Component(0, 11), CanonHostInfo::IPV4, 3,
 480       "C0A80001"},
 481     {"%30%78%63%30%2e%30%32%35%30.01%2e", L"%30%78%63%30%2e%30%32%35%30.01%2e",
 482       "192.168.0.1", Component(0, 11), CanonHostInfo::IPV4, 3,
 483       "C0A80001"},
 484       // Invalid escaping should trigger the regular host error handling.
 485     {"%3g%78%63%30%2e%30%32%35%30%2E.01", L"%3g%78%63%30%2e%30%32%35%30%2E.01", "%253gxc0.0250..01", Component(0, 17), CanonHostInfo::BROKEN, -1, ""},
 486       // Something that isn't exactly an IP should get treated as a host and
 487       // spaces escaped.
 488     {"192.168.0.1 hello", L"192.168.0.1 hello", "192.168.0.1%20hello", Component(0, 19), CanonHostInfo::NEUTRAL, -1, ""},
 489       // Fullwidth and escaped UTF-8 fullwidth should still be treated as IP.
 490       // These are "0Xc0.0250.01" in fullwidth.
 491     {"\xef\xbc\x90%Ef%bc\xb8%ef%Bd%83\xef\xbc\x90%EF%BC%8E\xef\xbc\x90\xef\xbc\x92\xef\xbc\x95\xef\xbc\x90\xef\xbc%8E\xef\xbc\x90\xef\xbc\x91", L"\xff10\xff38\xff43\xff10\xff0e\xff10\xff12\xff15\xff10\xff0e\xff10\xff11", "192.168.0.1", Component(0, 11), CanonHostInfo::IPV4, 3, "C0A80001"},
 492       // Broken IP addresses get marked as such.
 493     {"192.168.0.257", L"192.168.0.257", "192.168.0.257", Component(0, 13), CanonHostInfo::BROKEN, -1, ""},
 494     {"[google.com]", L"[google.com]", "[google.com]", Component(0, 12), CanonHostInfo::BROKEN, -1, ""},
 495       // Cyrillic letter followed by '(' should return punycode for '(' escaped
 496       // before punycode string was created. I.e.
 497       // if '(' is escaped after punycode is created we would get xn--%28-8tb
 498       // (incorrect).
 499     {"\xd1\x82(", L"\x0442(", "xn--%28-7ed", Component(0, 11),
 500       CanonHostInfo::NEUTRAL, -1, ""},
 501       // Address with all hexidecimal characters with leading number of 1<<32
 502       // or greater and should return NEUTRAL rather than BROKEN if not all
 503       // components are numbers.
 504     {"12345678912345.de", L"12345678912345.de", "12345678912345.de", Component(0, 17), CanonHostInfo::NEUTRAL, -1, ""},
 505     {"1.12345678912345.de", L"1.12345678912345.de", "1.12345678912345.de", Component(0, 19), CanonHostInfo::NEUTRAL, -1, ""},
 506     {"12345678912345.12345678912345.de", L"12345678912345.12345678912345.de", "12345678912345.12345678912345.de", Component(0, 32), CanonHostInfo::NEUTRAL, -1, ""},
 507     {"1.2.0xB3A73CE5B59.de", L"1.2.0xB3A73CE5B59.de", "1.2.0xb3a73ce5b59.de", Component(0, 20), CanonHostInfo::NEUTRAL, -1, ""},
 508     {"12345678912345.0xde", L"12345678912345.0xde", "12345678912345.0xde", Component(0, 19), CanonHostInfo::BROKEN, -1, ""},
 509     // A label that starts with "xn--" but contains non-ASCII characters should
 510     // be an error. Escape the invalid characters.
 511     {"xn--m\xc3\xbcnchen", L"xn--m\xfcnchen", "xn--m%C3%BCnchen", Component(0, 16), CanonHostInfo::BROKEN, -1, ""},
 512   };
 513
 514   // CanonicalizeHost() non-verbose.
 515   std::string out_str;
 516   for (size_t i = 0; i < arraysize(host_cases); i++) {
 517     // Narrow version.
 518     if (host_cases[i].input8) {
 519       int host_len = static_cast<int>(strlen(host_cases[i].input8));
 520       Component in_comp(0, host_len);
 521       Component out_comp;
 522
 523       out_str.clear();
 524       StdStringCanonOutput output(&out_str);
 525
 526       bool success = CanonicalizeHost(host_cases[i].input8, in_comp, &output,
 527                                       &out_comp);
 528       output.Complete();
 529
 530       EXPECT_EQ(host_cases[i].expected_family != CanonHostInfo::BROKEN,
 531                 success) << "for input: " << host_cases[i].input8;
 532       EXPECT_EQ(std::string(host_cases[i].expected), out_str) <<
 533                 "for input: " << host_cases[i].input8;
 534       EXPECT_EQ(host_cases[i].expected_component.begin, out_comp.begin) <<
 535                 "for input: " << host_cases[i].input8;
 536       EXPECT_EQ(host_cases[i].expected_component.len, out_comp.len) <<
 537                 "for input: " << host_cases[i].input8;
 538     }
 539
 540     // Wide version.
 541     if (host_cases[i].input16) {
 542       base::string16 input16(
 543           test_utils::TruncateWStringToUTF16(host_cases[i].input16));
 544       int host_len = static_cast<int>(input16.length());
 545       Component in_comp(0, host_len);
 546       Component out_comp;
 547
 548       out_str.clear();
 549       StdStringCanonOutput output(&out_str);
 550
 551       bool success = CanonicalizeHost(input16.c_str(), in_comp, &output,
 552                                       &out_comp);
 553       output.Complete();
 554
 555       EXPECT_EQ(host_cases[i].expected_family != CanonHostInfo::BROKEN,
 556                 success);
 557       EXPECT_EQ(std::string(host_cases[i].expected), out_str);
 558       EXPECT_EQ(host_cases[i].expected_component.begin, out_comp.begin);
 559       EXPECT_EQ(host_cases[i].expected_component.len, out_comp.len);
 560     }
 561   }
 562
 563   // CanonicalizeHostVerbose()
 564   for (size_t i = 0; i < arraysize(host_cases); i++) {
 565     // Narrow version.
 566     if (host_cases[i].input8) {
 567       int host_len = static_cast<int>(strlen(host_cases[i].input8));
 568       Component in_comp(0, host_len);
 569
 570       out_str.clear();
 571       StdStringCanonOutput output(&out_str);
 572       CanonHostInfo host_info;
 573
 574       CanonicalizeHostVerbose(host_cases[i].input8, in_comp, &output,
 575                               &host_info);
 576       output.Complete();
 577
 578       EXPECT_EQ(host_cases[i].expected_family, host_info.family);
 579       EXPECT_EQ(std::string(host_cases[i].expected), out_str);
 580       EXPECT_EQ(host_cases[i].expected_component.begin,
 581                 host_info.out_host.begin);
 582       EXPECT_EQ(host_cases[i].expected_component.len, host_info.out_host.len);
 583       EXPECT_EQ(std::string(host_cases[i].expected_address_hex),
 584                 BytesToHexString(host_info.address, host_info.AddressLength()));
 585       if (host_cases[i].expected_family == CanonHostInfo::IPV4) {
 586         EXPECT_EQ(host_cases[i].expected_num_ipv4_components,
 587                   host_info.num_ipv4_components);
 588       }
 589     }
 590
 591     // Wide version.
 592     if (host_cases[i].input16) {
 593       base::string16 input16(
 594           test_utils::TruncateWStringToUTF16(host_cases[i].input16));
 595       int host_len = static_cast<int>(input16.length());
 596       Component in_comp(0, host_len);
 597
 598       out_str.clear();
 599       StdStringCanonOutput output(&out_str);
 600       CanonHostInfo host_info;
 601
 602       CanonicalizeHostVerbose(input16.c_str(), in_comp, &output, &host_info);
 603       output.Complete();
 604
 605       EXPECT_EQ(host_cases[i].expected_family, host_info.family);
 606       EXPECT_EQ(std::string(host_cases[i].expected), out_str);
 607       EXPECT_EQ(host_cases[i].expected_component.begin,
 608                 host_info.out_host.begin);
 609       EXPECT_EQ(host_cases[i].expected_component.len, host_info.out_host.len);
 610       EXPECT_EQ(std::string(host_cases[i].expected_address_hex),
 611                 BytesToHexString(host_info.address, host_info.AddressLength()));
 612       if (host_cases[i].expected_family == CanonHostInfo::IPV4) {
 613         EXPECT_EQ(host_cases[i].expected_num_ipv4_components,
 614                   host_info.num_ipv4_components);
 615       }
 616     }
 617   }
 618 }
 619
 620 TEST(URLCanonTest, IPv4) {
 621   IPAddressCase cases[] = {
 622       // Empty is not an IP address.
 623     {"", L"", "", Component(), CanonHostInfo::NEUTRAL, -1, ""},
 624     {".", L".", "", Component(), CanonHostInfo::NEUTRAL, -1, ""},
 625       // Regular IP addresses in different bases.
 626     {"192.168.0.1", L"192.168.0.1", "192.168.0.1", Component(0, 11), CanonHostInfo::IPV4, 4, "C0A80001"},
 627     {"0300.0250.00.01", L"0300.0250.00.01", "192.168.0.1", Component(0, 11), CanonHostInfo::IPV4, 4, "C0A80001"},
 628     {"0xC0.0Xa8.0x0.0x1", L"0xC0.0Xa8.0x0.0x1", "192.168.0.1", Component(0, 11), CanonHostInfo::IPV4, 4, "C0A80001"},
 629       // Non-IP addresses due to invalid characters.
 630     {"192.168.9.com", L"192.168.9.com", "", Component(), CanonHostInfo::NEUTRAL, -1, ""},
 631       // Invalid characters for the base should be rejected.
 632     {"19a.168.0.1", L"19a.168.0.1", "", Component(), CanonHostInfo::NEUTRAL, -1, ""},
 633     {"0308.0250.00.01", L"0308.0250.00.01", "", Component(), CanonHostInfo::NEUTRAL, -1, ""},
 634     {"0xCG.0xA8.0x0.0x1", L"0xCG.0xA8.0x0.0x1", "", Component(), CanonHostInfo::NEUTRAL, -1, ""},
 635       // If there are not enough components, the last one should fill them out.
 636     {"192", L"192", "0.0.0.192", Component(0, 9), CanonHostInfo::IPV4, 1, "000000C0"},
 637     {"0xC0a80001", L"0xC0a80001", "192.168.0.1", Component(0, 11), CanonHostInfo::IPV4, 1, "C0A80001"},
 638     {"030052000001", L"030052000001", "192.168.0.1", Component(0, 11), CanonHostInfo::IPV4, 1, "C0A80001"},
 639     {"000030052000001", L"000030052000001", "192.168.0.1", Component(0, 11), CanonHostInfo::IPV4, 1, "C0A80001"},
 640     {"192.168", L"192.168", "192.0.0.168", Component(0, 11), CanonHostInfo::IPV4, 2, "C00000A8"},
 641     {"192.0x00A80001", L"192.0x000A80001", "192.168.0.1", Component(0, 11), CanonHostInfo::IPV4, 2, "C0A80001"},
 642     {"0xc0.052000001", L"0xc0.052000001", "192.168.0.1", Component(0, 11), CanonHostInfo::IPV4, 2, "C0A80001"},
 643     {"192.168.1", L"192.168.1", "192.168.0.1", Component(0, 11), CanonHostInfo::IPV4, 3, "C0A80001"},
 644       // Too many components means not an IP address.
 645     {"192.168.0.0.1", L"192.168.0.0.1", "", Component(), CanonHostInfo::NEUTRAL, -1, ""},
 646       // We allow a single trailing dot.
 647     {"192.168.0.1.", L"192.168.0.1.", "192.168.0.1", Component(0, 11), CanonHostInfo::IPV4, 4, "C0A80001"},
 648     {"192.168.0.1. hello", L"192.168.0.1. hello", "", Component(), CanonHostInfo::NEUTRAL, -1, ""},
 649     {"192.168.0.1..", L"192.168.0.1..", "", Component(), CanonHostInfo::NEUTRAL, -1, ""},
 650       // Two dots in a row means not an IP address.
 651     {"192.168..1", L"192.168..1", "", Component(), CanonHostInfo::NEUTRAL, -1, ""},
 652       // Any numerical overflow should be marked as BROKEN.
 653     {"0x100.0", L"0x100.0", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 654     {"0x100.0.0", L"0x100.0.0", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 655     {"0x100.0.0.0", L"0x100.0.0.0", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 656     {"0.0x100.0.0", L"0.0x100.0.0", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 657     {"0.0.0x100.0", L"0.0.0x100.0", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 658     {"0.0.0.0x100", L"0.0.0.0x100", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 659     {"0.0.0x10000", L"0.0.0x10000", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 660     {"0.0x1000000", L"0.0x1000000", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 661     {"0x100000000", L"0x100000000", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 662       // Repeat the previous tests, minus 1, to verify boundaries.
 663     {"0xFF.0", L"0xFF.0", "255.0.0.0", Component(0, 9), CanonHostInfo::IPV4, 2, "FF000000"},
 664     {"0xFF.0.0", L"0xFF.0.0", "255.0.0.0", Component(0, 9), CanonHostInfo::IPV4, 3, "FF000000"},
 665     {"0xFF.0.0.0", L"0xFF.0.0.0", "255.0.0.0", Component(0, 9), CanonHostInfo::IPV4, 4, "FF000000"},
 666     {"0.0xFF.0.0", L"0.0xFF.0.0", "0.255.0.0", Component(0, 9), CanonHostInfo::IPV4, 4, "00FF0000"},
 667     {"0.0.0xFF.0", L"0.0.0xFF.0", "0.0.255.0", Component(0, 9), CanonHostInfo::IPV4, 4, "0000FF00"},
 668     {"0.0.0.0xFF", L"0.0.0.0xFF", "0.0.0.255", Component(0, 9), CanonHostInfo::IPV4, 4, "000000FF"},
 669     {"0.0.0xFFFF", L"0.0.0xFFFF", "0.0.255.255", Component(0, 11), CanonHostInfo::IPV4, 3, "0000FFFF"},
 670     {"0.0xFFFFFF", L"0.0xFFFFFF", "0.255.255.255", Component(0, 13), CanonHostInfo::IPV4, 2, "00FFFFFF"},
 671     {"0xFFFFFFFF", L"0xFFFFFFFF", "255.255.255.255", Component(0, 15), CanonHostInfo::IPV4, 1, "FFFFFFFF"},
 672       // Old trunctations tests. They're all "BROKEN" now.
 673     {"276.256.0xf1a2.077777", L"276.256.0xf1a2.077777", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 674     {"192.168.0.257", L"192.168.0.257", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 675     {"192.168.0xa20001", L"192.168.0xa20001", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 676     {"192.015052000001", L"192.015052000001", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 677     {"0X12C0a80001", L"0X12C0a80001", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 678     {"276.1.2", L"276.1.2", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 679       // Spaces should be rejected.
 680     {"192.168.0.1 hello", L"192.168.0.1 hello", "", Component(), CanonHostInfo::NEUTRAL, -1, ""},
 681       // Very large numbers.
 682     {"0000000000000300.0x00000000000000fF.00000000000000001", L"0000000000000300.0x00000000000000fF.00000000000000001", "192.255.0.1", Component(0, 11), CanonHostInfo::IPV4, 3, "C0FF0001"},
 683     {"0000000000000300.0xffffffffFFFFFFFF.3022415481470977", L"0000000000000300.0xffffffffFFFFFFFF.3022415481470977", "", Component(0, 11), CanonHostInfo::BROKEN, -1, ""},
 684       // A number has no length limit, but long numbers can still overflow.
 685     {"00000000000000000001", L"00000000000000000001", "0.0.0.1", Component(0, 7), CanonHostInfo::IPV4, 1, "00000001"},
 686     {"0000000000000000100000000000000001", L"0000000000000000100000000000000001", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 687       // If a long component is non-numeric, it's a hostname, *not* a broken IP.
 688     {"0.0.0.000000000000000000z", L"0.0.0.000000000000000000z", "", Component(), CanonHostInfo::NEUTRAL, -1, ""},
 689     {"0.0.0.100000000000000000z", L"0.0.0.100000000000000000z", "", Component(), CanonHostInfo::NEUTRAL, -1, ""},
 690       // Truncation of all zeros should still result in 0.
 691     {"0.00.0x.0x0", L"0.00.0x.0x0", "0.0.0.0", Component(0, 7), CanonHostInfo::IPV4, 4, "00000000"},
 692   };
 693
 694   for (size_t i = 0; i < arraysize(cases); i++) {
 695     // 8-bit version.
 696     Component component(0, static_cast<int>(strlen(cases[i].input8)));
 697
 698     std::string out_str1;
 699     StdStringCanonOutput output1(&out_str1);
 700     CanonHostInfo host_info;
 701     CanonicalizeIPAddress(cases[i].input8, component, &output1, &host_info);
 702     output1.Complete();
 703
 704     EXPECT_EQ(cases[i].expected_family, host_info.family);
 705     EXPECT_EQ(std::string(cases[i].expected_address_hex),
 706               BytesToHexString(host_info.address, host_info.AddressLength()));
 707     if (host_info.family == CanonHostInfo::IPV4) {
 708       EXPECT_STREQ(cases[i].expected, out_str1.c_str());
 709       EXPECT_EQ(cases[i].expected_component.begin, host_info.out_host.begin);
 710       EXPECT_EQ(cases[i].expected_component.len, host_info.out_host.len);
 711       EXPECT_EQ(cases[i].expected_num_ipv4_components,
 712                 host_info.num_ipv4_components);
 713     }
 714
 715     // 16-bit version.
 716     base::string16 input16(
 717         test_utils::TruncateWStringToUTF16(cases[i].input16));
 718     component = Component(0, static_cast<int>(input16.length()));
 719
 720     std::string out_str2;
 721     StdStringCanonOutput output2(&out_str2);
 722     CanonicalizeIPAddress(input16.c_str(), component, &output2, &host_info);
 723     output2.Complete();
 724
 725     EXPECT_EQ(cases[i].expected_family, host_info.family);
 726     EXPECT_EQ(std::string(cases[i].expected_address_hex),
 727               BytesToHexString(host_info.address, host_info.AddressLength()));
 728     if (host_info.family == CanonHostInfo::IPV4) {
 729       EXPECT_STREQ(cases[i].expected, out_str2.c_str());
 730       EXPECT_EQ(cases[i].expected_component.begin, host_info.out_host.begin);
 731       EXPECT_EQ(cases[i].expected_component.len, host_info.out_host.len);
 732       EXPECT_EQ(cases[i].expected_num_ipv4_components,
 733                 host_info.num_ipv4_components);
 734     }
 735   }
 736 }
 737
 738 TEST(URLCanonTest, IPv6) {
 739   IPAddressCase cases[] = {
 740       // Empty is not an IP address.
 741     {"", L"", "", Component(), CanonHostInfo::NEUTRAL, -1, ""},
 742       // Non-IPs with [:] characters are marked BROKEN.
 743     {":", L":", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 744     {"[", L"[", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 745     {"[:", L"[:", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 746     {"]", L"]", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 747     {":]", L":]", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 748     {"[]", L"[]", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 749     {"[:]", L"[:]", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 750       // Regular IP address is invalid without bounding '[' and ']'.
 751     {"2001:db8::1", L"2001:db8::1", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 752     {"[2001:db8::1", L"[2001:db8::1", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 753     {"2001:db8::1]", L"2001:db8::1]", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 754       // Regular IP addresses.
 755     {"[::]", L"[::]", "[::]", Component(0,4), CanonHostInfo::IPV6, -1, "00000000000000000000000000000000"},
 756     {"[::1]", L"[::1]", "[::1]", Component(0,5), CanonHostInfo::IPV6, -1, "00000000000000000000000000000001"},
 757     {"[1::]", L"[1::]", "[1::]", Component(0,5), CanonHostInfo::IPV6, -1, "00010000000000000000000000000000"},
 758
 759     // Leading zeros should be stripped.
 760     {"[000:01:02:003:004:5:6:007]", L"[000:01:02:003:004:5:6:007]", "[0:1:2:3:4:5:6:7]", Component(0,17), CanonHostInfo::IPV6, -1, "00000001000200030004000500060007"},
 761
 762     // Upper case letters should be lowercased.
 763     {"[A:b:c:DE:fF:0:1:aC]", L"[A:b:c:DE:fF:0:1:aC]", "[a:b:c:de:ff:0:1:ac]", Component(0,20), CanonHostInfo::IPV6, -1, "000A000B000C00DE00FF0000000100AC"},
 764
 765     // The same address can be written with different contractions, but should
 766     // get canonicalized to the same thing.
 767     {"[1:0:0:2::3:0]", L"[1:0:0:2::3:0]", "[1::2:0:0:3:0]", Component(0,14), CanonHostInfo::IPV6, -1, "00010000000000020000000000030000"},
 768     {"[1::2:0:0:3:0]", L"[1::2:0:0:3:0]", "[1::2:0:0:3:0]", Component(0,14), CanonHostInfo::IPV6, -1, "00010000000000020000000000030000"},
 769
 770     // Addresses with embedded IPv4.
 771     {"[::192.168.0.1]", L"[::192.168.0.1]", "[::c0a8:1]", Component(0,10), CanonHostInfo::IPV6, -1, "000000000000000000000000C0A80001"},
 772     {"[::ffff:192.168.0.1]", L"[::ffff:192.168.0.1]", "[::ffff:c0a8:1]", Component(0,15), CanonHostInfo::IPV6, -1, "00000000000000000000FFFFC0A80001"},
 773     {"[::eeee:192.168.0.1]", L"[::eeee:192.168.0.1]", "[::eeee:c0a8:1]", Component(0, 15), CanonHostInfo::IPV6, -1, "00000000000000000000EEEEC0A80001"},
 774     {"[2001::192.168.0.1]", L"[2001::192.168.0.1]", "[2001::c0a8:1]", Component(0, 14), CanonHostInfo::IPV6, -1, "200100000000000000000000C0A80001"},
 775     {"[1:2:192.168.0.1:5:6]", L"[1:2:192.168.0.1:5:6]", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 776
 777     // IPv4 with last component missing.
 778     {"[::ffff:192.1.2]", L"[::ffff:192.1.2]", "[::ffff:c001:2]", Component(0,15), CanonHostInfo::IPV6, -1, "00000000000000000000FFFFC0010002"},
 779
 780     // IPv4 using hex.
 781     // TODO(eroman): Should this format be disallowed?
 782     {"[::ffff:0xC0.0Xa8.0x0.0x1]", L"[::ffff:0xC0.0Xa8.0x0.0x1]", "[::ffff:c0a8:1]", Component(0,15), CanonHostInfo::IPV6, -1, "00000000000000000000FFFFC0A80001"},
 783
 784     // There may be zeros surrounding the "::" contraction.
 785     {"[0:0::0:0:8]", L"[0:0::0:0:8]", "[::8]", Component(0,5), CanonHostInfo::IPV6, -1, "00000000000000000000000000000008"},
 786
 787     {"[2001:db8::1]", L"[2001:db8::1]", "[2001:db8::1]", Component(0,13), CanonHostInfo::IPV6, -1, "20010DB8000000000000000000000001"},
 788
 789     // Can only have one "::" contraction in an IPv6 string literal.
 790     {"[2001::db8::1]", L"[2001::db8::1]", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 791     // No more than 2 consecutive ':'s.
 792     {"[2001:db8:::1]", L"[2001:db8:::1]", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 793     {"[:::]", L"[:::]", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 794     // Non-IP addresses due to invalid characters.
 795     {"[2001::.com]", L"[2001::.com]", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 796     // If there are not enough components, the last one should fill them out.
 797     // ... omitted at this time ...
 798     // Too many components means not an IP address. Similarly, with too few
 799     // if using IPv4 compat or mapped addresses.
 800     {"[::192.168.0.0.1]", L"[::192.168.0.0.1]", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 801     {"[::ffff:192.168.0.0.1]", L"[::ffff:192.168.0.0.1]", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 802     {"[1:2:3:4:5:6:7:8:9]", L"[1:2:3:4:5:6:7:8:9]", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 803     // Too many bits (even though 8 comonents, the last one holds 32 bits).
 804     {"[0:0:0:0:0:0:0:192.168.0.1]", L"[0:0:0:0:0:0:0:192.168.0.1]", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 805
 806     // Too many bits specified -- the contraction would have to be zero-length
 807     // to not exceed 128 bits.
 808     {"[1:2:3:4:5:6::192.168.0.1]", L"[1:2:3:4:5:6::192.168.0.1]", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 809
 810     // The contraction is for 16 bits of zero.
 811     {"[1:2:3:4:5:6::8]", L"[1:2:3:4:5:6::8]", "[1:2:3:4:5:6:0:8]", Component(0,17), CanonHostInfo::IPV6, -1, "00010002000300040005000600000008"},
 812
 813     // Cannot have a trailing colon.
 814     {"[1:2:3:4:5:6:7:8:]", L"[1:2:3:4:5:6:7:8:]", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 815     {"[1:2:3:4:5:6:192.168.0.1:]", L"[1:2:3:4:5:6:192.168.0.1:]", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 816
 817     // Cannot have negative numbers.
 818     {"[-1:2:3:4:5:6:7:8]", L"[-1:2:3:4:5:6:7:8]", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 819
 820     // Scope ID -- the URL may contain an optional ["%" <scope_id>] section.
 821     // The scope_id should be included in the canonicalized URL, and is an
 822     // unsigned decimal number.
 823
 824     // Invalid because no ID was given after the percent.
 825
 826     // Don't allow scope-id
 827     {"[1::%1]", L"[1::%1]", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 828     {"[1::%eth0]", L"[1::%eth0]", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 829     {"[1::%]", L"[1::%]", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 830     {"[%]", L"[%]", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 831     {"[::%:]", L"[::%:]", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 832
 833     // Don't allow leading or trailing colons.
 834     {"[:0:0::0:0:8]", L"[:0:0::0:0:8]", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 835     {"[0:0::0:0:8:]", L"[0:0::0:0:8:]", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 836     {"[:0:0::0:0:8:]", L"[:0:0::0:0:8:]", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 837
 838       // We allow a single trailing dot.
 839     // ... omitted at this time ...
 840       // Two dots in a row means not an IP address.
 841     {"[::192.168..1]", L"[::192.168..1]", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 842       // Any non-first components get truncated to one byte.
 843     // ... omitted at this time ...
 844       // Spaces should be rejected.
 845     {"[::1 hello]", L"[::1 hello]", "", Component(), CanonHostInfo::BROKEN, -1, ""},
 846   };
 847
 848   for (size_t i = 0; i < arraysize(cases); i++) {
 849     // 8-bit version.
 850     Component component(0, static_cast<int>(strlen(cases[i].input8)));
 851
 852     std::string out_str1;
 853     StdStringCanonOutput output1(&out_str1);
 854     CanonHostInfo host_info;
 855     CanonicalizeIPAddress(cases[i].input8, component, &output1, &host_info);
 856     output1.Complete();
 857
 858     EXPECT_EQ(cases[i].expected_family, host_info.family);
 859     EXPECT_EQ(std::string(cases[i].expected_address_hex),
 860               BytesToHexString(host_info.address, host_info.AddressLength())) << "iter " << i << " host " << cases[i].input8;
 861     if (host_info.family == CanonHostInfo::IPV6) {
 862       EXPECT_STREQ(cases[i].expected, out_str1.c_str());
 863       EXPECT_EQ(cases[i].expected_component.begin,
 864                 host_info.out_host.begin);
 865       EXPECT_EQ(cases[i].expected_component.len, host_info.out_host.len);
 866     }
 867
 868     // 16-bit version.
 869     base::string16 input16(
 870         test_utils::TruncateWStringToUTF16(cases[i].input16));
 871     component = Component(0, static_cast<int>(input16.length()));
 872
 873     std::string out_str2;
 874     StdStringCanonOutput output2(&out_str2);
 875     CanonicalizeIPAddress(input16.c_str(), component, &output2, &host_info);
 876     output2.Complete();
 877
 878     EXPECT_EQ(cases[i].expected_family, host_info.family);
 879     EXPECT_EQ(std::string(cases[i].expected_address_hex),
 880               BytesToHexString(host_info.address, host_info.AddressLength()));
 881     if (host_info.family == CanonHostInfo::IPV6) {
 882       EXPECT_STREQ(cases[i].expected, out_str2.c_str());
 883       EXPECT_EQ(cases[i].expected_component.begin, host_info.out_host.begin);
 884       EXPECT_EQ(cases[i].expected_component.len, host_info.out_host.len);
 885     }
 886   }
 887 }
 888
 889 TEST(URLCanonTest, IPEmpty) {
 890   std::string out_str1;
 891   StdStringCanonOutput output1(&out_str1);
 892   CanonHostInfo host_info;
 893
 894   // This tests tests.
 895   const char spec[] = "192.168.0.1";
 896   CanonicalizeIPAddress(spec, Component(), &output1, &host_info);
 897   EXPECT_FALSE(host_info.IsIPAddress());
 898
 899   CanonicalizeIPAddress(spec, Component(0, 0), &output1, &host_info);
 900   EXPECT_FALSE(host_info.IsIPAddress());
 901 }
 902
 903 // Verifies that CanonicalizeHostSubstring produces the expected output and
 904 // does not "fix" IP addresses. Because this code is a subset of
 905 // CanonicalizeHost, the shared functionality is not tested.
 906 TEST(URLCanonTest, CanonicalizeHostSubstring) {
 907   // Basic sanity check.
 908   {
 909     std::string out_str;
 910     StdStringCanonOutput output(&out_str);
 911     EXPECT_TRUE(CanonicalizeHostSubstring("M\xc3\x9cNCHEN.com",
 912                                           Component(0, 12), &output));
 913     output.Complete();
 914     EXPECT_EQ("xn--mnchen-3ya.com", out_str);
 915   }
 916
 917   // Failure case.
 918   {
 919     std::string out_str;
 920     StdStringCanonOutput output(&out_str);
 921     EXPECT_FALSE(CanonicalizeHostSubstring(
 922         test_utils::TruncateWStringToUTF16(L"\xfdd0zyx.com").c_str(),
 923         Component(0, 8), &output));
 924     output.Complete();
 925     EXPECT_EQ("%EF%BF%BDzyx.com", out_str);
 926   }
 927
 928   // Should return true for empty input strings.
 929   {
 930     std::string out_str;
 931     StdStringCanonOutput output(&out_str);
 932     EXPECT_TRUE(CanonicalizeHostSubstring("", Component(0, 0), &output));
 933     output.Complete();
 934     EXPECT_EQ(std::string(), out_str);
 935   }
 936
 937   // Numbers that look like IP addresses should not be changed.
 938   {
 939     std::string out_str;
 940     StdStringCanonOutput output(&out_str);
 941     EXPECT_TRUE(
 942         CanonicalizeHostSubstring("01.02.03.04", Component(0, 11), &output));
 943     output.Complete();
 944     EXPECT_EQ("01.02.03.04", out_str);
 945   }
 946 }
 947
 948 TEST(URLCanonTest, UserInfo) {
 949   // Note that the canonicalizer should escape and treat empty components as
 950   // not being there.
 951
 952   // We actually parse a full input URL so we can get the initial components.
 953   struct UserComponentCase {
 954     const char* input;
 955     const char* expected;
 956     Component expected_username;
 957     Component expected_password;
 958     bool expected_success;
 959   } user_info_cases[] = {
 960     {"http://user:pass@host.com/", "user:pass@", Component(0, 4), Component(5, 4), true},
 961     {"http://@host.com/", "", Component(0, -1), Component(0, -1), true},
 962     {"http://:@host.com/", "", Component(0, -1), Component(0, -1), true},
 963     {"http://foo:@host.com/", "foo@", Component(0, 3), Component(0, -1), true},
 964     {"http://:foo@host.com/", ":foo@", Component(0, 0), Component(1, 3), true},
 965     {"http://^ :$\t@host.com/", "%5E%20:$%09@", Component(0, 6), Component(7, 4), true},
 966     {"http://user:pass@/", "user:pass@", Component(0, 4), Component(5, 4), true},
 967     {"http://%2540:bar@domain.com/", "%2540:bar@", Component(0, 5), Component(6, 3), true },
 968
 969       // IE7 compatibility: old versions allowed backslashes in usernames, but
 970       // IE7 does not. We disallow it as well.
 971     {"ftp://me\\mydomain:pass@foo.com/", "", Component(0, -1), Component(0, -1), true},
 972   };
 973
 974   for (size_t i = 0; i < arraysize(user_info_cases); i++) {
 975     int url_len = static_cast<int>(strlen(user_info_cases[i].input));
 976     Parsed parsed;
 977     ParseStandardURL(user_info_cases[i].input, url_len, &parsed);
 978     Component out_user, out_pass;
 979     std::string out_str;
 980     StdStringCanonOutput output1(&out_str);
 981
 982     bool success = CanonicalizeUserInfo(user_info_cases[i].input,
 983                                         parsed.username,
 984                                         user_info_cases[i].input,
 985                                         parsed.password,
 986                                         &output1,
 987                                         &out_user,
 988                                         &out_pass);
 989     output1.Complete();
 990
 991     EXPECT_EQ(user_info_cases[i].expected_success, success);
 992     EXPECT_EQ(std::string(user_info_cases[i].expected), out_str);
 993     EXPECT_EQ(user_info_cases[i].expected_username.begin, out_user.begin);
 994     EXPECT_EQ(user_info_cases[i].expected_username.len, out_user.len);
 995     EXPECT_EQ(user_info_cases[i].expected_password.begin, out_pass.begin);
 996     EXPECT_EQ(user_info_cases[i].expected_password.len, out_pass.len);
 997
 998     // Now try the wide version
 999     out_str.clear();
1000     StdStringCanonOutput output2(&out_str);
1001     base::string16 wide_input(base::UTF8ToUTF16(user_info_cases[i].input));
1002     success = CanonicalizeUserInfo(wide_input.c_str(),
1003                                    parsed.username,
1004                                    wide_input.c_str(),
1005                                    parsed.password,
1006                                    &output2,
1007                                    &out_user,
1008                                    &out_pass);
1009     output2.Complete();
1010
1011     EXPECT_EQ(user_info_cases[i].expected_success, success);
1012     EXPECT_EQ(std::string(user_info_cases[i].expected), out_str);
1013     EXPECT_EQ(user_info_cases[i].expected_username.begin, out_user.begin);
1014     EXPECT_EQ(user_info_cases[i].expected_username.len, out_user.len);
1015     EXPECT_EQ(user_info_cases[i].expected_password.begin, out_pass.begin);
1016     EXPECT_EQ(user_info_cases[i].expected_password.len, out_pass.len);
1017   }
1018 }
1019
1020 TEST(URLCanonTest, Port) {
1021   // We only need to test that the number gets properly put into the output
1022   // buffer. The parser unit tests will test scanning the number correctly.
1023   //
1024   // Note that the CanonicalizePort will always prepend a colon to the output
1025   // to separate it from the colon that it assumes precedes it.
1026   struct PortCase {
1027     const char* input;
1028     int default_port;
1029     const char* expected;
1030     Component expected_component;
1031     bool expected_success;
1032   } port_cases[] = {
1033       // Invalid input should be copied w/ failure.
1034     {"as df", 80, ":as%20df", Component(1, 7), false},
1035     {"-2", 80, ":-2", Component(1, 2), false},
1036       // Default port should be omitted.
1037     {"80", 80, "", Component(0, -1), true},
1038     {"8080", 80, ":8080", Component(1, 4), true},
1039       // PORT_UNSPECIFIED should mean always keep the port.
1040     {"80", PORT_UNSPECIFIED, ":80", Component(1, 2), true},
1041   };
1042
1043   for (size_t i = 0; i < arraysize(port_cases); i++) {
1044     int url_len = static_cast<int>(strlen(port_cases[i].input));
1045     Component in_comp(0, url_len);
1046     Component out_comp;
1047     std::string out_str;
1048     StdStringCanonOutput output1(&out_str);
1049     bool success = CanonicalizePort(port_cases[i].input,
1050                                     in_comp,
1051                                     port_cases[i].default_port,
1052                                     &output1,
1053                                     &out_comp);
1054     output1.Complete();
1055
1056     EXPECT_EQ(port_cases[i].expected_success, success);
1057     EXPECT_EQ(std::string(port_cases[i].expected), out_str);
1058     EXPECT_EQ(port_cases[i].expected_component.begin, out_comp.begin);
1059     EXPECT_EQ(port_cases[i].expected_component.len, out_comp.len);
1060
1061     // Now try the wide version
1062     out_str.clear();
1063     StdStringCanonOutput output2(&out_str);
1064     base::string16 wide_input(base::UTF8ToUTF16(port_cases[i].input));
1065     success = CanonicalizePort(wide_input.c_str(),
1066                                in_comp,
1067                                port_cases[i].default_port,
1068                                &output2,
1069                                &out_comp);
1070     output2.Complete();
1071
1072     EXPECT_EQ(port_cases[i].expected_success, success);
1073     EXPECT_EQ(std::string(port_cases[i].expected), out_str);
1074     EXPECT_EQ(port_cases[i].expected_component.begin, out_comp.begin);
1075     EXPECT_EQ(port_cases[i].expected_component.len, out_comp.len);
1076   }
1077 }
1078
1079 TEST(URLCanonTest, Path) {
1080   DualComponentCase path_cases[] = {
1081     // ----- path collapsing tests -----
1082     {"/././foo", L"/././foo", "/foo", Component(0, 4), true},
1083     {"/./.foo", L"/./.foo", "/.foo", Component(0, 5), true},
1084     {"/foo/.", L"/foo/.", "/foo/", Component(0, 5), true},
1085     {"/foo/./", L"/foo/./", "/foo/", Component(0, 5), true},
1086       // double dots followed by a slash or the end of the string count
1087     {"/foo/bar/..", L"/foo/bar/..", "/foo/", Component(0, 5), true},
1088     {"/foo/bar/../", L"/foo/bar/../", "/foo/", Component(0, 5), true},
1089       // don't count double dots when they aren't followed by a slash
1090     {"/foo/..bar", L"/foo/..bar", "/foo/..bar", Component(0, 10), true},
1091       // some in the middle
1092     {"/foo/bar/../ton", L"/foo/bar/../ton", "/foo/ton", Component(0, 8), true},
1093     {"/foo/bar/../ton/../../a", L"/foo/bar/../ton/../../a", "/a", Component(0, 2), true},
1094       // we should not be able to go above the root
1095     {"/foo/../../..", L"/foo/../../..", "/", Component(0, 1), true},
1096     {"/foo/../../../ton", L"/foo/../../../ton", "/ton", Component(0, 4), true},
1097       // escaped dots should be unescaped and treated the same as dots
1098     {"/foo/%2e", L"/foo/%2e", "/foo/", Component(0, 5), true},
1099     {"/foo/%2e%2", L"/foo/%2e%2", "/foo/.%2", Component(0, 8), true},
1100     {"/foo/%2e./%2e%2e/.%2e/%2e.bar", L"/foo/%2e./%2e%2e/.%2e/%2e.bar", "/..bar", Component(0, 6), true},
1101       // Multiple slashes in a row should be preserved and treated like empty
1102       // directory names.
1103     {"////../..", L"////../..", "//", Component(0, 2), true},
1104
1105     // ----- escaping tests -----
1106     {"/foo", L"/foo", "/foo", Component(0, 4), true},
1107       // Valid escape sequence
1108     {"/%20foo", L"/%20foo", "/%20foo", Component(0, 7), true},
1109       // Invalid escape sequence we should pass through unchanged.
1110     {"/foo%", L"/foo%", "/foo%", Component(0, 5), true},
1111     {"/foo%2", L"/foo%2", "/foo%2", Component(0, 6), true},
1112       // Invalid escape sequence: bad characters should be treated the same as
1113       // the sourrounding text, not as escaped (in this case, UTF-8).
1114     {"/foo%2zbar", L"/foo%2zbar", "/foo%2zbar", Component(0, 10), true},
1115     {"/foo%2\xc2\xa9zbar", NULL, "/foo%2%C2%A9zbar", Component(0, 16), true},
1116     {NULL, L"/foo%2\xc2\xa9zbar", "/foo%2%C3%82%C2%A9zbar", Component(0, 22), true},
1117       // Regular characters that are escaped should be unescaped
1118     {"/foo%41%7a", L"/foo%41%7a", "/fooAz", Component(0, 6), true},
1119       // Funny characters that are unescaped should be escaped
1120     {"/foo\x09\x91%91", NULL, "/foo%09%91%91", Component(0, 13), true},
1121     {NULL, L"/foo\x09\x91%91", "/foo%09%C2%91%91", Component(0, 16), true},
1122       // Invalid characters that are escaped should cause a failure.
1123     {"/foo%00%51", L"/foo%00%51", "/foo%00Q", Component(0, 8), false},
1124       // Some characters should be passed through unchanged regardless of esc.
1125     {"/(%28:%3A%29)", L"/(%28:%3A%29)", "/(%28:%3A%29)", Component(0, 13), true},
1126       // Characters that are properly escaped should not have the case changed
1127       // of hex letters.
1128     {"/%3A%3a%3C%3c", L"/%3A%3a%3C%3c", "/%3A%3a%3C%3c", Component(0, 13), true},
1129       // Funny characters that are unescaped should be escaped
1130     {"/foo\tbar", L"/foo\tbar", "/foo%09bar", Component(0, 10), true},
1131       // Backslashes should get converted to forward slashes
1132     {"\\foo\\bar", L"\\foo\\bar", "/foo/bar", Component(0, 8), true},
1133       // Hashes found in paths (possibly only when the caller explicitly sets
1134       // the path on an already-parsed URL) should be escaped.
1135     {"/foo#bar", L"/foo#bar", "/foo%23bar", Component(0, 10), true},
1136       // %7f should be allowed and %3D should not be unescaped (these were wrong
1137       // in a previous version).
1138     {"/%7Ffp3%3Eju%3Dduvgw%3Dd", L"/%7Ffp3%3Eju%3Dduvgw%3Dd", "/%7Ffp3%3Eju%3Dduvgw%3Dd", Component(0, 24), true},
1139       // @ should be passed through unchanged (escaped or unescaped).
1140     {"/@asdf%40", L"/@asdf%40", "/@asdf%40", Component(0, 9), true},
1141       // Nested escape sequences should result in escaping the leading '%' if
1142       // unescaping would result in a new escape sequence.
1143     {"/%A%42", L"/%A%42", "/%25AB", Component(0, 6), true},
1144     {"/%%41B", L"/%%41B", "/%25AB", Component(0, 6), true},
1145     {"/%%41%42", L"/%%41%42", "/%25AB", Component(0, 6), true},
1146       // Make sure truncated "nested" escapes don't result in reading off the
1147       // string end.
1148     {"/%%41", L"/%%41", "/%A", Component(0, 3), true},
1149       // Don't unescape the leading '%' if unescaping doesn't result in a valid
1150       // new escape sequence.
1151     {"/%%470", L"/%%470", "/%G0", Component(0, 4), true},
1152     {"/%%2D%41", L"/%%2D%41", "/%-A", Component(0, 4), true},
1153       // Don't erroneously downcast a UTF-16 charater in a way that makes it
1154       // look like part of an escape sequence.
1155     {NULL, L"/%%41\x0130", "/%A%C4%B0", Component(0, 9), true},
1156
1157     // ----- encoding tests -----
1158       // Basic conversions
1159     {"/\xe4\xbd\xa0\xe5\xa5\xbd\xe4\xbd\xa0\xe5\xa5\xbd", L"/\x4f60\x597d\x4f60\x597d", "/%E4%BD%A0%E5%A5%BD%E4%BD%A0%E5%A5%BD", Component(0, 37), true},
1160       // Invalid unicode characters should fail. We only do validation on
1161       // UTF-16 input, so this doesn't happen on 8-bit.
1162     {"/\xef\xb7\x90zyx", NULL, "/%EF%B7%90zyx", Component(0, 13), true},
1163     {NULL, L"/\xfdd0zyx", "/%EF%BF%BDzyx", Component(0, 13), false},
1164   };
1165
1166   for (size_t i = 0; i < arraysize(path_cases); i++) {
1167     if (path_cases[i].input8) {
1168       int len = static_cast<int>(strlen(path_cases[i].input8));
1169       Component in_comp(0, len);
1170       Component out_comp;
1171       std::string out_str;
1172       StdStringCanonOutput output(&out_str);
1173       bool success =
1174           CanonicalizePath(path_cases[i].input8, in_comp, &output, &out_comp);
1175       output.Complete();
1176
1177       EXPECT_EQ(path_cases[i].expected_success, success);
1178       EXPECT_EQ(path_cases[i].expected_component.begin, out_comp.begin);
1179       EXPECT_EQ(path_cases[i].expected_component.len, out_comp.len);
1180       EXPECT_EQ(path_cases[i].expected, out_str);
1181     }
1182
1183     if (path_cases[i].input16) {
1184       base::string16 input16(
1185           test_utils::TruncateWStringToUTF16(path_cases[i].input16));
1186       int len = static_cast<int>(input16.length());
1187       Component in_comp(0, len);
1188       Component out_comp;
1189       std::string out_str;
1190       StdStringCanonOutput output(&out_str);
1191
1192       bool success =
1193           CanonicalizePath(input16.c_str(), in_comp, &output, &out_comp);
1194       output.Complete();
1195
1196       EXPECT_EQ(path_cases[i].expected_success, success);
1197       EXPECT_EQ(path_cases[i].expected_component.begin, out_comp.begin);
1198       EXPECT_EQ(path_cases[i].expected_component.len, out_comp.len);
1199       EXPECT_EQ(path_cases[i].expected, out_str);
1200     }
1201   }
1202
1203   // Manual test: embedded NULLs should be escaped and the URL should be marked
1204   // as invalid.
1205   const char path_with_null[] = "/ab\0c";
1206   Component in_comp(0, 5);
1207   Component out_comp;
1208
1209   std::string out_str;
1210   StdStringCanonOutput output(&out_str);
1211   bool success = CanonicalizePath(path_with_null, in_comp, &output, &out_comp);
1212   output.Complete();
1213   EXPECT_FALSE(success);
1214   EXPECT_EQ("/ab%00c", out_str);
1215 }
1216
1217 TEST(URLCanonTest, Query) {
1218   struct QueryCase {
1219     const char* input8;
1220     const wchar_t* input16;
1221     const char* expected;
1222   } query_cases[] = {
1223       // Regular ASCII case.
1224     {"foo=bar", L"foo=bar", "?foo=bar"},
1225       // Allow question marks in the query without escaping
1226     {"as?df", L"as?df", "?as?df"},
1227       // Always escape '#' since it would mark the ref.
1228     {"as#df", L"as#df", "?as%23df"},
1229       // Escape some questionable 8-bit characters, but never unescape.
1230     {"\x02hello\x7f bye", L"\x02hello\x7f bye", "?%02hello%7F%20bye"},
1231     {"%40%41123", L"%40%41123", "?%40%41123"},
1232       // Chinese input/output
1233     {"q=\xe4\xbd\xa0\xe5\xa5\xbd", L"q=\x4f60\x597d", "?q=%E4%BD%A0%E5%A5%BD"},
1234       // Invalid UTF-8/16 input should be replaced with invalid characters.
1235     {"q=\xed\xed", L"q=\xd800\xd800", "?q=%EF%BF%BD%EF%BF%BD"},
1236       // Don't allow < or > because sometimes they are used for XSS if the
1237       // URL is echoed in content. Firefox does this, IE doesn't.
1238     {"q=<asdf>", L"q=<asdf>", "?q=%3Casdf%3E"},
1239       // Escape double quotemarks in the query.
1240     {"q=\"asdf\"", L"q=\"asdf\"", "?q=%22asdf%22"},
1241   };
1242
1243   for (size_t i = 0; i < arraysize(query_cases); i++) {
1244     Component out_comp;
1245
1246     if (query_cases[i].input8) {
1247       int len = static_cast<int>(strlen(query_cases[i].input8));
1248       Component in_comp(0, len);
1249       std::string out_str;
1250
1251       StdStringCanonOutput output(&out_str);
1252       CanonicalizeQuery(query_cases[i].input8, in_comp, NULL, &output,
1253                         &out_comp);
1254       output.Complete();
1255
1256       EXPECT_EQ(query_cases[i].expected, out_str);
1257     }
1258
1259     if (query_cases[i].input16) {
1260       base::string16 input16(
1261           test_utils::TruncateWStringToUTF16(query_cases[i].input16));
1262       int len = static_cast<int>(input16.length());
1263       Component in_comp(0, len);
1264       std::string out_str;
1265
1266       StdStringCanonOutput output(&out_str);
1267       CanonicalizeQuery(input16.c_str(), in_comp, NULL, &output, &out_comp);
1268       output.Complete();
1269
1270       EXPECT_EQ(query_cases[i].expected, out_str);
1271     }
1272   }
1273
1274   // Extra test for input with embedded NULL;
1275   std::string out_str;
1276   StdStringCanonOutput output(&out_str);
1277   Component out_comp;
1278   CanonicalizeQuery("a \x00z\x01", Component(0, 5), NULL, &output, &out_comp);
1279   output.Complete();
1280   EXPECT_EQ("?a%20%00z%01", out_str);
1281 }
1282
1283 TEST(URLCanonTest, Ref) {
1284   // Refs are trivial, it just checks the encoding.
1285   DualComponentCase ref_cases[] = {
1286       {"hello!", L"hello!", "#hello!", Component(1, 6), true},
1287       // We should escape spaces, double-quotes, angled braces, and backtics.
1288       {"hello, world", L"hello, world", "#hello,%20world", Component(1, 14),
1289        true},
1290       {"hello,\"world", L"hello,\"world", "#hello,%22world", Component(1, 14),
1291        true},
1292       {"hello,<world", L"hello,<world", "#hello,%3Cworld", Component(1, 14),
1293        true},
1294       {"hello,>world", L"hello,>world", "#hello,%3Eworld", Component(1, 14),
1295        true},
1296       {"hello,`world", L"hello,`world", "#hello,%60world", Component(1, 14),
1297        true},
1298       // UTF-8/wide input should be preserved
1299       {"\xc2\xa9", L"\xa9", "#%C2%A9", Component(1, 6), true},
1300       // Test a characer that takes > 16 bits (U+10300 = old italic letter A)
1301       {"\xF0\x90\x8C\x80ss", L"\xd800\xdf00ss", "#%F0%90%8C%80ss",
1302        Component(1, 14), true},
1303       // Escaping should be preserved unchanged, even invalid ones
1304       {"%41%a", L"%41%a", "#%41%a", Component(1, 5), true},
1305       // Invalid UTF-8/16 input should be flagged and the input made valid
1306       {"\xc2", NULL, "#%EF%BF%BD", Component(1, 9), true},
1307       {NULL, L"\xd800\x597d", "#%EF%BF%BD%E5%A5%BD", Component(1, 18), true},
1308       // Test a Unicode invalid character.
1309       {"a\xef\xb7\x90", L"a\xfdd0", "#a%EF%BF%BD", Component(1, 10), true},
1310       // Refs can have # signs and we should preserve them.
1311       {"asdf#qwer", L"asdf#qwer", "#asdf#qwer", Component(1, 9), true},
1312       {"#asdf", L"#asdf", "##asdf", Component(1, 5), true},
1313   };
1314
1315   for (size_t i = 0; i < arraysize(ref_cases); i++) {
1316     // 8-bit input
1317     if (ref_cases[i].input8) {
1318       int len = static_cast<int>(strlen(ref_cases[i].input8));
1319       Component in_comp(0, len);
1320       Component out_comp;
1321
1322       std::string out_str;
1323       StdStringCanonOutput output(&out_str);
1324       CanonicalizeRef(ref_cases[i].input8, in_comp, &output, &out_comp);
1325       output.Complete();
1326
1327       EXPECT_EQ(ref_cases[i].expected_component.begin, out_comp.begin);
1328       EXPECT_EQ(ref_cases[i].expected_component.len, out_comp.len);
1329       EXPECT_EQ(ref_cases[i].expected, out_str);
1330     }
1331
1332     // 16-bit input
1333     if (ref_cases[i].input16) {
1334       base::string16 input16(
1335           test_utils::TruncateWStringToUTF16(ref_cases[i].input16));
1336       int len = static_cast<int>(input16.length());
1337       Component in_comp(0, len);
1338       Component out_comp;
1339
1340       std::string out_str;
1341       StdStringCanonOutput output(&out_str);
1342       CanonicalizeRef(input16.c_str(), in_comp, &output, &out_comp);
1343       output.Complete();
1344
1345       EXPECT_EQ(ref_cases[i].expected_component.begin, out_comp.begin);
1346       EXPECT_EQ(ref_cases[i].expected_component.len, out_comp.len);
1347       EXPECT_EQ(ref_cases[i].expected, out_str);
1348     }
1349   }
1350
1351   // Try one with an embedded NULL. It should be stripped.
1352   const char null_input[5] = "ab\x00z";
1353   Component null_input_component(0, 4);
1354   Component out_comp;
1355
1356   std::string out_str;
1357   StdStringCanonOutput output(&out_str);
1358   CanonicalizeRef(null_input, null_input_component, &output, &out_comp);
1359   output.Complete();
1360
1361   EXPECT_EQ(1, out_comp.begin);
1362   EXPECT_EQ(3, out_comp.len);
1363   EXPECT_EQ("#abz", out_str);
1364 }
1365
1366 TEST(URLCanonTest, CanonicalizeStandardURL) {
1367   // The individual component canonicalize tests should have caught the cases
1368   // for each of those components. Here, we just need to test that the various
1369   // parts are included or excluded properly, and have the correct separators.
1370   struct URLCase {
1371     const char* input;
1372     const char* expected;
1373     bool expected_success;
1374   } cases[] = {
1375       {"http://www.google.com/foo?bar=baz#",
1376        "http://www.google.com/foo?bar=baz#", true},
1377       {"http://[www.google.com]/", "http://[www.google.com]/", false},
1378       {"ht\ttp:@www.google.com:80/;p?#", "ht%09tp://www.google.com:80/;p?#",
1379        false},
1380       {"http:////////user:@google.com:99?foo", "http://user@google.com:99/?foo",
1381        true},
1382       {"www.google.com", ":www.google.com/", false},
1383       {"http://192.0x00A80001", "http://192.168.0.1/", true},
1384       {"http://www/foo%2Ehtml", "http://www/foo.html", true},
1385       {"http://user:pass@/", "http://user:pass@/", false},
1386       {"http://%25DOMAIN:foobar@foodomain.com/",
1387        "http://%25DOMAIN:foobar@foodomain.com/", true},
1388
1389       // Backslashes should get converted to forward slashes.
1390       {"http:\\\\www.google.com\\foo", "http://www.google.com/foo", true},
1391
1392       // Busted refs shouldn't make the whole thing fail.
1393       {"http://www.google.com/asdf#\xc2",
1394        "http://www.google.com/asdf#%EF%BF%BD", true},
1395
1396       // Basic port tests.
1397       {"http://foo:80/", "http://foo/", true},
1398       {"http://foo:81/", "http://foo:81/", true},
1399       {"httpa://foo:80/", "httpa://foo:80/", true},
1400       {"http://foo:-80/", "http://foo:-80/", false},
1401
1402       {"https://foo:443/", "https://foo/", true},
1403       {"https://foo:80/", "https://foo:80/", true},
1404       {"ftp://foo:21/", "ftp://foo/", true},
1405       {"ftp://foo:80/", "ftp://foo:80/", true},
1406       {"gopher://foo:70/", "gopher://foo/", true},
1407       {"gopher://foo:443/", "gopher://foo:443/", true},
1408       {"ws://foo:80/", "ws://foo/", true},
1409       {"ws://foo:81/", "ws://foo:81/", true},
1410       {"ws://foo:443/", "ws://foo:443/", true},
1411       {"ws://foo:815/", "ws://foo:815/", true},
1412       {"wss://foo:80/", "wss://foo:80/", true},
1413       {"wss://foo:81/", "wss://foo:81/", true},
1414       {"wss://foo:443/", "wss://foo/", true},
1415       {"wss://foo:815/", "wss://foo:815/", true},
1416
1417       // This particular code path ends up "backing up" to replace an invalid
1418       // host ICU generated with an escaped version. Test that in the context
1419       // of a full URL to make sure the backing up doesn't mess up the non-host
1420       // parts of the URL. "EF B9 AA" is U+FE6A which is a type of percent that
1421       // ICU will convert to an ASCII one, generating "%81".
1422       {"ws:)W\x1eW\xef\xb9\xaa"
1423        "81:80/",
1424        "ws://%29w%1ew%81/", false},
1425   };
1426
1427   for (size_t i = 0; i < arraysize(cases); i++) {
1428     int url_len = static_cast<int>(strlen(cases[i].input));
1429     Parsed parsed;
1430     ParseStandardURL(cases[i].input, url_len, &parsed);
1431
1432     Parsed out_parsed;
1433     std::string out_str;
1434     StdStringCanonOutput output(&out_str);
1435     bool success = CanonicalizeStandardURL(
1436         cases[i].input, url_len, parsed,
1437         SCHEME_WITH_HOST_PORT_AND_USER_INFORMATION, NULL, &output, &out_parsed);
1438     output.Complete();
1439
1440     EXPECT_EQ(cases[i].expected_success, success);
1441     EXPECT_EQ(cases[i].expected, out_str);
1442   }
1443 }
1444
1445 // The codepath here is the same as for regular canonicalization, so we just
1446 // need to test that things are replaced or not correctly.
1447 TEST(URLCanonTest, ReplaceStandardURL) {
1448   ReplaceCase replace_cases[] = {
1449       // Common case of truncating the path.
1450     {"http://www.google.com/foo?bar=baz#ref", NULL, NULL, NULL, NULL, NULL, "/", kDeleteComp, kDeleteComp, "http://www.google.com/"},
1451       // Replace everything
1452     {"http://a:b@google.com:22/foo;bar?baz@cat", "https", "me", "pw", "host.com", "99", "/path", "query", "ref", "https://me:pw@host.com:99/path?query#ref"},
1453       // Replace nothing
1454     {"http://a:b@google.com:22/foo?baz@cat", NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, "http://a:b@google.com:22/foo?baz@cat"},
1455       // Replace scheme with filesystem. The result is garbage, but you asked
1456       // for it.
1457     {"http://a:b@google.com:22/foo?baz@cat", "filesystem", NULL, NULL, NULL, NULL, NULL, NULL, NULL, "filesystem://a:b@google.com:22/foo?baz@cat"},
1458   };
1459
1460   for (size_t i = 0; i < arraysize(replace_cases); i++) {
1461     const ReplaceCase& cur = replace_cases[i];
1462     int base_len = static_cast<int>(strlen(cur.base));
1463     Parsed parsed;
1464     ParseStandardURL(cur.base, base_len, &parsed);
1465
1466     Replacements<char> r;
1467     typedef Replacements<char> R;  // Clean up syntax.
1468
1469     // Note that for the scheme we pass in a different clear function since
1470     // there is no function to clear the scheme.
1471     SetupReplComp(&R::SetScheme, &R::ClearRef, &r, cur.scheme);
1472     SetupReplComp(&R::SetUsername, &R::ClearUsername, &r, cur.username);
1473     SetupReplComp(&R::SetPassword, &R::ClearPassword, &r, cur.password);
1474     SetupReplComp(&R::SetHost, &R::ClearHost, &r, cur.host);
1475     SetupReplComp(&R::SetPort, &R::ClearPort, &r, cur.port);
1476     SetupReplComp(&R::SetPath, &R::ClearPath, &r, cur.path);
1477     SetupReplComp(&R::SetQuery, &R::ClearQuery, &r, cur.query);
1478     SetupReplComp(&R::SetRef, &R::ClearRef, &r, cur.ref);
1479
1480     std::string out_str;
1481     StdStringCanonOutput output(&out_str);
1482     Parsed out_parsed;
1483     ReplaceStandardURL(replace_cases[i].base, parsed, r,
1484                        SCHEME_WITH_HOST_PORT_AND_USER_INFORMATION, NULL,
1485                        &output, &out_parsed);
1486     output.Complete();
1487
1488     EXPECT_EQ(replace_cases[i].expected, out_str);
1489   }
1490
1491   // The path pointer should be ignored if the address is invalid.
1492   {
1493     const char src[] = "http://www.google.com/here_is_the_path";
1494     int src_len = static_cast<int>(strlen(src));
1495
1496     Parsed parsed;
1497     ParseStandardURL(src, src_len, &parsed);
1498
1499     // Replace the path to 0 length string. By using 1 as the string address,
1500     // the test should get an access violation if it tries to dereference it.
1501     Replacements<char> r;
1502     r.SetPath(reinterpret_cast<char*>(0x00000001), Component(0, 0));
1503     std::string out_str1;
1504     StdStringCanonOutput output1(&out_str1);
1505     Parsed new_parsed;
1506     ReplaceStandardURL(src, parsed, r,
1507                        SCHEME_WITH_HOST_PORT_AND_USER_INFORMATION, NULL,
1508                        &output1, &new_parsed);
1509     output1.Complete();
1510     EXPECT_STREQ("http://www.google.com/", out_str1.c_str());
1511
1512     // Same with an "invalid" path.
1513     r.SetPath(reinterpret_cast<char*>(0x00000001), Component());
1514     std::string out_str2;
1515     StdStringCanonOutput output2(&out_str2);
1516     ReplaceStandardURL(src, parsed, r,
1517                        SCHEME_WITH_HOST_PORT_AND_USER_INFORMATION, NULL,
1518                        &output2, &new_parsed);
1519     output2.Complete();
1520     EXPECT_STREQ("http://www.google.com/", out_str2.c_str());
1521   }
1522 }
1523
1524 TEST(URLCanonTest, ReplaceFileURL) {
1525   ReplaceCase replace_cases[] = {
1526       // Replace everything
1527     {"file:///C:/gaba?query#ref", NULL, NULL, NULL, "filer", NULL, "/foo", "b", "c", "file://filer/foo?b#c"},
1528       // Replace nothing
1529     {"file:///C:/gaba?query#ref", NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, "file:///C:/gaba?query#ref"},
1530       // Clear non-path components (common)
1531     {"file:///C:/gaba?query#ref", NULL, NULL, NULL, NULL, NULL, NULL, kDeleteComp, kDeleteComp, "file:///C:/gaba"},
1532       // Replace path with something that doesn't begin with a slash and make
1533       // sure it gets added properly.
1534     {"file:///C:/gaba", NULL, NULL, NULL, NULL, NULL, "interesting/", NULL, NULL, "file:///interesting/"},
1535     {"file:///home/gaba?query#ref", NULL, NULL, NULL, "filer", NULL, "/foo", "b", "c", "file://filer/foo?b#c"},
1536     {"file:///home/gaba?query#ref", NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, "file:///home/gaba?query#ref"},
1537     {"file:///home/gaba?query#ref", NULL, NULL, NULL, NULL, NULL, NULL, kDeleteComp, kDeleteComp, "file:///home/gaba"},
1538     {"file:///home/gaba", NULL, NULL, NULL, NULL, NULL, "interesting/", NULL, NULL, "file:///interesting/"},
1539       // Replace scheme -- shouldn't do anything.
1540     {"file:///C:/gaba?query#ref", "http", NULL, NULL, NULL, NULL, NULL, NULL, NULL, "file:///C:/gaba?query#ref"},
1541   };
1542
1543   for (size_t i = 0; i < arraysize(replace_cases); i++) {
1544     const ReplaceCase& cur = replace_cases[i];
1545     int base_len = static_cast<int>(strlen(cur.base));
1546     Parsed parsed;
1547     ParseFileURL(cur.base, base_len, &parsed);
1548
1549     Replacements<char> r;
1550     typedef Replacements<char> R;  // Clean up syntax.
1551     SetupReplComp(&R::SetScheme, &R::ClearRef, &r, cur.scheme);
1552     SetupReplComp(&R::SetUsername, &R::ClearUsername, &r, cur.username);
1553     SetupReplComp(&R::SetPassword, &R::ClearPassword, &r, cur.password);
1554     SetupReplComp(&R::SetHost, &R::ClearHost, &r, cur.host);
1555     SetupReplComp(&R::SetPort, &R::ClearPort, &r, cur.port);
1556     SetupReplComp(&R::SetPath, &R::ClearPath, &r, cur.path);
1557     SetupReplComp(&R::SetQuery, &R::ClearQuery, &r, cur.query);
1558     SetupReplComp(&R::SetRef, &R::ClearRef, &r, cur.ref);
1559
1560     std::string out_str;
1561     StdStringCanonOutput output(&out_str);
1562     Parsed out_parsed;
1563     ReplaceFileURL(cur.base, parsed, r, NULL, &output, &out_parsed);
1564     output.Complete();
1565
1566     EXPECT_EQ(replace_cases[i].expected, out_str);
1567   }
1568 }
1569
1570 TEST(URLCanonTest, ReplaceFileSystemURL) {
1571   ReplaceCase replace_cases[] = {
1572       // Replace everything in the outer URL.
1573       {"filesystem:file:///temporary/gaba?query#ref", NULL, NULL, NULL, NULL,
1574        NULL, "/foo", "b", "c", "filesystem:file:///temporary/foo?b#c"},
1575       // Replace nothing
1576       {"filesystem:file:///temporary/gaba?query#ref", NULL, NULL, NULL, NULL,
1577        NULL, NULL, NULL, NULL, "filesystem:file:///temporary/gaba?query#ref"},
1578       // Clear non-path components (common)
1579       {"filesystem:file:///temporary/gaba?query#ref", NULL, NULL, NULL, NULL,
1580        NULL, NULL, kDeleteComp, kDeleteComp,
1581        "filesystem:file:///temporary/gaba"},
1582       // Replace path with something that doesn't begin with a slash and make
1583       // sure it gets added properly.
1584       {"filesystem:file:///temporary/gaba?query#ref", NULL, NULL, NULL, NULL,
1585        NULL, "interesting/", NULL, NULL,
1586        "filesystem:file:///temporary/interesting/?query#ref"},
1587       // Replace scheme -- shouldn't do anything except canonicalize.
1588       {"filesystem:http://u:p@bar.com/t/gaba?query#ref", "http", NULL, NULL,
1589        NULL, NULL, NULL, NULL, NULL,
1590        "filesystem:http://bar.com/t/gaba?query#ref"},
1591       // Replace username -- shouldn't do anything except canonicalize.
1592       {"filesystem:http://u:p@bar.com/t/gaba?query#ref", NULL, "u2", NULL, NULL,
1593        NULL, NULL, NULL, NULL, "filesystem:http://bar.com/t/gaba?query#ref"},
1594       // Replace password -- shouldn't do anything except canonicalize.
1595       {"filesystem:http://u:p@bar.com/t/gaba?query#ref", NULL, NULL, "pw2",
1596        NULL, NULL, NULL, NULL, NULL,
1597        "filesystem:http://bar.com/t/gaba?query#ref"},
1598       // Replace host -- shouldn't do anything except canonicalize.
1599       {"filesystem:http://u:p@bar.com:80/t/gaba?query#ref", NULL, NULL, NULL,
1600        "foo.com", NULL, NULL, NULL, NULL,
1601        "filesystem:http://bar.com/t/gaba?query#ref"},
1602       // Replace port -- shouldn't do anything except canonicalize.
1603       {"filesystem:http://u:p@bar.com:40/t/gaba?query#ref", NULL, NULL, NULL,
1604        NULL, "41", NULL, NULL, NULL,
1605        "filesystem:http://bar.com:40/t/gaba?query#ref"},
1606   };
1607
1608   for (size_t i = 0; i < arraysize(replace_cases); i++) {
1609     const ReplaceCase& cur = replace_cases[i];
1610     int base_len = static_cast<int>(strlen(cur.base));
1611     Parsed parsed;
1612     ParseFileSystemURL(cur.base, base_len, &parsed);
1613
1614     Replacements<char> r;
1615     typedef Replacements<char> R;  // Clean up syntax.
1616     SetupReplComp(&R::SetScheme, &R::ClearRef, &r, cur.scheme);
1617     SetupReplComp(&R::SetUsername, &R::ClearUsername, &r, cur.username);
1618     SetupReplComp(&R::SetPassword, &R::ClearPassword, &r, cur.password);
1619     SetupReplComp(&R::SetHost, &R::ClearHost, &r, cur.host);
1620     SetupReplComp(&R::SetPort, &R::ClearPort, &r, cur.port);
1621     SetupReplComp(&R::SetPath, &R::ClearPath, &r, cur.path);
1622     SetupReplComp(&R::SetQuery, &R::ClearQuery, &r, cur.query);
1623     SetupReplComp(&R::SetRef, &R::ClearRef, &r, cur.ref);
1624
1625     std::string out_str;
1626     StdStringCanonOutput output(&out_str);
1627     Parsed out_parsed;
1628     ReplaceFileSystemURL(cur.base, parsed, r, NULL, &output, &out_parsed);
1629     output.Complete();
1630
1631     EXPECT_EQ(replace_cases[i].expected, out_str);
1632   }
1633 }
1634
1635 TEST(URLCanonTest, ReplacePathURL) {
1636   ReplaceCase replace_cases[] = {
1637       // Replace everything
1638     {"data:foo", "javascript", NULL, NULL, NULL, NULL, "alert('foo?');", NULL, NULL, "javascript:alert('foo?');"},
1639       // Replace nothing
1640     {"data:foo", NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, "data:foo"},
1641       // Replace one or the other
1642     {"data:foo", "javascript", NULL, NULL, NULL, NULL, NULL, NULL, NULL, "javascript:foo"},
1643     {"data:foo", NULL, NULL, NULL, NULL, NULL, "bar", NULL, NULL, "data:bar"},
1644     {"data:foo", NULL, NULL, NULL, NULL, NULL, kDeleteComp, NULL, NULL, "data:"},
1645   };
1646
1647   for (size_t i = 0; i < arraysize(replace_cases); i++) {
1648     const ReplaceCase& cur = replace_cases[i];
1649     int base_len = static_cast<int>(strlen(cur.base));
1650     Parsed parsed;
1651     ParsePathURL(cur.base, base_len, false, &parsed);
1652
1653     Replacements<char> r;
1654     typedef Replacements<char> R;  // Clean up syntax.
1655     SetupReplComp(&R::SetScheme, &R::ClearRef, &r, cur.scheme);
1656     SetupReplComp(&R::SetUsername, &R::ClearUsername, &r, cur.username);
1657     SetupReplComp(&R::SetPassword, &R::ClearPassword, &r, cur.password);
1658     SetupReplComp(&R::SetHost, &R::ClearHost, &r, cur.host);
1659     SetupReplComp(&R::SetPort, &R::ClearPort, &r, cur.port);
1660     SetupReplComp(&R::SetPath, &R::ClearPath, &r, cur.path);
1661     SetupReplComp(&R::SetQuery, &R::ClearQuery, &r, cur.query);
1662     SetupReplComp(&R::SetRef, &R::ClearRef, &r, cur.ref);
1663
1664     std::string out_str;
1665     StdStringCanonOutput output(&out_str);
1666     Parsed out_parsed;
1667     ReplacePathURL(cur.base, parsed, r, &output, &out_parsed);
1668     output.Complete();
1669
1670     EXPECT_EQ(replace_cases[i].expected, out_str);
1671   }
1672 }
1673
1674 TEST(URLCanonTest, ReplaceMailtoURL) {
1675   ReplaceCase replace_cases[] = {
1676       // Replace everything
1677     {"mailto:jon@foo.com?body=sup", "mailto", NULL, NULL, NULL, NULL, "addr1", "to=tony", NULL, "mailto:addr1?to=tony"},
1678       // Replace nothing
1679     {"mailto:jon@foo.com?body=sup", NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, "mailto:jon@foo.com?body=sup"},
1680       // Replace the path
1681     {"mailto:jon@foo.com?body=sup", NULL, NULL, NULL, NULL, NULL, "jason", NULL, NULL, "mailto:jason?body=sup"},
1682       // Replace the query
1683     {"mailto:jon@foo.com?body=sup", NULL, NULL, NULL, NULL, NULL, NULL, "custom=1", NULL, "mailto:jon@foo.com?custom=1"},
1684       // Replace the path and query
1685     {"mailto:jon@foo.com?body=sup", NULL, NULL, NULL, NULL, NULL, "jason", "custom=1", NULL, "mailto:jason?custom=1"},
1686       // Set the query to empty (should leave trailing question mark)
1687     {"mailto:jon@foo.com?body=sup", NULL, NULL, NULL, NULL, NULL, NULL, "", NULL, "mailto:jon@foo.com?"},
1688       // Clear the query
1689     {"mailto:jon@foo.com?body=sup", NULL, NULL, NULL, NULL, NULL, NULL, "|", NULL, "mailto:jon@foo.com"},
1690       // Clear the path
1691     {"mailto:jon@foo.com?body=sup", NULL, NULL, NULL, NULL, NULL, "|", NULL, NULL, "mailto:?body=sup"},
1692       // Clear the path + query
1693     {"mailto:", NULL, NULL, NULL, NULL, NULL, "|", "|", NULL, "mailto:"},
1694       // Setting the ref should have no effect
1695     {"mailto:addr1", NULL, NULL, NULL, NULL, NULL, NULL, NULL, "BLAH", "mailto:addr1"},
1696   };
1697
1698   for (size_t i = 0; i < arraysize(replace_cases); i++) {
1699     const ReplaceCase& cur = replace_cases[i];
1700     int base_len = static_cast<int>(strlen(cur.base));
1701     Parsed parsed;
1702     ParseMailtoURL(cur.base, base_len, &parsed);
1703
1704     Replacements<char> r;
1705     typedef Replacements<char> R;
1706     SetupReplComp(&R::SetScheme, &R::ClearRef, &r, cur.scheme);
1707     SetupReplComp(&R::SetUsername, &R::ClearUsername, &r, cur.username);
1708     SetupReplComp(&R::SetPassword, &R::ClearPassword, &r, cur.password);
1709     SetupReplComp(&R::SetHost, &R::ClearHost, &r, cur.host);
1710     SetupReplComp(&R::SetPort, &R::ClearPort, &r, cur.port);
1711     SetupReplComp(&R::SetPath, &R::ClearPath, &r, cur.path);
1712     SetupReplComp(&R::SetQuery, &R::ClearQuery, &r, cur.query);
1713     SetupReplComp(&R::SetRef, &R::ClearRef, &r, cur.ref);
1714
1715     std::string out_str;
1716     StdStringCanonOutput output(&out_str);
1717     Parsed out_parsed;
1718     ReplaceMailtoURL(cur.base, parsed, r, &output, &out_parsed);
1719     output.Complete();
1720
1721     EXPECT_EQ(replace_cases[i].expected, out_str);
1722   }
1723 }
1724
1725 TEST(URLCanonTest, CanonicalizeFileURL) {
1726   struct URLCase {
1727     const char* input;
1728     const char* expected;
1729     bool expected_success;
1730     Component expected_host;
1731     Component expected_path;
1732   } cases[] = {
1733 #ifdef _WIN32
1734       // Windows-style paths
1735       {"file:c:\\foo\\bar.html", "file:///C:/foo/bar.html", true, Component(),
1736        Component(7, 16)},
1737       {"  File:c|////foo\\bar.html", "file:///C:////foo/bar.html", true,
1738        Component(), Component(7, 19)},
1739       {"file:", "file:///", true, Component(), Component(7, 1)},
1740       {"file:UNChost/path", "file://unchost/path", true, Component(7, 7),
1741        Component(14, 5)},
1742       // CanonicalizeFileURL supports absolute Windows style paths for IE
1743       // compatibility. Note that the caller must decide that this is a file
1744       // URL itself so it can call the file canonicalizer. This is usually
1745       // done automatically as part of relative URL resolving.
1746       {"c:\\foo\\bar", "file:///C:/foo/bar", true, Component(),
1747        Component(7, 11)},
1748       {"C|/foo/bar", "file:///C:/foo/bar", true, Component(), Component(7, 11)},
1749       {"/C|\\foo\\bar", "file:///C:/foo/bar", true, Component(),
1750        Component(7, 11)},
1751       {"//C|/foo/bar", "file:///C:/foo/bar", true, Component(),
1752        Component(7, 11)},
1753       {"//server/file", "file://server/file", true, Component(7, 6),
1754        Component(13, 5)},
1755       {"\\\\server\\file", "file://server/file", true, Component(7, 6),
1756        Component(13, 5)},
1757       {"/\\server/file", "file://server/file", true, Component(7, 6),
1758        Component(13, 5)},
1759       // We should preserve the number of slashes after the colon for IE
1760       // compatibility, except when there is none, in which case we should
1761       // add one.
1762       {"file:c:foo/bar.html", "file:///C:/foo/bar.html", true, Component(),
1763        Component(7, 16)},
1764       {"file:/\\/\\C:\\\\//foo\\bar.html", "file:///C:////foo/bar.html", true,
1765        Component(), Component(7, 19)},
1766       // Three slashes should be non-UNC, even if there is no drive spec (IE
1767       // does this, which makes the resulting request invalid).
1768       {"file:///foo/bar.txt", "file:///foo/bar.txt", true, Component(),
1769        Component(7, 12)},
1770       // TODO(brettw) we should probably fail for invalid host names, which
1771       // would change the expected result on this test. We also currently allow
1772       // colon even though it's probably invalid, because its currently the
1773       // "natural" result of the way the canonicalizer is written. There doesn't
1774       // seem to be a strong argument for why allowing it here would be bad, so
1775       // we just tolerate it and the load will fail later.
1776       {"FILE:/\\/\\7:\\\\//foo\\bar.html", "file://7:////foo/bar.html", false,
1777        Component(7, 2), Component(9, 16)},
1778       {"file:filer/home\\me", "file://filer/home/me", true, Component(7, 5),
1779        Component(12, 8)},
1780       // Make sure relative paths can't go above the "C:"
1781       {"file:///C:/foo/../../../bar.html", "file:///C:/bar.html", true,
1782        Component(), Component(7, 12)},
1783       // Busted refs shouldn't make the whole thing fail.
1784       {"file:///C:/asdf#\xc2", "file:///C:/asdf#%EF%BF%BD", true, Component(),
1785        Component(7, 8)},
1786 #else
1787       // Unix-style paths
1788     {"file:///home/me", "file:///home/me", true, Component(), Component(7, 8)},
1789       // Windowsy ones should get still treated as Unix-style.
1790     {"file:c:\\foo\\bar.html", "file:///c:/foo/bar.html", true, Component(), Component(7, 16)},
1791     {"file:c|//foo\\bar.html", "file:///c%7C//foo/bar.html", true, Component(), Component(7, 19)},
1792       // file: tests from WebKit (LayoutTests/fast/loader/url-parse-1.html)
1793     {"//", "file:///", true, Component(), Component(7, 1)},
1794     {"///", "file:///", true, Component(), Component(7, 1)},
1795     {"///test", "file:///test", true, Component(), Component(7, 5)},
1796     {"file://test", "file://test/", true, Component(7, 4), Component(11, 1)},
1797     {"file://localhost",  "file://localhost/", true, Component(7, 9), Component(16, 1)},
1798     {"file://localhost/", "file://localhost/", true, Component(7, 9), Component(16, 1)},
1799     {"file://localhost/test", "file://localhost/test", true, Component(7, 9), Component(16, 5)},
1800 #endif  // _WIN32
1801   };
1802
1803   for (size_t i = 0; i < arraysize(cases); i++) {
1804     int url_len = static_cast<int>(strlen(cases[i].input));
1805     Parsed parsed;
1806     ParseFileURL(cases[i].input, url_len, &parsed);
1807
1808     Parsed out_parsed;
1809     std::string out_str;
1810     StdStringCanonOutput output(&out_str);
1811     bool success = CanonicalizeFileURL(cases[i].input, url_len, parsed, NULL,
1812                                        &output, &out_parsed);
1813     output.Complete();
1814
1815     EXPECT_EQ(cases[i].expected_success, success);
1816     EXPECT_EQ(cases[i].expected, out_str);
1817
1818     // Make sure the spec was properly identified, the file canonicalizer has
1819     // different code for writing the spec.
1820     EXPECT_EQ(0, out_parsed.scheme.begin);
1821     EXPECT_EQ(4, out_parsed.scheme.len);
1822
1823     EXPECT_EQ(cases[i].expected_host.begin, out_parsed.host.begin);
1824     EXPECT_EQ(cases[i].expected_host.len, out_parsed.host.len);
1825
1826     EXPECT_EQ(cases[i].expected_path.begin, out_parsed.path.begin);
1827     EXPECT_EQ(cases[i].expected_path.len, out_parsed.path.len);
1828   }
1829 }
1830
1831 TEST(URLCanonTest, CanonicalizeFileSystemURL) {
1832   struct URLCase {
1833     const char* input;
1834     const char* expected;
1835     bool expected_success;
1836   } cases[] = {
1837     {"Filesystem:htTp://www.Foo.com:80/tempoRary", "filesystem:http://www.foo.com/tempoRary/", true},
1838     {"filesystem:httpS://www.foo.com/temporary/", "filesystem:https://www.foo.com/temporary/", true},
1839     {"filesystem:http://www.foo.com//", "filesystem:http://www.foo.com//", false},
1840     {"filesystem:http://www.foo.com/persistent/bob?query#ref", "filesystem:http://www.foo.com/persistent/bob?query#ref", true},
1841     {"filesystem:fIle://\\temporary/", "filesystem:file:///temporary/", true},
1842     {"filesystem:fiLe:///temporary", "filesystem:file:///temporary/", true},
1843     {"filesystem:File:///temporary/Bob?qUery#reF", "filesystem:file:///temporary/Bob?qUery#reF", true},
1844   };
1845
1846   for (size_t i = 0; i < arraysize(cases); i++) {
1847     int url_len = static_cast<int>(strlen(cases[i].input));
1848     Parsed parsed;
1849     ParseFileSystemURL(cases[i].input, url_len, &parsed);
1850
1851     Parsed out_parsed;
1852     std::string out_str;
1853     StdStringCanonOutput output(&out_str);
1854     bool success = CanonicalizeFileSystemURL(cases[i].input, url_len, parsed,
1855                                              NULL, &output, &out_parsed);
1856     output.Complete();
1857
1858     EXPECT_EQ(cases[i].expected_success, success);
1859     EXPECT_EQ(cases[i].expected, out_str);
1860
1861     // Make sure the spec was properly identified, the filesystem canonicalizer
1862     // has different code for writing the spec.
1863     EXPECT_EQ(0, out_parsed.scheme.begin);
1864     EXPECT_EQ(10, out_parsed.scheme.len);
1865     if (success)
1866       EXPECT_GT(out_parsed.path.len, 0);
1867   }
1868 }
1869
1870 TEST(URLCanonTest, CanonicalizePathURL) {
1871   // Path URLs should get canonicalized schemes but nothing else.
1872   struct PathCase {
1873     const char* input;
1874     const char* expected;
1875   } path_cases[] = {
1876     {"javascript:", "javascript:"},
1877     {"JavaScript:Foo", "javascript:Foo"},
1878     {"Foo:\":This /is interesting;?#", "foo:\":This /is interesting;?#"},
1879   };
1880
1881   for (size_t i = 0; i < arraysize(path_cases); i++) {
1882     int url_len = static_cast<int>(strlen(path_cases[i].input));
1883     Parsed parsed;
1884     ParsePathURL(path_cases[i].input, url_len, true, &parsed);
1885
1886     Parsed out_parsed;
1887     std::string out_str;
1888     StdStringCanonOutput output(&out_str);
1889     bool success = CanonicalizePathURL(path_cases[i].input, url_len, parsed,
1890                                        &output, &out_parsed);
1891     output.Complete();
1892
1893     EXPECT_TRUE(success);
1894     EXPECT_EQ(path_cases[i].expected, out_str);
1895
1896     EXPECT_EQ(0, out_parsed.host.begin);
1897     EXPECT_EQ(-1, out_parsed.host.len);
1898
1899     // When we end with a colon at the end, there should be no path.
1900     if (path_cases[i].input[url_len - 1] == ':') {
1901       EXPECT_EQ(0, out_parsed.GetContent().begin);
1902       EXPECT_EQ(-1, out_parsed.GetContent().len);
1903     }
1904   }
1905 }
1906
1907 TEST(URLCanonTest, CanonicalizeMailtoURL) {
1908   struct URLCase {
1909     const char* input;
1910     const char* expected;
1911     bool expected_success;
1912     Component expected_path;
1913     Component expected_query;
1914   } cases[] = {
1915     // Null character should be escaped to %00.
1916     // Keep this test first in the list as it is handled specially below.
1917     {"mailto:addr1\0addr2?foo",
1918      "mailto:addr1%00addr2?foo",
1919      true, Component(7, 13), Component(21, 3)},
1920     {"mailto:addr1",
1921      "mailto:addr1",
1922      true, Component(7, 5), Component()},
1923     {"mailto:addr1@foo.com",
1924      "mailto:addr1@foo.com",
1925      true, Component(7, 13), Component()},
1926     // Trailing whitespace is stripped.
1927     {"MaIlTo:addr1 \t ",
1928      "mailto:addr1",
1929      true, Component(7, 5), Component()},
1930     {"MaIlTo:addr1?to=jon",
1931      "mailto:addr1?to=jon",
1932      true, Component(7, 5), Component(13,6)},
1933     {"mailto:addr1,addr2",
1934      "mailto:addr1,addr2",
1935      true, Component(7, 11), Component()},
1936     // Embedded spaces must be encoded.
1937     {"mailto:addr1, addr2",
1938      "mailto:addr1,%20addr2",
1939      true, Component(7, 14), Component()},
1940     {"mailto:addr1, addr2?subject=one two ",
1941      "mailto:addr1,%20addr2?subject=one%20two",
1942      true, Component(7, 14), Component(22, 17)},
1943     {"mailto:addr1%2caddr2",
1944      "mailto:addr1%2caddr2",
1945      true, Component(7, 13), Component()},
1946     {"mailto:\xF0\x90\x8C\x80",
1947      "mailto:%F0%90%8C%80",
1948      true, Component(7, 12), Component()},
1949     // Invalid -- UTF-8 encoded surrogate value.
1950     {"mailto:\xed\xa0\x80",
1951      "mailto:%EF%BF%BD%EF%BF%BD%EF%BF%BD",
1952      false, Component(7, 27), Component()},
1953     {"mailto:addr1?",
1954      "mailto:addr1?",
1955      true, Component(7, 5), Component(13, 0)},
1956     // Certain characters have special meanings and must be encoded.
1957     {"mailto:! \x22$&()+,-./09:;<=>@AZ[\\]&_`az{|}~\x7f?Query! \x22$&()+,-./09:;<=>@AZ[\\]&_`az{|}~",
1958      "mailto:!%20%22$&()+,-./09:;%3C=%3E@AZ[\\]&_%60az%7B%7C%7D~%7F?Query!%20%22$&()+,-./09:;%3C=%3E@AZ[\\]&_`az{|}~",
1959      true, Component(7, 53), Component(61, 47)},
1960   };
1961
1962   // Define outside of loop to catch bugs where components aren't reset
1963   Parsed parsed;
1964   Parsed out_parsed;
1965
1966   for (size_t i = 0; i < arraysize(cases); i++) {
1967     int url_len = static_cast<int>(strlen(cases[i].input));
1968     if (i == 0) {
1969       // The first test case purposely has a '\0' in it -- don't count it
1970       // as the string terminator.
1971       url_len = 22;
1972     }
1973     ParseMailtoURL(cases[i].input, url_len, &parsed);
1974
1975     std::string out_str;
1976     StdStringCanonOutput output(&out_str);
1977     bool success = CanonicalizeMailtoURL(cases[i].input, url_len, parsed,
1978                                          &output, &out_parsed);
1979     output.Complete();
1980
1981     EXPECT_EQ(cases[i].expected_success, success);
1982     EXPECT_EQ(cases[i].expected, out_str);
1983
1984     // Make sure the spec was properly identified
1985     EXPECT_EQ(0, out_parsed.scheme.begin);
1986     EXPECT_EQ(6, out_parsed.scheme.len);
1987
1988     EXPECT_EQ(cases[i].expected_path.begin, out_parsed.path.begin);
1989     EXPECT_EQ(cases[i].expected_path.len, out_parsed.path.len);
1990
1991     EXPECT_EQ(cases[i].expected_query.begin, out_parsed.query.begin);
1992     EXPECT_EQ(cases[i].expected_query.len, out_parsed.query.len);
1993   }
1994 }
1995
1996 #ifndef WIN32
1997
1998 TEST(URLCanonTest, _itoa_s) {
1999   // We fill the buffer with 0xff to ensure that it's getting properly
2000   // null-terminated. We also allocate one byte more than what we tell
2001   // _itoa_s about, and ensure that the extra byte is untouched.
2002   char buf[6];
2003   memset(buf, 0xff, sizeof(buf));
2004   EXPECT_EQ(0, _itoa_s(12, buf, sizeof(buf) - 1, 10));
2005   EXPECT_STREQ("12", buf);
2006   EXPECT_EQ('\xFF', buf[3]);
2007
2008   // Test the edge cases - exactly the buffer size and one over
2009   memset(buf, 0xff, sizeof(buf));
2010   EXPECT_EQ(0, _itoa_s(1234, buf, sizeof(buf) - 1, 10));
2011   EXPECT_STREQ("1234", buf);
2012   EXPECT_EQ('\xFF', buf[5]);
2013
2014   memset(buf, 0xff, sizeof(buf));
2015   EXPECT_EQ(EINVAL, _itoa_s(12345, buf, sizeof(buf) - 1, 10));
2016   EXPECT_EQ('\xFF', buf[5]);  // should never write to this location
2017
2018   // Test the template overload (note that this will see the full buffer)
2019   memset(buf, 0xff, sizeof(buf));
2020   EXPECT_EQ(0, _itoa_s(12, buf, 10));
2021   EXPECT_STREQ("12", buf);
2022   EXPECT_EQ('\xFF', buf[3]);
2023
2024   memset(buf, 0xff, sizeof(buf));
2025   EXPECT_EQ(0, _itoa_s(12345, buf, 10));
2026   EXPECT_STREQ("12345", buf);
2027
2028   EXPECT_EQ(EINVAL, _itoa_s(123456, buf, 10));
2029
2030   // Test that radix 16 is supported.
2031   memset(buf, 0xff, sizeof(buf));
2032   EXPECT_EQ(0, _itoa_s(1234, buf, sizeof(buf) - 1, 16));
2033   EXPECT_STREQ("4d2", buf);
2034   EXPECT_EQ('\xFF', buf[5]);
2035 }
2036
2037 TEST(URLCanonTest, _itow_s) {
2038   // We fill the buffer with 0xff to ensure that it's getting properly
2039   // null-terminated. We also allocate one byte more than what we tell
2040   // _itoa_s about, and ensure that the extra byte is untouched.
2041   base::char16 buf[6];
2042   const char fill_mem = 0xff;
2043   const base::char16 fill_char = 0xffff;
2044   memset(buf, fill_mem, sizeof(buf));
2045   EXPECT_EQ(0, _itow_s(12, buf, sizeof(buf) / 2 - 1, 10));
2046   EXPECT_EQ(base::UTF8ToUTF16("12"), base::string16(buf));
2047   EXPECT_EQ(fill_char, buf[3]);
2048
2049   // Test the edge cases - exactly the buffer size and one over
2050   EXPECT_EQ(0, _itow_s(1234, buf, sizeof(buf) / 2 - 1, 10));
2051   EXPECT_EQ(base::UTF8ToUTF16("1234"), base::string16(buf));
2052   EXPECT_EQ(fill_char, buf[5]);
2053
2054   memset(buf, fill_mem, sizeof(buf));
2055   EXPECT_EQ(EINVAL, _itow_s(12345, buf, sizeof(buf) / 2 - 1, 10));
2056   EXPECT_EQ(fill_char, buf[5]);  // should never write to this location
2057
2058   // Test the template overload (note that this will see the full buffer)
2059   memset(buf, fill_mem, sizeof(buf));
2060   EXPECT_EQ(0, _itow_s(12, buf, 10));
2061   EXPECT_EQ(base::UTF8ToUTF16("12"),
2062             base::string16(buf));
2063   EXPECT_EQ(fill_char, buf[3]);
2064
2065   memset(buf, fill_mem, sizeof(buf));
2066   EXPECT_EQ(0, _itow_s(12345, buf, 10));
2067   EXPECT_EQ(base::UTF8ToUTF16("12345"), base::string16(buf));
2068
2069   EXPECT_EQ(EINVAL, _itow_s(123456, buf, 10));
2070 }
2071
2072 #endif  // !WIN32
2073
2074 // Returns true if the given two structures are the same.
2075 static bool ParsedIsEqual(const Parsed& a, const Parsed& b) {
2076   return a.scheme.begin == b.scheme.begin && a.scheme.len == b.scheme.len &&
2077          a.username.begin == b.username.begin && a.username.len == b.username.len &&
2078          a.password.begin == b.password.begin && a.password.len == b.password.len &&
2079          a.host.begin == b.host.begin && a.host.len == b.host.len &&
2080          a.port.begin == b.port.begin && a.port.len == b.port.len &&
2081          a.path.begin == b.path.begin && a.path.len == b.path.len &&
2082          a.query.begin == b.query.begin && a.query.len == b.query.len &&
2083          a.ref.begin == b.ref.begin && a.ref.len == b.ref.len;
2084 }
2085
2086 TEST(URLCanonTest, ResolveRelativeURL) {
2087   struct RelativeCase {
2088     const char* base;      // Input base URL: MUST BE CANONICAL
2089     bool is_base_hier;     // Is the base URL hierarchical
2090     bool is_base_file;     // Tells us if the base is a file URL.
2091     const char* test;      // Input URL to test against.
2092     bool succeed_relative; // Whether we expect IsRelativeURL to succeed
2093     bool is_rel;           // Whether we expect |test| to be relative or not.
2094     bool succeed_resolve;  // Whether we expect ResolveRelativeURL to succeed.
2095     const char* resolved;  // What we expect in the result when resolving.
2096   } rel_cases[] = {
2097       // Basic absolute input.
2098     {"http://host/a", true, false, "http://another/", true, false, false, NULL},
2099     {"http://host/a", true, false, "http:////another/", true, false, false, NULL},
2100       // Empty relative URLs should only remove the ref part of the URL,
2101       // leaving the rest unchanged.
2102     {"http://foo/bar", true, false, "", true, true, true, "http://foo/bar"},
2103     {"http://foo/bar#ref", true, false, "", true, true, true, "http://foo/bar"},
2104     {"http://foo/bar#", true, false, "", true, true, true, "http://foo/bar"},
2105       // Spaces at the ends of the relative path should be ignored.
2106     {"http://foo/bar", true, false, "  another  ", true, true, true, "http://foo/another"},
2107     {"http://foo/bar", true, false, "  .  ", true, true, true, "http://foo/"},
2108     {"http://foo/bar", true, false, " \t ", true, true, true, "http://foo/bar"},
2109       // Matching schemes without two slashes are treated as relative.
2110     {"http://host/a", true, false, "http:path", true, true, true, "http://host/path"},
2111     {"http://host/a/", true, false, "http:path", true, true, true, "http://host/a/path"},
2112     {"http://host/a", true, false, "http:/path", true, true, true, "http://host/path"},
2113     {"http://host/a", true, false, "HTTP:/path", true, true, true, "http://host/path"},
2114       // Nonmatching schemes are absolute.
2115     {"http://host/a", true, false, "https:host2", true, false, false, NULL},
2116     {"http://host/a", true, false, "htto:/host2", true, false, false, NULL},
2117       // Absolute path input
2118     {"http://host/a", true, false, "/b/c/d", true, true, true, "http://host/b/c/d"},
2119     {"http://host/a", true, false, "\\b\\c\\d", true, true, true, "http://host/b/c/d"},
2120     {"http://host/a", true, false, "/b/../c", true, true, true, "http://host/c"},
2121     {"http://host/a?b#c", true, false, "/b/../c", true, true, true, "http://host/c"},
2122     {"http://host/a", true, false, "\\b/../c?x#y", true, true, true, "http://host/c?x#y"},
2123     {"http://host/a?b#c", true, false, "/b/../c?x#y", true, true, true, "http://host/c?x#y"},
2124       // Relative path input
2125     {"http://host/a", true, false, "b", true, true, true, "http://host/b"},
2126     {"http://host/a", true, false, "bc/de", true, true, true, "http://host/bc/de"},
2127     {"http://host/a/", true, false, "bc/de?query#ref", true, true, true, "http://host/a/bc/de?query#ref"},
2128     {"http://host/a/", true, false, ".", true, true, true, "http://host/a/"},
2129     {"http://host/a/", true, false, "..", true, true, true, "http://host/"},
2130     {"http://host/a/", true, false, "./..", true, true, true, "http://host/"},
2131     {"http://host/a/", true, false, "../.", true, true, true, "http://host/"},
2132     {"http://host/a/", true, false, "././.", true, true, true, "http://host/a/"},
2133     {"http://host/a?query#ref", true, false, "../../../foo", true, true, true, "http://host/foo"},
2134       // Query input
2135     {"http://host/a", true, false, "?foo=bar", true, true, true, "http://host/a?foo=bar"},
2136     {"http://host/a?x=y#z", true, false, "?", true, true, true, "http://host/a?"},
2137     {"http://host/a?x=y#z", true, false, "?foo=bar#com", true, true, true, "http://host/a?foo=bar#com"},
2138       // Ref input
2139     {"http://host/a", true, false, "#ref", true, true, true, "http://host/a#ref"},
2140     {"http://host/a#b", true, false, "#", true, true, true, "http://host/a#"},
2141     {"http://host/a?foo=bar#hello", true, false, "#bye", true, true, true, "http://host/a?foo=bar#bye"},
2142       // Non-hierarchical base: no relative handling. Relative input should
2143       // error, and if a scheme is present, it should be treated as absolute.
2144     {"data:foobar", false, false, "baz.html", false, false, false, NULL},
2145     {"data:foobar", false, false, "data:baz", true, false, false, NULL},
2146     {"data:foobar", false, false, "data:/base", true, false, false, NULL},
2147       // Non-hierarchical base: absolute input should succeed.
2148     {"data:foobar", false, false, "http://host/", true, false, false, NULL},
2149     {"data:foobar", false, false, "http:host", true, false, false, NULL},
2150       // Non-hierarchical base: empty URL should give error.
2151     {"data:foobar", false, false, "", false, false, false, NULL},
2152       // Invalid schemes should be treated as relative.
2153     {"http://foo/bar", true, false, "./asd:fgh", true, true, true, "http://foo/asd:fgh"},
2154     {"http://foo/bar", true, false, ":foo", true, true, true, "http://foo/:foo"},
2155     {"http://foo/bar", true, false, " hello world", true, true, true, "http://foo/hello%20world"},
2156     {"data:asdf", false, false, ":foo", false, false, false, NULL},
2157     {"data:asdf", false, false, "bad(':foo')", false, false, false, NULL},
2158       // We should treat semicolons like any other character in URL resolving
2159     {"http://host/a", true, false, ";foo", true, true, true, "http://host/;foo"},
2160     {"http://host/a;", true, false, ";foo", true, true, true, "http://host/;foo"},
2161     {"http://host/a", true, false, ";/../bar", true, true, true, "http://host/bar"},
2162       // Relative URLs can also be written as "//foo/bar" which is relative to
2163       // the scheme. In this case, it would take the old scheme, so for http
2164       // the example would resolve to "http://foo/bar".
2165     {"http://host/a", true, false, "//another", true, true, true, "http://another/"},
2166     {"http://host/a", true, false, "//another/path?query#ref", true, true, true, "http://another/path?query#ref"},
2167     {"http://host/a", true, false, "///another/path", true, true, true, "http://another/path"},
2168     {"http://host/a", true, false, "//Another\\path", true, true, true, "http://another/path"},
2169     {"http://host/a", true, false, "//", true, true, false, "http:"},
2170       // IE will also allow one or the other to be a backslash to get the same
2171       // behavior.
2172     {"http://host/a", true, false, "\\/another/path", true, true, true, "http://another/path"},
2173     {"http://host/a", true, false, "/\\Another\\path", true, true, true, "http://another/path"},
2174 #ifdef WIN32
2175       // Resolving against Windows file base URLs.
2176     {"file:///C:/foo", true, true, "http://host/", true, false, false, NULL},
2177     {"file:///C:/foo", true, true, "bar", true, true, true, "file:///C:/bar"},
2178     {"file:///C:/foo", true, true, "../../../bar.html", true, true, true, "file:///C:/bar.html"},
2179     {"file:///C:/foo", true, true, "/../bar.html", true, true, true, "file:///C:/bar.html"},
2180       // But two backslashes on Windows should be UNC so should be treated
2181       // as absolute.
2182     {"http://host/a", true, false, "\\\\another\\path", true, false, false, NULL},
2183       // IE doesn't support drive specs starting with two slashes. It fails
2184       // immediately and doesn't even try to load. We fix it up to either
2185       // an absolute path or UNC depending on what it looks like.
2186     {"file:///C:/something", true, true, "//c:/foo", true, true, true, "file:///C:/foo"},
2187     {"file:///C:/something", true, true, "//localhost/c:/foo", true, true, true, "file:///C:/foo"},
2188       // Windows drive specs should be allowed and treated as absolute.
2189     {"file:///C:/foo", true, true, "c:", true, false, false, NULL},
2190     {"file:///C:/foo", true, true, "c:/foo", true, false, false, NULL},
2191     {"http://host/a", true, false, "c:\\foo", true, false, false, NULL},
2192       // Relative paths with drive letters should be allowed when the base is
2193       // also a file.
2194     {"file:///C:/foo", true, true, "/z:/bar", true, true, true, "file:///Z:/bar"},
2195       // Treat absolute paths as being off of the drive.
2196     {"file:///C:/foo", true, true, "/bar", true, true, true, "file:///C:/bar"},
2197     {"file://localhost/C:/foo", true, true, "/bar", true, true, true, "file://localhost/C:/bar"},
2198     {"file:///C:/foo/com/", true, true, "/bar", true, true, true, "file:///C:/bar"},
2199       // On Windows, two slashes without a drive letter when the base is a file
2200       // means that the path is UNC.
2201     {"file:///C:/something", true, true, "//somehost/path", true, true, true, "file://somehost/path"},
2202     {"file:///C:/something", true, true, "/\\//somehost/path", true, true, true, "file://somehost/path"},
2203 #else
2204       // On Unix we fall back to relative behavior since there's nothing else
2205       // reasonable to do.
2206     {"http://host/a", true, false, "\\\\Another\\path", true, true, true, "http://another/path"},
2207 #endif
2208       // Even on Windows, we don't allow relative drive specs when the base
2209       // is not file.
2210     {"http://host/a", true, false, "/c:\\foo", true, true, true, "http://host/c:/foo"},
2211     {"http://host/a", true, false, "//c:\\foo", true, true, true, "http://c/foo"},
2212       // Ensure that ports aren't allowed for hosts relative to a file url.
2213       // Although the result string shows a host:port portion, the call to
2214       // resolve the relative URL returns false, indicating parse failure,
2215       // which is what is required.
2216     {"file:///foo.txt", true, true, "//host:80/bar.txt", true, true, false, "file://host:80/bar.txt"},
2217       // Filesystem URL tests; filesystem URLs are only valid and relative if
2218       // they have no scheme, e.g. "./index.html". There's no valid equivalent
2219       // to http:index.html.
2220     {"filesystem:http://host/t/path", true, false, "filesystem:http://host/t/path2", true, false, false, NULL},
2221     {"filesystem:http://host/t/path", true, false, "filesystem:https://host/t/path2", true, false, false, NULL},
2222     {"filesystem:http://host/t/path", true, false, "http://host/t/path2", true, false, false, NULL},
2223     {"http://host/t/path", true, false, "filesystem:http://host/t/path2", true, false, false, NULL},
2224     {"filesystem:http://host/t/path", true, false, "./path2", true, true, true, "filesystem:http://host/t/path2"},
2225     {"filesystem:http://host/t/path/", true, false, "path2", true, true, true, "filesystem:http://host/t/path/path2"},
2226     {"filesystem:http://host/t/path", true, false, "filesystem:http:path2", true, false, false, NULL},
2227       // Absolute URLs are still not relative to a non-standard base URL.
2228     {"about:blank", false, false, "http://X/A", true, false, true, ""},
2229     {"about:blank", false, false, "content://content.Provider/", true, false, true, ""},
2230   };
2231
2232   for (size_t i = 0; i < arraysize(rel_cases); i++) {
2233     const RelativeCase& cur_case = rel_cases[i];
2234
2235     Parsed parsed;
2236     int base_len = static_cast<int>(strlen(cur_case.base));
2237     if (cur_case.is_base_file)
2238       ParseFileURL(cur_case.base, base_len, &parsed);
2239     else if (cur_case.is_base_hier)
2240       ParseStandardURL(cur_case.base, base_len, &parsed);
2241     else
2242       ParsePathURL(cur_case.base, base_len, false, &parsed);
2243
2244     // First see if it is relative.
2245     int test_len = static_cast<int>(strlen(cur_case.test));
2246     bool is_relative;
2247     Component relative_component;
2248     bool succeed_is_rel = IsRelativeURL(
2249         cur_case.base, parsed, cur_case.test, test_len, cur_case.is_base_hier,
2250         &is_relative, &relative_component);
2251
2252     EXPECT_EQ(cur_case.succeed_relative, succeed_is_rel) <<
2253         "succeed is rel failure on " << cur_case.test;
2254     EXPECT_EQ(cur_case.is_rel, is_relative) <<
2255         "is rel failure on " << cur_case.test;
2256     // Now resolve it.
2257     if (succeed_is_rel && is_relative && cur_case.is_rel) {
2258       std::string resolved;
2259       StdStringCanonOutput output(&resolved);
2260       Parsed resolved_parsed;
2261
2262       bool succeed_resolve = ResolveRelativeURL(
2263           cur_case.base, parsed, cur_case.is_base_file, cur_case.test,
2264           relative_component, NULL, &output, &resolved_parsed);
2265       output.Complete();
2266
2267       EXPECT_EQ(cur_case.succeed_resolve, succeed_resolve);
2268       EXPECT_EQ(cur_case.resolved, resolved) << " on " << cur_case.test;
2269
2270       // Verify that the output parsed structure is the same as parsing a
2271       // the URL freshly.
2272       Parsed ref_parsed;
2273       int resolved_len = static_cast<int>(resolved.size());
2274       if (cur_case.is_base_file) {
2275         ParseFileURL(resolved.c_str(), resolved_len, &ref_parsed);
2276       } else if (cur_case.is_base_hier) {
2277         ParseStandardURL(resolved.c_str(), resolved_len, &ref_parsed);
2278       } else {
2279         ParsePathURL(resolved.c_str(), resolved_len, false, &ref_parsed);
2280       }
2281       EXPECT_TRUE(ParsedIsEqual(ref_parsed, resolved_parsed));
2282     }
2283   }
2284 }
2285
2286 // It used to be the case that when we did a replacement with a long buffer of
2287 // UTF-16 characters, we would get invalid data in the URL. This is because the
2288 // buffer that it used to hold the UTF-8 data was resized, while some pointers
2289 // were still kept to the old buffer that was removed.
2290 TEST(URLCanonTest, ReplacementOverflow) {
2291   const char src[] = "file:///C:/foo/bar";
2292   int src_len = static_cast<int>(strlen(src));
2293   Parsed parsed;
2294   ParseFileURL(src, src_len, &parsed);
2295
2296   // Override two components, the path with something short, and the query with
2297   // something long enough to trigger the bug.
2298   Replacements<base::char16> repl;
2299   base::string16 new_query;
2300   for (int i = 0; i < 4800; i++)
2301     new_query.push_back('a');
2302
2303   base::string16 new_path(test_utils::TruncateWStringToUTF16(L"/foo"));
2304   repl.SetPath(new_path.c_str(), Component(0, 4));
2305   repl.SetQuery(new_query.c_str(),
2306                 Component(0, static_cast<int>(new_query.length())));
2307
2308   // Call ReplaceComponents on the string. It doesn't matter if we call it for
2309   // standard URLs, file URLs, etc, since they will go to the same replacement
2310   // function that was buggy.
2311   Parsed repl_parsed;
2312   std::string repl_str;
2313   StdStringCanonOutput repl_output(&repl_str);
2314   ReplaceFileURL(src, parsed, repl, NULL, &repl_output, &repl_parsed);
2315   repl_output.Complete();
2316
2317   // Generate the expected string and check.
2318   std::string expected("file:///foo?");
2319   for (size_t i = 0; i < new_query.length(); i++)
2320     expected.push_back('a');
2321   EXPECT_TRUE(expected == repl_str);
2322 }
2323
2324 TEST(URLCanonTest, DefaultPortForScheme) {
2325   struct TestCases {
2326     const char* scheme;
2327     const int expected_port;
2328   } cases[]{
2329       {"http", 80},
2330       {"https", 443},
2331       {"ftp", 21},
2332       {"ws", 80},
2333       {"wss", 443},
2334       {"gopher", 70},
2335       {"fake-scheme", PORT_UNSPECIFIED},
2336       {"HTTP", PORT_UNSPECIFIED},
2337       {"HTTPS", PORT_UNSPECIFIED},
2338       {"FTP", PORT_UNSPECIFIED},
2339       {"WS", PORT_UNSPECIFIED},
2340       {"WSS", PORT_UNSPECIFIED},
2341       {"GOPHER", PORT_UNSPECIFIED},
2342   };
2343
2344   for (auto& test_case : cases) {
2345     SCOPED_TRACE(test_case.scheme);
2346     EXPECT_EQ(test_case.expected_port,
2347               DefaultPortForScheme(test_case.scheme, strlen(test_case.scheme)));
2348   }
2349 }
2350
2351 TEST(URLCanonTest, IDNToASCII) {
2352   RawCanonOutputW<1024> output;
2353
2354   // Basic ASCII test.
2355   base::string16 str = base::UTF8ToUTF16("hello");
2356   EXPECT_TRUE(IDNToASCII(str.data(), str.length(), &output));
2357   EXPECT_EQ(base::UTF8ToUTF16("hello"), base::string16(output.data()));
2358   output.set_length(0);
2359
2360   // Mixed ASCII/non-ASCII.
2361   str = base::UTF8ToUTF16("hellö");
2362   EXPECT_TRUE(IDNToASCII(str.data(), str.length(), &output));
2363   EXPECT_EQ(base::UTF8ToUTF16("xn--hell-8qa"), base::string16(output.data()));
2364   output.set_length(0);
2365
2366   // All non-ASCII.
2367   str = base::UTF8ToUTF16("你好");
2368   EXPECT_TRUE(IDNToASCII(str.data(), str.length(), &output));
2369   EXPECT_EQ(base::UTF8ToUTF16("xn--6qq79v"), base::string16(output.data()));
2370   output.set_length(0);
2371
2372   // Characters that need mapping (the resulting Punycode is the encoding for
2373   // "1⁄4").
2374   str = base::UTF8ToUTF16("¼");
2375   EXPECT_TRUE(IDNToASCII(str.data(), str.length(), &output));
2376   EXPECT_EQ(base::UTF8ToUTF16("xn--14-c6t"), base::string16(output.data()));
2377   output.set_length(0);
2378
2379   // String to encode already starts with "xn--", and all ASCII. Should not
2380   // modify the string.
2381   str = base::UTF8ToUTF16("xn--hell-8qa");
2382   EXPECT_TRUE(IDNToASCII(str.data(), str.length(), &output));
2383   EXPECT_EQ(base::UTF8ToUTF16("xn--hell-8qa"), base::string16(output.data()));
2384   output.set_length(0);
2385
2386   // String to encode already starts with "xn--", and mixed ASCII/non-ASCII.
2387   // Should fail, due to a special case: if the label starts with "xn--", it
2388   // should be parsed as Punycode, which must be all ASCII.
2389   str = base::UTF8ToUTF16("xn--hellö");
2390   EXPECT_FALSE(IDNToASCII(str.data(), str.length(), &output));
2391   output.set_length(0);
2392
2393   // String to encode already starts with "xn--", and mixed ASCII/non-ASCII.
2394   // This tests that there is still an error for the character '⁄' (U+2044),
2395   // which would be a valid ASCII character, U+0044, if the high byte were
2396   // ignored.
2397   str = base::UTF8ToUTF16("xn--1⁄4");
2398   EXPECT_FALSE(IDNToASCII(str.data(), str.length(), &output));
2399   output.set_length(0);
2400 }
2401
2402 }  // namespace url