src/chrome/browser/net/predictor_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include <time.h>
   6
   7 #include <algorithm>
   8 #include <sstream>
   9 #include <string>
  10
  11 #include "base/memory/scoped_ptr.h"
  12 #include "base/message_loop/message_loop.h"
  13 #include "base/strings/string_number_conversions.h"
  14 #include "base/timer/timer.h"
  15 #include "base/values.h"
  16 #include "chrome/browser/net/predictor.h"
  17 #include "chrome/browser/net/spdyproxy/proxy_advisor.h"
  18 #include "chrome/browser/net/url_info.h"
  19 #include "chrome/common/net/predictor_common.h"
  20 #include "content/public/test/test_browser_thread.h"
  21 #include "net/base/address_list.h"
  22 #include "net/base/winsock_init.h"
  23 #include "net/dns/mock_host_resolver.h"
  24 #include "net/http/transport_security_state.h"
  25 #include "testing/gmock/include/gmock/gmock.h"
  26 #include "testing/gtest/include/gtest/gtest.h"
  27
  28 using base::Time;
  29 using base::TimeDelta;
  30 using content::BrowserThread;
  31
  32 namespace chrome_browser_net {
  33
  34 class WaitForResolutionHelper;
  35
  36 typedef base::RepeatingTimer<WaitForResolutionHelper> HelperTimer;
  37
  38 class WaitForResolutionHelper {
  39  public:
  40   WaitForResolutionHelper(Predictor* predictor, const UrlList& hosts,
  41                           HelperTimer* timer, int checks_until_quit)
  42       : predictor_(predictor),
  43         hosts_(hosts),
  44         timer_(timer),
  45         checks_until_quit_(checks_until_quit) {
  46   }
  47
  48   void CheckIfResolutionsDone() {
  49     if (--checks_until_quit_ > 0) {
  50       for (UrlList::const_iterator i = hosts_.begin(); i != hosts_.end(); ++i)
  51         if (predictor_->GetResolutionDuration(*i) ==
  52             UrlInfo::NullDuration())
  53           return;  // We don't have resolution for that host.
  54     }
  55
  56     // When all hostnames have been resolved, or we've hit the limit,
  57     // exit the loop.
  58     timer_->Stop();
  59     base::MessageLoop::current()->Quit();
  60     delete timer_;
  61     delete this;
  62   }
  63
  64  private:
  65   Predictor* predictor_;
  66   const UrlList hosts_;
  67   HelperTimer* timer_;
  68   int checks_until_quit_;
  69 };
  70
  71 class PredictorTest : public testing::Test {
  72  public:
  73   PredictorTest()
  74       : ui_thread_(BrowserThread::UI, &loop_),
  75         io_thread_(BrowserThread::IO, &loop_),
  76         host_resolver_(new net::MockCachingHostResolver()) {
  77   }
  78
  79  protected:
  80   virtual void SetUp() {
  81 #if defined(OS_WIN)
  82     net::EnsureWinsockInit();
  83 #endif
  84     Predictor::set_max_parallel_resolves(
  85         Predictor::kMaxSpeculativeParallelResolves);
  86     Predictor::set_max_queueing_delay(
  87         Predictor::kMaxSpeculativeResolveQueueDelayMs);
  88     // Since we are using a caching HostResolver, the following latencies will
  89     // only be incurred by the first request, after which the result will be
  90     // cached internally by |host_resolver_|.
  91     net::RuleBasedHostResolverProc* rules = host_resolver_->rules();
  92     rules->AddRuleWithLatency("www.google.com", "127.0.0.1", 50);
  93     rules->AddRuleWithLatency("gmail.google.com.com", "127.0.0.1", 70);
  94     rules->AddRuleWithLatency("mail.google.com", "127.0.0.1", 44);
  95     rules->AddRuleWithLatency("gmail.com", "127.0.0.1", 63);
  96   }
  97
  98   void WaitForResolution(Predictor* predictor, const UrlList& hosts) {
  99     HelperTimer* timer = new HelperTimer();
 100     // By default allow the loop to run for a minute -- 600 iterations.
 101     timer->Start(FROM_HERE, TimeDelta::FromMilliseconds(100),
 102                  new WaitForResolutionHelper(predictor, hosts, timer, 600),
 103                  &WaitForResolutionHelper::CheckIfResolutionsDone);
 104     base::MessageLoop::current()->Run();
 105   }
 106
 107   void WaitForResolutionWithLimit(
 108       Predictor* predictor, const UrlList& hosts, int limit) {
 109     HelperTimer* timer = new HelperTimer();
 110     timer->Start(FROM_HERE, TimeDelta::FromMilliseconds(100),
 111                  new WaitForResolutionHelper(predictor, hosts, timer, limit),
 112                  &WaitForResolutionHelper::CheckIfResolutionsDone);
 113     base::MessageLoop::current()->Run();
 114   }
 115
 116  private:
 117   // IMPORTANT: do not move this below |host_resolver_|; the host resolver
 118   // must not outlive the message loop, otherwise bad things can happen
 119   // (like posting to a deleted message loop).
 120   base::MessageLoopForUI loop_;
 121   content::TestBrowserThread ui_thread_;
 122   content::TestBrowserThread io_thread_;
 123
 124  protected:
 125   scoped_ptr<net::MockCachingHostResolver> host_resolver_;
 126 };
 127
 128 //------------------------------------------------------------------------------
 129
 130 TEST_F(PredictorTest, StartupShutdownTest) {
 131   Predictor testing_master(true);
 132   testing_master.Shutdown();
 133 }
 134
 135
 136 TEST_F(PredictorTest, ShutdownWhenResolutionIsPendingTest) {
 137   scoped_ptr<net::HostResolver> host_resolver(new net::HangingHostResolver());
 138
 139   Predictor testing_master(true);
 140   testing_master.SetHostResolver(host_resolver.get());
 141
 142   GURL localhost("http://localhost:80");
 143   UrlList names;
 144   names.push_back(localhost);
 145
 146   testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
 147
 148   base::MessageLoop::current()->PostDelayedTask(
 149       FROM_HERE,
 150       base::MessageLoop::QuitClosure(),
 151       base::TimeDelta::FromMilliseconds(500));
 152   base::MessageLoop::current()->Run();
 153
 154   EXPECT_FALSE(testing_master.WasFound(localhost));
 155
 156   testing_master.Shutdown();
 157
 158   // Clean up after ourselves.
 159   base::MessageLoop::current()->RunUntilIdle();
 160 }
 161
 162 TEST_F(PredictorTest, SingleLookupTest) {
 163   Predictor testing_master(true);
 164   testing_master.SetHostResolver(host_resolver_.get());
 165
 166   GURL goog("http://www.google.com:80");
 167
 168   UrlList names;
 169   names.push_back(goog);
 170
 171   // Try to flood the predictor with many concurrent requests.
 172   for (int i = 0; i < 10; i++)
 173     testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
 174
 175   WaitForResolution(&testing_master, names);
 176
 177   EXPECT_TRUE(testing_master.WasFound(goog));
 178
 179   base::MessageLoop::current()->RunUntilIdle();
 180
 181   EXPECT_GT(testing_master.peak_pending_lookups(), names.size() / 2);
 182   EXPECT_LE(testing_master.peak_pending_lookups(), names.size());
 183   EXPECT_LE(testing_master.peak_pending_lookups(),
 184             testing_master.max_concurrent_dns_lookups());
 185
 186   testing_master.Shutdown();
 187 }
 188
 189 TEST_F(PredictorTest, ConcurrentLookupTest) {
 190   host_resolver_->rules()->AddSimulatedFailure("*.notfound");
 191
 192   Predictor testing_master(true);
 193   testing_master.SetHostResolver(host_resolver_.get());
 194
 195   GURL goog("http://www.google.com:80"),
 196       goog2("http://gmail.google.com.com:80"),
 197       goog3("http://mail.google.com:80"),
 198       goog4("http://gmail.com:80");
 199   GURL bad1("http://bad1.notfound:80"),
 200       bad2("http://bad2.notfound:80");
 201
 202   UrlList names;
 203   names.push_back(goog);
 204   names.push_back(goog3);
 205   names.push_back(bad1);
 206   names.push_back(goog2);
 207   names.push_back(bad2);
 208   names.push_back(goog4);
 209   names.push_back(goog);
 210
 211   // Try to flood the predictor with many concurrent requests.
 212   for (int i = 0; i < 10; i++)
 213     testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
 214
 215   WaitForResolution(&testing_master, names);
 216
 217   EXPECT_TRUE(testing_master.WasFound(goog));
 218   EXPECT_TRUE(testing_master.WasFound(goog3));
 219   EXPECT_TRUE(testing_master.WasFound(goog2));
 220   EXPECT_TRUE(testing_master.WasFound(goog4));
 221   EXPECT_FALSE(testing_master.WasFound(bad1));
 222   EXPECT_FALSE(testing_master.WasFound(bad2));
 223
 224   base::MessageLoop::current()->RunUntilIdle();
 225
 226   EXPECT_FALSE(testing_master.WasFound(bad1));
 227   EXPECT_FALSE(testing_master.WasFound(bad2));
 228
 229   EXPECT_LE(testing_master.peak_pending_lookups(), names.size());
 230   EXPECT_LE(testing_master.peak_pending_lookups(),
 231             testing_master.max_concurrent_dns_lookups());
 232
 233   testing_master.Shutdown();
 234 }
 235
 236 TEST_F(PredictorTest, MassiveConcurrentLookupTest) {
 237   host_resolver_->rules()->AddSimulatedFailure("*.notfound");
 238
 239   Predictor testing_master(true);
 240   testing_master.SetHostResolver(host_resolver_.get());
 241
 242   UrlList names;
 243   for (int i = 0; i < 100; i++)
 244     names.push_back(GURL(
 245         "http://host" + base::IntToString(i) + ".notfound:80"));
 246
 247   // Try to flood the predictor with many concurrent requests.
 248   for (int i = 0; i < 10; i++)
 249     testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
 250
 251   WaitForResolution(&testing_master, names);
 252
 253   base::MessageLoop::current()->RunUntilIdle();
 254
 255   EXPECT_LE(testing_master.peak_pending_lookups(), names.size());
 256   EXPECT_LE(testing_master.peak_pending_lookups(),
 257             testing_master.max_concurrent_dns_lookups());
 258
 259   testing_master.Shutdown();
 260 }
 261
 262 //------------------------------------------------------------------------------
 263 // Functions to help synthesize and test serializations of subresource referrer
 264 // lists.
 265
 266 // Return a motivation_list if we can find one for the given motivating_host (or
 267 // NULL if a match is not found).
 268 static const base::ListValue* FindSerializationMotivation(
 269     const GURL& motivation,
 270     const base::ListValue* referral_list) {
 271   CHECK_LT(0u, referral_list->GetSize());  // Room for version.
 272   int format_version = -1;
 273   CHECK(referral_list->GetInteger(0, &format_version));
 274   CHECK_EQ(Predictor::kPredictorReferrerVersion, format_version);
 275   const base::ListValue* motivation_list(NULL);
 276   for (size_t i = 1; i < referral_list->GetSize(); ++i) {
 277     referral_list->GetList(i, &motivation_list);
 278     std::string existing_spec;
 279     EXPECT_TRUE(motivation_list->GetString(0, &existing_spec));
 280     if (motivation == GURL(existing_spec))
 281       return motivation_list;
 282   }
 283   return NULL;
 284 }
 285
 286 static base::ListValue* FindSerializationMotivation(
 287     const GURL& motivation,
 288     base::ListValue* referral_list) {
 289   return const_cast<base::ListValue*>(FindSerializationMotivation(
 290       motivation, static_cast<const base::ListValue*>(referral_list)));
 291 }
 292
 293 // Create a new empty serialization list.
 294 static base::ListValue* NewEmptySerializationList() {
 295   base::ListValue* list = new base::ListValue;
 296   list->Append(
 297       new base::FundamentalValue(Predictor::kPredictorReferrerVersion));
 298   return list;
 299 }
 300
 301 // Add a motivating_url and a subresource_url to a serialized list, using
 302 // this given latency. This is a helper function for quickly building these
 303 // lists.
 304 static void AddToSerializedList(const GURL& motivation,
 305                                 const GURL& subresource,
 306                                 double use_rate,
 307                                 base::ListValue* referral_list) {
 308   // Find the motivation if it is already used.
 309   base::ListValue* motivation_list = FindSerializationMotivation(motivation,
 310                                                            referral_list);
 311   if (!motivation_list) {
 312     // This is the first mention of this motivation, so build a list.
 313     motivation_list = new base::ListValue;
 314     motivation_list->Append(new base::StringValue(motivation.spec()));
 315     // Provide empty subresource list.
 316     motivation_list->Append(new base::ListValue());
 317
 318     // ...and make it part of the serialized referral_list.
 319     referral_list->Append(motivation_list);
 320   }
 321
 322   base::ListValue* subresource_list(NULL);
 323   // 0 == url; 1 == subresource_list.
 324   EXPECT_TRUE(motivation_list->GetList(1, &subresource_list));
 325
 326   // We won't bother to check for the subresource being there already.  Worst
 327   // case, during deserialization, the latency value we supply plus the
 328   // existing value(s) will be added to the referrer.
 329
 330   subresource_list->Append(new base::StringValue(subresource.spec()));
 331   subresource_list->Append(new base::FundamentalValue(use_rate));
 332 }
 333
 334 // For a given motivation, and subresource, find what latency is currently
 335 // listed.  This assume a well formed serialization, which has at most one such
 336 // entry for any pair of names.  If no such pair is found, then return false.
 337 // Data is written into use_rate arguments.
 338 static bool GetDataFromSerialization(const GURL& motivation,
 339                                      const GURL& subresource,
 340                                      const base::ListValue& referral_list,
 341                                      double* use_rate) {
 342   const base::ListValue* motivation_list =
 343       FindSerializationMotivation(motivation, &referral_list);
 344   if (!motivation_list)
 345     return false;
 346   const base::ListValue* subresource_list;
 347   EXPECT_TRUE(motivation_list->GetList(1, &subresource_list));
 348   for (size_t i = 0; i < subresource_list->GetSize();) {
 349     std::string url_spec;
 350     EXPECT_TRUE(subresource_list->GetString(i++, &url_spec));
 351     EXPECT_TRUE(subresource_list->GetDouble(i++, use_rate));
 352     if (subresource == GURL(url_spec)) {
 353       return true;
 354     }
 355   }
 356   return false;
 357 }
 358
 359 //------------------------------------------------------------------------------
 360
 361 // Make sure nil referral lists really have no entries, and no latency listed.
 362 TEST_F(PredictorTest, ReferrerSerializationNilTest) {
 363   Predictor predictor(true);
 364   predictor.SetHostResolver(host_resolver_.get());
 365
 366   scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
 367   predictor.SerializeReferrers(referral_list.get());
 368   EXPECT_EQ(1U, referral_list->GetSize());
 369   EXPECT_FALSE(GetDataFromSerialization(
 370     GURL("http://a.com:79"), GURL("http://b.com:78"),
 371       *referral_list.get(), NULL));
 372
 373   predictor.Shutdown();
 374 }
 375
 376 // Make sure that when a serialization list includes a value, that it can be
 377 // deserialized into the database, and can be extracted back out via
 378 // serialization without being changed.
 379 TEST_F(PredictorTest, ReferrerSerializationSingleReferrerTest) {
 380   Predictor predictor(true);
 381   predictor.SetHostResolver(host_resolver_.get());
 382   const GURL motivation_url("http://www.google.com:91");
 383   const GURL subresource_url("http://icons.google.com:90");
 384   const double kUseRate = 23.4;
 385   scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
 386
 387   AddToSerializedList(motivation_url, subresource_url,
 388       kUseRate, referral_list.get());
 389
 390   predictor.DeserializeReferrers(*referral_list.get());
 391
 392   base::ListValue recovered_referral_list;
 393   predictor.SerializeReferrers(&recovered_referral_list);
 394   EXPECT_EQ(2U, recovered_referral_list.GetSize());
 395   double rate;
 396   EXPECT_TRUE(GetDataFromSerialization(
 397       motivation_url, subresource_url, recovered_referral_list, &rate));
 398   EXPECT_EQ(rate, kUseRate);
 399
 400   predictor.Shutdown();
 401 }
 402
 403 // Check that GetHtmlReferrerLists() doesn't crash when given duplicated
 404 // domains for referring URL, and that it sorts the results in the
 405 // correct order.
 406 TEST_F(PredictorTest, GetHtmlReferrerLists) {
 407   Predictor predictor(true);
 408   predictor.SetHostResolver(host_resolver_.get());
 409   const double kUseRate = 23.4;
 410   scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
 411
 412   AddToSerializedList(
 413       GURL("http://d.google.com/x1"),
 414       GURL("http://foo.com/"),
 415       kUseRate, referral_list.get());
 416
 417   // Duplicated hostname (d.google.com). This should not cause any crashes
 418   // (i.e. crbug.com/116345)
 419   AddToSerializedList(
 420       GURL("http://d.google.com/x2"),
 421       GURL("http://foo.com/"),
 422       kUseRate, referral_list.get());
 423
 424   AddToSerializedList(
 425       GURL("http://a.yahoo.com/y"),
 426       GURL("http://foo1.com/"),
 427       kUseRate, referral_list.get());
 428
 429   AddToSerializedList(
 430       GURL("http://b.google.com/x3"),
 431       GURL("http://foo2.com/"),
 432       kUseRate, referral_list.get());
 433
 434   AddToSerializedList(
 435       GURL("http://d.yahoo.com/x5"),
 436       GURL("http://i.like.turtles/"),
 437       kUseRate, referral_list.get());
 438
 439   AddToSerializedList(
 440       GURL("http://c.yahoo.com/x4"),
 441       GURL("http://foo3.com/"),
 442       kUseRate, referral_list.get());
 443
 444   predictor.DeserializeReferrers(*referral_list.get());
 445
 446   std::string html;
 447   predictor.GetHtmlReferrerLists(&html);
 448
 449   // The lexicographic sorting of hostnames would be:
 450   //   a.yahoo.com
 451   //   b.google.com
 452   //   c.yahoo.com
 453   //   d.google.com
 454   //   d.yahoo.com
 455   //
 456   // However we expect to sort them by domain in the output:
 457   //   b.google.com
 458   //   d.google.com
 459   //   a.yahoo.com
 460   //   c.yahoo.com
 461   //   d.yahoo.com
 462
 463   size_t pos[] = {
 464       html.find("<td rowspan=1>http://b.google.com/x3"),
 465       html.find("<td rowspan=1>http://d.google.com/x1"),
 466       html.find("<td rowspan=1>http://d.google.com/x2"),
 467       html.find("<td rowspan=1>http://a.yahoo.com/y"),
 468       html.find("<td rowspan=1>http://c.yahoo.com/x4"),
 469       html.find("<td rowspan=1>http://d.yahoo.com/x5"),
 470   };
 471
 472   // Make sure things appeared in the expected order.
 473   for (size_t i = 1; i < arraysize(pos); ++i) {
 474     EXPECT_LT(pos[i - 1], pos[i]) << "Mismatch for pos[" << i << "]";
 475   }
 476
 477   predictor.Shutdown();
 478 }
 479
 480 // Verify that two floats are within 1% of each other in value.
 481 #define EXPECT_SIMILAR(a, b) do { \
 482     double espilon_ratio = 1.01;  \
 483     if ((a) < 0.)  \
 484       espilon_ratio = 1 / espilon_ratio;  \
 485     EXPECT_LT(a, espilon_ratio * (b));   \
 486     EXPECT_GT((a) * espilon_ratio, b);   \
 487     } while (0)
 488
 489
 490 // Make sure the Trim() functionality works as expected.
 491 TEST_F(PredictorTest, ReferrerSerializationTrimTest) {
 492   Predictor predictor(true);
 493   predictor.SetHostResolver(host_resolver_.get());
 494   GURL motivation_url("http://www.google.com:110");
 495
 496   GURL icon_subresource_url("http://icons.google.com:111");
 497   const double kRateIcon = 16.0 * Predictor::kDiscardableExpectedValue;
 498   GURL img_subresource_url("http://img.google.com:118");
 499   const double kRateImg = 8.0 * Predictor::kDiscardableExpectedValue;
 500
 501   scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
 502   AddToSerializedList(
 503       motivation_url, icon_subresource_url, kRateIcon, referral_list.get());
 504   AddToSerializedList(
 505       motivation_url, img_subresource_url, kRateImg, referral_list.get());
 506
 507   predictor.DeserializeReferrers(*referral_list.get());
 508
 509   base::ListValue recovered_referral_list;
 510   predictor.SerializeReferrers(&recovered_referral_list);
 511   EXPECT_EQ(2U, recovered_referral_list.GetSize());
 512   double rate;
 513   EXPECT_TRUE(GetDataFromSerialization(
 514       motivation_url, icon_subresource_url, recovered_referral_list,
 515       &rate));
 516   EXPECT_SIMILAR(rate, kRateIcon);
 517
 518   EXPECT_TRUE(GetDataFromSerialization(
 519       motivation_url, img_subresource_url, recovered_referral_list, &rate));
 520   EXPECT_SIMILAR(rate, kRateImg);
 521
 522   // Each time we Trim 24 times, the user_rate figures should reduce by a factor
 523   // of two,  until they are small, and then a trim will delete the whole entry.
 524   for (int i = 0; i < 24; ++i)
 525     predictor.TrimReferrersNow();
 526   predictor.SerializeReferrers(&recovered_referral_list);
 527   EXPECT_EQ(2U, recovered_referral_list.GetSize());
 528   EXPECT_TRUE(GetDataFromSerialization(
 529       motivation_url, icon_subresource_url, recovered_referral_list, &rate));
 530   EXPECT_SIMILAR(rate, kRateIcon / 2);
 531
 532   EXPECT_TRUE(GetDataFromSerialization(
 533       motivation_url, img_subresource_url, recovered_referral_list, &rate));
 534   EXPECT_SIMILAR(rate, kRateImg / 2);
 535
 536   for (int i = 0; i < 24; ++i)
 537     predictor.TrimReferrersNow();
 538   predictor.SerializeReferrers(&recovered_referral_list);
 539   EXPECT_EQ(2U, recovered_referral_list.GetSize());
 540   EXPECT_TRUE(GetDataFromSerialization(
 541       motivation_url, icon_subresource_url, recovered_referral_list, &rate));
 542   EXPECT_SIMILAR(rate, kRateIcon / 4);
 543   EXPECT_TRUE(GetDataFromSerialization(
 544       motivation_url, img_subresource_url, recovered_referral_list, &rate));
 545   EXPECT_SIMILAR(rate, kRateImg / 4);
 546
 547   for (int i = 0; i < 24; ++i)
 548     predictor.TrimReferrersNow();
 549   predictor.SerializeReferrers(&recovered_referral_list);
 550   EXPECT_EQ(2U, recovered_referral_list.GetSize());
 551   EXPECT_TRUE(GetDataFromSerialization(
 552       motivation_url, icon_subresource_url, recovered_referral_list, &rate));
 553   EXPECT_SIMILAR(rate, kRateIcon / 8);
 554
 555   // Img is below threshold, and so it gets deleted.
 556   EXPECT_FALSE(GetDataFromSerialization(
 557       motivation_url, img_subresource_url, recovered_referral_list, &rate));
 558
 559   for (int i = 0; i < 24; ++i)
 560     predictor.TrimReferrersNow();
 561   predictor.SerializeReferrers(&recovered_referral_list);
 562   // Icon is also trimmed away, so entire set gets discarded.
 563   EXPECT_EQ(1U, recovered_referral_list.GetSize());
 564   EXPECT_FALSE(GetDataFromSerialization(
 565       motivation_url, icon_subresource_url, recovered_referral_list, &rate));
 566   EXPECT_FALSE(GetDataFromSerialization(
 567       motivation_url, img_subresource_url, recovered_referral_list, &rate));
 568
 569   predictor.Shutdown();
 570 }
 571
 572
 573 TEST_F(PredictorTest, PriorityQueuePushPopTest) {
 574   Predictor::HostNameQueue queue;
 575
 576   GURL first("http://first:80"), second("http://second:90");
 577
 578   // First check high priority queue FIFO functionality.
 579   EXPECT_TRUE(queue.IsEmpty());
 580   queue.Push(first, UrlInfo::LEARNED_REFERAL_MOTIVATED);
 581   EXPECT_FALSE(queue.IsEmpty());
 582   queue.Push(second, UrlInfo::MOUSE_OVER_MOTIVATED);
 583   EXPECT_FALSE(queue.IsEmpty());
 584   EXPECT_EQ(queue.Pop(), first);
 585   EXPECT_FALSE(queue.IsEmpty());
 586   EXPECT_EQ(queue.Pop(), second);
 587   EXPECT_TRUE(queue.IsEmpty());
 588
 589   // Then check low priority queue FIFO functionality.
 590   queue.Push(first, UrlInfo::PAGE_SCAN_MOTIVATED);
 591   EXPECT_FALSE(queue.IsEmpty());
 592   queue.Push(second, UrlInfo::OMNIBOX_MOTIVATED);
 593   EXPECT_FALSE(queue.IsEmpty());
 594   EXPECT_EQ(queue.Pop(), first);
 595   EXPECT_FALSE(queue.IsEmpty());
 596   EXPECT_EQ(queue.Pop(), second);
 597   EXPECT_TRUE(queue.IsEmpty());
 598 }
 599
 600 TEST_F(PredictorTest, PriorityQueueReorderTest) {
 601   Predictor::HostNameQueue queue;
 602
 603   // Push all the low priority items.
 604   GURL low1("http://low1:80"),
 605       low2("http://low2:80"),
 606       low3("http://low3:443"),
 607       low4("http://low4:80"),
 608       low5("http://low5:80"),
 609       hi1("http://hi1:80"),
 610       hi2("http://hi2:80"),
 611       hi3("http://hi3:80");
 612
 613   EXPECT_TRUE(queue.IsEmpty());
 614   queue.Push(low1, UrlInfo::PAGE_SCAN_MOTIVATED);
 615   queue.Push(low2, UrlInfo::UNIT_TEST_MOTIVATED);
 616   queue.Push(low3, UrlInfo::LINKED_MAX_MOTIVATED);
 617   queue.Push(low4, UrlInfo::OMNIBOX_MOTIVATED);
 618   queue.Push(low5, UrlInfo::STARTUP_LIST_MOTIVATED);
 619   queue.Push(low4, UrlInfo::OMNIBOX_MOTIVATED);
 620
 621   // Push all the high prority items
 622   queue.Push(hi1, UrlInfo::LEARNED_REFERAL_MOTIVATED);
 623   queue.Push(hi2, UrlInfo::STATIC_REFERAL_MOTIVATED);
 624   queue.Push(hi3, UrlInfo::MOUSE_OVER_MOTIVATED);
 625
 626   // Check that high priority stuff comes out first, and in FIFO order.
 627   EXPECT_EQ(queue.Pop(), hi1);
 628   EXPECT_EQ(queue.Pop(), hi2);
 629   EXPECT_EQ(queue.Pop(), hi3);
 630
 631   // ...and then low priority strings.
 632   EXPECT_EQ(queue.Pop(), low1);
 633   EXPECT_EQ(queue.Pop(), low2);
 634   EXPECT_EQ(queue.Pop(), low3);
 635   EXPECT_EQ(queue.Pop(), low4);
 636   EXPECT_EQ(queue.Pop(), low5);
 637   EXPECT_EQ(queue.Pop(), low4);
 638
 639   EXPECT_TRUE(queue.IsEmpty());
 640 }
 641
 642 TEST_F(PredictorTest, CanonicalizeUrl) {
 643   // Base case, only handles HTTP and HTTPS.
 644   EXPECT_EQ(GURL(), Predictor::CanonicalizeUrl(GURL("ftp://anything")));
 645
 646   // Remove path testing.
 647   GURL long_url("http://host:999/path?query=value");
 648   EXPECT_EQ(Predictor::CanonicalizeUrl(long_url), long_url.GetWithEmptyPath());
 649
 650   // Default port cannoncalization.
 651   GURL implied_port("http://test");
 652   GURL explicit_port("http://test:80");
 653   EXPECT_EQ(Predictor::CanonicalizeUrl(implied_port),
 654             Predictor::CanonicalizeUrl(explicit_port));
 655
 656   // Port is still maintained.
 657   GURL port_80("http://test:80");
 658   GURL port_90("http://test:90");
 659   EXPECT_NE(Predictor::CanonicalizeUrl(port_80),
 660             Predictor::CanonicalizeUrl(port_90));
 661
 662   // Host is still maintained.
 663   GURL host_1("http://test_1");
 664   GURL host_2("http://test_2");
 665   EXPECT_NE(Predictor::CanonicalizeUrl(host_1),
 666             Predictor::CanonicalizeUrl(host_2));
 667
 668   // Scheme is maintained (mismatch identified).
 669   GURL http("http://test");
 670   GURL https("https://test");
 671   EXPECT_NE(Predictor::CanonicalizeUrl(http),
 672             Predictor::CanonicalizeUrl(https));
 673
 674   // Https works fine.
 675   GURL long_https("https://host:999/path?query=value");
 676   EXPECT_EQ(Predictor::CanonicalizeUrl(long_https),
 677             long_https.GetWithEmptyPath());
 678 }
 679
 680 TEST_F(PredictorTest, DiscardPredictorResults) {
 681   Predictor predictor(true);
 682   predictor.SetHostResolver(host_resolver_.get());
 683   base::ListValue referral_list;
 684   predictor.SerializeReferrers(&referral_list);
 685   EXPECT_EQ(1U, referral_list.GetSize());
 686
 687   GURL host_1("http://test_1");
 688   GURL host_2("http://test_2");
 689   predictor.LearnFromNavigation(host_1, host_2);
 690
 691   predictor.SerializeReferrers(&referral_list);
 692   EXPECT_EQ(2U, referral_list.GetSize());
 693
 694   predictor.DiscardAllResults();
 695   predictor.SerializeReferrers(&referral_list);
 696   EXPECT_EQ(1U, referral_list.GetSize());
 697
 698   predictor.Shutdown();
 699 }
 700
 701 class TestPredictorObserver : public PredictorObserver {
 702  public:
 703   // PredictorObserver implementation:
 704   virtual void OnPreconnectUrl(const GURL& url,
 705                                const GURL& first_party_for_cookies,
 706                                UrlInfo::ResolutionMotivation motivation,
 707                                int count) OVERRIDE {
 708     preconnected_urls_.push_back(url);
 709   }
 710
 711   std::vector<GURL> preconnected_urls_;
 712 };
 713
 714 // Tests that preconnects apply the HSTS list.
 715 TEST_F(PredictorTest, HSTSRedirect) {
 716   const GURL kHttpUrl("http://example.com");
 717   const GURL kHttpsUrl("https://example.com");
 718
 719   const base::Time expiry =
 720       base::Time::Now() + base::TimeDelta::FromSeconds(1000);
 721   net::TransportSecurityState state;
 722   state.AddHSTS(kHttpUrl.host(), expiry, false);
 723
 724   Predictor predictor(true);
 725   TestPredictorObserver observer;
 726   predictor.SetObserver(&observer);
 727   predictor.SetTransportSecurityState(&state);
 728
 729   predictor.PreconnectUrl(kHttpUrl, GURL(), UrlInfo::OMNIBOX_MOTIVATED, 2);
 730   ASSERT_EQ(1u, observer.preconnected_urls_.size());
 731   EXPECT_EQ(kHttpsUrl, observer.preconnected_urls_[0]);
 732
 733   predictor.Shutdown();
 734 }
 735
 736 // Tests that preconnecting a URL on the HSTS list preconnects the subresources
 737 // for the SSL version.
 738 TEST_F(PredictorTest, HSTSRedirectSubresources) {
 739   const GURL kHttpUrl("http://example.com");
 740   const GURL kHttpsUrl("https://example.com");
 741   const GURL kSubresourceUrl("https://images.example.com");
 742   const double kUseRate = 23.4;
 743
 744   const base::Time expiry =
 745       base::Time::Now() + base::TimeDelta::FromSeconds(1000);
 746   net::TransportSecurityState state;
 747   state.AddHSTS(kHttpUrl.host(), expiry, false);
 748
 749   Predictor predictor(true);
 750   TestPredictorObserver observer;
 751   predictor.SetObserver(&observer);
 752   predictor.SetTransportSecurityState(&state);
 753
 754   scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
 755   AddToSerializedList(
 756       kHttpsUrl, kSubresourceUrl, kUseRate, referral_list.get());
 757   predictor.DeserializeReferrers(*referral_list.get());
 758
 759   predictor.PreconnectUrlAndSubresources(kHttpUrl, GURL());
 760   ASSERT_EQ(2u, observer.preconnected_urls_.size());
 761   EXPECT_EQ(kHttpsUrl, observer.preconnected_urls_[0]);
 762   EXPECT_EQ(kSubresourceUrl, observer.preconnected_urls_[1]);
 763
 764   predictor.Shutdown();
 765 }
 766
 767 #if defined(OS_ANDROID) || defined(OS_IOS)
 768 // Tests for the predictor with a proxy advisor
 769
 770 class TestProxyAdvisor : public ProxyAdvisor {
 771  public:
 772   TestProxyAdvisor()
 773       : ProxyAdvisor(NULL, NULL),
 774         would_proxy_(false),
 775         advise_count_(0),
 776         would_proxy_count_(0) {
 777   }
 778
 779   virtual ~TestProxyAdvisor() {}
 780
 781   virtual void Advise(const GURL& url,
 782                       UrlInfo::ResolutionMotivation motivation,
 783                       bool is_preconnect) OVERRIDE {
 784     ++advise_count_;
 785   }
 786
 787   virtual bool WouldProxyURL(const GURL& url) OVERRIDE {
 788     ++would_proxy_count_;
 789     return would_proxy_;
 790   }
 791
 792   bool would_proxy_;
 793   int advise_count_;
 794   int would_proxy_count_;
 795 };
 796
 797 TEST_F(PredictorTest, SingleLookupTestWithDisabledAdvisor) {
 798   Predictor testing_master(true);
 799   TestProxyAdvisor* advisor = new TestProxyAdvisor();
 800   testing_master.SetHostResolver(host_resolver_.get());
 801   testing_master.proxy_advisor_.reset(advisor);
 802
 803   GURL goog("http://www.google.com:80");
 804
 805   advisor->would_proxy_ = false;
 806
 807   UrlList names;
 808   names.push_back(goog);
 809   testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
 810
 811   WaitForResolution(&testing_master, names);
 812   EXPECT_TRUE(testing_master.WasFound(goog));
 813   EXPECT_EQ(advisor->would_proxy_count_, 1);
 814   EXPECT_EQ(advisor->advise_count_, 1);
 815
 816   base::MessageLoop::current()->RunUntilIdle();
 817
 818   testing_master.Shutdown();
 819 }
 820
 821 TEST_F(PredictorTest, SingleLookupTestWithEnabledAdvisor) {
 822   Predictor testing_master(true);
 823   testing_master.SetHostResolver(host_resolver_.get());
 824   TestProxyAdvisor* advisor = new TestProxyAdvisor();
 825   testing_master.proxy_advisor_.reset(advisor);
 826
 827   GURL goog("http://www.google.com:80");
 828
 829   advisor->would_proxy_ = true;
 830
 831   UrlList names;
 832   names.push_back(goog);
 833
 834   testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
 835
 836   // Attempt to resolve a few times.
 837   WaitForResolutionWithLimit(&testing_master, names, 10);
 838
 839   // Because the advisor indicated that the url would be proxied,
 840   // no resolution should have occurred.
 841   EXPECT_FALSE(testing_master.WasFound(goog));
 842   EXPECT_EQ(advisor->would_proxy_count_, 1);
 843   EXPECT_EQ(advisor->advise_count_, 1);
 844
 845   base::MessageLoop::current()->RunUntilIdle();
 846
 847   testing_master.Shutdown();
 848 }
 849
 850 TEST_F(PredictorTest, TestSimplePreconnectAdvisor) {
 851   Predictor testing_master(true);
 852   testing_master.SetHostResolver(host_resolver_.get());
 853   TestProxyAdvisor* advisor = new TestProxyAdvisor();
 854   testing_master.proxy_advisor_.reset(advisor);
 855
 856   GURL goog("http://www.google.com:80");
 857
 858   testing_master.PreconnectUrl(goog, goog, UrlInfo::OMNIBOX_MOTIVATED, 2);
 859
 860   EXPECT_EQ(advisor->would_proxy_count_, 0);
 861   EXPECT_EQ(advisor->advise_count_, 1);
 862
 863   testing_master.Shutdown();
 864 }
 865
 866 #endif  // defined(OS_ANDROID) || defined(OS_IOS)
 867
 868 }  // namespace chrome_browser_net