src/chrome/browser/net/predictor_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include <time.h>
   6
   7 #include <algorithm>
   8 #include <sstream>
   9 #include <string>
  10
  11 #include "base/memory/scoped_ptr.h"
  12 #include "base/message_loop/message_loop.h"
  13 #include "base/strings/string_number_conversions.h"
  14 #include "base/timer/timer.h"
  15 #include "base/values.h"
  16 #include "chrome/browser/net/predictor.h"
  17 #include "chrome/browser/net/spdyproxy/proxy_advisor.h"
  18 #include "chrome/browser/net/url_info.h"
  19 #include "chrome/common/net/predictor_common.h"
  20 #include "content/public/test/test_browser_thread.h"
  21 #include "net/base/address_list.h"
  22 #include "net/base/load_flags.h"
  23 #include "net/base/net_errors.h"
  24 #include "net/base/winsock_init.h"
  25 #include "net/dns/mock_host_resolver.h"
  26 #include "net/http/transport_security_state.h"
  27 #include "net/proxy/proxy_config_service_fixed.h"
  28 #include "net/proxy/proxy_service.h"
  29 #include "testing/gmock/include/gmock/gmock.h"
  30 #include "testing/gtest/include/gtest/gtest.h"
  31
  32 using base::Time;
  33 using base::TimeDelta;
  34 using content::BrowserThread;
  35
  36 namespace chrome_browser_net {
  37
  38 class WaitForResolutionHelper;
  39
  40 typedef base::RepeatingTimer<WaitForResolutionHelper> HelperTimer;
  41
  42 class WaitForResolutionHelper {
  43  public:
  44   WaitForResolutionHelper(Predictor* predictor, const UrlList& hosts,
  45                           HelperTimer* timer, int checks_until_quit)
  46       : predictor_(predictor),
  47         hosts_(hosts),
  48         timer_(timer),
  49         checks_until_quit_(checks_until_quit) {
  50   }
  51
  52   void CheckIfResolutionsDone() {
  53     if (--checks_until_quit_ > 0) {
  54       for (UrlList::const_iterator i = hosts_.begin(); i != hosts_.end(); ++i)
  55         if (predictor_->GetResolutionDuration(*i) ==
  56             UrlInfo::NullDuration())
  57           return;  // We don't have resolution for that host.
  58     }
  59
  60     // When all hostnames have been resolved, or we've hit the limit,
  61     // exit the loop.
  62     timer_->Stop();
  63     base::MessageLoop::current()->Quit();
  64     delete timer_;
  65     delete this;
  66   }
  67
  68  private:
  69   Predictor* predictor_;
  70   const UrlList hosts_;
  71   HelperTimer* timer_;
  72   int checks_until_quit_;
  73 };
  74
  75 class PredictorTest : public testing::Test {
  76  public:
  77   PredictorTest()
  78       : ui_thread_(BrowserThread::UI, &loop_),
  79         io_thread_(BrowserThread::IO, &loop_),
  80         host_resolver_(new net::MockCachingHostResolver()) {
  81   }
  82
  83  protected:
  84   void SetUp() override {
  85 #if defined(OS_WIN)
  86     net::EnsureWinsockInit();
  87 #endif
  88     Predictor::set_max_parallel_resolves(
  89         Predictor::kMaxSpeculativeParallelResolves);
  90     Predictor::set_max_queueing_delay(
  91         Predictor::kMaxSpeculativeResolveQueueDelayMs);
  92     // Since we are using a caching HostResolver, the following latencies will
  93     // only be incurred by the first request, after which the result will be
  94     // cached internally by |host_resolver_|.
  95     net::RuleBasedHostResolverProc* rules = host_resolver_->rules();
  96     rules->AddRuleWithLatency("www.google.com", "127.0.0.1", 50);
  97     rules->AddRuleWithLatency("gmail.google.com.com", "127.0.0.1", 70);
  98     rules->AddRuleWithLatency("mail.google.com", "127.0.0.1", 44);
  99     rules->AddRuleWithLatency("gmail.com", "127.0.0.1", 63);
 100   }
 101
 102   void WaitForResolution(Predictor* predictor, const UrlList& hosts) {
 103     HelperTimer* timer = new HelperTimer();
 104     // By default allow the loop to run for a minute -- 600 iterations.
 105     timer->Start(FROM_HERE, TimeDelta::FromMilliseconds(100),
 106                  new WaitForResolutionHelper(predictor, hosts, timer, 600),
 107                  &WaitForResolutionHelper::CheckIfResolutionsDone);
 108     base::MessageLoop::current()->Run();
 109   }
 110
 111   void WaitForResolutionWithLimit(
 112       Predictor* predictor, const UrlList& hosts, int limit) {
 113     HelperTimer* timer = new HelperTimer();
 114     timer->Start(FROM_HERE, TimeDelta::FromMilliseconds(100),
 115                  new WaitForResolutionHelper(predictor, hosts, timer, limit),
 116                  &WaitForResolutionHelper::CheckIfResolutionsDone);
 117     base::MessageLoop::current()->Run();
 118   }
 119
 120  private:
 121   // IMPORTANT: do not move this below |host_resolver_|; the host resolver
 122   // must not outlive the message loop, otherwise bad things can happen
 123   // (like posting to a deleted message loop).
 124   base::MessageLoopForUI loop_;
 125   content::TestBrowserThread ui_thread_;
 126   content::TestBrowserThread io_thread_;
 127
 128  protected:
 129   scoped_ptr<net::MockCachingHostResolver> host_resolver_;
 130 };
 131
 132 //------------------------------------------------------------------------------
 133
 134 TEST_F(PredictorTest, StartupShutdownTest) {
 135   Predictor testing_master(true, true);
 136   testing_master.Shutdown();
 137 }
 138
 139
 140 TEST_F(PredictorTest, ShutdownWhenResolutionIsPendingTest) {
 141   scoped_ptr<net::HostResolver> host_resolver(new net::HangingHostResolver());
 142
 143   Predictor testing_master(true, true);
 144   testing_master.SetHostResolver(host_resolver.get());
 145
 146   GURL localhost("http://localhost:80");
 147   UrlList names;
 148   names.push_back(localhost);
 149
 150   testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
 151
 152   base::MessageLoop::current()->PostDelayedTask(
 153       FROM_HERE,
 154       base::MessageLoop::QuitClosure(),
 155       base::TimeDelta::FromMilliseconds(500));
 156   base::MessageLoop::current()->Run();
 157
 158   EXPECT_FALSE(testing_master.WasFound(localhost));
 159
 160   testing_master.Shutdown();
 161
 162   // Clean up after ourselves.
 163   base::MessageLoop::current()->RunUntilIdle();
 164 }
 165
 166 TEST_F(PredictorTest, SingleLookupTest) {
 167   Predictor testing_master(true, true);
 168   testing_master.SetHostResolver(host_resolver_.get());
 169
 170   GURL goog("http://www.google.com:80");
 171
 172   UrlList names;
 173   names.push_back(goog);
 174
 175   // Try to flood the predictor with many concurrent requests.
 176   for (int i = 0; i < 10; i++)
 177     testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
 178
 179   WaitForResolution(&testing_master, names);
 180
 181   EXPECT_TRUE(testing_master.WasFound(goog));
 182
 183   base::MessageLoop::current()->RunUntilIdle();
 184
 185   EXPECT_GT(testing_master.peak_pending_lookups(), names.size() / 2);
 186   EXPECT_LE(testing_master.peak_pending_lookups(), names.size());
 187   EXPECT_LE(testing_master.peak_pending_lookups(),
 188             testing_master.max_concurrent_dns_lookups());
 189
 190   testing_master.Shutdown();
 191 }
 192
 193 TEST_F(PredictorTest, ConcurrentLookupTest) {
 194   host_resolver_->rules()->AddSimulatedFailure("*.notfound");
 195
 196   Predictor testing_master(true, true);
 197   testing_master.SetHostResolver(host_resolver_.get());
 198
 199   GURL goog("http://www.google.com:80"),
 200       goog2("http://gmail.google.com.com:80"),
 201       goog3("http://mail.google.com:80"),
 202       goog4("http://gmail.com:80");
 203   GURL bad1("http://bad1.notfound:80"),
 204       bad2("http://bad2.notfound:80");
 205
 206   UrlList names;
 207   names.push_back(goog);
 208   names.push_back(goog3);
 209   names.push_back(bad1);
 210   names.push_back(goog2);
 211   names.push_back(bad2);
 212   names.push_back(goog4);
 213   names.push_back(goog);
 214
 215   // Try to flood the predictor with many concurrent requests.
 216   for (int i = 0; i < 10; i++)
 217     testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
 218
 219   WaitForResolution(&testing_master, names);
 220
 221   EXPECT_TRUE(testing_master.WasFound(goog));
 222   EXPECT_TRUE(testing_master.WasFound(goog3));
 223   EXPECT_TRUE(testing_master.WasFound(goog2));
 224   EXPECT_TRUE(testing_master.WasFound(goog4));
 225   EXPECT_FALSE(testing_master.WasFound(bad1));
 226   EXPECT_FALSE(testing_master.WasFound(bad2));
 227
 228   base::MessageLoop::current()->RunUntilIdle();
 229
 230   EXPECT_FALSE(testing_master.WasFound(bad1));
 231   EXPECT_FALSE(testing_master.WasFound(bad2));
 232
 233   EXPECT_LE(testing_master.peak_pending_lookups(), names.size());
 234   EXPECT_LE(testing_master.peak_pending_lookups(),
 235             testing_master.max_concurrent_dns_lookups());
 236
 237   testing_master.Shutdown();
 238 }
 239
 240 TEST_F(PredictorTest, MassiveConcurrentLookupTest) {
 241   host_resolver_->rules()->AddSimulatedFailure("*.notfound");
 242
 243   Predictor testing_master(true, true);
 244   testing_master.SetHostResolver(host_resolver_.get());
 245
 246   UrlList names;
 247   for (int i = 0; i < 100; i++)
 248     names.push_back(GURL(
 249         "http://host" + base::IntToString(i) + ".notfound:80"));
 250
 251   // Try to flood the predictor with many concurrent requests.
 252   for (int i = 0; i < 10; i++)
 253     testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
 254
 255   WaitForResolution(&testing_master, names);
 256
 257   base::MessageLoop::current()->RunUntilIdle();
 258
 259   EXPECT_LE(testing_master.peak_pending_lookups(), names.size());
 260   EXPECT_LE(testing_master.peak_pending_lookups(),
 261             testing_master.max_concurrent_dns_lookups());
 262
 263   testing_master.Shutdown();
 264 }
 265
 266 //------------------------------------------------------------------------------
 267 // Functions to help synthesize and test serializations of subresource referrer
 268 // lists.
 269
 270 // Return a motivation_list if we can find one for the given motivating_host (or
 271 // NULL if a match is not found).
 272 static const base::ListValue* FindSerializationMotivation(
 273     const GURL& motivation,
 274     const base::ListValue* referral_list) {
 275   CHECK_LT(0u, referral_list->GetSize());  // Room for version.
 276   int format_version = -1;
 277   CHECK(referral_list->GetInteger(0, &format_version));
 278   CHECK_EQ(Predictor::kPredictorReferrerVersion, format_version);
 279   const base::ListValue* motivation_list(NULL);
 280   for (size_t i = 1; i < referral_list->GetSize(); ++i) {
 281     referral_list->GetList(i, &motivation_list);
 282     std::string existing_spec;
 283     EXPECT_TRUE(motivation_list->GetString(0, &existing_spec));
 284     if (motivation == GURL(existing_spec))
 285       return motivation_list;
 286   }
 287   return NULL;
 288 }
 289
 290 static base::ListValue* FindSerializationMotivation(
 291     const GURL& motivation,
 292     base::ListValue* referral_list) {
 293   return const_cast<base::ListValue*>(FindSerializationMotivation(
 294       motivation, static_cast<const base::ListValue*>(referral_list)));
 295 }
 296
 297 // Create a new empty serialization list.
 298 static base::ListValue* NewEmptySerializationList() {
 299   base::ListValue* list = new base::ListValue;
 300   list->Append(
 301       new base::FundamentalValue(Predictor::kPredictorReferrerVersion));
 302   return list;
 303 }
 304
 305 // Add a motivating_url and a subresource_url to a serialized list, using
 306 // this given latency. This is a helper function for quickly building these
 307 // lists.
 308 static void AddToSerializedList(const GURL& motivation,
 309                                 const GURL& subresource,
 310                                 double use_rate,
 311                                 base::ListValue* referral_list) {
 312   // Find the motivation if it is already used.
 313   base::ListValue* motivation_list = FindSerializationMotivation(motivation,
 314                                                            referral_list);
 315   if (!motivation_list) {
 316     // This is the first mention of this motivation, so build a list.
 317     motivation_list = new base::ListValue;
 318     motivation_list->Append(new base::StringValue(motivation.spec()));
 319     // Provide empty subresource list.
 320     motivation_list->Append(new base::ListValue());
 321
 322     // ...and make it part of the serialized referral_list.
 323     referral_list->Append(motivation_list);
 324   }
 325
 326   base::ListValue* subresource_list(NULL);
 327   // 0 == url; 1 == subresource_list.
 328   EXPECT_TRUE(motivation_list->GetList(1, &subresource_list));
 329
 330   // We won't bother to check for the subresource being there already.  Worst
 331   // case, during deserialization, the latency value we supply plus the
 332   // existing value(s) will be added to the referrer.
 333
 334   subresource_list->Append(new base::StringValue(subresource.spec()));
 335   subresource_list->Append(new base::FundamentalValue(use_rate));
 336 }
 337
 338 // For a given motivation, and subresource, find what latency is currently
 339 // listed.  This assume a well formed serialization, which has at most one such
 340 // entry for any pair of names.  If no such pair is found, then return false.
 341 // Data is written into use_rate arguments.
 342 static bool GetDataFromSerialization(const GURL& motivation,
 343                                      const GURL& subresource,
 344                                      const base::ListValue& referral_list,
 345                                      double* use_rate) {
 346   const base::ListValue* motivation_list =
 347       FindSerializationMotivation(motivation, &referral_list);
 348   if (!motivation_list)
 349     return false;
 350   const base::ListValue* subresource_list;
 351   EXPECT_TRUE(motivation_list->GetList(1, &subresource_list));
 352   for (size_t i = 0; i < subresource_list->GetSize();) {
 353     std::string url_spec;
 354     EXPECT_TRUE(subresource_list->GetString(i++, &url_spec));
 355     EXPECT_TRUE(subresource_list->GetDouble(i++, use_rate));
 356     if (subresource == GURL(url_spec)) {
 357       return true;
 358     }
 359   }
 360   return false;
 361 }
 362
 363 //------------------------------------------------------------------------------
 364
 365 // Make sure nil referral lists really have no entries, and no latency listed.
 366 TEST_F(PredictorTest, ReferrerSerializationNilTest) {
 367   Predictor predictor(true, true);
 368   predictor.SetHostResolver(host_resolver_.get());
 369
 370   scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
 371   predictor.SerializeReferrers(referral_list.get());
 372   EXPECT_EQ(1U, referral_list->GetSize());
 373   EXPECT_FALSE(GetDataFromSerialization(
 374     GURL("http://a.com:79"), GURL("http://b.com:78"),
 375       *referral_list.get(), NULL));
 376
 377   predictor.Shutdown();
 378 }
 379
 380 // Make sure that when a serialization list includes a value, that it can be
 381 // deserialized into the database, and can be extracted back out via
 382 // serialization without being changed.
 383 TEST_F(PredictorTest, ReferrerSerializationSingleReferrerTest) {
 384   Predictor predictor(true, true);
 385   predictor.SetHostResolver(host_resolver_.get());
 386   const GURL motivation_url("http://www.google.com:91");
 387   const GURL subresource_url("http://icons.google.com:90");
 388   const double kUseRate = 23.4;
 389   scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
 390
 391   AddToSerializedList(motivation_url, subresource_url,
 392       kUseRate, referral_list.get());
 393
 394   predictor.DeserializeReferrers(*referral_list.get());
 395
 396   base::ListValue recovered_referral_list;
 397   predictor.SerializeReferrers(&recovered_referral_list);
 398   EXPECT_EQ(2U, recovered_referral_list.GetSize());
 399   double rate;
 400   EXPECT_TRUE(GetDataFromSerialization(
 401       motivation_url, subresource_url, recovered_referral_list, &rate));
 402   EXPECT_EQ(rate, kUseRate);
 403
 404   predictor.Shutdown();
 405 }
 406
 407 // Check that GetHtmlReferrerLists() doesn't crash when given duplicated
 408 // domains for referring URL, and that it sorts the results in the
 409 // correct order.
 410 TEST_F(PredictorTest, GetHtmlReferrerLists) {
 411   Predictor predictor(true, true);
 412   predictor.SetHostResolver(host_resolver_.get());
 413   const double kUseRate = 23.4;
 414   scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
 415
 416   AddToSerializedList(
 417       GURL("http://d.google.com/x1"),
 418       GURL("http://foo.com/"),
 419       kUseRate, referral_list.get());
 420
 421   // Duplicated hostname (d.google.com). This should not cause any crashes
 422   // (i.e. crbug.com/116345)
 423   AddToSerializedList(
 424       GURL("http://d.google.com/x2"),
 425       GURL("http://foo.com/"),
 426       kUseRate, referral_list.get());
 427
 428   AddToSerializedList(
 429       GURL("http://a.yahoo.com/y"),
 430       GURL("http://foo1.com/"),
 431       kUseRate, referral_list.get());
 432
 433   AddToSerializedList(
 434       GURL("http://b.google.com/x3"),
 435       GURL("http://foo2.com/"),
 436       kUseRate, referral_list.get());
 437
 438   AddToSerializedList(
 439       GURL("http://d.yahoo.com/x5"),
 440       GURL("http://i.like.turtles/"),
 441       kUseRate, referral_list.get());
 442
 443   AddToSerializedList(
 444       GURL("http://c.yahoo.com/x4"),
 445       GURL("http://foo3.com/"),
 446       kUseRate, referral_list.get());
 447
 448   predictor.DeserializeReferrers(*referral_list.get());
 449
 450   std::string html;
 451   predictor.GetHtmlReferrerLists(&html);
 452
 453   // The lexicographic sorting of hostnames would be:
 454   //   a.yahoo.com
 455   //   b.google.com
 456   //   c.yahoo.com
 457   //   d.google.com
 458   //   d.yahoo.com
 459   //
 460   // However we expect to sort them by domain in the output:
 461   //   b.google.com
 462   //   d.google.com
 463   //   a.yahoo.com
 464   //   c.yahoo.com
 465   //   d.yahoo.com
 466
 467   size_t pos[] = {
 468       html.find("<td rowspan=1>http://b.google.com/x3"),
 469       html.find("<td rowspan=1>http://d.google.com/x1"),
 470       html.find("<td rowspan=1>http://d.google.com/x2"),
 471       html.find("<td rowspan=1>http://a.yahoo.com/y"),
 472       html.find("<td rowspan=1>http://c.yahoo.com/x4"),
 473       html.find("<td rowspan=1>http://d.yahoo.com/x5"),
 474   };
 475
 476   // Make sure things appeared in the expected order.
 477   for (size_t i = 1; i < arraysize(pos); ++i) {
 478     EXPECT_LT(pos[i - 1], pos[i]) << "Mismatch for pos[" << i << "]";
 479   }
 480
 481   predictor.Shutdown();
 482 }
 483
 484 // Verify that two floats are within 1% of each other in value.
 485 #define EXPECT_SIMILAR(a, b) do { \
 486     double espilon_ratio = 1.01;  \
 487     if ((a) < 0.)  \
 488       espilon_ratio = 1 / espilon_ratio;  \
 489     EXPECT_LT(a, espilon_ratio * (b));   \
 490     EXPECT_GT((a) * espilon_ratio, b);   \
 491     } while (0)
 492
 493
 494 // Make sure the Trim() functionality works as expected.
 495 TEST_F(PredictorTest, ReferrerSerializationTrimTest) {
 496   Predictor predictor(true, true);
 497   predictor.SetHostResolver(host_resolver_.get());
 498   GURL motivation_url("http://www.google.com:110");
 499
 500   GURL icon_subresource_url("http://icons.google.com:111");
 501   const double kRateIcon = 16.0 * Predictor::kDiscardableExpectedValue;
 502   GURL img_subresource_url("http://img.google.com:118");
 503   const double kRateImg = 8.0 * Predictor::kDiscardableExpectedValue;
 504
 505   scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
 506   AddToSerializedList(
 507       motivation_url, icon_subresource_url, kRateIcon, referral_list.get());
 508   AddToSerializedList(
 509       motivation_url, img_subresource_url, kRateImg, referral_list.get());
 510
 511   predictor.DeserializeReferrers(*referral_list.get());
 512
 513   base::ListValue recovered_referral_list;
 514   predictor.SerializeReferrers(&recovered_referral_list);
 515   EXPECT_EQ(2U, recovered_referral_list.GetSize());
 516   double rate;
 517   EXPECT_TRUE(GetDataFromSerialization(
 518       motivation_url, icon_subresource_url, recovered_referral_list,
 519       &rate));
 520   EXPECT_SIMILAR(rate, kRateIcon);
 521
 522   EXPECT_TRUE(GetDataFromSerialization(
 523       motivation_url, img_subresource_url, recovered_referral_list, &rate));
 524   EXPECT_SIMILAR(rate, kRateImg);
 525
 526   // Each time we Trim 24 times, the user_rate figures should reduce by a factor
 527   // of two,  until they are small, and then a trim will delete the whole entry.
 528   for (int i = 0; i < 24; ++i)
 529     predictor.TrimReferrersNow();
 530   predictor.SerializeReferrers(&recovered_referral_list);
 531   EXPECT_EQ(2U, recovered_referral_list.GetSize());
 532   EXPECT_TRUE(GetDataFromSerialization(
 533       motivation_url, icon_subresource_url, recovered_referral_list, &rate));
 534   EXPECT_SIMILAR(rate, kRateIcon / 2);
 535
 536   EXPECT_TRUE(GetDataFromSerialization(
 537       motivation_url, img_subresource_url, recovered_referral_list, &rate));
 538   EXPECT_SIMILAR(rate, kRateImg / 2);
 539
 540   for (int i = 0; i < 24; ++i)
 541     predictor.TrimReferrersNow();
 542   predictor.SerializeReferrers(&recovered_referral_list);
 543   EXPECT_EQ(2U, recovered_referral_list.GetSize());
 544   EXPECT_TRUE(GetDataFromSerialization(
 545       motivation_url, icon_subresource_url, recovered_referral_list, &rate));
 546   EXPECT_SIMILAR(rate, kRateIcon / 4);
 547   EXPECT_TRUE(GetDataFromSerialization(
 548       motivation_url, img_subresource_url, recovered_referral_list, &rate));
 549   EXPECT_SIMILAR(rate, kRateImg / 4);
 550
 551   for (int i = 0; i < 24; ++i)
 552     predictor.TrimReferrersNow();
 553   predictor.SerializeReferrers(&recovered_referral_list);
 554   EXPECT_EQ(2U, recovered_referral_list.GetSize());
 555   EXPECT_TRUE(GetDataFromSerialization(
 556       motivation_url, icon_subresource_url, recovered_referral_list, &rate));
 557   EXPECT_SIMILAR(rate, kRateIcon / 8);
 558
 559   // Img is below threshold, and so it gets deleted.
 560   EXPECT_FALSE(GetDataFromSerialization(
 561       motivation_url, img_subresource_url, recovered_referral_list, &rate));
 562
 563   for (int i = 0; i < 24; ++i)
 564     predictor.TrimReferrersNow();
 565   predictor.SerializeReferrers(&recovered_referral_list);
 566   // Icon is also trimmed away, so entire set gets discarded.
 567   EXPECT_EQ(1U, recovered_referral_list.GetSize());
 568   EXPECT_FALSE(GetDataFromSerialization(
 569       motivation_url, icon_subresource_url, recovered_referral_list, &rate));
 570   EXPECT_FALSE(GetDataFromSerialization(
 571       motivation_url, img_subresource_url, recovered_referral_list, &rate));
 572
 573   predictor.Shutdown();
 574 }
 575
 576
 577 TEST_F(PredictorTest, PriorityQueuePushPopTest) {
 578   Predictor::HostNameQueue queue;
 579
 580   GURL first("http://first:80"), second("http://second:90");
 581
 582   // First check high priority queue FIFO functionality.
 583   EXPECT_TRUE(queue.IsEmpty());
 584   queue.Push(first, UrlInfo::LEARNED_REFERAL_MOTIVATED);
 585   EXPECT_FALSE(queue.IsEmpty());
 586   queue.Push(second, UrlInfo::MOUSE_OVER_MOTIVATED);
 587   EXPECT_FALSE(queue.IsEmpty());
 588   EXPECT_EQ(queue.Pop(), first);
 589   EXPECT_FALSE(queue.IsEmpty());
 590   EXPECT_EQ(queue.Pop(), second);
 591   EXPECT_TRUE(queue.IsEmpty());
 592
 593   // Then check low priority queue FIFO functionality.
 594   queue.Push(first, UrlInfo::PAGE_SCAN_MOTIVATED);
 595   EXPECT_FALSE(queue.IsEmpty());
 596   queue.Push(second, UrlInfo::OMNIBOX_MOTIVATED);
 597   EXPECT_FALSE(queue.IsEmpty());
 598   EXPECT_EQ(queue.Pop(), first);
 599   EXPECT_FALSE(queue.IsEmpty());
 600   EXPECT_EQ(queue.Pop(), second);
 601   EXPECT_TRUE(queue.IsEmpty());
 602 }
 603
 604 TEST_F(PredictorTest, PriorityQueueReorderTest) {
 605   Predictor::HostNameQueue queue;
 606
 607   // Push all the low priority items.
 608   GURL low1("http://low1:80"),
 609       low2("http://low2:80"),
 610       low3("http://low3:443"),
 611       low4("http://low4:80"),
 612       low5("http://low5:80"),
 613       hi1("http://hi1:80"),
 614       hi2("http://hi2:80"),
 615       hi3("http://hi3:80");
 616
 617   EXPECT_TRUE(queue.IsEmpty());
 618   queue.Push(low1, UrlInfo::PAGE_SCAN_MOTIVATED);
 619   queue.Push(low2, UrlInfo::UNIT_TEST_MOTIVATED);
 620   queue.Push(low3, UrlInfo::LINKED_MAX_MOTIVATED);
 621   queue.Push(low4, UrlInfo::OMNIBOX_MOTIVATED);
 622   queue.Push(low5, UrlInfo::STARTUP_LIST_MOTIVATED);
 623   queue.Push(low4, UrlInfo::OMNIBOX_MOTIVATED);
 624
 625   // Push all the high prority items
 626   queue.Push(hi1, UrlInfo::LEARNED_REFERAL_MOTIVATED);
 627   queue.Push(hi2, UrlInfo::STATIC_REFERAL_MOTIVATED);
 628   queue.Push(hi3, UrlInfo::MOUSE_OVER_MOTIVATED);
 629
 630   // Check that high priority stuff comes out first, and in FIFO order.
 631   EXPECT_EQ(queue.Pop(), hi1);
 632   EXPECT_EQ(queue.Pop(), hi2);
 633   EXPECT_EQ(queue.Pop(), hi3);
 634
 635   // ...and then low priority strings.
 636   EXPECT_EQ(queue.Pop(), low1);
 637   EXPECT_EQ(queue.Pop(), low2);
 638   EXPECT_EQ(queue.Pop(), low3);
 639   EXPECT_EQ(queue.Pop(), low4);
 640   EXPECT_EQ(queue.Pop(), low5);
 641   EXPECT_EQ(queue.Pop(), low4);
 642
 643   EXPECT_TRUE(queue.IsEmpty());
 644 }
 645
 646 TEST_F(PredictorTest, CanonicalizeUrl) {
 647   // Base case, only handles HTTP and HTTPS.
 648   EXPECT_EQ(GURL(), Predictor::CanonicalizeUrl(GURL("ftp://anything")));
 649
 650   // Remove path testing.
 651   GURL long_url("http://host:999/path?query=value");
 652   EXPECT_EQ(Predictor::CanonicalizeUrl(long_url), long_url.GetWithEmptyPath());
 653
 654   // Default port cannoncalization.
 655   GURL implied_port("http://test");
 656   GURL explicit_port("http://test:80");
 657   EXPECT_EQ(Predictor::CanonicalizeUrl(implied_port),
 658             Predictor::CanonicalizeUrl(explicit_port));
 659
 660   // Port is still maintained.
 661   GURL port_80("http://test:80");
 662   GURL port_90("http://test:90");
 663   EXPECT_NE(Predictor::CanonicalizeUrl(port_80),
 664             Predictor::CanonicalizeUrl(port_90));
 665
 666   // Host is still maintained.
 667   GURL host_1("http://test_1");
 668   GURL host_2("http://test_2");
 669   EXPECT_NE(Predictor::CanonicalizeUrl(host_1),
 670             Predictor::CanonicalizeUrl(host_2));
 671
 672   // Scheme is maintained (mismatch identified).
 673   GURL http("http://test");
 674   GURL https("https://test");
 675   EXPECT_NE(Predictor::CanonicalizeUrl(http),
 676             Predictor::CanonicalizeUrl(https));
 677
 678   // Https works fine.
 679   GURL long_https("https://host:999/path?query=value");
 680   EXPECT_EQ(Predictor::CanonicalizeUrl(long_https),
 681             long_https.GetWithEmptyPath());
 682 }
 683
 684 TEST_F(PredictorTest, DiscardPredictorResults) {
 685   SimplePredictor predictor(true, true);
 686   predictor.SetHostResolver(host_resolver_.get());
 687   base::ListValue referral_list;
 688   predictor.SerializeReferrers(&referral_list);
 689   EXPECT_EQ(1U, referral_list.GetSize());
 690
 691   GURL host_1("http://test_1");
 692   GURL host_2("http://test_2");
 693   predictor.LearnFromNavigation(host_1, host_2);
 694
 695   predictor.SerializeReferrers(&referral_list);
 696   EXPECT_EQ(2U, referral_list.GetSize());
 697
 698   predictor.DiscardAllResults();
 699   predictor.SerializeReferrers(&referral_list);
 700   EXPECT_EQ(1U, referral_list.GetSize());
 701
 702   predictor.Shutdown();
 703 }
 704
 705 class TestPredictorObserver : public PredictorObserver {
 706  public:
 707   // PredictorObserver implementation:
 708   void OnPreconnectUrl(const GURL& url,
 709                        const GURL& first_party_for_cookies,
 710                        UrlInfo::ResolutionMotivation motivation,
 711                        int count) override {
 712     preconnected_urls_.push_back(url);
 713   }
 714
 715   std::vector<GURL> preconnected_urls_;
 716 };
 717
 718 // Tests that preconnects apply the HSTS list.
 719 TEST_F(PredictorTest, HSTSRedirect) {
 720   const GURL kHttpUrl("http://example.com");
 721   const GURL kHttpsUrl("https://example.com");
 722
 723   const base::Time expiry =
 724       base::Time::Now() + base::TimeDelta::FromSeconds(1000);
 725   net::TransportSecurityState state;
 726   state.AddHSTS(kHttpUrl.host(), expiry, false);
 727
 728   Predictor predictor(true, true);
 729   TestPredictorObserver observer;
 730   predictor.SetObserver(&observer);
 731   predictor.SetTransportSecurityState(&state);
 732
 733   predictor.PreconnectUrl(kHttpUrl, GURL(), UrlInfo::OMNIBOX_MOTIVATED, 2);
 734   ASSERT_EQ(1u, observer.preconnected_urls_.size());
 735   EXPECT_EQ(kHttpsUrl, observer.preconnected_urls_[0]);
 736
 737   predictor.Shutdown();
 738 }
 739
 740 // Tests that preconnecting a URL on the HSTS list preconnects the subresources
 741 // for the SSL version.
 742 TEST_F(PredictorTest, HSTSRedirectSubresources) {
 743   const GURL kHttpUrl("http://example.com");
 744   const GURL kHttpsUrl("https://example.com");
 745   const GURL kSubresourceUrl("https://images.example.com");
 746   const double kUseRate = 23.4;
 747
 748   const base::Time expiry =
 749       base::Time::Now() + base::TimeDelta::FromSeconds(1000);
 750   net::TransportSecurityState state;
 751   state.AddHSTS(kHttpUrl.host(), expiry, false);
 752
 753   SimplePredictor predictor(true, true);
 754   TestPredictorObserver observer;
 755   predictor.SetObserver(&observer);
 756   predictor.SetTransportSecurityState(&state);
 757
 758   scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
 759   AddToSerializedList(
 760       kHttpsUrl, kSubresourceUrl, kUseRate, referral_list.get());
 761   predictor.DeserializeReferrers(*referral_list.get());
 762
 763   predictor.PreconnectUrlAndSubresources(kHttpUrl, GURL());
 764   ASSERT_EQ(2u, observer.preconnected_urls_.size());
 765   EXPECT_EQ(kHttpsUrl, observer.preconnected_urls_[0]);
 766   EXPECT_EQ(kSubresourceUrl, observer.preconnected_urls_[1]);
 767
 768   predictor.Shutdown();
 769 }
 770
 771 #if defined(OS_ANDROID) || defined(OS_IOS)
 772 // Tests for the predictor with a proxy advisor
 773
 774 class TestProxyAdvisor : public ProxyAdvisor {
 775  public:
 776   TestProxyAdvisor()
 777       : ProxyAdvisor(NULL, NULL),
 778         would_proxy_(false),
 779         advise_count_(0),
 780         would_proxy_count_(0) {
 781   }
 782
 783   virtual ~TestProxyAdvisor() {}
 784
 785   virtual void Advise(const GURL& url,
 786                       UrlInfo::ResolutionMotivation motivation,
 787                       bool is_preconnect) override {
 788     ++advise_count_;
 789   }
 790
 791   virtual bool WouldProxyURL(const GURL& url) override {
 792     ++would_proxy_count_;
 793     return would_proxy_;
 794   }
 795
 796   bool would_proxy_;
 797   int advise_count_;
 798   int would_proxy_count_;
 799 };
 800
 801 TEST_F(PredictorTest, SingleLookupTestWithDisabledAdvisor) {
 802   Predictor testing_master(true, true);
 803   TestProxyAdvisor* advisor = new TestProxyAdvisor();
 804   testing_master.SetHostResolver(host_resolver_.get());
 805   testing_master.proxy_advisor_.reset(advisor);
 806
 807   GURL goog("http://www.google.com:80");
 808
 809   advisor->would_proxy_ = false;
 810
 811   UrlList names;
 812   names.push_back(goog);
 813   testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
 814
 815   WaitForResolution(&testing_master, names);
 816   EXPECT_TRUE(testing_master.WasFound(goog));
 817   EXPECT_EQ(advisor->would_proxy_count_, 1);
 818   EXPECT_EQ(advisor->advise_count_, 1);
 819
 820   base::MessageLoop::current()->RunUntilIdle();
 821
 822   testing_master.Shutdown();
 823 }
 824
 825 TEST_F(PredictorTest, SingleLookupTestWithEnabledAdvisor) {
 826   Predictor testing_master(true, true);
 827   testing_master.SetHostResolver(host_resolver_.get());
 828   TestProxyAdvisor* advisor = new TestProxyAdvisor();
 829   testing_master.proxy_advisor_.reset(advisor);
 830
 831   GURL goog("http://www.google.com:80");
 832
 833   advisor->would_proxy_ = true;
 834
 835   UrlList names;
 836   names.push_back(goog);
 837
 838   testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
 839
 840   // Attempt to resolve a few times.
 841   WaitForResolutionWithLimit(&testing_master, names, 10);
 842
 843   // Because the advisor indicated that the url would be proxied,
 844   // no resolution should have occurred.
 845   EXPECT_FALSE(testing_master.WasFound(goog));
 846   EXPECT_EQ(advisor->would_proxy_count_, 1);
 847   EXPECT_EQ(advisor->advise_count_, 1);
 848
 849   base::MessageLoop::current()->RunUntilIdle();
 850
 851   testing_master.Shutdown();
 852 }
 853
 854 TEST_F(PredictorTest, TestSimplePreconnectAdvisor) {
 855   Predictor testing_master(true, true);
 856   testing_master.SetHostResolver(host_resolver_.get());
 857   TestProxyAdvisor* advisor = new TestProxyAdvisor();
 858   testing_master.proxy_advisor_.reset(advisor);
 859
 860   GURL goog("http://www.google.com:80");
 861
 862   testing_master.PreconnectUrl(goog, goog, UrlInfo::OMNIBOX_MOTIVATED, 2);
 863
 864   EXPECT_EQ(advisor->would_proxy_count_, 0);
 865   EXPECT_EQ(advisor->advise_count_, 1);
 866
 867   testing_master.Shutdown();
 868 }
 869
 870 #endif  // defined(OS_ANDROID) || defined(OS_IOS)
 871
 872 TEST_F(PredictorTest, NoProxyService) {
 873   // Don't actually try to resolve names.
 874   Predictor::set_max_parallel_resolves(0);
 875
 876   Predictor testing_master(true, true);
 877
 878   GURL goog("http://www.google.com:80");
 879   testing_master.Resolve(goog, UrlInfo::OMNIBOX_MOTIVATED);
 880   EXPECT_FALSE(testing_master.work_queue_.IsEmpty());
 881
 882   testing_master.Shutdown();
 883 }
 884
 885 TEST_F(PredictorTest, ProxyDefinitelyEnabled) {
 886   // Don't actually try to resolve names.
 887   Predictor::set_max_parallel_resolves(0);
 888
 889   Predictor testing_master(true, true);
 890
 891   net::ProxyConfig config;
 892   config.proxy_rules().ParseFromString("http=socks://localhost:12345");
 893   testing_master.proxy_service_ = net::ProxyService::CreateFixed(config);
 894
 895   GURL goog("http://www.google.com:80");
 896   testing_master.Resolve(goog, UrlInfo::OMNIBOX_MOTIVATED);
 897
 898   // Proxy is definitely in use, so there is no need to pre-resolve the domain.
 899   EXPECT_TRUE(testing_master.work_queue_.IsEmpty());
 900
 901   delete testing_master.proxy_service_;
 902   testing_master.Shutdown();
 903 }
 904
 905 TEST_F(PredictorTest, ProxyDefinitelyNotEnabled) {
 906   // Don't actually try to resolve names.
 907   Predictor::set_max_parallel_resolves(0);
 908
 909   Predictor testing_master(true, true);
 910   net::ProxyConfig config = net::ProxyConfig::CreateDirect();
 911   testing_master.proxy_service_ = net::ProxyService::CreateFixed(config);
 912
 913   GURL goog("http://www.google.com:80");
 914   testing_master.Resolve(goog, UrlInfo::OMNIBOX_MOTIVATED);
 915
 916   // Proxy is not in use, so the name has been registered for pre-resolve.
 917   EXPECT_FALSE(testing_master.work_queue_.IsEmpty());
 918
 919   delete testing_master.proxy_service_;
 920   testing_master.Shutdown();
 921 }
 922
 923 TEST_F(PredictorTest, ProxyMaybeEnabled) {
 924   // Don't actually try to resolve names.
 925   Predictor::set_max_parallel_resolves(0);
 926
 927   Predictor testing_master(true, true);
 928   net::ProxyConfig config = net::ProxyConfig::CreateFromCustomPacURL(GURL(
 929       "http://foopy/proxy.pac"));
 930   testing_master.proxy_service_ = net::ProxyService::CreateFixed(config);
 931
 932   GURL goog("http://www.google.com:80");
 933   testing_master.Resolve(goog, UrlInfo::OMNIBOX_MOTIVATED);
 934
 935   // Proxy may not be in use (the PAC script has not yet been evaluated), so the
 936   // name has been registered for pre-resolve.
 937   EXPECT_FALSE(testing_master.work_queue_.IsEmpty());
 938
 939   delete testing_master.proxy_service_;
 940   testing_master.Shutdown();
 941 }
 942
 943 }  // namespace chrome_browser_net