src/chrome/browser/safe_browsing/safe_browsing_database_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4 //
   5 // Unit tests for the SafeBrowsing storage system.
   6
   7 #include "chrome/browser/safe_browsing/safe_browsing_database.h"
   8
   9 #include "base/file_util.h"
  10 #include "base/files/scoped_temp_dir.h"
  11 #include "base/logging.h"
  12 #include "base/memory/scoped_vector.h"
  13 #include "base/message_loop/message_loop.h"
  14 #include "base/sha1.h"
  15 #include "base/strings/string_number_conversions.h"
  16 #include "base/strings/string_split.h"
  17 #include "base/time/time.h"
  18 #include "chrome/browser/safe_browsing/chunk.pb.h"
  19 #include "chrome/browser/safe_browsing/safe_browsing_store_file.h"
  20 #include "content/public/test/test_browser_thread_bundle.h"
  21 #include "crypto/sha2.h"
  22 #include "net/base/net_util.h"
  23 #include "sql/connection.h"
  24 #include "sql/statement.h"
  25 #include "testing/gtest/include/gtest/gtest.h"
  26 #include "testing/platform_test.h"
  27 #include "url/gurl.h"
  28
  29 using base::Time;
  30 using base::TimeDelta;
  31
  32 namespace {
  33
  34 const TimeDelta kCacheLifetime = TimeDelta::FromMinutes(45);
  35
  36 SBPrefix SBPrefixForString(const std::string& str) {
  37   return SBFullHashForString(str).prefix;
  38 }
  39
  40 // Construct a full hash which has the given prefix, with the given
  41 // suffix data coming after the prefix.
  42 SBFullHash SBFullHashForPrefixAndSuffix(SBPrefix prefix,
  43                                         const base::StringPiece& suffix) {
  44   SBFullHash full_hash;
  45   memset(&full_hash, 0, sizeof(SBFullHash));
  46   full_hash.prefix = prefix;
  47   CHECK_LE(suffix.size() + sizeof(SBPrefix), sizeof(SBFullHash));
  48   memcpy(full_hash.full_hash + sizeof(SBPrefix), suffix.data(), suffix.size());
  49   return full_hash;
  50 }
  51
  52 std::string HashedIpPrefix(const std::string& ip_prefix, size_t prefix_size) {
  53   net::IPAddressNumber ip_number;
  54   EXPECT_TRUE(net::ParseIPLiteralToNumber(ip_prefix, &ip_number));
  55   EXPECT_EQ(net::kIPv6AddressSize, ip_number.size());
  56   const std::string hashed_ip_prefix = base::SHA1HashString(
  57       net::IPAddressToPackedString(ip_number));
  58   std::string hash(crypto::kSHA256Length, '\0');
  59   hash.replace(0, hashed_ip_prefix.size(), hashed_ip_prefix);
  60   hash[base::kSHA1Length] = static_cast<char>(prefix_size);
  61   return hash;
  62 }
  63
  64 // Helper to build a chunk.  Caller takes ownership.
  65 SBChunkData* BuildChunk(int chunk_number,
  66                         safe_browsing::ChunkData::ChunkType chunk_type,
  67                         safe_browsing::ChunkData::PrefixType prefix_type,
  68                         const void* data, size_t data_size,
  69                         const std::vector<int>& add_chunk_numbers) {
  70   scoped_ptr<safe_browsing::ChunkData> raw_data(new safe_browsing::ChunkData);
  71   raw_data->set_chunk_number(chunk_number);
  72   raw_data->set_chunk_type(chunk_type);
  73   raw_data->set_prefix_type(prefix_type);
  74   raw_data->set_hashes(data, data_size);
  75   raw_data->clear_add_numbers();
  76   for (size_t i = 0; i < add_chunk_numbers.size(); ++i) {
  77     raw_data->add_add_numbers(add_chunk_numbers[i]);
  78   }
  79
  80   return new SBChunkData(raw_data.release());
  81 }
  82
  83 // Create add chunk with a single prefix.
  84 SBChunkData* AddChunkPrefix(int chunk_number,  SBPrefix prefix) {
  85   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
  86                     safe_browsing::ChunkData::PREFIX_4B,
  87                     &prefix, sizeof(prefix),
  88                     std::vector<int>());
  89 }
  90
  91 // Create add chunk with a single prefix generated from |value|.
  92 SBChunkData* AddChunkPrefixValue(int chunk_number,
  93                                  const std::string& value) {
  94   return AddChunkPrefix(chunk_number, SBPrefixForString(value));
  95 }
  96
  97 // Generate an add chunk with two prefixes.
  98 SBChunkData* AddChunkPrefix2Value(int chunk_number,
  99                                   const std::string& value1,
 100                                   const std::string& value2) {
 101   const SBPrefix prefixes[2] = {
 102     SBPrefixForString(value1),
 103     SBPrefixForString(value2),
 104   };
 105   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 106                     safe_browsing::ChunkData::PREFIX_4B,
 107                     &prefixes[0], sizeof(prefixes),
 108                     std::vector<int>());
 109 }
 110
 111 // Generate an add chunk with four prefixes.
 112 SBChunkData* AddChunkPrefix4Value(int chunk_number,
 113                                   const std::string& value1,
 114                                   const std::string& value2,
 115                                   const std::string& value3,
 116                                   const std::string& value4) {
 117   const SBPrefix prefixes[4] = {
 118     SBPrefixForString(value1),
 119     SBPrefixForString(value2),
 120     SBPrefixForString(value3),
 121     SBPrefixForString(value4),
 122   };
 123   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 124                     safe_browsing::ChunkData::PREFIX_4B,
 125                     &prefixes[0], sizeof(prefixes),
 126                     std::vector<int>());
 127 }
 128
 129 // Generate an add chunk with a full hash.
 130 SBChunkData* AddChunkFullHash(int chunk_number, SBFullHash full_hash) {
 131   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 132                     safe_browsing::ChunkData::FULL_32B,
 133                     &full_hash, sizeof(full_hash),
 134                     std::vector<int>());
 135 }
 136
 137 // Generate an add chunk with a full hash generated from |value|.
 138 SBChunkData* AddChunkFullHashValue(int chunk_number,
 139                                    const std::string& value) {
 140   return AddChunkFullHash(chunk_number, SBFullHashForString(value));
 141 }
 142
 143 // Generate an add chunk with two full hashes.
 144 SBChunkData* AddChunkFullHash2Value(int chunk_number,
 145                                    const std::string& value1,
 146                                    const std::string& value2) {
 147   const SBFullHash full_hashes[2] = {
 148     SBFullHashForString(value1),
 149     SBFullHashForString(value2),
 150   };
 151   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 152                     safe_browsing::ChunkData::FULL_32B,
 153                     &full_hashes[0], sizeof(full_hashes),
 154                     std::vector<int>());
 155 }
 156
 157 // Generate a sub chunk with a prefix generated from |value|.
 158 SBChunkData* SubChunkPrefixValue(int chunk_number,
 159                                  const std::string& value,
 160                                  int add_chunk_number) {
 161   const SBPrefix prefix = SBPrefixForString(value);
 162   return BuildChunk(chunk_number, safe_browsing::ChunkData::SUB,
 163                     safe_browsing::ChunkData::PREFIX_4B,
 164                     &prefix, sizeof(prefix),
 165                     std::vector<int>(1, add_chunk_number));
 166 }
 167
 168 // Generate a sub chunk with two prefixes.
 169 SBChunkData* SubChunkPrefix2Value(int chunk_number,
 170                                   const std::string& value1,
 171                                   int add_chunk_number1,
 172                                   const std::string& value2,
 173                                   int add_chunk_number2) {
 174   const SBPrefix prefixes[2] = {
 175     SBPrefixForString(value1),
 176     SBPrefixForString(value2),
 177   };
 178   std::vector<int> add_chunk_numbers;
 179   add_chunk_numbers.push_back(add_chunk_number1);
 180   add_chunk_numbers.push_back(add_chunk_number2);
 181   return BuildChunk(chunk_number, safe_browsing::ChunkData::SUB,
 182                     safe_browsing::ChunkData::PREFIX_4B,
 183                     &prefixes[0], sizeof(prefixes),
 184                     add_chunk_numbers);
 185 }
 186
 187 // Generate a sub chunk with a full hash.
 188 SBChunkData* SubChunkFullHash(int chunk_number,
 189                               SBFullHash full_hash,
 190                               int add_chunk_number) {
 191   return BuildChunk(chunk_number, safe_browsing::ChunkData::SUB,
 192                     safe_browsing::ChunkData::FULL_32B,
 193                     &full_hash, sizeof(full_hash),
 194                     std::vector<int>(1, add_chunk_number));
 195 }
 196
 197 // Generate a sub chunk with a full hash generated from |value|.
 198 SBChunkData* SubChunkFullHashValue(int chunk_number,
 199                                    const std::string& value,
 200                                    int add_chunk_number) {
 201   return SubChunkFullHash(chunk_number,
 202                           SBFullHashForString(value),
 203                           add_chunk_number);
 204 }
 205
 206 // Generate an add chunk with a single full hash for the ip blacklist.
 207 SBChunkData* AddChunkHashedIpValue(int chunk_number,
 208                                    const std::string& ip_str,
 209                                    size_t prefix_size) {
 210   const std::string full_hash_str = HashedIpPrefix(ip_str, prefix_size);
 211   EXPECT_EQ(sizeof(SBFullHash), full_hash_str.size());
 212   SBFullHash full_hash;
 213   std::memcpy(&(full_hash.full_hash), full_hash_str.data(), sizeof(SBFullHash));
 214   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 215                     safe_browsing::ChunkData::FULL_32B,
 216                     &full_hash, sizeof(full_hash),
 217                     std::vector<int>());
 218 }
 219
 220 // Prevent DCHECK from killing tests.
 221 // TODO(shess): Pawel disputes the use of this, so the test which uses
 222 // it is DISABLED.  http://crbug.com/56448
 223 class ScopedLogMessageIgnorer {
 224  public:
 225   ScopedLogMessageIgnorer() {
 226     logging::SetLogMessageHandler(&LogMessageIgnorer);
 227   }
 228   ~ScopedLogMessageIgnorer() {
 229     // TODO(shess): Would be better to verify whether anyone else
 230     // changed it, and then restore it to the previous value.
 231     logging::SetLogMessageHandler(NULL);
 232   }
 233
 234  private:
 235   static bool LogMessageIgnorer(int severity, const char* file, int line,
 236       size_t message_start, const std::string& str) {
 237     // Intercept FATAL, strip the stack backtrace, and log it without
 238     // the crash part.
 239     if (severity == logging::LOG_FATAL) {
 240       size_t newline = str.find('\n');
 241       if (newline != std::string::npos) {
 242         const std::string msg = str.substr(0, newline + 1);
 243         fprintf(stderr, "%s", msg.c_str());
 244         fflush(stderr);
 245       }
 246       return true;
 247     }
 248
 249     return false;
 250   }
 251 };
 252
 253 }  // namespace
 254
 255 class SafeBrowsingDatabaseTest : public PlatformTest {
 256  public:
 257   virtual void SetUp() {
 258     PlatformTest::SetUp();
 259
 260     // Setup a database in a temporary directory.
 261     ASSERT_TRUE(temp_dir_.CreateUniqueTempDir());
 262     database_.reset(new SafeBrowsingDatabaseNew);
 263     database_filename_ =
 264         temp_dir_.path().AppendASCII("SafeBrowsingTestDatabase");
 265     database_->Init(database_filename_);
 266   }
 267
 268   virtual void TearDown() {
 269     database_.reset();
 270
 271     PlatformTest::TearDown();
 272   }
 273
 274   void GetListsInfo(std::vector<SBListChunkRanges>* lists) {
 275     lists->clear();
 276     ASSERT_TRUE(database_->UpdateStarted(lists));
 277     database_->UpdateFinished(true);
 278   }
 279
 280   // Helper function to do an AddDel or SubDel command.
 281   void DelChunk(const std::string& list,
 282                 int chunk_id,
 283                 bool is_sub_del) {
 284     std::vector<SBChunkDelete> deletes;
 285     SBChunkDelete chunk_delete;
 286     chunk_delete.list_name = list;
 287     chunk_delete.is_sub_del = is_sub_del;
 288     chunk_delete.chunk_del.push_back(ChunkRange(chunk_id));
 289     deletes.push_back(chunk_delete);
 290     database_->DeleteChunks(deletes);
 291   }
 292
 293   void AddDelChunk(const std::string& list, int chunk_id) {
 294     DelChunk(list, chunk_id, false);
 295   }
 296
 297   void SubDelChunk(const std::string& list, int chunk_id) {
 298     DelChunk(list, chunk_id, true);
 299   }
 300
 301   // Utility function for setting up the database for the caching test.
 302   void PopulateDatabaseForCacheTest();
 303
 304   scoped_ptr<SafeBrowsingDatabaseNew> database_;
 305   base::FilePath database_filename_;
 306   base::ScopedTempDir temp_dir_;
 307 };
 308
 309 // Tests retrieving list name information.
 310 TEST_F(SafeBrowsingDatabaseTest, ListNameForBrowse) {
 311   std::vector<SBListChunkRanges> lists;
 312   ScopedVector<SBChunkData> chunks;
 313
 314   chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/malware.html"));
 315   chunks.push_back(AddChunkPrefixValue(2, "www.foo.com/malware.html"));
 316   chunks.push_back(AddChunkPrefixValue(3, "www.whatever.com/malware.html"));
 317
 318   ASSERT_TRUE(database_->UpdateStarted(&lists));
 319   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 320   database_->UpdateFinished(true);
 321
 322   GetListsInfo(&lists);
 323   ASSERT_LE(1U, lists.size());
 324   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 325   EXPECT_EQ("1-3", lists[0].adds);
 326   EXPECT_TRUE(lists[0].subs.empty());
 327
 328   // Insert a malware sub chunk.
 329   chunks.clear();
 330   chunks.push_back(SubChunkPrefixValue(7, "www.subbed.com/noteveil1.html", 19));
 331
 332   ASSERT_TRUE(database_->UpdateStarted(&lists));
 333   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 334   database_->UpdateFinished(true);
 335
 336   GetListsInfo(&lists);
 337   ASSERT_LE(1U, lists.size());
 338   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 339   EXPECT_EQ("1-3", lists[0].adds);
 340   EXPECT_EQ("7", lists[0].subs);
 341   if (lists.size() == 2) {
 342     // Old style database won't have the second entry since it creates the lists
 343     // when it receives an update containing that list. The filter-based
 344     // database has these values hard coded.
 345     EXPECT_EQ(safe_browsing_util::kPhishingList, lists[1].name);
 346     EXPECT_TRUE(lists[1].adds.empty());
 347     EXPECT_TRUE(lists[1].subs.empty());
 348   }
 349
 350   // Add phishing chunks.
 351   chunks.clear();
 352   chunks.push_back(AddChunkPrefixValue(47, "www.evil.com/phishing.html"));
 353   chunks.push_back(
 354       SubChunkPrefixValue(200, "www.phishy.com/notevil1.html", 1999));
 355   chunks.push_back(
 356       SubChunkPrefixValue(201, "www.phishy2.com/notevil1.html", 1999));
 357
 358   ASSERT_TRUE(database_->UpdateStarted(&lists));
 359   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks.get());
 360   database_->UpdateFinished(true);
 361
 362   GetListsInfo(&lists);
 363   ASSERT_EQ(2U, lists.size());
 364   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 365   EXPECT_EQ("1-3", lists[0].adds);
 366   EXPECT_EQ("7", lists[0].subs);
 367   EXPECT_EQ(safe_browsing_util::kPhishingList, lists[1].name);
 368   EXPECT_EQ("47", lists[1].adds);
 369   EXPECT_EQ("200-201", lists[1].subs);
 370 }
 371
 372 TEST_F(SafeBrowsingDatabaseTest, ListNameForBrowseAndDownload) {
 373   database_.reset();
 374   base::MessageLoop loop;
 375   SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile();
 376   SafeBrowsingStoreFile* download_store = new SafeBrowsingStoreFile();
 377   SafeBrowsingStoreFile* csd_whitelist_store = new SafeBrowsingStoreFile();
 378   SafeBrowsingStoreFile* download_whitelist_store = new SafeBrowsingStoreFile();
 379   SafeBrowsingStoreFile* extension_blacklist_store =
 380       new SafeBrowsingStoreFile();
 381   SafeBrowsingStoreFile* ip_blacklist_store = new SafeBrowsingStoreFile();
 382   database_.reset(new SafeBrowsingDatabaseNew(browse_store,
 383                                               download_store,
 384                                               csd_whitelist_store,
 385                                               download_whitelist_store,
 386                                               extension_blacklist_store,
 387                                               NULL,
 388                                               ip_blacklist_store));
 389   database_->Init(database_filename_);
 390
 391   ScopedVector<SBChunkData> chunks;
 392
 393   std::vector<SBListChunkRanges> lists;
 394   ASSERT_TRUE(database_->UpdateStarted(&lists));
 395
 396   // Insert malware, phish, binurl and bindownload add chunks.
 397   chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/malware.html"));
 398   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 399
 400   chunks.clear();
 401   chunks.push_back(AddChunkPrefixValue(2, "www.foo.com/malware.html"));
 402   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks.get());
 403
 404   chunks.clear();
 405   chunks.push_back(AddChunkPrefixValue(3, "www.whatever.com/download.html"));
 406   database_->InsertChunks(safe_browsing_util::kBinUrlList, chunks.get());
 407
 408   chunks.clear();
 409   chunks.push_back(AddChunkFullHashValue(5, "www.forwhitelist.com/a.html"));
 410   database_->InsertChunks(safe_browsing_util::kCsdWhiteList, chunks.get());
 411
 412   chunks.clear();
 413   chunks.push_back(AddChunkFullHashValue(6, "www.download.com/"));
 414   database_->InsertChunks(safe_browsing_util::kDownloadWhiteList, chunks.get());
 415
 416   chunks.clear();
 417   chunks.push_back(AddChunkFullHashValue(8,
 418                                          "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
 419                                          "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"));
 420   database_->InsertChunks(safe_browsing_util::kExtensionBlacklist,
 421                           chunks.get());
 422
 423   chunks.clear();
 424   chunks.push_back(AddChunkHashedIpValue(9, "::ffff:192.168.1.0", 120));
 425   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks.get());
 426
 427   database_->UpdateFinished(true);
 428
 429   GetListsInfo(&lists);
 430   ASSERT_EQ(7U, lists.size());
 431   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 432   EXPECT_EQ("1", lists[0].adds);
 433   EXPECT_TRUE(lists[0].subs.empty());
 434   EXPECT_EQ(safe_browsing_util::kPhishingList, lists[1].name);
 435   EXPECT_EQ("2", lists[1].adds);
 436   EXPECT_TRUE(lists[1].subs.empty());
 437   EXPECT_EQ(safe_browsing_util::kBinUrlList, lists[2].name);
 438   EXPECT_EQ("3", lists[2].adds);
 439   EXPECT_TRUE(lists[2].subs.empty());
 440   EXPECT_EQ(safe_browsing_util::kCsdWhiteList, lists[3].name);
 441   EXPECT_EQ("5", lists[3].adds);
 442   EXPECT_TRUE(lists[3].subs.empty());
 443   EXPECT_EQ(safe_browsing_util::kDownloadWhiteList, lists[4].name);
 444   EXPECT_EQ("6", lists[4].adds);
 445   EXPECT_TRUE(lists[4].subs.empty());
 446   EXPECT_EQ(safe_browsing_util::kExtensionBlacklist, lists[5].name);
 447   EXPECT_EQ("8", lists[5].adds);
 448   EXPECT_TRUE(lists[5].subs.empty());
 449   EXPECT_EQ(safe_browsing_util::kIPBlacklist, lists[6].name);
 450   EXPECT_EQ("9", lists[6].adds);
 451   EXPECT_TRUE(lists[6].subs.empty());
 452
 453   database_.reset();
 454 }
 455
 456 // Checks database reading and writing for browse.
 457 TEST_F(SafeBrowsingDatabaseTest, BrowseDatabase) {
 458   std::vector<SBListChunkRanges> lists;
 459   ScopedVector<SBChunkData> chunks;
 460
 461   chunks.push_back(AddChunkPrefix2Value(1,
 462                                         "www.evil.com/phishing.html",
 463                                         "www.evil.com/malware.html"));
 464   chunks.push_back(AddChunkPrefix4Value(2,
 465                                         "www.evil.com/notevil1.html",
 466                                         "www.evil.com/notevil2.html",
 467                                         "www.good.com/good1.html",
 468                                         "www.good.com/good2.html"));
 469   chunks.push_back(AddChunkPrefixValue(3, "192.168.0.1/malware.html"));
 470   chunks.push_back(AddChunkFullHashValue(7, "www.evil.com/evil.html"));
 471
 472   ASSERT_TRUE(database_->UpdateStarted(&lists));
 473   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 474   database_->UpdateFinished(true);
 475
 476   // Make sure they were added correctly.
 477   GetListsInfo(&lists);
 478   ASSERT_LE(1U, lists.size());
 479   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 480   EXPECT_EQ("1-3,7", lists[0].adds);
 481   EXPECT_TRUE(lists[0].subs.empty());
 482
 483   std::vector<SBPrefix> prefix_hits;
 484   std::vector<SBFullHashResult> cache_hits;
 485   EXPECT_TRUE(database_->ContainsBrowseUrl(
 486       GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 487   ASSERT_EQ(1U, prefix_hits.size());
 488   EXPECT_EQ(SBPrefixForString("www.evil.com/phishing.html"), prefix_hits[0]);
 489   EXPECT_TRUE(cache_hits.empty());
 490
 491   EXPECT_TRUE(database_->ContainsBrowseUrl(
 492       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 493
 494   EXPECT_TRUE(database_->ContainsBrowseUrl(
 495       GURL("http://www.evil.com/notevil1.html"), &prefix_hits, &cache_hits));
 496
 497   EXPECT_TRUE(database_->ContainsBrowseUrl(
 498       GURL("http://www.evil.com/notevil2.html"), &prefix_hits, &cache_hits));
 499
 500   EXPECT_TRUE(database_->ContainsBrowseUrl(
 501       GURL("http://www.good.com/good1.html"), &prefix_hits, &cache_hits));
 502
 503   EXPECT_TRUE(database_->ContainsBrowseUrl(
 504       GURL("http://www.good.com/good2.html"), &prefix_hits, &cache_hits));
 505
 506   EXPECT_TRUE(database_->ContainsBrowseUrl(
 507       GURL("http://192.168.0.1/malware.html"), &prefix_hits, &cache_hits));
 508
 509   EXPECT_FALSE(database_->ContainsBrowseUrl(
 510       GURL("http://www.evil.com/"), &prefix_hits, &cache_hits));
 511   EXPECT_TRUE(prefix_hits.empty());
 512   EXPECT_TRUE(cache_hits.empty());
 513
 514   EXPECT_FALSE(database_->ContainsBrowseUrl(
 515       GURL("http://www.evil.com/robots.txt"), &prefix_hits, &cache_hits));
 516
 517   EXPECT_TRUE(database_->ContainsBrowseUrl(
 518       GURL("http://www.evil.com/evil.html"), &prefix_hits, &cache_hits));
 519   ASSERT_EQ(1U, prefix_hits.size());
 520   EXPECT_EQ(SBPrefixForString("www.evil.com/evil.html"), prefix_hits[0]);
 521
 522   // Attempt to re-add the first chunk (should be a no-op).
 523   // see bug: http://code.google.com/p/chromium/issues/detail?id=4522
 524   chunks.clear();
 525   chunks.push_back(AddChunkPrefix2Value(1,
 526                                         "www.evil.com/phishing.html",
 527                                         "www.evil.com/malware.html"));
 528   ASSERT_TRUE(database_->UpdateStarted(&lists));
 529   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 530   database_->UpdateFinished(true);
 531
 532   GetListsInfo(&lists);
 533   ASSERT_LE(1U, lists.size());
 534   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 535   EXPECT_EQ("1-3,7", lists[0].adds);
 536   EXPECT_TRUE(lists[0].subs.empty());
 537
 538   // Test removing a single prefix from the add chunk.
 539   chunks.clear();
 540   chunks.push_back(SubChunkPrefixValue(4, "www.evil.com/notevil1.html", 2));
 541   ASSERT_TRUE(database_->UpdateStarted(&lists));
 542   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 543   database_->UpdateFinished(true);
 544
 545   EXPECT_TRUE(database_->ContainsBrowseUrl(
 546       GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 547   ASSERT_EQ(1U, prefix_hits.size());
 548   EXPECT_EQ(SBPrefixForString("www.evil.com/phishing.html"), prefix_hits[0]);
 549   EXPECT_TRUE(cache_hits.empty());
 550
 551   EXPECT_FALSE(database_->ContainsBrowseUrl(
 552       GURL("http://www.evil.com/notevil1.html"), &prefix_hits, &cache_hits));
 553   EXPECT_TRUE(prefix_hits.empty());
 554   EXPECT_TRUE(cache_hits.empty());
 555
 556   EXPECT_TRUE(database_->ContainsBrowseUrl(
 557       GURL("http://www.evil.com/notevil2.html"), &prefix_hits, &cache_hits));
 558
 559   EXPECT_TRUE(database_->ContainsBrowseUrl(
 560       GURL("http://www.good.com/good1.html"), &prefix_hits, &cache_hits));
 561
 562   EXPECT_TRUE(database_->ContainsBrowseUrl(
 563       GURL("http://www.good.com/good2.html"), &prefix_hits, &cache_hits));
 564
 565   GetListsInfo(&lists);
 566   ASSERT_LE(1U, lists.size());
 567   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 568   EXPECT_EQ("1-3,7", lists[0].adds);
 569   EXPECT_EQ("4", lists[0].subs);
 570
 571   // Test the same sub chunk again.  This should be a no-op.
 572   // see bug: http://code.google.com/p/chromium/issues/detail?id=4522
 573   chunks.clear();
 574   chunks.push_back(SubChunkPrefixValue(4, "www.evil.com/notevil1.html", 2));
 575
 576   ASSERT_TRUE(database_->UpdateStarted(&lists));
 577   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 578   database_->UpdateFinished(true);
 579
 580   GetListsInfo(&lists);
 581   ASSERT_LE(1U, lists.size());
 582   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 583   EXPECT_EQ("1-3,7", lists[0].adds);
 584   EXPECT_EQ("4", lists[0].subs);
 585
 586   // Test removing all the prefixes from an add chunk.
 587   ASSERT_TRUE(database_->UpdateStarted(&lists));
 588   AddDelChunk(safe_browsing_util::kMalwareList, 2);
 589   database_->UpdateFinished(true);
 590
 591   EXPECT_FALSE(database_->ContainsBrowseUrl(
 592       GURL("http://www.evil.com/notevil2.html"), &prefix_hits, &cache_hits));
 593
 594   EXPECT_FALSE(database_->ContainsBrowseUrl(
 595       GURL("http://www.good.com/good1.html"), &prefix_hits, &cache_hits));
 596
 597   EXPECT_FALSE(database_->ContainsBrowseUrl(
 598       GURL("http://www.good.com/good2.html"), &prefix_hits, &cache_hits));
 599
 600   GetListsInfo(&lists);
 601   ASSERT_LE(1U, lists.size());
 602   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 603   EXPECT_EQ("1,3,7", lists[0].adds);
 604   EXPECT_EQ("4", lists[0].subs);
 605
 606   // The adddel command exposed a bug in the transaction code where any
 607   // transaction after it would fail.  Add a dummy entry and remove it to
 608   // make sure the transcation works fine.
 609   chunks.clear();
 610   chunks.push_back(AddChunkPrefixValue(44, "www.redherring.com/index.html"));
 611   ASSERT_TRUE(database_->UpdateStarted(&lists));
 612   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 613
 614   // Now remove the dummy entry.  If there are any problems with the
 615   // transactions, asserts will fire.
 616   AddDelChunk(safe_browsing_util::kMalwareList, 44);
 617
 618   // Test the subdel command.
 619   SubDelChunk(safe_browsing_util::kMalwareList, 4);
 620   database_->UpdateFinished(true);
 621
 622   GetListsInfo(&lists);
 623   ASSERT_LE(1U, lists.size());
 624   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 625   EXPECT_EQ("1,3,7", lists[0].adds);
 626   EXPECT_TRUE(lists[0].subs.empty());
 627
 628   // Test a sub command coming in before the add.
 629   chunks.clear();
 630   chunks.push_back(SubChunkPrefix2Value(5,
 631                                         "www.notevilanymore.com/index.html",
 632                                         10,
 633                                         "www.notevilanymore.com/good.html",
 634                                         10));
 635   ASSERT_TRUE(database_->UpdateStarted(&lists));
 636   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 637   database_->UpdateFinished(true);
 638
 639   EXPECT_FALSE(database_->ContainsBrowseUrl(
 640       GURL("http://www.notevilanymore.com/index.html"),
 641       &prefix_hits,
 642       &cache_hits));
 643
 644   // Now insert the tardy add chunk and we don't expect them to appear
 645   // in database because of the previous sub chunk.
 646   chunks.clear();
 647   chunks.push_back(AddChunkPrefix2Value(10,
 648                                         "www.notevilanymore.com/index.html",
 649                                         "www.notevilanymore.com/good.html"));
 650   ASSERT_TRUE(database_->UpdateStarted(&lists));
 651   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 652   database_->UpdateFinished(true);
 653
 654   EXPECT_FALSE(database_->ContainsBrowseUrl(
 655       GURL("http://www.notevilanymore.com/index.html"),
 656       &prefix_hits,
 657       &cache_hits));
 658
 659   EXPECT_FALSE(database_->ContainsBrowseUrl(
 660       GURL("http://www.notevilanymore.com/good.html"),
 661       &prefix_hits,
 662       &cache_hits));
 663
 664   // Reset and reload the database.  The database will rely on the prefix set.
 665   database_.reset(new SafeBrowsingDatabaseNew);
 666   database_->Init(database_filename_);
 667
 668   // Check that a prefix still hits.
 669   EXPECT_TRUE(database_->ContainsBrowseUrl(
 670       GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 671   ASSERT_EQ(1U, prefix_hits.size());
 672   EXPECT_EQ(SBPrefixForString("www.evil.com/phishing.html"), prefix_hits[0]);
 673
 674   // Also check that it's not just always returning true in this case.
 675   EXPECT_FALSE(database_->ContainsBrowseUrl(
 676       GURL("http://www.evil.com/"), &prefix_hits, &cache_hits));
 677
 678   // Check that the full hash is still present.
 679   EXPECT_TRUE(database_->ContainsBrowseUrl(
 680       GURL("http://www.evil.com/evil.html"), &prefix_hits, &cache_hits));
 681   ASSERT_EQ(1U, prefix_hits.size());
 682   EXPECT_EQ(SBPrefixForString("www.evil.com/evil.html"), prefix_hits[0]);
 683 }
 684
 685 // Test adding zero length chunks to the database.
 686 TEST_F(SafeBrowsingDatabaseTest, ZeroSizeChunk) {
 687   std::vector<SBListChunkRanges> lists;
 688   ScopedVector<SBChunkData> chunks;
 689
 690   // Populate with a couple of normal chunks.
 691   chunks.push_back(AddChunkPrefix2Value(1,
 692                                         "www.test.com/test1.html",
 693                                         "www.test.com/test2.html"));
 694   chunks.push_back(AddChunkPrefix2Value(10,
 695                                         "www.random.com/random1.html",
 696                                         "www.random.com/random2.html"));
 697
 698   ASSERT_TRUE(database_->UpdateStarted(&lists));
 699   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 700   database_->UpdateFinished(true);
 701
 702   // Add an empty ADD and SUB chunk.
 703   GetListsInfo(&lists);
 704   ASSERT_LE(1U, lists.size());
 705   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 706   EXPECT_EQ("1,10", lists[0].adds);
 707   EXPECT_TRUE(lists[0].subs.empty());
 708
 709   chunks.clear();
 710   chunks.push_back(BuildChunk(19, safe_browsing::ChunkData::ADD,
 711                               safe_browsing::ChunkData::PREFIX_4B,
 712                               NULL, 0, std::vector<int>()));
 713   chunks.push_back(BuildChunk(7, safe_browsing::ChunkData::SUB,
 714                               safe_browsing::ChunkData::PREFIX_4B,
 715                               NULL, 0, std::vector<int>()));
 716
 717   ASSERT_TRUE(database_->UpdateStarted(&lists));
 718   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 719   database_->UpdateFinished(true);
 720
 721   GetListsInfo(&lists);
 722   ASSERT_LE(1U, lists.size());
 723   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 724   EXPECT_EQ("1,10,19", lists[0].adds);
 725   EXPECT_EQ("7", lists[0].subs);
 726
 727   // Add an empty chunk along with a couple that contain data. This should
 728   // result in the chunk range being reduced in size.
 729   chunks.clear();
 730   chunks.push_back(AddChunkPrefixValue(20, "www.notempty.com/full1.html"));
 731   chunks.push_back(BuildChunk(21, safe_browsing::ChunkData::ADD,
 732                               safe_browsing::ChunkData::PREFIX_4B,
 733                               NULL, 0, std::vector<int>()));
 734   chunks.push_back(AddChunkPrefixValue(22, "www.notempty.com/full2.html"));
 735
 736   ASSERT_TRUE(database_->UpdateStarted(&lists));
 737   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 738   database_->UpdateFinished(true);
 739
 740   std::vector<SBPrefix> prefix_hits;
 741   std::vector<SBFullHashResult> cache_hits;
 742   EXPECT_TRUE(database_->ContainsBrowseUrl(
 743       GURL("http://www.notempty.com/full1.html"), &prefix_hits, &cache_hits));
 744   EXPECT_TRUE(database_->ContainsBrowseUrl(
 745       GURL("http://www.notempty.com/full2.html"), &prefix_hits, &cache_hits));
 746
 747   GetListsInfo(&lists);
 748   ASSERT_LE(1U, lists.size());
 749   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 750   EXPECT_EQ("1,10,19-22", lists[0].adds);
 751   EXPECT_EQ("7", lists[0].subs);
 752
 753   // Handle AddDel and SubDel commands for empty chunks.
 754   ASSERT_TRUE(database_->UpdateStarted(&lists));
 755   AddDelChunk(safe_browsing_util::kMalwareList, 21);
 756   database_->UpdateFinished(true);
 757
 758   GetListsInfo(&lists);
 759   ASSERT_LE(1U, lists.size());
 760   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 761   EXPECT_EQ("1,10,19-20,22", lists[0].adds);
 762   EXPECT_EQ("7", lists[0].subs);
 763
 764   ASSERT_TRUE(database_->UpdateStarted(&lists));
 765   SubDelChunk(safe_browsing_util::kMalwareList, 7);
 766   database_->UpdateFinished(true);
 767
 768   GetListsInfo(&lists);
 769   ASSERT_LE(1U, lists.size());
 770   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 771   EXPECT_EQ("1,10,19-20,22", lists[0].adds);
 772   EXPECT_TRUE(lists[0].subs.empty());
 773 }
 774
 775 // Utility function for setting up the database for the caching test.
 776 void SafeBrowsingDatabaseTest::PopulateDatabaseForCacheTest() {
 777   // Add a couple prefixes.
 778   ScopedVector<SBChunkData> chunks;
 779   chunks.push_back(AddChunkPrefix2Value(1,
 780                                         "www.evil.com/phishing.html",
 781                                         "www.evil.com/malware.html"));
 782
 783   std::vector<SBListChunkRanges> lists;
 784   ASSERT_TRUE(database_->UpdateStarted(&lists));
 785   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 786   database_->UpdateFinished(true);
 787
 788   // Cache should be cleared after updating.
 789   EXPECT_TRUE(database_->browse_gethash_cache_.empty());
 790
 791   SBFullHashResult full_hash;
 792   full_hash.list_id = safe_browsing_util::MALWARE;
 793
 794   std::vector<SBFullHashResult> results;
 795   std::vector<SBPrefix> prefixes;
 796
 797   // Add a fullhash result for each prefix.
 798   full_hash.hash = SBFullHashForString("www.evil.com/phishing.html");
 799   results.push_back(full_hash);
 800   prefixes.push_back(full_hash.hash.prefix);
 801
 802   full_hash.hash = SBFullHashForString("www.evil.com/malware.html");
 803   results.push_back(full_hash);
 804   prefixes.push_back(full_hash.hash.prefix);
 805
 806   database_->CacheHashResults(prefixes, results, kCacheLifetime);
 807 }
 808
 809 TEST_F(SafeBrowsingDatabaseTest, HashCaching) {
 810   PopulateDatabaseForCacheTest();
 811
 812   // We should have both full hashes in the cache.
 813   EXPECT_EQ(2U, database_->browse_gethash_cache_.size());
 814
 815   // Test the cache lookup for the first prefix.
 816   std::vector<SBPrefix> prefix_hits;
 817   std::vector<SBFullHashResult> cache_hits;
 818   EXPECT_TRUE(database_->ContainsBrowseUrl(
 819       GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 820   EXPECT_TRUE(prefix_hits.empty());
 821   ASSERT_EQ(1U, cache_hits.size());
 822   EXPECT_TRUE(SBFullHashEqual(
 823       cache_hits[0].hash, SBFullHashForString("www.evil.com/phishing.html")));
 824
 825   prefix_hits.clear();
 826   cache_hits.clear();
 827
 828   // Test the cache lookup for the second prefix.
 829   EXPECT_TRUE(database_->ContainsBrowseUrl(
 830       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 831   EXPECT_TRUE(prefix_hits.empty());
 832   ASSERT_EQ(1U, cache_hits.size());
 833   EXPECT_TRUE(SBFullHashEqual(
 834       cache_hits[0].hash, SBFullHashForString("www.evil.com/malware.html")));
 835
 836   prefix_hits.clear();
 837   cache_hits.clear();
 838
 839   // Test removing a prefix via a sub chunk.
 840   ScopedVector<SBChunkData> chunks;
 841   chunks.push_back(SubChunkPrefixValue(2, "www.evil.com/phishing.html", 1));
 842
 843   std::vector<SBListChunkRanges> lists;
 844   ASSERT_TRUE(database_->UpdateStarted(&lists));
 845   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 846   database_->UpdateFinished(true);
 847
 848   // This prefix should still be there, but cached fullhash should be gone.
 849   EXPECT_TRUE(database_->ContainsBrowseUrl(
 850       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 851   ASSERT_EQ(1U, prefix_hits.size());
 852   EXPECT_EQ(SBPrefixForString("www.evil.com/malware.html"), prefix_hits[0]);
 853   EXPECT_TRUE(cache_hits.empty());
 854   prefix_hits.clear();
 855   cache_hits.clear();
 856
 857   // This prefix should be gone.
 858   EXPECT_FALSE(database_->ContainsBrowseUrl(
 859       GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 860   prefix_hits.clear();
 861   cache_hits.clear();
 862
 863   // Test that an AddDel for the original chunk removes the last cached entry.
 864   ASSERT_TRUE(database_->UpdateStarted(&lists));
 865   AddDelChunk(safe_browsing_util::kMalwareList, 1);
 866   database_->UpdateFinished(true);
 867   EXPECT_FALSE(database_->ContainsBrowseUrl(
 868       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 869   EXPECT_TRUE(database_->browse_gethash_cache_.empty());
 870   prefix_hits.clear();
 871   cache_hits.clear();
 872
 873   // Test that the cache won't return expired values. First we have to adjust
 874   // the cached entries' received time to make them older, since the database
 875   // cache insert uses Time::Now(). First, store some entries.
 876   PopulateDatabaseForCacheTest();
 877
 878   std::map<SBPrefix, SBCachedFullHashResult>* hash_cache =
 879       &database_->browse_gethash_cache_;
 880   EXPECT_EQ(2U, hash_cache->size());
 881
 882   // Now adjust one of the entries times to be in the past.
 883   const SBPrefix key = SBPrefixForString("www.evil.com/malware.html");
 884   std::map<SBPrefix, SBCachedFullHashResult>::iterator iter =
 885       hash_cache->find(key);
 886   ASSERT_TRUE(iter != hash_cache->end());
 887   iter->second.expire_after = Time::Now() - TimeDelta::FromMinutes(1);
 888
 889   EXPECT_TRUE(database_->ContainsBrowseUrl(
 890       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 891   EXPECT_EQ(1U, prefix_hits.size());
 892   EXPECT_TRUE(cache_hits.empty());
 893   // Expired entry should have been removed from cache.
 894   EXPECT_EQ(1U, hash_cache->size());
 895
 896   // This entry should still exist.
 897   EXPECT_TRUE(database_->ContainsBrowseUrl(
 898       GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 899   EXPECT_TRUE(prefix_hits.empty());
 900   EXPECT_EQ(1U, cache_hits.size());
 901
 902   // Testing prefix miss caching. First, we clear out the existing database,
 903   // Since PopulateDatabaseForCacheTest() doesn't handle adding duplicate
 904   // chunks.
 905   ASSERT_TRUE(database_->UpdateStarted(&lists));
 906   AddDelChunk(safe_browsing_util::kMalwareList, 1);
 907   database_->UpdateFinished(true);
 908
 909   // Cache should be cleared after updating.
 910   EXPECT_TRUE(hash_cache->empty());
 911
 912   std::vector<SBPrefix> prefix_misses;
 913   std::vector<SBFullHashResult> empty_full_hash;
 914   prefix_misses.push_back(SBPrefixForString("http://www.bad.com/malware.html"));
 915   prefix_misses.push_back(
 916       SBPrefixForString("http://www.bad.com/phishing.html"));
 917   database_->CacheHashResults(prefix_misses, empty_full_hash, kCacheLifetime);
 918
 919   // Prefixes with no full results are misses.
 920   EXPECT_EQ(hash_cache->size(), prefix_misses.size());
 921   ASSERT_TRUE(
 922       hash_cache->count(SBPrefixForString("http://www.bad.com/malware.html")));
 923   EXPECT_TRUE(
 924       hash_cache->find(SBPrefixForString("http://www.bad.com/malware.html"))
 925           ->second.full_hashes.empty());
 926   ASSERT_TRUE(
 927       hash_cache->count(SBPrefixForString("http://www.bad.com/phishing.html")));
 928   EXPECT_TRUE(
 929       hash_cache->find(SBPrefixForString("http://www.bad.com/phishing.html"))
 930           ->second.full_hashes.empty());
 931
 932   // Update the database.
 933   PopulateDatabaseForCacheTest();
 934
 935   // Cache a GetHash miss for a particular prefix, and even though the prefix is
 936   // in the database, it is flagged as a miss so looking up the associated URL
 937   // will not succeed.
 938   prefix_hits.clear();
 939   cache_hits.clear();
 940   prefix_misses.clear();
 941   empty_full_hash.clear();
 942   prefix_misses.push_back(SBPrefixForString("www.evil.com/phishing.html"));
 943   database_->CacheHashResults(prefix_misses, empty_full_hash, kCacheLifetime);
 944   EXPECT_FALSE(database_->ContainsBrowseUrl(
 945       GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 946   prefix_hits.clear();
 947   cache_hits.clear();
 948
 949   // Test receiving a full add chunk.
 950   chunks.clear();
 951   chunks.push_back(AddChunkFullHash2Value(20,
 952                                           "www.fullevil.com/bad1.html",
 953                                           "www.fullevil.com/bad2.html"));
 954   ASSERT_TRUE(database_->UpdateStarted(&lists));
 955   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 956   database_->UpdateFinished(true);
 957
 958   EXPECT_TRUE(database_->ContainsBrowseUrl(
 959       GURL("http://www.fullevil.com/bad1.html"), &prefix_hits, &cache_hits));
 960   ASSERT_EQ(1U, prefix_hits.size());
 961   EXPECT_EQ(SBPrefixForString("www.fullevil.com/bad1.html"), prefix_hits[0]);
 962   EXPECT_TRUE(cache_hits.empty());
 963   prefix_hits.clear();
 964   cache_hits.clear();
 965
 966   EXPECT_TRUE(database_->ContainsBrowseUrl(
 967       GURL("http://www.fullevil.com/bad2.html"), &prefix_hits, &cache_hits));
 968   ASSERT_EQ(1U, prefix_hits.size());
 969   EXPECT_EQ(SBPrefixForString("www.fullevil.com/bad2.html"), prefix_hits[0]);
 970   EXPECT_TRUE(cache_hits.empty());
 971   prefix_hits.clear();
 972   cache_hits.clear();
 973
 974   // Test receiving a full sub chunk, which will remove one of the full adds.
 975   chunks.clear();
 976   chunks.push_back(SubChunkFullHashValue(200,
 977                                          "www.fullevil.com/bad1.html",
 978                                          20));
 979   ASSERT_TRUE(database_->UpdateStarted(&lists));
 980   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 981   database_->UpdateFinished(true);
 982
 983   EXPECT_FALSE(database_->ContainsBrowseUrl(
 984       GURL("http://www.fullevil.com/bad1.html"), &prefix_hits, &cache_hits));
 985
 986   // There should be one remaining full add.
 987   EXPECT_TRUE(database_->ContainsBrowseUrl(
 988       GURL("http://www.fullevil.com/bad2.html"), &prefix_hits, &cache_hits));
 989   ASSERT_EQ(1U, prefix_hits.size());
 990   EXPECT_EQ(SBPrefixForString("www.fullevil.com/bad2.html"), prefix_hits[0]);
 991   EXPECT_TRUE(cache_hits.empty());
 992   prefix_hits.clear();
 993   cache_hits.clear();
 994
 995   // Now test an AddDel for the remaining full add.
 996   ASSERT_TRUE(database_->UpdateStarted(&lists));
 997   AddDelChunk(safe_browsing_util::kMalwareList, 20);
 998   database_->UpdateFinished(true);
 999
1000   EXPECT_FALSE(database_->ContainsBrowseUrl(
1001       GURL("http://www.fullevil.com/bad1.html"), &prefix_hits, &cache_hits));
1002   EXPECT_FALSE(database_->ContainsBrowseUrl(
1003       GURL("http://www.fullevil.com/bad2.html"), &prefix_hits, &cache_hits));
1004
1005   // Add a fullhash which has a prefix collision for a known url.
1006   static const char kExampleFine[] = "www.example.com/fine.html";
1007   static const char kExampleCollision[] =
1008       "www.example.com/3123364814/malware.htm";
1009   ASSERT_EQ(SBPrefixForString(kExampleFine),
1010             SBPrefixForString(kExampleCollision));
1011   ASSERT_TRUE(database_->UpdateStarted(&lists));
1012   {
1013     ScopedVector<SBChunkData> chunks;
1014     chunks.push_back(AddChunkPrefixValue(21, kExampleCollision));
1015     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1016   }
1017   database_->UpdateFinished(true);
1018
1019   // Expect a prefix hit due to the collision between |kExampleFine| and
1020   // |kExampleCollision|.
1021   EXPECT_TRUE(database_->ContainsBrowseUrl(
1022       GURL(std::string("http://") + kExampleFine), &prefix_hits, &cache_hits));
1023   ASSERT_EQ(1U, prefix_hits.size());
1024   EXPECT_EQ(SBPrefixForString(kExampleFine), prefix_hits[0]);
1025   EXPECT_TRUE(cache_hits.empty());
1026
1027   // Cache gethash response for |kExampleCollision|.
1028   {
1029     SBFullHashResult result;
1030     result.hash = SBFullHashForString(kExampleCollision);
1031     result.list_id = safe_browsing_util::MALWARE;
1032     database_->CacheHashResults(std::vector<SBPrefix>(1, result.hash.prefix),
1033                                 std::vector<SBFullHashResult>(1, result),
1034                                 kCacheLifetime);
1035   }
1036
1037   // The cached response means the collision no longer causes a hit.
1038   EXPECT_FALSE(database_->ContainsBrowseUrl(
1039       GURL(std::string("http://") + kExampleFine), &prefix_hits, &cache_hits));
1040 }
1041
1042 // Test that corrupt databases are appropriately handled, even if the
1043 // corruption is detected in the midst of the update.
1044 // TODO(shess): Disabled until ScopedLogMessageIgnorer resolved.
1045 // http://crbug.com/56448
1046 TEST_F(SafeBrowsingDatabaseTest, DISABLED_FileCorruptionHandling) {
1047   // Re-create the database in a captive message loop so that we can
1048   // influence task-posting.  Database specifically needs to the
1049   // file-backed.
1050   database_.reset();
1051   base::MessageLoop loop;
1052   SafeBrowsingStoreFile* store = new SafeBrowsingStoreFile();
1053   database_.reset(new SafeBrowsingDatabaseNew(store, NULL, NULL, NULL, NULL,
1054                                               NULL, NULL));
1055   database_->Init(database_filename_);
1056
1057   // This will cause an empty database to be created.
1058   std::vector<SBListChunkRanges> lists;
1059   ASSERT_TRUE(database_->UpdateStarted(&lists));
1060   database_->UpdateFinished(true);
1061
1062   // Create a sub chunk to insert.
1063   ScopedVector<SBChunkData> chunks;
1064   chunks.push_back(SubChunkPrefixValue(7,
1065                                        "www.subbed.com/notevil1.html",
1066                                        19));
1067
1068   // Corrupt the file by corrupting the checksum, which is not checked
1069   // until the entire table is read in |UpdateFinished()|.
1070   FILE* fp = base::OpenFile(database_filename_, "r+");
1071   ASSERT_TRUE(fp);
1072   ASSERT_NE(-1, fseek(fp, -8, SEEK_END));
1073   for (size_t i = 0; i < 8; ++i) {
1074     fputc('!', fp);
1075   }
1076   fclose(fp);
1077
1078   {
1079     // The following code will cause DCHECKs, so suppress the crashes.
1080     ScopedLogMessageIgnorer ignorer;
1081
1082     // Start an update.  The insert will fail due to corruption.
1083     ASSERT_TRUE(database_->UpdateStarted(&lists));
1084     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1085     database_->UpdateFinished(true);
1086
1087     // Database file still exists until the corruption handler has run.
1088     EXPECT_TRUE(base::PathExists(database_filename_));
1089
1090     // Flush through the corruption-handler task.
1091     VLOG(1) << "Expect failed check on: SafeBrowsing database reset";
1092     base::MessageLoop::current()->RunUntilIdle();
1093   }
1094
1095   // Database file should not exist.
1096   EXPECT_FALSE(base::PathExists(database_filename_));
1097
1098   // Run the update again successfully.
1099   ASSERT_TRUE(database_->UpdateStarted(&lists));
1100   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1101   database_->UpdateFinished(true);
1102   EXPECT_TRUE(base::PathExists(database_filename_));
1103
1104   database_.reset();
1105 }
1106
1107 // Checks database reading and writing.
1108 TEST_F(SafeBrowsingDatabaseTest, ContainsDownloadUrl) {
1109   database_.reset();
1110   base::MessageLoop loop;
1111   SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile();
1112   SafeBrowsingStoreFile* download_store = new SafeBrowsingStoreFile();
1113   SafeBrowsingStoreFile* csd_whitelist_store = new SafeBrowsingStoreFile();
1114   database_.reset(new SafeBrowsingDatabaseNew(browse_store,
1115                                               download_store,
1116                                               csd_whitelist_store,
1117                                               NULL,
1118                                               NULL,
1119                                               NULL,
1120                                               NULL));
1121   database_->Init(database_filename_);
1122
1123   const char kEvil1Url1[] = "www.evil1.com/download1/";
1124   const char kEvil1Url2[] = "www.evil1.com/download2.html";
1125
1126   // Add a simple chunk with one hostkey for download url list.
1127   ScopedVector<SBChunkData> chunks;
1128   chunks.push_back(AddChunkPrefix2Value(1, kEvil1Url1, kEvil1Url2));
1129
1130   std::vector<SBListChunkRanges> lists;
1131   ASSERT_TRUE(database_->UpdateStarted(&lists));
1132   database_->InsertChunks(safe_browsing_util::kBinUrlList, chunks.get());
1133   database_->UpdateFinished(true);
1134
1135   std::vector<SBPrefix> prefix_hits;
1136   std::vector<GURL> urls(1);
1137
1138   urls[0] = GURL(std::string("http://") + kEvil1Url1);
1139   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1140   ASSERT_EQ(1U, prefix_hits.size());
1141   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1142
1143   urls[0] = GURL(std::string("http://") + kEvil1Url2);
1144   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1145   ASSERT_EQ(1U, prefix_hits.size());
1146   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[0]);
1147
1148   urls[0] = GURL(std::string("https://") + kEvil1Url2);
1149   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1150   ASSERT_EQ(1U, prefix_hits.size());
1151   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[0]);
1152
1153   urls[0] = GURL(std::string("ftp://") + kEvil1Url2);
1154   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1155   ASSERT_EQ(1U, prefix_hits.size());
1156   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[0]);
1157
1158   urls[0] = GURL("http://www.randomevil.com");
1159   EXPECT_FALSE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1160
1161   // Should match with query args stripped.
1162   urls[0] = GURL(std::string("http://") + kEvil1Url2 + "?blah");
1163   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1164   ASSERT_EQ(1U, prefix_hits.size());
1165   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[0]);
1166
1167   // Should match with extra path stuff and query args stripped.
1168   urls[0] = GURL(std::string("http://") + kEvil1Url1 + "foo/bar?blah");
1169   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1170   ASSERT_EQ(1U, prefix_hits.size());
1171   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1172
1173   // First hit in redirect chain is malware.
1174   urls.clear();
1175   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1176   urls.push_back(GURL("http://www.randomevil.com"));
1177   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1178   ASSERT_EQ(1U, prefix_hits.size());
1179   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1180
1181   // Middle hit in redirect chain is malware.
1182   urls.clear();
1183   urls.push_back(GURL("http://www.randomevil.com"));
1184   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1185   urls.push_back(GURL("http://www.randomevil2.com"));
1186   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1187   ASSERT_EQ(1U, prefix_hits.size());
1188   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1189
1190   // Final hit in redirect chain is malware.
1191   urls.clear();
1192   urls.push_back(GURL("http://www.randomevil.com"));
1193   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1194   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1195   ASSERT_EQ(1U, prefix_hits.size());
1196   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1197
1198   // Multiple hits in redirect chain are in malware list.
1199   urls.clear();
1200   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1201   urls.push_back(GURL(std::string("https://") + kEvil1Url2));
1202   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1203   ASSERT_EQ(2U, prefix_hits.size());
1204   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1205   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[1]);
1206   database_.reset();
1207 }
1208
1209 // Checks that the whitelists are handled properly.
1210 TEST_F(SafeBrowsingDatabaseTest, Whitelists) {
1211   database_.reset();
1212
1213   // We expect all calls to ContainsCsdWhitelistedUrl in particular to be made
1214   // from the IO thread.  In general the whitelist lookups are thread-safe.
1215   content::TestBrowserThreadBundle thread_bundle_;
1216
1217   // If the whitelist is disabled everything should match the whitelist.
1218   database_.reset(new SafeBrowsingDatabaseNew(new SafeBrowsingStoreFile(),
1219                                               NULL, NULL, NULL, NULL, NULL,
1220                                               NULL));
1221   database_->Init(database_filename_);
1222   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1223       GURL(std::string("http://www.phishing.com/"))));
1224   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1225       GURL(std::string("http://www.phishing.com/"))));
1226   EXPECT_TRUE(database_->ContainsDownloadWhitelistedString("asdf"));
1227
1228   SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile();
1229   SafeBrowsingStoreFile* csd_whitelist_store = new SafeBrowsingStoreFile();
1230   SafeBrowsingStoreFile* download_whitelist_store = new SafeBrowsingStoreFile();
1231   SafeBrowsingStoreFile* extension_blacklist_store =
1232       new SafeBrowsingStoreFile();
1233   database_.reset(new SafeBrowsingDatabaseNew(browse_store, NULL,
1234                                               csd_whitelist_store,
1235                                               download_whitelist_store,
1236                                               extension_blacklist_store,
1237                                               NULL, NULL));
1238   database_->Init(database_filename_);
1239
1240   const char kGood1Host[] = "www.good1.com/";
1241   const char kGood1Url1[] = "www.good1.com/a/b.html";
1242   const char kGood1Url2[] = "www.good1.com/b/";
1243
1244   const char kGood2Url1[] = "www.good2.com/c";  // Should match '/c/bla'.
1245
1246   // good3.com/a/b/c/d/e/f/g/ should match because it's a whitelist.
1247   const char kGood3Url1[] = "good3.com/";
1248
1249   const char kGoodString[] = "good_string";
1250
1251   ScopedVector<SBChunkData> csd_chunks;
1252   ScopedVector<SBChunkData> download_chunks;
1253
1254   // Add two simple chunks to the csd whitelist.
1255   csd_chunks.push_back(AddChunkFullHash2Value(1, kGood1Url1, kGood1Url2));
1256   csd_chunks.push_back(AddChunkFullHashValue(2, kGood2Url1));
1257   download_chunks.push_back(AddChunkFullHashValue(2, kGood2Url1));
1258   download_chunks.push_back(AddChunkFullHashValue(3, kGoodString));
1259   download_chunks.push_back(AddChunkFullHashValue(4, kGood3Url1));
1260
1261   std::vector<SBListChunkRanges> lists;
1262   ASSERT_TRUE(database_->UpdateStarted(&lists));
1263   database_->InsertChunks(safe_browsing_util::kCsdWhiteList, csd_chunks.get());
1264   database_->InsertChunks(safe_browsing_util::kDownloadWhiteList,
1265                           download_chunks.get());
1266   database_->UpdateFinished(true);
1267
1268   EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl(
1269       GURL(std::string("http://") + kGood1Host)));
1270
1271   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1272       GURL(std::string("http://") + kGood1Url1)));
1273   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1274       GURL(std::string("http://") + kGood1Url1 + "?a=b")));
1275
1276   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1277       GURL(std::string("http://") + kGood1Url2)));
1278   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1279       GURL(std::string("http://") + kGood1Url2 + "/c.html")));
1280
1281   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1282       GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1283
1284   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1285       GURL(std::string("http://") + kGood2Url1 + "/c")));
1286   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1287       GURL(std::string("http://") + kGood2Url1 + "/c?bla")));
1288   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1289       GURL(std::string("http://") + kGood2Url1 + "/c/bla")));
1290
1291   EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl(
1292       GURL(std::string("http://www.google.com/"))));
1293
1294   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1295       GURL(std::string("http://") + kGood2Url1 + "/c")));
1296   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1297       GURL(std::string("http://") + kGood2Url1 + "/c?bla")));
1298   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1299       GURL(std::string("http://") + kGood2Url1 + "/c/bla")));
1300
1301   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1302       GURL(std::string("http://good3.com/a/b/c/d/e/f/g/"))));
1303   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1304       GURL(std::string("http://a.b.good3.com/"))));
1305
1306   EXPECT_FALSE(database_->ContainsDownloadWhitelistedString("asdf"));
1307   EXPECT_TRUE(database_->ContainsDownloadWhitelistedString(kGoodString));
1308
1309   EXPECT_FALSE(database_->ContainsDownloadWhitelistedUrl(
1310       GURL(std::string("http://www.google.com/"))));
1311
1312   // The CSD whitelist killswitch is not present.
1313   EXPECT_FALSE(database_->IsCsdWhitelistKillSwitchOn());
1314
1315   // Test only add the malware IP killswitch
1316   csd_chunks.clear();
1317   csd_chunks.push_back(AddChunkFullHashValue(
1318       15, "sb-ssl.google.com/safebrowsing/csd/killswitch_malware"));
1319
1320   ASSERT_TRUE(database_->UpdateStarted(&lists));
1321   database_->InsertChunks(safe_browsing_util::kCsdWhiteList, csd_chunks.get());
1322   database_->UpdateFinished(true);
1323
1324   EXPECT_TRUE(database_->IsMalwareIPMatchKillSwitchOn());
1325   // The CSD whitelist killswitch is not present.
1326   EXPECT_FALSE(database_->IsCsdWhitelistKillSwitchOn());
1327
1328   // Test that the kill-switch works as intended.
1329   csd_chunks.clear();
1330   download_chunks.clear();
1331   lists.clear();
1332   csd_chunks.push_back(AddChunkFullHashValue(
1333       5, "sb-ssl.google.com/safebrowsing/csd/killswitch"));
1334   download_chunks.push_back(AddChunkFullHashValue(
1335       5, "sb-ssl.google.com/safebrowsing/csd/killswitch"));
1336
1337   ASSERT_TRUE(database_->UpdateStarted(&lists));
1338   database_->InsertChunks(safe_browsing_util::kCsdWhiteList, csd_chunks.get());
1339   database_->InsertChunks(safe_browsing_util::kDownloadWhiteList,
1340                           download_chunks.get());
1341   database_->UpdateFinished(true);
1342
1343   // The CSD whitelist killswitch is present.
1344   EXPECT_TRUE(database_->IsCsdWhitelistKillSwitchOn());
1345   EXPECT_TRUE(database_->IsMalwareIPMatchKillSwitchOn());
1346   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1347       GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1348   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1349       GURL(std::string("http://www.google.com/"))));
1350   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1351       GURL(std::string("http://www.phishing_url.com/"))));
1352
1353   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1354       GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1355   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1356       GURL(std::string("http://www.google.com/"))));
1357   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1358       GURL(std::string("http://www.phishing_url.com/"))));
1359
1360   EXPECT_TRUE(database_->ContainsDownloadWhitelistedString("asdf"));
1361   EXPECT_TRUE(database_->ContainsDownloadWhitelistedString(kGoodString));
1362
1363   // Remove the kill-switch and verify that we can recover.
1364   csd_chunks.clear();
1365   download_chunks.clear();
1366   lists.clear();
1367   csd_chunks.push_back(SubChunkFullHashValue(
1368       1, "sb-ssl.google.com/safebrowsing/csd/killswitch", 5));
1369   csd_chunks.push_back(SubChunkFullHashValue(
1370       10, "sb-ssl.google.com/safebrowsing/csd/killswitch_malware", 15));
1371   download_chunks.push_back(SubChunkFullHashValue(
1372       1, "sb-ssl.google.com/safebrowsing/csd/killswitch", 5));
1373
1374   ASSERT_TRUE(database_->UpdateStarted(&lists));
1375   database_->InsertChunks(safe_browsing_util::kCsdWhiteList, csd_chunks.get());
1376   database_->InsertChunks(safe_browsing_util::kDownloadWhiteList,
1377                           download_chunks.get());
1378   database_->UpdateFinished(true);
1379
1380   EXPECT_FALSE(database_->IsMalwareIPMatchKillSwitchOn());
1381   EXPECT_FALSE(database_->IsCsdWhitelistKillSwitchOn());
1382   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1383       GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1384   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1385       GURL(std::string("https://") + kGood2Url1 + "/c/bla")));
1386   EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl(
1387       GURL(std::string("http://www.google.com/"))));
1388   EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl(
1389       GURL(std::string("http://www.phishing_url.com/"))));
1390
1391   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1392       GURL(std::string("https://") + kGood2Url1 + "/c/bla")));
1393   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1394       GURL(std::string("https://good3.com/"))));
1395   EXPECT_TRUE(database_->ContainsDownloadWhitelistedString(kGoodString));
1396   EXPECT_FALSE(database_->ContainsDownloadWhitelistedUrl(
1397       GURL(std::string("http://www.google.com/"))));
1398   EXPECT_FALSE(database_->ContainsDownloadWhitelistedUrl(
1399       GURL(std::string("http://www.phishing_url.com/"))));
1400   EXPECT_FALSE(database_->ContainsDownloadWhitelistedString("asdf"));
1401
1402   database_.reset();
1403 }
1404
1405 // Test to make sure we could insert chunk list that
1406 // contains entries for the same host.
1407 TEST_F(SafeBrowsingDatabaseTest, SameHostEntriesOkay) {
1408   ScopedVector<SBChunkData> chunks;
1409
1410   // Add a malware add chunk with two entries of the same host.
1411   chunks.push_back(AddChunkPrefix2Value(1,
1412                                         "www.evil.com/malware1.html",
1413                                         "www.evil.com/malware2.html"));
1414
1415   // Insert the testing chunks into database.
1416   std::vector<SBListChunkRanges> lists;
1417   ASSERT_TRUE(database_->UpdateStarted(&lists));
1418   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1419   database_->UpdateFinished(true);
1420
1421   GetListsInfo(&lists);
1422   ASSERT_LE(1U, lists.size());
1423   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
1424   EXPECT_EQ("1", lists[0].adds);
1425   EXPECT_TRUE(lists[0].subs.empty());
1426
1427   // Add a phishing add chunk with two entries of the same host.
1428   chunks.clear();
1429   chunks.push_back(AddChunkPrefix2Value(47,
1430                                         "www.evil.com/phishing1.html",
1431                                         "www.evil.com/phishing2.html"));
1432
1433   ASSERT_TRUE(database_->UpdateStarted(&lists));
1434   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks.get());
1435   database_->UpdateFinished(true);
1436
1437   GetListsInfo(&lists);
1438   ASSERT_EQ(2U, lists.size());
1439   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
1440   EXPECT_EQ("1", lists[0].adds);
1441   EXPECT_TRUE(lists[0].subs.empty());
1442   EXPECT_EQ(safe_browsing_util::kPhishingList, lists[1].name);
1443   EXPECT_EQ("47", lists[1].adds);
1444   EXPECT_TRUE(lists[1].subs.empty());
1445
1446   std::vector<SBPrefix> prefix_hits;
1447   std::vector<SBFullHashResult> cache_hits;
1448
1449   EXPECT_TRUE(database_->ContainsBrowseUrl(
1450       GURL("http://www.evil.com/malware1.html"), &prefix_hits, &cache_hits));
1451   EXPECT_TRUE(database_->ContainsBrowseUrl(
1452       GURL("http://www.evil.com/malware2.html"), &prefix_hits, &cache_hits));
1453   EXPECT_TRUE(database_->ContainsBrowseUrl(
1454       GURL("http://www.evil.com/phishing1.html"), &prefix_hits, &cache_hits));
1455   EXPECT_TRUE(database_->ContainsBrowseUrl(
1456       GURL("http://www.evil.com/phishing2.html"), &prefix_hits, &cache_hits));
1457
1458   // Test removing a single prefix from the add chunk.
1459   // Remove the prefix that added first.
1460   chunks.clear();
1461   chunks.push_back(SubChunkPrefixValue(4, "www.evil.com/malware1.html", 1));
1462   ASSERT_TRUE(database_->UpdateStarted(&lists));
1463   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1464   database_->UpdateFinished(true);
1465
1466   // Remove the prefix that added last.
1467   chunks.clear();
1468   chunks.push_back(SubChunkPrefixValue(5, "www.evil.com/phishing2.html", 47));
1469   ASSERT_TRUE(database_->UpdateStarted(&lists));
1470   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks.get());
1471   database_->UpdateFinished(true);
1472
1473   // Verify that the database contains urls expected.
1474   EXPECT_FALSE(database_->ContainsBrowseUrl(
1475       GURL("http://www.evil.com/malware1.html"), &prefix_hits, &cache_hits));
1476   EXPECT_TRUE(database_->ContainsBrowseUrl(
1477       GURL("http://www.evil.com/malware2.html"), &prefix_hits, &cache_hits));
1478   EXPECT_TRUE(database_->ContainsBrowseUrl(
1479       GURL("http://www.evil.com/phishing1.html"), &prefix_hits, &cache_hits));
1480   EXPECT_FALSE(database_->ContainsBrowseUrl(
1481       GURL("http://www.evil.com/phishing2.html"), &prefix_hits, &cache_hits));
1482 }
1483
1484 // Test that an empty update doesn't actually update the database.
1485 // This isn't a functionality requirement, but it is a useful
1486 // optimization.
1487 TEST_F(SafeBrowsingDatabaseTest, EmptyUpdate) {
1488   ScopedVector<SBChunkData> chunks;
1489
1490   base::FilePath filename = database_->BrowseDBFilename(database_filename_);
1491
1492   // Prime the database.
1493   std::vector<SBListChunkRanges> lists;
1494   ASSERT_TRUE(database_->UpdateStarted(&lists));
1495   chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/malware.html"));
1496   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1497   database_->UpdateFinished(true);
1498
1499   // Get an older time to reset the lastmod time for detecting whether
1500   // the file has been updated.
1501   base::File::Info before_info, after_info;
1502   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1503   const Time old_last_modified =
1504       before_info.last_modified - TimeDelta::FromSeconds(10);
1505
1506   // Inserting another chunk updates the database file.  The sleep is
1507   // needed because otherwise the entire test can finish w/in the
1508   // resolution of the lastmod time.
1509   ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1510   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1511   ASSERT_TRUE(database_->UpdateStarted(&lists));
1512   chunks.push_back(AddChunkPrefixValue(2, "www.foo.com/malware.html"));
1513   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1514   database_->UpdateFinished(true);
1515   ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1516   EXPECT_LT(before_info.last_modified, after_info.last_modified);
1517
1518   // Deleting a chunk updates the database file.
1519   ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1520   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1521   ASSERT_TRUE(database_->UpdateStarted(&lists));
1522   AddDelChunk(safe_browsing_util::kMalwareList, 2);
1523   database_->UpdateFinished(true);
1524   ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1525   EXPECT_LT(before_info.last_modified, after_info.last_modified);
1526
1527   // Simply calling |UpdateStarted()| then |UpdateFinished()| does not
1528   // update the database file.
1529   ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1530   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1531   ASSERT_TRUE(database_->UpdateStarted(&lists));
1532   database_->UpdateFinished(true);
1533   ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1534   EXPECT_EQ(before_info.last_modified, after_info.last_modified);
1535 }
1536
1537 // Test that a filter file is written out during update and read back
1538 // in during setup.
1539 TEST_F(SafeBrowsingDatabaseTest, FilterFile) {
1540   // Create a database with trivial example data and write it out.
1541   {
1542     // Prime the database.
1543     std::vector<SBListChunkRanges> lists;
1544     ASSERT_TRUE(database_->UpdateStarted(&lists));
1545
1546     ScopedVector<SBChunkData> chunks;
1547     chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/malware.html"));
1548     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1549     database_->UpdateFinished(true);
1550   }
1551
1552   // Find the malware url in the database, don't find a good url.
1553   std::vector<SBPrefix> prefix_hits;
1554   std::vector<SBFullHashResult> cache_hits;
1555   EXPECT_TRUE(database_->ContainsBrowseUrl(
1556       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
1557   EXPECT_FALSE(database_->ContainsBrowseUrl(
1558       GURL("http://www.good.com/goodware.html"), &prefix_hits, &cache_hits));
1559
1560   base::FilePath filter_file = database_->PrefixSetForFilename(
1561       database_->BrowseDBFilename(database_filename_));
1562
1563   // After re-creating the database, it should have a filter read from
1564   // a file, so it should find the same results.
1565   ASSERT_TRUE(base::PathExists(filter_file));
1566   database_.reset(new SafeBrowsingDatabaseNew);
1567   database_->Init(database_filename_);
1568   EXPECT_TRUE(database_->ContainsBrowseUrl(
1569       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
1570   EXPECT_FALSE(database_->ContainsBrowseUrl(
1571       GURL("http://www.good.com/goodware.html"), &prefix_hits, &cache_hits));
1572
1573   // If there is no filter file, the database cannot find malware urls.
1574   base::DeleteFile(filter_file, false);
1575   ASSERT_FALSE(base::PathExists(filter_file));
1576   database_.reset(new SafeBrowsingDatabaseNew);
1577   database_->Init(database_filename_);
1578   EXPECT_FALSE(database_->ContainsBrowseUrl(
1579       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
1580   EXPECT_FALSE(database_->ContainsBrowseUrl(
1581       GURL("http://www.good.com/goodware.html"), &prefix_hits, &cache_hits));
1582 }
1583
1584 TEST_F(SafeBrowsingDatabaseTest, CachedFullMiss) {
1585   const SBPrefix kPrefix1 = 1001U;
1586   const SBFullHash kFullHash1_1 =
1587       SBFullHashForPrefixAndSuffix(kPrefix1, "\x01");
1588
1589   const SBPrefix kPrefix2 = 1002U;
1590   const SBFullHash kFullHash2_1 =
1591       SBFullHashForPrefixAndSuffix(kPrefix2, "\x01");
1592
1593   // Insert prefix kPrefix1 and kPrefix2 into database.
1594   ScopedVector<SBChunkData> chunks;
1595   chunks.push_back(AddChunkPrefix(1, kPrefix1));
1596   chunks.push_back(AddChunkPrefix(2, kPrefix2));
1597
1598   std::vector<SBListChunkRanges> lists;
1599   ASSERT_TRUE(database_->UpdateStarted(&lists));
1600   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1601   database_->UpdateFinished(true);
1602
1603   {
1604     // Cache a full miss result for kPrefix1.
1605     std::vector<SBPrefix> prefixes(1, kPrefix1);
1606     std::vector<SBFullHashResult> cache_results;
1607     database_->CacheHashResults(prefixes, cache_results, kCacheLifetime);
1608   }
1609
1610   {
1611     // kFullHash1_1 gets no prefix hit because of the cached item, and also does
1612     // not have a cache hit.
1613     std::vector<SBFullHash> full_hashes(1, kFullHash1_1);
1614     std::vector<SBPrefix> prefix_hits;
1615     std::vector<SBFullHashResult> cache_hits;
1616     EXPECT_FALSE(database_->ContainsBrowseUrlHashes(
1617         full_hashes, &prefix_hits, &cache_hits));
1618
1619     // kFullHash2_1 gets a hit from the prefix in the database.
1620     full_hashes.push_back(kFullHash2_1);
1621     prefix_hits.clear();
1622     cache_hits.clear();
1623     EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1624         full_hashes, &prefix_hits, &cache_hits));
1625     ASSERT_EQ(1U, prefix_hits.size());
1626     EXPECT_EQ(kPrefix2, prefix_hits[0]);
1627     EXPECT_TRUE(cache_hits.empty());
1628   }
1629 }
1630
1631 TEST_F(SafeBrowsingDatabaseTest, CachedPrefixHitFullMiss) {
1632   const SBPrefix kPrefix1 = 1001U;
1633   const SBFullHash kFullHash1_1 =
1634       SBFullHashForPrefixAndSuffix(kPrefix1, "\x01");
1635   const SBFullHash kFullHash1_2 =
1636       SBFullHashForPrefixAndSuffix(kPrefix1, "\x02");
1637   const SBFullHash kFullHash1_3 =
1638       SBFullHashForPrefixAndSuffix(kPrefix1, "\x03");
1639
1640   const SBPrefix kPrefix2 = 1002U;
1641   const SBFullHash kFullHash2_1 =
1642       SBFullHashForPrefixAndSuffix(kPrefix2, "\x01");
1643
1644   const SBPrefix kPrefix3 = 1003U;
1645   const SBFullHash kFullHash3_1 =
1646       SBFullHashForPrefixAndSuffix(kPrefix3, "\x01");
1647
1648   // Insert prefix kPrefix1 and kPrefix2 into database.
1649   ScopedVector<SBChunkData> chunks;
1650   chunks.push_back(AddChunkPrefix(1, kPrefix1));
1651   chunks.push_back(AddChunkPrefix(2, kPrefix2));
1652
1653   std::vector<SBListChunkRanges> lists;
1654   ASSERT_TRUE(database_->UpdateStarted(&lists));
1655   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1656   database_->UpdateFinished(true);
1657
1658   {
1659     // kFullHash1_1 has a prefix hit of kPrefix1.
1660     std::vector<SBFullHash> full_hashes;
1661     full_hashes.push_back(kFullHash1_1);
1662     std::vector<SBPrefix> prefix_hits;
1663     std::vector<SBFullHashResult> cache_hits;
1664     EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1665         full_hashes, &prefix_hits, &cache_hits));
1666     ASSERT_EQ(1U, prefix_hits.size());
1667     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1668     EXPECT_TRUE(cache_hits.empty());
1669
1670     // kFullHash2_1 has a prefix hit of kPrefix2.
1671     full_hashes.push_back(kFullHash2_1);
1672     prefix_hits.clear();
1673     cache_hits.clear();
1674     EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1675         full_hashes, &prefix_hits, &cache_hits));
1676     ASSERT_EQ(2U, prefix_hits.size());
1677     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1678     EXPECT_EQ(kPrefix2, prefix_hits[1]);
1679     EXPECT_TRUE(cache_hits.empty());
1680
1681     // kFullHash3_1 has no hits.
1682     full_hashes.push_back(kFullHash3_1);
1683     prefix_hits.clear();
1684     cache_hits.clear();
1685     EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1686         full_hashes, &prefix_hits, &cache_hits));
1687     ASSERT_EQ(2U, prefix_hits.size());
1688     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1689     EXPECT_EQ(kPrefix2, prefix_hits[1]);
1690     EXPECT_TRUE(cache_hits.empty());
1691   }
1692
1693   {
1694     // Cache a fullhash result for two kPrefix1 full hashes.
1695     std::vector<SBPrefix> prefixes(1, kPrefix1);
1696     std::vector<SBFullHashResult> cache_results;
1697
1698     SBFullHashResult full_hash_result;
1699     full_hash_result.list_id = safe_browsing_util::MALWARE;
1700
1701     full_hash_result.hash = kFullHash1_1;
1702     cache_results.push_back(full_hash_result);
1703
1704     full_hash_result.hash = kFullHash1_3;
1705     cache_results.push_back(full_hash_result);
1706
1707     database_->CacheHashResults(prefixes, cache_results, kCacheLifetime);
1708   }
1709
1710   {
1711     // kFullHash1_1 should now see a cache hit.
1712     std::vector<SBFullHash> full_hashes(1, kFullHash1_1);
1713     std::vector<SBPrefix> prefix_hits;
1714     std::vector<SBFullHashResult> cache_hits;
1715     EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1716         full_hashes, &prefix_hits, &cache_hits));
1717     EXPECT_TRUE(prefix_hits.empty());
1718     ASSERT_EQ(1U, cache_hits.size());
1719     EXPECT_TRUE(SBFullHashEqual(kFullHash1_1, cache_hits[0].hash));
1720
1721     // Adding kFullHash2_1 will see the existing cache hit plus the prefix hit
1722     // for kPrefix2.
1723     full_hashes.push_back(kFullHash2_1);
1724     prefix_hits.clear();
1725     cache_hits.clear();
1726     EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1727         full_hashes, &prefix_hits, &cache_hits));
1728     ASSERT_EQ(1U, prefix_hits.size());
1729     EXPECT_EQ(kPrefix2, prefix_hits[0]);
1730     ASSERT_EQ(1U, cache_hits.size());
1731     EXPECT_TRUE(SBFullHashEqual(kFullHash1_1, cache_hits[0].hash));
1732
1733     // kFullHash1_3 also gets a cache hit.
1734     full_hashes.push_back(kFullHash1_3);
1735     prefix_hits.clear();
1736     cache_hits.clear();
1737     EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1738         full_hashes, &prefix_hits, &cache_hits));
1739     ASSERT_EQ(1U, prefix_hits.size());
1740     EXPECT_EQ(kPrefix2, prefix_hits[0]);
1741     ASSERT_EQ(2U, cache_hits.size());
1742     EXPECT_TRUE(SBFullHashEqual(kFullHash1_1, cache_hits[0].hash));
1743     EXPECT_TRUE(SBFullHashEqual(kFullHash1_3, cache_hits[1].hash));
1744   }
1745
1746   {
1747     // Check if DB contains only kFullHash1_3. Should return a cache hit.
1748     std::vector<SBFullHash> full_hashes(1, kFullHash1_3);
1749     std::vector<SBPrefix> prefix_hits;
1750     std::vector<SBFullHashResult> cache_hits;
1751     EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1752         full_hashes, &prefix_hits, &cache_hits));
1753     EXPECT_TRUE(prefix_hits.empty());
1754     ASSERT_EQ(1U, cache_hits.size());
1755     EXPECT_TRUE(SBFullHashEqual(kFullHash1_3, cache_hits[0].hash));
1756   }
1757
1758   {
1759     // kFullHash1_2 has no cache hit, and no prefix hit because of the cache for
1760     // kPrefix1.
1761     std::vector<SBFullHash> full_hashes(1, kFullHash1_2);
1762     std::vector<SBPrefix> prefix_hits;
1763     std::vector<SBFullHashResult> cache_hits;
1764     EXPECT_FALSE(database_->ContainsBrowseUrlHashes(
1765         full_hashes, &prefix_hits, &cache_hits));
1766
1767     // Other prefix hits possible when kFullHash1_2 hits nothing.
1768     full_hashes.push_back(kFullHash2_1);
1769     prefix_hits.clear();
1770     cache_hits.clear();
1771     EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1772         full_hashes, &prefix_hits, &cache_hits));
1773     ASSERT_EQ(1U, prefix_hits.size());
1774     EXPECT_EQ(kPrefix2, prefix_hits[0]);
1775     EXPECT_TRUE(cache_hits.empty());
1776   }
1777 }
1778
1779 TEST_F(SafeBrowsingDatabaseTest, BrowseFullHashMatching) {
1780   const SBPrefix kPrefix1 = 1001U;
1781   const SBFullHash kFullHash1_1 =
1782       SBFullHashForPrefixAndSuffix(kPrefix1, "\x01");
1783   const SBFullHash kFullHash1_2 =
1784       SBFullHashForPrefixAndSuffix(kPrefix1, "\x02");
1785   const SBFullHash kFullHash1_3 =
1786       SBFullHashForPrefixAndSuffix(kPrefix1, "\x03");
1787
1788   // Insert two full hashes with a shared prefix.
1789   ScopedVector<SBChunkData> chunks;
1790   chunks.push_back(AddChunkFullHash(1, kFullHash1_1));
1791   chunks.push_back(AddChunkFullHash(2, kFullHash1_2));
1792
1793   std::vector<SBListChunkRanges> lists;
1794   ASSERT_TRUE(database_->UpdateStarted(&lists));
1795   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1796   database_->UpdateFinished(true);
1797
1798   {
1799     // Check a full hash which isn't present.
1800     std::vector<SBFullHash> full_hashes(1, kFullHash1_3);
1801     std::vector<SBPrefix> prefix_hits;
1802     std::vector<SBFullHashResult> cache_hits;
1803     EXPECT_FALSE(database_->ContainsBrowseUrlHashes(
1804         full_hashes, &prefix_hits, &cache_hits));
1805
1806     // Also one which is present, should have a prefix hit.
1807     full_hashes.push_back(kFullHash1_1);
1808     prefix_hits.clear();
1809     cache_hits.clear();
1810     EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1811         full_hashes, &prefix_hits, &cache_hits));
1812     ASSERT_EQ(1U, prefix_hits.size());
1813     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1814     EXPECT_TRUE(cache_hits.empty());
1815
1816     // Two full hash matches with the same prefix should return one prefix hit.
1817     full_hashes.push_back(kFullHash1_2);
1818     prefix_hits.clear();
1819     cache_hits.clear();
1820     EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1821         full_hashes, &prefix_hits, &cache_hits));
1822     ASSERT_EQ(1U, prefix_hits.size());
1823     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1824     EXPECT_TRUE(cache_hits.empty());
1825   }
1826
1827   {
1828     // Cache a gethash result for kFullHash1_2.
1829     SBFullHashResult full_hash_result;
1830     full_hash_result.list_id = safe_browsing_util::MALWARE;
1831     full_hash_result.hash = kFullHash1_2;
1832
1833     std::vector<SBPrefix> prefixes(1, kPrefix1);
1834     std::vector<SBFullHashResult> cache_results(1, full_hash_result);
1835
1836     database_->CacheHashResults(prefixes, cache_results, kCacheLifetime);
1837   }
1838
1839   {
1840     // kFullHash1_3 should still return false, because the cached
1841     // result for kPrefix1 doesn't contain kFullHash1_3.
1842     std::vector<SBFullHash> full_hashes(1, kFullHash1_3);
1843     std::vector<SBPrefix> prefix_hits;
1844     std::vector<SBFullHashResult> cache_hits;
1845     EXPECT_FALSE(database_->ContainsBrowseUrlHashes(
1846         full_hashes, &prefix_hits, &cache_hits));
1847
1848     // kFullHash1_1 is also not in the cached result, which takes
1849     // priority over the database.
1850     prefix_hits.clear();
1851     full_hashes.push_back(kFullHash1_1);
1852     cache_hits.clear();
1853     EXPECT_FALSE(database_->ContainsBrowseUrlHashes(
1854         full_hashes, &prefix_hits, &cache_hits));
1855
1856     // kFullHash1_2 is in the cached result.
1857     full_hashes.push_back(kFullHash1_2);
1858     prefix_hits.clear();
1859     cache_hits.clear();
1860     EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1861         full_hashes, &prefix_hits, &cache_hits));
1862     EXPECT_TRUE(prefix_hits.empty());
1863     ASSERT_EQ(1U, cache_hits.size());
1864     EXPECT_TRUE(SBFullHashEqual(kFullHash1_2, cache_hits[0].hash));
1865   }
1866
1867   // Remove kFullHash1_1 from the database.
1868   chunks.clear();
1869   chunks.push_back(SubChunkFullHash(11, kFullHash1_1, 1));
1870
1871   ASSERT_TRUE(database_->UpdateStarted(&lists));
1872   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1873   database_->UpdateFinished(true);
1874
1875   // Cache should be cleared after updating.
1876   EXPECT_TRUE(database_->browse_gethash_cache_.empty());
1877
1878   {
1879     // Now the database doesn't contain kFullHash1_1.
1880     std::vector<SBFullHash> full_hashes(1, kFullHash1_1);
1881     std::vector<SBPrefix> prefix_hits;
1882     std::vector<SBFullHashResult> cache_hits;
1883     EXPECT_FALSE(database_->ContainsBrowseUrlHashes(
1884         full_hashes, &prefix_hits, &cache_hits));
1885
1886     // Nor kFullHash1_3.
1887     full_hashes.push_back(kFullHash1_3);
1888     prefix_hits.clear();
1889     cache_hits.clear();
1890     EXPECT_FALSE(database_->ContainsBrowseUrlHashes(
1891         full_hashes, &prefix_hits, &cache_hits));
1892
1893     // Still has kFullHash1_2.
1894     full_hashes.push_back(kFullHash1_2);
1895     prefix_hits.clear();
1896     cache_hits.clear();
1897     EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1898         full_hashes, &prefix_hits, &cache_hits));
1899     ASSERT_EQ(1U, prefix_hits.size());
1900     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1901     EXPECT_TRUE(cache_hits.empty());
1902   }
1903
1904   // Remove kFullHash1_2 from the database.
1905   chunks.clear();
1906   chunks.push_back(SubChunkFullHash(12, kFullHash1_2, 2));
1907
1908   ASSERT_TRUE(database_->UpdateStarted(&lists));
1909   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1910   database_->UpdateFinished(true);
1911
1912   // Cache should be cleared after updating.
1913   EXPECT_TRUE(database_->browse_gethash_cache_.empty());
1914
1915   {
1916     // None are present.
1917     std::vector<SBFullHash> full_hashes;
1918     std::vector<SBPrefix> prefix_hits;
1919     std::vector<SBFullHashResult> cache_hits;
1920     full_hashes.push_back(kFullHash1_1);
1921     full_hashes.push_back(kFullHash1_2);
1922     full_hashes.push_back(kFullHash1_3);
1923     EXPECT_FALSE(database_->ContainsBrowseUrlHashes(
1924         full_hashes, &prefix_hits, &cache_hits));
1925   }
1926 }
1927
1928 TEST_F(SafeBrowsingDatabaseTest, BrowseFullHashAndPrefixMatching) {
1929   const SBPrefix kPrefix1 = 1001U;
1930   const SBFullHash kFullHash1_1 =
1931       SBFullHashForPrefixAndSuffix(kPrefix1, "\x01");
1932   const SBFullHash kFullHash1_2 =
1933       SBFullHashForPrefixAndSuffix(kPrefix1, "\x02");
1934
1935   ScopedVector<SBChunkData> chunks;
1936   chunks.push_back(AddChunkFullHash(1, kFullHash1_1));
1937
1938   std::vector<SBListChunkRanges> lists;
1939   ASSERT_TRUE(database_->UpdateStarted(&lists));
1940   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1941   database_->UpdateFinished(true);
1942
1943   {
1944     // kFullHash1_2 does not match kFullHash1_1.
1945     std::vector<SBFullHash> full_hashes(1, kFullHash1_2);
1946     std::vector<SBPrefix> prefix_hits;
1947     std::vector<SBFullHashResult> cache_hits;
1948     EXPECT_FALSE(database_->ContainsBrowseUrlHashes(
1949         full_hashes, &prefix_hits, &cache_hits));
1950   }
1951
1952   // Add a prefix match.
1953   chunks.clear();
1954   chunks.push_back(AddChunkPrefix(2, kPrefix1));
1955
1956   ASSERT_TRUE(database_->UpdateStarted(&lists));
1957   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1958   database_->UpdateFinished(true);
1959
1960   {
1961     // kFullHash1_2 does match kPrefix1.
1962     std::vector<SBFullHash> full_hashes(1, kFullHash1_2);
1963     std::vector<SBPrefix> prefix_hits;
1964     std::vector<SBFullHashResult> cache_hits;
1965     EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1966         full_hashes, &prefix_hits, &cache_hits));
1967     ASSERT_EQ(1U, prefix_hits.size());
1968     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1969     EXPECT_TRUE(cache_hits.empty());
1970   }
1971
1972   // Remove the full hash.
1973   chunks.clear();
1974   chunks.push_back(SubChunkFullHash(11, kFullHash1_1, 1));
1975
1976   ASSERT_TRUE(database_->UpdateStarted(&lists));
1977   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1978   database_->UpdateFinished(true);
1979
1980   {
1981     // kFullHash1_2 still returns true due to the prefix hit.
1982     std::vector<SBFullHash> full_hashes(1, kFullHash1_2);
1983     std::vector<SBPrefix> prefix_hits;
1984     std::vector<SBFullHashResult> cache_hits;
1985     EXPECT_TRUE(database_->ContainsBrowseUrlHashes(
1986         full_hashes, &prefix_hits, &cache_hits));
1987     ASSERT_EQ(1U, prefix_hits.size());
1988     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1989     EXPECT_TRUE(cache_hits.empty());
1990   }
1991 }
1992
1993 TEST_F(SafeBrowsingDatabaseTest, MalwareIpBlacklist) {
1994   database_.reset();
1995   SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile();
1996   SafeBrowsingStoreFile* ip_blacklist_store = new SafeBrowsingStoreFile();
1997   database_.reset(new SafeBrowsingDatabaseNew(browse_store,
1998                                               NULL,
1999                                               NULL,
2000                                               NULL,
2001                                               NULL,
2002                                               NULL,
2003                                               ip_blacklist_store));
2004   database_->Init(database_filename_);
2005   std::vector<SBListChunkRanges> lists;
2006   ASSERT_TRUE(database_->UpdateStarted(&lists));
2007
2008   ScopedVector<SBChunkData> chunks;
2009
2010   // IPv4 prefix match for ::ffff:192.168.1.0/120.
2011   chunks.push_back(AddChunkHashedIpValue(1, "::ffff:192.168.1.0", 120));
2012
2013   // IPv4 exact match for ::ffff:192.1.1.1.
2014   chunks.push_back(AddChunkHashedIpValue(2, "::ffff:192.1.1.1", 128));
2015
2016   // IPv6 exact match for: fe80::31a:a0ff:fe10:786e/128.
2017   chunks.push_back(AddChunkHashedIpValue(3, "fe80::31a:a0ff:fe10:786e", 128));
2018
2019   // IPv6 prefix match for: 2620:0:1000:3103::/64.
2020   chunks.push_back(AddChunkHashedIpValue(4, "2620:0:1000:3103::", 64));
2021
2022   // IPv4 prefix match for ::ffff:192.1.122.0/119.
2023   chunks.push_back(AddChunkHashedIpValue(5, "::ffff:192.1.122.0", 119));
2024
2025   // IPv4 prefix match for ::ffff:192.1.128.0/113.
2026   chunks.push_back(AddChunkHashedIpValue(6, "::ffff:192.1.128.0", 113));
2027
2028   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks.get());
2029   database_->UpdateFinished(true);
2030
2031   EXPECT_FALSE(database_->ContainsMalwareIP("192.168.0.255"));
2032   EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.0"));
2033   EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.255"));
2034   EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.10"));
2035   EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.168.1.2"));
2036   EXPECT_FALSE(database_->ContainsMalwareIP("192.168.2.0"));
2037
2038   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.1.0"));
2039   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.1.1"));
2040   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.1.2"));
2041
2042   EXPECT_FALSE(database_->ContainsMalwareIP(
2043       "2620:0:1000:3102:ffff:ffff:ffff:ffff"));
2044   EXPECT_TRUE(database_->ContainsMalwareIP("2620:0:1000:3103::"));
2045   EXPECT_TRUE(database_->ContainsMalwareIP(
2046       "2620:0:1000:3103:ffff:ffff:ffff:ffff"));
2047   EXPECT_FALSE(database_->ContainsMalwareIP("2620:0:1000:3104::"));
2048
2049   EXPECT_FALSE(database_->ContainsMalwareIP("fe80::21a:a0ff:fe10:786d"));
2050   EXPECT_TRUE(database_->ContainsMalwareIP("fe80::31a:a0ff:fe10:786e"));
2051   EXPECT_FALSE(database_->ContainsMalwareIP("fe80::21a:a0ff:fe10:786f"));
2052
2053   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.121.255"));
2054   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.122.0"));
2055   EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.1.122.1"));
2056   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.122.255"));
2057   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.123.0"));
2058   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.123.255"));
2059   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.124.0"));
2060
2061   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.127.255"));
2062   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.128.0"));
2063   EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.1.128.1"));
2064   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.128.255"));
2065   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.255.0"));
2066   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.255.255"));
2067   EXPECT_FALSE(database_->ContainsMalwareIP("192.2.0.0"));
2068 }
2069
2070 TEST_F(SafeBrowsingDatabaseTest, ContainsBrowseURL) {
2071   std::vector<SBListChunkRanges> lists;
2072   ASSERT_TRUE(database_->UpdateStarted(&lists));
2073
2074   // Add a host-level hit.
2075   {
2076     ScopedVector<SBChunkData> chunks;
2077     chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/"));
2078     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2079   }
2080
2081   // Add a specific fullhash.
2082   static const char kWhateverMalware[] = "www.whatever.com/malware.html";
2083   {
2084     ScopedVector<SBChunkData> chunks;
2085     chunks.push_back(AddChunkFullHashValue(2, kWhateverMalware));
2086     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2087   }
2088
2089   // Add a fullhash which has a prefix collision for a known url.
2090   static const char kExampleFine[] = "www.example.com/fine.html";
2091   static const char kExampleCollision[] =
2092       "www.example.com/3123364814/malware.htm";
2093   ASSERT_EQ(SBPrefixForString(kExampleFine),
2094             SBPrefixForString(kExampleCollision));
2095   {
2096     ScopedVector<SBChunkData> chunks;
2097     chunks.push_back(AddChunkFullHashValue(3, kExampleCollision));
2098     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2099   }
2100
2101   database_->UpdateFinished(true);
2102
2103   std::vector<SBPrefix> prefix_hits;
2104   std::vector<SBFullHashResult> cache_hits;
2105
2106   // Anything will hit the host prefix.
2107   EXPECT_TRUE(database_->ContainsBrowseUrl(
2108       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
2109   ASSERT_EQ(1U, prefix_hits.size());
2110   EXPECT_EQ(SBPrefixForString("www.evil.com/"), prefix_hits[0]);
2111   EXPECT_TRUE(cache_hits.empty());
2112
2113   // Hit the specific URL prefix.
2114   EXPECT_TRUE(database_->ContainsBrowseUrl(
2115       GURL(std::string("http://") + kWhateverMalware),
2116       &prefix_hits, &cache_hits));
2117   ASSERT_EQ(1U, prefix_hits.size());
2118   EXPECT_EQ(SBPrefixForString(kWhateverMalware), prefix_hits[0]);
2119   EXPECT_TRUE(cache_hits.empty());
2120
2121   // Other URLs at that host are fine.
2122   EXPECT_FALSE(database_->ContainsBrowseUrl(
2123       GURL("http://www.whatever.com/fine.html"), &prefix_hits, &cache_hits));
2124   EXPECT_TRUE(prefix_hits.empty());
2125   EXPECT_TRUE(cache_hits.empty());
2126
2127   // Hit the specific URL full hash.
2128   EXPECT_TRUE(database_->ContainsBrowseUrl(
2129       GURL(std::string("http://") + kExampleCollision),
2130       &prefix_hits, &cache_hits));
2131   ASSERT_EQ(1U, prefix_hits.size());
2132   EXPECT_EQ(SBPrefixForString(kExampleCollision), prefix_hits[0]);
2133   EXPECT_TRUE(cache_hits.empty());
2134
2135   // This prefix collides, but no full hash match.
2136   EXPECT_FALSE(database_->ContainsBrowseUrl(
2137       GURL(std::string("http://") + kExampleFine), &prefix_hits, &cache_hits));
2138 }