src/chrome/browser/safe_browsing/safe_browsing_database_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4 //
   5 // Unit tests for the SafeBrowsing storage system.
   6
   7 #include "chrome/browser/safe_browsing/safe_browsing_database.h"
   8
   9 #include "base/files/file_util.h"
  10 #include "base/files/scoped_temp_dir.h"
  11 #include "base/logging.h"
  12 #include "base/memory/scoped_vector.h"
  13 #include "base/message_loop/message_loop.h"
  14 #include "base/sha1.h"
  15 #include "base/strings/string_number_conversions.h"
  16 #include "base/strings/string_split.h"
  17 #include "base/time/time.h"
  18 #include "chrome/browser/safe_browsing/chunk.pb.h"
  19 #include "chrome/browser/safe_browsing/safe_browsing_store_file.h"
  20 #include "content/public/test/test_browser_thread_bundle.h"
  21 #include "crypto/sha2.h"
  22 #include "net/base/net_util.h"
  23 #include "sql/connection.h"
  24 #include "sql/statement.h"
  25 #include "testing/gtest/include/gtest/gtest.h"
  26 #include "testing/platform_test.h"
  27 #include "url/gurl.h"
  28
  29 using base::Time;
  30 using base::TimeDelta;
  31
  32 namespace {
  33
  34 const TimeDelta kCacheLifetime = TimeDelta::FromMinutes(45);
  35
  36 SBPrefix SBPrefixForString(const std::string& str) {
  37   return SBFullHashForString(str).prefix;
  38 }
  39
  40 // Construct a full hash which has the given prefix, with the given
  41 // suffix data coming after the prefix.
  42 SBFullHash SBFullHashForPrefixAndSuffix(SBPrefix prefix,
  43                                         const base::StringPiece& suffix) {
  44   SBFullHash full_hash;
  45   memset(&full_hash, 0, sizeof(SBFullHash));
  46   full_hash.prefix = prefix;
  47   CHECK_LE(suffix.size() + sizeof(SBPrefix), sizeof(SBFullHash));
  48   memcpy(full_hash.full_hash + sizeof(SBPrefix), suffix.data(), suffix.size());
  49   return full_hash;
  50 }
  51
  52 std::string HashedIpPrefix(const std::string& ip_prefix, size_t prefix_size) {
  53   net::IPAddressNumber ip_number;
  54   EXPECT_TRUE(net::ParseIPLiteralToNumber(ip_prefix, &ip_number));
  55   EXPECT_EQ(net::kIPv6AddressSize, ip_number.size());
  56   const std::string hashed_ip_prefix = base::SHA1HashString(
  57       net::IPAddressToPackedString(ip_number));
  58   std::string hash(crypto::kSHA256Length, '\0');
  59   hash.replace(0, hashed_ip_prefix.size(), hashed_ip_prefix);
  60   hash[base::kSHA1Length] = static_cast<char>(prefix_size);
  61   return hash;
  62 }
  63
  64 // Helper to build a chunk.  Caller takes ownership.
  65 SBChunkData* BuildChunk(int chunk_number,
  66                         safe_browsing::ChunkData::ChunkType chunk_type,
  67                         safe_browsing::ChunkData::PrefixType prefix_type,
  68                         const void* data, size_t data_size,
  69                         const std::vector<int>& add_chunk_numbers) {
  70   scoped_ptr<safe_browsing::ChunkData> raw_data(new safe_browsing::ChunkData);
  71   raw_data->set_chunk_number(chunk_number);
  72   raw_data->set_chunk_type(chunk_type);
  73   raw_data->set_prefix_type(prefix_type);
  74   raw_data->set_hashes(data, data_size);
  75   raw_data->clear_add_numbers();
  76   for (size_t i = 0; i < add_chunk_numbers.size(); ++i) {
  77     raw_data->add_add_numbers(add_chunk_numbers[i]);
  78   }
  79
  80   return new SBChunkData(raw_data.release());
  81 }
  82
  83 // Create add chunk with a single prefix.
  84 SBChunkData* AddChunkPrefix(int chunk_number, SBPrefix prefix) {
  85   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
  86                     safe_browsing::ChunkData::PREFIX_4B,
  87                     &prefix, sizeof(prefix),
  88                     std::vector<int>());
  89 }
  90
  91 // Create add chunk with a single prefix generated from |value|.
  92 SBChunkData* AddChunkPrefixValue(int chunk_number,
  93                                  const std::string& value) {
  94   return AddChunkPrefix(chunk_number, SBPrefixForString(value));
  95 }
  96
  97 // Generate an add chunk with two prefixes.
  98 SBChunkData* AddChunkPrefix2Value(int chunk_number,
  99                                   const std::string& value1,
 100                                   const std::string& value2) {
 101   const SBPrefix prefixes[2] = {
 102     SBPrefixForString(value1),
 103     SBPrefixForString(value2),
 104   };
 105   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 106                     safe_browsing::ChunkData::PREFIX_4B,
 107                     &prefixes[0], sizeof(prefixes),
 108                     std::vector<int>());
 109 }
 110
 111 // Generate an add chunk with four prefixes.
 112 SBChunkData* AddChunkPrefix4Value(int chunk_number,
 113                                   const std::string& value1,
 114                                   const std::string& value2,
 115                                   const std::string& value3,
 116                                   const std::string& value4) {
 117   const SBPrefix prefixes[4] = {
 118     SBPrefixForString(value1),
 119     SBPrefixForString(value2),
 120     SBPrefixForString(value3),
 121     SBPrefixForString(value4),
 122   };
 123   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 124                     safe_browsing::ChunkData::PREFIX_4B,
 125                     &prefixes[0], sizeof(prefixes),
 126                     std::vector<int>());
 127 }
 128
 129 // Generate an add chunk with a full hash.
 130 SBChunkData* AddChunkFullHash(int chunk_number, SBFullHash full_hash) {
 131   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 132                     safe_browsing::ChunkData::FULL_32B,
 133                     &full_hash, sizeof(full_hash),
 134                     std::vector<int>());
 135 }
 136
 137 // Generate an add chunk with a full hash generated from |value|.
 138 SBChunkData* AddChunkFullHashValue(int chunk_number,
 139                                    const std::string& value) {
 140   return AddChunkFullHash(chunk_number, SBFullHashForString(value));
 141 }
 142
 143 // Generate an add chunk with two full hashes.
 144 SBChunkData* AddChunkFullHash2Value(int chunk_number,
 145                                    const std::string& value1,
 146                                    const std::string& value2) {
 147   const SBFullHash full_hashes[2] = {
 148     SBFullHashForString(value1),
 149     SBFullHashForString(value2),
 150   };
 151   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 152                     safe_browsing::ChunkData::FULL_32B,
 153                     &full_hashes[0], sizeof(full_hashes),
 154                     std::vector<int>());
 155 }
 156
 157 // Generate a sub chunk with a prefix generated from |value|.
 158 SBChunkData* SubChunkPrefixValue(int chunk_number,
 159                                  const std::string& value,
 160                                  int add_chunk_number) {
 161   const SBPrefix prefix = SBPrefixForString(value);
 162   return BuildChunk(chunk_number, safe_browsing::ChunkData::SUB,
 163                     safe_browsing::ChunkData::PREFIX_4B,
 164                     &prefix, sizeof(prefix),
 165                     std::vector<int>(1, add_chunk_number));
 166 }
 167
 168 // Generate a sub chunk with two prefixes.
 169 SBChunkData* SubChunkPrefix2Value(int chunk_number,
 170                                   const std::string& value1,
 171                                   int add_chunk_number1,
 172                                   const std::string& value2,
 173                                   int add_chunk_number2) {
 174   const SBPrefix prefixes[2] = {
 175     SBPrefixForString(value1),
 176     SBPrefixForString(value2),
 177   };
 178   std::vector<int> add_chunk_numbers;
 179   add_chunk_numbers.push_back(add_chunk_number1);
 180   add_chunk_numbers.push_back(add_chunk_number2);
 181   return BuildChunk(chunk_number, safe_browsing::ChunkData::SUB,
 182                     safe_browsing::ChunkData::PREFIX_4B,
 183                     &prefixes[0], sizeof(prefixes),
 184                     add_chunk_numbers);
 185 }
 186
 187 // Generate a sub chunk with a full hash.
 188 SBChunkData* SubChunkFullHash(int chunk_number,
 189                               SBFullHash full_hash,
 190                               int add_chunk_number) {
 191   return BuildChunk(chunk_number, safe_browsing::ChunkData::SUB,
 192                     safe_browsing::ChunkData::FULL_32B,
 193                     &full_hash, sizeof(full_hash),
 194                     std::vector<int>(1, add_chunk_number));
 195 }
 196
 197 // Generate a sub chunk with a full hash generated from |value|.
 198 SBChunkData* SubChunkFullHashValue(int chunk_number,
 199                                    const std::string& value,
 200                                    int add_chunk_number) {
 201   return SubChunkFullHash(chunk_number,
 202                           SBFullHashForString(value),
 203                           add_chunk_number);
 204 }
 205
 206 // Generate an add chunk with a single full hash for the ip blacklist.
 207 SBChunkData* AddChunkHashedIpValue(int chunk_number,
 208                                    const std::string& ip_str,
 209                                    size_t prefix_size) {
 210   const std::string full_hash_str = HashedIpPrefix(ip_str, prefix_size);
 211   EXPECT_EQ(sizeof(SBFullHash), full_hash_str.size());
 212   SBFullHash full_hash;
 213   std::memcpy(&(full_hash.full_hash), full_hash_str.data(), sizeof(SBFullHash));
 214   return BuildChunk(chunk_number, safe_browsing::ChunkData::ADD,
 215                     safe_browsing::ChunkData::FULL_32B,
 216                     &full_hash, sizeof(full_hash),
 217                     std::vector<int>());
 218 }
 219
 220 // Prevent DCHECK from killing tests.
 221 // TODO(shess): Pawel disputes the use of this, so the test which uses
 222 // it is DISABLED.  http://crbug.com/56448
 223 class ScopedLogMessageIgnorer {
 224  public:
 225   ScopedLogMessageIgnorer() {
 226     logging::SetLogMessageHandler(&LogMessageIgnorer);
 227   }
 228   ~ScopedLogMessageIgnorer() {
 229     // TODO(shess): Would be better to verify whether anyone else
 230     // changed it, and then restore it to the previous value.
 231     logging::SetLogMessageHandler(NULL);
 232   }
 233
 234  private:
 235   static bool LogMessageIgnorer(int severity, const char* file, int line,
 236       size_t message_start, const std::string& str) {
 237     // Intercept FATAL, strip the stack backtrace, and log it without
 238     // the crash part.
 239     if (severity == logging::LOG_FATAL) {
 240       size_t newline = str.find('\n');
 241       if (newline != std::string::npos) {
 242         const std::string msg = str.substr(0, newline + 1);
 243         fprintf(stderr, "%s", msg.c_str());
 244         fflush(stderr);
 245       }
 246       return true;
 247     }
 248
 249     return false;
 250   }
 251 };
 252
 253 }  // namespace
 254
 255 class SafeBrowsingDatabaseTest : public PlatformTest {
 256  public:
 257   void SetUp() override {
 258     PlatformTest::SetUp();
 259
 260     // Setup a database in a temporary directory.
 261     ASSERT_TRUE(temp_dir_.CreateUniqueTempDir());
 262     database_filename_ =
 263         temp_dir_.path().AppendASCII("SafeBrowsingTestDatabase");
 264
 265     ResetAndReloadFullDatabase();
 266   }
 267
 268   void TearDown() override {
 269     database_.reset();
 270
 271     PlatformTest::TearDown();
 272   }
 273
 274   // Reloads the |database_| in a new SafeBrowsingDatabaseNew object with all
 275   // stores enabled.
 276   void ResetAndReloadFullDatabase() {
 277     SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile();
 278     SafeBrowsingStoreFile* download_store = new SafeBrowsingStoreFile();
 279     SafeBrowsingStoreFile* csd_whitelist_store = new SafeBrowsingStoreFile();
 280     SafeBrowsingStoreFile* download_whitelist_store =
 281         new SafeBrowsingStoreFile();
 282     SafeBrowsingStoreFile* extension_blacklist_store =
 283         new SafeBrowsingStoreFile();
 284     SafeBrowsingStoreFile* side_effect_free_whitelist_store =
 285         new SafeBrowsingStoreFile();
 286     SafeBrowsingStoreFile* ip_blacklist_store = new SafeBrowsingStoreFile();
 287     SafeBrowsingStoreFile* unwanted_software_store =
 288         new SafeBrowsingStoreFile();
 289     database_.reset(
 290         new SafeBrowsingDatabaseNew(browse_store,
 291                                     download_store,
 292                                     csd_whitelist_store,
 293                                     download_whitelist_store,
 294                                     extension_blacklist_store,
 295                                     side_effect_free_whitelist_store,
 296                                     ip_blacklist_store,
 297                                     unwanted_software_store));
 298     database_->Init(database_filename_);
 299   }
 300
 301   void GetListsInfo(std::vector<SBListChunkRanges>* lists) {
 302     lists->clear();
 303     ASSERT_TRUE(database_->UpdateStarted(lists));
 304     database_->UpdateFinished(true);
 305   }
 306
 307   // Helper function to do an AddDel or SubDel command.
 308   void DelChunk(const std::string& list,
 309                 int chunk_id,
 310                 bool is_sub_del) {
 311     std::vector<SBChunkDelete> deletes;
 312     SBChunkDelete chunk_delete;
 313     chunk_delete.list_name = list;
 314     chunk_delete.is_sub_del = is_sub_del;
 315     chunk_delete.chunk_del.push_back(ChunkRange(chunk_id));
 316     deletes.push_back(chunk_delete);
 317     database_->DeleteChunks(deletes);
 318   }
 319
 320   void AddDelChunk(const std::string& list, int chunk_id) {
 321     DelChunk(list, chunk_id, false);
 322   }
 323
 324   void SubDelChunk(const std::string& list, int chunk_id) {
 325     DelChunk(list, chunk_id, true);
 326   }
 327
 328   // Utility function for setting up the database for the caching test.
 329   void PopulateDatabaseForCacheTest();
 330
 331   scoped_ptr<SafeBrowsingDatabaseNew> database_;
 332   base::FilePath database_filename_;
 333   base::ScopedTempDir temp_dir_;
 334 };
 335
 336 // Tests retrieving list name information.
 337 TEST_F(SafeBrowsingDatabaseTest, ListNameForBrowse) {
 338   std::vector<SBListChunkRanges> lists;
 339   ScopedVector<SBChunkData> chunks;
 340
 341   chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/malware.html"));
 342   chunks.push_back(AddChunkPrefixValue(2, "www.foo.com/malware.html"));
 343   chunks.push_back(AddChunkPrefixValue(3, "www.whatever.com/malware.html"));
 344
 345   ASSERT_TRUE(database_->UpdateStarted(&lists));
 346   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 347   database_->UpdateFinished(true);
 348
 349   GetListsInfo(&lists);
 350   ASSERT_LE(1U, lists.size());
 351   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 352   EXPECT_EQ("1-3", lists[0].adds);
 353   EXPECT_TRUE(lists[0].subs.empty());
 354
 355   // Insert a malware sub chunk.
 356   chunks.clear();
 357   chunks.push_back(SubChunkPrefixValue(7, "www.subbed.com/noteveil1.html", 19));
 358
 359   ASSERT_TRUE(database_->UpdateStarted(&lists));
 360   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 361   database_->UpdateFinished(true);
 362
 363   GetListsInfo(&lists);
 364   ASSERT_LE(1U, lists.size());
 365   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 366   EXPECT_EQ("1-3", lists[0].adds);
 367   EXPECT_EQ("7", lists[0].subs);
 368   if (lists.size() == 2) {
 369     // Old style database won't have the second entry since it creates the lists
 370     // when it receives an update containing that list. The filter-based
 371     // database has these values hard coded.
 372     EXPECT_EQ(safe_browsing_util::kPhishingList, lists[1].name);
 373     EXPECT_TRUE(lists[1].adds.empty());
 374     EXPECT_TRUE(lists[1].subs.empty());
 375   }
 376
 377   // Add phishing chunks.
 378   chunks.clear();
 379   chunks.push_back(AddChunkPrefixValue(47, "www.evil.com/phishing.html"));
 380   chunks.push_back(
 381       SubChunkPrefixValue(200, "www.phishy.com/notevil1.html", 1999));
 382   chunks.push_back(
 383       SubChunkPrefixValue(201, "www.phishy2.com/notevil1.html", 1999));
 384
 385   ASSERT_TRUE(database_->UpdateStarted(&lists));
 386   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks.get());
 387   database_->UpdateFinished(true);
 388
 389   GetListsInfo(&lists);
 390   ASSERT_LE(2U, lists.size());
 391   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 392   EXPECT_EQ("1-3", lists[0].adds);
 393   EXPECT_EQ("7", lists[0].subs);
 394   EXPECT_EQ(safe_browsing_util::kPhishingList, lists[1].name);
 395   EXPECT_EQ("47", lists[1].adds);
 396   EXPECT_EQ("200-201", lists[1].subs);
 397 }
 398
 399 TEST_F(SafeBrowsingDatabaseTest, ListNameForBrowseAndDownload) {
 400   ScopedVector<SBChunkData> chunks;
 401
 402   std::vector<SBListChunkRanges> lists;
 403   ASSERT_TRUE(database_->UpdateStarted(&lists));
 404
 405   // Insert malware, phish, binurl and bindownload add chunks.
 406   chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/malware.html"));
 407   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 408
 409   chunks.clear();
 410   chunks.push_back(AddChunkPrefixValue(2, "www.foo.com/malware.html"));
 411   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks.get());
 412
 413   chunks.clear();
 414   chunks.push_back(AddChunkPrefixValue(3, "www.whatever.com/download.html"));
 415   database_->InsertChunks(safe_browsing_util::kBinUrlList, chunks.get());
 416
 417   chunks.clear();
 418   chunks.push_back(AddChunkFullHashValue(5, "www.forwhitelist.com/a.html"));
 419   database_->InsertChunks(safe_browsing_util::kCsdWhiteList, chunks.get());
 420
 421   chunks.clear();
 422   chunks.push_back(AddChunkFullHashValue(6, "www.download.com/"));
 423   database_->InsertChunks(safe_browsing_util::kDownloadWhiteList, chunks.get());
 424
 425   chunks.clear();
 426   chunks.push_back(AddChunkFullHashValue(8,
 427                                          "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
 428                                          "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"));
 429   database_->InsertChunks(safe_browsing_util::kExtensionBlacklist,
 430                           chunks.get());
 431
 432   chunks.clear();
 433   chunks.push_back(AddChunkHashedIpValue(9, "::ffff:192.168.1.0", 120));
 434   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks.get());
 435
 436   chunks.clear();
 437   chunks.push_back(AddChunkPrefixValue(10, "www.unwanted.com/software.html"));
 438   database_->InsertChunks(safe_browsing_util::kUnwantedUrlList, chunks.get());
 439
 440   database_->UpdateFinished(true);
 441
 442   GetListsInfo(&lists);
 443   ASSERT_EQ(9U, lists.size());
 444   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 445   EXPECT_EQ("1", lists[0].adds);
 446   EXPECT_TRUE(lists[0].subs.empty());
 447   EXPECT_EQ(safe_browsing_util::kPhishingList, lists[1].name);
 448   EXPECT_EQ("2", lists[1].adds);
 449   EXPECT_TRUE(lists[1].subs.empty());
 450   EXPECT_EQ(safe_browsing_util::kBinUrlList, lists[2].name);
 451   EXPECT_EQ("3", lists[2].adds);
 452   EXPECT_TRUE(lists[2].subs.empty());
 453   EXPECT_EQ(safe_browsing_util::kCsdWhiteList, lists[3].name);
 454   EXPECT_EQ("5", lists[3].adds);
 455   EXPECT_TRUE(lists[3].subs.empty());
 456   EXPECT_EQ(safe_browsing_util::kDownloadWhiteList, lists[4].name);
 457   EXPECT_EQ("6", lists[4].adds);
 458   EXPECT_TRUE(lists[4].subs.empty());
 459   EXPECT_EQ(safe_browsing_util::kExtensionBlacklist, lists[5].name);
 460   EXPECT_EQ("8", lists[5].adds);
 461   EXPECT_TRUE(lists[5].subs.empty());
 462   EXPECT_EQ(safe_browsing_util::kIPBlacklist, lists[7].name);
 463   EXPECT_EQ("9", lists[7].adds);
 464   EXPECT_TRUE(lists[7].subs.empty());
 465   EXPECT_EQ(safe_browsing_util::kUnwantedUrlList, lists[8].name);
 466   EXPECT_EQ("10", lists[8].adds);
 467   EXPECT_TRUE(lists[8].subs.empty());
 468
 469   database_.reset();
 470 }
 471
 472 // Checks database reading and writing for browse and unwanted PrefixSets.
 473 TEST_F(SafeBrowsingDatabaseTest, BrowseAndUnwantedDatabasesAndPrefixSets) {
 474   struct TestCase {
 475     using TestListContainsBadUrl = bool (SafeBrowsingDatabase::*)(
 476         const GURL& url,
 477         std::vector<SBPrefix>* prefix_hits,
 478         std::vector<SBFullHashResult>* cache_hits);
 479
 480     const char* test_list_name;
 481     size_t expected_list_index;
 482     TestListContainsBadUrl test_list_contains_bad_url;
 483   } const kTestCases[] {
 484     { safe_browsing_util::kMalwareList, 0U,
 485       &SafeBrowsingDatabase::ContainsBrowseUrl },
 486     { safe_browsing_util::kPhishingList, 1U,
 487       &SafeBrowsingDatabase::ContainsBrowseUrl },
 488     { safe_browsing_util::kUnwantedUrlList, 8U,
 489       &SafeBrowsingDatabase::ContainsUnwantedSoftwareUrl },
 490   };
 491
 492   for (const auto& test_case : kTestCases) {
 493     SCOPED_TRACE(std::string("Tested list at fault => ") +
 494                  test_case.test_list_name);
 495
 496     std::vector<SBListChunkRanges> lists;
 497     ScopedVector<SBChunkData> chunks;
 498
 499     chunks.push_back(AddChunkPrefix2Value(1,
 500                                           "www.evil.com/phishing.html",
 501                                           "www.evil.com/malware.html"));
 502     chunks.push_back(AddChunkPrefix4Value(2,
 503                                           "www.evil.com/notevil1.html",
 504                                           "www.evil.com/notevil2.html",
 505                                           "www.good.com/good1.html",
 506                                           "www.good.com/good2.html"));
 507     chunks.push_back(AddChunkPrefixValue(3, "192.168.0.1/malware.html"));
 508     chunks.push_back(AddChunkFullHashValue(7, "www.evil.com/evil.html"));
 509
 510     ASSERT_TRUE(database_->UpdateStarted(&lists));
 511     database_->InsertChunks(test_case.test_list_name, chunks.get());
 512     database_->UpdateFinished(true);
 513
 514     // Make sure they were added correctly.
 515     GetListsInfo(&lists);
 516
 517     ASSERT_LE(1U, lists.size());
 518     EXPECT_EQ(test_case.test_list_name,
 519               lists[test_case.expected_list_index].name);
 520     EXPECT_EQ("1-3,7", lists[test_case.expected_list_index].adds);
 521     EXPECT_TRUE(lists[test_case.expected_list_index].subs.empty());
 522
 523     std::vector<SBPrefix> prefix_hits;
 524     std::vector<SBFullHashResult> cache_hits;
 525     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 526         GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 527     ASSERT_EQ(1U, prefix_hits.size());
 528     EXPECT_EQ(SBPrefixForString("www.evil.com/phishing.html"), prefix_hits[0]);
 529     EXPECT_TRUE(cache_hits.empty());
 530
 531     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 532         GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 533
 534     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 535         GURL("http://www.evil.com/notevil1.html"), &prefix_hits, &cache_hits));
 536
 537     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 538         GURL("http://www.evil.com/notevil2.html"), &prefix_hits, &cache_hits));
 539
 540     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 541         GURL("http://www.good.com/good1.html"), &prefix_hits, &cache_hits));
 542
 543     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 544         GURL("http://www.good.com/good2.html"), &prefix_hits, &cache_hits));
 545
 546     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 547         GURL("http://192.168.0.1/malware.html"), &prefix_hits, &cache_hits));
 548
 549     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 550         GURL("http://www.evil.com/"), &prefix_hits, &cache_hits));
 551     EXPECT_TRUE(prefix_hits.empty());
 552     EXPECT_TRUE(cache_hits.empty());
 553
 554     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 555         GURL("http://www.evil.com/robots.txt"), &prefix_hits, &cache_hits));
 556
 557     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 558         GURL("http://www.evil.com/evil.html"), &prefix_hits, &cache_hits));
 559     ASSERT_EQ(1U, prefix_hits.size());
 560     EXPECT_EQ(SBPrefixForString("www.evil.com/evil.html"), prefix_hits[0]);
 561
 562     // Attempt to re-add the first chunk (should be a no-op).
 563     // see bug: http://code.google.com/p/chromium/issues/detail?id=4522
 564     chunks.clear();
 565     chunks.push_back(AddChunkPrefix2Value(1,
 566                                           "www.evil.com/phishing.html",
 567                                           "www.evil.com/malware.html"));
 568     ASSERT_TRUE(database_->UpdateStarted(&lists));
 569     database_->InsertChunks(test_case.test_list_name, chunks.get());
 570     database_->UpdateFinished(true);
 571
 572     GetListsInfo(&lists);
 573     ASSERT_LE(1U, lists.size());
 574     EXPECT_EQ(test_case.test_list_name,
 575               lists[test_case.expected_list_index].name);
 576     EXPECT_EQ("1-3,7", lists[test_case.expected_list_index].adds);
 577     EXPECT_TRUE(lists[test_case.expected_list_index].subs.empty());
 578
 579     // Test removing a single prefix from the add chunk.
 580     chunks.clear();
 581     chunks.push_back(SubChunkPrefixValue(4, "www.evil.com/notevil1.html", 2));
 582     ASSERT_TRUE(database_->UpdateStarted(&lists));
 583     database_->InsertChunks(test_case.test_list_name, chunks.get());
 584     database_->UpdateFinished(true);
 585
 586     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 587         GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 588     ASSERT_EQ(1U, prefix_hits.size());
 589     EXPECT_EQ(SBPrefixForString("www.evil.com/phishing.html"), prefix_hits[0]);
 590     EXPECT_TRUE(cache_hits.empty());
 591
 592     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 593         GURL("http://www.evil.com/notevil1.html"), &prefix_hits, &cache_hits));
 594     EXPECT_TRUE(prefix_hits.empty());
 595     EXPECT_TRUE(cache_hits.empty());
 596
 597     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 598         GURL("http://www.evil.com/notevil2.html"), &prefix_hits, &cache_hits));
 599
 600     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 601         GURL("http://www.good.com/good1.html"), &prefix_hits, &cache_hits));
 602
 603     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 604         GURL("http://www.good.com/good2.html"), &prefix_hits, &cache_hits));
 605
 606     GetListsInfo(&lists);
 607     ASSERT_LE(1U, lists.size());
 608     EXPECT_EQ(test_case.test_list_name,
 609               lists[test_case.expected_list_index].name);
 610     EXPECT_EQ("1-3,7", lists[test_case.expected_list_index].adds);
 611     EXPECT_EQ("4", lists[test_case.expected_list_index].subs);
 612
 613     // Test the same sub chunk again.  This should be a no-op.
 614     // see bug: http://code.google.com/p/chromium/issues/detail?id=4522
 615     chunks.clear();
 616     chunks.push_back(SubChunkPrefixValue(4, "www.evil.com/notevil1.html", 2));
 617
 618     ASSERT_TRUE(database_->UpdateStarted(&lists));
 619     database_->InsertChunks(test_case.test_list_name, chunks.get());
 620     database_->UpdateFinished(true);
 621
 622     GetListsInfo(&lists);
 623     ASSERT_LE(1U, lists.size());
 624     EXPECT_EQ(test_case.test_list_name,
 625               lists[test_case.expected_list_index].name);
 626     EXPECT_EQ("1-3,7", lists[test_case.expected_list_index].adds);
 627     EXPECT_EQ("4", lists[test_case.expected_list_index].subs);
 628
 629     // Test removing all the prefixes from an add chunk.
 630     ASSERT_TRUE(database_->UpdateStarted(&lists));
 631     AddDelChunk(test_case.test_list_name, 2);
 632     database_->UpdateFinished(true);
 633
 634     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 635         GURL("http://www.evil.com/notevil2.html"), &prefix_hits, &cache_hits));
 636
 637     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 638         GURL("http://www.good.com/good1.html"), &prefix_hits, &cache_hits));
 639
 640     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 641         GURL("http://www.good.com/good2.html"), &prefix_hits, &cache_hits));
 642
 643     GetListsInfo(&lists);
 644     ASSERT_LE(1U, lists.size());
 645     EXPECT_EQ(test_case.test_list_name,
 646               lists[test_case.expected_list_index].name);
 647     EXPECT_EQ("1,3,7", lists[test_case.expected_list_index].adds);
 648     EXPECT_EQ("4", lists[test_case.expected_list_index].subs);
 649
 650     // The adddel command exposed a bug in the transaction code where any
 651     // transaction after it would fail.  Add a dummy entry and remove it to
 652     // make sure the transcation works fine.
 653     chunks.clear();
 654     chunks.push_back(AddChunkPrefixValue(44, "www.redherring.com/index.html"));
 655     ASSERT_TRUE(database_->UpdateStarted(&lists));
 656     database_->InsertChunks(test_case.test_list_name, chunks.get());
 657
 658     // Now remove the dummy entry.  If there are any problems with the
 659     // transactions, asserts will fire.
 660     AddDelChunk(test_case.test_list_name, 44);
 661
 662     // Test the subdel command.
 663     SubDelChunk(test_case.test_list_name, 4);
 664     database_->UpdateFinished(true);
 665
 666     GetListsInfo(&lists);
 667     ASSERT_LE(1U, lists.size());
 668     EXPECT_EQ(test_case.test_list_name,
 669               lists[test_case.expected_list_index].name);
 670     EXPECT_EQ("1,3,7", lists[test_case.expected_list_index].adds);
 671     EXPECT_TRUE(lists[test_case.expected_list_index].subs.empty());
 672
 673     // Test a sub command coming in before the add.
 674     chunks.clear();
 675     chunks.push_back(SubChunkPrefix2Value(5,
 676                                           "www.notevilanymore.com/index.html",
 677                                           10,
 678                                           "www.notevilanymore.com/good.html",
 679                                           10));
 680     ASSERT_TRUE(database_->UpdateStarted(&lists));
 681     database_->InsertChunks(test_case.test_list_name, chunks.get());
 682     database_->UpdateFinished(true);
 683
 684     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 685         GURL("http://www.notevilanymore.com/index.html"),
 686         &prefix_hits,
 687         &cache_hits));
 688
 689     // Now insert the tardy add chunk and we don't expect them to appear
 690     // in database because of the previous sub chunk.
 691     chunks.clear();
 692     chunks.push_back(AddChunkPrefix2Value(10,
 693                                           "www.notevilanymore.com/index.html",
 694                                           "www.notevilanymore.com/good.html"));
 695     ASSERT_TRUE(database_->UpdateStarted(&lists));
 696     database_->InsertChunks(test_case.test_list_name, chunks.get());
 697     database_->UpdateFinished(true);
 698
 699     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 700         GURL("http://www.notevilanymore.com/index.html"),
 701         &prefix_hits,
 702         &cache_hits));
 703
 704     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 705         GURL("http://www.notevilanymore.com/good.html"),
 706         &prefix_hits,
 707         &cache_hits));
 708
 709     // Reset and reload the database.  The database will rely on the prefix set.
 710     ResetAndReloadFullDatabase();
 711
 712     // Check that a prefix still hits.
 713     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 714         GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 715     ASSERT_EQ(1U, prefix_hits.size());
 716     EXPECT_EQ(SBPrefixForString("www.evil.com/phishing.html"), prefix_hits[0]);
 717
 718     // Also check that it's not just always returning true in this case.
 719     EXPECT_FALSE((database_.get()->*test_case.test_list_contains_bad_url)(
 720         GURL("http://www.evil.com/"), &prefix_hits, &cache_hits));
 721
 722     // Check that the full hash is still present.
 723     EXPECT_TRUE((database_.get()->*test_case.test_list_contains_bad_url)(
 724         GURL("http://www.evil.com/evil.html"), &prefix_hits, &cache_hits));
 725     ASSERT_EQ(1U, prefix_hits.size());
 726     EXPECT_EQ(SBPrefixForString("www.evil.com/evil.html"), prefix_hits[0]);
 727   }
 728 }
 729
 730 // Test adding zero length chunks to the database.
 731 TEST_F(SafeBrowsingDatabaseTest, ZeroSizeChunk) {
 732   std::vector<SBListChunkRanges> lists;
 733   ScopedVector<SBChunkData> chunks;
 734
 735   // Populate with a couple of normal chunks.
 736   chunks.push_back(AddChunkPrefix2Value(1,
 737                                         "www.test.com/test1.html",
 738                                         "www.test.com/test2.html"));
 739   chunks.push_back(AddChunkPrefix2Value(10,
 740                                         "www.random.com/random1.html",
 741                                         "www.random.com/random2.html"));
 742
 743   ASSERT_TRUE(database_->UpdateStarted(&lists));
 744   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 745   database_->UpdateFinished(true);
 746
 747   // Add an empty ADD and SUB chunk.
 748   GetListsInfo(&lists);
 749   ASSERT_LE(1U, lists.size());
 750   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 751   EXPECT_EQ("1,10", lists[0].adds);
 752   EXPECT_TRUE(lists[0].subs.empty());
 753
 754   chunks.clear();
 755   chunks.push_back(BuildChunk(19, safe_browsing::ChunkData::ADD,
 756                               safe_browsing::ChunkData::PREFIX_4B,
 757                               NULL, 0, std::vector<int>()));
 758   chunks.push_back(BuildChunk(7, safe_browsing::ChunkData::SUB,
 759                               safe_browsing::ChunkData::PREFIX_4B,
 760                               NULL, 0, std::vector<int>()));
 761
 762   ASSERT_TRUE(database_->UpdateStarted(&lists));
 763   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 764   database_->UpdateFinished(true);
 765
 766   GetListsInfo(&lists);
 767   ASSERT_LE(1U, lists.size());
 768   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 769   EXPECT_EQ("1,10,19", lists[0].adds);
 770   EXPECT_EQ("7", lists[0].subs);
 771
 772   // Add an empty chunk along with a couple that contain data. This should
 773   // result in the chunk range being reduced in size.
 774   chunks.clear();
 775   chunks.push_back(AddChunkPrefixValue(20, "www.notempty.com/full1.html"));
 776   chunks.push_back(BuildChunk(21, safe_browsing::ChunkData::ADD,
 777                               safe_browsing::ChunkData::PREFIX_4B,
 778                               NULL, 0, std::vector<int>()));
 779   chunks.push_back(AddChunkPrefixValue(22, "www.notempty.com/full2.html"));
 780
 781   ASSERT_TRUE(database_->UpdateStarted(&lists));
 782   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 783   database_->UpdateFinished(true);
 784
 785   std::vector<SBPrefix> prefix_hits;
 786   std::vector<SBFullHashResult> cache_hits;
 787   EXPECT_TRUE(database_->ContainsBrowseUrl(
 788       GURL("http://www.notempty.com/full1.html"), &prefix_hits, &cache_hits));
 789   EXPECT_TRUE(database_->ContainsBrowseUrl(
 790       GURL("http://www.notempty.com/full2.html"), &prefix_hits, &cache_hits));
 791
 792   GetListsInfo(&lists);
 793   ASSERT_LE(1U, lists.size());
 794   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 795   EXPECT_EQ("1,10,19-22", lists[0].adds);
 796   EXPECT_EQ("7", lists[0].subs);
 797
 798   // Handle AddDel and SubDel commands for empty chunks.
 799   ASSERT_TRUE(database_->UpdateStarted(&lists));
 800   AddDelChunk(safe_browsing_util::kMalwareList, 21);
 801   database_->UpdateFinished(true);
 802
 803   GetListsInfo(&lists);
 804   ASSERT_LE(1U, lists.size());
 805   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 806   EXPECT_EQ("1,10,19-20,22", lists[0].adds);
 807   EXPECT_EQ("7", lists[0].subs);
 808
 809   ASSERT_TRUE(database_->UpdateStarted(&lists));
 810   SubDelChunk(safe_browsing_util::kMalwareList, 7);
 811   database_->UpdateFinished(true);
 812
 813   GetListsInfo(&lists);
 814   ASSERT_LE(1U, lists.size());
 815   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
 816   EXPECT_EQ("1,10,19-20,22", lists[0].adds);
 817   EXPECT_TRUE(lists[0].subs.empty());
 818 }
 819
 820 // Utility function for setting up the database for the caching test.
 821 void SafeBrowsingDatabaseTest::PopulateDatabaseForCacheTest() {
 822   // Add a couple prefixes.
 823   ScopedVector<SBChunkData> chunks;
 824   chunks.push_back(AddChunkPrefix2Value(1,
 825                                         "www.evil.com/phishing.html",
 826                                         "www.evil.com/malware.html"));
 827
 828   std::vector<SBListChunkRanges> lists;
 829   ASSERT_TRUE(database_->UpdateStarted(&lists));
 830   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 831   database_->UpdateFinished(true);
 832
 833   // Cache should be cleared after updating.
 834   EXPECT_TRUE(database_->prefix_gethash_cache_.empty());
 835
 836   SBFullHashResult full_hash;
 837   full_hash.list_id = safe_browsing_util::MALWARE;
 838
 839   std::vector<SBFullHashResult> results;
 840   std::vector<SBPrefix> prefixes;
 841
 842   // Add a fullhash result for each prefix.
 843   full_hash.hash = SBFullHashForString("www.evil.com/phishing.html");
 844   results.push_back(full_hash);
 845   prefixes.push_back(full_hash.hash.prefix);
 846
 847   full_hash.hash = SBFullHashForString("www.evil.com/malware.html");
 848   results.push_back(full_hash);
 849   prefixes.push_back(full_hash.hash.prefix);
 850
 851   database_->CacheHashResults(prefixes, results, kCacheLifetime);
 852 }
 853
 854 TEST_F(SafeBrowsingDatabaseTest, HashCaching) {
 855   PopulateDatabaseForCacheTest();
 856
 857   // We should have both full hashes in the cache.
 858   EXPECT_EQ(2U, database_->prefix_gethash_cache_.size());
 859
 860   // Test the cache lookup for the first prefix.
 861   std::vector<SBPrefix> prefix_hits;
 862   std::vector<SBFullHashResult> cache_hits;
 863   EXPECT_TRUE(database_->ContainsBrowseUrl(
 864       GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 865   EXPECT_TRUE(prefix_hits.empty());
 866   ASSERT_EQ(1U, cache_hits.size());
 867   EXPECT_TRUE(SBFullHashEqual(
 868       cache_hits[0].hash, SBFullHashForString("www.evil.com/phishing.html")));
 869
 870   prefix_hits.clear();
 871   cache_hits.clear();
 872
 873   // Test the cache lookup for the second prefix.
 874   EXPECT_TRUE(database_->ContainsBrowseUrl(
 875       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 876   EXPECT_TRUE(prefix_hits.empty());
 877   ASSERT_EQ(1U, cache_hits.size());
 878   EXPECT_TRUE(SBFullHashEqual(
 879       cache_hits[0].hash, SBFullHashForString("www.evil.com/malware.html")));
 880
 881   prefix_hits.clear();
 882   cache_hits.clear();
 883
 884   // Test removing a prefix via a sub chunk.
 885   ScopedVector<SBChunkData> chunks;
 886   chunks.push_back(SubChunkPrefixValue(2, "www.evil.com/phishing.html", 1));
 887
 888   std::vector<SBListChunkRanges> lists;
 889   ASSERT_TRUE(database_->UpdateStarted(&lists));
 890   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
 891   database_->UpdateFinished(true);
 892
 893   // This prefix should still be there, but cached fullhash should be gone.
 894   EXPECT_TRUE(database_->ContainsBrowseUrl(
 895       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 896   ASSERT_EQ(1U, prefix_hits.size());
 897   EXPECT_EQ(SBPrefixForString("www.evil.com/malware.html"), prefix_hits[0]);
 898   EXPECT_TRUE(cache_hits.empty());
 899   prefix_hits.clear();
 900   cache_hits.clear();
 901
 902   // This prefix should be gone.
 903   EXPECT_FALSE(database_->ContainsBrowseUrl(
 904       GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 905   prefix_hits.clear();
 906   cache_hits.clear();
 907
 908   // Test that an AddDel for the original chunk removes the last cached entry.
 909   ASSERT_TRUE(database_->UpdateStarted(&lists));
 910   AddDelChunk(safe_browsing_util::kMalwareList, 1);
 911   database_->UpdateFinished(true);
 912   EXPECT_FALSE(database_->ContainsBrowseUrl(
 913       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 914   EXPECT_TRUE(database_->prefix_gethash_cache_.empty());
 915   prefix_hits.clear();
 916   cache_hits.clear();
 917
 918   // Test that the cache won't return expired values. First we have to adjust
 919   // the cached entries' received time to make them older, since the database
 920   // cache insert uses Time::Now(). First, store some entries.
 921   PopulateDatabaseForCacheTest();
 922
 923   std::map<SBPrefix, SBCachedFullHashResult>* hash_cache =
 924       &database_->prefix_gethash_cache_;
 925   EXPECT_EQ(2U, hash_cache->size());
 926
 927   // Now adjust one of the entries times to be in the past.
 928   const SBPrefix key = SBPrefixForString("www.evil.com/malware.html");
 929   std::map<SBPrefix, SBCachedFullHashResult>::iterator iter =
 930       hash_cache->find(key);
 931   ASSERT_TRUE(iter != hash_cache->end());
 932   iter->second.expire_after = Time::Now() - TimeDelta::FromMinutes(1);
 933
 934   EXPECT_TRUE(database_->ContainsBrowseUrl(
 935       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
 936   EXPECT_EQ(1U, prefix_hits.size());
 937   EXPECT_TRUE(cache_hits.empty());
 938   // Expired entry should have been removed from cache.
 939   EXPECT_EQ(1U, hash_cache->size());
 940
 941   // This entry should still exist.
 942   EXPECT_TRUE(database_->ContainsBrowseUrl(
 943       GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 944   EXPECT_TRUE(prefix_hits.empty());
 945   EXPECT_EQ(1U, cache_hits.size());
 946
 947   // Testing prefix miss caching. First, we clear out the existing database,
 948   // Since PopulateDatabaseForCacheTest() doesn't handle adding duplicate
 949   // chunks.
 950   ASSERT_TRUE(database_->UpdateStarted(&lists));
 951   AddDelChunk(safe_browsing_util::kMalwareList, 1);
 952   database_->UpdateFinished(true);
 953
 954   // Cache should be cleared after updating.
 955   EXPECT_TRUE(hash_cache->empty());
 956
 957   std::vector<SBPrefix> prefix_misses;
 958   std::vector<SBFullHashResult> empty_full_hash;
 959   prefix_misses.push_back(SBPrefixForString("http://www.bad.com/malware.html"));
 960   prefix_misses.push_back(
 961       SBPrefixForString("http://www.bad.com/phishing.html"));
 962   database_->CacheHashResults(prefix_misses, empty_full_hash, kCacheLifetime);
 963
 964   // Prefixes with no full results are misses.
 965   EXPECT_EQ(hash_cache->size(), prefix_misses.size());
 966   ASSERT_TRUE(
 967       hash_cache->count(SBPrefixForString("http://www.bad.com/malware.html")));
 968   EXPECT_TRUE(
 969       hash_cache->find(SBPrefixForString("http://www.bad.com/malware.html"))
 970           ->second.full_hashes.empty());
 971   ASSERT_TRUE(
 972       hash_cache->count(SBPrefixForString("http://www.bad.com/phishing.html")));
 973   EXPECT_TRUE(
 974       hash_cache->find(SBPrefixForString("http://www.bad.com/phishing.html"))
 975           ->second.full_hashes.empty());
 976
 977   // Update the database.
 978   PopulateDatabaseForCacheTest();
 979
 980   // Cache a GetHash miss for a particular prefix, and even though the prefix is
 981   // in the database, it is flagged as a miss so looking up the associated URL
 982   // will not succeed.
 983   prefix_hits.clear();
 984   cache_hits.clear();
 985   prefix_misses.clear();
 986   empty_full_hash.clear();
 987   prefix_misses.push_back(SBPrefixForString("www.evil.com/phishing.html"));
 988   database_->CacheHashResults(prefix_misses, empty_full_hash, kCacheLifetime);
 989   EXPECT_FALSE(database_->ContainsBrowseUrl(
 990       GURL("http://www.evil.com/phishing.html"), &prefix_hits, &cache_hits));
 991   prefix_hits.clear();
 992   cache_hits.clear();
 993
 994   // Test receiving a full add chunk.
 995   chunks.clear();
 996   chunks.push_back(AddChunkFullHash2Value(20,
 997                                           "www.fullevil.com/bad1.html",
 998                                           "www.fullevil.com/bad2.html"));
 999   ASSERT_TRUE(database_->UpdateStarted(&lists));
1000   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1001   database_->UpdateFinished(true);
1002
1003   EXPECT_TRUE(database_->ContainsBrowseUrl(
1004       GURL("http://www.fullevil.com/bad1.html"), &prefix_hits, &cache_hits));
1005   ASSERT_EQ(1U, prefix_hits.size());
1006   EXPECT_EQ(SBPrefixForString("www.fullevil.com/bad1.html"), prefix_hits[0]);
1007   EXPECT_TRUE(cache_hits.empty());
1008   prefix_hits.clear();
1009   cache_hits.clear();
1010
1011   EXPECT_TRUE(database_->ContainsBrowseUrl(
1012       GURL("http://www.fullevil.com/bad2.html"), &prefix_hits, &cache_hits));
1013   ASSERT_EQ(1U, prefix_hits.size());
1014   EXPECT_EQ(SBPrefixForString("www.fullevil.com/bad2.html"), prefix_hits[0]);
1015   EXPECT_TRUE(cache_hits.empty());
1016   prefix_hits.clear();
1017   cache_hits.clear();
1018
1019   // Test receiving a full sub chunk, which will remove one of the full adds.
1020   chunks.clear();
1021   chunks.push_back(SubChunkFullHashValue(200,
1022                                          "www.fullevil.com/bad1.html",
1023                                          20));
1024   ASSERT_TRUE(database_->UpdateStarted(&lists));
1025   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1026   database_->UpdateFinished(true);
1027
1028   EXPECT_FALSE(database_->ContainsBrowseUrl(
1029       GURL("http://www.fullevil.com/bad1.html"), &prefix_hits, &cache_hits));
1030
1031   // There should be one remaining full add.
1032   EXPECT_TRUE(database_->ContainsBrowseUrl(
1033       GURL("http://www.fullevil.com/bad2.html"), &prefix_hits, &cache_hits));
1034   ASSERT_EQ(1U, prefix_hits.size());
1035   EXPECT_EQ(SBPrefixForString("www.fullevil.com/bad2.html"), prefix_hits[0]);
1036   EXPECT_TRUE(cache_hits.empty());
1037   prefix_hits.clear();
1038   cache_hits.clear();
1039
1040   // Now test an AddDel for the remaining full add.
1041   ASSERT_TRUE(database_->UpdateStarted(&lists));
1042   AddDelChunk(safe_browsing_util::kMalwareList, 20);
1043   database_->UpdateFinished(true);
1044
1045   EXPECT_FALSE(database_->ContainsBrowseUrl(
1046       GURL("http://www.fullevil.com/bad1.html"), &prefix_hits, &cache_hits));
1047   EXPECT_FALSE(database_->ContainsBrowseUrl(
1048       GURL("http://www.fullevil.com/bad2.html"), &prefix_hits, &cache_hits));
1049
1050   // Add a fullhash which has a prefix collision for a known url.
1051   static const char kExampleFine[] = "www.example.com/fine.html";
1052   static const char kExampleCollision[] =
1053       "www.example.com/3123364814/malware.htm";
1054   ASSERT_EQ(SBPrefixForString(kExampleFine),
1055             SBPrefixForString(kExampleCollision));
1056   ASSERT_TRUE(database_->UpdateStarted(&lists));
1057   {
1058     ScopedVector<SBChunkData> chunks;
1059     chunks.push_back(AddChunkPrefixValue(21, kExampleCollision));
1060     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1061   }
1062   database_->UpdateFinished(true);
1063
1064   // Expect a prefix hit due to the collision between |kExampleFine| and
1065   // |kExampleCollision|.
1066   EXPECT_TRUE(database_->ContainsBrowseUrl(
1067       GURL(std::string("http://") + kExampleFine), &prefix_hits, &cache_hits));
1068   ASSERT_EQ(1U, prefix_hits.size());
1069   EXPECT_EQ(SBPrefixForString(kExampleFine), prefix_hits[0]);
1070   EXPECT_TRUE(cache_hits.empty());
1071
1072   // Cache gethash response for |kExampleCollision|.
1073   {
1074     SBFullHashResult result;
1075     result.hash = SBFullHashForString(kExampleCollision);
1076     result.list_id = safe_browsing_util::MALWARE;
1077     database_->CacheHashResults(std::vector<SBPrefix>(1, result.hash.prefix),
1078                                 std::vector<SBFullHashResult>(1, result),
1079                                 kCacheLifetime);
1080   }
1081
1082   // The cached response means the collision no longer causes a hit.
1083   EXPECT_FALSE(database_->ContainsBrowseUrl(
1084       GURL(std::string("http://") + kExampleFine), &prefix_hits, &cache_hits));
1085 }
1086
1087 // Test that corrupt databases are appropriately handled, even if the
1088 // corruption is detected in the midst of the update.
1089 // TODO(shess): Disabled until ScopedLogMessageIgnorer resolved.
1090 // http://crbug.com/56448
1091 TEST_F(SafeBrowsingDatabaseTest, DISABLED_FileCorruptionHandling) {
1092   // Re-create the database in a captive message loop so that we can
1093   // influence task-posting.  Database specifically needs to the
1094   // file-backed.
1095   database_.reset();
1096   base::MessageLoop loop;
1097   SafeBrowsingStoreFile* store = new SafeBrowsingStoreFile();
1098   database_.reset(new SafeBrowsingDatabaseNew(store, NULL, NULL, NULL, NULL,
1099                                               NULL, NULL, NULL));
1100   database_->Init(database_filename_);
1101
1102   // This will cause an empty database to be created.
1103   std::vector<SBListChunkRanges> lists;
1104   ASSERT_TRUE(database_->UpdateStarted(&lists));
1105   database_->UpdateFinished(true);
1106
1107   // Create a sub chunk to insert.
1108   ScopedVector<SBChunkData> chunks;
1109   chunks.push_back(SubChunkPrefixValue(7,
1110                                        "www.subbed.com/notevil1.html",
1111                                        19));
1112
1113   // Corrupt the file by corrupting the checksum, which is not checked
1114   // until the entire table is read in |UpdateFinished()|.
1115   FILE* fp = base::OpenFile(database_filename_, "r+");
1116   ASSERT_TRUE(fp);
1117   ASSERT_NE(-1, fseek(fp, -8, SEEK_END));
1118   for (size_t i = 0; i < 8; ++i) {
1119     fputc('!', fp);
1120   }
1121   fclose(fp);
1122
1123   {
1124     // The following code will cause DCHECKs, so suppress the crashes.
1125     ScopedLogMessageIgnorer ignorer;
1126
1127     // Start an update.  The insert will fail due to corruption.
1128     ASSERT_TRUE(database_->UpdateStarted(&lists));
1129     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1130     database_->UpdateFinished(true);
1131
1132     // Database file still exists until the corruption handler has run.
1133     EXPECT_TRUE(base::PathExists(database_filename_));
1134
1135     // Flush through the corruption-handler task.
1136     DVLOG(1) << "Expect failed check on: SafeBrowsing database reset";
1137     base::MessageLoop::current()->RunUntilIdle();
1138   }
1139
1140   // Database file should not exist.
1141   EXPECT_FALSE(base::PathExists(database_filename_));
1142
1143   // Run the update again successfully.
1144   ASSERT_TRUE(database_->UpdateStarted(&lists));
1145   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1146   database_->UpdateFinished(true);
1147   EXPECT_TRUE(base::PathExists(database_filename_));
1148
1149   database_.reset();
1150 }
1151
1152 // Checks database reading and writing.
1153 TEST_F(SafeBrowsingDatabaseTest, ContainsDownloadUrl) {
1154   const char kEvil1Url1[] = "www.evil1.com/download1/";
1155   const char kEvil1Url2[] = "www.evil1.com/download2.html";
1156
1157   // Add a simple chunk with one hostkey for download url list.
1158   ScopedVector<SBChunkData> chunks;
1159   chunks.push_back(AddChunkPrefix2Value(1, kEvil1Url1, kEvil1Url2));
1160
1161   std::vector<SBListChunkRanges> lists;
1162   ASSERT_TRUE(database_->UpdateStarted(&lists));
1163   database_->InsertChunks(safe_browsing_util::kBinUrlList, chunks.get());
1164   database_->UpdateFinished(true);
1165
1166   std::vector<SBPrefix> prefix_hits;
1167   std::vector<GURL> urls(1);
1168
1169   urls[0] = GURL(std::string("http://") + kEvil1Url1);
1170   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1171   ASSERT_EQ(1U, prefix_hits.size());
1172   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1173
1174   urls[0] = GURL(std::string("http://") + kEvil1Url2);
1175   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1176   ASSERT_EQ(1U, prefix_hits.size());
1177   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[0]);
1178
1179   urls[0] = GURL(std::string("https://") + kEvil1Url2);
1180   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1181   ASSERT_EQ(1U, prefix_hits.size());
1182   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[0]);
1183
1184   urls[0] = GURL(std::string("ftp://") + kEvil1Url2);
1185   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1186   ASSERT_EQ(1U, prefix_hits.size());
1187   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[0]);
1188
1189   urls[0] = GURL("http://www.randomevil.com");
1190   EXPECT_FALSE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1191
1192   // Should match with query args stripped.
1193   urls[0] = GURL(std::string("http://") + kEvil1Url2 + "?blah");
1194   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1195   ASSERT_EQ(1U, prefix_hits.size());
1196   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[0]);
1197
1198   // Should match with extra path stuff and query args stripped.
1199   urls[0] = GURL(std::string("http://") + kEvil1Url1 + "foo/bar?blah");
1200   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1201   ASSERT_EQ(1U, prefix_hits.size());
1202   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1203
1204   // First hit in redirect chain is malware.
1205   urls.clear();
1206   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1207   urls.push_back(GURL("http://www.randomevil.com"));
1208   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1209   ASSERT_EQ(1U, prefix_hits.size());
1210   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1211
1212   // Middle hit in redirect chain is malware.
1213   urls.clear();
1214   urls.push_back(GURL("http://www.randomevil.com"));
1215   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1216   urls.push_back(GURL("http://www.randomevil2.com"));
1217   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1218   ASSERT_EQ(1U, prefix_hits.size());
1219   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1220
1221   // Final hit in redirect chain is malware.
1222   urls.clear();
1223   urls.push_back(GURL("http://www.randomevil.com"));
1224   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1225   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1226   ASSERT_EQ(1U, prefix_hits.size());
1227   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1228
1229   // Multiple hits in redirect chain are in malware list.
1230   urls.clear();
1231   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
1232   urls.push_back(GURL(std::string("https://") + kEvil1Url2));
1233   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
1234   ASSERT_EQ(2U, prefix_hits.size());
1235   EXPECT_EQ(SBPrefixForString(kEvil1Url1), prefix_hits[0]);
1236   EXPECT_EQ(SBPrefixForString(kEvil1Url2), prefix_hits[1]);
1237   database_.reset();
1238 }
1239
1240 // Checks that the whitelists are handled properly.
1241 TEST_F(SafeBrowsingDatabaseTest, Whitelists) {
1242   database_.reset();
1243
1244   // We expect all calls to ContainsCsdWhitelistedUrl in particular to be made
1245   // from the IO thread.  In general the whitelist lookups are thread-safe.
1246   content::TestBrowserThreadBundle thread_bundle_;
1247
1248   // If the whitelist is disabled everything should match the whitelist.
1249   database_.reset(new SafeBrowsingDatabaseNew(new SafeBrowsingStoreFile(),
1250                                               NULL, NULL, NULL, NULL, NULL,
1251                                               NULL, NULL));
1252   database_->Init(database_filename_);
1253   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1254       GURL(std::string("http://www.phishing.com/"))));
1255   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1256       GURL(std::string("http://www.phishing.com/"))));
1257   EXPECT_TRUE(database_->ContainsDownloadWhitelistedString("asdf"));
1258
1259   ResetAndReloadFullDatabase();
1260
1261   const char kGood1Host[] = "www.good1.com/";
1262   const char kGood1Url1[] = "www.good1.com/a/b.html";
1263   const char kGood1Url2[] = "www.good1.com/b/";
1264
1265   const char kGood2Url1[] = "www.good2.com/c";  // Should match '/c/bla'.
1266
1267   // good3.com/a/b/c/d/e/f/g/ should match because it's a whitelist.
1268   const char kGood3Url1[] = "good3.com/";
1269
1270   const char kGoodString[] = "good_string";
1271
1272   ScopedVector<SBChunkData> csd_chunks;
1273   ScopedVector<SBChunkData> download_chunks;
1274
1275   // Add two simple chunks to the csd whitelist.
1276   csd_chunks.push_back(AddChunkFullHash2Value(1, kGood1Url1, kGood1Url2));
1277   csd_chunks.push_back(AddChunkFullHashValue(2, kGood2Url1));
1278   download_chunks.push_back(AddChunkFullHashValue(2, kGood2Url1));
1279   download_chunks.push_back(AddChunkFullHashValue(3, kGoodString));
1280   download_chunks.push_back(AddChunkFullHashValue(4, kGood3Url1));
1281
1282   std::vector<SBListChunkRanges> lists;
1283   ASSERT_TRUE(database_->UpdateStarted(&lists));
1284   database_->InsertChunks(safe_browsing_util::kCsdWhiteList, csd_chunks.get());
1285   database_->InsertChunks(safe_browsing_util::kDownloadWhiteList,
1286                           download_chunks.get());
1287   database_->UpdateFinished(true);
1288
1289   EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl(
1290       GURL(std::string("http://") + kGood1Host)));
1291
1292   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1293       GURL(std::string("http://") + kGood1Url1)));
1294   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1295       GURL(std::string("http://") + kGood1Url1 + "?a=b")));
1296
1297   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1298       GURL(std::string("http://") + kGood1Url2)));
1299   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1300       GURL(std::string("http://") + kGood1Url2 + "/c.html")));
1301
1302   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1303       GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1304
1305   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1306       GURL(std::string("http://") + kGood2Url1 + "/c")));
1307   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1308       GURL(std::string("http://") + kGood2Url1 + "/c?bla")));
1309   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1310       GURL(std::string("http://") + kGood2Url1 + "/c/bla")));
1311
1312   EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl(
1313       GURL(std::string("http://www.google.com/"))));
1314
1315   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1316       GURL(std::string("http://") + kGood2Url1 + "/c")));
1317   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1318       GURL(std::string("http://") + kGood2Url1 + "/c?bla")));
1319   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1320       GURL(std::string("http://") + kGood2Url1 + "/c/bla")));
1321
1322   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1323       GURL(std::string("http://good3.com/a/b/c/d/e/f/g/"))));
1324   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1325       GURL(std::string("http://a.b.good3.com/"))));
1326
1327   EXPECT_FALSE(database_->ContainsDownloadWhitelistedString("asdf"));
1328   EXPECT_TRUE(database_->ContainsDownloadWhitelistedString(kGoodString));
1329
1330   EXPECT_FALSE(database_->ContainsDownloadWhitelistedUrl(
1331       GURL(std::string("http://www.google.com/"))));
1332
1333   // The CSD whitelist killswitch is not present.
1334   EXPECT_FALSE(database_->IsCsdWhitelistKillSwitchOn());
1335
1336   // Test only add the malware IP killswitch
1337   csd_chunks.clear();
1338   csd_chunks.push_back(AddChunkFullHashValue(
1339       15, "sb-ssl.google.com/safebrowsing/csd/killswitch_malware"));
1340
1341   ASSERT_TRUE(database_->UpdateStarted(&lists));
1342   database_->InsertChunks(safe_browsing_util::kCsdWhiteList, csd_chunks.get());
1343   database_->UpdateFinished(true);
1344
1345   EXPECT_TRUE(database_->IsMalwareIPMatchKillSwitchOn());
1346   // The CSD whitelist killswitch is not present.
1347   EXPECT_FALSE(database_->IsCsdWhitelistKillSwitchOn());
1348
1349   // Test that the kill-switch works as intended.
1350   csd_chunks.clear();
1351   download_chunks.clear();
1352   lists.clear();
1353   csd_chunks.push_back(AddChunkFullHashValue(
1354       5, "sb-ssl.google.com/safebrowsing/csd/killswitch"));
1355   download_chunks.push_back(AddChunkFullHashValue(
1356       5, "sb-ssl.google.com/safebrowsing/csd/killswitch"));
1357
1358   ASSERT_TRUE(database_->UpdateStarted(&lists));
1359   database_->InsertChunks(safe_browsing_util::kCsdWhiteList, csd_chunks.get());
1360   database_->InsertChunks(safe_browsing_util::kDownloadWhiteList,
1361                           download_chunks.get());
1362   database_->UpdateFinished(true);
1363
1364   // The CSD whitelist killswitch is present.
1365   EXPECT_TRUE(database_->IsCsdWhitelistKillSwitchOn());
1366   EXPECT_TRUE(database_->IsMalwareIPMatchKillSwitchOn());
1367   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1368       GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1369   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1370       GURL(std::string("http://www.google.com/"))));
1371   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1372       GURL(std::string("http://www.phishing_url.com/"))));
1373
1374   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1375       GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1376   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1377       GURL(std::string("http://www.google.com/"))));
1378   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1379       GURL(std::string("http://www.phishing_url.com/"))));
1380
1381   EXPECT_TRUE(database_->ContainsDownloadWhitelistedString("asdf"));
1382   EXPECT_TRUE(database_->ContainsDownloadWhitelistedString(kGoodString));
1383
1384   // Remove the kill-switch and verify that we can recover.
1385   csd_chunks.clear();
1386   download_chunks.clear();
1387   lists.clear();
1388   csd_chunks.push_back(SubChunkFullHashValue(
1389       1, "sb-ssl.google.com/safebrowsing/csd/killswitch", 5));
1390   csd_chunks.push_back(SubChunkFullHashValue(
1391       10, "sb-ssl.google.com/safebrowsing/csd/killswitch_malware", 15));
1392   download_chunks.push_back(SubChunkFullHashValue(
1393       1, "sb-ssl.google.com/safebrowsing/csd/killswitch", 5));
1394
1395   ASSERT_TRUE(database_->UpdateStarted(&lists));
1396   database_->InsertChunks(safe_browsing_util::kCsdWhiteList, csd_chunks.get());
1397   database_->InsertChunks(safe_browsing_util::kDownloadWhiteList,
1398                           download_chunks.get());
1399   database_->UpdateFinished(true);
1400
1401   EXPECT_FALSE(database_->IsMalwareIPMatchKillSwitchOn());
1402   EXPECT_FALSE(database_->IsCsdWhitelistKillSwitchOn());
1403   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1404       GURL(std::string("https://") + kGood1Url2 + "/c.html")));
1405   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
1406       GURL(std::string("https://") + kGood2Url1 + "/c/bla")));
1407   EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl(
1408       GURL(std::string("http://www.google.com/"))));
1409   EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl(
1410       GURL(std::string("http://www.phishing_url.com/"))));
1411
1412   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1413       GURL(std::string("https://") + kGood2Url1 + "/c/bla")));
1414   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
1415       GURL(std::string("https://good3.com/"))));
1416   EXPECT_TRUE(database_->ContainsDownloadWhitelistedString(kGoodString));
1417   EXPECT_FALSE(database_->ContainsDownloadWhitelistedUrl(
1418       GURL(std::string("http://www.google.com/"))));
1419   EXPECT_FALSE(database_->ContainsDownloadWhitelistedUrl(
1420       GURL(std::string("http://www.phishing_url.com/"))));
1421   EXPECT_FALSE(database_->ContainsDownloadWhitelistedString("asdf"));
1422
1423   database_.reset();
1424 }
1425
1426 // Test to make sure we could insert chunk list that
1427 // contains entries for the same host.
1428 TEST_F(SafeBrowsingDatabaseTest, SameHostEntriesOkay) {
1429   ScopedVector<SBChunkData> chunks;
1430
1431   // Add a malware add chunk with two entries of the same host.
1432   chunks.push_back(AddChunkPrefix2Value(1,
1433                                         "www.evil.com/malware1.html",
1434                                         "www.evil.com/malware2.html"));
1435
1436   // Insert the testing chunks into database.
1437   std::vector<SBListChunkRanges> lists;
1438   ASSERT_TRUE(database_->UpdateStarted(&lists));
1439   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1440   database_->UpdateFinished(true);
1441
1442   GetListsInfo(&lists);
1443   ASSERT_LE(1U, lists.size());
1444   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
1445   EXPECT_EQ("1", lists[0].adds);
1446   EXPECT_TRUE(lists[0].subs.empty());
1447
1448   // Add a phishing add chunk with two entries of the same host.
1449   chunks.clear();
1450   chunks.push_back(AddChunkPrefix2Value(47,
1451                                         "www.evil.com/phishing1.html",
1452                                         "www.evil.com/phishing2.html"));
1453
1454   ASSERT_TRUE(database_->UpdateStarted(&lists));
1455   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks.get());
1456   database_->UpdateFinished(true);
1457
1458   GetListsInfo(&lists);
1459   ASSERT_LE(2U, lists.size());
1460   EXPECT_EQ(safe_browsing_util::kMalwareList, lists[0].name);
1461   EXPECT_EQ("1", lists[0].adds);
1462   EXPECT_TRUE(lists[0].subs.empty());
1463   EXPECT_EQ(safe_browsing_util::kPhishingList, lists[1].name);
1464   EXPECT_EQ("47", lists[1].adds);
1465   EXPECT_TRUE(lists[1].subs.empty());
1466
1467   std::vector<SBPrefix> prefix_hits;
1468   std::vector<SBFullHashResult> cache_hits;
1469
1470   EXPECT_TRUE(database_->ContainsBrowseUrl(
1471       GURL("http://www.evil.com/malware1.html"), &prefix_hits, &cache_hits));
1472   EXPECT_TRUE(database_->ContainsBrowseUrl(
1473       GURL("http://www.evil.com/malware2.html"), &prefix_hits, &cache_hits));
1474   EXPECT_TRUE(database_->ContainsBrowseUrl(
1475       GURL("http://www.evil.com/phishing1.html"), &prefix_hits, &cache_hits));
1476   EXPECT_TRUE(database_->ContainsBrowseUrl(
1477       GURL("http://www.evil.com/phishing2.html"), &prefix_hits, &cache_hits));
1478
1479   // Test removing a single prefix from the add chunk.
1480   // Remove the prefix that added first.
1481   chunks.clear();
1482   chunks.push_back(SubChunkPrefixValue(4, "www.evil.com/malware1.html", 1));
1483   ASSERT_TRUE(database_->UpdateStarted(&lists));
1484   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1485   database_->UpdateFinished(true);
1486
1487   // Remove the prefix that added last.
1488   chunks.clear();
1489   chunks.push_back(SubChunkPrefixValue(5, "www.evil.com/phishing2.html", 47));
1490   ASSERT_TRUE(database_->UpdateStarted(&lists));
1491   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks.get());
1492   database_->UpdateFinished(true);
1493
1494   // Verify that the database contains urls expected.
1495   EXPECT_FALSE(database_->ContainsBrowseUrl(
1496       GURL("http://www.evil.com/malware1.html"), &prefix_hits, &cache_hits));
1497   EXPECT_TRUE(database_->ContainsBrowseUrl(
1498       GURL("http://www.evil.com/malware2.html"), &prefix_hits, &cache_hits));
1499   EXPECT_TRUE(database_->ContainsBrowseUrl(
1500       GURL("http://www.evil.com/phishing1.html"), &prefix_hits, &cache_hits));
1501   EXPECT_FALSE(database_->ContainsBrowseUrl(
1502       GURL("http://www.evil.com/phishing2.html"), &prefix_hits, &cache_hits));
1503 }
1504
1505 // Test that an empty update doesn't actually update the database.
1506 // This isn't a functionality requirement, but it is a useful
1507 // optimization.
1508 TEST_F(SafeBrowsingDatabaseTest, EmptyUpdate) {
1509   ScopedVector<SBChunkData> chunks;
1510
1511   base::FilePath filename = database_->BrowseDBFilename(database_filename_);
1512
1513   // Prime the database.
1514   std::vector<SBListChunkRanges> lists;
1515   ASSERT_TRUE(database_->UpdateStarted(&lists));
1516   chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/malware.html"));
1517   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1518   database_->UpdateFinished(true);
1519
1520   // Get an older time to reset the lastmod time for detecting whether
1521   // the file has been updated.
1522   base::File::Info before_info, after_info;
1523   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1524   const Time old_last_modified =
1525       before_info.last_modified - TimeDelta::FromSeconds(10);
1526
1527   // Inserting another chunk updates the database file.  The sleep is
1528   // needed because otherwise the entire test can finish w/in the
1529   // resolution of the lastmod time.
1530   ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1531   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1532   ASSERT_TRUE(database_->UpdateStarted(&lists));
1533   chunks.push_back(AddChunkPrefixValue(2, "www.foo.com/malware.html"));
1534   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1535   database_->UpdateFinished(true);
1536   ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1537   EXPECT_LT(before_info.last_modified, after_info.last_modified);
1538
1539   // Deleting a chunk updates the database file.
1540   ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1541   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1542   ASSERT_TRUE(database_->UpdateStarted(&lists));
1543   AddDelChunk(safe_browsing_util::kMalwareList, 2);
1544   database_->UpdateFinished(true);
1545   ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1546   EXPECT_LT(before_info.last_modified, after_info.last_modified);
1547
1548   // Simply calling |UpdateStarted()| then |UpdateFinished()| does not
1549   // update the database file.
1550   ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
1551   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
1552   ASSERT_TRUE(database_->UpdateStarted(&lists));
1553   database_->UpdateFinished(true);
1554   ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
1555   EXPECT_EQ(before_info.last_modified, after_info.last_modified);
1556 }
1557
1558 // Test that a filter file is written out during update and read back
1559 // in during setup.
1560 TEST_F(SafeBrowsingDatabaseTest, FilterFile) {
1561   // Create a database with trivial example data and write it out.
1562   {
1563     // Prime the database.
1564     std::vector<SBListChunkRanges> lists;
1565     ASSERT_TRUE(database_->UpdateStarted(&lists));
1566
1567     ScopedVector<SBChunkData> chunks;
1568     chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/malware.html"));
1569     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1570     database_->UpdateFinished(true);
1571   }
1572
1573   // Find the malware url in the database, don't find a good url.
1574   std::vector<SBPrefix> prefix_hits;
1575   std::vector<SBFullHashResult> cache_hits;
1576   EXPECT_TRUE(database_->ContainsBrowseUrl(
1577       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
1578   EXPECT_FALSE(database_->ContainsBrowseUrl(
1579       GURL("http://www.good.com/goodware.html"), &prefix_hits, &cache_hits));
1580
1581   base::FilePath filter_file = database_->PrefixSetForFilename(
1582       database_->BrowseDBFilename(database_filename_));
1583
1584   // After re-creating the database, it should have a filter read from
1585   // a file, so it should find the same results.
1586   ASSERT_TRUE(base::PathExists(filter_file));
1587   ResetAndReloadFullDatabase();
1588   EXPECT_TRUE(database_->ContainsBrowseUrl(
1589       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
1590   EXPECT_FALSE(database_->ContainsBrowseUrl(
1591       GURL("http://www.good.com/goodware.html"), &prefix_hits, &cache_hits));
1592
1593   // If there is no filter file, the database cannot find malware urls.
1594   base::DeleteFile(filter_file, false);
1595   ASSERT_FALSE(base::PathExists(filter_file));
1596   ResetAndReloadFullDatabase();
1597   EXPECT_FALSE(database_->ContainsBrowseUrl(
1598       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
1599   EXPECT_FALSE(database_->ContainsBrowseUrl(
1600       GURL("http://www.good.com/goodware.html"), &prefix_hits, &cache_hits));
1601 }
1602
1603 TEST_F(SafeBrowsingDatabaseTest, CachedFullMiss) {
1604   const SBPrefix kPrefix1 = 1001U;
1605   const SBFullHash kFullHash1_1 =
1606       SBFullHashForPrefixAndSuffix(kPrefix1, "\x01");
1607
1608   const SBPrefix kPrefix2 = 1002U;
1609   const SBFullHash kFullHash2_1 =
1610       SBFullHashForPrefixAndSuffix(kPrefix2, "\x01");
1611
1612   // Insert prefix kPrefix1 and kPrefix2 into database.
1613   ScopedVector<SBChunkData> chunks;
1614   chunks.push_back(AddChunkPrefix(1, kPrefix1));
1615   chunks.push_back(AddChunkPrefix(2, kPrefix2));
1616
1617   std::vector<SBListChunkRanges> lists;
1618   ASSERT_TRUE(database_->UpdateStarted(&lists));
1619   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1620   database_->UpdateFinished(true);
1621
1622   {
1623     // Cache a full miss result for kPrefix1.
1624     std::vector<SBPrefix> prefixes(1, kPrefix1);
1625     std::vector<SBFullHashResult> cache_results;
1626     database_->CacheHashResults(prefixes, cache_results, kCacheLifetime);
1627   }
1628
1629   {
1630     // kFullHash1_1 gets no prefix hit because of the cached item, and also does
1631     // not have a cache hit.
1632     std::vector<SBFullHash> full_hashes(1, kFullHash1_1);
1633     std::vector<SBPrefix> prefix_hits;
1634     std::vector<SBFullHashResult> cache_hits;
1635     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1636         full_hashes, &prefix_hits, &cache_hits));
1637
1638     // kFullHash2_1 gets a hit from the prefix in the database.
1639     full_hashes.push_back(kFullHash2_1);
1640     prefix_hits.clear();
1641     cache_hits.clear();
1642     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1643         full_hashes, &prefix_hits, &cache_hits));
1644     ASSERT_EQ(1U, prefix_hits.size());
1645     EXPECT_EQ(kPrefix2, prefix_hits[0]);
1646     EXPECT_TRUE(cache_hits.empty());
1647   }
1648 }
1649
1650 TEST_F(SafeBrowsingDatabaseTest, CachedPrefixHitFullMiss) {
1651   const SBPrefix kPrefix1 = 1001U;
1652   const SBFullHash kFullHash1_1 =
1653       SBFullHashForPrefixAndSuffix(kPrefix1, "\x01");
1654   const SBFullHash kFullHash1_2 =
1655       SBFullHashForPrefixAndSuffix(kPrefix1, "\x02");
1656   const SBFullHash kFullHash1_3 =
1657       SBFullHashForPrefixAndSuffix(kPrefix1, "\x03");
1658
1659   const SBPrefix kPrefix2 = 1002U;
1660   const SBFullHash kFullHash2_1 =
1661       SBFullHashForPrefixAndSuffix(kPrefix2, "\x01");
1662
1663   const SBPrefix kPrefix3 = 1003U;
1664   const SBFullHash kFullHash3_1 =
1665       SBFullHashForPrefixAndSuffix(kPrefix3, "\x01");
1666
1667   // Insert prefix kPrefix1 and kPrefix2 into database.
1668   ScopedVector<SBChunkData> chunks;
1669   chunks.push_back(AddChunkPrefix(1, kPrefix1));
1670   chunks.push_back(AddChunkPrefix(2, kPrefix2));
1671
1672   std::vector<SBListChunkRanges> lists;
1673   ASSERT_TRUE(database_->UpdateStarted(&lists));
1674   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1675   database_->UpdateFinished(true);
1676
1677   {
1678     // kFullHash1_1 has a prefix hit of kPrefix1.
1679     std::vector<SBFullHash> full_hashes;
1680     full_hashes.push_back(kFullHash1_1);
1681     std::vector<SBPrefix> prefix_hits;
1682     std::vector<SBFullHashResult> cache_hits;
1683     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1684         full_hashes, &prefix_hits, &cache_hits));
1685     ASSERT_EQ(1U, prefix_hits.size());
1686     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1687     EXPECT_TRUE(cache_hits.empty());
1688
1689     // kFullHash2_1 has a prefix hit of kPrefix2.
1690     full_hashes.push_back(kFullHash2_1);
1691     prefix_hits.clear();
1692     cache_hits.clear();
1693     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1694         full_hashes, &prefix_hits, &cache_hits));
1695     ASSERT_EQ(2U, prefix_hits.size());
1696     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1697     EXPECT_EQ(kPrefix2, prefix_hits[1]);
1698     EXPECT_TRUE(cache_hits.empty());
1699
1700     // kFullHash3_1 has no hits.
1701     full_hashes.push_back(kFullHash3_1);
1702     prefix_hits.clear();
1703     cache_hits.clear();
1704     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1705         full_hashes, &prefix_hits, &cache_hits));
1706     ASSERT_EQ(2U, prefix_hits.size());
1707     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1708     EXPECT_EQ(kPrefix2, prefix_hits[1]);
1709     EXPECT_TRUE(cache_hits.empty());
1710   }
1711
1712   {
1713     // Cache a fullhash result for two kPrefix1 full hashes.
1714     std::vector<SBPrefix> prefixes(1, kPrefix1);
1715     std::vector<SBFullHashResult> cache_results;
1716
1717     SBFullHashResult full_hash_result;
1718     full_hash_result.list_id = safe_browsing_util::MALWARE;
1719
1720     full_hash_result.hash = kFullHash1_1;
1721     cache_results.push_back(full_hash_result);
1722
1723     full_hash_result.hash = kFullHash1_3;
1724     cache_results.push_back(full_hash_result);
1725
1726     database_->CacheHashResults(prefixes, cache_results, kCacheLifetime);
1727   }
1728
1729   {
1730     // kFullHash1_1 should now see a cache hit.
1731     std::vector<SBFullHash> full_hashes(1, kFullHash1_1);
1732     std::vector<SBPrefix> prefix_hits;
1733     std::vector<SBFullHashResult> cache_hits;
1734     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1735         full_hashes, &prefix_hits, &cache_hits));
1736     EXPECT_TRUE(prefix_hits.empty());
1737     ASSERT_EQ(1U, cache_hits.size());
1738     EXPECT_TRUE(SBFullHashEqual(kFullHash1_1, cache_hits[0].hash));
1739
1740     // Adding kFullHash2_1 will see the existing cache hit plus the prefix hit
1741     // for kPrefix2.
1742     full_hashes.push_back(kFullHash2_1);
1743     prefix_hits.clear();
1744     cache_hits.clear();
1745     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1746         full_hashes, &prefix_hits, &cache_hits));
1747     ASSERT_EQ(1U, prefix_hits.size());
1748     EXPECT_EQ(kPrefix2, prefix_hits[0]);
1749     ASSERT_EQ(1U, cache_hits.size());
1750     EXPECT_TRUE(SBFullHashEqual(kFullHash1_1, cache_hits[0].hash));
1751
1752     // kFullHash1_3 also gets a cache hit.
1753     full_hashes.push_back(kFullHash1_3);
1754     prefix_hits.clear();
1755     cache_hits.clear();
1756     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1757         full_hashes, &prefix_hits, &cache_hits));
1758     ASSERT_EQ(1U, prefix_hits.size());
1759     EXPECT_EQ(kPrefix2, prefix_hits[0]);
1760     ASSERT_EQ(2U, cache_hits.size());
1761     EXPECT_TRUE(SBFullHashEqual(kFullHash1_1, cache_hits[0].hash));
1762     EXPECT_TRUE(SBFullHashEqual(kFullHash1_3, cache_hits[1].hash));
1763   }
1764
1765   {
1766     // Check if DB contains only kFullHash1_3. Should return a cache hit.
1767     std::vector<SBFullHash> full_hashes(1, kFullHash1_3);
1768     std::vector<SBPrefix> prefix_hits;
1769     std::vector<SBFullHashResult> cache_hits;
1770     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1771         full_hashes, &prefix_hits, &cache_hits));
1772     EXPECT_TRUE(prefix_hits.empty());
1773     ASSERT_EQ(1U, cache_hits.size());
1774     EXPECT_TRUE(SBFullHashEqual(kFullHash1_3, cache_hits[0].hash));
1775   }
1776
1777   {
1778     // kFullHash1_2 has no cache hit, and no prefix hit because of the cache for
1779     // kPrefix1.
1780     std::vector<SBFullHash> full_hashes(1, kFullHash1_2);
1781     std::vector<SBPrefix> prefix_hits;
1782     std::vector<SBFullHashResult> cache_hits;
1783     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1784         full_hashes, &prefix_hits, &cache_hits));
1785
1786     // Other prefix hits possible when kFullHash1_2 hits nothing.
1787     full_hashes.push_back(kFullHash2_1);
1788     prefix_hits.clear();
1789     cache_hits.clear();
1790     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1791         full_hashes, &prefix_hits, &cache_hits));
1792     ASSERT_EQ(1U, prefix_hits.size());
1793     EXPECT_EQ(kPrefix2, prefix_hits[0]);
1794     EXPECT_TRUE(cache_hits.empty());
1795   }
1796 }
1797
1798 TEST_F(SafeBrowsingDatabaseTest, BrowseFullHashMatching) {
1799   const SBPrefix kPrefix1 = 1001U;
1800   const SBFullHash kFullHash1_1 =
1801       SBFullHashForPrefixAndSuffix(kPrefix1, "\x01");
1802   const SBFullHash kFullHash1_2 =
1803       SBFullHashForPrefixAndSuffix(kPrefix1, "\x02");
1804   const SBFullHash kFullHash1_3 =
1805       SBFullHashForPrefixAndSuffix(kPrefix1, "\x03");
1806
1807   // Insert two full hashes with a shared prefix.
1808   ScopedVector<SBChunkData> chunks;
1809   chunks.push_back(AddChunkFullHash(1, kFullHash1_1));
1810   chunks.push_back(AddChunkFullHash(2, kFullHash1_2));
1811
1812   std::vector<SBListChunkRanges> lists;
1813   ASSERT_TRUE(database_->UpdateStarted(&lists));
1814   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1815   database_->UpdateFinished(true);
1816
1817   {
1818     // Check a full hash which isn't present.
1819     std::vector<SBFullHash> full_hashes(1, kFullHash1_3);
1820     std::vector<SBPrefix> prefix_hits;
1821     std::vector<SBFullHashResult> cache_hits;
1822     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1823         full_hashes, &prefix_hits, &cache_hits));
1824
1825     // Also one which is present, should have a prefix hit.
1826     full_hashes.push_back(kFullHash1_1);
1827     prefix_hits.clear();
1828     cache_hits.clear();
1829     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1830         full_hashes, &prefix_hits, &cache_hits));
1831     ASSERT_EQ(1U, prefix_hits.size());
1832     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1833     EXPECT_TRUE(cache_hits.empty());
1834
1835     // Two full hash matches with the same prefix should return one prefix hit.
1836     full_hashes.push_back(kFullHash1_2);
1837     prefix_hits.clear();
1838     cache_hits.clear();
1839     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1840         full_hashes, &prefix_hits, &cache_hits));
1841     ASSERT_EQ(1U, prefix_hits.size());
1842     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1843     EXPECT_TRUE(cache_hits.empty());
1844   }
1845
1846   {
1847     // Cache a gethash result for kFullHash1_2.
1848     SBFullHashResult full_hash_result;
1849     full_hash_result.list_id = safe_browsing_util::MALWARE;
1850     full_hash_result.hash = kFullHash1_2;
1851
1852     std::vector<SBPrefix> prefixes(1, kPrefix1);
1853     std::vector<SBFullHashResult> cache_results(1, full_hash_result);
1854
1855     database_->CacheHashResults(prefixes, cache_results, kCacheLifetime);
1856   }
1857
1858   {
1859     // kFullHash1_3 should still return false, because the cached
1860     // result for kPrefix1 doesn't contain kFullHash1_3.
1861     std::vector<SBFullHash> full_hashes(1, kFullHash1_3);
1862     std::vector<SBPrefix> prefix_hits;
1863     std::vector<SBFullHashResult> cache_hits;
1864     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1865         full_hashes, &prefix_hits, &cache_hits));
1866
1867     // kFullHash1_1 is also not in the cached result, which takes
1868     // priority over the database.
1869     prefix_hits.clear();
1870     full_hashes.push_back(kFullHash1_1);
1871     cache_hits.clear();
1872     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1873         full_hashes, &prefix_hits, &cache_hits));
1874
1875     // kFullHash1_2 is in the cached result.
1876     full_hashes.push_back(kFullHash1_2);
1877     prefix_hits.clear();
1878     cache_hits.clear();
1879     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1880         full_hashes, &prefix_hits, &cache_hits));
1881     EXPECT_TRUE(prefix_hits.empty());
1882     ASSERT_EQ(1U, cache_hits.size());
1883     EXPECT_TRUE(SBFullHashEqual(kFullHash1_2, cache_hits[0].hash));
1884   }
1885
1886   // Remove kFullHash1_1 from the database.
1887   chunks.clear();
1888   chunks.push_back(SubChunkFullHash(11, kFullHash1_1, 1));
1889
1890   ASSERT_TRUE(database_->UpdateStarted(&lists));
1891   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1892   database_->UpdateFinished(true);
1893
1894   // Cache should be cleared after updating.
1895   EXPECT_TRUE(database_->prefix_gethash_cache_.empty());
1896
1897   {
1898     // Now the database doesn't contain kFullHash1_1.
1899     std::vector<SBFullHash> full_hashes(1, kFullHash1_1);
1900     std::vector<SBPrefix> prefix_hits;
1901     std::vector<SBFullHashResult> cache_hits;
1902     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1903         full_hashes, &prefix_hits, &cache_hits));
1904
1905     // Nor kFullHash1_3.
1906     full_hashes.push_back(kFullHash1_3);
1907     prefix_hits.clear();
1908     cache_hits.clear();
1909     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1910         full_hashes, &prefix_hits, &cache_hits));
1911
1912     // Still has kFullHash1_2.
1913     full_hashes.push_back(kFullHash1_2);
1914     prefix_hits.clear();
1915     cache_hits.clear();
1916     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1917         full_hashes, &prefix_hits, &cache_hits));
1918     ASSERT_EQ(1U, prefix_hits.size());
1919     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1920     EXPECT_TRUE(cache_hits.empty());
1921   }
1922
1923   // Remove kFullHash1_2 from the database.
1924   chunks.clear();
1925   chunks.push_back(SubChunkFullHash(12, kFullHash1_2, 2));
1926
1927   ASSERT_TRUE(database_->UpdateStarted(&lists));
1928   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1929   database_->UpdateFinished(true);
1930
1931   // Cache should be cleared after updating.
1932   EXPECT_TRUE(database_->prefix_gethash_cache_.empty());
1933
1934   {
1935     // None are present.
1936     std::vector<SBFullHash> full_hashes;
1937     std::vector<SBPrefix> prefix_hits;
1938     std::vector<SBFullHashResult> cache_hits;
1939     full_hashes.push_back(kFullHash1_1);
1940     full_hashes.push_back(kFullHash1_2);
1941     full_hashes.push_back(kFullHash1_3);
1942     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1943         full_hashes, &prefix_hits, &cache_hits));
1944   }
1945 }
1946
1947 TEST_F(SafeBrowsingDatabaseTest, BrowseFullHashAndPrefixMatching) {
1948   const SBPrefix kPrefix1 = 1001U;
1949   const SBFullHash kFullHash1_1 =
1950       SBFullHashForPrefixAndSuffix(kPrefix1, "\x01");
1951   const SBFullHash kFullHash1_2 =
1952       SBFullHashForPrefixAndSuffix(kPrefix1, "\x02");
1953
1954   ScopedVector<SBChunkData> chunks;
1955   chunks.push_back(AddChunkFullHash(1, kFullHash1_1));
1956
1957   std::vector<SBListChunkRanges> lists;
1958   ASSERT_TRUE(database_->UpdateStarted(&lists));
1959   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1960   database_->UpdateFinished(true);
1961
1962   {
1963     // kFullHash1_2 does not match kFullHash1_1.
1964     std::vector<SBFullHash> full_hashes(1, kFullHash1_2);
1965     std::vector<SBPrefix> prefix_hits;
1966     std::vector<SBFullHashResult> cache_hits;
1967     EXPECT_FALSE(database_->ContainsBrowseUrlHashesForTesting(
1968         full_hashes, &prefix_hits, &cache_hits));
1969   }
1970
1971   // Add a prefix match.
1972   chunks.clear();
1973   chunks.push_back(AddChunkPrefix(2, kPrefix1));
1974
1975   ASSERT_TRUE(database_->UpdateStarted(&lists));
1976   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1977   database_->UpdateFinished(true);
1978
1979   {
1980     // kFullHash1_2 does match kPrefix1.
1981     std::vector<SBFullHash> full_hashes(1, kFullHash1_2);
1982     std::vector<SBPrefix> prefix_hits;
1983     std::vector<SBFullHashResult> cache_hits;
1984     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
1985         full_hashes, &prefix_hits, &cache_hits));
1986     ASSERT_EQ(1U, prefix_hits.size());
1987     EXPECT_EQ(kPrefix1, prefix_hits[0]);
1988     EXPECT_TRUE(cache_hits.empty());
1989   }
1990
1991   // Remove the full hash.
1992   chunks.clear();
1993   chunks.push_back(SubChunkFullHash(11, kFullHash1_1, 1));
1994
1995   ASSERT_TRUE(database_->UpdateStarted(&lists));
1996   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
1997   database_->UpdateFinished(true);
1998
1999   {
2000     // kFullHash1_2 still returns true due to the prefix hit.
2001     std::vector<SBFullHash> full_hashes(1, kFullHash1_2);
2002     std::vector<SBPrefix> prefix_hits;
2003     std::vector<SBFullHashResult> cache_hits;
2004     EXPECT_TRUE(database_->ContainsBrowseUrlHashesForTesting(
2005         full_hashes, &prefix_hits, &cache_hits));
2006     ASSERT_EQ(1U, prefix_hits.size());
2007     EXPECT_EQ(kPrefix1, prefix_hits[0]);
2008     EXPECT_TRUE(cache_hits.empty());
2009   }
2010 }
2011
2012 TEST_F(SafeBrowsingDatabaseTest, MalwareIpBlacklist) {
2013   std::vector<SBListChunkRanges> lists;
2014   ASSERT_TRUE(database_->UpdateStarted(&lists));
2015
2016   ScopedVector<SBChunkData> chunks;
2017
2018   // IPv4 prefix match for ::ffff:192.168.1.0/120.
2019   chunks.push_back(AddChunkHashedIpValue(1, "::ffff:192.168.1.0", 120));
2020
2021   // IPv4 exact match for ::ffff:192.1.1.1.
2022   chunks.push_back(AddChunkHashedIpValue(2, "::ffff:192.1.1.1", 128));
2023
2024   // IPv6 exact match for: fe80::31a:a0ff:fe10:786e/128.
2025   chunks.push_back(AddChunkHashedIpValue(3, "fe80::31a:a0ff:fe10:786e", 128));
2026
2027   // IPv6 prefix match for: 2620:0:1000:3103::/64.
2028   chunks.push_back(AddChunkHashedIpValue(4, "2620:0:1000:3103::", 64));
2029
2030   // IPv4 prefix match for ::ffff:192.1.122.0/119.
2031   chunks.push_back(AddChunkHashedIpValue(5, "::ffff:192.1.122.0", 119));
2032
2033   // IPv4 prefix match for ::ffff:192.1.128.0/113.
2034   chunks.push_back(AddChunkHashedIpValue(6, "::ffff:192.1.128.0", 113));
2035
2036   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks.get());
2037   database_->UpdateFinished(true);
2038
2039   EXPECT_FALSE(database_->ContainsMalwareIP("192.168.0.255"));
2040   EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.0"));
2041   EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.255"));
2042   EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.10"));
2043   EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.168.1.2"));
2044   EXPECT_FALSE(database_->ContainsMalwareIP("192.168.2.0"));
2045
2046   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.1.0"));
2047   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.1.1"));
2048   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.1.2"));
2049
2050   EXPECT_FALSE(database_->ContainsMalwareIP(
2051       "2620:0:1000:3102:ffff:ffff:ffff:ffff"));
2052   EXPECT_TRUE(database_->ContainsMalwareIP("2620:0:1000:3103::"));
2053   EXPECT_TRUE(database_->ContainsMalwareIP(
2054       "2620:0:1000:3103:ffff:ffff:ffff:ffff"));
2055   EXPECT_FALSE(database_->ContainsMalwareIP("2620:0:1000:3104::"));
2056
2057   EXPECT_FALSE(database_->ContainsMalwareIP("fe80::21a:a0ff:fe10:786d"));
2058   EXPECT_TRUE(database_->ContainsMalwareIP("fe80::31a:a0ff:fe10:786e"));
2059   EXPECT_FALSE(database_->ContainsMalwareIP("fe80::21a:a0ff:fe10:786f"));
2060
2061   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.121.255"));
2062   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.122.0"));
2063   EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.1.122.1"));
2064   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.122.255"));
2065   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.123.0"));
2066   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.123.255"));
2067   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.124.0"));
2068
2069   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.127.255"));
2070   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.128.0"));
2071   EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.1.128.1"));
2072   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.128.255"));
2073   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.255.0"));
2074   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.255.255"));
2075   EXPECT_FALSE(database_->ContainsMalwareIP("192.2.0.0"));
2076 }
2077
2078 TEST_F(SafeBrowsingDatabaseTest, ContainsBrowseURL) {
2079   std::vector<SBListChunkRanges> lists;
2080   ASSERT_TRUE(database_->UpdateStarted(&lists));
2081
2082   // Add a host-level hit.
2083   {
2084     ScopedVector<SBChunkData> chunks;
2085     chunks.push_back(AddChunkPrefixValue(1, "www.evil.com/"));
2086     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2087   }
2088
2089   // Add a specific fullhash.
2090   static const char kWhateverMalware[] = "www.whatever.com/malware.html";
2091   {
2092     ScopedVector<SBChunkData> chunks;
2093     chunks.push_back(AddChunkFullHashValue(2, kWhateverMalware));
2094     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2095   }
2096
2097   // Add a fullhash which has a prefix collision for a known url.
2098   static const char kExampleFine[] = "www.example.com/fine.html";
2099   static const char kExampleCollision[] =
2100       "www.example.com/3123364814/malware.htm";
2101   ASSERT_EQ(SBPrefixForString(kExampleFine),
2102             SBPrefixForString(kExampleCollision));
2103   {
2104     ScopedVector<SBChunkData> chunks;
2105     chunks.push_back(AddChunkFullHashValue(3, kExampleCollision));
2106     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks.get());
2107   }
2108
2109   database_->UpdateFinished(true);
2110
2111   std::vector<SBPrefix> prefix_hits;
2112   std::vector<SBFullHashResult> cache_hits;
2113
2114   // Anything will hit the host prefix.
2115   EXPECT_TRUE(database_->ContainsBrowseUrl(
2116       GURL("http://www.evil.com/malware.html"), &prefix_hits, &cache_hits));
2117   ASSERT_EQ(1U, prefix_hits.size());
2118   EXPECT_EQ(SBPrefixForString("www.evil.com/"), prefix_hits[0]);
2119   EXPECT_TRUE(cache_hits.empty());
2120
2121   // Hit the specific URL prefix.
2122   EXPECT_TRUE(database_->ContainsBrowseUrl(
2123       GURL(std::string("http://") + kWhateverMalware),
2124       &prefix_hits, &cache_hits));
2125   ASSERT_EQ(1U, prefix_hits.size());
2126   EXPECT_EQ(SBPrefixForString(kWhateverMalware), prefix_hits[0]);
2127   EXPECT_TRUE(cache_hits.empty());
2128
2129   // Other URLs at that host are fine.
2130   EXPECT_FALSE(database_->ContainsBrowseUrl(
2131       GURL("http://www.whatever.com/fine.html"), &prefix_hits, &cache_hits));
2132   EXPECT_TRUE(prefix_hits.empty());
2133   EXPECT_TRUE(cache_hits.empty());
2134
2135   // Hit the specific URL full hash.
2136   EXPECT_TRUE(database_->ContainsBrowseUrl(
2137       GURL(std::string("http://") + kExampleCollision),
2138       &prefix_hits, &cache_hits));
2139   ASSERT_EQ(1U, prefix_hits.size());
2140   EXPECT_EQ(SBPrefixForString(kExampleCollision), prefix_hits[0]);
2141   EXPECT_TRUE(cache_hits.empty());
2142
2143   // This prefix collides, but no full hash match.
2144   EXPECT_FALSE(database_->ContainsBrowseUrl(
2145       GURL(std::string("http://") + kExampleFine), &prefix_hits, &cache_hits));
2146 }