src/chrome/browser/safe_browsing/database_manager.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/safe_browsing/database_manager.h"
   6
   7 #include <algorithm>
   8
   9 #include "base/bind.h"
  10 #include "base/bind_helpers.h"
  11 #include "base/callback.h"
  12 #include "base/command_line.h"
  13 #include "base/debug/leak_tracker.h"
  14 #include "base/path_service.h"
  15 #include "base/stl_util.h"
  16 #include "base/strings/string_util.h"
  17 #include "base/threading/thread.h"
  18 #include "base/threading/thread_restrictions.h"
  19 #include "chrome/browser/browser_process.h"
  20 #include "chrome/browser/chrome_notification_types.h"
  21 #include "chrome/browser/prerender/prerender_field_trial.h"
  22 #include "chrome/browser/safe_browsing/client_side_detection_service.h"
  23 #include "chrome/browser/safe_browsing/download_protection_service.h"
  24 #include "chrome/browser/safe_browsing/malware_details.h"
  25 #include "chrome/browser/safe_browsing/protocol_manager.h"
  26 #include "chrome/browser/safe_browsing/safe_browsing_database.h"
  27 #include "chrome/browser/safe_browsing/safe_browsing_service.h"
  28 #include "chrome/browser/safe_browsing/ui_manager.h"
  29 #include "chrome/common/chrome_constants.h"
  30 #include "chrome/common/chrome_paths.h"
  31 #include "chrome/common/chrome_switches.h"
  32 #include "components/metrics/metrics_service.h"
  33 #include "components/startup_metric_utils/startup_metric_utils.h"
  34 #include "content/public/browser/browser_thread.h"
  35 #include "content/public/browser/notification_service.h"
  36 #include "url/url_constants.h"
  37
  38 using content::BrowserThread;
  39
  40 namespace {
  41
  42 // Timeout for match checks, e.g. download URLs, hashes.
  43 const int kCheckTimeoutMs = 10000;
  44
  45 // Records disposition information about the check.  |hit| should be
  46 // |true| if there were any prefix hits in |full_hashes|.
  47 void RecordGetHashCheckStatus(
  48     bool hit,
  49     safe_browsing_util::ListType check_type,
  50     const std::vector<SBFullHashResult>& full_hashes) {
  51   SafeBrowsingProtocolManager::ResultType result;
  52   if (full_hashes.empty()) {
  53     result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_EMPTY;
  54   } else if (hit) {
  55     result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_HIT;
  56   } else {
  57     result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_MISS;
  58   }
  59   bool is_download = check_type == safe_browsing_util::BINURL;
  60   SafeBrowsingProtocolManager::RecordGetHashResult(is_download, result);
  61 }
  62
  63 bool IsExpectedThreat(
  64     const SBThreatType threat_type,
  65     const std::vector<SBThreatType>& expected_threats) {
  66   return expected_threats.end() != std::find(expected_threats.begin(),
  67                                              expected_threats.end(),
  68                                              threat_type);
  69 }
  70
  71 // Return the list id from the first result in |full_hashes| which matches
  72 // |hash|, or INVALID if none match.
  73 safe_browsing_util::ListType GetHashThreatListType(
  74     const SBFullHash& hash,
  75     const std::vector<SBFullHashResult>& full_hashes,
  76     size_t* index) {
  77   for (size_t i = 0; i < full_hashes.size(); ++i) {
  78     if (SBFullHashEqual(hash, full_hashes[i].hash)) {
  79       if (index)
  80         *index = i;
  81       return static_cast<safe_browsing_util::ListType>(full_hashes[i].list_id);
  82     }
  83   }
  84   return safe_browsing_util::INVALID;
  85 }
  86
  87 // Given a URL, compare all the possible host + path full hashes to the set of
  88 // provided full hashes.  Returns the list id of the a matching result from
  89 // |full_hashes|, or INVALID if none match.
  90 safe_browsing_util::ListType GetUrlThreatListType(
  91     const GURL& url,
  92     const std::vector<SBFullHashResult>& full_hashes,
  93     size_t* index) {
  94   if (full_hashes.empty())
  95     return safe_browsing_util::INVALID;
  96
  97   std::vector<std::string> patterns;
  98   safe_browsing_util::GeneratePatternsToCheck(url, &patterns);
  99
 100   for (size_t i = 0; i < patterns.size(); ++i) {
 101     safe_browsing_util::ListType threat = GetHashThreatListType(
 102         SBFullHashForString(patterns[i]), full_hashes, index);
 103     if (threat != safe_browsing_util::INVALID)
 104       return threat;
 105   }
 106   return safe_browsing_util::INVALID;
 107 }
 108
 109 SBThreatType GetThreatTypeFromListType(safe_browsing_util::ListType list_type) {
 110   switch (list_type) {
 111     case safe_browsing_util::PHISH:
 112       return SB_THREAT_TYPE_URL_PHISHING;
 113     case safe_browsing_util::MALWARE:
 114       return SB_THREAT_TYPE_URL_MALWARE;
 115     case safe_browsing_util::BINURL:
 116       return SB_THREAT_TYPE_BINARY_MALWARE_URL;
 117     case safe_browsing_util::EXTENSIONBLACKLIST:
 118       return SB_THREAT_TYPE_EXTENSION;
 119     default:
 120       DVLOG(1) << "Unknown safe browsing list id " << list_type;
 121       return SB_THREAT_TYPE_SAFE;
 122   }
 123 }
 124
 125 }  // namespace
 126
 127 // static
 128 SBThreatType SafeBrowsingDatabaseManager::GetHashThreatType(
 129     const SBFullHash& hash,
 130     const std::vector<SBFullHashResult>& full_hashes) {
 131   return GetThreatTypeFromListType(
 132       GetHashThreatListType(hash, full_hashes, NULL));
 133 }
 134
 135 // static
 136 SBThreatType SafeBrowsingDatabaseManager::GetUrlThreatType(
 137     const GURL& url,
 138     const std::vector<SBFullHashResult>& full_hashes,
 139     size_t* index) {
 140   return GetThreatTypeFromListType(
 141       GetUrlThreatListType(url, full_hashes, index));
 142 }
 143
 144 SafeBrowsingDatabaseManager::SafeBrowsingCheck::SafeBrowsingCheck(
 145     const std::vector<GURL>& urls,
 146     const std::vector<SBFullHash>& full_hashes,
 147     Client* client,
 148     safe_browsing_util::ListType check_type,
 149     const std::vector<SBThreatType>& expected_threats)
 150     : urls(urls),
 151       url_results(urls.size(), SB_THREAT_TYPE_SAFE),
 152       url_metadata(urls.size()),
 153       full_hashes(full_hashes),
 154       full_hash_results(full_hashes.size(), SB_THREAT_TYPE_SAFE),
 155       client(client),
 156       need_get_hash(false),
 157       check_type(check_type),
 158       expected_threats(expected_threats) {
 159   DCHECK_EQ(urls.empty(), !full_hashes.empty())
 160       << "Exactly one of urls and full_hashes must be set";
 161 }
 162
 163 SafeBrowsingDatabaseManager::SafeBrowsingCheck::~SafeBrowsingCheck() {}
 164
 165 void SafeBrowsingDatabaseManager::Client::OnSafeBrowsingResult(
 166     const SafeBrowsingCheck& check) {
 167   DCHECK_EQ(check.urls.size(), check.url_results.size());
 168   DCHECK_EQ(check.full_hashes.size(), check.full_hash_results.size());
 169   if (!check.urls.empty()) {
 170     DCHECK(check.full_hashes.empty());
 171     switch (check.check_type) {
 172       case safe_browsing_util::MALWARE:
 173       case safe_browsing_util::PHISH:
 174         DCHECK_EQ(1u, check.urls.size());
 175         OnCheckBrowseUrlResult(
 176             check.urls[0], check.url_results[0], check.url_metadata[0]);
 177         break;
 178       case safe_browsing_util::BINURL:
 179         DCHECK_EQ(check.urls.size(), check.url_results.size());
 180         OnCheckDownloadUrlResult(
 181             check.urls,
 182             *std::max_element(check.url_results.begin(),
 183                               check.url_results.end()));
 184         break;
 185       default:
 186         NOTREACHED();
 187     }
 188   } else if (!check.full_hashes.empty()) {
 189     switch (check.check_type) {
 190       case safe_browsing_util::EXTENSIONBLACKLIST: {
 191         std::set<std::string> unsafe_extension_ids;
 192         for (size_t i = 0; i < check.full_hashes.size(); ++i) {
 193           std::string extension_id =
 194               safe_browsing_util::SBFullHashToString(check.full_hashes[i]);
 195           if (check.full_hash_results[i] == SB_THREAT_TYPE_EXTENSION)
 196             unsafe_extension_ids.insert(extension_id);
 197         }
 198         OnCheckExtensionsResult(unsafe_extension_ids);
 199         break;
 200       }
 201       default:
 202         NOTREACHED();
 203     }
 204   } else {
 205     NOTREACHED();
 206   }
 207 }
 208
 209 SafeBrowsingDatabaseManager::SafeBrowsingDatabaseManager(
 210     const scoped_refptr<SafeBrowsingService>& service)
 211     : sb_service_(service),
 212       database_(NULL),
 213       enabled_(false),
 214       enable_download_protection_(false),
 215       enable_csd_whitelist_(false),
 216       enable_download_whitelist_(false),
 217       enable_extension_blacklist_(false),
 218       enable_side_effect_free_whitelist_(false),
 219       enable_ip_blacklist_(false),
 220       update_in_progress_(false),
 221       database_update_in_progress_(false),
 222       closing_database_(false),
 223       check_timeout_(base::TimeDelta::FromMilliseconds(kCheckTimeoutMs)) {
 224   DCHECK(sb_service_.get() != NULL);
 225
 226   // Android only supports a subset of FULL_SAFE_BROWSING.
 227   // TODO(shess): This shouldn't be OS-driven <http://crbug.com/394379>
 228 #if !defined(OS_ANDROID)
 229   CommandLine* cmdline = CommandLine::ForCurrentProcess();
 230   enable_download_protection_ =
 231       !cmdline->HasSwitch(switches::kSbDisableDownloadProtection);
 232
 233   // We only download the csd-whitelist if client-side phishing detection is
 234   // enabled.
 235   enable_csd_whitelist_ =
 236       !cmdline->HasSwitch(switches::kDisableClientSidePhishingDetection);
 237
 238   // TODO(noelutz): remove this boolean variable since it should always be true
 239   // if SafeBrowsing is enabled.  Unfortunately, we have no test data for this
 240   // list right now.  This means that we need to be able to disable this list
 241   // for the SafeBrowsing test to pass.
 242   enable_download_whitelist_ = enable_csd_whitelist_;
 243
 244   // TODO(kalman): there really shouldn't be a flag for this.
 245   enable_extension_blacklist_ =
 246       !cmdline->HasSwitch(switches::kSbDisableExtensionBlacklist);
 247
 248   enable_side_effect_free_whitelist_ =
 249       prerender::IsSideEffectFreeWhitelistEnabled() &&
 250       !cmdline->HasSwitch(switches::kSbDisableSideEffectFreeWhitelist);
 251
 252   // The client-side IP blacklist feature is tightly integrated with client-side
 253   // phishing protection for now.
 254   enable_ip_blacklist_ = enable_csd_whitelist_;
 255
 256   enum SideEffectFreeWhitelistStatus {
 257     SIDE_EFFECT_FREE_WHITELIST_ENABLED,
 258     SIDE_EFFECT_FREE_WHITELIST_DISABLED,
 259     SIDE_EFFECT_FREE_WHITELIST_STATUS_MAX
 260   };
 261
 262   SideEffectFreeWhitelistStatus side_effect_free_whitelist_status =
 263       enable_side_effect_free_whitelist_ ? SIDE_EFFECT_FREE_WHITELIST_ENABLED :
 264       SIDE_EFFECT_FREE_WHITELIST_DISABLED;
 265
 266   UMA_HISTOGRAM_ENUMERATION("SB2.SideEffectFreeWhitelistStatus",
 267                             side_effect_free_whitelist_status,
 268                             SIDE_EFFECT_FREE_WHITELIST_STATUS_MAX);
 269 #endif
 270 }
 271
 272 SafeBrowsingDatabaseManager::~SafeBrowsingDatabaseManager() {
 273   // We should have already been shut down. If we're still enabled, then the
 274   // database isn't going to be closed properly, which could lead to corruption.
 275   DCHECK(!enabled_);
 276 }
 277
 278 bool SafeBrowsingDatabaseManager::CanCheckUrl(const GURL& url) const {
 279   return url.SchemeIs(url::kFtpScheme) ||
 280          url.SchemeIs(url::kHttpScheme) ||
 281          url.SchemeIs(url::kHttpsScheme);
 282 }
 283
 284 bool SafeBrowsingDatabaseManager::CheckDownloadUrl(
 285     const std::vector<GURL>& url_chain,
 286     Client* client) {
 287   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 288   if (!enabled_ || !enable_download_protection_)
 289     return true;
 290
 291   // We need to check the database for url prefix, and later may fetch the url
 292   // from the safebrowsing backends. These need to be asynchronous.
 293   SafeBrowsingCheck* check =
 294       new SafeBrowsingCheck(url_chain,
 295                             std::vector<SBFullHash>(),
 296                             client,
 297                             safe_browsing_util::BINURL,
 298                             std::vector<SBThreatType>(1,
 299                                 SB_THREAT_TYPE_BINARY_MALWARE_URL));
 300   StartSafeBrowsingCheck(
 301       check,
 302       base::Bind(&SafeBrowsingDatabaseManager::CheckDownloadUrlOnSBThread, this,
 303                  check));
 304   return false;
 305 }
 306
 307 bool SafeBrowsingDatabaseManager::CheckExtensionIDs(
 308     const std::set<std::string>& extension_ids,
 309     Client* client) {
 310   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 311
 312   if (!enabled_ || !enable_extension_blacklist_)
 313     return true;
 314
 315   std::vector<SBFullHash> extension_id_hashes;
 316   std::transform(extension_ids.begin(), extension_ids.end(),
 317                  std::back_inserter(extension_id_hashes),
 318                  safe_browsing_util::StringToSBFullHash);
 319
 320   SafeBrowsingCheck* check = new SafeBrowsingCheck(
 321       std::vector<GURL>(),
 322       extension_id_hashes,
 323       client,
 324       safe_browsing_util::EXTENSIONBLACKLIST,
 325       std::vector<SBThreatType>(1, SB_THREAT_TYPE_EXTENSION));
 326
 327   StartSafeBrowsingCheck(
 328       check,
 329       base::Bind(&SafeBrowsingDatabaseManager::CheckExtensionIDsOnSBThread,
 330                  this,
 331                  check));
 332   return false;
 333 }
 334
 335 bool SafeBrowsingDatabaseManager::CheckSideEffectFreeWhitelistUrl(
 336     const GURL& url) {
 337   if (!enabled_)
 338     return false;
 339
 340   if (!CanCheckUrl(url))
 341     return false;
 342
 343   return database_->ContainsSideEffectFreeWhitelistUrl(url);
 344 }
 345
 346 bool SafeBrowsingDatabaseManager::MatchMalwareIP(
 347     const std::string& ip_address) {
 348   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 349   if (!enabled_ || !enable_ip_blacklist_ || !MakeDatabaseAvailable()) {
 350     return false;  // Fail open.
 351   }
 352   return database_->ContainsMalwareIP(ip_address);
 353 }
 354
 355 bool SafeBrowsingDatabaseManager::MatchCsdWhitelistUrl(const GURL& url) {
 356   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 357   if (!enabled_ || !enable_csd_whitelist_ || !MakeDatabaseAvailable()) {
 358     // There is something funky going on here -- for example, perhaps the user
 359     // has not restarted since enabling metrics reporting, so we haven't
 360     // enabled the csd whitelist yet.  Just to be safe we return true in this
 361     // case.
 362     return true;
 363   }
 364   return database_->ContainsCsdWhitelistedUrl(url);
 365 }
 366
 367 bool SafeBrowsingDatabaseManager::MatchDownloadWhitelistUrl(const GURL& url) {
 368   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 369   if (!enabled_ || !enable_download_whitelist_ || !MakeDatabaseAvailable()) {
 370     return true;
 371   }
 372   return database_->ContainsDownloadWhitelistedUrl(url);
 373 }
 374
 375 bool SafeBrowsingDatabaseManager::MatchDownloadWhitelistString(
 376     const std::string& str) {
 377   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 378   if (!enabled_ || !enable_download_whitelist_ || !MakeDatabaseAvailable()) {
 379     return true;
 380   }
 381   return database_->ContainsDownloadWhitelistedString(str);
 382 }
 383
 384 bool SafeBrowsingDatabaseManager::IsMalwareKillSwitchOn() {
 385   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 386   if (!enabled_ || !MakeDatabaseAvailable()) {
 387     return true;
 388   }
 389   return database_->IsMalwareIPMatchKillSwitchOn();
 390 }
 391
 392 bool SafeBrowsingDatabaseManager::IsCsdWhitelistKillSwitchOn() {
 393   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 394   if (!enabled_ || !MakeDatabaseAvailable()) {
 395     return true;
 396   }
 397   return database_->IsCsdWhitelistKillSwitchOn();
 398 }
 399
 400 bool SafeBrowsingDatabaseManager::CheckBrowseUrl(const GURL& url,
 401                                                  Client* client) {
 402   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 403   if (!enabled_)
 404     return true;
 405
 406   if (!CanCheckUrl(url))
 407     return true;
 408
 409   std::vector<SBThreatType> expected_threats;
 410   expected_threats.push_back(SB_THREAT_TYPE_URL_MALWARE);
 411   expected_threats.push_back(SB_THREAT_TYPE_URL_PHISHING);
 412
 413   const base::TimeTicks start = base::TimeTicks::Now();
 414   if (!MakeDatabaseAvailable()) {
 415     QueuedCheck queued_check(safe_browsing_util::MALWARE,  // or PHISH
 416                              client,
 417                              url,
 418                              expected_threats,
 419                              start);
 420     queued_checks_.push_back(queued_check);
 421     return false;
 422   }
 423
 424   std::vector<SBPrefix> prefix_hits;
 425   std::vector<SBFullHashResult> cache_hits;
 426
 427   bool prefix_match =
 428       database_->ContainsBrowseUrl(url, &prefix_hits, &cache_hits);
 429
 430   UMA_HISTOGRAM_TIMES("SB2.FilterCheck", base::TimeTicks::Now() - start);
 431
 432   if (!prefix_match)
 433     return true;  // URL is okay.
 434
 435   // Needs to be asynchronous, since we could be in the constructor of a
 436   // ResourceDispatcherHost event handler which can't pause there.
 437   SafeBrowsingCheck* check = new SafeBrowsingCheck(std::vector<GURL>(1, url),
 438                                                    std::vector<SBFullHash>(),
 439                                                    client,
 440                                                    safe_browsing_util::MALWARE,
 441                                                    expected_threats);
 442   check->need_get_hash = cache_hits.empty();
 443   check->prefix_hits.swap(prefix_hits);
 444   check->cache_hits.swap(cache_hits);
 445   checks_.insert(check);
 446
 447   BrowserThread::PostTask(
 448       BrowserThread::IO, FROM_HERE,
 449       base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone, this, check));
 450
 451   return false;
 452 }
 453
 454 void SafeBrowsingDatabaseManager::CancelCheck(Client* client) {
 455   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 456   for (CurrentChecks::iterator i = checks_.begin(); i != checks_.end(); ++i) {
 457     // We can't delete matching checks here because the db thread has a copy of
 458     // the pointer.  Instead, we simply NULL out the client, and when the db
 459     // thread calls us back, we'll clean up the check.
 460     if ((*i)->client == client)
 461       (*i)->client = NULL;
 462   }
 463
 464   // Scan the queued clients store. Clients may be here if they requested a URL
 465   // check before the database has finished loading.
 466   for (std::deque<QueuedCheck>::iterator it(queued_checks_.begin());
 467        it != queued_checks_.end(); ) {
 468     // In this case it's safe to delete matches entirely since nothing has a
 469     // pointer to them.
 470     if (it->client == client)
 471       it = queued_checks_.erase(it);
 472     else
 473       ++it;
 474   }
 475 }
 476
 477 void SafeBrowsingDatabaseManager::HandleGetHashResults(
 478     SafeBrowsingCheck* check,
 479     const std::vector<SBFullHashResult>& full_hashes,
 480     const base::TimeDelta& cache_lifetime) {
 481   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 482
 483   if (!enabled_)
 484     return;
 485
 486   // If the service has been shut down, |check| should have been deleted.
 487   DCHECK(checks_.find(check) != checks_.end());
 488
 489   // |start| is set before calling |GetFullHash()|, which should be
 490   // the only path which gets to here.
 491   DCHECK(!check->start.is_null());
 492   UMA_HISTOGRAM_LONG_TIMES("SB2.Network",
 493                            base::TimeTicks::Now() - check->start);
 494
 495   std::vector<SBPrefix> prefixes = check->prefix_hits;
 496   OnHandleGetHashResults(check, full_hashes);  // 'check' is deleted here.
 497
 498   // Cache the GetHash results.
 499   if (cache_lifetime != base::TimeDelta() && MakeDatabaseAvailable())
 500     database_->CacheHashResults(prefixes, full_hashes, cache_lifetime);
 501 }
 502
 503 void SafeBrowsingDatabaseManager::GetChunks(GetChunksCallback callback) {
 504   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 505   DCHECK(enabled_);
 506   DCHECK(!callback.is_null());
 507   safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind(
 508       &SafeBrowsingDatabaseManager::GetAllChunksFromDatabase, this, callback));
 509 }
 510
 511 void SafeBrowsingDatabaseManager::AddChunks(
 512     const std::string& list,
 513     scoped_ptr<ScopedVector<SBChunkData> > chunks,
 514     AddChunksCallback callback) {
 515   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 516   DCHECK(enabled_);
 517   DCHECK(!callback.is_null());
 518   safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind(
 519       &SafeBrowsingDatabaseManager::AddDatabaseChunks, this, list,
 520       base::Passed(&chunks), callback));
 521 }
 522
 523 void SafeBrowsingDatabaseManager::DeleteChunks(
 524     scoped_ptr<std::vector<SBChunkDelete> > chunk_deletes) {
 525   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 526   DCHECK(enabled_);
 527   safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind(
 528       &SafeBrowsingDatabaseManager::DeleteDatabaseChunks, this,
 529       base::Passed(&chunk_deletes)));
 530 }
 531
 532 void SafeBrowsingDatabaseManager::UpdateStarted() {
 533   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 534   DCHECK(enabled_);
 535   DCHECK(!update_in_progress_);
 536   update_in_progress_ = true;
 537 }
 538
 539 void SafeBrowsingDatabaseManager::UpdateFinished(bool update_succeeded) {
 540   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 541   DCHECK(enabled_);
 542   if (update_in_progress_) {
 543     update_in_progress_ = false;
 544     safe_browsing_thread_->message_loop()->PostTask(FROM_HERE,
 545       base::Bind(&SafeBrowsingDatabaseManager::DatabaseUpdateFinished,
 546                  this, update_succeeded));
 547   }
 548 }
 549
 550 void SafeBrowsingDatabaseManager::ResetDatabase() {
 551   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 552   DCHECK(enabled_);
 553   safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind(
 554       &SafeBrowsingDatabaseManager::OnResetDatabase, this));
 555 }
 556
 557 void SafeBrowsingDatabaseManager::LogPauseDelay(base::TimeDelta time) {
 558   UMA_HISTOGRAM_LONG_TIMES("SB2.Delay", time);
 559 }
 560
 561 void SafeBrowsingDatabaseManager::StartOnIOThread() {
 562   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 563   if (enabled_)
 564     return;
 565
 566   DCHECK(!safe_browsing_thread_.get());
 567   safe_browsing_thread_.reset(new base::Thread("Chrome_SafeBrowsingThread"));
 568   if (!safe_browsing_thread_->Start())
 569     return;
 570   enabled_ = true;
 571
 572   MakeDatabaseAvailable();
 573 }
 574
 575 void SafeBrowsingDatabaseManager::StopOnIOThread(bool shutdown) {
 576   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 577
 578   DoStopOnIOThread();
 579   if (shutdown) {
 580     sb_service_ = NULL;
 581   }
 582 }
 583
 584 void SafeBrowsingDatabaseManager::NotifyDatabaseUpdateFinished(
 585     bool update_succeeded) {
 586   content::NotificationService::current()->Notify(
 587       chrome::NOTIFICATION_SAFE_BROWSING_UPDATE_COMPLETE,
 588       content::Source<SafeBrowsingDatabaseManager>(this),
 589       content::Details<bool>(&update_succeeded));
 590 }
 591
 592 SafeBrowsingDatabaseManager::QueuedCheck::QueuedCheck(
 593     const safe_browsing_util::ListType check_type,
 594     Client* client,
 595     const GURL& url,
 596     const std::vector<SBThreatType>& expected_threats,
 597     const base::TimeTicks& start)
 598     : check_type(check_type),
 599       client(client),
 600       url(url),
 601       expected_threats(expected_threats),
 602       start(start) {
 603 }
 604
 605 SafeBrowsingDatabaseManager::QueuedCheck::~QueuedCheck() {
 606 }
 607
 608 void SafeBrowsingDatabaseManager::DoStopOnIOThread() {
 609   if (!enabled_)
 610     return;
 611
 612   enabled_ = false;
 613
 614   // Delete queued checks, calling back any clients with 'SB_THREAT_TYPE_SAFE'.
 615   while (!queued_checks_.empty()) {
 616     QueuedCheck queued = queued_checks_.front();
 617     if (queued.client) {
 618       SafeBrowsingCheck sb_check(std::vector<GURL>(1, queued.url),
 619                                  std::vector<SBFullHash>(),
 620                                  queued.client,
 621                                  queued.check_type,
 622                                  queued.expected_threats);
 623       queued.client->OnSafeBrowsingResult(sb_check);
 624     }
 625     queued_checks_.pop_front();
 626   }
 627
 628   // Close the database.  Cases to avoid:
 629   //  * If |closing_database_| is true, continuing will queue up a second
 630   //    request, |closing_database_| will be reset after handling the first
 631   //    request, and if any functions on the db thread recreate the database, we
 632   //    could start using it on the IO thread and then have the second request
 633   //    handler delete it out from under us.
 634   //  * If |database_| is NULL, then either no creation request is in flight, in
 635   //    which case we don't need to do anything, or one is in flight, in which
 636   //    case the database will be recreated before our deletion request is
 637   //    handled, and could be used on the IO thread in that time period, leading
 638   //    to the same problem as above.
 639   // Checking DatabaseAvailable() avoids both of these.
 640   if (DatabaseAvailable()) {
 641     closing_database_ = true;
 642     safe_browsing_thread_->message_loop()->PostTask(FROM_HERE,
 643         base::Bind(&SafeBrowsingDatabaseManager::OnCloseDatabase, this));
 644   }
 645
 646   // Flush the database thread. Any in-progress database check results will be
 647   // ignored and cleaned up below.
 648   //
 649   // Note that to avoid leaking the database, we rely on the fact that no new
 650   // tasks will be added to the db thread between the call above and this one.
 651   // See comments on the declaration of |safe_browsing_thread_|.
 652   {
 653     // A ScopedAllowIO object is required to join the thread when calling Stop.
 654     // See http://crbug.com/72696. Note that we call Stop() first to clear out
 655     // any remaining tasks before clearing safe_browsing_thread_.
 656     base::ThreadRestrictions::ScopedAllowIO allow_io_for_thread_join;
 657     safe_browsing_thread_->Stop();
 658     safe_browsing_thread_.reset();
 659   }
 660
 661   // Delete pending checks, calling back any clients with 'SB_THREAT_TYPE_SAFE'.
 662   // We have to do this after the db thread returns because methods on it can
 663   // have copies of these pointers, so deleting them might lead to accessing
 664   // garbage.
 665   for (CurrentChecks::iterator it = checks_.begin();
 666        it != checks_.end(); ++it) {
 667     SafeBrowsingCheck* check = *it;
 668     if (check->client)
 669       check->client->OnSafeBrowsingResult(*check);
 670   }
 671   STLDeleteElements(&checks_);
 672
 673   gethash_requests_.clear();
 674 }
 675
 676 bool SafeBrowsingDatabaseManager::DatabaseAvailable() const {
 677   base::AutoLock lock(database_lock_);
 678   return !closing_database_ && (database_ != NULL);
 679 }
 680
 681 bool SafeBrowsingDatabaseManager::MakeDatabaseAvailable() {
 682   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 683   DCHECK(enabled_);
 684   if (DatabaseAvailable())
 685     return true;
 686   safe_browsing_thread_->message_loop()->PostTask(
 687       FROM_HERE,
 688       base::Bind(base::IgnoreResult(&SafeBrowsingDatabaseManager::GetDatabase),
 689                  this));
 690   return false;
 691 }
 692
 693 SafeBrowsingDatabase* SafeBrowsingDatabaseManager::GetDatabase() {
 694   DCHECK_EQ(base::MessageLoop::current(),
 695             safe_browsing_thread_->message_loop());
 696   if (database_)
 697     return database_;
 698   startup_metric_utils::ScopedSlowStartupUMA
 699       scoped_timer("Startup.SlowStartupSafeBrowsingGetDatabase");
 700   const base::TimeTicks before = base::TimeTicks::Now();
 701
 702   SafeBrowsingDatabase* database =
 703       SafeBrowsingDatabase::Create(enable_download_protection_,
 704                                    enable_csd_whitelist_,
 705                                    enable_download_whitelist_,
 706                                    enable_extension_blacklist_,
 707                                    enable_side_effect_free_whitelist_,
 708                                    enable_ip_blacklist_);
 709
 710   database->Init(SafeBrowsingService::GetBaseFilename());
 711   {
 712     // Acquiring the lock here guarantees correct ordering between the writes to
 713     // the new database object above, and the setting of |databse_| below.
 714     base::AutoLock lock(database_lock_);
 715     database_ = database;
 716   }
 717
 718   BrowserThread::PostTask(
 719       BrowserThread::IO, FROM_HERE,
 720       base::Bind(&SafeBrowsingDatabaseManager::DatabaseLoadComplete, this));
 721
 722   UMA_HISTOGRAM_TIMES("SB2.DatabaseOpen", base::TimeTicks::Now() - before);
 723   return database_;
 724 }
 725
 726 void SafeBrowsingDatabaseManager::OnCheckDone(SafeBrowsingCheck* check) {
 727   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 728
 729   if (!enabled_)
 730     return;
 731
 732   // If the service has been shut down, |check| should have been deleted.
 733   DCHECK(checks_.find(check) != checks_.end());
 734
 735   if (check->client && check->need_get_hash) {
 736     // We have a partial match so we need to query Google for the full hash.
 737     // Clean up will happen in HandleGetHashResults.
 738
 739     // See if we have a GetHash request already in progress for this particular
 740     // prefix. If so, we just append ourselves to the list of interested parties
 741     // when the results arrive. We only do this for checks involving one prefix,
 742     // since that is the common case (multiple prefixes will issue the request
 743     // as normal).
 744     if (check->prefix_hits.size() == 1) {
 745       SBPrefix prefix = check->prefix_hits[0];
 746       GetHashRequests::iterator it = gethash_requests_.find(prefix);
 747       if (it != gethash_requests_.end()) {
 748         // There's already a request in progress.
 749         it->second.push_back(check);
 750         return;
 751       }
 752
 753       // No request in progress, so we're the first for this prefix.
 754       GetHashRequestors requestors;
 755       requestors.push_back(check);
 756       gethash_requests_[prefix] = requestors;
 757     }
 758
 759     // Reset the start time so that we can measure the network time without the
 760     // database time.
 761     check->start = base::TimeTicks::Now();
 762     // Note: If |this| is deleted or stopped, the protocol_manager will
 763     // be destroyed as well - hence it's OK to do unretained in this case.
 764     bool is_download = check->check_type == safe_browsing_util::BINURL;
 765     sb_service_->protocol_manager()->GetFullHash(
 766         check->prefix_hits,
 767         base::Bind(&SafeBrowsingDatabaseManager::HandleGetHashResults,
 768                    base::Unretained(this),
 769                    check),
 770         is_download);
 771   } else {
 772     // We may have cached results for previous GetHash queries.  Since
 773     // this data comes from cache, don't histogram hits.
 774     bool is_threat = HandleOneCheck(check, check->cache_hits);
 775     // cache_hits should only contain hits for a fullhash we searched for, so if
 776     // we got to this point it should always result in a threat match.
 777     DCHECK(is_threat);
 778   }
 779 }
 780
 781 void SafeBrowsingDatabaseManager::GetAllChunksFromDatabase(
 782     GetChunksCallback callback) {
 783   DCHECK_EQ(base::MessageLoop::current(),
 784             safe_browsing_thread_->message_loop());
 785
 786   bool database_error = true;
 787   std::vector<SBListChunkRanges> lists;
 788   DCHECK(!database_update_in_progress_);
 789   database_update_in_progress_ = true;
 790   GetDatabase();  // This guarantees that |database_| is non-NULL.
 791   if (database_->UpdateStarted(&lists)) {
 792     database_error = false;
 793   } else {
 794     database_->UpdateFinished(false);
 795   }
 796
 797   BrowserThread::PostTask(
 798       BrowserThread::IO, FROM_HERE,
 799       base::Bind(&SafeBrowsingDatabaseManager::OnGetAllChunksFromDatabase,
 800                  this, lists, database_error, callback));
 801 }
 802
 803 void SafeBrowsingDatabaseManager::OnGetAllChunksFromDatabase(
 804     const std::vector<SBListChunkRanges>& lists, bool database_error,
 805     GetChunksCallback callback) {
 806   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 807   if (enabled_)
 808     callback.Run(lists, database_error);
 809 }
 810
 811 void SafeBrowsingDatabaseManager::OnAddChunksComplete(
 812     AddChunksCallback callback) {
 813   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 814   if (enabled_)
 815     callback.Run();
 816 }
 817
 818 void SafeBrowsingDatabaseManager::DatabaseLoadComplete() {
 819   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 820   if (!enabled_)
 821     return;
 822
 823   LOCAL_HISTOGRAM_COUNTS("SB.QueueDepth", queued_checks_.size());
 824   if (queued_checks_.empty())
 825     return;
 826
 827   // If the database isn't already available, calling CheckUrl() in the loop
 828   // below will add the check back to the queue, and we'll infinite-loop.
 829   DCHECK(DatabaseAvailable());
 830   while (!queued_checks_.empty()) {
 831     QueuedCheck check = queued_checks_.front();
 832     DCHECK(!check.start.is_null());
 833     LOCAL_HISTOGRAM_TIMES("SB.QueueDelay",
 834                           base::TimeTicks::Now() - check.start);
 835     // If CheckUrl() determines the URL is safe immediately, it doesn't call the
 836     // client's handler function (because normally it's being directly called by
 837     // the client).  Since we're not the client, we have to convey this result.
 838     if (check.client && CheckBrowseUrl(check.url, check.client)) {
 839       SafeBrowsingCheck sb_check(std::vector<GURL>(1, check.url),
 840                                  std::vector<SBFullHash>(),
 841                                  check.client,
 842                                  check.check_type,
 843                                  check.expected_threats);
 844       check.client->OnSafeBrowsingResult(sb_check);
 845     }
 846     queued_checks_.pop_front();
 847   }
 848 }
 849
 850 void SafeBrowsingDatabaseManager::AddDatabaseChunks(
 851     const std::string& list_name,
 852     scoped_ptr<ScopedVector<SBChunkData> > chunks,
 853     AddChunksCallback callback) {
 854   DCHECK_EQ(base::MessageLoop::current(),
 855             safe_browsing_thread_->message_loop());
 856   if (chunks)
 857     GetDatabase()->InsertChunks(list_name, chunks->get());
 858   BrowserThread::PostTask(
 859       BrowserThread::IO, FROM_HERE,
 860       base::Bind(&SafeBrowsingDatabaseManager::OnAddChunksComplete, this,
 861                  callback));
 862 }
 863
 864 void SafeBrowsingDatabaseManager::DeleteDatabaseChunks(
 865     scoped_ptr<std::vector<SBChunkDelete> > chunk_deletes) {
 866   DCHECK_EQ(base::MessageLoop::current(),
 867             safe_browsing_thread_->message_loop());
 868   if (chunk_deletes)
 869     GetDatabase()->DeleteChunks(*chunk_deletes);
 870 }
 871
 872 void SafeBrowsingDatabaseManager::DatabaseUpdateFinished(
 873     bool update_succeeded) {
 874   DCHECK_EQ(base::MessageLoop::current(),
 875             safe_browsing_thread_->message_loop());
 876   GetDatabase()->UpdateFinished(update_succeeded);
 877   DCHECK(database_update_in_progress_);
 878   database_update_in_progress_ = false;
 879   BrowserThread::PostTask(
 880       BrowserThread::UI, FROM_HERE,
 881       base::Bind(&SafeBrowsingDatabaseManager::NotifyDatabaseUpdateFinished,
 882                  this, update_succeeded));
 883 }
 884
 885 void SafeBrowsingDatabaseManager::OnCloseDatabase() {
 886   DCHECK_EQ(base::MessageLoop::current(),
 887             safe_browsing_thread_->message_loop());
 888   DCHECK(closing_database_);
 889
 890   // Because |closing_database_| is true, nothing on the IO thread will be
 891   // accessing the database, so it's safe to delete and then NULL the pointer.
 892   delete database_;
 893   database_ = NULL;
 894
 895   // Acquiring the lock here guarantees correct ordering between the resetting
 896   // of |database_| above and of |closing_database_| below, which ensures there
 897   // won't be a window during which the IO thread falsely believes the database
 898   // is available.
 899   base::AutoLock lock(database_lock_);
 900   closing_database_ = false;
 901 }
 902
 903 void SafeBrowsingDatabaseManager::OnResetDatabase() {
 904   DCHECK_EQ(base::MessageLoop::current(),
 905             safe_browsing_thread_->message_loop());
 906   GetDatabase()->ResetDatabase();
 907 }
 908
 909 void SafeBrowsingDatabaseManager::OnHandleGetHashResults(
 910     SafeBrowsingCheck* check,
 911     const std::vector<SBFullHashResult>& full_hashes) {
 912   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 913   safe_browsing_util::ListType check_type = check->check_type;
 914   SBPrefix prefix = check->prefix_hits[0];
 915   GetHashRequests::iterator it = gethash_requests_.find(prefix);
 916   if (check->prefix_hits.size() > 1 || it == gethash_requests_.end()) {
 917     const bool hit = HandleOneCheck(check, full_hashes);
 918     RecordGetHashCheckStatus(hit, check_type, full_hashes);
 919     return;
 920   }
 921
 922   // Call back all interested parties, noting if any has a hit.
 923   GetHashRequestors& requestors = it->second;
 924   bool hit = false;
 925   for (GetHashRequestors::iterator r = requestors.begin();
 926        r != requestors.end(); ++r) {
 927     if (HandleOneCheck(*r, full_hashes))
 928       hit = true;
 929   }
 930   RecordGetHashCheckStatus(hit, check_type, full_hashes);
 931
 932   gethash_requests_.erase(it);
 933 }
 934
 935 bool SafeBrowsingDatabaseManager::HandleOneCheck(
 936     SafeBrowsingCheck* check,
 937     const std::vector<SBFullHashResult>& full_hashes) {
 938   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 939   DCHECK(check);
 940
 941   bool is_threat = false;
 942
 943   // TODO(shess): GetHashThreadListType() contains a loop,
 944   // GetUrlThreatListType() a loop around that loop.  Having another loop out
 945   // here concerns me.  It is likely that SAFE is an expected outcome, which
 946   // means all of those loops run to completion.  Refactoring this to generate a
 947   // set of sorted items to compare in sequence would probably improve things.
 948   //
 949   // Additionally, the set of patterns generated from the urls is very similar
 950   // to the patterns generated in ContainsBrowseUrl() and other database checks,
 951   // which are called from this code.  Refactoring that across the checks could
 952   // interact well with batching the checks here.
 953
 954   for (size_t i = 0; i < check->urls.size(); ++i) {
 955     size_t threat_index;
 956     SBThreatType threat =
 957         GetUrlThreatType(check->urls[i], full_hashes, &threat_index);
 958     if (threat != SB_THREAT_TYPE_SAFE &&
 959         IsExpectedThreat(threat, check->expected_threats)) {
 960       check->url_results[i] = threat;
 961       check->url_metadata[i] = full_hashes[threat_index].metadata;
 962       is_threat = true;
 963     }
 964   }
 965
 966   for (size_t i = 0; i < check->full_hashes.size(); ++i) {
 967     SBThreatType threat = GetHashThreatType(check->full_hashes[i], full_hashes);
 968     if (threat != SB_THREAT_TYPE_SAFE &&
 969         IsExpectedThreat(threat, check->expected_threats)) {
 970       check->full_hash_results[i] = threat;
 971       is_threat = true;
 972     }
 973   }
 974
 975   SafeBrowsingCheckDone(check);
 976   return is_threat;
 977 }
 978
 979 void SafeBrowsingDatabaseManager::CheckDownloadUrlOnSBThread(
 980     SafeBrowsingCheck* check) {
 981   DCHECK_EQ(base::MessageLoop::current(),
 982             safe_browsing_thread_->message_loop());
 983   DCHECK(enable_download_protection_);
 984
 985   std::vector<SBPrefix> prefix_hits;
 986
 987   if (!database_->ContainsDownloadUrl(check->urls, &prefix_hits)) {
 988     // Good, we don't have hash for this url prefix.
 989     BrowserThread::PostTask(
 990         BrowserThread::IO, FROM_HERE,
 991         base::Bind(&SafeBrowsingDatabaseManager::CheckDownloadUrlDone, this,
 992                    check));
 993     return;
 994   }
 995
 996   check->need_get_hash = true;
 997   check->prefix_hits.clear();
 998   check->prefix_hits = prefix_hits;
 999   BrowserThread::PostTask(
1000       BrowserThread::IO, FROM_HERE,
1001       base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone, this, check));
1002 }
1003
1004 void SafeBrowsingDatabaseManager::CheckExtensionIDsOnSBThread(
1005     SafeBrowsingCheck* check) {
1006   DCHECK_EQ(base::MessageLoop::current(),
1007             safe_browsing_thread_->message_loop());
1008
1009   std::vector<SBPrefix> prefixes;
1010   for (std::vector<SBFullHash>::iterator it = check->full_hashes.begin();
1011        it != check->full_hashes.end(); ++it) {
1012     prefixes.push_back((*it).prefix);
1013   }
1014   database_->ContainsExtensionPrefixes(prefixes, &check->prefix_hits);
1015
1016   if (check->prefix_hits.empty()) {
1017     // No matches for any extensions.
1018     BrowserThread::PostTask(
1019         BrowserThread::IO,
1020         FROM_HERE,
1021         base::Bind(&SafeBrowsingDatabaseManager::SafeBrowsingCheckDone, this,
1022                    check));
1023   } else {
1024     // Some prefixes matched, we need to ask Google whether they're legit.
1025     check->need_get_hash = true;
1026     BrowserThread::PostTask(
1027         BrowserThread::IO,
1028         FROM_HERE,
1029         base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone, this, check));
1030   }
1031 }
1032
1033 void SafeBrowsingDatabaseManager::TimeoutCallback(SafeBrowsingCheck* check) {
1034   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
1035   DCHECK(check);
1036
1037   if (!enabled_)
1038     return;
1039
1040   DCHECK(checks_.find(check) != checks_.end());
1041   if (check->client) {
1042     check->client->OnSafeBrowsingResult(*check);
1043     check->client = NULL;
1044   }
1045 }
1046
1047 void SafeBrowsingDatabaseManager::CheckDownloadUrlDone(
1048     SafeBrowsingCheck* check) {
1049   DCHECK(enable_download_protection_);
1050   SafeBrowsingCheckDone(check);
1051 }
1052
1053 void SafeBrowsingDatabaseManager::SafeBrowsingCheckDone(
1054     SafeBrowsingCheck* check) {
1055   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
1056   DCHECK(check);
1057
1058   if (!enabled_)
1059     return;
1060
1061   VLOG(1) << "SafeBrowsingCheckDone";
1062   DCHECK(checks_.find(check) != checks_.end());
1063   if (check->client)
1064     check->client->OnSafeBrowsingResult(*check);
1065   checks_.erase(check);
1066   delete check;
1067 }
1068
1069 void SafeBrowsingDatabaseManager::StartSafeBrowsingCheck(
1070     SafeBrowsingCheck* check,
1071     const base::Closure& task) {
1072   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
1073   check->timeout_factory_.reset(
1074       new base::WeakPtrFactory<SafeBrowsingDatabaseManager>(this));
1075   checks_.insert(check);
1076
1077   safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, task);
1078
1079   base::MessageLoop::current()->PostDelayedTask(FROM_HERE,
1080       base::Bind(&SafeBrowsingDatabaseManager::TimeoutCallback,
1081                  check->timeout_factory_->GetWeakPtr(), check),
1082       check_timeout_);
1083 }