src/chrome/browser/history/top_sites_impl.cc

   1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/history/top_sites_impl.h"
   6
   7 #include <algorithm>
   8 #include <set>
   9
  10 #include "base/bind.h"
  11 #include "base/bind_helpers.h"
  12 #include "base/logging.h"
  13 #include "base/md5.h"
  14 #include "base/memory/ref_counted_memory.h"
  15 #include "base/message_loop/message_loop_proxy.h"
  16 #include "base/metrics/histogram.h"
  17 #include "base/prefs/pref_service.h"
  18 #include "base/prefs/scoped_user_pref_update.h"
  19 #include "base/strings/string_util.h"
  20 #include "base/strings/utf_string_conversions.h"
  21 #include "base/task_runner.h"
  22 #include "base/values.h"
  23 #include "chrome/browser/chrome_notification_types.h"
  24 #include "chrome/browser/history/history_backend.h"
  25 #include "chrome/browser/history/history_db_task.h"
  26 #include "chrome/browser/history/history_notifications.h"
  27 #include "chrome/browser/history/history_service_factory.h"
  28 #include "chrome/browser/history/page_usage_data.h"
  29 #include "chrome/browser/history/top_sites_cache.h"
  30 #include "chrome/browser/history/url_utils.h"
  31 #include "chrome/browser/profiles/profile.h"
  32 #include "chrome/browser/ui/webui/ntp/most_visited_handler.h"
  33 #include "chrome/browser/ui/webui/ntp/new_tab_ui.h"
  34 #include "chrome/common/pref_names.h"
  35 #include "chrome/common/thumbnail_score.h"
  36 #include "content/public/browser/browser_thread.h"
  37 #include "content/public/browser/navigation_controller.h"
  38 #include "content/public/browser/navigation_details.h"
  39 #include "content/public/browser/navigation_entry.h"
  40 #include "content/public/browser/notification_service.h"
  41 #include "content/public/browser/web_contents.h"
  42 #include "grit/locale_settings.h"
  43 #include "ui/base/l10n/l10n_util.h"
  44 #include "ui/base/layout.h"
  45 #include "ui/base/resource/resource_bundle.h"
  46 #include "ui/gfx/image/image_util.h"
  47
  48 using base::DictionaryValue;
  49 using content::BrowserThread;
  50 using content::NavigationController;
  51
  52 namespace history {
  53
  54 namespace {
  55
  56 void RunOrPostGetMostVisitedURLsCallback(
  57     base::TaskRunner* task_runner,
  58     const TopSitesImpl::GetMostVisitedURLsCallback& callback,
  59     const MostVisitedURLList& urls) {
  60   if (task_runner->RunsTasksOnCurrentThread())
  61     callback.Run(urls);
  62   else
  63     task_runner->PostTask(FROM_HERE, base::Bind(callback, urls));
  64 }
  65
  66 }  // namespace
  67
  68 // How many top sites to store in the cache.
  69 static const size_t kTopSitesNumber = 20;
  70
  71 // Max number of temporary images we'll cache. See comment above
  72 // temp_images_ for details.
  73 static const size_t kMaxTempTopImages = 8;
  74
  75 static const int kDaysOfHistory = 90;
  76 // Time from startup to first HistoryService query.
  77 static const int64 kUpdateIntervalSecs = 15;
  78 // Intervals between requests to HistoryService.
  79 static const int64 kMinUpdateIntervalMinutes = 1;
  80 static const int64 kMaxUpdateIntervalMinutes = 60;
  81
  82 // Use 100 quality (highest quality) because we're very sensitive to
  83 // artifacts for these small sized, highly detailed images.
  84 static const int kTopSitesImageQuality = 100;
  85
  86 TopSitesImpl::TopSitesImpl(Profile* profile)
  87     : backend_(NULL),
  88       cache_(new TopSitesCache()),
  89       thread_safe_cache_(new TopSitesCache()),
  90       profile_(profile),
  91       last_num_urls_changed_(0),
  92       loaded_(false) {
  93   if (!profile_)
  94     return;
  95
  96   if (content::NotificationService::current()) {
  97     registrar_.Add(this, chrome::NOTIFICATION_HISTORY_URLS_DELETED,
  98                    content::Source<Profile>(profile_));
  99     // Listen for any nav commits. We'll ignore those not related to this
 100     // profile when we get the notification.
 101     registrar_.Add(this, content::NOTIFICATION_NAV_ENTRY_COMMITTED,
 102                    content::NotificationService::AllSources());
 103   }
 104   for (size_t i = 0; i < arraysize(kPrepopulatedPages); i++) {
 105     int url_id = kPrepopulatedPages[i].url_id;
 106     prepopulated_page_urls_.push_back(
 107         GURL(l10n_util::GetStringUTF8(url_id)));
 108   }
 109 }
 110
 111 void TopSitesImpl::Init(const base::FilePath& db_name) {
 112   // Create the backend here, rather than in the constructor, so that
 113   // unit tests that do not need the backend can run without a problem.
 114   backend_ = new TopSitesBackend;
 115   backend_->Init(db_name);
 116   backend_->GetMostVisitedThumbnails(
 117       base::Bind(&TopSitesImpl::OnGotMostVisitedThumbnails,
 118                  base::Unretained(this)),
 119       &cancelable_task_tracker_);
 120 }
 121
 122 bool TopSitesImpl::SetPageThumbnail(const GURL& url,
 123                                     const gfx::Image& thumbnail,
 124                                     const ThumbnailScore& score) {
 125   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 126
 127   if (!loaded_) {
 128     // TODO(sky): I need to cache these and apply them after the load
 129     // completes.
 130     return false;
 131   }
 132
 133   bool add_temp_thumbnail = false;
 134   if (!IsKnownURL(url)) {
 135     if (!IsFull()) {
 136       add_temp_thumbnail = true;
 137     } else {
 138       return false;  // This URL is not known to us.
 139     }
 140   }
 141
 142   if (!HistoryService::CanAddURL(url))
 143     return false;  // It's not a real webpage.
 144
 145   scoped_refptr<base::RefCountedBytes> thumbnail_data;
 146   if (!EncodeBitmap(thumbnail, &thumbnail_data))
 147     return false;
 148
 149   if (add_temp_thumbnail) {
 150     // Always remove the existing entry and then add it back. That way if we end
 151     // up with too many temp thumbnails we'll prune the oldest first.
 152     RemoveTemporaryThumbnailByURL(url);
 153     AddTemporaryThumbnail(url, thumbnail_data.get(), score);
 154     return true;
 155   }
 156
 157   return SetPageThumbnailEncoded(url, thumbnail_data.get(), score);
 158 }
 159
 160 bool TopSitesImpl::SetPageThumbnailToJPEGBytes(
 161     const GURL& url,
 162     const base::RefCountedMemory* memory,
 163     const ThumbnailScore& score) {
 164   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 165
 166   if (!loaded_) {
 167     // TODO(sky): I need to cache these and apply them after the load
 168     // completes.
 169     return false;
 170   }
 171
 172   bool add_temp_thumbnail = false;
 173   if (!IsKnownURL(url)) {
 174     if (!IsFull()) {
 175       add_temp_thumbnail = true;
 176     } else {
 177       return false;  // This URL is not known to us.
 178     }
 179   }
 180
 181   if (!HistoryService::CanAddURL(url))
 182     return false;  // It's not a real webpage.
 183
 184   if (add_temp_thumbnail) {
 185     // Always remove the existing entry and then add it back. That way if we end
 186     // up with too many temp thumbnails we'll prune the oldest first.
 187     RemoveTemporaryThumbnailByURL(url);
 188     AddTemporaryThumbnail(url, memory, score);
 189     return true;
 190   }
 191
 192   return SetPageThumbnailEncoded(url, memory, score);
 193 }
 194
 195 // WARNING: this function may be invoked on any thread.
 196 void TopSitesImpl::GetMostVisitedURLs(
 197     const GetMostVisitedURLsCallback& callback) {
 198   MostVisitedURLList filtered_urls;
 199   {
 200     base::AutoLock lock(lock_);
 201     if (!loaded_) {
 202       // A request came in before we finished loading. Store the callback and
 203       // we'll run it on current thread when we finish loading.
 204       pending_callbacks_.push_back(
 205           base::Bind(&RunOrPostGetMostVisitedURLsCallback,
 206                      base::MessageLoopProxy::current(),
 207                      callback));
 208       return;
 209     }
 210     filtered_urls = thread_safe_cache_->top_sites();
 211   }
 212   callback.Run(filtered_urls);
 213 }
 214
 215 bool TopSitesImpl::GetPageThumbnail(
 216     const GURL& url,
 217     bool prefix_match,
 218     scoped_refptr<base::RefCountedMemory>* bytes) {
 219   // WARNING: this may be invoked on any thread.
 220   // Perform exact match.
 221   {
 222     base::AutoLock lock(lock_);
 223     if (thread_safe_cache_->GetPageThumbnail(url, bytes))
 224       return true;
 225   }
 226
 227   // Resource bundle is thread safe.
 228   for (size_t i = 0; i < arraysize(kPrepopulatedPages); i++) {
 229     if (url == prepopulated_page_urls_[i]) {
 230       *bytes = ResourceBundle::GetSharedInstance().
 231           LoadDataResourceBytesForScale(
 232               kPrepopulatedPages[i].thumbnail_id,
 233               ui::SCALE_FACTOR_100P);
 234       return true;
 235     }
 236   }
 237
 238   if (prefix_match) {
 239     // If http or https, search with |url| first, then try the other one.
 240     std::vector<GURL> url_list;
 241     url_list.push_back(url);
 242     if (url.SchemeIsHTTPOrHTTPS())
 243       url_list.push_back(ToggleHTTPAndHTTPS(url));
 244
 245     for (std::vector<GURL>::iterator it = url_list.begin();
 246          it != url_list.end(); ++it) {
 247       base::AutoLock lock(lock_);
 248
 249       GURL canonical_url;
 250       // Test whether |url| is prefix of any stored URL.
 251       canonical_url = thread_safe_cache_->GetSpecializedCanonicalURL(*it);
 252       if (!canonical_url.is_empty() &&
 253           thread_safe_cache_->GetPageThumbnail(canonical_url, bytes)) {
 254         return true;
 255       }
 256
 257       // Test whether any stored URL is a prefix of |url|.
 258       canonical_url = thread_safe_cache_->GetGeneralizedCanonicalURL(*it);
 259       if (!canonical_url.is_empty() &&
 260           thread_safe_cache_->GetPageThumbnail(canonical_url, bytes)) {
 261         return true;
 262       }
 263     }
 264   }
 265
 266   return false;
 267 }
 268
 269 bool TopSitesImpl::GetPageThumbnailScore(const GURL& url,
 270                                          ThumbnailScore* score) {
 271   // WARNING: this may be invoked on any thread.
 272   base::AutoLock lock(lock_);
 273   return thread_safe_cache_->GetPageThumbnailScore(url, score);
 274 }
 275
 276 bool TopSitesImpl::GetTemporaryPageThumbnailScore(const GURL& url,
 277                                                   ThumbnailScore* score) {
 278   for (TempImages::iterator i = temp_images_.begin(); i != temp_images_.end();
 279        ++i) {
 280     if (i->first == url) {
 281       *score = i->second.thumbnail_score;
 282       return true;
 283     }
 284   }
 285   return false;
 286 }
 287
 288
 289 // Returns the index of |url| in |urls|, or -1 if not found.
 290 static int IndexOf(const MostVisitedURLList& urls, const GURL& url) {
 291   for (size_t i = 0; i < urls.size(); i++) {
 292     if (urls[i].url == url)
 293       return i;
 294   }
 295   return -1;
 296 }
 297
 298 void TopSitesImpl::SyncWithHistory() {
 299   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 300   if (loaded_ && temp_images_.size()) {
 301     // If we have temporary thumbnails it means there isn't much data, and most
 302     // likely the user is first running Chrome. During this time we throttle
 303     // updating from history by 30 seconds. If the user creates a new tab page
 304     // during this window of time we force updating from history so that the new
 305     // tab page isn't so far out of date.
 306     timer_.Stop();
 307     StartQueryForMostVisited();
 308   }
 309 }
 310
 311 bool TopSitesImpl::HasBlacklistedItems() const {
 312   const DictionaryValue* blacklist =
 313       profile_->GetPrefs()->GetDictionary(prefs::kNtpMostVisitedURLsBlacklist);
 314   return blacklist && !blacklist->empty();
 315 }
 316
 317 void TopSitesImpl::AddBlacklistedURL(const GURL& url) {
 318   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 319
 320   Value* dummy = Value::CreateNullValue();
 321   {
 322     DictionaryPrefUpdate update(profile_->GetPrefs(),
 323                                 prefs::kNtpMostVisitedURLsBlacklist);
 324     DictionaryValue* blacklist = update.Get();
 325     blacklist->SetWithoutPathExpansion(GetURLHash(url), dummy);
 326   }
 327
 328   ResetThreadSafeCache();
 329   NotifyTopSitesChanged();
 330 }
 331
 332 void TopSitesImpl::RemoveBlacklistedURL(const GURL& url) {
 333   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 334   {
 335     DictionaryPrefUpdate update(profile_->GetPrefs(),
 336                                 prefs::kNtpMostVisitedURLsBlacklist);
 337     DictionaryValue* blacklist = update.Get();
 338     blacklist->RemoveWithoutPathExpansion(GetURLHash(url), NULL);
 339   }
 340   ResetThreadSafeCache();
 341   NotifyTopSitesChanged();
 342 }
 343
 344 bool TopSitesImpl::IsBlacklisted(const GURL& url) {
 345   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 346   const DictionaryValue* blacklist =
 347       profile_->GetPrefs()->GetDictionary(prefs::kNtpMostVisitedURLsBlacklist);
 348   return blacklist && blacklist->HasKey(GetURLHash(url));
 349 }
 350
 351 void TopSitesImpl::ClearBlacklistedURLs() {
 352   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 353   {
 354     DictionaryPrefUpdate update(profile_->GetPrefs(),
 355                                 prefs::kNtpMostVisitedURLsBlacklist);
 356     DictionaryValue* blacklist = update.Get();
 357     blacklist->Clear();
 358   }
 359   ResetThreadSafeCache();
 360   NotifyTopSitesChanged();
 361 }
 362
 363 void TopSitesImpl::Shutdown() {
 364   profile_ = NULL;
 365   // Cancel all requests so that the service doesn't callback to us after we've
 366   // invoked Shutdown (this could happen if we have a pending request and
 367   // Shutdown is invoked).
 368   history_consumer_.CancelAllRequests();
 369   backend_->Shutdown();
 370 }
 371
 372 // static
 373 void TopSitesImpl::DiffMostVisited(const MostVisitedURLList& old_list,
 374                                    const MostVisitedURLList& new_list,
 375                                    TopSitesDelta* delta) {
 376   // Add all the old URLs for quick lookup. This maps URLs to the corresponding
 377   // index in the input.
 378   std::map<GURL, size_t> all_old_urls;
 379   for (size_t i = 0; i < old_list.size(); i++)
 380     all_old_urls[old_list[i].url] = i;
 381
 382   // Check all the URLs in the new set to see which ones are new or just moved.
 383   // When we find a match in the old set, we'll reset its index to our special
 384   // marker. This allows us to quickly identify the deleted ones in a later
 385   // pass.
 386   const size_t kAlreadyFoundMarker = static_cast<size_t>(-1);
 387   for (size_t i = 0; i < new_list.size(); i++) {
 388     std::map<GURL, size_t>::iterator found = all_old_urls.find(new_list[i].url);
 389     if (found == all_old_urls.end()) {
 390       MostVisitedURLWithRank added;
 391       added.url = new_list[i];
 392       added.rank = i;
 393       delta->added.push_back(added);
 394     } else {
 395       if (found->second != i) {
 396         MostVisitedURLWithRank moved;
 397         moved.url = new_list[i];
 398         moved.rank = i;
 399         delta->moved.push_back(moved);
 400       }
 401       found->second = kAlreadyFoundMarker;
 402     }
 403   }
 404
 405   // Any member without the special marker in the all_old_urls list means that
 406   // there wasn't a "new" URL that mapped to it, so it was deleted.
 407   for (std::map<GURL, size_t>::const_iterator i = all_old_urls.begin();
 408        i != all_old_urls.end(); ++i) {
 409     if (i->second != kAlreadyFoundMarker)
 410       delta->deleted.push_back(old_list[i->second]);
 411   }
 412 }
 413
 414 CancelableRequestProvider::Handle TopSitesImpl::StartQueryForMostVisited() {
 415   DCHECK(loaded_);
 416   if (!profile_)
 417     return 0;
 418
 419   HistoryService* hs = HistoryServiceFactory::GetForProfile(
 420       profile_, Profile::EXPLICIT_ACCESS);
 421   // |hs| may be null during unit tests.
 422   if (hs) {
 423     return hs->QueryMostVisitedURLs(
 424         num_results_to_request_from_history(),
 425         kDaysOfHistory,
 426         &history_consumer_,
 427         base::Bind(&TopSitesImpl::OnTopSitesAvailableFromHistory,
 428                    base::Unretained(this)));
 429   }
 430   return 0;
 431 }
 432
 433 bool TopSitesImpl::IsKnownURL(const GURL& url) {
 434   return loaded_ && cache_->IsKnownURL(url);
 435 }
 436
 437 const std::string& TopSitesImpl::GetCanonicalURLString(const GURL& url) const {
 438   return cache_->GetCanonicalURL(url).spec();
 439 }
 440
 441 bool TopSitesImpl::IsFull() {
 442   return loaded_ && cache_->top_sites().size() >= kTopSitesNumber;
 443 }
 444
 445 TopSitesImpl::~TopSitesImpl() {
 446 }
 447
 448 bool TopSitesImpl::SetPageThumbnailNoDB(
 449     const GURL& url,
 450     const base::RefCountedMemory* thumbnail_data,
 451     const ThumbnailScore& score) {
 452   // This should only be invoked when we know about the url.
 453   DCHECK(cache_->IsKnownURL(url));
 454
 455   const MostVisitedURL& most_visited =
 456       cache_->top_sites()[cache_->GetURLIndex(url)];
 457   Images* image = cache_->GetImage(url);
 458
 459   // When comparing the thumbnail scores, we need to take into account the
 460   // redirect hops, which are not generated when the thumbnail is because the
 461   // redirects weren't known. We fill that in here since we know the redirects.
 462   ThumbnailScore new_score_with_redirects(score);
 463   new_score_with_redirects.redirect_hops_from_dest =
 464       GetRedirectDistanceForURL(most_visited, url);
 465
 466   if (!ShouldReplaceThumbnailWith(image->thumbnail_score,
 467                                   new_score_with_redirects) &&
 468       image->thumbnail.get())
 469     return false;  // The one we already have is better.
 470
 471   image->thumbnail = const_cast<base::RefCountedMemory*>(thumbnail_data);
 472   image->thumbnail_score = new_score_with_redirects;
 473
 474   ResetThreadSafeImageCache();
 475   return true;
 476 }
 477
 478 bool TopSitesImpl::SetPageThumbnailEncoded(
 479     const GURL& url,
 480     const base::RefCountedMemory* thumbnail,
 481     const ThumbnailScore& score) {
 482   if (!SetPageThumbnailNoDB(url, thumbnail, score))
 483     return false;
 484
 485   // Update the database.
 486   if (!cache_->IsKnownURL(url))
 487     return false;
 488
 489   size_t index = cache_->GetURLIndex(url);
 490   const MostVisitedURL& most_visited = cache_->top_sites()[index];
 491   backend_->SetPageThumbnail(most_visited,
 492                              index,
 493                              *(cache_->GetImage(most_visited.url)));
 494   return true;
 495 }
 496
 497 // static
 498 bool TopSitesImpl::EncodeBitmap(const gfx::Image& bitmap,
 499                                 scoped_refptr<base::RefCountedBytes>* bytes) {
 500   if (bitmap.IsEmpty())
 501     return false;
 502   *bytes = new base::RefCountedBytes();
 503   std::vector<unsigned char> data;
 504   if (!gfx::JPEG1xEncodedDataFromImage(bitmap, kTopSitesImageQuality, &data))
 505     return false;
 506
 507   // As we're going to cache this data, make sure the vector is only as big as
 508   // it needs to be, as JPEGCodec::Encode() over-allocates data.capacity().
 509   // (In a C++0x future, we can just call shrink_to_fit() in Encode())
 510   (*bytes)->data() = data;
 511   return true;
 512 }
 513
 514 void TopSitesImpl::RemoveTemporaryThumbnailByURL(const GURL& url) {
 515   for (TempImages::iterator i = temp_images_.begin(); i != temp_images_.end();
 516        ++i) {
 517     if (i->first == url) {
 518       temp_images_.erase(i);
 519       return;
 520     }
 521   }
 522 }
 523
 524 void TopSitesImpl::AddTemporaryThumbnail(
 525     const GURL& url,
 526     const base::RefCountedMemory* thumbnail,
 527     const ThumbnailScore& score) {
 528   if (temp_images_.size() == kMaxTempTopImages)
 529     temp_images_.erase(temp_images_.begin());
 530
 531   TempImage image;
 532   image.first = url;
 533   image.second.thumbnail = const_cast<base::RefCountedMemory*>(thumbnail);
 534   image.second.thumbnail_score = score;
 535   temp_images_.push_back(image);
 536 }
 537
 538 void TopSitesImpl::TimerFired() {
 539   StartQueryForMostVisited();
 540 }
 541
 542 // static
 543 int TopSitesImpl::GetRedirectDistanceForURL(const MostVisitedURL& most_visited,
 544                                             const GURL& url) {
 545   for (size_t i = 0; i < most_visited.redirects.size(); i++) {
 546     if (most_visited.redirects[i] == url)
 547       return static_cast<int>(most_visited.redirects.size() - i - 1);
 548   }
 549   NOTREACHED() << "URL should always be found.";
 550   return 0;
 551 }
 552
 553 MostVisitedURLList TopSitesImpl::GetPrepopulatePages() {
 554   MostVisitedURLList urls;
 555   urls.resize(arraysize(kPrepopulatedPages));
 556   for (size_t i = 0; i < urls.size(); ++i) {
 557     MostVisitedURL& url = urls[i];
 558     url.url = GURL(prepopulated_page_urls_[i]);
 559     url.redirects.push_back(url.url);
 560     url.title = l10n_util::GetStringUTF16(kPrepopulatedPages[i].title_id);
 561   }
 562   return urls;
 563 }
 564
 565 bool TopSitesImpl::loaded() const {
 566   return loaded_;
 567 }
 568
 569 bool TopSitesImpl::AddPrepopulatedPages(MostVisitedURLList* urls) {
 570   bool added = false;
 571   MostVisitedURLList prepopulate_urls = GetPrepopulatePages();
 572   for (size_t i = 0; i < prepopulate_urls.size(); ++i) {
 573     if (urls->size() < kTopSitesNumber &&
 574         IndexOf(*urls, prepopulate_urls[i].url) == -1) {
 575       urls->push_back(prepopulate_urls[i]);
 576       added = true;
 577     }
 578   }
 579   return added;
 580 }
 581
 582 void TopSitesImpl::ApplyBlacklist(const MostVisitedURLList& urls,
 583                                   MostVisitedURLList* out) {
 584   // Log the number of times ApplyBlacklist is called so we can compute the
 585   // average number of blacklisted items per user.
 586   const DictionaryValue* blacklist =
 587       profile_->GetPrefs()->GetDictionary(prefs::kNtpMostVisitedURLsBlacklist);
 588   UMA_HISTOGRAM_BOOLEAN("TopSites.NumberOfApplyBlacklist", true);
 589   UMA_HISTOGRAM_COUNTS_100("TopSites.NumberOfBlacklistedItems",
 590       (blacklist ? blacklist->size() : 0));
 591   for (size_t i = 0; i < urls.size() && i < kTopSitesNumber; ++i) {
 592     if (!IsBlacklisted(urls[i].url))
 593       out->push_back(urls[i]);
 594   }
 595 }
 596
 597 std::string TopSitesImpl::GetURLHash(const GURL& url) {
 598   // We don't use canonical URLs here to be able to blacklist only one of
 599   // the two 'duplicate' sites, e.g. 'gmail.com' and 'mail.google.com'.
 600   return base::MD5String(url.spec());
 601 }
 602
 603 base::TimeDelta TopSitesImpl::GetUpdateDelay() {
 604   if (cache_->top_sites().size() <= arraysize(kPrepopulatedPages))
 605     return base::TimeDelta::FromSeconds(30);
 606
 607   int64 range = kMaxUpdateIntervalMinutes - kMinUpdateIntervalMinutes;
 608   int64 minutes = kMaxUpdateIntervalMinutes -
 609       last_num_urls_changed_ * range / cache_->top_sites().size();
 610   return base::TimeDelta::FromMinutes(minutes);
 611 }
 612
 613 void TopSitesImpl::Observe(int type,
 614                            const content::NotificationSource& source,
 615                            const content::NotificationDetails& details) {
 616   if (!loaded_)
 617     return;
 618
 619   if (type == chrome::NOTIFICATION_HISTORY_URLS_DELETED) {
 620     content::Details<history::URLsDeletedDetails> deleted_details(details);
 621     if (deleted_details->all_history) {
 622       SetTopSites(MostVisitedURLList());
 623       backend_->ResetDatabase();
 624     } else {
 625       std::set<size_t> indices_to_delete;  // Indices into top_sites_.
 626       for (URLRows::const_iterator i = deleted_details->rows.begin();
 627            i != deleted_details->rows.end(); ++i) {
 628         if (cache_->IsKnownURL(i->url()))
 629           indices_to_delete.insert(cache_->GetURLIndex(i->url()));
 630       }
 631
 632       if (indices_to_delete.empty())
 633         return;
 634
 635       MostVisitedURLList new_top_sites(cache_->top_sites());
 636       for (std::set<size_t>::reverse_iterator i = indices_to_delete.rbegin();
 637            i != indices_to_delete.rend(); i++) {
 638         new_top_sites.erase(new_top_sites.begin() + *i);
 639       }
 640       SetTopSites(new_top_sites);
 641     }
 642     StartQueryForMostVisited();
 643   } else if (type == content::NOTIFICATION_NAV_ENTRY_COMMITTED) {
 644     NavigationController* controller =
 645         content::Source<NavigationController>(source).ptr();
 646     Profile* profile = Profile::FromBrowserContext(
 647         controller->GetWebContents()->GetBrowserContext());
 648     if (profile == profile_ && !IsFull()) {
 649       content::LoadCommittedDetails* load_details =
 650           content::Details<content::LoadCommittedDetails>(details).ptr();
 651       if (!load_details)
 652         return;
 653       const GURL& url = load_details->entry->GetURL();
 654       if (!cache_->IsKnownURL(url) && HistoryService::CanAddURL(url)) {
 655         // To avoid slamming history we throttle requests when the url updates.
 656         // To do otherwise negatively impacts perf tests.
 657         RestartQueryForTopSitesTimer(GetUpdateDelay());
 658       }
 659     }
 660   }
 661 }
 662
 663 void TopSitesImpl::SetTopSites(const MostVisitedURLList& new_top_sites) {
 664   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 665
 666   MostVisitedURLList top_sites(new_top_sites);
 667   AddPrepopulatedPages(&top_sites);
 668
 669   TopSitesDelta delta;
 670   DiffMostVisited(cache_->top_sites(), top_sites, &delta);
 671   if (!delta.deleted.empty() || !delta.added.empty() || !delta.moved.empty()) {
 672     backend_->UpdateTopSites(delta);
 673   }
 674
 675   last_num_urls_changed_ = delta.added.size() + delta.moved.size();
 676
 677   // We always do the following steps (setting top sites in cache, and resetting
 678   // thread safe cache ...) as this method is invoked during startup at which
 679   // point the caches haven't been updated yet.
 680   cache_->SetTopSites(top_sites);
 681
 682   // See if we have any tmp thumbnails for the new sites.
 683   if (!temp_images_.empty()) {
 684     for (size_t i = 0; i < top_sites.size(); ++i) {
 685       const MostVisitedURL& mv = top_sites[i];
 686       GURL canonical_url = cache_->GetCanonicalURL(mv.url);
 687       // At the time we get the thumbnail redirects aren't known, so we have to
 688       // iterate through all the images.
 689       for (TempImages::iterator it = temp_images_.begin();
 690            it != temp_images_.end(); ++it) {
 691         if (canonical_url == cache_->GetCanonicalURL(it->first)) {
 692           SetPageThumbnailEncoded(
 693               mv.url, it->second.thumbnail.get(), it->second.thumbnail_score);
 694           temp_images_.erase(it);
 695           break;
 696         }
 697       }
 698     }
 699   }
 700
 701   if (top_sites.size() >= kTopSitesNumber)
 702     temp_images_.clear();
 703
 704   ResetThreadSafeCache();
 705   ResetThreadSafeImageCache();
 706   NotifyTopSitesChanged();
 707
 708   // Restart the timer that queries history for top sites. This is done to
 709   // ensure we stay in sync with history.
 710   RestartQueryForTopSitesTimer(GetUpdateDelay());
 711 }
 712
 713 int TopSitesImpl::num_results_to_request_from_history() const {
 714   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 715
 716   const DictionaryValue* blacklist =
 717       profile_->GetPrefs()->GetDictionary(prefs::kNtpMostVisitedURLsBlacklist);
 718   return kTopSitesNumber + (blacklist ? blacklist->size() : 0);
 719 }
 720
 721 void TopSitesImpl::MoveStateToLoaded() {
 722   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 723
 724   MostVisitedURLList filtered_urls;
 725   PendingCallbacks pending_callbacks;
 726   {
 727     base::AutoLock lock(lock_);
 728
 729     if (loaded_)
 730       return;  // Don't do anything if we're already loaded.
 731     loaded_ = true;
 732
 733     // Now that we're loaded we can service the queued up callbacks. Copy them
 734     // here and service them outside the lock.
 735     if (!pending_callbacks_.empty()) {
 736       filtered_urls = thread_safe_cache_->top_sites();
 737       pending_callbacks.swap(pending_callbacks_);
 738     }
 739   }
 740
 741   for (size_t i = 0; i < pending_callbacks.size(); i++)
 742     pending_callbacks[i].Run(filtered_urls);
 743
 744   content::NotificationService::current()->Notify(
 745       chrome::NOTIFICATION_TOP_SITES_LOADED,
 746       content::Source<Profile>(profile_),
 747       content::Details<TopSites>(this));
 748 }
 749
 750 void TopSitesImpl::ResetThreadSafeCache() {
 751   base::AutoLock lock(lock_);
 752   MostVisitedURLList cached;
 753   ApplyBlacklist(cache_->top_sites(), &cached);
 754   thread_safe_cache_->SetTopSites(cached);
 755 }
 756
 757 void TopSitesImpl::ResetThreadSafeImageCache() {
 758   base::AutoLock lock(lock_);
 759   thread_safe_cache_->SetThumbnails(cache_->images());
 760 }
 761
 762 void TopSitesImpl::NotifyTopSitesChanged() {
 763   content::NotificationService::current()->Notify(
 764       chrome::NOTIFICATION_TOP_SITES_CHANGED,
 765       content::Source<TopSites>(this),
 766       content::NotificationService::NoDetails());
 767 }
 768
 769 void TopSitesImpl::RestartQueryForTopSitesTimer(base::TimeDelta delta) {
 770   if (timer_.IsRunning() && ((timer_start_time_ + timer_.GetCurrentDelay()) <
 771                              (base::TimeTicks::Now() + delta))) {
 772     return;
 773   }
 774
 775   timer_start_time_ = base::TimeTicks::Now();
 776   timer_.Stop();
 777   timer_.Start(FROM_HERE, delta, this, &TopSitesImpl::TimerFired);
 778 }
 779
 780 void TopSitesImpl::OnGotMostVisitedThumbnails(
 781     const scoped_refptr<MostVisitedThumbnails>& thumbnails) {
 782   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 783
 784   // Set the top sites directly in the cache so that SetTopSites diffs
 785   // correctly.
 786   cache_->SetTopSites(thumbnails->most_visited);
 787   SetTopSites(thumbnails->most_visited);
 788   cache_->SetThumbnails(thumbnails->url_to_images_map);
 789
 790   ResetThreadSafeImageCache();
 791
 792   MoveStateToLoaded();
 793
 794   // Start a timer that refreshes top sites from history.
 795   RestartQueryForTopSitesTimer(
 796       base::TimeDelta::FromSeconds(kUpdateIntervalSecs));
 797 }
 798
 799 void TopSitesImpl::OnTopSitesAvailableFromHistory(
 800     CancelableRequestProvider::Handle handle,
 801     MostVisitedURLList pages) {
 802   SetTopSites(pages);
 803
 804   // Used only in testing.
 805   content::NotificationService::current()->Notify(
 806       chrome::NOTIFICATION_TOP_SITES_UPDATED,
 807       content::Source<TopSitesImpl>(this),
 808       content::Details<CancelableRequestProvider::Handle>(&handle));
 809 }
 810
 811 }  // namespace history