src/chrome/browser/metrics/thread_watcher_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include <math.h>
   6
   7 #include "base/basictypes.h"
   8 #include "base/bind.h"
   9 #include "base/logging.h"
  10 #include "base/memory/scoped_ptr.h"
  11 #include "base/message_loop/message_loop.h"
  12 #include "base/message_loop/message_loop_proxy.h"
  13 #include "base/run_loop.h"
  14 #include "base/strings/string_number_conversions.h"
  15 #include "base/strings/string_split.h"
  16 #include "base/strings/string_tokenizer.h"
  17 #include "base/synchronization/condition_variable.h"
  18 #include "base/synchronization/lock.h"
  19 #include "base/threading/platform_thread.h"
  20 #include "base/time/time.h"
  21 #include "build/build_config.h"
  22 #include "chrome/browser/metrics/thread_watcher.h"
  23 #include "chrome/common/chrome_switches.h"
  24 #include "content/public/test/test_browser_thread.h"
  25 #include "testing/gtest/include/gtest/gtest.h"
  26 #include "testing/platform_test.h"
  27
  28 using base::TimeDelta;
  29 using base::TimeTicks;
  30 using content::BrowserThread;
  31
  32 enum State {
  33   INITIALIZED,        // Created ThreadWatch object.
  34   ACTIVATED,          // Thread watching activated.
  35   SENT_PING,          // Sent ping message to watched thread.
  36   RECEIVED_PONG,      // Received Pong message.
  37   DEACTIVATED,        // Thread watching de-activated.
  38 };
  39
  40 enum WaitState {
  41   UNINITIALIZED,
  42   STARTED_WAITING,    // Start waiting for state_ to change to expected_state.
  43   STOPPED_WAITING,    // Done with the waiting.
  44   ALL_DONE,           // Done with waiting for STOPPED_WAITING.
  45 };
  46
  47 enum CheckResponseState {
  48   UNKNOWN,
  49   SUCCESSFUL,         // CheckResponse was successful.
  50   FAILED,             // CheckResponse has failed.
  51 };
  52
  53 // This class helps to track and manipulate thread state during tests. This
  54 // class also has utility method to simulate hanging of watched thread by making
  55 // the watched thread wait for a very long time by posting a task on watched
  56 // thread that keeps it busy. It also has an utility method to block running of
  57 // tests until ThreadWatcher object's post-condition state changes to an
  58 // expected state.
  59 class CustomThreadWatcher : public ThreadWatcher {
  60  public:
  61   base::Lock custom_lock_;
  62   base::ConditionVariable state_changed_;
  63   State thread_watcher_state_;
  64   WaitState wait_state_;
  65   CheckResponseState check_response_state_;
  66   uint64 ping_sent_;
  67   uint64 pong_received_;
  68   base::subtle::Atomic32 success_response_;
  69   base::subtle::Atomic32 failed_response_;
  70   base::TimeTicks saved_ping_time_;
  71   uint64 saved_ping_sequence_number_;
  72
  73   CustomThreadWatcher(const BrowserThread::ID thread_id,
  74                       const std::string thread_name,
  75                       const TimeDelta& sleep_time,
  76                       const TimeDelta& unresponsive_time)
  77       : ThreadWatcher(WatchingParams(thread_id, thread_name, sleep_time,
  78                       unresponsive_time, ThreadWatcherList::kUnresponsiveCount,
  79                       true, ThreadWatcherList::kLiveThreadsThreshold)),
  80         state_changed_(&custom_lock_),
  81         thread_watcher_state_(INITIALIZED),
  82         wait_state_(UNINITIALIZED),
  83         check_response_state_(UNKNOWN),
  84         ping_sent_(0),
  85         pong_received_(0),
  86         success_response_(0),
  87         failed_response_(0),
  88         saved_ping_time_(base::TimeTicks::Now()),
  89         saved_ping_sequence_number_(0) {
  90   }
  91
  92   State UpdateState(State new_state) {
  93     State old_state;
  94     {
  95       base::AutoLock auto_lock(custom_lock_);
  96       old_state = thread_watcher_state_;
  97       if (old_state != DEACTIVATED)
  98         thread_watcher_state_ = new_state;
  99       if (new_state == SENT_PING)
 100         ++ping_sent_;
 101       if (new_state == RECEIVED_PONG)
 102         ++pong_received_;
 103       saved_ping_time_ = ping_time();
 104       saved_ping_sequence_number_ = ping_sequence_number();
 105     }
 106     state_changed_.Broadcast();
 107     return old_state;
 108   }
 109
 110   WaitState UpdateWaitState(WaitState new_state) {
 111     WaitState old_state;
 112     {
 113       base::AutoLock auto_lock(custom_lock_);
 114       old_state = wait_state_;
 115       wait_state_ = new_state;
 116     }
 117     state_changed_.Broadcast();
 118     return old_state;
 119   }
 120
 121   void ActivateThreadWatching() override {
 122     State old_state = UpdateState(ACTIVATED);
 123     EXPECT_EQ(old_state, INITIALIZED);
 124     ThreadWatcher::ActivateThreadWatching();
 125   }
 126
 127   void DeActivateThreadWatching() override {
 128     State old_state = UpdateState(DEACTIVATED);
 129     EXPECT_TRUE(old_state == ACTIVATED || old_state == SENT_PING ||
 130                 old_state == RECEIVED_PONG);
 131     ThreadWatcher::DeActivateThreadWatching();
 132   }
 133
 134   void PostPingMessage() override {
 135     State old_state = UpdateState(SENT_PING);
 136     EXPECT_TRUE(old_state == ACTIVATED || old_state == RECEIVED_PONG);
 137     ThreadWatcher::PostPingMessage();
 138   }
 139
 140   void OnPongMessage(uint64 ping_sequence_number) override {
 141     State old_state = UpdateState(RECEIVED_PONG);
 142     EXPECT_TRUE(old_state == SENT_PING || old_state == DEACTIVATED);
 143     ThreadWatcher::OnPongMessage(ping_sequence_number);
 144   }
 145
 146   void OnCheckResponsiveness(uint64 ping_sequence_number) override {
 147     ThreadWatcher::OnCheckResponsiveness(ping_sequence_number);
 148     {
 149       base::AutoLock auto_lock(custom_lock_);
 150       if (responsive_) {
 151         base::subtle::Release_Store(&success_response_,
 152             base::subtle::Acquire_Load(&success_response_) + 1);
 153         check_response_state_ = SUCCESSFUL;
 154       } else {
 155         base::subtle::Release_Store(&failed_response_,
 156             base::subtle::Acquire_Load(&failed_response_) + 1);
 157         check_response_state_ = FAILED;
 158       }
 159     }
 160     // Broadcast to indicate we have checked responsiveness of the thread that
 161     // is watched.
 162     state_changed_.Broadcast();
 163   }
 164
 165   void WaitForWaitStateChange(TimeDelta wait_time, WaitState expected_state) {
 166     DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
 167     TimeTicks end_time = TimeTicks::Now() + wait_time;
 168     {
 169       base::AutoLock auto_lock(custom_lock_);
 170       while (wait_state_ != expected_state && TimeTicks::Now() < end_time)
 171         state_changed_.TimedWait(end_time - TimeTicks::Now());
 172     }
 173   }
 174
 175   void VeryLongMethod(TimeDelta wait_time) {
 176     DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
 177     WaitForWaitStateChange(wait_time, STOPPED_WAITING);
 178     UpdateWaitState(ALL_DONE);
 179   }
 180
 181   State WaitForStateChange(const TimeDelta& wait_time, State expected_state) {
 182     DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
 183     UpdateWaitState(STARTED_WAITING);
 184
 185     State exit_state = INITIALIZED;
 186     // Keep the thread that is running the tests waiting until ThreadWatcher
 187     // object's state changes to the expected_state or until wait_time elapses.
 188     for (uint32 i = 0; i < unresponsive_threshold_; ++i) {
 189         TimeTicks end_time = TimeTicks::Now() + wait_time;
 190         {
 191           base::AutoLock auto_lock(custom_lock_);
 192           while (thread_watcher_state_ != expected_state &&
 193                  TimeTicks::Now() < end_time) {
 194             TimeDelta state_change_wait_time = end_time - TimeTicks::Now();
 195             state_changed_.TimedWait(state_change_wait_time);
 196           }
 197           // Capture the thread_watcher_state_ before it changes and return it
 198           // to the caller.
 199           exit_state = thread_watcher_state_;
 200           if (exit_state == expected_state)
 201             break;
 202         }
 203     }
 204     UpdateWaitState(STOPPED_WAITING);
 205     return exit_state;
 206   }
 207
 208   CheckResponseState WaitForCheckResponse(const TimeDelta& wait_time,
 209                                           CheckResponseState expected_state) {
 210     DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
 211     UpdateWaitState(STARTED_WAITING);
 212
 213     CheckResponseState exit_state = UNKNOWN;
 214     // Keep the thread that is running the tests waiting until ThreadWatcher
 215     // object's check_response_state_ changes to the expected_state or until
 216     // wait_time elapses.
 217     for (uint32 i = 0; i < unresponsive_threshold_; ++i) {
 218         TimeTicks end_time = TimeTicks::Now() + wait_time;
 219         {
 220           base::AutoLock auto_lock(custom_lock_);
 221           while (check_response_state_ != expected_state &&
 222                  TimeTicks::Now() < end_time) {
 223             TimeDelta state_change_wait_time = end_time - TimeTicks::Now();
 224             state_changed_.TimedWait(state_change_wait_time);
 225           }
 226           // Capture the check_response_state_ before it changes and return it
 227           // to the caller.
 228           exit_state = check_response_state_;
 229           if (exit_state == expected_state)
 230             break;
 231         }
 232     }
 233     UpdateWaitState(STOPPED_WAITING);
 234     return exit_state;
 235   }
 236 };
 237
 238 class ThreadWatcherTest : public ::testing::Test {
 239  public:
 240   static const TimeDelta kSleepTime;
 241   static const TimeDelta kUnresponsiveTime;
 242   static const BrowserThread::ID io_thread_id;
 243   static const std::string io_thread_name;
 244   static const BrowserThread::ID db_thread_id;
 245   static const std::string db_thread_name;
 246   static const std::string crash_on_hang_seconds;
 247   static const std::string crash_on_hang_thread_names;
 248   static const std::string thread_names_and_live_threshold;
 249   static const std::string crash_on_hang_thread_data;
 250   CustomThreadWatcher* io_watcher_;
 251   CustomThreadWatcher* db_watcher_;
 252   ThreadWatcherList* thread_watcher_list_;
 253
 254   ThreadWatcherTest()
 255       : setup_complete_(&lock_),
 256         initialized_(false) {
 257     db_thread_.reset(new content::TestBrowserThread(BrowserThread::DB));
 258     io_thread_.reset(new content::TestBrowserThread(BrowserThread::IO));
 259     watchdog_thread_.reset(new WatchDogThread());
 260     db_thread_->Start();
 261     io_thread_->Start();
 262     watchdog_thread_->Start();
 263
 264     WatchDogThread::PostTask(
 265         FROM_HERE,
 266         base::Bind(&ThreadWatcherTest::SetUpObjects, base::Unretained(this)));
 267
 268     WaitForSetUp(TimeDelta::FromMinutes(1));
 269   }
 270
 271   void SetUpObjects() {
 272     DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 273
 274     // Setup the registry for thread watchers.
 275     thread_watcher_list_ = new ThreadWatcherList();
 276
 277     // Create thread watcher object for the IO thread.
 278     io_watcher_ = new CustomThreadWatcher(io_thread_id, io_thread_name,
 279                                           kSleepTime, kUnresponsiveTime);
 280     EXPECT_EQ(io_watcher_, thread_watcher_list_->Find(io_thread_id));
 281
 282     // Create thread watcher object for the DB thread.
 283     db_watcher_ = new CustomThreadWatcher(
 284         db_thread_id, db_thread_name, kSleepTime, kUnresponsiveTime);
 285     EXPECT_EQ(db_watcher_, thread_watcher_list_->Find(db_thread_id));
 286
 287     {
 288       base::AutoLock lock(lock_);
 289       initialized_ = true;
 290     }
 291     setup_complete_.Signal();
 292   }
 293
 294   void WaitForSetUp(TimeDelta wait_time) {
 295     DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
 296     TimeTicks end_time = TimeTicks::Now() + wait_time;
 297     {
 298       base::AutoLock auto_lock(lock_);
 299       while (!initialized_ && TimeTicks::Now() < end_time)
 300         setup_complete_.TimedWait(end_time - TimeTicks::Now());
 301     }
 302   }
 303
 304   ~ThreadWatcherTest() override {
 305     ThreadWatcherList::DeleteAll();
 306     io_watcher_ = NULL;
 307     db_watcher_ = NULL;
 308     io_thread_.reset();
 309     db_thread_.reset();
 310     watchdog_thread_.reset();
 311     thread_watcher_list_ = NULL;
 312   }
 313
 314  private:
 315   base::Lock lock_;
 316   base::ConditionVariable setup_complete_;
 317   bool initialized_;
 318   scoped_ptr<content::TestBrowserThread> db_thread_;
 319   scoped_ptr<content::TestBrowserThread> io_thread_;
 320   scoped_ptr<WatchDogThread> watchdog_thread_;
 321 };
 322
 323 // Define static constants.
 324 const TimeDelta ThreadWatcherTest::kSleepTime =
 325     TimeDelta::FromMilliseconds(50);
 326 const TimeDelta ThreadWatcherTest::kUnresponsiveTime =
 327     TimeDelta::FromMilliseconds(500);
 328 const BrowserThread::ID ThreadWatcherTest::io_thread_id = BrowserThread::IO;
 329 const std::string ThreadWatcherTest::io_thread_name = "IO";
 330 const BrowserThread::ID ThreadWatcherTest::db_thread_id = BrowserThread::DB;
 331 const std::string ThreadWatcherTest::db_thread_name = "DB";
 332 const std::string ThreadWatcherTest::crash_on_hang_thread_names = "UI,IO";
 333 const std::string ThreadWatcherTest::thread_names_and_live_threshold =
 334     "UI:4,IO:4";
 335 const std::string ThreadWatcherTest::crash_on_hang_thread_data =
 336     "UI:5:12,IO:5:12,FILE:5:12";
 337
 338 TEST_F(ThreadWatcherTest, ThreadNamesOnlyArgs) {
 339   // Setup command_line arguments.
 340   CommandLine command_line(CommandLine::NO_PROGRAM);
 341   command_line.AppendSwitchASCII(switches::kCrashOnHangThreads,
 342                                  crash_on_hang_thread_names);
 343
 344   // Parse command_line arguments.
 345   ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads;
 346   uint32 unresponsive_threshold;
 347   ThreadWatcherList::ParseCommandLine(command_line,
 348                                       &unresponsive_threshold,
 349                                       &crash_on_hang_threads);
 350
 351   // Verify the data.
 352   base::StringTokenizer tokens(crash_on_hang_thread_names, ",");
 353   std::vector<std::string> values;
 354   while (tokens.GetNext()) {
 355     const std::string& token = tokens.token();
 356     base::SplitString(token, ':', &values);
 357     std::string thread_name = values[0];
 358
 359     ThreadWatcherList::CrashOnHangThreadMap::iterator it =
 360         crash_on_hang_threads.find(thread_name);
 361     bool crash_on_hang = (it != crash_on_hang_threads.end());
 362     EXPECT_TRUE(crash_on_hang);
 363     EXPECT_LT(0u, it->second.live_threads_threshold);
 364     EXPECT_LT(0u, it->second.unresponsive_threshold);
 365   }
 366 }
 367
 368 TEST_F(ThreadWatcherTest, ThreadNamesAndLiveThresholdArgs) {
 369   // Setup command_line arguments.
 370   CommandLine command_line(CommandLine::NO_PROGRAM);
 371   command_line.AppendSwitchASCII(switches::kCrashOnHangThreads,
 372                                  thread_names_and_live_threshold);
 373
 374   // Parse command_line arguments.
 375   ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads;
 376   uint32 unresponsive_threshold;
 377   ThreadWatcherList::ParseCommandLine(command_line,
 378                                       &unresponsive_threshold,
 379                                       &crash_on_hang_threads);
 380
 381   // Verify the data.
 382   base::StringTokenizer tokens(thread_names_and_live_threshold, ",");
 383   std::vector<std::string> values;
 384   while (tokens.GetNext()) {
 385     const std::string& token = tokens.token();
 386     base::SplitString(token, ':', &values);
 387     std::string thread_name = values[0];
 388
 389     ThreadWatcherList::CrashOnHangThreadMap::iterator it =
 390         crash_on_hang_threads.find(thread_name);
 391     bool crash_on_hang = (it != crash_on_hang_threads.end());
 392     EXPECT_TRUE(crash_on_hang);
 393     EXPECT_EQ(4u, it->second.live_threads_threshold);
 394     EXPECT_LT(0u, it->second.unresponsive_threshold);
 395   }
 396 }
 397
 398 TEST_F(ThreadWatcherTest, CrashOnHangThreadsAllArgs) {
 399   // Setup command_line arguments.
 400   CommandLine command_line(CommandLine::NO_PROGRAM);
 401   command_line.AppendSwitchASCII(switches::kCrashOnHangThreads,
 402                                  crash_on_hang_thread_data);
 403
 404   // Parse command_line arguments.
 405   ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads;
 406   uint32 unresponsive_threshold;
 407   ThreadWatcherList::ParseCommandLine(command_line,
 408                                       &unresponsive_threshold,
 409                                       &crash_on_hang_threads);
 410
 411   // Verify the data.
 412   base::StringTokenizer tokens(crash_on_hang_thread_data, ",");
 413   std::vector<std::string> values;
 414   while (tokens.GetNext()) {
 415     const std::string& token = tokens.token();
 416     base::SplitString(token, ':', &values);
 417     std::string thread_name = values[0];
 418
 419     ThreadWatcherList::CrashOnHangThreadMap::iterator it =
 420         crash_on_hang_threads.find(thread_name);
 421
 422     bool crash_on_hang = (it != crash_on_hang_threads.end());
 423     EXPECT_TRUE(crash_on_hang);
 424
 425     uint32 crash_live_threads_threshold = it->second.live_threads_threshold;
 426     EXPECT_EQ(5u, crash_live_threads_threshold);
 427
 428     uint32 crash_unresponsive_threshold = it->second.unresponsive_threshold;
 429     uint32 crash_on_unresponsive_seconds =
 430         ThreadWatcherList::kUnresponsiveSeconds * crash_unresponsive_threshold;
 431     EXPECT_EQ(12u, crash_on_unresponsive_seconds);
 432   }
 433 }
 434
 435 // Test registration. When thread_watcher_list_ goes out of scope after
 436 // TearDown, all thread watcher objects will be deleted.
 437 TEST_F(ThreadWatcherTest, Registration) {
 438   // Check ThreadWatcher object has all correct parameters.
 439   EXPECT_EQ(io_thread_id, io_watcher_->thread_id());
 440   EXPECT_EQ(io_thread_name, io_watcher_->thread_name());
 441   EXPECT_EQ(kSleepTime, io_watcher_->sleep_time());
 442   EXPECT_EQ(kUnresponsiveTime, io_watcher_->unresponsive_time());
 443   EXPECT_FALSE(io_watcher_->active());
 444
 445   // Check ThreadWatcher object of watched DB thread has correct data.
 446   EXPECT_EQ(db_thread_id, db_watcher_->thread_id());
 447   EXPECT_EQ(db_thread_name, db_watcher_->thread_name());
 448   EXPECT_EQ(kSleepTime, db_watcher_->sleep_time());
 449   EXPECT_EQ(kUnresponsiveTime, db_watcher_->unresponsive_time());
 450   EXPECT_FALSE(db_watcher_->active());
 451 }
 452
 453 // Test ActivateThreadWatching and DeActivateThreadWatching of IO thread. This
 454 // method also checks that pong message was sent by the watched thread and pong
 455 // message was received by the WatchDogThread. It also checks that
 456 // OnCheckResponsiveness has verified the ping-pong mechanism and the watched
 457 // thread is not hung.
 458 TEST_F(ThreadWatcherTest, ThreadResponding) {
 459   TimeTicks time_before_ping = TimeTicks::Now();
 460   // Activate watching IO thread.
 461   WatchDogThread::PostTask(
 462       FROM_HERE,
 463       base::Bind(&ThreadWatcher::ActivateThreadWatching,
 464                  base::Unretained(io_watcher_)));
 465
 466   // Activate would have started ping/pong messaging. Expect atleast one
 467   // ping/pong messaging sequence to happen.
 468   io_watcher_->WaitForStateChange(kSleepTime + TimeDelta::FromMinutes(1),
 469                                   RECEIVED_PONG);
 470   EXPECT_GT(io_watcher_->ping_sent_, static_cast<uint64>(0));
 471   EXPECT_GT(io_watcher_->pong_received_, static_cast<uint64>(0));
 472   EXPECT_TRUE(io_watcher_->active());
 473   EXPECT_GE(io_watcher_->saved_ping_time_, time_before_ping);
 474   EXPECT_GE(io_watcher_->saved_ping_sequence_number_, static_cast<uint64>(0));
 475
 476   // Verify watched thread is responding with ping/pong messaging.
 477   io_watcher_->WaitForCheckResponse(
 478       kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL);
 479   EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)),
 480       static_cast<base::subtle::Atomic32>(0));
 481   EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)),
 482       static_cast<base::subtle::Atomic32>(0));
 483
 484   // DeActivate thread watching for shutdown.
 485   WatchDogThread::PostTask(
 486       FROM_HERE,
 487       base::Bind(&ThreadWatcher::DeActivateThreadWatching,
 488       base::Unretained(io_watcher_)));
 489 }
 490
 491 // This test posts a task on watched thread that takes very long time (this is
 492 // to simulate hanging of watched thread). It then checks for
 493 // OnCheckResponsiveness raising an alert (OnCheckResponsiveness returns false
 494 // if the watched thread is not responding).
 495 TEST_F(ThreadWatcherTest, ThreadNotResponding) {
 496   // Simulate hanging of watched thread by making the watched thread wait for a
 497   // very long time by posting a task on watched thread that keeps it busy.
 498   // It is safe to use base::Unretained because test is waiting for the method
 499   // to finish.
 500   BrowserThread::PostTask(
 501       io_thread_id,
 502       FROM_HERE,
 503       base::Bind(&CustomThreadWatcher::VeryLongMethod,
 504                  base::Unretained(io_watcher_),
 505                  kUnresponsiveTime * 10));
 506
 507   // Activate thread watching.
 508   WatchDogThread::PostTask(
 509       FROM_HERE,
 510       base::Bind(&ThreadWatcher::ActivateThreadWatching,
 511                  base::Unretained(io_watcher_)));
 512
 513   // Verify watched thread is not responding for ping messages.
 514   io_watcher_->WaitForCheckResponse(
 515       kUnresponsiveTime + TimeDelta::FromMinutes(1), FAILED);
 516   EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)),
 517       static_cast<base::subtle::Atomic32>(0));
 518   EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)),
 519       static_cast<base::subtle::Atomic32>(0));
 520
 521   // DeActivate thread watching for shutdown.
 522   WatchDogThread::PostTask(
 523       FROM_HERE,
 524       base::Bind(&ThreadWatcher::DeActivateThreadWatching,
 525                  base::Unretained(io_watcher_)));
 526
 527   // Wait for the io_watcher_'s VeryLongMethod to finish.
 528   io_watcher_->WaitForWaitStateChange(kUnresponsiveTime * 10, ALL_DONE);
 529 }
 530
 531 // Test watching of multiple threads with all threads not responding.
 532 TEST_F(ThreadWatcherTest, MultipleThreadsResponding) {
 533   // Check for DB thread to perform ping/pong messaging.
 534   WatchDogThread::PostTask(
 535       FROM_HERE,
 536       base::Bind(&ThreadWatcher::ActivateThreadWatching,
 537                  base::Unretained(db_watcher_)));
 538
 539   // Check for IO thread to perform ping/pong messaging.
 540   WatchDogThread::PostTask(
 541       FROM_HERE,
 542       base::Bind(&ThreadWatcher::ActivateThreadWatching,
 543                  base::Unretained(io_watcher_)));
 544
 545   // Verify DB thread is responding with ping/pong messaging.
 546   db_watcher_->WaitForCheckResponse(
 547       kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL);
 548   EXPECT_GT(db_watcher_->ping_sent_, static_cast<uint64>(0));
 549   EXPECT_GT(db_watcher_->pong_received_, static_cast<uint64>(0));
 550   EXPECT_GE(db_watcher_->ping_sequence_number_, static_cast<uint64>(0));
 551   EXPECT_GT(base::subtle::NoBarrier_Load(&(db_watcher_->success_response_)),
 552       static_cast<base::subtle::Atomic32>(0));
 553   EXPECT_EQ(base::subtle::NoBarrier_Load(&(db_watcher_->failed_response_)),
 554       static_cast<base::subtle::Atomic32>(0));
 555
 556   // Verify IO thread is responding with ping/pong messaging.
 557   io_watcher_->WaitForCheckResponse(
 558       kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL);
 559   EXPECT_GT(io_watcher_->ping_sent_, static_cast<uint64>(0));
 560   EXPECT_GT(io_watcher_->pong_received_, static_cast<uint64>(0));
 561   EXPECT_GE(io_watcher_->ping_sequence_number_, static_cast<uint64>(0));
 562   EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)),
 563       static_cast<base::subtle::Atomic32>(0));
 564   EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)),
 565       static_cast<base::subtle::Atomic32>(0));
 566
 567   // DeActivate thread watching for shutdown.
 568   WatchDogThread::PostTask(
 569       FROM_HERE,
 570       base::Bind(&ThreadWatcher::DeActivateThreadWatching,
 571                  base::Unretained(io_watcher_)));
 572
 573   WatchDogThread::PostTask(
 574       FROM_HERE,
 575       base::Bind(&ThreadWatcher::DeActivateThreadWatching,
 576                  base::Unretained(db_watcher_)));
 577 }
 578
 579 // Test watching of multiple threads with one of the threads not responding.
 580 TEST_F(ThreadWatcherTest, MultipleThreadsNotResponding) {
 581   // Simulate hanging of watched thread by making the watched thread wait for a
 582   // very long time by posting a task on watched thread that keeps it busy.
 583   // It is safe ot use base::Unretained because test is waiting for the method
 584   // to finish.
 585   BrowserThread::PostTask(
 586       io_thread_id,
 587       FROM_HERE,
 588       base::Bind(&CustomThreadWatcher::VeryLongMethod,
 589                  base::Unretained(io_watcher_),
 590                  kUnresponsiveTime * 10));
 591
 592   // Activate watching of DB thread.
 593   WatchDogThread::PostTask(
 594       FROM_HERE,
 595       base::Bind(&ThreadWatcher::ActivateThreadWatching,
 596                  base::Unretained(db_watcher_)));
 597
 598   // Activate watching of IO thread.
 599   WatchDogThread::PostTask(
 600       FROM_HERE,
 601       base::Bind(&ThreadWatcher::ActivateThreadWatching,
 602                  base::Unretained(io_watcher_)));
 603
 604   // Verify DB thread is responding with ping/pong messaging.
 605   db_watcher_->WaitForCheckResponse(
 606       kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL);
 607   EXPECT_GT(base::subtle::NoBarrier_Load(&(db_watcher_->success_response_)),
 608       static_cast<base::subtle::Atomic32>(0));
 609   EXPECT_EQ(base::subtle::NoBarrier_Load(&(db_watcher_->failed_response_)),
 610       static_cast<base::subtle::Atomic32>(0));
 611
 612   // Verify IO thread is not responding for ping messages.
 613   io_watcher_->WaitForCheckResponse(
 614       kUnresponsiveTime + TimeDelta::FromMinutes(1), FAILED);
 615   EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)),
 616       static_cast<base::subtle::Atomic32>(0));
 617   EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)),
 618       static_cast<base::subtle::Atomic32>(0));
 619
 620   // DeActivate thread watching for shutdown.
 621   WatchDogThread::PostTask(
 622       FROM_HERE,
 623       base::Bind(&ThreadWatcher::DeActivateThreadWatching,
 624                  base::Unretained(io_watcher_)));
 625   WatchDogThread::PostTask(
 626       FROM_HERE,
 627       base::Bind(&ThreadWatcher::DeActivateThreadWatching,
 628                  base::Unretained(db_watcher_)));
 629
 630   // Wait for the io_watcher_'s VeryLongMethod to finish.
 631   io_watcher_->WaitForWaitStateChange(kUnresponsiveTime * 10, ALL_DONE);
 632 }
 633
 634 class ThreadWatcherListTest : public ::testing::Test {
 635  protected:
 636   ThreadWatcherListTest()
 637       : done_(&lock_),
 638         state_available_(false),
 639         has_thread_watcher_list_(false),
 640         stopped_(false) {
 641   }
 642
 643   void ReadStateOnWatchDogThread() {
 644     CHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 645     {
 646       base::AutoLock auto_lock(lock_);
 647       has_thread_watcher_list_ =
 648           ThreadWatcherList::g_thread_watcher_list_ != NULL;
 649       stopped_ = ThreadWatcherList::g_stopped_;
 650       state_available_ = true;
 651     }
 652     done_.Signal();
 653   }
 654
 655   void CheckState(bool has_thread_watcher_list,
 656                   bool stopped,
 657                   const char* const msg) {
 658     CHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
 659     {
 660       base::AutoLock auto_lock(lock_);
 661       state_available_ = false;
 662     }
 663
 664     WatchDogThread::PostTask(
 665         FROM_HERE,
 666         base::Bind(&ThreadWatcherListTest::ReadStateOnWatchDogThread,
 667                    base::Unretained(this)));
 668     {
 669       base::AutoLock auto_lock(lock_);
 670       while (!state_available_)
 671         done_.Wait();
 672
 673       EXPECT_EQ(has_thread_watcher_list, has_thread_watcher_list_) << msg;
 674       EXPECT_EQ(stopped, stopped_) << msg;
 675     }
 676   }
 677
 678   base::Lock lock_;
 679   base::ConditionVariable done_;
 680
 681   bool state_available_;
 682   bool has_thread_watcher_list_;
 683   bool stopped_;
 684 };
 685
 686 TEST_F(ThreadWatcherListTest, Restart) {
 687   ThreadWatcherList::g_initialize_delay_seconds = 1;
 688
 689   base::MessageLoopForUI message_loop_for_ui;
 690   content::TestBrowserThread ui_thread(BrowserThread::UI, &message_loop_for_ui);
 691
 692   scoped_ptr<WatchDogThread> watchdog_thread_(new WatchDogThread());
 693   watchdog_thread_->Start();
 694
 695   // See http://crbug.com/347887.
 696   // StartWatchingAll() will PostDelayedTask to create g_thread_watcher_list_,
 697   // whilst StopWatchingAll() will just PostTask to destroy it.
 698   // Ensure that when Stop is called, Start will NOT create
 699   // g_thread_watcher_list_ later on.
 700   ThreadWatcherList::StartWatchingAll(*CommandLine::ForCurrentProcess());
 701   ThreadWatcherList::StopWatchingAll();
 702   message_loop_for_ui.PostDelayedTask(
 703       FROM_HERE,
 704       message_loop_for_ui.QuitClosure(),
 705       base::TimeDelta::FromSeconds(
 706           ThreadWatcherList::g_initialize_delay_seconds));
 707   message_loop_for_ui.Run();
 708
 709   CheckState(false /* has_thread_watcher_list */,
 710              true /* stopped */,
 711              "Start / Stopped");
 712
 713   // Proceed with just |StartWatchingAll| and ensure it'll be started.
 714   ThreadWatcherList::StartWatchingAll(*CommandLine::ForCurrentProcess());
 715   message_loop_for_ui.PostDelayedTask(
 716       FROM_HERE,
 717       message_loop_for_ui.QuitClosure(),
 718       base::TimeDelta::FromSeconds(
 719           ThreadWatcherList::g_initialize_delay_seconds + 1));
 720   message_loop_for_ui.Run();
 721
 722   CheckState(true /* has_thread_watcher_list */,
 723              false /* stopped */,
 724              "Started");
 725
 726   // Finally, StopWatchingAll() must stop.
 727   ThreadWatcherList::StopWatchingAll();
 728   message_loop_for_ui.PostDelayedTask(
 729       FROM_HERE,
 730       message_loop_for_ui.QuitClosure(),
 731       base::TimeDelta::FromSeconds(
 732           ThreadWatcherList::g_initialize_delay_seconds));
 733   message_loop_for_ui.Run();
 734
 735   CheckState(false /* has_thread_watcher_list */,
 736              true /* stopped */,
 737              "Stopped");
 738 }