base/observer_list_threadsafe_unittest.cc

   1 // Copyright 2018 The Chromium Authors
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/observer_list_threadsafe.h"
   6
   7 #include <memory>
   8 #include <utility>
   9 #include <vector>
  10
  11 #include "base/compiler_specific.h"
  12 #include "base/functional/bind.h"
  13 #include "base/location.h"
  14 #include "base/logging.h"
  15 #include "base/memory/raw_ptr.h"
  16 #include "base/memory/weak_ptr.h"
  17 #include "base/run_loop.h"
  18 #include "base/synchronization/waitable_event.h"
  19 #include "base/task/sequenced_task_runner.h"
  20 #include "base/task/single_thread_task_runner.h"
  21 #include "base/task/thread_pool.h"
  22 #include "base/task/thread_pool/thread_pool_instance.h"
  23 #include "base/test/bind.h"
  24 #include "base/test/gtest_util.h"
  25 #include "base/test/task_environment.h"
  26 #include "base/test/test_waitable_event.h"
  27 #include "base/threading/platform_thread.h"
  28 #include "base/threading/thread_restrictions.h"
  29 #include "build/build_config.h"
  30 #include "testing/gtest/include/gtest/gtest.h"
  31 #include "third_party/abseil-cpp/absl/types/optional.h"
  32
  33 namespace base {
  34 namespace {
  35
  36 constexpr int kThreadRunTime = 1000;  // ms to run the multi-threaded test.
  37
  38 class Foo {
  39  public:
  40   virtual void Observe(int x) = 0;
  41   virtual ~Foo() = default;
  42   virtual int GetValue() const { return 0; }
  43 };
  44
  45 class Adder : public Foo {
  46  public:
  47   explicit Adder(int scaler) : total(0), scaler_(scaler) {}
  48   ~Adder() override = default;
  49
  50   void Observe(int x) override { total += x * scaler_; }
  51   int GetValue() const override { return total; }
  52
  53   int total;
  54
  55  private:
  56   int scaler_;
  57 };
  58
  59 class AddInObserve : public Foo {
  60  public:
  61   explicit AddInObserve(ObserverListThreadSafe<Foo>* observer_list)
  62       : observer_list(observer_list), to_add_() {}
  63
  64   void SetToAdd(Foo* to_add) { to_add_ = to_add; }
  65
  66   void Observe(int x) override {
  67     if (to_add_) {
  68       observer_list->AddObserver(to_add_.get());
  69       to_add_ = nullptr;
  70     }
  71   }
  72
  73   raw_ptr<ObserverListThreadSafe<Foo>> observer_list;
  74   raw_ptr<Foo> to_add_;
  75 };
  76
  77 // A task for use in the ThreadSafeObserver test which will add and remove
  78 // itself from the notification list repeatedly.
  79 template <RemoveObserverPolicy RemovePolicy =
  80               RemoveObserverPolicy::kAnySequence>
  81 class AddRemoveThread : public Foo {
  82   using Self = AddRemoveThread<RemovePolicy>;
  83   using ObserverList = ObserverListThreadSafe<Foo, RemovePolicy>;
  84
  85  public:
  86   AddRemoveThread(ObserverList* list,
  87                   bool notify,
  88                   scoped_refptr<SingleThreadTaskRunner> removal_task_runner)
  89       : list_(list),
  90         task_runner_(ThreadPool::CreateSingleThreadTaskRunner(
  91             {},
  92             SingleThreadTaskRunnerThreadMode::DEDICATED)),
  93         removal_task_runner_(std::move(removal_task_runner)),
  94         in_list_(false),
  95         start_(Time::Now()),
  96         do_notifies_(notify) {
  97     task_runner_->PostTask(
  98         FROM_HERE, base::BindOnce(&Self::AddTask, weak_factory_.GetWeakPtr()));
  99   }
 100
 101   ~AddRemoveThread() override = default;
 102
 103   // This task just keeps posting to itself in an attempt to race with the
 104   // notifier.
 105   void AddTask() {
 106     if ((Time::Now() - start_).InMilliseconds() > kThreadRunTime) {
 107       VLOG(1) << "DONE!";
 108       return;
 109     }
 110
 111     if (!in_list_) {
 112       list_->AddObserver(this);
 113       in_list_ = true;
 114     }
 115
 116     if (do_notifies_) {
 117       list_->Notify(FROM_HERE, &Foo::Observe, 10);
 118     }
 119
 120     SingleThreadTaskRunner::GetCurrentDefault()->PostTask(
 121         FROM_HERE, base::BindOnce(&Self::AddTask, weak_factory_.GetWeakPtr()));
 122   }
 123
 124   void RemoveTask() {
 125     list_->RemoveObserver(this);
 126     in_list_ = false;
 127   }
 128
 129   void Observe(int x) override {
 130     // If we're getting called after we removed ourselves from the list, that is
 131     // very bad!
 132     EXPECT_TRUE(in_list_);
 133
 134     // This callback should fire on the appropriate thread
 135     EXPECT_TRUE(task_runner_->BelongsToCurrentThread());
 136
 137     if (removal_task_runner_) {
 138       // Remove the observer on a different thread, blocking the current thread
 139       // until it's removed. Unretained is safe since the pointers are valid
 140       // until the thread is unblocked.
 141       base::TestWaitableEvent event;
 142       removal_task_runner_->PostTask(
 143           FROM_HERE, base::BindOnce(&Self::RemoveTask, base::Unretained(this))
 144                          .Then(base::BindOnce(&base::TestWaitableEvent::Signal,
 145                                               base::Unretained(&event))));
 146       event.Wait();
 147     } else {
 148       // Remove the observer on the same thread.
 149       RemoveTask();
 150     }
 151   }
 152
 153   scoped_refptr<SingleThreadTaskRunner> task_runner() const {
 154     return task_runner_;
 155   }
 156
 157  private:
 158   raw_ptr<ObserverList> list_;
 159   scoped_refptr<SingleThreadTaskRunner> task_runner_;
 160   // Optional task runner used to remove observers. This will be the main task
 161   // runner of a different AddRemoveThread.
 162   scoped_refptr<SingleThreadTaskRunner> removal_task_runner_;
 163   bool in_list_;  // Are we currently registered for notifications.
 164                   // in_list_ is only used on |this| thread.
 165   Time start_;    // The time we started the test.
 166
 167   bool do_notifies_;    // Whether these threads should do notifications.
 168
 169   base::WeakPtrFactory<Self> weak_factory_{this};
 170 };
 171
 172 }  // namespace
 173
 174 TEST(ObserverListThreadSafeTest, BasicTest) {
 175   using List = ObserverListThreadSafe<Foo>;
 176   test::TaskEnvironment task_environment;
 177
 178   scoped_refptr<List> observer_list(new List);
 179   Adder a(1);
 180   Adder b(-1);
 181   Adder c(1);
 182   Adder d(-1);
 183
 184   List::AddObserverResult result;
 185
 186   result = observer_list->AddObserver(&a);
 187   EXPECT_EQ(result, List::AddObserverResult::kBecameNonEmpty);
 188   result = observer_list->AddObserver(&b);
 189   EXPECT_EQ(result, List::AddObserverResult::kWasAlreadyNonEmpty);
 190
 191   observer_list->Notify(FROM_HERE, &Foo::Observe, 10);
 192   RunLoop().RunUntilIdle();
 193
 194   result = observer_list->AddObserver(&c);
 195   EXPECT_EQ(result, List::AddObserverResult::kWasAlreadyNonEmpty);
 196   result = observer_list->AddObserver(&d);
 197   EXPECT_EQ(result, List::AddObserverResult::kWasAlreadyNonEmpty);
 198
 199   observer_list->Notify(FROM_HERE, &Foo::Observe, 10);
 200   observer_list->RemoveObserver(&c);
 201   RunLoop().RunUntilIdle();
 202
 203   EXPECT_EQ(20, a.total);
 204   EXPECT_EQ(-20, b.total);
 205   EXPECT_EQ(0, c.total);
 206   EXPECT_EQ(-10, d.total);
 207 }
 208
 209 TEST(ObserverListThreadSafeTest, RemoveObserver) {
 210   using List = ObserverListThreadSafe<Foo>;
 211   test::TaskEnvironment task_environment;
 212
 213   scoped_refptr<List> observer_list(new List);
 214   Adder a(1), b(1);
 215
 216   // A workaround for the compiler bug. See http://crbug.com/121960.
 217   EXPECT_NE(&a, &b);
 218
 219   List::RemoveObserverResult result;
 220
 221   // Should do nothing.
 222   result = observer_list->RemoveObserver(&a);
 223   EXPECT_EQ(result, List::RemoveObserverResult::kWasOrBecameEmpty);
 224   result = observer_list->RemoveObserver(&b);
 225   EXPECT_EQ(result, List::RemoveObserverResult::kWasOrBecameEmpty);
 226
 227   observer_list->Notify(FROM_HERE, &Foo::Observe, 10);
 228   RunLoop().RunUntilIdle();
 229
 230   EXPECT_EQ(0, a.total);
 231   EXPECT_EQ(0, b.total);
 232
 233   observer_list->AddObserver(&a);
 234
 235   // Should also do nothing.
 236   result = observer_list->RemoveObserver(&b);
 237   EXPECT_EQ(result, List::RemoveObserverResult::kRemainsNonEmpty);
 238
 239   observer_list->Notify(FROM_HERE, &Foo::Observe, 10);
 240   RunLoop().RunUntilIdle();
 241
 242   EXPECT_EQ(10, a.total);
 243   EXPECT_EQ(0, b.total);
 244
 245   result = observer_list->RemoveObserver(&a);
 246   EXPECT_EQ(result, List::RemoveObserverResult::kWasOrBecameEmpty);
 247 }
 248
 249 class FooRemover : public Foo {
 250  public:
 251   explicit FooRemover(ObserverListThreadSafe<Foo>* list) : list_(list) {}
 252   ~FooRemover() override = default;
 253
 254   void AddFooToRemove(Foo* foo) { foos_.push_back(foo); }
 255
 256   void Observe(int x) override {
 257     std::vector<Foo*> tmp;
 258     tmp.swap(foos_);
 259     for (auto* it : tmp) {
 260       list_->RemoveObserver(it);
 261     }
 262   }
 263
 264  private:
 265   const scoped_refptr<ObserverListThreadSafe<Foo>> list_;
 266   std::vector<Foo*> foos_;
 267 };
 268
 269 TEST(ObserverListThreadSafeTest, RemoveMultipleObservers) {
 270   test::TaskEnvironment task_environment;
 271   scoped_refptr<ObserverListThreadSafe<Foo>> observer_list(
 272       new ObserverListThreadSafe<Foo>);
 273
 274   FooRemover a(observer_list.get());
 275   Adder b(1);
 276
 277   observer_list->AddObserver(&a);
 278   observer_list->AddObserver(&b);
 279
 280   a.AddFooToRemove(&a);
 281   a.AddFooToRemove(&b);
 282
 283   observer_list->Notify(FROM_HERE, &Foo::Observe, 1);
 284   RunLoop().RunUntilIdle();
 285 }
 286
 287 // A test driver for a multi-threaded notification loop.  Runs a number of
 288 // observer threads, each of which constantly adds/removes itself from the
 289 // observer list.  Optionally, if `cross_thread_notifies` is set to true, the
 290 // observer threads will also trigger notifications to all observers, and if
 291 // `cross_thread_removes` is set to true, the observer threads will also remove
 292 // observers added by other threads.
 293 template <
 294     RemoveObserverPolicy RemovePolicy = RemoveObserverPolicy::kAnySequence>
 295 static void ThreadSafeObserverHarness(int num_threads,
 296                                       bool cross_thread_notifies = false,
 297                                       bool cross_thread_removes = false) {
 298   test::TaskEnvironment task_environment;
 299
 300   auto observer_list =
 301       base::MakeRefCounted<ObserverListThreadSafe<Foo, RemovePolicy>>();
 302
 303   Adder a(1);
 304   Adder b(-1);
 305
 306   observer_list->AddObserver(&a);
 307   observer_list->AddObserver(&b);
 308
 309   using TestThread = AddRemoveThread<RemovePolicy>;
 310   std::vector<std::unique_ptr<TestThread>> threaded_observers;
 311   threaded_observers.reserve(num_threads);
 312   scoped_refptr<SingleThreadTaskRunner> removal_task_runner;
 313   for (int index = 0; index < num_threads; index++) {
 314     auto add_remove_thread =
 315         std::make_unique<TestThread>(observer_list.get(), cross_thread_notifies,
 316                                      std::move(removal_task_runner));
 317     if (cross_thread_removes) {
 318       // Save the task runner to pass to the next thread.
 319       removal_task_runner = add_remove_thread->task_runner();
 320     }
 321     threaded_observers.push_back(std::move(add_remove_thread));
 322   }
 323   ASSERT_EQ(static_cast<size_t>(num_threads), threaded_observers.size());
 324
 325   Time start = Time::Now();
 326   while (true) {
 327     if ((Time::Now() - start).InMilliseconds() > kThreadRunTime)
 328       break;
 329
 330     observer_list->Notify(FROM_HERE, &Foo::Observe, 10);
 331
 332     RunLoop().RunUntilIdle();
 333   }
 334
 335   task_environment.RunUntilIdle();
 336 }
 337
 338 TEST(ObserverListThreadSafeTest, CrossThreadObserver) {
 339   // Use 7 observer threads.  Notifications only come from the main thread.
 340   ThreadSafeObserverHarness(7);
 341 }
 342
 343 TEST(ObserverListThreadSafeTest, CrossThreadNotifications) {
 344   // Use 3 observer threads.  Notifications will fire from the main thread and
 345   // all 3 observer threads.
 346   ThreadSafeObserverHarness(3, /*cross_thread_notifies=*/true);
 347 }
 348
 349 TEST(ObserverListThreadSafeTest, CrossThreadRemoval) {
 350   // Use 3 observer threads. Observers can be removed from any thread.
 351   ThreadSafeObserverHarness(3, /*cross_thread_notifies=*/true,
 352                             /*cross_thread_removes=*/true);
 353 }
 354
 355 TEST(ObserverListThreadSafeTest, CrossThreadRemovalRestricted) {
 356   // Use 3 observer threads. Observers must be removed from the thread that
 357   // added them. This should succeed because the test doesn't break that
 358   // restriction.
 359   ThreadSafeObserverHarness<RemoveObserverPolicy::kAddingSequenceOnly>(
 360       3, /*cross_thread_notifies=*/true, /*cross_thread_removes=*/false);
 361 }
 362
 363 TEST(ObserverListThreadSafeDeathTest, CrossThreadRemovalRestricted) {
 364   // Use 3 observer threads. Observers must be removed from the thread that
 365   // added them. This should CHECK because the test breaks that restriction.
 366   EXPECT_CHECK_DEATH(
 367       ThreadSafeObserverHarness<RemoveObserverPolicy::kAddingSequenceOnly>(
 368           3, /*cross_thread_notifies=*/true, /*cross_thread_removes=*/true));
 369 }
 370
 371 TEST(ObserverListThreadSafeTest, OutlivesTaskEnvironment) {
 372   absl::optional<test::TaskEnvironment> task_environment(absl::in_place);
 373   auto observer_list = base::MakeRefCounted<ObserverListThreadSafe<Foo>>();
 374
 375   Adder a(1);
 376   observer_list->AddObserver(&a);
 377   task_environment.reset();
 378   // Test passes if we don't crash here.
 379   observer_list->Notify(FROM_HERE, &Foo::Observe, 1);
 380   observer_list->RemoveObserver(&a);
 381 }
 382
 383 TEST(ObserverListThreadSafeTest, OutlivesTaskEnvironmentRemovalRestricted) {
 384   absl::optional<test::TaskEnvironment> task_environment(absl::in_place);
 385   auto observer_list = base::MakeRefCounted<
 386       ObserverListThreadSafe<Foo, RemoveObserverPolicy::kAddingSequenceOnly>>();
 387
 388   Adder a(1);
 389   observer_list->AddObserver(&a);
 390   task_environment.reset();
 391   // Test passes if we don't crash here.
 392   observer_list->Notify(FROM_HERE, &Foo::Observe, 1);
 393   observer_list->RemoveObserver(&a);
 394 }
 395
 396 namespace {
 397
 398 class SequenceVerificationObserver : public Foo {
 399  public:
 400   explicit SequenceVerificationObserver(
 401       scoped_refptr<SequencedTaskRunner> task_runner)
 402       : task_runner_(std::move(task_runner)) {}
 403   SequenceVerificationObserver(const SequenceVerificationObserver&) = delete;
 404   SequenceVerificationObserver& operator=(const SequenceVerificationObserver&) =
 405       delete;
 406   ~SequenceVerificationObserver() override = default;
 407
 408   void Observe(int x) override {
 409     called_on_valid_sequence_ = task_runner_->RunsTasksInCurrentSequence();
 410   }
 411
 412   bool called_on_valid_sequence() const { return called_on_valid_sequence_; }
 413
 414  private:
 415   const scoped_refptr<SequencedTaskRunner> task_runner_;
 416   bool called_on_valid_sequence_ = false;
 417 };
 418
 419 }  // namespace
 420
 421 // Verify that observers are notified on the correct sequence.
 422 TEST(ObserverListThreadSafeTest, NotificationOnValidSequence) {
 423   test::TaskEnvironment task_environment;
 424
 425   auto task_runner_1 = ThreadPool::CreateSequencedTaskRunner({});
 426   auto task_runner_2 = ThreadPool::ThreadPool::CreateSequencedTaskRunner({});
 427
 428   auto observer_list = MakeRefCounted<ObserverListThreadSafe<Foo>>();
 429
 430   SequenceVerificationObserver observer_1(task_runner_1);
 431   SequenceVerificationObserver observer_2(task_runner_2);
 432
 433   task_runner_1->PostTask(
 434       FROM_HERE,
 435       BindOnce(base::IgnoreResult(&ObserverListThreadSafe<Foo>::AddObserver),
 436                observer_list, Unretained(&observer_1)));
 437   task_runner_2->PostTask(
 438       FROM_HERE,
 439       BindOnce(base::IgnoreResult(&ObserverListThreadSafe<Foo>::AddObserver),
 440                observer_list, Unretained(&observer_2)));
 441
 442   ThreadPoolInstance::Get()->FlushForTesting();
 443
 444   observer_list->Notify(FROM_HERE, &Foo::Observe, 1);
 445
 446   ThreadPoolInstance::Get()->FlushForTesting();
 447
 448   EXPECT_TRUE(observer_1.called_on_valid_sequence());
 449   EXPECT_TRUE(observer_2.called_on_valid_sequence());
 450 }
 451
 452 // Verify that when an observer is added to a NOTIFY_ALL ObserverListThreadSafe
 453 // from a notification, it is itself notified.
 454 TEST(ObserverListThreadSafeTest, AddObserverFromNotificationNotifyAll) {
 455   test::TaskEnvironment task_environment;
 456   auto observer_list = MakeRefCounted<ObserverListThreadSafe<Foo>>();
 457
 458   Adder observer_added_from_notification(1);
 459
 460   AddInObserve initial_observer(observer_list.get());
 461   initial_observer.SetToAdd(&observer_added_from_notification);
 462   observer_list->AddObserver(&initial_observer);
 463
 464   observer_list->Notify(FROM_HERE, &Foo::Observe, 1);
 465
 466   base::RunLoop().RunUntilIdle();
 467
 468   EXPECT_EQ(1, observer_added_from_notification.GetValue());
 469 }
 470
 471 namespace {
 472
 473 class RemoveWhileNotificationIsRunningObserver : public Foo {
 474  public:
 475   RemoveWhileNotificationIsRunningObserver()
 476       : notification_running_(WaitableEvent::ResetPolicy::AUTOMATIC,
 477                               WaitableEvent::InitialState::NOT_SIGNALED),
 478         barrier_(WaitableEvent::ResetPolicy::AUTOMATIC,
 479                  WaitableEvent::InitialState::NOT_SIGNALED) {}
 480   RemoveWhileNotificationIsRunningObserver(
 481       const RemoveWhileNotificationIsRunningObserver&) = delete;
 482   RemoveWhileNotificationIsRunningObserver& operator=(
 483       const RemoveWhileNotificationIsRunningObserver&) = delete;
 484   ~RemoveWhileNotificationIsRunningObserver() override = default;
 485
 486   void Observe(int x) override {
 487     notification_running_.Signal();
 488     ScopedAllowBaseSyncPrimitivesForTesting allow_base_sync_primitives;
 489     barrier_.Wait();
 490   }
 491
 492   void WaitForNotificationRunning() { notification_running_.Wait(); }
 493   void Unblock() { barrier_.Signal(); }
 494
 495  private:
 496   WaitableEvent notification_running_;
 497   WaitableEvent barrier_;
 498 };
 499
 500 }  // namespace
 501
 502 // Verify that there is no crash when an observer is removed while it is being
 503 // notified.
 504 TEST(ObserverListThreadSafeTest, RemoveWhileNotificationIsRunning) {
 505   auto observer_list = MakeRefCounted<ObserverListThreadSafe<Foo>>();
 506   RemoveWhileNotificationIsRunningObserver observer;
 507
 508   WaitableEvent task_running(WaitableEvent::ResetPolicy::AUTOMATIC,
 509                              WaitableEvent::InitialState::NOT_SIGNALED);
 510   WaitableEvent barrier(WaitableEvent::ResetPolicy::AUTOMATIC,
 511                         WaitableEvent::InitialState::NOT_SIGNALED);
 512
 513   // This must be after the declaration of |barrier| so that tasks posted to
 514   // ThreadPool can safely use |barrier|.
 515   test::TaskEnvironment task_environment;
 516
 517   ThreadPool::CreateSequencedTaskRunner({MayBlock()})
 518       ->PostTask(FROM_HERE,
 519                  base::BindOnce(base::IgnoreResult(
 520                                     &ObserverListThreadSafe<Foo>::AddObserver),
 521                                 observer_list, Unretained(&observer)));
 522   ThreadPoolInstance::Get()->FlushForTesting();
 523
 524   observer_list->Notify(FROM_HERE, &Foo::Observe, 1);
 525   observer.WaitForNotificationRunning();
 526   observer_list->RemoveObserver(&observer);
 527
 528   observer.Unblock();
 529 }
 530
 531 TEST(ObserverListThreadSafeTest, AddRemoveWithPendingNotifications) {
 532   test::TaskEnvironment task_environment;
 533
 534   scoped_refptr<ObserverListThreadSafe<Foo>> observer_list(
 535       new ObserverListThreadSafe<Foo>);
 536   Adder a(1);
 537   Adder b(1);
 538
 539   observer_list->AddObserver(&a);
 540   observer_list->AddObserver(&b);
 541
 542   // Remove observer `a` while there is a pending notification for observer `a`.
 543   observer_list->Notify(FROM_HERE, &Foo::Observe, 10);
 544   observer_list->RemoveObserver(&a);
 545   RunLoop().RunUntilIdle();
 546   observer_list->AddObserver(&a);
 547
 548   EXPECT_EQ(0, a.total);
 549   EXPECT_EQ(10, b.total);
 550
 551   // Remove and re-adding observer `a` while there is a pending notification for
 552   // observer `a`. The notification to `a` must not be executed since it was
 553   // sent before the removal of `a`.
 554   observer_list->Notify(FROM_HERE, &Foo::Observe, 10);
 555   observer_list->RemoveObserver(&a);
 556   observer_list->AddObserver(&a);
 557   RunLoop().RunUntilIdle();
 558
 559   EXPECT_EQ(0, a.total);
 560   EXPECT_EQ(20, b.total);
 561
 562   // Observer `a` and `b` are present and should both receive a notification.
 563   observer_list->RemoveObserver(&a);
 564   observer_list->AddObserver(&a);
 565   observer_list->Notify(FROM_HERE, &Foo::Observe, 10);
 566   RunLoop().RunUntilIdle();
 567
 568   EXPECT_EQ(10, a.total);
 569   EXPECT_EQ(30, b.total);
 570 }
 571
 572 // Same as ObserverListTest.Existing, but for ObserverListThreadSafe
 573 TEST(ObserverListThreadSafeTest, Existing) {
 574   test::TaskEnvironment task_environment;
 575   scoped_refptr<ObserverListThreadSafe<Foo>> observer_list(
 576       new ObserverListThreadSafe<Foo>(ObserverListPolicy::EXISTING_ONLY));
 577   Adder a(1);
 578   AddInObserve b(observer_list.get());
 579   Adder c(1);
 580   b.SetToAdd(&c);
 581
 582   observer_list->AddObserver(&a);
 583   observer_list->AddObserver(&b);
 584
 585   observer_list->Notify(FROM_HERE, &Foo::Observe, 1);
 586   RunLoop().RunUntilIdle();
 587
 588   EXPECT_FALSE(b.to_add_);
 589   // B's adder should not have been notified because it was added during
 590   // notification.
 591   EXPECT_EQ(0, c.total);
 592
 593   // Notify again to make sure b's adder is notified.
 594   observer_list->Notify(FROM_HERE, &Foo::Observe, 1);
 595   RunLoop().RunUntilIdle();
 596   EXPECT_EQ(1, c.total);
 597 }
 598
 599 }  // namespace base