src/sync/engine/sync_scheduler_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/bind.h"
   6 #include "base/callback.h"
   7 #include "base/compiler_specific.h"
   8 #include "base/memory/weak_ptr.h"
   9 #include "base/message_loop/message_loop.h"
  10 #include "base/test/test_timeouts.h"
  11 #include "sync/engine/backoff_delay_provider.h"
  12 #include "sync/engine/sync_scheduler_impl.h"
  13 #include "sync/engine/syncer.h"
  14 #include "sync/internal_api/public/base/cancelation_signal.h"
  15 #include "sync/internal_api/public/base/model_type_test_util.h"
  16 #include "sync/sessions/test_util.h"
  17 #include "sync/test/callback_counter.h"
  18 #include "sync/test/engine/fake_model_worker.h"
  19 #include "sync/test/engine/mock_connection_manager.h"
  20 #include "sync/test/engine/mock_nudge_handler.h"
  21 #include "sync/test/engine/test_directory_setter_upper.h"
  22 #include "sync/test/mock_invalidation.h"
  23 #include "sync/util/extensions_activity.h"
  24 #include "testing/gmock/include/gmock/gmock.h"
  25 #include "testing/gtest/include/gtest/gtest.h"
  26
  27 using base::TimeDelta;
  28 using base::TimeTicks;
  29 using testing::_;
  30 using testing::AtLeast;
  31 using testing::DoAll;
  32 using testing::Invoke;
  33 using testing::Mock;
  34 using testing::Return;
  35 using testing::WithArg;
  36 using testing::WithArgs;
  37 using testing::WithoutArgs;
  38
  39 namespace syncer {
  40 using sessions::SyncSession;
  41 using sessions::SyncSessionContext;
  42 using sync_pb::GetUpdatesCallerInfo;
  43
  44 class MockSyncer : public Syncer {
  45  public:
  46   MockSyncer();
  47   MOCK_METHOD3(NormalSyncShare, bool(ModelTypeSet,
  48                                      const sessions::NudgeTracker&,
  49                                      sessions::SyncSession*));
  50   MOCK_METHOD3(ConfigureSyncShare,
  51                bool(ModelTypeSet,
  52                     sync_pb::GetUpdatesCallerInfo::GetUpdatesSource,
  53                     SyncSession*));
  54   MOCK_METHOD2(PollSyncShare, bool(ModelTypeSet, sessions::SyncSession*));
  55 };
  56
  57 MockSyncer::MockSyncer()
  58   : Syncer(NULL) {}
  59
  60 typedef std::vector<TimeTicks> SyncShareTimes;
  61
  62 void QuitLoopNow() {
  63   // We use QuitNow() instead of Quit() as the latter may get stalled
  64   // indefinitely in the presence of repeated timers with low delays
  65   // and a slow test (e.g., ThrottlingDoesThrottle [which has a poll
  66   // delay of 5ms] run under TSAN on the trybots).
  67   base::MessageLoop::current()->QuitNow();
  68 }
  69
  70 void RunLoop() {
  71   base::MessageLoop::current()->Run();
  72 }
  73
  74 void PumpLoop() {
  75   // Do it this way instead of RunAllPending to pump loop exactly once
  76   // (necessary in the presence of timers; see comment in
  77   // QuitLoopNow).
  78   base::MessageLoop::current()->PostTask(FROM_HERE, base::Bind(&QuitLoopNow));
  79   RunLoop();
  80 }
  81
  82 void PumpLoopFor(base::TimeDelta time) {
  83   // Allow the loop to run for the specified amount of time.
  84   base::MessageLoop::current()->PostDelayedTask(
  85       FROM_HERE, base::Bind(&QuitLoopNow), time);
  86   RunLoop();
  87 }
  88
  89 ModelSafeRoutingInfo TypesToRoutingInfo(ModelTypeSet types) {
  90   ModelSafeRoutingInfo routes;
  91   for (ModelTypeSet::Iterator iter = types.First(); iter.Good(); iter.Inc()) {
  92     routes[iter.Get()] = GROUP_PASSIVE;
  93   }
  94   return routes;
  95 }
  96
  97
  98 static const size_t kMinNumSamples = 5;
  99
 100 // Test harness for the SyncScheduler.  Test the delays and backoff timers used
 101 // in response to various events.
 102 //
 103 // These tests execute in real time with real timers.  We try to keep the
 104 // delays short, but there is a limit to how short we can make them.  The
 105 // timers on some platforms (ie. Windows) have a timer resolution greater than
 106 // 1ms.  Using 1ms delays may result in test flakiness.
 107 //
 108 // See crbug.com/402212 for more info.
 109 class SyncSchedulerTest : public testing::Test {
 110  public:
 111   SyncSchedulerTest() : syncer_(NULL), delay_(NULL), weak_ptr_factory_(this) {}
 112
 113   class MockDelayProvider : public BackoffDelayProvider {
 114    public:
 115     MockDelayProvider() : BackoffDelayProvider(
 116         TimeDelta::FromSeconds(kInitialBackoffRetrySeconds),
 117         TimeDelta::FromSeconds(kInitialBackoffImmediateRetrySeconds)) {
 118     }
 119
 120     MOCK_METHOD1(GetDelay, TimeDelta(const TimeDelta&));
 121   };
 122
 123   virtual void SetUp() {
 124     dir_maker_.SetUp();
 125     syncer_ = new testing::StrictMock<MockSyncer>();
 126     delay_ = NULL;
 127     extensions_activity_ = new ExtensionsActivity();
 128
 129     routing_info_[BOOKMARKS] = GROUP_UI;
 130     routing_info_[AUTOFILL] = GROUP_DB;
 131     routing_info_[THEMES] = GROUP_UI;
 132     routing_info_[NIGORI] = GROUP_PASSIVE;
 133
 134     workers_.clear();
 135     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_UI)));
 136     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_DB)));
 137     workers_.push_back(make_scoped_refptr(new FakeModelWorker(GROUP_PASSIVE)));
 138
 139     connection_.reset(new MockConnectionManager(directory(),
 140                                                 &cancelation_signal_));
 141     connection_->SetServerReachable();
 142
 143     model_type_registry_.reset(
 144         new ModelTypeRegistry(workers_, directory(), &mock_nudge_handler_));
 145
 146     context_.reset(new SyncSessionContext(
 147             connection_.get(), directory(),
 148             extensions_activity_.get(),
 149             std::vector<SyncEngineEventListener*>(), NULL,
 150             model_type_registry_.get(),
 151             true,  // enable keystore encryption
 152             false,  // force enable pre-commit GU avoidance
 153             "fake_invalidator_client_id"));
 154     context_->SetRoutingInfo(routing_info_);
 155     context_->set_notifications_enabled(true);
 156     context_->set_account_name("Test");
 157     scheduler_.reset(
 158         new SyncSchedulerImpl("TestSyncScheduler",
 159             BackoffDelayProvider::FromDefaults(),
 160             context(),
 161             syncer_));
 162   }
 163
 164   SyncSchedulerImpl* scheduler() { return scheduler_.get(); }
 165   const ModelSafeRoutingInfo& routing_info() { return routing_info_; }
 166   MockSyncer* syncer() { return syncer_; }
 167   MockDelayProvider* delay() { return delay_; }
 168   MockConnectionManager* connection() { return connection_.get(); }
 169   TimeDelta zero() { return TimeDelta::FromSeconds(0); }
 170   TimeDelta timeout() {
 171     return TestTimeouts::action_timeout();
 172   }
 173
 174   virtual void TearDown() {
 175     PumpLoop();
 176     scheduler_.reset();
 177     PumpLoop();
 178     dir_maker_.TearDown();
 179   }
 180
 181   void AnalyzePollRun(const SyncShareTimes& times, size_t min_num_samples,
 182       const TimeTicks& optimal_start, const TimeDelta& poll_interval) {
 183     EXPECT_GE(times.size(), min_num_samples);
 184     for (size_t i = 0; i < times.size(); i++) {
 185       SCOPED_TRACE(testing::Message() << "SyncShare # (" << i << ")");
 186       TimeTicks optimal_next_sync = optimal_start + poll_interval * i;
 187       EXPECT_GE(times[i], optimal_next_sync);
 188     }
 189   }
 190
 191   void DoQuitLoopNow() {
 192     QuitLoopNow();
 193   }
 194
 195   void StartSyncScheduler(SyncScheduler::Mode mode) {
 196     scheduler()->Start(mode);
 197   }
 198
 199   // This stops the scheduler synchronously.
 200   void StopSyncScheduler() {
 201     base::MessageLoop::current()->PostTask(
 202         FROM_HERE,
 203         base::Bind(&SyncSchedulerTest::DoQuitLoopNow,
 204                    weak_ptr_factory_.GetWeakPtr()));
 205     RunLoop();
 206   }
 207
 208   bool RunAndGetBackoff() {
 209     ModelTypeSet nudge_types(BOOKMARKS);
 210     StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 211
 212     scheduler()->ScheduleLocalNudge(zero(), nudge_types, FROM_HERE);
 213     RunLoop();
 214
 215     return scheduler()->IsBackingOff();
 216   }
 217
 218   void UseMockDelayProvider() {
 219     delay_ = new MockDelayProvider();
 220     scheduler_->delay_provider_.reset(delay_);
 221   }
 222
 223   SyncSessionContext* context() { return context_.get(); }
 224
 225   ModelTypeSet GetThrottledTypes() {
 226     return scheduler_->nudge_tracker_.GetThrottledTypes();
 227   }
 228
 229   base::TimeDelta GetRetryTimerDelay() {
 230     EXPECT_TRUE(scheduler_->retry_timer_.IsRunning());
 231     return scheduler_->retry_timer_.GetCurrentDelay();
 232   }
 233
 234   static scoped_ptr<InvalidationInterface> BuildInvalidation(
 235       int64 version,
 236       const std::string& payload) {
 237     return MockInvalidation::Build(version, payload)
 238         .PassAs<InvalidationInterface>();
 239   }
 240
 241  private:
 242   syncable::Directory* directory() {
 243     return dir_maker_.directory();
 244   }
 245
 246   base::MessageLoop loop_;
 247   TestDirectorySetterUpper dir_maker_;
 248   CancelationSignal cancelation_signal_;
 249   scoped_ptr<MockConnectionManager> connection_;
 250   scoped_ptr<ModelTypeRegistry> model_type_registry_;
 251   scoped_ptr<SyncSessionContext> context_;
 252   scoped_ptr<SyncSchedulerImpl> scheduler_;
 253   MockNudgeHandler mock_nudge_handler_;
 254   MockSyncer* syncer_;
 255   MockDelayProvider* delay_;
 256   std::vector<scoped_refptr<ModelSafeWorker> > workers_;
 257   scoped_refptr<ExtensionsActivity> extensions_activity_;
 258   ModelSafeRoutingInfo routing_info_;
 259   base::WeakPtrFactory<SyncSchedulerTest> weak_ptr_factory_;
 260 };
 261
 262 void RecordSyncShareImpl(SyncShareTimes* times) {
 263   times->push_back(TimeTicks::Now());
 264 }
 265
 266 ACTION_P(RecordSyncShare, times) {
 267   RecordSyncShareImpl(times);
 268   if (base::MessageLoop::current()->is_running())
 269     QuitLoopNow();
 270   return true;
 271 }
 272
 273 ACTION_P2(RecordSyncShareMultiple, times, quit_after) {
 274   RecordSyncShareImpl(times);
 275   EXPECT_LE(times->size(), quit_after);
 276   if (times->size() >= quit_after &&
 277       base::MessageLoop::current()->is_running()) {
 278     QuitLoopNow();
 279   }
 280   return true;
 281 }
 282
 283 ACTION_P(StopScheduler, scheduler) {
 284   scheduler->Stop();
 285 }
 286
 287 ACTION(AddFailureAndQuitLoopNow) {
 288   ADD_FAILURE();
 289   QuitLoopNow();
 290   return true;
 291 }
 292
 293 ACTION(QuitLoopNowAction) {
 294   QuitLoopNow();
 295   return true;
 296 }
 297
 298 // Test nudge scheduling.
 299 TEST_F(SyncSchedulerTest, Nudge) {
 300   SyncShareTimes times;
 301   ModelTypeSet model_types(BOOKMARKS);
 302
 303   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 304       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 305                       RecordSyncShare(&times)))
 306       .RetiresOnSaturation();
 307
 308   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 309
 310   scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
 311   RunLoop();
 312
 313   Mock::VerifyAndClearExpectations(syncer());
 314
 315   // Make sure a second, later, nudge is unaffected by first (no coalescing).
 316   SyncShareTimes times2;
 317   model_types.Remove(BOOKMARKS);
 318   model_types.Put(AUTOFILL);
 319   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 320       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 321                       RecordSyncShare(&times2)));
 322   scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
 323   RunLoop();
 324 }
 325
 326 // Make sure a regular config command is scheduled fine in the absence of any
 327 // errors.
 328 TEST_F(SyncSchedulerTest, Config) {
 329   SyncShareTimes times;
 330   const ModelTypeSet model_types(BOOKMARKS);
 331
 332   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 333       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 334                       RecordSyncShare(&times)));
 335
 336   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 337
 338   CallbackCounter ready_counter;
 339   CallbackCounter retry_counter;
 340   ConfigurationParams params(
 341       GetUpdatesCallerInfo::RECONFIGURATION,
 342       model_types,
 343       TypesToRoutingInfo(model_types),
 344       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 345       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 346   scheduler()->ScheduleConfiguration(params);
 347   PumpLoop();
 348   ASSERT_EQ(1, ready_counter.times_called());
 349   ASSERT_EQ(0, retry_counter.times_called());
 350 }
 351
 352 // Simulate a failure and make sure the config request is retried.
 353 TEST_F(SyncSchedulerTest, ConfigWithBackingOff) {
 354   UseMockDelayProvider();
 355   EXPECT_CALL(*delay(), GetDelay(_))
 356       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(20)));
 357   SyncShareTimes times;
 358   const ModelTypeSet model_types(BOOKMARKS);
 359
 360   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 361
 362   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 363       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 364                       RecordSyncShare(&times)))
 365       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 366                       RecordSyncShare(&times)));
 367
 368   CallbackCounter ready_counter;
 369   CallbackCounter retry_counter;
 370   ConfigurationParams params(
 371       GetUpdatesCallerInfo::RECONFIGURATION,
 372       model_types,
 373       TypesToRoutingInfo(model_types),
 374       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 375       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 376   scheduler()->ScheduleConfiguration(params);
 377   RunLoop();
 378   ASSERT_EQ(0, ready_counter.times_called());
 379   ASSERT_EQ(1, retry_counter.times_called());
 380
 381   // RunLoop() will trigger TryCanaryJob which will retry configuration.
 382   // Since retry_task was already called it shouldn't be called again.
 383   RunLoop();
 384   ASSERT_EQ(0, ready_counter.times_called());
 385   ASSERT_EQ(1, retry_counter.times_called());
 386
 387   Mock::VerifyAndClearExpectations(syncer());
 388
 389   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 390       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 391                       RecordSyncShare(&times)));
 392   RunLoop();
 393
 394   ASSERT_EQ(1, ready_counter.times_called());
 395 }
 396
 397 // Simuilate SyncSchedulerImpl::Stop being called in the middle of Configure.
 398 // This can happen if server returns NOT_MY_BIRTHDAY.
 399 TEST_F(SyncSchedulerTest, ConfigWithStop) {
 400   UseMockDelayProvider();
 401   EXPECT_CALL(*delay(), GetDelay(_))
 402       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(20)));
 403   SyncShareTimes times;
 404   const ModelTypeSet model_types(BOOKMARKS);
 405
 406   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 407
 408   // Make ConfigureSyncShare call scheduler->Stop(). It is not supposed to call
 409   // retry_task or dereference configuration params.
 410   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 411       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 412                       StopScheduler(scheduler()),
 413                       RecordSyncShare(&times)));
 414
 415   CallbackCounter ready_counter;
 416   CallbackCounter retry_counter;
 417   ConfigurationParams params(
 418       GetUpdatesCallerInfo::RECONFIGURATION,
 419       model_types,
 420       TypesToRoutingInfo(model_types),
 421       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 422       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 423   scheduler()->ScheduleConfiguration(params);
 424   PumpLoop();
 425   ASSERT_EQ(0, ready_counter.times_called());
 426   ASSERT_EQ(0, retry_counter.times_called());
 427 }
 428
 429 // Issue a nudge when the config has failed. Make sure both the config and
 430 // nudge are executed.
 431 TEST_F(SyncSchedulerTest, NudgeWithConfigWithBackingOff) {
 432   const ModelTypeSet model_types(BOOKMARKS);
 433   UseMockDelayProvider();
 434   EXPECT_CALL(*delay(), GetDelay(_))
 435       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(50)));
 436   SyncShareTimes times;
 437
 438   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 439
 440   // Request a configure and make sure it fails.
 441   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 442       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 443                       RecordSyncShare(&times)));
 444   CallbackCounter ready_counter;
 445   CallbackCounter retry_counter;
 446   ConfigurationParams params(
 447       GetUpdatesCallerInfo::RECONFIGURATION,
 448       model_types,
 449       TypesToRoutingInfo(model_types),
 450       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 451       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 452   scheduler()->ScheduleConfiguration(params);
 453   RunLoop();
 454   ASSERT_EQ(0, ready_counter.times_called());
 455   ASSERT_EQ(1, retry_counter.times_called());
 456   Mock::VerifyAndClearExpectations(syncer());
 457
 458   // Ask for a nudge while dealing with repeated configure failure.
 459   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 460       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureFailed),
 461                       RecordSyncShare(&times)));
 462   scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
 463   RunLoop();
 464   // Note that we're not RunLoop()ing for the NUDGE we just scheduled, but
 465   // for the first retry attempt from the config job (after
 466   // waiting ~+/- 50ms).
 467   Mock::VerifyAndClearExpectations(syncer());
 468   ASSERT_EQ(0, ready_counter.times_called());
 469
 470   // Let the next configure retry succeed.
 471   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 472       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 473                       RecordSyncShare(&times)));
 474   RunLoop();
 475
 476   // Now change the mode so nudge can execute.
 477   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 478       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 479                       RecordSyncShare(&times)));
 480   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 481   PumpLoop();
 482 }
 483
 484 // Test that nudges are coalesced.
 485 TEST_F(SyncSchedulerTest, NudgeCoalescing) {
 486   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 487
 488   SyncShareTimes times;
 489   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 490       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 491                       RecordSyncShare(&times)));
 492   const ModelTypeSet types1(BOOKMARKS), types2(AUTOFILL), types3(THEMES);
 493   TimeDelta delay = zero();
 494   TimeTicks optimal_time = TimeTicks::Now() + delay;
 495   scheduler()->ScheduleLocalNudge(delay, types1, FROM_HERE);
 496   scheduler()->ScheduleLocalNudge(zero(), types2, FROM_HERE);
 497   RunLoop();
 498
 499   ASSERT_EQ(1U, times.size());
 500   EXPECT_GE(times[0], optimal_time);
 501
 502   Mock::VerifyAndClearExpectations(syncer());
 503
 504   SyncShareTimes times2;
 505   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 506       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 507                       RecordSyncShare(&times2)));
 508   scheduler()->ScheduleLocalNudge(zero(), types3, FROM_HERE);
 509   RunLoop();
 510 }
 511
 512 // Test that nudges are coalesced.
 513 TEST_F(SyncSchedulerTest, NudgeCoalescingWithDifferentTimings) {
 514   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 515
 516   SyncShareTimes times;
 517   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 518       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 519                       RecordSyncShare(&times)));
 520   ModelTypeSet types1(BOOKMARKS), types2(AUTOFILL), types3;
 521
 522   // Create a huge time delay.
 523   TimeDelta delay = TimeDelta::FromDays(1);
 524
 525   scheduler()->ScheduleLocalNudge(delay, types1, FROM_HERE);
 526   scheduler()->ScheduleLocalNudge(zero(), types2, FROM_HERE);
 527
 528   TimeTicks min_time = TimeTicks::Now();
 529   TimeTicks max_time = TimeTicks::Now() + delay;
 530
 531   RunLoop();
 532   Mock::VerifyAndClearExpectations(syncer());
 533
 534   // Make sure the sync happened at the right time.
 535   ASSERT_EQ(1U, times.size());
 536   EXPECT_GE(times[0], min_time);
 537   EXPECT_LE(times[0], max_time);
 538 }
 539
 540 // Test nudge scheduling.
 541 TEST_F(SyncSchedulerTest, NudgeWithStates) {
 542   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 543
 544   SyncShareTimes times1;
 545   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 546       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 547                       RecordSyncShare(&times1)))
 548       .RetiresOnSaturation();
 549   scheduler()->ScheduleInvalidationNudge(
 550       zero(), BOOKMARKS, BuildInvalidation(10, "test"), FROM_HERE);
 551   RunLoop();
 552
 553   Mock::VerifyAndClearExpectations(syncer());
 554
 555   // Make sure a second, later, nudge is unaffected by first (no coalescing).
 556   SyncShareTimes times2;
 557   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 558       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 559                       RecordSyncShare(&times2)));
 560   scheduler()->ScheduleInvalidationNudge(
 561       zero(), AUTOFILL, BuildInvalidation(10, "test2"), FROM_HERE);
 562   RunLoop();
 563 }
 564
 565 // Test that polling works as expected.
 566 TEST_F(SyncSchedulerTest, Polling) {
 567   SyncShareTimes times;
 568   TimeDelta poll_interval(TimeDelta::FromMilliseconds(30));
 569   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
 570       .WillRepeatedly(
 571           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 572                 RecordSyncShareMultiple(&times, kMinNumSamples)));
 573
 574   scheduler()->OnReceivedLongPollIntervalUpdate(poll_interval);
 575
 576   TimeTicks optimal_start = TimeTicks::Now() + poll_interval;
 577   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 578
 579   // Run again to wait for polling.
 580   RunLoop();
 581
 582   StopSyncScheduler();
 583   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll_interval);
 584 }
 585
 586 // Test that the short poll interval is used.
 587 TEST_F(SyncSchedulerTest, PollNotificationsDisabled) {
 588   SyncShareTimes times;
 589   TimeDelta poll_interval(TimeDelta::FromMilliseconds(30));
 590   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
 591       .WillRepeatedly(
 592           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 593                 RecordSyncShareMultiple(&times, kMinNumSamples)));
 594
 595   scheduler()->OnReceivedShortPollIntervalUpdate(poll_interval);
 596   scheduler()->SetNotificationsEnabled(false);
 597
 598   TimeTicks optimal_start = TimeTicks::Now() + poll_interval;
 599   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 600
 601   // Run again to wait for polling.
 602   RunLoop();
 603
 604   StopSyncScheduler();
 605   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll_interval);
 606 }
 607
 608 // Test that polling intervals are updated when needed.
 609 TEST_F(SyncSchedulerTest, PollIntervalUpdate) {
 610   SyncShareTimes times;
 611   TimeDelta poll1(TimeDelta::FromMilliseconds(120));
 612   TimeDelta poll2(TimeDelta::FromMilliseconds(30));
 613   scheduler()->OnReceivedLongPollIntervalUpdate(poll1);
 614   EXPECT_CALL(*syncer(), PollSyncShare(_,_)).Times(AtLeast(kMinNumSamples))
 615       .WillOnce(DoAll(
 616           WithArgs<0,1>(
 617               sessions::test_util::SimulatePollIntervalUpdate(poll2)),
 618           Return(true)))
 619       .WillRepeatedly(
 620           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 621                 WithArg<1>(
 622                     RecordSyncShareMultiple(&times, kMinNumSamples))));
 623
 624   TimeTicks optimal_start = TimeTicks::Now() + poll1 + poll2;
 625   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 626
 627   // Run again to wait for polling.
 628   RunLoop();
 629
 630   StopSyncScheduler();
 631   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll2);
 632 }
 633
 634 // Test that the sessions commit delay is updated when needed.
 635 TEST_F(SyncSchedulerTest, SessionsCommitDelay) {
 636   SyncShareTimes times;
 637   TimeDelta delay1(TimeDelta::FromMilliseconds(120));
 638   TimeDelta delay2(TimeDelta::FromMilliseconds(30));
 639   scheduler()->OnReceivedSessionsCommitDelay(delay1);
 640
 641   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 642       .WillOnce(
 643           DoAll(
 644               WithArgs<0,1,2>(
 645                   sessions::test_util::SimulateSessionsCommitDelayUpdate(
 646                       delay2)),
 647               Invoke(sessions::test_util::SimulateNormalSuccess),
 648               QuitLoopNowAction()));
 649
 650   EXPECT_EQ(delay1, scheduler()->GetSessionsCommitDelay());
 651   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 652
 653   EXPECT_EQ(delay1, scheduler()->GetSessionsCommitDelay());
 654   const ModelTypeSet model_types(BOOKMARKS);
 655   scheduler()->ScheduleLocalNudge(zero(), model_types, FROM_HERE);
 656   RunLoop();
 657
 658   EXPECT_EQ(delay2, scheduler()->GetSessionsCommitDelay());
 659   StopSyncScheduler();
 660 }
 661
 662 // Test that no syncing occurs when throttled.
 663 TEST_F(SyncSchedulerTest, ThrottlingDoesThrottle) {
 664   const ModelTypeSet types(BOOKMARKS);
 665   TimeDelta poll(TimeDelta::FromMilliseconds(20));
 666   TimeDelta throttle(TimeDelta::FromMinutes(10));
 667   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 668
 669   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 670       .WillOnce(DoAll(
 671           WithArg<2>(sessions::test_util::SimulateThrottled(throttle)),
 672           Return(true)))
 673       .WillRepeatedly(AddFailureAndQuitLoopNow());
 674
 675   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 676
 677   scheduler()->ScheduleLocalNudge(
 678       TimeDelta::FromMicroseconds(1), types, FROM_HERE);
 679   PumpLoop();
 680
 681   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 682
 683   CallbackCounter ready_counter;
 684   CallbackCounter retry_counter;
 685   ConfigurationParams params(
 686       GetUpdatesCallerInfo::RECONFIGURATION,
 687       types,
 688       TypesToRoutingInfo(types),
 689       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 690       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 691   scheduler()->ScheduleConfiguration(params);
 692   PumpLoop();
 693   ASSERT_EQ(0, ready_counter.times_called());
 694   ASSERT_EQ(1, retry_counter.times_called());
 695
 696 }
 697
 698 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromPoll) {
 699   SyncShareTimes times;
 700   TimeDelta poll(TimeDelta::FromMilliseconds(15));
 701   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
 702   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 703
 704   ::testing::InSequence seq;
 705   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
 706       .WillOnce(DoAll(
 707           WithArg<1>(sessions::test_util::SimulateThrottled(throttle1)),
 708           Return(true)))
 709       .RetiresOnSaturation();
 710   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
 711       .WillRepeatedly(
 712           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
 713                 RecordSyncShareMultiple(&times, kMinNumSamples)));
 714
 715   TimeTicks optimal_start = TimeTicks::Now() + poll + throttle1;
 716   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 717
 718   // Run again to wait for polling.
 719   RunLoop();
 720
 721   StopSyncScheduler();
 722   AnalyzePollRun(times, kMinNumSamples, optimal_start, poll);
 723 }
 724
 725 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromNudge) {
 726   SyncShareTimes times;
 727   TimeDelta poll(TimeDelta::FromDays(1));
 728   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
 729   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 730
 731   ::testing::InSequence seq;
 732   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 733       .WillOnce(DoAll(
 734           WithArg<2>(sessions::test_util::SimulateThrottled(throttle1)),
 735           Return(true)))
 736       .RetiresOnSaturation();
 737   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 738       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 739                       QuitLoopNowAction()));
 740
 741   const ModelTypeSet types(BOOKMARKS);
 742   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 743   scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
 744
 745   PumpLoop(); // To get PerformDelayedNudge called.
 746   PumpLoop(); // To get TrySyncSessionJob called
 747   EXPECT_TRUE(scheduler()->IsCurrentlyThrottled());
 748   RunLoop();
 749   EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
 750
 751   StopSyncScheduler();
 752 }
 753
 754 TEST_F(SyncSchedulerTest, ThrottlingExpiresFromConfigure) {
 755   SyncShareTimes times;
 756   TimeDelta poll(TimeDelta::FromDays(1));
 757   TimeDelta throttle1(TimeDelta::FromMilliseconds(150));
 758   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 759
 760   ::testing::InSequence seq;
 761   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 762       .WillOnce(DoAll(
 763           WithArg<2>(sessions::test_util::SimulateThrottled(throttle1)),
 764           Return(true)))
 765       .RetiresOnSaturation();
 766   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 767       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 768                       QuitLoopNowAction()));
 769
 770   const ModelTypeSet types(BOOKMARKS);
 771   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 772
 773   CallbackCounter ready_counter;
 774   CallbackCounter retry_counter;
 775   ConfigurationParams params(
 776       GetUpdatesCallerInfo::RECONFIGURATION,
 777       types,
 778       TypesToRoutingInfo(types),
 779       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 780       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 781   scheduler()->ScheduleConfiguration(params);
 782   PumpLoop();
 783   EXPECT_EQ(0, ready_counter.times_called());
 784   EXPECT_EQ(1, retry_counter.times_called());
 785   EXPECT_TRUE(scheduler()->IsCurrentlyThrottled());
 786
 787   RunLoop();
 788   EXPECT_FALSE(scheduler()->IsCurrentlyThrottled());
 789
 790   StopSyncScheduler();
 791 }
 792
 793 TEST_F(SyncSchedulerTest, TypeThrottlingBlocksNudge) {
 794   UseMockDelayProvider();
 795   EXPECT_CALL(*delay(), GetDelay(_))
 796       .WillRepeatedly(Return(zero()));
 797
 798   TimeDelta poll(TimeDelta::FromDays(1));
 799   TimeDelta throttle1(TimeDelta::FromSeconds(60));
 800   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 801
 802   const ModelTypeSet types(BOOKMARKS);
 803
 804   ::testing::InSequence seq;
 805   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 806       .WillOnce(DoAll(
 807           WithArg<2>(
 808               sessions::test_util::SimulateTypesThrottled(types, throttle1)),
 809           Return(true)))
 810       .RetiresOnSaturation();
 811
 812   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 813   scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
 814   PumpLoop(); // To get PerformDelayedNudge called.
 815   PumpLoop(); // To get TrySyncSessionJob called
 816   EXPECT_TRUE(GetThrottledTypes().HasAll(types));
 817
 818   // This won't cause a sync cycle because the types are throttled.
 819   scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
 820   PumpLoop();
 821
 822   StopSyncScheduler();
 823 }
 824
 825 TEST_F(SyncSchedulerTest, TypeThrottlingDoesBlockOtherSources) {
 826   UseMockDelayProvider();
 827   EXPECT_CALL(*delay(), GetDelay(_))
 828       .WillRepeatedly(Return(zero()));
 829
 830   SyncShareTimes times;
 831   TimeDelta poll(TimeDelta::FromDays(1));
 832   TimeDelta throttle1(TimeDelta::FromSeconds(60));
 833   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 834
 835   const ModelTypeSet throttled_types(BOOKMARKS);
 836   const ModelTypeSet unthrottled_types(PREFERENCES);
 837
 838   ::testing::InSequence seq;
 839   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 840       .WillOnce(DoAll(
 841           WithArg<2>(
 842               sessions::test_util::SimulateTypesThrottled(
 843                   throttled_types, throttle1)),
 844           Return(true)))
 845       .RetiresOnSaturation();
 846
 847   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 848   scheduler()->ScheduleLocalNudge(zero(), throttled_types, FROM_HERE);
 849   PumpLoop(); // To get PerformDelayedNudge called.
 850   PumpLoop(); // To get TrySyncSessionJob called
 851   EXPECT_TRUE(GetThrottledTypes().HasAll(throttled_types));
 852
 853   // Ignore invalidations for throttled types.
 854   scheduler()->ScheduleInvalidationNudge(
 855       zero(), BOOKMARKS, BuildInvalidation(10, "test"), FROM_HERE);
 856   PumpLoop();
 857
 858   // Ignore refresh requests for throttled types.
 859   scheduler()->ScheduleLocalRefreshRequest(zero(), throttled_types, FROM_HERE);
 860   PumpLoop();
 861
 862   Mock::VerifyAndClearExpectations(syncer());
 863
 864   // Local nudges for non-throttled types will trigger a sync.
 865   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 866       .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 867                             RecordSyncShare(&times)));
 868   scheduler()->ScheduleLocalNudge(zero(), unthrottled_types, FROM_HERE);
 869   RunLoop();
 870   Mock::VerifyAndClearExpectations(syncer());
 871
 872   StopSyncScheduler();
 873 }
 874
 875 // Test nudges / polls don't run in config mode and config tasks do.
 876 TEST_F(SyncSchedulerTest, ConfigurationMode) {
 877   TimeDelta poll(TimeDelta::FromMilliseconds(15));
 878   SyncShareTimes times;
 879   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
 880
 881   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 882
 883   const ModelTypeSet nudge_types(AUTOFILL);
 884   scheduler()->ScheduleLocalNudge(zero(), nudge_types, FROM_HERE);
 885   scheduler()->ScheduleLocalNudge(zero(), nudge_types, FROM_HERE);
 886
 887   const ModelTypeSet config_types(BOOKMARKS);
 888
 889   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 890       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConfigureSuccess),
 891                       RecordSyncShare(&times)))
 892       .RetiresOnSaturation();
 893   CallbackCounter ready_counter;
 894   CallbackCounter retry_counter;
 895   ConfigurationParams params(
 896       GetUpdatesCallerInfo::RECONFIGURATION,
 897       config_types,
 898       TypesToRoutingInfo(config_types),
 899       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
 900       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
 901   scheduler()->ScheduleConfiguration(params);
 902   RunLoop();
 903   ASSERT_EQ(1, ready_counter.times_called());
 904   ASSERT_EQ(0, retry_counter.times_called());
 905
 906   Mock::VerifyAndClearExpectations(syncer());
 907
 908   // Switch to NORMAL_MODE to ensure NUDGES were properly saved and run.
 909   scheduler()->OnReceivedLongPollIntervalUpdate(TimeDelta::FromDays(1));
 910   SyncShareTimes times2;
 911   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 912       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 913                       RecordSyncShare(&times2)));
 914
 915   // TODO(tim): Figure out how to remove this dangerous need to reset
 916   // routing info between mode switches.
 917   context()->SetRoutingInfo(routing_info());
 918   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
 919
 920   RunLoop();
 921   Mock::VerifyAndClearExpectations(syncer());
 922 }
 923
 924 class BackoffTriggersSyncSchedulerTest : public SyncSchedulerTest {
 925   virtual void SetUp() {
 926     SyncSchedulerTest::SetUp();
 927     UseMockDelayProvider();
 928     EXPECT_CALL(*delay(), GetDelay(_))
 929         .WillRepeatedly(Return(TimeDelta::FromMilliseconds(10)));
 930   }
 931
 932   virtual void TearDown() {
 933     StopSyncScheduler();
 934     SyncSchedulerTest::TearDown();
 935   }
 936 };
 937
 938 // Have the sycner fail during commit.  Expect that the scheduler enters
 939 // backoff.
 940 TEST_F(BackoffTriggersSyncSchedulerTest, FailCommitOnce) {
 941   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 942       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
 943                       QuitLoopNowAction()));
 944   EXPECT_TRUE(RunAndGetBackoff());
 945 }
 946
 947 // Have the syncer fail during download updates and succeed on the first
 948 // retry.  Expect that this clears the backoff state.
 949 TEST_F(BackoffTriggersSyncSchedulerTest, FailDownloadOnceThenSucceed) {
 950   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 951       .WillOnce(DoAll(
 952           Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
 953           Return(true)))
 954       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 955                       QuitLoopNowAction()));
 956   EXPECT_FALSE(RunAndGetBackoff());
 957 }
 958
 959 // Have the syncer fail during commit and succeed on the first retry.  Expect
 960 // that this clears the backoff state.
 961 TEST_F(BackoffTriggersSyncSchedulerTest, FailCommitOnceThenSucceed) {
 962   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 963       .WillOnce(DoAll(
 964           Invoke(sessions::test_util::SimulateCommitFailed),
 965           Return(true)))
 966       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
 967                       QuitLoopNowAction()));
 968   EXPECT_FALSE(RunAndGetBackoff());
 969 }
 970
 971 // Have the syncer fail to download updates and fail again on the retry.
 972 // Expect this will leave the scheduler in backoff.
 973 TEST_F(BackoffTriggersSyncSchedulerTest, FailDownloadTwice) {
 974   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
 975       .WillOnce(DoAll(
 976           Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
 977           Return(true)))
 978       .WillRepeatedly(DoAll(
 979               Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
 980               QuitLoopNowAction()));
 981   EXPECT_TRUE(RunAndGetBackoff());
 982 }
 983
 984 // Have the syncer fail to get the encryption key yet succeed in downloading
 985 // updates. Expect this will leave the scheduler in backoff.
 986 TEST_F(BackoffTriggersSyncSchedulerTest, FailGetEncryptionKey) {
 987   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
 988       .WillOnce(DoAll(
 989           Invoke(sessions::test_util::SimulateGetEncryptionKeyFailed),
 990           Return(true)))
 991       .WillRepeatedly(DoAll(
 992               Invoke(sessions::test_util::SimulateGetEncryptionKeyFailed),
 993               QuitLoopNowAction()));
 994   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
 995
 996   ModelTypeSet types(BOOKMARKS);
 997   CallbackCounter ready_counter;
 998   CallbackCounter retry_counter;
 999   ConfigurationParams params(
1000       GetUpdatesCallerInfo::RECONFIGURATION,
1001       types,
1002       TypesToRoutingInfo(types),
1003       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
1004       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
1005   scheduler()->ScheduleConfiguration(params);
1006   RunLoop();
1007
1008   EXPECT_TRUE(scheduler()->IsBackingOff());
1009 }
1010
1011 // Test that no polls or extraneous nudges occur when in backoff.
1012 TEST_F(SyncSchedulerTest, BackoffDropsJobs) {
1013   SyncShareTimes times;
1014   TimeDelta poll(TimeDelta::FromMilliseconds(10));
1015   const ModelTypeSet types(BOOKMARKS);
1016   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
1017   UseMockDelayProvider();
1018
1019   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1020       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
1021                       RecordSyncShareMultiple(&times, 1U)));
1022   EXPECT_CALL(*delay(), GetDelay(_)).
1023       WillRepeatedly(Return(TimeDelta::FromDays(1)));
1024
1025   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1026
1027   // This nudge should fail and put us into backoff.  Thanks to our mock
1028   // GetDelay() setup above, this will be a long backoff.
1029   scheduler()->ScheduleLocalNudge(zero(), types, FROM_HERE);
1030   RunLoop();
1031
1032   // From this point forward, no SyncShare functions should be invoked.
1033   Mock::VerifyAndClearExpectations(syncer());
1034
1035   // Wait a while (10x poll interval) so a few poll jobs will be attempted.
1036   PumpLoopFor(poll * 10);
1037
1038   // Try (and fail) to schedule a nudge.
1039   scheduler()->ScheduleLocalNudge(
1040       base::TimeDelta::FromMilliseconds(10),
1041       types,
1042       FROM_HERE);
1043
1044   Mock::VerifyAndClearExpectations(syncer());
1045   Mock::VerifyAndClearExpectations(delay());
1046
1047   EXPECT_CALL(*delay(), GetDelay(_)).Times(0);
1048
1049   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
1050
1051   CallbackCounter ready_counter;
1052   CallbackCounter retry_counter;
1053   ConfigurationParams params(
1054       GetUpdatesCallerInfo::RECONFIGURATION,
1055       types,
1056       TypesToRoutingInfo(types),
1057       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
1058       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
1059   scheduler()->ScheduleConfiguration(params);
1060   PumpLoop();
1061   ASSERT_EQ(0, ready_counter.times_called());
1062   ASSERT_EQ(1, retry_counter.times_called());
1063
1064 }
1065
1066 // Test that backoff is shaping traffic properly with consecutive errors.
1067 TEST_F(SyncSchedulerTest, BackoffElevation) {
1068   SyncShareTimes times;
1069   UseMockDelayProvider();
1070
1071   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_)).Times(kMinNumSamples)
1072       .WillRepeatedly(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
1073           RecordSyncShareMultiple(&times, kMinNumSamples)));
1074
1075   const TimeDelta first = TimeDelta::FromSeconds(kInitialBackoffRetrySeconds);
1076   const TimeDelta second = TimeDelta::FromMilliseconds(20);
1077   const TimeDelta third = TimeDelta::FromMilliseconds(30);
1078   const TimeDelta fourth = TimeDelta::FromMilliseconds(40);
1079   const TimeDelta fifth = TimeDelta::FromMilliseconds(50);
1080   const TimeDelta sixth = TimeDelta::FromDays(1);
1081
1082   EXPECT_CALL(*delay(), GetDelay(first)).WillOnce(Return(second))
1083           .RetiresOnSaturation();
1084   EXPECT_CALL(*delay(), GetDelay(second)).WillOnce(Return(third))
1085           .RetiresOnSaturation();
1086   EXPECT_CALL(*delay(), GetDelay(third)).WillOnce(Return(fourth))
1087           .RetiresOnSaturation();
1088   EXPECT_CALL(*delay(), GetDelay(fourth)).WillOnce(Return(fifth))
1089           .RetiresOnSaturation();
1090   EXPECT_CALL(*delay(), GetDelay(fifth)).WillOnce(Return(sixth));
1091
1092   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1093
1094   // Run again with a nudge.
1095   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1096   RunLoop();
1097
1098   ASSERT_EQ(kMinNumSamples, times.size());
1099   EXPECT_GE(times[1] - times[0], second);
1100   EXPECT_GE(times[2] - times[1], third);
1101   EXPECT_GE(times[3] - times[2], fourth);
1102   EXPECT_GE(times[4] - times[3], fifth);
1103 }
1104
1105 // Test that things go back to normal once a retry makes forward progress.
1106 TEST_F(SyncSchedulerTest, BackoffRelief) {
1107   SyncShareTimes times;
1108   const TimeDelta poll(TimeDelta::FromMilliseconds(10));
1109   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
1110   UseMockDelayProvider();
1111
1112   const TimeDelta backoff = TimeDelta::FromMilliseconds(10);
1113   EXPECT_CALL(*delay(), GetDelay(_)).WillOnce(Return(backoff));
1114
1115   // Optimal start for the post-backoff poll party.
1116   TimeTicks optimal_start = TimeTicks::Now();
1117   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1118
1119   // Kick off the test with a failed nudge.
1120   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1121       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateCommitFailed),
1122                       RecordSyncShare(&times)));
1123   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1124   RunLoop();
1125   Mock::VerifyAndClearExpectations(syncer());
1126   TimeTicks optimal_job_time = optimal_start;
1127   ASSERT_EQ(1U, times.size());
1128   EXPECT_GE(times[0], optimal_job_time);
1129
1130   // The retry succeeds.
1131   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1132       .WillOnce(DoAll(
1133               Invoke(sessions::test_util::SimulateNormalSuccess),
1134               RecordSyncShare(&times)));
1135   RunLoop();
1136   Mock::VerifyAndClearExpectations(syncer());
1137   optimal_job_time = optimal_job_time + backoff;
1138   ASSERT_EQ(2U, times.size());
1139   EXPECT_GE(times[1], optimal_job_time);
1140
1141   // Now let the Poll timer do its thing.
1142   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1143       .WillRepeatedly(DoAll(
1144               Invoke(sessions::test_util::SimulatePollSuccess),
1145               RecordSyncShareMultiple(&times, kMinNumSamples)));
1146   RunLoop();
1147   Mock::VerifyAndClearExpectations(syncer());
1148   ASSERT_EQ(kMinNumSamples, times.size());
1149   for (size_t i = 2; i < times.size(); i++) {
1150     optimal_job_time = optimal_job_time + poll;
1151     SCOPED_TRACE(testing::Message() << "SyncShare # (" << i << ")");
1152     EXPECT_GE(times[i], optimal_job_time);
1153   }
1154
1155   StopSyncScheduler();
1156 }
1157
1158 // Test that poll failures are ignored.  They should have no effect on
1159 // subsequent poll attempts, nor should they trigger a backoff/retry.
1160 TEST_F(SyncSchedulerTest, TransientPollFailure) {
1161   SyncShareTimes times;
1162   const TimeDelta poll_interval(TimeDelta::FromMilliseconds(10));
1163   scheduler()->OnReceivedLongPollIntervalUpdate(poll_interval);
1164   UseMockDelayProvider(); // Will cause test failure if backoff is initiated.
1165
1166   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1167       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollFailed),
1168                       RecordSyncShare(&times)))
1169       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
1170                       RecordSyncShare(&times)));
1171
1172   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1173
1174   // Run the unsucessful poll. The failed poll should not trigger backoff.
1175   RunLoop();
1176   EXPECT_FALSE(scheduler()->IsBackingOff());
1177
1178   // Run the successful poll.
1179   RunLoop();
1180   EXPECT_FALSE(scheduler()->IsBackingOff());
1181 }
1182
1183 // Test that starting the syncer thread without a valid connection doesn't
1184 // break things when a connection is detected.
1185 TEST_F(SyncSchedulerTest, StartWhenNotConnected) {
1186   connection()->SetServerNotReachable();
1187   connection()->UpdateConnectionStatus();
1188   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1189     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1190                     Return(true)))
1191     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1192                     Return(true)));
1193   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1194
1195   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1196   // Should save the nudge for until after the server is reachable.
1197   base::MessageLoop::current()->RunUntilIdle();
1198
1199   scheduler()->OnConnectionStatusChange();
1200   connection()->SetServerReachable();
1201   connection()->UpdateConnectionStatus();
1202   base::MessageLoop::current()->RunUntilIdle();
1203 }
1204
1205 TEST_F(SyncSchedulerTest, ServerConnectionChangeDuringBackoff) {
1206   UseMockDelayProvider();
1207   EXPECT_CALL(*delay(), GetDelay(_))
1208       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(0)));
1209
1210   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1211   connection()->SetServerNotReachable();
1212   connection()->UpdateConnectionStatus();
1213
1214   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1215     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1216                     Return(true)))
1217     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1218                     Return(true)));
1219
1220   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1221   PumpLoop(); // To get PerformDelayedNudge called.
1222   PumpLoop(); // Run the nudge, that will fail and schedule a quick retry.
1223   ASSERT_TRUE(scheduler()->IsBackingOff());
1224
1225   // Before we run the scheduled canary, trigger a server connection change.
1226   scheduler()->OnConnectionStatusChange();
1227   connection()->SetServerReachable();
1228   connection()->UpdateConnectionStatus();
1229   base::MessageLoop::current()->RunUntilIdle();
1230 }
1231
1232 // This was supposed to test the scenario where we receive a nudge while a
1233 // connection change canary is scheduled, but has not run yet.  Since we've made
1234 // the connection change canary synchronous, this is no longer possible.
1235 TEST_F(SyncSchedulerTest, ConnectionChangeCanaryPreemptedByNudge) {
1236   UseMockDelayProvider();
1237   EXPECT_CALL(*delay(), GetDelay(_))
1238       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(0)));
1239
1240   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1241   connection()->SetServerNotReachable();
1242   connection()->UpdateConnectionStatus();
1243
1244   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1245     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateConnectionFailure),
1246                     Return(true)))
1247     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1248                     Return(true)))
1249     .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1250                     QuitLoopNowAction()));
1251
1252   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1253
1254   PumpLoop(); // To get PerformDelayedNudge called.
1255   PumpLoop(); // Run the nudge, that will fail and schedule a quick retry.
1256   ASSERT_TRUE(scheduler()->IsBackingOff());
1257
1258   // Before we run the scheduled canary, trigger a server connection change.
1259   scheduler()->OnConnectionStatusChange();
1260   PumpLoop();
1261   connection()->SetServerReachable();
1262   connection()->UpdateConnectionStatus();
1263   scheduler()->ScheduleLocalNudge(zero(), ModelTypeSet(BOOKMARKS), FROM_HERE);
1264   base::MessageLoop::current()->RunUntilIdle();
1265 }
1266
1267 // Tests that we don't crash trying to run two canaries at once if we receive
1268 // extra connection status change notifications.  See crbug.com/190085.
1269 TEST_F(SyncSchedulerTest, DoubleCanaryInConfigure) {
1270   EXPECT_CALL(*syncer(), ConfigureSyncShare(_,_,_))
1271       .WillRepeatedly(DoAll(
1272               Invoke(sessions::test_util::SimulateConfigureConnectionFailure),
1273               Return(true)));
1274   StartSyncScheduler(SyncScheduler::CONFIGURATION_MODE);
1275   connection()->SetServerNotReachable();
1276   connection()->UpdateConnectionStatus();
1277
1278   ModelTypeSet model_types(BOOKMARKS);
1279   CallbackCounter ready_counter;
1280   CallbackCounter retry_counter;
1281   ConfigurationParams params(
1282       GetUpdatesCallerInfo::RECONFIGURATION,
1283       model_types,
1284       TypesToRoutingInfo(model_types),
1285       base::Bind(&CallbackCounter::Callback, base::Unretained(&ready_counter)),
1286       base::Bind(&CallbackCounter::Callback, base::Unretained(&retry_counter)));
1287   scheduler()->ScheduleConfiguration(params);
1288
1289   scheduler()->OnConnectionStatusChange();
1290   scheduler()->OnConnectionStatusChange();
1291
1292   PumpLoop();  // Run the nudge, that will fail and schedule a quick retry.
1293 }
1294
1295 TEST_F(SyncSchedulerTest, PollFromCanaryAfterAuthError) {
1296   SyncShareTimes times;
1297   TimeDelta poll(TimeDelta::FromMilliseconds(15));
1298   scheduler()->OnReceivedLongPollIntervalUpdate(poll);
1299
1300   ::testing::InSequence seq;
1301   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1302       .WillRepeatedly(
1303           DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
1304                 RecordSyncShareMultiple(&times, kMinNumSamples)));
1305
1306   connection()->SetServerStatus(HttpResponse::SYNC_AUTH_ERROR);
1307   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1308
1309   // Run to wait for polling.
1310   RunLoop();
1311
1312   // Normally OnCredentialsUpdated calls TryCanaryJob that doesn't run Poll,
1313   // but after poll finished with auth error from poll timer it should retry
1314   // poll once more
1315   EXPECT_CALL(*syncer(), PollSyncShare(_,_))
1316       .WillOnce(DoAll(Invoke(sessions::test_util::SimulatePollSuccess),
1317                       RecordSyncShare(&times)));
1318   scheduler()->OnCredentialsUpdated();
1319   connection()->SetServerStatus(HttpResponse::SERVER_CONNECTION_OK);
1320   RunLoop();
1321   StopSyncScheduler();
1322 }
1323
1324 TEST_F(SyncSchedulerTest, SuccessfulRetry) {
1325   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1326
1327   SyncShareTimes times;
1328   base::TimeDelta delay = base::TimeDelta::FromMilliseconds(10);
1329   scheduler()->OnReceivedGuRetryDelay(delay);
1330   EXPECT_EQ(delay, GetRetryTimerDelay());
1331
1332   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1333       .WillOnce(
1334           DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1335                 RecordSyncShare(&times)));
1336
1337   // Run to wait for retrying.
1338   RunLoop();
1339
1340   StopSyncScheduler();
1341 }
1342
1343 TEST_F(SyncSchedulerTest, FailedRetry) {
1344   UseMockDelayProvider();
1345   EXPECT_CALL(*delay(), GetDelay(_))
1346       .WillRepeatedly(Return(TimeDelta::FromMilliseconds(10)));
1347
1348   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1349
1350   base::TimeDelta delay = base::TimeDelta::FromMilliseconds(10);
1351   scheduler()->OnReceivedGuRetryDelay(delay);
1352
1353   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1354       .WillOnce(
1355           DoAll(Invoke(sessions::test_util::SimulateDownloadUpdatesFailed),
1356                 QuitLoopNowAction()));
1357
1358   // Run to wait for retrying.
1359   RunLoop();
1360
1361   EXPECT_TRUE(scheduler()->IsBackingOff());
1362   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1363       .WillOnce(
1364           DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1365                 QuitLoopNowAction()));
1366
1367   // Run to wait for second retrying.
1368   RunLoop();
1369
1370   StopSyncScheduler();
1371 }
1372
1373 ACTION_P2(VerifyRetryTimerDelay, scheduler_test, expected_delay) {
1374   EXPECT_EQ(expected_delay, scheduler_test->GetRetryTimerDelay());
1375 }
1376
1377 TEST_F(SyncSchedulerTest, ReceiveNewRetryDelay) {
1378   StartSyncScheduler(SyncScheduler::NORMAL_MODE);
1379
1380   SyncShareTimes times;
1381   base::TimeDelta delay1 = base::TimeDelta::FromMilliseconds(100);
1382   base::TimeDelta delay2 = base::TimeDelta::FromMilliseconds(200);
1383
1384   scheduler()->ScheduleLocalRefreshRequest(zero(), ModelTypeSet(BOOKMARKS),
1385                                            FROM_HERE);
1386   scheduler()->OnReceivedGuRetryDelay(delay1);
1387   EXPECT_EQ(delay1, GetRetryTimerDelay());
1388
1389   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1390       .WillOnce(DoAll(
1391           WithoutArgs(VerifyRetryTimerDelay(this, delay1)),
1392           WithArg<2>(sessions::test_util::SimulateGuRetryDelayCommand(delay2)),
1393           RecordSyncShare(&times)));
1394
1395   // Run nudge GU.
1396   RunLoop();
1397   EXPECT_EQ(delay2, GetRetryTimerDelay());
1398
1399   EXPECT_CALL(*syncer(), NormalSyncShare(_,_,_))
1400       .WillOnce(DoAll(Invoke(sessions::test_util::SimulateNormalSuccess),
1401                       RecordSyncShare(&times)));
1402
1403   // Run to wait for retrying.
1404   RunLoop();
1405
1406   StopSyncScheduler();
1407 }
1408
1409 }  // namespace syncer