src/net/socket/client_socket_pool_base.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/socket/client_socket_pool_base.h"
   6
   7 #include "base/compiler_specific.h"
   8 #include "base/format_macros.h"
   9 #include "base/logging.h"
  10 #include "base/message_loop/message_loop.h"
  11 #include "base/metrics/stats_counters.h"
  12 #include "base/stl_util.h"
  13 #include "base/strings/string_util.h"
  14 #include "base/time/time.h"
  15 #include "base/values.h"
  16 #include "net/base/net_errors.h"
  17 #include "net/base/net_log.h"
  18 #include "net/socket/client_socket_handle.h"
  19
  20 using base::TimeDelta;
  21
  22 namespace net {
  23
  24 namespace {
  25
  26 // Indicate whether we should enable idle socket cleanup timer. When timer is
  27 // disabled, sockets are closed next time a socket request is made.
  28 bool g_cleanup_timer_enabled = true;
  29
  30 // The timeout value, in seconds, used to clean up idle sockets that can't be
  31 // reused.
  32 //
  33 // Note: It's important to close idle sockets that have received data as soon
  34 // as possible because the received data may cause BSOD on Windows XP under
  35 // some conditions.  See http://crbug.com/4606.
  36 const int kCleanupInterval = 10;  // DO NOT INCREASE THIS TIMEOUT.
  37
  38 // Indicate whether or not we should establish a new transport layer connection
  39 // after a certain timeout has passed without receiving an ACK.
  40 bool g_connect_backup_jobs_enabled = true;
  41
  42 }  // namespace
  43
  44 ConnectJob::ConnectJob(const std::string& group_name,
  45                        base::TimeDelta timeout_duration,
  46                        RequestPriority priority,
  47                        Delegate* delegate,
  48                        const BoundNetLog& net_log)
  49     : group_name_(group_name),
  50       timeout_duration_(timeout_duration),
  51       priority_(priority),
  52       delegate_(delegate),
  53       net_log_(net_log),
  54       idle_(true) {
  55   DCHECK(!group_name.empty());
  56   DCHECK(delegate);
  57   net_log.BeginEvent(NetLog::TYPE_SOCKET_POOL_CONNECT_JOB,
  58                      NetLog::StringCallback("group_name", &group_name_));
  59 }
  60
  61 ConnectJob::~ConnectJob() {
  62   net_log().EndEvent(NetLog::TYPE_SOCKET_POOL_CONNECT_JOB);
  63 }
  64
  65 scoped_ptr<StreamSocket> ConnectJob::PassSocket() {
  66   return socket_.Pass();
  67 }
  68
  69 int ConnectJob::Connect() {
  70   if (timeout_duration_ != base::TimeDelta())
  71     timer_.Start(FROM_HERE, timeout_duration_, this, &ConnectJob::OnTimeout);
  72
  73   idle_ = false;
  74
  75   LogConnectStart();
  76
  77   int rv = ConnectInternal();
  78
  79   if (rv != ERR_IO_PENDING) {
  80     LogConnectCompletion(rv);
  81     delegate_ = NULL;
  82   }
  83
  84   return rv;
  85 }
  86
  87 void ConnectJob::SetSocket(scoped_ptr<StreamSocket> socket) {
  88   if (socket) {
  89     net_log().AddEvent(NetLog::TYPE_CONNECT_JOB_SET_SOCKET,
  90                        socket->NetLog().source().ToEventParametersCallback());
  91   }
  92   socket_ = socket.Pass();
  93 }
  94
  95 void ConnectJob::NotifyDelegateOfCompletion(int rv) {
  96   // The delegate will own |this|.
  97   Delegate* delegate = delegate_;
  98   delegate_ = NULL;
  99
 100   LogConnectCompletion(rv);
 101   delegate->OnConnectJobComplete(rv, this);
 102 }
 103
 104 void ConnectJob::ResetTimer(base::TimeDelta remaining_time) {
 105   timer_.Stop();
 106   timer_.Start(FROM_HERE, remaining_time, this, &ConnectJob::OnTimeout);
 107 }
 108
 109 void ConnectJob::LogConnectStart() {
 110   connect_timing_.connect_start = base::TimeTicks::Now();
 111   net_log().BeginEvent(NetLog::TYPE_SOCKET_POOL_CONNECT_JOB_CONNECT);
 112 }
 113
 114 void ConnectJob::LogConnectCompletion(int net_error) {
 115   connect_timing_.connect_end = base::TimeTicks::Now();
 116   net_log().EndEventWithNetErrorCode(
 117       NetLog::TYPE_SOCKET_POOL_CONNECT_JOB_CONNECT, net_error);
 118 }
 119
 120 void ConnectJob::OnTimeout() {
 121   // Make sure the socket is NULL before calling into |delegate|.
 122   SetSocket(scoped_ptr<StreamSocket>());
 123
 124   net_log_.AddEvent(NetLog::TYPE_SOCKET_POOL_CONNECT_JOB_TIMED_OUT);
 125
 126   NotifyDelegateOfCompletion(ERR_TIMED_OUT);
 127 }
 128
 129 namespace internal {
 130
 131 ClientSocketPoolBaseHelper::Request::Request(
 132     ClientSocketHandle* handle,
 133     const CompletionCallback& callback,
 134     RequestPriority priority,
 135     bool ignore_limits,
 136     Flags flags,
 137     const BoundNetLog& net_log)
 138     : handle_(handle),
 139       callback_(callback),
 140       priority_(priority),
 141       ignore_limits_(ignore_limits),
 142       flags_(flags),
 143       net_log_(net_log) {}
 144
 145 ClientSocketPoolBaseHelper::Request::~Request() {}
 146
 147 ClientSocketPoolBaseHelper::ClientSocketPoolBaseHelper(
 148     HigherLayeredPool* pool,
 149     int max_sockets,
 150     int max_sockets_per_group,
 151     base::TimeDelta unused_idle_socket_timeout,
 152     base::TimeDelta used_idle_socket_timeout,
 153     ConnectJobFactory* connect_job_factory)
 154     : idle_socket_count_(0),
 155       connecting_socket_count_(0),
 156       handed_out_socket_count_(0),
 157       max_sockets_(max_sockets),
 158       max_sockets_per_group_(max_sockets_per_group),
 159       use_cleanup_timer_(g_cleanup_timer_enabled),
 160       unused_idle_socket_timeout_(unused_idle_socket_timeout),
 161       used_idle_socket_timeout_(used_idle_socket_timeout),
 162       connect_job_factory_(connect_job_factory),
 163       connect_backup_jobs_enabled_(false),
 164       pool_generation_number_(0),
 165       pool_(pool),
 166       weak_factory_(this) {
 167   DCHECK_LE(0, max_sockets_per_group);
 168   DCHECK_LE(max_sockets_per_group, max_sockets);
 169
 170   NetworkChangeNotifier::AddIPAddressObserver(this);
 171 }
 172
 173 ClientSocketPoolBaseHelper::~ClientSocketPoolBaseHelper() {
 174   // Clean up any idle sockets and pending connect jobs.  Assert that we have no
 175   // remaining active sockets or pending requests.  They should have all been
 176   // cleaned up prior to |this| being destroyed.
 177   FlushWithError(ERR_ABORTED);
 178   DCHECK(group_map_.empty());
 179   DCHECK(pending_callback_map_.empty());
 180   DCHECK_EQ(0, connecting_socket_count_);
 181   CHECK(higher_pools_.empty());
 182
 183   NetworkChangeNotifier::RemoveIPAddressObserver(this);
 184
 185   // Remove from lower layer pools.
 186   for (std::set<LowerLayeredPool*>::iterator it = lower_pools_.begin();
 187        it != lower_pools_.end();
 188        ++it) {
 189     (*it)->RemoveHigherLayeredPool(pool_);
 190   }
 191 }
 192
 193 ClientSocketPoolBaseHelper::CallbackResultPair::CallbackResultPair()
 194     : result(OK) {
 195 }
 196
 197 ClientSocketPoolBaseHelper::CallbackResultPair::CallbackResultPair(
 198     const CompletionCallback& callback_in, int result_in)
 199     : callback(callback_in),
 200       result(result_in) {
 201 }
 202
 203 ClientSocketPoolBaseHelper::CallbackResultPair::~CallbackResultPair() {}
 204
 205 bool ClientSocketPoolBaseHelper::IsStalled() const {
 206   // If a lower layer pool is stalled, consider |this| stalled as well.
 207   for (std::set<LowerLayeredPool*>::const_iterator it = lower_pools_.begin();
 208        it != lower_pools_.end();
 209        ++it) {
 210     if ((*it)->IsStalled())
 211       return true;
 212   }
 213
 214   // If fewer than |max_sockets_| are in use, then clearly |this| is not
 215   // stalled.
 216   if ((handed_out_socket_count_ + connecting_socket_count_) < max_sockets_)
 217     return false;
 218   // So in order to be stalled, |this| must be using at least |max_sockets_| AND
 219   // |this| must have a request that is actually stalled on the global socket
 220   // limit.  To find such a request, look for a group that has more requests
 221   // than jobs AND where the number of sockets is less than
 222   // |max_sockets_per_group_|.  (If the number of sockets is equal to
 223   // |max_sockets_per_group_|, then the request is stalled on the group limit,
 224   // which does not count.)
 225   for (GroupMap::const_iterator it = group_map_.begin();
 226        it != group_map_.end(); ++it) {
 227     if (it->second->IsStalledOnPoolMaxSockets(max_sockets_per_group_))
 228       return true;
 229   }
 230   return false;
 231 }
 232
 233 void ClientSocketPoolBaseHelper::AddLowerLayeredPool(
 234     LowerLayeredPool* lower_pool) {
 235   DCHECK(pool_);
 236   CHECK(!ContainsKey(lower_pools_, lower_pool));
 237   lower_pools_.insert(lower_pool);
 238   lower_pool->AddHigherLayeredPool(pool_);
 239 }
 240
 241 void ClientSocketPoolBaseHelper::AddHigherLayeredPool(
 242     HigherLayeredPool* higher_pool) {
 243   CHECK(higher_pool);
 244   CHECK(!ContainsKey(higher_pools_, higher_pool));
 245   higher_pools_.insert(higher_pool);
 246 }
 247
 248 void ClientSocketPoolBaseHelper::RemoveHigherLayeredPool(
 249     HigherLayeredPool* higher_pool) {
 250   CHECK(higher_pool);
 251   CHECK(ContainsKey(higher_pools_, higher_pool));
 252   higher_pools_.erase(higher_pool);
 253 }
 254
 255 int ClientSocketPoolBaseHelper::RequestSocket(
 256     const std::string& group_name,
 257     scoped_ptr<const Request> request) {
 258   CHECK(!request->callback().is_null());
 259   CHECK(request->handle());
 260
 261   // Cleanup any timed-out idle sockets if no timer is used.
 262   if (!use_cleanup_timer_)
 263     CleanupIdleSockets(false);
 264
 265   request->net_log().BeginEvent(NetLog::TYPE_SOCKET_POOL);
 266   Group* group = GetOrCreateGroup(group_name);
 267
 268   int rv = RequestSocketInternal(group_name, *request);
 269   if (rv != ERR_IO_PENDING) {
 270     request->net_log().EndEventWithNetErrorCode(NetLog::TYPE_SOCKET_POOL, rv);
 271     CHECK(!request->handle()->is_initialized());
 272     request.reset();
 273   } else {
 274     group->InsertPendingRequest(request.Pass());
 275     // Have to do this asynchronously, as closing sockets in higher level pools
 276     // call back in to |this|, which will cause all sorts of fun and exciting
 277     // re-entrancy issues if the socket pool is doing something else at the
 278     // time.
 279     if (group->IsStalledOnPoolMaxSockets(max_sockets_per_group_)) {
 280       base::MessageLoop::current()->PostTask(
 281           FROM_HERE,
 282           base::Bind(
 283               &ClientSocketPoolBaseHelper::TryToCloseSocketsInLayeredPools,
 284               weak_factory_.GetWeakPtr()));
 285     }
 286   }
 287   return rv;
 288 }
 289
 290 void ClientSocketPoolBaseHelper::RequestSockets(
 291     const std::string& group_name,
 292     const Request& request,
 293     int num_sockets) {
 294   DCHECK(request.callback().is_null());
 295   DCHECK(!request.handle());
 296
 297   // Cleanup any timed out idle sockets if no timer is used.
 298   if (!use_cleanup_timer_)
 299     CleanupIdleSockets(false);
 300
 301   if (num_sockets > max_sockets_per_group_) {
 302     num_sockets = max_sockets_per_group_;
 303   }
 304
 305   request.net_log().BeginEvent(
 306       NetLog::TYPE_SOCKET_POOL_CONNECTING_N_SOCKETS,
 307       NetLog::IntegerCallback("num_sockets", num_sockets));
 308
 309   Group* group = GetOrCreateGroup(group_name);
 310
 311   // RequestSocketsInternal() may delete the group.
 312   bool deleted_group = false;
 313
 314   int rv = OK;
 315   for (int num_iterations_left = num_sockets;
 316        group->NumActiveSocketSlots() < num_sockets &&
 317        num_iterations_left > 0 ; num_iterations_left--) {
 318     rv = RequestSocketInternal(group_name, request);
 319     if (rv < 0 && rv != ERR_IO_PENDING) {
 320       // We're encountering a synchronous error.  Give up.
 321       if (!ContainsKey(group_map_, group_name))
 322         deleted_group = true;
 323       break;
 324     }
 325     if (!ContainsKey(group_map_, group_name)) {
 326       // Unexpected.  The group should only be getting deleted on synchronous
 327       // error.
 328       NOTREACHED();
 329       deleted_group = true;
 330       break;
 331     }
 332   }
 333
 334   if (!deleted_group && group->IsEmpty())
 335     RemoveGroup(group_name);
 336
 337   if (rv == ERR_IO_PENDING)
 338     rv = OK;
 339   request.net_log().EndEventWithNetErrorCode(
 340       NetLog::TYPE_SOCKET_POOL_CONNECTING_N_SOCKETS, rv);
 341 }
 342
 343 int ClientSocketPoolBaseHelper::RequestSocketInternal(
 344     const std::string& group_name,
 345     const Request& request) {
 346   ClientSocketHandle* const handle = request.handle();
 347   const bool preconnecting = !handle;
 348   Group* group = GetOrCreateGroup(group_name);
 349
 350   if (!(request.flags() & NO_IDLE_SOCKETS)) {
 351     // Try to reuse a socket.
 352     if (AssignIdleSocketToRequest(request, group))
 353       return OK;
 354   }
 355
 356   // If there are more ConnectJobs than pending requests, don't need to do
 357   // anything.  Can just wait for the extra job to connect, and then assign it
 358   // to the request.
 359   if (!preconnecting && group->TryToUseUnassignedConnectJob())
 360     return ERR_IO_PENDING;
 361
 362   // Can we make another active socket now?
 363   if (!group->HasAvailableSocketSlot(max_sockets_per_group_) &&
 364       !request.ignore_limits()) {
 365     // TODO(willchan): Consider whether or not we need to close a socket in a
 366     // higher layered group. I don't think this makes sense since we would just
 367     // reuse that socket then if we needed one and wouldn't make it down to this
 368     // layer.
 369     request.net_log().AddEvent(
 370         NetLog::TYPE_SOCKET_POOL_STALLED_MAX_SOCKETS_PER_GROUP);
 371     return ERR_IO_PENDING;
 372   }
 373
 374   if (ReachedMaxSocketsLimit() && !request.ignore_limits()) {
 375     // NOTE(mmenke):  Wonder if we really need different code for each case
 376     // here.  Only reason for them now seems to be preconnects.
 377     if (idle_socket_count() > 0) {
 378       // There's an idle socket in this pool. Either that's because there's
 379       // still one in this group, but we got here due to preconnecting bypassing
 380       // idle sockets, or because there's an idle socket in another group.
 381       bool closed = CloseOneIdleSocketExceptInGroup(group);
 382       if (preconnecting && !closed)
 383         return ERR_PRECONNECT_MAX_SOCKET_LIMIT;
 384     } else {
 385       // We could check if we really have a stalled group here, but it requires
 386       // a scan of all groups, so just flip a flag here, and do the check later.
 387       request.net_log().AddEvent(NetLog::TYPE_SOCKET_POOL_STALLED_MAX_SOCKETS);
 388       return ERR_IO_PENDING;
 389     }
 390   }
 391
 392   // We couldn't find a socket to reuse, and there's space to allocate one,
 393   // so allocate and connect a new one.
 394   scoped_ptr<ConnectJob> connect_job(
 395       connect_job_factory_->NewConnectJob(group_name, request, this));
 396
 397   int rv = connect_job->Connect();
 398   if (rv == OK) {
 399     LogBoundConnectJobToRequest(connect_job->net_log().source(), request);
 400     if (!preconnecting) {
 401       HandOutSocket(connect_job->PassSocket(), false /* not reused */,
 402                     connect_job->connect_timing(), handle, base::TimeDelta(),
 403                     group, request.net_log());
 404     } else {
 405       AddIdleSocket(connect_job->PassSocket(), group);
 406     }
 407   } else if (rv == ERR_IO_PENDING) {
 408     // If we don't have any sockets in this group, set a timer for potentially
 409     // creating a new one.  If the SYN is lost, this backup socket may complete
 410     // before the slow socket, improving end user latency.
 411     if (connect_backup_jobs_enabled_ && group->IsEmpty()) {
 412       group->StartBackupJobTimer(group_name, this);
 413     }
 414
 415     connecting_socket_count_++;
 416
 417     group->AddJob(connect_job.Pass(), preconnecting);
 418   } else {
 419     LogBoundConnectJobToRequest(connect_job->net_log().source(), request);
 420     scoped_ptr<StreamSocket> error_socket;
 421     if (!preconnecting) {
 422       DCHECK(handle);
 423       connect_job->GetAdditionalErrorState(handle);
 424       error_socket = connect_job->PassSocket();
 425     }
 426     if (error_socket) {
 427       HandOutSocket(error_socket.Pass(), false /* not reused */,
 428                     connect_job->connect_timing(), handle, base::TimeDelta(),
 429                     group, request.net_log());
 430     } else if (group->IsEmpty()) {
 431       RemoveGroup(group_name);
 432     }
 433   }
 434
 435   return rv;
 436 }
 437
 438 bool ClientSocketPoolBaseHelper::AssignIdleSocketToRequest(
 439     const Request& request, Group* group) {
 440   std::list<IdleSocket>* idle_sockets = group->mutable_idle_sockets();
 441   std::list<IdleSocket>::iterator idle_socket_it = idle_sockets->end();
 442
 443   // Iterate through the idle sockets forwards (oldest to newest)
 444   //   * Delete any disconnected ones.
 445   //   * If we find a used idle socket, assign to |idle_socket|.  At the end,
 446   //   the |idle_socket_it| will be set to the newest used idle socket.
 447   for (std::list<IdleSocket>::iterator it = idle_sockets->begin();
 448        it != idle_sockets->end();) {
 449     if (!it->socket->IsConnectedAndIdle()) {
 450       DecrementIdleCount();
 451       delete it->socket;
 452       it = idle_sockets->erase(it);
 453       continue;
 454     }
 455
 456     if (it->socket->WasEverUsed()) {
 457       // We found one we can reuse!
 458       idle_socket_it = it;
 459     }
 460
 461     ++it;
 462   }
 463
 464   // If we haven't found an idle socket, that means there are no used idle
 465   // sockets.  Pick the oldest (first) idle socket (FIFO).
 466
 467   if (idle_socket_it == idle_sockets->end() && !idle_sockets->empty())
 468     idle_socket_it = idle_sockets->begin();
 469
 470   if (idle_socket_it != idle_sockets->end()) {
 471     DecrementIdleCount();
 472     base::TimeDelta idle_time =
 473         base::TimeTicks::Now() - idle_socket_it->start_time;
 474     IdleSocket idle_socket = *idle_socket_it;
 475     idle_sockets->erase(idle_socket_it);
 476     HandOutSocket(
 477         scoped_ptr<StreamSocket>(idle_socket.socket),
 478         idle_socket.socket->WasEverUsed(),
 479         LoadTimingInfo::ConnectTiming(),
 480         request.handle(),
 481         idle_time,
 482         group,
 483         request.net_log());
 484     return true;
 485   }
 486
 487   return false;
 488 }
 489
 490 // static
 491 void ClientSocketPoolBaseHelper::LogBoundConnectJobToRequest(
 492     const NetLog::Source& connect_job_source, const Request& request) {
 493   request.net_log().AddEvent(NetLog::TYPE_SOCKET_POOL_BOUND_TO_CONNECT_JOB,
 494                              connect_job_source.ToEventParametersCallback());
 495 }
 496
 497 void ClientSocketPoolBaseHelper::CancelRequest(
 498     const std::string& group_name, ClientSocketHandle* handle) {
 499   PendingCallbackMap::iterator callback_it = pending_callback_map_.find(handle);
 500   if (callback_it != pending_callback_map_.end()) {
 501     int result = callback_it->second.result;
 502     pending_callback_map_.erase(callback_it);
 503     scoped_ptr<StreamSocket> socket = handle->PassSocket();
 504     if (socket) {
 505       if (result != OK)
 506         socket->Disconnect();
 507       ReleaseSocket(handle->group_name(), socket.Pass(), handle->id());
 508     }
 509     return;
 510   }
 511
 512   CHECK(ContainsKey(group_map_, group_name));
 513
 514   Group* group = GetOrCreateGroup(group_name);
 515
 516   // Search pending_requests for matching handle.
 517   scoped_ptr<const Request> request =
 518       group->FindAndRemovePendingRequest(handle);
 519   if (request) {
 520     request->net_log().AddEvent(NetLog::TYPE_CANCELLED);
 521     request->net_log().EndEvent(NetLog::TYPE_SOCKET_POOL);
 522
 523     // We let the job run, unless we're at the socket limit and there is
 524     // not another request waiting on the job.
 525     if (group->jobs().size() > group->pending_request_count() &&
 526         ReachedMaxSocketsLimit()) {
 527       RemoveConnectJob(*group->jobs().begin(), group);
 528       CheckForStalledSocketGroups();
 529     }
 530   }
 531 }
 532
 533 bool ClientSocketPoolBaseHelper::HasGroup(const std::string& group_name) const {
 534   return ContainsKey(group_map_, group_name);
 535 }
 536
 537 void ClientSocketPoolBaseHelper::CloseIdleSockets() {
 538   CleanupIdleSockets(true);
 539   DCHECK_EQ(0, idle_socket_count_);
 540 }
 541
 542 int ClientSocketPoolBaseHelper::IdleSocketCountInGroup(
 543     const std::string& group_name) const {
 544   GroupMap::const_iterator i = group_map_.find(group_name);
 545   CHECK(i != group_map_.end());
 546
 547   return i->second->idle_sockets().size();
 548 }
 549
 550 LoadState ClientSocketPoolBaseHelper::GetLoadState(
 551     const std::string& group_name,
 552     const ClientSocketHandle* handle) const {
 553   if (ContainsKey(pending_callback_map_, handle))
 554     return LOAD_STATE_CONNECTING;
 555
 556   if (!ContainsKey(group_map_, group_name)) {
 557     NOTREACHED() << "ClientSocketPool does not contain group: " << group_name
 558                  << " for handle: " << handle;
 559     return LOAD_STATE_IDLE;
 560   }
 561
 562   // Can't use operator[] since it is non-const.
 563   const Group& group = *group_map_.find(group_name)->second;
 564
 565   if (group.HasConnectJobForHandle(handle)) {
 566     // Just return the state  of the farthest along ConnectJob for the first
 567     // group.jobs().size() pending requests.
 568     LoadState max_state = LOAD_STATE_IDLE;
 569     for (ConnectJobSet::const_iterator job_it = group.jobs().begin();
 570          job_it != group.jobs().end(); ++job_it) {
 571       max_state = std::max(max_state, (*job_it)->GetLoadState());
 572     }
 573     return max_state;
 574   }
 575
 576   if (group.IsStalledOnPoolMaxSockets(max_sockets_per_group_))
 577     return LOAD_STATE_WAITING_FOR_STALLED_SOCKET_POOL;
 578   return LOAD_STATE_WAITING_FOR_AVAILABLE_SOCKET;
 579 }
 580
 581 base::DictionaryValue* ClientSocketPoolBaseHelper::GetInfoAsValue(
 582     const std::string& name, const std::string& type) const {
 583   base::DictionaryValue* dict = new base::DictionaryValue();
 584   dict->SetString("name", name);
 585   dict->SetString("type", type);
 586   dict->SetInteger("handed_out_socket_count", handed_out_socket_count_);
 587   dict->SetInteger("connecting_socket_count", connecting_socket_count_);
 588   dict->SetInteger("idle_socket_count", idle_socket_count_);
 589   dict->SetInteger("max_socket_count", max_sockets_);
 590   dict->SetInteger("max_sockets_per_group", max_sockets_per_group_);
 591   dict->SetInteger("pool_generation_number", pool_generation_number_);
 592
 593   if (group_map_.empty())
 594     return dict;
 595
 596   base::DictionaryValue* all_groups_dict = new base::DictionaryValue();
 597   for (GroupMap::const_iterator it = group_map_.begin();
 598        it != group_map_.end(); it++) {
 599     const Group* group = it->second;
 600     base::DictionaryValue* group_dict = new base::DictionaryValue();
 601
 602     group_dict->SetInteger("pending_request_count",
 603                            group->pending_request_count());
 604     if (group->has_pending_requests()) {
 605       group_dict->SetString(
 606           "top_pending_priority",
 607           RequestPriorityToString(group->TopPendingPriority()));
 608     }
 609
 610     group_dict->SetInteger("active_socket_count", group->active_socket_count());
 611
 612     base::ListValue* idle_socket_list = new base::ListValue();
 613     std::list<IdleSocket>::const_iterator idle_socket;
 614     for (idle_socket = group->idle_sockets().begin();
 615          idle_socket != group->idle_sockets().end();
 616          idle_socket++) {
 617       int source_id = idle_socket->socket->NetLog().source().id;
 618       idle_socket_list->Append(new base::FundamentalValue(source_id));
 619     }
 620     group_dict->Set("idle_sockets", idle_socket_list);
 621
 622     base::ListValue* connect_jobs_list = new base::ListValue();
 623     std::set<ConnectJob*>::const_iterator job = group->jobs().begin();
 624     for (job = group->jobs().begin(); job != group->jobs().end(); job++) {
 625       int source_id = (*job)->net_log().source().id;
 626       connect_jobs_list->Append(new base::FundamentalValue(source_id));
 627     }
 628     group_dict->Set("connect_jobs", connect_jobs_list);
 629
 630     group_dict->SetBoolean("is_stalled",
 631                            group->IsStalledOnPoolMaxSockets(
 632                                max_sockets_per_group_));
 633     group_dict->SetBoolean("backup_job_timer_is_running",
 634                            group->BackupJobTimerIsRunning());
 635
 636     all_groups_dict->SetWithoutPathExpansion(it->first, group_dict);
 637   }
 638   dict->Set("groups", all_groups_dict);
 639   return dict;
 640 }
 641
 642 bool ClientSocketPoolBaseHelper::IdleSocket::ShouldCleanup(
 643     base::TimeTicks now,
 644     base::TimeDelta timeout) const {
 645   bool timed_out = (now - start_time) >= timeout;
 646   if (timed_out)
 647     return true;
 648   if (socket->WasEverUsed())
 649     return !socket->IsConnectedAndIdle();
 650   return !socket->IsConnected();
 651 }
 652
 653 void ClientSocketPoolBaseHelper::CleanupIdleSockets(bool force) {
 654   if (idle_socket_count_ == 0)
 655     return;
 656
 657   // Current time value. Retrieving it once at the function start rather than
 658   // inside the inner loop, since it shouldn't change by any meaningful amount.
 659   base::TimeTicks now = base::TimeTicks::Now();
 660
 661   GroupMap::iterator i = group_map_.begin();
 662   while (i != group_map_.end()) {
 663     Group* group = i->second;
 664
 665     std::list<IdleSocket>::iterator j = group->mutable_idle_sockets()->begin();
 666     while (j != group->idle_sockets().end()) {
 667       base::TimeDelta timeout =
 668           j->socket->WasEverUsed() ?
 669           used_idle_socket_timeout_ : unused_idle_socket_timeout_;
 670       if (force || j->ShouldCleanup(now, timeout)) {
 671         delete j->socket;
 672         j = group->mutable_idle_sockets()->erase(j);
 673         DecrementIdleCount();
 674       } else {
 675         ++j;
 676       }
 677     }
 678
 679     // Delete group if no longer needed.
 680     if (group->IsEmpty()) {
 681       RemoveGroup(i++);
 682     } else {
 683       ++i;
 684     }
 685   }
 686 }
 687
 688 ClientSocketPoolBaseHelper::Group* ClientSocketPoolBaseHelper::GetOrCreateGroup(
 689     const std::string& group_name) {
 690   GroupMap::iterator it = group_map_.find(group_name);
 691   if (it != group_map_.end())
 692     return it->second;
 693   Group* group = new Group;
 694   group_map_[group_name] = group;
 695   return group;
 696 }
 697
 698 void ClientSocketPoolBaseHelper::RemoveGroup(const std::string& group_name) {
 699   GroupMap::iterator it = group_map_.find(group_name);
 700   CHECK(it != group_map_.end());
 701
 702   RemoveGroup(it);
 703 }
 704
 705 void ClientSocketPoolBaseHelper::RemoveGroup(GroupMap::iterator it) {
 706   delete it->second;
 707   group_map_.erase(it);
 708 }
 709
 710 // static
 711 bool ClientSocketPoolBaseHelper::connect_backup_jobs_enabled() {
 712   return g_connect_backup_jobs_enabled;
 713 }
 714
 715 // static
 716 bool ClientSocketPoolBaseHelper::set_connect_backup_jobs_enabled(bool enabled) {
 717   bool old_value = g_connect_backup_jobs_enabled;
 718   g_connect_backup_jobs_enabled = enabled;
 719   return old_value;
 720 }
 721
 722 void ClientSocketPoolBaseHelper::EnableConnectBackupJobs() {
 723   connect_backup_jobs_enabled_ = g_connect_backup_jobs_enabled;
 724 }
 725
 726 void ClientSocketPoolBaseHelper::IncrementIdleCount() {
 727   if (++idle_socket_count_ == 1 && use_cleanup_timer_)
 728     StartIdleSocketTimer();
 729 }
 730
 731 void ClientSocketPoolBaseHelper::DecrementIdleCount() {
 732   if (--idle_socket_count_ == 0)
 733     timer_.Stop();
 734 }
 735
 736 // static
 737 bool ClientSocketPoolBaseHelper::cleanup_timer_enabled() {
 738   return g_cleanup_timer_enabled;
 739 }
 740
 741 // static
 742 bool ClientSocketPoolBaseHelper::set_cleanup_timer_enabled(bool enabled) {
 743   bool old_value = g_cleanup_timer_enabled;
 744   g_cleanup_timer_enabled = enabled;
 745   return old_value;
 746 }
 747
 748 void ClientSocketPoolBaseHelper::StartIdleSocketTimer() {
 749   timer_.Start(FROM_HERE, TimeDelta::FromSeconds(kCleanupInterval), this,
 750                &ClientSocketPoolBaseHelper::OnCleanupTimerFired);
 751 }
 752
 753 void ClientSocketPoolBaseHelper::ReleaseSocket(const std::string& group_name,
 754                                                scoped_ptr<StreamSocket> socket,
 755                                                int id) {
 756   GroupMap::iterator i = group_map_.find(group_name);
 757   CHECK(i != group_map_.end());
 758
 759   Group* group = i->second;
 760
 761   CHECK_GT(handed_out_socket_count_, 0);
 762   handed_out_socket_count_--;
 763
 764   CHECK_GT(group->active_socket_count(), 0);
 765   group->DecrementActiveSocketCount();
 766
 767   const bool can_reuse = socket->IsConnectedAndIdle() &&
 768       id == pool_generation_number_;
 769   if (can_reuse) {
 770     // Add it to the idle list.
 771     AddIdleSocket(socket.Pass(), group);
 772     OnAvailableSocketSlot(group_name, group);
 773   } else {
 774     socket.reset();
 775   }
 776
 777   CheckForStalledSocketGroups();
 778 }
 779
 780 void ClientSocketPoolBaseHelper::CheckForStalledSocketGroups() {
 781   // If we have idle sockets, see if we can give one to the top-stalled group.
 782   std::string top_group_name;
 783   Group* top_group = NULL;
 784   if (!FindTopStalledGroup(&top_group, &top_group_name)) {
 785     // There may still be a stalled group in a lower level pool.
 786     for (std::set<LowerLayeredPool*>::iterator it = lower_pools_.begin();
 787          it != lower_pools_.end();
 788          ++it) {
 789        if ((*it)->IsStalled()) {
 790          CloseOneIdleSocket();
 791          break;
 792        }
 793     }
 794     return;
 795   }
 796
 797   if (ReachedMaxSocketsLimit()) {
 798     if (idle_socket_count() > 0) {
 799       CloseOneIdleSocket();
 800     } else {
 801       // We can't activate more sockets since we're already at our global
 802       // limit.
 803       return;
 804     }
 805   }
 806
 807   // Note:  we don't loop on waking stalled groups.  If the stalled group is at
 808   //        its limit, may be left with other stalled groups that could be
 809   //        woken.  This isn't optimal, but there is no starvation, so to avoid
 810   //        the looping we leave it at this.
 811   OnAvailableSocketSlot(top_group_name, top_group);
 812 }
 813
 814 // Search for the highest priority pending request, amongst the groups that
 815 // are not at the |max_sockets_per_group_| limit. Note: for requests with
 816 // the same priority, the winner is based on group hash ordering (and not
 817 // insertion order).
 818 bool ClientSocketPoolBaseHelper::FindTopStalledGroup(
 819     Group** group,
 820     std::string* group_name) const {
 821   CHECK((group && group_name) || (!group && !group_name));
 822   Group* top_group = NULL;
 823   const std::string* top_group_name = NULL;
 824   bool has_stalled_group = false;
 825   for (GroupMap::const_iterator i = group_map_.begin();
 826        i != group_map_.end(); ++i) {
 827     Group* curr_group = i->second;
 828     if (!curr_group->has_pending_requests())
 829       continue;
 830     if (curr_group->IsStalledOnPoolMaxSockets(max_sockets_per_group_)) {
 831       if (!group)
 832         return true;
 833       has_stalled_group = true;
 834       bool has_higher_priority = !top_group ||
 835           curr_group->TopPendingPriority() > top_group->TopPendingPriority();
 836       if (has_higher_priority) {
 837         top_group = curr_group;
 838         top_group_name = &i->first;
 839       }
 840     }
 841   }
 842
 843   if (top_group) {
 844     CHECK(group);
 845     *group = top_group;
 846     *group_name = *top_group_name;
 847   } else {
 848     CHECK(!has_stalled_group);
 849   }
 850   return has_stalled_group;
 851 }
 852
 853 void ClientSocketPoolBaseHelper::OnConnectJobComplete(
 854     int result, ConnectJob* job) {
 855   DCHECK_NE(ERR_IO_PENDING, result);
 856   const std::string group_name = job->group_name();
 857   GroupMap::iterator group_it = group_map_.find(group_name);
 858   CHECK(group_it != group_map_.end());
 859   Group* group = group_it->second;
 860
 861   scoped_ptr<StreamSocket> socket = job->PassSocket();
 862
 863   // Copies of these are needed because |job| may be deleted before they are
 864   // accessed.
 865   BoundNetLog job_log = job->net_log();
 866   LoadTimingInfo::ConnectTiming connect_timing = job->connect_timing();
 867
 868   // RemoveConnectJob(job, _) must be called by all branches below;
 869   // otherwise, |job| will be leaked.
 870
 871   if (result == OK) {
 872     DCHECK(socket.get());
 873     RemoveConnectJob(job, group);
 874     scoped_ptr<const Request> request = group->PopNextPendingRequest();
 875     if (request) {
 876       LogBoundConnectJobToRequest(job_log.source(), *request);
 877       HandOutSocket(
 878           socket.Pass(), false /* unused socket */, connect_timing,
 879           request->handle(), base::TimeDelta(), group, request->net_log());
 880       request->net_log().EndEvent(NetLog::TYPE_SOCKET_POOL);
 881       InvokeUserCallbackLater(request->handle(), request->callback(), result);
 882     } else {
 883       AddIdleSocket(socket.Pass(), group);
 884       OnAvailableSocketSlot(group_name, group);
 885       CheckForStalledSocketGroups();
 886     }
 887   } else {
 888     // If we got a socket, it must contain error information so pass that
 889     // up so that the caller can retrieve it.
 890     bool handed_out_socket = false;
 891     scoped_ptr<const Request> request = group->PopNextPendingRequest();
 892     if (request) {
 893       LogBoundConnectJobToRequest(job_log.source(), *request);
 894       job->GetAdditionalErrorState(request->handle());
 895       RemoveConnectJob(job, group);
 896       if (socket.get()) {
 897         handed_out_socket = true;
 898         HandOutSocket(socket.Pass(), false /* unused socket */,
 899                       connect_timing, request->handle(), base::TimeDelta(),
 900                       group, request->net_log());
 901       }
 902       request->net_log().EndEventWithNetErrorCode(
 903           NetLog::TYPE_SOCKET_POOL, result);
 904       InvokeUserCallbackLater(request->handle(), request->callback(), result);
 905     } else {
 906       RemoveConnectJob(job, group);
 907     }
 908     if (!handed_out_socket) {
 909       OnAvailableSocketSlot(group_name, group);
 910       CheckForStalledSocketGroups();
 911     }
 912   }
 913 }
 914
 915 void ClientSocketPoolBaseHelper::OnIPAddressChanged() {
 916   FlushWithError(ERR_NETWORK_CHANGED);
 917 }
 918
 919 void ClientSocketPoolBaseHelper::FlushWithError(int error) {
 920   pool_generation_number_++;
 921   CancelAllConnectJobs();
 922   CloseIdleSockets();
 923   CancelAllRequestsWithError(error);
 924 }
 925
 926 void ClientSocketPoolBaseHelper::RemoveConnectJob(ConnectJob* job,
 927                                                   Group* group) {
 928   CHECK_GT(connecting_socket_count_, 0);
 929   connecting_socket_count_--;
 930
 931   DCHECK(group);
 932   group->RemoveJob(job);
 933 }
 934
 935 void ClientSocketPoolBaseHelper::OnAvailableSocketSlot(
 936     const std::string& group_name, Group* group) {
 937   DCHECK(ContainsKey(group_map_, group_name));
 938   if (group->IsEmpty()) {
 939     RemoveGroup(group_name);
 940   } else if (group->has_pending_requests()) {
 941     ProcessPendingRequest(group_name, group);
 942   }
 943 }
 944
 945 void ClientSocketPoolBaseHelper::ProcessPendingRequest(
 946     const std::string& group_name, Group* group) {
 947   const Request* next_request = group->GetNextPendingRequest();
 948   DCHECK(next_request);
 949   int rv = RequestSocketInternal(group_name, *next_request);
 950   if (rv != ERR_IO_PENDING) {
 951     scoped_ptr<const Request> request = group->PopNextPendingRequest();
 952     DCHECK(request);
 953     if (group->IsEmpty())
 954       RemoveGroup(group_name);
 955
 956     request->net_log().EndEventWithNetErrorCode(NetLog::TYPE_SOCKET_POOL, rv);
 957     InvokeUserCallbackLater(request->handle(), request->callback(), rv);
 958   }
 959 }
 960
 961 void ClientSocketPoolBaseHelper::HandOutSocket(
 962     scoped_ptr<StreamSocket> socket,
 963     bool reused,
 964     const LoadTimingInfo::ConnectTiming& connect_timing,
 965     ClientSocketHandle* handle,
 966     base::TimeDelta idle_time,
 967     Group* group,
 968     const BoundNetLog& net_log) {
 969   DCHECK(socket);
 970   handle->SetSocket(socket.Pass());
 971   handle->set_is_reused(reused);
 972   handle->set_idle_time(idle_time);
 973   handle->set_pool_id(pool_generation_number_);
 974   handle->set_connect_timing(connect_timing);
 975
 976   if (reused) {
 977     net_log.AddEvent(
 978         NetLog::TYPE_SOCKET_POOL_REUSED_AN_EXISTING_SOCKET,
 979         NetLog::IntegerCallback(
 980             "idle_ms", static_cast<int>(idle_time.InMilliseconds())));
 981   }
 982
 983   net_log.AddEvent(
 984       NetLog::TYPE_SOCKET_POOL_BOUND_TO_SOCKET,
 985       handle->socket()->NetLog().source().ToEventParametersCallback());
 986
 987   handed_out_socket_count_++;
 988   group->IncrementActiveSocketCount();
 989 }
 990
 991 void ClientSocketPoolBaseHelper::AddIdleSocket(
 992     scoped_ptr<StreamSocket> socket,
 993     Group* group) {
 994   DCHECK(socket);
 995   IdleSocket idle_socket;
 996   idle_socket.socket = socket.release();
 997   idle_socket.start_time = base::TimeTicks::Now();
 998
 999   group->mutable_idle_sockets()->push_back(idle_socket);
1000   IncrementIdleCount();
1001 }
1002
1003 void ClientSocketPoolBaseHelper::CancelAllConnectJobs() {
1004   for (GroupMap::iterator i = group_map_.begin(); i != group_map_.end();) {
1005     Group* group = i->second;
1006     connecting_socket_count_ -= group->jobs().size();
1007     group->RemoveAllJobs();
1008
1009     // Delete group if no longer needed.
1010     if (group->IsEmpty()) {
1011       // RemoveGroup() will call .erase() which will invalidate the iterator,
1012       // but i will already have been incremented to a valid iterator before
1013       // RemoveGroup() is called.
1014       RemoveGroup(i++);
1015     } else {
1016       ++i;
1017     }
1018   }
1019   DCHECK_EQ(0, connecting_socket_count_);
1020 }
1021
1022 void ClientSocketPoolBaseHelper::CancelAllRequestsWithError(int error) {
1023   for (GroupMap::iterator i = group_map_.begin(); i != group_map_.end();) {
1024     Group* group = i->second;
1025
1026     while (true) {
1027       scoped_ptr<const Request> request = group->PopNextPendingRequest();
1028       if (!request)
1029         break;
1030       InvokeUserCallbackLater(request->handle(), request->callback(), error);
1031     }
1032
1033     // Delete group if no longer needed.
1034     if (group->IsEmpty()) {
1035       // RemoveGroup() will call .erase() which will invalidate the iterator,
1036       // but i will already have been incremented to a valid iterator before
1037       // RemoveGroup() is called.
1038       RemoveGroup(i++);
1039     } else {
1040       ++i;
1041     }
1042   }
1043 }
1044
1045 bool ClientSocketPoolBaseHelper::ReachedMaxSocketsLimit() const {
1046   // Each connecting socket will eventually connect and be handed out.
1047   int total = handed_out_socket_count_ + connecting_socket_count_ +
1048       idle_socket_count();
1049   // There can be more sockets than the limit since some requests can ignore
1050   // the limit
1051   if (total < max_sockets_)
1052     return false;
1053   return true;
1054 }
1055
1056 bool ClientSocketPoolBaseHelper::CloseOneIdleSocket() {
1057   if (idle_socket_count() == 0)
1058     return false;
1059   return CloseOneIdleSocketExceptInGroup(NULL);
1060 }
1061
1062 bool ClientSocketPoolBaseHelper::CloseOneIdleSocketExceptInGroup(
1063     const Group* exception_group) {
1064   CHECK_GT(idle_socket_count(), 0);
1065
1066   for (GroupMap::iterator i = group_map_.begin(); i != group_map_.end(); ++i) {
1067     Group* group = i->second;
1068     if (exception_group == group)
1069       continue;
1070     std::list<IdleSocket>* idle_sockets = group->mutable_idle_sockets();
1071
1072     if (!idle_sockets->empty()) {
1073       delete idle_sockets->front().socket;
1074       idle_sockets->pop_front();
1075       DecrementIdleCount();
1076       if (group->IsEmpty())
1077         RemoveGroup(i);
1078
1079       return true;
1080     }
1081   }
1082
1083   return false;
1084 }
1085
1086 bool ClientSocketPoolBaseHelper::CloseOneIdleConnectionInHigherLayeredPool() {
1087   // This pool doesn't have any idle sockets. It's possible that a pool at a
1088   // higher layer is holding one of this sockets active, but it's actually idle.
1089   // Query the higher layers.
1090   for (std::set<HigherLayeredPool*>::const_iterator it = higher_pools_.begin();
1091        it != higher_pools_.end(); ++it) {
1092     if ((*it)->CloseOneIdleConnection())
1093       return true;
1094   }
1095   return false;
1096 }
1097
1098 void ClientSocketPoolBaseHelper::InvokeUserCallbackLater(
1099     ClientSocketHandle* handle, const CompletionCallback& callback, int rv) {
1100   CHECK(!ContainsKey(pending_callback_map_, handle));
1101   pending_callback_map_[handle] = CallbackResultPair(callback, rv);
1102   base::MessageLoop::current()->PostTask(
1103       FROM_HERE,
1104       base::Bind(&ClientSocketPoolBaseHelper::InvokeUserCallback,
1105                  weak_factory_.GetWeakPtr(), handle));
1106 }
1107
1108 void ClientSocketPoolBaseHelper::InvokeUserCallback(
1109     ClientSocketHandle* handle) {
1110   PendingCallbackMap::iterator it = pending_callback_map_.find(handle);
1111
1112   // Exit if the request has already been cancelled.
1113   if (it == pending_callback_map_.end())
1114     return;
1115
1116   CHECK(!handle->is_initialized());
1117   CompletionCallback callback = it->second.callback;
1118   int result = it->second.result;
1119   pending_callback_map_.erase(it);
1120   callback.Run(result);
1121 }
1122
1123 void ClientSocketPoolBaseHelper::TryToCloseSocketsInLayeredPools() {
1124   while (IsStalled()) {
1125     // Closing a socket will result in calling back into |this| to use the freed
1126     // socket slot, so nothing else is needed.
1127     if (!CloseOneIdleConnectionInHigherLayeredPool())
1128       return;
1129   }
1130 }
1131
1132 ClientSocketPoolBaseHelper::Group::Group()
1133     : unassigned_job_count_(0),
1134       // The number of priorities is doubled since requests with
1135       // |ignore_limits| are prioritized over other requests.
1136       pending_requests_(2 * NUM_PRIORITIES),
1137       active_socket_count_(0) {}
1138
1139 ClientSocketPoolBaseHelper::Group::~Group() {
1140   DCHECK_EQ(0u, unassigned_job_count_);
1141 }
1142
1143 void ClientSocketPoolBaseHelper::Group::StartBackupJobTimer(
1144     const std::string& group_name,
1145     ClientSocketPoolBaseHelper* pool) {
1146   // Only allow one timer to run at a time.
1147   if (BackupJobTimerIsRunning())
1148     return;
1149
1150   // Unretained here is okay because |backup_job_timer_| is
1151   // automatically cancelled when it's destroyed.
1152   backup_job_timer_.Start(
1153       FROM_HERE, pool->ConnectRetryInterval(),
1154       base::Bind(&Group::OnBackupJobTimerFired, base::Unretained(this),
1155                  group_name, pool));
1156 }
1157
1158 bool ClientSocketPoolBaseHelper::Group::BackupJobTimerIsRunning() const {
1159   return backup_job_timer_.IsRunning();
1160 }
1161
1162 bool ClientSocketPoolBaseHelper::Group::TryToUseUnassignedConnectJob() {
1163   SanityCheck();
1164
1165   if (unassigned_job_count_ == 0)
1166     return false;
1167   --unassigned_job_count_;
1168   return true;
1169 }
1170
1171 void ClientSocketPoolBaseHelper::Group::AddJob(scoped_ptr<ConnectJob> job,
1172                                                bool is_preconnect) {
1173   SanityCheck();
1174
1175   if (is_preconnect)
1176     ++unassigned_job_count_;
1177   jobs_.insert(job.release());
1178 }
1179
1180 void ClientSocketPoolBaseHelper::Group::RemoveJob(ConnectJob* job) {
1181   scoped_ptr<ConnectJob> owned_job(job);
1182   SanityCheck();
1183
1184   std::set<ConnectJob*>::iterator it = jobs_.find(job);
1185   if (it != jobs_.end()) {
1186     jobs_.erase(it);
1187   } else {
1188     NOTREACHED();
1189   }
1190   size_t job_count = jobs_.size();
1191   if (job_count < unassigned_job_count_)
1192     unassigned_job_count_ = job_count;
1193
1194   // If we've got no more jobs for this group, then we no longer need a
1195   // backup job either.
1196   if (jobs_.empty())
1197     backup_job_timer_.Stop();
1198 }
1199
1200 void ClientSocketPoolBaseHelper::Group::OnBackupJobTimerFired(
1201     std::string group_name,
1202     ClientSocketPoolBaseHelper* pool) {
1203   // If there are no more jobs pending, there is no work to do.
1204   // If we've done our cleanups correctly, this should not happen.
1205   if (jobs_.empty()) {
1206     NOTREACHED();
1207     return;
1208   }
1209
1210   // If our old job is waiting on DNS, or if we can't create any sockets
1211   // right now due to limits, just reset the timer.
1212   if (pool->ReachedMaxSocketsLimit() ||
1213       !HasAvailableSocketSlot(pool->max_sockets_per_group_) ||
1214       (*jobs_.begin())->GetLoadState() == LOAD_STATE_RESOLVING_HOST) {
1215     StartBackupJobTimer(group_name, pool);
1216     return;
1217   }
1218
1219   if (pending_requests_.empty())
1220     return;
1221
1222   scoped_ptr<ConnectJob> backup_job =
1223       pool->connect_job_factory_->NewConnectJob(
1224           group_name, *pending_requests_.FirstMax().value(), pool);
1225   backup_job->net_log().AddEvent(NetLog::TYPE_BACKUP_CONNECT_JOB_CREATED);
1226   SIMPLE_STATS_COUNTER("socket.backup_created");
1227   int rv = backup_job->Connect();
1228   pool->connecting_socket_count_++;
1229   ConnectJob* raw_backup_job = backup_job.get();
1230   AddJob(backup_job.Pass(), false);
1231   if (rv != ERR_IO_PENDING)
1232     pool->OnConnectJobComplete(rv, raw_backup_job);
1233 }
1234
1235 void ClientSocketPoolBaseHelper::Group::SanityCheck() {
1236   DCHECK_LE(unassigned_job_count_, jobs_.size());
1237 }
1238
1239 void ClientSocketPoolBaseHelper::Group::RemoveAllJobs() {
1240   SanityCheck();
1241
1242   // Delete active jobs.
1243   STLDeleteElements(&jobs_);
1244   unassigned_job_count_ = 0;
1245
1246   // Stop backup job timer.
1247   backup_job_timer_.Stop();
1248 }
1249
1250 const ClientSocketPoolBaseHelper::Request*
1251 ClientSocketPoolBaseHelper::Group::GetNextPendingRequest() const {
1252   return
1253       pending_requests_.empty() ? NULL : pending_requests_.FirstMax().value();
1254 }
1255
1256 bool ClientSocketPoolBaseHelper::Group::HasConnectJobForHandle(
1257     const ClientSocketHandle* handle) const {
1258   // Search the first |jobs_.size()| pending requests for |handle|.
1259   // If it's farther back in the deque than that, it doesn't have a
1260   // corresponding ConnectJob.
1261   size_t i = 0;
1262   for (RequestQueue::Pointer pointer = pending_requests_.FirstMax();
1263        !pointer.is_null() && i < jobs_.size();
1264        pointer = pending_requests_.GetNextTowardsLastMin(pointer), ++i) {
1265     if (pointer.value()->handle() == handle)
1266       return true;
1267   }
1268   return false;
1269 }
1270
1271 void ClientSocketPoolBaseHelper::Group::InsertPendingRequest(
1272     scoped_ptr<const Request> request) {
1273   RequestQueue::Priority queue_priority = request->priority();
1274   // Prioritize requests with |ignore_limits| over ones that don't.
1275   if (request->ignore_limits())
1276     queue_priority += NUM_PRIORITIES;
1277   pending_requests_.Insert(request.release(), queue_priority);
1278 }
1279
1280 scoped_ptr<const ClientSocketPoolBaseHelper::Request>
1281 ClientSocketPoolBaseHelper::Group::PopNextPendingRequest() {
1282   if (pending_requests_.empty())
1283     return scoped_ptr<const ClientSocketPoolBaseHelper::Request>();
1284   return RemovePendingRequest(pending_requests_.FirstMax());
1285 }
1286
1287 scoped_ptr<const ClientSocketPoolBaseHelper::Request>
1288 ClientSocketPoolBaseHelper::Group::FindAndRemovePendingRequest(
1289     ClientSocketHandle* handle) {
1290   for (RequestQueue::Pointer pointer = pending_requests_.FirstMax();
1291        !pointer.is_null();
1292        pointer = pending_requests_.GetNextTowardsLastMin(pointer)) {
1293     if (pointer.value()->handle() == handle) {
1294       scoped_ptr<const Request> request = RemovePendingRequest(pointer);
1295       return request.Pass();
1296     }
1297   }
1298   return scoped_ptr<const ClientSocketPoolBaseHelper::Request>();
1299 }
1300
1301 scoped_ptr<const ClientSocketPoolBaseHelper::Request>
1302 ClientSocketPoolBaseHelper::Group::RemovePendingRequest(
1303     const RequestQueue::Pointer& pointer) {
1304   scoped_ptr<const Request> request(pointer.value());
1305   pending_requests_.Erase(pointer);
1306   // If there are no more requests, kill the backup timer.
1307   if (pending_requests_.empty())
1308     backup_job_timer_.Stop();
1309   return request.Pass();
1310 }
1311
1312 }  // namespace internal
1313
1314 }  // namespace net