src/net/url_request/url_fetcher_core.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/url_request/url_fetcher_core.h"
   6
   7 #include <stdint.h>
   8
   9 #include "base/bind.h"
  10 #include "base/logging.h"
  11 #include "base/metrics/histogram.h"
  12 #include "base/sequenced_task_runner.h"
  13 #include "base/single_thread_task_runner.h"
  14 #include "base/stl_util.h"
  15 #include "base/thread_task_runner_handle.h"
  16 #include "base/tracked_objects.h"
  17 #include "net/base/io_buffer.h"
  18 #include "net/base/load_flags.h"
  19 #include "net/base/net_errors.h"
  20 #include "net/base/request_priority.h"
  21 #include "net/base/upload_bytes_element_reader.h"
  22 #include "net/base/upload_data_stream.h"
  23 #include "net/base/upload_file_element_reader.h"
  24 #include "net/http/http_response_headers.h"
  25 #include "net/url_request/redirect_info.h"
  26 #include "net/url_request/url_fetcher_delegate.h"
  27 #include "net/url_request/url_fetcher_response_writer.h"
  28 #include "net/url_request/url_request_context.h"
  29 #include "net/url_request/url_request_context_getter.h"
  30 #include "net/url_request/url_request_throttler_manager.h"
  31
  32 namespace {
  33
  34 const int kBufferSize = 4096;
  35 const int kUploadProgressTimerInterval = 100;
  36 bool g_interception_enabled = false;
  37 bool g_ignore_certificate_requests = false;
  38
  39 void EmptyCompletionCallback(int result) {}
  40
  41 }  // namespace
  42
  43 namespace net {
  44
  45 // URLFetcherCore::Registry ---------------------------------------------------
  46
  47 URLFetcherCore::Registry::Registry() {}
  48 URLFetcherCore::Registry::~Registry() {}
  49
  50 void URLFetcherCore::Registry::AddURLFetcherCore(URLFetcherCore* core) {
  51   DCHECK(!ContainsKey(fetchers_, core));
  52   fetchers_.insert(core);
  53 }
  54
  55 void URLFetcherCore::Registry::RemoveURLFetcherCore(URLFetcherCore* core) {
  56   DCHECK(ContainsKey(fetchers_, core));
  57   fetchers_.erase(core);
  58 }
  59
  60 void URLFetcherCore::Registry::CancelAll() {
  61   while (!fetchers_.empty())
  62     (*fetchers_.begin())->CancelURLRequest(ERR_ABORTED);
  63 }
  64
  65 // URLFetcherCore -------------------------------------------------------------
  66
  67 // static
  68 base::LazyInstance<URLFetcherCore::Registry>
  69     URLFetcherCore::g_registry = LAZY_INSTANCE_INITIALIZER;
  70
  71 URLFetcherCore::URLFetcherCore(URLFetcher* fetcher,
  72                                const GURL& original_url,
  73                                URLFetcher::RequestType request_type,
  74                                URLFetcherDelegate* d)
  75     : fetcher_(fetcher),
  76       original_url_(original_url),
  77       request_type_(request_type),
  78       delegate_(d),
  79       delegate_task_runner_(base::ThreadTaskRunnerHandle::Get()),
  80       load_flags_(LOAD_NORMAL),
  81       response_code_(URLFetcher::RESPONSE_CODE_INVALID),
  82       buffer_(new IOBuffer(kBufferSize)),
  83       url_request_data_key_(NULL),
  84       was_fetched_via_proxy_(false),
  85       upload_content_set_(false),
  86       upload_range_offset_(0),
  87       upload_range_length_(0),
  88       referrer_policy_(
  89           URLRequest::CLEAR_REFERRER_ON_TRANSITION_FROM_SECURE_TO_INSECURE),
  90       is_chunked_upload_(false),
  91       was_cancelled_(false),
  92       stop_on_redirect_(false),
  93       stopped_on_redirect_(false),
  94       automatically_retry_on_5xx_(true),
  95       num_retries_on_5xx_(0),
  96       max_retries_on_5xx_(0),
  97       num_retries_on_network_changes_(0),
  98       max_retries_on_network_changes_(0),
  99       current_upload_bytes_(-1),
 100       current_response_bytes_(0),
 101       total_response_bytes_(-1) {
 102   CHECK(original_url_.is_valid());
 103 }
 104
 105 void URLFetcherCore::Start() {
 106   DCHECK(delegate_task_runner_.get());
 107   DCHECK(request_context_getter_.get()) << "We need an URLRequestContext!";
 108   if (network_task_runner_.get()) {
 109     DCHECK_EQ(network_task_runner_,
 110               request_context_getter_->GetNetworkTaskRunner());
 111   } else {
 112     network_task_runner_ = request_context_getter_->GetNetworkTaskRunner();
 113   }
 114   DCHECK(network_task_runner_.get()) << "We need an IO task runner";
 115
 116   network_task_runner_->PostTask(
 117       FROM_HERE, base::Bind(&URLFetcherCore::StartOnIOThread, this));
 118 }
 119
 120 void URLFetcherCore::Stop() {
 121   if (delegate_task_runner_.get())  // May be NULL in tests.
 122     DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 123
 124   delegate_ = NULL;
 125   fetcher_ = NULL;
 126   if (!network_task_runner_.get())
 127     return;
 128   if (network_task_runner_->RunsTasksOnCurrentThread()) {
 129     CancelURLRequest(ERR_ABORTED);
 130   } else {
 131     network_task_runner_->PostTask(
 132         FROM_HERE,
 133         base::Bind(&URLFetcherCore::CancelURLRequest, this, ERR_ABORTED));
 134   }
 135 }
 136
 137 void URLFetcherCore::SetUploadData(const std::string& upload_content_type,
 138                                    const std::string& upload_content) {
 139   DCHECK(!is_chunked_upload_);
 140   DCHECK(!upload_content_set_);
 141   DCHECK(upload_content_.empty());
 142   DCHECK(upload_file_path_.empty());
 143   DCHECK(upload_content_type_.empty());
 144
 145   // Empty |upload_content_type| is allowed iff the |upload_content| is empty.
 146   DCHECK(upload_content.empty() || !upload_content_type.empty());
 147
 148   upload_content_type_ = upload_content_type;
 149   upload_content_ = upload_content;
 150   upload_content_set_ = true;
 151 }
 152
 153 void URLFetcherCore::SetUploadFilePath(
 154     const std::string& upload_content_type,
 155     const base::FilePath& file_path,
 156     uint64 range_offset,
 157     uint64 range_length,
 158     scoped_refptr<base::TaskRunner> file_task_runner) {
 159   DCHECK(!is_chunked_upload_);
 160   DCHECK(!upload_content_set_);
 161   DCHECK(upload_content_.empty());
 162   DCHECK(upload_file_path_.empty());
 163   DCHECK_EQ(upload_range_offset_, 0ULL);
 164   DCHECK_EQ(upload_range_length_, 0ULL);
 165   DCHECK(upload_content_type_.empty());
 166   DCHECK(!upload_content_type.empty());
 167
 168   upload_content_type_ = upload_content_type;
 169   upload_file_path_ = file_path;
 170   upload_range_offset_ = range_offset;
 171   upload_range_length_ = range_length;
 172   upload_file_task_runner_ = file_task_runner;
 173   upload_content_set_ = true;
 174 }
 175
 176 void URLFetcherCore::SetChunkedUpload(const std::string& content_type) {
 177   DCHECK(is_chunked_upload_ ||
 178          (upload_content_type_.empty() &&
 179           upload_content_.empty()));
 180
 181   // Empty |content_type| is not allowed here, because it is impossible
 182   // to ensure non-empty upload content as it is not yet supplied.
 183   DCHECK(!content_type.empty());
 184
 185   upload_content_type_ = content_type;
 186   upload_content_.clear();
 187   is_chunked_upload_ = true;
 188 }
 189
 190 void URLFetcherCore::AppendChunkToUpload(const std::string& content,
 191                                          bool is_last_chunk) {
 192   DCHECK(delegate_task_runner_.get());
 193   DCHECK(network_task_runner_.get());
 194   network_task_runner_->PostTask(
 195       FROM_HERE,
 196       base::Bind(&URLFetcherCore::CompleteAddingUploadDataChunk, this, content,
 197                  is_last_chunk));
 198 }
 199
 200 void URLFetcherCore::SetLoadFlags(int load_flags) {
 201   load_flags_ = load_flags;
 202 }
 203
 204 int URLFetcherCore::GetLoadFlags() const {
 205   return load_flags_;
 206 }
 207
 208 void URLFetcherCore::SetReferrer(const std::string& referrer) {
 209   referrer_ = referrer;
 210 }
 211
 212 void URLFetcherCore::SetReferrerPolicy(
 213     URLRequest::ReferrerPolicy referrer_policy) {
 214   referrer_policy_ = referrer_policy;
 215 }
 216
 217 void URLFetcherCore::SetExtraRequestHeaders(
 218     const std::string& extra_request_headers) {
 219   extra_request_headers_.Clear();
 220   extra_request_headers_.AddHeadersFromString(extra_request_headers);
 221 }
 222
 223 void URLFetcherCore::AddExtraRequestHeader(const std::string& header_line) {
 224   extra_request_headers_.AddHeaderFromString(header_line);
 225 }
 226
 227 void URLFetcherCore::SetRequestContext(
 228     URLRequestContextGetter* request_context_getter) {
 229   DCHECK(!request_context_getter_.get());
 230   DCHECK(request_context_getter);
 231   request_context_getter_ = request_context_getter;
 232 }
 233
 234 void URLFetcherCore::SetFirstPartyForCookies(
 235     const GURL& first_party_for_cookies) {
 236   DCHECK(first_party_for_cookies_.is_empty());
 237   first_party_for_cookies_ = first_party_for_cookies;
 238 }
 239
 240 void URLFetcherCore::SetURLRequestUserData(
 241     const void* key,
 242     const URLFetcher::CreateDataCallback& create_data_callback) {
 243   DCHECK(key);
 244   DCHECK(!create_data_callback.is_null());
 245   url_request_data_key_ = key;
 246   url_request_create_data_callback_ = create_data_callback;
 247 }
 248
 249 void URLFetcherCore::SetStopOnRedirect(bool stop_on_redirect) {
 250   stop_on_redirect_ = stop_on_redirect;
 251 }
 252
 253 void URLFetcherCore::SetAutomaticallyRetryOn5xx(bool retry) {
 254   automatically_retry_on_5xx_ = retry;
 255 }
 256
 257 void URLFetcherCore::SetMaxRetriesOn5xx(int max_retries) {
 258   max_retries_on_5xx_ = max_retries;
 259 }
 260
 261 int URLFetcherCore::GetMaxRetriesOn5xx() const {
 262   return max_retries_on_5xx_;
 263 }
 264
 265 base::TimeDelta URLFetcherCore::GetBackoffDelay() const {
 266   return backoff_delay_;
 267 }
 268
 269 void URLFetcherCore::SetAutomaticallyRetryOnNetworkChanges(int max_retries) {
 270   max_retries_on_network_changes_ = max_retries;
 271 }
 272
 273 void URLFetcherCore::SaveResponseToFileAtPath(
 274     const base::FilePath& file_path,
 275     scoped_refptr<base::SequencedTaskRunner> file_task_runner) {
 276   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 277   SaveResponseWithWriter(scoped_ptr<URLFetcherResponseWriter>(
 278       new URLFetcherFileWriter(file_task_runner, file_path)));
 279 }
 280
 281 void URLFetcherCore::SaveResponseToTemporaryFile(
 282     scoped_refptr<base::SequencedTaskRunner> file_task_runner) {
 283   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 284   SaveResponseWithWriter(scoped_ptr<URLFetcherResponseWriter>(
 285       new URLFetcherFileWriter(file_task_runner, base::FilePath())));
 286 }
 287
 288 void URLFetcherCore::SaveResponseWithWriter(
 289     scoped_ptr<URLFetcherResponseWriter> response_writer) {
 290   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 291   response_writer_ = response_writer.Pass();
 292 }
 293
 294 HttpResponseHeaders* URLFetcherCore::GetResponseHeaders() const {
 295   return response_headers_.get();
 296 }
 297
 298 // TODO(panayiotis): socket_address_ is written in the IO thread,
 299 // if this is accessed in the UI thread, this could result in a race.
 300 // Same for response_headers_ above and was_fetched_via_proxy_ below.
 301 HostPortPair URLFetcherCore::GetSocketAddress() const {
 302   return socket_address_;
 303 }
 304
 305 bool URLFetcherCore::WasFetchedViaProxy() const {
 306   return was_fetched_via_proxy_;
 307 }
 308
 309 const GURL& URLFetcherCore::GetOriginalURL() const {
 310   return original_url_;
 311 }
 312
 313 const GURL& URLFetcherCore::GetURL() const {
 314   return url_;
 315 }
 316
 317 const URLRequestStatus& URLFetcherCore::GetStatus() const {
 318   return status_;
 319 }
 320
 321 int URLFetcherCore::GetResponseCode() const {
 322   return response_code_;
 323 }
 324
 325 const ResponseCookies& URLFetcherCore::GetCookies() const {
 326   return cookies_;
 327 }
 328
 329 void URLFetcherCore::ReceivedContentWasMalformed() {
 330   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 331   if (network_task_runner_.get()) {
 332     network_task_runner_->PostTask(
 333         FROM_HERE, base::Bind(&URLFetcherCore::NotifyMalformedContent, this));
 334   }
 335 }
 336
 337 bool URLFetcherCore::GetResponseAsString(
 338     std::string* out_response_string) const {
 339   URLFetcherStringWriter* string_writer =
 340       response_writer_ ? response_writer_->AsStringWriter() : NULL;
 341   if (!string_writer)
 342     return false;
 343
 344   *out_response_string = string_writer->data();
 345   UMA_HISTOGRAM_MEMORY_KB("UrlFetcher.StringResponseSize",
 346                           (string_writer->data().length() / 1024));
 347   return true;
 348 }
 349
 350 bool URLFetcherCore::GetResponseAsFilePath(bool take_ownership,
 351                                            base::FilePath* out_response_path) {
 352   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 353
 354   URLFetcherFileWriter* file_writer =
 355       response_writer_ ? response_writer_->AsFileWriter() : NULL;
 356   if (!file_writer)
 357     return false;
 358
 359   *out_response_path = file_writer->file_path();
 360
 361   if (take_ownership) {
 362     // Intentionally calling a file_writer_ method directly without posting
 363     // the task to network_task_runner_.
 364     //
 365     // This is for correctly handling the case when file_writer_->DisownFile()
 366     // is soon followed by URLFetcherCore::Stop(). We have to make sure that
 367     // DisownFile takes effect before Stop deletes file_writer_.
 368     //
 369     // This direct call should be thread-safe, since DisownFile itself does no
 370     // file operation. It just flips the state to be referred in destruction.
 371     file_writer->DisownFile();
 372   }
 373   return true;
 374 }
 375
 376 void URLFetcherCore::OnReceivedRedirect(URLRequest* request,
 377                                         const RedirectInfo& redirect_info,
 378                                         bool* defer_redirect) {
 379   DCHECK_EQ(request, request_.get());
 380   DCHECK(network_task_runner_->BelongsToCurrentThread());
 381   if (stop_on_redirect_) {
 382     stopped_on_redirect_ = true;
 383     url_ = redirect_info.new_url;
 384     response_code_ = request_->GetResponseCode();
 385     was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
 386     request->Cancel();
 387     OnReadCompleted(request, 0);
 388   }
 389 }
 390
 391 void URLFetcherCore::OnResponseStarted(URLRequest* request) {
 392   DCHECK_EQ(request, request_.get());
 393   DCHECK(network_task_runner_->BelongsToCurrentThread());
 394   if (request_->status().is_success()) {
 395     response_code_ = request_->GetResponseCode();
 396     response_headers_ = request_->response_headers();
 397     socket_address_ = request_->GetSocketAddress();
 398     was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
 399     total_response_bytes_ = request_->GetExpectedContentSize();
 400   }
 401
 402   ReadResponse();
 403 }
 404
 405 void URLFetcherCore::OnCertificateRequested(
 406     URLRequest* request,
 407     SSLCertRequestInfo* cert_request_info) {
 408   DCHECK_EQ(request, request_.get());
 409   DCHECK(network_task_runner_->BelongsToCurrentThread());
 410
 411   if (g_ignore_certificate_requests) {
 412     request->ContinueWithCertificate(NULL);
 413   } else {
 414     request->Cancel();
 415   }
 416 }
 417
 418 void URLFetcherCore::OnReadCompleted(URLRequest* request,
 419                                      int bytes_read) {
 420   DCHECK(request == request_);
 421   DCHECK(network_task_runner_->BelongsToCurrentThread());
 422
 423   if (!stopped_on_redirect_)
 424     url_ = request->url();
 425   URLRequestThrottlerManager* throttler_manager =
 426       request->context()->throttler_manager();
 427   if (throttler_manager) {
 428     url_throttler_entry_ = throttler_manager->RegisterRequestUrl(url_);
 429   }
 430
 431   do {
 432     if (!request_->status().is_success() || bytes_read <= 0)
 433       break;
 434
 435     current_response_bytes_ += bytes_read;
 436     InformDelegateDownloadProgress();
 437
 438     const int result =
 439         WriteBuffer(new DrainableIOBuffer(buffer_.get(), bytes_read));
 440     if (result < 0) {
 441       // Write failed or waiting for write completion.
 442       return;
 443     }
 444   } while (request_->Read(buffer_.get(), kBufferSize, &bytes_read));
 445
 446   const URLRequestStatus status = request_->status();
 447
 448   if (status.is_success())
 449     request_->GetResponseCookies(&cookies_);
 450
 451   // See comments re: HEAD requests in ReadResponse().
 452   if (!status.is_io_pending() || request_type_ == URLFetcher::HEAD) {
 453     status_ = status;
 454     ReleaseRequest();
 455
 456     // No more data to write.
 457     const int result = response_writer_->Finish(
 458         base::Bind(&URLFetcherCore::DidFinishWriting, this));
 459     if (result != ERR_IO_PENDING)
 460       DidFinishWriting(result);
 461   }
 462 }
 463
 464 void URLFetcherCore::CancelAll() {
 465   g_registry.Get().CancelAll();
 466 }
 467
 468 int URLFetcherCore::GetNumFetcherCores() {
 469   return g_registry.Get().size();
 470 }
 471
 472 void URLFetcherCore::SetEnableInterceptionForTests(bool enabled) {
 473   g_interception_enabled = enabled;
 474 }
 475
 476 void URLFetcherCore::SetIgnoreCertificateRequests(bool ignored) {
 477   g_ignore_certificate_requests = ignored;
 478 }
 479
 480 URLFetcherCore::~URLFetcherCore() {
 481   // |request_| should be NULL. If not, it's unsafe to delete it here since we
 482   // may not be on the IO thread.
 483   DCHECK(!request_.get());
 484 }
 485
 486 void URLFetcherCore::StartOnIOThread() {
 487   DCHECK(network_task_runner_->BelongsToCurrentThread());
 488
 489   if (!response_writer_)
 490     response_writer_.reset(new URLFetcherStringWriter);
 491
 492   const int result = response_writer_->Initialize(
 493       base::Bind(&URLFetcherCore::DidInitializeWriter, this));
 494   if (result != ERR_IO_PENDING)
 495     DidInitializeWriter(result);
 496 }
 497
 498 void URLFetcherCore::StartURLRequest() {
 499   DCHECK(network_task_runner_->BelongsToCurrentThread());
 500
 501   if (was_cancelled_) {
 502     // Since StartURLRequest() is posted as a *delayed* task, it may
 503     // run after the URLFetcher was already stopped.
 504     return;
 505   }
 506
 507   DCHECK(request_context_getter_.get());
 508   DCHECK(!request_.get());
 509
 510   g_registry.Get().AddURLFetcherCore(this);
 511   current_response_bytes_ = 0;
 512   request_ = request_context_getter_->GetURLRequestContext()->CreateRequest(
 513       original_url_, DEFAULT_PRIORITY, this, NULL);
 514   request_->set_stack_trace(stack_trace_);
 515   int flags = request_->load_flags() | load_flags_;
 516   if (!g_interception_enabled)
 517     flags = flags | LOAD_DISABLE_INTERCEPT;
 518
 519   if (is_chunked_upload_)
 520     request_->EnableChunkedUpload();
 521   request_->SetLoadFlags(flags);
 522   request_->SetReferrer(referrer_);
 523   request_->set_referrer_policy(referrer_policy_);
 524   request_->set_first_party_for_cookies(first_party_for_cookies_.is_empty() ?
 525       original_url_ : first_party_for_cookies_);
 526   if (url_request_data_key_ && !url_request_create_data_callback_.is_null()) {
 527     request_->SetUserData(url_request_data_key_,
 528                           url_request_create_data_callback_.Run());
 529   }
 530
 531   switch (request_type_) {
 532     case URLFetcher::GET:
 533       break;
 534
 535     case URLFetcher::POST:
 536     case URLFetcher::PUT:
 537     case URLFetcher::PATCH:
 538       // Upload content must be set.
 539       DCHECK(is_chunked_upload_ || upload_content_set_);
 540
 541       request_->set_method(
 542           request_type_ == URLFetcher::POST ? "POST" :
 543           request_type_ == URLFetcher::PUT ? "PUT" : "PATCH");
 544       if (!upload_content_type_.empty()) {
 545         extra_request_headers_.SetHeader(HttpRequestHeaders::kContentType,
 546                                          upload_content_type_);
 547       }
 548       if (!upload_content_.empty()) {
 549         scoped_ptr<UploadElementReader> reader(new UploadBytesElementReader(
 550             upload_content_.data(), upload_content_.size()));
 551         request_->set_upload(make_scoped_ptr(
 552             UploadDataStream::CreateWithReader(reader.Pass(), 0)));
 553       } else if (!upload_file_path_.empty()) {
 554         scoped_ptr<UploadElementReader> reader(
 555             new UploadFileElementReader(upload_file_task_runner_.get(),
 556                                         upload_file_path_,
 557                                         upload_range_offset_,
 558                                         upload_range_length_,
 559                                         base::Time()));
 560         request_->set_upload(make_scoped_ptr(
 561             UploadDataStream::CreateWithReader(reader.Pass(), 0)));
 562       }
 563
 564       current_upload_bytes_ = -1;
 565       // TODO(kinaba): http://crbug.com/118103. Implement upload callback in the
 566       //  layer and avoid using timer here.
 567       upload_progress_checker_timer_.reset(
 568           new base::RepeatingTimer<URLFetcherCore>());
 569       upload_progress_checker_timer_->Start(
 570           FROM_HERE,
 571           base::TimeDelta::FromMilliseconds(kUploadProgressTimerInterval),
 572           this,
 573           &URLFetcherCore::InformDelegateUploadProgress);
 574       break;
 575
 576     case URLFetcher::HEAD:
 577       request_->set_method("HEAD");
 578       break;
 579
 580     case URLFetcher::DELETE_REQUEST:
 581       request_->set_method("DELETE");
 582       break;
 583
 584     default:
 585       NOTREACHED();
 586   }
 587
 588   if (!extra_request_headers_.IsEmpty())
 589     request_->SetExtraRequestHeaders(extra_request_headers_);
 590
 591   request_->Start();
 592 }
 593
 594 void URLFetcherCore::DidInitializeWriter(int result) {
 595   if (result != OK) {
 596     CancelURLRequest(result);
 597     delegate_task_runner_->PostTask(
 598         FROM_HERE,
 599         base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
 600     return;
 601   }
 602   StartURLRequestWhenAppropriate();
 603 }
 604
 605 void URLFetcherCore::StartURLRequestWhenAppropriate() {
 606   DCHECK(network_task_runner_->BelongsToCurrentThread());
 607
 608   if (was_cancelled_)
 609     return;
 610
 611   DCHECK(request_context_getter_.get());
 612
 613   int64 delay = INT64_C(0);
 614   if (!original_url_throttler_entry_.get()) {
 615     URLRequestThrottlerManager* manager =
 616         request_context_getter_->GetURLRequestContext()->throttler_manager();
 617     if (manager) {
 618       original_url_throttler_entry_ =
 619           manager->RegisterRequestUrl(original_url_);
 620     }
 621   }
 622   if (original_url_throttler_entry_.get()) {
 623     delay = original_url_throttler_entry_->ReserveSendingTimeForNextRequest(
 624         GetBackoffReleaseTime());
 625   }
 626
 627   if (delay == INT64_C(0)) {
 628     StartURLRequest();
 629   } else {
 630     base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
 631         FROM_HERE, base::Bind(&URLFetcherCore::StartURLRequest, this),
 632         base::TimeDelta::FromMilliseconds(delay));
 633   }
 634 }
 635
 636 void URLFetcherCore::CancelURLRequest(int error) {
 637   DCHECK(network_task_runner_->BelongsToCurrentThread());
 638
 639   if (request_.get()) {
 640     request_->CancelWithError(error);
 641     ReleaseRequest();
 642   }
 643
 644   // Set the error manually.
 645   // Normally, calling URLRequest::CancelWithError() results in calling
 646   // OnReadCompleted() with bytes_read = -1 via an asynchronous task posted by
 647   // URLRequestJob::NotifyDone(). But, because the request was released
 648   // immediately after being canceled, the request could not call
 649   // OnReadCompleted() which overwrites |status_| with the error status.
 650   status_.set_status(URLRequestStatus::CANCELED);
 651   status_.set_error(error);
 652
 653   // Release the reference to the request context. There could be multiple
 654   // references to URLFetcher::Core at this point so it may take a while to
 655   // delete the object, but we cannot delay the destruction of the request
 656   // context.
 657   request_context_getter_ = NULL;
 658   first_party_for_cookies_ = GURL();
 659   url_request_data_key_ = NULL;
 660   url_request_create_data_callback_.Reset();
 661   was_cancelled_ = true;
 662 }
 663
 664 void URLFetcherCore::OnCompletedURLRequest(
 665     base::TimeDelta backoff_delay) {
 666   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 667
 668   // Save the status and backoff_delay so that delegates can read it.
 669   if (delegate_) {
 670     backoff_delay_ = backoff_delay;
 671     InformDelegateFetchIsComplete();
 672   }
 673 }
 674
 675 void URLFetcherCore::InformDelegateFetchIsComplete() {
 676   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 677   if (delegate_)
 678     delegate_->OnURLFetchComplete(fetcher_);
 679 }
 680
 681 void URLFetcherCore::NotifyMalformedContent() {
 682   DCHECK(network_task_runner_->BelongsToCurrentThread());
 683   if (url_throttler_entry_.get()) {
 684     int status_code = response_code_;
 685     if (status_code == URLFetcher::RESPONSE_CODE_INVALID) {
 686       // The status code will generally be known by the time clients
 687       // call the |ReceivedContentWasMalformed()| function (which ends up
 688       // calling the current function) but if it's not, we need to assume
 689       // the response was successful so that the total failure count
 690       // used to calculate exponential back-off goes up.
 691       status_code = 200;
 692     }
 693     url_throttler_entry_->ReceivedContentWasMalformed(status_code);
 694   }
 695 }
 696
 697 void URLFetcherCore::DidFinishWriting(int result) {
 698   if (result != OK) {
 699     CancelURLRequest(result);
 700     delegate_task_runner_->PostTask(
 701         FROM_HERE,
 702         base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
 703     return;
 704   }
 705   // If the file was successfully closed, then the URL request is complete.
 706   RetryOrCompleteUrlFetch();
 707 }
 708
 709 void URLFetcherCore::RetryOrCompleteUrlFetch() {
 710   DCHECK(network_task_runner_->BelongsToCurrentThread());
 711   base::TimeDelta backoff_delay;
 712
 713   // Checks the response from server.
 714   if (response_code_ >= 500 ||
 715       status_.error() == ERR_TEMPORARILY_THROTTLED) {
 716     // When encountering a server error, we will send the request again
 717     // after backoff time.
 718     ++num_retries_on_5xx_;
 719
 720     // Note that backoff_delay may be 0 because (a) the
 721     // URLRequestThrottlerManager and related code does not
 722     // necessarily back off on the first error, (b) it only backs off
 723     // on some of the 5xx status codes, (c) not all URLRequestContexts
 724     // have a throttler manager.
 725     base::TimeTicks backoff_release_time = GetBackoffReleaseTime();
 726     backoff_delay = backoff_release_time - base::TimeTicks::Now();
 727     if (backoff_delay < base::TimeDelta())
 728       backoff_delay = base::TimeDelta();
 729
 730     if (automatically_retry_on_5xx_ &&
 731         num_retries_on_5xx_ <= max_retries_on_5xx_) {
 732       StartOnIOThread();
 733       return;
 734     }
 735   } else {
 736     backoff_delay = base::TimeDelta();
 737   }
 738
 739   // Retry if the request failed due to network changes.
 740   if (status_.error() == ERR_NETWORK_CHANGED &&
 741       num_retries_on_network_changes_ < max_retries_on_network_changes_) {
 742     ++num_retries_on_network_changes_;
 743
 744     // Retry soon, after flushing all the current tasks which may include
 745     // further network change observers.
 746     network_task_runner_->PostTask(
 747         FROM_HERE, base::Bind(&URLFetcherCore::StartOnIOThread, this));
 748     return;
 749   }
 750
 751   request_context_getter_ = NULL;
 752   first_party_for_cookies_ = GURL();
 753   url_request_data_key_ = NULL;
 754   url_request_create_data_callback_.Reset();
 755   bool posted = delegate_task_runner_->PostTask(
 756       FROM_HERE,
 757       base::Bind(&URLFetcherCore::OnCompletedURLRequest, this, backoff_delay));
 758
 759   // If the delegate message loop does not exist any more, then the delegate
 760   // should be gone too.
 761   DCHECK(posted || !delegate_);
 762 }
 763
 764 void URLFetcherCore::ReleaseRequest() {
 765   upload_progress_checker_timer_.reset();
 766   request_.reset();
 767   g_registry.Get().RemoveURLFetcherCore(this);
 768 }
 769
 770 base::TimeTicks URLFetcherCore::GetBackoffReleaseTime() {
 771   DCHECK(network_task_runner_->BelongsToCurrentThread());
 772
 773   if (!original_url_throttler_entry_.get())
 774     return base::TimeTicks();
 775
 776   base::TimeTicks original_url_backoff =
 777       original_url_throttler_entry_->GetExponentialBackoffReleaseTime();
 778   base::TimeTicks destination_url_backoff;
 779   if (url_throttler_entry_.get() &&
 780       original_url_throttler_entry_.get() != url_throttler_entry_.get()) {
 781     destination_url_backoff =
 782         url_throttler_entry_->GetExponentialBackoffReleaseTime();
 783   }
 784
 785   return original_url_backoff > destination_url_backoff ?
 786       original_url_backoff : destination_url_backoff;
 787 }
 788
 789 void URLFetcherCore::CompleteAddingUploadDataChunk(
 790     const std::string& content, bool is_last_chunk) {
 791   if (was_cancelled_) {
 792     // Since CompleteAddingUploadDataChunk() is posted as a *delayed* task, it
 793     // may run after the URLFetcher was already stopped.
 794     return;
 795   }
 796   DCHECK(is_chunked_upload_);
 797   DCHECK(request_.get());
 798   DCHECK(!content.empty());
 799   request_->AppendChunkToUpload(content.data(),
 800                                 static_cast<int>(content.length()),
 801                                 is_last_chunk);
 802 }
 803
 804 int URLFetcherCore::WriteBuffer(scoped_refptr<DrainableIOBuffer> data) {
 805   while (data->BytesRemaining() > 0) {
 806     const int result = response_writer_->Write(
 807         data.get(),
 808         data->BytesRemaining(),
 809         base::Bind(&URLFetcherCore::DidWriteBuffer, this, data));
 810     if (result < 0) {
 811       if (result != ERR_IO_PENDING)
 812         DidWriteBuffer(data, result);
 813       return result;
 814     }
 815     data->DidConsume(result);
 816   }
 817   return OK;
 818 }
 819
 820 void URLFetcherCore::DidWriteBuffer(scoped_refptr<DrainableIOBuffer> data,
 821                                     int result) {
 822   if (result < 0) {  // Handle errors.
 823     CancelURLRequest(result);
 824     response_writer_->Finish(base::Bind(&EmptyCompletionCallback));
 825     delegate_task_runner_->PostTask(
 826         FROM_HERE,
 827         base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
 828     return;
 829   }
 830
 831   // Continue writing.
 832   data->DidConsume(result);
 833   if (WriteBuffer(data) < 0)
 834     return;
 835
 836   // Finished writing buffer_. Read some more, unless the request has been
 837   // cancelled and deleted.
 838   DCHECK_EQ(0, data->BytesRemaining());
 839   if (request_.get())
 840     ReadResponse();
 841 }
 842
 843 void URLFetcherCore::ReadResponse() {
 844   // Some servers may treat HEAD requests as GET requests. To free up the
 845   // network connection as soon as possible, signal that the request has
 846   // completed immediately, without trying to read any data back (all we care
 847   // about is the response code and headers, which we already have).
 848   int bytes_read = 0;
 849   if (request_->status().is_success() &&
 850       (request_type_ != URLFetcher::HEAD)) {
 851     if (!request_->Read(buffer_.get(), kBufferSize, &bytes_read))
 852       bytes_read = -1;  // Match OnReadCompleted() interface contract.
 853   }
 854   OnReadCompleted(request_.get(), bytes_read);
 855 }
 856
 857 void URLFetcherCore::InformDelegateUploadProgress() {
 858   DCHECK(network_task_runner_->BelongsToCurrentThread());
 859   if (request_.get()) {
 860     int64 current = request_->GetUploadProgress().position();
 861     if (current_upload_bytes_ != current) {
 862       current_upload_bytes_ = current;
 863       int64 total = -1;
 864       if (!is_chunked_upload_) {
 865         total = static_cast<int64>(request_->GetUploadProgress().size());
 866         // Total may be zero if the UploadDataStream::Init has not been called
 867         // yet. Don't send the upload progress until the size is initialized.
 868         if (!total)
 869           return;
 870       }
 871       delegate_task_runner_->PostTask(
 872           FROM_HERE,
 873           base::Bind(
 874               &URLFetcherCore::InformDelegateUploadProgressInDelegateThread,
 875               this, current, total));
 876     }
 877   }
 878 }
 879
 880 void URLFetcherCore::InformDelegateUploadProgressInDelegateThread(
 881     int64 current, int64 total) {
 882   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 883   if (delegate_)
 884     delegate_->OnURLFetchUploadProgress(fetcher_, current, total);
 885 }
 886
 887 void URLFetcherCore::InformDelegateDownloadProgress() {
 888   DCHECK(network_task_runner_->BelongsToCurrentThread());
 889   delegate_task_runner_->PostTask(
 890       FROM_HERE,
 891       base::Bind(
 892           &URLFetcherCore::InformDelegateDownloadProgressInDelegateThread,
 893           this, current_response_bytes_, total_response_bytes_));
 894 }
 895
 896 void URLFetcherCore::InformDelegateDownloadProgressInDelegateThread(
 897     int64 current, int64 total) {
 898   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 899   if (delegate_)
 900     delegate_->OnURLFetchDownloadProgress(fetcher_, current, total);
 901 }
 902
 903 }  // namespace net