src/net/url_request/url_fetcher_core.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/url_request/url_fetcher_core.h"
   6
   7 #include "base/bind.h"
   8 #include "base/logging.h"
   9 #include "base/metrics/histogram.h"
  10 #include "base/sequenced_task_runner.h"
  11 #include "base/single_thread_task_runner.h"
  12 #include "base/stl_util.h"
  13 #include "base/thread_task_runner_handle.h"
  14 #include "base/tracked_objects.h"
  15 #include "net/base/io_buffer.h"
  16 #include "net/base/load_flags.h"
  17 #include "net/base/net_errors.h"
  18 #include "net/base/request_priority.h"
  19 #include "net/base/upload_bytes_element_reader.h"
  20 #include "net/base/upload_data_stream.h"
  21 #include "net/base/upload_file_element_reader.h"
  22 #include "net/http/http_response_headers.h"
  23 #include "net/url_request/redirect_info.h"
  24 #include "net/url_request/url_fetcher_delegate.h"
  25 #include "net/url_request/url_fetcher_response_writer.h"
  26 #include "net/url_request/url_request_context.h"
  27 #include "net/url_request/url_request_context_getter.h"
  28 #include "net/url_request/url_request_throttler_manager.h"
  29
  30 namespace {
  31
  32 const int kBufferSize = 4096;
  33 const int kUploadProgressTimerInterval = 100;
  34 bool g_interception_enabled = false;
  35 bool g_ignore_certificate_requests = false;
  36
  37 void EmptyCompletionCallback(int result) {}
  38
  39 }  // namespace
  40
  41 namespace net {
  42
  43 // URLFetcherCore::Registry ---------------------------------------------------
  44
  45 URLFetcherCore::Registry::Registry() {}
  46 URLFetcherCore::Registry::~Registry() {}
  47
  48 void URLFetcherCore::Registry::AddURLFetcherCore(URLFetcherCore* core) {
  49   DCHECK(!ContainsKey(fetchers_, core));
  50   fetchers_.insert(core);
  51 }
  52
  53 void URLFetcherCore::Registry::RemoveURLFetcherCore(URLFetcherCore* core) {
  54   DCHECK(ContainsKey(fetchers_, core));
  55   fetchers_.erase(core);
  56 }
  57
  58 void URLFetcherCore::Registry::CancelAll() {
  59   while (!fetchers_.empty())
  60     (*fetchers_.begin())->CancelURLRequest(ERR_ABORTED);
  61 }
  62
  63 // URLFetcherCore -------------------------------------------------------------
  64
  65 // static
  66 base::LazyInstance<URLFetcherCore::Registry>
  67     URLFetcherCore::g_registry = LAZY_INSTANCE_INITIALIZER;
  68
  69 URLFetcherCore::URLFetcherCore(URLFetcher* fetcher,
  70                                const GURL& original_url,
  71                                URLFetcher::RequestType request_type,
  72                                URLFetcherDelegate* d)
  73     : fetcher_(fetcher),
  74       original_url_(original_url),
  75       request_type_(request_type),
  76       delegate_(d),
  77       delegate_task_runner_(base::ThreadTaskRunnerHandle::Get()),
  78       load_flags_(LOAD_NORMAL),
  79       response_code_(URLFetcher::RESPONSE_CODE_INVALID),
  80       buffer_(new IOBuffer(kBufferSize)),
  81       url_request_data_key_(NULL),
  82       was_fetched_via_proxy_(false),
  83       upload_content_set_(false),
  84       upload_range_offset_(0),
  85       upload_range_length_(0),
  86       referrer_policy_(
  87           URLRequest::CLEAR_REFERRER_ON_TRANSITION_FROM_SECURE_TO_INSECURE),
  88       is_chunked_upload_(false),
  89       was_cancelled_(false),
  90       stop_on_redirect_(false),
  91       stopped_on_redirect_(false),
  92       automatically_retry_on_5xx_(true),
  93       num_retries_on_5xx_(0),
  94       max_retries_on_5xx_(0),
  95       num_retries_on_network_changes_(0),
  96       max_retries_on_network_changes_(0),
  97       current_upload_bytes_(-1),
  98       current_response_bytes_(0),
  99       total_response_bytes_(-1) {
 100   CHECK(original_url_.is_valid());
 101 }
 102
 103 void URLFetcherCore::Start() {
 104   DCHECK(delegate_task_runner_.get());
 105   DCHECK(request_context_getter_.get()) << "We need an URLRequestContext!";
 106   if (network_task_runner_.get()) {
 107     DCHECK_EQ(network_task_runner_,
 108               request_context_getter_->GetNetworkTaskRunner());
 109   } else {
 110     network_task_runner_ = request_context_getter_->GetNetworkTaskRunner();
 111   }
 112   DCHECK(network_task_runner_.get()) << "We need an IO task runner";
 113
 114   network_task_runner_->PostTask(
 115       FROM_HERE, base::Bind(&URLFetcherCore::StartOnIOThread, this));
 116 }
 117
 118 void URLFetcherCore::Stop() {
 119   if (delegate_task_runner_.get())  // May be NULL in tests.
 120     DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 121
 122   delegate_ = NULL;
 123   fetcher_ = NULL;
 124   if (!network_task_runner_.get())
 125     return;
 126   if (network_task_runner_->RunsTasksOnCurrentThread()) {
 127     CancelURLRequest(ERR_ABORTED);
 128   } else {
 129     network_task_runner_->PostTask(
 130         FROM_HERE,
 131         base::Bind(&URLFetcherCore::CancelURLRequest, this, ERR_ABORTED));
 132   }
 133 }
 134
 135 void URLFetcherCore::SetUploadData(const std::string& upload_content_type,
 136                                    const std::string& upload_content) {
 137   DCHECK(!is_chunked_upload_);
 138   DCHECK(!upload_content_set_);
 139   DCHECK(upload_content_.empty());
 140   DCHECK(upload_file_path_.empty());
 141   DCHECK(upload_content_type_.empty());
 142
 143   // Empty |upload_content_type| is allowed iff the |upload_content| is empty.
 144   DCHECK(upload_content.empty() || !upload_content_type.empty());
 145
 146   upload_content_type_ = upload_content_type;
 147   upload_content_ = upload_content;
 148   upload_content_set_ = true;
 149 }
 150
 151 void URLFetcherCore::SetUploadFilePath(
 152     const std::string& upload_content_type,
 153     const base::FilePath& file_path,
 154     uint64 range_offset,
 155     uint64 range_length,
 156     scoped_refptr<base::TaskRunner> file_task_runner) {
 157   DCHECK(!is_chunked_upload_);
 158   DCHECK(!upload_content_set_);
 159   DCHECK(upload_content_.empty());
 160   DCHECK(upload_file_path_.empty());
 161   DCHECK_EQ(upload_range_offset_, 0ULL);
 162   DCHECK_EQ(upload_range_length_, 0ULL);
 163   DCHECK(upload_content_type_.empty());
 164   DCHECK(!upload_content_type.empty());
 165
 166   upload_content_type_ = upload_content_type;
 167   upload_file_path_ = file_path;
 168   upload_range_offset_ = range_offset;
 169   upload_range_length_ = range_length;
 170   upload_file_task_runner_ = file_task_runner;
 171   upload_content_set_ = true;
 172 }
 173
 174 void URLFetcherCore::SetChunkedUpload(const std::string& content_type) {
 175   DCHECK(is_chunked_upload_ ||
 176          (upload_content_type_.empty() &&
 177           upload_content_.empty()));
 178
 179   // Empty |content_type| is not allowed here, because it is impossible
 180   // to ensure non-empty upload content as it is not yet supplied.
 181   DCHECK(!content_type.empty());
 182
 183   upload_content_type_ = content_type;
 184   upload_content_.clear();
 185   is_chunked_upload_ = true;
 186 }
 187
 188 void URLFetcherCore::AppendChunkToUpload(const std::string& content,
 189                                          bool is_last_chunk) {
 190   DCHECK(delegate_task_runner_.get());
 191   DCHECK(network_task_runner_.get());
 192   network_task_runner_->PostTask(
 193       FROM_HERE,
 194       base::Bind(&URLFetcherCore::CompleteAddingUploadDataChunk, this, content,
 195                  is_last_chunk));
 196 }
 197
 198 void URLFetcherCore::SetLoadFlags(int load_flags) {
 199   load_flags_ = load_flags;
 200 }
 201
 202 int URLFetcherCore::GetLoadFlags() const {
 203   return load_flags_;
 204 }
 205
 206 void URLFetcherCore::SetReferrer(const std::string& referrer) {
 207   referrer_ = referrer;
 208 }
 209
 210 void URLFetcherCore::SetReferrerPolicy(
 211     URLRequest::ReferrerPolicy referrer_policy) {
 212   referrer_policy_ = referrer_policy;
 213 }
 214
 215 void URLFetcherCore::SetExtraRequestHeaders(
 216     const std::string& extra_request_headers) {
 217   extra_request_headers_.Clear();
 218   extra_request_headers_.AddHeadersFromString(extra_request_headers);
 219 }
 220
 221 void URLFetcherCore::AddExtraRequestHeader(const std::string& header_line) {
 222   extra_request_headers_.AddHeaderFromString(header_line);
 223 }
 224
 225 void URLFetcherCore::SetRequestContext(
 226     URLRequestContextGetter* request_context_getter) {
 227   DCHECK(!request_context_getter_.get());
 228   DCHECK(request_context_getter);
 229   request_context_getter_ = request_context_getter;
 230 }
 231
 232 void URLFetcherCore::SetFirstPartyForCookies(
 233     const GURL& first_party_for_cookies) {
 234   DCHECK(first_party_for_cookies_.is_empty());
 235   first_party_for_cookies_ = first_party_for_cookies;
 236 }
 237
 238 void URLFetcherCore::SetURLRequestUserData(
 239     const void* key,
 240     const URLFetcher::CreateDataCallback& create_data_callback) {
 241   DCHECK(key);
 242   DCHECK(!create_data_callback.is_null());
 243   url_request_data_key_ = key;
 244   url_request_create_data_callback_ = create_data_callback;
 245 }
 246
 247 void URLFetcherCore::SetStopOnRedirect(bool stop_on_redirect) {
 248   stop_on_redirect_ = stop_on_redirect;
 249 }
 250
 251 void URLFetcherCore::SetAutomaticallyRetryOn5xx(bool retry) {
 252   automatically_retry_on_5xx_ = retry;
 253 }
 254
 255 void URLFetcherCore::SetMaxRetriesOn5xx(int max_retries) {
 256   max_retries_on_5xx_ = max_retries;
 257 }
 258
 259 int URLFetcherCore::GetMaxRetriesOn5xx() const {
 260   return max_retries_on_5xx_;
 261 }
 262
 263 base::TimeDelta URLFetcherCore::GetBackoffDelay() const {
 264   return backoff_delay_;
 265 }
 266
 267 void URLFetcherCore::SetAutomaticallyRetryOnNetworkChanges(int max_retries) {
 268   max_retries_on_network_changes_ = max_retries;
 269 }
 270
 271 void URLFetcherCore::SaveResponseToFileAtPath(
 272     const base::FilePath& file_path,
 273     scoped_refptr<base::SequencedTaskRunner> file_task_runner) {
 274   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 275   SaveResponseWithWriter(scoped_ptr<URLFetcherResponseWriter>(
 276       new URLFetcherFileWriter(file_task_runner, file_path)));
 277 }
 278
 279 void URLFetcherCore::SaveResponseToTemporaryFile(
 280     scoped_refptr<base::SequencedTaskRunner> file_task_runner) {
 281   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 282   SaveResponseWithWriter(scoped_ptr<URLFetcherResponseWriter>(
 283       new URLFetcherFileWriter(file_task_runner, base::FilePath())));
 284 }
 285
 286 void URLFetcherCore::SaveResponseWithWriter(
 287     scoped_ptr<URLFetcherResponseWriter> response_writer) {
 288   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 289   response_writer_ = response_writer.Pass();
 290 }
 291
 292 HttpResponseHeaders* URLFetcherCore::GetResponseHeaders() const {
 293   return response_headers_.get();
 294 }
 295
 296 // TODO(panayiotis): socket_address_ is written in the IO thread,
 297 // if this is accessed in the UI thread, this could result in a race.
 298 // Same for response_headers_ above and was_fetched_via_proxy_ below.
 299 HostPortPair URLFetcherCore::GetSocketAddress() const {
 300   return socket_address_;
 301 }
 302
 303 bool URLFetcherCore::WasFetchedViaProxy() const {
 304   return was_fetched_via_proxy_;
 305 }
 306
 307 const GURL& URLFetcherCore::GetOriginalURL() const {
 308   return original_url_;
 309 }
 310
 311 const GURL& URLFetcherCore::GetURL() const {
 312   return url_;
 313 }
 314
 315 const URLRequestStatus& URLFetcherCore::GetStatus() const {
 316   return status_;
 317 }
 318
 319 int URLFetcherCore::GetResponseCode() const {
 320   return response_code_;
 321 }
 322
 323 const ResponseCookies& URLFetcherCore::GetCookies() const {
 324   return cookies_;
 325 }
 326
 327 void URLFetcherCore::ReceivedContentWasMalformed() {
 328   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 329   if (network_task_runner_.get()) {
 330     network_task_runner_->PostTask(
 331         FROM_HERE, base::Bind(&URLFetcherCore::NotifyMalformedContent, this));
 332   }
 333 }
 334
 335 bool URLFetcherCore::GetResponseAsString(
 336     std::string* out_response_string) const {
 337   URLFetcherStringWriter* string_writer =
 338       response_writer_ ? response_writer_->AsStringWriter() : NULL;
 339   if (!string_writer)
 340     return false;
 341
 342   *out_response_string = string_writer->data();
 343   UMA_HISTOGRAM_MEMORY_KB("UrlFetcher.StringResponseSize",
 344                           (string_writer->data().length() / 1024));
 345   return true;
 346 }
 347
 348 bool URLFetcherCore::GetResponseAsFilePath(bool take_ownership,
 349                                            base::FilePath* out_response_path) {
 350   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 351
 352   URLFetcherFileWriter* file_writer =
 353       response_writer_ ? response_writer_->AsFileWriter() : NULL;
 354   if (!file_writer)
 355     return false;
 356
 357   *out_response_path = file_writer->file_path();
 358
 359   if (take_ownership) {
 360     // Intentionally calling a file_writer_ method directly without posting
 361     // the task to network_task_runner_.
 362     //
 363     // This is for correctly handling the case when file_writer_->DisownFile()
 364     // is soon followed by URLFetcherCore::Stop(). We have to make sure that
 365     // DisownFile takes effect before Stop deletes file_writer_.
 366     //
 367     // This direct call should be thread-safe, since DisownFile itself does no
 368     // file operation. It just flips the state to be referred in destruction.
 369     file_writer->DisownFile();
 370   }
 371   return true;
 372 }
 373
 374 void URLFetcherCore::OnReceivedRedirect(URLRequest* request,
 375                                         const RedirectInfo& redirect_info,
 376                                         bool* defer_redirect) {
 377   DCHECK_EQ(request, request_.get());
 378   DCHECK(network_task_runner_->BelongsToCurrentThread());
 379   if (stop_on_redirect_) {
 380     stopped_on_redirect_ = true;
 381     url_ = redirect_info.new_url;
 382     response_code_ = request_->GetResponseCode();
 383     was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
 384     request->Cancel();
 385     OnReadCompleted(request, 0);
 386   }
 387 }
 388
 389 void URLFetcherCore::OnResponseStarted(URLRequest* request) {
 390   DCHECK_EQ(request, request_.get());
 391   DCHECK(network_task_runner_->BelongsToCurrentThread());
 392   if (request_->status().is_success()) {
 393     response_code_ = request_->GetResponseCode();
 394     response_headers_ = request_->response_headers();
 395     socket_address_ = request_->GetSocketAddress();
 396     was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
 397     total_response_bytes_ = request_->GetExpectedContentSize();
 398   }
 399
 400   ReadResponse();
 401 }
 402
 403 void URLFetcherCore::OnCertificateRequested(
 404     URLRequest* request,
 405     SSLCertRequestInfo* cert_request_info) {
 406   DCHECK_EQ(request, request_.get());
 407   DCHECK(network_task_runner_->BelongsToCurrentThread());
 408
 409   if (g_ignore_certificate_requests) {
 410     request->ContinueWithCertificate(NULL);
 411   } else {
 412     request->Cancel();
 413   }
 414 }
 415
 416 void URLFetcherCore::OnReadCompleted(URLRequest* request,
 417                                      int bytes_read) {
 418   DCHECK(request == request_);
 419   DCHECK(network_task_runner_->BelongsToCurrentThread());
 420
 421   if (!stopped_on_redirect_)
 422     url_ = request->url();
 423   URLRequestThrottlerManager* throttler_manager =
 424       request->context()->throttler_manager();
 425   if (throttler_manager) {
 426     url_throttler_entry_ = throttler_manager->RegisterRequestUrl(url_);
 427   }
 428
 429   do {
 430     if (!request_->status().is_success() || bytes_read <= 0)
 431       break;
 432
 433     current_response_bytes_ += bytes_read;
 434     InformDelegateDownloadProgress();
 435
 436     const int result =
 437         WriteBuffer(new DrainableIOBuffer(buffer_.get(), bytes_read));
 438     if (result < 0) {
 439       // Write failed or waiting for write completion.
 440       return;
 441     }
 442   } while (request_->Read(buffer_.get(), kBufferSize, &bytes_read));
 443
 444   const URLRequestStatus status = request_->status();
 445
 446   if (status.is_success())
 447     request_->GetResponseCookies(&cookies_);
 448
 449   // See comments re: HEAD requests in ReadResponse().
 450   if (!status.is_io_pending() || request_type_ == URLFetcher::HEAD) {
 451     status_ = status;
 452     ReleaseRequest();
 453
 454     // No more data to write.
 455     const int result = response_writer_->Finish(
 456         base::Bind(&URLFetcherCore::DidFinishWriting, this));
 457     if (result != ERR_IO_PENDING)
 458       DidFinishWriting(result);
 459   }
 460 }
 461
 462 void URLFetcherCore::CancelAll() {
 463   g_registry.Get().CancelAll();
 464 }
 465
 466 int URLFetcherCore::GetNumFetcherCores() {
 467   return g_registry.Get().size();
 468 }
 469
 470 void URLFetcherCore::SetEnableInterceptionForTests(bool enabled) {
 471   g_interception_enabled = enabled;
 472 }
 473
 474 void URLFetcherCore::SetIgnoreCertificateRequests(bool ignored) {
 475   g_ignore_certificate_requests = ignored;
 476 }
 477
 478 URLFetcherCore::~URLFetcherCore() {
 479   // |request_| should be NULL.  If not, it's unsafe to delete it here since we
 480   // may not be on the IO thread.
 481   DCHECK(!request_.get());
 482 }
 483
 484 void URLFetcherCore::StartOnIOThread() {
 485   DCHECK(network_task_runner_->BelongsToCurrentThread());
 486
 487   if (!response_writer_)
 488     response_writer_.reset(new URLFetcherStringWriter);
 489
 490   const int result = response_writer_->Initialize(
 491       base::Bind(&URLFetcherCore::DidInitializeWriter, this));
 492   if (result != ERR_IO_PENDING)
 493     DidInitializeWriter(result);
 494 }
 495
 496 void URLFetcherCore::StartURLRequest() {
 497   DCHECK(network_task_runner_->BelongsToCurrentThread());
 498
 499   if (was_cancelled_) {
 500     // Since StartURLRequest() is posted as a *delayed* task, it may
 501     // run after the URLFetcher was already stopped.
 502     return;
 503   }
 504
 505   DCHECK(request_context_getter_.get());
 506   DCHECK(!request_.get());
 507
 508   g_registry.Get().AddURLFetcherCore(this);
 509   current_response_bytes_ = 0;
 510   request_ = request_context_getter_->GetURLRequestContext()->CreateRequest(
 511       original_url_, DEFAULT_PRIORITY, this, NULL);
 512   request_->set_stack_trace(stack_trace_);
 513   int flags = request_->load_flags() | load_flags_;
 514   if (!g_interception_enabled)
 515     flags = flags | LOAD_DISABLE_INTERCEPT;
 516
 517   if (is_chunked_upload_)
 518     request_->EnableChunkedUpload();
 519   request_->SetLoadFlags(flags);
 520   request_->SetReferrer(referrer_);
 521   request_->set_referrer_policy(referrer_policy_);
 522   request_->set_first_party_for_cookies(first_party_for_cookies_.is_empty() ?
 523       original_url_ : first_party_for_cookies_);
 524   if (url_request_data_key_ && !url_request_create_data_callback_.is_null()) {
 525     request_->SetUserData(url_request_data_key_,
 526                           url_request_create_data_callback_.Run());
 527   }
 528
 529   switch (request_type_) {
 530     case URLFetcher::GET:
 531       break;
 532
 533     case URLFetcher::POST:
 534     case URLFetcher::PUT:
 535     case URLFetcher::PATCH:
 536       // Upload content must be set.
 537       DCHECK(is_chunked_upload_ || upload_content_set_);
 538
 539       request_->set_method(
 540           request_type_ == URLFetcher::POST ? "POST" :
 541           request_type_ == URLFetcher::PUT ? "PUT" : "PATCH");
 542       if (!upload_content_type_.empty()) {
 543         extra_request_headers_.SetHeader(HttpRequestHeaders::kContentType,
 544                                          upload_content_type_);
 545       }
 546       if (!upload_content_.empty()) {
 547         scoped_ptr<UploadElementReader> reader(new UploadBytesElementReader(
 548             upload_content_.data(), upload_content_.size()));
 549         request_->set_upload(make_scoped_ptr(
 550             UploadDataStream::CreateWithReader(reader.Pass(), 0)));
 551       } else if (!upload_file_path_.empty()) {
 552         scoped_ptr<UploadElementReader> reader(
 553             new UploadFileElementReader(upload_file_task_runner_.get(),
 554                                         upload_file_path_,
 555                                         upload_range_offset_,
 556                                         upload_range_length_,
 557                                         base::Time()));
 558         request_->set_upload(make_scoped_ptr(
 559             UploadDataStream::CreateWithReader(reader.Pass(), 0)));
 560       }
 561
 562       current_upload_bytes_ = -1;
 563       // TODO(kinaba): http://crbug.com/118103. Implement upload callback in the
 564       //  layer and avoid using timer here.
 565       upload_progress_checker_timer_.reset(
 566           new base::RepeatingTimer<URLFetcherCore>());
 567       upload_progress_checker_timer_->Start(
 568           FROM_HERE,
 569           base::TimeDelta::FromMilliseconds(kUploadProgressTimerInterval),
 570           this,
 571           &URLFetcherCore::InformDelegateUploadProgress);
 572       break;
 573
 574     case URLFetcher::HEAD:
 575       request_->set_method("HEAD");
 576       break;
 577
 578     case URLFetcher::DELETE_REQUEST:
 579       request_->set_method("DELETE");
 580       break;
 581
 582     default:
 583       NOTREACHED();
 584   }
 585
 586   if (!extra_request_headers_.IsEmpty())
 587     request_->SetExtraRequestHeaders(extra_request_headers_);
 588
 589   request_->Start();
 590 }
 591
 592 void URLFetcherCore::DidInitializeWriter(int result) {
 593   if (result != OK) {
 594     CancelURLRequest(result);
 595     delegate_task_runner_->PostTask(
 596         FROM_HERE,
 597         base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
 598     return;
 599   }
 600   StartURLRequestWhenAppropriate();
 601 }
 602
 603 void URLFetcherCore::StartURLRequestWhenAppropriate() {
 604   DCHECK(network_task_runner_->BelongsToCurrentThread());
 605
 606   if (was_cancelled_)
 607     return;
 608
 609   DCHECK(request_context_getter_.get());
 610
 611   int64 delay = 0LL;
 612   if (original_url_throttler_entry_.get() == NULL) {
 613     URLRequestThrottlerManager* manager =
 614         request_context_getter_->GetURLRequestContext()->throttler_manager();
 615     if (manager) {
 616       original_url_throttler_entry_ =
 617           manager->RegisterRequestUrl(original_url_);
 618     }
 619   }
 620   if (original_url_throttler_entry_.get() != NULL) {
 621     delay = original_url_throttler_entry_->ReserveSendingTimeForNextRequest(
 622         GetBackoffReleaseTime());
 623   }
 624
 625   if (delay == 0) {
 626     StartURLRequest();
 627   } else {
 628     base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
 629         FROM_HERE, base::Bind(&URLFetcherCore::StartURLRequest, this),
 630         base::TimeDelta::FromMilliseconds(delay));
 631   }
 632 }
 633
 634 void URLFetcherCore::CancelURLRequest(int error) {
 635   DCHECK(network_task_runner_->BelongsToCurrentThread());
 636
 637   if (request_.get()) {
 638     request_->CancelWithError(error);
 639     ReleaseRequest();
 640   }
 641
 642   // Set the error manually.
 643   // Normally, calling URLRequest::CancelWithError() results in calling
 644   // OnReadCompleted() with bytes_read = -1 via an asynchronous task posted by
 645   // URLRequestJob::NotifyDone(). But, because the request was released
 646   // immediately after being canceled, the request could not call
 647   // OnReadCompleted() which overwrites |status_| with the error status.
 648   status_.set_status(URLRequestStatus::CANCELED);
 649   status_.set_error(error);
 650
 651   // Release the reference to the request context. There could be multiple
 652   // references to URLFetcher::Core at this point so it may take a while to
 653   // delete the object, but we cannot delay the destruction of the request
 654   // context.
 655   request_context_getter_ = NULL;
 656   first_party_for_cookies_ = GURL();
 657   url_request_data_key_ = NULL;
 658   url_request_create_data_callback_.Reset();
 659   was_cancelled_ = true;
 660 }
 661
 662 void URLFetcherCore::OnCompletedURLRequest(
 663     base::TimeDelta backoff_delay) {
 664   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 665
 666   // Save the status and backoff_delay so that delegates can read it.
 667   if (delegate_) {
 668     backoff_delay_ = backoff_delay;
 669     InformDelegateFetchIsComplete();
 670   }
 671 }
 672
 673 void URLFetcherCore::InformDelegateFetchIsComplete() {
 674   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 675   if (delegate_)
 676     delegate_->OnURLFetchComplete(fetcher_);
 677 }
 678
 679 void URLFetcherCore::NotifyMalformedContent() {
 680   DCHECK(network_task_runner_->BelongsToCurrentThread());
 681   if (url_throttler_entry_.get() != NULL) {
 682     int status_code = response_code_;
 683     if (status_code == URLFetcher::RESPONSE_CODE_INVALID) {
 684       // The status code will generally be known by the time clients
 685       // call the |ReceivedContentWasMalformed()| function (which ends up
 686       // calling the current function) but if it's not, we need to assume
 687       // the response was successful so that the total failure count
 688       // used to calculate exponential back-off goes up.
 689       status_code = 200;
 690     }
 691     url_throttler_entry_->ReceivedContentWasMalformed(status_code);
 692   }
 693 }
 694
 695 void URLFetcherCore::DidFinishWriting(int result) {
 696   if (result != OK) {
 697     CancelURLRequest(result);
 698     delegate_task_runner_->PostTask(
 699         FROM_HERE,
 700         base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
 701     return;
 702   }
 703   // If the file was successfully closed, then the URL request is complete.
 704   RetryOrCompleteUrlFetch();
 705 }
 706
 707 void URLFetcherCore::RetryOrCompleteUrlFetch() {
 708   DCHECK(network_task_runner_->BelongsToCurrentThread());
 709   base::TimeDelta backoff_delay;
 710
 711   // Checks the response from server.
 712   if (response_code_ >= 500 ||
 713       status_.error() == ERR_TEMPORARILY_THROTTLED) {
 714     // When encountering a server error, we will send the request again
 715     // after backoff time.
 716     ++num_retries_on_5xx_;
 717
 718     // Note that backoff_delay may be 0 because (a) the
 719     // URLRequestThrottlerManager and related code does not
 720     // necessarily back off on the first error, (b) it only backs off
 721     // on some of the 5xx status codes, (c) not all URLRequestContexts
 722     // have a throttler manager.
 723     base::TimeTicks backoff_release_time = GetBackoffReleaseTime();
 724     backoff_delay = backoff_release_time - base::TimeTicks::Now();
 725     if (backoff_delay < base::TimeDelta())
 726       backoff_delay = base::TimeDelta();
 727
 728     if (automatically_retry_on_5xx_ &&
 729         num_retries_on_5xx_ <= max_retries_on_5xx_) {
 730       StartOnIOThread();
 731       return;
 732     }
 733   } else {
 734     backoff_delay = base::TimeDelta();
 735   }
 736
 737   // Retry if the request failed due to network changes.
 738   if (status_.error() == ERR_NETWORK_CHANGED &&
 739       num_retries_on_network_changes_ < max_retries_on_network_changes_) {
 740     ++num_retries_on_network_changes_;
 741
 742     // Retry soon, after flushing all the current tasks which may include
 743     // further network change observers.
 744     network_task_runner_->PostTask(
 745         FROM_HERE, base::Bind(&URLFetcherCore::StartOnIOThread, this));
 746     return;
 747   }
 748
 749   request_context_getter_ = NULL;
 750   first_party_for_cookies_ = GURL();
 751   url_request_data_key_ = NULL;
 752   url_request_create_data_callback_.Reset();
 753   bool posted = delegate_task_runner_->PostTask(
 754       FROM_HERE,
 755       base::Bind(&URLFetcherCore::OnCompletedURLRequest, this, backoff_delay));
 756
 757   // If the delegate message loop does not exist any more, then the delegate
 758   // should be gone too.
 759   DCHECK(posted || !delegate_);
 760 }
 761
 762 void URLFetcherCore::ReleaseRequest() {
 763   upload_progress_checker_timer_.reset();
 764   request_.reset();
 765   g_registry.Get().RemoveURLFetcherCore(this);
 766 }
 767
 768 base::TimeTicks URLFetcherCore::GetBackoffReleaseTime() {
 769   DCHECK(network_task_runner_->BelongsToCurrentThread());
 770
 771   if (original_url_throttler_entry_.get()) {
 772     base::TimeTicks original_url_backoff =
 773         original_url_throttler_entry_->GetExponentialBackoffReleaseTime();
 774     base::TimeTicks destination_url_backoff;
 775     if (url_throttler_entry_.get() != NULL &&
 776         original_url_throttler_entry_.get() != url_throttler_entry_.get()) {
 777       destination_url_backoff =
 778           url_throttler_entry_->GetExponentialBackoffReleaseTime();
 779     }
 780
 781     return original_url_backoff > destination_url_backoff ?
 782         original_url_backoff : destination_url_backoff;
 783   } else {
 784     return base::TimeTicks();
 785   }
 786 }
 787
 788 void URLFetcherCore::CompleteAddingUploadDataChunk(
 789     const std::string& content, bool is_last_chunk) {
 790   if (was_cancelled_) {
 791     // Since CompleteAddingUploadDataChunk() is posted as a *delayed* task, it
 792     // may run after the URLFetcher was already stopped.
 793     return;
 794   }
 795   DCHECK(is_chunked_upload_);
 796   DCHECK(request_.get());
 797   DCHECK(!content.empty());
 798   request_->AppendChunkToUpload(content.data(),
 799                                 static_cast<int>(content.length()),
 800                                 is_last_chunk);
 801 }
 802
 803 int URLFetcherCore::WriteBuffer(scoped_refptr<DrainableIOBuffer> data) {
 804   while (data->BytesRemaining() > 0) {
 805     const int result = response_writer_->Write(
 806         data.get(),
 807         data->BytesRemaining(),
 808         base::Bind(&URLFetcherCore::DidWriteBuffer, this, data));
 809     if (result < 0) {
 810       if (result != ERR_IO_PENDING)
 811         DidWriteBuffer(data, result);
 812       return result;
 813     }
 814     data->DidConsume(result);
 815   }
 816   return OK;
 817 }
 818
 819 void URLFetcherCore::DidWriteBuffer(scoped_refptr<DrainableIOBuffer> data,
 820                                     int result) {
 821   if (result < 0) {  // Handle errors.
 822     CancelURLRequest(result);
 823     response_writer_->Finish(base::Bind(&EmptyCompletionCallback));
 824     delegate_task_runner_->PostTask(
 825         FROM_HERE,
 826         base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
 827     return;
 828   }
 829
 830   // Continue writing.
 831   data->DidConsume(result);
 832   if (WriteBuffer(data) < 0)
 833     return;
 834
 835   // Finished writing buffer_. Read some more, unless the request has been
 836   // cancelled and deleted.
 837   DCHECK_EQ(0, data->BytesRemaining());
 838   if (request_.get())
 839     ReadResponse();
 840 }
 841
 842 void URLFetcherCore::ReadResponse() {
 843   // Some servers may treat HEAD requests as GET requests.  To free up the
 844   // network connection as soon as possible, signal that the request has
 845   // completed immediately, without trying to read any data back (all we care
 846   // about is the response code and headers, which we already have).
 847   int bytes_read = 0;
 848   if (request_->status().is_success() &&
 849       (request_type_ != URLFetcher::HEAD))
 850     request_->Read(buffer_.get(), kBufferSize, &bytes_read);
 851   OnReadCompleted(request_.get(), bytes_read);
 852 }
 853
 854 void URLFetcherCore::InformDelegateUploadProgress() {
 855   DCHECK(network_task_runner_->BelongsToCurrentThread());
 856   if (request_.get()) {
 857     int64 current = request_->GetUploadProgress().position();
 858     if (current_upload_bytes_ != current) {
 859       current_upload_bytes_ = current;
 860       int64 total = -1;
 861       if (!is_chunked_upload_) {
 862         total = static_cast<int64>(request_->GetUploadProgress().size());
 863         // Total may be zero if the UploadDataStream::Init has not been called
 864         // yet.  Don't send the upload progress until the size is initialized.
 865         if (!total)
 866           return;
 867       }
 868       delegate_task_runner_->PostTask(
 869           FROM_HERE,
 870           base::Bind(
 871               &URLFetcherCore::InformDelegateUploadProgressInDelegateThread,
 872               this, current, total));
 873     }
 874   }
 875 }
 876
 877 void URLFetcherCore::InformDelegateUploadProgressInDelegateThread(
 878     int64 current, int64 total) {
 879   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 880   if (delegate_)
 881     delegate_->OnURLFetchUploadProgress(fetcher_, current, total);
 882 }
 883
 884 void URLFetcherCore::InformDelegateDownloadProgress() {
 885   DCHECK(network_task_runner_->BelongsToCurrentThread());
 886   delegate_task_runner_->PostTask(
 887       FROM_HERE,
 888       base::Bind(
 889           &URLFetcherCore::InformDelegateDownloadProgressInDelegateThread,
 890           this, current_response_bytes_, total_response_bytes_));
 891 }
 892
 893 void URLFetcherCore::InformDelegateDownloadProgressInDelegateThread(
 894     int64 current, int64 total) {
 895   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 896   if (delegate_)
 897     delegate_->OnURLFetchDownloadProgress(fetcher_, current, total);
 898 }
 899
 900 }  // namespace net