src/net/url_request/url_fetcher_core.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/url_request/url_fetcher_core.h"
   6
   7 #include "base/bind.h"
   8 #include "base/logging.h"
   9 #include "base/metrics/histogram.h"
  10 #include "base/single_thread_task_runner.h"
  11 #include "base/stl_util.h"
  12 #include "base/thread_task_runner_handle.h"
  13 #include "base/tracked_objects.h"
  14 #include "net/base/io_buffer.h"
  15 #include "net/base/load_flags.h"
  16 #include "net/base/net_errors.h"
  17 #include "net/base/request_priority.h"
  18 #include "net/base/upload_bytes_element_reader.h"
  19 #include "net/base/upload_data_stream.h"
  20 #include "net/base/upload_file_element_reader.h"
  21 #include "net/http/http_response_headers.h"
  22 #include "net/url_request/url_fetcher_delegate.h"
  23 #include "net/url_request/url_fetcher_response_writer.h"
  24 #include "net/url_request/url_request_context.h"
  25 #include "net/url_request/url_request_context_getter.h"
  26 #include "net/url_request/url_request_throttler_manager.h"
  27
  28 namespace {
  29
  30 const int kBufferSize = 4096;
  31 const int kUploadProgressTimerInterval = 100;
  32 bool g_interception_enabled = false;
  33 bool g_ignore_certificate_requests = false;
  34
  35 void EmptyCompletionCallback(int result) {}
  36
  37 }  // namespace
  38
  39 namespace net {
  40
  41 // URLFetcherCore::Registry ---------------------------------------------------
  42
  43 URLFetcherCore::Registry::Registry() {}
  44 URLFetcherCore::Registry::~Registry() {}
  45
  46 void URLFetcherCore::Registry::AddURLFetcherCore(URLFetcherCore* core) {
  47   DCHECK(!ContainsKey(fetchers_, core));
  48   fetchers_.insert(core);
  49 }
  50
  51 void URLFetcherCore::Registry::RemoveURLFetcherCore(URLFetcherCore* core) {
  52   DCHECK(ContainsKey(fetchers_, core));
  53   fetchers_.erase(core);
  54 }
  55
  56 void URLFetcherCore::Registry::CancelAll() {
  57   while (!fetchers_.empty())
  58     (*fetchers_.begin())->CancelURLRequest(ERR_ABORTED);
  59 }
  60
  61 // URLFetcherCore -------------------------------------------------------------
  62
  63 // static
  64 base::LazyInstance<URLFetcherCore::Registry>
  65     URLFetcherCore::g_registry = LAZY_INSTANCE_INITIALIZER;
  66
  67 URLFetcherCore::URLFetcherCore(URLFetcher* fetcher,
  68                                const GURL& original_url,
  69                                URLFetcher::RequestType request_type,
  70                                URLFetcherDelegate* d)
  71     : fetcher_(fetcher),
  72       original_url_(original_url),
  73       request_type_(request_type),
  74       delegate_(d),
  75       delegate_task_runner_(base::ThreadTaskRunnerHandle::Get()),
  76       load_flags_(LOAD_NORMAL),
  77       response_code_(URLFetcher::RESPONSE_CODE_INVALID),
  78       buffer_(new IOBuffer(kBufferSize)),
  79       url_request_data_key_(NULL),
  80       was_fetched_via_proxy_(false),
  81       upload_content_set_(false),
  82       upload_range_offset_(0),
  83       upload_range_length_(0),
  84       is_chunked_upload_(false),
  85       was_cancelled_(false),
  86       stop_on_redirect_(false),
  87       stopped_on_redirect_(false),
  88       automatically_retry_on_5xx_(true),
  89       num_retries_on_5xx_(0),
  90       max_retries_on_5xx_(0),
  91       num_retries_on_network_changes_(0),
  92       max_retries_on_network_changes_(0),
  93       current_upload_bytes_(-1),
  94       current_response_bytes_(0),
  95       total_response_bytes_(-1) {
  96   CHECK(original_url_.is_valid());
  97 }
  98
  99 void URLFetcherCore::Start() {
 100   DCHECK(delegate_task_runner_.get());
 101   DCHECK(request_context_getter_.get()) << "We need an URLRequestContext!";
 102   if (network_task_runner_.get()) {
 103     DCHECK_EQ(network_task_runner_,
 104               request_context_getter_->GetNetworkTaskRunner());
 105   } else {
 106     network_task_runner_ = request_context_getter_->GetNetworkTaskRunner();
 107   }
 108   DCHECK(network_task_runner_.get()) << "We need an IO task runner";
 109
 110   network_task_runner_->PostTask(
 111       FROM_HERE, base::Bind(&URLFetcherCore::StartOnIOThread, this));
 112 }
 113
 114 void URLFetcherCore::Stop() {
 115   if (delegate_task_runner_.get())  // May be NULL in tests.
 116     DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 117
 118   delegate_ = NULL;
 119   fetcher_ = NULL;
 120   if (!network_task_runner_.get())
 121     return;
 122   if (network_task_runner_->RunsTasksOnCurrentThread()) {
 123     CancelURLRequest(ERR_ABORTED);
 124   } else {
 125     network_task_runner_->PostTask(
 126         FROM_HERE,
 127         base::Bind(&URLFetcherCore::CancelURLRequest, this, ERR_ABORTED));
 128   }
 129 }
 130
 131 void URLFetcherCore::SetUploadData(const std::string& upload_content_type,
 132                                    const std::string& upload_content) {
 133   DCHECK(!is_chunked_upload_);
 134   DCHECK(!upload_content_set_);
 135   DCHECK(upload_content_.empty());
 136   DCHECK(upload_file_path_.empty());
 137   DCHECK(upload_content_type_.empty());
 138
 139   // Empty |upload_content_type| is allowed iff the |upload_content| is empty.
 140   DCHECK(upload_content.empty() || !upload_content_type.empty());
 141
 142   upload_content_type_ = upload_content_type;
 143   upload_content_ = upload_content;
 144   upload_content_set_ = true;
 145 }
 146
 147 void URLFetcherCore::SetUploadFilePath(
 148     const std::string& upload_content_type,
 149     const base::FilePath& file_path,
 150     uint64 range_offset,
 151     uint64 range_length,
 152     scoped_refptr<base::TaskRunner> file_task_runner) {
 153   DCHECK(!is_chunked_upload_);
 154   DCHECK(!upload_content_set_);
 155   DCHECK(upload_content_.empty());
 156   DCHECK(upload_file_path_.empty());
 157   DCHECK_EQ(upload_range_offset_, 0ULL);
 158   DCHECK_EQ(upload_range_length_, 0ULL);
 159   DCHECK(upload_content_type_.empty());
 160   DCHECK(!upload_content_type.empty());
 161
 162   upload_content_type_ = upload_content_type;
 163   upload_file_path_ = file_path;
 164   upload_range_offset_ = range_offset;
 165   upload_range_length_ = range_length;
 166   upload_file_task_runner_ = file_task_runner;
 167   upload_content_set_ = true;
 168 }
 169
 170 void URLFetcherCore::SetChunkedUpload(const std::string& content_type) {
 171   DCHECK(is_chunked_upload_ ||
 172          (upload_content_type_.empty() &&
 173           upload_content_.empty()));
 174
 175   // Empty |content_type| is not allowed here, because it is impossible
 176   // to ensure non-empty upload content as it is not yet supplied.
 177   DCHECK(!content_type.empty());
 178
 179   upload_content_type_ = content_type;
 180   upload_content_.clear();
 181   is_chunked_upload_ = true;
 182 }
 183
 184 void URLFetcherCore::AppendChunkToUpload(const std::string& content,
 185                                          bool is_last_chunk) {
 186   DCHECK(delegate_task_runner_.get());
 187   DCHECK(network_task_runner_.get());
 188   network_task_runner_->PostTask(
 189       FROM_HERE,
 190       base::Bind(&URLFetcherCore::CompleteAddingUploadDataChunk, this, content,
 191                  is_last_chunk));
 192 }
 193
 194 void URLFetcherCore::SetLoadFlags(int load_flags) {
 195   load_flags_ = load_flags;
 196 }
 197
 198 int URLFetcherCore::GetLoadFlags() const {
 199   return load_flags_;
 200 }
 201
 202 void URLFetcherCore::SetReferrer(const std::string& referrer) {
 203   referrer_ = referrer;
 204 }
 205
 206 void URLFetcherCore::SetExtraRequestHeaders(
 207     const std::string& extra_request_headers) {
 208   extra_request_headers_.Clear();
 209   extra_request_headers_.AddHeadersFromString(extra_request_headers);
 210 }
 211
 212 void URLFetcherCore::AddExtraRequestHeader(const std::string& header_line) {
 213   extra_request_headers_.AddHeaderFromString(header_line);
 214 }
 215
 216 void URLFetcherCore::GetExtraRequestHeaders(
 217     HttpRequestHeaders* headers) const {
 218   headers->CopyFrom(extra_request_headers_);
 219 }
 220
 221 void URLFetcherCore::SetRequestContext(
 222     URLRequestContextGetter* request_context_getter) {
 223   DCHECK(!request_context_getter_.get());
 224   DCHECK(request_context_getter);
 225   request_context_getter_ = request_context_getter;
 226 }
 227
 228 void URLFetcherCore::SetFirstPartyForCookies(
 229     const GURL& first_party_for_cookies) {
 230   DCHECK(first_party_for_cookies_.is_empty());
 231   first_party_for_cookies_ = first_party_for_cookies;
 232 }
 233
 234 void URLFetcherCore::SetURLRequestUserData(
 235     const void* key,
 236     const URLFetcher::CreateDataCallback& create_data_callback) {
 237   DCHECK(key);
 238   DCHECK(!create_data_callback.is_null());
 239   url_request_data_key_ = key;
 240   url_request_create_data_callback_ = create_data_callback;
 241 }
 242
 243 void URLFetcherCore::SetStopOnRedirect(bool stop_on_redirect) {
 244   stop_on_redirect_ = stop_on_redirect;
 245 }
 246
 247 void URLFetcherCore::SetAutomaticallyRetryOn5xx(bool retry) {
 248   automatically_retry_on_5xx_ = retry;
 249 }
 250
 251 void URLFetcherCore::SetMaxRetriesOn5xx(int max_retries) {
 252   max_retries_on_5xx_ = max_retries;
 253 }
 254
 255 int URLFetcherCore::GetMaxRetriesOn5xx() const {
 256   return max_retries_on_5xx_;
 257 }
 258
 259 base::TimeDelta URLFetcherCore::GetBackoffDelay() const {
 260   return backoff_delay_;
 261 }
 262
 263 void URLFetcherCore::SetAutomaticallyRetryOnNetworkChanges(int max_retries) {
 264   max_retries_on_network_changes_ = max_retries;
 265 }
 266
 267 void URLFetcherCore::SaveResponseToFileAtPath(
 268     const base::FilePath& file_path,
 269     scoped_refptr<base::TaskRunner> file_task_runner) {
 270   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 271   SaveResponseWithWriter(scoped_ptr<URLFetcherResponseWriter>(
 272       new URLFetcherFileWriter(file_task_runner, file_path)));
 273 }
 274
 275 void URLFetcherCore::SaveResponseToTemporaryFile(
 276     scoped_refptr<base::TaskRunner> file_task_runner) {
 277   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 278   SaveResponseWithWriter(scoped_ptr<URLFetcherResponseWriter>(
 279       new URLFetcherFileWriter(file_task_runner, base::FilePath())));
 280 }
 281
 282 void URLFetcherCore::SaveResponseWithWriter(
 283     scoped_ptr<URLFetcherResponseWriter> response_writer) {
 284   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 285   response_writer_ = response_writer.Pass();
 286 }
 287
 288 HttpResponseHeaders* URLFetcherCore::GetResponseHeaders() const {
 289   return response_headers_.get();
 290 }
 291
 292 // TODO(panayiotis): socket_address_ is written in the IO thread,
 293 // if this is accessed in the UI thread, this could result in a race.
 294 // Same for response_headers_ above and was_fetched_via_proxy_ below.
 295 HostPortPair URLFetcherCore::GetSocketAddress() const {
 296   return socket_address_;
 297 }
 298
 299 bool URLFetcherCore::WasFetchedViaProxy() const {
 300   return was_fetched_via_proxy_;
 301 }
 302
 303 const GURL& URLFetcherCore::GetOriginalURL() const {
 304   return original_url_;
 305 }
 306
 307 const GURL& URLFetcherCore::GetURL() const {
 308   return url_;
 309 }
 310
 311 const URLRequestStatus& URLFetcherCore::GetStatus() const {
 312   return status_;
 313 }
 314
 315 int URLFetcherCore::GetResponseCode() const {
 316   return response_code_;
 317 }
 318
 319 const ResponseCookies& URLFetcherCore::GetCookies() const {
 320   return cookies_;
 321 }
 322
 323 void URLFetcherCore::ReceivedContentWasMalformed() {
 324   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 325   if (network_task_runner_.get()) {
 326     network_task_runner_->PostTask(
 327         FROM_HERE, base::Bind(&URLFetcherCore::NotifyMalformedContent, this));
 328   }
 329 }
 330
 331 bool URLFetcherCore::GetResponseAsString(
 332     std::string* out_response_string) const {
 333   URLFetcherStringWriter* string_writer =
 334       response_writer_ ? response_writer_->AsStringWriter() : NULL;
 335   if (!string_writer)
 336     return false;
 337
 338   *out_response_string = string_writer->data();
 339   UMA_HISTOGRAM_MEMORY_KB("UrlFetcher.StringResponseSize",
 340                           (string_writer->data().length() / 1024));
 341   return true;
 342 }
 343
 344 bool URLFetcherCore::GetResponseAsFilePath(bool take_ownership,
 345                                            base::FilePath* out_response_path) {
 346   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 347
 348   URLFetcherFileWriter* file_writer =
 349       response_writer_ ? response_writer_->AsFileWriter() : NULL;
 350   if (!file_writer)
 351     return false;
 352
 353   *out_response_path = file_writer->file_path();
 354
 355   if (take_ownership) {
 356     // Intentionally calling a file_writer_ method directly without posting
 357     // the task to network_task_runner_.
 358     //
 359     // This is for correctly handling the case when file_writer_->DisownFile()
 360     // is soon followed by URLFetcherCore::Stop(). We have to make sure that
 361     // DisownFile takes effect before Stop deletes file_writer_.
 362     //
 363     // This direct call should be thread-safe, since DisownFile itself does no
 364     // file operation. It just flips the state to be referred in destruction.
 365     file_writer->DisownFile();
 366   }
 367   return true;
 368 }
 369
 370 void URLFetcherCore::OnReceivedRedirect(URLRequest* request,
 371                                         const GURL& new_url,
 372                                         bool* defer_redirect) {
 373   DCHECK_EQ(request, request_.get());
 374   DCHECK(network_task_runner_->BelongsToCurrentThread());
 375   if (stop_on_redirect_) {
 376     stopped_on_redirect_ = true;
 377     url_ = new_url;
 378     response_code_ = request_->GetResponseCode();
 379     was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
 380     request->Cancel();
 381     OnReadCompleted(request, 0);
 382   }
 383 }
 384
 385 void URLFetcherCore::OnResponseStarted(URLRequest* request) {
 386   DCHECK_EQ(request, request_.get());
 387   DCHECK(network_task_runner_->BelongsToCurrentThread());
 388   if (request_->status().is_success()) {
 389     response_code_ = request_->GetResponseCode();
 390     response_headers_ = request_->response_headers();
 391     socket_address_ = request_->GetSocketAddress();
 392     was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
 393     total_response_bytes_ = request_->GetExpectedContentSize();
 394   }
 395
 396   ReadResponse();
 397 }
 398
 399 void URLFetcherCore::OnCertificateRequested(
 400     URLRequest* request,
 401     SSLCertRequestInfo* cert_request_info) {
 402   DCHECK_EQ(request, request_.get());
 403   DCHECK(network_task_runner_->BelongsToCurrentThread());
 404
 405   if (g_ignore_certificate_requests) {
 406     request->ContinueWithCertificate(NULL);
 407   } else {
 408     request->Cancel();
 409   }
 410 }
 411
 412 void URLFetcherCore::OnReadCompleted(URLRequest* request,
 413                                      int bytes_read) {
 414   DCHECK(request == request_);
 415   DCHECK(network_task_runner_->BelongsToCurrentThread());
 416
 417   if (!stopped_on_redirect_)
 418     url_ = request->url();
 419   URLRequestThrottlerManager* throttler_manager =
 420       request->context()->throttler_manager();
 421   if (throttler_manager) {
 422     url_throttler_entry_ = throttler_manager->RegisterRequestUrl(url_);
 423   }
 424
 425   do {
 426     if (!request_->status().is_success() || bytes_read <= 0)
 427       break;
 428
 429     current_response_bytes_ += bytes_read;
 430     InformDelegateDownloadProgress();
 431
 432     const int result =
 433         WriteBuffer(new DrainableIOBuffer(buffer_.get(), bytes_read));
 434     if (result < 0) {
 435       // Write failed or waiting for write completion.
 436       return;
 437     }
 438   } while (request_->Read(buffer_.get(), kBufferSize, &bytes_read));
 439
 440   const URLRequestStatus status = request_->status();
 441
 442   if (status.is_success())
 443     request_->GetResponseCookies(&cookies_);
 444
 445   // See comments re: HEAD requests in ReadResponse().
 446   if (!status.is_io_pending() || request_type_ == URLFetcher::HEAD) {
 447     status_ = status;
 448     ReleaseRequest();
 449
 450     // No more data to write.
 451     const int result = response_writer_->Finish(
 452         base::Bind(&URLFetcherCore::DidFinishWriting, this));
 453     if (result != ERR_IO_PENDING)
 454       DidFinishWriting(result);
 455   }
 456 }
 457
 458 void URLFetcherCore::CancelAll() {
 459   g_registry.Get().CancelAll();
 460 }
 461
 462 int URLFetcherCore::GetNumFetcherCores() {
 463   return g_registry.Get().size();
 464 }
 465
 466 void URLFetcherCore::SetEnableInterceptionForTests(bool enabled) {
 467   g_interception_enabled = enabled;
 468 }
 469
 470 void URLFetcherCore::SetIgnoreCertificateRequests(bool ignored) {
 471   g_ignore_certificate_requests = ignored;
 472 }
 473
 474 URLFetcherCore::~URLFetcherCore() {
 475   // |request_| should be NULL.  If not, it's unsafe to delete it here since we
 476   // may not be on the IO thread.
 477   DCHECK(!request_.get());
 478 }
 479
 480 void URLFetcherCore::StartOnIOThread() {
 481   DCHECK(network_task_runner_->BelongsToCurrentThread());
 482
 483   if (!response_writer_)
 484     response_writer_.reset(new URLFetcherStringWriter);
 485
 486   const int result = response_writer_->Initialize(
 487       base::Bind(&URLFetcherCore::DidInitializeWriter, this));
 488   if (result != ERR_IO_PENDING)
 489     DidInitializeWriter(result);
 490 }
 491
 492 void URLFetcherCore::StartURLRequest() {
 493   DCHECK(network_task_runner_->BelongsToCurrentThread());
 494
 495   if (was_cancelled_) {
 496     // Since StartURLRequest() is posted as a *delayed* task, it may
 497     // run after the URLFetcher was already stopped.
 498     return;
 499   }
 500
 501   DCHECK(request_context_getter_.get());
 502   DCHECK(!request_.get());
 503
 504   g_registry.Get().AddURLFetcherCore(this);
 505   current_response_bytes_ = 0;
 506   request_ = request_context_getter_->GetURLRequestContext()->CreateRequest(
 507       original_url_, DEFAULT_PRIORITY, this);
 508   request_->set_stack_trace(stack_trace_);
 509   int flags = request_->load_flags() | load_flags_;
 510   if (!g_interception_enabled)
 511     flags = flags | LOAD_DISABLE_INTERCEPT;
 512
 513   if (is_chunked_upload_)
 514     request_->EnableChunkedUpload();
 515   request_->set_load_flags(flags);
 516   request_->SetReferrer(referrer_);
 517   request_->set_first_party_for_cookies(first_party_for_cookies_.is_empty() ?
 518       original_url_ : first_party_for_cookies_);
 519   if (url_request_data_key_ && !url_request_create_data_callback_.is_null()) {
 520     request_->SetUserData(url_request_data_key_,
 521                           url_request_create_data_callback_.Run());
 522   }
 523
 524   switch (request_type_) {
 525     case URLFetcher::GET:
 526       break;
 527
 528     case URLFetcher::POST:
 529     case URLFetcher::PUT:
 530     case URLFetcher::PATCH:
 531       // Upload content must be set.
 532       DCHECK(is_chunked_upload_ || upload_content_set_);
 533
 534       request_->set_method(
 535           request_type_ == URLFetcher::POST ? "POST" :
 536           request_type_ == URLFetcher::PUT ? "PUT" : "PATCH");
 537       if (!upload_content_type_.empty()) {
 538         extra_request_headers_.SetHeader(HttpRequestHeaders::kContentType,
 539                                          upload_content_type_);
 540       }
 541       if (!upload_content_.empty()) {
 542         scoped_ptr<UploadElementReader> reader(new UploadBytesElementReader(
 543             upload_content_.data(), upload_content_.size()));
 544         request_->set_upload(make_scoped_ptr(
 545             UploadDataStream::CreateWithReader(reader.Pass(), 0)));
 546       } else if (!upload_file_path_.empty()) {
 547         scoped_ptr<UploadElementReader> reader(
 548             new UploadFileElementReader(upload_file_task_runner_.get(),
 549                                         upload_file_path_,
 550                                         upload_range_offset_,
 551                                         upload_range_length_,
 552                                         base::Time()));
 553         request_->set_upload(make_scoped_ptr(
 554             UploadDataStream::CreateWithReader(reader.Pass(), 0)));
 555       }
 556
 557       current_upload_bytes_ = -1;
 558       // TODO(kinaba): http://crbug.com/118103. Implement upload callback in the
 559       //  layer and avoid using timer here.
 560       upload_progress_checker_timer_.reset(
 561           new base::RepeatingTimer<URLFetcherCore>());
 562       upload_progress_checker_timer_->Start(
 563           FROM_HERE,
 564           base::TimeDelta::FromMilliseconds(kUploadProgressTimerInterval),
 565           this,
 566           &URLFetcherCore::InformDelegateUploadProgress);
 567       break;
 568
 569     case URLFetcher::HEAD:
 570       request_->set_method("HEAD");
 571       break;
 572
 573     case URLFetcher::DELETE_REQUEST:
 574       request_->set_method("DELETE");
 575       break;
 576
 577     default:
 578       NOTREACHED();
 579   }
 580
 581   if (!extra_request_headers_.IsEmpty())
 582     request_->SetExtraRequestHeaders(extra_request_headers_);
 583
 584   request_->Start();
 585 }
 586
 587 void URLFetcherCore::DidInitializeWriter(int result) {
 588   if (result != OK) {
 589     CancelURLRequest(result);
 590     delegate_task_runner_->PostTask(
 591         FROM_HERE,
 592         base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
 593     return;
 594   }
 595   StartURLRequestWhenAppropriate();
 596 }
 597
 598 void URLFetcherCore::StartURLRequestWhenAppropriate() {
 599   DCHECK(network_task_runner_->BelongsToCurrentThread());
 600
 601   if (was_cancelled_)
 602     return;
 603
 604   DCHECK(request_context_getter_.get());
 605
 606   int64 delay = 0LL;
 607   if (original_url_throttler_entry_.get() == NULL) {
 608     URLRequestThrottlerManager* manager =
 609         request_context_getter_->GetURLRequestContext()->throttler_manager();
 610     if (manager) {
 611       original_url_throttler_entry_ =
 612           manager->RegisterRequestUrl(original_url_);
 613     }
 614   }
 615   if (original_url_throttler_entry_.get() != NULL) {
 616     delay = original_url_throttler_entry_->ReserveSendingTimeForNextRequest(
 617         GetBackoffReleaseTime());
 618   }
 619
 620   if (delay == 0) {
 621     StartURLRequest();
 622   } else {
 623     base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
 624         FROM_HERE, base::Bind(&URLFetcherCore::StartURLRequest, this),
 625         base::TimeDelta::FromMilliseconds(delay));
 626   }
 627 }
 628
 629 void URLFetcherCore::CancelURLRequest(int error) {
 630   DCHECK(network_task_runner_->BelongsToCurrentThread());
 631
 632   if (request_.get()) {
 633     request_->CancelWithError(error);
 634     ReleaseRequest();
 635   }
 636
 637   // Set the error manually.
 638   // Normally, calling URLRequest::CancelWithError() results in calling
 639   // OnReadCompleted() with bytes_read = -1 via an asynchronous task posted by
 640   // URLRequestJob::NotifyDone(). But, because the request was released
 641   // immediately after being canceled, the request could not call
 642   // OnReadCompleted() which overwrites |status_| with the error status.
 643   status_.set_status(URLRequestStatus::CANCELED);
 644   status_.set_error(error);
 645
 646   // Release the reference to the request context. There could be multiple
 647   // references to URLFetcher::Core at this point so it may take a while to
 648   // delete the object, but we cannot delay the destruction of the request
 649   // context.
 650   request_context_getter_ = NULL;
 651   first_party_for_cookies_ = GURL();
 652   url_request_data_key_ = NULL;
 653   url_request_create_data_callback_.Reset();
 654   was_cancelled_ = true;
 655 }
 656
 657 void URLFetcherCore::OnCompletedURLRequest(
 658     base::TimeDelta backoff_delay) {
 659   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 660
 661   // Save the status and backoff_delay so that delegates can read it.
 662   if (delegate_) {
 663     backoff_delay_ = backoff_delay;
 664     InformDelegateFetchIsComplete();
 665   }
 666 }
 667
 668 void URLFetcherCore::InformDelegateFetchIsComplete() {
 669   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 670   if (delegate_)
 671     delegate_->OnURLFetchComplete(fetcher_);
 672 }
 673
 674 void URLFetcherCore::NotifyMalformedContent() {
 675   DCHECK(network_task_runner_->BelongsToCurrentThread());
 676   if (url_throttler_entry_.get() != NULL) {
 677     int status_code = response_code_;
 678     if (status_code == URLFetcher::RESPONSE_CODE_INVALID) {
 679       // The status code will generally be known by the time clients
 680       // call the |ReceivedContentWasMalformed()| function (which ends up
 681       // calling the current function) but if it's not, we need to assume
 682       // the response was successful so that the total failure count
 683       // used to calculate exponential back-off goes up.
 684       status_code = 200;
 685     }
 686     url_throttler_entry_->ReceivedContentWasMalformed(status_code);
 687   }
 688 }
 689
 690 void URLFetcherCore::DidFinishWriting(int result) {
 691   if (result != OK) {
 692     CancelURLRequest(result);
 693     delegate_task_runner_->PostTask(
 694         FROM_HERE,
 695         base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
 696     return;
 697   }
 698   // If the file was successfully closed, then the URL request is complete.
 699   RetryOrCompleteUrlFetch();
 700 }
 701
 702 void URLFetcherCore::RetryOrCompleteUrlFetch() {
 703   DCHECK(network_task_runner_->BelongsToCurrentThread());
 704   base::TimeDelta backoff_delay;
 705
 706   // Checks the response from server.
 707   if (response_code_ >= 500 ||
 708       status_.error() == ERR_TEMPORARILY_THROTTLED) {
 709     // When encountering a server error, we will send the request again
 710     // after backoff time.
 711     ++num_retries_on_5xx_;
 712
 713     // Note that backoff_delay may be 0 because (a) the
 714     // URLRequestThrottlerManager and related code does not
 715     // necessarily back off on the first error, (b) it only backs off
 716     // on some of the 5xx status codes, (c) not all URLRequestContexts
 717     // have a throttler manager.
 718     base::TimeTicks backoff_release_time = GetBackoffReleaseTime();
 719     backoff_delay = backoff_release_time - base::TimeTicks::Now();
 720     if (backoff_delay < base::TimeDelta())
 721       backoff_delay = base::TimeDelta();
 722
 723     if (automatically_retry_on_5xx_ &&
 724         num_retries_on_5xx_ <= max_retries_on_5xx_) {
 725       StartOnIOThread();
 726       return;
 727     }
 728   } else {
 729     backoff_delay = base::TimeDelta();
 730   }
 731
 732   // Retry if the request failed due to network changes.
 733   if (status_.error() == ERR_NETWORK_CHANGED &&
 734       num_retries_on_network_changes_ < max_retries_on_network_changes_) {
 735     ++num_retries_on_network_changes_;
 736
 737     // Retry soon, after flushing all the current tasks which may include
 738     // further network change observers.
 739     network_task_runner_->PostTask(
 740         FROM_HERE, base::Bind(&URLFetcherCore::StartOnIOThread, this));
 741     return;
 742   }
 743
 744   request_context_getter_ = NULL;
 745   first_party_for_cookies_ = GURL();
 746   url_request_data_key_ = NULL;
 747   url_request_create_data_callback_.Reset();
 748   bool posted = delegate_task_runner_->PostTask(
 749       FROM_HERE,
 750       base::Bind(&URLFetcherCore::OnCompletedURLRequest, this, backoff_delay));
 751
 752   // If the delegate message loop does not exist any more, then the delegate
 753   // should be gone too.
 754   DCHECK(posted || !delegate_);
 755 }
 756
 757 void URLFetcherCore::ReleaseRequest() {
 758   upload_progress_checker_timer_.reset();
 759   request_.reset();
 760   g_registry.Get().RemoveURLFetcherCore(this);
 761 }
 762
 763 base::TimeTicks URLFetcherCore::GetBackoffReleaseTime() {
 764   DCHECK(network_task_runner_->BelongsToCurrentThread());
 765
 766   if (original_url_throttler_entry_.get()) {
 767     base::TimeTicks original_url_backoff =
 768         original_url_throttler_entry_->GetExponentialBackoffReleaseTime();
 769     base::TimeTicks destination_url_backoff;
 770     if (url_throttler_entry_.get() != NULL &&
 771         original_url_throttler_entry_.get() != url_throttler_entry_.get()) {
 772       destination_url_backoff =
 773           url_throttler_entry_->GetExponentialBackoffReleaseTime();
 774     }
 775
 776     return original_url_backoff > destination_url_backoff ?
 777         original_url_backoff : destination_url_backoff;
 778   } else {
 779     return base::TimeTicks();
 780   }
 781 }
 782
 783 void URLFetcherCore::CompleteAddingUploadDataChunk(
 784     const std::string& content, bool is_last_chunk) {
 785   if (was_cancelled_) {
 786     // Since CompleteAddingUploadDataChunk() is posted as a *delayed* task, it
 787     // may run after the URLFetcher was already stopped.
 788     return;
 789   }
 790   DCHECK(is_chunked_upload_);
 791   DCHECK(request_.get());
 792   DCHECK(!content.empty());
 793   request_->AppendChunkToUpload(content.data(),
 794                                 static_cast<int>(content.length()),
 795                                 is_last_chunk);
 796 }
 797
 798 int URLFetcherCore::WriteBuffer(scoped_refptr<DrainableIOBuffer> data) {
 799   while (data->BytesRemaining() > 0) {
 800     const int result = response_writer_->Write(
 801         data.get(),
 802         data->BytesRemaining(),
 803         base::Bind(&URLFetcherCore::DidWriteBuffer, this, data));
 804     if (result < 0) {
 805       if (result != ERR_IO_PENDING)
 806         DidWriteBuffer(data, result);
 807       return result;
 808     }
 809     data->DidConsume(result);
 810   }
 811   return OK;
 812 }
 813
 814 void URLFetcherCore::DidWriteBuffer(scoped_refptr<DrainableIOBuffer> data,
 815                                     int result) {
 816   if (result < 0) {  // Handle errors.
 817     CancelURLRequest(result);
 818     response_writer_->Finish(base::Bind(&EmptyCompletionCallback));
 819     delegate_task_runner_->PostTask(
 820         FROM_HERE,
 821         base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
 822     return;
 823   }
 824
 825   // Continue writing.
 826   data->DidConsume(result);
 827   if (WriteBuffer(data) < 0)
 828     return;
 829
 830   // Finished writing buffer_. Read some more.
 831   DCHECK_EQ(0, data->BytesRemaining());
 832   ReadResponse();
 833 }
 834
 835 void URLFetcherCore::ReadResponse() {
 836   // Some servers may treat HEAD requests as GET requests.  To free up the
 837   // network connection as soon as possible, signal that the request has
 838   // completed immediately, without trying to read any data back (all we care
 839   // about is the response code and headers, which we already have).
 840   int bytes_read = 0;
 841   if (request_->status().is_success() &&
 842       (request_type_ != URLFetcher::HEAD))
 843     request_->Read(buffer_.get(), kBufferSize, &bytes_read);
 844   OnReadCompleted(request_.get(), bytes_read);
 845 }
 846
 847 void URLFetcherCore::InformDelegateUploadProgress() {
 848   DCHECK(network_task_runner_->BelongsToCurrentThread());
 849   if (request_.get()) {
 850     int64 current = request_->GetUploadProgress().position();
 851     if (current_upload_bytes_ != current) {
 852       current_upload_bytes_ = current;
 853       int64 total = -1;
 854       if (!is_chunked_upload_) {
 855         total = static_cast<int64>(request_->GetUploadProgress().size());
 856         // Total may be zero if the UploadDataStream::Init has not been called
 857         // yet.  Don't send the upload progress until the size is initialized.
 858         if (!total)
 859           return;
 860       }
 861       delegate_task_runner_->PostTask(
 862           FROM_HERE,
 863           base::Bind(
 864               &URLFetcherCore::InformDelegateUploadProgressInDelegateThread,
 865               this, current, total));
 866     }
 867   }
 868 }
 869
 870 void URLFetcherCore::InformDelegateUploadProgressInDelegateThread(
 871     int64 current, int64 total) {
 872   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 873   if (delegate_)
 874     delegate_->OnURLFetchUploadProgress(fetcher_, current, total);
 875 }
 876
 877 void URLFetcherCore::InformDelegateDownloadProgress() {
 878   DCHECK(network_task_runner_->BelongsToCurrentThread());
 879   delegate_task_runner_->PostTask(
 880       FROM_HERE,
 881       base::Bind(
 882           &URLFetcherCore::InformDelegateDownloadProgressInDelegateThread,
 883           this, current_response_bytes_, total_response_bytes_));
 884 }
 885
 886 void URLFetcherCore::InformDelegateDownloadProgressInDelegateThread(
 887     int64 current, int64 total) {
 888   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
 889   if (delegate_)
 890     delegate_->OnURLFetchDownloadProgress(fetcher_, current, total);
 891 }
 892
 893 }  // namespace net