src/net/url_request/url_request_http_job.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/url_request/url_request_http_job.h"
   6
   7 #include "base/base_switches.h"
   8 #include "base/bind.h"
   9 #include "base/bind_helpers.h"
  10 #include "base/command_line.h"
  11 #include "base/compiler_specific.h"
  12 #include "base/file_version_info.h"
  13 #include "base/message_loop/message_loop.h"
  14 #include "base/metrics/field_trial.h"
  15 #include "base/metrics/histogram.h"
  16 #include "base/profiler/scoped_tracker.h"
  17 #include "base/rand_util.h"
  18 #include "base/strings/string_util.h"
  19 #include "base/time/time.h"
  20 #include "net/base/host_port_pair.h"
  21 #include "net/base/load_flags.h"
  22 #include "net/base/mime_util.h"
  23 #include "net/base/net_errors.h"
  24 #include "net/base/net_util.h"
  25 #include "net/base/network_delegate.h"
  26 #include "net/base/sdch_manager.h"
  27 #include "net/cert/cert_status_flags.h"
  28 #include "net/cookies/cookie_store.h"
  29 #include "net/http/http_content_disposition.h"
  30 #include "net/http/http_network_session.h"
  31 #include "net/http/http_request_headers.h"
  32 #include "net/http/http_response_headers.h"
  33 #include "net/http/http_response_info.h"
  34 #include "net/http/http_status_code.h"
  35 #include "net/http/http_transaction.h"
  36 #include "net/http/http_transaction_factory.h"
  37 #include "net/http/http_util.h"
  38 #include "net/proxy/proxy_info.h"
  39 #include "net/ssl/ssl_cert_request_info.h"
  40 #include "net/ssl/ssl_config_service.h"
  41 #include "net/url_request/fraudulent_certificate_reporter.h"
  42 #include "net/url_request/http_user_agent_settings.h"
  43 #include "net/url_request/url_request.h"
  44 #include "net/url_request/url_request_context.h"
  45 #include "net/url_request/url_request_error_job.h"
  46 #include "net/url_request/url_request_job_factory.h"
  47 #include "net/url_request/url_request_redirect_job.h"
  48 #include "net/url_request/url_request_throttler_header_adapter.h"
  49 #include "net/url_request/url_request_throttler_manager.h"
  50 #include "net/websockets/websocket_handshake_stream_base.h"
  51
  52 static const char kAvailDictionaryHeader[] = "Avail-Dictionary";
  53
  54 namespace net {
  55
  56 class URLRequestHttpJob::HttpFilterContext : public FilterContext {
  57  public:
  58   explicit HttpFilterContext(URLRequestHttpJob* job);
  59   ~HttpFilterContext() override;
  60
  61   // FilterContext implementation.
  62   bool GetMimeType(std::string* mime_type) const override;
  63   bool GetURL(GURL* gurl) const override;
  64   bool GetContentDisposition(std::string* disposition) const override;
  65   base::Time GetRequestTime() const override;
  66   bool IsCachedContent() const override;
  67   bool IsDownload() const override;
  68   bool SdchResponseExpected() const override;
  69   int64 GetByteReadCount() const override;
  70   int GetResponseCode() const override;
  71   const URLRequestContext* GetURLRequestContext() const override;
  72   void RecordPacketStats(StatisticSelector statistic) const override;
  73
  74   // Method to allow us to reset filter context for a response that should have
  75   // been SDCH encoded when there is an update due to an explicit HTTP header.
  76   void ResetSdchResponseToFalse();
  77
  78  private:
  79   URLRequestHttpJob* job_;
  80
  81   DISALLOW_COPY_AND_ASSIGN(HttpFilterContext);
  82 };
  83
  84 URLRequestHttpJob::HttpFilterContext::HttpFilterContext(URLRequestHttpJob* job)
  85     : job_(job) {
  86   DCHECK(job_);
  87 }
  88
  89 URLRequestHttpJob::HttpFilterContext::~HttpFilterContext() {
  90 }
  91
  92 bool URLRequestHttpJob::HttpFilterContext::GetMimeType(
  93     std::string* mime_type) const {
  94   return job_->GetMimeType(mime_type);
  95 }
  96
  97 bool URLRequestHttpJob::HttpFilterContext::GetURL(GURL* gurl) const {
  98   if (!job_->request())
  99     return false;
 100   *gurl = job_->request()->url();
 101   return true;
 102 }
 103
 104 bool URLRequestHttpJob::HttpFilterContext::GetContentDisposition(
 105     std::string* disposition) const {
 106   HttpResponseHeaders* headers = job_->GetResponseHeaders();
 107   void *iter = NULL;
 108   return headers->EnumerateHeader(&iter, "Content-Disposition", disposition);
 109 }
 110
 111 base::Time URLRequestHttpJob::HttpFilterContext::GetRequestTime() const {
 112   return job_->request() ? job_->request()->request_time() : base::Time();
 113 }
 114
 115 bool URLRequestHttpJob::HttpFilterContext::IsCachedContent() const {
 116   return job_->is_cached_content_;
 117 }
 118
 119 bool URLRequestHttpJob::HttpFilterContext::IsDownload() const {
 120   return (job_->request_info_.load_flags & LOAD_IS_DOWNLOAD) != 0;
 121 }
 122
 123 void URLRequestHttpJob::HttpFilterContext::ResetSdchResponseToFalse() {
 124   DCHECK(job_->sdch_dictionary_advertised_);
 125   job_->sdch_dictionary_advertised_ = false;
 126 }
 127
 128 bool URLRequestHttpJob::HttpFilterContext::SdchResponseExpected() const {
 129   return job_->sdch_dictionary_advertised_;
 130 }
 131
 132 int64 URLRequestHttpJob::HttpFilterContext::GetByteReadCount() const {
 133   return job_->filter_input_byte_count();
 134 }
 135
 136 int URLRequestHttpJob::HttpFilterContext::GetResponseCode() const {
 137   return job_->GetResponseCode();
 138 }
 139
 140 const URLRequestContext*
 141 URLRequestHttpJob::HttpFilterContext::GetURLRequestContext() const {
 142   return job_->request() ? job_->request()->context() : NULL;
 143 }
 144
 145 void URLRequestHttpJob::HttpFilterContext::RecordPacketStats(
 146     StatisticSelector statistic) const {
 147   job_->RecordPacketStats(statistic);
 148 }
 149
 150 // TODO(darin): make sure the port blocking code is not lost
 151 // static
 152 URLRequestJob* URLRequestHttpJob::Factory(URLRequest* request,
 153                                           NetworkDelegate* network_delegate,
 154                                           const std::string& scheme) {
 155   DCHECK(scheme == "http" || scheme == "https" || scheme == "ws" ||
 156          scheme == "wss");
 157
 158   if (!request->context()->http_transaction_factory()) {
 159     NOTREACHED() << "requires a valid context";
 160     return new URLRequestErrorJob(
 161         request, network_delegate, ERR_INVALID_ARGUMENT);
 162   }
 163
 164   GURL redirect_url;
 165   if (request->GetHSTSRedirect(&redirect_url)) {
 166     return new URLRequestRedirectJob(
 167         request, network_delegate, redirect_url,
 168         // Use status code 307 to preserve the method, so POST requests work.
 169         URLRequestRedirectJob::REDIRECT_307_TEMPORARY_REDIRECT, "HSTS");
 170   }
 171   return new URLRequestHttpJob(request,
 172                                network_delegate,
 173                                request->context()->http_user_agent_settings());
 174 }
 175
 176 URLRequestHttpJob::URLRequestHttpJob(
 177     URLRequest* request,
 178     NetworkDelegate* network_delegate,
 179     const HttpUserAgentSettings* http_user_agent_settings)
 180     : URLRequestJob(request, network_delegate),
 181       priority_(DEFAULT_PRIORITY),
 182       response_info_(NULL),
 183       response_cookies_save_index_(0),
 184       proxy_auth_state_(AUTH_STATE_DONT_NEED_AUTH),
 185       server_auth_state_(AUTH_STATE_DONT_NEED_AUTH),
 186       start_callback_(base::Bind(&URLRequestHttpJob::OnStartCompleted,
 187                                  base::Unretained(this))),
 188       notify_before_headers_sent_callback_(
 189           base::Bind(&URLRequestHttpJob::NotifyBeforeSendHeadersCallback,
 190                      base::Unretained(this))),
 191       read_in_progress_(false),
 192       throttling_entry_(NULL),
 193       sdch_dictionary_advertised_(false),
 194       sdch_test_activated_(false),
 195       sdch_test_control_(false),
 196       is_cached_content_(false),
 197       request_creation_time_(),
 198       packet_timing_enabled_(false),
 199       done_(false),
 200       bytes_observed_in_packets_(0),
 201       request_time_snapshot_(),
 202       final_packet_time_(),
 203       filter_context_(new HttpFilterContext(this)),
 204       on_headers_received_callback_(
 205           base::Bind(&URLRequestHttpJob::OnHeadersReceivedCallback,
 206                      base::Unretained(this))),
 207       awaiting_callback_(false),
 208       http_user_agent_settings_(http_user_agent_settings),
 209       weak_factory_(this) {
 210   URLRequestThrottlerManager* manager = request->context()->throttler_manager();
 211   if (manager)
 212     throttling_entry_ = manager->RegisterRequestUrl(request->url());
 213
 214   ResetTimer();
 215 }
 216
 217 URLRequestHttpJob::~URLRequestHttpJob() {
 218   CHECK(!awaiting_callback_);
 219
 220   DCHECK(!sdch_test_control_ || !sdch_test_activated_);
 221   if (!is_cached_content_) {
 222     if (sdch_test_control_)
 223       RecordPacketStats(FilterContext::SDCH_EXPERIMENT_HOLDBACK);
 224     if (sdch_test_activated_)
 225       RecordPacketStats(FilterContext::SDCH_EXPERIMENT_DECODE);
 226   }
 227   // Make sure SDCH filters are told to emit histogram data while
 228   // filter_context_ is still alive.
 229   DestroyFilters();
 230
 231   DoneWithRequest(ABORTED);
 232 }
 233
 234 void URLRequestHttpJob::SetPriority(RequestPriority priority) {
 235   priority_ = priority;
 236   if (transaction_)
 237     transaction_->SetPriority(priority_);
 238 }
 239
 240 void URLRequestHttpJob::Start() {
 241   DCHECK(!transaction_.get());
 242
 243   // URLRequest::SetReferrer ensures that we do not send username and password
 244   // fields in the referrer.
 245   GURL referrer(request_->referrer());
 246
 247   request_info_.url = request_->url();
 248   request_info_.method = request_->method();
 249   request_info_.load_flags = request_->load_flags();
 250   // Enable privacy mode if cookie settings or flags tell us not send or
 251   // save cookies.
 252   bool enable_privacy_mode =
 253       (request_info_.load_flags & LOAD_DO_NOT_SEND_COOKIES) ||
 254       (request_info_.load_flags & LOAD_DO_NOT_SAVE_COOKIES) ||
 255       CanEnablePrivacyMode();
 256   // Privacy mode could still be disabled in OnCookiesLoaded if we are going
 257   // to send previously saved cookies.
 258   request_info_.privacy_mode = enable_privacy_mode ?
 259       PRIVACY_MODE_ENABLED : PRIVACY_MODE_DISABLED;
 260
 261   // Strip Referer from request_info_.extra_headers to prevent, e.g., plugins
 262   // from overriding headers that are controlled using other means. Otherwise a
 263   // plugin could set a referrer although sending the referrer is inhibited.
 264   request_info_.extra_headers.RemoveHeader(HttpRequestHeaders::kReferer);
 265
 266   // Our consumer should have made sure that this is a safe referrer.  See for
 267   // instance WebCore::FrameLoader::HideReferrer.
 268   if (referrer.is_valid()) {
 269     request_info_.extra_headers.SetHeader(HttpRequestHeaders::kReferer,
 270                                           referrer.spec());
 271   }
 272
 273   request_info_.extra_headers.SetHeaderIfMissing(
 274       HttpRequestHeaders::kUserAgent,
 275       http_user_agent_settings_ ?
 276           http_user_agent_settings_->GetUserAgent() : std::string());
 277
 278   AddExtraHeaders();
 279   AddCookieHeaderAndStart();
 280 }
 281
 282 void URLRequestHttpJob::Kill() {
 283   if (!transaction_.get())
 284     return;
 285
 286   weak_factory_.InvalidateWeakPtrs();
 287   DestroyTransaction();
 288   URLRequestJob::Kill();
 289 }
 290
 291 void URLRequestHttpJob::NotifyBeforeSendProxyHeadersCallback(
 292     const ProxyInfo& proxy_info,
 293     HttpRequestHeaders* request_headers) {
 294   DCHECK(request_headers);
 295   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 296   if (network_delegate()) {
 297     network_delegate()->NotifyBeforeSendProxyHeaders(
 298         request_,
 299         proxy_info,
 300         request_headers);
 301   }
 302 }
 303
 304 void URLRequestHttpJob::NotifyHeadersComplete() {
 305   DCHECK(!response_info_);
 306
 307   response_info_ = transaction_->GetResponseInfo();
 308
 309   // Save boolean, as we'll need this info at destruction time, and filters may
 310   // also need this info.
 311   is_cached_content_ = response_info_->was_cached;
 312
 313   if (!is_cached_content_ && throttling_entry_.get()) {
 314     URLRequestThrottlerHeaderAdapter response_adapter(GetResponseHeaders());
 315     throttling_entry_->UpdateWithResponse(request_info_.url.host(),
 316                                           &response_adapter);
 317   }
 318
 319   // The ordering of these calls is not important.
 320   ProcessStrictTransportSecurityHeader();
 321   ProcessPublicKeyPinsHeader();
 322
 323   SdchManager* sdch_manager(request()->context()->sdch_manager());
 324   if (sdch_manager && sdch_manager->IsInSupportedDomain(request_->url())) {
 325     const std::string name = "Get-Dictionary";
 326     std::string url_text;
 327     void* iter = NULL;
 328     // TODO(jar): We need to not fetch dictionaries the first time they are
 329     // seen, but rather wait until we can justify their usefulness.
 330     // For now, we will only fetch the first dictionary, which will at least
 331     // require multiple suggestions before we get additional ones for this site.
 332     // Eventually we should wait until a dictionary is requested several times
 333     // before we even download it (so that we don't waste memory or bandwidth).
 334     if (GetResponseHeaders()->EnumerateHeader(&iter, name, &url_text)) {
 335       // Resolve suggested URL relative to request url.
 336       GURL sdch_dictionary_url = request_->url().Resolve(url_text);
 337       if (sdch_dictionary_url.is_valid()) {
 338         sdch_manager->OnGetDictionary(request_->url(), sdch_dictionary_url);
 339       }
 340     }
 341   }
 342
 343   // The HTTP transaction may be restarted several times for the purposes
 344   // of sending authorization information. Each time it restarts, we get
 345   // notified of the headers completion so that we can update the cookie store.
 346   if (transaction_->IsReadyToRestartForAuth()) {
 347     DCHECK(!response_info_->auth_challenge.get());
 348     // TODO(battre): This breaks the webrequest API for
 349     // URLRequestTestHTTP.BasicAuthWithCookies
 350     // where OnBeforeSendHeaders -> OnSendHeaders -> OnBeforeSendHeaders
 351     // occurs.
 352     RestartTransactionWithAuth(AuthCredentials());
 353     return;
 354   }
 355
 356   URLRequestJob::NotifyHeadersComplete();
 357 }
 358
 359 void URLRequestHttpJob::NotifyDone(const URLRequestStatus& status) {
 360   DoneWithRequest(FINISHED);
 361   URLRequestJob::NotifyDone(status);
 362 }
 363
 364 void URLRequestHttpJob::DestroyTransaction() {
 365   DCHECK(transaction_.get());
 366
 367   DoneWithRequest(ABORTED);
 368   transaction_.reset();
 369   response_info_ = NULL;
 370   receive_headers_end_ = base::TimeTicks();
 371 }
 372
 373 void URLRequestHttpJob::StartTransaction() {
 374   if (network_delegate()) {
 375     OnCallToDelegate();
 376     int rv = network_delegate()->NotifyBeforeSendHeaders(
 377         request_, notify_before_headers_sent_callback_,
 378         &request_info_.extra_headers);
 379     // If an extension blocks the request, we rely on the callback to
 380     // MaybeStartTransactionInternal().
 381     if (rv == ERR_IO_PENDING)
 382       return;
 383     MaybeStartTransactionInternal(rv);
 384     return;
 385   }
 386   StartTransactionInternal();
 387 }
 388
 389 void URLRequestHttpJob::NotifyBeforeSendHeadersCallback(int result) {
 390   // Check that there are no callbacks to already canceled requests.
 391   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 392
 393   MaybeStartTransactionInternal(result);
 394 }
 395
 396 void URLRequestHttpJob::MaybeStartTransactionInternal(int result) {
 397   OnCallToDelegateComplete();
 398   if (result == OK) {
 399     StartTransactionInternal();
 400   } else {
 401     std::string source("delegate");
 402     request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 403                                  NetLog::StringCallback("source", &source));
 404     NotifyCanceled();
 405     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 406   }
 407 }
 408
 409 void URLRequestHttpJob::StartTransactionInternal() {
 410   // NOTE: This method assumes that request_info_ is already setup properly.
 411
 412   // If we already have a transaction, then we should restart the transaction
 413   // with auth provided by auth_credentials_.
 414
 415   int rv;
 416
 417   if (network_delegate()) {
 418     network_delegate()->NotifySendHeaders(
 419         request_, request_info_.extra_headers);
 420   }
 421
 422   if (transaction_.get()) {
 423     rv = transaction_->RestartWithAuth(auth_credentials_, start_callback_);
 424     auth_credentials_ = AuthCredentials();
 425   } else {
 426     DCHECK(request_->context()->http_transaction_factory());
 427
 428     rv = request_->context()->http_transaction_factory()->CreateTransaction(
 429         priority_, &transaction_);
 430
 431     if (rv == OK && request_info_.url.SchemeIsWSOrWSS()) {
 432       base::SupportsUserData::Data* data = request_->GetUserData(
 433           WebSocketHandshakeStreamBase::CreateHelper::DataKey());
 434       if (data) {
 435         transaction_->SetWebSocketHandshakeStreamCreateHelper(
 436             static_cast<WebSocketHandshakeStreamBase::CreateHelper*>(data));
 437       } else {
 438         rv = ERR_DISALLOWED_URL_SCHEME;
 439       }
 440     }
 441
 442     if (rv == OK) {
 443       transaction_->SetBeforeNetworkStartCallback(
 444           base::Bind(&URLRequestHttpJob::NotifyBeforeNetworkStart,
 445                      base::Unretained(this)));
 446       transaction_->SetBeforeProxyHeadersSentCallback(
 447           base::Bind(&URLRequestHttpJob::NotifyBeforeSendProxyHeadersCallback,
 448                      base::Unretained(this)));
 449
 450       if (!throttling_entry_.get() ||
 451           !throttling_entry_->ShouldRejectRequest(*request_,
 452                                                   network_delegate())) {
 453         rv = transaction_->Start(
 454             &request_info_, start_callback_, request_->net_log());
 455         start_time_ = base::TimeTicks::Now();
 456       } else {
 457         // Special error code for the exponential back-off module.
 458         rv = ERR_TEMPORARILY_THROTTLED;
 459       }
 460     }
 461   }
 462
 463   if (rv == ERR_IO_PENDING)
 464     return;
 465
 466   // The transaction started synchronously, but we need to notify the
 467   // URLRequest delegate via the message loop.
 468   base::MessageLoop::current()->PostTask(
 469       FROM_HERE,
 470       base::Bind(&URLRequestHttpJob::OnStartCompleted,
 471                  weak_factory_.GetWeakPtr(), rv));
 472 }
 473
 474 void URLRequestHttpJob::AddExtraHeaders() {
 475   SdchManager* sdch_manager = request()->context()->sdch_manager();
 476
 477   // Supply Accept-Encoding field only if it is not already provided.
 478   // It should be provided IF the content is known to have restrictions on
 479   // potential encoding, such as streaming multi-media.
 480   // For details see bug 47381.
 481   // TODO(jar, enal): jpeg files etc. should set up a request header if
 482   // possible. Right now it is done only by buffered_resource_loader and
 483   // simple_data_source.
 484   if (!request_info_.extra_headers.HasHeader(
 485       HttpRequestHeaders::kAcceptEncoding)) {
 486     bool advertise_sdch = sdch_manager &&
 487         // We don't support SDCH responses to POST as there is a possibility
 488         // of having SDCH encoded responses returned (e.g. by the cache)
 489         // which we cannot decode, and in those situations, we will need
 490         // to retransmit the request without SDCH, which is illegal for a POST.
 491         request()->method() != "POST" &&
 492         sdch_manager->IsInSupportedDomain(request_->url());
 493     std::string avail_dictionaries;
 494     if (advertise_sdch) {
 495       sdch_manager->GetAvailDictionaryList(request_->url(),
 496                                            &avail_dictionaries);
 497
 498       // The AllowLatencyExperiment() is only true if we've successfully done a
 499       // full SDCH compression recently in this browser session for this host.
 500       // Note that for this path, there might be no applicable dictionaries,
 501       // and hence we can't participate in the experiment.
 502       if (!avail_dictionaries.empty() &&
 503           sdch_manager->AllowLatencyExperiment(request_->url())) {
 504         // We are participating in the test (or control), and hence we'll
 505         // eventually record statistics via either SDCH_EXPERIMENT_DECODE or
 506         // SDCH_EXPERIMENT_HOLDBACK, and we'll need some packet timing data.
 507         packet_timing_enabled_ = true;
 508         if (base::RandDouble() < .01) {
 509           sdch_test_control_ = true;  // 1% probability.
 510           advertise_sdch = false;
 511         } else {
 512           sdch_test_activated_ = true;
 513         }
 514       }
 515     }
 516
 517     // Supply Accept-Encoding headers first so that it is more likely that they
 518     // will be in the first transmitted packet.  This can sometimes make it
 519     // easier to filter and analyze the streams to assure that a proxy has not
 520     // damaged these headers.  Some proxies deliberately corrupt Accept-Encoding
 521     // headers.
 522     if (!advertise_sdch) {
 523       // Tell the server what compression formats we support (other than SDCH).
 524       request_info_.extra_headers.SetHeader(
 525           HttpRequestHeaders::kAcceptEncoding, "gzip, deflate");
 526     } else {
 527       // Include SDCH in acceptable list.
 528       request_info_.extra_headers.SetHeader(
 529           HttpRequestHeaders::kAcceptEncoding, "gzip, deflate, sdch");
 530       if (!avail_dictionaries.empty()) {
 531         request_info_.extra_headers.SetHeader(
 532             kAvailDictionaryHeader,
 533             avail_dictionaries);
 534         sdch_dictionary_advertised_ = true;
 535         // Since we're tagging this transaction as advertising a dictionary,
 536         // we'll definitely employ an SDCH filter (or tentative sdch filter)
 537         // when we get a response.  When done, we'll record histograms via
 538         // SDCH_DECODE or SDCH_PASSTHROUGH.  Hence we need to record packet
 539         // arrival times.
 540         packet_timing_enabled_ = true;
 541       }
 542     }
 543   }
 544
 545   if (http_user_agent_settings_) {
 546     // Only add default Accept-Language if the request didn't have it
 547     // specified.
 548     std::string accept_language =
 549         http_user_agent_settings_->GetAcceptLanguage();
 550     if (!accept_language.empty()) {
 551       request_info_.extra_headers.SetHeaderIfMissing(
 552           HttpRequestHeaders::kAcceptLanguage,
 553           accept_language);
 554     }
 555   }
 556 }
 557
 558 void URLRequestHttpJob::AddCookieHeaderAndStart() {
 559   // No matter what, we want to report our status as IO pending since we will
 560   // be notifying our consumer asynchronously via OnStartCompleted.
 561   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
 562
 563   // If the request was destroyed, then there is no more work to do.
 564   if (!request_)
 565     return;
 566
 567   CookieStore* cookie_store = GetCookieStore();
 568   if (cookie_store && !(request_info_.load_flags & LOAD_DO_NOT_SEND_COOKIES)) {
 569     cookie_store->GetAllCookiesForURLAsync(
 570         request_->url(),
 571         base::Bind(&URLRequestHttpJob::CheckCookiePolicyAndLoad,
 572                    weak_factory_.GetWeakPtr()));
 573   } else {
 574     DoStartTransaction();
 575   }
 576 }
 577
 578 void URLRequestHttpJob::DoLoadCookies() {
 579   CookieOptions options;
 580   options.set_include_httponly();
 581   GetCookieStore()->GetCookiesWithOptionsAsync(
 582       request_->url(), options,
 583       base::Bind(&URLRequestHttpJob::OnCookiesLoaded,
 584                  weak_factory_.GetWeakPtr()));
 585 }
 586
 587 void URLRequestHttpJob::CheckCookiePolicyAndLoad(
 588     const CookieList& cookie_list) {
 589   if (CanGetCookies(cookie_list))
 590     DoLoadCookies();
 591   else
 592     DoStartTransaction();
 593 }
 594
 595 void URLRequestHttpJob::OnCookiesLoaded(const std::string& cookie_line) {
 596   if (!cookie_line.empty()) {
 597     request_info_.extra_headers.SetHeader(
 598         HttpRequestHeaders::kCookie, cookie_line);
 599     // Disable privacy mode as we are sending cookies anyway.
 600     request_info_.privacy_mode = PRIVACY_MODE_DISABLED;
 601   }
 602   DoStartTransaction();
 603 }
 604
 605 void URLRequestHttpJob::DoStartTransaction() {
 606   // We may have been canceled while retrieving cookies.
 607   if (GetStatus().is_success()) {
 608     StartTransaction();
 609   } else {
 610     NotifyCanceled();
 611   }
 612 }
 613
 614 void URLRequestHttpJob::SaveCookiesAndNotifyHeadersComplete(int result) {
 615   // End of the call started in OnStartCompleted.
 616   OnCallToDelegateComplete();
 617
 618   if (result != net::OK) {
 619     std::string source("delegate");
 620     request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 621                                  NetLog::StringCallback("source", &source));
 622     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 623     return;
 624   }
 625
 626   DCHECK(transaction_.get());
 627
 628   const HttpResponseInfo* response_info = transaction_->GetResponseInfo();
 629   DCHECK(response_info);
 630
 631   response_cookies_.clear();
 632   response_cookies_save_index_ = 0;
 633
 634   FetchResponseCookies(&response_cookies_);
 635
 636   if (!GetResponseHeaders()->GetDateValue(&response_date_))
 637     response_date_ = base::Time();
 638
 639   // Now, loop over the response cookies, and attempt to persist each.
 640   SaveNextCookie();
 641 }
 642
 643 // If the save occurs synchronously, SaveNextCookie will loop and save the next
 644 // cookie. If the save is deferred, the callback is responsible for continuing
 645 // to iterate through the cookies.
 646 // TODO(erikwright): Modify the CookieStore API to indicate via return value
 647 // whether it completed synchronously or asynchronously.
 648 // See http://crbug.com/131066.
 649 void URLRequestHttpJob::SaveNextCookie() {
 650   // No matter what, we want to report our status as IO pending since we will
 651   // be notifying our consumer asynchronously via OnStartCompleted.
 652   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
 653
 654   // Used to communicate with the callback. See the implementation of
 655   // OnCookieSaved.
 656   scoped_refptr<SharedBoolean> callback_pending = new SharedBoolean(false);
 657   scoped_refptr<SharedBoolean> save_next_cookie_running =
 658       new SharedBoolean(true);
 659
 660   if (!(request_info_.load_flags & LOAD_DO_NOT_SAVE_COOKIES) &&
 661       GetCookieStore() && response_cookies_.size() > 0) {
 662     CookieOptions options;
 663     options.set_include_httponly();
 664     options.set_server_time(response_date_);
 665
 666     net::CookieStore::SetCookiesCallback callback(
 667         base::Bind(&URLRequestHttpJob::OnCookieSaved,
 668                    weak_factory_.GetWeakPtr(),
 669                    save_next_cookie_running,
 670                    callback_pending));
 671
 672     // Loop through the cookies as long as SetCookieWithOptionsAsync completes
 673     // synchronously.
 674     while (!callback_pending->data &&
 675            response_cookies_save_index_ < response_cookies_.size()) {
 676       if (CanSetCookie(
 677           response_cookies_[response_cookies_save_index_], &options)) {
 678         callback_pending->data = true;
 679         GetCookieStore()->SetCookieWithOptionsAsync(
 680             request_->url(), response_cookies_[response_cookies_save_index_],
 681             options, callback);
 682       }
 683       ++response_cookies_save_index_;
 684     }
 685   }
 686
 687   save_next_cookie_running->data = false;
 688
 689   if (!callback_pending->data) {
 690     response_cookies_.clear();
 691     response_cookies_save_index_ = 0;
 692     SetStatus(URLRequestStatus());  // Clear the IO_PENDING status
 693     NotifyHeadersComplete();
 694     return;
 695   }
 696 }
 697
 698 // |save_next_cookie_running| is true when the callback is bound and set to
 699 // false when SaveNextCookie exits, allowing the callback to determine if the
 700 // save occurred synchronously or asynchronously.
 701 // |callback_pending| is false when the callback is invoked and will be set to
 702 // true by the callback, allowing SaveNextCookie to detect whether the save
 703 // occurred synchronously.
 704 // See SaveNextCookie() for more information.
 705 void URLRequestHttpJob::OnCookieSaved(
 706     scoped_refptr<SharedBoolean> save_next_cookie_running,
 707     scoped_refptr<SharedBoolean> callback_pending,
 708     bool cookie_status) {
 709   callback_pending->data = false;
 710
 711   // If we were called synchronously, return.
 712   if (save_next_cookie_running->data) {
 713     return;
 714   }
 715
 716   // We were called asynchronously, so trigger the next save.
 717   // We may have been canceled within OnSetCookie.
 718   if (GetStatus().is_success()) {
 719     SaveNextCookie();
 720   } else {
 721     NotifyCanceled();
 722   }
 723 }
 724
 725 void URLRequestHttpJob::FetchResponseCookies(
 726     std::vector<std::string>* cookies) {
 727   const std::string name = "Set-Cookie";
 728   std::string value;
 729
 730   void* iter = NULL;
 731   HttpResponseHeaders* headers = GetResponseHeaders();
 732   while (headers->EnumerateHeader(&iter, name, &value)) {
 733     if (!value.empty())
 734       cookies->push_back(value);
 735   }
 736 }
 737
 738 // NOTE: |ProcessStrictTransportSecurityHeader| and
 739 // |ProcessPublicKeyPinsHeader| have very similar structures, by design.
 740 void URLRequestHttpJob::ProcessStrictTransportSecurityHeader() {
 741   DCHECK(response_info_);
 742   TransportSecurityState* security_state =
 743       request_->context()->transport_security_state();
 744   const SSLInfo& ssl_info = response_info_->ssl_info;
 745
 746   // Only accept HSTS headers on HTTPS connections that have no
 747   // certificate errors.
 748   if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
 749       !security_state)
 750     return;
 751
 752   // http://tools.ietf.org/html/draft-ietf-websec-strict-transport-sec:
 753   //
 754   //   If a UA receives more than one STS header field in a HTTP response
 755   //   message over secure transport, then the UA MUST process only the
 756   //   first such header field.
 757   HttpResponseHeaders* headers = GetResponseHeaders();
 758   std::string value;
 759   if (headers->EnumerateHeader(NULL, "Strict-Transport-Security", &value))
 760     security_state->AddHSTSHeader(request_info_.url.host(), value);
 761 }
 762
 763 void URLRequestHttpJob::ProcessPublicKeyPinsHeader() {
 764   DCHECK(response_info_);
 765   TransportSecurityState* security_state =
 766       request_->context()->transport_security_state();
 767   const SSLInfo& ssl_info = response_info_->ssl_info;
 768
 769   // Only accept HPKP headers on HTTPS connections that have no
 770   // certificate errors.
 771   if (!ssl_info.is_valid() || IsCertStatusError(ssl_info.cert_status) ||
 772       !security_state)
 773     return;
 774
 775   // http://tools.ietf.org/html/draft-ietf-websec-key-pinning:
 776   //
 777   //   If a UA receives more than one PKP header field in an HTTP
 778   //   response message over secure transport, then the UA MUST process
 779   //   only the first such header field.
 780   HttpResponseHeaders* headers = GetResponseHeaders();
 781   std::string value;
 782   if (headers->EnumerateHeader(NULL, "Public-Key-Pins", &value))
 783     security_state->AddHPKPHeader(request_info_.url.host(), value, ssl_info);
 784 }
 785
 786 void URLRequestHttpJob::OnStartCompleted(int result) {
 787   // TODO(vadimt): Remove ScopedTracker below once crbug.com/424359 is fixed.
 788   tracked_objects::ScopedTracker tracking_profile(
 789       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 790           "424359 URLRequestHttpJob::OnStartCompleted"));
 791
 792   RecordTimer();
 793
 794   // If the request was destroyed, then there is no more work to do.
 795   if (!request_)
 796     return;
 797
 798   // If the job is done (due to cancellation), can just ignore this
 799   // notification.
 800   if (done_)
 801     return;
 802
 803   receive_headers_end_ = base::TimeTicks::Now();
 804
 805   // Clear the IO_PENDING status
 806   SetStatus(URLRequestStatus());
 807
 808   const URLRequestContext* context = request_->context();
 809
 810   if (result == ERR_SSL_PINNED_KEY_NOT_IN_CERT_CHAIN &&
 811       transaction_->GetResponseInfo() != NULL) {
 812     FraudulentCertificateReporter* reporter =
 813       context->fraudulent_certificate_reporter();
 814     if (reporter != NULL) {
 815       const SSLInfo& ssl_info = transaction_->GetResponseInfo()->ssl_info;
 816       const std::string& host = request_->url().host();
 817
 818       reporter->SendReport(host, ssl_info);
 819     }
 820   }
 821
 822   if (result == OK) {
 823     if (transaction_ && transaction_->GetResponseInfo()) {
 824       SetProxyServer(transaction_->GetResponseInfo()->proxy_server);
 825     }
 826     scoped_refptr<HttpResponseHeaders> headers = GetResponseHeaders();
 827     if (network_delegate()) {
 828       // Note that |this| may not be deleted until
 829       // |on_headers_received_callback_| or
 830       // |NetworkDelegate::URLRequestDestroyed()| has been called.
 831       OnCallToDelegate();
 832       allowed_unsafe_redirect_url_ = GURL();
 833       int error = network_delegate()->NotifyHeadersReceived(
 834           request_,
 835           on_headers_received_callback_,
 836           headers.get(),
 837           &override_response_headers_,
 838           &allowed_unsafe_redirect_url_);
 839       if (error != net::OK) {
 840         if (error == net::ERR_IO_PENDING) {
 841           awaiting_callback_ = true;
 842         } else {
 843           std::string source("delegate");
 844           request_->net_log().AddEvent(NetLog::TYPE_CANCELLED,
 845                                        NetLog::StringCallback("source",
 846                                                               &source));
 847           OnCallToDelegateComplete();
 848           NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, error));
 849         }
 850         return;
 851       }
 852     }
 853
 854     SaveCookiesAndNotifyHeadersComplete(net::OK);
 855   } else if (IsCertificateError(result)) {
 856     // We encountered an SSL certificate error.
 857     if (result == ERR_SSL_WEAK_SERVER_EPHEMERAL_DH_KEY ||
 858         result == ERR_SSL_PINNED_KEY_NOT_IN_CERT_CHAIN) {
 859       // These are hard failures. They're handled separately and don't have
 860       // the correct cert status, so set it here.
 861       SSLInfo info(transaction_->GetResponseInfo()->ssl_info);
 862       info.cert_status = MapNetErrorToCertStatus(result);
 863       NotifySSLCertificateError(info, true);
 864     } else {
 865       // Maybe overridable, maybe not. Ask the delegate to decide.
 866       const URLRequestContext* context = request_->context();
 867       TransportSecurityState* state = context->transport_security_state();
 868       const bool fatal =
 869           state && state->ShouldSSLErrorsBeFatal(request_info_.url.host());
 870       NotifySSLCertificateError(
 871           transaction_->GetResponseInfo()->ssl_info, fatal);
 872     }
 873   } else if (result == ERR_SSL_CLIENT_AUTH_CERT_NEEDED) {
 874     NotifyCertificateRequested(
 875         transaction_->GetResponseInfo()->cert_request_info.get());
 876   } else {
 877     // Even on an error, there may be useful information in the response
 878     // info (e.g. whether there's a cached copy).
 879     if (transaction_.get())
 880       response_info_ = transaction_->GetResponseInfo();
 881     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
 882   }
 883 }
 884
 885 void URLRequestHttpJob::OnHeadersReceivedCallback(int result) {
 886   awaiting_callback_ = false;
 887
 888   // Check that there are no callbacks to already canceled requests.
 889   DCHECK_NE(URLRequestStatus::CANCELED, GetStatus().status());
 890
 891   SaveCookiesAndNotifyHeadersComplete(result);
 892 }
 893
 894 void URLRequestHttpJob::OnReadCompleted(int result) {
 895   // TODO(vadimt): Remove ScopedTracker below once crbug.com/424359 is fixed.
 896   tracked_objects::ScopedTracker tracking_profile(
 897       FROM_HERE_WITH_EXPLICIT_FUNCTION(
 898           "424359 URLRequestHttpJob::OnReadCompleted"));
 899
 900   read_in_progress_ = false;
 901
 902   if (ShouldFixMismatchedContentLength(result))
 903     result = OK;
 904
 905   if (result == OK) {
 906     NotifyDone(URLRequestStatus());
 907   } else if (result < 0) {
 908     NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, result));
 909   } else {
 910     // Clear the IO_PENDING status
 911     SetStatus(URLRequestStatus());
 912   }
 913
 914   NotifyReadComplete(result);
 915 }
 916
 917 void URLRequestHttpJob::RestartTransactionWithAuth(
 918     const AuthCredentials& credentials) {
 919   auth_credentials_ = credentials;
 920
 921   // These will be reset in OnStartCompleted.
 922   response_info_ = NULL;
 923   receive_headers_end_ = base::TimeTicks();
 924   response_cookies_.clear();
 925
 926   ResetTimer();
 927
 928   // Update the cookies, since the cookie store may have been updated from the
 929   // headers in the 401/407. Since cookies were already appended to
 930   // extra_headers, we need to strip them out before adding them again.
 931   request_info_.extra_headers.RemoveHeader(HttpRequestHeaders::kCookie);
 932
 933   AddCookieHeaderAndStart();
 934 }
 935
 936 void URLRequestHttpJob::SetUpload(UploadDataStream* upload) {
 937   DCHECK(!transaction_.get()) << "cannot change once started";
 938   request_info_.upload_data_stream = upload;
 939 }
 940
 941 void URLRequestHttpJob::SetExtraRequestHeaders(
 942     const HttpRequestHeaders& headers) {
 943   DCHECK(!transaction_.get()) << "cannot change once started";
 944   request_info_.extra_headers.CopyFrom(headers);
 945 }
 946
 947 LoadState URLRequestHttpJob::GetLoadState() const {
 948   return transaction_.get() ?
 949       transaction_->GetLoadState() : LOAD_STATE_IDLE;
 950 }
 951
 952 UploadProgress URLRequestHttpJob::GetUploadProgress() const {
 953   return transaction_.get() ?
 954       transaction_->GetUploadProgress() : UploadProgress();
 955 }
 956
 957 bool URLRequestHttpJob::GetMimeType(std::string* mime_type) const {
 958   DCHECK(transaction_.get());
 959
 960   if (!response_info_)
 961     return false;
 962
 963   return GetResponseHeaders()->GetMimeType(mime_type);
 964 }
 965
 966 bool URLRequestHttpJob::GetCharset(std::string* charset) {
 967   DCHECK(transaction_.get());
 968
 969   if (!response_info_)
 970     return false;
 971
 972   return GetResponseHeaders()->GetCharset(charset);
 973 }
 974
 975 void URLRequestHttpJob::GetResponseInfo(HttpResponseInfo* info) {
 976   DCHECK(request_);
 977
 978   if (response_info_) {
 979     DCHECK(transaction_.get());
 980
 981     *info = *response_info_;
 982     if (override_response_headers_.get())
 983       info->headers = override_response_headers_;
 984   }
 985 }
 986
 987 void URLRequestHttpJob::GetLoadTimingInfo(
 988     LoadTimingInfo* load_timing_info) const {
 989   // If haven't made it far enough to receive any headers, don't return
 990   // anything.  This makes for more consistent behavior in the case of errors.
 991   if (!transaction_ || receive_headers_end_.is_null())
 992     return;
 993   if (transaction_->GetLoadTimingInfo(load_timing_info))
 994     load_timing_info->receive_headers_end = receive_headers_end_;
 995 }
 996
 997 bool URLRequestHttpJob::GetResponseCookies(std::vector<std::string>* cookies) {
 998   DCHECK(transaction_.get());
 999
1000   if (!response_info_)
1001     return false;
1002
1003   // TODO(darin): Why are we extracting response cookies again?  Perhaps we
1004   // should just leverage response_cookies_.
1005
1006   cookies->clear();
1007   FetchResponseCookies(cookies);
1008   return true;
1009 }
1010
1011 int URLRequestHttpJob::GetResponseCode() const {
1012   DCHECK(transaction_.get());
1013
1014   if (!response_info_)
1015     return -1;
1016
1017   return GetResponseHeaders()->response_code();
1018 }
1019
1020 Filter* URLRequestHttpJob::SetupFilter() const {
1021   DCHECK(transaction_.get());
1022   if (!response_info_)
1023     return NULL;
1024
1025   std::vector<Filter::FilterType> encoding_types;
1026   std::string encoding_type;
1027   HttpResponseHeaders* headers = GetResponseHeaders();
1028   void* iter = NULL;
1029   while (headers->EnumerateHeader(&iter, "Content-Encoding", &encoding_type)) {
1030     encoding_types.push_back(Filter::ConvertEncodingToType(encoding_type));
1031   }
1032
1033   if (filter_context_->SdchResponseExpected()) {
1034     // We are wary of proxies that discard or damage SDCH encoding.  If a server
1035     // explicitly states that this is not SDCH content, then we can correct our
1036     // assumption that this is an SDCH response, and avoid the need to recover
1037     // as though the content is corrupted (when we discover it is not SDCH
1038     // encoded).
1039     std::string sdch_response_status;
1040     iter = NULL;
1041     while (headers->EnumerateHeader(&iter, "X-Sdch-Encode",
1042                                     &sdch_response_status)) {
1043       if (sdch_response_status == "0") {
1044         filter_context_->ResetSdchResponseToFalse();
1045         break;
1046       }
1047     }
1048   }
1049
1050   // Even if encoding types are empty, there is a chance that we need to add
1051   // some decoding, as some proxies strip encoding completely. In such cases,
1052   // we may need to add (for example) SDCH filtering (when the context suggests
1053   // it is appropriate).
1054   Filter::FixupEncodingTypes(*filter_context_, &encoding_types);
1055
1056   return !encoding_types.empty()
1057       ? Filter::Factory(encoding_types, *filter_context_) : NULL;
1058 }
1059
1060 bool URLRequestHttpJob::CopyFragmentOnRedirect(const GURL& location) const {
1061   // Allow modification of reference fragments by default, unless
1062   // |allowed_unsafe_redirect_url_| is set and equal to the redirect URL.
1063   // When this is the case, we assume that the network delegate has set the
1064   // desired redirect URL (with or without fragment), so it must not be changed
1065   // any more.
1066   return !allowed_unsafe_redirect_url_.is_valid() ||
1067        allowed_unsafe_redirect_url_ != location;
1068 }
1069
1070 bool URLRequestHttpJob::IsSafeRedirect(const GURL& location) {
1071   // HTTP is always safe.
1072   // TODO(pauljensen): Remove once crbug.com/146591 is fixed.
1073   if (location.is_valid() &&
1074       (location.scheme() == "http" || location.scheme() == "https")) {
1075     return true;
1076   }
1077   // Delegates may mark a URL as safe for redirection.
1078   if (allowed_unsafe_redirect_url_.is_valid() &&
1079       allowed_unsafe_redirect_url_ == location) {
1080     return true;
1081   }
1082   // Query URLRequestJobFactory as to whether |location| would be safe to
1083   // redirect to.
1084   return request_->context()->job_factory() &&
1085       request_->context()->job_factory()->IsSafeRedirectTarget(location);
1086 }
1087
1088 bool URLRequestHttpJob::NeedsAuth() {
1089   int code = GetResponseCode();
1090   if (code == -1)
1091     return false;
1092
1093   // Check if we need either Proxy or WWW Authentication.  This could happen
1094   // because we either provided no auth info, or provided incorrect info.
1095   switch (code) {
1096     case 407:
1097       if (proxy_auth_state_ == AUTH_STATE_CANCELED)
1098         return false;
1099       proxy_auth_state_ = AUTH_STATE_NEED_AUTH;
1100       return true;
1101     case 401:
1102       if (server_auth_state_ == AUTH_STATE_CANCELED)
1103         return false;
1104       server_auth_state_ = AUTH_STATE_NEED_AUTH;
1105       return true;
1106   }
1107   return false;
1108 }
1109
1110 void URLRequestHttpJob::GetAuthChallengeInfo(
1111     scoped_refptr<AuthChallengeInfo>* result) {
1112   DCHECK(transaction_.get());
1113   DCHECK(response_info_);
1114
1115   // sanity checks:
1116   DCHECK(proxy_auth_state_ == AUTH_STATE_NEED_AUTH ||
1117          server_auth_state_ == AUTH_STATE_NEED_AUTH);
1118   DCHECK((GetResponseHeaders()->response_code() == HTTP_UNAUTHORIZED) ||
1119          (GetResponseHeaders()->response_code() ==
1120           HTTP_PROXY_AUTHENTICATION_REQUIRED));
1121
1122   *result = response_info_->auth_challenge;
1123 }
1124
1125 void URLRequestHttpJob::SetAuth(const AuthCredentials& credentials) {
1126   DCHECK(transaction_.get());
1127
1128   // Proxy gets set first, then WWW.
1129   if (proxy_auth_state_ == AUTH_STATE_NEED_AUTH) {
1130     proxy_auth_state_ = AUTH_STATE_HAVE_AUTH;
1131   } else {
1132     DCHECK_EQ(server_auth_state_, AUTH_STATE_NEED_AUTH);
1133     server_auth_state_ = AUTH_STATE_HAVE_AUTH;
1134   }
1135
1136   RestartTransactionWithAuth(credentials);
1137 }
1138
1139 void URLRequestHttpJob::CancelAuth() {
1140   // Proxy gets set first, then WWW.
1141   if (proxy_auth_state_ == AUTH_STATE_NEED_AUTH) {
1142     proxy_auth_state_ = AUTH_STATE_CANCELED;
1143   } else {
1144     DCHECK_EQ(server_auth_state_, AUTH_STATE_NEED_AUTH);
1145     server_auth_state_ = AUTH_STATE_CANCELED;
1146   }
1147
1148   // These will be reset in OnStartCompleted.
1149   response_info_ = NULL;
1150   receive_headers_end_ = base::TimeTicks::Now();
1151   response_cookies_.clear();
1152
1153   ResetTimer();
1154
1155   // OK, let the consumer read the error page...
1156   //
1157   // Because we set the AUTH_STATE_CANCELED flag, NeedsAuth will return false,
1158   // which will cause the consumer to receive OnResponseStarted instead of
1159   // OnAuthRequired.
1160   //
1161   // We have to do this via InvokeLater to avoid "recursing" the consumer.
1162   //
1163   base::MessageLoop::current()->PostTask(
1164       FROM_HERE,
1165       base::Bind(&URLRequestHttpJob::OnStartCompleted,
1166                  weak_factory_.GetWeakPtr(), OK));
1167 }
1168
1169 void URLRequestHttpJob::ContinueWithCertificate(
1170     X509Certificate* client_cert) {
1171   DCHECK(transaction_.get());
1172
1173   DCHECK(!response_info_) << "should not have a response yet";
1174   receive_headers_end_ = base::TimeTicks();
1175
1176   ResetTimer();
1177
1178   // No matter what, we want to report our status as IO pending since we will
1179   // be notifying our consumer asynchronously via OnStartCompleted.
1180   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1181
1182   int rv = transaction_->RestartWithCertificate(client_cert, start_callback_);
1183   if (rv == ERR_IO_PENDING)
1184     return;
1185
1186   // The transaction started synchronously, but we need to notify the
1187   // URLRequest delegate via the message loop.
1188   base::MessageLoop::current()->PostTask(
1189       FROM_HERE,
1190       base::Bind(&URLRequestHttpJob::OnStartCompleted,
1191                  weak_factory_.GetWeakPtr(), rv));
1192 }
1193
1194 void URLRequestHttpJob::ContinueDespiteLastError() {
1195   // If the transaction was destroyed, then the job was cancelled.
1196   if (!transaction_.get())
1197     return;
1198
1199   DCHECK(!response_info_) << "should not have a response yet";
1200   receive_headers_end_ = base::TimeTicks();
1201
1202   ResetTimer();
1203
1204   // No matter what, we want to report our status as IO pending since we will
1205   // be notifying our consumer asynchronously via OnStartCompleted.
1206   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1207
1208   int rv = transaction_->RestartIgnoringLastError(start_callback_);
1209   if (rv == ERR_IO_PENDING)
1210     return;
1211
1212   // The transaction started synchronously, but we need to notify the
1213   // URLRequest delegate via the message loop.
1214   base::MessageLoop::current()->PostTask(
1215       FROM_HERE,
1216       base::Bind(&URLRequestHttpJob::OnStartCompleted,
1217                  weak_factory_.GetWeakPtr(), rv));
1218 }
1219
1220 void URLRequestHttpJob::ResumeNetworkStart() {
1221   DCHECK(transaction_.get());
1222   transaction_->ResumeNetworkStart();
1223 }
1224
1225 bool URLRequestHttpJob::ShouldFixMismatchedContentLength(int rv) const {
1226   // Some servers send the body compressed, but specify the content length as
1227   // the uncompressed size.  Although this violates the HTTP spec we want to
1228   // support it (as IE and FireFox do), but *only* for an exact match.
1229   // See http://crbug.com/79694.
1230   if (rv == net::ERR_CONTENT_LENGTH_MISMATCH ||
1231       rv == net::ERR_INCOMPLETE_CHUNKED_ENCODING) {
1232     if (request_ && request_->response_headers()) {
1233       int64 expected_length = request_->response_headers()->GetContentLength();
1234       VLOG(1) << __FUNCTION__ << "() "
1235               << "\"" << request_->url().spec() << "\""
1236               << " content-length = " << expected_length
1237               << " pre total = " << prefilter_bytes_read()
1238               << " post total = " << postfilter_bytes_read();
1239       if (postfilter_bytes_read() == expected_length) {
1240         // Clear the error.
1241         return true;
1242       }
1243     }
1244   }
1245   return false;
1246 }
1247
1248 bool URLRequestHttpJob::ReadRawData(IOBuffer* buf, int buf_size,
1249                                     int* bytes_read) {
1250   DCHECK_NE(buf_size, 0);
1251   DCHECK(bytes_read);
1252   DCHECK(!read_in_progress_);
1253
1254   int rv = transaction_->Read(
1255       buf, buf_size,
1256       base::Bind(&URLRequestHttpJob::OnReadCompleted, base::Unretained(this)));
1257
1258   if (ShouldFixMismatchedContentLength(rv))
1259     rv = 0;
1260
1261   if (rv >= 0) {
1262     *bytes_read = rv;
1263     if (!rv)
1264       DoneWithRequest(FINISHED);
1265     return true;
1266   }
1267
1268   if (rv == ERR_IO_PENDING) {
1269     read_in_progress_ = true;
1270     SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
1271   } else {
1272     NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, rv));
1273   }
1274
1275   return false;
1276 }
1277
1278 void URLRequestHttpJob::StopCaching() {
1279   if (transaction_.get())
1280     transaction_->StopCaching();
1281 }
1282
1283 bool URLRequestHttpJob::GetFullRequestHeaders(
1284     HttpRequestHeaders* headers) const {
1285   if (!transaction_)
1286     return false;
1287
1288   return transaction_->GetFullRequestHeaders(headers);
1289 }
1290
1291 int64 URLRequestHttpJob::GetTotalReceivedBytes() const {
1292   if (!transaction_)
1293     return 0;
1294
1295   return transaction_->GetTotalReceivedBytes();
1296 }
1297
1298 void URLRequestHttpJob::DoneReading() {
1299   if (transaction_) {
1300     transaction_->DoneReading();
1301   }
1302   DoneWithRequest(FINISHED);
1303 }
1304
1305 void URLRequestHttpJob::DoneReadingRedirectResponse() {
1306   if (transaction_) {
1307     if (transaction_->GetResponseInfo()->headers->IsRedirect(NULL)) {
1308       // If the original headers indicate a redirect, go ahead and cache the
1309       // response, even if the |override_response_headers_| are a redirect to
1310       // another location.
1311       transaction_->DoneReading();
1312     } else {
1313       // Otherwise, |override_response_headers_| must be non-NULL and contain
1314       // bogus headers indicating a redirect.
1315       DCHECK(override_response_headers_.get());
1316       DCHECK(override_response_headers_->IsRedirect(NULL));
1317       transaction_->StopCaching();
1318     }
1319   }
1320   DoneWithRequest(FINISHED);
1321 }
1322
1323 HostPortPair URLRequestHttpJob::GetSocketAddress() const {
1324   return response_info_ ? response_info_->socket_address : HostPortPair();
1325 }
1326
1327 void URLRequestHttpJob::RecordTimer() {
1328   if (request_creation_time_.is_null()) {
1329     NOTREACHED()
1330         << "The same transaction shouldn't start twice without new timing.";
1331     return;
1332   }
1333
1334   base::TimeDelta to_start = base::Time::Now() - request_creation_time_;
1335   request_creation_time_ = base::Time();
1336
1337   UMA_HISTOGRAM_MEDIUM_TIMES("Net.HttpTimeToFirstByte", to_start);
1338 }
1339
1340 void URLRequestHttpJob::ResetTimer() {
1341   if (!request_creation_time_.is_null()) {
1342     NOTREACHED()
1343         << "The timer was reset before it was recorded.";
1344     return;
1345   }
1346   request_creation_time_ = base::Time::Now();
1347 }
1348
1349 void URLRequestHttpJob::UpdatePacketReadTimes() {
1350   if (!packet_timing_enabled_)
1351     return;
1352
1353   if (filter_input_byte_count() <= bytes_observed_in_packets_) {
1354     DCHECK_EQ(filter_input_byte_count(), bytes_observed_in_packets_);
1355     return;  // No new bytes have arrived.
1356   }
1357
1358   base::Time now(base::Time::Now());
1359   if (!bytes_observed_in_packets_)
1360     request_time_snapshot_ = now;
1361   final_packet_time_ = now;
1362
1363   bytes_observed_in_packets_ = filter_input_byte_count();
1364 }
1365
1366 void URLRequestHttpJob::RecordPacketStats(
1367     FilterContext::StatisticSelector statistic) const {
1368   if (!packet_timing_enabled_ || (final_packet_time_ == base::Time()))
1369     return;
1370
1371   base::TimeDelta duration = final_packet_time_ - request_time_snapshot_;
1372   switch (statistic) {
1373     case FilterContext::SDCH_DECODE: {
1374       UMA_HISTOGRAM_CUSTOM_COUNTS("Sdch3.Network_Decode_Bytes_Processed_b",
1375           static_cast<int>(bytes_observed_in_packets_), 500, 100000, 100);
1376       return;
1377     }
1378     case FilterContext::SDCH_PASSTHROUGH: {
1379       // Despite advertising a dictionary, we handled non-sdch compressed
1380       // content.
1381       return;
1382     }
1383
1384     case FilterContext::SDCH_EXPERIMENT_DECODE: {
1385       UMA_HISTOGRAM_CUSTOM_TIMES("Sdch3.Experiment3_Decode",
1386                                   duration,
1387                                   base::TimeDelta::FromMilliseconds(20),
1388                                   base::TimeDelta::FromMinutes(10), 100);
1389       return;
1390     }
1391     case FilterContext::SDCH_EXPERIMENT_HOLDBACK: {
1392       UMA_HISTOGRAM_CUSTOM_TIMES("Sdch3.Experiment3_Holdback",
1393                                   duration,
1394                                   base::TimeDelta::FromMilliseconds(20),
1395                                   base::TimeDelta::FromMinutes(10), 100);
1396       return;
1397     }
1398     default:
1399       NOTREACHED();
1400       return;
1401   }
1402 }
1403
1404 // The common type of histogram we use for all compression-tracking histograms.
1405 #define COMPRESSION_HISTOGRAM(name, sample) \
1406     do { \
1407       UMA_HISTOGRAM_CUSTOM_COUNTS("Net.Compress." name, sample, \
1408                                   500, 1000000, 100); \
1409     } while (0)
1410
1411 void URLRequestHttpJob::RecordCompressionHistograms() {
1412   DCHECK(request_);
1413   if (!request_)
1414     return;
1415
1416   if (is_cached_content_ ||                // Don't record cached content
1417       !GetStatus().is_success() ||         // Don't record failed content
1418       !IsCompressibleContent() ||          // Only record compressible content
1419       !prefilter_bytes_read())       // Zero-byte responses aren't useful.
1420     return;
1421
1422   // Miniature requests aren't really compressible.  Don't count them.
1423   const int kMinSize = 16;
1424   if (prefilter_bytes_read() < kMinSize)
1425     return;
1426
1427   // Only record for http or https urls.
1428   bool is_http = request_->url().SchemeIs("http");
1429   bool is_https = request_->url().SchemeIs("https");
1430   if (!is_http && !is_https)
1431     return;
1432
1433   int compressed_B = prefilter_bytes_read();
1434   int decompressed_B = postfilter_bytes_read();
1435   bool was_filtered = HasFilter();
1436
1437   // We want to record how often downloaded resources are compressed.
1438   // But, we recognize that different protocols may have different
1439   // properties.  So, for each request, we'll put it into one of 3
1440   // groups:
1441   //      a) SSL resources
1442   //         Proxies cannot tamper with compression headers with SSL.
1443   //      b) Non-SSL, loaded-via-proxy resources
1444   //         In this case, we know a proxy might have interfered.
1445   //      c) Non-SSL, loaded-without-proxy resources
1446   //         In this case, we know there was no explicit proxy.  However,
1447   //         it is possible that a transparent proxy was still interfering.
1448   //
1449   // For each group, we record the same 3 histograms.
1450
1451   if (is_https) {
1452     if (was_filtered) {
1453       COMPRESSION_HISTOGRAM("SSL.BytesBeforeCompression", compressed_B);
1454       COMPRESSION_HISTOGRAM("SSL.BytesAfterCompression", decompressed_B);
1455     } else {
1456       COMPRESSION_HISTOGRAM("SSL.ShouldHaveBeenCompressed", decompressed_B);
1457     }
1458     return;
1459   }
1460
1461   if (request_->was_fetched_via_proxy()) {
1462     if (was_filtered) {
1463       COMPRESSION_HISTOGRAM("Proxy.BytesBeforeCompression", compressed_B);
1464       COMPRESSION_HISTOGRAM("Proxy.BytesAfterCompression", decompressed_B);
1465     } else {
1466       COMPRESSION_HISTOGRAM("Proxy.ShouldHaveBeenCompressed", decompressed_B);
1467     }
1468     return;
1469   }
1470
1471   if (was_filtered) {
1472     COMPRESSION_HISTOGRAM("NoProxy.BytesBeforeCompression", compressed_B);
1473     COMPRESSION_HISTOGRAM("NoProxy.BytesAfterCompression", decompressed_B);
1474   } else {
1475     COMPRESSION_HISTOGRAM("NoProxy.ShouldHaveBeenCompressed", decompressed_B);
1476   }
1477 }
1478
1479 bool URLRequestHttpJob::IsCompressibleContent() const {
1480   std::string mime_type;
1481   return GetMimeType(&mime_type) &&
1482       (IsSupportedJavascriptMimeType(mime_type.c_str()) ||
1483        IsSupportedNonImageMimeType(mime_type.c_str()));
1484 }
1485
1486 void URLRequestHttpJob::RecordPerfHistograms(CompletionCause reason) {
1487   if (start_time_.is_null())
1488     return;
1489
1490   base::TimeDelta total_time = base::TimeTicks::Now() - start_time_;
1491   UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTime", total_time);
1492
1493   if (reason == FINISHED) {
1494     UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeSuccess", total_time);
1495   } else {
1496     UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeCancel", total_time);
1497   }
1498
1499   if (response_info_) {
1500     if (response_info_->was_cached) {
1501       UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeCached", total_time);
1502     } else  {
1503       UMA_HISTOGRAM_TIMES("Net.HttpJob.TotalTimeNotCached", total_time);
1504     }
1505   }
1506
1507   if (request_info_.load_flags & LOAD_PREFETCH && !request_->was_cached())
1508     UMA_HISTOGRAM_COUNTS("Net.Prefetch.PrefilterBytesReadFromNetwork",
1509                          prefilter_bytes_read());
1510
1511   start_time_ = base::TimeTicks();
1512 }
1513
1514 void URLRequestHttpJob::DoneWithRequest(CompletionCause reason) {
1515   if (done_)
1516     return;
1517   done_ = true;
1518   RecordPerfHistograms(reason);
1519   if (reason == FINISHED) {
1520     request_->set_received_response_content_length(prefilter_bytes_read());
1521     RecordCompressionHistograms();
1522   }
1523 }
1524
1525 HttpResponseHeaders* URLRequestHttpJob::GetResponseHeaders() const {
1526   DCHECK(transaction_.get());
1527   DCHECK(transaction_->GetResponseInfo());
1528   return override_response_headers_.get() ?
1529              override_response_headers_.get() :
1530              transaction_->GetResponseInfo()->headers.get();
1531 }
1532
1533 void URLRequestHttpJob::NotifyURLRequestDestroyed() {
1534   awaiting_callback_ = false;
1535 }
1536
1537 }  // namespace net