1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/browser/loader/resource_loader.h"
7 #include "base/command_line.h"
8 #include "base/message_loop/message_loop.h"
9 #include "base/metrics/histogram.h"
10 #include "base/time/time.h"
11 #include "content/browser/appcache/appcache_interceptor.h"
12 #include "content/browser/child_process_security_policy_impl.h"
13 #include "content/browser/loader/cross_site_resource_handler.h"
14 #include "content/browser/loader/detachable_resource_handler.h"
15 #include "content/browser/loader/resource_loader_delegate.h"
16 #include "content/browser/loader/resource_request_info_impl.h"
17 #include "content/browser/service_worker/service_worker_request_handler.h"
18 #include "content/browser/ssl/ssl_client_auth_handler.h"
19 #include "content/browser/ssl/ssl_manager.h"
20 #include "content/common/ssl_status_serialization.h"
21 #include "content/public/browser/cert_store.h"
22 #include "content/public/browser/resource_context.h"
23 #include "content/public/browser/resource_dispatcher_host_login_delegate.h"
24 #include "content/public/browser/signed_certificate_timestamp_store.h"
25 #include "content/public/common/content_client.h"
26 #include "content/public/common/content_switches.h"
27 #include "content/public/common/process_type.h"
28 #include "content/public/common/resource_response.h"
29 #include "net/base/io_buffer.h"
30 #include "net/base/load_flags.h"
31 #include "net/http/http_response_headers.h"
32 #include "net/ssl/client_cert_store.h"
33 #include "net/url_request/redirect_info.h"
34 #include "net/url_request/url_request_status.h"
36 using base::TimeDelta;
37 using base::TimeTicks;
42 void PopulateResourceResponse(ResourceRequestInfoImpl* info,
43 net::URLRequest* request,
44 ResourceResponse* response) {
45 response->head.request_time = request->request_time();
46 response->head.response_time = request->response_time();
47 response->head.headers = request->response_headers();
48 request->GetCharset(&response->head.charset);
49 response->head.content_length = request->GetExpectedContentSize();
50 request->GetMimeType(&response->head.mime_type);
51 net::HttpResponseInfo response_info = request->response_info();
52 response->head.was_fetched_via_spdy = response_info.was_fetched_via_spdy;
53 response->head.was_npn_negotiated = response_info.was_npn_negotiated;
54 response->head.npn_negotiated_protocol =
55 response_info.npn_negotiated_protocol;
56 response->head.connection_info = response_info.connection_info;
57 response->head.was_fetched_via_proxy = request->was_fetched_via_proxy();
58 response->head.socket_address = request->GetSocketAddress();
59 if (ServiceWorkerRequestHandler* handler =
60 ServiceWorkerRequestHandler::GetHandler(request)) {
61 handler->GetExtraResponseInfo(
62 &response->head.was_fetched_via_service_worker,
63 &response->head.original_url_via_service_worker,
64 &response->head.service_worker_fetch_start,
65 &response->head.service_worker_fetch_ready,
66 &response->head.service_worker_fetch_end);
68 AppCacheInterceptor::GetExtraResponseInfo(
70 &response->head.appcache_id,
71 &response->head.appcache_manifest_url);
72 if (info->is_load_timing_enabled())
73 request->GetLoadTimingInfo(&response->head.load_timing);
78 ResourceLoader::ResourceLoader(scoped_ptr<net::URLRequest> request,
79 scoped_ptr<ResourceHandler> handler,
80 ResourceLoaderDelegate* delegate)
81 : deferred_stage_(DEFERRED_NONE),
82 request_(request.Pass()),
83 handler_(handler.Pass()),
85 last_upload_position_(0),
86 waiting_for_upload_progress_ack_(false),
87 is_transferring_(false),
88 weak_ptr_factory_(this) {
89 request_->set_delegate(this);
90 handler_->SetController(this);
93 ResourceLoader::~ResourceLoader() {
94 if (login_delegate_.get())
95 login_delegate_->OnRequestCancelled();
96 if (ssl_client_auth_handler_.get())
97 ssl_client_auth_handler_->OnRequestCancelled();
99 // Run ResourceHandler destructor before we tear-down the rest of our state
100 // as the ResourceHandler may want to inspect the URLRequest and other state.
104 void ResourceLoader::StartRequest() {
105 if (delegate_->HandleExternalProtocol(this, request_->url())) {
110 // Give the handler a chance to delay the URLRequest from being started.
111 bool defer_start = false;
112 if (!handler_->OnWillStart(request_->url(), &defer_start)) {
118 deferred_stage_ = DEFERRED_START;
120 StartRequestInternal();
124 void ResourceLoader::CancelRequest(bool from_renderer) {
125 CancelRequestInternal(net::ERR_ABORTED, from_renderer);
128 void ResourceLoader::CancelAndIgnore() {
129 ResourceRequestInfoImpl* info = GetRequestInfo();
130 info->set_was_ignored_by_handler(true);
131 CancelRequest(false);
134 void ResourceLoader::CancelWithError(int error_code) {
135 CancelRequestInternal(error_code, false);
138 void ResourceLoader::ReportUploadProgress() {
139 if (waiting_for_upload_progress_ack_)
140 return; // Send one progress event at a time.
142 net::UploadProgress progress = request_->GetUploadProgress();
143 if (!progress.size())
144 return; // Nothing to upload.
146 if (progress.position() == last_upload_position_)
147 return; // No progress made since last time.
149 const uint64 kHalfPercentIncrements = 200;
150 const TimeDelta kOneSecond = TimeDelta::FromMilliseconds(1000);
152 uint64 amt_since_last = progress.position() - last_upload_position_;
153 TimeDelta time_since_last = TimeTicks::Now() - last_upload_ticks_;
155 bool is_finished = (progress.size() == progress.position());
156 bool enough_new_progress =
157 (amt_since_last > (progress.size() / kHalfPercentIncrements));
158 bool too_much_time_passed = time_since_last > kOneSecond;
160 if (is_finished || enough_new_progress || too_much_time_passed) {
161 if (request_->load_flags() & net::LOAD_ENABLE_UPLOAD_PROGRESS) {
162 handler_->OnUploadProgress(progress.position(), progress.size());
163 waiting_for_upload_progress_ack_ = true;
165 last_upload_ticks_ = TimeTicks::Now();
166 last_upload_position_ = progress.position();
170 void ResourceLoader::MarkAsTransferring() {
171 CHECK(IsResourceTypeFrame(GetRequestInfo()->GetResourceType()))
172 << "Can only transfer for navigations";
173 is_transferring_ = true;
176 void ResourceLoader::CompleteTransfer() {
177 // Although CrossSiteResourceHandler defers at OnResponseStarted
178 // (DEFERRED_READ), it may be seeing a replay of events via
179 // BufferedResourceHandler, and so the request itself is actually deferred at
180 // a later read stage.
181 DCHECK(DEFERRED_READ == deferred_stage_ ||
182 DEFERRED_RESPONSE_COMPLETE == deferred_stage_);
184 is_transferring_ = false;
185 GetRequestInfo()->cross_site_handler()->ResumeResponse();
188 ResourceRequestInfoImpl* ResourceLoader::GetRequestInfo() {
189 return ResourceRequestInfoImpl::ForRequest(request_.get());
192 void ResourceLoader::ClearLoginDelegate() {
193 login_delegate_ = NULL;
196 void ResourceLoader::ClearSSLClientAuthHandler() {
197 ssl_client_auth_handler_ = NULL;
200 void ResourceLoader::OnUploadProgressACK() {
201 waiting_for_upload_progress_ack_ = false;
204 void ResourceLoader::OnReceivedRedirect(net::URLRequest* unused,
205 const net::RedirectInfo& redirect_info,
207 DCHECK_EQ(request_.get(), unused);
209 VLOG(1) << "OnReceivedRedirect: " << request_->url().spec();
210 DCHECK(request_->status().is_success());
212 ResourceRequestInfoImpl* info = GetRequestInfo();
214 if (info->GetProcessType() != PROCESS_TYPE_PLUGIN &&
215 !ChildProcessSecurityPolicyImpl::GetInstance()->
216 CanRequestURL(info->GetChildID(), redirect_info.new_url)) {
217 VLOG(1) << "Denied unauthorized request for "
218 << redirect_info.new_url.possibly_invalid_spec();
220 // Tell the renderer that this request was disallowed.
225 delegate_->DidReceiveRedirect(this, redirect_info.new_url);
227 if (delegate_->HandleExternalProtocol(this, redirect_info.new_url)) {
228 // The request is complete so we can remove it.
233 scoped_refptr<ResourceResponse> response(new ResourceResponse());
234 PopulateResourceResponse(info, request_.get(), response.get());
236 if (!handler_->OnRequestRedirected(redirect_info, response.get(), defer)) {
239 deferred_stage_ = DEFERRED_REDIRECT; // Follow redirect when resumed.
243 void ResourceLoader::OnAuthRequired(net::URLRequest* unused,
244 net::AuthChallengeInfo* auth_info) {
245 DCHECK_EQ(request_.get(), unused);
247 if (request_->load_flags() & net::LOAD_DO_NOT_PROMPT_FOR_LOGIN) {
248 request_->CancelAuth();
252 // Create a login dialog on the UI thread to get authentication data, or pull
253 // from cache and continue on the IO thread.
255 DCHECK(!login_delegate_.get())
256 << "OnAuthRequired called with login_delegate pending";
257 login_delegate_ = delegate_->CreateLoginDelegate(this, auth_info);
258 if (!login_delegate_.get())
259 request_->CancelAuth();
262 void ResourceLoader::OnCertificateRequested(
263 net::URLRequest* unused,
264 net::SSLCertRequestInfo* cert_info) {
265 DCHECK_EQ(request_.get(), unused);
267 if (request_->load_flags() & net::LOAD_PREFETCH) {
272 DCHECK(!ssl_client_auth_handler_.get())
273 << "OnCertificateRequested called with ssl_client_auth_handler pending";
274 ssl_client_auth_handler_ = new SSLClientAuthHandler(
275 GetRequestInfo()->GetContext()->CreateClientCertStore(),
278 ssl_client_auth_handler_->SelectCertificate();
281 void ResourceLoader::OnSSLCertificateError(net::URLRequest* request,
282 const net::SSLInfo& ssl_info,
284 ResourceRequestInfoImpl* info = GetRequestInfo();
286 int render_process_id;
288 if (!info->GetAssociatedRenderFrame(&render_process_id, &render_frame_id))
291 SSLManager::OnSSLCertificateError(
292 weak_ptr_factory_.GetWeakPtr(),
293 info->GetGlobalRequestID(),
294 info->GetResourceType(),
302 void ResourceLoader::OnBeforeNetworkStart(net::URLRequest* unused,
304 DCHECK_EQ(request_.get(), unused);
306 // Give the handler a chance to delay the URLRequest from using the network.
307 if (!handler_->OnBeforeNetworkStart(request_->url(), defer)) {
311 deferred_stage_ = DEFERRED_NETWORK_START;
315 void ResourceLoader::OnResponseStarted(net::URLRequest* unused) {
316 DCHECK_EQ(request_.get(), unused);
318 VLOG(1) << "OnResponseStarted: " << request_->url().spec();
320 // The CanLoadPage check should take place after any server redirects have
321 // finished, at the point in time that we know a page will commit in the
323 ResourceRequestInfoImpl* info = GetRequestInfo();
324 ChildProcessSecurityPolicyImpl* policy =
325 ChildProcessSecurityPolicyImpl::GetInstance();
326 if (!policy->CanLoadPage(info->GetChildID(),
328 info->GetResourceType())) {
333 if (!request_->status().is_success()) {
338 // We want to send a final upload progress message prior to sending the
339 // response complete message even if we're waiting for an ack to to a
340 // previous upload progress message.
341 waiting_for_upload_progress_ack_ = false;
342 ReportUploadProgress();
344 CompleteResponseStarted();
349 if (request_->status().is_success()) {
350 StartReading(false); // Read the first chunk.
356 void ResourceLoader::OnReadCompleted(net::URLRequest* unused, int bytes_read) {
357 DCHECK_EQ(request_.get(), unused);
358 VLOG(1) << "OnReadCompleted: \"" << request_->url().spec() << "\""
359 << " bytes_read = " << bytes_read;
361 // bytes_read == -1 always implies an error.
362 if (bytes_read == -1 || !request_->status().is_success()) {
367 CompleteRead(bytes_read);
369 // If the handler cancelled or deferred the request, do not continue
370 // processing the read. If cancelled, the URLRequest has already been
371 // cancelled and will schedule an erroring OnReadCompleted later. If deferred,
372 // do nothing until resumed.
374 // Note: if bytes_read is 0 (EOF) and the handler defers, resumption will call
375 // ResponseCompleted().
376 if (is_deferred() || !request_->status().is_success())
379 if (bytes_read > 0) {
380 StartReading(true); // Read the next chunk.
382 // URLRequest reported an EOF. Call ResponseCompleted.
383 DCHECK_EQ(0, bytes_read);
388 void ResourceLoader::CancelSSLRequest(const GlobalRequestID& id,
390 const net::SSLInfo* ssl_info) {
391 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
393 // The request can be NULL if it was cancelled by the renderer (as the
394 // request of the user navigating to a new page from the location bar).
395 if (!request_->is_pending())
397 DVLOG(1) << "CancelSSLRequest() url: " << request_->url().spec();
400 request_->CancelWithSSLError(error, *ssl_info);
402 request_->CancelWithError(error);
406 void ResourceLoader::ContinueSSLRequest(const GlobalRequestID& id) {
407 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
409 DVLOG(1) << "ContinueSSLRequest() url: " << request_->url().spec();
411 request_->ContinueDespiteLastError();
414 void ResourceLoader::Resume() {
415 DCHECK(!is_transferring_);
417 DeferredStage stage = deferred_stage_;
418 deferred_stage_ = DEFERRED_NONE;
424 StartRequestInternal();
426 case DEFERRED_NETWORK_START:
427 request_->ResumeNetworkStart();
429 case DEFERRED_REDIRECT:
430 request_->FollowDeferredRedirect();
433 base::MessageLoop::current()->PostTask(
435 base::Bind(&ResourceLoader::ResumeReading,
436 weak_ptr_factory_.GetWeakPtr()));
438 case DEFERRED_RESPONSE_COMPLETE:
439 base::MessageLoop::current()->PostTask(
441 base::Bind(&ResourceLoader::ResponseCompleted,
442 weak_ptr_factory_.GetWeakPtr()));
444 case DEFERRED_FINISH:
445 // Delay self-destruction since we don't know how we were reached.
446 base::MessageLoop::current()->PostTask(
448 base::Bind(&ResourceLoader::CallDidFinishLoading,
449 weak_ptr_factory_.GetWeakPtr()));
454 void ResourceLoader::Cancel() {
455 CancelRequest(false);
458 void ResourceLoader::StartRequestInternal() {
459 DCHECK(!request_->is_pending());
461 if (!request_->status().is_success()) {
467 delegate_->DidStartRequest(this);
470 void ResourceLoader::CancelRequestInternal(int error, bool from_renderer) {
471 VLOG(1) << "CancelRequestInternal: " << request_->url().spec();
473 ResourceRequestInfoImpl* info = GetRequestInfo();
475 // WebKit will send us a cancel for downloads since it no longer handles
476 // them. In this case, ignore the cancel since we handle downloads in the
478 if (from_renderer && (info->IsDownload() || info->is_stream()))
481 if (from_renderer && info->detachable_handler()) {
482 // TODO(davidben): Fix Blink handling of prefetches so they are not
483 // cancelled on navigate away and end up in the local cache.
484 info->detachable_handler()->Detach();
488 // TODO(darin): Perhaps we should really be looking to see if the status is
490 bool was_pending = request_->is_pending();
492 if (login_delegate_.get()) {
493 login_delegate_->OnRequestCancelled();
494 login_delegate_ = NULL;
496 if (ssl_client_auth_handler_.get()) {
497 ssl_client_auth_handler_->OnRequestCancelled();
498 ssl_client_auth_handler_ = NULL;
501 request_->CancelWithError(error);
504 // If the request isn't in flight, then we won't get an asynchronous
505 // notification from the request, so we have to signal ourselves to finish
507 base::MessageLoop::current()->PostTask(
509 base::Bind(&ResourceLoader::ResponseCompleted,
510 weak_ptr_factory_.GetWeakPtr()));
514 void ResourceLoader::StoreSignedCertificateTimestamps(
515 const net::SignedCertificateTimestampAndStatusList& sct_list,
517 SignedCertificateTimestampIDStatusList* sct_ids) {
518 SignedCertificateTimestampStore* sct_store(
519 SignedCertificateTimestampStore::GetInstance());
521 for (net::SignedCertificateTimestampAndStatusList::const_iterator iter =
522 sct_list.begin(); iter != sct_list.end(); ++iter) {
523 const int sct_id(sct_store->Store(iter->sct.get(), process_id));
525 SignedCertificateTimestampIDAndStatus(sct_id, iter->status));
529 void ResourceLoader::CompleteResponseStarted() {
530 ResourceRequestInfoImpl* info = GetRequestInfo();
532 scoped_refptr<ResourceResponse> response(new ResourceResponse());
533 PopulateResourceResponse(info, request_.get(), response.get());
535 if (request_->ssl_info().cert.get()) {
536 int cert_id = CertStore::GetInstance()->StoreCert(
537 request_->ssl_info().cert.get(), info->GetChildID());
539 SignedCertificateTimestampIDStatusList signed_certificate_timestamp_ids;
540 StoreSignedCertificateTimestamps(
541 request_->ssl_info().signed_certificate_timestamps,
543 &signed_certificate_timestamp_ids);
545 response->head.security_info = SerializeSecurityInfo(
547 request_->ssl_info().cert_status,
548 request_->ssl_info().security_bits,
549 request_->ssl_info().connection_status,
550 signed_certificate_timestamp_ids);
552 // We should not have any SSL state.
553 DCHECK(!request_->ssl_info().cert_status &&
554 request_->ssl_info().security_bits == -1 &&
555 !request_->ssl_info().connection_status);
558 delegate_->DidReceiveResponse(this);
561 if (!handler_->OnResponseStarted(response.get(), &defer)) {
564 read_deferral_start_time_ = base::TimeTicks::Now();
565 deferred_stage_ = DEFERRED_READ; // Read first chunk when resumed.
569 void ResourceLoader::StartReading(bool is_continuation) {
571 ReadMore(&bytes_read);
573 // If IO is pending, wait for the URLRequest to call OnReadCompleted.
574 if (request_->status().is_io_pending())
577 if (!is_continuation || bytes_read <= 0) {
578 OnReadCompleted(request_.get(), bytes_read);
580 // Else, trigger OnReadCompleted asynchronously to avoid starving the IO
581 // thread in case the URLRequest can provide data synchronously.
582 base::MessageLoop::current()->PostTask(
584 base::Bind(&ResourceLoader::OnReadCompleted,
585 weak_ptr_factory_.GetWeakPtr(),
591 void ResourceLoader::ResumeReading() {
592 DCHECK(!is_deferred());
594 if (!read_deferral_start_time_.is_null()) {
595 UMA_HISTOGRAM_TIMES("Net.ResourceLoader.ReadDeferral",
596 base::TimeTicks::Now() - read_deferral_start_time_);
597 read_deferral_start_time_ = base::TimeTicks();
599 if (request_->status().is_success()) {
600 StartReading(false); // Read the next chunk (OK to complete synchronously).
606 void ResourceLoader::ReadMore(int* bytes_read) {
607 DCHECK(!is_deferred());
609 // Make sure we track the buffer in at least one place. This ensures it gets
610 // deleted even in the case the request has already finished its job and
611 // doesn't use the buffer.
612 scoped_refptr<net::IOBuffer> buf;
614 if (!handler_->OnWillRead(&buf, &buf_size, -1)) {
620 DCHECK(buf_size > 0);
622 request_->Read(buf.get(), buf_size, bytes_read);
624 // No need to check the return value here as we'll detect errors by
625 // inspecting the URLRequest's status.
628 void ResourceLoader::CompleteRead(int bytes_read) {
629 DCHECK(bytes_read >= 0);
630 DCHECK(request_->status().is_success());
633 if (!handler_->OnReadCompleted(bytes_read, &defer)) {
637 bytes_read > 0 ? DEFERRED_READ : DEFERRED_RESPONSE_COMPLETE;
640 // Note: the request may still have been cancelled while OnReadCompleted
641 // returns true if OnReadCompleted caused request to get cancelled
642 // out-of-band. (In AwResourceDispatcherHostDelegate::DownloadStarting, for
646 void ResourceLoader::ResponseCompleted() {
647 VLOG(1) << "ResponseCompleted: " << request_->url().spec();
649 ResourceRequestInfoImpl* info = GetRequestInfo();
651 std::string security_info;
652 const net::SSLInfo& ssl_info = request_->ssl_info();
653 if (ssl_info.cert.get() != NULL) {
654 int cert_id = CertStore::GetInstance()->StoreCert(ssl_info.cert.get(),
656 SignedCertificateTimestampIDStatusList signed_certificate_timestamp_ids;
657 StoreSignedCertificateTimestamps(ssl_info.signed_certificate_timestamps,
659 &signed_certificate_timestamp_ids);
661 security_info = SerializeSecurityInfo(
662 cert_id, ssl_info.cert_status, ssl_info.security_bits,
663 ssl_info.connection_status, signed_certificate_timestamp_ids);
667 handler_->OnResponseCompleted(request_->status(), security_info, &defer);
669 // The handler is not ready to die yet. We will call DidFinishLoading when
671 deferred_stage_ = DEFERRED_FINISH;
673 // This will result in our destruction.
674 CallDidFinishLoading();
678 void ResourceLoader::CallDidFinishLoading() {
679 delegate_->DidFinishLoading(this);
682 void ResourceLoader::RecordHistograms() {
683 ResourceRequestInfoImpl* info = GetRequestInfo();
685 if (info->GetResourceType() == RESOURCE_TYPE_PREFETCH) {
686 PrefetchStatus status = STATUS_UNDEFINED;
687 TimeDelta total_time = base::TimeTicks::Now() - request_->creation_time();
689 switch (request_->status().status()) {
690 case net::URLRequestStatus::SUCCESS:
691 if (request_->was_cached()) {
692 status = STATUS_SUCCESS_FROM_CACHE;
693 UMA_HISTOGRAM_TIMES("Net.Prefetch.TimeSpentPrefetchingFromCache",
696 status = STATUS_SUCCESS_FROM_NETWORK;
697 UMA_HISTOGRAM_TIMES("Net.Prefetch.TimeSpentPrefetchingFromNetwork",
701 case net::URLRequestStatus::CANCELED:
702 status = STATUS_CANCELED;
703 UMA_HISTOGRAM_TIMES("Net.Prefetch.TimeBeforeCancel", total_time);
705 case net::URLRequestStatus::IO_PENDING:
706 case net::URLRequestStatus::FAILED:
707 status = STATUS_UNDEFINED;
711 UMA_HISTOGRAM_ENUMERATION("Net.Prefetch.Pattern", status, STATUS_MAX);
715 } // namespace content