1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "net/url_request/url_request_job.h"
8 #include "base/compiler_specific.h"
9 #include "base/message_loop/message_loop.h"
10 #include "base/power_monitor/power_monitor.h"
11 #include "base/strings/string_number_conversions.h"
12 #include "base/strings/string_util.h"
13 #include "net/base/auth.h"
14 #include "net/base/host_port_pair.h"
15 #include "net/base/io_buffer.h"
16 #include "net/base/load_states.h"
17 #include "net/base/net_errors.h"
18 #include "net/base/network_delegate.h"
19 #include "net/filter/filter.h"
20 #include "net/http/http_response_headers.h"
21 #include "net/url_request/url_request.h"
25 URLRequestJob::URLRequestJob(URLRequest* request,
26 NetworkDelegate* network_delegate)
29 prefilter_bytes_read_(0),
30 postfilter_bytes_read_(0),
31 filter_input_byte_count_(0),
32 filter_needs_more_output_space_(false),
33 filtered_read_buffer_len_(0),
34 has_handled_response_(false),
35 expected_content_size_(-1),
36 deferred_redirect_status_code_(-1),
37 network_delegate_(network_delegate),
39 base::PowerMonitor* power_monitor = base::PowerMonitor::Get();
41 power_monitor->AddObserver(this);
44 void URLRequestJob::SetUpload(UploadDataStream* upload) {
47 void URLRequestJob::SetExtraRequestHeaders(const HttpRequestHeaders& headers) {
50 void URLRequestJob::SetPriority(RequestPriority priority) {
53 void URLRequestJob::Kill() {
54 weak_factory_.InvalidateWeakPtrs();
55 // Make sure the request is notified that we are done. We assume that the
56 // request took care of setting its error status before calling Kill.
61 void URLRequestJob::DetachRequest() {
65 // This function calls ReadData to get stream data. If a filter exists, passes
66 // the data to the attached filter. Then returns the output from filter back to
68 bool URLRequestJob::Read(IOBuffer* buf, int buf_size, int *bytes_read) {
71 DCHECK_LT(buf_size, 1000000); // Sanity check.
74 DCHECK(filtered_read_buffer_.get() == NULL);
75 DCHECK_EQ(0, filtered_read_buffer_len_);
79 // Skip Filter if not present.
81 rv = ReadRawDataHelper(buf, buf_size, bytes_read);
83 // Save the caller's buffers while we do IO
84 // in the filter's buffers.
85 filtered_read_buffer_ = buf;
86 filtered_read_buffer_len_ = buf_size;
88 if (ReadFilteredData(bytes_read)) {
89 rv = true; // We have data to return.
91 // It is fine to call DoneReading even if ReadFilteredData receives 0
92 // bytes from the net, but we avoid making that call if we know for
93 // sure that's the case (ReadRawDataHelper path).
97 rv = false; // Error, or a new IO is pending.
100 if (rv && *bytes_read == 0)
101 NotifyDone(URLRequestStatus());
105 void URLRequestJob::StopCaching() {
106 // Nothing to do here.
109 bool URLRequestJob::GetFullRequestHeaders(HttpRequestHeaders* headers) const {
110 // Most job types don't send request headers.
114 int64 URLRequestJob::GetTotalReceivedBytes() const {
118 LoadState URLRequestJob::GetLoadState() const {
119 return LOAD_STATE_IDLE;
122 UploadProgress URLRequestJob::GetUploadProgress() const {
123 return UploadProgress();
126 bool URLRequestJob::GetCharset(std::string* charset) {
130 void URLRequestJob::GetResponseInfo(HttpResponseInfo* info) {
133 void URLRequestJob::GetLoadTimingInfo(LoadTimingInfo* load_timing_info) const {
134 // Only certain request types return more than just request start times.
137 bool URLRequestJob::GetResponseCookies(std::vector<std::string>* cookies) {
141 Filter* URLRequestJob::SetupFilter() const {
145 bool URLRequestJob::IsRedirectResponse(GURL* location,
146 int* http_status_code) {
147 // For non-HTTP jobs, headers will be null.
148 HttpResponseHeaders* headers = request_->response_headers();
153 if (!headers->IsRedirect(&value))
156 *location = request_->url().Resolve(value);
157 *http_status_code = headers->response_code();
161 bool URLRequestJob::CopyFragmentOnRedirect(const GURL& location) const {
165 bool URLRequestJob::IsSafeRedirect(const GURL& location) {
169 bool URLRequestJob::NeedsAuth() {
173 void URLRequestJob::GetAuthChallengeInfo(
174 scoped_refptr<AuthChallengeInfo>* auth_info) {
175 // This will only be called if NeedsAuth() returns true, in which
176 // case the derived class should implement this!
180 void URLRequestJob::SetAuth(const AuthCredentials& credentials) {
181 // This will only be called if NeedsAuth() returns true, in which
182 // case the derived class should implement this!
186 void URLRequestJob::CancelAuth() {
187 // This will only be called if NeedsAuth() returns true, in which
188 // case the derived class should implement this!
192 void URLRequestJob::ContinueWithCertificate(
193 X509Certificate* client_cert) {
194 // The derived class should implement this!
198 void URLRequestJob::ContinueDespiteLastError() {
199 // Implementations should know how to recover from errors they generate.
200 // If this code was reached, we are trying to recover from an error that
201 // we don't know how to recover from.
205 void URLRequestJob::FollowDeferredRedirect() {
206 DCHECK(deferred_redirect_status_code_ != -1);
208 // NOTE: deferred_redirect_url_ may be invalid, and attempting to redirect to
209 // such an URL will fail inside FollowRedirect. The DCHECK above asserts
210 // that we called OnReceivedRedirect.
212 // It is also possible that FollowRedirect will drop the last reference to
213 // this job, so we need to reset our members before calling it.
215 GURL redirect_url = deferred_redirect_url_;
216 int redirect_status_code = deferred_redirect_status_code_;
218 deferred_redirect_url_ = GURL();
219 deferred_redirect_status_code_ = -1;
221 FollowRedirect(redirect_url, redirect_status_code);
224 void URLRequestJob::ResumeNetworkStart() {
225 // This should only be called for HTTP Jobs, and implemented in the derived
230 bool URLRequestJob::GetMimeType(std::string* mime_type) const {
234 int URLRequestJob::GetResponseCode() const {
238 HostPortPair URLRequestJob::GetSocketAddress() const {
239 return HostPortPair();
242 void URLRequestJob::OnSuspend() {
246 void URLRequestJob::NotifyURLRequestDestroyed() {
249 URLRequestJob::~URLRequestJob() {
250 base::PowerMonitor* power_monitor = base::PowerMonitor::Get();
252 power_monitor->RemoveObserver(this);
255 void URLRequestJob::NotifyCertificateRequested(
256 SSLCertRequestInfo* cert_request_info) {
258 return; // The request was destroyed, so there is no more work to do.
260 request_->NotifyCertificateRequested(cert_request_info);
263 void URLRequestJob::NotifySSLCertificateError(const SSLInfo& ssl_info,
266 return; // The request was destroyed, so there is no more work to do.
268 request_->NotifySSLCertificateError(ssl_info, fatal);
271 bool URLRequestJob::CanGetCookies(const CookieList& cookie_list) const {
273 return false; // The request was destroyed, so there is no more work to do.
275 return request_->CanGetCookies(cookie_list);
278 bool URLRequestJob::CanSetCookie(const std::string& cookie_line,
279 CookieOptions* options) const {
281 return false; // The request was destroyed, so there is no more work to do.
283 return request_->CanSetCookie(cookie_line, options);
286 bool URLRequestJob::CanEnablePrivacyMode() const {
288 return false; // The request was destroyed, so there is no more work to do.
290 return request_->CanEnablePrivacyMode();
293 CookieStore* URLRequestJob::GetCookieStore() const {
296 return request_->cookie_store();
299 void URLRequestJob::NotifyBeforeNetworkStart(bool* defer) {
303 request_->NotifyBeforeNetworkStart(defer);
306 void URLRequestJob::NotifyHeadersComplete() {
307 if (!request_ || !request_->has_delegate())
308 return; // The request was destroyed, so there is no more work to do.
310 if (has_handled_response_)
313 DCHECK(!request_->status().is_io_pending());
315 // Initialize to the current time, and let the subclass optionally override
316 // the time stamps if it has that information. The default request_time is
317 // set by URLRequest before it calls our Start method.
318 request_->response_info_.response_time = base::Time::Now();
319 GetResponseInfo(&request_->response_info_);
321 // When notifying the delegate, the delegate can release the request
322 // (and thus release 'this'). After calling to the delgate, we must
323 // check the request pointer to see if it still exists, and return
324 // immediately if it has been destroyed. self_preservation ensures our
325 // survival until we can get out of this method.
326 scoped_refptr<URLRequestJob> self_preservation(this);
329 request_->OnHeadersComplete();
332 int http_status_code;
333 if (IsRedirectResponse(&new_location, &http_status_code)) {
334 // Redirect response bodies are not read. Notify the transaction
335 // so it does not treat being stopped as an error.
336 DoneReadingRedirectResponse();
338 const GURL& url = request_->url();
340 // Move the reference fragment of the old location to the new one if the
341 // new one has none. This duplicates mozilla's behavior.
342 if (url.is_valid() && url.has_ref() && !new_location.has_ref() &&
343 CopyFragmentOnRedirect(new_location)) {
344 GURL::Replacements replacements;
345 // Reference the |ref| directly out of the original URL to avoid a
347 replacements.SetRef(url.spec().data(),
348 url.parsed_for_possibly_invalid_spec().ref);
349 new_location = new_location.ReplaceComponents(replacements);
352 bool defer_redirect = false;
353 request_->NotifyReceivedRedirect(new_location, &defer_redirect);
355 // Ensure that the request wasn't detached or destroyed in
356 // NotifyReceivedRedirect
357 if (!request_ || !request_->has_delegate())
360 // If we were not cancelled, then maybe follow the redirect.
361 if (request_->status().is_success()) {
362 if (defer_redirect) {
363 deferred_redirect_url_ = new_location;
364 deferred_redirect_status_code_ = http_status_code;
366 FollowRedirect(new_location, http_status_code);
370 } else if (NeedsAuth()) {
371 scoped_refptr<AuthChallengeInfo> auth_info;
372 GetAuthChallengeInfo(&auth_info);
373 // Need to check for a NULL auth_info because the server may have failed
374 // to send a challenge with the 401 response.
375 if (auth_info.get()) {
376 request_->NotifyAuthRequired(auth_info.get());
377 // Wait for SetAuth or CancelAuth to be called.
382 has_handled_response_ = true;
383 if (request_->status().is_success())
384 filter_.reset(SetupFilter());
386 if (!filter_.get()) {
387 std::string content_length;
388 request_->GetResponseHeaderByName("content-length", &content_length);
389 if (!content_length.empty())
390 base::StringToInt64(content_length, &expected_content_size_);
393 request_->NotifyResponseStarted();
396 void URLRequestJob::NotifyReadComplete(int bytes_read) {
397 if (!request_ || !request_->has_delegate())
398 return; // The request was destroyed, so there is no more work to do.
400 // TODO(darin): Bug 1004233. Re-enable this test once all of the chrome
401 // unit_tests have been fixed to not trip this.
402 //DCHECK(!request_->status().is_io_pending());
404 // The headers should be complete before reads complete
405 DCHECK(has_handled_response_);
407 OnRawReadComplete(bytes_read);
409 // Don't notify if we had an error.
410 if (!request_->status().is_success())
413 // When notifying the delegate, the delegate can release the request
414 // (and thus release 'this'). After calling to the delegate, we must
415 // check the request pointer to see if it still exists, and return
416 // immediately if it has been destroyed. self_preservation ensures our
417 // survival until we can get out of this method.
418 scoped_refptr<URLRequestJob> self_preservation(this);
421 // Tell the filter that it has more data
422 FilteredDataRead(bytes_read);
425 int filter_bytes_read = 0;
426 if (ReadFilteredData(&filter_bytes_read)) {
427 if (!filter_bytes_read)
429 request_->NotifyReadCompleted(filter_bytes_read);
432 request_->NotifyReadCompleted(bytes_read);
434 DVLOG(1) << __FUNCTION__ << "() "
435 << "\"" << (request_ ? request_->url().spec() : "???") << "\""
436 << " pre bytes read = " << bytes_read
437 << " pre total = " << prefilter_bytes_read_
438 << " post total = " << postfilter_bytes_read_;
441 void URLRequestJob::NotifyStartError(const URLRequestStatus &status) {
442 DCHECK(!has_handled_response_);
443 has_handled_response_ = true;
445 // There may be relevant information in the response info even in the
447 GetResponseInfo(&request_->response_info_);
449 request_->set_status(status);
450 request_->NotifyResponseStarted();
451 // We may have been deleted.
455 void URLRequestJob::NotifyDone(const URLRequestStatus &status) {
456 DCHECK(!done_) << "Job sending done notification twice";
461 // Unless there was an error, we should have at least tried to handle
462 // the response before getting here.
463 DCHECK(has_handled_response_ || !status.is_success());
465 // As with NotifyReadComplete, we need to take care to notice if we were
466 // destroyed during a delegate callback.
468 request_->set_is_pending(false);
469 // With async IO, it's quite possible to have a few outstanding
470 // requests. We could receive a request to Cancel, followed shortly
471 // by a successful IO. For tracking the status(), once there is
472 // an error, we do not change the status back to success. To
473 // enforce this, only set the status if the job is so far
475 if (request_->status().is_success()) {
476 if (status.status() == URLRequestStatus::FAILED) {
477 request_->net_log().AddEventWithNetErrorCode(NetLog::TYPE_FAILED,
480 request_->set_status(status);
484 // Complete this notification later. This prevents us from re-entering the
485 // delegate if we're done because of a synchronous call.
486 base::MessageLoop::current()->PostTask(
488 base::Bind(&URLRequestJob::CompleteNotifyDone,
489 weak_factory_.GetWeakPtr()));
492 void URLRequestJob::CompleteNotifyDone() {
493 // Check if we should notify the delegate that we're done because of an error.
495 !request_->status().is_success() &&
496 request_->has_delegate()) {
497 // We report the error differently depending on whether we've called
498 // OnResponseStarted yet.
499 if (has_handled_response_) {
500 // We signal the error by calling OnReadComplete with a bytes_read of -1.
501 request_->NotifyReadCompleted(-1);
503 has_handled_response_ = true;
504 request_->NotifyResponseStarted();
509 void URLRequestJob::NotifyCanceled() {
511 NotifyDone(URLRequestStatus(URLRequestStatus::CANCELED, ERR_ABORTED));
515 void URLRequestJob::NotifyRestartRequired() {
516 DCHECK(!has_handled_response_);
517 if (GetStatus().status() != URLRequestStatus::CANCELED)
521 void URLRequestJob::OnCallToDelegate() {
522 request_->OnCallToDelegate();
525 void URLRequestJob::OnCallToDelegateComplete() {
526 request_->OnCallToDelegateComplete();
529 bool URLRequestJob::ReadRawData(IOBuffer* buf, int buf_size,
536 void URLRequestJob::DoneReading() {
540 void URLRequestJob::DoneReadingRedirectResponse() {
543 void URLRequestJob::FilteredDataRead(int bytes_read) {
545 filter_->FlushStreamBuffer(bytes_read);
548 bool URLRequestJob::ReadFilteredData(int* bytes_read) {
550 DCHECK(filtered_read_buffer_);
551 DCHECK_GT(filtered_read_buffer_len_, 0);
552 DCHECK_LT(filtered_read_buffer_len_, 1000000); // Sanity check.
553 DCHECK(!raw_read_buffer_);
562 if (!filter_needs_more_output_space_ && !filter_->stream_data_len()) {
563 // We don't have any raw data to work with, so read from the transaction.
564 int filtered_data_read;
565 if (ReadRawDataForFilter(&filtered_data_read)) {
566 if (filtered_data_read > 0) {
567 // Give data to filter.
568 filter_->FlushStreamBuffer(filtered_data_read);
573 return false; // IO Pending (or error).
577 if ((filter_->stream_data_len() || filter_needs_more_output_space_) &&
579 // Get filtered data.
580 int filtered_data_len = filtered_read_buffer_len_;
581 int output_buffer_size = filtered_data_len;
582 Filter::FilterStatus status =
583 filter_->ReadData(filtered_read_buffer_->data(), &filtered_data_len);
585 if (filter_needs_more_output_space_ && !filtered_data_len) {
586 // filter_needs_more_output_space_ was mistaken... there are no more
587 // bytes and we should have at least tried to fill up the filter's input
588 // buffer. Correct the state, and try again.
589 filter_needs_more_output_space_ = false;
592 filter_needs_more_output_space_ =
593 (filtered_data_len == output_buffer_size);
596 case Filter::FILTER_DONE: {
597 filter_needs_more_output_space_ = false;
598 *bytes_read = filtered_data_len;
599 postfilter_bytes_read_ += filtered_data_len;
603 case Filter::FILTER_NEED_MORE_DATA: {
604 // We have finished filtering all data currently in the buffer.
605 // There might be some space left in the output buffer. One can
606 // consider reading more data from the stream to feed the filter
607 // and filling up the output buffer. This leads to more complicated
608 // buffer management and data notification mechanisms.
609 // We can revisit this issue if there is a real perf need.
610 if (filtered_data_len > 0) {
611 *bytes_read = filtered_data_len;
612 postfilter_bytes_read_ += filtered_data_len;
615 // Read again since we haven't received enough data yet (e.g., we
616 // may not have a complete gzip header yet).
621 case Filter::FILTER_OK: {
622 *bytes_read = filtered_data_len;
623 postfilter_bytes_read_ += filtered_data_len;
627 case Filter::FILTER_ERROR: {
628 DVLOG(1) << __FUNCTION__ << "() "
629 << "\"" << (request_ ? request_->url().spec() : "???")
630 << "\"" << " Filter Error";
631 filter_needs_more_output_space_ = false;
632 NotifyDone(URLRequestStatus(URLRequestStatus::FAILED,
633 ERR_CONTENT_DECODING_FAILED));
639 filter_needs_more_output_space_ = false;
645 // If logging all bytes is enabled, log the filtered bytes read.
646 if (rv && request() && request()->net_log().IsLoggingBytes() &&
647 filtered_data_len > 0) {
648 request()->net_log().AddByteTransferEvent(
649 NetLog::TYPE_URL_REQUEST_JOB_FILTERED_BYTES_READ,
650 filtered_data_len, filtered_read_buffer_->data());
653 // we are done, or there is no data left.
660 // When we successfully finished a read, we no longer need to save the
661 // caller's buffers. Release our reference.
662 filtered_read_buffer_ = NULL;
663 filtered_read_buffer_len_ = 0;
668 void URLRequestJob::DestroyFilters() {
672 const URLRequestStatus URLRequestJob::GetStatus() {
674 return request_->status();
675 // If the request is gone, we must be cancelled.
676 return URLRequestStatus(URLRequestStatus::CANCELED,
680 void URLRequestJob::SetStatus(const URLRequestStatus &status) {
682 request_->set_status(status);
685 bool URLRequestJob::ReadRawDataForFilter(int* bytes_read) {
689 DCHECK(filter_.get());
693 // Get more pre-filtered data if needed.
694 // TODO(mbelshe): is it possible that the filter needs *MORE* data
695 // when there is some data already in the buffer?
696 if (!filter_->stream_data_len() && !is_done()) {
697 IOBuffer* stream_buffer = filter_->stream_buffer();
698 int stream_buffer_size = filter_->stream_buffer_size();
699 rv = ReadRawDataHelper(stream_buffer, stream_buffer_size, bytes_read);
704 bool URLRequestJob::ReadRawDataHelper(IOBuffer* buf, int buf_size,
706 DCHECK(!request_->status().is_io_pending());
707 DCHECK(raw_read_buffer_.get() == NULL);
709 // Keep a pointer to the read buffer, so we have access to it in the
710 // OnRawReadComplete() callback in the event that the read completes
712 raw_read_buffer_ = buf;
713 bool rv = ReadRawData(buf, buf_size, bytes_read);
715 if (!request_->status().is_io_pending()) {
716 // If the read completes synchronously, either success or failure,
717 // invoke the OnRawReadComplete callback so we can account for the
719 OnRawReadComplete(*bytes_read);
724 void URLRequestJob::FollowRedirect(const GURL& location, int http_status_code) {
725 int rv = request_->Redirect(location, http_status_code);
727 NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, rv));
730 void URLRequestJob::OnRawReadComplete(int bytes_read) {
731 DCHECK(raw_read_buffer_.get());
732 // If |filter_| is non-NULL, bytes will be logged after it is applied instead.
733 if (!filter_.get() && request() && request()->net_log().IsLoggingBytes() &&
735 request()->net_log().AddByteTransferEvent(
736 NetLog::TYPE_URL_REQUEST_JOB_BYTES_READ,
737 bytes_read, raw_read_buffer_->data());
740 if (bytes_read > 0) {
741 RecordBytesRead(bytes_read);
743 raw_read_buffer_ = NULL;
746 void URLRequestJob::RecordBytesRead(int bytes_read) {
747 filter_input_byte_count_ += bytes_read;
748 prefilter_bytes_read_ += bytes_read;
750 postfilter_bytes_read_ += bytes_read;
751 DVLOG(2) << __FUNCTION__ << "() "
752 << "\"" << (request_ ? request_->url().spec() : "???") << "\""
753 << " pre bytes read = " << bytes_read
754 << " pre total = " << prefilter_bytes_read_
755 << " post total = " << postfilter_bytes_read_;
756 UpdatePacketReadTimes(); // Facilitate stats recording if it is active.
757 if (network_delegate_)
758 network_delegate_->NotifyRawBytesRead(*request_, bytes_read);
761 bool URLRequestJob::FilterHasData() {
762 return filter_.get() && filter_->stream_data_len();
765 void URLRequestJob::UpdatePacketReadTimes() {