1 /****************************************************************************
3 ** Copyright (C) 2012 Nokia Corporation and/or its subsidiary(-ies).
4 ** Contact: http://www.qt-project.org/
6 ** This file is part of the QtNetwork module of the Qt Toolkit.
8 ** $QT_BEGIN_LICENSE:LGPL$
9 ** GNU Lesser General Public License Usage
10 ** This file may be used under the terms of the GNU Lesser General Public
11 ** License version 2.1 as published by the Free Software Foundation and
12 ** appearing in the file LICENSE.LGPL included in the packaging of this
13 ** file. Please review the following information to ensure the GNU Lesser
14 ** General Public License version 2.1 requirements will be met:
15 ** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
17 ** In addition, as a special exception, Nokia gives you certain additional
18 ** rights. These rights are described in the Nokia Qt LGPL Exception
19 ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
21 ** GNU General Public License Usage
22 ** Alternatively, this file may be used under the terms of the GNU General
23 ** Public License version 3.0 as published by the Free Software Foundation
24 ** and appearing in the file LICENSE.GPL included in the packaging of this
25 ** file. Please review the following information to ensure the GNU General
26 ** Public License version 3.0 requirements will be met:
27 ** http://www.gnu.org/copyleft/gpl.html.
30 ** Alternatively, this file may be used in accordance with the terms and
31 ** conditions contained in a signed written agreement between you and Nokia.
40 ****************************************************************************/
42 //#define QNETWORKACCESSHTTPBACKEND_DEBUG
44 #include "qnetworkreplyhttpimpl_p.h"
45 #include "qnetworkaccessmanager_p.h"
46 #include "qnetworkaccesscache_p.h"
47 #include "qabstractnetworkcache.h"
48 #include "qnetworkrequest.h"
49 #include "qnetworkreply.h"
50 #include "qnetworkrequest_p.h"
51 #include "qnetworkcookie.h"
52 #include "qnetworkcookie_p.h"
53 #include "QtCore/qdatetime.h"
54 #include "QtCore/qelapsedtimer.h"
55 #include "QtNetwork/qsslconfiguration.h"
56 #include "qhttpthreaddelegate_p.h"
58 #include "QtCore/qcoreapplication.h"
60 #include "qnetworkcookiejar.h"
64 #include <string.h> // for strchr
66 Q_DECLARE_METATYPE(QSharedPointer<char>)
72 static inline bool isSeparator(register char c)
74 static const char separators[] = "()<>@,;:\\\"/[]?={}";
75 return isLWS(c) || strchr(separators, c) != 0;
78 // ### merge with nextField in cookiejar.cpp
79 static QHash<QByteArray, QByteArray> parseHttpOptionHeader(const QByteArray &header)
81 // The HTTP header is of the form:
82 // header = #1(directives)
83 // directives = token | value-directive
84 // value-directive = token "=" (token | quoted-string)
85 QHash<QByteArray, QByteArray> result;
90 pos = nextNonWhitespace(header, pos);
91 if (pos == header.length())
92 return result; // end of parsing
94 // pos points to a non-whitespace
95 int comma = header.indexOf(',', pos);
96 int equal = header.indexOf('=', pos);
97 if (comma == pos || equal == pos)
98 // huh? Broken header.
101 // The key name is delimited by either a comma, an equal sign or the end
102 // of the header, whichever comes first
105 end = header.length();
106 if (equal != -1 && end > equal)
107 end = equal; // equal sign comes before comma/end
108 QByteArray key = QByteArray(header.constData() + pos, end - pos).trimmed().toLower();
111 if (uint(equal) < uint(comma)) {
112 // case: token "=" (token | quoted-string)
114 pos = nextNonWhitespace(header, pos);
115 if (pos == header.length())
116 // huh? Broken header
120 value.reserve(header.length() - pos);
121 if (header.at(pos) == '"') {
122 // case: quoted-string
123 // quoted-string = ( <"> *(qdtext | quoted-pair ) <"> )
124 // qdtext = <any TEXT except <">>
125 // quoted-pair = "\" CHAR
127 while (pos < header.length()) {
128 register char c = header.at(pos);
130 // end of quoted text
132 } else if (c == '\\') {
134 if (pos >= header.length())
145 while (pos < header.length()) {
146 register char c = header.at(pos);
154 result.insert(key, value);
156 // find the comma now:
157 comma = header.indexOf(',', pos);
159 return result; // end of parsing
163 // key is already set
164 result.insert(key, QByteArray());
169 QNetworkReplyHttpImpl::QNetworkReplyHttpImpl(QNetworkAccessManager* const manager,
170 const QNetworkRequest& request,
171 QNetworkAccessManager::Operation& operation,
172 QIODevice* outgoingData)
173 : QNetworkReply(*new QNetworkReplyHttpImplPrivate, manager)
175 Q_D(QNetworkReplyHttpImpl);
176 d->manager = manager;
177 d->managerPrivate = manager->d_func();
178 d->request = request;
179 d->operation = operation;
180 d->outgoingData = outgoingData;
181 d->url = request.url();
183 d->sslConfiguration = request.sslConfiguration();
186 // FIXME Later maybe set to Unbuffered, especially if it is zerocopy or from cache?
187 QIODevice::open(QIODevice::ReadOnly);
190 // Internal code that does a HTTP reply for the synchronous Ajax
192 QVariant synchronousHttpAttribute = request.attribute(
193 static_cast<QNetworkRequest::Attribute>(QNetworkRequest::SynchronousRequestAttribute));
194 if (synchronousHttpAttribute.isValid()) {
195 d->synchronous = synchronousHttpAttribute.toBool();
196 if (d->synchronous && outgoingData) {
197 // The synchronous HTTP is a corner case, we will put all upload data in one big QByteArray in the outgoingDataBuffer.
198 // Yes, this is not the most efficient thing to do, but on the other hand synchronous XHR needs to die anyway.
199 d->outgoingDataBuffer = QSharedPointer<QRingBuffer>(new QRingBuffer());
200 qint64 previousDataSize = 0;
202 previousDataSize = d->outgoingDataBuffer->size();
203 d->outgoingDataBuffer->append(d->outgoingData->readAll());
204 } while (d->outgoingDataBuffer->size() != previousDataSize);
205 d->_q_startOperation();
212 // there is data to be uploaded, e.g. HTTP POST.
214 if (!d->outgoingData->isSequential()) {
215 // fixed size non-sequential (random-access)
216 // just start the operation
217 QMetaObject::invokeMethod(this, "_q_startOperation", Qt::QueuedConnection);
218 // FIXME make direct call?
220 bool bufferingDisallowed =
221 request.attribute(QNetworkRequest::DoNotBufferUploadDataAttribute,
224 if (bufferingDisallowed) {
225 // if a valid content-length header for the request was supplied, we can disable buffering
226 // if not, we will buffer anyway
227 if (request.header(QNetworkRequest::ContentLengthHeader).isValid()) {
228 QMetaObject::invokeMethod(this, "_q_startOperation", Qt::QueuedConnection);
229 // FIXME make direct call?
231 d->state = d->Buffering;
232 QMetaObject::invokeMethod(this, "_q_bufferOutgoingData", Qt::QueuedConnection);
235 // _q_startOperation will be called when the buffering has finished.
236 d->state = d->Buffering;
237 QMetaObject::invokeMethod(this, "_q_bufferOutgoingData", Qt::QueuedConnection);
241 // No outgoing data (POST, ..)
242 d->_q_startOperation();
246 QNetworkReplyHttpImpl::~QNetworkReplyHttpImpl()
248 // Most work is done in private destructor
251 void QNetworkReplyHttpImpl::close()
253 Q_D(QNetworkReplyHttpImpl);
255 if (d->state == QNetworkReplyHttpImplPrivate::Aborted ||
256 d->state == QNetworkReplyHttpImplPrivate::Finished)
259 // According to the documentation close only stops the download
260 // by closing we can ignore the download part and continue uploading.
261 QNetworkReply::close();
263 // call finished which will emit signals
264 // FIXME shouldn't this be emitted Queued?
265 d->error(OperationCanceledError, tr("Operation canceled"));
269 void QNetworkReplyHttpImpl::abort()
271 Q_D(QNetworkReplyHttpImpl);
273 if (d->state == QNetworkReplyHttpImplPrivate::Finished || d->state == QNetworkReplyHttpImplPrivate::Aborted)
276 QNetworkReply::close();
278 if (d->state != QNetworkReplyHttpImplPrivate::Finished) {
279 // call finished which will emit signals
280 // FIXME shouldn't this be emitted Queued?
281 d->error(OperationCanceledError, tr("Operation canceled"));
285 d->state = QNetworkReplyHttpImplPrivate::Aborted;
287 emit abortHttpRequest();
290 qint64 QNetworkReplyHttpImpl::bytesAvailable() const
292 Q_D(const QNetworkReplyHttpImpl);
294 // if we load from cache device
295 if (d->cacheLoadDevice) {
296 return QNetworkReply::bytesAvailable() + d->cacheLoadDevice->bytesAvailable() + d->downloadMultiBuffer.byteAmount();
300 if (d->downloadZerocopyBuffer) {
301 return QNetworkReply::bytesAvailable() + d->downloadBufferCurrentSize - d->downloadBufferReadPosition;
305 return QNetworkReply::bytesAvailable() + d->downloadMultiBuffer.byteAmount();
308 bool QNetworkReplyHttpImpl::isSequential () const
310 // FIXME In the cache of a cached load or the zero-copy buffer we could actually be non-sequential.
311 // FIXME however this requires us to implement stuff like seek() too.
315 qint64 QNetworkReplyHttpImpl::size() const
317 // FIXME At some point, this could return a proper value, e.g. if we're non-sequential.
318 return QNetworkReply::size();
321 qint64 QNetworkReplyHttpImpl::readData(char* data, qint64 maxlen)
323 Q_D(QNetworkReplyHttpImpl);
326 if (d->cacheLoadDevice) {
327 // FIXME bytesdownloaded, position etc?
329 // There is something already in the buffer we buffered before because the user did not read()
330 // anything, so we read there first:
331 if (!d->downloadMultiBuffer.isEmpty()) {
332 return d->downloadMultiBuffer.read(data, maxlen);
335 qint64 ret = d->cacheLoadDevice->read(data, maxlen);
340 if (d->downloadZerocopyBuffer) {
341 // FIXME bytesdownloaded, position etc?
343 qint64 howMuch = qMin(maxlen, (d->downloadBufferCurrentSize - d->downloadBufferReadPosition));
344 memcpy(data, d->downloadZerocopyBuffer + d->downloadBufferReadPosition, howMuch);
345 d->downloadBufferReadPosition += howMuch;
351 if (d->downloadMultiBuffer.isEmpty()) {
352 if (d->state == d->Finished || d->state == d->Aborted)
358 // optimization for getChar()
359 *data = d->downloadMultiBuffer.getChar();
360 if (readBufferSize())
361 emit readBufferFreed(1);
365 maxlen = qMin<qint64>(maxlen, d->downloadMultiBuffer.byteAmount());
366 qint64 bytesRead = d->downloadMultiBuffer.read(data, maxlen);
367 if (readBufferSize())
368 emit readBufferFreed(bytesRead);
372 void QNetworkReplyHttpImpl::setReadBufferSize(qint64 size)
374 QNetworkReply::setReadBufferSize(size);
375 emit readBufferSizeChanged(size);
379 bool QNetworkReplyHttpImpl::canReadLine () const
381 Q_D(const QNetworkReplyHttpImpl);
383 if (QNetworkReply::canReadLine())
386 if (d->cacheLoadDevice)
387 return d->cacheLoadDevice->canReadLine() || d->downloadMultiBuffer.canReadLine();
389 if (d->downloadZerocopyBuffer)
390 return memchr(d->downloadZerocopyBuffer + d->downloadBufferReadPosition, '\n', d->downloadBufferCurrentSize - d->downloadBufferReadPosition);
392 return d->downloadMultiBuffer.canReadLine();
396 void QNetworkReplyHttpImpl::ignoreSslErrors()
398 Q_D(QNetworkReplyHttpImpl);
400 d->pendingIgnoreAllSslErrors = true;
403 void QNetworkReplyHttpImpl::ignoreSslErrorsImplementation(const QList<QSslError> &errors)
405 Q_D(QNetworkReplyHttpImpl);
407 // the pending list is set if QNetworkReply::ignoreSslErrors(const QList<QSslError> &errors)
408 // is called before QNetworkAccessManager::get() (or post(), etc.)
409 d->pendingIgnoreSslErrorsList = errors;
412 void QNetworkReplyHttpImpl::setSslConfigurationImplementation(const QSslConfiguration &newconfig)
414 // Setting a SSL configuration on a reply is not supported. The user needs to set
415 // her/his QSslConfiguration on the QNetworkRequest.
419 void QNetworkReplyHttpImpl::sslConfigurationImplementation(QSslConfiguration &configuration) const
421 Q_D(const QNetworkReplyHttpImpl);
422 configuration = d->sslConfiguration;
426 QNetworkReplyHttpImplPrivate::QNetworkReplyHttpImplPrivate()
427 : QNetworkReplyPrivate()
436 , loadingFromCache(false)
438 , cacheEnabled(false)
440 , preMigrationDownloaded(-1)
442 , downloadBufferReadPosition(0)
443 , downloadBufferCurrentSize(0)
444 , downloadZerocopyBuffer(0)
445 , pendingDownloadDataEmissions(new QAtomicInt())
446 , pendingDownloadProgressEmissions(new QAtomicInt())
448 , pendingIgnoreAllSslErrors(false)
454 QNetworkReplyHttpImplPrivate::~QNetworkReplyHttpImplPrivate()
456 Q_Q(QNetworkReplyHttpImpl);
457 // This will do nothing if the request was already finished or aborted
458 emit q->abortHttpRequest();
462 For a given httpRequest
463 1) If AlwaysNetwork, return
464 2) If we have a cache entry for this url populate headers so the server can return 304
465 3) Calculate if response_is_fresh and if so send the cache and set loadedFromCache to true
467 bool QNetworkReplyHttpImplPrivate::loadFromCacheIfAllowed(QHttpNetworkRequest &httpRequest)
469 QNetworkRequest::CacheLoadControl CacheLoadControlAttribute =
470 (QNetworkRequest::CacheLoadControl)request.attribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferNetwork).toInt();
471 if (CacheLoadControlAttribute == QNetworkRequest::AlwaysNetwork) {
472 // If the request does not already specify preferred cache-control
473 // force reload from the network and tell any caching proxy servers to reload too
474 if (!request.rawHeaderList().contains("Cache-Control")) {
475 httpRequest.setHeaderField("Cache-Control", "no-cache");
476 httpRequest.setHeaderField("Pragma", "no-cache");
481 // The disk cache API does not currently support partial content retrieval.
482 // That is why we don't use the disk cache for any such requests.
483 if (request.hasRawHeader("Range"))
486 QAbstractNetworkCache *nc = managerPrivate->networkCache;
488 return false; // no local cache
490 QNetworkCacheMetaData metaData = nc->metaData(request.url());
491 if (!metaData.isValid())
492 return false; // not in cache
494 if (!metaData.saveToDisk())
497 QNetworkHeadersPrivate cacheHeaders;
498 QNetworkHeadersPrivate::RawHeadersList::ConstIterator it;
499 cacheHeaders.setAllRawHeaders(metaData.rawHeaders());
501 it = cacheHeaders.findRawHeader("etag");
502 if (it != cacheHeaders.rawHeaders.constEnd())
503 httpRequest.setHeaderField("If-None-Match", it->second);
505 QDateTime lastModified = metaData.lastModified();
506 if (lastModified.isValid())
507 httpRequest.setHeaderField("If-Modified-Since", QNetworkHeadersPrivate::toHttpDate(lastModified));
509 it = cacheHeaders.findRawHeader("Cache-Control");
510 if (it != cacheHeaders.rawHeaders.constEnd()) {
511 QHash<QByteArray, QByteArray> cacheControl = parseHttpOptionHeader(it->second);
512 if (cacheControl.contains("must-revalidate"))
516 QDateTime currentDateTime = QDateTime::currentDateTime();
517 QDateTime expirationDate = metaData.expirationDate();
519 bool response_is_fresh;
520 if (!expirationDate.isValid()) {
523 * is the value of Age: header received by the cache with
526 * is the value of the origin server's Date: header
528 * is the (local) time when the cache made the request
529 * that resulted in this cached response
531 * is the (local) time when the cache received the
534 * is the current (local) time
537 it = cacheHeaders.findRawHeader("age");
538 if (it != cacheHeaders.rawHeaders.constEnd())
539 age_value = it->second.toInt();
541 QDateTime dateHeader;
543 it = cacheHeaders.findRawHeader("date");
544 if (it != cacheHeaders.rawHeaders.constEnd()) {
545 dateHeader = QNetworkHeadersPrivate::fromHttpDate(it->second);
546 date_value = dateHeader.toTime_t();
549 int now = currentDateTime.toUTC().toTime_t();
550 int request_time = now;
551 int response_time = now;
553 // Algorithm from RFC 2616 section 13.2.3
554 int apparent_age = qMax(0, response_time - date_value);
555 int corrected_received_age = qMax(apparent_age, age_value);
556 int response_delay = response_time - request_time;
557 int corrected_initial_age = corrected_received_age + response_delay;
558 int resident_time = now - response_time;
559 int current_age = corrected_initial_age + resident_time;
561 // RFC 2616 13.2.4 Expiration Calculations
562 if (!expirationDate.isValid()) {
563 if (lastModified.isValid()) {
564 int diff = currentDateTime.secsTo(lastModified);
565 expirationDate = lastModified;
566 expirationDate.addSecs(diff / 10);
567 if (httpRequest.headerField("Warning").isEmpty()) {
569 dt.setTime_t(current_age);
570 if (dt.daysTo(currentDateTime) > 1)
571 httpRequest.setHeaderField("Warning", "113");
576 // the cache-saving code below sets the expirationDate with date+max_age
577 // if "max-age" is present, or to Expires otherwise
578 int freshness_lifetime = dateHeader.secsTo(expirationDate);
579 response_is_fresh = (freshness_lifetime > current_age);
581 // expiration date was calculated earlier (e.g. when storing object to the cache)
582 response_is_fresh = currentDateTime.secsTo(expirationDate) >= 0;
585 if (!response_is_fresh)
588 #if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
589 qDebug() << "response_is_fresh" << CacheLoadControlAttribute;
591 return sendCacheContents(metaData);
594 QHttpNetworkRequest::Priority QNetworkReplyHttpImplPrivate::convert(const QNetworkRequest::Priority& prio)
597 case QNetworkRequest::LowPriority:
598 return QHttpNetworkRequest::LowPriority;
599 case QNetworkRequest::HighPriority:
600 return QHttpNetworkRequest::HighPriority;
601 case QNetworkRequest::NormalPriority:
603 return QHttpNetworkRequest::NormalPriority;
607 void QNetworkReplyHttpImplPrivate::postRequest()
609 Q_Q(QNetworkReplyHttpImpl);
613 // A synchronous HTTP request uses its own thread
614 thread = new QThread();
615 thread->setObjectName(QStringLiteral("httpReply"));
616 QObject::connect(thread, SIGNAL(finished()), thread, SLOT(deleteLater()));
618 } else if (!managerPrivate->httpThread) {
619 // We use the manager-global thread.
620 // At some point we could switch to having multiple threads if it makes sense.
621 managerPrivate->httpThread = new QThread();
622 managerPrivate->httpThread->setObjectName(QStringLiteral("httpThread"));
623 QObject::connect(managerPrivate->httpThread, SIGNAL(finished()), managerPrivate->httpThread, SLOT(deleteLater()));
624 managerPrivate->httpThread->start();
626 thread = managerPrivate->httpThread;
628 // Asynchronous request, thread already exists
629 thread = managerPrivate->httpThread;
632 QUrl url = request.url();
633 httpRequest.setUrl(url);
635 bool ssl = url.scheme().toLower() == QLatin1String("https");
636 q->setAttribute(QNetworkRequest::ConnectionEncryptedAttribute, ssl);
637 httpRequest.setSsl(ssl);
640 #ifndef QT_NO_NETWORKPROXY
641 QNetworkProxy transparentProxy, cacheProxy;
643 // FIXME the proxy stuff should be done in the HTTP thread
644 foreach (const QNetworkProxy &p, managerPrivate->queryProxy(QNetworkProxyQuery(request.url()))) {
645 // use the first proxy that works
646 // for non-encrypted connections, any transparent or HTTP proxy
647 // for encrypted, only transparent proxies
649 && (p.capabilities() & QNetworkProxy::CachingCapability)
650 && (p.type() == QNetworkProxy::HttpProxy ||
651 p.type() == QNetworkProxy::HttpCachingProxy)) {
653 transparentProxy = QNetworkProxy::NoProxy;
656 if (p.isTransparentProxy()) {
657 transparentProxy = p;
658 cacheProxy = QNetworkProxy::NoProxy;
663 // check if at least one of the proxies
664 if (transparentProxy.type() == QNetworkProxy::DefaultProxy &&
665 cacheProxy.type() == QNetworkProxy::DefaultProxy) {
666 // unsuitable proxies
667 QMetaObject::invokeMethod(q, "_q_error", synchronous ? Qt::DirectConnection : Qt::QueuedConnection,
668 Q_ARG(QNetworkReply::NetworkError, QNetworkReply::ProxyNotFoundError),
669 Q_ARG(QString, q->tr("No suitable proxy found")));
670 QMetaObject::invokeMethod(q, "_q_finished", synchronous ? Qt::DirectConnection : Qt::QueuedConnection);
676 bool loadedFromCache = false;
677 httpRequest.setPriority(convert(request.priority()));
680 case QNetworkAccessManager::GetOperation:
681 httpRequest.setOperation(QHttpNetworkRequest::Get);
682 loadedFromCache = loadFromCacheIfAllowed(httpRequest);
685 case QNetworkAccessManager::HeadOperation:
686 httpRequest.setOperation(QHttpNetworkRequest::Head);
687 loadedFromCache = loadFromCacheIfAllowed(httpRequest);
690 case QNetworkAccessManager::PostOperation:
692 httpRequest.setOperation(QHttpNetworkRequest::Post);
693 createUploadByteDevice();
696 case QNetworkAccessManager::PutOperation:
698 httpRequest.setOperation(QHttpNetworkRequest::Put);
699 createUploadByteDevice();
702 case QNetworkAccessManager::DeleteOperation:
704 httpRequest.setOperation(QHttpNetworkRequest::Delete);
707 case QNetworkAccessManager::CustomOperation:
708 invalidateCache(); // for safety reasons, we don't know what the operation does
709 httpRequest.setOperation(QHttpNetworkRequest::Custom);
710 createUploadByteDevice();
711 httpRequest.setCustomVerb(request.attribute(
712 QNetworkRequest::CustomVerbAttribute).toByteArray());
716 break; // can't happen
719 if (loadedFromCache) {
720 return; // no need to send the request! :)
723 QList<QByteArray> headers = request.rawHeaderList();
724 if (resumeOffset != 0) {
725 if (headers.contains("Range")) {
726 // Need to adjust resume offset for user specified range
728 headers.removeOne("Range");
730 // We've already verified that requestRange starts with "bytes=", see canResume.
731 QByteArray requestRange = request.rawHeader("Range").mid(6);
733 int index = requestRange.indexOf('-');
735 quint64 requestStartOffset = requestRange.left(index).toULongLong();
736 quint64 requestEndOffset = requestRange.mid(index + 1).toULongLong();
738 requestRange = "bytes=" + QByteArray::number(resumeOffset + requestStartOffset) +
739 '-' + QByteArray::number(requestEndOffset);
741 httpRequest.setHeaderField("Range", requestRange);
743 httpRequest.setHeaderField("Range", "bytes=" + QByteArray::number(resumeOffset) + '-');
747 foreach (const QByteArray &header, headers)
748 httpRequest.setHeaderField(header, request.rawHeader(header));
750 if (request.attribute(QNetworkRequest::HttpPipeliningAllowedAttribute).toBool() == true)
751 httpRequest.setPipeliningAllowed(true);
753 if (static_cast<QNetworkRequest::LoadControl>
754 (request.attribute(QNetworkRequest::AuthenticationReuseAttribute,
755 QNetworkRequest::Automatic).toInt()) == QNetworkRequest::Manual)
756 httpRequest.setWithCredentials(false);
759 // Create the HTTP thread delegate
760 QHttpThreadDelegate *delegate = new QHttpThreadDelegate;
761 #ifndef QT_NO_BEARERMANAGEMENT
762 delegate->networkSession = managerPrivate->getNetworkSession();
765 // For the synchronous HTTP, this is the normal way the delegate gets deleted
766 // For the asynchronous HTTP this is a safety measure, the delegate deletes itself when HTTP is finished
767 QObject::connect(thread, SIGNAL(finished()), delegate, SLOT(deleteLater()));
769 // Set the properties it needs
770 delegate->httpRequest = httpRequest;
771 #ifndef QT_NO_NETWORKPROXY
772 delegate->cacheProxy = cacheProxy;
773 delegate->transparentProxy = transparentProxy;
778 delegate->incomingSslConfiguration = request.sslConfiguration();
781 // Do we use synchronous HTTP?
782 delegate->synchronous = synchronous;
784 // The authentication manager is used to avoid the BlockingQueuedConnection communication
785 // from HTTP thread to user thread in some cases.
786 delegate->authenticationManager = managerPrivate->authenticationManager;
789 // Tell our zerocopy policy to the delegate
790 QVariant downloadBufferMaximumSizeAttribute = request.attribute(QNetworkRequest::MaximumDownloadBufferSizeAttribute);
791 if (downloadBufferMaximumSizeAttribute.isValid()) {
792 delegate->downloadBufferMaximumSize = downloadBufferMaximumSizeAttribute.toLongLong();
794 // If there is no MaximumDownloadBufferSizeAttribute set (which is for the majority
795 // of QNetworkRequest) then we can assume we'll do it anyway for small HTTP replies.
796 // This helps with performance and memory fragmentation.
797 delegate->downloadBufferMaximumSize = 128*1024;
801 // These atomic integers are used for signal compression
802 delegate->pendingDownloadData = pendingDownloadDataEmissions;
803 delegate->pendingDownloadProgress = pendingDownloadProgressEmissions;
805 // Connect the signals of the delegate to us
806 QObject::connect(delegate, SIGNAL(downloadData(QByteArray)),
807 q, SLOT(replyDownloadData(QByteArray)),
808 Qt::QueuedConnection);
809 QObject::connect(delegate, SIGNAL(downloadFinished()),
810 q, SLOT(replyFinished()),
811 Qt::QueuedConnection);
812 QObject::connect(delegate, SIGNAL(downloadMetaData(QList<QPair<QByteArray,QByteArray> >,int,QString,bool,QSharedPointer<char>,qint64)),
813 q, SLOT(replyDownloadMetaData(QList<QPair<QByteArray,QByteArray> >,int,QString,bool,QSharedPointer<char>,qint64)),
814 Qt::QueuedConnection);
815 QObject::connect(delegate, SIGNAL(downloadProgress(qint64,qint64)),
816 q, SLOT(replyDownloadProgressSlot(qint64,qint64)),
817 Qt::QueuedConnection);
818 QObject::connect(delegate, SIGNAL(error(QNetworkReply::NetworkError,QString)),
819 q, SLOT(httpError(QNetworkReply::NetworkError, const QString)),
820 Qt::QueuedConnection);
822 QObject::connect(delegate, SIGNAL(sslConfigurationChanged(QSslConfiguration)),
823 q, SLOT(replySslConfigurationChanged(QSslConfiguration)),
824 Qt::QueuedConnection);
826 // Those need to report back, therefire BlockingQueuedConnection
827 QObject::connect(delegate, SIGNAL(authenticationRequired(QHttpNetworkRequest,QAuthenticator*)),
828 q, SLOT(httpAuthenticationRequired(QHttpNetworkRequest,QAuthenticator*)),
829 Qt::BlockingQueuedConnection);
830 #ifndef QT_NO_NETWORKPROXY
831 QObject::connect(delegate, SIGNAL(proxyAuthenticationRequired(QNetworkProxy,QAuthenticator*)),
832 q, SLOT(proxyAuthenticationRequired(QNetworkProxy,QAuthenticator*)),
833 Qt::BlockingQueuedConnection);
836 QObject::connect(delegate, SIGNAL(sslErrors(QList<QSslError>,bool*,QList<QSslError>*)),
837 q, SLOT(replySslErrors(const QList<QSslError> &, bool *, QList<QSslError> *)),
838 Qt::BlockingQueuedConnection);
840 // This signal we will use to start the request.
841 QObject::connect(q, SIGNAL(startHttpRequest()), delegate, SLOT(startRequest()));
842 QObject::connect(q, SIGNAL(abortHttpRequest()), delegate, SLOT(abortRequest()));
844 // To throttle the connection.
845 QObject::connect(q, SIGNAL(readBufferSizeChanged(qint64)), delegate, SLOT(readBufferSizeChanged(qint64)));
846 QObject::connect(q, SIGNAL(readBufferFreed(qint64)), delegate, SLOT(readBufferFreed(qint64)));
848 if (uploadByteDevice) {
849 QNonContiguousByteDeviceThreadForwardImpl *forwardUploadDevice =
850 new QNonContiguousByteDeviceThreadForwardImpl(uploadByteDevice->atEnd(), uploadByteDevice->size());
851 if (uploadByteDevice->isResetDisabled())
852 forwardUploadDevice->disableReset();
853 forwardUploadDevice->setParent(delegate); // needed to make sure it is moved on moveToThread()
854 delegate->httpRequest.setUploadByteDevice(forwardUploadDevice);
856 // From main thread to user thread:
857 QObject::connect(q, SIGNAL(haveUploadData(QByteArray, bool, qint64)),
858 forwardUploadDevice, SLOT(haveDataSlot(QByteArray, bool, qint64)), Qt::QueuedConnection);
859 QObject::connect(uploadByteDevice.data(), SIGNAL(readyRead()),
860 forwardUploadDevice, SIGNAL(readyRead()),
861 Qt::QueuedConnection);
863 // From http thread to user thread:
864 QObject::connect(forwardUploadDevice, SIGNAL(wantData(qint64)),
865 q, SLOT(wantUploadDataSlot(qint64)));
866 QObject::connect(forwardUploadDevice, SIGNAL(processedData(qint64)),
867 q, SLOT(sentUploadDataSlot(qint64)));
868 QObject::connect(forwardUploadDevice, SIGNAL(resetData(bool*)),
869 q, SLOT(resetUploadDataSlot(bool*)),
870 Qt::BlockingQueuedConnection); // this is the only one with BlockingQueued!
872 } else if (synchronous) {
873 QObject::connect(q, SIGNAL(startHttpRequestSynchronously()), delegate, SLOT(startRequestSynchronously()), Qt::BlockingQueuedConnection);
875 if (uploadByteDevice) {
876 // For the synchronous HTTP use case the use thread (this one here) is blocked
877 // so we cannot use the asynchronous upload architecture.
878 // We therefore won't use the QNonContiguousByteDeviceThreadForwardImpl but directly
879 // use the uploadByteDevice provided to us by the QNetworkReplyImpl.
880 // The code that is in start() makes sure it is safe to use from a thread
881 // since it only wraps a QRingBuffer
882 delegate->httpRequest.setUploadByteDevice(uploadByteDevice.data());
887 // Move the delegate to the http thread
888 delegate->moveToThread(thread);
889 // This call automatically moves the uploadDevice too for the asynchronous case.
891 // Prepare timers for progress notifications
892 downloadProgressSignalChoke.start();
893 uploadProgressSignalChoke.invalidate();
895 // Send an signal to the delegate so it starts working in the other thread
897 emit q->startHttpRequestSynchronously(); // This one is BlockingQueuedConnection, so it will return when all work is done
899 if (delegate->incomingErrorCode != QNetworkReply::NoError) {
900 replyDownloadMetaData
901 (delegate->incomingHeaders,
902 delegate->incomingStatusCode,
903 delegate->incomingReasonPhrase,
904 delegate->isPipeliningUsed,
905 QSharedPointer<char>(),
906 delegate->incomingContentLength);
907 replyDownloadData(delegate->synchronousDownloadData);
908 httpError(delegate->incomingErrorCode, delegate->incomingErrorDetail);
910 replyDownloadMetaData
911 (delegate->incomingHeaders,
912 delegate->incomingStatusCode,
913 delegate->incomingReasonPhrase,
914 delegate->isPipeliningUsed,
915 QSharedPointer<char>(),
916 delegate->incomingContentLength);
917 replyDownloadData(delegate->synchronousDownloadData);
920 // End the thread. It will delete itself from the finished() signal
926 emit q->startHttpRequest(); // Signal to the HTTP thread and go back to user.
930 void QNetworkReplyHttpImplPrivate::invalidateCache()
932 QAbstractNetworkCache *nc = managerPrivate->networkCache;
934 nc->remove(request.url());
937 void QNetworkReplyHttpImplPrivate::initCacheSaveDevice()
939 Q_Q(QNetworkReplyHttpImpl);
941 // The disk cache does not support partial content, so don't even try to
942 // save any such content into the cache.
943 if (q->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt() == 206) {
944 cacheEnabled = false;
948 // save the meta data
949 QNetworkCacheMetaData metaData;
950 metaData.setUrl(url);
951 metaData = fetchCacheMetaData(metaData);
953 // save the redirect request also in the cache
954 QVariant redirectionTarget = q->attribute(QNetworkRequest::RedirectionTargetAttribute);
955 if (redirectionTarget.isValid()) {
956 QNetworkCacheMetaData::AttributesMap attributes = metaData.attributes();
957 attributes.insert(QNetworkRequest::RedirectionTargetAttribute, redirectionTarget);
958 metaData.setAttributes(attributes);
961 cacheSaveDevice = managerPrivate->networkCache->prepare(metaData);
963 if (!cacheSaveDevice || (cacheSaveDevice && !cacheSaveDevice->isOpen())) {
964 if (cacheSaveDevice && !cacheSaveDevice->isOpen())
965 qCritical("QNetworkReplyImpl: network cache returned a device that is not open -- "
966 "class %s probably needs to be fixed",
967 managerPrivate->networkCache->metaObject()->className());
969 managerPrivate->networkCache->remove(url);
971 cacheEnabled = false;
975 void QNetworkReplyHttpImplPrivate::replyDownloadData(QByteArray d)
977 Q_Q(QNetworkReplyHttpImpl);
979 // If we're closed just ignore this data
983 int pendingSignals = (int)pendingDownloadDataEmissions->fetchAndAddAcquire(-1) - 1;
985 if (pendingSignals > 0) {
986 // Some more signal emissions to this slot are pending.
987 // Instead of writing the downstream data, we wait
988 // and do it in the next call we get
989 // (signal comppression)
990 pendingDownloadData.append(d);
994 pendingDownloadData.append(d);
996 // We need to usa a copy for calling writeDownstreamData as we could
997 // possibly recurse into this this function when we call
998 // appendDownstreamDataSignalEmissions because the user might call
999 // processEvents() or spin an event loop when this occur.
1000 QByteDataBuffer pendingDownloadDataCopy = pendingDownloadData;
1001 pendingDownloadData.clear();
1003 if (cacheEnabled && !cacheSaveDevice) {
1004 initCacheSaveDevice();
1007 qint64 bytesWritten = 0;
1008 for (int i = 0; i < pendingDownloadDataCopy.bufferCount(); i++) {
1009 QByteArray const &item = pendingDownloadDataCopy[i];
1011 if (cacheSaveDevice)
1012 cacheSaveDevice->write(item.constData(), item.size());
1013 downloadMultiBuffer.append(item);
1015 bytesWritten += item.size();
1017 pendingDownloadDataCopy.clear();
1019 bytesDownloaded += bytesWritten;
1022 QVariant totalSize = cookedHeaders.value(QNetworkRequest::ContentLengthHeader);
1023 if (preMigrationDownloaded != Q_INT64_C(-1))
1024 totalSize = totalSize.toLongLong() + preMigrationDownloaded;
1026 emit q->readyRead();
1027 // emit readyRead before downloadProgress incase this will cause events to be
1028 // processed and we get into a recursive call (as in QProgressDialog).
1029 if (downloadProgressSignalChoke.elapsed() >= progressSignalInterval) {
1030 downloadProgressSignalChoke.restart();
1031 emit q->downloadProgress(bytesDownloaded,
1032 totalSize.isNull() ? Q_INT64_C(-1) : totalSize.toLongLong());
1037 void QNetworkReplyHttpImplPrivate::replyFinished()
1039 // We are already loading from cache, we still however
1040 // got this signal because it was posted already
1041 if (loadingFromCache)
1047 void QNetworkReplyHttpImplPrivate::checkForRedirect(const int statusCode)
1049 Q_Q(QNetworkReplyHttpImpl);
1050 switch (statusCode) {
1051 case 301: // Moved Permanently
1053 case 303: // See Other
1054 case 307: // Temporary Redirect
1055 // What do we do about the caching of the HTML note?
1056 // The response to a 303 MUST NOT be cached, while the response to
1057 // all of the others is cacheable if the headers indicate it to be
1058 QByteArray header = q->rawHeader("location");
1059 QUrl url = QUrl(QString::fromUtf8(header));
1061 url = QUrl(QLatin1String(header));
1062 q->setAttribute(QNetworkRequest::RedirectionTargetAttribute, url);
1066 void QNetworkReplyHttpImplPrivate::replyDownloadMetaData
1067 (QList<QPair<QByteArray,QByteArray> > hm,
1068 int sc,QString rp,bool pu,
1069 QSharedPointer<char> db,
1070 qint64 contentLength)
1072 Q_Q(QNetworkReplyHttpImpl);
1073 Q_UNUSED(contentLength);
1080 downloadBufferPointer = db;
1081 downloadZerocopyBuffer = downloadBufferPointer.data();
1082 downloadBufferCurrentSize = 0;
1083 q->setAttribute(QNetworkRequest::DownloadBufferAttribute, QVariant::fromValue<QSharedPointer<char> > (downloadBufferPointer));
1086 q->setAttribute(QNetworkRequest::HttpPipeliningWasUsedAttribute, pu);
1088 // reconstruct the HTTP header
1089 QList<QPair<QByteArray, QByteArray> > headerMap = hm;
1090 QList<QPair<QByteArray, QByteArray> >::ConstIterator it = headerMap.constBegin(),
1091 end = headerMap.constEnd();
1092 for (; it != end; ++it) {
1093 QByteArray value = q->rawHeader(it->first);
1094 if (!value.isEmpty()) {
1095 if (qstricmp(it->first.constData(), "set-cookie") == 0)
1100 value += it->second;
1101 q->setRawHeader(it->first, value);
1104 q->setAttribute(QNetworkRequest::HttpStatusCodeAttribute, statusCode);
1105 q->setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, reasonPhrase);
1107 // is it a redirection?
1108 checkForRedirect(statusCode);
1110 if (statusCode >= 500 && statusCode < 600) {
1111 QAbstractNetworkCache *nc = managerPrivate->networkCache;
1113 QNetworkCacheMetaData metaData = nc->metaData(request.url());
1114 QNetworkHeadersPrivate cacheHeaders;
1115 cacheHeaders.setAllRawHeaders(metaData.rawHeaders());
1116 QNetworkHeadersPrivate::RawHeadersList::ConstIterator it;
1117 it = cacheHeaders.findRawHeader("Cache-Control");
1118 bool mustReValidate = false;
1119 if (it != cacheHeaders.rawHeaders.constEnd()) {
1120 QHash<QByteArray, QByteArray> cacheControl = parseHttpOptionHeader(it->second);
1121 if (cacheControl.contains("must-revalidate"))
1122 mustReValidate = true;
1124 if (!mustReValidate && sendCacheContents(metaData))
1129 if (statusCode == 304) {
1130 #if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
1131 qDebug() << "Received a 304 from" << url();
1133 QAbstractNetworkCache *nc = managerPrivate->networkCache;
1135 QNetworkCacheMetaData oldMetaData = nc->metaData(request.url());
1136 QNetworkCacheMetaData metaData = fetchCacheMetaData(oldMetaData);
1137 if (oldMetaData != metaData)
1138 nc->updateMetaData(metaData);
1139 if (sendCacheContents(metaData))
1145 if (statusCode != 304 && statusCode != 303) {
1146 if (!isCachingEnabled())
1147 setCachingEnabled(true);
1153 void QNetworkReplyHttpImplPrivate::replyDownloadProgressSlot(qint64 bytesReceived, qint64 bytesTotal)
1155 Q_Q(QNetworkReplyHttpImpl);
1157 // If we're closed just ignore this data
1161 // we can be sure here that there is a download buffer
1163 int pendingSignals = (int)pendingDownloadProgressEmissions->fetchAndAddAcquire(-1) - 1;
1164 if (pendingSignals > 0) {
1165 // Let's ignore this signal and look at the next one coming in
1166 // (signal comppression)
1173 if (cacheEnabled && bytesReceived == bytesTotal) {
1174 // Write everything in one go if we use a download buffer. might be more performant.
1175 initCacheSaveDevice();
1176 // need to check again if cache enabled and device exists
1177 if (cacheSaveDevice && cacheEnabled)
1178 cacheSaveDevice->write(downloadZerocopyBuffer, bytesTotal);
1179 // FIXME where is it closed?
1182 bytesDownloaded = bytesReceived;
1184 downloadBufferCurrentSize = bytesReceived;
1186 // Only emit readyRead when actual data is there
1187 // emit readyRead before downloadProgress incase this will cause events to be
1188 // processed and we get into a recursive call (as in QProgressDialog).
1189 if (bytesDownloaded > 0)
1190 emit q->readyRead();
1191 if (downloadProgressSignalChoke.elapsed() >= progressSignalInterval) {
1192 downloadProgressSignalChoke.restart();
1193 emit q->downloadProgress(bytesDownloaded, bytesTotal);
1197 void QNetworkReplyHttpImplPrivate::httpAuthenticationRequired(const QHttpNetworkRequest &request,
1198 QAuthenticator *auth)
1200 managerPrivate->authenticationRequired(auth, q_func(), synchronous, url, &urlForLastAuthentication, request.withCredentials());
1203 #ifndef QT_NO_NETWORKPROXY
1204 void QNetworkReplyHttpImplPrivate::proxyAuthenticationRequired(const QNetworkProxy &proxy,
1205 QAuthenticator *authenticator)
1207 managerPrivate->proxyAuthenticationRequired(proxy, synchronous, authenticator, &lastProxyAuthentication);
1211 void QNetworkReplyHttpImplPrivate::httpError(QNetworkReply::NetworkError errorCode,
1212 const QString &errorString)
1214 #if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
1215 qDebug() << "http error!" << errorCode << errorString;
1219 error(errorCode, errorString);
1223 void QNetworkReplyHttpImplPrivate::replySslErrors(
1224 const QList<QSslError> &list, bool *ignoreAll, QList<QSslError> *toBeIgnored)
1226 Q_Q(QNetworkReplyHttpImpl);
1227 emit q->sslErrors(list);
1228 // Check if the callback set any ignore and return this here to http thread
1229 if (pendingIgnoreAllSslErrors)
1231 if (!pendingIgnoreSslErrorsList.isEmpty())
1232 *toBeIgnored = pendingIgnoreSslErrorsList;
1235 void QNetworkReplyHttpImplPrivate::replySslConfigurationChanged(const QSslConfiguration &sslConfiguration)
1237 // Receiving the used SSL configuration from the HTTP thread
1238 this->sslConfiguration = sslConfiguration;
1242 // Coming from QNonContiguousByteDeviceThreadForwardImpl in HTTP thread
1243 void QNetworkReplyHttpImplPrivate::resetUploadDataSlot(bool *r)
1245 *r = uploadByteDevice->reset();
1248 // Coming from QNonContiguousByteDeviceThreadForwardImpl in HTTP thread
1249 void QNetworkReplyHttpImplPrivate::sentUploadDataSlot(qint64 amount)
1251 uploadByteDevice->advanceReadPointer(amount);
1254 // Coming from QNonContiguousByteDeviceThreadForwardImpl in HTTP thread
1255 void QNetworkReplyHttpImplPrivate::wantUploadDataSlot(qint64 maxSize)
1257 Q_Q(QNetworkReplyHttpImpl);
1260 qint64 currentUploadDataLength = 0;
1261 char *data = const_cast<char*>(uploadByteDevice->readPointer(maxSize, currentUploadDataLength));
1262 // Let's make a copy of this data
1263 QByteArray dataArray(data, currentUploadDataLength);
1265 // Communicate back to HTTP thread
1266 emit q->haveUploadData(dataArray, uploadByteDevice->atEnd(), uploadByteDevice->size());
1270 A simple web page that can be used to test us: http://www.procata.com/cachetest/
1272 bool QNetworkReplyHttpImplPrivate::sendCacheContents(const QNetworkCacheMetaData &metaData)
1274 Q_Q(QNetworkReplyHttpImpl);
1276 setCachingEnabled(false);
1277 if (!metaData.isValid())
1280 QAbstractNetworkCache *nc = managerPrivate->networkCache;
1282 QIODevice *contents = nc->data(url);
1284 #if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
1285 qDebug() << "Can not send cache, the contents are 0" << url;
1289 contents->setParent(q);
1291 QNetworkCacheMetaData::AttributesMap attributes = metaData.attributes();
1292 int status = attributes.value(QNetworkRequest::HttpStatusCodeAttribute).toInt();
1294 status = 200; // fake it
1296 q->setAttribute(QNetworkRequest::HttpStatusCodeAttribute, status);
1297 q->setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, attributes.value(QNetworkRequest::HttpReasonPhraseAttribute));
1298 q->setAttribute(QNetworkRequest::SourceIsFromCacheAttribute, true);
1300 QNetworkCacheMetaData::RawHeaderList rawHeaders = metaData.rawHeaders();
1301 QNetworkCacheMetaData::RawHeaderList::ConstIterator it = rawHeaders.constBegin(),
1302 end = rawHeaders.constEnd();
1303 for ( ; it != end; ++it)
1304 setRawHeader(it->first, it->second);
1306 checkForRedirect(status);
1308 cacheLoadDevice = contents;
1309 q->connect(cacheLoadDevice, SIGNAL(readyRead()), SLOT(_q_cacheLoadReadyRead()));
1310 q->connect(cacheLoadDevice, SIGNAL(readChannelFinished()), SLOT(_q_cacheLoadReadyRead()));
1312 // This needs to be emitted in the event loop because it can be reached at
1313 // the direct code path of qnam.get(...) before the user has a chance
1314 // to connect any signals.
1315 QMetaObject::invokeMethod(q, "metaDataChanged", Qt::QueuedConnection);
1316 QMetaObject::invokeMethod(q, "_q_cacheLoadReadyRead", Qt::QueuedConnection);
1319 #if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
1320 qDebug() << "Successfully sent cache:" << url << contents->size() << "bytes";
1323 // Set the following flag so we can ignore some signals from HTTP thread
1324 // that would still come
1325 loadingFromCache = true;
1329 QNetworkCacheMetaData QNetworkReplyHttpImplPrivate::fetchCacheMetaData(const QNetworkCacheMetaData &oldMetaData) const
1331 Q_Q(const QNetworkReplyHttpImpl);
1333 QNetworkCacheMetaData metaData = oldMetaData;
1335 QNetworkHeadersPrivate cacheHeaders;
1336 cacheHeaders.setAllRawHeaders(metaData.rawHeaders());
1337 QNetworkHeadersPrivate::RawHeadersList::ConstIterator it;
1339 QList<QByteArray> newHeaders = q->rawHeaderList();
1340 foreach (QByteArray header, newHeaders) {
1341 QByteArray originalHeader = header;
1342 header = header.toLower();
1344 (header == "connection"
1345 || header == "keep-alive"
1346 || header == "proxy-authenticate"
1347 || header == "proxy-authorization"
1349 || header == "trailers"
1350 || header == "transfer-encoding"
1351 || header == "upgrade");
1355 // for 4.6.0, we were planning to not store the date header in the
1356 // cached resource; through that we planned to reduce the number
1357 // of writes to disk when using a QNetworkDiskCache (i.e. don't
1358 // write to disk when only the date changes).
1359 // However, without the date we cannot calculate the age of the page
1361 //if (header == "date")
1364 // Don't store Warning 1xx headers
1365 if (header == "warning") {
1366 QByteArray v = q->rawHeader(header);
1369 && v[1] >= '0' && v[1] <= '9'
1370 && v[2] >= '0' && v[2] <= '9')
1374 it = cacheHeaders.findRawHeader(header);
1375 if (it != cacheHeaders.rawHeaders.constEnd()) {
1376 // Match the behavior of Firefox and assume Cache-Control: "no-transform"
1377 if (header == "content-encoding"
1378 || header == "content-range"
1379 || header == "content-type")
1382 // For MS servers that send "Content-Length: 0" on 304 responses
1384 if (header == "content-length")
1388 #if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
1389 QByteArray n = rawHeader(header);
1391 if (it != cacheHeaders.rawHeaders.constEnd())
1393 if (n != o && header != "date") {
1394 qDebug() << "replacing" << header;
1395 qDebug() << "new" << n;
1396 qDebug() << "old" << o;
1399 cacheHeaders.setRawHeader(originalHeader, q->rawHeader(header));
1401 metaData.setRawHeaders(cacheHeaders.rawHeaders);
1403 bool checkExpired = true;
1405 QHash<QByteArray, QByteArray> cacheControl;
1406 it = cacheHeaders.findRawHeader("Cache-Control");
1407 if (it != cacheHeaders.rawHeaders.constEnd()) {
1408 cacheControl = parseHttpOptionHeader(it->second);
1409 QByteArray maxAge = cacheControl.value("max-age");
1410 if (!maxAge.isEmpty()) {
1411 checkExpired = false;
1412 QDateTime dt = QDateTime::currentDateTime();
1413 dt = dt.addSecs(maxAge.toInt());
1414 metaData.setExpirationDate(dt);
1418 it = cacheHeaders.findRawHeader("expires");
1419 if (it != cacheHeaders.rawHeaders.constEnd()) {
1420 QDateTime expiredDateTime = QNetworkHeadersPrivate::fromHttpDate(it->second);
1421 metaData.setExpirationDate(expiredDateTime);
1425 it = cacheHeaders.findRawHeader("last-modified");
1426 if (it != cacheHeaders.rawHeaders.constEnd())
1427 metaData.setLastModified(QNetworkHeadersPrivate::fromHttpDate(it->second));
1430 // only cache GET replies by default, all other replies (POST, PUT, DELETE)
1431 // are not cacheable by default (according to RFC 2616 section 9)
1432 if (httpRequest.operation() == QHttpNetworkRequest::Get) {
1434 canDiskCache = true;
1436 // HTTP/1.1 caches SHOULD treat "Pragma: no-cache" as if the client
1437 // had sent "Cache-Control: no-cache".
1438 it = cacheHeaders.findRawHeader("pragma");
1439 if (it != cacheHeaders.rawHeaders.constEnd()
1440 && it->second == "no-cache")
1441 canDiskCache = false;
1443 // HTTP/1.1. Check the Cache-Control header
1444 if (cacheControl.contains("no-cache"))
1445 canDiskCache = false;
1446 else if (cacheControl.contains("no-store"))
1447 canDiskCache = false;
1449 // responses to POST might be cacheable
1450 } else if (httpRequest.operation() == QHttpNetworkRequest::Post) {
1452 canDiskCache = false;
1453 // some pages contain "expires:" and "cache-control: no-cache" field,
1454 // so we only might cache POST requests if we get "cache-control: max-age ..."
1455 if (cacheControl.contains("max-age"))
1456 canDiskCache = true;
1458 // responses to PUT and DELETE are not cacheable
1460 canDiskCache = false;
1463 metaData.setSaveToDisk(canDiskCache);
1464 QNetworkCacheMetaData::AttributesMap attributes;
1465 if (statusCode != 304) {
1466 // update the status code
1467 attributes.insert(QNetworkRequest::HttpStatusCodeAttribute, statusCode);
1468 attributes.insert(QNetworkRequest::HttpReasonPhraseAttribute, reasonPhrase);
1470 // this is a redirection, keep the attributes intact
1471 attributes = oldMetaData.attributes();
1473 metaData.setAttributes(attributes);
1477 bool QNetworkReplyHttpImplPrivate::canResume() const
1479 Q_Q(const QNetworkReplyHttpImpl);
1481 // Only GET operation supports resuming.
1482 if (operation != QNetworkAccessManager::GetOperation)
1485 // Can only resume if server/resource supports Range header.
1486 QByteArray acceptRangesheaderName("Accept-Ranges");
1487 if (!q->hasRawHeader(acceptRangesheaderName) || q->rawHeader(acceptRangesheaderName) == "none")
1490 // We only support resuming for byte ranges.
1491 if (request.hasRawHeader("Range")) {
1492 QByteArray range = request.rawHeader("Range");
1493 if (!range.startsWith("bytes="))
1497 // If we're using a download buffer then we don't support resuming/migration
1498 // right now. Too much trouble.
1499 if (downloadZerocopyBuffer)
1505 void QNetworkReplyHttpImplPrivate::setResumeOffset(quint64 offset)
1507 resumeOffset = offset;
1511 Starts the backend. Returns true if the backend is started. Returns false if the backend
1512 could not be started due to an unopened or roaming session. The caller should recall this
1513 function once the session has been opened or the roaming process has finished.
1515 bool QNetworkReplyHttpImplPrivate::start()
1517 #ifndef QT_NO_BEARERMANAGEMENT
1518 QSharedPointer<QNetworkSession> networkSession(managerPrivate->getNetworkSession());
1519 if (!networkSession) {
1523 #ifndef QT_NO_BEARERMANAGEMENT
1526 // This is not ideal.
1527 const QString host = url.host();
1528 if (host == QLatin1String("localhost") ||
1529 QHostAddress(host).isLoopback()) {
1530 // Don't need an open session for localhost access.
1535 if (networkSession->isOpen() &&
1536 networkSession->state() == QNetworkSession::Connected) {
1537 Q_Q(QNetworkReplyHttpImpl);
1538 QObject::connect(networkSession.data(), SIGNAL(usagePoliciesChanged(QNetworkSession::UsagePolicies)),
1539 q, SLOT(_q_networkSessionUsagePoliciesChanged(QNetworkSession::UsagePolicies)));
1548 void QNetworkReplyHttpImplPrivate::_q_startOperation()
1550 Q_Q(QNetworkReplyHttpImpl);
1552 // ensure this function is only being called once
1553 if (state == Working) {
1554 qDebug("QNetworkReplyImpl::_q_startOperation was called more than once");
1559 #ifndef QT_NO_BEARERMANAGEMENT
1560 // Do not start background requests if they are not allowed by session policy
1561 QSharedPointer<QNetworkSession> session(manager->d_func()->getNetworkSession());
1562 QVariant isBackground = request.attribute(QNetworkRequest::BackgroundRequestAttribute, QVariant::fromValue(false));
1563 if (isBackground.toBool() && session && session->usagePolicies().testFlag(QNetworkSession::NoBackgroundTrafficPolicy)) {
1564 QMetaObject::invokeMethod(q, "_q_error", synchronous ? Qt::DirectConnection : Qt::QueuedConnection,
1565 Q_ARG(QNetworkReply::NetworkError, QNetworkReply::BackgroundRequestNotAllowedError),
1566 Q_ARG(QString, QCoreApplication::translate("QNetworkReply", "Background request not allowed.")));
1567 QMetaObject::invokeMethod(q, "_q_finished", synchronous ? Qt::DirectConnection : Qt::QueuedConnection);
1573 #ifndef QT_NO_BEARERMANAGEMENT
1574 // backend failed to start because the session state is not Connected.
1575 // QNetworkAccessManager will call reply->backend->start() again for us when the session
1577 state = WaitingForSession;
1580 QObject::connect(session.data(), SIGNAL(error(QNetworkSession::SessionError)),
1581 q, SLOT(_q_networkSessionFailed()), Qt::QueuedConnection);
1583 if (!session->isOpen()) {
1584 session->setSessionProperty(QStringLiteral("ConnectInBackground"), isBackground);
1588 qWarning("Backend is waiting for QNetworkSession to connect, but there is none!");
1589 QMetaObject::invokeMethod(q, "_q_error", synchronous ? Qt::DirectConnection : Qt::QueuedConnection,
1590 Q_ARG(QNetworkReply::NetworkError, QNetworkReply::NetworkSessionFailedError),
1591 Q_ARG(QString, QCoreApplication::translate("QNetworkReply", "Network session error.")));
1592 QMetaObject::invokeMethod(q, "_q_finished", synchronous ? Qt::DirectConnection : Qt::QueuedConnection);
1596 qWarning("Backend start failed");
1597 QMetaObject::invokeMethod(q, "_q_error", synchronous ? Qt::DirectConnection : Qt::QueuedConnection,
1598 Q_ARG(QNetworkReply::NetworkError, QNetworkReply::UnknownNetworkError),
1599 Q_ARG(QString, QCoreApplication::translate("QNetworkReply", "backend start error.")));
1600 QMetaObject::invokeMethod(q, "_q_finished", synchronous ? Qt::DirectConnection : Qt::QueuedConnection);
1607 q_func()->setFinished(true);
1609 if (state != Finished) {
1615 void QNetworkReplyHttpImplPrivate::_q_cacheLoadReadyRead()
1617 Q_Q(QNetworkReplyHttpImpl);
1619 if (state != Working)
1621 if (!cacheLoadDevice || !q->isOpen() || !cacheLoadDevice->bytesAvailable())
1624 // FIXME Optimize to use zerocopy download buffer if it is a QBuffer.
1625 // Needs to be done where sendCacheContents() (?) of HTTP is emitting
1626 // metaDataChanged ?
1629 QVariant totalSize = cookedHeaders.value(QNetworkRequest::ContentLengthHeader);
1631 // emit readyRead before downloadProgress incase this will cause events to be
1632 // processed and we get into a recursive call (as in QProgressDialog).
1634 // This readyRead() goes to the user. The user then may or may not read() anything.
1635 emit q->readyRead();
1636 if (downloadProgressSignalChoke.elapsed() >= progressSignalInterval) {
1637 downloadProgressSignalChoke.restart();
1638 emit q->downloadProgress(bytesDownloaded,
1639 totalSize.isNull() ? Q_INT64_C(-1) : totalSize.toLongLong());
1642 // If there are still bytes available in the cacheLoadDevice then the user did not read
1643 // in response to the readyRead() signal. This means we have to load from the cacheLoadDevice
1644 // and buffer that stuff. This is needed to be able to properly emit finished() later.
1645 while (cacheLoadDevice->bytesAvailable()) {
1646 downloadMultiBuffer.append(cacheLoadDevice->readAll());
1649 if (cacheLoadDevice->isSequential()) {
1650 // check if end and we can read the EOF -1
1652 qint64 actualCount = cacheLoadDevice->read(&c, 1);
1653 if (actualCount < 0) {
1654 cacheLoadDevice->deleteLater();
1655 cacheLoadDevice = 0;
1656 QMetaObject::invokeMethod(q, "_q_finished", Qt::QueuedConnection);
1657 } else if (actualCount == 1) {
1658 // This is most probably not happening since most QIODevice returned something proper for bytesAvailable()
1659 // and had already been "emptied".
1660 cacheLoadDevice->ungetChar(c);
1662 } else if ((!cacheLoadDevice->isSequential() && cacheLoadDevice->atEnd())) {
1663 // This codepath is in case the cache device is a QBuffer, e.g. from QNetworkDiskCache.
1664 cacheLoadDevice->deleteLater();
1665 cacheLoadDevice = 0;
1666 QMetaObject::invokeMethod(q, "_q_finished", Qt::QueuedConnection);
1672 void QNetworkReplyHttpImplPrivate::_q_bufferOutgoingDataFinished()
1674 Q_Q(QNetworkReplyHttpImpl);
1676 // make sure this is only called once, ever.
1677 //_q_bufferOutgoingData may call it or the readChannelFinished emission
1678 if (state != Buffering)
1681 // disconnect signals
1682 QObject::disconnect(outgoingData, SIGNAL(readyRead()), q, SLOT(_q_bufferOutgoingData()));
1683 QObject::disconnect(outgoingData, SIGNAL(readChannelFinished()), q, SLOT(_q_bufferOutgoingDataFinished()));
1685 // finally, start the request
1686 QMetaObject::invokeMethod(q, "_q_startOperation", Qt::QueuedConnection);
1689 void QNetworkReplyHttpImplPrivate::_q_bufferOutgoingData()
1691 Q_Q(QNetworkReplyHttpImpl);
1693 if (!outgoingDataBuffer) {
1694 // first call, create our buffer
1695 outgoingDataBuffer = QSharedPointer<QRingBuffer>(new QRingBuffer());
1697 QObject::connect(outgoingData, SIGNAL(readyRead()), q, SLOT(_q_bufferOutgoingData()));
1698 QObject::connect(outgoingData, SIGNAL(readChannelFinished()), q, SLOT(_q_bufferOutgoingDataFinished()));
1701 qint64 bytesBuffered = 0;
1702 qint64 bytesToBuffer = 0;
1704 // read data into our buffer
1706 bytesToBuffer = outgoingData->bytesAvailable();
1707 // unknown? just try 2 kB, this also ensures we always try to read the EOF
1708 if (bytesToBuffer <= 0)
1709 bytesToBuffer = 2*1024;
1711 char *dst = outgoingDataBuffer->reserve(bytesToBuffer);
1712 bytesBuffered = outgoingData->read(dst, bytesToBuffer);
1714 if (bytesBuffered == -1) {
1715 // EOF has been reached.
1716 outgoingDataBuffer->chop(bytesToBuffer);
1718 _q_bufferOutgoingDataFinished();
1720 } else if (bytesBuffered == 0) {
1721 // nothing read right now, just wait until we get called again
1722 outgoingDataBuffer->chop(bytesToBuffer);
1726 // don't break, try to read() again
1727 outgoingDataBuffer->chop(bytesToBuffer - bytesBuffered);
1732 #ifndef QT_NO_BEARERMANAGEMENT
1733 void QNetworkReplyHttpImplPrivate::_q_networkSessionConnected()
1735 Q_Q(QNetworkReplyHttpImpl);
1740 QSharedPointer<QNetworkSession> session = managerPrivate->getNetworkSession();
1744 if (session->state() != QNetworkSession::Connected)
1748 case QNetworkReplyImplPrivate::Buffering:
1749 case QNetworkReplyImplPrivate::Working:
1750 case QNetworkReplyImplPrivate::Reconnecting:
1751 // Migrate existing downloads to new network connection.
1754 case QNetworkReplyImplPrivate::WaitingForSession:
1755 // Start waiting requests.
1756 QMetaObject::invokeMethod(q, "_q_startOperation", Qt::QueuedConnection);
1763 void QNetworkReplyHttpImplPrivate::_q_networkSessionFailed()
1765 // Abort waiting and working replies.
1766 if (state == WaitingForSession || state == Working) {
1768 QSharedPointer<QNetworkSession> session(manager->d_func()->getNetworkSession());
1771 errorStr = session->errorString();
1773 errorStr = QCoreApplication::translate("QNetworkReply", "Network session error.");
1774 error(QNetworkReplyImpl::NetworkSessionFailedError, errorStr);
1779 void QNetworkReplyHttpImplPrivate::_q_networkSessionUsagePoliciesChanged(QNetworkSession::UsagePolicies newPolicies)
1781 if (request.attribute(QNetworkRequest::BackgroundRequestAttribute).toBool()) {
1782 if (newPolicies & QNetworkSession::NoBackgroundTrafficPolicy) {
1783 // Abort waiting and working replies.
1784 if (state == WaitingForSession || state == Working) {
1786 error(QNetworkReply::BackgroundRequestNotAllowedError,
1787 QCoreApplication::translate("QNetworkReply", "Background request not allowed."));
1790 // ### if canResume(), then we could resume automatically
1798 // need to have this function since the reply is a private member variable
1799 // and the special backends need to access this.
1800 void QNetworkReplyHttpImplPrivate::emitReplyUploadProgress(qint64 bytesSent, qint64 bytesTotal)
1802 Q_Q(QNetworkReplyHttpImpl);
1806 //choke signal emissions, except the first and last signals which are unconditional
1807 if (uploadProgressSignalChoke.isValid()) {
1808 if (bytesSent != bytesTotal && uploadProgressSignalChoke.elapsed() < progressSignalInterval) {
1811 uploadProgressSignalChoke.restart();
1813 uploadProgressSignalChoke.start();
1816 emit q->uploadProgress(bytesSent, bytesTotal);
1819 QNonContiguousByteDevice* QNetworkReplyHttpImplPrivate::createUploadByteDevice()
1821 Q_Q(QNetworkReplyHttpImpl);
1823 if (outgoingDataBuffer)
1824 uploadByteDevice = QSharedPointer<QNonContiguousByteDevice>(QNonContiguousByteDeviceFactory::create(outgoingDataBuffer));
1825 else if (outgoingData) {
1826 uploadByteDevice = QSharedPointer<QNonContiguousByteDevice>(QNonContiguousByteDeviceFactory::create(outgoingData));
1831 bool bufferDisallowed =
1832 request.attribute(QNetworkRequest::DoNotBufferUploadDataAttribute,
1833 QVariant(false)) == QVariant(true);
1834 if (bufferDisallowed)
1835 uploadByteDevice->disableReset();
1837 // We want signal emissions only for normal asynchronous uploads
1839 QObject::connect(uploadByteDevice.data(), SIGNAL(readProgress(qint64,qint64)),
1840 q, SLOT(emitReplyUploadProgress(qint64,qint64)));
1842 return uploadByteDevice.data();
1845 void QNetworkReplyHttpImplPrivate::_q_finished()
1847 // This gets called queued, just forward to real call then
1851 void QNetworkReplyHttpImplPrivate::finished()
1853 Q_Q(QNetworkReplyHttpImpl);
1855 if (state == Finished || state == Aborted || state == WaitingForSession)
1858 QVariant totalSize = cookedHeaders.value(QNetworkRequest::ContentLengthHeader);
1859 if (preMigrationDownloaded != Q_INT64_C(-1))
1860 totalSize = totalSize.toLongLong() + preMigrationDownloaded;
1863 #ifndef QT_NO_BEARERMANAGEMENT
1864 QSharedPointer<QNetworkSession> session = managerPrivate->getNetworkSession();
1865 if (session && session->state() == QNetworkSession::Roaming &&
1866 state == Working && errorCode != QNetworkReply::OperationCanceledError) {
1867 // only content with a known size will fail with a temporary network failure error
1868 if (!totalSize.isNull()) {
1869 if (bytesDownloaded != totalSize) {
1870 if (migrateBackend()) {
1871 // either we are migrating or the request is finished/aborted
1872 if (state == Reconnecting || state == WaitingForSession) {
1873 return; // exit early if we are migrating.
1876 error(QNetworkReply::TemporaryNetworkFailureError,
1877 QNetworkReply::tr("Temporary network failure."));
1886 q->setFinished(true);
1888 if (totalSize.isNull() || totalSize == -1) {
1889 emit q->downloadProgress(bytesDownloaded, bytesDownloaded);
1891 emit q->downloadProgress(bytesDownloaded, totalSize.toLongLong());
1894 if (bytesUploaded == -1 && (outgoingData || outgoingDataBuffer))
1895 emit q->uploadProgress(0, 0);
1897 // if we don't know the total size of or we received everything save the cache
1898 if (totalSize.isNull() || totalSize == -1 || bytesDownloaded == totalSize)
1899 completeCacheSave();
1901 emit q->readChannelFinished();
1905 void QNetworkReplyHttpImplPrivate::_q_error(QNetworkReplyImpl::NetworkError code, const QString &errorMessage)
1907 this->error(code, errorMessage);
1911 void QNetworkReplyHttpImplPrivate::error(QNetworkReplyImpl::NetworkError code, const QString &errorMessage)
1913 Q_Q(QNetworkReplyHttpImpl);
1914 // Can't set and emit multiple errors.
1915 if (errorCode != QNetworkReply::NoError) {
1916 qWarning("QNetworkReplyImplPrivate::error: Internal problem, this method must only be called once.");
1921 q->setErrorString(errorMessage);
1923 // note: might not be a good idea, since users could decide to delete us
1924 // which would delete the backend too...
1925 // maybe we should protect the backend
1926 emit q->error(code);
1929 void QNetworkReplyHttpImplPrivate::metaDataChanged()
1931 // FIXME merge this with replyDownloadMetaData(); ?
1933 Q_Q(QNetworkReplyHttpImpl);
1934 // 1. do we have cookies?
1935 // 2. are we allowed to set them?
1936 if (cookedHeaders.contains(QNetworkRequest::SetCookieHeader) && manager
1937 && (static_cast<QNetworkRequest::LoadControl>
1938 (request.attribute(QNetworkRequest::CookieSaveControlAttribute,
1939 QNetworkRequest::Automatic).toInt()) == QNetworkRequest::Automatic)) {
1940 QList<QNetworkCookie> cookies =
1941 qvariant_cast<QList<QNetworkCookie> >(cookedHeaders.value(QNetworkRequest::SetCookieHeader));
1942 QNetworkCookieJar *jar = manager->cookieJar();
1944 jar->setCookiesFromUrl(cookies, url);
1946 emit q->metaDataChanged();
1950 Migrates the backend of the QNetworkReply to a new network connection if required. Returns
1951 true if the reply is migrated or it is not required; otherwise returns false.
1953 bool QNetworkReplyHttpImplPrivate::migrateBackend()
1955 Q_Q(QNetworkReplyHttpImpl);
1957 // Network reply is already finished or aborted, don't need to migrate.
1958 if (state == Finished || state == Aborted)
1961 // Backend does not support resuming download.
1965 // Request has outgoing data, not migrating.
1969 // Request is serviced from the cache, don't need to migrate.
1970 if (cacheLoadDevice)
1973 state = Reconnecting;
1975 cookedHeaders.clear();
1978 preMigrationDownloaded = bytesDownloaded;
1980 setResumeOffset(bytesDownloaded);
1982 emit q->abortHttpRequest();
1984 QMetaObject::invokeMethod(q, "_q_startOperation", Qt::QueuedConnection);
1990 void QNetworkReplyHttpImplPrivate::createCache()
1992 // check if we can save and if we're allowed to
1993 if (!managerPrivate->networkCache
1994 || !request.attribute(QNetworkRequest::CacheSaveControlAttribute, true).toBool())
1996 cacheEnabled = true;
1999 bool QNetworkReplyHttpImplPrivate::isCachingEnabled() const
2001 return (cacheEnabled && managerPrivate->networkCache != 0);
2004 void QNetworkReplyHttpImplPrivate::setCachingEnabled(bool enable)
2006 if (!enable && !cacheEnabled)
2007 return; // nothing to do
2008 if (enable && cacheEnabled)
2009 return; // nothing to do either!
2012 if (bytesDownloaded) {
2013 qDebug("setCachingEnabled: %lld bytesDownloaded", bytesDownloaded);
2014 // refuse to enable in this case
2015 qCritical("QNetworkReplyImpl: backend error: caching was enabled after some bytes had been written");
2021 // someone told us to turn on, then back off?
2022 // ok... but you should make up your mind
2023 qDebug("QNetworkReplyImpl: setCachingEnabled(true) called after setCachingEnabled(false)");
2024 managerPrivate->networkCache->remove(url);
2025 cacheSaveDevice = 0;
2026 cacheEnabled = false;
2030 void QNetworkReplyHttpImplPrivate::completeCacheSave()
2032 if (cacheEnabled && errorCode != QNetworkReplyImpl::NoError) {
2033 managerPrivate->networkCache->remove(url);
2034 } else if (cacheEnabled && cacheSaveDevice) {
2035 managerPrivate->networkCache->insert(cacheSaveDevice);
2037 cacheSaveDevice = 0;
2038 cacheEnabled = false;
2043 #endif // QT_NO_HTTP