1 /****************************************************************************
3 ** Copyright (C) 2012 Nokia Corporation and/or its subsidiary(-ies).
4 ** Contact: http://www.qt-project.org/
6 ** This file is part of the QtNetwork module of the Qt Toolkit.
8 ** $QT_BEGIN_LICENSE:LGPL$
9 ** GNU Lesser General Public License Usage
10 ** This file may be used under the terms of the GNU Lesser General Public
11 ** License version 2.1 as published by the Free Software Foundation and
12 ** appearing in the file LICENSE.LGPL included in the packaging of this
13 ** file. Please review the following information to ensure the GNU Lesser
14 ** General Public License version 2.1 requirements will be met:
15 ** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
17 ** In addition, as a special exception, Nokia gives you certain additional
18 ** rights. These rights are described in the Nokia Qt LGPL Exception
19 ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
21 ** GNU General Public License Usage
22 ** Alternatively, this file may be used under the terms of the GNU General
23 ** Public License version 3.0 as published by the Free Software Foundation
24 ** and appearing in the file LICENSE.GPL included in the packaging of this
25 ** file. Please review the following information to ensure the GNU General
26 ** Public License version 3.0 requirements will be met:
27 ** http://www.gnu.org/copyleft/gpl.html.
30 ** Alternatively, this file may be used in accordance with the terms and
31 ** conditions contained in a signed written agreement between you and Nokia.
40 ****************************************************************************/
42 //#define QNETWORKDISKCACHE_DEBUG
45 #include "qnetworkdiskcache.h"
46 #include "qnetworkdiskcache_p.h"
47 #include "QtCore/qscopedpointer.h"
51 #include <qdatetime.h>
52 #include <qdiriterator.h>
54 #include <qcryptographichash.h>
57 #define CACHE_POSTFIX QLatin1String(".d")
58 #define PREPARED_SLASH QLatin1String("prepared/")
59 #define CACHE_VERSION 7
60 #define DATA_DIR QLatin1String("data")
62 #define MAX_COMPRESSION_SIZE (1024 * 1024 * 3)
64 #ifndef QT_NO_NETWORKDISKCACHE
69 \class QNetworkDiskCache
73 \brief The QNetworkDiskCache class provides a very basic disk cache.
75 QNetworkDiskCache stores each url in its own file inside of the
76 cacheDirectory using QDataStream. Files with a text MimeType
77 are compressed using qCompress. Each cache file starts with "cache_"
78 and ends in ".cache". Data is written to disk only in insert()
81 Currently you can not share the same cache files with more then
84 QNetworkDiskCache by default limits the amount of space that the cache will
85 use on the system to 50MB.
87 Note you have to set the cache directory before it will work.
89 A network disk cache can be enabled by:
91 \snippet code/src_network_access_qnetworkdiskcache.cpp 0
93 When sending requests, to control the preference of when to use the cache
94 and when to use the network, consider the following:
96 \snippet code/src_network_access_qnetworkdiskcache.cpp 1
98 To check whether the response came from the cache or from the network, the
99 following can be applied:
101 \snippet code/src_network_access_qnetworkdiskcache.cpp 2
105 Creates a new disk cache. The \a parent argument is passed to
106 QAbstractNetworkCache's constructor.
108 QNetworkDiskCache::QNetworkDiskCache(QObject *parent)
109 : QAbstractNetworkCache(*new QNetworkDiskCachePrivate, parent)
114 Destroys the cache object. This does not clear the disk cache.
116 QNetworkDiskCache::~QNetworkDiskCache()
118 Q_D(QNetworkDiskCache);
119 QHashIterator<QIODevice*, QCacheItem*> it(d->inserting);
120 while (it.hasNext()) {
127 Returns the location where cached files will be stored.
129 QString QNetworkDiskCache::cacheDirectory() const
131 Q_D(const QNetworkDiskCache);
132 return d->cacheDirectory;
136 Sets the directory where cached files will be stored to \a cacheDir
138 QNetworkDiskCache will create this directory if it does not exists.
140 Prepared cache items will be stored in the new cache directory when
143 \sa QDesktopServices::CacheLocation
145 void QNetworkDiskCache::setCacheDirectory(const QString &cacheDir)
147 #if defined(QNETWORKDISKCACHE_DEBUG)
148 qDebug() << "QNetworkDiskCache::setCacheDirectory()" << cacheDir;
150 Q_D(QNetworkDiskCache);
151 if (cacheDir.isEmpty())
153 d->cacheDirectory = cacheDir;
154 QDir dir(d->cacheDirectory);
155 d->cacheDirectory = dir.absolutePath();
156 if (!d->cacheDirectory.endsWith(QLatin1Char('/')))
157 d->cacheDirectory += QLatin1Char('/');
159 d->dataDirectory = d->cacheDirectory + DATA_DIR + QString::number(CACHE_VERSION) + QLatin1Char('/');
166 qint64 QNetworkDiskCache::cacheSize() const
168 #if defined(QNETWORKDISKCACHE_DEBUG)
169 qDebug() << "QNetworkDiskCache::cacheSize()";
171 Q_D(const QNetworkDiskCache);
172 if (d->cacheDirectory.isEmpty())
174 if (d->currentCacheSize < 0) {
175 QNetworkDiskCache *that = const_cast<QNetworkDiskCache*>(this);
176 that->d_func()->currentCacheSize = that->expire();
178 return d->currentCacheSize;
184 QIODevice *QNetworkDiskCache::prepare(const QNetworkCacheMetaData &metaData)
186 #if defined(QNETWORKDISKCACHE_DEBUG)
187 qDebug() << "QNetworkDiskCache::prepare()" << metaData.url();
189 Q_D(QNetworkDiskCache);
190 if (!metaData.isValid() || !metaData.url().isValid() || !metaData.saveToDisk())
193 if (d->cacheDirectory.isEmpty()) {
194 qWarning() << "QNetworkDiskCache::prepare() The cache directory is not set";
198 foreach (QNetworkCacheMetaData::RawHeader header, metaData.rawHeaders()) {
199 if (header.first.toLower() == "content-length") {
200 qint64 size = header.second.toInt();
201 if (size > (maximumCacheSize() * 3)/4)
206 QScopedPointer<QCacheItem> cacheItem(new QCacheItem);
207 cacheItem->metaData = metaData;
209 QIODevice *device = 0;
210 if (cacheItem->canCompress()) {
211 cacheItem->data.open(QBuffer::ReadWrite);
212 device = &(cacheItem->data);
214 QString templateName = d->tmpCacheFileName();
216 cacheItem->file = new QTemporaryFile(templateName, &cacheItem->data);
220 if (!cacheItem->file || !cacheItem->file->open()) {
221 qWarning() << "QNetworkDiskCache::prepare() unable to open temporary file";
225 cacheItem->writeHeader(cacheItem->file);
226 device = cacheItem->file;
228 d->inserting[device] = cacheItem.take();
235 void QNetworkDiskCache::insert(QIODevice *device)
237 #if defined(QNETWORKDISKCACHE_DEBUG)
238 qDebug() << "QNetworkDiskCache::insert()" << device;
240 Q_D(QNetworkDiskCache);
241 QHash<QIODevice*, QCacheItem*>::iterator it = d->inserting.find(device);
242 if (it == d->inserting.end()) {
243 qWarning() << "QNetworkDiskCache::insert() called on a device we don't know about" << device;
247 d->storeItem(it.value());
249 d->inserting.erase(it);
254 Create subdirectories and other housekeeping on the filesystem.
255 Prevents too many files from being present in any single directory.
257 void QNetworkDiskCachePrivate::prepareLayout()
260 helper.mkpath(cacheDirectory + PREPARED_SLASH);
262 //Create directory and subdirectories 0-F
263 helper.mkpath(dataDirectory);
264 for (uint i = 0; i < 16 ; i++) {
265 QString str = QString::number(i, 16);
266 QString subdir = dataDirectory + str;
267 helper.mkdir(subdir);
272 void QNetworkDiskCachePrivate::storeItem(QCacheItem *cacheItem)
274 Q_Q(QNetworkDiskCache);
275 Q_ASSERT(cacheItem->metaData.saveToDisk());
277 QString fileName = cacheFileName(cacheItem->metaData.url());
278 Q_ASSERT(!fileName.isEmpty());
280 if (QFile::exists(fileName)) {
281 if (!QFile::remove(fileName)) {
282 qWarning() << "QNetworkDiskCache: couldn't remove the cache file " << fileName;
287 if (currentCacheSize > 0)
288 currentCacheSize += 1024 + cacheItem->size();
289 currentCacheSize = q->expire();
290 if (!cacheItem->file) {
291 QString templateName = tmpCacheFileName();
292 cacheItem->file = new QTemporaryFile(templateName, &cacheItem->data);
293 if (cacheItem->file->open()) {
294 cacheItem->writeHeader(cacheItem->file);
295 cacheItem->writeCompressedData(cacheItem->file);
300 && cacheItem->file->isOpen()
301 && cacheItem->file->error() == QFile::NoError) {
302 cacheItem->file->setAutoRemove(false);
303 // ### use atomic rename rather then remove & rename
304 if (cacheItem->file->rename(fileName))
305 currentCacheSize += cacheItem->file->size();
307 cacheItem->file->setAutoRemove(true);
309 if (cacheItem->metaData.url() == lastItem.metaData.url())
316 bool QNetworkDiskCache::remove(const QUrl &url)
318 #if defined(QNETWORKDISKCACHE_DEBUG)
319 qDebug() << "QNetworkDiskCache::remove()" << url;
321 Q_D(QNetworkDiskCache);
323 // remove is also used to cancel insertions, not a common operation
324 QHashIterator<QIODevice*, QCacheItem*> it(d->inserting);
325 while (it.hasNext()) {
327 QCacheItem *item = it.value();
328 if (item && item->metaData.url() == url) {
330 d->inserting.remove(it.key());
335 if (d->lastItem.metaData.url() == url)
337 return d->removeFile(d->cacheFileName(url));
341 Put all of the misc file removing into one function to be extra safe
343 bool QNetworkDiskCachePrivate::removeFile(const QString &file)
345 #if defined(QNETWORKDISKCACHE_DEBUG)
346 qDebug() << "QNetworkDiskCache::removFile()" << file;
350 QFileInfo info(file);
351 QString fileName = info.fileName();
352 if (!fileName.endsWith(CACHE_POSTFIX))
354 qint64 size = info.size();
355 if (QFile::remove(file)) {
356 currentCacheSize -= size;
365 QNetworkCacheMetaData QNetworkDiskCache::metaData(const QUrl &url)
367 #if defined(QNETWORKDISKCACHE_DEBUG)
368 qDebug() << "QNetworkDiskCache::metaData()" << url;
370 Q_D(QNetworkDiskCache);
371 if (d->lastItem.metaData.url() == url)
372 return d->lastItem.metaData;
373 return fileMetaData(d->cacheFileName(url));
377 Returns the QNetworkCacheMetaData for the cache file \a fileName.
379 If \a fileName is not a cache file QNetworkCacheMetaData will be invalid.
381 QNetworkCacheMetaData QNetworkDiskCache::fileMetaData(const QString &fileName) const
383 #if defined(QNETWORKDISKCACHE_DEBUG)
384 qDebug() << "QNetworkDiskCache::fileMetaData()" << fileName;
386 Q_D(const QNetworkDiskCache);
387 QFile file(fileName);
388 if (!file.open(QFile::ReadOnly))
389 return QNetworkCacheMetaData();
390 if (!d->lastItem.read(&file, false)) {
392 QNetworkDiskCachePrivate *that = const_cast<QNetworkDiskCachePrivate*>(d);
393 that->removeFile(fileName);
395 return d->lastItem.metaData;
401 QIODevice *QNetworkDiskCache::data(const QUrl &url)
403 #if defined(QNETWORKDISKCACHE_DEBUG)
404 qDebug() << "QNetworkDiskCache::data()" << url;
406 Q_D(QNetworkDiskCache);
407 QScopedPointer<QBuffer> buffer;
410 if (d->lastItem.metaData.url() == url && d->lastItem.data.isOpen()) {
411 buffer.reset(new QBuffer);
412 buffer->setData(d->lastItem.data.data());
414 QScopedPointer<QFile> file(new QFile(d->cacheFileName(url)));
415 if (!file->open(QFile::ReadOnly | QIODevice::Unbuffered))
418 if (!d->lastItem.read(file.data(), true)) {
423 if (d->lastItem.data.isOpen()) {
425 buffer.reset(new QBuffer);
426 buffer->setData(d->lastItem.data.data());
428 buffer.reset(new QBuffer);
429 // ### verify that QFile uses the fd size and not the file name
430 qint64 size = file->size() - file->pos();
432 #if !defined(Q_OS_WINCE) && !defined(Q_OS_INTEGRITY)
433 p = file->map(file->pos(), size);
436 buffer->setData((const char *)p, size);
437 file.take()->setParent(buffer.data());
439 buffer->setData(file->readAll());
443 buffer->open(QBuffer::ReadOnly);
444 return buffer.take();
450 void QNetworkDiskCache::updateMetaData(const QNetworkCacheMetaData &metaData)
452 #if defined(QNETWORKDISKCACHE_DEBUG)
453 qDebug() << "QNetworkDiskCache::updateMetaData()" << metaData.url();
455 QUrl url = metaData.url();
456 QIODevice *oldDevice = data(url);
458 #if defined(QNETWORKDISKCACHE_DEBUG)
459 qDebug() << "QNetworkDiskCache::updateMetaData(), no device!";
464 QIODevice *newDevice = prepare(metaData);
466 #if defined(QNETWORKDISKCACHE_DEBUG)
467 qDebug() << "QNetworkDiskCache::updateMetaData(), no new device!" << url;
472 while (!oldDevice->atEnd()) {
473 qint64 s = oldDevice->read(data, 1024);
474 newDevice->write(data, s);
481 Returns the current maximum size for the disk cache.
483 \sa setMaximumCacheSize()
485 qint64 QNetworkDiskCache::maximumCacheSize() const
487 Q_D(const QNetworkDiskCache);
488 return d->maximumCacheSize;
492 Sets the maximum size of the disk cache to be \a size.
494 If the new size is smaller then the current cache size then the cache will call expire().
496 \sa maximumCacheSize()
498 void QNetworkDiskCache::setMaximumCacheSize(qint64 size)
500 Q_D(QNetworkDiskCache);
501 bool expireCache = (size < d->maximumCacheSize);
502 d->maximumCacheSize = size;
504 d->currentCacheSize = expire();
508 Cleans the cache so that its size is under the maximum cache size.
509 Returns the current size of the cache.
511 When the current size of the cache is greater than the maximumCacheSize()
512 older cache files are removed until the total size is less then 90% of
513 maximumCacheSize() starting with the oldest ones first using the file
514 creation date to determine how old a cache file is.
516 Subclasses can reimplement this function to change the order that cache
517 files are removed taking into account information in the application
518 knows about that QNetworkDiskCache does not, for example the number of times
521 Note: cacheSize() calls expire if the current cache size is unknown.
523 \sa maximumCacheSize(), fileMetaData()
525 qint64 QNetworkDiskCache::expire()
527 Q_D(QNetworkDiskCache);
528 if (d->currentCacheSize >= 0 && d->currentCacheSize < maximumCacheSize())
529 return d->currentCacheSize;
531 if (cacheDirectory().isEmpty()) {
532 qWarning() << "QNetworkDiskCache::expire() The cache directory is not set";
536 // close file handle to prevent "in use" error when QFile::remove() is called
539 QDir::Filters filters = QDir::AllDirs | QDir:: Files | QDir::NoDotAndDotDot;
540 QDirIterator it(cacheDirectory(), filters, QDirIterator::Subdirectories);
542 QMultiMap<QDateTime, QString> cacheItems;
543 qint64 totalSize = 0;
544 while (it.hasNext()) {
545 QString path = it.next();
546 QFileInfo info = it.fileInfo();
547 QString fileName = info.fileName();
548 if (fileName.endsWith(CACHE_POSTFIX)) {
549 cacheItems.insert(info.created(), path);
550 totalSize += info.size();
554 int removedFiles = 0;
555 qint64 goal = (maximumCacheSize() * 9) / 10;
556 QMultiMap<QDateTime, QString>::const_iterator i = cacheItems.constBegin();
557 while (i != cacheItems.constEnd()) {
558 if (totalSize < goal)
560 QString name = i.value();
562 qint64 size = file.size();
568 #if defined(QNETWORKDISKCACHE_DEBUG)
569 if (removedFiles > 0) {
570 qDebug() << "QNetworkDiskCache::expire()"
571 << "Removed:" << removedFiles
572 << "Kept:" << cacheItems.count() - removedFiles;
581 void QNetworkDiskCache::clear()
583 #if defined(QNETWORKDISKCACHE_DEBUG)
584 qDebug() << "QNetworkDiskCache::clear()";
586 Q_D(QNetworkDiskCache);
587 qint64 size = d->maximumCacheSize;
588 d->maximumCacheSize = 0;
589 d->currentCacheSize = expire();
590 d->maximumCacheSize = size;
594 Given a URL, generates a unique enough filename (and subdirectory)
596 QString QNetworkDiskCachePrivate::uniqueFileName(const QUrl &url)
599 cleanUrl.setPassword(QString());
600 cleanUrl.setFragment(QString());
602 QCryptographicHash hash(QCryptographicHash::Sha1);
603 hash.addData(cleanUrl.toEncoded());
604 // convert sha1 to base36 form and return first 8 bytes for use as string
605 QByteArray id = QByteArray::number(*(qlonglong*)hash.result().data(), 36).left(8);
606 // generates <one-char subdir>/<8-char filname.d>
607 uint code = (uint)id.at(id.length()-1) % 16;
608 QString pathFragment = QString::number(code, 16) + QLatin1Char('/')
609 + QLatin1String(id) + CACHE_POSTFIX;
614 QString QNetworkDiskCachePrivate::tmpCacheFileName() const
616 //The subdirectory is presumed to be already read for use.
617 return cacheDirectory + PREPARED_SLASH + QLatin1String("XXXXXX") + CACHE_POSTFIX;
621 Generates fully qualified path of cached resource from a URL.
623 QString QNetworkDiskCachePrivate::cacheFileName(const QUrl &url) const
628 QString fullpath = dataDirectory + uniqueFileName(url);
633 We compress small text and JavaScript files.
635 bool QCacheItem::canCompress() const
639 foreach (QNetworkCacheMetaData::RawHeader header, metaData.rawHeaders()) {
640 if (header.first.toLower() == "content-length") {
641 qint64 size = header.second.toLongLong();
642 if (size > MAX_COMPRESSION_SIZE)
648 if (header.first.toLower() == "content-type") {
649 QByteArray type = header.second;
650 if (type.startsWith("text/")
651 || (type.startsWith("application/")
652 && (type.endsWith("javascript") || type.endsWith("ecmascript"))))
657 if (sizeOk && typeOk)
666 CurrentCacheVersion = CACHE_VERSION
669 void QCacheItem::writeHeader(QFile *device) const
671 QDataStream out(device);
673 out << qint32(CacheMagic);
674 out << qint32(CurrentCacheVersion);
676 bool compressed = canCompress();
680 void QCacheItem::writeCompressedData(QFile *device) const
682 QDataStream out(device);
684 out << qCompress(data.data());
688 Returns false if the file is a cache file,
689 but is an older version and should be removed otherwise true.
691 bool QCacheItem::read(QFile *device, bool readData)
695 QDataStream in(device);
701 if (marker != CacheMagic)
704 // If the cache magic is correct, but the version is not we should remove it
705 if (v != CurrentCacheVersion)
712 if (readData && compressed) {
714 data.setData(qUncompress(dataBA));
715 data.open(QBuffer::ReadOnly);
718 // quick and dirty check if metadata's URL field and the file's name are in synch
719 QString expectedFilename = QNetworkDiskCachePrivate::uniqueFileName(metaData.url());
720 if (!device->fileName().endsWith(expectedFilename))
723 return metaData.isValid();
728 #endif // QT_NO_NETWORKDISKCACHE