From 093a92fb03e7c163882d1ba5fbdaf9fabb5bd969 Mon Sep 17 00:00:00 2001 From: Robert Hogan Date: Tue, 24 May 2011 11:04:27 +0200 Subject: [PATCH] Add QUrl::topLevelDomain() and move TLD table from QtNetwork to QtCore Move Qt's copy of the Mozilla public suffix list from QtNetwork to QtCore and use it to expose a new API function QUrl::topLevelDomain(). This function returns the section of the url that is a registrar-controlled top level domain. QtCore now exports a couple of functions to the other Qt modules: qTopLevelDomain, a helper function for QUrl::topLevelDomain(); and qIsEffectiveTLD(), a helper function for QNetworkCookeieJar. The motivation for this new API is to allow QtWebKit implement a Third-Party Cookie blocking policy. For this QtWebKit needs to know the element of the url that is the registry-controlled TLD. Without this knowledge it would end up blocking third-party cookies per host rather than per registry-controlled domain. See also https://bugs.webkit.org/show_bug.cgi?id=45455 Merge-request: 1205 Task-number: QTBUG-13601 Reviewed-by: Peter Hartmann (cherry picked from commit 154402f56dcf8303a6ce601a52215226af8d31ba) --- src/corelib/io/io.pri | 3 + src/corelib/io/qtldurl.cpp | 117 +++++++++++++++++++++ src/corelib/io/qtldurl_p.h | 66 ++++++++++++ src/corelib/io/qurl.cpp | 19 +++- src/corelib/io/qurl.h | 3 + .../io/qurltlds_p.h} | 10 +- .../io/qurltlds_p.h.INFO} | 0 src/network/access/access.pri | 1 - src/network/access/qnetworkcookiejar.cpp | 43 +------- src/network/access/qnetworkcookiejar_p.h | 3 - .../qnetworkcookiejar/tst_qnetworkcookiejar.cpp | 4 +- tests/auto/qurl/tst_qurl.cpp | 26 +++++ .../qurl-generateTLDs}/main.cpp | 4 +- .../qurl-generateTLDs/qurl-generateTLDs.pro} | 0 14 files changed, 244 insertions(+), 55 deletions(-) create mode 100644 src/corelib/io/qtldurl.cpp create mode 100644 src/corelib/io/qtldurl_p.h rename src/{network/access/qnetworkcookiejartlds_p.h => corelib/io/qurltlds_p.h} (99%) rename src/{network/access/qnetworkcookiejartlds_p.h.INFO => corelib/io/qurltlds_p.h.INFO} (100%) rename util/{network/cookiejar-generateTLDs => corelib/qurl-generateTLDs}/main.cpp (97%) rename util/{network/cookiejar-generateTLDs/cookiejar-generateTLDs.pro => corelib/qurl-generateTLDs/qurl-generateTLDs.pro} (100%) diff --git a/src/corelib/io/io.pri b/src/corelib/io/io.pri index f67600d..e411f8f 100644 --- a/src/corelib/io/io.pri +++ b/src/corelib/io/io.pri @@ -24,6 +24,8 @@ HEADERS += \ io/qresource_p.h \ io/qresource_iterator_p.h \ io/qurl.h \ + io/qurltlds_p.h \ + io/qtldurl_p.h \ io/qsettings.h \ io/qsettings_p.h \ io/qfsfileengine.h \ @@ -41,6 +43,7 @@ SOURCES += \ io/qbuffer.cpp \ io/qdatastream.cpp \ io/qdataurl.cpp \ + io/qtldurl.cpp \ io/qdebug.cpp \ io/qdir.cpp \ io/qdiriterator.cpp \ diff --git a/src/corelib/io/qtldurl.cpp b/src/corelib/io/qtldurl.cpp new file mode 100644 index 0000000..7db4bbd --- /dev/null +++ b/src/corelib/io/qtldurl.cpp @@ -0,0 +1,117 @@ +/**************************************************************************** +** +** Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies). +** All rights reserved. +** Contact: Nokia Corporation (qt-info@nokia.com) +** +** This file is part of the QtCore module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** No Commercial Usage +** This file contains pre-release code and may not be distributed. +** You may use this file in accordance with the terms and conditions +** contained in the Technology Preview License Agreement accompanying +** this package. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 2.1 requirements +** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Nokia gives you certain additional +** rights. These rights are described in the Nokia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** If you have questions regarding the use of this file, please contact +** Nokia at qt-info@nokia.com. +** +** +** +** +** +** +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "qplatformdefs.h" +#include "qurl.h" +#include "private/qurltlds_p.h" +#include "private/qtldurl_p.h" +#include "QtCore/qstringlist.h" + +QT_BEGIN_NAMESPACE + +static bool containsTLDEntry(const QString &entry) +{ + int index = qHash(entry) % tldCount; + int currentDomainIndex = tldIndices[index]; + while (currentDomainIndex < tldIndices[index+1]) { + QString currentEntry = QString::fromUtf8(tldData + currentDomainIndex); + if (currentEntry == entry) + return true; + currentDomainIndex += qstrlen(tldData + currentDomainIndex) + 1; // +1 for the ending \0 + } + return false; +} + +/*! + \internal + + Return the top-level-domain per Qt's copy of the Mozilla public suffix list of + \a domain. +*/ + +Q_CORE_EXPORT QString qTopLevelDomain(const QString &domain) +{ + QStringList sections = domain.toLower().split(QLatin1Char('.'), QString::SkipEmptyParts); + if (sections.isEmpty()) + return QString(); + + QString level, tld; + for (int j = sections.count() - 1; j >= 0; --j) { + level.prepend(QLatin1Char('.') + sections.at(j)); + if (qIsEffectiveTLD(level.right(level.size() - 1))) + tld = level; + } + return tld; +} + +/*! + \internal + + Return true if \a domain is a top-level-domain per Qt's copy of the Mozilla public suffix list. +*/ + +Q_CORE_EXPORT bool qIsEffectiveTLD(const QString &domain) +{ + // for domain 'foo.bar.com': + // 1. return if TLD table contains 'foo.bar.com' + if (containsTLDEntry(domain)) + return true; + + if (domain.contains(QLatin1Char('.'))) { + int count = domain.size() - domain.indexOf(QLatin1Char('.')); + QString wildCardDomain; + wildCardDomain.reserve(count + 1); + wildCardDomain.append(QLatin1Char('*')); + wildCardDomain.append(domain.right(count)); + // 2. if table contains '*.bar.com', + // test if table contains '!foo.bar.com' + if (containsTLDEntry(wildCardDomain)) { + QString exceptionDomain; + exceptionDomain.reserve(domain.size() + 1); + exceptionDomain.append(QLatin1Char('!')); + exceptionDomain.append(domain); + return (! containsTLDEntry(exceptionDomain)); + } + } + return false; +} + +QT_END_NAMESPACE diff --git a/src/corelib/io/qtldurl_p.h b/src/corelib/io/qtldurl_p.h new file mode 100644 index 0000000..152ffa0 --- /dev/null +++ b/src/corelib/io/qtldurl_p.h @@ -0,0 +1,66 @@ +/**************************************************************************** +** +** Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies). +** All rights reserved. +** Contact: Nokia Corporation (qt-info@nokia.com) +** +** This file is part of the QtCore module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** No Commercial Usage +** This file contains pre-release code and may not be distributed. +** You may use this file in accordance with the terms and conditions +** contained in the Technology Preview License Agreement accompanying +** this package. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 2.1 requirements +** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Nokia gives you certain additional +** rights. These rights are described in the Nokia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** If you have questions regarding the use of this file, please contact +** Nokia at qt-info@nokia.com. +** +** +** +** +** +** +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#ifndef QTLDURL_P_H +#define QTLDURL_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists for the convenience +// of qDecodeDataUrl. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "QtCore/qurl.h" +#include "QtCore/qstring.h" + +QT_BEGIN_NAMESPACE + +Q_CORE_EXPORT QString qTopLevelDomain(const QString &domain); +Q_CORE_EXPORT bool qIsEffectiveTLD(const QString &domain); + +QT_END_NAMESPACE + +#endif // QDATAURL_P_H diff --git a/src/corelib/io/qurl.cpp b/src/corelib/io/qurl.cpp index efd3f45..c433d35 100644 --- a/src/corelib/io/qurl.cpp +++ b/src/corelib/io/qurl.cpp @@ -192,7 +192,9 @@ #if defined QT3_SUPPORT #include "qfileinfo.h" #endif - +#ifndef QT_BOOTSTRAPPED +#include "qtldurl_p.h" +#endif #if defined(Q_OS_WINCE_WM) #pragma optimize("g", off) #endif @@ -5593,6 +5595,21 @@ bool QUrl::hasFragment() const } /*! + \since 4.8 + + Returns the TLD (Top-Level Domain) of the URL, (e.g. .co.uk, .net). + Note that the return value is prefixed with a '.' unless the + URL does not contain a valid TLD, in which case the function returns + an empty string. +*/ +#ifndef QT_BOOTSTRAPPED +QString QUrl::topLevelDomain() const +{ + return qTopLevelDomain(host()); +} +#endif + +/*! Returns the result of the merge of this URL with \a relative. This URL is used as a base to convert \a relative to an absolute URL. diff --git a/src/corelib/io/qurl.h b/src/corelib/io/qurl.h index 7534b8d..07b0b4e 100644 --- a/src/corelib/io/qurl.h +++ b/src/corelib/io/qurl.h @@ -181,6 +181,9 @@ public: void setEncodedFragment(const QByteArray &fragment); QByteArray encodedFragment() const; bool hasFragment() const; +#ifndef QT_BOOTSTRAPPED + QString topLevelDomain() const; +#endif QUrl resolved(const QUrl &relative) const; diff --git a/src/network/access/qnetworkcookiejartlds_p.h b/src/corelib/io/qurltlds_p.h similarity index 99% rename from src/network/access/qnetworkcookiejartlds_p.h rename to src/corelib/io/qurltlds_p.h index b06d881..f4f525c 100644 --- a/src/network/access/qnetworkcookiejartlds_p.h +++ b/src/corelib/io/qurltlds_p.h @@ -38,15 +38,15 @@ // the terms of any one of the MPL, the GPL or the LGPL. // -#ifndef QNETWORKCOOKIEJARTLD_P_H -#define QNETWORKCOOKIEJARTLD_P_H +#ifndef QURLTLD_P_H +#define QURLTLD_P_H // // W A R N I N G // ------------- // // This file is not part of the Qt API. It exists for the convenience -// of the Network Access framework. This header file may change from +// of the Network Access and Core framework. This header file may change from // version to version without notice, or even be removed. // // We mean it. @@ -57,7 +57,7 @@ QT_BEGIN_NAMESPACE // note to maintainer: // this file should be updated before each release -> // for instructions see the program at -// util/network/cookiejar-generateTLDs +// util/corelib/qurl-generateTLDs static const quint16 tldCount = 3949; static const quint16 tldIndices[] = { @@ -6478,4 +6478,4 @@ static const char tldData[] = { QT_END_NAMESPACE -#endif // QNETWORKCOOKIEJARTLD_P_H +#endif // QURLTLD_P_H diff --git a/src/network/access/qnetworkcookiejartlds_p.h.INFO b/src/corelib/io/qurltlds_p.h.INFO similarity index 100% rename from src/network/access/qnetworkcookiejartlds_p.h.INFO rename to src/corelib/io/qurltlds_p.h.INFO diff --git a/src/network/access/access.pri b/src/network/access/access.pri index 0f901b8..3d5558d 100644 --- a/src/network/access/access.pri +++ b/src/network/access/access.pri @@ -21,7 +21,6 @@ HEADERS += \ access/qnetworkcookie_p.h \ access/qnetworkcookiejar.h \ access/qnetworkcookiejar_p.h \ - access/qnetworkcookiejartlds_p.h \ access/qnetworkrequest.h \ access/qnetworkrequest_p.h \ access/qnetworkreply.h \ diff --git a/src/network/access/qnetworkcookiejar.cpp b/src/network/access/qnetworkcookiejar.cpp index 53fab9f..f2b1714 100644 --- a/src/network/access/qnetworkcookiejar.cpp +++ b/src/network/access/qnetworkcookiejar.cpp @@ -40,12 +40,12 @@ ****************************************************************************/ #include "qnetworkcookiejar.h" -#include "qnetworkcookiejartlds_p.h" #include "qnetworkcookiejar_p.h" #include "QtNetwork/qnetworkcookie.h" #include "QtCore/qurl.h" #include "QtCore/qdatetime.h" +#include "private/qtldurl_p.h" QT_BEGIN_NAMESPACE @@ -216,7 +216,7 @@ bool QNetworkCookieJar::setCookiesFromUrl(const QList &cookieLis // the check for effective TLDs makes the "embedded dot" rule from RFC 2109 section 4.3.2 // redundant; the "leading dot" rule has been relaxed anyway, see above // we remove the leading dot for this check - if (QNetworkCookieJarPrivate::isEffectiveTLD(domain.remove(0, 1))) + if (qIsEffectiveTLD(domain.remove(0, 1))) continue; // not accepted } @@ -304,43 +304,4 @@ QList QNetworkCookieJar::cookiesForUrl(const QUrl &url) const return result; } -bool QNetworkCookieJarPrivate::isEffectiveTLD(const QString &domain) -{ - // for domain 'foo.bar.com': - // 1. return if TLD table contains 'foo.bar.com' - if (containsTLDEntry(domain)) - return true; - - if (domain.contains(QLatin1Char('.'))) { - int count = domain.size() - domain.indexOf(QLatin1Char('.')); - QString wildCardDomain; - wildCardDomain.reserve(count + 1); - wildCardDomain.append(QLatin1Char('*')); - wildCardDomain.append(domain.right(count)); - // 2. if table contains '*.bar.com', - // test if table contains '!foo.bar.com' - if (containsTLDEntry(wildCardDomain)) { - QString exceptionDomain; - exceptionDomain.reserve(domain.size() + 1); - exceptionDomain.append(QLatin1Char('!')); - exceptionDomain.append(domain); - return (! containsTLDEntry(exceptionDomain)); - } - } - return false; -} - -bool QNetworkCookieJarPrivate::containsTLDEntry(const QString &entry) -{ - int index = qHash(entry) % tldCount; - int currentDomainIndex = tldIndices[index]; - while (currentDomainIndex < tldIndices[index+1]) { - QString currentEntry = QString::fromUtf8(tldData + currentDomainIndex); - if (currentEntry == entry) - return true; - currentDomainIndex += qstrlen(tldData + currentDomainIndex) + 1; // +1 for the ending \0 - } - return false; -} - QT_END_NAMESPACE diff --git a/src/network/access/qnetworkcookiejar_p.h b/src/network/access/qnetworkcookiejar_p.h index d6dc450..ce5a87a 100644 --- a/src/network/access/qnetworkcookiejar_p.h +++ b/src/network/access/qnetworkcookiejar_p.h @@ -63,9 +63,6 @@ class QNetworkCookieJarPrivate: public QObjectPrivate public: QList allCookies; - static bool Q_AUTOTEST_EXPORT isEffectiveTLD(const QString &domain); - static bool containsTLDEntry(const QString &entry); - Q_DECLARE_PUBLIC(QNetworkCookieJar) }; diff --git a/tests/auto/qnetworkcookiejar/tst_qnetworkcookiejar.cpp b/tests/auto/qnetworkcookiejar/tst_qnetworkcookiejar.cpp index 178f6d8..1f7c269 100644 --- a/tests/auto/qnetworkcookiejar/tst_qnetworkcookiejar.cpp +++ b/tests/auto/qnetworkcookiejar/tst_qnetworkcookiejar.cpp @@ -42,7 +42,7 @@ #include #include -#include "private/qnetworkcookiejar_p.h" +#include "private/qtldurl_p.h" class tst_QNetworkCookieJar: public QObject { @@ -438,7 +438,7 @@ void tst_QNetworkCookieJar::effectiveTLDs() #endif QFETCH(QString, domain); QFETCH(bool, isTLD); - QCOMPARE(QNetworkCookieJarPrivate::isEffectiveTLD(domain), isTLD); + QCOMPARE(qIsEffectiveTLD(domain), isTLD); } QTEST_MAIN(tst_QNetworkCookieJar) diff --git a/tests/auto/qurl/tst_qurl.cpp b/tests/auto/qurl/tst_qurl.cpp index 336ee36..85aae1f 100644 --- a/tests/auto/qurl/tst_qurl.cpp +++ b/tests/auto/qurl/tst_qurl.cpp @@ -49,6 +49,7 @@ #include #include #include +#include "private/qtldurl_p.h" // For testsuites #define IDNA_ACE_PREFIX "xn--" @@ -88,6 +89,8 @@ public slots: void init(); void cleanup(); private slots: + void effectiveTLDs_data(); + void effectiveTLDs(); void getSetCheck(); void constructing(); void assignment(); @@ -4005,5 +4008,28 @@ void tst_QUrl::taskQTBUG_8701() QCOMPARE(foo_uni_bar, QUrl(foo_uni_bar, QUrl::StrictMode).toString()); } +void tst_QUrl::effectiveTLDs_data() +{ + QTest::addColumn("domain"); + QTest::addColumn("TLD"); + + QTest::newRow("yes0") << QUrl::fromEncoded("http://test.co.uk") << ".co.uk"; + QTest::newRow("yes1") << QUrl::fromEncoded("http://test.com") << ".com"; + QTest::newRow("yes2") << QUrl::fromEncoded("http://www.test.de") << ".de"; + QTest::newRow("yes3") << QUrl::fromEncoded("http://test.ulm.museum") << ".ulm.museum"; + QTest::newRow("yes4") << QUrl::fromEncoded("http://www.com.krodsherad.no") << ".krodsherad.no"; + QTest::newRow("yes5") << QUrl::fromEncoded("http://www.co.uk.1.bg") << ".1.bg"; + QTest::newRow("yes6") << QUrl::fromEncoded("http://www.com.com.cn") << ".com.cn"; + QTest::newRow("yes7") << QUrl::fromEncoded("http://www.test.org.ws") << ".org.ws"; + QTest::newRow("yes9") << QUrl::fromEncoded("http://www.com.co.uk.wallonie.museum") << ".wallonie.museum"; +} + +void tst_QUrl::effectiveTLDs() +{ + QFETCH(QUrl, domain); + QFETCH(QString, TLD); + QCOMPARE(domain.topLevelDomain(), TLD); +} + QTEST_MAIN(tst_QUrl) #include "tst_qurl.moc" diff --git a/util/network/cookiejar-generateTLDs/main.cpp b/util/corelib/qurl-generateTLDs/main.cpp similarity index 97% rename from util/network/cookiejar-generateTLDs/main.cpp rename to util/corelib/qurl-generateTLDs/main.cpp index 5cdc97c..baaf256 100644 --- a/util/network/cookiejar-generateTLDs/main.cpp +++ b/util/corelib/qurl-generateTLDs/main.cpp @@ -77,7 +77,7 @@ int main(int argc, char **argv) { printf(" wget http://mxr.mozilla.org/mozilla-central/source/netwerk/dns/effective_tld_names.dat?raw=1 -O effective_tld_names.dat\n"); printf(" grep '^[^\\/\\/]' effective_tld_names.dat > effective_tld_names.dat.trimmed\n"); printf(" %s effective_tld_names.dat.trimmed effective_tld_names.dat.qt\n\n", argv[0]); - printf("Now copy the data from effective_tld_names.dat.qt to the file src/network/access/qnetworkcookiejartlds_p.h in your Qt repo\n\n"); + printf("Now copy the data from effective_tld_names.dat.qt to the file src/corelib/io/qurltlds_p.h in your Qt repo\n\n"); exit(1); } QFile file(argv[1]); @@ -156,6 +156,6 @@ int main(int argc, char **argv) { outFile.write(outDataBufferBA); outFile.write("};\n"); outFile.close(); - printf("data generated to %s . Now copy the data from this file to src/network/access/qnetworkcookiejartlds_p.h in your Qt repo\n", argv[2]); + printf("data generated to %s . Now copy the data from this file to src/corelib/io/qurltlds_p.h in your Qt repo\n", argv[2]); exit(0); } diff --git a/util/network/cookiejar-generateTLDs/cookiejar-generateTLDs.pro b/util/corelib/qurl-generateTLDs/qurl-generateTLDs.pro similarity index 100% rename from util/network/cookiejar-generateTLDs/cookiejar-generateTLDs.pro rename to util/corelib/qurl-generateTLDs/qurl-generateTLDs.pro -- 2.7.4