From 741edd350ae39d55e0c7237227885435334e8de9 Mon Sep 17 00:00:00 2001 From: Dominik Heidler Date: Wed, 3 Nov 2010 16:04:39 +0100 Subject: [PATCH] add mirrorlist-support a metalink (must contain /metalink) or line-seperated mirrorlist will be parsed, if there is no baseurl set --- tests/data/Mirrorlist/remote-site/metalink.xml | 34 ++++++++ tests/data/Mirrorlist/remote-site/mirrors.txt | 5 ++ tests/repo/CMakeLists.txt | 2 +- tests/repo/MirrorList_test.cc | 34 ++++++++ tests/zypp/RepoInfo_test.cc | 19 ++++- zypp/CMakeLists.txt | 2 + zypp/RepoInfo.cc | 108 ++++++++++++++++++------- zypp/repo/RepoMirrorList.cc | 89 ++++++++++++++++++++ zypp/repo/RepoMirrorList.h | 37 +++++++++ 9 files changed, 298 insertions(+), 32 deletions(-) create mode 100644 tests/data/Mirrorlist/remote-site/metalink.xml create mode 100644 tests/data/Mirrorlist/remote-site/mirrors.txt create mode 100644 tests/repo/MirrorList_test.cc create mode 100644 zypp/repo/RepoMirrorList.cc create mode 100644 zypp/repo/RepoMirrorList.h diff --git a/tests/data/Mirrorlist/remote-site/metalink.xml b/tests/data/Mirrorlist/remote-site/metalink.xml new file mode 100644 index 0000000..5df588b --- /dev/null +++ b/tests/data/Mirrorlist/remote-site/metalink.xml @@ -0,0 +1,34 @@ + + + + + 1288734483 + 4834 + + 8fd7745c38277ac8b5618107edb72b7e + de06c2b34f5b13fe6029da59475359675931eb9d + d6f8135c9d5ac370fafd258c89cfb574989cd8557044d9551eefd1aaf2c54c48 + f43d747fa0134e9990297f98771f74a7a449f1b5a53a6936949af5e0b51f3b3834b69af3e25ae4dd7c97415d5570abb5c0a4588ddfae673228f46a83f200b5cd + + + + 1288642986 + 4834 + + bd32127a8b2ae7a370be3dbf876afbc7 + b4b9ea2494aefd82ff08915c28f6548a31625744 + b0d8ae9c43bfd06829092273b409cb0f6e2b581fafd370f11752842c0cd0c2c6 + 78d2eff26e6a12bb1b1bbc78956f548734e921335943afabcfbf4f3ba6d164e8820eb2e685fad78088d2b42b7ec75fe4bde8ad6f5d7e9cf44e70837ce23fb758 + + + + + http://ftp-stud.hs-esslingen.de/pub/fedora/linux/updates/13/x86_64/repodata/repomd.xml + ftp://ftp-stud.hs-esslingen.de/pub/fedora/linux/updates/13/x86_64/repodata/repomd.xml + rsync://ftp-stud.hs-esslingen.de/fedora/linux/updates/13/x86_64/repodata/repomd.xml + http://mirror.eurid.eu/fedora/linux/updates/13/x86_64/repodata/repomd.xml + ftp://mirror.eurid.eu/fedora/linux/updates/13/x86_64/repodata/repomd.xml + + + + diff --git a/tests/data/Mirrorlist/remote-site/mirrors.txt b/tests/data/Mirrorlist/remote-site/mirrors.txt new file mode 100644 index 0000000..20a2168 --- /dev/null +++ b/tests/data/Mirrorlist/remote-site/mirrors.txt @@ -0,0 +1,5 @@ +http://ftp-stud.hs-esslingen.de/pub/fedora/linux/updates/13/x86_64/repodata/repomd.xml +ftp://ftp-stud.hs-esslingen.de/pub/fedora/linux/updates/13/x86_64/repodata/repomd.xml +rsync://ftp-stud.hs-esslingen.de/fedora/linux/updates/13/x86_64/repodata/repomd.xml +http://mirror.eurid.eu/fedora/linux/updates/13/x86_64/repodata/repomd.xml +ftp://mirror.eurid.eu/fedora/linux/updates/13/x86_64/repodata/repomd.xml diff --git a/tests/repo/CMakeLists.txt b/tests/repo/CMakeLists.txt index 55bb85e..3f06bc0 100644 --- a/tests/repo/CMakeLists.txt +++ b/tests/repo/CMakeLists.txt @@ -4,4 +4,4 @@ ADD_SUBDIRECTORY( susetags ) # to find the KeyRingTest receiver INCLUDE_DIRECTORIES( ${LIBZYPP_SOURCE_DIR}/tests/zypp ) -ADD_TESTS(RepoVariables ExtendedMetadata PluginServices) +ADD_TESTS(RepoVariables ExtendedMetadata PluginServices MirrorList) diff --git a/tests/repo/MirrorList_test.cc b/tests/repo/MirrorList_test.cc new file mode 100644 index 0000000..40c38a6 --- /dev/null +++ b/tests/repo/MirrorList_test.cc @@ -0,0 +1,34 @@ +#include +#include +#include + +#include "WebServer.h" + +#include "zypp/repo/RepoMirrorList.cc" + +using namespace std; +using namespace zypp; +using namespace zypp::repo; + +BOOST_AUTO_TEST_CASE(get_mirrorlist) +{ + WebServer web((Pathname(TESTS_SRC_DIR) + "/data/Mirrorlist/remote-site").c_str(), 10001); + web.start(); + + Url weburl1 (web.url()); + Url weburl2 (web.url()); + + weburl1.setPathName("/metalink.xml"); + weburl2.setPathName("/mirrors.txt"); + + RepoMirrorList rml1 (weburl1); + RepoMirrorList rml2 (weburl2); + + BOOST_CHECK(rml1.getUrls().begin()->asString() == "http://ftp-stud.hs-esslingen.de/pub/fedora/linux/updates/13/x86_64/"); + BOOST_CHECK(rml2.getUrls().begin()->asString() == "http://ftp-stud.hs-esslingen.de/pub/fedora/linux/updates/13/x86_64/"); + + BOOST_CHECK(rml1.getUrls().size() == 4); + BOOST_CHECK(rml2.getUrls().size() == 4); + + web.stop(); +} diff --git a/tests/zypp/RepoInfo_test.cc b/tests/zypp/RepoInfo_test.cc index c1bad6b..55ca91c 100644 --- a/tests/zypp/RepoInfo_test.cc +++ b/tests/zypp/RepoInfo_test.cc @@ -15,6 +15,8 @@ #include "KeyRingTestReceiver.h" +#include "WebServer.h" + using boost::unit_test::test_suite; using boost::unit_test::test_case; using namespace boost::unit_test::log; @@ -26,7 +28,22 @@ using namespace zypp::repo; BOOST_AUTO_TEST_CASE(repoinfo_test) { + WebServer web((Pathname(TESTS_SRC_DIR) + "/data/Mirrorlist/remote-site").c_str(), 10001); + web.start(); -} + Url weburl (web.url()); + weburl.setPathName("/metalink.xml"); + + RepoInfo ri; + + ri.setMirrorListUrl(weburl); + BOOST_CHECK(ri.url().asString() == "ftp://ftp-stud.hs-esslingen.de/pub/fedora/linux/updates/13/x86_64/"); + ostringstream ostr; + ri.dumpAsIniOn(ostr); + + BOOST_CHECK( ostr.str().find("baseurl=") == string::npos ); + + web.stop(); +} diff --git a/zypp/CMakeLists.txt b/zypp/CMakeLists.txt index c2b6463..d4f4682 100644 --- a/zypp/CMakeLists.txt +++ b/zypp/CMakeLists.txt @@ -702,6 +702,7 @@ INSTALL( FILES SET( zypp_repo_SRCS repo/RepoException.cc + repo/RepoMirrorList.cc repo/RepoType.cc repo/ServiceType.cc repo/PackageProvider.cc @@ -721,6 +722,7 @@ SET( zypp_repo_SRCS SET( zypp_repo_HEADERS repo/RepoException.h + repo/RepoMirrorList.h repo/RepoType.h repo/ServiceType.h repo/PackageProvider.h diff --git a/zypp/RepoInfo.cc b/zypp/RepoInfo.cc index 79c7a7a..c86b19f 100644 --- a/zypp/RepoInfo.cc +++ b/zypp/RepoInfo.cc @@ -10,6 +10,7 @@ * */ #include +#include #include "zypp/base/Logger.h" #include "zypp/base/DefaultIntegral.h" @@ -17,6 +18,7 @@ #include "zypp/RepoInfo.h" #include "zypp/repo/RepoInfoBaseImpl.h" +#include "zypp/repo/RepoMirrorList.h" #include "zypp/ExternalProgram.h" #include "zypp/media/MediaAccess.h" @@ -39,6 +41,7 @@ namespace zypp , gpgcheck(indeterminate) , keeppackages(indeterminate) , type(repo::RepoType::NONE_e) + , emptybaseurls(false) {} ~Impl() @@ -61,23 +64,57 @@ namespace zypp Pathname licenseTgz() const { return metadatapath.empty() ? Pathname() : metadatapath / path / "license.tar.gz"; } + Url getmirrorListUrl() const + { + repo::RepoVariablesUrlReplacer replacer; + return replacer(mirrorlist_url); + } + + Url &setmirrorListUrl() + { + return mirrorlist_url; + } + + const std::set &baseUrls() const + { + if ( _baseUrls.empty() && ! (getmirrorListUrl().asString().empty()) ) + { + emptybaseurls = true; + repo::RepoMirrorList rmirrorlist (getmirrorListUrl()); + std::vector rmurls = rmirrorlist.getUrls(); + _baseUrls.insert(rmurls.begin(), rmurls.end()); + } + return _baseUrls; + } + + std::set &baseUrls() + { + return _baseUrls; + } + + bool baseurl2dump() const + { + return !emptybaseurls && !_baseUrls.empty(); + } + public: TriBool gpgcheck; TriBool keeppackages; Url gpgkey_url; repo::RepoType type; - Url mirrorlist_url; - std::set baseUrls; Pathname path; std::string service; std::string targetDistro; Pathname metadatapath; Pathname packagespath; DefaultIntegral priority; - public: + mutable bool emptybaseurls; private: + Url mirrorlist_url; + mutable std::set _baseUrls; + friend Impl * rwcowClone( const Impl * rhs ); /** clone for RWCOW_pointer */ Impl * clone() const @@ -134,7 +171,7 @@ namespace zypp void RepoInfo::setMirrorListUrl( const Url &url ) { - _pimpl->mirrorlist_url = url; + _pimpl->setmirrorListUrl() = url; } void RepoInfo::setGpgKeyUrl( const Url &url ) @@ -144,12 +181,12 @@ namespace zypp void RepoInfo::addBaseUrl( const Url &url ) { - _pimpl->baseUrls.insert(url); + _pimpl->baseUrls().insert(url); } void RepoInfo::setBaseUrl( const Url &url ) { - _pimpl->baseUrls.clear(); + _pimpl->baseUrls().clear(); addBaseUrl(url); } @@ -207,8 +244,7 @@ namespace zypp Url RepoInfo::mirrorListUrl() const { - repo::RepoVariablesUrlReplacer replacer; - return replacer(_pimpl->mirrorlist_url); + return _pimpl->getmirrorListUrl(); } Url RepoInfo::gpgKeyUrl() const @@ -218,8 +254,8 @@ namespace zypp { RepoInfo::url_set replaced_urls; repo::RepoVariablesUrlReplacer replacer; - for ( url_set::const_iterator it = _pimpl->baseUrls.begin(); - it != _pimpl->baseUrls.end(); + for ( url_set::const_iterator it = _pimpl->baseUrls().begin(); + it != _pimpl->baseUrls().end(); ++it ) { replaced_urls.insert(replacer(*it)); @@ -238,7 +274,7 @@ namespace zypp RepoInfo::urls_const_iterator RepoInfo::baseUrlsBegin() const { - return make_transform_iterator( _pimpl->baseUrls.begin(), + return make_transform_iterator( _pimpl->baseUrls().begin(), repo::RepoVariablesUrlReplacer() ); //return _pimpl->baseUrls.begin(); } @@ -246,22 +282,22 @@ namespace zypp RepoInfo::urls_const_iterator RepoInfo::baseUrlsEnd() const { //return _pimpl->baseUrls.end(); - return make_transform_iterator( _pimpl->baseUrls.end(), + return make_transform_iterator( _pimpl->baseUrls().end(), repo::RepoVariablesUrlReplacer() ); } RepoInfo::urls_size_type RepoInfo::baseUrlsSize() const - { return _pimpl->baseUrls.size(); } + { return _pimpl->baseUrls().size(); } bool RepoInfo::baseUrlsEmpty() const - { return _pimpl->baseUrls.empty(); } + { return _pimpl->baseUrls().empty(); } // false by default (if not set by setKeepPackages) bool RepoInfo::keepPackages() const { if (indeterminate(_pimpl->keeppackages)) { - if (_pimpl->baseUrls.empty()) + if (_pimpl->baseUrls().empty()) return false; else if ( baseUrlsBegin()->schemeIsDownloading() ) return true; @@ -365,11 +401,18 @@ namespace zypp std::ostream & RepoInfo::dumpOn( std::ostream & str ) const { RepoInfoBase::dumpOn(str); - for ( urls_const_iterator it = baseUrlsBegin(); - it != baseUrlsEnd(); - ++it ) + if ( _pimpl->baseurl2dump() ) { - str << "- url : " << *it << std::endl; + for ( urls_const_iterator it = baseUrlsBegin(); + it != baseUrlsEnd(); + ++it ) + { + str << "- url : " << *it << std::endl; + } + } + if ( ! (_pimpl->getmirrorListUrl().asString().empty()) ) + { + str << "- mirrorlist : " << _pimpl->getmirrorListUrl() << std::endl; } str << "- path : " << path() << std::endl; str << "- type : " << type() << std::endl; @@ -396,20 +439,22 @@ namespace zypp { RepoInfoBase::dumpAsIniOn(str); - if ( ! _pimpl->baseUrls.empty() ) - str << "baseurl="; - for ( url_set::const_iterator it = _pimpl->baseUrls.begin(); - it != _pimpl->baseUrls.end(); - ++it ) + if ( _pimpl->baseurl2dump() ) { - str << *it << endl; + str << "baseurl="; + for ( url_set::const_iterator it = _pimpl->baseUrls().begin(); + it != _pimpl->baseUrls().end(); + ++it ) + { + str << *it << endl; + } } if ( ! _pimpl->path.empty() ) str << "path="<< path() << endl; - if ( ! (_pimpl->mirrorlist_url.asString().empty()) ) - str << "mirrorlist=" << _pimpl->mirrorlist_url << endl; + if ( ! (_pimpl->getmirrorListUrl().asString().empty()) ) + str << "mirrorlist=" << _pimpl->getmirrorListUrl() << endl; str << "type=" << type().asString() << endl; @@ -452,9 +497,12 @@ namespace zypp str << " mirrorlist=\"" << escape(tmpstr) << "\""; str << ">" << endl; - for (RepoInfo::urls_const_iterator urlit = baseUrlsBegin(); - urlit != baseUrlsEnd(); ++urlit) - str << "" << escape(urlit->asString()) << "" << endl; + if ( _pimpl->baseurl2dump() ) + { + for (RepoInfo::urls_const_iterator urlit = baseUrlsBegin(); + urlit != baseUrlsEnd(); ++urlit) + str << "" << escape(urlit->asString()) << "" << endl; + } str << "" << endl; return str; diff --git a/zypp/repo/RepoMirrorList.cc b/zypp/repo/RepoMirrorList.cc new file mode 100644 index 0000000..750b607 --- /dev/null +++ b/zypp/repo/RepoMirrorList.cc @@ -0,0 +1,89 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ +/** \file zypp/repo/RepoMirrorList.cc + * +*/ + +#include +#include +#include "zypp/repo/RepoMirrorList.h" +#include "zypp/media/MetaLinkParser.h" +#include "zypp/MediaSetAccess.h" +#include "zypp/base/LogTools.h" + +using namespace std; + +/////////////////////////////////////////////////////////////////// +namespace zypp +{ ///////////////////////////////////////////////////////////////// + /////////////////////////////////////////////////////////////////// + namespace repo + { ///////////////////////////////////////////////////////////////// + + RepoMirrorList::RepoMirrorList( const Url &url ) + { + Pathname filepath (url.getPathName()); + Url abs_url (url); + std::vector my_urls; + + DBG << "Getting MirrorList from URL: " << abs_url << endl; + + abs_url.setPathName(""); + abs_url.setQueryParam("mediahandler", "curl"); + + MediaSetAccess access (abs_url); + Pathname tmpfile = access.provideFile(filepath); + + InputStream tmpfstream (tmpfile); + + if ( url.asString().find("/metalink") != string::npos ) + { + media::MetaLinkParser metalink; + metalink.parse(tmpfstream); + my_urls = metalink.getUrls(); + } + else + { + string tmpurl; + while (getline(tmpfstream.stream(), tmpurl)) + { + my_urls.push_back(Url(tmpurl)); + } + } + + int valid_urls = 0; + for (std::vector::iterator it = my_urls.begin() ; it != my_urls.end() and valid_urls < 4 ; ++it) + { + if ( it->getScheme() != "rsync" ) + { + size_t delpos = it->getPathName().find("repodata/repomd.xml"); + if( delpos != string::npos ) + { + it->setPathName( it->getPathName().erase(delpos) ); + } + urls.push_back(*it); + ++valid_urls; + } + } + } + + std::vector RepoMirrorList::getUrls() const + { + return urls; + } + + RepoMirrorList::~RepoMirrorList() + {} + + ///////////////////////////////////////////////////////////////// + } // namespace repo + /////////////////////////////////////////////////////////////////// + ///////////////////////////////////////////////////////////////// +} // namespace zypp +/////////////////////////////////////////////////////////////////// diff --git a/zypp/repo/RepoMirrorList.h b/zypp/repo/RepoMirrorList.h new file mode 100644 index 0000000..49a0efd --- /dev/null +++ b/zypp/repo/RepoMirrorList.h @@ -0,0 +1,37 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ + +#ifndef ZYPP_REPO_MIRRORLIST_H_ +#define ZYPP_REPO_MIRRORLIST_H_ + +#include +#include "zypp/Url.h" + +namespace zypp +{ + namespace repo + { + class RepoMirrorList + { + public: + RepoMirrorList( const Url &url ); + virtual ~RepoMirrorList(); + + std::vector getUrls() const; + + private: + std::vector urls; + }; + + } // ns repo +} // ns zypp + +#endif + +// vim: set ts=2 sts=2 sw=2 et ai: -- 2.7.4