add mirrorlist-support

a metalink (must contain /metalink) or line-seperated mirrorlist
will be parsed, if there is no baseurl set

diff --git a/tests/data/Mirrorlist/remote-site/metalink.xml b/tests/data/Mirrorlist/remote-site/metalink.xml
new file mode 100644 (file)
index 0000000..5df588b
--- /dev/null
+++ b/tests/data/Mirrorlist/remote-site/metalink.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="utf-8"?>
+<metalink version="3.0" xmlns="http://www.metalinker.org/" type="dynamic" pubdate="Wed, 03 Nov 2010 13:30:11 GMT" generator="mirrormanager" xmlns:mm0="http://fedorahosted.org/mirrormanager">
+  <files>
+    <file name="repomd.xml">
+      <mm0:timestamp>1288734483</mm0:timestamp>
+      <size>4834</size>
+      <verification>
+        <hash type="md5">8fd7745c38277ac8b5618107edb72b7e</hash>
+        <hash type="sha1">de06c2b34f5b13fe6029da59475359675931eb9d</hash>
+        <hash type="sha256">d6f8135c9d5ac370fafd258c89cfb574989cd8557044d9551eefd1aaf2c54c48</hash>
+        <hash type="sha512">f43d747fa0134e9990297f98771f74a7a449f1b5a53a6936949af5e0b51f3b3834b69af3e25ae4dd7c97415d5570abb5c0a4588ddfae673228f46a83f200b5cd</hash>
+      </verification>
+      <mm0:alternates>
+        <mm0:alternate>
+            <mm0:timestamp>1288642986</mm0:timestamp>
+            <size>4834</size>
+            <verification>
+              <hash type="md5">bd32127a8b2ae7a370be3dbf876afbc7</hash>
+              <hash type="sha1">b4b9ea2494aefd82ff08915c28f6548a31625744</hash>
+              <hash type="sha256">b0d8ae9c43bfd06829092273b409cb0f6e2b581fafd370f11752842c0cd0c2c6</hash>
+              <hash type="sha512">78d2eff26e6a12bb1b1bbc78956f548734e921335943afabcfbf4f3ba6d164e8820eb2e685fad78088d2b42b7ec75fe4bde8ad6f5d7e9cf44e70837ce23fb758</hash>
+            </verification>
+        </mm0:alternate>
+      </mm0:alternates>
+      <resources maxconnections="1">
+        <url protocol="http" type="http" location="DE" preference="100" >http://ftp-stud.hs-esslingen.de/pub/fedora/linux/updates/13/x86_64/repodata/repomd.xml</url>
+        <url protocol="ftp" type="ftp" location="DE" preference="100" >ftp://ftp-stud.hs-esslingen.de/pub/fedora/linux/updates/13/x86_64/repodata/repomd.xml</url>
+        <url protocol="rsync" type="rsync" location="DE" preference="100" >rsync://ftp-stud.hs-esslingen.de/fedora/linux/updates/13/x86_64/repodata/repomd.xml</url>
+        <url protocol="http" type="http" location="BE" preference="45" >http://mirror.eurid.eu/fedora/linux/updates/13/x86_64/repodata/repomd.xml</url>
+        <url protocol="ftp" type="ftp" location="BE" preference="45" >ftp://mirror.eurid.eu/fedora/linux/updates/13/x86_64/repodata/repomd.xml</url>
+      </resources>
+    </file>
+  </files>
+</metalink>

diff --git a/tests/data/Mirrorlist/remote-site/mirrors.txt b/tests/data/Mirrorlist/remote-site/mirrors.txt
new file mode 100644 (file)
index 0000000..20a2168
--- /dev/null
+++ b/tests/data/Mirrorlist/remote-site/mirrors.txt
@@ -0,0 +1,5 @@
+http://ftp-stud.hs-esslingen.de/pub/fedora/linux/updates/13/x86_64/repodata/repomd.xml
+ftp://ftp-stud.hs-esslingen.de/pub/fedora/linux/updates/13/x86_64/repodata/repomd.xml
+rsync://ftp-stud.hs-esslingen.de/fedora/linux/updates/13/x86_64/repodata/repomd.xml
+http://mirror.eurid.eu/fedora/linux/updates/13/x86_64/repodata/repomd.xml
+ftp://mirror.eurid.eu/fedora/linux/updates/13/x86_64/repodata/repomd.xml

diff --git a/tests/repo/CMakeLists.txt b/tests/repo/CMakeLists.txt
index 55bb85e..3f06bc0 100644 (file)
--- a/tests/repo/CMakeLists.txt
+++ b/tests/repo/CMakeLists.txt
@@ -4,4 +4,4 @@ ADD_SUBDIRECTORY( susetags )
 # to find the KeyRingTest receiver
 INCLUDE_DIRECTORIES( ${LIBZYPP_SOURCE_DIR}/tests/zypp )
 
-ADD_TESTS(RepoVariables ExtendedMetadata PluginServices)
+ADD_TESTS(RepoVariables ExtendedMetadata PluginServices MirrorList)

diff --git a/tests/repo/MirrorList_test.cc b/tests/repo/MirrorList_test.cc
new file mode 100644 (file)
index 0000000..40c38a6
--- /dev/null
+++ b/tests/repo/MirrorList_test.cc
@@ -0,0 +1,34 @@
+#include <iostream>
+#include <vector>
+#include <boost/test/auto_unit_test.hpp>
+
+#include "WebServer.h"
+
+#include "zypp/repo/RepoMirrorList.cc"
+
+using namespace std;
+using namespace zypp;
+using namespace zypp::repo;
+
+BOOST_AUTO_TEST_CASE(get_mirrorlist)
+{
+  WebServer web((Pathname(TESTS_SRC_DIR) + "/data/Mirrorlist/remote-site").c_str(), 10001);
+  web.start();
+
+  Url weburl1 (web.url());
+  Url weburl2 (web.url());
+
+  weburl1.setPathName("/metalink.xml");
+  weburl2.setPathName("/mirrors.txt");
+
+  RepoMirrorList rml1 (weburl1);
+  RepoMirrorList rml2 (weburl2);
+
+  BOOST_CHECK(rml1.getUrls().begin()->asString() == "http://ftp-stud.hs-esslingen.de/pub/fedora/linux/updates/13/x86_64/");
+  BOOST_CHECK(rml2.getUrls().begin()->asString() == "http://ftp-stud.hs-esslingen.de/pub/fedora/linux/updates/13/x86_64/");
+
+  BOOST_CHECK(rml1.getUrls().size() == 4);
+  BOOST_CHECK(rml2.getUrls().size() == 4);
+
+  web.stop();
+}

diff --git a/tests/zypp/RepoInfo_test.cc b/tests/zypp/RepoInfo_test.cc
index c1bad6b..55ca91c 100644 (file)
--- a/tests/zypp/RepoInfo_test.cc
+++ b/tests/zypp/RepoInfo_test.cc
@@ -15,6 +15,8 @@
 
 #include "KeyRingTestReceiver.h"
 
+#include "WebServer.h"
+
 using boost::unit_test::test_suite;
 using boost::unit_test::test_case;
 using namespace boost::unit_test::log;
@@ -26,7 +28,22 @@ using namespace zypp::repo;
 
 BOOST_AUTO_TEST_CASE(repoinfo_test)
 {
+  WebServer web((Pathname(TESTS_SRC_DIR) + "/data/Mirrorlist/remote-site").c_str(), 10001);
+  web.start();
 
-}
+  Url weburl (web.url());
+  weburl.setPathName("/metalink.xml");
+
+  RepoInfo ri;
+
+  ri.setMirrorListUrl(weburl);
 
+  BOOST_CHECK(ri.url().asString() == "ftp://ftp-stud.hs-esslingen.de/pub/fedora/linux/updates/13/x86_64/");
 
+  ostringstream ostr;
+  ri.dumpAsIniOn(ostr);
+
+  BOOST_CHECK( ostr.str().find("baseurl=") == string::npos );
+
+  web.stop();
+}

diff --git a/zypp/CMakeLists.txt b/zypp/CMakeLists.txt
index c2b6463..d4f4682 100644 (file)
--- a/zypp/CMakeLists.txt
+++ b/zypp/CMakeLists.txt
@@ -702,6 +702,7 @@ INSTALL(  FILES
 
 SET( zypp_repo_SRCS
   repo/RepoException.cc
+  repo/RepoMirrorList.cc
   repo/RepoType.cc
   repo/ServiceType.cc
   repo/PackageProvider.cc
@@ -721,6 +722,7 @@ SET( zypp_repo_SRCS
 
 SET( zypp_repo_HEADERS
   repo/RepoException.h
+  repo/RepoMirrorList.h
   repo/RepoType.h
   repo/ServiceType.h
   repo/PackageProvider.h

diff --git a/zypp/RepoInfo.cc b/zypp/RepoInfo.cc
index 79c7a7a..c86b19f 100644 (file)
--- a/zypp/RepoInfo.cc
+++ b/zypp/RepoInfo.cc
@@ -10,6 +10,7 @@
  *
 */
 #include <iostream>
+#include <vector>
 
 #include "zypp/base/Logger.h"
 #include "zypp/base/DefaultIntegral.h"
@@ -17,6 +18,7 @@
 
 #include "zypp/RepoInfo.h"
 #include "zypp/repo/RepoInfoBaseImpl.h"
+#include "zypp/repo/RepoMirrorList.h"
 #include "zypp/ExternalProgram.h"
 #include "zypp/media/MediaAccess.h"
 
@@ -39,6 +41,7 @@ namespace zypp
       , gpgcheck(indeterminate)
       ,        keeppackages(indeterminate)
       , type(repo::RepoType::NONE_e)
+      , emptybaseurls(false)
     {}
 
     ~Impl()
@@ -61,23 +64,57 @@ namespace zypp
     Pathname licenseTgz() const
     { return metadatapath.empty() ? Pathname() : metadatapath / path / "license.tar.gz"; }
 
+    Url getmirrorListUrl() const
+    {
+      repo::RepoVariablesUrlReplacer replacer;
+      return replacer(mirrorlist_url);
+    }
+
+    Url &setmirrorListUrl()
+    {
+      return mirrorlist_url;
+    }
+
+    const std::set<Url> &baseUrls() const
+    {
+      if ( _baseUrls.empty() && ! (getmirrorListUrl().asString().empty()) )
+      {
+        emptybaseurls = true;
+        repo::RepoMirrorList rmirrorlist (getmirrorListUrl());
+        std::vector<Url> rmurls = rmirrorlist.getUrls();
+        _baseUrls.insert(rmurls.begin(), rmurls.end());
+      }
+      return _baseUrls;
+    }
+
+    std::set<Url> &baseUrls()
+    {
+      return _baseUrls;
+    }
+
+    bool baseurl2dump() const
+    {
+      return !emptybaseurls && !_baseUrls.empty();
+    }
+
 
   public:
     TriBool gpgcheck;
     TriBool keeppackages;
     Url gpgkey_url;
     repo::RepoType type;
-    Url mirrorlist_url;
-    std::set<Url> baseUrls;
     Pathname path;
     std::string service;
     std::string targetDistro;
     Pathname metadatapath;
     Pathname packagespath;
     DefaultIntegral<unsigned,defaultPriority> priority;
-  public:
+    mutable bool emptybaseurls;
 
   private:
+    Url mirrorlist_url;
+    mutable std::set<Url> _baseUrls;
+
     friend Impl * rwcowClone<Impl>( const Impl * rhs );
     /** clone for RWCOW_pointer */
     Impl * clone() const
@@ -134,7 +171,7 @@ namespace zypp
 
   void RepoInfo::setMirrorListUrl( const Url &url )
   {
-    _pimpl->mirrorlist_url = url;
+    _pimpl->setmirrorListUrl() = url;
   }
 
   void RepoInfo::setGpgKeyUrl( const Url &url )
@@ -144,12 +181,12 @@ namespace zypp
 
   void RepoInfo::addBaseUrl( const Url &url )
   {
-    _pimpl->baseUrls.insert(url);
+    _pimpl->baseUrls().insert(url);
   }
 
   void RepoInfo::setBaseUrl( const Url &url )
   {
-    _pimpl->baseUrls.clear();
+    _pimpl->baseUrls().clear();
     addBaseUrl(url);
   }
 
@@ -207,8 +244,7 @@ namespace zypp
 
   Url RepoInfo::mirrorListUrl() const
   {
-    repo::RepoVariablesUrlReplacer replacer;
-    return replacer(_pimpl->mirrorlist_url);
+    return _pimpl->getmirrorListUrl();
   }
 
   Url RepoInfo::gpgKeyUrl() const
@@ -218,8 +254,8 @@ namespace zypp
   {
     RepoInfo::url_set replaced_urls;
     repo::RepoVariablesUrlReplacer replacer;
-    for ( url_set::const_iterator it = _pimpl->baseUrls.begin();
-          it != _pimpl->baseUrls.end();
+    for ( url_set::const_iterator it = _pimpl->baseUrls().begin();
+          it != _pimpl->baseUrls().end();
           ++it )
     {
       replaced_urls.insert(replacer(*it));
@@ -238,7 +274,7 @@ namespace zypp
 
   RepoInfo::urls_const_iterator RepoInfo::baseUrlsBegin() const
   {
-    return make_transform_iterator( _pimpl->baseUrls.begin(),
+    return make_transform_iterator( _pimpl->baseUrls().begin(),
                                     repo::RepoVariablesUrlReplacer() );
     //return _pimpl->baseUrls.begin();
   }
@@ -246,22 +282,22 @@ namespace zypp
   RepoInfo::urls_const_iterator RepoInfo::baseUrlsEnd() const
   {
     //return _pimpl->baseUrls.end();
-    return make_transform_iterator( _pimpl->baseUrls.end(),
+    return make_transform_iterator( _pimpl->baseUrls().end(),
                                     repo::RepoVariablesUrlReplacer() );
   }
 
   RepoInfo::urls_size_type RepoInfo::baseUrlsSize() const
-  { return _pimpl->baseUrls.size(); }
+  { return _pimpl->baseUrls().size(); }
 
   bool RepoInfo::baseUrlsEmpty() const
-  { return _pimpl->baseUrls.empty(); }
+  { return _pimpl->baseUrls().empty(); }
 
   // false by default (if not set by setKeepPackages)
   bool RepoInfo::keepPackages() const
   {
     if (indeterminate(_pimpl->keeppackages))
     {
-      if (_pimpl->baseUrls.empty())
+      if (_pimpl->baseUrls().empty())
         return false;
       else if ( baseUrlsBegin()->schemeIsDownloading() )
         return true;
@@ -365,11 +401,18 @@ namespace zypp
   std::ostream & RepoInfo::dumpOn( std::ostream & str ) const
   {
     RepoInfoBase::dumpOn(str);
-    for ( urls_const_iterator it = baseUrlsBegin();
-          it != baseUrlsEnd();
-          ++it )
+    if ( _pimpl->baseurl2dump() )
     {
-      str << "- url         : " << *it << std::endl;
+      for ( urls_const_iterator it = baseUrlsBegin();
+            it != baseUrlsEnd();
+            ++it )
+      {
+        str << "- url         : " << *it << std::endl;
+      }
+    }
+    if ( ! (_pimpl->getmirrorListUrl().asString().empty())  )
+    {
+      str << "- mirrorlist  : " << _pimpl->getmirrorListUrl() << std::endl;
     }
     str << "- path        : " << path() << std::endl;
     str << "- type        : " << type() << std::endl;
@@ -396,20 +439,22 @@ namespace zypp
   {
     RepoInfoBase::dumpAsIniOn(str);
 
-    if ( ! _pimpl->baseUrls.empty() )
-      str << "baseurl=";
-    for ( url_set::const_iterator it = _pimpl->baseUrls.begin();
-          it != _pimpl->baseUrls.end();
-          ++it )
+    if ( _pimpl->baseurl2dump() )
     {
-      str << *it << endl;
+      str << "baseurl=";
+      for ( url_set::const_iterator it = _pimpl->baseUrls().begin();
+            it != _pimpl->baseUrls().end();
+            ++it )
+      {
+        str << *it << endl;
+      }
     }
 
     if ( ! _pimpl->path.empty() )
       str << "path="<< path() << endl;
 
-    if ( ! (_pimpl->mirrorlist_url.asString().empty()) )
-      str << "mirrorlist=" << _pimpl->mirrorlist_url << endl;
+    if ( ! (_pimpl->getmirrorListUrl().asString().empty()) )
+      str << "mirrorlist=" << _pimpl->getmirrorListUrl() << endl;
 
     str << "type=" << type().asString() << endl;
 
@@ -452,9 +497,12 @@ namespace zypp
       str << " mirrorlist=\"" << escape(tmpstr) << "\"";
     str << ">" << endl;
 
-    for (RepoInfo::urls_const_iterator urlit = baseUrlsBegin();
-         urlit != baseUrlsEnd(); ++urlit)
-      str << "<url>" << escape(urlit->asString()) << "</url>" << endl;
+    if ( _pimpl->baseurl2dump() )
+    {
+      for (RepoInfo::urls_const_iterator urlit = baseUrlsBegin();
+           urlit != baseUrlsEnd(); ++urlit)
+        str << "<url>" << escape(urlit->asString()) << "</url>" << endl;
+    }
 
     str << "</repo>" << endl;
     return str;

diff --git a/zypp/repo/RepoMirrorList.cc b/zypp/repo/RepoMirrorList.cc
new file mode 100644 (file)
index 0000000..750b607
--- /dev/null
+++ b/zypp/repo/RepoMirrorList.cc
@@ -0,0 +1,89 @@
+/*---------------------------------------------------------------------\
+|                          ____ _   __ __ ___                          |
+|                         |__  / \ / / . \ . \                         |
+|                           / / \ V /|  _/  _/                         |
+|                          / /__ | | | | | |                           |
+|                         /_____||_| |_| |_|                           |
+|                                                                      |
+\---------------------------------------------------------------------*/
+/** \file      zypp/repo/RepoMirrorList.cc
+ *
+*/
+
+#include <iostream>
+#include <vector>
+#include "zypp/repo/RepoMirrorList.h"
+#include "zypp/media/MetaLinkParser.h"
+#include "zypp/MediaSetAccess.h"
+#include "zypp/base/LogTools.h"
+
+using namespace std;
+
+///////////////////////////////////////////////////////////////////
+namespace zypp
+{ /////////////////////////////////////////////////////////////////
+  ///////////////////////////////////////////////////////////////////
+  namespace repo
+  { /////////////////////////////////////////////////////////////////
+
+    RepoMirrorList::RepoMirrorList( const Url &url )
+    {
+      Pathname filepath (url.getPathName());
+      Url abs_url (url);
+      std::vector<Url> my_urls;
+
+      DBG << "Getting MirrorList from URL: " << abs_url << endl;
+
+      abs_url.setPathName("");
+      abs_url.setQueryParam("mediahandler", "curl");
+
+      MediaSetAccess access (abs_url);
+      Pathname tmpfile = access.provideFile(filepath);
+
+      InputStream tmpfstream (tmpfile);
+
+      if ( url.asString().find("/metalink") != string::npos )
+      {
+        media::MetaLinkParser metalink;
+        metalink.parse(tmpfstream);
+        my_urls = metalink.getUrls();
+      }
+      else
+      {
+        string tmpurl;
+        while (getline(tmpfstream.stream(), tmpurl))
+        {
+          my_urls.push_back(Url(tmpurl));
+        }
+      }
+
+      int valid_urls = 0;
+      for (std::vector<Url>::iterator it = my_urls.begin() ; it != my_urls.end() and valid_urls < 4 ; ++it)
+      {
+        if ( it->getScheme() != "rsync" )
+        {
+          size_t delpos = it->getPathName().find("repodata/repomd.xml");
+          if( delpos != string::npos )
+          {
+            it->setPathName( it->getPathName().erase(delpos)  );
+          }
+          urls.push_back(*it);
+          ++valid_urls;
+        }
+      }
+    }
+    
+    std::vector<Url> RepoMirrorList::getUrls() const
+    {
+      return urls;
+    }
+
+    RepoMirrorList::~RepoMirrorList()
+    {}
+
+   /////////////////////////////////////////////////////////////////
+  } // namespace repo
+  ///////////////////////////////////////////////////////////////////
+  /////////////////////////////////////////////////////////////////
+} // namespace zypp
+///////////////////////////////////////////////////////////////////

diff --git a/zypp/repo/RepoMirrorList.h b/zypp/repo/RepoMirrorList.h
new file mode 100644 (file)
index 0000000..49a0efd
--- /dev/null
+++ b/zypp/repo/RepoMirrorList.h
@@ -0,0 +1,37 @@
+/*---------------------------------------------------------------------\
+|                          ____ _   __ __ ___                          |
+|                         |__  / \ / / . \ . \                         |
+|                           / / \ V /|  _/  _/                         |
+|                          / /__ | | | | | |                           |
+|                         /_____||_| |_| |_|                           |
+|                                                                      |
+\---------------------------------------------------------------------*/
+
+#ifndef ZYPP_REPO_MIRRORLIST_H_
+#define ZYPP_REPO_MIRRORLIST_H_
+
+#include <vector>
+#include "zypp/Url.h"
+
+namespace zypp
+{
+  namespace repo
+  {
+    class RepoMirrorList
+    {
+      public:
+        RepoMirrorList( const Url &url );
+        virtual ~RepoMirrorList();
+        
+        std::vector<Url> getUrls() const;
+
+      private:
+        std::vector<Url> urls;
+    };
+
+  } // ns repo
+} // ns zypp
+
+#endif
+
+// vim: set ts=2 sts=2 sw=2 et ai: