zypp/RepoManager.cc

   1 /*---------------------------------------------------------------------\
   2 |                          ____ _   __ __ ___                          |
   3 |                         |__  / \ / / . \ . \                         |
   4 |                           / / \ V /|  _/  _/                         |
   5 |                          / /__ | | | | | |                           |
   6 |                         /_____||_| |_| |_|                           |
   7 |                                                                      |
   8 \---------------------------------------------------------------------*/
   9 /** \file       zypp/RepoManager.cc
  10  *
  11 */
  12
  13 #include <cstdlib>
  14 #include <iostream>
  15 #include <fstream>
  16 #include <sstream>
  17 #include <list>
  18 #include <algorithm>
  19 #include "zypp/base/InputStream.h"
  20 #include "zypp/base/Logger.h"
  21 #include "zypp/base/Gettext.h"
  22 #include "zypp/base/Function.h"
  23 #include "zypp/base/Regex.h"
  24 #include "zypp/PathInfo.h"
  25 #include "zypp/TmpPath.h"
  26
  27 #include "zypp/repo/RepoException.h"
  28 #include "zypp/RepoManager.h"
  29
  30 #include "zypp/cache/SolvStore.h"
  31 #include "zypp/repo/cached/RepoImpl.h"
  32 #include "zypp/media/MediaManager.h"
  33 #include "zypp/MediaSetAccess.h"
  34 #include "zypp/ExternalProgram.h"
  35
  36 #include "zypp/parser/RepoFileReader.h"
  37 #include "zypp/repo/yum/Downloader.h"
  38 #include "zypp/parser/yum/RepoParser.h"
  39 //#include "zypp/parser/plaindir/RepoParser.h"
  40 #include "zypp/repo/susetags/Downloader.h"
  41 #include "zypp/parser/susetags/RepoParser.h"
  42
  43 #include "zypp/ZYppCallbacks.h"
  44
  45 #include "sat/Pool.h"
  46 #include "satsolver/pool.h"
  47 #include "satsolver/repo.h"
  48 #include "satsolver/repo_solv.h"
  49
  50 using namespace std;
  51 using namespace zypp;
  52 using namespace zypp::repo;
  53 using namespace zypp::filesystem;
  54
  55 using namespace zypp::repo;
  56
  57 ///////////////////////////////////////////////////////////////////
  58 namespace zypp
  59 { /////////////////////////////////////////////////////////////////
  60
  61   ///////////////////////////////////////////////////////////////////
  62   //
  63   //    CLASS NAME : RepoManagerOptions
  64   //
  65   ///////////////////////////////////////////////////////////////////
  66
  67   RepoManagerOptions::RepoManagerOptions()
  68   {
  69     repoCachePath    = ZConfig::instance().repoCachePath();
  70     repoRawCachePath = ZConfig::instance().repoMetadataPath();
  71     knownReposPath   = ZConfig::instance().knownReposPath();
  72   }
  73
  74   ////////////////////////////////////////////////////////////////////////////
  75
  76   /**
  77     * \short Simple callback to collect the results
  78     *
  79     * Classes like RepoFileParser call the callback
  80     * once per each repo in a file.
  81     *
  82     * Passing this functor as callback, you can collect
  83     * all resuls at the end, without dealing with async
  84     * code.
  85     */
  86     struct RepoCollector
  87     {
  88       RepoCollector()
  89       {
  90         MIL << endl;
  91       }
  92
  93       ~RepoCollector()
  94       {
  95         MIL << endl;
  96       }
  97
  98       bool collect( const RepoInfo &repo )
  99       {
 100         //MIL << "here in collector: " << repo.alias() << endl;
 101         repos.push_back(repo);
 102         //MIL << "added: " << repo.alias() << endl;
 103         return true;
 104       }
 105
 106       RepoInfoList repos;
 107     };
 108
 109   ////////////////////////////////////////////////////////////////////////////
 110
 111    /**
 112     * \short Internal version of clean cache
 113     *
 114     * Takes an extra SolvStore reference, so we avoid internally
 115     * having 2 SolvStores writing to the same database.
 116     */
 117   static void cleanCacheInternal( cache::SolvStore &store,
 118                                   const RepoInfo &info,
 119                                   const ProgressData::ReceiverFnc & progressrcv = ProgressData::ReceiverFnc() )
 120   {
 121     ProgressData progress;
 122     callback::SendReport<ProgressReport> report;
 123     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
 124     progress.name(str::form(_("Cleaning repository '%s' cache"), info.name().c_str()));
 125
 126     if ( !store.isCached(info.alias()) )
 127       return;
 128
 129     MIL << info.alias() << " cleaning cache..." << endl;
 130
 131     CombinedProgressData subprogrcv(progress);
 132
 133     store.cleanRepository(info.alias(), subprogrcv);
 134   }
 135
 136   ////////////////////////////////////////////////////////////////////////////
 137
 138   /**
 139    * Reads RepoInfo's from a repo file.
 140    *
 141    * \param file pathname of the file to read.
 142    */
 143   static std::list<RepoInfo> repositories_in_file( const Pathname & file )
 144   {
 145     MIL << "repo file: " << file << endl;
 146     RepoCollector collector;
 147     parser::RepoFileReader parser( file, bind( &RepoCollector::collect, &collector, _1 ) );
 148     return collector.repos;
 149   }
 150
 151   ////////////////////////////////////////////////////////////////////////////
 152
 153   std::list<RepoInfo> readRepoFile(const Url & repo_file)
 154    {
 155      // no interface to download a specific file, using workaround:
 156      //! \todo add MediaManager::provideFile(Url file_url) to easily access any file URLs? (no need for media access id or media_nr)
 157      Url url(repo_file);
 158      Pathname path(url.getPathName());
 159      url.setPathName ("/");
 160      MediaSetAccess access(url);
 161      Pathname local = access.provideFile(path);
 162
 163      DBG << "reading repo file " << repo_file << ", local path: " << local << endl;
 164
 165      return repositories_in_file(local);
 166    }
 167
 168   ////////////////////////////////////////////////////////////////////////////
 169
 170   /**
 171    * \short List of RepoInfo's from a directory
 172    *
 173    * Goes trough every file ending with ".repo" in a directory and adds all
 174    * RepoInfo's contained in that file.
 175    *
 176    * \param dir pathname of the directory to read.
 177    */
 178   static std::list<RepoInfo> repositories_in_dir( const Pathname &dir )
 179   {
 180     MIL << "directory " << dir << endl;
 181     list<RepoInfo> repos;
 182     list<Pathname> entries;
 183     if ( filesystem::readdir( entries, Pathname(dir), false ) != 0 )
 184       ZYPP_THROW(Exception("failed to read directory"));
 185
 186     str::regex allowedRepoExt("^\\.repo(_[0-9]+)?$");
 187     for ( list<Pathname>::const_iterator it = entries.begin(); it != entries.end(); ++it )
 188     {
 189       if (str::regex_match(it->extension(), allowedRepoExt))
 190       {
 191         list<RepoInfo> tmp = repositories_in_file( *it );
 192         repos.insert( repos.end(), tmp.begin(), tmp.end() );
 193
 194         //std::copy( collector.repos.begin(), collector.repos.end(), std::back_inserter(repos));
 195         //MIL << "ok" << endl;
 196       }
 197     }
 198     return repos;
 199   }
 200
 201   ////////////////////////////////////////////////////////////////////////////
 202
 203   static void assert_alias( const RepoInfo &info )
 204   {
 205     if (info.alias().empty())
 206         ZYPP_THROW(RepoNoAliasException());
 207   }
 208
 209   ////////////////////////////////////////////////////////////////////////////
 210
 211   static void assert_urls( const RepoInfo &info )
 212   {
 213     if (info.baseUrlsEmpty())
 214         ZYPP_THROW(RepoNoUrlException());
 215   }
 216
 217   ////////////////////////////////////////////////////////////////////////////
 218
 219   /**
 220    * \short Calculates the raw cache path for a repository
 221    */
 222   static Pathname rawcache_path_for_repoinfo( const RepoManagerOptions &opt, const RepoInfo &info )
 223   {
 224     assert_alias(info);
 225     return opt.repoRawCachePath + info.alias();
 226   }
 227
 228   ///////////////////////////////////////////////////////////////////
 229   //
 230   //    CLASS NAME : RepoManager::Impl
 231   //
 232   ///////////////////////////////////////////////////////////////////
 233
 234   /**
 235    * \short RepoManager implementation.
 236    */
 237   struct RepoManager::Impl
 238   {
 239     Impl( const RepoManagerOptions &opt )
 240       : options(opt)
 241     {
 242
 243     }
 244
 245     Impl()
 246     {
 247
 248     }
 249
 250     RepoManagerOptions options;
 251
 252   public:
 253     /** Offer default Impl. */
 254     static shared_ptr<Impl> nullimpl()
 255     {
 256       static shared_ptr<Impl> _nullimpl( new Impl );
 257       return _nullimpl;
 258     }
 259
 260   private:
 261     friend Impl * rwcowClone<Impl>( const Impl * rhs );
 262     /** clone for RWCOW_pointer */
 263     Impl * clone() const
 264     { return new Impl( *this ); }
 265   };
 266
 267   ///////////////////////////////////////////////////////////////////
 268
 269   /** \relates RepoManager::Impl Stream output */
 270   inline std::ostream & operator<<( std::ostream & str, const RepoManager::Impl & obj )
 271   {
 272     return str << "RepoManager::Impl";
 273   }
 274
 275   ///////////////////////////////////////////////////////////////////
 276   //
 277   //    CLASS NAME : RepoManager
 278   //
 279   ///////////////////////////////////////////////////////////////////
 280
 281   RepoManager::RepoManager( const RepoManagerOptions &opt )
 282   : _pimpl( new Impl(opt) )
 283   {}
 284
 285   ////////////////////////////////////////////////////////////////////////////
 286
 287   RepoManager::~RepoManager()
 288   {}
 289
 290   ////////////////////////////////////////////////////////////////////////////
 291
 292   std::list<RepoInfo> RepoManager::knownRepositories() const
 293   {
 294     MIL << endl;
 295
 296     if ( PathInfo(_pimpl->options.knownReposPath).isExist() )
 297     {
 298       RepoInfoList repos = repositories_in_dir(_pimpl->options.knownReposPath);
 299       for ( RepoInfoList::iterator it = repos.begin();
 300             it != repos.end();
 301             ++it )
 302       {
 303         // set the metadata path for the repo
 304         Pathname metadata_path = rawcache_path_for_repoinfo(_pimpl->options, (*it));
 305         (*it).setMetadataPath(metadata_path);
 306       }
 307       return repos;
 308     }
 309     else
 310       return std::list<RepoInfo>();
 311
 312     MIL << endl;
 313   }
 314
 315   ////////////////////////////////////////////////////////////////////////////
 316
 317   Pathname RepoManager::metadataPath( const RepoInfo &info ) const
 318   {
 319     return rawcache_path_for_repoinfo(_pimpl->options, info );
 320   }
 321
 322   ////////////////////////////////////////////////////////////////////////////
 323
 324   RepoStatus RepoManager::metadataStatus( const RepoInfo &info ) const
 325   {
 326     Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 327     RepoType repokind = info.type();
 328     RepoStatus status;
 329
 330     switch ( repokind.toEnum() )
 331     {
 332       case RepoType::NONE_e:
 333       // unknown, probe the local metadata
 334         repokind = probe(rawpath.asUrl());
 335       break;
 336       default:
 337       break;
 338     }
 339
 340     switch ( repokind.toEnum() )
 341     {
 342       case RepoType::RPMMD_e :
 343       {
 344         status = RepoStatus( rawpath + "/repodata/repomd.xml");
 345       }
 346       break;
 347
 348       case RepoType::YAST2_e :
 349       {
 350         // the order of RepoStatus && RepoStatus matters! (#304310)
 351         status = RepoStatus( rawpath + "/content") && (RepoStatus( rawpath + "/media.1/media"));
 352       }
 353       break;
 354
 355       case RepoType::RPMPLAINDIR_e :
 356       {
 357         if ( PathInfo(Pathname(rawpath + "/cookie")).isExist() )
 358           status = RepoStatus( rawpath + "/cookie");
 359       }
 360       break;
 361
 362       case RepoType::NONE_e :
 363         // Return default RepoStatus in case of RepoType::NONE
 364         // indicating it should be created?
 365         // ZYPP_THROW(RepoUnknownTypeException());
 366         break;
 367     }
 368     return status;
 369   }
 370
 371   void RepoManager::touchIndexFile(const RepoInfo & info)
 372   {
 373     Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 374
 375     RepoType repokind = info.type();
 376     if ( repokind.toEnum() == RepoType::NONE_e )
 377       // unknown, probe the local metadata
 378       repokind = probe(rawpath.asUrl());
 379     // if still unknown, just return
 380     if (repokind == RepoType::NONE_e)
 381       return;
 382
 383     Pathname p;
 384     switch ( repokind.toEnum() )
 385     {
 386       case RepoType::RPMMD_e :
 387         p = Pathname(rawpath + "/repodata/repomd.xml");
 388         break;
 389
 390       case RepoType::YAST2_e :
 391         p = Pathname(rawpath + "/content");
 392         break;
 393
 394       case RepoType::RPMPLAINDIR_e :
 395         p = Pathname(rawpath + "/cookie");
 396         break;
 397
 398       case RepoType::NONE_e :
 399       default:
 400         break;
 401     }
 402
 403     // touch the file, ignore error (they are logged anyway)
 404     filesystem::touch(p);
 405   }
 406
 407   bool RepoManager::checkIfToRefreshMetadata( const RepoInfo &info,
 408                                               const Url &url,
 409                                               RawMetadataRefreshPolicy policy )
 410   {
 411     assert_alias(info);
 412
 413     RepoStatus oldstatus;
 414     RepoStatus newstatus;
 415
 416     try
 417     {
 418       MIL << "Going to try to check whether refresh is needed for " << url << endl;
 419
 420       repo::RepoType repokind = info.type();
 421
 422       // if the type is unknown, try probing.
 423       switch ( repokind.toEnum() )
 424       {
 425         case RepoType::NONE_e:
 426           // unknown, probe it
 427           repokind = probe(url);
 428         break;
 429         default:
 430         break;
 431       }
 432
 433       Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 434       filesystem::assert_dir(rawpath);
 435       oldstatus = metadataStatus(info);
 436
 437       // now we've got the old (cached) status, we can decide repo.refresh.delay
 438       if (policy != RefreshForced)
 439       {
 440         // difference in seconds
 441         double diff = difftime(
 442           (Date::ValueType)Date::now(),
 443           (Date::ValueType)oldstatus.timestamp()) / 60;
 444
 445         DBG << "oldstatus: " << (Date::ValueType)oldstatus.timestamp() << endl;
 446         DBG << "current time: " << (Date::ValueType)Date::now() << endl;
 447         DBG << "last refresh = " << diff << " minutes ago" << endl;
 448
 449         if (diff < ZConfig::instance().repo_refresh_delay())
 450         {
 451           MIL << "Repository '" << info.alias()
 452               << "' has been refreshed less than repo.refresh.delay ("
 453               << ZConfig::instance().repo_refresh_delay()
 454               << ") minutes ago. Advising to skip refresh" << endl;
 455           return false;
 456         }
 457       }
 458
 459       // create temp dir as sibling of rawpath
 460       filesystem::TmpDir tmpdir( filesystem::TmpDir::makeSibling( rawpath ) );
 461
 462       if ( ( repokind.toEnum() == RepoType::RPMMD_e ) ||
 463            ( repokind.toEnum() == RepoType::YAST2_e ) )
 464       {
 465         MediaSetAccess media(url);
 466         shared_ptr<repo::Downloader> downloader_ptr;
 467
 468         if ( repokind.toEnum() == RepoType::RPMMD_e )
 469           downloader_ptr.reset(new yum::Downloader(info.path()));
 470         else
 471           downloader_ptr.reset( new susetags::Downloader(info.path()));
 472
 473         RepoStatus newstatus = downloader_ptr->status(media);
 474         bool refresh = false;
 475         if ( oldstatus.checksum() == newstatus.checksum() )
 476         {
 477           MIL << "repo has not changed" << endl;
 478           if ( policy == RefreshForced )
 479           {
 480             MIL << "refresh set to forced" << endl;
 481             refresh = true;
 482           }
 483         }
 484         else
 485         {
 486           MIL << "repo has changed, going to refresh" << endl;
 487           refresh = true;
 488         }
 489
 490         if (!refresh)
 491           touchIndexFile(info);
 492
 493         return refresh;
 494       }
 495 #if 0
 496       else if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 497       {
 498         RepoStatus newstatus = parser::plaindir::dirStatus(url.getPathName());
 499         bool refresh = false;
 500         if ( oldstatus.checksum() == newstatus.checksum() )
 501         {
 502           MIL << "repo has not changed" << endl;
 503           if ( policy == RefreshForced )
 504           {
 505             MIL << "refresh set to forced" << endl;
 506             refresh = true;
 507           }
 508         }
 509         else
 510         {
 511           MIL << "repo has changed, going to refresh" << endl;
 512           refresh = true;
 513         }
 514
 515         if (!refresh)
 516           touchIndexFile(info);
 517
 518         return refresh;
 519       }
 520 #endif
 521       else
 522       {
 523         ZYPP_THROW(RepoUnknownTypeException());
 524       }
 525     }
 526     catch ( const Exception &e )
 527     {
 528       ZYPP_CAUGHT(e);
 529       ERR << "refresh check failed for " << url << endl;
 530       ZYPP_RETHROW(e);
 531     }
 532
 533     return true; // default
 534   }
 535
 536   void RepoManager::refreshMetadata( const RepoInfo &info,
 537                                      RawMetadataRefreshPolicy policy,
 538                                      const ProgressData::ReceiverFnc & progress )
 539   {
 540     assert_alias(info);
 541     assert_urls(info);
 542
 543     // we will throw this later if no URL checks out fine
 544     RepoException rexception(_("Valid metadata not found at specified URL(s)"));
 545
 546     // try urls one by one
 547     for ( RepoInfo::urls_const_iterator it = info.baseUrlsBegin(); it != info.baseUrlsEnd(); ++it )
 548     {
 549       try
 550       {
 551         Url url(*it);
 552
 553         // check whether to refresh metadata
 554         // if the check fails for this url, it throws, so another url will be checked
 555         if (!checkIfToRefreshMetadata(info, url, policy))
 556           return;
 557
 558         MIL << "Going to refresh metadata from " << url << endl;
 559
 560         repo::RepoType repokind = info.type();
 561
 562         // if the type is unknown, try probing.
 563         switch ( repokind.toEnum() )
 564         {
 565           case RepoType::NONE_e:
 566             // unknown, probe it
 567             repokind = probe(*it);
 568           break;
 569           default:
 570           break;
 571         }
 572
 573         Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 574         filesystem::assert_dir(rawpath);
 575
 576         // create temp dir as sibling of rawpath
 577         filesystem::TmpDir tmpdir( filesystem::TmpDir::makeSibling( rawpath ) );
 578
 579         if ( ( repokind.toEnum() == RepoType::RPMMD_e ) ||
 580              ( repokind.toEnum() == RepoType::YAST2_e ) )
 581         {
 582           MediaSetAccess media(url);
 583           shared_ptr<repo::Downloader> downloader_ptr;
 584
 585           if ( repokind.toEnum() == RepoType::RPMMD_e )
 586             downloader_ptr.reset(new yum::Downloader(info.path()));
 587           else
 588             downloader_ptr.reset( new susetags::Downloader(info.path()));
 589
 590           /**
 591            * Given a downloader, sets the other repos raw metadata
 592            * path as cache paths for the fetcher, so if another
 593            * repo has the same file, it will not download it
 594            * but copy it from the other repository
 595            */
 596           std::list<RepoInfo> repos = knownRepositories();
 597           for ( std::list<RepoInfo>::const_iterator it = repos.begin();
 598                 it != repos.end();
 599                 ++it )
 600           {
 601             downloader_ptr->addCachePath(rawcache_path_for_repoinfo( _pimpl->options, *it ));
 602           }
 603
 604           downloader_ptr->download( media, tmpdir.path());
 605         }
 606 #if 0
 607         else if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 608         {
 609           RepoStatus newstatus = parser::plaindir::dirStatus(url.getPathName());
 610
 611           std::ofstream file(( tmpdir.path() + "/cookie").c_str());
 612           if (!file) {
 613             ZYPP_THROW (Exception( "Can't open " + tmpdir.path().asString() + "/cookie" ) );
 614           }
 615           file << url << endl;
 616           file << newstatus.checksum() << endl;
 617
 618           file.close();
 619         }
 620 #endif
 621         else
 622         {
 623           ZYPP_THROW(RepoUnknownTypeException());
 624         }
 625
 626         // ok we have the metadata, now exchange
 627         // the contents
 628         TmpDir oldmetadata( TmpDir::makeSibling( rawpath ) );
 629         filesystem::rename( rawpath, oldmetadata.path() );
 630         // move the just downloaded there
 631         filesystem::rename( tmpdir.path(), rawpath );
 632         // we are done.
 633         return;
 634       }
 635       catch ( const Exception &e )
 636       {
 637         ZYPP_CAUGHT(e);
 638         ERR << "Trying another url..." << endl;
 639
 640         // remember the exception caught for the *first URL*
 641         // if all other URLs fail, the rexception will be thrown with the
 642         // cause of the problem of the first URL remembered
 643         if (it == info.baseUrlsBegin())
 644           rexception.remember(e);
 645       }
 646     } // for every url
 647     ERR << "No more urls..." << endl;
 648     ZYPP_THROW(rexception);
 649   }
 650
 651   ////////////////////////////////////////////////////////////////////////////
 652
 653   void RepoManager::cleanMetadata( const RepoInfo &info,
 654                                    const ProgressData::ReceiverFnc & progressfnc )
 655   {
 656     ProgressData progress(100);
 657     progress.sendTo(progressfnc);
 658
 659     filesystem::recursive_rmdir(rawcache_path_for_repoinfo(_pimpl->options, info));
 660     progress.toMax();
 661   }
 662
 663   void RepoManager::buildCache( const RepoInfo &info,
 664                                 CacheBuildPolicy policy,
 665                                 const ProgressData::ReceiverFnc & progressrcv )
 666   {
 667     assert_alias(info);
 668     Pathname rawpath = rawcache_path_for_repoinfo(_pimpl->options, info);
 669
 670     Pathname base = _pimpl->options.repoCachePath + info.alias();
 671     Pathname solvfile = base.extend(".solv");
 672
 673     //cache::SolvStore store(_pimpl->options.repoCachePath);
 674
 675     RepoStatus raw_metadata_status = metadataStatus(info);
 676     if ( raw_metadata_status.empty() )
 677     {
 678       ZYPP_THROW(RepoMetadataException(info));
 679     }
 680
 681     bool needs_cleaning = false;
 682     if ( isCached( info ) )
 683     {
 684       MIL << info.alias() << " is already cached." << endl;
 685       //data::RecordId id = store.lookupRepository(info.alias());
 686       RepoStatus cache_status = cacheStatus(info);
 687
 688       if ( cache_status.checksum() == raw_metadata_status.checksum() )
 689       {
 690         MIL << info.alias() << " cache is up to date with metadata." << endl;
 691         if ( policy == BuildIfNeeded ) {
 692           return;
 693         }
 694         else {
 695           MIL << info.alias() << " cache rebuild is forced" << endl;
 696         }
 697       }
 698
 699       needs_cleaning = true;
 700     }
 701
 702     ProgressData progress(100);
 703     callback::SendReport<ProgressReport> report;
 704     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
 705     progress.name(str::form(_("Building repository '%s' cache"), info.name().c_str()));
 706     progress.toMin();
 707
 708     if (needs_cleaning)
 709     {
 710 //       Pathname name = _pimpl->options.repoCachePath;
 711 //       //data::RecordId id = store.lookupRepository(info.alias());
 712 //       ostringstream os;
 713 //       os << id.get();
 714 //       name += os.str() + ".solv";
 715 //       unlink (name);
 716 //       cleanCacheInternal( store, info);
 717       cleanCache(info);
 718     }
 719
 720     MIL << info.alias() << " building cache..." << endl;
 721     //data::RecordId id = store.lookupOrAppendRepository(info.alias());
 722     // do we have type?
 723     repo::RepoType repokind = info.type();
 724
 725     // if the type is unknown, try probing.
 726     switch ( repokind.toEnum() )
 727     {
 728       case RepoType::NONE_e:
 729         // unknown, probe the local metadata
 730         repokind = probe(rawpath.asUrl());
 731       break;
 732       default:
 733       break;
 734     }
 735
 736     MIL << "repo type is " << repokind << endl;
 737
 738     switch ( repokind.toEnum() )
 739     {
 740       case RepoType::RPMMD_e :
 741       case RepoType::YAST2_e :
 742       {
 743 //         string cmd = "repo2solv.sh \"";
 744 //      cmd += rawpath.asString() + "\" > " + solvfile.asString();
 745 //      int ret = system (cmd.c_str());
 746 //         if (WIFEXITED (ret) && WEXITSTATUS (ret) != 0)
 747 //        ZYPP_THROW(RepoUnknownTypeException());
 748         MIL << "Executing solv converter" << endl;
 749         string cmd( str::form( "/usr/bin/repo2solv.sh \"%s\" > %s", rawpath.asString().c_str(), solvfile.asString().c_str() ) );
 750         ExternalProgram prog( cmd, ExternalProgram::Stderr_To_Stdout );
 751         for ( string output( prog.receiveLine() ); output.length(); output = prog.receiveLine() ) {
 752           MIL << "  " << output;
 753         }
 754         int ret = prog.close();
 755       }
 756       break;
 757       default:
 758         ZYPP_THROW(Exception("Unhandled repostory type"));
 759       break;
 760     }
 761 #if 0
 762     switch ( repokind.toEnum() )
 763     {
 764       case RepoType::RPMMD_e :
 765       if (0)
 766       {
 767         CombinedProgressData subprogrcv( progress, 100);
 768         parser::yum::RepoParser parser(id, store, parser::yum::RepoParserOpts(), subprogrcv);
 769         parser.parse(rawpath);
 770           // no error
 771       }
 772       break;
 773       case RepoType::YAST2_e :
 774       if (0)
 775       {
 776         CombinedProgressData subprogrcv( progress, 100);
 777         parser::susetags::RepoParser parser(id, store, subprogrcv);
 778         parser.parse(rawpath);
 779         // no error
 780       }
 781       break;
 782 #endif
 783 #if 0
 784       case RepoType::RPMPLAINDIR_e :
 785       {
 786         CombinedProgressData subprogrcv( progress, 100);
 787         InputStream is(rawpath + "cookie");
 788         string buffer;
 789         getline( is.stream(), buffer);
 790         Url url(buffer);
 791         parser::plaindir::RepoParser parser(id, store, subprogrcv);
 792         parser.parse(url.getPathName());
 793       }
 794       break;
 795
 796       default:
 797         ZYPP_THROW(RepoUnknownTypeException());
 798     }
 799 #endif
 800     // update timestamp and checksum
 801     //store.updateRepositoryStatus(id, raw_metadata_status);
 802     setCacheStatus(info.alias(), raw_metadata_status);
 803     MIL << "Commit cache.." << endl;
 804     //store.commit();
 805     //progress.toMax();
 806   }
 807
 808   ////////////////////////////////////////////////////////////////////////////
 809
 810   repo::RepoType RepoManager::probe( const Url &url ) const
 811   {
 812     if ( url.getScheme() == "dir" && ! PathInfo( url.getPathName() ).isDir() )
 813     {
 814       // Handle non existing local directory in advance, as
 815       // MediaSetAccess does not support it.
 816       return repo::RepoType::NONE;
 817     }
 818
 819     try
 820     {
 821       MediaSetAccess access(url);
 822       if ( access.doesFileExist("/repodata/repomd.xml") )
 823         return repo::RepoType::RPMMD;
 824       if ( access.doesFileExist("/content") )
 825         return repo::RepoType::YAST2;
 826
 827       // if it is a local url of type dir
 828       if ( (! media::MediaManager::downloads(url)) && ( url.getScheme() == "dir" ) )
 829       {
 830         Pathname path = Pathname(url.getPathName());
 831         if ( PathInfo(path).isDir() )
 832         {
 833           // allow empty dirs for now
 834           return repo::RepoType::RPMPLAINDIR;
 835         }
 836       }
 837     }
 838     catch ( const media::MediaException &e )
 839     {
 840       ZYPP_CAUGHT(e);
 841       RepoException enew("Error trying to read from " + url.asString());
 842       enew.remember(e);
 843       ZYPP_THROW(enew);
 844     }
 845     catch ( const Exception &e )
 846     {
 847       ZYPP_CAUGHT(e);
 848       Exception enew("Unknown error reading from " + url.asString());
 849       enew.remember(e);
 850       ZYPP_THROW(enew);
 851     }
 852
 853     return repo::RepoType::NONE;
 854   }
 855
 856   ////////////////////////////////////////////////////////////////////////////
 857
 858   void RepoManager::cleanCache( const RepoInfo &info,
 859                                 const ProgressData::ReceiverFnc & progressrcv )
 860   {
 861     Pathname name = _pimpl->options.repoCachePath;
 862     name += info.alias() + ".solv";
 863     unlink (name);
 864   }
 865
 866   ////////////////////////////////////////////////////////////////////////////
 867
 868   bool RepoManager::isCached( const RepoInfo &info ) const
 869   {
 870     Pathname name = _pimpl->options.repoCachePath;
 871     return PathInfo(name + Pathname(info.alias()).extend(".solv")).isExist();
 872   }
 873
 874   RepoStatus RepoManager::cacheStatus( const RepoInfo &info ) const
 875   {
 876     RepoStatus status;
 877     Pathname base = _pimpl->options.repoCachePath + info.alias();
 878     Pathname solvfile = base.extend(".solv");
 879     Pathname cookiefile = base.extend(".cookie");
 880
 881     std::ifstream file(cookiefile.c_str());
 882     if (!file) {
 883       ZYPP_THROW (Exception( "Can't open " + cookiefile.asString() ) );
 884     }
 885
 886     std::string buffer;
 887     while(file && !file.eof()) {
 888       getline(file, buffer);
 889     }
 890
 891     std::vector<std::string> words;
 892     if ( str::split( buffer, std::back_inserter(words) ) != 2 )
 893       ZYPP_THROW (Exception( "corrupt file " + cookiefile.asString() ) );
 894
 895     status.setTimestamp(Date(str::strtonum<time_t>(words[1])));
 896     status.setChecksum(words[0]);
 897     return status;
 898   }
 899
 900   void RepoManager::setCacheStatus( const string &alias, const RepoStatus &status )
 901   {
 902     Pathname base = _pimpl->options.repoCachePath + alias;
 903     Pathname cookiefile = base.extend(".cookie");
 904
 905     std::ofstream file(cookiefile.c_str());
 906     if (!file) {
 907       ZYPP_THROW (Exception( "Can't open " + cookiefile.asString() ) );
 908     }
 909     file << status;
 910     file.close();
 911   }
 912
 913   map<data::RecordId, Repo *> repo2solv;
 914
 915   Repository RepoManager::createFromCache( const RepoInfo &info,
 916                                            const ProgressData::ReceiverFnc & progressrcv )
 917   {
 918     callback::SendReport<ProgressReport> report;
 919     ProgressData progress;
 920     progress.sendTo(ProgressReportAdaptor( progressrcv, report ));
 921     //progress.sendTo( progressrcv );
 922     progress.name(str::form(_("Reading repository '%s' cache"), info.name().c_str()));
 923
 924     //_pimpl->options.repoCachePath
 925     if ( ! isCached( info ) )
 926       ZYPP_THROW(RepoNotCachedException());
 927
 928     MIL << "Repository " << info.alias() << " is cached" << endl;
 929
 930     CombinedProgressData subprogrcv(progress);
 931
 932     repo::cached::RepoOptions opts( info, _pimpl->options.repoCachePath );
 933     opts.readingResolvablesProgress = subprogrcv;
 934     //opts.repo = repo;
 935     repo::cached::RepoImpl::Ptr repoimpl =
 936          new repo::cached::RepoImpl( opts );
 937
 938     //repoimpl->createResolvables();
 939     repoimpl->resolvables();
 940     // read the resolvables from cache
 941     //return Repository::noRepository;
 942     return Repository(repoimpl);
 943   }
 944
 945   ////////////////////////////////////////////////////////////////////////////
 946
 947   /**
 948    * Generate a non existing filename in a directory, using a base
 949    * name. For example if a directory contains 3 files
 950    *
 951    * |-- bar
 952    * |-- foo
 953    * `-- moo
 954    *
 955    * If you try to generate a unique filename for this directory,
 956    * based on "ruu" you will get "ruu", but if you use the base
 957    * "foo" you will get "foo_1"
 958    *
 959    * \param dir Directory where the file needs to be unique
 960    * \param basefilename string to base the filename on.
 961    */
 962   static Pathname generate_non_existing_name( const Pathname &dir,
 963                                               const std::string &basefilename )
 964   {
 965     string final_filename = basefilename;
 966     int counter = 1;
 967     while ( PathInfo(dir + final_filename).isExist() )
 968     {
 969       final_filename = basefilename + "_" + str::numstring(counter);
 970       counter++;
 971     }
 972     return dir + Pathname(final_filename);
 973   }
 974
 975   ////////////////////////////////////////////////////////////////////////////
 976
 977   /**
 978    * \short Generate a related filename from a repo info
 979    *
 980    * From a repo info, it will try to use the alias as a filename
 981    * escaping it if necessary. Other fallbacks can be added to
 982    * this function in case there is no way to use the alias
 983    */
 984   static std::string generate_filename( const RepoInfo &info )
 985   {
 986     std::string fnd="/";
 987     std::string rep="_";
 988     std::string filename = info.alias();
 989     // replace slashes with underscores
 990     size_t pos = filename.find(fnd);
 991     while(pos!=string::npos)
 992     {
 993       filename.replace(pos,fnd.length(),rep);
 994       pos = filename.find(fnd,pos+rep.length());
 995     }
 996     filename = Pathname(filename).extend(".repo").asString();
 997     MIL << "generating filename for repo [" << info.alias() << "] : '" << filename << "'" << endl;
 998     return filename;
 999   }
1000
1001
1002   ////////////////////////////////////////////////////////////////////////////
1003
1004   void RepoManager::addRepository( const RepoInfo &info,
1005                                    const ProgressData::ReceiverFnc & progressrcv )
1006   {
1007     assert_alias(info);
1008
1009     ProgressData progress(100);
1010     callback::SendReport<ProgressReport> report;
1011     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
1012     progress.name(str::form(_("Adding repository '%s'"), info.name().c_str()));
1013     progress.toMin();
1014
1015     std::list<RepoInfo> repos = knownRepositories();
1016     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1017           it != repos.end();
1018           ++it )
1019     {
1020       if ( info.alias() == (*it).alias() )
1021         ZYPP_THROW(RepoAlreadyExistsException(info.alias()));
1022     }
1023
1024     RepoInfo tosave = info;
1025
1026     // check the first url for now
1027     if ( ZConfig::instance().repo_add_probe()
1028         || ( tosave.type() == RepoType::NONE && tosave.enabled()) )
1029     {
1030       DBG << "unknown repository type, probing" << endl;
1031
1032       RepoType probedtype;
1033       probedtype = probe(*tosave.baseUrlsBegin());
1034       if ( tosave.baseUrlsSize() > 0 )
1035       {
1036         if ( probedtype == RepoType::NONE )
1037           ZYPP_THROW(RepoUnknownTypeException());
1038         else
1039           tosave.setType(probedtype);
1040       }
1041     }
1042
1043     progress.set(50);
1044
1045     // assert the directory exists
1046     filesystem::assert_dir(_pimpl->options.knownReposPath);
1047
1048     Pathname repofile = generate_non_existing_name(_pimpl->options.knownReposPath,
1049                                                     generate_filename(tosave));
1050     // now we have a filename that does not exists
1051     MIL << "Saving repo in " << repofile << endl;
1052
1053     std::ofstream file(repofile.c_str());
1054     if (!file) {
1055       ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
1056     }
1057
1058     tosave.dumpRepoOn(file);
1059     progress.toMax();
1060     MIL << "done" << endl;
1061   }
1062
1063   void RepoManager::addRepositories( const Url &url,
1064                                      const ProgressData::ReceiverFnc & progressrcv )
1065   {
1066     std::list<RepoInfo> knownrepos = knownRepositories();
1067     std::list<RepoInfo> repos = readRepoFile(url);
1068     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1069           it != repos.end();
1070           ++it )
1071     {
1072       // look if the alias is in the known repos.
1073       for ( std::list<RepoInfo>::const_iterator kit = knownrepos.begin();
1074           kit != knownrepos.end();
1075           ++kit )
1076       {
1077         if ( (*it).alias() == (*kit).alias() )
1078         {
1079           ERR << "To be added repo " << (*it).alias() << " conflicts with existing repo " << (*kit).alias() << endl;
1080           ZYPP_THROW(RepoAlreadyExistsException((*it).alias()));
1081         }
1082       }
1083     }
1084
1085     string filename = Pathname(url.getPathName()).basename();
1086
1087     if ( filename == Pathname() )
1088       ZYPP_THROW(RepoException("Invalid repo file name at " + url.asString() ));
1089
1090     // assert the directory exists
1091     filesystem::assert_dir(_pimpl->options.knownReposPath);
1092
1093     Pathname repofile = generate_non_existing_name(_pimpl->options.knownReposPath, filename);
1094     // now we have a filename that does not exists
1095     MIL << "Saving " << repos.size() << " repo" << ( repos.size() ? "s" : "" ) << " in " << repofile << endl;
1096
1097     std::ofstream file(repofile.c_str());
1098     if (!file) {
1099       ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
1100     }
1101
1102     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1103           it != repos.end();
1104           ++it )
1105     {
1106       MIL << "Saving " << (*it).alias() << endl;
1107       (*it).dumpRepoOn(file);
1108     }
1109     MIL << "done" << endl;
1110   }
1111
1112   ////////////////////////////////////////////////////////////////////////////
1113
1114   void RepoManager::removeRepository( const RepoInfo & info,
1115                                       const ProgressData::ReceiverFnc & progressrcv)
1116   {
1117     ProgressData progress;
1118     callback::SendReport<ProgressReport> report;
1119     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
1120     progress.name(str::form(_("Removing repository '%s'"), info.name().c_str()));
1121
1122     MIL << "Going to delete repo " << info.alias() << endl;
1123
1124     std::list<RepoInfo> repos = knownRepositories();
1125     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1126           it != repos.end();
1127           ++it )
1128     {
1129       // they can be the same only if the provided is empty, that means
1130       // the provided repo has no alias
1131       // then skip
1132       if ( (!info.alias().empty()) && ( info.alias() != (*it).alias() ) )
1133         continue;
1134
1135       // TODO match by url
1136
1137       // we have a matcing repository, now we need to know
1138       // where it does come from.
1139       RepoInfo todelete = *it;
1140       if (todelete.filepath().empty())
1141       {
1142         ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
1143       }
1144       else
1145       {
1146         // figure how many repos are there in the file:
1147         std::list<RepoInfo> filerepos = repositories_in_file(todelete.filepath());
1148         if ( (filerepos.size() == 1) && ( filerepos.front().alias() == todelete.alias() ) )
1149         {
1150           // easy, only this one, just delete the file
1151           if ( filesystem::unlink(todelete.filepath()) != 0 )
1152           {
1153             ZYPP_THROW(RepoException("Can't delete " + todelete.filepath().asString()));
1154           }
1155           MIL << todelete.alias() << " sucessfully deleted." << endl;
1156         }
1157         else
1158         {
1159           // there are more repos in the same file
1160           // write them back except the deleted one.
1161           //TmpFile tmp;
1162           //std::ofstream file(tmp.path().c_str());
1163
1164           // assert the directory exists
1165           filesystem::assert_dir(todelete.filepath().dirname());
1166
1167           std::ofstream file(todelete.filepath().c_str());
1168           if (!file) {
1169             //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
1170             ZYPP_THROW (Exception( "Can't open " + todelete.filepath().asString() ) );
1171           }
1172           for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
1173                 fit != filerepos.end();
1174                 ++fit )
1175           {
1176             if ( (*fit).alias() != todelete.alias() )
1177               (*fit).dumpRepoOn(file);
1178           }
1179         }
1180
1181         CombinedProgressData subprogrcv(progress, 70);
1182         CombinedProgressData cleansubprogrcv(progress, 30);
1183         // now delete it from cache
1184         if ( isCached(todelete) )
1185           cleanCache( todelete, subprogrcv);
1186         // now delete metadata (#301037)
1187         cleanMetadata( todelete, cleansubprogrcv);
1188         MIL << todelete.alias() << " sucessfully deleted." << endl;
1189         return;
1190       } // else filepath is empty
1191
1192     }
1193     // should not be reached on a sucess workflow
1194     ZYPP_THROW(RepoNotFoundException(info));
1195   }
1196
1197   ////////////////////////////////////////////////////////////////////////////
1198
1199   void RepoManager::modifyRepository( const std::string &alias,
1200                                       const RepoInfo & newinfo,
1201                                       const ProgressData::ReceiverFnc & progressrcv )
1202   {
1203     RepoInfo toedit = getRepositoryInfo(alias);
1204
1205     if (toedit.filepath().empty())
1206     {
1207       ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
1208     }
1209     else
1210     {
1211       // figure how many repos are there in the file:
1212       std::list<RepoInfo> filerepos = repositories_in_file(toedit.filepath());
1213
1214       // there are more repos in the same file
1215       // write them back except the deleted one.
1216       //TmpFile tmp;
1217       //std::ofstream file(tmp.path().c_str());
1218
1219       // assert the directory exists
1220       filesystem::assert_dir(toedit.filepath().dirname());
1221
1222       std::ofstream file(toedit.filepath().c_str());
1223       if (!file) {
1224         //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
1225         ZYPP_THROW (Exception( "Can't open " + toedit.filepath().asString() ) );
1226       }
1227       for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
1228             fit != filerepos.end();
1229             ++fit )
1230       {
1231           // if the alias is different, dump the original
1232           // if it is the same, dump the provided one
1233           if ( (*fit).alias() != toedit.alias() )
1234             (*fit).dumpRepoOn(file);
1235           else
1236             newinfo.dumpRepoOn(file);
1237       }
1238     }
1239   }
1240
1241   ////////////////////////////////////////////////////////////////////////////
1242
1243   RepoInfo RepoManager::getRepositoryInfo( const std::string &alias,
1244                                            const ProgressData::ReceiverFnc & progressrcv )
1245   {
1246     std::list<RepoInfo> repos = knownRepositories();
1247     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1248           it != repos.end();
1249           ++it )
1250     {
1251       if ( (*it).alias() == alias )
1252         return *it;
1253     }
1254     RepoInfo info;
1255     info.setAlias(info.alias());
1256     ZYPP_THROW(RepoNotFoundException(info));
1257   }
1258
1259   ////////////////////////////////////////////////////////////////////////////
1260
1261   RepoInfo RepoManager::getRepositoryInfo( const Url & url,
1262                                            const url::ViewOption & urlview,
1263                                            const ProgressData::ReceiverFnc & progressrcv )
1264   {
1265     std::list<RepoInfo> repos = knownRepositories();
1266     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1267           it != repos.end();
1268           ++it )
1269     {
1270       for(RepoInfo::urls_const_iterator urlit = (*it).baseUrlsBegin();
1271           urlit != (*it).baseUrlsEnd();
1272           ++urlit)
1273       {
1274         if ((*urlit).asString(urlview) == url.asString(urlview))
1275           return *it;
1276       }
1277     }
1278     RepoInfo info;
1279     info.setAlias(info.alias());
1280     info.setBaseUrl(url);
1281     ZYPP_THROW(RepoNotFoundException(info));
1282   }
1283
1284   ////////////////////////////////////////////////////////////////////////////
1285
1286   std::ostream & operator<<( std::ostream & str, const RepoManager & obj )
1287   {
1288     return str << *obj._pimpl;
1289   }
1290
1291   /////////////////////////////////////////////////////////////////
1292 } // namespace zypp
1293 ///////////////////////////////////////////////////////////////////