zypp/RepoManager.cc

   1 /*---------------------------------------------------------------------\
   2 |                          ____ _   __ __ ___                          |
   3 |                         |__  / \ / / . \ . \                         |
   4 |                           / / \ V /|  _/  _/                         |
   5 |                          / /__ | | | | | |                           |
   6 |                         /_____||_| |_| |_|                           |
   7 |                                                                      |
   8 \---------------------------------------------------------------------*/
   9 /** \file       zypp/RepoManager.cc
  10  *
  11 */
  12
  13 #include <cstdlib>
  14 #include <iostream>
  15 #include <fstream>
  16 #include <sstream>
  17 #include <list>
  18 #include <map>
  19 #include <algorithm>
  20 #include "zypp/base/InputStream.h"
  21 #include "zypp/base/Logger.h"
  22 #include "zypp/base/Gettext.h"
  23 #include "zypp/base/Function.h"
  24 #include "zypp/base/Regex.h"
  25 #include "zypp/PathInfo.h"
  26 #include "zypp/TmpPath.h"
  27
  28 #include "zypp/repo/RepoException.h"
  29 #include "zypp/RepoManager.h"
  30
  31 #include "zypp/media/MediaManager.h"
  32 #include "zypp/MediaSetAccess.h"
  33 #include "zypp/ExternalProgram.h"
  34 #include "zypp/ManagedFile.h"
  35
  36 #include "zypp/parser/RepoFileReader.h"
  37 #include "zypp/repo/yum/Downloader.h"
  38 #include "zypp/parser/yum/RepoParser.h"
  39 #include "zypp/repo/susetags/Downloader.h"
  40 #include "zypp/parser/susetags/RepoParser.h"
  41 #include "zypp/parser/plaindir/RepoParser.h"
  42
  43 #include "zypp/ZYppCallbacks.h"
  44
  45 #include "sat/Pool.h"
  46 #include "satsolver/pool.h"
  47 #include "satsolver/repo.h"
  48 #include "satsolver/repo_solv.h"
  49
  50 using namespace std;
  51 using namespace zypp;
  52 using namespace zypp::repo;
  53 using namespace zypp::filesystem;
  54
  55 using namespace zypp::repo;
  56
  57 ///////////////////////////////////////////////////////////////////
  58 namespace zypp
  59 { /////////////////////////////////////////////////////////////////
  60
  61   ///////////////////////////////////////////////////////////////////
  62   //
  63   //    CLASS NAME : RepoManagerOptions
  64   //
  65   ///////////////////////////////////////////////////////////////////
  66
  67   RepoManagerOptions::RepoManagerOptions()
  68   {
  69     repoCachePath    = ZConfig::instance().repoCachePath();
  70     repoRawCachePath = ZConfig::instance().repoMetadataPath();
  71     repoPackagesCachePath = ZConfig::instance().repoPackagesPath();
  72     knownReposPath   = ZConfig::instance().knownReposPath();
  73     probe            = ZConfig::instance().repo_add_probe();
  74   }
  75
  76   ////////////////////////////////////////////////////////////////////////////
  77
  78   /**
  79     * \short Simple callback to collect the results
  80     *
  81     * Classes like RepoFileParser call the callback
  82     * once per each repo in a file.
  83     *
  84     * Passing this functor as callback, you can collect
  85     * all resuls at the end, without dealing with async
  86     * code.
  87     */
  88     struct RepoCollector
  89     {
  90       RepoCollector()
  91       {}
  92
  93       ~RepoCollector()
  94       {}
  95
  96       bool collect( const RepoInfo &repo )
  97       {
  98         repos.push_back(repo);
  99         return true;
 100       }
 101
 102       RepoInfoList repos;
 103     };
 104
 105   ////////////////////////////////////////////////////////////////////////////
 106
 107   /**
 108    * Reads RepoInfo's from a repo file.
 109    *
 110    * \param file pathname of the file to read.
 111    */
 112   static std::list<RepoInfo> repositories_in_file( const Pathname & file )
 113   {
 114     MIL << "repo file: " << file << endl;
 115     RepoCollector collector;
 116     parser::RepoFileReader parser( file, bind( &RepoCollector::collect, &collector, _1 ) );
 117     return collector.repos;
 118   }
 119
 120   ////////////////////////////////////////////////////////////////////////////
 121
 122   std::list<RepoInfo> readRepoFile(const Url & repo_file)
 123    {
 124      // no interface to download a specific file, using workaround:
 125      //! \todo add MediaManager::provideFile(Url file_url) to easily access any file URLs? (no need for media access id or media_nr)
 126      Url url(repo_file);
 127      Pathname path(url.getPathName());
 128      url.setPathName ("/");
 129      MediaSetAccess access(url);
 130      Pathname local = access.provideFile(path);
 131
 132      DBG << "reading repo file " << repo_file << ", local path: " << local << endl;
 133
 134      return repositories_in_file(local);
 135    }
 136
 137   ////////////////////////////////////////////////////////////////////////////
 138
 139   /**
 140    * \short List of RepoInfo's from a directory
 141    *
 142    * Goes trough every file ending with ".repo" in a directory and adds all
 143    * RepoInfo's contained in that file.
 144    *
 145    * \param dir pathname of the directory to read.
 146    */
 147   static std::list<RepoInfo> repositories_in_dir( const Pathname &dir )
 148   {
 149     MIL << "directory " << dir << endl;
 150     list<RepoInfo> repos;
 151     list<Pathname> entries;
 152     if ( filesystem::readdir( entries, Pathname(dir), false ) != 0 )
 153       ZYPP_THROW(Exception("failed to read directory"));
 154
 155     str::regex allowedRepoExt("^\\.repo(_[0-9]+)?$");
 156     for ( list<Pathname>::const_iterator it = entries.begin(); it != entries.end(); ++it )
 157     {
 158       if (str::regex_match(it->extension(), allowedRepoExt))
 159       {
 160         list<RepoInfo> tmp = repositories_in_file( *it );
 161         repos.insert( repos.end(), tmp.begin(), tmp.end() );
 162
 163         //std::copy( collector.repos.begin(), collector.repos.end(), std::back_inserter(repos));
 164         //MIL << "ok" << endl;
 165       }
 166     }
 167     return repos;
 168   }
 169
 170   ////////////////////////////////////////////////////////////////////////////
 171
 172   static void assert_alias( const RepoInfo &info )
 173   {
 174     if (info.alias().empty())
 175         ZYPP_THROW(RepoNoAliasException());
 176   }
 177
 178   ////////////////////////////////////////////////////////////////////////////
 179
 180   static void assert_urls( const RepoInfo &info )
 181   {
 182     if (info.baseUrlsEmpty())
 183         ZYPP_THROW(RepoNoUrlException());
 184   }
 185
 186   ////////////////////////////////////////////////////////////////////////////
 187
 188   /**
 189    * \short Calculates the raw cache path for a repository
 190    */
 191   static Pathname rawcache_path_for_repoinfo( const RepoManagerOptions &opt, const RepoInfo &info )
 192   {
 193     assert_alias(info);
 194     return opt.repoRawCachePath / info.escaped_alias();
 195   }
 196
 197   /**
 198    * \short Calculates the packages cache path for a repository
 199    */
 200   static Pathname packagescache_path_for_repoinfo( const RepoManagerOptions &opt, const RepoInfo &info )
 201   {
 202     assert_alias(info);
 203     return opt.repoPackagesCachePath / info.escaped_alias();
 204   }
 205
 206   ///////////////////////////////////////////////////////////////////
 207   //
 208   //    CLASS NAME : RepoManager::Impl
 209   //
 210   ///////////////////////////////////////////////////////////////////
 211
 212   /**
 213    * \short RepoManager implementation.
 214    */
 215   struct RepoManager::Impl
 216   {
 217     Impl( const RepoManagerOptions &opt )
 218       : options(opt)
 219     {
 220
 221     }
 222
 223     Impl()
 224     {
 225
 226     }
 227
 228     RepoManagerOptions options;
 229
 230     map<string, RepoInfo> _repoinfos;
 231
 232   public:
 233     /** Offer default Impl. */
 234     static shared_ptr<Impl> nullimpl()
 235     {
 236       static shared_ptr<Impl> _nullimpl( new Impl );
 237       return _nullimpl;
 238     }
 239
 240   private:
 241     friend Impl * rwcowClone<Impl>( const Impl * rhs );
 242     /** clone for RWCOW_pointer */
 243     Impl * clone() const
 244     { return new Impl( *this ); }
 245   };
 246
 247   ///////////////////////////////////////////////////////////////////
 248
 249   /** \relates RepoManager::Impl Stream output */
 250   inline std::ostream & operator<<( std::ostream & str, const RepoManager::Impl & obj )
 251   {
 252     return str << "RepoManager::Impl";
 253   }
 254
 255   ///////////////////////////////////////////////////////////////////
 256   //
 257   //    CLASS NAME : RepoManager
 258   //
 259   ///////////////////////////////////////////////////////////////////
 260
 261   RepoManager::RepoManager( const RepoManagerOptions &opt )
 262   : _pimpl( new Impl(opt) )
 263   {}
 264
 265   ////////////////////////////////////////////////////////////////////////////
 266
 267   RepoManager::~RepoManager()
 268   {}
 269
 270   ////////////////////////////////////////////////////////////////////////////
 271
 272   std::list<RepoInfo> RepoManager::knownRepositories() const
 273   {
 274     MIL << endl;
 275
 276     if ( PathInfo(_pimpl->options.knownReposPath).isExist() )
 277     {
 278       RepoInfoList repos = repositories_in_dir(_pimpl->options.knownReposPath);
 279       for ( RepoInfoList::iterator it = repos.begin();
 280             it != repos.end();
 281             ++it )
 282       {
 283         // set the metadata path for the repo
 284         Pathname metadata_path = rawcache_path_for_repoinfo(_pimpl->options, (*it));
 285         (*it).setMetadataPath(metadata_path);
 286
 287         // set the downloaded packages path for the repo
 288         Pathname packages_path = packagescache_path_for_repoinfo(_pimpl->options, (*it));
 289         (*it).setPackagesPath(packages_path);
 290       }
 291       return repos;
 292     }
 293     else
 294       return std::list<RepoInfo>();
 295
 296     MIL << endl;
 297   }
 298
 299   ////////////////////////////////////////////////////////////////////////////
 300
 301   Pathname RepoManager::metadataPath( const RepoInfo &info ) const
 302   {
 303     return rawcache_path_for_repoinfo(_pimpl->options, info );
 304   }
 305
 306   ////////////////////////////////////////////////////////////////////////////
 307
 308   RepoStatus RepoManager::metadataStatus( const RepoInfo &info ) const
 309   {
 310     Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 311     RepoType repokind = info.type();
 312     RepoStatus status;
 313
 314     switch ( repokind.toEnum() )
 315     {
 316       case RepoType::NONE_e:
 317       // unknown, probe the local metadata
 318         repokind = probe(rawpath.asUrl());
 319       break;
 320       default:
 321       break;
 322     }
 323
 324     switch ( repokind.toEnum() )
 325     {
 326       case RepoType::RPMMD_e :
 327       {
 328         status = RepoStatus( rawpath + "/repodata/repomd.xml");
 329       }
 330       break;
 331
 332       case RepoType::YAST2_e :
 333       {
 334         status = RepoStatus( rawpath + "/content") && (RepoStatus( rawpath + "/media.1/media"));
 335       }
 336       break;
 337
 338       case RepoType::RPMPLAINDIR_e :
 339       {
 340         if ( PathInfo(Pathname(rawpath + "/cookie")).isExist() )
 341           status = RepoStatus( rawpath + "/cookie");
 342       }
 343       break;
 344
 345       case RepoType::NONE_e :
 346         // Return default RepoStatus in case of RepoType::NONE
 347         // indicating it should be created?
 348         // ZYPP_THROW(RepoUnknownTypeException());
 349         break;
 350     }
 351     return status;
 352   }
 353
 354   void RepoManager::touchIndexFile(const RepoInfo & info)
 355   {
 356     Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 357
 358     RepoType repokind = info.type();
 359     if ( repokind.toEnum() == RepoType::NONE_e )
 360       // unknown, probe the local metadata
 361       repokind = probe(rawpath.asUrl());
 362     // if still unknown, just return
 363     if (repokind == RepoType::NONE_e)
 364       return;
 365
 366     Pathname p;
 367     switch ( repokind.toEnum() )
 368     {
 369       case RepoType::RPMMD_e :
 370         p = Pathname(rawpath + "/repodata/repomd.xml");
 371         break;
 372
 373       case RepoType::YAST2_e :
 374         p = Pathname(rawpath + "/content");
 375         break;
 376
 377       case RepoType::RPMPLAINDIR_e :
 378         p = Pathname(rawpath + "/cookie");
 379         break;
 380
 381       case RepoType::NONE_e :
 382       default:
 383         break;
 384     }
 385
 386     // touch the file, ignore error (they are logged anyway)
 387     filesystem::touch(p);
 388   }
 389
 390   RepoManager::RefreshCheckStatus RepoManager::checkIfToRefreshMetadata(
 391                                               const RepoInfo &info,
 392                                               const Url &url,
 393                                               RawMetadataRefreshPolicy policy )
 394   {
 395     assert_alias(info);
 396
 397     RepoStatus oldstatus;
 398     RepoStatus newstatus;
 399
 400     try
 401     {
 402       MIL << "Going to try to check whether refresh is needed for " << url << endl;
 403
 404       repo::RepoType repokind = info.type();
 405
 406       // if the type is unknown, try probing.
 407       switch ( repokind.toEnum() )
 408       {
 409         case RepoType::NONE_e:
 410           // unknown, probe it
 411           repokind = probe(url);
 412         break;
 413         default:
 414         break;
 415       }
 416
 417       Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 418       filesystem::assert_dir(rawpath);
 419       oldstatus = metadataStatus(info);
 420
 421       // now we've got the old (cached) status, we can decide repo.refresh.delay
 422       if (policy != RefreshForced)
 423       {
 424         // difference in seconds
 425         double diff = difftime(
 426           (Date::ValueType)Date::now(),
 427           (Date::ValueType)oldstatus.timestamp()) / 60;
 428
 429         DBG << "oldstatus: " << (Date::ValueType)oldstatus.timestamp() << endl;
 430         DBG << "current time: " << (Date::ValueType)Date::now() << endl;
 431         DBG << "last refresh = " << diff << " minutes ago" << endl;
 432
 433         if (diff < ZConfig::instance().repo_refresh_delay())
 434         {
 435           MIL << "Repository '" << info.alias()
 436               << "' has been refreshed less than repo.refresh.delay ("
 437               << ZConfig::instance().repo_refresh_delay()
 438               << ") minutes ago. Advising to skip refresh" << endl;
 439           return REPO_CHECK_DELAYED;
 440         }
 441       }
 442
 443       // create temp dir as sibling of rawpath
 444       filesystem::TmpDir tmpdir( filesystem::TmpDir::makeSibling( rawpath ) );
 445
 446       if ( ( repokind.toEnum() == RepoType::RPMMD_e ) ||
 447            ( repokind.toEnum() == RepoType::YAST2_e ) )
 448       {
 449         MediaSetAccess media(url);
 450         shared_ptr<repo::Downloader> downloader_ptr;
 451
 452         if ( repokind.toEnum() == RepoType::RPMMD_e )
 453           downloader_ptr.reset(new yum::Downloader(info.path()));
 454         else
 455           downloader_ptr.reset( new susetags::Downloader(info.path()));
 456
 457         RepoStatus newstatus = downloader_ptr->status(media);
 458         bool refresh = false;
 459         if ( oldstatus.checksum() == newstatus.checksum() )
 460         {
 461           MIL << "repo has not changed" << endl;
 462           if ( policy == RefreshForced )
 463           {
 464             MIL << "refresh set to forced" << endl;
 465             refresh = true;
 466           }
 467         }
 468         else
 469         {
 470           MIL << "repo has changed, going to refresh" << endl;
 471           refresh = true;
 472         }
 473
 474         if (!refresh)
 475           touchIndexFile(info);
 476
 477         return refresh ? REFRESH_NEEDED : REPO_UP_TO_DATE;
 478       }
 479       else if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 480       {
 481         RepoStatus newstatus = parser::plaindir::dirStatus(url.getPathName());
 482         bool refresh = false;
 483         if ( oldstatus.checksum() == newstatus.checksum() )
 484         {
 485           MIL << "repo has not changed" << endl;
 486           if ( policy == RefreshForced )
 487           {
 488             MIL << "refresh set to forced" << endl;
 489             refresh = true;
 490           }
 491         }
 492         else
 493         {
 494           MIL << "repo has changed, going to refresh" << endl;
 495           refresh = true;
 496         }
 497
 498         if (!refresh)
 499           touchIndexFile(info);
 500
 501         return refresh ? REFRESH_NEEDED : REPO_UP_TO_DATE;
 502       }
 503       else
 504       {
 505         ZYPP_THROW(RepoUnknownTypeException());
 506       }
 507     }
 508     catch ( const Exception &e )
 509     {
 510       ZYPP_CAUGHT(e);
 511       ERR << "refresh check failed for " << url << endl;
 512       ZYPP_RETHROW(e);
 513     }
 514
 515     return REFRESH_NEEDED; // default
 516   }
 517
 518   void RepoManager::refreshMetadata( const RepoInfo &info,
 519                                      RawMetadataRefreshPolicy policy,
 520                                      const ProgressData::ReceiverFnc & progress )
 521   {
 522     assert_alias(info);
 523     assert_urls(info);
 524
 525     // we will throw this later if no URL checks out fine
 526     RepoException rexception(_("Valid metadata not found at specified URL(s)"));
 527
 528     // try urls one by one
 529     for ( RepoInfo::urls_const_iterator it = info.baseUrlsBegin(); it != info.baseUrlsEnd(); ++it )
 530     {
 531       try
 532       {
 533         Url url(*it);
 534
 535         // check whether to refresh metadata
 536         // if the check fails for this url, it throws, so another url will be checked
 537         if (checkIfToRefreshMetadata(info, url, policy)!=REFRESH_NEEDED)
 538           return;
 539
 540         MIL << "Going to refresh metadata from " << url << endl;
 541
 542         repo::RepoType repokind = info.type();
 543
 544         // if the type is unknown, try probing.
 545         switch ( repokind.toEnum() )
 546         {
 547           case RepoType::NONE_e:
 548             // unknown, probe it
 549             repokind = probe(*it);
 550
 551             if (repokind.toEnum() != RepoType::NONE_e)
 552             {
 553               //save probed type only for repos in system
 554               std::list<RepoInfo> repos = knownRepositories();
 555               for ( std::list<RepoInfo>::const_iterator it = repos.begin();
 556                    it != repos.end(); ++it )
 557               {
 558                 if ( info.alias() == (*it).alias() )
 559                 {
 560                   RepoInfo modifiedrepo = info;
 561                   modifiedrepo.setType(repokind);
 562                   modifyRepository(info.alias(),modifiedrepo);
 563                 }
 564               }
 565             }
 566           break;
 567           default:
 568           break;
 569         }
 570
 571         Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 572         filesystem::assert_dir(rawpath);
 573
 574         // create temp dir as sibling of rawpath
 575         filesystem::TmpDir tmpdir( filesystem::TmpDir::makeSibling( rawpath ) );
 576
 577         if ( ( repokind.toEnum() == RepoType::RPMMD_e ) ||
 578              ( repokind.toEnum() == RepoType::YAST2_e ) )
 579         {
 580           MediaSetAccess media(url);
 581           shared_ptr<repo::Downloader> downloader_ptr;
 582
 583           if ( repokind.toEnum() == RepoType::RPMMD_e )
 584             downloader_ptr.reset(new yum::Downloader(info.path()));
 585           else
 586             downloader_ptr.reset( new susetags::Downloader(info.path()));
 587
 588           /**
 589            * Given a downloader, sets the other repos raw metadata
 590            * path as cache paths for the fetcher, so if another
 591            * repo has the same file, it will not download it
 592            * but copy it from the other repository
 593            */
 594           std::list<RepoInfo> repos = knownRepositories();
 595           for ( std::list<RepoInfo>::const_iterator it = repos.begin();
 596                 it != repos.end();
 597                 ++it )
 598           {
 599             Pathname cachepath(rawcache_path_for_repoinfo( _pimpl->options, *it ));
 600             if ( PathInfo(cachepath).isExist() )
 601               downloader_ptr->addCachePath(cachepath);
 602           }
 603
 604           downloader_ptr->download( media, tmpdir.path());
 605         }
 606         else if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 607         {
 608           RepoStatus newstatus = parser::plaindir::dirStatus(url.getPathName());
 609
 610           std::ofstream file(( tmpdir.path() + "/cookie").c_str());
 611           if (!file) {
 612             ZYPP_THROW (Exception( "Can't open " + tmpdir.path().asString() + "/cookie" ) );
 613           }
 614           file << url << endl;
 615           file << newstatus.checksum() << endl;
 616
 617           file.close();
 618         }
 619         else
 620         {
 621           ZYPP_THROW(RepoUnknownTypeException());
 622         }
 623
 624         // ok we have the metadata, now exchange
 625         // the contents
 626
 627         TmpDir oldmetadata( TmpDir::makeSibling( rawpath ) );
 628         filesystem::rename( rawpath, oldmetadata.path() );
 629         // move the just downloaded there
 630         filesystem::rename( tmpdir.path(), rawpath );
 631
 632         // we are done.
 633         return;
 634       }
 635       catch ( const Exception &e )
 636       {
 637         ZYPP_CAUGHT(e);
 638         ERR << "Trying another url..." << endl;
 639
 640         // remember the exception caught for the *first URL*
 641         // if all other URLs fail, the rexception will be thrown with the
 642         // cause of the problem of the first URL remembered
 643         if (it == info.baseUrlsBegin())
 644           rexception.remember(e);
 645       }
 646     } // for every url
 647     ERR << "No more urls..." << endl;
 648     ZYPP_THROW(rexception);
 649   }
 650
 651   ////////////////////////////////////////////////////////////////////////////
 652
 653   void RepoManager::cleanMetadata( const RepoInfo &info,
 654                                    const ProgressData::ReceiverFnc & progressfnc )
 655   {
 656     ProgressData progress(100);
 657     progress.sendTo(progressfnc);
 658
 659     filesystem::recursive_rmdir(rawcache_path_for_repoinfo(_pimpl->options, info));
 660     progress.toMax();
 661   }
 662
 663   void RepoManager::cleanPackages( const RepoInfo &info,
 664                                    const ProgressData::ReceiverFnc & progressfnc )
 665   {
 666     ProgressData progress(100);
 667     progress.sendTo(progressfnc);
 668
 669     filesystem::recursive_rmdir(packagescache_path_for_repoinfo(_pimpl->options, info));
 670     progress.toMax();
 671   }
 672
 673   void RepoManager::buildCache( const RepoInfo &info,
 674                                 CacheBuildPolicy policy,
 675                                 const ProgressData::ReceiverFnc & progressrcv )
 676   {
 677     assert_alias(info);
 678     Pathname rawpath = rawcache_path_for_repoinfo(_pimpl->options, info);
 679
 680     filesystem::assert_dir(_pimpl->options.repoCachePath);
 681     Pathname base = _pimpl->options.repoCachePath + info.escaped_alias();
 682     Pathname solvfile = base.extend(".solv");
 683
 684
 685     RepoStatus raw_metadata_status = metadataStatus(info);
 686     if ( raw_metadata_status.empty() )
 687     {
 688        /* if there is no cache at this point, we refresh the raw
 689           in case this is the first time - if it's !autorefresh,
 690           we may still refresh */
 691       refreshMetadata(info, RefreshIfNeeded, progressrcv );
 692       raw_metadata_status = metadataStatus(info);
 693     }
 694
 695     bool needs_cleaning = false;
 696     if ( isCached( info ) )
 697     {
 698       MIL << info.alias() << " is already cached." << endl;
 699       //data::RecordId id = store.lookupRepository(info.alias());
 700       RepoStatus cache_status = cacheStatus(info);
 701
 702       if ( cache_status.checksum() == raw_metadata_status.checksum() )
 703       {
 704         MIL << info.alias() << " cache is up to date with metadata." << endl;
 705         if ( policy == BuildIfNeeded ) {
 706           return;
 707         }
 708         else {
 709           MIL << info.alias() << " cache rebuild is forced" << endl;
 710         }
 711       }
 712
 713       needs_cleaning = true;
 714     }
 715
 716     ProgressData progress(100);
 717     callback::SendReport<ProgressReport> report;
 718     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
 719     progress.name(str::form(_("Building repository '%s' cache"), info.name().c_str()));
 720     progress.toMin();
 721
 722     if (needs_cleaning)
 723     {
 724       cleanCache(info);
 725     }
 726
 727     MIL << info.alias() << " building cache..." << endl;
 728     // do we have type?
 729     repo::RepoType repokind = info.type();
 730
 731     // if the type is unknown, try probing.
 732     switch ( repokind.toEnum() )
 733     {
 734       case RepoType::NONE_e:
 735         // unknown, probe the local metadata
 736         repokind = probe(rawpath.asUrl());
 737       break;
 738       default:
 739       break;
 740     }
 741
 742     MIL << "repo type is " << repokind << endl;
 743
 744     switch ( repokind.toEnum() )
 745     {
 746       case RepoType::RPMMD_e :
 747       case RepoType::YAST2_e :
 748       case RepoType::RPMPLAINDIR_e :
 749       {
 750         // Take care we unlink the solvfile on exception
 751         ManagedFile guard( solvfile, filesystem::unlink );
 752
 753         ostringstream cmd;
 754         if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 755         {
 756           cmd << str::form( "repo2solv.sh \"%s\" > \"%s\"", info.baseUrlsBegin()->getPathName().c_str(), solvfile.c_str() );
 757         } else
 758           cmd << str::form( "repo2solv.sh \"%s\" > \"%s\"", rawpath.c_str(), solvfile.c_str() );
 759
 760         MIL << "Executing: " << cmd.str() << endl;
 761         ExternalProgram prog( cmd.str(), ExternalProgram::Stderr_To_Stdout );
 762
 763         cmd << endl;
 764         for ( string output( prog.receiveLine() ); output.length(); output = prog.receiveLine() ) {
 765           WAR << "  " << output;
 766           cmd << "     " << output;
 767         }
 768
 769         int ret = prog.close();
 770         if ( ret != 0 )
 771         {
 772           RepoException ex(str::form("Failed to cache repo (%d).", ret));
 773           ex.remember( cmd.str() );
 774           ZYPP_THROW(ex);
 775         }
 776
 777         // We keep it.
 778         guard.resetDispose();
 779       }
 780       break;
 781       default:
 782         ZYPP_THROW(RepoUnknownTypeException("Unhandled repository type"));
 783       break;
 784     }
 785 #if 0
 786     switch ( repokind.toEnum() )
 787     {
 788       case RepoType::RPMMD_e :
 789       if (0)
 790       {
 791         CombinedProgressData subprogrcv( progress, 100);
 792         parser::yum::RepoParser parser(id, store, parser::yum::RepoParserOpts(), subprogrcv);
 793         parser.parse(rawpath);
 794           // no error
 795       }
 796       break;
 797       case RepoType::YAST2_e :
 798       if (0)
 799       {
 800         CombinedProgressData subprogrcv( progress, 100);
 801         parser::susetags::RepoParser parser(id, store, subprogrcv);
 802         parser.parse(rawpath);
 803         // no error
 804       }
 805       break;
 806
 807       default:
 808         ZYPP_THROW(RepoUnknownTypeException());
 809     }
 810 #endif
 811     // update timestamp and checksum
 812     //store.updateRepositoryStatus(id, raw_metadata_status);
 813     setCacheStatus(info.escaped_alias(), raw_metadata_status);
 814     MIL << "Commit cache.." << endl;
 815     //store.commit();
 816     //progress.toMax();
 817   }
 818
 819   ////////////////////////////////////////////////////////////////////////////
 820
 821   repo::RepoType RepoManager::probe( const Url &url ) const
 822   {
 823     if ( url.getScheme() == "dir" && ! PathInfo( url.getPathName() ).isDir() )
 824     {
 825       // Handle non existing local directory in advance, as
 826       // MediaSetAccess does not support it.
 827       return repo::RepoType::NONE;
 828     }
 829
 830     try
 831     {
 832       MediaSetAccess access(url);
 833       if ( access.doesFileExist("/repodata/repomd.xml") )
 834         return repo::RepoType::RPMMD;
 835       if ( access.doesFileExist("/content") )
 836         return repo::RepoType::YAST2;
 837
 838       // if it is a local url of type dir
 839       if ( (! media::MediaManager::downloads(url)) && ( url.getScheme() == "dir" ) )
 840       {
 841         Pathname path = Pathname(url.getPathName());
 842         if ( PathInfo(path).isDir() )
 843         {
 844           // allow empty dirs for now
 845           return repo::RepoType::RPMPLAINDIR;
 846         }
 847       }
 848     }
 849     catch ( const media::MediaException &e )
 850     {
 851       ZYPP_CAUGHT(e);
 852       RepoException enew("Error trying to read from " + url.asString());
 853       enew.remember(e);
 854       ZYPP_THROW(enew);
 855     }
 856     catch ( const Exception &e )
 857     {
 858       ZYPP_CAUGHT(e);
 859       Exception enew("Unknown error reading from " + url.asString());
 860       enew.remember(e);
 861       ZYPP_THROW(enew);
 862     }
 863
 864     return repo::RepoType::NONE;
 865   }
 866
 867   ////////////////////////////////////////////////////////////////////////////
 868
 869   void RepoManager::cleanCache( const RepoInfo &info,
 870                                 const ProgressData::ReceiverFnc & progressrcv )
 871   {
 872     ProgressData progress(100);
 873     progress.sendTo(progressrcv);
 874     progress.toMin();
 875
 876     unlink (_pimpl->options.repoCachePath / (info.escaped_alias() + ".solv"));
 877     progress.set(99);
 878     unlink (_pimpl->options.repoCachePath / (info.escaped_alias() + ".cookie"));
 879
 880     progress.toMax();
 881   }
 882
 883   void RepoManager::cleanTargetCache(const ProgressData::ReceiverFnc & progressrcv)
 884   {
 885     unlink (_pimpl->options.repoCachePath / (sat::Pool::systemRepoName() + ".solv"));
 886     unlink (_pimpl->options.repoCachePath / (sat::Pool::systemRepoName() + ".cookie"));
 887   }
 888
 889   ////////////////////////////////////////////////////////////////////////////
 890
 891   bool RepoManager::isCached( const RepoInfo &info ) const
 892   {
 893     Pathname name = _pimpl->options.repoCachePath;
 894     return PathInfo(name + Pathname(info.escaped_alias()).extend(".solv")).isExist();
 895   }
 896
 897   RepoStatus RepoManager::cacheStatus( const RepoInfo &info ) const
 898   {
 899
 900     Pathname base = _pimpl->options.repoCachePath + info.escaped_alias();
 901     Pathname cookiefile = base.extend(".cookie");
 902
 903     return RepoStatus::fromCookieFile(cookiefile);
 904   }
 905
 906   void RepoManager::setCacheStatus( const string &alias, const RepoStatus &status )
 907   {
 908     filesystem::assert_dir(_pimpl->options.repoCachePath);
 909     Pathname base = _pimpl->options.repoCachePath + alias;
 910     Pathname cookiefile = base.extend(".cookie");
 911
 912     status.saveToCookieFile(cookiefile);
 913   }
 914
 915   void RepoManager::loadFromCache( const RepoInfo & info,
 916                                    const ProgressData::ReceiverFnc & progressrcv )
 917   {
 918     assert_alias(info);
 919     string alias = info.escaped_alias();
 920     Pathname solvfile = (_pimpl->options.repoCachePath / alias).extend(".solv");
 921
 922     if ( ! PathInfo(solvfile).isExist() )
 923       ZYPP_THROW(RepoNotCachedException());
 924
 925     try
 926     {
 927       sat::Pool::instance().addRepoSolv( solvfile, info );
 928     }
 929     catch ( const Exception & exp )
 930     {
 931       ZYPP_CAUGHT( exp );
 932       MIL << "Try to handle exception by rebuilding the solv-file" << endl;
 933       cleanCache( info, progressrcv );
 934       buildCache( info, BuildIfNeeded, progressrcv );
 935
 936       sat::Pool::instance().addRepoSolv( solvfile, info );
 937     }
 938   }
 939
 940   ////////////////////////////////////////////////////////////////////////////
 941
 942   /**
 943    * Generate a non existing filename in a directory, using a base
 944    * name. For example if a directory contains 3 files
 945    *
 946    * |-- bar
 947    * |-- foo
 948    * `-- moo
 949    *
 950    * If you try to generate a unique filename for this directory,
 951    * based on "ruu" you will get "ruu", but if you use the base
 952    * "foo" you will get "foo_1"
 953    *
 954    * \param dir Directory where the file needs to be unique
 955    * \param basefilename string to base the filename on.
 956    */
 957   static Pathname generate_non_existing_name( const Pathname &dir,
 958                                               const std::string &basefilename )
 959   {
 960     string final_filename = basefilename;
 961     int counter = 1;
 962     while ( PathInfo(dir + final_filename).isExist() )
 963     {
 964       final_filename = basefilename + "_" + str::numstring(counter);
 965       counter++;
 966     }
 967     return dir + Pathname(final_filename);
 968   }
 969
 970   ////////////////////////////////////////////////////////////////////////////
 971
 972   /**
 973    * \short Generate a related filename from a repo info
 974    *
 975    * From a repo info, it will try to use the alias as a filename
 976    * escaping it if necessary. Other fallbacks can be added to
 977    * this function in case there is no way to use the alias
 978    */
 979   static std::string generate_filename( const RepoInfo &info )
 980   {
 981     std::string fnd="/";
 982     std::string rep="_";
 983     std::string filename = info.alias();
 984     // replace slashes with underscores
 985     size_t pos = filename.find(fnd);
 986     while(pos!=string::npos)
 987     {
 988       filename.replace(pos,fnd.length(),rep);
 989       pos = filename.find(fnd,pos+rep.length());
 990     }
 991     filename = Pathname(filename).extend(".repo").asString();
 992     MIL << "generating filename for repo [" << info.alias() << "] : '" << filename << "'" << endl;
 993     return filename;
 994   }
 995
 996
 997   ////////////////////////////////////////////////////////////////////////////
 998
 999   void RepoManager::addRepository( const RepoInfo &info,
1000                                    const ProgressData::ReceiverFnc & progressrcv )
1001   {
1002     assert_alias(info);
1003
1004     ProgressData progress(100);
1005     callback::SendReport<ProgressReport> report;
1006     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
1007     progress.name(str::form(_("Adding repository '%s'"), info.name().c_str()));
1008     progress.toMin();
1009
1010     std::list<RepoInfo> repos = knownRepositories();
1011     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1012           it != repos.end();
1013           ++it )
1014     {
1015       if ( info.alias() == (*it).alias() )
1016         ZYPP_THROW(RepoAlreadyExistsException(info.alias()));
1017     }
1018
1019     RepoInfo tosave = info;
1020
1021     // check the first url for now
1022     if ( _pimpl->options.probe )
1023     {
1024       DBG << "unknown repository type, probing" << endl;
1025
1026       RepoType probedtype;
1027       probedtype = probe(*tosave.baseUrlsBegin());
1028       if ( tosave.baseUrlsSize() > 0 )
1029       {
1030         if ( probedtype == RepoType::NONE )
1031           ZYPP_THROW(RepoUnknownTypeException());
1032         else
1033           tosave.setType(probedtype);
1034       }
1035     }
1036
1037     progress.set(50);
1038
1039     // assert the directory exists
1040     filesystem::assert_dir(_pimpl->options.knownReposPath);
1041
1042     Pathname repofile = generate_non_existing_name(_pimpl->options.knownReposPath,
1043                                                     generate_filename(tosave));
1044     // now we have a filename that does not exists
1045     MIL << "Saving repo in " << repofile << endl;
1046
1047     std::ofstream file(repofile.c_str());
1048     if (!file) {
1049       ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
1050     }
1051
1052     tosave.dumpRepoOn(file);
1053     progress.toMax();
1054     MIL << "done" << endl;
1055   }
1056
1057   void RepoManager::addRepositories( const Url &url,
1058                                      const ProgressData::ReceiverFnc & progressrcv )
1059   {
1060     std::list<RepoInfo> knownrepos = knownRepositories();
1061     std::list<RepoInfo> repos = readRepoFile(url);
1062     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1063           it != repos.end();
1064           ++it )
1065     {
1066       // look if the alias is in the known repos.
1067       for ( std::list<RepoInfo>::const_iterator kit = knownrepos.begin();
1068           kit != knownrepos.end();
1069           ++kit )
1070       {
1071         if ( (*it).alias() == (*kit).alias() )
1072         {
1073           ERR << "To be added repo " << (*it).alias() << " conflicts with existing repo " << (*kit).alias() << endl;
1074           ZYPP_THROW(RepoAlreadyExistsException((*it).alias()));
1075         }
1076       }
1077     }
1078
1079     string filename = Pathname(url.getPathName()).basename();
1080
1081     if ( filename == Pathname() )
1082       ZYPP_THROW(RepoException("Invalid repo file name at " + url.asString() ));
1083
1084     // assert the directory exists
1085     filesystem::assert_dir(_pimpl->options.knownReposPath);
1086
1087     Pathname repofile = generate_non_existing_name(_pimpl->options.knownReposPath, filename);
1088     // now we have a filename that does not exists
1089     MIL << "Saving " << repos.size() << " repo" << ( repos.size() ? "s" : "" ) << " in " << repofile << endl;
1090
1091     std::ofstream file(repofile.c_str());
1092     if (!file) {
1093       ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
1094     }
1095
1096     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1097           it != repos.end();
1098           ++it )
1099     {
1100       MIL << "Saving " << (*it).alias() << endl;
1101       (*it).dumpRepoOn(file);
1102     }
1103     MIL << "done" << endl;
1104   }
1105
1106   ////////////////////////////////////////////////////////////////////////////
1107
1108   void RepoManager::removeRepository( const RepoInfo & info,
1109                                       const ProgressData::ReceiverFnc & progressrcv)
1110   {
1111     ProgressData progress;
1112     callback::SendReport<ProgressReport> report;
1113     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
1114     progress.name(str::form(_("Removing repository '%s'"), info.name().c_str()));
1115
1116     MIL << "Going to delete repo " << info.alias() << endl;
1117
1118     std::list<RepoInfo> repos = knownRepositories();
1119     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1120           it != repos.end();
1121           ++it )
1122     {
1123       // they can be the same only if the provided is empty, that means
1124       // the provided repo has no alias
1125       // then skip
1126       if ( (!info.alias().empty()) && ( info.alias() != (*it).alias() ) )
1127         continue;
1128
1129       // TODO match by url
1130
1131       // we have a matcing repository, now we need to know
1132       // where it does come from.
1133       RepoInfo todelete = *it;
1134       if (todelete.filepath().empty())
1135       {
1136         ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
1137       }
1138       else
1139       {
1140         // figure how many repos are there in the file:
1141         std::list<RepoInfo> filerepos = repositories_in_file(todelete.filepath());
1142         if ( (filerepos.size() == 1) && ( filerepos.front().alias() == todelete.alias() ) )
1143         {
1144           // easy, only this one, just delete the file
1145           if ( filesystem::unlink(todelete.filepath()) != 0 )
1146           {
1147             ZYPP_THROW(RepoException("Can't delete " + todelete.filepath().asString()));
1148           }
1149           MIL << todelete.alias() << " sucessfully deleted." << endl;
1150         }
1151         else
1152         {
1153           // there are more repos in the same file
1154           // write them back except the deleted one.
1155           //TmpFile tmp;
1156           //std::ofstream file(tmp.path().c_str());
1157
1158           // assert the directory exists
1159           filesystem::assert_dir(todelete.filepath().dirname());
1160
1161           std::ofstream file(todelete.filepath().c_str());
1162           if (!file) {
1163             //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
1164             ZYPP_THROW (Exception( "Can't open " + todelete.filepath().asString() ) );
1165           }
1166           for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
1167                 fit != filerepos.end();
1168                 ++fit )
1169           {
1170             if ( (*fit).alias() != todelete.alias() )
1171               (*fit).dumpRepoOn(file);
1172           }
1173         }
1174
1175         CombinedProgressData subprogrcv(progress, 70);
1176         CombinedProgressData cleansubprogrcv(progress, 30);
1177         // now delete it from cache
1178         if ( isCached(todelete) )
1179           cleanCache( todelete, subprogrcv);
1180         // now delete metadata (#301037)
1181         cleanMetadata( todelete, cleansubprogrcv);
1182         MIL << todelete.alias() << " sucessfully deleted." << endl;
1183         return;
1184       } // else filepath is empty
1185
1186     }
1187     // should not be reached on a sucess workflow
1188     ZYPP_THROW(RepoNotFoundException(info));
1189   }
1190
1191   ////////////////////////////////////////////////////////////////////////////
1192
1193   void RepoManager::modifyRepository( const std::string &alias,
1194                                       const RepoInfo & newinfo,
1195                                       const ProgressData::ReceiverFnc & progressrcv )
1196   {
1197     RepoInfo toedit = getRepositoryInfo(alias);
1198
1199     // check if the new alias already exists when renaming the repo
1200     if (alias != newinfo.alias())
1201     {
1202       std::list<RepoInfo> repos = knownRepositories();
1203       for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1204             it != repos.end();
1205             ++it )
1206       {
1207         if ( newinfo.alias() == (*it).alias() )
1208           ZYPP_THROW(RepoAlreadyExistsException(newinfo.alias()));
1209       }
1210     }
1211
1212     if (toedit.filepath().empty())
1213     {
1214       ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
1215     }
1216     else
1217     {
1218       // figure how many repos are there in the file:
1219       std::list<RepoInfo> filerepos = repositories_in_file(toedit.filepath());
1220
1221       // there are more repos in the same file
1222       // write them back except the deleted one.
1223       //TmpFile tmp;
1224       //std::ofstream file(tmp.path().c_str());
1225
1226       // assert the directory exists
1227       filesystem::assert_dir(toedit.filepath().dirname());
1228
1229       std::ofstream file(toedit.filepath().c_str());
1230       if (!file) {
1231         //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
1232         ZYPP_THROW (Exception( "Can't open " + toedit.filepath().asString() ) );
1233       }
1234       for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
1235             fit != filerepos.end();
1236             ++fit )
1237       {
1238           // if the alias is different, dump the original
1239           // if it is the same, dump the provided one
1240           if ( (*fit).alias() != toedit.alias() )
1241             (*fit).dumpRepoOn(file);
1242           else
1243             newinfo.dumpRepoOn(file);
1244       }
1245     }
1246   }
1247
1248   ////////////////////////////////////////////////////////////////////////////
1249
1250   RepoInfo RepoManager::getRepositoryInfo( const std::string &alias,
1251                                            const ProgressData::ReceiverFnc & progressrcv )
1252   {
1253     std::list<RepoInfo> repos = knownRepositories();
1254     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1255           it != repos.end();
1256           ++it )
1257     {
1258       if ( (*it).alias() == alias )
1259         return *it;
1260     }
1261     RepoInfo info;
1262     info.setAlias(info.alias());
1263     ZYPP_THROW(RepoNotFoundException(info));
1264   }
1265
1266   ////////////////////////////////////////////////////////////////////////////
1267
1268   RepoInfo RepoManager::getRepositoryInfo( const Url & url,
1269                                            const url::ViewOption & urlview,
1270                                            const ProgressData::ReceiverFnc & progressrcv )
1271   {
1272     std::list<RepoInfo> repos = knownRepositories();
1273     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1274           it != repos.end();
1275           ++it )
1276     {
1277       for(RepoInfo::urls_const_iterator urlit = (*it).baseUrlsBegin();
1278           urlit != (*it).baseUrlsEnd();
1279           ++urlit)
1280       {
1281         if ((*urlit).asString(urlview) == url.asString(urlview))
1282           return *it;
1283       }
1284     }
1285     RepoInfo info;
1286     info.setAlias(info.alias());
1287     info.setBaseUrl(url);
1288     ZYPP_THROW(RepoNotFoundException(info));
1289   }
1290
1291   ////////////////////////////////////////////////////////////////////////////
1292
1293   std::ostream & operator<<( std::ostream & str, const RepoManager & obj )
1294   {
1295     return str << *obj._pimpl;
1296   }
1297
1298   /////////////////////////////////////////////////////////////////
1299 } // namespace zypp
1300 ///////////////////////////////////////////////////////////////////