zypp/RepoManager.cc

   1 /*---------------------------------------------------------------------\
   2 |                          ____ _   __ __ ___                          |
   3 |                         |__  / \ / / . \ . \                         |
   4 |                           / / \ V /|  _/  _/                         |
   5 |                          / /__ | | | | | |                           |
   6 |                         /_____||_| |_| |_|                           |
   7 |                                                                      |
   8 \---------------------------------------------------------------------*/
   9 /** \file       zypp/RepoManager.cc
  10  *
  11 */
  12
  13 #include <cstdlib>
  14 #include <iostream>
  15 #include <fstream>
  16 #include <sstream>
  17 #include <list>
  18 #include <map>
  19 #include <algorithm>
  20 #include "zypp/base/InputStream.h"
  21 #include "zypp/base/Logger.h"
  22 #include "zypp/base/Gettext.h"
  23 #include "zypp/base/Function.h"
  24 #include "zypp/base/Regex.h"
  25 #include "zypp/PathInfo.h"
  26 #include "zypp/TmpPath.h"
  27
  28 #include "zypp/repo/RepoException.h"
  29 #include "zypp/RepoManager.h"
  30
  31 #include "zypp/media/MediaManager.h"
  32 #include "zypp/MediaSetAccess.h"
  33 #include "zypp/ExternalProgram.h"
  34 #include "zypp/ManagedFile.h"
  35
  36 #include "zypp/parser/RepoFileReader.h"
  37 #include "zypp/repo/yum/Downloader.h"
  38 #include "zypp/parser/yum/RepoParser.h"
  39 #include "zypp/repo/susetags/Downloader.h"
  40 #include "zypp/parser/susetags/RepoParser.h"
  41 #include "zypp/parser/plaindir/RepoParser.h"
  42
  43 #include "zypp/ZYppCallbacks.h"
  44
  45 #include "sat/Pool.h"
  46 #include "satsolver/pool.h"
  47 #include "satsolver/repo.h"
  48 #include "satsolver/repo_solv.h"
  49
  50 using namespace std;
  51 using namespace zypp;
  52 using namespace zypp::repo;
  53 using namespace zypp::filesystem;
  54
  55 using namespace zypp::repo;
  56
  57 ///////////////////////////////////////////////////////////////////
  58 namespace zypp
  59 { /////////////////////////////////////////////////////////////////
  60
  61   ///////////////////////////////////////////////////////////////////
  62   //
  63   //    CLASS NAME : RepoManagerOptions
  64   //
  65   ///////////////////////////////////////////////////////////////////
  66
  67   RepoManagerOptions::RepoManagerOptions( const Pathname & root_r )
  68   {
  69     repoCachePath         = Pathname::assertprefix( root_r, ZConfig::instance().repoCachePath() );
  70     repoRawCachePath      = Pathname::assertprefix( root_r, ZConfig::instance().repoMetadataPath() );
  71     repoSolvCachePath     = Pathname::assertprefix( root_r, ZConfig::instance().repoSolvfilesPath() );
  72     repoPackagesCachePath = Pathname::assertprefix( root_r, ZConfig::instance().repoPackagesPath() );
  73     knownReposPath        = Pathname::assertprefix( root_r, ZConfig::instance().knownReposPath() );
  74     probe                 = ZConfig::instance().repo_add_probe();
  75   }
  76
  77   RepoManagerOptions RepoManagerOptions::makeTestSetup( const Pathname & root_r )
  78   {
  79     RepoManagerOptions ret;
  80     ret.repoCachePath         = root_r;
  81     ret.repoRawCachePath      = root_r/"raw";
  82     ret.repoSolvCachePath     = root_r/"solv";
  83     ret.repoPackagesCachePath = root_r/"packages";
  84     ret.knownReposPath        = root_r/"repos.d";
  85     return ret;
  86   }
  87
  88   ////////////////////////////////////////////////////////////////////////////
  89
  90   /**
  91     * \short Simple callback to collect the results
  92     *
  93     * Classes like RepoFileParser call the callback
  94     * once per each repo in a file.
  95     *
  96     * Passing this functor as callback, you can collect
  97     * all resuls at the end, without dealing with async
  98     * code.
  99     */
 100     struct RepoCollector
 101     {
 102       RepoCollector()
 103       {}
 104
 105       ~RepoCollector()
 106       {}
 107
 108       bool collect( const RepoInfo &repo )
 109       {
 110         repos.push_back(repo);
 111         return true;
 112       }
 113
 114       RepoInfoList repos;
 115     };
 116
 117   ////////////////////////////////////////////////////////////////////////////
 118
 119   /**
 120    * Reads RepoInfo's from a repo file.
 121    *
 122    * \param file pathname of the file to read.
 123    */
 124   static std::list<RepoInfo> repositories_in_file( const Pathname & file )
 125   {
 126     MIL << "repo file: " << file << endl;
 127     RepoCollector collector;
 128     parser::RepoFileReader parser( file, bind( &RepoCollector::collect, &collector, _1 ) );
 129     return collector.repos;
 130   }
 131
 132   ////////////////////////////////////////////////////////////////////////////
 133
 134   std::list<RepoInfo> readRepoFile(const Url & repo_file)
 135    {
 136      // no interface to download a specific file, using workaround:
 137      //! \todo add MediaManager::provideFile(Url file_url) to easily access any file URLs? (no need for media access id or media_nr)
 138      Url url(repo_file);
 139      Pathname path(url.getPathName());
 140      url.setPathName ("/");
 141      MediaSetAccess access(url);
 142      Pathname local = access.provideFile(path);
 143
 144      DBG << "reading repo file " << repo_file << ", local path: " << local << endl;
 145
 146      return repositories_in_file(local);
 147    }
 148
 149   ////////////////////////////////////////////////////////////////////////////
 150
 151   /**
 152    * \short List of RepoInfo's from a directory
 153    *
 154    * Goes trough every file ending with ".repo" in a directory and adds all
 155    * RepoInfo's contained in that file.
 156    *
 157    * \param dir pathname of the directory to read.
 158    */
 159   static std::list<RepoInfo> repositories_in_dir( const Pathname &dir )
 160   {
 161     MIL << "directory " << dir << endl;
 162     list<RepoInfo> repos;
 163     list<Pathname> entries;
 164     if ( filesystem::readdir( entries, Pathname(dir), false ) != 0 )
 165       ZYPP_THROW(Exception("failed to read directory"));
 166
 167     str::regex allowedRepoExt("^\\.repo(_[0-9]+)?$");
 168     for ( list<Pathname>::const_iterator it = entries.begin(); it != entries.end(); ++it )
 169     {
 170       if (str::regex_match(it->extension(), allowedRepoExt))
 171       {
 172         list<RepoInfo> tmp = repositories_in_file( *it );
 173         repos.insert( repos.end(), tmp.begin(), tmp.end() );
 174
 175         //std::copy( collector.repos.begin(), collector.repos.end(), std::back_inserter(repos));
 176         //MIL << "ok" << endl;
 177       }
 178     }
 179     return repos;
 180   }
 181
 182   ////////////////////////////////////////////////////////////////////////////
 183
 184   static void assert_alias( const RepoInfo &info )
 185   {
 186     if (info.alias().empty())
 187         ZYPP_THROW(RepoNoAliasException());
 188   }
 189
 190   ////////////////////////////////////////////////////////////////////////////
 191
 192   static void assert_urls( const RepoInfo &info )
 193   {
 194     if (info.baseUrlsEmpty())
 195         ZYPP_THROW(RepoNoUrlException());
 196   }
 197
 198   ////////////////////////////////////////////////////////////////////////////
 199
 200   /**
 201    * \short Calculates the raw cache path for a repository
 202    */
 203   static Pathname rawcache_path_for_repoinfo( const RepoManagerOptions &opt, const RepoInfo &info )
 204   {
 205     assert_alias(info);
 206     return opt.repoRawCachePath / info.escaped_alias();
 207   }
 208
 209   /**
 210    * \short Calculates the packages cache path for a repository
 211    */
 212   static Pathname packagescache_path_for_repoinfo( const RepoManagerOptions &opt, const RepoInfo &info )
 213   {
 214     assert_alias(info);
 215     return opt.repoPackagesCachePath / info.escaped_alias();
 216   }
 217
 218   static Pathname solv_path_for_repoinfo( const RepoManagerOptions &opt, const RepoInfo &info)
 219   {
 220     assert_alias(info);
 221     return opt.repoSolvCachePath / info.escaped_alias();
 222   }
 223
 224   ///////////////////////////////////////////////////////////////////
 225   //
 226   //    CLASS NAME : RepoManager::Impl
 227   //
 228   ///////////////////////////////////////////////////////////////////
 229
 230   /**
 231    * \short RepoManager implementation.
 232    */
 233   struct RepoManager::Impl
 234   {
 235     Impl( const RepoManagerOptions &opt )
 236       : options(opt)
 237     {
 238
 239     }
 240
 241     Impl()
 242     {
 243
 244     }
 245
 246     RepoManagerOptions options;
 247
 248     map<string, RepoInfo> _repoinfos;
 249
 250   public:
 251     /** Offer default Impl. */
 252     static shared_ptr<Impl> nullimpl()
 253     {
 254       static shared_ptr<Impl> _nullimpl( new Impl );
 255       return _nullimpl;
 256     }
 257
 258   private:
 259     friend Impl * rwcowClone<Impl>( const Impl * rhs );
 260     /** clone for RWCOW_pointer */
 261     Impl * clone() const
 262     { return new Impl( *this ); }
 263   };
 264
 265   ///////////////////////////////////////////////////////////////////
 266
 267   /** \relates RepoManager::Impl Stream output */
 268   inline std::ostream & operator<<( std::ostream & str, const RepoManager::Impl & obj )
 269   {
 270     return str << "RepoManager::Impl";
 271   }
 272
 273   ///////////////////////////////////////////////////////////////////
 274   //
 275   //    CLASS NAME : RepoManager
 276   //
 277   ///////////////////////////////////////////////////////////////////
 278
 279   RepoManager::RepoManager( const RepoManagerOptions &opt )
 280   : _pimpl( new Impl(opt) )
 281   {}
 282
 283   ////////////////////////////////////////////////////////////////////////////
 284
 285   RepoManager::~RepoManager()
 286   {}
 287
 288   ////////////////////////////////////////////////////////////////////////////
 289
 290   std::list<RepoInfo> RepoManager::knownRepositories() const
 291   {
 292     MIL << endl;
 293
 294     if ( PathInfo(_pimpl->options.knownReposPath).isExist() )
 295     {
 296       RepoInfoList repos = repositories_in_dir(_pimpl->options.knownReposPath);
 297       for ( RepoInfoList::iterator it = repos.begin();
 298             it != repos.end();
 299             ++it )
 300       {
 301         // set the metadata path for the repo
 302         Pathname metadata_path = rawcache_path_for_repoinfo(_pimpl->options, (*it));
 303         (*it).setMetadataPath(metadata_path);
 304
 305         // set the downloaded packages path for the repo
 306         Pathname packages_path = packagescache_path_for_repoinfo(_pimpl->options, (*it));
 307         (*it).setPackagesPath(packages_path);
 308       }
 309       return repos;
 310     }
 311     else
 312       return std::list<RepoInfo>();
 313
 314     MIL << endl;
 315   }
 316
 317   ////////////////////////////////////////////////////////////////////////////
 318
 319   Pathname RepoManager::metadataPath( const RepoInfo &info ) const
 320   {
 321     return rawcache_path_for_repoinfo(_pimpl->options, info );
 322   }
 323
 324   ////////////////////////////////////////////////////////////////////////////
 325
 326   RepoStatus RepoManager::metadataStatus( const RepoInfo &info ) const
 327   {
 328     Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 329     RepoType repokind = info.type();
 330     RepoStatus status;
 331
 332     switch ( repokind.toEnum() )
 333     {
 334       case RepoType::NONE_e:
 335       // unknown, probe the local metadata
 336         repokind = probe(rawpath.asUrl());
 337       break;
 338       default:
 339       break;
 340     }
 341
 342     switch ( repokind.toEnum() )
 343     {
 344       case RepoType::RPMMD_e :
 345       {
 346         status = RepoStatus( rawpath + "/repodata/repomd.xml");
 347       }
 348       break;
 349
 350       case RepoType::YAST2_e :
 351       {
 352         status = RepoStatus( rawpath + "/content") && (RepoStatus( rawpath + "/media.1/media"));
 353       }
 354       break;
 355
 356       case RepoType::RPMPLAINDIR_e :
 357       {
 358         if ( PathInfo(Pathname(rawpath + "/cookie")).isExist() )
 359           status = RepoStatus( rawpath + "/cookie");
 360       }
 361       break;
 362
 363       case RepoType::NONE_e :
 364         // Return default RepoStatus in case of RepoType::NONE
 365         // indicating it should be created?
 366         // ZYPP_THROW(RepoUnknownTypeException());
 367         break;
 368     }
 369     return status;
 370   }
 371
 372   void RepoManager::touchIndexFile(const RepoInfo & info)
 373   {
 374     Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 375
 376     RepoType repokind = info.type();
 377     if ( repokind.toEnum() == RepoType::NONE_e )
 378       // unknown, probe the local metadata
 379       repokind = probe(rawpath.asUrl());
 380     // if still unknown, just return
 381     if (repokind == RepoType::NONE_e)
 382       return;
 383
 384     Pathname p;
 385     switch ( repokind.toEnum() )
 386     {
 387       case RepoType::RPMMD_e :
 388         p = Pathname(rawpath + "/repodata/repomd.xml");
 389         break;
 390
 391       case RepoType::YAST2_e :
 392         p = Pathname(rawpath + "/content");
 393         break;
 394
 395       case RepoType::RPMPLAINDIR_e :
 396         p = Pathname(rawpath + "/cookie");
 397         break;
 398
 399       case RepoType::NONE_e :
 400       default:
 401         break;
 402     }
 403
 404     // touch the file, ignore error (they are logged anyway)
 405     filesystem::touch(p);
 406   }
 407
 408   RepoManager::RefreshCheckStatus RepoManager::checkIfToRefreshMetadata(
 409                                               const RepoInfo &info,
 410                                               const Url &url,
 411                                               RawMetadataRefreshPolicy policy )
 412   {
 413     assert_alias(info);
 414
 415     RepoStatus oldstatus;
 416     RepoStatus newstatus;
 417
 418     try
 419     {
 420       MIL << "Going to try to check whether refresh is needed for " << url << endl;
 421
 422       repo::RepoType repokind = info.type();
 423
 424       // if the type is unknown, try probing.
 425       switch ( repokind.toEnum() )
 426       {
 427         case RepoType::NONE_e:
 428           // unknown, probe it
 429           repokind = probe(url);
 430         break;
 431         default:
 432         break;
 433       }
 434
 435       Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 436       filesystem::assert_dir(rawpath);
 437       oldstatus = metadataStatus(info);
 438
 439       // now we've got the old (cached) status, we can decide repo.refresh.delay
 440       if (policy != RefreshForced && policy != RefreshIfNeededIgnoreDelay)
 441       {
 442         // difference in seconds
 443         double diff = difftime(
 444           (Date::ValueType)Date::now(),
 445           (Date::ValueType)oldstatus.timestamp()) / 60;
 446
 447         DBG << "oldstatus: " << (Date::ValueType)oldstatus.timestamp() << endl;
 448         DBG << "current time: " << (Date::ValueType)Date::now() << endl;
 449         DBG << "last refresh = " << diff << " minutes ago" << endl;
 450
 451         if (diff < ZConfig::instance().repo_refresh_delay())
 452         {
 453           MIL << "Repository '" << info.alias()
 454               << "' has been refreshed less than repo.refresh.delay ("
 455               << ZConfig::instance().repo_refresh_delay()
 456               << ") minutes ago. Advising to skip refresh" << endl;
 457           return REPO_CHECK_DELAYED;
 458         }
 459       }
 460
 461       // create temp dir as sibling of rawpath
 462       filesystem::TmpDir tmpdir( filesystem::TmpDir::makeSibling( rawpath ) );
 463
 464       if ( ( repokind.toEnum() == RepoType::RPMMD_e ) ||
 465            ( repokind.toEnum() == RepoType::YAST2_e ) )
 466       {
 467         MediaSetAccess media(url);
 468         shared_ptr<repo::Downloader> downloader_ptr;
 469
 470         if ( repokind.toEnum() == RepoType::RPMMD_e )
 471           downloader_ptr.reset(new yum::Downloader(info.path()));
 472         else
 473           downloader_ptr.reset( new susetags::Downloader(info.path()));
 474
 475         RepoStatus newstatus = downloader_ptr->status(media);
 476         bool refresh = false;
 477         if ( oldstatus.checksum() == newstatus.checksum() )
 478         {
 479           MIL << "repo has not changed" << endl;
 480           if ( policy == RefreshForced )
 481           {
 482             MIL << "refresh set to forced" << endl;
 483             refresh = true;
 484           }
 485         }
 486         else
 487         {
 488           MIL << "repo has changed, going to refresh" << endl;
 489           refresh = true;
 490         }
 491
 492         if (!refresh)
 493           touchIndexFile(info);
 494
 495         return refresh ? REFRESH_NEEDED : REPO_UP_TO_DATE;
 496       }
 497       else if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 498       {
 499         RepoStatus newstatus = parser::plaindir::dirStatus(url.getPathName());
 500         bool refresh = false;
 501         if ( oldstatus.checksum() == newstatus.checksum() )
 502         {
 503           MIL << "repo has not changed" << endl;
 504           if ( policy == RefreshForced )
 505           {
 506             MIL << "refresh set to forced" << endl;
 507             refresh = true;
 508           }
 509         }
 510         else
 511         {
 512           MIL << "repo has changed, going to refresh" << endl;
 513           refresh = true;
 514         }
 515
 516         if (!refresh)
 517           touchIndexFile(info);
 518
 519         return refresh ? REFRESH_NEEDED : REPO_UP_TO_DATE;
 520       }
 521       else
 522       {
 523         ZYPP_THROW(RepoUnknownTypeException(info));
 524       }
 525     }
 526     catch ( const Exception &e )
 527     {
 528       ZYPP_CAUGHT(e);
 529       ERR << "refresh check failed for " << url << endl;
 530       ZYPP_RETHROW(e);
 531     }
 532
 533     return REFRESH_NEEDED; // default
 534   }
 535
 536   void RepoManager::refreshMetadata( const RepoInfo &info,
 537                                      RawMetadataRefreshPolicy policy,
 538                                      const ProgressData::ReceiverFnc & progress )
 539   {
 540     assert_alias(info);
 541     assert_urls(info);
 542
 543     // we will throw this later if no URL checks out fine
 544     RepoException rexception(_("Valid metadata not found at specified URL(s)"));
 545
 546     // try urls one by one
 547     for ( RepoInfo::urls_const_iterator it = info.baseUrlsBegin(); it != info.baseUrlsEnd(); ++it )
 548     {
 549       try
 550       {
 551         Url url(*it);
 552
 553         // check whether to refresh metadata
 554         // if the check fails for this url, it throws, so another url will be checked
 555         if (checkIfToRefreshMetadata(info, url, policy)!=REFRESH_NEEDED)
 556           return;
 557
 558         MIL << "Going to refresh metadata from " << url << endl;
 559
 560         repo::RepoType repokind = info.type();
 561
 562         // if the type is unknown, try probing.
 563         switch ( repokind.toEnum() )
 564         {
 565           case RepoType::NONE_e:
 566             // unknown, probe it
 567             repokind = probe(*it);
 568
 569             if (repokind.toEnum() != RepoType::NONE_e)
 570             {
 571               // Adjust the probed type in RepoInfo
 572               info.setProbedType( repokind ); // lazy init!
 573               //save probed type only for repos in system
 574               std::list<RepoInfo> repos = knownRepositories();
 575               for ( std::list<RepoInfo>::const_iterator it = repos.begin();
 576                    it != repos.end(); ++it )
 577               {
 578                 if ( info.alias() == (*it).alias() )
 579                 {
 580                   RepoInfo modifiedrepo = info;
 581                   modifiedrepo.setType(repokind);
 582                   modifyRepository(info.alias(),modifiedrepo);
 583                 }
 584               }
 585             }
 586           break;
 587           default:
 588           break;
 589         }
 590
 591         Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 592         filesystem::assert_dir(rawpath);
 593
 594         // create temp dir as sibling of rawpath
 595         filesystem::TmpDir tmpdir( filesystem::TmpDir::makeSibling( rawpath ) );
 596
 597         if ( ( repokind.toEnum() == RepoType::RPMMD_e ) ||
 598              ( repokind.toEnum() == RepoType::YAST2_e ) )
 599         {
 600           MediaSetAccess media(url);
 601           shared_ptr<repo::Downloader> downloader_ptr;
 602
 603           if ( repokind.toEnum() == RepoType::RPMMD_e )
 604             downloader_ptr.reset(new yum::Downloader(info.path()));
 605           else
 606             downloader_ptr.reset( new susetags::Downloader(info.path()));
 607
 608           /**
 609            * Given a downloader, sets the other repos raw metadata
 610            * path as cache paths for the fetcher, so if another
 611            * repo has the same file, it will not download it
 612            * but copy it from the other repository
 613            */
 614           std::list<RepoInfo> repos = knownRepositories();
 615           for ( std::list<RepoInfo>::const_iterator it = repos.begin();
 616                 it != repos.end();
 617                 ++it )
 618           {
 619             Pathname cachepath(rawcache_path_for_repoinfo( _pimpl->options, *it ));
 620             if ( PathInfo(cachepath).isExist() )
 621               downloader_ptr->addCachePath(cachepath);
 622           }
 623
 624           downloader_ptr->download( media, tmpdir.path());
 625         }
 626         else if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 627         {
 628           RepoStatus newstatus = parser::plaindir::dirStatus(url.getPathName());
 629
 630           std::ofstream file(( tmpdir.path() + "/cookie").c_str());
 631           if (!file) {
 632             ZYPP_THROW (Exception( "Can't open " + tmpdir.path().asString() + "/cookie" ) );
 633           }
 634           file << url << endl;
 635           file << newstatus.checksum() << endl;
 636
 637           file.close();
 638         }
 639         else
 640         {
 641           ZYPP_THROW(RepoUnknownTypeException());
 642         }
 643
 644         // ok we have the metadata, now exchange
 645         // the contents
 646
 647         TmpDir oldmetadata( TmpDir::makeSibling( rawpath ) );
 648         filesystem::rename( rawpath, oldmetadata.path() );
 649         // move the just downloaded there
 650         filesystem::rename( tmpdir.path(), rawpath );
 651
 652         // we are done.
 653         return;
 654       }
 655       catch ( const Exception &e )
 656       {
 657         ZYPP_CAUGHT(e);
 658         ERR << "Trying another url..." << endl;
 659
 660         // remember the exception caught for the *first URL*
 661         // if all other URLs fail, the rexception will be thrown with the
 662         // cause of the problem of the first URL remembered
 663         if (it == info.baseUrlsBegin())
 664           rexception.remember(e);
 665       }
 666     } // for every url
 667     ERR << "No more urls..." << endl;
 668     ZYPP_THROW(rexception);
 669   }
 670
 671   ////////////////////////////////////////////////////////////////////////////
 672
 673   void RepoManager::cleanMetadata( const RepoInfo &info,
 674                                    const ProgressData::ReceiverFnc & progressfnc )
 675   {
 676     ProgressData progress(100);
 677     progress.sendTo(progressfnc);
 678
 679     filesystem::recursive_rmdir(rawcache_path_for_repoinfo(_pimpl->options, info));
 680     progress.toMax();
 681   }
 682
 683   void RepoManager::cleanPackages( const RepoInfo &info,
 684                                    const ProgressData::ReceiverFnc & progressfnc )
 685   {
 686     ProgressData progress(100);
 687     progress.sendTo(progressfnc);
 688
 689     filesystem::recursive_rmdir(packagescache_path_for_repoinfo(_pimpl->options, info));
 690     progress.toMax();
 691   }
 692
 693   void RepoManager::buildCache( const RepoInfo &info,
 694                                 CacheBuildPolicy policy,
 695                                 const ProgressData::ReceiverFnc & progressrcv )
 696   {
 697     assert_alias(info);
 698     Pathname rawpath = rawcache_path_for_repoinfo(_pimpl->options, info);
 699
 700     filesystem::assert_dir(_pimpl->options.repoCachePath);
 701     RepoStatus raw_metadata_status = metadataStatus(info);
 702     if ( raw_metadata_status.empty() )
 703     {
 704        /* if there is no cache at this point, we refresh the raw
 705           in case this is the first time - if it's !autorefresh,
 706           we may still refresh */
 707       refreshMetadata(info, RefreshIfNeeded, progressrcv );
 708       raw_metadata_status = metadataStatus(info);
 709     }
 710
 711     bool needs_cleaning = false;
 712     if ( isCached( info ) )
 713     {
 714       MIL << info.alias() << " is already cached." << endl;
 715       //data::RecordId id = store.lookupRepository(info.alias());
 716       RepoStatus cache_status = cacheStatus(info);
 717
 718       if ( cache_status.checksum() == raw_metadata_status.checksum() )
 719       {
 720         MIL << info.alias() << " cache is up to date with metadata." << endl;
 721         if ( policy == BuildIfNeeded ) {
 722           return;
 723         }
 724         else {
 725           MIL << info.alias() << " cache rebuild is forced" << endl;
 726         }
 727       }
 728
 729       needs_cleaning = true;
 730     }
 731
 732     ProgressData progress(100);
 733     callback::SendReport<ProgressReport> report;
 734     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
 735     progress.name(str::form(_("Building repository '%s' cache"), info.name().c_str()));
 736     progress.toMin();
 737
 738     if (needs_cleaning)
 739     {
 740       cleanCache(info);
 741     }
 742
 743     MIL << info.alias() << " building cache..." << endl;
 744
 745     Pathname base = solv_path_for_repoinfo( _pimpl->options, info);
 746     filesystem::assert_dir(base);
 747     Pathname solvfile = base / "solv";
 748
 749     // do we have type?
 750     repo::RepoType repokind = info.type();
 751
 752     // if the type is unknown, try probing.
 753     switch ( repokind.toEnum() )
 754     {
 755       case RepoType::NONE_e:
 756         // unknown, probe the local metadata
 757         repokind = probe(rawpath.asUrl());
 758       break;
 759       default:
 760       break;
 761     }
 762
 763     MIL << "repo type is " << repokind << endl;
 764
 765     switch ( repokind.toEnum() )
 766     {
 767       case RepoType::RPMMD_e :
 768       case RepoType::YAST2_e :
 769       case RepoType::RPMPLAINDIR_e :
 770       {
 771         // Take care we unlink the solvfile on exception
 772         ManagedFile guard( solvfile, filesystem::unlink );
 773
 774         ostringstream cmd;
 775         std::string toFile( str::gsub(solvfile.asString(),"\"","\\\"") );
 776         if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 777         {
 778           cmd << str::form( "repo2solv.sh \"%s\" > \"%s\"",
 779                             str::gsub( info.baseUrlsBegin()->getPathName(),"\"","\\\"" ).c_str(),
 780                             toFile.c_str() );
 781         }
 782         else
 783         {
 784           cmd << str::form( "repo2solv.sh \"%s\" > \"%s\"",
 785                             str::gsub( rawpath.asString(),"\"","\\\"" ).c_str(),
 786                             toFile.c_str() );
 787         }
 788         MIL << "Executing: " << cmd.str() << endl;
 789         ExternalProgram prog( cmd.str(), ExternalProgram::Stderr_To_Stdout );
 790
 791         cmd << endl;
 792         for ( string output( prog.receiveLine() ); output.length(); output = prog.receiveLine() ) {
 793           WAR << "  " << output;
 794           cmd << "     " << output;
 795         }
 796
 797         int ret = prog.close();
 798         if ( ret != 0 )
 799         {
 800           RepoException ex(str::form("Failed to cache repo (%d).", ret));
 801           ex.remember( cmd.str() );
 802           ZYPP_THROW(ex);
 803         }
 804
 805         // We keep it.
 806         guard.resetDispose();
 807       }
 808       break;
 809       default:
 810         ZYPP_THROW(RepoUnknownTypeException("Unhandled repository type"));
 811       break;
 812     }
 813 #if 0
 814     switch ( repokind.toEnum() )
 815     {
 816       case RepoType::RPMMD_e :
 817       if (0)
 818       {
 819         CombinedProgressData subprogrcv( progress, 100);
 820         parser::yum::RepoParser parser(id, store, parser::yum::RepoParserOpts(), subprogrcv);
 821         parser.parse(rawpath);
 822           // no error
 823       }
 824       break;
 825       case RepoType::YAST2_e :
 826       if (0)
 827       {
 828         CombinedProgressData subprogrcv( progress, 100);
 829         parser::susetags::RepoParser parser(id, store, subprogrcv);
 830         parser.parse(rawpath);
 831         // no error
 832       }
 833       break;
 834
 835       default:
 836         ZYPP_THROW(RepoUnknownTypeException());
 837     }
 838 #endif
 839     // update timestamp and checksum
 840     //store.updateRepositoryStatus(id, raw_metadata_status);
 841     setCacheStatus(info, raw_metadata_status);
 842     MIL << "Commit cache.." << endl;
 843     //store.commit();
 844     //progress.toMax();
 845   }
 846
 847   ////////////////////////////////////////////////////////////////////////////
 848
 849   repo::RepoType RepoManager::probe( const Url &url ) const
 850   {
 851     if ( url.getScheme() == "dir" && ! PathInfo( url.getPathName() ).isDir() )
 852     {
 853       // Handle non existing local directory in advance, as
 854       // MediaSetAccess does not support it.
 855       return repo::RepoType::NONE;
 856     }
 857
 858     try
 859     {
 860       MediaSetAccess access(url);
 861       if ( access.doesFileExist("/repodata/repomd.xml") )
 862         return repo::RepoType::RPMMD;
 863       if ( access.doesFileExist("/content") )
 864         return repo::RepoType::YAST2;
 865
 866       // if it is a local url of type dir
 867       if ( (! media::MediaManager::downloads(url)) && ( url.getScheme() == "dir" ) )
 868       {
 869         Pathname path = Pathname(url.getPathName());
 870         if ( PathInfo(path).isDir() )
 871         {
 872           // allow empty dirs for now
 873           return repo::RepoType::RPMPLAINDIR;
 874         }
 875       }
 876     }
 877     catch ( const media::MediaException &e )
 878     {
 879       ZYPP_CAUGHT(e);
 880       RepoException enew("Error trying to read from " + url.asString());
 881       enew.remember(e);
 882       ZYPP_THROW(enew);
 883     }
 884     catch ( const Exception &e )
 885     {
 886       ZYPP_CAUGHT(e);
 887       Exception enew("Unknown error reading from " + url.asString());
 888       enew.remember(e);
 889       ZYPP_THROW(enew);
 890     }
 891
 892     return repo::RepoType::NONE;
 893   }
 894
 895   ////////////////////////////////////////////////////////////////////////////
 896
 897   void RepoManager::cleanCache( const RepoInfo &info,
 898                                 const ProgressData::ReceiverFnc & progressrcv )
 899   {
 900     ProgressData progress(100);
 901     progress.sendTo(progressrcv);
 902     progress.toMin();
 903
 904     filesystem::recursive_rmdir(solv_path_for_repoinfo(_pimpl->options, info));
 905
 906     progress.toMax();
 907   }
 908
 909   ////////////////////////////////////////////////////////////////////////////
 910
 911   bool RepoManager::isCached( const RepoInfo &info ) const
 912   {
 913     return PathInfo(solv_path_for_repoinfo( _pimpl->options, info ) / "solv").isExist();
 914   }
 915
 916   RepoStatus RepoManager::cacheStatus( const RepoInfo &info ) const
 917   {
 918
 919     Pathname cookiefile = solv_path_for_repoinfo(_pimpl->options, info) / "cookie";
 920
 921     return RepoStatus::fromCookieFile(cookiefile);
 922   }
 923
 924   void RepoManager::setCacheStatus( const RepoInfo &info, const RepoStatus &status )
 925   {
 926     Pathname base = solv_path_for_repoinfo(_pimpl->options, info);
 927     filesystem::assert_dir(base);
 928     Pathname cookiefile = base / "cookie";
 929
 930     status.saveToCookieFile(cookiefile);
 931   }
 932
 933   void RepoManager::loadFromCache( const RepoInfo & info,
 934                                    const ProgressData::ReceiverFnc & progressrcv )
 935   {
 936     assert_alias(info);
 937     Pathname solvfile = solv_path_for_repoinfo(_pimpl->options, info) / "solv";
 938
 939     if ( ! PathInfo(solvfile).isExist() )
 940       ZYPP_THROW(RepoNotCachedException(info));
 941
 942     try
 943     {
 944       sat::Pool::instance().addRepoSolv( solvfile, info );
 945     }
 946     catch ( const Exception & exp )
 947     {
 948       ZYPP_CAUGHT( exp );
 949       MIL << "Try to handle exception by rebuilding the solv-file" << endl;
 950       cleanCache( info, progressrcv );
 951       buildCache( info, BuildIfNeeded, progressrcv );
 952
 953       sat::Pool::instance().addRepoSolv( solvfile, info );
 954     }
 955   }
 956
 957   ////////////////////////////////////////////////////////////////////////////
 958
 959   /**
 960    * Generate a non existing filename in a directory, using a base
 961    * name. For example if a directory contains 3 files
 962    *
 963    * |-- bar
 964    * |-- foo
 965    * `-- moo
 966    *
 967    * If you try to generate a unique filename for this directory,
 968    * based on "ruu" you will get "ruu", but if you use the base
 969    * "foo" you will get "foo_1"
 970    *
 971    * \param dir Directory where the file needs to be unique
 972    * \param basefilename string to base the filename on.
 973    */
 974   static Pathname generate_non_existing_name( const Pathname &dir,
 975                                               const std::string &basefilename )
 976   {
 977     string final_filename = basefilename;
 978     int counter = 1;
 979     while ( PathInfo(dir + final_filename).isExist() )
 980     {
 981       final_filename = basefilename + "_" + str::numstring(counter);
 982       counter++;
 983     }
 984     return dir + Pathname(final_filename);
 985   }
 986
 987   ////////////////////////////////////////////////////////////////////////////
 988
 989   /**
 990    * \short Generate a related filename from a repo info
 991    *
 992    * From a repo info, it will try to use the alias as a filename
 993    * escaping it if necessary. Other fallbacks can be added to
 994    * this function in case there is no way to use the alias
 995    */
 996   static std::string generate_filename( const RepoInfo &info )
 997   {
 998     std::string fnd="/";
 999     std::string rep="_";
1000     std::string filename = info.alias();
1001     // replace slashes with underscores
1002     size_t pos = filename.find(fnd);
1003     while(pos!=string::npos)
1004     {
1005       filename.replace(pos,fnd.length(),rep);
1006       pos = filename.find(fnd,pos+rep.length());
1007     }
1008     filename = Pathname(filename).extend(".repo").asString();
1009     MIL << "generating filename for repo [" << info.alias() << "] : '" << filename << "'" << endl;
1010     return filename;
1011   }
1012
1013
1014   ////////////////////////////////////////////////////////////////////////////
1015
1016   void RepoManager::addRepository( const RepoInfo &info,
1017                                    const ProgressData::ReceiverFnc & progressrcv )
1018   {
1019     assert_alias(info);
1020
1021     ProgressData progress(100);
1022     callback::SendReport<ProgressReport> report;
1023     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
1024     progress.name(str::form(_("Adding repository '%s'"), info.name().c_str()));
1025     progress.toMin();
1026
1027     std::list<RepoInfo> repos = knownRepositories();
1028     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1029           it != repos.end();
1030           ++it )
1031     {
1032       if ( info.alias() == (*it).alias() )
1033         ZYPP_THROW(RepoAlreadyExistsException(info));
1034     }
1035
1036     RepoInfo tosave = info;
1037
1038     // check the first url for now
1039     if ( _pimpl->options.probe )
1040     {
1041       DBG << "unknown repository type, probing" << endl;
1042
1043       RepoType probedtype;
1044       probedtype = probe(*tosave.baseUrlsBegin());
1045       if ( tosave.baseUrlsSize() > 0 )
1046       {
1047         if ( probedtype == RepoType::NONE )
1048           ZYPP_THROW(RepoUnknownTypeException());
1049         else
1050           tosave.setType(probedtype);
1051       }
1052     }
1053
1054     progress.set(50);
1055
1056     // assert the directory exists
1057     filesystem::assert_dir(_pimpl->options.knownReposPath);
1058
1059     Pathname repofile = generate_non_existing_name(_pimpl->options.knownReposPath,
1060                                                     generate_filename(tosave));
1061     // now we have a filename that does not exists
1062     MIL << "Saving repo in " << repofile << endl;
1063
1064     std::ofstream file(repofile.c_str());
1065     if (!file) {
1066       ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
1067     }
1068
1069     tosave.dumpRepoOn(file);
1070     progress.toMax();
1071     MIL << "done" << endl;
1072   }
1073
1074   void RepoManager::addRepositories( const Url &url,
1075                                      const ProgressData::ReceiverFnc & progressrcv )
1076   {
1077     std::list<RepoInfo> knownrepos = knownRepositories();
1078     std::list<RepoInfo> repos = readRepoFile(url);
1079     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1080           it != repos.end();
1081           ++it )
1082     {
1083       // look if the alias is in the known repos.
1084       for ( std::list<RepoInfo>::const_iterator kit = knownrepos.begin();
1085           kit != knownrepos.end();
1086           ++kit )
1087       {
1088         if ( (*it).alias() == (*kit).alias() )
1089         {
1090           ERR << "To be added repo " << (*it).alias() << " conflicts with existing repo " << (*kit).alias() << endl;
1091           ZYPP_THROW(RepoAlreadyExistsException(*it));
1092         }
1093       }
1094     }
1095
1096     string filename = Pathname(url.getPathName()).basename();
1097
1098     if ( filename == Pathname() )
1099       ZYPP_THROW(RepoException("Invalid repo file name at " + url.asString() ));
1100
1101     // assert the directory exists
1102     filesystem::assert_dir(_pimpl->options.knownReposPath);
1103
1104     Pathname repofile = generate_non_existing_name(_pimpl->options.knownReposPath, filename);
1105     // now we have a filename that does not exists
1106     MIL << "Saving " << repos.size() << " repo" << ( repos.size() ? "s" : "" ) << " in " << repofile << endl;
1107
1108     std::ofstream file(repofile.c_str());
1109     if (!file) {
1110       ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
1111     }
1112
1113     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1114           it != repos.end();
1115           ++it )
1116     {
1117       MIL << "Saving " << (*it).alias() << endl;
1118       (*it).dumpRepoOn(file);
1119     }
1120     MIL << "done" << endl;
1121   }
1122
1123   ////////////////////////////////////////////////////////////////////////////
1124
1125   void RepoManager::removeRepository( const RepoInfo & info,
1126                                       const ProgressData::ReceiverFnc & progressrcv)
1127   {
1128     ProgressData progress;
1129     callback::SendReport<ProgressReport> report;
1130     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
1131     progress.name(str::form(_("Removing repository '%s'"), info.name().c_str()));
1132
1133     MIL << "Going to delete repo " << info.alias() << endl;
1134
1135     std::list<RepoInfo> repos = knownRepositories();
1136     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1137           it != repos.end();
1138           ++it )
1139     {
1140       // they can be the same only if the provided is empty, that means
1141       // the provided repo has no alias
1142       // then skip
1143       if ( (!info.alias().empty()) && ( info.alias() != (*it).alias() ) )
1144         continue;
1145
1146       // TODO match by url
1147
1148       // we have a matcing repository, now we need to know
1149       // where it does come from.
1150       RepoInfo todelete = *it;
1151       if (todelete.filepath().empty())
1152       {
1153         ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
1154       }
1155       else
1156       {
1157         // figure how many repos are there in the file:
1158         std::list<RepoInfo> filerepos = repositories_in_file(todelete.filepath());
1159         if ( (filerepos.size() == 1) && ( filerepos.front().alias() == todelete.alias() ) )
1160         {
1161           // easy, only this one, just delete the file
1162           if ( filesystem::unlink(todelete.filepath()) != 0 )
1163           {
1164             ZYPP_THROW(RepoException("Can't delete " + todelete.filepath().asString()));
1165           }
1166           MIL << todelete.alias() << " sucessfully deleted." << endl;
1167         }
1168         else
1169         {
1170           // there are more repos in the same file
1171           // write them back except the deleted one.
1172           //TmpFile tmp;
1173           //std::ofstream file(tmp.path().c_str());
1174
1175           // assert the directory exists
1176           filesystem::assert_dir(todelete.filepath().dirname());
1177
1178           std::ofstream file(todelete.filepath().c_str());
1179           if (!file) {
1180             //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
1181             ZYPP_THROW (Exception( "Can't open " + todelete.filepath().asString() ) );
1182           }
1183           for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
1184                 fit != filerepos.end();
1185                 ++fit )
1186           {
1187             if ( (*fit).alias() != todelete.alias() )
1188               (*fit).dumpRepoOn(file);
1189           }
1190         }
1191
1192         CombinedProgressData subprogrcv(progress, 70);
1193         CombinedProgressData cleansubprogrcv(progress, 30);
1194         // now delete it from cache
1195         if ( isCached(todelete) )
1196           cleanCache( todelete, subprogrcv);
1197         // now delete metadata (#301037)
1198         cleanMetadata( todelete, cleansubprogrcv);
1199         MIL << todelete.alias() << " sucessfully deleted." << endl;
1200         return;
1201       } // else filepath is empty
1202
1203     }
1204     // should not be reached on a sucess workflow
1205     ZYPP_THROW(RepoNotFoundException(info));
1206   }
1207
1208   ////////////////////////////////////////////////////////////////////////////
1209
1210   void RepoManager::modifyRepository( const std::string &alias,
1211                                       const RepoInfo & newinfo,
1212                                       const ProgressData::ReceiverFnc & progressrcv )
1213   {
1214     RepoInfo toedit = getRepositoryInfo(alias);
1215
1216     // check if the new alias already exists when renaming the repo
1217     if (alias != newinfo.alias())
1218     {
1219       std::list<RepoInfo> repos = knownRepositories();
1220       for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1221             it != repos.end();
1222             ++it )
1223       {
1224         if ( newinfo.alias() == (*it).alias() )
1225           ZYPP_THROW(RepoAlreadyExistsException(newinfo));
1226       }
1227     }
1228
1229     if (toedit.filepath().empty())
1230     {
1231       ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
1232     }
1233     else
1234     {
1235       // figure how many repos are there in the file:
1236       std::list<RepoInfo> filerepos = repositories_in_file(toedit.filepath());
1237
1238       // there are more repos in the same file
1239       // write them back except the deleted one.
1240       //TmpFile tmp;
1241       //std::ofstream file(tmp.path().c_str());
1242
1243       // assert the directory exists
1244       filesystem::assert_dir(toedit.filepath().dirname());
1245
1246       std::ofstream file(toedit.filepath().c_str());
1247       if (!file) {
1248         //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
1249         ZYPP_THROW (Exception( "Can't open " + toedit.filepath().asString() ) );
1250       }
1251       for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
1252             fit != filerepos.end();
1253             ++fit )
1254       {
1255           // if the alias is different, dump the original
1256           // if it is the same, dump the provided one
1257           if ( (*fit).alias() != toedit.alias() )
1258             (*fit).dumpRepoOn(file);
1259           else
1260             newinfo.dumpRepoOn(file);
1261       }
1262     }
1263   }
1264
1265   ////////////////////////////////////////////////////////////////////////////
1266
1267   RepoInfo RepoManager::getRepositoryInfo( const std::string &alias,
1268                                            const ProgressData::ReceiverFnc & progressrcv )
1269   {
1270     std::list<RepoInfo> repos = knownRepositories();
1271     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1272           it != repos.end();
1273           ++it )
1274     {
1275       if ( (*it).alias() == alias )
1276         return *it;
1277     }
1278     RepoInfo info;
1279     info.setAlias(info.alias());
1280     ZYPP_THROW(RepoNotFoundException(info));
1281   }
1282
1283   ////////////////////////////////////////////////////////////////////////////
1284
1285   RepoInfo RepoManager::getRepositoryInfo( const Url & url,
1286                                            const url::ViewOption & urlview,
1287                                            const ProgressData::ReceiverFnc & progressrcv )
1288   {
1289     std::list<RepoInfo> repos = knownRepositories();
1290     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1291           it != repos.end();
1292           ++it )
1293     {
1294       for(RepoInfo::urls_const_iterator urlit = (*it).baseUrlsBegin();
1295           urlit != (*it).baseUrlsEnd();
1296           ++urlit)
1297       {
1298         if ((*urlit).asString(urlview) == url.asString(urlview))
1299           return *it;
1300       }
1301     }
1302     RepoInfo info;
1303     info.setAlias(info.alias());
1304     info.setBaseUrl(url);
1305     ZYPP_THROW(RepoNotFoundException(info));
1306   }
1307
1308   ////////////////////////////////////////////////////////////////////////////
1309
1310   std::ostream & operator<<( std::ostream & str, const RepoManager & obj )
1311   {
1312     return str << *obj._pimpl;
1313   }
1314
1315   /////////////////////////////////////////////////////////////////
1316 } // namespace zypp
1317 ///////////////////////////////////////////////////////////////////