zypp/RepoManager.cc

   1 /*---------------------------------------------------------------------\
   2 |                          ____ _   __ __ ___                          |
   3 |                         |__  / \ / / . \ . \                         |
   4 |                           / / \ V /|  _/  _/                         |
   5 |                          / /__ | | | | | |                           |
   6 |                         /_____||_| |_| |_|                           |
   7 |                                                                      |
   8 \---------------------------------------------------------------------*/
   9 /** \file       zypp/RepoManager.cc
  10  *
  11 */
  12
  13 #include <cstdlib>
  14 #include <iostream>
  15 #include <fstream>
  16 #include <sstream>
  17 #include <list>
  18 #include <map>
  19 #include <algorithm>
  20 #include "zypp/base/InputStream.h"
  21 #include "zypp/base/Logger.h"
  22 #include "zypp/base/Gettext.h"
  23 #include "zypp/base/Function.h"
  24 #include "zypp/base/Regex.h"
  25 #include "zypp/PathInfo.h"
  26 #include "zypp/TmpPath.h"
  27
  28 #include "zypp/repo/RepoException.h"
  29 #include "zypp/RepoManager.h"
  30
  31 #include "zypp/media/MediaManager.h"
  32 #include "zypp/MediaSetAccess.h"
  33 #include "zypp/ExternalProgram.h"
  34 #include "zypp/ManagedFile.h"
  35
  36 #include "zypp/parser/RepoFileReader.h"
  37 #include "zypp/repo/yum/Downloader.h"
  38 #include "zypp/parser/yum/RepoParser.h"
  39 #include "zypp/repo/susetags/Downloader.h"
  40 #include "zypp/parser/susetags/RepoParser.h"
  41 #include "zypp/parser/plaindir/RepoParser.h"
  42
  43 #include "zypp/ZYppCallbacks.h"
  44
  45 #include "sat/Pool.h"
  46 #include "satsolver/pool.h"
  47 #include "satsolver/repo.h"
  48 #include "satsolver/repo_solv.h"
  49
  50 using namespace std;
  51 using namespace zypp;
  52 using namespace zypp::repo;
  53 using namespace zypp::filesystem;
  54
  55 using namespace zypp::repo;
  56
  57 ///////////////////////////////////////////////////////////////////
  58 namespace zypp
  59 { /////////////////////////////////////////////////////////////////
  60
  61   ///////////////////////////////////////////////////////////////////
  62   //
  63   //    CLASS NAME : RepoManagerOptions
  64   //
  65   ///////////////////////////////////////////////////////////////////
  66
  67   RepoManagerOptions::RepoManagerOptions()
  68   {
  69     repoCachePath    = ZConfig::instance().repoCachePath();
  70     repoRawCachePath = ZConfig::instance().repoMetadataPath();
  71     repoPackagesCachePath = ZConfig::instance().repoPackagesPath();
  72     knownReposPath   = ZConfig::instance().knownReposPath();
  73     probe            = ZConfig::instance().repo_add_probe();
  74   }
  75
  76   ////////////////////////////////////////////////////////////////////////////
  77
  78   /**
  79     * \short Simple callback to collect the results
  80     *
  81     * Classes like RepoFileParser call the callback
  82     * once per each repo in a file.
  83     *
  84     * Passing this functor as callback, you can collect
  85     * all resuls at the end, without dealing with async
  86     * code.
  87     */
  88     struct RepoCollector
  89     {
  90       RepoCollector()
  91       {}
  92
  93       ~RepoCollector()
  94       {}
  95
  96       bool collect( const RepoInfo &repo )
  97       {
  98         repos.push_back(repo);
  99         return true;
 100       }
 101
 102       RepoInfoList repos;
 103     };
 104
 105   ////////////////////////////////////////////////////////////////////////////
 106
 107   /**
 108    * Reads RepoInfo's from a repo file.
 109    *
 110    * \param file pathname of the file to read.
 111    */
 112   static std::list<RepoInfo> repositories_in_file( const Pathname & file )
 113   {
 114     MIL << "repo file: " << file << endl;
 115     RepoCollector collector;
 116     parser::RepoFileReader parser( file, bind( &RepoCollector::collect, &collector, _1 ) );
 117     return collector.repos;
 118   }
 119
 120   ////////////////////////////////////////////////////////////////////////////
 121
 122   std::list<RepoInfo> readRepoFile(const Url & repo_file)
 123    {
 124      // no interface to download a specific file, using workaround:
 125      //! \todo add MediaManager::provideFile(Url file_url) to easily access any file URLs? (no need for media access id or media_nr)
 126      Url url(repo_file);
 127      Pathname path(url.getPathName());
 128      url.setPathName ("/");
 129      MediaSetAccess access(url);
 130      Pathname local = access.provideFile(path);
 131
 132      DBG << "reading repo file " << repo_file << ", local path: " << local << endl;
 133
 134      return repositories_in_file(local);
 135    }
 136
 137   ////////////////////////////////////////////////////////////////////////////
 138
 139   /**
 140    * \short List of RepoInfo's from a directory
 141    *
 142    * Goes trough every file ending with ".repo" in a directory and adds all
 143    * RepoInfo's contained in that file.
 144    *
 145    * \param dir pathname of the directory to read.
 146    */
 147   static std::list<RepoInfo> repositories_in_dir( const Pathname &dir )
 148   {
 149     MIL << "directory " << dir << endl;
 150     list<RepoInfo> repos;
 151     list<Pathname> entries;
 152     if ( filesystem::readdir( entries, Pathname(dir), false ) != 0 )
 153       ZYPP_THROW(Exception("failed to read directory"));
 154
 155     str::regex allowedRepoExt("^\\.repo(_[0-9]+)?$");
 156     for ( list<Pathname>::const_iterator it = entries.begin(); it != entries.end(); ++it )
 157     {
 158       if (str::regex_match(it->extension(), allowedRepoExt))
 159       {
 160         list<RepoInfo> tmp = repositories_in_file( *it );
 161         repos.insert( repos.end(), tmp.begin(), tmp.end() );
 162
 163         //std::copy( collector.repos.begin(), collector.repos.end(), std::back_inserter(repos));
 164         //MIL << "ok" << endl;
 165       }
 166     }
 167     return repos;
 168   }
 169
 170   ////////////////////////////////////////////////////////////////////////////
 171
 172   static void assert_alias( const RepoInfo &info )
 173   {
 174     if (info.alias().empty())
 175         ZYPP_THROW(RepoNoAliasException());
 176   }
 177
 178   ////////////////////////////////////////////////////////////////////////////
 179
 180   static void assert_urls( const RepoInfo &info )
 181   {
 182     if (info.baseUrlsEmpty())
 183         ZYPP_THROW(RepoNoUrlException());
 184   }
 185
 186   ////////////////////////////////////////////////////////////////////////////
 187
 188   /**
 189    * \short Calculates the raw cache path for a repository
 190    */
 191   static Pathname rawcache_path_for_repoinfo( const RepoManagerOptions &opt, const RepoInfo &info )
 192   {
 193     assert_alias(info);
 194     return opt.repoRawCachePath / info.escaped_alias();
 195   }
 196
 197   /**
 198    * \short Calculates the packages cache path for a repository
 199    */
 200   static Pathname packagescache_path_for_repoinfo( const RepoManagerOptions &opt, const RepoInfo &info )
 201   {
 202     assert_alias(info);
 203     return opt.repoPackagesCachePath / info.escaped_alias();
 204   }
 205
 206   static Pathname solv_path_for_repoinfo( const RepoManagerOptions &opt, const RepoInfo &info)
 207   {
 208     assert_alias(info);
 209     return opt.repoCachePath / "solv" / info.escaped_alias();
 210   }
 211
 212   ///////////////////////////////////////////////////////////////////
 213   //
 214   //    CLASS NAME : RepoManager::Impl
 215   //
 216   ///////////////////////////////////////////////////////////////////
 217
 218   /**
 219    * \short RepoManager implementation.
 220    */
 221   struct RepoManager::Impl
 222   {
 223     Impl( const RepoManagerOptions &opt )
 224       : options(opt)
 225     {
 226
 227     }
 228
 229     Impl()
 230     {
 231
 232     }
 233
 234     RepoManagerOptions options;
 235
 236     map<string, RepoInfo> _repoinfos;
 237
 238   public:
 239     /** Offer default Impl. */
 240     static shared_ptr<Impl> nullimpl()
 241     {
 242       static shared_ptr<Impl> _nullimpl( new Impl );
 243       return _nullimpl;
 244     }
 245
 246   private:
 247     friend Impl * rwcowClone<Impl>( const Impl * rhs );
 248     /** clone for RWCOW_pointer */
 249     Impl * clone() const
 250     { return new Impl( *this ); }
 251   };
 252
 253   ///////////////////////////////////////////////////////////////////
 254
 255   /** \relates RepoManager::Impl Stream output */
 256   inline std::ostream & operator<<( std::ostream & str, const RepoManager::Impl & obj )
 257   {
 258     return str << "RepoManager::Impl";
 259   }
 260
 261   ///////////////////////////////////////////////////////////////////
 262   //
 263   //    CLASS NAME : RepoManager
 264   //
 265   ///////////////////////////////////////////////////////////////////
 266
 267   RepoManager::RepoManager( const RepoManagerOptions &opt )
 268   : _pimpl( new Impl(opt) )
 269   {}
 270
 271   ////////////////////////////////////////////////////////////////////////////
 272
 273   RepoManager::~RepoManager()
 274   {}
 275
 276   ////////////////////////////////////////////////////////////////////////////
 277
 278   std::list<RepoInfo> RepoManager::knownRepositories() const
 279   {
 280     MIL << endl;
 281
 282     if ( PathInfo(_pimpl->options.knownReposPath).isExist() )
 283     {
 284       RepoInfoList repos = repositories_in_dir(_pimpl->options.knownReposPath);
 285       for ( RepoInfoList::iterator it = repos.begin();
 286             it != repos.end();
 287             ++it )
 288       {
 289         // set the metadata path for the repo
 290         Pathname metadata_path = rawcache_path_for_repoinfo(_pimpl->options, (*it));
 291         (*it).setMetadataPath(metadata_path);
 292
 293         // set the downloaded packages path for the repo
 294         Pathname packages_path = packagescache_path_for_repoinfo(_pimpl->options, (*it));
 295         (*it).setPackagesPath(packages_path);
 296       }
 297       return repos;
 298     }
 299     else
 300       return std::list<RepoInfo>();
 301
 302     MIL << endl;
 303   }
 304
 305   ////////////////////////////////////////////////////////////////////////////
 306
 307   Pathname RepoManager::metadataPath( const RepoInfo &info ) const
 308   {
 309     return rawcache_path_for_repoinfo(_pimpl->options, info );
 310   }
 311
 312   ////////////////////////////////////////////////////////////////////////////
 313
 314   RepoStatus RepoManager::metadataStatus( const RepoInfo &info ) const
 315   {
 316     Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 317     RepoType repokind = info.type();
 318     RepoStatus status;
 319
 320     switch ( repokind.toEnum() )
 321     {
 322       case RepoType::NONE_e:
 323       // unknown, probe the local metadata
 324         repokind = probe(rawpath.asUrl());
 325       break;
 326       default:
 327       break;
 328     }
 329
 330     switch ( repokind.toEnum() )
 331     {
 332       case RepoType::RPMMD_e :
 333       {
 334         status = RepoStatus( rawpath + "/repodata/repomd.xml");
 335       }
 336       break;
 337
 338       case RepoType::YAST2_e :
 339       {
 340         status = RepoStatus( rawpath + "/content") && (RepoStatus( rawpath + "/media.1/media"));
 341       }
 342       break;
 343
 344       case RepoType::RPMPLAINDIR_e :
 345       {
 346         if ( PathInfo(Pathname(rawpath + "/cookie")).isExist() )
 347           status = RepoStatus( rawpath + "/cookie");
 348       }
 349       break;
 350
 351       case RepoType::NONE_e :
 352         // Return default RepoStatus in case of RepoType::NONE
 353         // indicating it should be created?
 354         // ZYPP_THROW(RepoUnknownTypeException());
 355         break;
 356     }
 357     return status;
 358   }
 359
 360   void RepoManager::touchIndexFile(const RepoInfo & info)
 361   {
 362     Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 363
 364     RepoType repokind = info.type();
 365     if ( repokind.toEnum() == RepoType::NONE_e )
 366       // unknown, probe the local metadata
 367       repokind = probe(rawpath.asUrl());
 368     // if still unknown, just return
 369     if (repokind == RepoType::NONE_e)
 370       return;
 371
 372     Pathname p;
 373     switch ( repokind.toEnum() )
 374     {
 375       case RepoType::RPMMD_e :
 376         p = Pathname(rawpath + "/repodata/repomd.xml");
 377         break;
 378
 379       case RepoType::YAST2_e :
 380         p = Pathname(rawpath + "/content");
 381         break;
 382
 383       case RepoType::RPMPLAINDIR_e :
 384         p = Pathname(rawpath + "/cookie");
 385         break;
 386
 387       case RepoType::NONE_e :
 388       default:
 389         break;
 390     }
 391
 392     // touch the file, ignore error (they are logged anyway)
 393     filesystem::touch(p);
 394   }
 395
 396   RepoManager::RefreshCheckStatus RepoManager::checkIfToRefreshMetadata(
 397                                               const RepoInfo &info,
 398                                               const Url &url,
 399                                               RawMetadataRefreshPolicy policy )
 400   {
 401     assert_alias(info);
 402
 403     RepoStatus oldstatus;
 404     RepoStatus newstatus;
 405
 406     try
 407     {
 408       MIL << "Going to try to check whether refresh is needed for " << url << endl;
 409
 410       repo::RepoType repokind = info.type();
 411
 412       // if the type is unknown, try probing.
 413       switch ( repokind.toEnum() )
 414       {
 415         case RepoType::NONE_e:
 416           // unknown, probe it
 417           repokind = probe(url);
 418         break;
 419         default:
 420         break;
 421       }
 422
 423       Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 424       filesystem::assert_dir(rawpath);
 425       oldstatus = metadataStatus(info);
 426
 427       // now we've got the old (cached) status, we can decide repo.refresh.delay
 428       if (policy != RefreshForced && policy != RefreshIfNeededIgnoreDelay)
 429       {
 430         // difference in seconds
 431         double diff = difftime(
 432           (Date::ValueType)Date::now(),
 433           (Date::ValueType)oldstatus.timestamp()) / 60;
 434
 435         DBG << "oldstatus: " << (Date::ValueType)oldstatus.timestamp() << endl;
 436         DBG << "current time: " << (Date::ValueType)Date::now() << endl;
 437         DBG << "last refresh = " << diff << " minutes ago" << endl;
 438
 439         if (diff < ZConfig::instance().repo_refresh_delay())
 440         {
 441           MIL << "Repository '" << info.alias()
 442               << "' has been refreshed less than repo.refresh.delay ("
 443               << ZConfig::instance().repo_refresh_delay()
 444               << ") minutes ago. Advising to skip refresh" << endl;
 445           return REPO_CHECK_DELAYED;
 446         }
 447       }
 448
 449       // create temp dir as sibling of rawpath
 450       filesystem::TmpDir tmpdir( filesystem::TmpDir::makeSibling( rawpath ) );
 451
 452       if ( ( repokind.toEnum() == RepoType::RPMMD_e ) ||
 453            ( repokind.toEnum() == RepoType::YAST2_e ) )
 454       {
 455         MediaSetAccess media(url);
 456         shared_ptr<repo::Downloader> downloader_ptr;
 457
 458         if ( repokind.toEnum() == RepoType::RPMMD_e )
 459           downloader_ptr.reset(new yum::Downloader(info.path()));
 460         else
 461           downloader_ptr.reset( new susetags::Downloader(info.path()));
 462
 463         RepoStatus newstatus = downloader_ptr->status(media);
 464         bool refresh = false;
 465         if ( oldstatus.checksum() == newstatus.checksum() )
 466         {
 467           MIL << "repo has not changed" << endl;
 468           if ( policy == RefreshForced )
 469           {
 470             MIL << "refresh set to forced" << endl;
 471             refresh = true;
 472           }
 473         }
 474         else
 475         {
 476           MIL << "repo has changed, going to refresh" << endl;
 477           refresh = true;
 478         }
 479
 480         if (!refresh)
 481           touchIndexFile(info);
 482
 483         return refresh ? REFRESH_NEEDED : REPO_UP_TO_DATE;
 484       }
 485       else if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 486       {
 487         RepoStatus newstatus = parser::plaindir::dirStatus(url.getPathName());
 488         bool refresh = false;
 489         if ( oldstatus.checksum() == newstatus.checksum() )
 490         {
 491           MIL << "repo has not changed" << endl;
 492           if ( policy == RefreshForced )
 493           {
 494             MIL << "refresh set to forced" << endl;
 495             refresh = true;
 496           }
 497         }
 498         else
 499         {
 500           MIL << "repo has changed, going to refresh" << endl;
 501           refresh = true;
 502         }
 503
 504         if (!refresh)
 505           touchIndexFile(info);
 506
 507         return refresh ? REFRESH_NEEDED : REPO_UP_TO_DATE;
 508       }
 509       else
 510       {
 511         ZYPP_THROW(RepoUnknownTypeException());
 512       }
 513     }
 514     catch ( const Exception &e )
 515     {
 516       ZYPP_CAUGHT(e);
 517       ERR << "refresh check failed for " << url << endl;
 518       ZYPP_RETHROW(e);
 519     }
 520
 521     return REFRESH_NEEDED; // default
 522   }
 523
 524   void RepoManager::refreshMetadata( const RepoInfo &info,
 525                                      RawMetadataRefreshPolicy policy,
 526                                      const ProgressData::ReceiverFnc & progress )
 527   {
 528     assert_alias(info);
 529     assert_urls(info);
 530
 531     // we will throw this later if no URL checks out fine
 532     RepoException rexception(_("Valid metadata not found at specified URL(s)"));
 533
 534     // try urls one by one
 535     for ( RepoInfo::urls_const_iterator it = info.baseUrlsBegin(); it != info.baseUrlsEnd(); ++it )
 536     {
 537       try
 538       {
 539         Url url(*it);
 540
 541         // check whether to refresh metadata
 542         // if the check fails for this url, it throws, so another url will be checked
 543         if (checkIfToRefreshMetadata(info, url, policy)!=REFRESH_NEEDED)
 544           return;
 545
 546         MIL << "Going to refresh metadata from " << url << endl;
 547
 548         repo::RepoType repokind = info.type();
 549
 550         // if the type is unknown, try probing.
 551         switch ( repokind.toEnum() )
 552         {
 553           case RepoType::NONE_e:
 554             // unknown, probe it
 555             repokind = probe(*it);
 556
 557             if (repokind.toEnum() != RepoType::NONE_e)
 558             {
 559               //save probed type only for repos in system
 560               std::list<RepoInfo> repos = knownRepositories();
 561               for ( std::list<RepoInfo>::const_iterator it = repos.begin();
 562                    it != repos.end(); ++it )
 563               {
 564                 if ( info.alias() == (*it).alias() )
 565                 {
 566                   RepoInfo modifiedrepo = info;
 567                   modifiedrepo.setType(repokind);
 568                   modifyRepository(info.alias(),modifiedrepo);
 569                 }
 570               }
 571             }
 572           break;
 573           default:
 574           break;
 575         }
 576
 577         Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 578         filesystem::assert_dir(rawpath);
 579
 580         // create temp dir as sibling of rawpath
 581         filesystem::TmpDir tmpdir( filesystem::TmpDir::makeSibling( rawpath ) );
 582
 583         if ( ( repokind.toEnum() == RepoType::RPMMD_e ) ||
 584              ( repokind.toEnum() == RepoType::YAST2_e ) )
 585         {
 586           MediaSetAccess media(url);
 587           shared_ptr<repo::Downloader> downloader_ptr;
 588
 589           if ( repokind.toEnum() == RepoType::RPMMD_e )
 590             downloader_ptr.reset(new yum::Downloader(info.path()));
 591           else
 592             downloader_ptr.reset( new susetags::Downloader(info.path()));
 593
 594           /**
 595            * Given a downloader, sets the other repos raw metadata
 596            * path as cache paths for the fetcher, so if another
 597            * repo has the same file, it will not download it
 598            * but copy it from the other repository
 599            */
 600           std::list<RepoInfo> repos = knownRepositories();
 601           for ( std::list<RepoInfo>::const_iterator it = repos.begin();
 602                 it != repos.end();
 603                 ++it )
 604           {
 605             Pathname cachepath(rawcache_path_for_repoinfo( _pimpl->options, *it ));
 606             if ( PathInfo(cachepath).isExist() )
 607               downloader_ptr->addCachePath(cachepath);
 608           }
 609
 610           downloader_ptr->download( media, tmpdir.path());
 611         }
 612         else if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 613         {
 614           RepoStatus newstatus = parser::plaindir::dirStatus(url.getPathName());
 615
 616           std::ofstream file(( tmpdir.path() + "/cookie").c_str());
 617           if (!file) {
 618             ZYPP_THROW (Exception( "Can't open " + tmpdir.path().asString() + "/cookie" ) );
 619           }
 620           file << url << endl;
 621           file << newstatus.checksum() << endl;
 622
 623           file.close();
 624         }
 625         else
 626         {
 627           ZYPP_THROW(RepoUnknownTypeException());
 628         }
 629
 630         // ok we have the metadata, now exchange
 631         // the contents
 632
 633         TmpDir oldmetadata( TmpDir::makeSibling( rawpath ) );
 634         filesystem::rename( rawpath, oldmetadata.path() );
 635         // move the just downloaded there
 636         filesystem::rename( tmpdir.path(), rawpath );
 637
 638         // we are done.
 639         return;
 640       }
 641       catch ( const Exception &e )
 642       {
 643         ZYPP_CAUGHT(e);
 644         ERR << "Trying another url..." << endl;
 645
 646         // remember the exception caught for the *first URL*
 647         // if all other URLs fail, the rexception will be thrown with the
 648         // cause of the problem of the first URL remembered
 649         if (it == info.baseUrlsBegin())
 650           rexception.remember(e);
 651       }
 652     } // for every url
 653     ERR << "No more urls..." << endl;
 654     ZYPP_THROW(rexception);
 655   }
 656
 657   ////////////////////////////////////////////////////////////////////////////
 658
 659   void RepoManager::cleanMetadata( const RepoInfo &info,
 660                                    const ProgressData::ReceiverFnc & progressfnc )
 661   {
 662     ProgressData progress(100);
 663     progress.sendTo(progressfnc);
 664
 665     filesystem::recursive_rmdir(rawcache_path_for_repoinfo(_pimpl->options, info));
 666     progress.toMax();
 667   }
 668
 669   void RepoManager::cleanPackages( const RepoInfo &info,
 670                                    const ProgressData::ReceiverFnc & progressfnc )
 671   {
 672     ProgressData progress(100);
 673     progress.sendTo(progressfnc);
 674
 675     filesystem::recursive_rmdir(packagescache_path_for_repoinfo(_pimpl->options, info));
 676     progress.toMax();
 677   }
 678
 679   void RepoManager::buildCache( const RepoInfo &info,
 680                                 CacheBuildPolicy policy,
 681                                 const ProgressData::ReceiverFnc & progressrcv )
 682   {
 683     assert_alias(info);
 684     Pathname rawpath = rawcache_path_for_repoinfo(_pimpl->options, info);
 685
 686     filesystem::assert_dir(_pimpl->options.repoCachePath);
 687     Pathname base = solv_path_for_repoinfo( _pimpl->options, info);
 688     filesystem::assert_dir(base);
 689     Pathname solvfile = base / "solv";
 690
 691     RepoStatus raw_metadata_status = metadataStatus(info);
 692     if ( raw_metadata_status.empty() )
 693     {
 694        /* if there is no cache at this point, we refresh the raw
 695           in case this is the first time - if it's !autorefresh,
 696           we may still refresh */
 697       refreshMetadata(info, RefreshIfNeeded, progressrcv );
 698       raw_metadata_status = metadataStatus(info);
 699     }
 700
 701     bool needs_cleaning = false;
 702     if ( isCached( info ) )
 703     {
 704       MIL << info.alias() << " is already cached." << endl;
 705       //data::RecordId id = store.lookupRepository(info.alias());
 706       RepoStatus cache_status = cacheStatus(info);
 707
 708       if ( cache_status.checksum() == raw_metadata_status.checksum() )
 709       {
 710         MIL << info.alias() << " cache is up to date with metadata." << endl;
 711         if ( policy == BuildIfNeeded ) {
 712           return;
 713         }
 714         else {
 715           MIL << info.alias() << " cache rebuild is forced" << endl;
 716         }
 717       }
 718
 719       needs_cleaning = true;
 720     }
 721
 722     ProgressData progress(100);
 723     callback::SendReport<ProgressReport> report;
 724     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
 725     progress.name(str::form(_("Building repository '%s' cache"), info.name().c_str()));
 726     progress.toMin();
 727
 728     if (needs_cleaning)
 729     {
 730       cleanCache(info);
 731     }
 732
 733     MIL << info.alias() << " building cache..." << endl;
 734     // do we have type?
 735     repo::RepoType repokind = info.type();
 736
 737     // if the type is unknown, try probing.
 738     switch ( repokind.toEnum() )
 739     {
 740       case RepoType::NONE_e:
 741         // unknown, probe the local metadata
 742         repokind = probe(rawpath.asUrl());
 743       break;
 744       default:
 745       break;
 746     }
 747
 748     MIL << "repo type is " << repokind << endl;
 749
 750     switch ( repokind.toEnum() )
 751     {
 752       case RepoType::RPMMD_e :
 753       case RepoType::YAST2_e :
 754       case RepoType::RPMPLAINDIR_e :
 755       {
 756         // Take care we unlink the solvfile on exception
 757         ManagedFile guard( solvfile, filesystem::unlink );
 758
 759         ostringstream cmd;
 760         if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 761         {
 762           cmd << str::form( "repo2solv.sh \"%s\" > \"%s\"", info.baseUrlsBegin()->getPathName().c_str(), solvfile.c_str() );
 763         } else
 764           cmd << str::form( "repo2solv.sh \"%s\" > \"%s\"", rawpath.c_str(), solvfile.c_str() );
 765
 766         MIL << "Executing: " << cmd.str() << endl;
 767         ExternalProgram prog( cmd.str(), ExternalProgram::Stderr_To_Stdout );
 768
 769         cmd << endl;
 770         for ( string output( prog.receiveLine() ); output.length(); output = prog.receiveLine() ) {
 771           WAR << "  " << output;
 772           cmd << "     " << output;
 773         }
 774
 775         int ret = prog.close();
 776         if ( ret != 0 )
 777         {
 778           RepoException ex(str::form("Failed to cache repo (%d).", ret));
 779           ex.remember( cmd.str() );
 780           ZYPP_THROW(ex);
 781         }
 782
 783         // We keep it.
 784         guard.resetDispose();
 785       }
 786       break;
 787       default:
 788         ZYPP_THROW(RepoUnknownTypeException("Unhandled repository type"));
 789       break;
 790     }
 791 #if 0
 792     switch ( repokind.toEnum() )
 793     {
 794       case RepoType::RPMMD_e :
 795       if (0)
 796       {
 797         CombinedProgressData subprogrcv( progress, 100);
 798         parser::yum::RepoParser parser(id, store, parser::yum::RepoParserOpts(), subprogrcv);
 799         parser.parse(rawpath);
 800           // no error
 801       }
 802       break;
 803       case RepoType::YAST2_e :
 804       if (0)
 805       {
 806         CombinedProgressData subprogrcv( progress, 100);
 807         parser::susetags::RepoParser parser(id, store, subprogrcv);
 808         parser.parse(rawpath);
 809         // no error
 810       }
 811       break;
 812
 813       default:
 814         ZYPP_THROW(RepoUnknownTypeException());
 815     }
 816 #endif
 817     // update timestamp and checksum
 818     //store.updateRepositoryStatus(id, raw_metadata_status);
 819     setCacheStatus(info, raw_metadata_status);
 820     MIL << "Commit cache.." << endl;
 821     //store.commit();
 822     //progress.toMax();
 823   }
 824
 825   ////////////////////////////////////////////////////////////////////////////
 826
 827   repo::RepoType RepoManager::probe( const Url &url ) const
 828   {
 829     if ( url.getScheme() == "dir" && ! PathInfo( url.getPathName() ).isDir() )
 830     {
 831       // Handle non existing local directory in advance, as
 832       // MediaSetAccess does not support it.
 833       return repo::RepoType::NONE;
 834     }
 835
 836     try
 837     {
 838       MediaSetAccess access(url);
 839       if ( access.doesFileExist("/repodata/repomd.xml") )
 840         return repo::RepoType::RPMMD;
 841       if ( access.doesFileExist("/content") )
 842         return repo::RepoType::YAST2;
 843
 844       // if it is a local url of type dir
 845       if ( (! media::MediaManager::downloads(url)) && ( url.getScheme() == "dir" ) )
 846       {
 847         Pathname path = Pathname(url.getPathName());
 848         if ( PathInfo(path).isDir() )
 849         {
 850           // allow empty dirs for now
 851           return repo::RepoType::RPMPLAINDIR;
 852         }
 853       }
 854     }
 855     catch ( const media::MediaException &e )
 856     {
 857       ZYPP_CAUGHT(e);
 858       RepoException enew("Error trying to read from " + url.asString());
 859       enew.remember(e);
 860       ZYPP_THROW(enew);
 861     }
 862     catch ( const Exception &e )
 863     {
 864       ZYPP_CAUGHT(e);
 865       Exception enew("Unknown error reading from " + url.asString());
 866       enew.remember(e);
 867       ZYPP_THROW(enew);
 868     }
 869
 870     return repo::RepoType::NONE;
 871   }
 872
 873   ////////////////////////////////////////////////////////////////////////////
 874
 875   void RepoManager::cleanCache( const RepoInfo &info,
 876                                 const ProgressData::ReceiverFnc & progressrcv )
 877   {
 878     ProgressData progress(100);
 879     progress.sendTo(progressrcv);
 880     progress.toMin();
 881
 882     filesystem::recursive_rmdir(solv_path_for_repoinfo(_pimpl->options, info));
 883
 884     progress.toMax();
 885   }
 886
 887   void RepoManager::cleanTargetCache(const ProgressData::ReceiverFnc & progressrcv)
 888   {
 889     unlink (_pimpl->options.repoCachePath / (sat::Pool::systemRepoName() + ".solv"));
 890     unlink (_pimpl->options.repoCachePath / (sat::Pool::systemRepoName() + ".cookie"));
 891   }
 892
 893   ////////////////////////////////////////////////////////////////////////////
 894
 895   bool RepoManager::isCached( const RepoInfo &info ) const
 896   {
 897     return PathInfo(solv_path_for_repoinfo( _pimpl->options, info ) / "solv").isExist();
 898   }
 899
 900   RepoStatus RepoManager::cacheStatus( const RepoInfo &info ) const
 901   {
 902
 903     Pathname cookiefile = solv_path_for_repoinfo(_pimpl->options, info) / "cookie";
 904
 905     return RepoStatus::fromCookieFile(cookiefile);
 906   }
 907
 908   void RepoManager::setCacheStatus( const RepoInfo &info, const RepoStatus &status )
 909   {
 910     Pathname base = solv_path_for_repoinfo(_pimpl->options, info);
 911     filesystem::assert_dir(base);
 912     Pathname cookiefile = base / "cookie";
 913
 914     status.saveToCookieFile(cookiefile);
 915   }
 916
 917   void RepoManager::loadFromCache( const RepoInfo & info,
 918                                    const ProgressData::ReceiverFnc & progressrcv )
 919   {
 920     assert_alias(info);
 921     Pathname solvfile = solv_path_for_repoinfo(_pimpl->options, info) / "solv";
 922
 923     if ( ! PathInfo(solvfile).isExist() )
 924       ZYPP_THROW(RepoNotCachedException(info));
 925
 926     try
 927     {
 928       sat::Pool::instance().addRepoSolv( solvfile, info );
 929     }
 930     catch ( const Exception & exp )
 931     {
 932       ZYPP_CAUGHT( exp );
 933       MIL << "Try to handle exception by rebuilding the solv-file" << endl;
 934       cleanCache( info, progressrcv );
 935       buildCache( info, BuildIfNeeded, progressrcv );
 936
 937       sat::Pool::instance().addRepoSolv( solvfile, info );
 938     }
 939   }
 940
 941   ////////////////////////////////////////////////////////////////////////////
 942
 943   /**
 944    * Generate a non existing filename in a directory, using a base
 945    * name. For example if a directory contains 3 files
 946    *
 947    * |-- bar
 948    * |-- foo
 949    * `-- moo
 950    *
 951    * If you try to generate a unique filename for this directory,
 952    * based on "ruu" you will get "ruu", but if you use the base
 953    * "foo" you will get "foo_1"
 954    *
 955    * \param dir Directory where the file needs to be unique
 956    * \param basefilename string to base the filename on.
 957    */
 958   static Pathname generate_non_existing_name( const Pathname &dir,
 959                                               const std::string &basefilename )
 960   {
 961     string final_filename = basefilename;
 962     int counter = 1;
 963     while ( PathInfo(dir + final_filename).isExist() )
 964     {
 965       final_filename = basefilename + "_" + str::numstring(counter);
 966       counter++;
 967     }
 968     return dir + Pathname(final_filename);
 969   }
 970
 971   ////////////////////////////////////////////////////////////////////////////
 972
 973   /**
 974    * \short Generate a related filename from a repo info
 975    *
 976    * From a repo info, it will try to use the alias as a filename
 977    * escaping it if necessary. Other fallbacks can be added to
 978    * this function in case there is no way to use the alias
 979    */
 980   static std::string generate_filename( const RepoInfo &info )
 981   {
 982     std::string fnd="/";
 983     std::string rep="_";
 984     std::string filename = info.alias();
 985     // replace slashes with underscores
 986     size_t pos = filename.find(fnd);
 987     while(pos!=string::npos)
 988     {
 989       filename.replace(pos,fnd.length(),rep);
 990       pos = filename.find(fnd,pos+rep.length());
 991     }
 992     filename = Pathname(filename).extend(".repo").asString();
 993     MIL << "generating filename for repo [" << info.alias() << "] : '" << filename << "'" << endl;
 994     return filename;
 995   }
 996
 997
 998   ////////////////////////////////////////////////////////////////////////////
 999
1000   void RepoManager::addRepository( const RepoInfo &info,
1001                                    const ProgressData::ReceiverFnc & progressrcv )
1002   {
1003     assert_alias(info);
1004
1005     ProgressData progress(100);
1006     callback::SendReport<ProgressReport> report;
1007     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
1008     progress.name(str::form(_("Adding repository '%s'"), info.name().c_str()));
1009     progress.toMin();
1010
1011     std::list<RepoInfo> repos = knownRepositories();
1012     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1013           it != repos.end();
1014           ++it )
1015     {
1016       if ( info.alias() == (*it).alias() )
1017         ZYPP_THROW(RepoAlreadyExistsException(info));
1018     }
1019
1020     RepoInfo tosave = info;
1021
1022     // check the first url for now
1023     if ( _pimpl->options.probe )
1024     {
1025       DBG << "unknown repository type, probing" << endl;
1026
1027       RepoType probedtype;
1028       probedtype = probe(*tosave.baseUrlsBegin());
1029       if ( tosave.baseUrlsSize() > 0 )
1030       {
1031         if ( probedtype == RepoType::NONE )
1032           ZYPP_THROW(RepoUnknownTypeException());
1033         else
1034           tosave.setType(probedtype);
1035       }
1036     }
1037
1038     progress.set(50);
1039
1040     // assert the directory exists
1041     filesystem::assert_dir(_pimpl->options.knownReposPath);
1042
1043     Pathname repofile = generate_non_existing_name(_pimpl->options.knownReposPath,
1044                                                     generate_filename(tosave));
1045     // now we have a filename that does not exists
1046     MIL << "Saving repo in " << repofile << endl;
1047
1048     std::ofstream file(repofile.c_str());
1049     if (!file) {
1050       ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
1051     }
1052
1053     tosave.dumpRepoOn(file);
1054     progress.toMax();
1055     MIL << "done" << endl;
1056   }
1057
1058   void RepoManager::addRepositories( const Url &url,
1059                                      const ProgressData::ReceiverFnc & progressrcv )
1060   {
1061     std::list<RepoInfo> knownrepos = knownRepositories();
1062     std::list<RepoInfo> repos = readRepoFile(url);
1063     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1064           it != repos.end();
1065           ++it )
1066     {
1067       // look if the alias is in the known repos.
1068       for ( std::list<RepoInfo>::const_iterator kit = knownrepos.begin();
1069           kit != knownrepos.end();
1070           ++kit )
1071       {
1072         if ( (*it).alias() == (*kit).alias() )
1073         {
1074           ERR << "To be added repo " << (*it).alias() << " conflicts with existing repo " << (*kit).alias() << endl;
1075           ZYPP_THROW(RepoAlreadyExistsException(*it));
1076         }
1077       }
1078     }
1079
1080     string filename = Pathname(url.getPathName()).basename();
1081
1082     if ( filename == Pathname() )
1083       ZYPP_THROW(RepoException("Invalid repo file name at " + url.asString() ));
1084
1085     // assert the directory exists
1086     filesystem::assert_dir(_pimpl->options.knownReposPath);
1087
1088     Pathname repofile = generate_non_existing_name(_pimpl->options.knownReposPath, filename);
1089     // now we have a filename that does not exists
1090     MIL << "Saving " << repos.size() << " repo" << ( repos.size() ? "s" : "" ) << " in " << repofile << endl;
1091
1092     std::ofstream file(repofile.c_str());
1093     if (!file) {
1094       ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
1095     }
1096
1097     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1098           it != repos.end();
1099           ++it )
1100     {
1101       MIL << "Saving " << (*it).alias() << endl;
1102       (*it).dumpRepoOn(file);
1103     }
1104     MIL << "done" << endl;
1105   }
1106
1107   ////////////////////////////////////////////////////////////////////////////
1108
1109   void RepoManager::removeRepository( const RepoInfo & info,
1110                                       const ProgressData::ReceiverFnc & progressrcv)
1111   {
1112     ProgressData progress;
1113     callback::SendReport<ProgressReport> report;
1114     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
1115     progress.name(str::form(_("Removing repository '%s'"), info.name().c_str()));
1116
1117     MIL << "Going to delete repo " << info.alias() << endl;
1118
1119     std::list<RepoInfo> repos = knownRepositories();
1120     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1121           it != repos.end();
1122           ++it )
1123     {
1124       // they can be the same only if the provided is empty, that means
1125       // the provided repo has no alias
1126       // then skip
1127       if ( (!info.alias().empty()) && ( info.alias() != (*it).alias() ) )
1128         continue;
1129
1130       // TODO match by url
1131
1132       // we have a matcing repository, now we need to know
1133       // where it does come from.
1134       RepoInfo todelete = *it;
1135       if (todelete.filepath().empty())
1136       {
1137         ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
1138       }
1139       else
1140       {
1141         // figure how many repos are there in the file:
1142         std::list<RepoInfo> filerepos = repositories_in_file(todelete.filepath());
1143         if ( (filerepos.size() == 1) && ( filerepos.front().alias() == todelete.alias() ) )
1144         {
1145           // easy, only this one, just delete the file
1146           if ( filesystem::unlink(todelete.filepath()) != 0 )
1147           {
1148             ZYPP_THROW(RepoException("Can't delete " + todelete.filepath().asString()));
1149           }
1150           MIL << todelete.alias() << " sucessfully deleted." << endl;
1151         }
1152         else
1153         {
1154           // there are more repos in the same file
1155           // write them back except the deleted one.
1156           //TmpFile tmp;
1157           //std::ofstream file(tmp.path().c_str());
1158
1159           // assert the directory exists
1160           filesystem::assert_dir(todelete.filepath().dirname());
1161
1162           std::ofstream file(todelete.filepath().c_str());
1163           if (!file) {
1164             //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
1165             ZYPP_THROW (Exception( "Can't open " + todelete.filepath().asString() ) );
1166           }
1167           for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
1168                 fit != filerepos.end();
1169                 ++fit )
1170           {
1171             if ( (*fit).alias() != todelete.alias() )
1172               (*fit).dumpRepoOn(file);
1173           }
1174         }
1175
1176         CombinedProgressData subprogrcv(progress, 70);
1177         CombinedProgressData cleansubprogrcv(progress, 30);
1178         // now delete it from cache
1179         if ( isCached(todelete) )
1180           cleanCache( todelete, subprogrcv);
1181         // now delete metadata (#301037)
1182         cleanMetadata( todelete, cleansubprogrcv);
1183         MIL << todelete.alias() << " sucessfully deleted." << endl;
1184         return;
1185       } // else filepath is empty
1186
1187     }
1188     // should not be reached on a sucess workflow
1189     ZYPP_THROW(RepoNotFoundException(info));
1190   }
1191
1192   ////////////////////////////////////////////////////////////////////////////
1193
1194   void RepoManager::modifyRepository( const std::string &alias,
1195                                       const RepoInfo & newinfo,
1196                                       const ProgressData::ReceiverFnc & progressrcv )
1197   {
1198     RepoInfo toedit = getRepositoryInfo(alias);
1199
1200     // check if the new alias already exists when renaming the repo
1201     if (alias != newinfo.alias())
1202     {
1203       std::list<RepoInfo> repos = knownRepositories();
1204       for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1205             it != repos.end();
1206             ++it )
1207       {
1208         if ( newinfo.alias() == (*it).alias() )
1209           ZYPP_THROW(RepoAlreadyExistsException(newinfo));
1210       }
1211     }
1212
1213     if (toedit.filepath().empty())
1214     {
1215       ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
1216     }
1217     else
1218     {
1219       // figure how many repos are there in the file:
1220       std::list<RepoInfo> filerepos = repositories_in_file(toedit.filepath());
1221
1222       // there are more repos in the same file
1223       // write them back except the deleted one.
1224       //TmpFile tmp;
1225       //std::ofstream file(tmp.path().c_str());
1226
1227       // assert the directory exists
1228       filesystem::assert_dir(toedit.filepath().dirname());
1229
1230       std::ofstream file(toedit.filepath().c_str());
1231       if (!file) {
1232         //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
1233         ZYPP_THROW (Exception( "Can't open " + toedit.filepath().asString() ) );
1234       }
1235       for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
1236             fit != filerepos.end();
1237             ++fit )
1238       {
1239           // if the alias is different, dump the original
1240           // if it is the same, dump the provided one
1241           if ( (*fit).alias() != toedit.alias() )
1242             (*fit).dumpRepoOn(file);
1243           else
1244             newinfo.dumpRepoOn(file);
1245       }
1246     }
1247   }
1248
1249   ////////////////////////////////////////////////////////////////////////////
1250
1251   RepoInfo RepoManager::getRepositoryInfo( const std::string &alias,
1252                                            const ProgressData::ReceiverFnc & progressrcv )
1253   {
1254     std::list<RepoInfo> repos = knownRepositories();
1255     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1256           it != repos.end();
1257           ++it )
1258     {
1259       if ( (*it).alias() == alias )
1260         return *it;
1261     }
1262     RepoInfo info;
1263     info.setAlias(info.alias());
1264     ZYPP_THROW(RepoNotFoundException(info));
1265   }
1266
1267   ////////////////////////////////////////////////////////////////////////////
1268
1269   RepoInfo RepoManager::getRepositoryInfo( const Url & url,
1270                                            const url::ViewOption & urlview,
1271                                            const ProgressData::ReceiverFnc & progressrcv )
1272   {
1273     std::list<RepoInfo> repos = knownRepositories();
1274     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1275           it != repos.end();
1276           ++it )
1277     {
1278       for(RepoInfo::urls_const_iterator urlit = (*it).baseUrlsBegin();
1279           urlit != (*it).baseUrlsEnd();
1280           ++urlit)
1281       {
1282         if ((*urlit).asString(urlview) == url.asString(urlview))
1283           return *it;
1284       }
1285     }
1286     RepoInfo info;
1287     info.setAlias(info.alias());
1288     info.setBaseUrl(url);
1289     ZYPP_THROW(RepoNotFoundException(info));
1290   }
1291
1292   ////////////////////////////////////////////////////////////////////////////
1293
1294   std::ostream & operator<<( std::ostream & str, const RepoManager & obj )
1295   {
1296     return str << *obj._pimpl;
1297   }
1298
1299   /////////////////////////////////////////////////////////////////
1300 } // namespace zypp
1301 ///////////////////////////////////////////////////////////////////