zypp/RepoManager.cc

   1 /*---------------------------------------------------------------------\
   2 |                          ____ _   __ __ ___                          |
   3 |                         |__  / \ / / . \ . \                         |
   4 |                           / / \ V /|  _/  _/                         |
   5 |                          / /__ | | | | | |                           |
   6 |                         /_____||_| |_| |_|                           |
   7 |                                                                      |
   8 \---------------------------------------------------------------------*/
   9 /** \file       zypp/RepoManager.cc
  10  *
  11 */
  12
  13 #include <cstdlib>
  14 #include <iostream>
  15 #include <fstream>
  16 #include <sstream>
  17 #include <list>
  18 #include <map>
  19 #include <algorithm>
  20 #include "zypp/base/InputStream.h"
  21 #include "zypp/base/Logger.h"
  22 #include "zypp/base/Gettext.h"
  23 #include "zypp/base/Function.h"
  24 #include "zypp/base/Regex.h"
  25 #include "zypp/PathInfo.h"
  26 #include "zypp/TmpPath.h"
  27
  28 #include "zypp/repo/RepoException.h"
  29 #include "zypp/RepoManager.h"
  30
  31 #include "zypp/media/MediaManager.h"
  32 #include "zypp/MediaSetAccess.h"
  33 #include "zypp/ExternalProgram.h"
  34 #include "zypp/ManagedFile.h"
  35
  36 #include "zypp/parser/RepoFileReader.h"
  37 #include "zypp/repo/yum/Downloader.h"
  38 #include "zypp/parser/yum/RepoParser.h"
  39 #include "zypp/repo/susetags/Downloader.h"
  40 #include "zypp/parser/susetags/RepoParser.h"
  41 #include "zypp/parser/plaindir/RepoParser.h"
  42
  43 #include "zypp/ZYppCallbacks.h"
  44
  45 #include "sat/Pool.h"
  46 #include "satsolver/pool.h"
  47 #include "satsolver/repo.h"
  48 #include "satsolver/repo_solv.h"
  49
  50 using namespace std;
  51 using namespace zypp;
  52 using namespace zypp::repo;
  53 using namespace zypp::filesystem;
  54
  55 using namespace zypp::repo;
  56
  57 ///////////////////////////////////////////////////////////////////
  58 namespace zypp
  59 { /////////////////////////////////////////////////////////////////
  60
  61   ///////////////////////////////////////////////////////////////////
  62   //
  63   //    CLASS NAME : RepoManagerOptions
  64   //
  65   ///////////////////////////////////////////////////////////////////
  66
  67   RepoManagerOptions::RepoManagerOptions()
  68   {
  69     repoCachePath         = ZConfig::instance().repoCachePath();
  70     repoRawCachePath      = ZConfig::instance().repoMetadataPath();
  71     repoSolvCachePath     = ZConfig::instance().repoSolvfilesPath();
  72     repoPackagesCachePath = ZConfig::instance().repoPackagesPath();
  73     knownReposPath        = ZConfig::instance().knownReposPath();
  74     probe                 = ZConfig::instance().repo_add_probe();
  75   }
  76
  77   ////////////////////////////////////////////////////////////////////////////
  78
  79   /**
  80     * \short Simple callback to collect the results
  81     *
  82     * Classes like RepoFileParser call the callback
  83     * once per each repo in a file.
  84     *
  85     * Passing this functor as callback, you can collect
  86     * all resuls at the end, without dealing with async
  87     * code.
  88     */
  89     struct RepoCollector
  90     {
  91       RepoCollector()
  92       {}
  93
  94       ~RepoCollector()
  95       {}
  96
  97       bool collect( const RepoInfo &repo )
  98       {
  99         repos.push_back(repo);
 100         return true;
 101       }
 102
 103       RepoInfoList repos;
 104     };
 105
 106   ////////////////////////////////////////////////////////////////////////////
 107
 108   /**
 109    * Reads RepoInfo's from a repo file.
 110    *
 111    * \param file pathname of the file to read.
 112    */
 113   static std::list<RepoInfo> repositories_in_file( const Pathname & file )
 114   {
 115     MIL << "repo file: " << file << endl;
 116     RepoCollector collector;
 117     parser::RepoFileReader parser( file, bind( &RepoCollector::collect, &collector, _1 ) );
 118     return collector.repos;
 119   }
 120
 121   ////////////////////////////////////////////////////////////////////////////
 122
 123   std::list<RepoInfo> readRepoFile(const Url & repo_file)
 124    {
 125      // no interface to download a specific file, using workaround:
 126      //! \todo add MediaManager::provideFile(Url file_url) to easily access any file URLs? (no need for media access id or media_nr)
 127      Url url(repo_file);
 128      Pathname path(url.getPathName());
 129      url.setPathName ("/");
 130      MediaSetAccess access(url);
 131      Pathname local = access.provideFile(path);
 132
 133      DBG << "reading repo file " << repo_file << ", local path: " << local << endl;
 134
 135      return repositories_in_file(local);
 136    }
 137
 138   ////////////////////////////////////////////////////////////////////////////
 139
 140   /**
 141    * \short List of RepoInfo's from a directory
 142    *
 143    * Goes trough every file ending with ".repo" in a directory and adds all
 144    * RepoInfo's contained in that file.
 145    *
 146    * \param dir pathname of the directory to read.
 147    */
 148   static std::list<RepoInfo> repositories_in_dir( const Pathname &dir )
 149   {
 150     MIL << "directory " << dir << endl;
 151     list<RepoInfo> repos;
 152     list<Pathname> entries;
 153     if ( filesystem::readdir( entries, Pathname(dir), false ) != 0 )
 154       ZYPP_THROW(Exception("failed to read directory"));
 155
 156     str::regex allowedRepoExt("^\\.repo(_[0-9]+)?$");
 157     for ( list<Pathname>::const_iterator it = entries.begin(); it != entries.end(); ++it )
 158     {
 159       if (str::regex_match(it->extension(), allowedRepoExt))
 160       {
 161         list<RepoInfo> tmp = repositories_in_file( *it );
 162         repos.insert( repos.end(), tmp.begin(), tmp.end() );
 163
 164         //std::copy( collector.repos.begin(), collector.repos.end(), std::back_inserter(repos));
 165         //MIL << "ok" << endl;
 166       }
 167     }
 168     return repos;
 169   }
 170
 171   ////////////////////////////////////////////////////////////////////////////
 172
 173   static void assert_alias( const RepoInfo &info )
 174   {
 175     if (info.alias().empty())
 176         ZYPP_THROW(RepoNoAliasException());
 177   }
 178
 179   ////////////////////////////////////////////////////////////////////////////
 180
 181   static void assert_urls( const RepoInfo &info )
 182   {
 183     if (info.baseUrlsEmpty())
 184         ZYPP_THROW(RepoNoUrlException());
 185   }
 186
 187   ////////////////////////////////////////////////////////////////////////////
 188
 189   /**
 190    * \short Calculates the raw cache path for a repository
 191    */
 192   static Pathname rawcache_path_for_repoinfo( const RepoManagerOptions &opt, const RepoInfo &info )
 193   {
 194     assert_alias(info);
 195     return opt.repoRawCachePath / info.escaped_alias();
 196   }
 197
 198   /**
 199    * \short Calculates the packages cache path for a repository
 200    */
 201   static Pathname packagescache_path_for_repoinfo( const RepoManagerOptions &opt, const RepoInfo &info )
 202   {
 203     assert_alias(info);
 204     return opt.repoPackagesCachePath / info.escaped_alias();
 205   }
 206
 207   static Pathname solv_path_for_repoinfo( const RepoManagerOptions &opt, const RepoInfo &info)
 208   {
 209     assert_alias(info);
 210     return opt.repoSolvCachePath / info.escaped_alias();
 211   }
 212
 213   ///////////////////////////////////////////////////////////////////
 214   //
 215   //    CLASS NAME : RepoManager::Impl
 216   //
 217   ///////////////////////////////////////////////////////////////////
 218
 219   /**
 220    * \short RepoManager implementation.
 221    */
 222   struct RepoManager::Impl
 223   {
 224     Impl( const RepoManagerOptions &opt )
 225       : options(opt)
 226     {
 227
 228     }
 229
 230     Impl()
 231     {
 232
 233     }
 234
 235     RepoManagerOptions options;
 236
 237     map<string, RepoInfo> _repoinfos;
 238
 239   public:
 240     /** Offer default Impl. */
 241     static shared_ptr<Impl> nullimpl()
 242     {
 243       static shared_ptr<Impl> _nullimpl( new Impl );
 244       return _nullimpl;
 245     }
 246
 247   private:
 248     friend Impl * rwcowClone<Impl>( const Impl * rhs );
 249     /** clone for RWCOW_pointer */
 250     Impl * clone() const
 251     { return new Impl( *this ); }
 252   };
 253
 254   ///////////////////////////////////////////////////////////////////
 255
 256   /** \relates RepoManager::Impl Stream output */
 257   inline std::ostream & operator<<( std::ostream & str, const RepoManager::Impl & obj )
 258   {
 259     return str << "RepoManager::Impl";
 260   }
 261
 262   ///////////////////////////////////////////////////////////////////
 263   //
 264   //    CLASS NAME : RepoManager
 265   //
 266   ///////////////////////////////////////////////////////////////////
 267
 268   RepoManager::RepoManager( const RepoManagerOptions &opt )
 269   : _pimpl( new Impl(opt) )
 270   {}
 271
 272   ////////////////////////////////////////////////////////////////////////////
 273
 274   RepoManager::~RepoManager()
 275   {}
 276
 277   ////////////////////////////////////////////////////////////////////////////
 278
 279   std::list<RepoInfo> RepoManager::knownRepositories() const
 280   {
 281     MIL << endl;
 282
 283     if ( PathInfo(_pimpl->options.knownReposPath).isExist() )
 284     {
 285       RepoInfoList repos = repositories_in_dir(_pimpl->options.knownReposPath);
 286       for ( RepoInfoList::iterator it = repos.begin();
 287             it != repos.end();
 288             ++it )
 289       {
 290         // set the metadata path for the repo
 291         Pathname metadata_path = rawcache_path_for_repoinfo(_pimpl->options, (*it));
 292         (*it).setMetadataPath(metadata_path);
 293
 294         // set the downloaded packages path for the repo
 295         Pathname packages_path = packagescache_path_for_repoinfo(_pimpl->options, (*it));
 296         (*it).setPackagesPath(packages_path);
 297       }
 298       return repos;
 299     }
 300     else
 301       return std::list<RepoInfo>();
 302
 303     MIL << endl;
 304   }
 305
 306   ////////////////////////////////////////////////////////////////////////////
 307
 308   Pathname RepoManager::metadataPath( const RepoInfo &info ) const
 309   {
 310     return rawcache_path_for_repoinfo(_pimpl->options, info );
 311   }
 312
 313   ////////////////////////////////////////////////////////////////////////////
 314
 315   RepoStatus RepoManager::metadataStatus( const RepoInfo &info ) const
 316   {
 317     Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 318     RepoType repokind = info.type();
 319     RepoStatus status;
 320
 321     switch ( repokind.toEnum() )
 322     {
 323       case RepoType::NONE_e:
 324       // unknown, probe the local metadata
 325         repokind = probe(rawpath.asUrl());
 326       break;
 327       default:
 328       break;
 329     }
 330
 331     switch ( repokind.toEnum() )
 332     {
 333       case RepoType::RPMMD_e :
 334       {
 335         status = RepoStatus( rawpath + "/repodata/repomd.xml");
 336       }
 337       break;
 338
 339       case RepoType::YAST2_e :
 340       {
 341         status = RepoStatus( rawpath + "/content") && (RepoStatus( rawpath + "/media.1/media"));
 342       }
 343       break;
 344
 345       case RepoType::RPMPLAINDIR_e :
 346       {
 347         if ( PathInfo(Pathname(rawpath + "/cookie")).isExist() )
 348           status = RepoStatus( rawpath + "/cookie");
 349       }
 350       break;
 351
 352       case RepoType::NONE_e :
 353         // Return default RepoStatus in case of RepoType::NONE
 354         // indicating it should be created?
 355         // ZYPP_THROW(RepoUnknownTypeException());
 356         break;
 357     }
 358     return status;
 359   }
 360
 361   void RepoManager::touchIndexFile(const RepoInfo & info)
 362   {
 363     Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 364
 365     RepoType repokind = info.type();
 366     if ( repokind.toEnum() == RepoType::NONE_e )
 367       // unknown, probe the local metadata
 368       repokind = probe(rawpath.asUrl());
 369     // if still unknown, just return
 370     if (repokind == RepoType::NONE_e)
 371       return;
 372
 373     Pathname p;
 374     switch ( repokind.toEnum() )
 375     {
 376       case RepoType::RPMMD_e :
 377         p = Pathname(rawpath + "/repodata/repomd.xml");
 378         break;
 379
 380       case RepoType::YAST2_e :
 381         p = Pathname(rawpath + "/content");
 382         break;
 383
 384       case RepoType::RPMPLAINDIR_e :
 385         p = Pathname(rawpath + "/cookie");
 386         break;
 387
 388       case RepoType::NONE_e :
 389       default:
 390         break;
 391     }
 392
 393     // touch the file, ignore error (they are logged anyway)
 394     filesystem::touch(p);
 395   }
 396
 397   RepoManager::RefreshCheckStatus RepoManager::checkIfToRefreshMetadata(
 398                                               const RepoInfo &info,
 399                                               const Url &url,
 400                                               RawMetadataRefreshPolicy policy )
 401   {
 402     assert_alias(info);
 403
 404     RepoStatus oldstatus;
 405     RepoStatus newstatus;
 406
 407     try
 408     {
 409       MIL << "Going to try to check whether refresh is needed for " << url << endl;
 410
 411       repo::RepoType repokind = info.type();
 412
 413       // if the type is unknown, try probing.
 414       switch ( repokind.toEnum() )
 415       {
 416         case RepoType::NONE_e:
 417           // unknown, probe it
 418           repokind = probe(url);
 419         break;
 420         default:
 421         break;
 422       }
 423
 424       Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 425       filesystem::assert_dir(rawpath);
 426       oldstatus = metadataStatus(info);
 427
 428       // now we've got the old (cached) status, we can decide repo.refresh.delay
 429       if (policy != RefreshForced && policy != RefreshIfNeededIgnoreDelay)
 430       {
 431         // difference in seconds
 432         double diff = difftime(
 433           (Date::ValueType)Date::now(),
 434           (Date::ValueType)oldstatus.timestamp()) / 60;
 435
 436         DBG << "oldstatus: " << (Date::ValueType)oldstatus.timestamp() << endl;
 437         DBG << "current time: " << (Date::ValueType)Date::now() << endl;
 438         DBG << "last refresh = " << diff << " minutes ago" << endl;
 439
 440         if (diff < ZConfig::instance().repo_refresh_delay())
 441         {
 442           MIL << "Repository '" << info.alias()
 443               << "' has been refreshed less than repo.refresh.delay ("
 444               << ZConfig::instance().repo_refresh_delay()
 445               << ") minutes ago. Advising to skip refresh" << endl;
 446           return REPO_CHECK_DELAYED;
 447         }
 448       }
 449
 450       // create temp dir as sibling of rawpath
 451       filesystem::TmpDir tmpdir( filesystem::TmpDir::makeSibling( rawpath ) );
 452
 453       if ( ( repokind.toEnum() == RepoType::RPMMD_e ) ||
 454            ( repokind.toEnum() == RepoType::YAST2_e ) )
 455       {
 456         MediaSetAccess media(url);
 457         shared_ptr<repo::Downloader> downloader_ptr;
 458
 459         if ( repokind.toEnum() == RepoType::RPMMD_e )
 460           downloader_ptr.reset(new yum::Downloader(info.path()));
 461         else
 462           downloader_ptr.reset( new susetags::Downloader(info.path()));
 463
 464         RepoStatus newstatus = downloader_ptr->status(media);
 465         bool refresh = false;
 466         if ( oldstatus.checksum() == newstatus.checksum() )
 467         {
 468           MIL << "repo has not changed" << endl;
 469           if ( policy == RefreshForced )
 470           {
 471             MIL << "refresh set to forced" << endl;
 472             refresh = true;
 473           }
 474         }
 475         else
 476         {
 477           MIL << "repo has changed, going to refresh" << endl;
 478           refresh = true;
 479         }
 480
 481         if (!refresh)
 482           touchIndexFile(info);
 483
 484         return refresh ? REFRESH_NEEDED : REPO_UP_TO_DATE;
 485       }
 486       else if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 487       {
 488         RepoStatus newstatus = parser::plaindir::dirStatus(url.getPathName());
 489         bool refresh = false;
 490         if ( oldstatus.checksum() == newstatus.checksum() )
 491         {
 492           MIL << "repo has not changed" << endl;
 493           if ( policy == RefreshForced )
 494           {
 495             MIL << "refresh set to forced" << endl;
 496             refresh = true;
 497           }
 498         }
 499         else
 500         {
 501           MIL << "repo has changed, going to refresh" << endl;
 502           refresh = true;
 503         }
 504
 505         if (!refresh)
 506           touchIndexFile(info);
 507
 508         return refresh ? REFRESH_NEEDED : REPO_UP_TO_DATE;
 509       }
 510       else
 511       {
 512         ZYPP_THROW(RepoUnknownTypeException());
 513       }
 514     }
 515     catch ( const Exception &e )
 516     {
 517       ZYPP_CAUGHT(e);
 518       ERR << "refresh check failed for " << url << endl;
 519       ZYPP_RETHROW(e);
 520     }
 521
 522     return REFRESH_NEEDED; // default
 523   }
 524
 525   void RepoManager::refreshMetadata( const RepoInfo &info,
 526                                      RawMetadataRefreshPolicy policy,
 527                                      const ProgressData::ReceiverFnc & progress )
 528   {
 529     assert_alias(info);
 530     assert_urls(info);
 531
 532     // we will throw this later if no URL checks out fine
 533     RepoException rexception(_("Valid metadata not found at specified URL(s)"));
 534
 535     // try urls one by one
 536     for ( RepoInfo::urls_const_iterator it = info.baseUrlsBegin(); it != info.baseUrlsEnd(); ++it )
 537     {
 538       try
 539       {
 540         Url url(*it);
 541
 542         // check whether to refresh metadata
 543         // if the check fails for this url, it throws, so another url will be checked
 544         if (checkIfToRefreshMetadata(info, url, policy)!=REFRESH_NEEDED)
 545           return;
 546
 547         MIL << "Going to refresh metadata from " << url << endl;
 548
 549         repo::RepoType repokind = info.type();
 550
 551         // if the type is unknown, try probing.
 552         switch ( repokind.toEnum() )
 553         {
 554           case RepoType::NONE_e:
 555             // unknown, probe it
 556             repokind = probe(*it);
 557
 558             if (repokind.toEnum() != RepoType::NONE_e)
 559             {
 560               //save probed type only for repos in system
 561               std::list<RepoInfo> repos = knownRepositories();
 562               for ( std::list<RepoInfo>::const_iterator it = repos.begin();
 563                    it != repos.end(); ++it )
 564               {
 565                 if ( info.alias() == (*it).alias() )
 566                 {
 567                   RepoInfo modifiedrepo = info;
 568                   modifiedrepo.setType(repokind);
 569                   modifyRepository(info.alias(),modifiedrepo);
 570                 }
 571               }
 572             }
 573           break;
 574           default:
 575           break;
 576         }
 577
 578         Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 579         filesystem::assert_dir(rawpath);
 580
 581         // create temp dir as sibling of rawpath
 582         filesystem::TmpDir tmpdir( filesystem::TmpDir::makeSibling( rawpath ) );
 583
 584         if ( ( repokind.toEnum() == RepoType::RPMMD_e ) ||
 585              ( repokind.toEnum() == RepoType::YAST2_e ) )
 586         {
 587           MediaSetAccess media(url);
 588           shared_ptr<repo::Downloader> downloader_ptr;
 589
 590           if ( repokind.toEnum() == RepoType::RPMMD_e )
 591             downloader_ptr.reset(new yum::Downloader(info.path()));
 592           else
 593             downloader_ptr.reset( new susetags::Downloader(info.path()));
 594
 595           /**
 596            * Given a downloader, sets the other repos raw metadata
 597            * path as cache paths for the fetcher, so if another
 598            * repo has the same file, it will not download it
 599            * but copy it from the other repository
 600            */
 601           std::list<RepoInfo> repos = knownRepositories();
 602           for ( std::list<RepoInfo>::const_iterator it = repos.begin();
 603                 it != repos.end();
 604                 ++it )
 605           {
 606             Pathname cachepath(rawcache_path_for_repoinfo( _pimpl->options, *it ));
 607             if ( PathInfo(cachepath).isExist() )
 608               downloader_ptr->addCachePath(cachepath);
 609           }
 610
 611           downloader_ptr->download( media, tmpdir.path());
 612         }
 613         else if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 614         {
 615           RepoStatus newstatus = parser::plaindir::dirStatus(url.getPathName());
 616
 617           std::ofstream file(( tmpdir.path() + "/cookie").c_str());
 618           if (!file) {
 619             ZYPP_THROW (Exception( "Can't open " + tmpdir.path().asString() + "/cookie" ) );
 620           }
 621           file << url << endl;
 622           file << newstatus.checksum() << endl;
 623
 624           file.close();
 625         }
 626         else
 627         {
 628           ZYPP_THROW(RepoUnknownTypeException());
 629         }
 630
 631         // ok we have the metadata, now exchange
 632         // the contents
 633
 634         TmpDir oldmetadata( TmpDir::makeSibling( rawpath ) );
 635         filesystem::rename( rawpath, oldmetadata.path() );
 636         // move the just downloaded there
 637         filesystem::rename( tmpdir.path(), rawpath );
 638
 639         // we are done.
 640         return;
 641       }
 642       catch ( const Exception &e )
 643       {
 644         ZYPP_CAUGHT(e);
 645         ERR << "Trying another url..." << endl;
 646
 647         // remember the exception caught for the *first URL*
 648         // if all other URLs fail, the rexception will be thrown with the
 649         // cause of the problem of the first URL remembered
 650         if (it == info.baseUrlsBegin())
 651           rexception.remember(e);
 652       }
 653     } // for every url
 654     ERR << "No more urls..." << endl;
 655     ZYPP_THROW(rexception);
 656   }
 657
 658   ////////////////////////////////////////////////////////////////////////////
 659
 660   void RepoManager::cleanMetadata( const RepoInfo &info,
 661                                    const ProgressData::ReceiverFnc & progressfnc )
 662   {
 663     ProgressData progress(100);
 664     progress.sendTo(progressfnc);
 665
 666     filesystem::recursive_rmdir(rawcache_path_for_repoinfo(_pimpl->options, info));
 667     progress.toMax();
 668   }
 669
 670   void RepoManager::cleanPackages( const RepoInfo &info,
 671                                    const ProgressData::ReceiverFnc & progressfnc )
 672   {
 673     ProgressData progress(100);
 674     progress.sendTo(progressfnc);
 675
 676     filesystem::recursive_rmdir(packagescache_path_for_repoinfo(_pimpl->options, info));
 677     progress.toMax();
 678   }
 679
 680   void RepoManager::buildCache( const RepoInfo &info,
 681                                 CacheBuildPolicy policy,
 682                                 const ProgressData::ReceiverFnc & progressrcv )
 683   {
 684     assert_alias(info);
 685     Pathname rawpath = rawcache_path_for_repoinfo(_pimpl->options, info);
 686
 687     filesystem::assert_dir(_pimpl->options.repoCachePath);
 688     RepoStatus raw_metadata_status = metadataStatus(info);
 689     if ( raw_metadata_status.empty() )
 690     {
 691        /* if there is no cache at this point, we refresh the raw
 692           in case this is the first time - if it's !autorefresh,
 693           we may still refresh */
 694       refreshMetadata(info, RefreshIfNeeded, progressrcv );
 695       raw_metadata_status = metadataStatus(info);
 696     }
 697
 698     bool needs_cleaning = false;
 699     if ( isCached( info ) )
 700     {
 701       MIL << info.alias() << " is already cached." << endl;
 702       //data::RecordId id = store.lookupRepository(info.alias());
 703       RepoStatus cache_status = cacheStatus(info);
 704
 705       if ( cache_status.checksum() == raw_metadata_status.checksum() )
 706       {
 707         MIL << info.alias() << " cache is up to date with metadata." << endl;
 708         if ( policy == BuildIfNeeded ) {
 709           return;
 710         }
 711         else {
 712           MIL << info.alias() << " cache rebuild is forced" << endl;
 713         }
 714       }
 715
 716       needs_cleaning = true;
 717     }
 718
 719     ProgressData progress(100);
 720     callback::SendReport<ProgressReport> report;
 721     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
 722     progress.name(str::form(_("Building repository '%s' cache"), info.name().c_str()));
 723     progress.toMin();
 724
 725     if (needs_cleaning)
 726     {
 727       cleanCache(info);
 728     }
 729
 730     MIL << info.alias() << " building cache..." << endl;
 731
 732     Pathname base = solv_path_for_repoinfo( _pimpl->options, info);
 733     filesystem::assert_dir(base);
 734     Pathname solvfile = base / "solv";
 735
 736     // do we have type?
 737     repo::RepoType repokind = info.type();
 738
 739     // if the type is unknown, try probing.
 740     switch ( repokind.toEnum() )
 741     {
 742       case RepoType::NONE_e:
 743         // unknown, probe the local metadata
 744         repokind = probe(rawpath.asUrl());
 745       break;
 746       default:
 747       break;
 748     }
 749
 750     MIL << "repo type is " << repokind << endl;
 751
 752     switch ( repokind.toEnum() )
 753     {
 754       case RepoType::RPMMD_e :
 755       case RepoType::YAST2_e :
 756       case RepoType::RPMPLAINDIR_e :
 757       {
 758         // Take care we unlink the solvfile on exception
 759         ManagedFile guard( solvfile, filesystem::unlink );
 760
 761         ostringstream cmd;
 762         if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 763         {
 764           cmd << str::form( "repo2solv.sh \"%s\" > \"%s\"", info.baseUrlsBegin()->getPathName().c_str(), solvfile.c_str() );
 765         } else
 766           cmd << str::form( "repo2solv.sh \"%s\" > \"%s\"", rawpath.c_str(), solvfile.c_str() );
 767
 768         MIL << "Executing: " << cmd.str() << endl;
 769         ExternalProgram prog( cmd.str(), ExternalProgram::Stderr_To_Stdout );
 770
 771         cmd << endl;
 772         for ( string output( prog.receiveLine() ); output.length(); output = prog.receiveLine() ) {
 773           WAR << "  " << output;
 774           cmd << "     " << output;
 775         }
 776
 777         int ret = prog.close();
 778         if ( ret != 0 )
 779         {
 780           RepoException ex(str::form("Failed to cache repo (%d).", ret));
 781           ex.remember( cmd.str() );
 782           ZYPP_THROW(ex);
 783         }
 784
 785         // We keep it.
 786         guard.resetDispose();
 787       }
 788       break;
 789       default:
 790         ZYPP_THROW(RepoUnknownTypeException("Unhandled repository type"));
 791       break;
 792     }
 793 #if 0
 794     switch ( repokind.toEnum() )
 795     {
 796       case RepoType::RPMMD_e :
 797       if (0)
 798       {
 799         CombinedProgressData subprogrcv( progress, 100);
 800         parser::yum::RepoParser parser(id, store, parser::yum::RepoParserOpts(), subprogrcv);
 801         parser.parse(rawpath);
 802           // no error
 803       }
 804       break;
 805       case RepoType::YAST2_e :
 806       if (0)
 807       {
 808         CombinedProgressData subprogrcv( progress, 100);
 809         parser::susetags::RepoParser parser(id, store, subprogrcv);
 810         parser.parse(rawpath);
 811         // no error
 812       }
 813       break;
 814
 815       default:
 816         ZYPP_THROW(RepoUnknownTypeException());
 817     }
 818 #endif
 819     // update timestamp and checksum
 820     //store.updateRepositoryStatus(id, raw_metadata_status);
 821     setCacheStatus(info, raw_metadata_status);
 822     MIL << "Commit cache.." << endl;
 823     //store.commit();
 824     //progress.toMax();
 825   }
 826
 827   ////////////////////////////////////////////////////////////////////////////
 828
 829   repo::RepoType RepoManager::probe( const Url &url ) const
 830   {
 831     if ( url.getScheme() == "dir" && ! PathInfo( url.getPathName() ).isDir() )
 832     {
 833       // Handle non existing local directory in advance, as
 834       // MediaSetAccess does not support it.
 835       return repo::RepoType::NONE;
 836     }
 837
 838     try
 839     {
 840       MediaSetAccess access(url);
 841       if ( access.doesFileExist("/repodata/repomd.xml") )
 842         return repo::RepoType::RPMMD;
 843       if ( access.doesFileExist("/content") )
 844         return repo::RepoType::YAST2;
 845
 846       // if it is a local url of type dir
 847       if ( (! media::MediaManager::downloads(url)) && ( url.getScheme() == "dir" ) )
 848       {
 849         Pathname path = Pathname(url.getPathName());
 850         if ( PathInfo(path).isDir() )
 851         {
 852           // allow empty dirs for now
 853           return repo::RepoType::RPMPLAINDIR;
 854         }
 855       }
 856     }
 857     catch ( const media::MediaException &e )
 858     {
 859       ZYPP_CAUGHT(e);
 860       RepoException enew("Error trying to read from " + url.asString());
 861       enew.remember(e);
 862       ZYPP_THROW(enew);
 863     }
 864     catch ( const Exception &e )
 865     {
 866       ZYPP_CAUGHT(e);
 867       Exception enew("Unknown error reading from " + url.asString());
 868       enew.remember(e);
 869       ZYPP_THROW(enew);
 870     }
 871
 872     return repo::RepoType::NONE;
 873   }
 874
 875   ////////////////////////////////////////////////////////////////////////////
 876
 877   void RepoManager::cleanCache( const RepoInfo &info,
 878                                 const ProgressData::ReceiverFnc & progressrcv )
 879   {
 880     ProgressData progress(100);
 881     progress.sendTo(progressrcv);
 882     progress.toMin();
 883
 884     filesystem::recursive_rmdir(solv_path_for_repoinfo(_pimpl->options, info));
 885
 886     progress.toMax();
 887   }
 888
 889   ////////////////////////////////////////////////////////////////////////////
 890
 891   bool RepoManager::isCached( const RepoInfo &info ) const
 892   {
 893     return PathInfo(solv_path_for_repoinfo( _pimpl->options, info ) / "solv").isExist();
 894   }
 895
 896   RepoStatus RepoManager::cacheStatus( const RepoInfo &info ) const
 897   {
 898
 899     Pathname cookiefile = solv_path_for_repoinfo(_pimpl->options, info) / "cookie";
 900
 901     return RepoStatus::fromCookieFile(cookiefile);
 902   }
 903
 904   void RepoManager::setCacheStatus( const RepoInfo &info, const RepoStatus &status )
 905   {
 906     Pathname base = solv_path_for_repoinfo(_pimpl->options, info);
 907     filesystem::assert_dir(base);
 908     Pathname cookiefile = base / "cookie";
 909
 910     status.saveToCookieFile(cookiefile);
 911   }
 912
 913   void RepoManager::loadFromCache( const RepoInfo & info,
 914                                    const ProgressData::ReceiverFnc & progressrcv )
 915   {
 916     assert_alias(info);
 917     Pathname solvfile = solv_path_for_repoinfo(_pimpl->options, info) / "solv";
 918
 919     if ( ! PathInfo(solvfile).isExist() )
 920       ZYPP_THROW(RepoNotCachedException(info));
 921
 922     try
 923     {
 924       sat::Pool::instance().addRepoSolv( solvfile, info );
 925     }
 926     catch ( const Exception & exp )
 927     {
 928       ZYPP_CAUGHT( exp );
 929       MIL << "Try to handle exception by rebuilding the solv-file" << endl;
 930       cleanCache( info, progressrcv );
 931       buildCache( info, BuildIfNeeded, progressrcv );
 932
 933       sat::Pool::instance().addRepoSolv( solvfile, info );
 934     }
 935   }
 936
 937   ////////////////////////////////////////////////////////////////////////////
 938
 939   /**
 940    * Generate a non existing filename in a directory, using a base
 941    * name. For example if a directory contains 3 files
 942    *
 943    * |-- bar
 944    * |-- foo
 945    * `-- moo
 946    *
 947    * If you try to generate a unique filename for this directory,
 948    * based on "ruu" you will get "ruu", but if you use the base
 949    * "foo" you will get "foo_1"
 950    *
 951    * \param dir Directory where the file needs to be unique
 952    * \param basefilename string to base the filename on.
 953    */
 954   static Pathname generate_non_existing_name( const Pathname &dir,
 955                                               const std::string &basefilename )
 956   {
 957     string final_filename = basefilename;
 958     int counter = 1;
 959     while ( PathInfo(dir + final_filename).isExist() )
 960     {
 961       final_filename = basefilename + "_" + str::numstring(counter);
 962       counter++;
 963     }
 964     return dir + Pathname(final_filename);
 965   }
 966
 967   ////////////////////////////////////////////////////////////////////////////
 968
 969   /**
 970    * \short Generate a related filename from a repo info
 971    *
 972    * From a repo info, it will try to use the alias as a filename
 973    * escaping it if necessary. Other fallbacks can be added to
 974    * this function in case there is no way to use the alias
 975    */
 976   static std::string generate_filename( const RepoInfo &info )
 977   {
 978     std::string fnd="/";
 979     std::string rep="_";
 980     std::string filename = info.alias();
 981     // replace slashes with underscores
 982     size_t pos = filename.find(fnd);
 983     while(pos!=string::npos)
 984     {
 985       filename.replace(pos,fnd.length(),rep);
 986       pos = filename.find(fnd,pos+rep.length());
 987     }
 988     filename = Pathname(filename).extend(".repo").asString();
 989     MIL << "generating filename for repo [" << info.alias() << "] : '" << filename << "'" << endl;
 990     return filename;
 991   }
 992
 993
 994   ////////////////////////////////////////////////////////////////////////////
 995
 996   void RepoManager::addRepository( const RepoInfo &info,
 997                                    const ProgressData::ReceiverFnc & progressrcv )
 998   {
 999     assert_alias(info);
1000
1001     ProgressData progress(100);
1002     callback::SendReport<ProgressReport> report;
1003     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
1004     progress.name(str::form(_("Adding repository '%s'"), info.name().c_str()));
1005     progress.toMin();
1006
1007     std::list<RepoInfo> repos = knownRepositories();
1008     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1009           it != repos.end();
1010           ++it )
1011     {
1012       if ( info.alias() == (*it).alias() )
1013         ZYPP_THROW(RepoAlreadyExistsException(info));
1014     }
1015
1016     RepoInfo tosave = info;
1017
1018     // check the first url for now
1019     if ( _pimpl->options.probe )
1020     {
1021       DBG << "unknown repository type, probing" << endl;
1022
1023       RepoType probedtype;
1024       probedtype = probe(*tosave.baseUrlsBegin());
1025       if ( tosave.baseUrlsSize() > 0 )
1026       {
1027         if ( probedtype == RepoType::NONE )
1028           ZYPP_THROW(RepoUnknownTypeException());
1029         else
1030           tosave.setType(probedtype);
1031       }
1032     }
1033
1034     progress.set(50);
1035
1036     // assert the directory exists
1037     filesystem::assert_dir(_pimpl->options.knownReposPath);
1038
1039     Pathname repofile = generate_non_existing_name(_pimpl->options.knownReposPath,
1040                                                     generate_filename(tosave));
1041     // now we have a filename that does not exists
1042     MIL << "Saving repo in " << repofile << endl;
1043
1044     std::ofstream file(repofile.c_str());
1045     if (!file) {
1046       ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
1047     }
1048
1049     tosave.dumpRepoOn(file);
1050     progress.toMax();
1051     MIL << "done" << endl;
1052   }
1053
1054   void RepoManager::addRepositories( const Url &url,
1055                                      const ProgressData::ReceiverFnc & progressrcv )
1056   {
1057     std::list<RepoInfo> knownrepos = knownRepositories();
1058     std::list<RepoInfo> repos = readRepoFile(url);
1059     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1060           it != repos.end();
1061           ++it )
1062     {
1063       // look if the alias is in the known repos.
1064       for ( std::list<RepoInfo>::const_iterator kit = knownrepos.begin();
1065           kit != knownrepos.end();
1066           ++kit )
1067       {
1068         if ( (*it).alias() == (*kit).alias() )
1069         {
1070           ERR << "To be added repo " << (*it).alias() << " conflicts with existing repo " << (*kit).alias() << endl;
1071           ZYPP_THROW(RepoAlreadyExistsException(*it));
1072         }
1073       }
1074     }
1075
1076     string filename = Pathname(url.getPathName()).basename();
1077
1078     if ( filename == Pathname() )
1079       ZYPP_THROW(RepoException("Invalid repo file name at " + url.asString() ));
1080
1081     // assert the directory exists
1082     filesystem::assert_dir(_pimpl->options.knownReposPath);
1083
1084     Pathname repofile = generate_non_existing_name(_pimpl->options.knownReposPath, filename);
1085     // now we have a filename that does not exists
1086     MIL << "Saving " << repos.size() << " repo" << ( repos.size() ? "s" : "" ) << " in " << repofile << endl;
1087
1088     std::ofstream file(repofile.c_str());
1089     if (!file) {
1090       ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
1091     }
1092
1093     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1094           it != repos.end();
1095           ++it )
1096     {
1097       MIL << "Saving " << (*it).alias() << endl;
1098       (*it).dumpRepoOn(file);
1099     }
1100     MIL << "done" << endl;
1101   }
1102
1103   ////////////////////////////////////////////////////////////////////////////
1104
1105   void RepoManager::removeRepository( const RepoInfo & info,
1106                                       const ProgressData::ReceiverFnc & progressrcv)
1107   {
1108     ProgressData progress;
1109     callback::SendReport<ProgressReport> report;
1110     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
1111     progress.name(str::form(_("Removing repository '%s'"), info.name().c_str()));
1112
1113     MIL << "Going to delete repo " << info.alias() << endl;
1114
1115     std::list<RepoInfo> repos = knownRepositories();
1116     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1117           it != repos.end();
1118           ++it )
1119     {
1120       // they can be the same only if the provided is empty, that means
1121       // the provided repo has no alias
1122       // then skip
1123       if ( (!info.alias().empty()) && ( info.alias() != (*it).alias() ) )
1124         continue;
1125
1126       // TODO match by url
1127
1128       // we have a matcing repository, now we need to know
1129       // where it does come from.
1130       RepoInfo todelete = *it;
1131       if (todelete.filepath().empty())
1132       {
1133         ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
1134       }
1135       else
1136       {
1137         // figure how many repos are there in the file:
1138         std::list<RepoInfo> filerepos = repositories_in_file(todelete.filepath());
1139         if ( (filerepos.size() == 1) && ( filerepos.front().alias() == todelete.alias() ) )
1140         {
1141           // easy, only this one, just delete the file
1142           if ( filesystem::unlink(todelete.filepath()) != 0 )
1143           {
1144             ZYPP_THROW(RepoException("Can't delete " + todelete.filepath().asString()));
1145           }
1146           MIL << todelete.alias() << " sucessfully deleted." << endl;
1147         }
1148         else
1149         {
1150           // there are more repos in the same file
1151           // write them back except the deleted one.
1152           //TmpFile tmp;
1153           //std::ofstream file(tmp.path().c_str());
1154
1155           // assert the directory exists
1156           filesystem::assert_dir(todelete.filepath().dirname());
1157
1158           std::ofstream file(todelete.filepath().c_str());
1159           if (!file) {
1160             //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
1161             ZYPP_THROW (Exception( "Can't open " + todelete.filepath().asString() ) );
1162           }
1163           for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
1164                 fit != filerepos.end();
1165                 ++fit )
1166           {
1167             if ( (*fit).alias() != todelete.alias() )
1168               (*fit).dumpRepoOn(file);
1169           }
1170         }
1171
1172         CombinedProgressData subprogrcv(progress, 70);
1173         CombinedProgressData cleansubprogrcv(progress, 30);
1174         // now delete it from cache
1175         if ( isCached(todelete) )
1176           cleanCache( todelete, subprogrcv);
1177         // now delete metadata (#301037)
1178         cleanMetadata( todelete, cleansubprogrcv);
1179         MIL << todelete.alias() << " sucessfully deleted." << endl;
1180         return;
1181       } // else filepath is empty
1182
1183     }
1184     // should not be reached on a sucess workflow
1185     ZYPP_THROW(RepoNotFoundException(info));
1186   }
1187
1188   ////////////////////////////////////////////////////////////////////////////
1189
1190   void RepoManager::modifyRepository( const std::string &alias,
1191                                       const RepoInfo & newinfo,
1192                                       const ProgressData::ReceiverFnc & progressrcv )
1193   {
1194     RepoInfo toedit = getRepositoryInfo(alias);
1195
1196     // check if the new alias already exists when renaming the repo
1197     if (alias != newinfo.alias())
1198     {
1199       std::list<RepoInfo> repos = knownRepositories();
1200       for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1201             it != repos.end();
1202             ++it )
1203       {
1204         if ( newinfo.alias() == (*it).alias() )
1205           ZYPP_THROW(RepoAlreadyExistsException(newinfo));
1206       }
1207     }
1208
1209     if (toedit.filepath().empty())
1210     {
1211       ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
1212     }
1213     else
1214     {
1215       // figure how many repos are there in the file:
1216       std::list<RepoInfo> filerepos = repositories_in_file(toedit.filepath());
1217
1218       // there are more repos in the same file
1219       // write them back except the deleted one.
1220       //TmpFile tmp;
1221       //std::ofstream file(tmp.path().c_str());
1222
1223       // assert the directory exists
1224       filesystem::assert_dir(toedit.filepath().dirname());
1225
1226       std::ofstream file(toedit.filepath().c_str());
1227       if (!file) {
1228         //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
1229         ZYPP_THROW (Exception( "Can't open " + toedit.filepath().asString() ) );
1230       }
1231       for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
1232             fit != filerepos.end();
1233             ++fit )
1234       {
1235           // if the alias is different, dump the original
1236           // if it is the same, dump the provided one
1237           if ( (*fit).alias() != toedit.alias() )
1238             (*fit).dumpRepoOn(file);
1239           else
1240             newinfo.dumpRepoOn(file);
1241       }
1242     }
1243   }
1244
1245   ////////////////////////////////////////////////////////////////////////////
1246
1247   RepoInfo RepoManager::getRepositoryInfo( const std::string &alias,
1248                                            const ProgressData::ReceiverFnc & progressrcv )
1249   {
1250     std::list<RepoInfo> repos = knownRepositories();
1251     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1252           it != repos.end();
1253           ++it )
1254     {
1255       if ( (*it).alias() == alias )
1256         return *it;
1257     }
1258     RepoInfo info;
1259     info.setAlias(info.alias());
1260     ZYPP_THROW(RepoNotFoundException(info));
1261   }
1262
1263   ////////////////////////////////////////////////////////////////////////////
1264
1265   RepoInfo RepoManager::getRepositoryInfo( const Url & url,
1266                                            const url::ViewOption & urlview,
1267                                            const ProgressData::ReceiverFnc & progressrcv )
1268   {
1269     std::list<RepoInfo> repos = knownRepositories();
1270     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1271           it != repos.end();
1272           ++it )
1273     {
1274       for(RepoInfo::urls_const_iterator urlit = (*it).baseUrlsBegin();
1275           urlit != (*it).baseUrlsEnd();
1276           ++urlit)
1277       {
1278         if ((*urlit).asString(urlview) == url.asString(urlview))
1279           return *it;
1280       }
1281     }
1282     RepoInfo info;
1283     info.setAlias(info.alias());
1284     info.setBaseUrl(url);
1285     ZYPP_THROW(RepoNotFoundException(info));
1286   }
1287
1288   ////////////////////////////////////////////////////////////////////////////
1289
1290   std::ostream & operator<<( std::ostream & str, const RepoManager & obj )
1291   {
1292     return str << *obj._pimpl;
1293   }
1294
1295   /////////////////////////////////////////////////////////////////
1296 } // namespace zypp
1297 ///////////////////////////////////////////////////////////////////