zypp/RepoManager.cc

   1 /*---------------------------------------------------------------------\
   2 |                          ____ _   __ __ ___                          |
   3 |                         |__  / \ / / . \ . \                         |
   4 |                           / / \ V /|  _/  _/                         |
   5 |                          / /__ | | | | | |                           |
   6 |                         /_____||_| |_| |_|                           |
   7 |                                                                      |
   8 \---------------------------------------------------------------------*/
   9 /** \file       zypp/RepoManager.cc
  10  *
  11 */
  12
  13 #include <cstdlib>
  14 #include <iostream>
  15 #include <fstream>
  16 #include <sstream>
  17 #include <list>
  18 #include <map>
  19 #include <algorithm>
  20 #include "zypp/base/InputStream.h"
  21 #include "zypp/base/Logger.h"
  22 #include "zypp/base/Gettext.h"
  23 #include "zypp/base/Function.h"
  24 #include "zypp/base/Regex.h"
  25 #include "zypp/PathInfo.h"
  26 #include "zypp/TmpPath.h"
  27
  28 #include "zypp/repo/RepoException.h"
  29 #include "zypp/RepoManager.h"
  30
  31 #include "zypp/media/MediaManager.h"
  32 #include "zypp/MediaSetAccess.h"
  33 #include "zypp/ExternalProgram.h"
  34 #include "zypp/ManagedFile.h"
  35
  36 #include "zypp/parser/RepoFileReader.h"
  37 #include "zypp/repo/yum/Downloader.h"
  38 #include "zypp/parser/yum/RepoParser.h"
  39 #include "zypp/repo/susetags/Downloader.h"
  40 #include "zypp/parser/susetags/RepoParser.h"
  41
  42 #include "zypp/ZYppCallbacks.h"
  43
  44 #include "sat/Pool.h"
  45 #include "satsolver/pool.h"
  46 #include "satsolver/repo.h"
  47 #include "satsolver/repo_solv.h"
  48
  49 using namespace std;
  50 using namespace zypp;
  51 using namespace zypp::repo;
  52 using namespace zypp::filesystem;
  53
  54 using namespace zypp::repo;
  55
  56 ///////////////////////////////////////////////////////////////////
  57 namespace zypp
  58 { /////////////////////////////////////////////////////////////////
  59
  60   ///////////////////////////////////////////////////////////////////
  61   //
  62   //    CLASS NAME : RepoManagerOptions
  63   //
  64   ///////////////////////////////////////////////////////////////////
  65
  66   RepoManagerOptions::RepoManagerOptions()
  67   {
  68     repoCachePath    = ZConfig::instance().repoCachePath();
  69     repoRawCachePath = ZConfig::instance().repoMetadataPath();
  70     repoPackagesCachePath = ZConfig::instance().repoPackagesPath();
  71     knownReposPath   = ZConfig::instance().knownReposPath();
  72     probe            = ZConfig::instance().repo_add_probe();
  73   }
  74
  75   ////////////////////////////////////////////////////////////////////////////
  76
  77   /**
  78     * \short Simple callback to collect the results
  79     *
  80     * Classes like RepoFileParser call the callback
  81     * once per each repo in a file.
  82     *
  83     * Passing this functor as callback, you can collect
  84     * all resuls at the end, without dealing with async
  85     * code.
  86     */
  87     struct RepoCollector
  88     {
  89       RepoCollector()
  90       {}
  91
  92       ~RepoCollector()
  93       {}
  94
  95       bool collect( const RepoInfo &repo )
  96       {
  97         repos.push_back(repo);
  98         return true;
  99       }
 100
 101       RepoInfoList repos;
 102     };
 103
 104   ////////////////////////////////////////////////////////////////////////////
 105
 106   /**
 107    * Reads RepoInfo's from a repo file.
 108    *
 109    * \param file pathname of the file to read.
 110    */
 111   static std::list<RepoInfo> repositories_in_file( const Pathname & file )
 112   {
 113     MIL << "repo file: " << file << endl;
 114     RepoCollector collector;
 115     parser::RepoFileReader parser( file, bind( &RepoCollector::collect, &collector, _1 ) );
 116     return collector.repos;
 117   }
 118
 119   ////////////////////////////////////////////////////////////////////////////
 120
 121   std::list<RepoInfo> readRepoFile(const Url & repo_file)
 122    {
 123      // no interface to download a specific file, using workaround:
 124      //! \todo add MediaManager::provideFile(Url file_url) to easily access any file URLs? (no need for media access id or media_nr)
 125      Url url(repo_file);
 126      Pathname path(url.getPathName());
 127      url.setPathName ("/");
 128      MediaSetAccess access(url);
 129      Pathname local = access.provideFile(path);
 130
 131      DBG << "reading repo file " << repo_file << ", local path: " << local << endl;
 132
 133      return repositories_in_file(local);
 134    }
 135
 136   ////////////////////////////////////////////////////////////////////////////
 137
 138   /**
 139    * \short List of RepoInfo's from a directory
 140    *
 141    * Goes trough every file ending with ".repo" in a directory and adds all
 142    * RepoInfo's contained in that file.
 143    *
 144    * \param dir pathname of the directory to read.
 145    */
 146   static std::list<RepoInfo> repositories_in_dir( const Pathname &dir )
 147   {
 148     MIL << "directory " << dir << endl;
 149     list<RepoInfo> repos;
 150     list<Pathname> entries;
 151     if ( filesystem::readdir( entries, Pathname(dir), false ) != 0 )
 152       ZYPP_THROW(Exception("failed to read directory"));
 153
 154     str::regex allowedRepoExt("^\\.repo(_[0-9]+)?$");
 155     for ( list<Pathname>::const_iterator it = entries.begin(); it != entries.end(); ++it )
 156     {
 157       if (str::regex_match(it->extension(), allowedRepoExt))
 158       {
 159         list<RepoInfo> tmp = repositories_in_file( *it );
 160         repos.insert( repos.end(), tmp.begin(), tmp.end() );
 161
 162         //std::copy( collector.repos.begin(), collector.repos.end(), std::back_inserter(repos));
 163         //MIL << "ok" << endl;
 164       }
 165     }
 166     return repos;
 167   }
 168
 169   ////////////////////////////////////////////////////////////////////////////
 170
 171   static void assert_alias( const RepoInfo &info )
 172   {
 173     if (info.alias().empty())
 174         ZYPP_THROW(RepoNoAliasException());
 175   }
 176
 177   ////////////////////////////////////////////////////////////////////////////
 178
 179   static void assert_urls( const RepoInfo &info )
 180   {
 181     if (info.baseUrlsEmpty())
 182         ZYPP_THROW(RepoNoUrlException());
 183   }
 184
 185   ////////////////////////////////////////////////////////////////////////////
 186
 187   /**
 188    * \short Calculates the raw cache path for a repository
 189    */
 190   static Pathname rawcache_path_for_repoinfo( const RepoManagerOptions &opt, const RepoInfo &info )
 191   {
 192     assert_alias(info);
 193     return opt.repoRawCachePath / info.escaped_alias();
 194   }
 195
 196   /**
 197    * \short Calculates the packages cache path for a repository
 198    */
 199   static Pathname packagescache_path_for_repoinfo( const RepoManagerOptions &opt, const RepoInfo &info )
 200   {
 201     assert_alias(info);
 202     return opt.repoPackagesCachePath / info.escaped_alias();
 203   }
 204
 205   ///////////////////////////////////////////////////////////////////
 206   //
 207   //    CLASS NAME : RepoManager::Impl
 208   //
 209   ///////////////////////////////////////////////////////////////////
 210
 211   /**
 212    * \short RepoManager implementation.
 213    */
 214   struct RepoManager::Impl
 215   {
 216     Impl( const RepoManagerOptions &opt )
 217       : options(opt)
 218     {
 219
 220     }
 221
 222     Impl()
 223     {
 224
 225     }
 226
 227     RepoManagerOptions options;
 228
 229     map<string, RepoInfo> _repoinfos;
 230
 231   public:
 232     /** Offer default Impl. */
 233     static shared_ptr<Impl> nullimpl()
 234     {
 235       static shared_ptr<Impl> _nullimpl( new Impl );
 236       return _nullimpl;
 237     }
 238
 239   private:
 240     friend Impl * rwcowClone<Impl>( const Impl * rhs );
 241     /** clone for RWCOW_pointer */
 242     Impl * clone() const
 243     { return new Impl( *this ); }
 244   };
 245
 246   ///////////////////////////////////////////////////////////////////
 247
 248   /** \relates RepoManager::Impl Stream output */
 249   inline std::ostream & operator<<( std::ostream & str, const RepoManager::Impl & obj )
 250   {
 251     return str << "RepoManager::Impl";
 252   }
 253
 254   ///////////////////////////////////////////////////////////////////
 255   //
 256   //    CLASS NAME : RepoManager
 257   //
 258   ///////////////////////////////////////////////////////////////////
 259
 260   RepoManager::RepoManager( const RepoManagerOptions &opt )
 261   : _pimpl( new Impl(opt) )
 262   {}
 263
 264   ////////////////////////////////////////////////////////////////////////////
 265
 266   RepoManager::~RepoManager()
 267   {}
 268
 269   ////////////////////////////////////////////////////////////////////////////
 270
 271   std::list<RepoInfo> RepoManager::knownRepositories() const
 272   {
 273     MIL << endl;
 274
 275     if ( PathInfo(_pimpl->options.knownReposPath).isExist() )
 276     {
 277       RepoInfoList repos = repositories_in_dir(_pimpl->options.knownReposPath);
 278       for ( RepoInfoList::iterator it = repos.begin();
 279             it != repos.end();
 280             ++it )
 281       {
 282         // set the metadata path for the repo
 283         Pathname metadata_path = rawcache_path_for_repoinfo(_pimpl->options, (*it));
 284         (*it).setMetadataPath(metadata_path);
 285
 286         // set the downloaded packages path for the repo
 287         Pathname packages_path = packagescache_path_for_repoinfo(_pimpl->options, (*it));
 288         (*it).setPackagesPath(packages_path);
 289       }
 290       return repos;
 291     }
 292     else
 293       return std::list<RepoInfo>();
 294
 295     MIL << endl;
 296   }
 297
 298   ////////////////////////////////////////////////////////////////////////////
 299
 300   Pathname RepoManager::metadataPath( const RepoInfo &info ) const
 301   {
 302     return rawcache_path_for_repoinfo(_pimpl->options, info );
 303   }
 304
 305   ////////////////////////////////////////////////////////////////////////////
 306
 307   RepoStatus RepoManager::metadataStatus( const RepoInfo &info ) const
 308   {
 309     Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 310     RepoType repokind = info.type();
 311     RepoStatus status;
 312
 313     switch ( repokind.toEnum() )
 314     {
 315       case RepoType::NONE_e:
 316       // unknown, probe the local metadata
 317         repokind = probe(rawpath.asUrl());
 318       break;
 319       default:
 320       break;
 321     }
 322
 323     switch ( repokind.toEnum() )
 324     {
 325       case RepoType::RPMMD_e :
 326       {
 327         status = RepoStatus( rawpath + "/repodata/repomd.xml");
 328       }
 329       break;
 330
 331       case RepoType::YAST2_e :
 332       {
 333         status = RepoStatus( rawpath + "/content") && (RepoStatus( rawpath + "/media.1/media"));
 334       }
 335       break;
 336
 337       case RepoType::RPMPLAINDIR_e :
 338       {
 339         if ( PathInfo(Pathname(rawpath + "/cookie")).isExist() )
 340           status = RepoStatus( rawpath + "/cookie");
 341       }
 342       break;
 343
 344       case RepoType::NONE_e :
 345         // Return default RepoStatus in case of RepoType::NONE
 346         // indicating it should be created?
 347         // ZYPP_THROW(RepoUnknownTypeException());
 348         break;
 349     }
 350     return status;
 351   }
 352
 353   void RepoManager::touchIndexFile(const RepoInfo & info)
 354   {
 355     Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 356
 357     RepoType repokind = info.type();
 358     if ( repokind.toEnum() == RepoType::NONE_e )
 359       // unknown, probe the local metadata
 360       repokind = probe(rawpath.asUrl());
 361     // if still unknown, just return
 362     if (repokind == RepoType::NONE_e)
 363       return;
 364
 365     Pathname p;
 366     switch ( repokind.toEnum() )
 367     {
 368       case RepoType::RPMMD_e :
 369         p = Pathname(rawpath + "/repodata/repomd.xml");
 370         break;
 371
 372       case RepoType::YAST2_e :
 373         p = Pathname(rawpath + "/content");
 374         break;
 375
 376       case RepoType::RPMPLAINDIR_e :
 377         p = Pathname(rawpath + "/cookie");
 378         break;
 379
 380       case RepoType::NONE_e :
 381       default:
 382         break;
 383     }
 384
 385     // touch the file, ignore error (they are logged anyway)
 386     filesystem::touch(p);
 387   }
 388
 389   RepoManager::RefreshCheckStatus RepoManager::checkIfToRefreshMetadata(
 390                                               const RepoInfo &info,
 391                                               const Url &url,
 392                                               RawMetadataRefreshPolicy policy )
 393   {
 394     assert_alias(info);
 395
 396     RepoStatus oldstatus;
 397     RepoStatus newstatus;
 398
 399     try
 400     {
 401       MIL << "Going to try to check whether refresh is needed for " << url << endl;
 402
 403       repo::RepoType repokind = info.type();
 404
 405       // if the type is unknown, try probing.
 406       switch ( repokind.toEnum() )
 407       {
 408         case RepoType::NONE_e:
 409           // unknown, probe it
 410           repokind = probe(url);
 411         break;
 412         default:
 413         break;
 414       }
 415
 416       Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 417       filesystem::assert_dir(rawpath);
 418       oldstatus = metadataStatus(info);
 419
 420       // now we've got the old (cached) status, we can decide repo.refresh.delay
 421       if (policy != RefreshForced)
 422       {
 423         // difference in seconds
 424         double diff = difftime(
 425           (Date::ValueType)Date::now(),
 426           (Date::ValueType)oldstatus.timestamp()) / 60;
 427
 428         DBG << "oldstatus: " << (Date::ValueType)oldstatus.timestamp() << endl;
 429         DBG << "current time: " << (Date::ValueType)Date::now() << endl;
 430         DBG << "last refresh = " << diff << " minutes ago" << endl;
 431
 432         if (diff < ZConfig::instance().repo_refresh_delay())
 433         {
 434           MIL << "Repository '" << info.alias()
 435               << "' has been refreshed less than repo.refresh.delay ("
 436               << ZConfig::instance().repo_refresh_delay()
 437               << ") minutes ago. Advising to skip refresh" << endl;
 438           return REPO_DELAYED;
 439         }
 440       }
 441
 442       // create temp dir as sibling of rawpath
 443       filesystem::TmpDir tmpdir( filesystem::TmpDir::makeSibling( rawpath ) );
 444
 445       if ( ( repokind.toEnum() == RepoType::RPMMD_e ) ||
 446            ( repokind.toEnum() == RepoType::YAST2_e ) )
 447       {
 448         MediaSetAccess media(url);
 449         shared_ptr<repo::Downloader> downloader_ptr;
 450
 451         if ( repokind.toEnum() == RepoType::RPMMD_e )
 452           downloader_ptr.reset(new yum::Downloader(info.path()));
 453         else
 454           downloader_ptr.reset( new susetags::Downloader(info.path()));
 455
 456         RepoStatus newstatus = downloader_ptr->status(media);
 457         bool refresh = false;
 458         if ( oldstatus.checksum() == newstatus.checksum() )
 459         {
 460           MIL << "repo has not changed" << endl;
 461           if ( policy == RefreshForced )
 462           {
 463             MIL << "refresh set to forced" << endl;
 464             refresh = true;
 465           }
 466         }
 467         else
 468         {
 469           MIL << "repo has changed, going to refresh" << endl;
 470           refresh = true;
 471         }
 472
 473         if (!refresh)
 474           touchIndexFile(info);
 475
 476         return refresh ? REFRESH_NEEDED : REPO_UP_TO_DATE;
 477       }
 478 #if 0
 479       else if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 480       {
 481         RepoStatus newstatus = parser::plaindir::dirStatus(url.getPathName());
 482         bool refresh = false;
 483         if ( oldstatus.checksum() == newstatus.checksum() )
 484         {
 485           MIL << "repo has not changed" << endl;
 486           if ( policy == RefreshForced )
 487           {
 488             MIL << "refresh set to forced" << endl;
 489             refresh = true;
 490           }
 491         }
 492         else
 493         {
 494           MIL << "repo has changed, going to refresh" << endl;
 495           refresh = true;
 496         }
 497
 498         if (!refresh)
 499           touchIndexFile(info);
 500
 501         return refresh;
 502       }
 503 #endif
 504       else
 505       {
 506         ZYPP_THROW(RepoUnknownTypeException());
 507       }
 508     }
 509     catch ( const Exception &e )
 510     {
 511       ZYPP_CAUGHT(e);
 512       ERR << "refresh check failed for " << url << endl;
 513       ZYPP_RETHROW(e);
 514     }
 515
 516     return REFRESH_NEEDED; // default
 517   }
 518
 519   void RepoManager::refreshMetadata( const RepoInfo &info,
 520                                      RawMetadataRefreshPolicy policy,
 521                                      const ProgressData::ReceiverFnc & progress )
 522   {
 523     assert_alias(info);
 524     assert_urls(info);
 525
 526     // we will throw this later if no URL checks out fine
 527     RepoException rexception(_("Valid metadata not found at specified URL(s)"));
 528
 529     // try urls one by one
 530     for ( RepoInfo::urls_const_iterator it = info.baseUrlsBegin(); it != info.baseUrlsEnd(); ++it )
 531     {
 532       try
 533       {
 534         Url url(*it);
 535
 536         // check whether to refresh metadata
 537         // if the check fails for this url, it throws, so another url will be checked
 538         if (checkIfToRefreshMetadata(info, url, policy)!=REFRESH_NEEDED)
 539           return;
 540
 541         MIL << "Going to refresh metadata from " << url << endl;
 542
 543         repo::RepoType repokind = info.type();
 544
 545         // if the type is unknown, try probing.
 546         switch ( repokind.toEnum() )
 547         {
 548           case RepoType::NONE_e:
 549             // unknown, probe it
 550             repokind = probe(*it);
 551
 552             if (repokind.toEnum() != RepoType::NONE_e)
 553             {
 554               //save probed type only for repos in system
 555               std::list<RepoInfo> repos = knownRepositories();
 556               for ( std::list<RepoInfo>::const_iterator it = repos.begin();
 557                    it != repos.end(); ++it )
 558               {
 559                 if ( info.alias() == (*it).alias() )
 560                 {
 561                   RepoInfo modifiedrepo = info;
 562                   modifiedrepo.setType(repokind);
 563                   modifyRepository(info.alias(),modifiedrepo);
 564                 }
 565               }
 566             }
 567           break;
 568           default:
 569           break;
 570         }
 571
 572         Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 573         filesystem::assert_dir(rawpath);
 574
 575         // create temp dir as sibling of rawpath
 576         filesystem::TmpDir tmpdir( filesystem::TmpDir::makeSibling( rawpath ) );
 577
 578         if ( ( repokind.toEnum() == RepoType::RPMMD_e ) ||
 579              ( repokind.toEnum() == RepoType::YAST2_e ) )
 580         {
 581           MediaSetAccess media(url);
 582           shared_ptr<repo::Downloader> downloader_ptr;
 583
 584           if ( repokind.toEnum() == RepoType::RPMMD_e )
 585             downloader_ptr.reset(new yum::Downloader(info.path()));
 586           else
 587             downloader_ptr.reset( new susetags::Downloader(info.path()));
 588
 589           /**
 590            * Given a downloader, sets the other repos raw metadata
 591            * path as cache paths for the fetcher, so if another
 592            * repo has the same file, it will not download it
 593            * but copy it from the other repository
 594            */
 595           std::list<RepoInfo> repos = knownRepositories();
 596           for ( std::list<RepoInfo>::const_iterator it = repos.begin();
 597                 it != repos.end();
 598                 ++it )
 599           {
 600             Pathname cachepath(rawcache_path_for_repoinfo( _pimpl->options, *it ));
 601             if ( PathInfo(cachepath).isExist() )
 602               downloader_ptr->addCachePath(cachepath);
 603           }
 604
 605           downloader_ptr->download( media, tmpdir.path());
 606         }
 607 #if 0
 608         else if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 609         {
 610           RepoStatus newstatus = parser::plaindir::dirStatus(url.getPathName());
 611
 612           std::ofstream file(( tmpdir.path() + "/cookie").c_str());
 613           if (!file) {
 614             ZYPP_THROW (Exception( "Can't open " + tmpdir.path().asString() + "/cookie" ) );
 615           }
 616           file << url << endl;
 617           file << newstatus.checksum() << endl;
 618
 619           file.close();
 620         }
 621 #endif
 622         else
 623         {
 624           ZYPP_THROW(RepoUnknownTypeException());
 625         }
 626
 627         // ok we have the metadata, now exchange
 628         // the contents
 629
 630         TmpDir oldmetadata( TmpDir::makeSibling( rawpath ) );
 631         filesystem::rename( rawpath, oldmetadata.path() );
 632         // move the just downloaded there
 633         filesystem::rename( tmpdir.path(), rawpath );
 634
 635         // we are done.
 636         return;
 637       }
 638       catch ( const Exception &e )
 639       {
 640         ZYPP_CAUGHT(e);
 641         ERR << "Trying another url..." << endl;
 642
 643         // remember the exception caught for the *first URL*
 644         // if all other URLs fail, the rexception will be thrown with the
 645         // cause of the problem of the first URL remembered
 646         if (it == info.baseUrlsBegin())
 647           rexception.remember(e);
 648       }
 649     } // for every url
 650     ERR << "No more urls..." << endl;
 651     ZYPP_THROW(rexception);
 652   }
 653
 654   ////////////////////////////////////////////////////////////////////////////
 655
 656   void RepoManager::cleanMetadata( const RepoInfo &info,
 657                                    const ProgressData::ReceiverFnc & progressfnc )
 658   {
 659     ProgressData progress(100);
 660     progress.sendTo(progressfnc);
 661
 662     filesystem::recursive_rmdir(rawcache_path_for_repoinfo(_pimpl->options, info));
 663     progress.toMax();
 664   }
 665
 666   void RepoManager::cleanPackages( const RepoInfo &info,
 667                                    const ProgressData::ReceiverFnc & progressfnc )
 668   {
 669     ProgressData progress(100);
 670     progress.sendTo(progressfnc);
 671
 672     filesystem::recursive_rmdir(packagescache_path_for_repoinfo(_pimpl->options, info));
 673     progress.toMax();
 674   }
 675
 676   void RepoManager::buildCache( const RepoInfo &info,
 677                                 CacheBuildPolicy policy,
 678                                 const ProgressData::ReceiverFnc & progressrcv )
 679   {
 680     assert_alias(info);
 681     Pathname rawpath = rawcache_path_for_repoinfo(_pimpl->options, info);
 682
 683     Pathname base = _pimpl->options.repoCachePath + info.escaped_alias();
 684     Pathname solvfile = base.extend(".solv");
 685
 686
 687     RepoStatus raw_metadata_status = metadataStatus(info);
 688     if ( raw_metadata_status.empty() )
 689     {
 690        /* if there is no cache at this point, we refresh the raw
 691           in case this is the first time - if it's !autorefresh,
 692           we may still refresh */
 693       refreshMetadata(info, RefreshIfNeeded, progressrcv );
 694       raw_metadata_status = metadataStatus(info);
 695     }
 696
 697     bool needs_cleaning = false;
 698     if ( isCached( info ) )
 699     {
 700       MIL << info.alias() << " is already cached." << endl;
 701       //data::RecordId id = store.lookupRepository(info.alias());
 702       RepoStatus cache_status = cacheStatus(info);
 703
 704       if ( cache_status.checksum() == raw_metadata_status.checksum() )
 705       {
 706         MIL << info.alias() << " cache is up to date with metadata." << endl;
 707         if ( policy == BuildIfNeeded ) {
 708           return;
 709         }
 710         else {
 711           MIL << info.alias() << " cache rebuild is forced" << endl;
 712         }
 713       }
 714
 715       needs_cleaning = true;
 716     }
 717
 718     ProgressData progress(100);
 719     callback::SendReport<ProgressReport> report;
 720     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
 721     progress.name(str::form(_("Building repository '%s' cache"), info.name().c_str()));
 722     progress.toMin();
 723
 724     if (needs_cleaning)
 725     {
 726       cleanCache(info);
 727     }
 728
 729     MIL << info.alias() << " building cache..." << endl;
 730     // do we have type?
 731     repo::RepoType repokind = info.type();
 732
 733     // if the type is unknown, try probing.
 734     switch ( repokind.toEnum() )
 735     {
 736       case RepoType::NONE_e:
 737         // unknown, probe the local metadata
 738         repokind = probe(rawpath.asUrl());
 739       break;
 740       default:
 741       break;
 742     }
 743
 744     MIL << "repo type is " << repokind << endl;
 745
 746     switch ( repokind.toEnum() )
 747     {
 748       case RepoType::RPMMD_e :
 749       case RepoType::YAST2_e :
 750       {
 751         // Take care we unlink the solvfile on exception
 752         ManagedFile guard( solvfile, filesystem::unlink );
 753
 754         ostringstream cmd;
 755         cmd << str::form( "repo2solv.sh \"%s\" > \"%s\"", rawpath.c_str(), solvfile.c_str() );
 756
 757         MIL << "Executing: " << cmd.str() << endl;
 758         ExternalProgram prog( cmd.str(), ExternalProgram::Stderr_To_Stdout );
 759
 760         cmd << endl;
 761         for ( string output( prog.receiveLine() ); output.length(); output = prog.receiveLine() ) {
 762           WAR << "  " << output;
 763           cmd << "     " << output;
 764         }
 765
 766         int ret = prog.close();
 767         if ( ret != 0 )
 768         {
 769           RepoException ex(str::form("Failed to cache repo (%d).", ret));
 770           ex.remember( cmd.str() );
 771           ZYPP_THROW(ex);
 772         }
 773
 774         // We keep it.
 775         guard.resetDispose();
 776       }
 777       break;
 778       default:
 779         ZYPP_THROW(RepoUnknownTypeException("Unhandled repository type"));
 780       break;
 781     }
 782 #if 0
 783     switch ( repokind.toEnum() )
 784     {
 785       case RepoType::RPMMD_e :
 786       if (0)
 787       {
 788         CombinedProgressData subprogrcv( progress, 100);
 789         parser::yum::RepoParser parser(id, store, parser::yum::RepoParserOpts(), subprogrcv);
 790         parser.parse(rawpath);
 791           // no error
 792       }
 793       break;
 794       case RepoType::YAST2_e :
 795       if (0)
 796       {
 797         CombinedProgressData subprogrcv( progress, 100);
 798         parser::susetags::RepoParser parser(id, store, subprogrcv);
 799         parser.parse(rawpath);
 800         // no error
 801       }
 802       break;
 803 #endif
 804 #if 0
 805       case RepoType::RPMPLAINDIR_e :
 806       {
 807         CombinedProgressData subprogrcv( progress, 100);
 808         InputStream is(rawpath + "cookie");
 809         string buffer;
 810         getline( is.stream(), buffer);
 811         Url url(buffer);
 812         parser::plaindir::RepoParser parser(id, store, subprogrcv);
 813         parser.parse(url.getPathName());
 814       }
 815       break;
 816
 817       default:
 818         ZYPP_THROW(RepoUnknownTypeException());
 819     }
 820 #endif
 821     // update timestamp and checksum
 822     //store.updateRepositoryStatus(id, raw_metadata_status);
 823     setCacheStatus(info.escaped_alias(), raw_metadata_status);
 824     MIL << "Commit cache.." << endl;
 825     //store.commit();
 826     //progress.toMax();
 827   }
 828
 829   ////////////////////////////////////////////////////////////////////////////
 830
 831   repo::RepoType RepoManager::probe( const Url &url ) const
 832   {
 833     if ( url.getScheme() == "dir" && ! PathInfo( url.getPathName() ).isDir() )
 834     {
 835       // Handle non existing local directory in advance, as
 836       // MediaSetAccess does not support it.
 837       return repo::RepoType::NONE;
 838     }
 839
 840     try
 841     {
 842       MediaSetAccess access(url);
 843       if ( access.doesFileExist("/repodata/repomd.xml") )
 844         return repo::RepoType::RPMMD;
 845       if ( access.doesFileExist("/content") )
 846         return repo::RepoType::YAST2;
 847
 848       // if it is a local url of type dir
 849       if ( (! media::MediaManager::downloads(url)) && ( url.getScheme() == "dir" ) )
 850       {
 851         Pathname path = Pathname(url.getPathName());
 852         if ( PathInfo(path).isDir() )
 853         {
 854           // allow empty dirs for now
 855           return repo::RepoType::RPMPLAINDIR;
 856         }
 857       }
 858     }
 859     catch ( const media::MediaException &e )
 860     {
 861       ZYPP_CAUGHT(e);
 862       RepoException enew("Error trying to read from " + url.asString());
 863       enew.remember(e);
 864       ZYPP_THROW(enew);
 865     }
 866     catch ( const Exception &e )
 867     {
 868       ZYPP_CAUGHT(e);
 869       Exception enew("Unknown error reading from " + url.asString());
 870       enew.remember(e);
 871       ZYPP_THROW(enew);
 872     }
 873
 874     return repo::RepoType::NONE;
 875   }
 876
 877   ////////////////////////////////////////////////////////////////////////////
 878
 879   void RepoManager::cleanCache( const RepoInfo &info,
 880                                 const ProgressData::ReceiverFnc & progressrcv )
 881   {
 882     ProgressData progress(100);
 883     progress.sendTo(progressrcv);
 884     progress.toMin();
 885
 886     unlink (_pimpl->options.repoCachePath / (info.escaped_alias() + ".solv"));
 887     progress.set(99);
 888     unlink (_pimpl->options.repoCachePath / (info.escaped_alias() + ".cookie"));
 889
 890     progress.toMax();
 891   }
 892
 893   void RepoManager::cleanTargetCache(const ProgressData::ReceiverFnc & progressrcv)
 894   {
 895     unlink (_pimpl->options.repoCachePath / (sat::Pool::systemRepoName() + ".solv"));
 896     unlink (_pimpl->options.repoCachePath / (sat::Pool::systemRepoName() + ".cookie"));
 897   }
 898
 899   ////////////////////////////////////////////////////////////////////////////
 900
 901   bool RepoManager::isCached( const RepoInfo &info ) const
 902   {
 903     Pathname name = _pimpl->options.repoCachePath;
 904     return PathInfo(name + Pathname(info.escaped_alias()).extend(".solv")).isExist();
 905   }
 906
 907   RepoStatus RepoManager::cacheStatus( const RepoInfo &info ) const
 908   {
 909
 910     Pathname base = _pimpl->options.repoCachePath + info.escaped_alias();
 911     Pathname cookiefile = base.extend(".cookie");
 912
 913     return RepoStatus::fromCookieFile(cookiefile);
 914   }
 915
 916   void RepoManager::setCacheStatus( const string &alias, const RepoStatus &status )
 917   {
 918     Pathname base = _pimpl->options.repoCachePath + alias;
 919     Pathname cookiefile = base.extend(".cookie");
 920
 921     status.saveToCookieFile(cookiefile);
 922   }
 923
 924   void RepoManager::loadFromCache( const RepoInfo & info,
 925                                    const ProgressData::ReceiverFnc & progressrcv )
 926   {
 927     assert_alias(info);
 928     string alias = info.escaped_alias();
 929     Pathname solvfile = (_pimpl->options.repoCachePath / alias).extend(".solv");
 930
 931     if ( ! PathInfo(solvfile).isExist() )
 932       ZYPP_THROW(RepoNotCachedException());
 933
 934     try
 935     {
 936       sat::Pool::instance().addRepoSolv( solvfile, info );
 937     }
 938     catch ( const Exception & exp )
 939     {
 940       ZYPP_CAUGHT( exp );
 941       MIL << "Try to handle exception by rebuilding the solv-file" << endl;
 942       cleanCache( info, progressrcv );
 943       buildCache( info, BuildIfNeeded, progressrcv );
 944
 945       sat::Pool::instance().addRepoSolv( solvfile, info );
 946     }
 947   }
 948
 949   ////////////////////////////////////////////////////////////////////////////
 950
 951   /**
 952    * Generate a non existing filename in a directory, using a base
 953    * name. For example if a directory contains 3 files
 954    *
 955    * |-- bar
 956    * |-- foo
 957    * `-- moo
 958    *
 959    * If you try to generate a unique filename for this directory,
 960    * based on "ruu" you will get "ruu", but if you use the base
 961    * "foo" you will get "foo_1"
 962    *
 963    * \param dir Directory where the file needs to be unique
 964    * \param basefilename string to base the filename on.
 965    */
 966   static Pathname generate_non_existing_name( const Pathname &dir,
 967                                               const std::string &basefilename )
 968   {
 969     string final_filename = basefilename;
 970     int counter = 1;
 971     while ( PathInfo(dir + final_filename).isExist() )
 972     {
 973       final_filename = basefilename + "_" + str::numstring(counter);
 974       counter++;
 975     }
 976     return dir + Pathname(final_filename);
 977   }
 978
 979   ////////////////////////////////////////////////////////////////////////////
 980
 981   /**
 982    * \short Generate a related filename from a repo info
 983    *
 984    * From a repo info, it will try to use the alias as a filename
 985    * escaping it if necessary. Other fallbacks can be added to
 986    * this function in case there is no way to use the alias
 987    */
 988   static std::string generate_filename( const RepoInfo &info )
 989   {
 990     std::string fnd="/";
 991     std::string rep="_";
 992     std::string filename = info.alias();
 993     // replace slashes with underscores
 994     size_t pos = filename.find(fnd);
 995     while(pos!=string::npos)
 996     {
 997       filename.replace(pos,fnd.length(),rep);
 998       pos = filename.find(fnd,pos+rep.length());
 999     }
1000     filename = Pathname(filename).extend(".repo").asString();
1001     MIL << "generating filename for repo [" << info.alias() << "] : '" << filename << "'" << endl;
1002     return filename;
1003   }
1004
1005
1006   ////////////////////////////////////////////////////////////////////////////
1007
1008   void RepoManager::addRepository( const RepoInfo &info,
1009                                    const ProgressData::ReceiverFnc & progressrcv )
1010   {
1011     assert_alias(info);
1012
1013     ProgressData progress(100);
1014     callback::SendReport<ProgressReport> report;
1015     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
1016     progress.name(str::form(_("Adding repository '%s'"), info.name().c_str()));
1017     progress.toMin();
1018
1019     std::list<RepoInfo> repos = knownRepositories();
1020     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1021           it != repos.end();
1022           ++it )
1023     {
1024       if ( info.alias() == (*it).alias() )
1025         ZYPP_THROW(RepoAlreadyExistsException(info.alias()));
1026     }
1027
1028     RepoInfo tosave = info;
1029
1030     // check the first url for now
1031     if ( _pimpl->options.probe )
1032     {
1033       DBG << "unknown repository type, probing" << endl;
1034
1035       RepoType probedtype;
1036       probedtype = probe(*tosave.baseUrlsBegin());
1037       if ( tosave.baseUrlsSize() > 0 )
1038       {
1039         if ( probedtype == RepoType::NONE )
1040           ZYPP_THROW(RepoUnknownTypeException());
1041         else
1042           tosave.setType(probedtype);
1043       }
1044     }
1045
1046     progress.set(50);
1047
1048     // assert the directory exists
1049     filesystem::assert_dir(_pimpl->options.knownReposPath);
1050
1051     Pathname repofile = generate_non_existing_name(_pimpl->options.knownReposPath,
1052                                                     generate_filename(tosave));
1053     // now we have a filename that does not exists
1054     MIL << "Saving repo in " << repofile << endl;
1055
1056     std::ofstream file(repofile.c_str());
1057     if (!file) {
1058       ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
1059     }
1060
1061     tosave.dumpRepoOn(file);
1062     progress.toMax();
1063     MIL << "done" << endl;
1064   }
1065
1066   void RepoManager::addRepositories( const Url &url,
1067                                      const ProgressData::ReceiverFnc & progressrcv )
1068   {
1069     std::list<RepoInfo> knownrepos = knownRepositories();
1070     std::list<RepoInfo> repos = readRepoFile(url);
1071     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1072           it != repos.end();
1073           ++it )
1074     {
1075       // look if the alias is in the known repos.
1076       for ( std::list<RepoInfo>::const_iterator kit = knownrepos.begin();
1077           kit != knownrepos.end();
1078           ++kit )
1079       {
1080         if ( (*it).alias() == (*kit).alias() )
1081         {
1082           ERR << "To be added repo " << (*it).alias() << " conflicts with existing repo " << (*kit).alias() << endl;
1083           ZYPP_THROW(RepoAlreadyExistsException((*it).alias()));
1084         }
1085       }
1086     }
1087
1088     string filename = Pathname(url.getPathName()).basename();
1089
1090     if ( filename == Pathname() )
1091       ZYPP_THROW(RepoException("Invalid repo file name at " + url.asString() ));
1092
1093     // assert the directory exists
1094     filesystem::assert_dir(_pimpl->options.knownReposPath);
1095
1096     Pathname repofile = generate_non_existing_name(_pimpl->options.knownReposPath, filename);
1097     // now we have a filename that does not exists
1098     MIL << "Saving " << repos.size() << " repo" << ( repos.size() ? "s" : "" ) << " in " << repofile << endl;
1099
1100     std::ofstream file(repofile.c_str());
1101     if (!file) {
1102       ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
1103     }
1104
1105     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1106           it != repos.end();
1107           ++it )
1108     {
1109       MIL << "Saving " << (*it).alias() << endl;
1110       (*it).dumpRepoOn(file);
1111     }
1112     MIL << "done" << endl;
1113   }
1114
1115   ////////////////////////////////////////////////////////////////////////////
1116
1117   void RepoManager::removeRepository( const RepoInfo & info,
1118                                       const ProgressData::ReceiverFnc & progressrcv)
1119   {
1120     ProgressData progress;
1121     callback::SendReport<ProgressReport> report;
1122     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
1123     progress.name(str::form(_("Removing repository '%s'"), info.name().c_str()));
1124
1125     MIL << "Going to delete repo " << info.alias() << endl;
1126
1127     std::list<RepoInfo> repos = knownRepositories();
1128     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1129           it != repos.end();
1130           ++it )
1131     {
1132       // they can be the same only if the provided is empty, that means
1133       // the provided repo has no alias
1134       // then skip
1135       if ( (!info.alias().empty()) && ( info.alias() != (*it).alias() ) )
1136         continue;
1137
1138       // TODO match by url
1139
1140       // we have a matcing repository, now we need to know
1141       // where it does come from.
1142       RepoInfo todelete = *it;
1143       if (todelete.filepath().empty())
1144       {
1145         ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
1146       }
1147       else
1148       {
1149         // figure how many repos are there in the file:
1150         std::list<RepoInfo> filerepos = repositories_in_file(todelete.filepath());
1151         if ( (filerepos.size() == 1) && ( filerepos.front().alias() == todelete.alias() ) )
1152         {
1153           // easy, only this one, just delete the file
1154           if ( filesystem::unlink(todelete.filepath()) != 0 )
1155           {
1156             ZYPP_THROW(RepoException("Can't delete " + todelete.filepath().asString()));
1157           }
1158           MIL << todelete.alias() << " sucessfully deleted." << endl;
1159         }
1160         else
1161         {
1162           // there are more repos in the same file
1163           // write them back except the deleted one.
1164           //TmpFile tmp;
1165           //std::ofstream file(tmp.path().c_str());
1166
1167           // assert the directory exists
1168           filesystem::assert_dir(todelete.filepath().dirname());
1169
1170           std::ofstream file(todelete.filepath().c_str());
1171           if (!file) {
1172             //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
1173             ZYPP_THROW (Exception( "Can't open " + todelete.filepath().asString() ) );
1174           }
1175           for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
1176                 fit != filerepos.end();
1177                 ++fit )
1178           {
1179             if ( (*fit).alias() != todelete.alias() )
1180               (*fit).dumpRepoOn(file);
1181           }
1182         }
1183
1184         CombinedProgressData subprogrcv(progress, 70);
1185         CombinedProgressData cleansubprogrcv(progress, 30);
1186         // now delete it from cache
1187         if ( isCached(todelete) )
1188           cleanCache( todelete, subprogrcv);
1189         // now delete metadata (#301037)
1190         cleanMetadata( todelete, cleansubprogrcv);
1191         MIL << todelete.alias() << " sucessfully deleted." << endl;
1192         return;
1193       } // else filepath is empty
1194
1195     }
1196     // should not be reached on a sucess workflow
1197     ZYPP_THROW(RepoNotFoundException(info));
1198   }
1199
1200   ////////////////////////////////////////////////////////////////////////////
1201
1202   void RepoManager::modifyRepository( const std::string &alias,
1203                                       const RepoInfo & newinfo,
1204                                       const ProgressData::ReceiverFnc & progressrcv )
1205   {
1206     RepoInfo toedit = getRepositoryInfo(alias);
1207
1208     // check if the new alias already exists when renaming the repo
1209     if (alias != newinfo.alias())
1210     {
1211       std::list<RepoInfo> repos = knownRepositories();
1212       for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1213             it != repos.end();
1214             ++it )
1215       {
1216         if ( newinfo.alias() == (*it).alias() )
1217           ZYPP_THROW(RepoAlreadyExistsException(newinfo.alias()));
1218       }
1219     }
1220
1221     if (toedit.filepath().empty())
1222     {
1223       ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
1224     }
1225     else
1226     {
1227       // figure how many repos are there in the file:
1228       std::list<RepoInfo> filerepos = repositories_in_file(toedit.filepath());
1229
1230       // there are more repos in the same file
1231       // write them back except the deleted one.
1232       //TmpFile tmp;
1233       //std::ofstream file(tmp.path().c_str());
1234
1235       // assert the directory exists
1236       filesystem::assert_dir(toedit.filepath().dirname());
1237
1238       std::ofstream file(toedit.filepath().c_str());
1239       if (!file) {
1240         //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
1241         ZYPP_THROW (Exception( "Can't open " + toedit.filepath().asString() ) );
1242       }
1243       for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
1244             fit != filerepos.end();
1245             ++fit )
1246       {
1247           // if the alias is different, dump the original
1248           // if it is the same, dump the provided one
1249           if ( (*fit).alias() != toedit.alias() )
1250             (*fit).dumpRepoOn(file);
1251           else
1252             newinfo.dumpRepoOn(file);
1253       }
1254     }
1255   }
1256
1257   ////////////////////////////////////////////////////////////////////////////
1258
1259   RepoInfo RepoManager::getRepositoryInfo( const std::string &alias,
1260                                            const ProgressData::ReceiverFnc & progressrcv )
1261   {
1262     std::list<RepoInfo> repos = knownRepositories();
1263     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1264           it != repos.end();
1265           ++it )
1266     {
1267       if ( (*it).alias() == alias )
1268         return *it;
1269     }
1270     RepoInfo info;
1271     info.setAlias(info.alias());
1272     ZYPP_THROW(RepoNotFoundException(info));
1273   }
1274
1275   ////////////////////////////////////////////////////////////////////////////
1276
1277   RepoInfo RepoManager::getRepositoryInfo( const Url & url,
1278                                            const url::ViewOption & urlview,
1279                                            const ProgressData::ReceiverFnc & progressrcv )
1280   {
1281     std::list<RepoInfo> repos = knownRepositories();
1282     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1283           it != repos.end();
1284           ++it )
1285     {
1286       for(RepoInfo::urls_const_iterator urlit = (*it).baseUrlsBegin();
1287           urlit != (*it).baseUrlsEnd();
1288           ++urlit)
1289       {
1290         if ((*urlit).asString(urlview) == url.asString(urlview))
1291           return *it;
1292       }
1293     }
1294     RepoInfo info;
1295     info.setAlias(info.alias());
1296     info.setBaseUrl(url);
1297     ZYPP_THROW(RepoNotFoundException(info));
1298   }
1299
1300   ////////////////////////////////////////////////////////////////////////////
1301
1302   std::ostream & operator<<( std::ostream & str, const RepoManager & obj )
1303   {
1304     return str << *obj._pimpl;
1305   }
1306
1307   /////////////////////////////////////////////////////////////////
1308 } // namespace zypp
1309 ///////////////////////////////////////////////////////////////////