zypp/RepoManager.cc

   1 /*---------------------------------------------------------------------\
   2 |                          ____ _   __ __ ___                          |
   3 |                         |__  / \ / / . \ . \                         |
   4 |                           / / \ V /|  _/  _/                         |
   5 |                          / /__ | | | | | |                           |
   6 |                         /_____||_| |_| |_|                           |
   7 |                                                                      |
   8 \---------------------------------------------------------------------*/
   9 /** \file       zypp/RepoManager.cc
  10  *
  11 */
  12
  13 #include <cstdlib>
  14 #include <iostream>
  15 #include <fstream>
  16 #include <sstream>
  17 #include <list>
  18 #include <map>
  19 #include <algorithm>
  20 #include "zypp/base/InputStream.h"
  21 #include "zypp/base/Logger.h"
  22 #include "zypp/base/Gettext.h"
  23 #include "zypp/base/Function.h"
  24 #include "zypp/base/Regex.h"
  25 #include "zypp/PathInfo.h"
  26 #include "zypp/TmpPath.h"
  27
  28 #include "zypp/repo/RepoException.h"
  29 #include "zypp/RepoManager.h"
  30
  31 #include "zypp/media/MediaManager.h"
  32 #include "zypp/MediaSetAccess.h"
  33 #include "zypp/ExternalProgram.h"
  34 #include "zypp/ManagedFile.h"
  35
  36 #include "zypp/parser/RepoFileReader.h"
  37 #include "zypp/repo/yum/Downloader.h"
  38 #include "zypp/parser/yum/RepoParser.h"
  39 #include "zypp/repo/susetags/Downloader.h"
  40 #include "zypp/parser/susetags/RepoParser.h"
  41 #include "zypp/parser/plaindir/RepoParser.h"
  42
  43 #include "zypp/ZYppCallbacks.h"
  44
  45 #include "sat/Pool.h"
  46 #include "satsolver/pool.h"
  47 #include "satsolver/repo.h"
  48 #include "satsolver/repo_solv.h"
  49
  50 using namespace std;
  51 using namespace zypp;
  52 using namespace zypp::repo;
  53 using namespace zypp::filesystem;
  54
  55 using namespace zypp::repo;
  56
  57 ///////////////////////////////////////////////////////////////////
  58 namespace zypp
  59 { /////////////////////////////////////////////////////////////////
  60
  61   ///////////////////////////////////////////////////////////////////
  62   //
  63   //    CLASS NAME : RepoManagerOptions
  64   //
  65   ///////////////////////////////////////////////////////////////////
  66
  67   RepoManagerOptions::RepoManagerOptions()
  68   {
  69     repoCachePath    = ZConfig::instance().repoCachePath();
  70     repoRawCachePath = ZConfig::instance().repoMetadataPath();
  71     repoPackagesCachePath = ZConfig::instance().repoPackagesPath();
  72     knownReposPath   = ZConfig::instance().knownReposPath();
  73     probe            = ZConfig::instance().repo_add_probe();
  74   }
  75
  76   ////////////////////////////////////////////////////////////////////////////
  77
  78   /**
  79     * \short Simple callback to collect the results
  80     *
  81     * Classes like RepoFileParser call the callback
  82     * once per each repo in a file.
  83     *
  84     * Passing this functor as callback, you can collect
  85     * all resuls at the end, without dealing with async
  86     * code.
  87     */
  88     struct RepoCollector
  89     {
  90       RepoCollector()
  91       {}
  92
  93       ~RepoCollector()
  94       {}
  95
  96       bool collect( const RepoInfo &repo )
  97       {
  98         repos.push_back(repo);
  99         return true;
 100       }
 101
 102       RepoInfoList repos;
 103     };
 104
 105   ////////////////////////////////////////////////////////////////////////////
 106
 107   /**
 108    * Reads RepoInfo's from a repo file.
 109    *
 110    * \param file pathname of the file to read.
 111    */
 112   static std::list<RepoInfo> repositories_in_file( const Pathname & file )
 113   {
 114     MIL << "repo file: " << file << endl;
 115     RepoCollector collector;
 116     parser::RepoFileReader parser( file, bind( &RepoCollector::collect, &collector, _1 ) );
 117     return collector.repos;
 118   }
 119
 120   ////////////////////////////////////////////////////////////////////////////
 121
 122   std::list<RepoInfo> readRepoFile(const Url & repo_file)
 123    {
 124      // no interface to download a specific file, using workaround:
 125      //! \todo add MediaManager::provideFile(Url file_url) to easily access any file URLs? (no need for media access id or media_nr)
 126      Url url(repo_file);
 127      Pathname path(url.getPathName());
 128      url.setPathName ("/");
 129      MediaSetAccess access(url);
 130      Pathname local = access.provideFile(path);
 131
 132      DBG << "reading repo file " << repo_file << ", local path: " << local << endl;
 133
 134      return repositories_in_file(local);
 135    }
 136
 137   ////////////////////////////////////////////////////////////////////////////
 138
 139   /**
 140    * \short List of RepoInfo's from a directory
 141    *
 142    * Goes trough every file ending with ".repo" in a directory and adds all
 143    * RepoInfo's contained in that file.
 144    *
 145    * \param dir pathname of the directory to read.
 146    */
 147   static std::list<RepoInfo> repositories_in_dir( const Pathname &dir )
 148   {
 149     MIL << "directory " << dir << endl;
 150     list<RepoInfo> repos;
 151     list<Pathname> entries;
 152     if ( filesystem::readdir( entries, Pathname(dir), false ) != 0 )
 153       ZYPP_THROW(Exception("failed to read directory"));
 154
 155     str::regex allowedRepoExt("^\\.repo(_[0-9]+)?$");
 156     for ( list<Pathname>::const_iterator it = entries.begin(); it != entries.end(); ++it )
 157     {
 158       if (str::regex_match(it->extension(), allowedRepoExt))
 159       {
 160         list<RepoInfo> tmp = repositories_in_file( *it );
 161         repos.insert( repos.end(), tmp.begin(), tmp.end() );
 162
 163         //std::copy( collector.repos.begin(), collector.repos.end(), std::back_inserter(repos));
 164         //MIL << "ok" << endl;
 165       }
 166     }
 167     return repos;
 168   }
 169
 170   ////////////////////////////////////////////////////////////////////////////
 171
 172   static void assert_alias( const RepoInfo &info )
 173   {
 174     if (info.alias().empty())
 175         ZYPP_THROW(RepoNoAliasException());
 176   }
 177
 178   ////////////////////////////////////////////////////////////////////////////
 179
 180   static void assert_urls( const RepoInfo &info )
 181   {
 182     if (info.baseUrlsEmpty())
 183         ZYPP_THROW(RepoNoUrlException());
 184   }
 185
 186   ////////////////////////////////////////////////////////////////////////////
 187
 188   /**
 189    * \short Calculates the raw cache path for a repository
 190    */
 191   static Pathname rawcache_path_for_repoinfo( const RepoManagerOptions &opt, const RepoInfo &info )
 192   {
 193     assert_alias(info);
 194     return opt.repoRawCachePath / info.escaped_alias();
 195   }
 196
 197   /**
 198    * \short Calculates the packages cache path for a repository
 199    */
 200   static Pathname packagescache_path_for_repoinfo( const RepoManagerOptions &opt, const RepoInfo &info )
 201   {
 202     assert_alias(info);
 203     return opt.repoPackagesCachePath / info.escaped_alias();
 204   }
 205
 206   ///////////////////////////////////////////////////////////////////
 207   //
 208   //    CLASS NAME : RepoManager::Impl
 209   //
 210   ///////////////////////////////////////////////////////////////////
 211
 212   /**
 213    * \short RepoManager implementation.
 214    */
 215   struct RepoManager::Impl
 216   {
 217     Impl( const RepoManagerOptions &opt )
 218       : options(opt)
 219     {
 220
 221     }
 222
 223     Impl()
 224     {
 225
 226     }
 227
 228     RepoManagerOptions options;
 229
 230     map<string, RepoInfo> _repoinfos;
 231
 232   public:
 233     /** Offer default Impl. */
 234     static shared_ptr<Impl> nullimpl()
 235     {
 236       static shared_ptr<Impl> _nullimpl( new Impl );
 237       return _nullimpl;
 238     }
 239
 240   private:
 241     friend Impl * rwcowClone<Impl>( const Impl * rhs );
 242     /** clone for RWCOW_pointer */
 243     Impl * clone() const
 244     { return new Impl( *this ); }
 245   };
 246
 247   ///////////////////////////////////////////////////////////////////
 248
 249   /** \relates RepoManager::Impl Stream output */
 250   inline std::ostream & operator<<( std::ostream & str, const RepoManager::Impl & obj )
 251   {
 252     return str << "RepoManager::Impl";
 253   }
 254
 255   ///////////////////////////////////////////////////////////////////
 256   //
 257   //    CLASS NAME : RepoManager
 258   //
 259   ///////////////////////////////////////////////////////////////////
 260
 261   RepoManager::RepoManager( const RepoManagerOptions &opt )
 262   : _pimpl( new Impl(opt) )
 263   {}
 264
 265   ////////////////////////////////////////////////////////////////////////////
 266
 267   RepoManager::~RepoManager()
 268   {}
 269
 270   ////////////////////////////////////////////////////////////////////////////
 271
 272   std::list<RepoInfo> RepoManager::knownRepositories() const
 273   {
 274     MIL << endl;
 275
 276     if ( PathInfo(_pimpl->options.knownReposPath).isExist() )
 277     {
 278       RepoInfoList repos = repositories_in_dir(_pimpl->options.knownReposPath);
 279       for ( RepoInfoList::iterator it = repos.begin();
 280             it != repos.end();
 281             ++it )
 282       {
 283         // set the metadata path for the repo
 284         Pathname metadata_path = rawcache_path_for_repoinfo(_pimpl->options, (*it));
 285         (*it).setMetadataPath(metadata_path);
 286
 287         // set the downloaded packages path for the repo
 288         Pathname packages_path = packagescache_path_for_repoinfo(_pimpl->options, (*it));
 289         (*it).setPackagesPath(packages_path);
 290       }
 291       return repos;
 292     }
 293     else
 294       return std::list<RepoInfo>();
 295
 296     MIL << endl;
 297   }
 298
 299   ////////////////////////////////////////////////////////////////////////////
 300
 301   Pathname RepoManager::metadataPath( const RepoInfo &info ) const
 302   {
 303     return rawcache_path_for_repoinfo(_pimpl->options, info );
 304   }
 305
 306   ////////////////////////////////////////////////////////////////////////////
 307
 308   RepoStatus RepoManager::metadataStatus( const RepoInfo &info ) const
 309   {
 310     Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 311     RepoType repokind = info.type();
 312     RepoStatus status;
 313
 314     switch ( repokind.toEnum() )
 315     {
 316       case RepoType::NONE_e:
 317       // unknown, probe the local metadata
 318         repokind = probe(rawpath.asUrl());
 319       break;
 320       default:
 321       break;
 322     }
 323
 324     switch ( repokind.toEnum() )
 325     {
 326       case RepoType::RPMMD_e :
 327       {
 328         status = RepoStatus( rawpath + "/repodata/repomd.xml");
 329       }
 330       break;
 331
 332       case RepoType::YAST2_e :
 333       {
 334         status = RepoStatus( rawpath + "/content") && (RepoStatus( rawpath + "/media.1/media"));
 335       }
 336       break;
 337
 338       case RepoType::RPMPLAINDIR_e :
 339       {
 340         if ( PathInfo(Pathname(rawpath + "/cookie")).isExist() )
 341           status = RepoStatus( rawpath + "/cookie");
 342       }
 343       break;
 344
 345       case RepoType::NONE_e :
 346         // Return default RepoStatus in case of RepoType::NONE
 347         // indicating it should be created?
 348         // ZYPP_THROW(RepoUnknownTypeException());
 349         break;
 350     }
 351     return status;
 352   }
 353
 354   void RepoManager::touchIndexFile(const RepoInfo & info)
 355   {
 356     Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 357
 358     RepoType repokind = info.type();
 359     if ( repokind.toEnum() == RepoType::NONE_e )
 360       // unknown, probe the local metadata
 361       repokind = probe(rawpath.asUrl());
 362     // if still unknown, just return
 363     if (repokind == RepoType::NONE_e)
 364       return;
 365
 366     Pathname p;
 367     switch ( repokind.toEnum() )
 368     {
 369       case RepoType::RPMMD_e :
 370         p = Pathname(rawpath + "/repodata/repomd.xml");
 371         break;
 372
 373       case RepoType::YAST2_e :
 374         p = Pathname(rawpath + "/content");
 375         break;
 376
 377       case RepoType::RPMPLAINDIR_e :
 378         p = Pathname(rawpath + "/cookie");
 379         break;
 380
 381       case RepoType::NONE_e :
 382       default:
 383         break;
 384     }
 385
 386     // touch the file, ignore error (they are logged anyway)
 387     filesystem::touch(p);
 388   }
 389
 390   RepoManager::RefreshCheckStatus RepoManager::checkIfToRefreshMetadata(
 391                                               const RepoInfo &info,
 392                                               const Url &url,
 393                                               RawMetadataRefreshPolicy policy )
 394   {
 395     assert_alias(info);
 396
 397     RepoStatus oldstatus;
 398     RepoStatus newstatus;
 399
 400     try
 401     {
 402       MIL << "Going to try to check whether refresh is needed for " << url << endl;
 403
 404       repo::RepoType repokind = info.type();
 405
 406       // if the type is unknown, try probing.
 407       switch ( repokind.toEnum() )
 408       {
 409         case RepoType::NONE_e:
 410           // unknown, probe it
 411           repokind = probe(url);
 412         break;
 413         default:
 414         break;
 415       }
 416
 417       Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 418       filesystem::assert_dir(rawpath);
 419       oldstatus = metadataStatus(info);
 420
 421       // now we've got the old (cached) status, we can decide repo.refresh.delay
 422       if (policy != RefreshForced)
 423       {
 424         // difference in seconds
 425         double diff = difftime(
 426           (Date::ValueType)Date::now(),
 427           (Date::ValueType)oldstatus.timestamp()) / 60;
 428
 429         DBG << "oldstatus: " << (Date::ValueType)oldstatus.timestamp() << endl;
 430         DBG << "current time: " << (Date::ValueType)Date::now() << endl;
 431         DBG << "last refresh = " << diff << " minutes ago" << endl;
 432
 433         if (diff < ZConfig::instance().repo_refresh_delay())
 434         {
 435           MIL << "Repository '" << info.alias()
 436               << "' has been refreshed less than repo.refresh.delay ("
 437               << ZConfig::instance().repo_refresh_delay()
 438               << ") minutes ago. Advising to skip refresh" << endl;
 439           return REPO_CHECK_DELAYED;
 440         }
 441       }
 442
 443       // create temp dir as sibling of rawpath
 444       filesystem::TmpDir tmpdir( filesystem::TmpDir::makeSibling( rawpath ) );
 445
 446       if ( ( repokind.toEnum() == RepoType::RPMMD_e ) ||
 447            ( repokind.toEnum() == RepoType::YAST2_e ) )
 448       {
 449         MediaSetAccess media(url);
 450         shared_ptr<repo::Downloader> downloader_ptr;
 451
 452         if ( repokind.toEnum() == RepoType::RPMMD_e )
 453           downloader_ptr.reset(new yum::Downloader(info.path()));
 454         else
 455           downloader_ptr.reset( new susetags::Downloader(info.path()));
 456
 457         RepoStatus newstatus = downloader_ptr->status(media);
 458         bool refresh = false;
 459         if ( oldstatus.checksum() == newstatus.checksum() )
 460         {
 461           MIL << "repo has not changed" << endl;
 462           if ( policy == RefreshForced )
 463           {
 464             MIL << "refresh set to forced" << endl;
 465             refresh = true;
 466           }
 467         }
 468         else
 469         {
 470           MIL << "repo has changed, going to refresh" << endl;
 471           refresh = true;
 472         }
 473
 474         if (!refresh)
 475           touchIndexFile(info);
 476
 477         return refresh ? REFRESH_NEEDED : REPO_UP_TO_DATE;
 478       }
 479       else if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 480       {
 481         RepoStatus newstatus = parser::plaindir::dirStatus(url.getPathName());
 482         bool refresh = false;
 483         if ( oldstatus.checksum() == newstatus.checksum() )
 484         {
 485           MIL << "repo has not changed" << endl;
 486           if ( policy == RefreshForced )
 487           {
 488             MIL << "refresh set to forced" << endl;
 489             refresh = true;
 490           }
 491         }
 492         else
 493         {
 494           MIL << "repo has changed, going to refresh" << endl;
 495           refresh = true;
 496         }
 497
 498         if (!refresh)
 499           touchIndexFile(info);
 500
 501         return refresh ? REFRESH_NEEDED : REPO_UP_TO_DATE;
 502       }
 503       else
 504       {
 505         ZYPP_THROW(RepoUnknownTypeException());
 506       }
 507     }
 508     catch ( const Exception &e )
 509     {
 510       ZYPP_CAUGHT(e);
 511       ERR << "refresh check failed for " << url << endl;
 512       ZYPP_RETHROW(e);
 513     }
 514
 515     return REFRESH_NEEDED; // default
 516   }
 517
 518   void RepoManager::refreshMetadata( const RepoInfo &info,
 519                                      RawMetadataRefreshPolicy policy,
 520                                      const ProgressData::ReceiverFnc & progress )
 521   {
 522     assert_alias(info);
 523     assert_urls(info);
 524
 525     // we will throw this later if no URL checks out fine
 526     RepoException rexception(_("Valid metadata not found at specified URL(s)"));
 527
 528     // try urls one by one
 529     for ( RepoInfo::urls_const_iterator it = info.baseUrlsBegin(); it != info.baseUrlsEnd(); ++it )
 530     {
 531       try
 532       {
 533         Url url(*it);
 534
 535         // check whether to refresh metadata
 536         // if the check fails for this url, it throws, so another url will be checked
 537         if (checkIfToRefreshMetadata(info, url, policy)!=REFRESH_NEEDED)
 538           return;
 539
 540         MIL << "Going to refresh metadata from " << url << endl;
 541
 542         repo::RepoType repokind = info.type();
 543
 544         // if the type is unknown, try probing.
 545         switch ( repokind.toEnum() )
 546         {
 547           case RepoType::NONE_e:
 548             // unknown, probe it
 549             repokind = probe(*it);
 550
 551             if (repokind.toEnum() != RepoType::NONE_e)
 552             {
 553               //save probed type only for repos in system
 554               std::list<RepoInfo> repos = knownRepositories();
 555               for ( std::list<RepoInfo>::const_iterator it = repos.begin();
 556                    it != repos.end(); ++it )
 557               {
 558                 if ( info.alias() == (*it).alias() )
 559                 {
 560                   RepoInfo modifiedrepo = info;
 561                   modifiedrepo.setType(repokind);
 562                   modifyRepository(info.alias(),modifiedrepo);
 563                 }
 564               }
 565             }
 566           break;
 567           default:
 568           break;
 569         }
 570
 571         Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 572         filesystem::assert_dir(rawpath);
 573
 574         // create temp dir as sibling of rawpath
 575         filesystem::TmpDir tmpdir( filesystem::TmpDir::makeSibling( rawpath ) );
 576
 577         if ( ( repokind.toEnum() == RepoType::RPMMD_e ) ||
 578              ( repokind.toEnum() == RepoType::YAST2_e ) )
 579         {
 580           MediaSetAccess media(url);
 581           shared_ptr<repo::Downloader> downloader_ptr;
 582
 583           if ( repokind.toEnum() == RepoType::RPMMD_e )
 584             downloader_ptr.reset(new yum::Downloader(info.path()));
 585           else
 586             downloader_ptr.reset( new susetags::Downloader(info.path()));
 587
 588           /**
 589            * Given a downloader, sets the other repos raw metadata
 590            * path as cache paths for the fetcher, so if another
 591            * repo has the same file, it will not download it
 592            * but copy it from the other repository
 593            */
 594           std::list<RepoInfo> repos = knownRepositories();
 595           for ( std::list<RepoInfo>::const_iterator it = repos.begin();
 596                 it != repos.end();
 597                 ++it )
 598           {
 599             Pathname cachepath(rawcache_path_for_repoinfo( _pimpl->options, *it ));
 600             if ( PathInfo(cachepath).isExist() )
 601               downloader_ptr->addCachePath(cachepath);
 602           }
 603
 604           downloader_ptr->download( media, tmpdir.path());
 605         }
 606         else if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 607         {
 608           RepoStatus newstatus = parser::plaindir::dirStatus(url.getPathName());
 609
 610           std::ofstream file(( tmpdir.path() + "/cookie").c_str());
 611           if (!file) {
 612             ZYPP_THROW (Exception( "Can't open " + tmpdir.path().asString() + "/cookie" ) );
 613           }
 614           file << url << endl;
 615           file << newstatus.checksum() << endl;
 616
 617           file.close();
 618         }
 619         else
 620         {
 621           ZYPP_THROW(RepoUnknownTypeException());
 622         }
 623
 624         // ok we have the metadata, now exchange
 625         // the contents
 626
 627         TmpDir oldmetadata( TmpDir::makeSibling( rawpath ) );
 628         filesystem::rename( rawpath, oldmetadata.path() );
 629         // move the just downloaded there
 630         filesystem::rename( tmpdir.path(), rawpath );
 631
 632         // we are done.
 633         return;
 634       }
 635       catch ( const Exception &e )
 636       {
 637         ZYPP_CAUGHT(e);
 638         ERR << "Trying another url..." << endl;
 639
 640         // remember the exception caught for the *first URL*
 641         // if all other URLs fail, the rexception will be thrown with the
 642         // cause of the problem of the first URL remembered
 643         if (it == info.baseUrlsBegin())
 644           rexception.remember(e);
 645       }
 646     } // for every url
 647     ERR << "No more urls..." << endl;
 648     ZYPP_THROW(rexception);
 649   }
 650
 651   ////////////////////////////////////////////////////////////////////////////
 652
 653   void RepoManager::cleanMetadata( const RepoInfo &info,
 654                                    const ProgressData::ReceiverFnc & progressfnc )
 655   {
 656     ProgressData progress(100);
 657     progress.sendTo(progressfnc);
 658
 659     filesystem::recursive_rmdir(rawcache_path_for_repoinfo(_pimpl->options, info));
 660     progress.toMax();
 661   }
 662
 663   void RepoManager::cleanPackages( const RepoInfo &info,
 664                                    const ProgressData::ReceiverFnc & progressfnc )
 665   {
 666     ProgressData progress(100);
 667     progress.sendTo(progressfnc);
 668
 669     filesystem::recursive_rmdir(packagescache_path_for_repoinfo(_pimpl->options, info));
 670     progress.toMax();
 671   }
 672
 673   void RepoManager::buildCache( const RepoInfo &info,
 674                                 CacheBuildPolicy policy,
 675                                 const ProgressData::ReceiverFnc & progressrcv )
 676   {
 677     assert_alias(info);
 678     Pathname rawpath = rawcache_path_for_repoinfo(_pimpl->options, info);
 679
 680     Pathname base = _pimpl->options.repoCachePath + info.escaped_alias();
 681     Pathname solvfile = base.extend(".solv");
 682
 683
 684     RepoStatus raw_metadata_status = metadataStatus(info);
 685     if ( raw_metadata_status.empty() )
 686     {
 687        /* if there is no cache at this point, we refresh the raw
 688           in case this is the first time - if it's !autorefresh,
 689           we may still refresh */
 690       refreshMetadata(info, RefreshIfNeeded, progressrcv );
 691       raw_metadata_status = metadataStatus(info);
 692     }
 693
 694     bool needs_cleaning = false;
 695     if ( isCached( info ) )
 696     {
 697       MIL << info.alias() << " is already cached." << endl;
 698       //data::RecordId id = store.lookupRepository(info.alias());
 699       RepoStatus cache_status = cacheStatus(info);
 700
 701       if ( cache_status.checksum() == raw_metadata_status.checksum() )
 702       {
 703         MIL << info.alias() << " cache is up to date with metadata." << endl;
 704         if ( policy == BuildIfNeeded ) {
 705           return;
 706         }
 707         else {
 708           MIL << info.alias() << " cache rebuild is forced" << endl;
 709         }
 710       }
 711
 712       needs_cleaning = true;
 713     }
 714
 715     ProgressData progress(100);
 716     callback::SendReport<ProgressReport> report;
 717     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
 718     progress.name(str::form(_("Building repository '%s' cache"), info.name().c_str()));
 719     progress.toMin();
 720
 721     if (needs_cleaning)
 722     {
 723       cleanCache(info);
 724     }
 725
 726     MIL << info.alias() << " building cache..." << endl;
 727     // do we have type?
 728     repo::RepoType repokind = info.type();
 729
 730     // if the type is unknown, try probing.
 731     switch ( repokind.toEnum() )
 732     {
 733       case RepoType::NONE_e:
 734         // unknown, probe the local metadata
 735         repokind = probe(rawpath.asUrl());
 736       break;
 737       default:
 738       break;
 739     }
 740
 741     MIL << "repo type is " << repokind << endl;
 742
 743     switch ( repokind.toEnum() )
 744     {
 745       case RepoType::RPMMD_e :
 746       case RepoType::YAST2_e :
 747       case RepoType::RPMPLAINDIR_e :
 748       {
 749         // Take care we unlink the solvfile on exception
 750         ManagedFile guard( solvfile, filesystem::unlink );
 751
 752         ostringstream cmd;
 753         if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 754         {
 755           cmd << str::form( "repo2solv.sh \"%s\" > \"%s\"", info.baseUrlsBegin()->getPathName().c_str(), solvfile.c_str() );
 756         } else
 757           cmd << str::form( "repo2solv.sh \"%s\" > \"%s\"", rawpath.c_str(), solvfile.c_str() );
 758
 759         MIL << "Executing: " << cmd.str() << endl;
 760         ExternalProgram prog( cmd.str(), ExternalProgram::Stderr_To_Stdout );
 761
 762         cmd << endl;
 763         for ( string output( prog.receiveLine() ); output.length(); output = prog.receiveLine() ) {
 764           WAR << "  " << output;
 765           cmd << "     " << output;
 766         }
 767
 768         int ret = prog.close();
 769         if ( ret != 0 )
 770         {
 771           RepoException ex(str::form("Failed to cache repo (%d).", ret));
 772           ex.remember( cmd.str() );
 773           ZYPP_THROW(ex);
 774         }
 775
 776         // We keep it.
 777         guard.resetDispose();
 778       }
 779       break;
 780       default:
 781         ZYPP_THROW(RepoUnknownTypeException("Unhandled repository type"));
 782       break;
 783     }
 784 #if 0
 785     switch ( repokind.toEnum() )
 786     {
 787       case RepoType::RPMMD_e :
 788       if (0)
 789       {
 790         CombinedProgressData subprogrcv( progress, 100);
 791         parser::yum::RepoParser parser(id, store, parser::yum::RepoParserOpts(), subprogrcv);
 792         parser.parse(rawpath);
 793           // no error
 794       }
 795       break;
 796       case RepoType::YAST2_e :
 797       if (0)
 798       {
 799         CombinedProgressData subprogrcv( progress, 100);
 800         parser::susetags::RepoParser parser(id, store, subprogrcv);
 801         parser.parse(rawpath);
 802         // no error
 803       }
 804       break;
 805
 806       default:
 807         ZYPP_THROW(RepoUnknownTypeException());
 808     }
 809 #endif
 810     // update timestamp and checksum
 811     //store.updateRepositoryStatus(id, raw_metadata_status);
 812     setCacheStatus(info.escaped_alias(), raw_metadata_status);
 813     MIL << "Commit cache.." << endl;
 814     //store.commit();
 815     //progress.toMax();
 816   }
 817
 818   ////////////////////////////////////////////////////////////////////////////
 819
 820   repo::RepoType RepoManager::probe( const Url &url ) const
 821   {
 822     if ( url.getScheme() == "dir" && ! PathInfo( url.getPathName() ).isDir() )
 823     {
 824       // Handle non existing local directory in advance, as
 825       // MediaSetAccess does not support it.
 826       return repo::RepoType::NONE;
 827     }
 828
 829     try
 830     {
 831       MediaSetAccess access(url);
 832       if ( access.doesFileExist("/repodata/repomd.xml") )
 833         return repo::RepoType::RPMMD;
 834       if ( access.doesFileExist("/content") )
 835         return repo::RepoType::YAST2;
 836
 837       // if it is a local url of type dir
 838       if ( (! media::MediaManager::downloads(url)) && ( url.getScheme() == "dir" ) )
 839       {
 840         Pathname path = Pathname(url.getPathName());
 841         if ( PathInfo(path).isDir() )
 842         {
 843           // allow empty dirs for now
 844           return repo::RepoType::RPMPLAINDIR;
 845         }
 846       }
 847     }
 848     catch ( const media::MediaException &e )
 849     {
 850       ZYPP_CAUGHT(e);
 851       RepoException enew("Error trying to read from " + url.asString());
 852       enew.remember(e);
 853       ZYPP_THROW(enew);
 854     }
 855     catch ( const Exception &e )
 856     {
 857       ZYPP_CAUGHT(e);
 858       Exception enew("Unknown error reading from " + url.asString());
 859       enew.remember(e);
 860       ZYPP_THROW(enew);
 861     }
 862
 863     return repo::RepoType::NONE;
 864   }
 865
 866   ////////////////////////////////////////////////////////////////////////////
 867
 868   void RepoManager::cleanCache( const RepoInfo &info,
 869                                 const ProgressData::ReceiverFnc & progressrcv )
 870   {
 871     ProgressData progress(100);
 872     progress.sendTo(progressrcv);
 873     progress.toMin();
 874
 875     unlink (_pimpl->options.repoCachePath / (info.escaped_alias() + ".solv"));
 876     progress.set(99);
 877     unlink (_pimpl->options.repoCachePath / (info.escaped_alias() + ".cookie"));
 878
 879     progress.toMax();
 880   }
 881
 882   void RepoManager::cleanTargetCache(const ProgressData::ReceiverFnc & progressrcv)
 883   {
 884     unlink (_pimpl->options.repoCachePath / (sat::Pool::systemRepoName() + ".solv"));
 885     unlink (_pimpl->options.repoCachePath / (sat::Pool::systemRepoName() + ".cookie"));
 886   }
 887
 888   ////////////////////////////////////////////////////////////////////////////
 889
 890   bool RepoManager::isCached( const RepoInfo &info ) const
 891   {
 892     Pathname name = _pimpl->options.repoCachePath;
 893     return PathInfo(name + Pathname(info.escaped_alias()).extend(".solv")).isExist();
 894   }
 895
 896   RepoStatus RepoManager::cacheStatus( const RepoInfo &info ) const
 897   {
 898
 899     Pathname base = _pimpl->options.repoCachePath + info.escaped_alias();
 900     Pathname cookiefile = base.extend(".cookie");
 901
 902     return RepoStatus::fromCookieFile(cookiefile);
 903   }
 904
 905   void RepoManager::setCacheStatus( const string &alias, const RepoStatus &status )
 906   {
 907     Pathname base = _pimpl->options.repoCachePath + alias;
 908     Pathname cookiefile = base.extend(".cookie");
 909
 910     status.saveToCookieFile(cookiefile);
 911   }
 912
 913   void RepoManager::loadFromCache( const RepoInfo & info,
 914                                    const ProgressData::ReceiverFnc & progressrcv )
 915   {
 916     assert_alias(info);
 917     string alias = info.escaped_alias();
 918     Pathname solvfile = (_pimpl->options.repoCachePath / alias).extend(".solv");
 919
 920     if ( ! PathInfo(solvfile).isExist() )
 921       ZYPP_THROW(RepoNotCachedException());
 922
 923     try
 924     {
 925       sat::Pool::instance().addRepoSolv( solvfile, info );
 926     }
 927     catch ( const Exception & exp )
 928     {
 929       ZYPP_CAUGHT( exp );
 930       MIL << "Try to handle exception by rebuilding the solv-file" << endl;
 931       cleanCache( info, progressrcv );
 932       buildCache( info, BuildIfNeeded, progressrcv );
 933
 934       sat::Pool::instance().addRepoSolv( solvfile, info );
 935     }
 936   }
 937
 938   ////////////////////////////////////////////////////////////////////////////
 939
 940   /**
 941    * Generate a non existing filename in a directory, using a base
 942    * name. For example if a directory contains 3 files
 943    *
 944    * |-- bar
 945    * |-- foo
 946    * `-- moo
 947    *
 948    * If you try to generate a unique filename for this directory,
 949    * based on "ruu" you will get "ruu", but if you use the base
 950    * "foo" you will get "foo_1"
 951    *
 952    * \param dir Directory where the file needs to be unique
 953    * \param basefilename string to base the filename on.
 954    */
 955   static Pathname generate_non_existing_name( const Pathname &dir,
 956                                               const std::string &basefilename )
 957   {
 958     string final_filename = basefilename;
 959     int counter = 1;
 960     while ( PathInfo(dir + final_filename).isExist() )
 961     {
 962       final_filename = basefilename + "_" + str::numstring(counter);
 963       counter++;
 964     }
 965     return dir + Pathname(final_filename);
 966   }
 967
 968   ////////////////////////////////////////////////////////////////////////////
 969
 970   /**
 971    * \short Generate a related filename from a repo info
 972    *
 973    * From a repo info, it will try to use the alias as a filename
 974    * escaping it if necessary. Other fallbacks can be added to
 975    * this function in case there is no way to use the alias
 976    */
 977   static std::string generate_filename( const RepoInfo &info )
 978   {
 979     std::string fnd="/";
 980     std::string rep="_";
 981     std::string filename = info.alias();
 982     // replace slashes with underscores
 983     size_t pos = filename.find(fnd);
 984     while(pos!=string::npos)
 985     {
 986       filename.replace(pos,fnd.length(),rep);
 987       pos = filename.find(fnd,pos+rep.length());
 988     }
 989     filename = Pathname(filename).extend(".repo").asString();
 990     MIL << "generating filename for repo [" << info.alias() << "] : '" << filename << "'" << endl;
 991     return filename;
 992   }
 993
 994
 995   ////////////////////////////////////////////////////////////////////////////
 996
 997   void RepoManager::addRepository( const RepoInfo &info,
 998                                    const ProgressData::ReceiverFnc & progressrcv )
 999   {
1000     assert_alias(info);
1001
1002     ProgressData progress(100);
1003     callback::SendReport<ProgressReport> report;
1004     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
1005     progress.name(str::form(_("Adding repository '%s'"), info.name().c_str()));
1006     progress.toMin();
1007
1008     std::list<RepoInfo> repos = knownRepositories();
1009     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1010           it != repos.end();
1011           ++it )
1012     {
1013       if ( info.alias() == (*it).alias() )
1014         ZYPP_THROW(RepoAlreadyExistsException(info.alias()));
1015     }
1016
1017     RepoInfo tosave = info;
1018
1019     // check the first url for now
1020     if ( _pimpl->options.probe )
1021     {
1022       DBG << "unknown repository type, probing" << endl;
1023
1024       RepoType probedtype;
1025       probedtype = probe(*tosave.baseUrlsBegin());
1026       if ( tosave.baseUrlsSize() > 0 )
1027       {
1028         if ( probedtype == RepoType::NONE )
1029           ZYPP_THROW(RepoUnknownTypeException());
1030         else
1031           tosave.setType(probedtype);
1032       }
1033     }
1034
1035     progress.set(50);
1036
1037     // assert the directory exists
1038     filesystem::assert_dir(_pimpl->options.knownReposPath);
1039
1040     Pathname repofile = generate_non_existing_name(_pimpl->options.knownReposPath,
1041                                                     generate_filename(tosave));
1042     // now we have a filename that does not exists
1043     MIL << "Saving repo in " << repofile << endl;
1044
1045     std::ofstream file(repofile.c_str());
1046     if (!file) {
1047       ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
1048     }
1049
1050     tosave.dumpRepoOn(file);
1051     progress.toMax();
1052     MIL << "done" << endl;
1053   }
1054
1055   void RepoManager::addRepositories( const Url &url,
1056                                      const ProgressData::ReceiverFnc & progressrcv )
1057   {
1058     std::list<RepoInfo> knownrepos = knownRepositories();
1059     std::list<RepoInfo> repos = readRepoFile(url);
1060     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1061           it != repos.end();
1062           ++it )
1063     {
1064       // look if the alias is in the known repos.
1065       for ( std::list<RepoInfo>::const_iterator kit = knownrepos.begin();
1066           kit != knownrepos.end();
1067           ++kit )
1068       {
1069         if ( (*it).alias() == (*kit).alias() )
1070         {
1071           ERR << "To be added repo " << (*it).alias() << " conflicts with existing repo " << (*kit).alias() << endl;
1072           ZYPP_THROW(RepoAlreadyExistsException((*it).alias()));
1073         }
1074       }
1075     }
1076
1077     string filename = Pathname(url.getPathName()).basename();
1078
1079     if ( filename == Pathname() )
1080       ZYPP_THROW(RepoException("Invalid repo file name at " + url.asString() ));
1081
1082     // assert the directory exists
1083     filesystem::assert_dir(_pimpl->options.knownReposPath);
1084
1085     Pathname repofile = generate_non_existing_name(_pimpl->options.knownReposPath, filename);
1086     // now we have a filename that does not exists
1087     MIL << "Saving " << repos.size() << " repo" << ( repos.size() ? "s" : "" ) << " in " << repofile << endl;
1088
1089     std::ofstream file(repofile.c_str());
1090     if (!file) {
1091       ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
1092     }
1093
1094     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1095           it != repos.end();
1096           ++it )
1097     {
1098       MIL << "Saving " << (*it).alias() << endl;
1099       (*it).dumpRepoOn(file);
1100     }
1101     MIL << "done" << endl;
1102   }
1103
1104   ////////////////////////////////////////////////////////////////////////////
1105
1106   void RepoManager::removeRepository( const RepoInfo & info,
1107                                       const ProgressData::ReceiverFnc & progressrcv)
1108   {
1109     ProgressData progress;
1110     callback::SendReport<ProgressReport> report;
1111     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
1112     progress.name(str::form(_("Removing repository '%s'"), info.name().c_str()));
1113
1114     MIL << "Going to delete repo " << info.alias() << endl;
1115
1116     std::list<RepoInfo> repos = knownRepositories();
1117     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1118           it != repos.end();
1119           ++it )
1120     {
1121       // they can be the same only if the provided is empty, that means
1122       // the provided repo has no alias
1123       // then skip
1124       if ( (!info.alias().empty()) && ( info.alias() != (*it).alias() ) )
1125         continue;
1126
1127       // TODO match by url
1128
1129       // we have a matcing repository, now we need to know
1130       // where it does come from.
1131       RepoInfo todelete = *it;
1132       if (todelete.filepath().empty())
1133       {
1134         ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
1135       }
1136       else
1137       {
1138         // figure how many repos are there in the file:
1139         std::list<RepoInfo> filerepos = repositories_in_file(todelete.filepath());
1140         if ( (filerepos.size() == 1) && ( filerepos.front().alias() == todelete.alias() ) )
1141         {
1142           // easy, only this one, just delete the file
1143           if ( filesystem::unlink(todelete.filepath()) != 0 )
1144           {
1145             ZYPP_THROW(RepoException("Can't delete " + todelete.filepath().asString()));
1146           }
1147           MIL << todelete.alias() << " sucessfully deleted." << endl;
1148         }
1149         else
1150         {
1151           // there are more repos in the same file
1152           // write them back except the deleted one.
1153           //TmpFile tmp;
1154           //std::ofstream file(tmp.path().c_str());
1155
1156           // assert the directory exists
1157           filesystem::assert_dir(todelete.filepath().dirname());
1158
1159           std::ofstream file(todelete.filepath().c_str());
1160           if (!file) {
1161             //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
1162             ZYPP_THROW (Exception( "Can't open " + todelete.filepath().asString() ) );
1163           }
1164           for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
1165                 fit != filerepos.end();
1166                 ++fit )
1167           {
1168             if ( (*fit).alias() != todelete.alias() )
1169               (*fit).dumpRepoOn(file);
1170           }
1171         }
1172
1173         CombinedProgressData subprogrcv(progress, 70);
1174         CombinedProgressData cleansubprogrcv(progress, 30);
1175         // now delete it from cache
1176         if ( isCached(todelete) )
1177           cleanCache( todelete, subprogrcv);
1178         // now delete metadata (#301037)
1179         cleanMetadata( todelete, cleansubprogrcv);
1180         MIL << todelete.alias() << " sucessfully deleted." << endl;
1181         return;
1182       } // else filepath is empty
1183
1184     }
1185     // should not be reached on a sucess workflow
1186     ZYPP_THROW(RepoNotFoundException(info));
1187   }
1188
1189   ////////////////////////////////////////////////////////////////////////////
1190
1191   void RepoManager::modifyRepository( const std::string &alias,
1192                                       const RepoInfo & newinfo,
1193                                       const ProgressData::ReceiverFnc & progressrcv )
1194   {
1195     RepoInfo toedit = getRepositoryInfo(alias);
1196
1197     // check if the new alias already exists when renaming the repo
1198     if (alias != newinfo.alias())
1199     {
1200       std::list<RepoInfo> repos = knownRepositories();
1201       for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1202             it != repos.end();
1203             ++it )
1204       {
1205         if ( newinfo.alias() == (*it).alias() )
1206           ZYPP_THROW(RepoAlreadyExistsException(newinfo.alias()));
1207       }
1208     }
1209
1210     if (toedit.filepath().empty())
1211     {
1212       ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
1213     }
1214     else
1215     {
1216       // figure how many repos are there in the file:
1217       std::list<RepoInfo> filerepos = repositories_in_file(toedit.filepath());
1218
1219       // there are more repos in the same file
1220       // write them back except the deleted one.
1221       //TmpFile tmp;
1222       //std::ofstream file(tmp.path().c_str());
1223
1224       // assert the directory exists
1225       filesystem::assert_dir(toedit.filepath().dirname());
1226
1227       std::ofstream file(toedit.filepath().c_str());
1228       if (!file) {
1229         //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
1230         ZYPP_THROW (Exception( "Can't open " + toedit.filepath().asString() ) );
1231       }
1232       for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
1233             fit != filerepos.end();
1234             ++fit )
1235       {
1236           // if the alias is different, dump the original
1237           // if it is the same, dump the provided one
1238           if ( (*fit).alias() != toedit.alias() )
1239             (*fit).dumpRepoOn(file);
1240           else
1241             newinfo.dumpRepoOn(file);
1242       }
1243     }
1244   }
1245
1246   ////////////////////////////////////////////////////////////////////////////
1247
1248   RepoInfo RepoManager::getRepositoryInfo( const std::string &alias,
1249                                            const ProgressData::ReceiverFnc & progressrcv )
1250   {
1251     std::list<RepoInfo> repos = knownRepositories();
1252     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1253           it != repos.end();
1254           ++it )
1255     {
1256       if ( (*it).alias() == alias )
1257         return *it;
1258     }
1259     RepoInfo info;
1260     info.setAlias(info.alias());
1261     ZYPP_THROW(RepoNotFoundException(info));
1262   }
1263
1264   ////////////////////////////////////////////////////////////////////////////
1265
1266   RepoInfo RepoManager::getRepositoryInfo( const Url & url,
1267                                            const url::ViewOption & urlview,
1268                                            const ProgressData::ReceiverFnc & progressrcv )
1269   {
1270     std::list<RepoInfo> repos = knownRepositories();
1271     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1272           it != repos.end();
1273           ++it )
1274     {
1275       for(RepoInfo::urls_const_iterator urlit = (*it).baseUrlsBegin();
1276           urlit != (*it).baseUrlsEnd();
1277           ++urlit)
1278       {
1279         if ((*urlit).asString(urlview) == url.asString(urlview))
1280           return *it;
1281       }
1282     }
1283     RepoInfo info;
1284     info.setAlias(info.alias());
1285     info.setBaseUrl(url);
1286     ZYPP_THROW(RepoNotFoundException(info));
1287   }
1288
1289   ////////////////////////////////////////////////////////////////////////////
1290
1291   std::ostream & operator<<( std::ostream & str, const RepoManager & obj )
1292   {
1293     return str << *obj._pimpl;
1294   }
1295
1296   /////////////////////////////////////////////////////////////////
1297 } // namespace zypp
1298 ///////////////////////////////////////////////////////////////////