zypp/RepoManager.cc

   1 /*---------------------------------------------------------------------\
   2 |                          ____ _   __ __ ___                          |
   3 |                         |__  / \ / / . \ . \                         |
   4 |                           / / \ V /|  _/  _/                         |
   5 |                          / /__ | | | | | |                           |
   6 |                         /_____||_| |_| |_|                           |
   7 |                                                                      |
   8 \---------------------------------------------------------------------*/
   9 /** \file       zypp/RepoManager.cc
  10  *
  11 */
  12
  13 #include <cstdlib>
  14 #include <iostream>
  15 #include <fstream>
  16 #include <sstream>
  17 #include <list>
  18 #include <algorithm>
  19 #include "zypp/base/InputStream.h"
  20 #include "zypp/base/Logger.h"
  21 #include "zypp/base/Gettext.h"
  22 #include "zypp/base/Function.h"
  23 #include "zypp/base/Regex.h"
  24 #include "zypp/PathInfo.h"
  25 #include "zypp/TmpPath.h"
  26
  27 #include "zypp/repo/RepoException.h"
  28 #include "zypp/RepoManager.h"
  29
  30 #include "zypp/cache/SolvStore.h"
  31 #include "zypp/repo/cached/RepoImpl.h"
  32 #include "zypp/media/MediaManager.h"
  33 #include "zypp/MediaSetAccess.h"
  34
  35 #include "zypp/parser/RepoFileReader.h"
  36 #include "zypp/repo/yum/Downloader.h"
  37 #include "zypp/parser/yum/RepoParser.h"
  38 //#include "zypp/parser/plaindir/RepoParser.h"
  39 #include "zypp/repo/susetags/Downloader.h"
  40 #include "zypp/parser/susetags/RepoParser.h"
  41
  42 #include "zypp/ZYppCallbacks.h"
  43
  44 #include "satsolver/pool.h"
  45 #include "satsolver/repo_solv.h"
  46
  47 using namespace std;
  48 using namespace zypp;
  49 using namespace zypp::repo;
  50 using namespace zypp::filesystem;
  51
  52 using namespace zypp::repo;
  53
  54 ///////////////////////////////////////////////////////////////////
  55 namespace zypp
  56 { /////////////////////////////////////////////////////////////////
  57
  58   ///////////////////////////////////////////////////////////////////
  59   //
  60   //    CLASS NAME : RepoManagerOptions
  61   //
  62   ///////////////////////////////////////////////////////////////////
  63
  64   RepoManagerOptions::RepoManagerOptions()
  65   {
  66     repoCachePath    = ZConfig::instance().repoCachePath();
  67     repoRawCachePath = ZConfig::instance().repoMetadataPath();
  68     knownReposPath   = ZConfig::instance().knownReposPath();
  69   }
  70
  71   ////////////////////////////////////////////////////////////////////////////
  72
  73   /**
  74     * \short Simple callback to collect the results
  75     *
  76     * Classes like RepoFileParser call the callback
  77     * once per each repo in a file.
  78     *
  79     * Passing this functor as callback, you can collect
  80     * all resuls at the end, without dealing with async
  81     * code.
  82     */
  83     struct RepoCollector
  84     {
  85       RepoCollector()
  86       {
  87         MIL << endl;
  88       }
  89
  90       ~RepoCollector()
  91       {
  92         MIL << endl;
  93       }
  94
  95       bool collect( const RepoInfo &repo )
  96       {
  97         //MIL << "here in collector: " << repo.alias() << endl;
  98         repos.push_back(repo);
  99         //MIL << "added: " << repo.alias() << endl;
 100         return true;
 101       }
 102
 103       RepoInfoList repos;
 104     };
 105
 106   ////////////////////////////////////////////////////////////////////////////
 107
 108    /**
 109     * \short Internal version of clean cache
 110     *
 111     * Takes an extra SolvStore reference, so we avoid internally
 112     * having 2 SolvStores writing to the same database.
 113     */
 114   static void cleanCacheInternal( cache::SolvStore &store,
 115                                   const RepoInfo &info,
 116                                   const ProgressData::ReceiverFnc & progressrcv = ProgressData::ReceiverFnc() )
 117   {
 118     ProgressData progress;
 119     callback::SendReport<ProgressReport> report;
 120     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
 121     progress.name(str::form(_("Cleaning repository '%s' cache"), info.name().c_str()));
 122
 123     if ( !store.isCached(info.alias()) )
 124       return;
 125
 126     MIL << info.alias() << " cleaning cache..." << endl;
 127     data::RecordId id = store.lookupRepository(info.alias());
 128
 129     CombinedProgressData subprogrcv(progress);
 130
 131     store.cleanRepository(id, subprogrcv);
 132   }
 133
 134   ////////////////////////////////////////////////////////////////////////////
 135
 136   /**
 137    * Reads RepoInfo's from a repo file.
 138    *
 139    * \param file pathname of the file to read.
 140    */
 141   static std::list<RepoInfo> repositories_in_file( const Pathname & file )
 142   {
 143     MIL << "repo file: " << file << endl;
 144     RepoCollector collector;
 145     parser::RepoFileReader parser( file, bind( &RepoCollector::collect, &collector, _1 ) );
 146     return collector.repos;
 147   }
 148
 149   ////////////////////////////////////////////////////////////////////////////
 150
 151   std::list<RepoInfo> readRepoFile(const Url & repo_file)
 152    {
 153      // no interface to download a specific file, using workaround:
 154      //! \todo add MediaManager::provideFile(Url file_url) to easily access any file URLs? (no need for media access id or media_nr)
 155      Url url(repo_file);
 156      Pathname path(url.getPathName());
 157      url.setPathName ("/");
 158      MediaSetAccess access(url);
 159      Pathname local = access.provideFile(path);
 160
 161      DBG << "reading repo file " << repo_file << ", local path: " << local << endl;
 162
 163      return repositories_in_file(local);
 164    }
 165
 166   ////////////////////////////////////////////////////////////////////////////
 167
 168   /**
 169    * \short List of RepoInfo's from a directory
 170    *
 171    * Goes trough every file ending with ".repo" in a directory and adds all
 172    * RepoInfo's contained in that file.
 173    *
 174    * \param dir pathname of the directory to read.
 175    */
 176   static std::list<RepoInfo> repositories_in_dir( const Pathname &dir )
 177   {
 178     MIL << "directory " << dir << endl;
 179     list<RepoInfo> repos;
 180     list<Pathname> entries;
 181     if ( filesystem::readdir( entries, Pathname(dir), false ) != 0 )
 182       ZYPP_THROW(Exception("failed to read directory"));
 183
 184     str::regex allowedRepoExt("^\\.repo(_[0-9]+)?$");
 185     for ( list<Pathname>::const_iterator it = entries.begin(); it != entries.end(); ++it )
 186     {
 187       if (str::regex_match(it->extension(), allowedRepoExt))
 188       {
 189         list<RepoInfo> tmp = repositories_in_file( *it );
 190         repos.insert( repos.end(), tmp.begin(), tmp.end() );
 191
 192         //std::copy( collector.repos.begin(), collector.repos.end(), std::back_inserter(repos));
 193         //MIL << "ok" << endl;
 194       }
 195     }
 196     return repos;
 197   }
 198
 199   ////////////////////////////////////////////////////////////////////////////
 200
 201   static void assert_alias( const RepoInfo &info )
 202   {
 203     if (info.alias().empty())
 204         ZYPP_THROW(RepoNoAliasException());
 205   }
 206
 207   ////////////////////////////////////////////////////////////////////////////
 208
 209   static void assert_urls( const RepoInfo &info )
 210   {
 211     if (info.baseUrlsEmpty())
 212         ZYPP_THROW(RepoNoUrlException());
 213   }
 214
 215   ////////////////////////////////////////////////////////////////////////////
 216
 217   /**
 218    * \short Calculates the raw cache path for a repository
 219    */
 220   static Pathname rawcache_path_for_repoinfo( const RepoManagerOptions &opt, const RepoInfo &info )
 221   {
 222     assert_alias(info);
 223     return opt.repoRawCachePath + info.alias();
 224   }
 225
 226   ///////////////////////////////////////////////////////////////////
 227   //
 228   //    CLASS NAME : RepoManager::Impl
 229   //
 230   ///////////////////////////////////////////////////////////////////
 231
 232   /**
 233    * \short RepoManager implementation.
 234    */
 235   struct RepoManager::Impl
 236   {
 237     Impl( const RepoManagerOptions &opt )
 238       : options(opt)
 239     {
 240
 241     }
 242
 243     Impl()
 244     {
 245
 246     }
 247
 248     RepoManagerOptions options;
 249
 250   public:
 251     /** Offer default Impl. */
 252     static shared_ptr<Impl> nullimpl()
 253     {
 254       static shared_ptr<Impl> _nullimpl( new Impl );
 255       return _nullimpl;
 256     }
 257
 258   private:
 259     friend Impl * rwcowClone<Impl>( const Impl * rhs );
 260     /** clone for RWCOW_pointer */
 261     Impl * clone() const
 262     { return new Impl( *this ); }
 263   };
 264
 265   ///////////////////////////////////////////////////////////////////
 266
 267   /** \relates RepoManager::Impl Stream output */
 268   inline std::ostream & operator<<( std::ostream & str, const RepoManager::Impl & obj )
 269   {
 270     return str << "RepoManager::Impl";
 271   }
 272
 273   ///////////////////////////////////////////////////////////////////
 274   //
 275   //    CLASS NAME : RepoManager
 276   //
 277   ///////////////////////////////////////////////////////////////////
 278
 279   RepoManager::RepoManager( const RepoManagerOptions &opt )
 280   : _pimpl( new Impl(opt) )
 281   {}
 282
 283   ////////////////////////////////////////////////////////////////////////////
 284
 285   RepoManager::~RepoManager()
 286   {}
 287
 288   ////////////////////////////////////////////////////////////////////////////
 289
 290   std::list<RepoInfo> RepoManager::knownRepositories() const
 291   {
 292     MIL << endl;
 293
 294     if ( PathInfo(_pimpl->options.knownReposPath).isExist() )
 295     {
 296       RepoInfoList repos = repositories_in_dir(_pimpl->options.knownReposPath);
 297       for ( RepoInfoList::iterator it = repos.begin();
 298             it != repos.end();
 299             ++it )
 300       {
 301         // set the metadata path for the repo
 302         Pathname metadata_path = rawcache_path_for_repoinfo(_pimpl->options, (*it));
 303         (*it).setMetadataPath(metadata_path);
 304       }
 305       return repos;
 306     }
 307     else
 308       return std::list<RepoInfo>();
 309
 310     MIL << endl;
 311   }
 312
 313   ////////////////////////////////////////////////////////////////////////////
 314
 315   Pathname RepoManager::metadataPath( const RepoInfo &info ) const
 316   {
 317     return rawcache_path_for_repoinfo(_pimpl->options, info );
 318   }
 319
 320   ////////////////////////////////////////////////////////////////////////////
 321
 322   RepoStatus RepoManager::metadataStatus( const RepoInfo &info ) const
 323   {
 324     Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 325     RepoType repokind = info.type();
 326     RepoStatus status;
 327
 328     switch ( repokind.toEnum() )
 329     {
 330       case RepoType::NONE_e:
 331       // unknown, probe the local metadata
 332         repokind = probe(rawpath.asUrl());
 333       break;
 334       default:
 335       break;
 336     }
 337
 338     switch ( repokind.toEnum() )
 339     {
 340       case RepoType::RPMMD_e :
 341       {
 342         status = RepoStatus( rawpath + "/repodata/repomd.xml");
 343       }
 344       break;
 345
 346       case RepoType::YAST2_e :
 347       {
 348         // the order of RepoStatus && RepoStatus matters! (#304310)
 349         status = RepoStatus( rawpath + "/content") && (RepoStatus( rawpath + "/media.1/media"));
 350       }
 351       break;
 352
 353       case RepoType::RPMPLAINDIR_e :
 354       {
 355         if ( PathInfo(Pathname(rawpath + "/cookie")).isExist() )
 356           status = RepoStatus( rawpath + "/cookie");
 357       }
 358       break;
 359
 360       case RepoType::NONE_e :
 361         // Return default RepoStatus in case of RepoType::NONE
 362         // indicating it should be created?
 363         // ZYPP_THROW(RepoUnknownTypeException());
 364         break;
 365     }
 366     return status;
 367   }
 368
 369   void RepoManager::touchIndexFile(const RepoInfo & info)
 370   {
 371     Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 372
 373     RepoType repokind = info.type();
 374     if ( repokind.toEnum() == RepoType::NONE_e )
 375       // unknown, probe the local metadata
 376       repokind = probe(rawpath.asUrl());
 377     // if still unknown, just return
 378     if (repokind == RepoType::NONE_e)
 379       return;
 380
 381     Pathname p;
 382     switch ( repokind.toEnum() )
 383     {
 384       case RepoType::RPMMD_e :
 385         p = Pathname(rawpath + "/repodata/repomd.xml");
 386         break;
 387
 388       case RepoType::YAST2_e :
 389         p = Pathname(rawpath + "/content");
 390         break;
 391
 392       case RepoType::RPMPLAINDIR_e :
 393         p = Pathname(rawpath + "/cookie");
 394         break;
 395
 396       case RepoType::NONE_e :
 397       default:
 398         break;
 399     }
 400
 401     // touch the file, ignore error (they are logged anyway)
 402     filesystem::touch(p);
 403   }
 404
 405   bool RepoManager::checkIfToRefreshMetadata( const RepoInfo &info,
 406                                               const Url &url,
 407                                               RawMetadataRefreshPolicy policy )
 408   {
 409     assert_alias(info);
 410
 411     RepoStatus oldstatus;
 412     RepoStatus newstatus;
 413
 414     try
 415     {
 416       MIL << "Going to try to check whether refresh is needed for " << url << endl;
 417
 418       repo::RepoType repokind = info.type();
 419
 420       // if the type is unknown, try probing.
 421       switch ( repokind.toEnum() )
 422       {
 423         case RepoType::NONE_e:
 424           // unknown, probe it
 425           repokind = probe(url);
 426         break;
 427         default:
 428         break;
 429       }
 430
 431       Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 432       filesystem::assert_dir(rawpath);
 433       oldstatus = metadataStatus(info);
 434
 435       // now we've got the old (cached) status, we can decide repo.refresh.delay
 436       if (policy != RefreshForced)
 437       {
 438         // difference in seconds
 439         double diff = difftime(
 440           (Date::ValueType)Date::now(),
 441           (Date::ValueType)oldstatus.timestamp()) / 60;
 442
 443         DBG << "oldstatus: " << (Date::ValueType)oldstatus.timestamp() << endl;
 444         DBG << "current time: " << (Date::ValueType)Date::now() << endl;
 445         DBG << "last refresh = " << diff << " minutes ago" << endl;
 446
 447         if (diff < ZConfig::instance().repo_refresh_delay())
 448         {
 449           MIL << "Repository '" << info.alias()
 450               << "' has been refreshed less than repo.refresh.delay ("
 451               << ZConfig::instance().repo_refresh_delay()
 452               << ") minutes ago. Advising to skip refresh" << endl;
 453           return false;
 454         }
 455       }
 456
 457       // create temp dir as sibling of rawpath
 458       filesystem::TmpDir tmpdir( filesystem::TmpDir::makeSibling( rawpath ) );
 459
 460       if ( ( repokind.toEnum() == RepoType::RPMMD_e ) ||
 461            ( repokind.toEnum() == RepoType::YAST2_e ) )
 462       {
 463         MediaSetAccess media(url);
 464         shared_ptr<repo::Downloader> downloader_ptr;
 465
 466         if ( repokind.toEnum() == RepoType::RPMMD_e )
 467           downloader_ptr.reset(new yum::Downloader(info.path()));
 468         else
 469           downloader_ptr.reset( new susetags::Downloader(info.path()));
 470
 471         RepoStatus newstatus = downloader_ptr->status(media);
 472         bool refresh = false;
 473         if ( oldstatus.checksum() == newstatus.checksum() )
 474         {
 475           MIL << "repo has not changed" << endl;
 476           if ( policy == RefreshForced )
 477           {
 478             MIL << "refresh set to forced" << endl;
 479             refresh = true;
 480           }
 481         }
 482         else
 483         {
 484           MIL << "repo has changed, going to refresh" << endl;
 485           refresh = true;
 486         }
 487
 488         if (!refresh)
 489           touchIndexFile(info);
 490
 491         return refresh;
 492       }
 493 #if 0
 494       else if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 495       {
 496         RepoStatus newstatus = parser::plaindir::dirStatus(url.getPathName());
 497         bool refresh = false;
 498         if ( oldstatus.checksum() == newstatus.checksum() )
 499         {
 500           MIL << "repo has not changed" << endl;
 501           if ( policy == RefreshForced )
 502           {
 503             MIL << "refresh set to forced" << endl;
 504             refresh = true;
 505           }
 506         }
 507         else
 508         {
 509           MIL << "repo has changed, going to refresh" << endl;
 510           refresh = true;
 511         }
 512
 513         if (!refresh)
 514           touchIndexFile(info);
 515
 516         return refresh;
 517       }
 518 #endif
 519       else
 520       {
 521         ZYPP_THROW(RepoUnknownTypeException());
 522       }
 523     }
 524     catch ( const Exception &e )
 525     {
 526       ZYPP_CAUGHT(e);
 527       ERR << "refresh check failed for " << url << endl;
 528       ZYPP_RETHROW(e);
 529     }
 530
 531     return true; // default
 532   }
 533
 534   void RepoManager::refreshMetadata( const RepoInfo &info,
 535                                      RawMetadataRefreshPolicy policy,
 536                                      const ProgressData::ReceiverFnc & progress )
 537   {
 538     assert_alias(info);
 539     assert_urls(info);
 540
 541     // we will throw this later if no URL checks out fine
 542     RepoException rexception(_("Valid metadata not found at specified URL(s)"));
 543
 544     // try urls one by one
 545     for ( RepoInfo::urls_const_iterator it = info.baseUrlsBegin(); it != info.baseUrlsEnd(); ++it )
 546     {
 547       try
 548       {
 549         Url url(*it);
 550
 551         // check whether to refresh metadata
 552         // if the check fails for this url, it throws, so another url will be checked
 553         if (!checkIfToRefreshMetadata(info, url, policy))
 554           return;
 555
 556         MIL << "Going to refresh metadata from " << url << endl;
 557
 558         repo::RepoType repokind = info.type();
 559
 560         // if the type is unknown, try probing.
 561         switch ( repokind.toEnum() )
 562         {
 563           case RepoType::NONE_e:
 564             // unknown, probe it
 565             repokind = probe(*it);
 566           break;
 567           default:
 568           break;
 569         }
 570
 571         Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 572         filesystem::assert_dir(rawpath);
 573
 574         // create temp dir as sibling of rawpath
 575         filesystem::TmpDir tmpdir( filesystem::TmpDir::makeSibling( rawpath ) );
 576
 577         if ( ( repokind.toEnum() == RepoType::RPMMD_e ) ||
 578              ( repokind.toEnum() == RepoType::YAST2_e ) )
 579         {
 580           MediaSetAccess media(url);
 581           shared_ptr<repo::Downloader> downloader_ptr;
 582
 583           if ( repokind.toEnum() == RepoType::RPMMD_e )
 584             downloader_ptr.reset(new yum::Downloader(info.path()));
 585           else
 586             downloader_ptr.reset( new susetags::Downloader(info.path()));
 587
 588           /**
 589            * Given a downloader, sets the other repos raw metadata
 590            * path as cache paths for the fetcher, so if another
 591            * repo has the same file, it will not download it
 592            * but copy it from the other repository
 593            */
 594           std::list<RepoInfo> repos = knownRepositories();
 595           for ( std::list<RepoInfo>::const_iterator it = repos.begin();
 596                 it != repos.end();
 597                 ++it )
 598           {
 599             downloader_ptr->addCachePath(rawcache_path_for_repoinfo( _pimpl->options, *it ));
 600           }
 601
 602           downloader_ptr->download( media, tmpdir.path());
 603         }
 604 #if 0
 605         else if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 606         {
 607           RepoStatus newstatus = parser::plaindir::dirStatus(url.getPathName());
 608
 609           std::ofstream file(( tmpdir.path() + "/cookie").c_str());
 610           if (!file) {
 611             ZYPP_THROW (Exception( "Can't open " + tmpdir.path().asString() + "/cookie" ) );
 612           }
 613           file << url << endl;
 614           file << newstatus.checksum() << endl;
 615
 616           file.close();
 617         }
 618 #endif
 619         else
 620         {
 621           ZYPP_THROW(RepoUnknownTypeException());
 622         }
 623
 624         // ok we have the metadata, now exchange
 625         // the contents
 626         TmpDir oldmetadata( TmpDir::makeSibling( rawpath ) );
 627         filesystem::rename( rawpath, oldmetadata.path() );
 628         // move the just downloaded there
 629         filesystem::rename( tmpdir.path(), rawpath );
 630         // we are done.
 631         return;
 632       }
 633       catch ( const Exception &e )
 634       {
 635         ZYPP_CAUGHT(e);
 636         ERR << "Trying another url..." << endl;
 637
 638         // remember the exception caught for the *first URL*
 639         // if all other URLs fail, the rexception will be thrown with the
 640         // cause of the problem of the first URL remembered
 641         if (it == info.baseUrlsBegin())
 642           rexception.remember(e);
 643       }
 644     } // for every url
 645     ERR << "No more urls..." << endl;
 646     ZYPP_THROW(rexception);
 647   }
 648
 649   ////////////////////////////////////////////////////////////////////////////
 650
 651   void RepoManager::cleanMetadata( const RepoInfo &info,
 652                                    const ProgressData::ReceiverFnc & progressfnc )
 653   {
 654     ProgressData progress(100);
 655     progress.sendTo(progressfnc);
 656
 657     filesystem::recursive_rmdir(rawcache_path_for_repoinfo(_pimpl->options, info));
 658     progress.toMax();
 659   }
 660
 661   void RepoManager::buildCache( const RepoInfo &info,
 662                                 CacheBuildPolicy policy,
 663                                 const ProgressData::ReceiverFnc & progressrcv )
 664   {
 665     assert_alias(info);
 666     Pathname rawpath = rawcache_path_for_repoinfo(_pimpl->options, info);
 667
 668     cache::SolvStore store(_pimpl->options.repoCachePath);
 669
 670     RepoStatus raw_metadata_status = metadataStatus(info);
 671     if ( raw_metadata_status.empty() )
 672     {
 673       ZYPP_THROW(RepoMetadataException(info));
 674     }
 675
 676     bool needs_cleaning = false;
 677     if ( store.isCached( info.alias() ) )
 678     {
 679       MIL << info.alias() << " is already cached." << endl;
 680       data::RecordId id = store.lookupRepository(info.alias());
 681       RepoStatus cache_status = store.repositoryStatus(info.alias());
 682
 683       if ( cache_status.checksum() == raw_metadata_status.checksum() )
 684       {
 685         MIL << info.alias() << " cache is up to date with metadata." << endl;
 686         if ( policy == BuildIfNeeded ) {
 687           return;
 688         }
 689         else {
 690           MIL << info.alias() << " cache rebuild is forced" << endl;
 691         }
 692       }
 693
 694       needs_cleaning = true;
 695     }
 696
 697     ProgressData progress(100);
 698     callback::SendReport<ProgressReport> report;
 699     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
 700     progress.name(str::form(_("Building repository '%s' cache"), info.name().c_str()));
 701     progress.toMin();
 702
 703     if (needs_cleaning)
 704     {
 705       Pathname name = _pimpl->options.repoCachePath;
 706       data::RecordId id = store.lookupRepository(info.alias());
 707       ostringstream os;
 708       os << id.get();
 709       name += os.str() + ".solv";
 710       unlink (name);
 711       cleanCacheInternal( store, info);
 712     }
 713
 714     MIL << info.alias() << " building cache..." << endl;
 715     data::RecordId id = store.lookupOrAppendRepository(info.alias());
 716     // do we have type?
 717     repo::RepoType repokind = info.type();
 718
 719     // if the type is unknown, try probing.
 720     switch ( repokind.toEnum() )
 721     {
 722       case RepoType::NONE_e:
 723         // unknown, probe the local metadata
 724         repokind = probe(rawpath.asUrl());
 725       break;
 726       default:
 727       break;
 728     }
 729
 730
 731     switch ( repokind.toEnum() )
 732     {
 733       case RepoType::RPMMD_e :
 734       case RepoType::YAST2_e :
 735       {
 736         Pathname name = _pimpl->options.repoCachePath;
 737         ostringstream os;
 738         os << id.get();
 739         name += os.str() + ".solv";
 740         string cmd = "repo2solv.sh \"";
 741         cmd += rawpath.asString() + "\" > " + name.asString();
 742         int ret = system (cmd.c_str());
 743         if (WIFEXITED (ret) && WEXITSTATUS (ret) != 0)
 744           ZYPP_THROW(RepoUnknownTypeException());
 745       }
 746       break;
 747       default:
 748       break;
 749     }
 750
 751     switch ( repokind.toEnum() )
 752     {
 753       case RepoType::RPMMD_e :
 754       if (0)
 755       {
 756         CombinedProgressData subprogrcv( progress, 100);
 757         parser::yum::RepoParser parser(id, store, parser::yum::RepoParserOpts(), subprogrcv);
 758         parser.parse(rawpath);
 759           // no error
 760       }
 761       break;
 762       case RepoType::YAST2_e :
 763       if (0)
 764       {
 765         CombinedProgressData subprogrcv( progress, 100);
 766         parser::susetags::RepoParser parser(id, store, subprogrcv);
 767         parser.parse(rawpath);
 768         // no error
 769       }
 770       break;
 771 #if 0
 772       case RepoType::RPMPLAINDIR_e :
 773       {
 774         CombinedProgressData subprogrcv( progress, 100);
 775         InputStream is(rawpath + "cookie");
 776         string buffer;
 777         getline( is.stream(), buffer);
 778         Url url(buffer);
 779         parser::plaindir::RepoParser parser(id, store, subprogrcv);
 780         parser.parse(url.getPathName());
 781       }
 782       break;
 783 #endif
 784       default:
 785         ZYPP_THROW(RepoUnknownTypeException());
 786     }
 787
 788     // update timestamp and checksum
 789     store.updateRepositoryStatus(id, raw_metadata_status);
 790
 791     MIL << "Commit cache.." << endl;
 792     store.commit();
 793     //progress.toMax();
 794   }
 795
 796   ////////////////////////////////////////////////////////////////////////////
 797
 798   repo::RepoType RepoManager::probe( const Url &url ) const
 799   {
 800     if ( url.getScheme() == "dir" && ! PathInfo( url.getPathName() ).isDir() )
 801     {
 802       // Handle non existing local directory in advance, as
 803       // MediaSetAccess does not support it.
 804       return repo::RepoType::NONE;
 805     }
 806
 807     try
 808     {
 809       MediaSetAccess access(url);
 810       if ( access.doesFileExist("/repodata/repomd.xml") )
 811         return repo::RepoType::RPMMD;
 812       if ( access.doesFileExist("/content") )
 813         return repo::RepoType::YAST2;
 814
 815       // if it is a local url of type dir
 816       if ( (! media::MediaManager::downloads(url)) && ( url.getScheme() == "dir" ) )
 817       {
 818         Pathname path = Pathname(url.getPathName());
 819         if ( PathInfo(path).isDir() )
 820         {
 821           // allow empty dirs for now
 822           return repo::RepoType::RPMPLAINDIR;
 823         }
 824       }
 825     }
 826     catch ( const media::MediaException &e )
 827     {
 828       ZYPP_CAUGHT(e);
 829       RepoException enew("Error trying to read from " + url.asString());
 830       enew.remember(e);
 831       ZYPP_THROW(enew);
 832     }
 833     catch ( const Exception &e )
 834     {
 835       ZYPP_CAUGHT(e);
 836       Exception enew("Unknown error reading from " + url.asString());
 837       enew.remember(e);
 838       ZYPP_THROW(enew);
 839     }
 840
 841     return repo::RepoType::NONE;
 842   }
 843
 844   ////////////////////////////////////////////////////////////////////////////
 845
 846   void RepoManager::cleanCache( const RepoInfo &info,
 847                                 const ProgressData::ReceiverFnc & progressrcv )
 848   {
 849     Pathname name = _pimpl->options.repoCachePath;
 850     name += info.alias() + ".solv";
 851     unlink (name);
 852   }
 853
 854   ////////////////////////////////////////////////////////////////////////////
 855
 856   bool RepoManager::isCached( const RepoInfo &info ) const
 857   {
 858     Pathname name = _pimpl->options.repoCachePath;
 859     return PathInfo(name + Pathname(info.alias()).extend(".solv")).isExist();
 860   }
 861
 862   RepoStatus RepoManager::cacheStatus( const RepoInfo &info ) const
 863   {
 864     Pathname name = _pimpl->options.repoCachePath;
 865     return RepoStatus(name + Pathname(info.alias()).extend(".solv"));
 866   }
 867
 868   map<data::RecordId, Repo *> repo2solv;
 869
 870   Repository RepoManager::createFromCache( const RepoInfo &info,
 871                                            const ProgressData::ReceiverFnc & progressrcv )
 872   {
 873     callback::SendReport<ProgressReport> report;
 874     ProgressData progress;
 875     progress.sendTo(ProgressReportAdaptor( progressrcv, report ));
 876     //progress.sendTo( progressrcv );
 877     progress.name(str::form(_("Reading repository '%s' cache"), info.name().c_str()));
 878
 879     //_pimpl->options.repoCachePath
 880     if ( ! isCached( info ) )
 881       ZYPP_THROW(RepoNotCachedException());
 882
 883     MIL << "Repository " << info.alias() << " is cached" << endl;
 884
 885     //sat::Pool satpool( sat::Pool::instance() );
 886
 887     //Pathname name = _pimpl->options.repoCachePath + Pathname(info.alias()).extend(".solv");
 888
 889     try
 890     {
 891       //satpool.addRepoSolv(name);
 892     }
 893     catch ( const Exception &e )
 894     {
 895
 896     }
 897
 898     CombinedProgressData subprogrcv(progress);
 899
 900 //     repo::cached::RepoOptions opts( info, _pimpl->options.repoCachePath, id );
 901 //     opts.readingResolvablesProgress = subprogrcv;
 902 //     //opts.repo = repo;
 903 //     repo::cached::RepoImpl::Ptr repoimpl =
 904 //         new repo::cached::RepoImpl( opts );
 905 //
 906 //     repoimpl->resolvables();
 907     // read the resolvables from cache
 908     return Repository::noRepository;
 909   }
 910
 911   ////////////////////////////////////////////////////////////////////////////
 912
 913   /**
 914    * Generate a non existing filename in a directory, using a base
 915    * name. For example if a directory contains 3 files
 916    *
 917    * |-- bar
 918    * |-- foo
 919    * `-- moo
 920    *
 921    * If you try to generate a unique filename for this directory,
 922    * based on "ruu" you will get "ruu", but if you use the base
 923    * "foo" you will get "foo_1"
 924    *
 925    * \param dir Directory where the file needs to be unique
 926    * \param basefilename string to base the filename on.
 927    */
 928   static Pathname generate_non_existing_name( const Pathname &dir,
 929                                               const std::string &basefilename )
 930   {
 931     string final_filename = basefilename;
 932     int counter = 1;
 933     while ( PathInfo(dir + final_filename).isExist() )
 934     {
 935       final_filename = basefilename + "_" + str::numstring(counter);
 936       counter++;
 937     }
 938     return dir + Pathname(final_filename);
 939   }
 940
 941   ////////////////////////////////////////////////////////////////////////////
 942
 943   /**
 944    * \short Generate a related filename from a repo info
 945    *
 946    * From a repo info, it will try to use the alias as a filename
 947    * escaping it if necessary. Other fallbacks can be added to
 948    * this function in case there is no way to use the alias
 949    */
 950   static std::string generate_filename( const RepoInfo &info )
 951   {
 952     std::string fnd="/";
 953     std::string rep="_";
 954     std::string filename = info.alias();
 955     // replace slashes with underscores
 956     size_t pos = filename.find(fnd);
 957     while(pos!=string::npos)
 958     {
 959       filename.replace(pos,fnd.length(),rep);
 960       pos = filename.find(fnd,pos+rep.length());
 961     }
 962     filename = Pathname(filename).extend(".repo").asString();
 963     MIL << "generating filename for repo [" << info.alias() << "] : '" << filename << "'" << endl;
 964     return filename;
 965   }
 966
 967
 968   ////////////////////////////////////////////////////////////////////////////
 969
 970   void RepoManager::addRepository( const RepoInfo &info,
 971                                    const ProgressData::ReceiverFnc & progressrcv )
 972   {
 973     assert_alias(info);
 974
 975     ProgressData progress(100);
 976     callback::SendReport<ProgressReport> report;
 977     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
 978     progress.name(str::form(_("Adding repository '%s'"), info.name().c_str()));
 979     progress.toMin();
 980
 981     std::list<RepoInfo> repos = knownRepositories();
 982     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
 983           it != repos.end();
 984           ++it )
 985     {
 986       if ( info.alias() == (*it).alias() )
 987         ZYPP_THROW(RepoAlreadyExistsException(info.alias()));
 988     }
 989
 990     RepoInfo tosave = info;
 991
 992     // check the first url for now
 993     if ( ZConfig::instance().repo_add_probe()
 994         || ( tosave.type() == RepoType::NONE && tosave.enabled()) )
 995     {
 996       DBG << "unknown repository type, probing" << endl;
 997
 998       RepoType probedtype;
 999       probedtype = probe(*tosave.baseUrlsBegin());
1000       if ( tosave.baseUrlsSize() > 0 )
1001       {
1002         if ( probedtype == RepoType::NONE )
1003           ZYPP_THROW(RepoUnknownTypeException());
1004         else
1005           tosave.setType(probedtype);
1006       }
1007     }
1008
1009     progress.set(50);
1010
1011     // assert the directory exists
1012     filesystem::assert_dir(_pimpl->options.knownReposPath);
1013
1014     Pathname repofile = generate_non_existing_name(_pimpl->options.knownReposPath,
1015                                                     generate_filename(tosave));
1016     // now we have a filename that does not exists
1017     MIL << "Saving repo in " << repofile << endl;
1018
1019     std::ofstream file(repofile.c_str());
1020     if (!file) {
1021       ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
1022     }
1023
1024     tosave.dumpRepoOn(file);
1025     progress.toMax();
1026     MIL << "done" << endl;
1027   }
1028
1029   void RepoManager::addRepositories( const Url &url,
1030                                      const ProgressData::ReceiverFnc & progressrcv )
1031   {
1032     std::list<RepoInfo> knownrepos = knownRepositories();
1033     std::list<RepoInfo> repos = readRepoFile(url);
1034     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1035           it != repos.end();
1036           ++it )
1037     {
1038       // look if the alias is in the known repos.
1039       for ( std::list<RepoInfo>::const_iterator kit = knownrepos.begin();
1040           kit != knownrepos.end();
1041           ++kit )
1042       {
1043         if ( (*it).alias() == (*kit).alias() )
1044         {
1045           ERR << "To be added repo " << (*it).alias() << " conflicts with existing repo " << (*kit).alias() << endl;
1046           ZYPP_THROW(RepoAlreadyExistsException((*it).alias()));
1047         }
1048       }
1049     }
1050
1051     string filename = Pathname(url.getPathName()).basename();
1052
1053     if ( filename == Pathname() )
1054       ZYPP_THROW(RepoException("Invalid repo file name at " + url.asString() ));
1055
1056     // assert the directory exists
1057     filesystem::assert_dir(_pimpl->options.knownReposPath);
1058
1059     Pathname repofile = generate_non_existing_name(_pimpl->options.knownReposPath, filename);
1060     // now we have a filename that does not exists
1061     MIL << "Saving " << repos.size() << " repo" << ( repos.size() ? "s" : "" ) << " in " << repofile << endl;
1062
1063     std::ofstream file(repofile.c_str());
1064     if (!file) {
1065       ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
1066     }
1067
1068     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1069           it != repos.end();
1070           ++it )
1071     {
1072       MIL << "Saving " << (*it).alias() << endl;
1073       (*it).dumpRepoOn(file);
1074     }
1075     MIL << "done" << endl;
1076   }
1077
1078   ////////////////////////////////////////////////////////////////////////////
1079
1080   void RepoManager::removeRepository( const RepoInfo & info,
1081                                       const ProgressData::ReceiverFnc & progressrcv)
1082   {
1083     ProgressData progress;
1084     callback::SendReport<ProgressReport> report;
1085     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
1086     progress.name(str::form(_("Removing repository '%s'"), info.name().c_str()));
1087
1088     MIL << "Going to delete repo " << info.alias() << endl;
1089
1090     std::list<RepoInfo> repos = knownRepositories();
1091     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1092           it != repos.end();
1093           ++it )
1094     {
1095       // they can be the same only if the provided is empty, that means
1096       // the provided repo has no alias
1097       // then skip
1098       if ( (!info.alias().empty()) && ( info.alias() != (*it).alias() ) )
1099         continue;
1100
1101       // TODO match by url
1102
1103       // we have a matcing repository, now we need to know
1104       // where it does come from.
1105       RepoInfo todelete = *it;
1106       if (todelete.filepath().empty())
1107       {
1108         ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
1109       }
1110       else
1111       {
1112         // figure how many repos are there in the file:
1113         std::list<RepoInfo> filerepos = repositories_in_file(todelete.filepath());
1114         if ( (filerepos.size() == 1) && ( filerepos.front().alias() == todelete.alias() ) )
1115         {
1116           // easy, only this one, just delete the file
1117           if ( filesystem::unlink(todelete.filepath()) != 0 )
1118           {
1119             ZYPP_THROW(RepoException("Can't delete " + todelete.filepath().asString()));
1120           }
1121           MIL << todelete.alias() << " sucessfully deleted." << endl;
1122         }
1123         else
1124         {
1125           // there are more repos in the same file
1126           // write them back except the deleted one.
1127           //TmpFile tmp;
1128           //std::ofstream file(tmp.path().c_str());
1129
1130           // assert the directory exists
1131           filesystem::assert_dir(todelete.filepath().dirname());
1132
1133           std::ofstream file(todelete.filepath().c_str());
1134           if (!file) {
1135             //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
1136             ZYPP_THROW (Exception( "Can't open " + todelete.filepath().asString() ) );
1137           }
1138           for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
1139                 fit != filerepos.end();
1140                 ++fit )
1141           {
1142             if ( (*fit).alias() != todelete.alias() )
1143               (*fit).dumpRepoOn(file);
1144           }
1145         }
1146
1147         CombinedProgressData subprogrcv(progress, 70);
1148         CombinedProgressData cleansubprogrcv(progress, 30);
1149         // now delete it from cache
1150         cleanCache( todelete, subprogrcv);
1151         // now delete metadata (#301037)
1152         cleanMetadata( todelete, cleansubprogrcv);
1153         MIL << todelete.alias() << " sucessfully deleted." << endl;
1154         return;
1155       } // else filepath is empty
1156
1157     }
1158     // should not be reached on a sucess workflow
1159     ZYPP_THROW(RepoNotFoundException(info));
1160   }
1161
1162   ////////////////////////////////////////////////////////////////////////////
1163
1164   void RepoManager::modifyRepository( const std::string &alias,
1165                                       const RepoInfo & newinfo,
1166                                       const ProgressData::ReceiverFnc & progressrcv )
1167   {
1168     RepoInfo toedit = getRepositoryInfo(alias);
1169
1170     if (toedit.filepath().empty())
1171     {
1172       ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
1173     }
1174     else
1175     {
1176       // figure how many repos are there in the file:
1177       std::list<RepoInfo> filerepos = repositories_in_file(toedit.filepath());
1178
1179       // there are more repos in the same file
1180       // write them back except the deleted one.
1181       //TmpFile tmp;
1182       //std::ofstream file(tmp.path().c_str());
1183
1184       // assert the directory exists
1185       filesystem::assert_dir(toedit.filepath().dirname());
1186
1187       std::ofstream file(toedit.filepath().c_str());
1188       if (!file) {
1189         //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
1190         ZYPP_THROW (Exception( "Can't open " + toedit.filepath().asString() ) );
1191       }
1192       for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
1193             fit != filerepos.end();
1194             ++fit )
1195       {
1196           // if the alias is different, dump the original
1197           // if it is the same, dump the provided one
1198           if ( (*fit).alias() != toedit.alias() )
1199             (*fit).dumpRepoOn(file);
1200           else
1201             newinfo.dumpRepoOn(file);
1202       }
1203     }
1204   }
1205
1206   ////////////////////////////////////////////////////////////////////////////
1207
1208   RepoInfo RepoManager::getRepositoryInfo( const std::string &alias,
1209                                            const ProgressData::ReceiverFnc & progressrcv )
1210   {
1211     std::list<RepoInfo> repos = knownRepositories();
1212     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1213           it != repos.end();
1214           ++it )
1215     {
1216       if ( (*it).alias() == alias )
1217         return *it;
1218     }
1219     RepoInfo info;
1220     info.setAlias(info.alias());
1221     ZYPP_THROW(RepoNotFoundException(info));
1222   }
1223
1224   ////////////////////////////////////////////////////////////////////////////
1225
1226   RepoInfo RepoManager::getRepositoryInfo( const Url & url,
1227                                            const url::ViewOption & urlview,
1228                                            const ProgressData::ReceiverFnc & progressrcv )
1229   {
1230     std::list<RepoInfo> repos = knownRepositories();
1231     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1232           it != repos.end();
1233           ++it )
1234     {
1235       for(RepoInfo::urls_const_iterator urlit = (*it).baseUrlsBegin();
1236           urlit != (*it).baseUrlsEnd();
1237           ++urlit)
1238       {
1239         if ((*urlit).asString(urlview) == url.asString(urlview))
1240           return *it;
1241       }
1242     }
1243     RepoInfo info;
1244     info.setAlias(info.alias());
1245     info.setBaseUrl(url);
1246     ZYPP_THROW(RepoNotFoundException(info));
1247   }
1248
1249   ////////////////////////////////////////////////////////////////////////////
1250
1251   std::ostream & operator<<( std::ostream & str, const RepoManager & obj )
1252   {
1253     return str << *obj._pimpl;
1254   }
1255
1256   /////////////////////////////////////////////////////////////////
1257 } // namespace zypp
1258 ///////////////////////////////////////////////////////////////////