zypp/RepoManager.cc

   1 /*---------------------------------------------------------------------\
   2 |                          ____ _   __ __ ___                          |
   3 |                         |__  / \ / / . \ . \                         |
   4 |                           / / \ V /|  _/  _/                         |
   5 |                          / /__ | | | | | |                           |
   6 |                         /_____||_| |_| |_|                           |
   7 |                                                                      |
   8 \---------------------------------------------------------------------*/
   9 /** \file       zypp/RepoManager.cc
  10  *
  11 */
  12
  13 #include <iostream>
  14 #include <fstream>
  15 #include <list>
  16 #include <algorithm>
  17 #include "zypp/base/InputStream.h"
  18 #include "zypp/base/Logger.h"
  19 #include "zypp/base/Gettext.h"
  20 #include "zypp/base/Function.h"
  21 #include "zypp/base/Regex.h"
  22 #include "zypp/PathInfo.h"
  23 #include "zypp/TmpPath.h"
  24
  25 #include "zypp/repo/RepoException.h"
  26 #include "zypp/RepoManager.h"
  27
  28 #include "zypp/cache/CacheStore.h"
  29 #include "zypp/repo/cached/RepoImpl.h"
  30 #include "zypp/media/MediaManager.h"
  31 #include "zypp/MediaSetAccess.h"
  32
  33 #include "zypp/parser/RepoFileReader.h"
  34 #include "zypp/repo/yum/Downloader.h"
  35 #include "zypp/parser/yum/RepoParser.h"
  36 #include "zypp/parser/plaindir/RepoParser.h"
  37 #include "zypp/repo/susetags/Downloader.h"
  38 #include "zypp/parser/susetags/RepoParser.h"
  39
  40 #include "zypp/ZYppCallbacks.h"
  41
  42 using namespace std;
  43 using namespace zypp;
  44 using namespace zypp::repo;
  45 using namespace zypp::filesystem;
  46
  47 using namespace zypp::repo;
  48
  49 ///////////////////////////////////////////////////////////////////
  50 namespace zypp
  51 { /////////////////////////////////////////////////////////////////
  52
  53   ///////////////////////////////////////////////////////////////////
  54   //
  55   //    CLASS NAME : RepoManagerOptions
  56   //
  57   ///////////////////////////////////////////////////////////////////
  58
  59   RepoManagerOptions::RepoManagerOptions()
  60   {
  61     repoCachePath    = ZConfig::instance().repoCachePath();
  62     repoRawCachePath = ZConfig::instance().repoMetadataPath();
  63     knownReposPath   = ZConfig::instance().knownReposPath();
  64   }
  65
  66   ////////////////////////////////////////////////////////////////////////////
  67
  68   /**
  69     * \short Simple callback to collect the results
  70     *
  71     * Classes like RepoFileParser call the callback
  72     * once per each repo in a file.
  73     *
  74     * Passing this functor as callback, you can collect
  75     * all resuls at the end, without dealing with async
  76     * code.
  77     */
  78     struct RepoCollector
  79     {
  80       RepoCollector()
  81       {
  82         MIL << endl;
  83       }
  84
  85       ~RepoCollector()
  86       {
  87         MIL << endl;
  88       }
  89
  90       bool collect( const RepoInfo &repo )
  91       {
  92         //MIL << "here in collector: " << repo.alias() << endl;
  93         repos.push_back(repo);
  94         //MIL << "added: " << repo.alias() << endl;
  95         return true;
  96       }
  97
  98       RepoInfoList repos;
  99     };
 100
 101   ////////////////////////////////////////////////////////////////////////////
 102
 103    /**
 104     * \short Internal version of clean cache
 105     *
 106     * Takes an extra CacheStore reference, so we avoid internally
 107     * having 2 CacheStores writing to the same database.
 108     */
 109   static void cleanCacheInternal( cache::CacheStore &store,
 110                                   const RepoInfo &info,
 111                                   const ProgressData::ReceiverFnc & progressrcv = ProgressData::ReceiverFnc() )
 112   {
 113     ProgressData progress;
 114     callback::SendReport<ProgressReport> report;
 115     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
 116     progress.name(str::form(_("Cleaning repository '%s' cache"), info.name().c_str()));
 117
 118     if ( !store.isCached(info.alias()) )
 119       return;
 120
 121     MIL << info.alias() << " cleaning cache..." << endl;
 122     data::RecordId id = store.lookupRepository(info.alias());
 123
 124     CombinedProgressData subprogrcv(progress);
 125
 126     store.cleanRepository(id, subprogrcv);
 127   }
 128
 129   ////////////////////////////////////////////////////////////////////////////
 130
 131   /**
 132    * Reads RepoInfo's from a repo file.
 133    *
 134    * \param file pathname of the file to read.
 135    */
 136   static std::list<RepoInfo> repositories_in_file( const Pathname & file )
 137   {
 138     MIL << "repo file: " << file << endl;
 139     RepoCollector collector;
 140     parser::RepoFileReader parser( file, bind( &RepoCollector::collect, &collector, _1 ) );
 141     return collector.repos;
 142   }
 143
 144   ////////////////////////////////////////////////////////////////////////////
 145
 146   std::list<RepoInfo> readRepoFile(const Url & repo_file)
 147    {
 148      // no interface to download a specific file, using workaround:
 149      //! \todo add MediaManager::provideFile(Url file_url) to easily access any file URLs? (no need for media access id or media_nr)
 150      Url url(repo_file);
 151      Pathname path(url.getPathName());
 152      url.setPathName ("/");
 153      MediaSetAccess access(url);
 154      Pathname local = access.provideFile(path);
 155
 156      DBG << "reading repo file " << repo_file << ", local path: " << local << endl;
 157
 158      return repositories_in_file(local);
 159    }
 160
 161   ////////////////////////////////////////////////////////////////////////////
 162
 163   /**
 164    * \short List of RepoInfo's from a directory
 165    *
 166    * Goes trough every file ending with ".repo" in a directory and adds all
 167    * RepoInfo's contained in that file.
 168    *
 169    * \param dir pathname of the directory to read.
 170    */
 171   static std::list<RepoInfo> repositories_in_dir( const Pathname &dir )
 172   {
 173     MIL << "directory " << dir << endl;
 174     list<RepoInfo> repos;
 175     list<Pathname> entries;
 176     if ( filesystem::readdir( entries, Pathname(dir), false ) != 0 )
 177       ZYPP_THROW(Exception("failed to read directory"));
 178
 179     str::regex allowedRepoExt("^\\.repo(_[0-9]+)?$");
 180     for ( list<Pathname>::const_iterator it = entries.begin(); it != entries.end(); ++it )
 181     {
 182       if (str::regex_match(it->extension(), allowedRepoExt))
 183       {
 184         list<RepoInfo> tmp = repositories_in_file( *it );
 185         repos.insert( repos.end(), tmp.begin(), tmp.end() );
 186
 187         //std::copy( collector.repos.begin(), collector.repos.end(), std::back_inserter(repos));
 188         //MIL << "ok" << endl;
 189       }
 190     }
 191     return repos;
 192   }
 193
 194   ////////////////////////////////////////////////////////////////////////////
 195
 196   static void assert_alias( const RepoInfo &info )
 197   {
 198     if (info.alias().empty())
 199         ZYPP_THROW(RepoNoAliasException());
 200   }
 201
 202   ////////////////////////////////////////////////////////////////////////////
 203
 204   static void assert_urls( const RepoInfo &info )
 205   {
 206     if (info.baseUrlsEmpty())
 207         ZYPP_THROW(RepoNoUrlException());
 208   }
 209
 210   ////////////////////////////////////////////////////////////////////////////
 211
 212   /**
 213    * \short Calculates the raw cache path for a repository
 214    */
 215   static Pathname rawcache_path_for_repoinfo( const RepoManagerOptions &opt, const RepoInfo &info )
 216   {
 217     assert_alias(info);
 218     return opt.repoRawCachePath + info.alias();
 219   }
 220
 221   ///////////////////////////////////////////////////////////////////
 222   //
 223   //    CLASS NAME : RepoManager::Impl
 224   //
 225   ///////////////////////////////////////////////////////////////////
 226
 227   /**
 228    * \short RepoManager implementation.
 229    */
 230   struct RepoManager::Impl
 231   {
 232     Impl( const RepoManagerOptions &opt )
 233       : options(opt)
 234     {
 235
 236     }
 237
 238     Impl()
 239     {
 240
 241     }
 242
 243     RepoManagerOptions options;
 244
 245   public:
 246     /** Offer default Impl. */
 247     static shared_ptr<Impl> nullimpl()
 248     {
 249       static shared_ptr<Impl> _nullimpl( new Impl );
 250       return _nullimpl;
 251     }
 252
 253   private:
 254     friend Impl * rwcowClone<Impl>( const Impl * rhs );
 255     /** clone for RWCOW_pointer */
 256     Impl * clone() const
 257     { return new Impl( *this ); }
 258   };
 259   ///////////////////////////////////////////////////////////////////
 260
 261   /** \relates RepoManager::Impl Stream output */
 262   inline std::ostream & operator<<( std::ostream & str, const RepoManager::Impl & obj )
 263   {
 264     return str << "RepoManager::Impl";
 265   }
 266
 267   ///////////////////////////////////////////////////////////////////
 268   //
 269   //    CLASS NAME : RepoManager
 270   //
 271   ///////////////////////////////////////////////////////////////////
 272
 273   RepoManager::RepoManager( const RepoManagerOptions &opt )
 274   : _pimpl( new Impl(opt) )
 275   {}
 276
 277   ////////////////////////////////////////////////////////////////////////////
 278
 279   RepoManager::~RepoManager()
 280   {}
 281
 282   ////////////////////////////////////////////////////////////////////////////
 283
 284   std::list<RepoInfo> RepoManager::knownRepositories() const
 285   {
 286     MIL << endl;
 287
 288     if ( PathInfo(_pimpl->options.knownReposPath).isExist() )
 289     {
 290       RepoInfoList repos = repositories_in_dir(_pimpl->options.knownReposPath);
 291       for ( RepoInfoList::iterator it = repos.begin();
 292             it != repos.end();
 293             ++it )
 294       {
 295         // set the metadata path for the repo
 296         Pathname metadata_path = rawcache_path_for_repoinfo(_pimpl->options, (*it));
 297         (*it).setMetadataPath(metadata_path);
 298       }
 299       return repos;
 300     }
 301     else
 302       return std::list<RepoInfo>();
 303
 304     MIL << endl;
 305   }
 306
 307   ////////////////////////////////////////////////////////////////////////////
 308
 309   Pathname RepoManager::metadataPath( const RepoInfo &info ) const
 310   {
 311     return rawcache_path_for_repoinfo(_pimpl->options, info );
 312   }
 313
 314   ////////////////////////////////////////////////////////////////////////////
 315
 316   RepoStatus RepoManager::metadataStatus( const RepoInfo &info ) const
 317   {
 318     Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 319     RepoType repokind = info.type();
 320     RepoStatus status;
 321
 322     switch ( repokind.toEnum() )
 323     {
 324       case RepoType::NONE_e:
 325       // unknown, probe the local metadata
 326         repokind = probe(rawpath.asUrl());
 327       break;
 328       default:
 329       break;
 330     }
 331
 332     switch ( repokind.toEnum() )
 333     {
 334       case RepoType::RPMMD_e :
 335       {
 336         status = RepoStatus( rawpath + "/repodata/repomd.xml");
 337       }
 338       break;
 339
 340       case RepoType::YAST2_e :
 341       {
 342         // the order of RepoStatus && RepoStatus matters! (#304310)
 343         status = RepoStatus( rawpath + "/content") && (RepoStatus( rawpath + "/media.1/media"));
 344       }
 345       break;
 346
 347       case RepoType::RPMPLAINDIR_e :
 348       {
 349         if ( PathInfo(Pathname(rawpath + "/cookie")).isExist() )
 350           status = RepoStatus( rawpath + "/cookie");
 351       }
 352       break;
 353
 354       case RepoType::NONE_e :
 355         // Return default RepoStatus in case of RepoType::NONE
 356         // indicating it should be created?
 357         // ZYPP_THROW(RepoUnknownTypeException());
 358         break;
 359     }
 360     return status;
 361   }
 362
 363   void RepoManager::touchIndexFile(const RepoInfo & info)
 364   {
 365     Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 366
 367     RepoType repokind = info.type();
 368     if ( repokind.toEnum() == RepoType::NONE_e )
 369       // unknown, probe the local metadata
 370       repokind = probe(rawpath.asUrl());
 371     // if still unknown, just return
 372     if (repokind == RepoType::NONE_e)
 373       return;
 374
 375     Pathname p;
 376     switch ( repokind.toEnum() )
 377     {
 378       case RepoType::RPMMD_e :
 379         p = Pathname(rawpath + "/repodata/repomd.xml");
 380         break;
 381
 382       case RepoType::YAST2_e :
 383         p = Pathname(rawpath + "/content");
 384         break;
 385
 386       case RepoType::RPMPLAINDIR_e :
 387         p = Pathname(rawpath + "/cookie");
 388         break;
 389
 390       case RepoType::NONE_e :
 391       default:
 392         break;
 393     }
 394
 395     // touch the file, ignore error (they are logged anyway)
 396     filesystem::touch(p);
 397   }
 398
 399   bool RepoManager::checkIfToRefreshMetadata( const RepoInfo &info,
 400                                               const Url &url,
 401                                               RawMetadataRefreshPolicy policy )
 402   {
 403     assert_alias(info);
 404
 405     RepoStatus oldstatus;
 406     RepoStatus newstatus;
 407
 408     try
 409     {
 410       MIL << "Going to try to check whether refresh is needed for " << url << endl;
 411
 412       repo::RepoType repokind = info.type();
 413
 414       // if the type is unknown, try probing.
 415       switch ( repokind.toEnum() )
 416       {
 417         case RepoType::NONE_e:
 418           // unknown, probe it
 419           repokind = probe(url);
 420         break;
 421         default:
 422         break;
 423       }
 424
 425       Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 426       filesystem::assert_dir(rawpath);
 427       oldstatus = metadataStatus(info);
 428
 429       // now we've got the old (cached) status, we can decide repo.refresh.delay
 430       if (policy != RefreshForced)
 431       {
 432         // difference in seconds
 433         double diff = difftime(
 434           (Date::ValueType)Date::now(),
 435           (Date::ValueType)oldstatus.timestamp()) / 60;
 436
 437         DBG << "oldstatus: " << (Date::ValueType)oldstatus.timestamp() << endl;
 438         DBG << "current time: " << (Date::ValueType)Date::now() << endl;
 439         DBG << "last refresh = " << diff << " minutes ago" << endl;
 440
 441         if (diff < ZConfig::instance().repo_refresh_delay())
 442         {
 443           MIL << "Repository '" << info.alias()
 444               << "' has been refreshed less than repo.refresh.delay ("
 445               << ZConfig::instance().repo_refresh_delay()
 446               << ") minutes ago. Advising to skip refresh" << endl;
 447           return false;
 448         }
 449       }
 450
 451       // create temp dir as sibling of rawpath
 452       filesystem::TmpDir tmpdir( filesystem::TmpDir::makeSibling( rawpath ) );
 453
 454       if ( ( repokind.toEnum() == RepoType::RPMMD_e ) ||
 455            ( repokind.toEnum() == RepoType::YAST2_e ) )
 456       {
 457         MediaSetAccess media(url);
 458         shared_ptr<repo::Downloader> downloader_ptr;
 459
 460         if ( repokind.toEnum() == RepoType::RPMMD_e )
 461           downloader_ptr.reset(new yum::Downloader(info.path()));
 462         else
 463           downloader_ptr.reset( new susetags::Downloader(info.path()));
 464
 465         RepoStatus newstatus = downloader_ptr->status(media);
 466         bool refresh = false;
 467         if ( oldstatus.checksum() == newstatus.checksum() )
 468         {
 469           MIL << "repo has not changed" << endl;
 470           if ( policy == RefreshForced )
 471           {
 472             MIL << "refresh set to forced" << endl;
 473             refresh = true;
 474           }
 475         }
 476         else
 477         {
 478           MIL << "repo has changed, going to refresh" << endl;
 479           refresh = true;
 480         }
 481
 482         if (!refresh)
 483           touchIndexFile(info);
 484
 485         return refresh;
 486       }
 487       else if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 488       {
 489         RepoStatus newstatus = parser::plaindir::dirStatus(url.getPathName());
 490         bool refresh = false;
 491         if ( oldstatus.checksum() == newstatus.checksum() )
 492         {
 493           MIL << "repo has not changed" << endl;
 494           if ( policy == RefreshForced )
 495           {
 496             MIL << "refresh set to forced" << endl;
 497             refresh = true;
 498           }
 499         }
 500         else
 501         {
 502           MIL << "repo has changed, going to refresh" << endl;
 503           refresh = true;
 504         }
 505
 506         if (!refresh)
 507           touchIndexFile(info);
 508
 509         return refresh;
 510       }
 511       else
 512       {
 513         ZYPP_THROW(RepoUnknownTypeException());
 514       }
 515     }
 516     catch ( const Exception &e )
 517     {
 518       ZYPP_CAUGHT(e);
 519       ERR << "refresh check failed for " << url << endl;
 520       ZYPP_RETHROW(e);
 521     }
 522
 523     return true; // default
 524   }
 525
 526   void RepoManager::refreshMetadata( const RepoInfo &info,
 527                                      RawMetadataRefreshPolicy policy,
 528                                      const ProgressData::ReceiverFnc & progress )
 529   {
 530     assert_alias(info);
 531     assert_urls(info);
 532
 533     // we will throw this later if no URL checks out fine
 534     RepoException rexception(_("Valid metadata not found at specified URL(s)"));
 535
 536     // try urls one by one
 537     for ( RepoInfo::urls_const_iterator it = info.baseUrlsBegin(); it != info.baseUrlsEnd(); ++it )
 538     {
 539       try
 540       {
 541         Url url(*it);
 542
 543         // check whether to refresh metadata
 544         // if the check fails for this url, it throws, so another url will be checked
 545         if (!checkIfToRefreshMetadata(info, url, policy))
 546           return;
 547
 548         MIL << "Going to refresh metadata from " << url << endl;
 549
 550         repo::RepoType repokind = info.type();
 551
 552         // if the type is unknown, try probing.
 553         switch ( repokind.toEnum() )
 554         {
 555           case RepoType::NONE_e:
 556             // unknown, probe it
 557             repokind = probe(*it);
 558           break;
 559           default:
 560           break;
 561         }
 562
 563         Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 564         filesystem::assert_dir(rawpath);
 565
 566         // create temp dir as sibling of rawpath
 567         filesystem::TmpDir tmpdir( filesystem::TmpDir::makeSibling( rawpath ) );
 568
 569         if ( ( repokind.toEnum() == RepoType::RPMMD_e ) ||
 570              ( repokind.toEnum() == RepoType::YAST2_e ) )
 571         {
 572           MediaSetAccess media(url);
 573           shared_ptr<repo::Downloader> downloader_ptr;
 574
 575           if ( repokind.toEnum() == RepoType::RPMMD_e )
 576             downloader_ptr.reset(new yum::Downloader(info.path()));
 577           else
 578             downloader_ptr.reset( new susetags::Downloader(info.path()));
 579
 580           /**
 581            * Given a downloader, sets the other repos raw metadata
 582            * path as cache paths for the fetcher, so if another
 583            * repo has the same file, it will not download it
 584            * but copy it from the other repository
 585            */
 586           std::list<RepoInfo> repos = knownRepositories();
 587           for ( std::list<RepoInfo>::const_iterator it = repos.begin();
 588                 it != repos.end();
 589                 ++it )
 590           {
 591             downloader_ptr->addCachePath(rawcache_path_for_repoinfo( _pimpl->options, *it ));
 592           }
 593
 594           downloader_ptr->download( media, tmpdir.path());
 595         }
 596         else if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 597         {
 598           RepoStatus newstatus = parser::plaindir::dirStatus(url.getPathName());
 599
 600           std::ofstream file(( tmpdir.path() + "/cookie").c_str());
 601           if (!file) {
 602             ZYPP_THROW (Exception( "Can't open " + tmpdir.path().asString() + "/cookie" ) );
 603           }
 604           file << url << endl;
 605           file << newstatus.checksum() << endl;
 606
 607           file.close();
 608         }
 609         else
 610         {
 611           ZYPP_THROW(RepoUnknownTypeException());
 612         }
 613
 614         // ok we have the metadata, now exchange
 615         // the contents
 616         TmpDir oldmetadata( TmpDir::makeSibling( rawpath ) );
 617         filesystem::rename( rawpath, oldmetadata.path() );
 618         // move the just downloaded there
 619         filesystem::rename( tmpdir.path(), rawpath );
 620         // we are done.
 621         return;
 622       }
 623       catch ( const Exception &e )
 624       {
 625         ZYPP_CAUGHT(e);
 626         ERR << "Trying another url..." << endl;
 627
 628         // remember the exception caught for the *first URL*
 629         // if all other URLs fail, the rexception will be thrown with the
 630         // cause of the problem of the first URL remembered
 631         if (it == info.baseUrlsBegin())
 632           rexception.remember(e);
 633       }
 634     } // for every url
 635     ERR << "No more urls..." << endl;
 636     ZYPP_THROW(rexception);
 637   }
 638
 639   ////////////////////////////////////////////////////////////////////////////
 640
 641   void RepoManager::cleanMetadata( const RepoInfo &info,
 642                                    const ProgressData::ReceiverFnc & progressfnc )
 643   {
 644     ProgressData progress(100);
 645     progress.sendTo(progressfnc);
 646
 647     filesystem::recursive_rmdir(rawcache_path_for_repoinfo(_pimpl->options, info));
 648     progress.toMax();
 649   }
 650
 651   void RepoManager::buildCache( const RepoInfo &info,
 652                                 CacheBuildPolicy policy,
 653                                 const ProgressData::ReceiverFnc & progressrcv )
 654   {
 655     assert_alias(info);
 656     Pathname rawpath = rawcache_path_for_repoinfo(_pimpl->options, info);
 657
 658     cache::CacheStore store(_pimpl->options.repoCachePath);
 659
 660     RepoStatus raw_metadata_status = metadataStatus(info);
 661     if ( raw_metadata_status.empty() )
 662     {
 663       ZYPP_THROW(RepoMetadataException(info));
 664     }
 665
 666     bool needs_cleaning = false;
 667     if ( store.isCached( info.alias() ) )
 668     {
 669       MIL << info.alias() << " is already cached." << endl;
 670       data::RecordId id = store.lookupRepository(info.alias());
 671       RepoStatus cache_status = store.repositoryStatus(id);
 672
 673       if ( cache_status.checksum() == raw_metadata_status.checksum() )
 674       {
 675         MIL << info.alias() << " cache is up to date with metadata." << endl;
 676         if ( policy == BuildIfNeeded ) {
 677           return;
 678         }
 679         else {
 680           MIL << info.alias() << " cache rebuild is forced" << endl;
 681         }
 682       }
 683
 684       needs_cleaning = true;
 685     }
 686
 687     ProgressData progress(100);
 688     callback::SendReport<ProgressReport> report;
 689     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
 690     progress.name(str::form(_("Building repository '%s' cache"), info.name().c_str()));
 691     progress.toMin();
 692
 693     if (needs_cleaning)
 694       cleanCacheInternal( store, info);
 695
 696     MIL << info.alias() << " building cache..." << endl;
 697     data::RecordId id = store.lookupOrAppendRepository(info.alias());
 698     // do we have type?
 699     repo::RepoType repokind = info.type();
 700
 701     // if the type is unknown, try probing.
 702     switch ( repokind.toEnum() )
 703     {
 704       case RepoType::NONE_e:
 705         // unknown, probe the local metadata
 706         repokind = probe(rawpath.asUrl());
 707       break;
 708       default:
 709       break;
 710     }
 711
 712
 713     switch ( repokind.toEnum() )
 714     {
 715       case RepoType::RPMMD_e :
 716       {
 717         CombinedProgressData subprogrcv( progress, 100);
 718         parser::yum::RepoParser parser(id, store, parser::yum::RepoParserOpts(), subprogrcv);
 719         parser.parse(rawpath);
 720           // no error
 721       }
 722       break;
 723       case RepoType::YAST2_e :
 724       {
 725         CombinedProgressData subprogrcv( progress, 100);
 726         parser::susetags::RepoParser parser(id, store, subprogrcv);
 727         parser.parse(rawpath);
 728         // no error
 729       }
 730       break;
 731       case RepoType::RPMPLAINDIR_e :
 732       {
 733         CombinedProgressData subprogrcv( progress, 100);
 734         InputStream is(rawpath + "cookie");
 735         string buffer;
 736         getline( is.stream(), buffer);
 737         Url url(buffer);
 738         parser::plaindir::RepoParser parser(id, store, subprogrcv);
 739         parser.parse(url.getPathName());
 740       }
 741       break;
 742       default:
 743         ZYPP_THROW(RepoUnknownTypeException());
 744     }
 745
 746     // update timestamp and checksum
 747     store.updateRepositoryStatus(id, raw_metadata_status);
 748
 749     MIL << "Commit cache.." << endl;
 750     store.commit();
 751     //progress.toMax();
 752   }
 753
 754   ////////////////////////////////////////////////////////////////////////////
 755
 756   repo::RepoType RepoManager::probe( const Url &url ) const
 757   {
 758     if ( url.getScheme() == "dir" && ! PathInfo( url.getPathName() ).isDir() )
 759     {
 760       // Handle non existing local directory in advance, as
 761       // MediaSetAccess does not support it.
 762       return repo::RepoType::NONE;
 763     }
 764
 765     try
 766     {
 767       MediaSetAccess access(url);
 768       if ( access.doesFileExist("/repodata/repomd.xml") )
 769         return repo::RepoType::RPMMD;
 770       if ( access.doesFileExist("/content") )
 771         return repo::RepoType::YAST2;
 772
 773       // if it is a local url of type dir
 774       if ( (! media::MediaManager::downloads(url)) && ( url.getScheme() == "dir" ) )
 775       {
 776         Pathname path = Pathname(url.getPathName());
 777         if ( PathInfo(path).isDir() )
 778         {
 779           // allow empty dirs for now
 780           return repo::RepoType::RPMPLAINDIR;
 781         }
 782       }
 783     }
 784     catch ( const media::MediaException &e )
 785     {
 786       ZYPP_CAUGHT(e);
 787       RepoException enew("Error trying to read from " + url.asString());
 788       enew.remember(e);
 789       ZYPP_THROW(enew);
 790     }
 791     catch ( const Exception &e )
 792     {
 793       ZYPP_CAUGHT(e);
 794       Exception enew("Unknown error reading from " + url.asString());
 795       enew.remember(e);
 796       ZYPP_THROW(enew);
 797     }
 798
 799     return repo::RepoType::NONE;
 800   }
 801
 802   ////////////////////////////////////////////////////////////////////////////
 803
 804   void RepoManager::cleanCache( const RepoInfo &info,
 805                                 const ProgressData::ReceiverFnc & progressrcv )
 806   {
 807     cache::CacheStore store(_pimpl->options.repoCachePath);
 808     cleanCacheInternal( store, info, progressrcv );
 809     store.commit();
 810   }
 811
 812   ////////////////////////////////////////////////////////////////////////////
 813
 814   bool RepoManager::isCached( const RepoInfo &info ) const
 815   {
 816     cache::CacheStore store(_pimpl->options.repoCachePath);
 817     return store.isCached(info.alias());
 818   }
 819
 820   RepoStatus RepoManager::cacheStatus( const RepoInfo &info ) const
 821   {
 822     cache::CacheStore store(_pimpl->options.repoCachePath);
 823     data::RecordId id = store.lookupRepository(info.alias());
 824     RepoStatus cache_status = store.repositoryStatus(id);
 825     return cache_status;
 826   }
 827
 828   Repository RepoManager::createFromCache( const RepoInfo &info,
 829                                            const ProgressData::ReceiverFnc & progressrcv )
 830   {
 831     callback::SendReport<ProgressReport> report;
 832     ProgressData progress;
 833     progress.sendTo(ProgressReportAdaptor( progressrcv, report ));
 834     //progress.sendTo( progressrcv );
 835     progress.name(str::form(_("Reading repository '%s' cache"), info.name().c_str()));
 836
 837     cache::CacheStore store(_pimpl->options.repoCachePath);
 838
 839     if ( ! store.isCached( info.alias() ) )
 840       ZYPP_THROW(RepoNotCachedException());
 841
 842     MIL << "Repository " << info.alias() << " is cached" << endl;
 843
 844     data::RecordId id = store.lookupRepository(info.alias());
 845
 846     CombinedProgressData subprogrcv(progress);
 847
 848     repo::cached::RepoOptions opts( info, _pimpl->options.repoCachePath, id );
 849     opts.readingResolvablesProgress = subprogrcv;
 850     repo::cached::RepoImpl::Ptr repoimpl =
 851         new repo::cached::RepoImpl( opts );
 852
 853     repoimpl->resolvables();
 854     // read the resolvables from cache
 855     return Repository(repoimpl);
 856   }
 857
 858   ////////////////////////////////////////////////////////////////////////////
 859
 860   /**
 861    * Generate a non existing filename in a directory, using a base
 862    * name. For example if a directory contains 3 files
 863    *
 864    * |-- bar
 865    * |-- foo
 866    * `-- moo
 867    *
 868    * If you try to generate a unique filename for this directory,
 869    * based on "ruu" you will get "ruu", but if you use the base
 870    * "foo" you will get "foo_1"
 871    *
 872    * \param dir Directory where the file needs to be unique
 873    * \param basefilename string to base the filename on.
 874    */
 875   static Pathname generate_non_existing_name( const Pathname &dir,
 876                                               const std::string &basefilename )
 877   {
 878     string final_filename = basefilename;
 879     int counter = 1;
 880     while ( PathInfo(dir + final_filename).isExist() )
 881     {
 882       final_filename = basefilename + "_" + str::numstring(counter);
 883       counter++;
 884     }
 885     return dir + Pathname(final_filename);
 886   }
 887
 888   ////////////////////////////////////////////////////////////////////////////
 889
 890   /**
 891    * \short Generate a related filename from a repo info
 892    *
 893    * From a repo info, it will try to use the alias as a filename
 894    * escaping it if necessary. Other fallbacks can be added to
 895    * this function in case there is no way to use the alias
 896    */
 897   static std::string generate_filename( const RepoInfo &info )
 898   {
 899     std::string fnd="/";
 900     std::string rep="_";
 901     std::string filename = info.alias();
 902     // replace slashes with underscores
 903     size_t pos = filename.find(fnd);
 904     while(pos!=string::npos)
 905     {
 906       filename.replace(pos,fnd.length(),rep);
 907       pos = filename.find(fnd,pos+rep.length());
 908     }
 909     filename = Pathname(filename).extend(".repo").asString();
 910     MIL << "generating filename for repo [" << info.alias() << "] : '" << filename << "'" << endl;
 911     return filename;
 912   }
 913
 914
 915   ////////////////////////////////////////////////////////////////////////////
 916
 917   void RepoManager::addRepository( const RepoInfo &info,
 918                                    const ProgressData::ReceiverFnc & progressrcv )
 919   {
 920     assert_alias(info);
 921
 922     ProgressData progress(100);
 923     callback::SendReport<ProgressReport> report;
 924     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
 925     progress.name(str::form(_("Adding repository '%s'"), info.name().c_str()));
 926     progress.toMin();
 927
 928     std::list<RepoInfo> repos = knownRepositories();
 929     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
 930           it != repos.end();
 931           ++it )
 932     {
 933       if ( info.alias() == (*it).alias() )
 934         ZYPP_THROW(RepoAlreadyExistsException(info.alias()));
 935     }
 936
 937     RepoInfo tosave = info;
 938
 939     // check the first url for now
 940     if ( ZConfig::instance().repo_add_probe()
 941         || ( tosave.type() == RepoType::NONE && tosave.enabled()) )
 942     {
 943       DBG << "unknown repository type, probing" << endl;
 944
 945       RepoType probedtype;
 946       probedtype = probe(*tosave.baseUrlsBegin());
 947       if ( tosave.baseUrlsSize() > 0 )
 948       {
 949         if ( probedtype == RepoType::NONE )
 950           ZYPP_THROW(RepoUnknownTypeException());
 951         else
 952           tosave.setType(probedtype);
 953       }
 954     }
 955
 956     progress.set(50);
 957
 958     // assert the directory exists
 959     filesystem::assert_dir(_pimpl->options.knownReposPath);
 960
 961     Pathname repofile = generate_non_existing_name(_pimpl->options.knownReposPath,
 962                                                     generate_filename(tosave));
 963     // now we have a filename that does not exists
 964     MIL << "Saving repo in " << repofile << endl;
 965
 966     std::ofstream file(repofile.c_str());
 967     if (!file) {
 968       ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
 969     }
 970
 971     tosave.dumpRepoOn(file);
 972     progress.toMax();
 973     MIL << "done" << endl;
 974   }
 975
 976   void RepoManager::addRepositories( const Url &url,
 977                                      const ProgressData::ReceiverFnc & progressrcv )
 978   {
 979     std::list<RepoInfo> knownrepos = knownRepositories();
 980     std::list<RepoInfo> repos = readRepoFile(url);
 981     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
 982           it != repos.end();
 983           ++it )
 984     {
 985       // look if the alias is in the known repos.
 986       for ( std::list<RepoInfo>::const_iterator kit = knownrepos.begin();
 987           kit != knownrepos.end();
 988           ++kit )
 989       {
 990         if ( (*it).alias() == (*kit).alias() )
 991         {
 992           ERR << "To be added repo " << (*it).alias() << " conflicts with existing repo " << (*kit).alias() << endl;
 993           ZYPP_THROW(RepoAlreadyExistsException((*it).alias()));
 994         }
 995       }
 996     }
 997
 998     string filename = Pathname(url.getPathName()).basename();
 999
1000     if ( filename == Pathname() )
1001       ZYPP_THROW(RepoException("Invalid repo file name at " + url.asString() ));
1002
1003     // assert the directory exists
1004     filesystem::assert_dir(_pimpl->options.knownReposPath);
1005
1006     Pathname repofile = generate_non_existing_name(_pimpl->options.knownReposPath, filename);
1007     // now we have a filename that does not exists
1008     MIL << "Saving " << repos.size() << " repo" << ( repos.size() ? "s" : "" ) << " in " << repofile << endl;
1009
1010     std::ofstream file(repofile.c_str());
1011     if (!file) {
1012       ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
1013     }
1014
1015     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1016           it != repos.end();
1017           ++it )
1018     {
1019       MIL << "Saving " << (*it).alias() << endl;
1020       (*it).dumpRepoOn(file);
1021     }
1022     MIL << "done" << endl;
1023   }
1024
1025   ////////////////////////////////////////////////////////////////////////////
1026
1027   void RepoManager::removeRepository( const RepoInfo & info,
1028                                       const ProgressData::ReceiverFnc & progressrcv)
1029   {
1030     ProgressData progress;
1031     callback::SendReport<ProgressReport> report;
1032     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
1033     progress.name(str::form(_("Removing repository '%s'"), info.name().c_str()));
1034
1035     MIL << "Going to delete repo " << info.alias() << endl;
1036
1037     std::list<RepoInfo> repos = knownRepositories();
1038     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1039           it != repos.end();
1040           ++it )
1041     {
1042       // they can be the same only if the provided is empty, that means
1043       // the provided repo has no alias
1044       // then skip
1045       if ( (!info.alias().empty()) && ( info.alias() != (*it).alias() ) )
1046         continue;
1047
1048       // TODO match by url
1049
1050       // we have a matcing repository, now we need to know
1051       // where it does come from.
1052       RepoInfo todelete = *it;
1053       if (todelete.filepath().empty())
1054       {
1055         ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
1056       }
1057       else
1058       {
1059         // figure how many repos are there in the file:
1060         std::list<RepoInfo> filerepos = repositories_in_file(todelete.filepath());
1061         if ( (filerepos.size() == 1) && ( filerepos.front().alias() == todelete.alias() ) )
1062         {
1063           // easy, only this one, just delete the file
1064           if ( filesystem::unlink(todelete.filepath()) != 0 )
1065           {
1066             ZYPP_THROW(RepoException("Can't delete " + todelete.filepath().asString()));
1067           }
1068           MIL << todelete.alias() << " sucessfully deleted." << endl;
1069         }
1070         else
1071         {
1072           // there are more repos in the same file
1073           // write them back except the deleted one.
1074           //TmpFile tmp;
1075           //std::ofstream file(tmp.path().c_str());
1076
1077           // assert the directory exists
1078           filesystem::assert_dir(todelete.filepath().dirname());
1079
1080           std::ofstream file(todelete.filepath().c_str());
1081           if (!file) {
1082             //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
1083             ZYPP_THROW (Exception( "Can't open " + todelete.filepath().asString() ) );
1084           }
1085           for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
1086                 fit != filerepos.end();
1087                 ++fit )
1088           {
1089             if ( (*fit).alias() != todelete.alias() )
1090               (*fit).dumpRepoOn(file);
1091           }
1092         }
1093
1094         CombinedProgressData subprogrcv(progress, 70);
1095         CombinedProgressData cleansubprogrcv(progress, 30);
1096         // now delete it from cache
1097         cleanCache( todelete, subprogrcv);
1098         // now delete metadata (#301037)
1099         cleanMetadata( todelete, cleansubprogrcv);
1100         MIL << todelete.alias() << " sucessfully deleted." << endl;
1101         return;
1102       } // else filepath is empty
1103
1104     }
1105     // should not be reached on a sucess workflow
1106     ZYPP_THROW(RepoNotFoundException(info));
1107   }
1108
1109   ////////////////////////////////////////////////////////////////////////////
1110
1111   void RepoManager::modifyRepository( const std::string &alias,
1112                                       const RepoInfo & newinfo,
1113                                       const ProgressData::ReceiverFnc & progressrcv )
1114   {
1115     RepoInfo toedit = getRepositoryInfo(alias);
1116
1117     // check if the new alias already exists when renaming the repo
1118     if (alias != newinfo.alias())
1119     {
1120       std::list<RepoInfo> repos = knownRepositories();
1121       for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1122             it != repos.end();
1123             ++it )
1124       {
1125         if ( newinfo.alias() == (*it).alias() )
1126           ZYPP_THROW(RepoAlreadyExistsException(newinfo.alias()));
1127       }
1128     }
1129
1130     if (toedit.filepath().empty())
1131     {
1132       ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
1133     }
1134     else
1135     {
1136       // figure how many repos are there in the file:
1137       std::list<RepoInfo> filerepos = repositories_in_file(toedit.filepath());
1138
1139       // there are more repos in the same file
1140       // write them back except the deleted one.
1141       //TmpFile tmp;
1142       //std::ofstream file(tmp.path().c_str());
1143
1144       // assert the directory exists
1145       filesystem::assert_dir(toedit.filepath().dirname());
1146
1147       std::ofstream file(toedit.filepath().c_str());
1148       if (!file) {
1149         //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
1150         ZYPP_THROW (Exception( "Can't open " + toedit.filepath().asString() ) );
1151       }
1152       for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
1153             fit != filerepos.end();
1154             ++fit )
1155       {
1156           // if the alias is different, dump the original
1157           // if it is the same, dump the provided one
1158           if ( (*fit).alias() != toedit.alias() )
1159             (*fit).dumpRepoOn(file);
1160           else
1161             newinfo.dumpRepoOn(file);
1162       }
1163     }
1164   }
1165
1166   ////////////////////////////////////////////////////////////////////////////
1167
1168   RepoInfo RepoManager::getRepositoryInfo( const std::string &alias,
1169                                            const ProgressData::ReceiverFnc & progressrcv )
1170   {
1171     std::list<RepoInfo> repos = knownRepositories();
1172     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1173           it != repos.end();
1174           ++it )
1175     {
1176       if ( (*it).alias() == alias )
1177         return *it;
1178     }
1179     RepoInfo info;
1180     info.setAlias(info.alias());
1181     ZYPP_THROW(RepoNotFoundException(info));
1182   }
1183
1184   ////////////////////////////////////////////////////////////////////////////
1185
1186   RepoInfo RepoManager::getRepositoryInfo( const Url & url,
1187                                            const url::ViewOption & urlview,
1188                                            const ProgressData::ReceiverFnc & progressrcv )
1189   {
1190     std::list<RepoInfo> repos = knownRepositories();
1191     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1192           it != repos.end();
1193           ++it )
1194     {
1195       for(RepoInfo::urls_const_iterator urlit = (*it).baseUrlsBegin();
1196           urlit != (*it).baseUrlsEnd();
1197           ++urlit)
1198       {
1199         if ((*urlit).asString(urlview) == url.asString(urlview))
1200           return *it;
1201       }
1202     }
1203     RepoInfo info;
1204     info.setAlias(info.alias());
1205     info.setBaseUrl(url);
1206     ZYPP_THROW(RepoNotFoundException(info));
1207   }
1208
1209   ////////////////////////////////////////////////////////////////////////////
1210
1211   std::ostream & operator<<( std::ostream & str, const RepoManager & obj )
1212   {
1213     return str << *obj._pimpl;
1214   }
1215
1216   /////////////////////////////////////////////////////////////////
1217 } // namespace zypp
1218 ///////////////////////////////////////////////////////////////////