zypp/RepoManager.cc

   1 /*---------------------------------------------------------------------\
   2 |                          ____ _   __ __ ___                          |
   3 |                         |__  / \ / / . \ . \                         |
   4 |                           / / \ V /|  _/  _/                         |
   5 |                          / /__ | | | | | |                           |
   6 |                         /_____||_| |_| |_|                           |
   7 |                                                                      |
   8 \---------------------------------------------------------------------*/
   9 /** \file       zypp/RepoManager.cc
  10  *
  11 */
  12
  13 #include <cstdlib>
  14 #include <iostream>
  15 #include <fstream>
  16 #include <sstream>
  17 #include <list>
  18 #include <map>
  19 #include <algorithm>
  20 #include "zypp/base/InputStream.h"
  21 #include "zypp/base/Logger.h"
  22 #include "zypp/base/Gettext.h"
  23 #include "zypp/base/Function.h"
  24 #include "zypp/base/Regex.h"
  25 #include "zypp/PathInfo.h"
  26 #include "zypp/TmpPath.h"
  27
  28 #include "zypp/repo/RepoException.h"
  29 #include "zypp/RepoManager.h"
  30
  31 #include "zypp/media/MediaManager.h"
  32 #include "zypp/MediaSetAccess.h"
  33 #include "zypp/ExternalProgram.h"
  34 #include "zypp/ManagedFile.h"
  35
  36 #include "zypp/parser/RepoFileReader.h"
  37 #include "zypp/repo/yum/Downloader.h"
  38 #include "zypp/parser/yum/RepoParser.h"
  39 #include "zypp/repo/susetags/Downloader.h"
  40 #include "zypp/parser/susetags/RepoParser.h"
  41 #include "zypp/parser/plaindir/RepoParser.h"
  42
  43 #include "zypp/ZYppCallbacks.h"
  44
  45 #include "sat/Pool.h"
  46 #include "satsolver/pool.h"
  47 #include "satsolver/repo.h"
  48 #include "satsolver/repo_solv.h"
  49
  50 using namespace std;
  51 using namespace zypp;
  52 using namespace zypp::repo;
  53 using namespace zypp::filesystem;
  54
  55 using namespace zypp::repo;
  56
  57 ///////////////////////////////////////////////////////////////////
  58 namespace zypp
  59 { /////////////////////////////////////////////////////////////////
  60
  61   ///////////////////////////////////////////////////////////////////
  62   //
  63   //    CLASS NAME : RepoManagerOptions
  64   //
  65   ///////////////////////////////////////////////////////////////////
  66
  67   RepoManagerOptions::RepoManagerOptions( const Pathname & root_r )
  68   {
  69     repoCachePath         = Pathname::assertprefix( root_r, ZConfig::instance().repoCachePath() );
  70     repoRawCachePath      = Pathname::assertprefix( root_r, ZConfig::instance().repoMetadataPath() );
  71     repoSolvCachePath     = Pathname::assertprefix( root_r, ZConfig::instance().repoSolvfilesPath() );
  72     repoPackagesCachePath = Pathname::assertprefix( root_r, ZConfig::instance().repoPackagesPath() );
  73     knownReposPath        = Pathname::assertprefix( root_r, ZConfig::instance().knownReposPath() );
  74     probe                 = ZConfig::instance().repo_add_probe();
  75   }
  76
  77   RepoManagerOptions RepoManagerOptions::makeTestSetup( const Pathname & root_r )
  78   {
  79     RepoManagerOptions ret;
  80     ret.repoCachePath         = root_r;
  81     ret.repoRawCachePath      = root_r/"raw";
  82     ret.repoSolvCachePath     = root_r/"solv";
  83     ret.repoPackagesCachePath = root_r/"packages";
  84     ret.knownReposPath        = root_r/"repos.d";
  85     return ret;
  86   }
  87
  88   ////////////////////////////////////////////////////////////////////////////
  89
  90   /**
  91     * \short Simple callback to collect the results
  92     *
  93     * Classes like RepoFileParser call the callback
  94     * once per each repo in a file.
  95     *
  96     * Passing this functor as callback, you can collect
  97     * all resuls at the end, without dealing with async
  98     * code.
  99     */
 100     struct RepoCollector
 101     {
 102       RepoCollector()
 103       {}
 104
 105       ~RepoCollector()
 106       {}
 107
 108       bool collect( const RepoInfo &repo )
 109       {
 110         repos.push_back(repo);
 111         return true;
 112       }
 113
 114       RepoInfoList repos;
 115     };
 116
 117   ////////////////////////////////////////////////////////////////////////////
 118
 119   /**
 120    * Reads RepoInfo's from a repo file.
 121    *
 122    * \param file pathname of the file to read.
 123    */
 124   static std::list<RepoInfo> repositories_in_file( const Pathname & file )
 125   {
 126     MIL << "repo file: " << file << endl;
 127     RepoCollector collector;
 128     parser::RepoFileReader parser( file, bind( &RepoCollector::collect, &collector, _1 ) );
 129     return collector.repos;
 130   }
 131
 132   ////////////////////////////////////////////////////////////////////////////
 133
 134   std::list<RepoInfo> readRepoFile(const Url & repo_file)
 135    {
 136      // no interface to download a specific file, using workaround:
 137      //! \todo add MediaManager::provideFile(Url file_url) to easily access any file URLs? (no need for media access id or media_nr)
 138      Url url(repo_file);
 139      Pathname path(url.getPathName());
 140      url.setPathName ("/");
 141      MediaSetAccess access(url);
 142      Pathname local = access.provideFile(path);
 143
 144      DBG << "reading repo file " << repo_file << ", local path: " << local << endl;
 145
 146      return repositories_in_file(local);
 147    }
 148
 149   ////////////////////////////////////////////////////////////////////////////
 150
 151   /**
 152    * \short List of RepoInfo's from a directory
 153    *
 154    * Goes trough every file ending with ".repo" in a directory and adds all
 155    * RepoInfo's contained in that file.
 156    *
 157    * \param dir pathname of the directory to read.
 158    */
 159   static std::list<RepoInfo> repositories_in_dir( const Pathname &dir )
 160   {
 161     MIL << "directory " << dir << endl;
 162     list<RepoInfo> repos;
 163     list<Pathname> entries;
 164     if ( filesystem::readdir( entries, Pathname(dir), false ) != 0 )
 165       ZYPP_THROW(Exception("failed to read directory"));
 166
 167     str::regex allowedRepoExt("^\\.repo(_[0-9]+)?$");
 168     for ( list<Pathname>::const_iterator it = entries.begin(); it != entries.end(); ++it )
 169     {
 170       if (str::regex_match(it->extension(), allowedRepoExt))
 171       {
 172         list<RepoInfo> tmp = repositories_in_file( *it );
 173         repos.insert( repos.end(), tmp.begin(), tmp.end() );
 174
 175         //std::copy( collector.repos.begin(), collector.repos.end(), std::back_inserter(repos));
 176         //MIL << "ok" << endl;
 177       }
 178     }
 179     return repos;
 180   }
 181
 182   ////////////////////////////////////////////////////////////////////////////
 183
 184   static void assert_alias( const RepoInfo &info )
 185   {
 186     if (info.alias().empty())
 187         ZYPP_THROW(RepoNoAliasException());
 188   }
 189
 190   ////////////////////////////////////////////////////////////////////////////
 191
 192   static void assert_urls( const RepoInfo &info )
 193   {
 194     if (info.baseUrlsEmpty())
 195         ZYPP_THROW(RepoNoUrlException());
 196   }
 197
 198   ////////////////////////////////////////////////////////////////////////////
 199
 200   /**
 201    * \short Calculates the raw cache path for a repository
 202    */
 203   static Pathname rawcache_path_for_repoinfo( const RepoManagerOptions &opt, const RepoInfo &info )
 204   {
 205     assert_alias(info);
 206     return opt.repoRawCachePath / info.escaped_alias();
 207   }
 208
 209   /**
 210    * \short Calculates the packages cache path for a repository
 211    */
 212   static Pathname packagescache_path_for_repoinfo( const RepoManagerOptions &opt, const RepoInfo &info )
 213   {
 214     assert_alias(info);
 215     return opt.repoPackagesCachePath / info.escaped_alias();
 216   }
 217
 218   static Pathname solv_path_for_repoinfo( const RepoManagerOptions &opt, const RepoInfo &info)
 219   {
 220     assert_alias(info);
 221     return opt.repoSolvCachePath / info.escaped_alias();
 222   }
 223
 224   ///////////////////////////////////////////////////////////////////
 225   //
 226   //    CLASS NAME : RepoManager::Impl
 227   //
 228   ///////////////////////////////////////////////////////////////////
 229
 230   /**
 231    * \short RepoManager implementation.
 232    */
 233   struct RepoManager::Impl
 234   {
 235     Impl( const RepoManagerOptions &opt )
 236       : options(opt)
 237     {
 238
 239     }
 240
 241     Impl()
 242     {
 243
 244     }
 245
 246     RepoManagerOptions options;
 247
 248     map<string, RepoInfo> _repoinfos;
 249
 250   public:
 251     /** Offer default Impl. */
 252     static shared_ptr<Impl> nullimpl()
 253     {
 254       static shared_ptr<Impl> _nullimpl( new Impl );
 255       return _nullimpl;
 256     }
 257
 258   private:
 259     friend Impl * rwcowClone<Impl>( const Impl * rhs );
 260     /** clone for RWCOW_pointer */
 261     Impl * clone() const
 262     { return new Impl( *this ); }
 263   };
 264
 265   ///////////////////////////////////////////////////////////////////
 266
 267   /** \relates RepoManager::Impl Stream output */
 268   inline std::ostream & operator<<( std::ostream & str, const RepoManager::Impl & obj )
 269   {
 270     return str << "RepoManager::Impl";
 271   }
 272
 273   ///////////////////////////////////////////////////////////////////
 274   //
 275   //    CLASS NAME : RepoManager
 276   //
 277   ///////////////////////////////////////////////////////////////////
 278
 279   RepoManager::RepoManager( const RepoManagerOptions &opt )
 280   : _pimpl( new Impl(opt) )
 281   {}
 282
 283   ////////////////////////////////////////////////////////////////////////////
 284
 285   RepoManager::~RepoManager()
 286   {}
 287
 288   ////////////////////////////////////////////////////////////////////////////
 289
 290   std::list<RepoInfo> RepoManager::knownRepositories() const
 291   {
 292     MIL << endl;
 293
 294     if ( PathInfo(_pimpl->options.knownReposPath).isExist() )
 295     {
 296       RepoInfoList repos = repositories_in_dir(_pimpl->options.knownReposPath);
 297       for ( RepoInfoList::iterator it = repos.begin();
 298             it != repos.end();
 299             ++it )
 300       {
 301         // set the metadata path for the repo
 302         Pathname metadata_path = rawcache_path_for_repoinfo(_pimpl->options, (*it));
 303         (*it).setMetadataPath(metadata_path);
 304
 305         // set the downloaded packages path for the repo
 306         Pathname packages_path = packagescache_path_for_repoinfo(_pimpl->options, (*it));
 307         (*it).setPackagesPath(packages_path);
 308       }
 309       return repos;
 310     }
 311     else
 312       return std::list<RepoInfo>();
 313
 314     MIL << endl;
 315   }
 316
 317   ////////////////////////////////////////////////////////////////////////////
 318
 319   Pathname RepoManager::metadataPath( const RepoInfo &info ) const
 320   {
 321     return rawcache_path_for_repoinfo(_pimpl->options, info );
 322   }
 323
 324   Pathname RepoManager::packagesPath( const RepoInfo &info ) const
 325   {
 326     return packagescache_path_for_repoinfo(_pimpl->options, info );
 327   }
 328
 329   ////////////////////////////////////////////////////////////////////////////
 330
 331   RepoStatus RepoManager::metadataStatus( const RepoInfo &info ) const
 332   {
 333     Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 334     RepoType repokind = info.type();
 335     RepoStatus status;
 336
 337     switch ( repokind.toEnum() )
 338     {
 339       case RepoType::NONE_e:
 340       // unknown, probe the local metadata
 341         repokind = probe(rawpath.asUrl());
 342       break;
 343       default:
 344       break;
 345     }
 346
 347     switch ( repokind.toEnum() )
 348     {
 349       case RepoType::RPMMD_e :
 350       {
 351         status = RepoStatus( rawpath + "/repodata/repomd.xml");
 352       }
 353       break;
 354
 355       case RepoType::YAST2_e :
 356       {
 357         status = RepoStatus( rawpath + "/content") && (RepoStatus( rawpath + "/media.1/media"));
 358       }
 359       break;
 360
 361       case RepoType::RPMPLAINDIR_e :
 362       {
 363         if ( PathInfo(Pathname(rawpath + "/cookie")).isExist() )
 364           status = RepoStatus( rawpath + "/cookie");
 365       }
 366       break;
 367
 368       case RepoType::NONE_e :
 369         // Return default RepoStatus in case of RepoType::NONE
 370         // indicating it should be created?
 371         // ZYPP_THROW(RepoUnknownTypeException());
 372         break;
 373     }
 374     return status;
 375   }
 376
 377   void RepoManager::touchIndexFile(const RepoInfo & info)
 378   {
 379     Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 380
 381     RepoType repokind = info.type();
 382     if ( repokind.toEnum() == RepoType::NONE_e )
 383       // unknown, probe the local metadata
 384       repokind = probe(rawpath.asUrl());
 385     // if still unknown, just return
 386     if (repokind == RepoType::NONE_e)
 387       return;
 388
 389     Pathname p;
 390     switch ( repokind.toEnum() )
 391     {
 392       case RepoType::RPMMD_e :
 393         p = Pathname(rawpath + "/repodata/repomd.xml");
 394         break;
 395
 396       case RepoType::YAST2_e :
 397         p = Pathname(rawpath + "/content");
 398         break;
 399
 400       case RepoType::RPMPLAINDIR_e :
 401         p = Pathname(rawpath + "/cookie");
 402         break;
 403
 404       case RepoType::NONE_e :
 405       default:
 406         break;
 407     }
 408
 409     // touch the file, ignore error (they are logged anyway)
 410     filesystem::touch(p);
 411   }
 412
 413   RepoManager::RefreshCheckStatus RepoManager::checkIfToRefreshMetadata(
 414                                               const RepoInfo &info,
 415                                               const Url &url,
 416                                               RawMetadataRefreshPolicy policy )
 417   {
 418     assert_alias(info);
 419
 420     RepoStatus oldstatus;
 421     RepoStatus newstatus;
 422
 423     try
 424     {
 425       MIL << "Going to try to check whether refresh is needed for " << url << endl;
 426
 427       repo::RepoType repokind = info.type();
 428
 429       // if the type is unknown, try probing.
 430       switch ( repokind.toEnum() )
 431       {
 432         case RepoType::NONE_e:
 433           // unknown, probe it
 434           repokind = probe(url);
 435         break;
 436         default:
 437         break;
 438       }
 439
 440       Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 441       filesystem::assert_dir(rawpath);
 442       oldstatus = metadataStatus(info);
 443
 444       // now we've got the old (cached) status, we can decide repo.refresh.delay
 445       if (policy != RefreshForced && policy != RefreshIfNeededIgnoreDelay)
 446       {
 447         // difference in seconds
 448         double diff = difftime(
 449           (Date::ValueType)Date::now(),
 450           (Date::ValueType)oldstatus.timestamp()) / 60;
 451
 452         DBG << "oldstatus: " << (Date::ValueType)oldstatus.timestamp() << endl;
 453         DBG << "current time: " << (Date::ValueType)Date::now() << endl;
 454         DBG << "last refresh = " << diff << " minutes ago" << endl;
 455
 456         if (diff < ZConfig::instance().repo_refresh_delay())
 457         {
 458           MIL << "Repository '" << info.alias()
 459               << "' has been refreshed less than repo.refresh.delay ("
 460               << ZConfig::instance().repo_refresh_delay()
 461               << ") minutes ago. Advising to skip refresh" << endl;
 462           return REPO_CHECK_DELAYED;
 463         }
 464       }
 465
 466       // create temp dir as sibling of rawpath
 467       filesystem::TmpDir tmpdir( filesystem::TmpDir::makeSibling( rawpath ) );
 468
 469       if ( ( repokind.toEnum() == RepoType::RPMMD_e ) ||
 470            ( repokind.toEnum() == RepoType::YAST2_e ) )
 471       {
 472         MediaSetAccess media(url);
 473         shared_ptr<repo::Downloader> downloader_ptr;
 474
 475         if ( repokind.toEnum() == RepoType::RPMMD_e )
 476           downloader_ptr.reset(new yum::Downloader(info.path()));
 477         else
 478           downloader_ptr.reset( new susetags::Downloader(info.path()));
 479
 480         RepoStatus newstatus = downloader_ptr->status(media);
 481         bool refresh = false;
 482         if ( oldstatus.checksum() == newstatus.checksum() )
 483         {
 484           MIL << "repo has not changed" << endl;
 485           if ( policy == RefreshForced )
 486           {
 487             MIL << "refresh set to forced" << endl;
 488             refresh = true;
 489           }
 490         }
 491         else
 492         {
 493           MIL << "repo has changed, going to refresh" << endl;
 494           refresh = true;
 495         }
 496
 497         if (!refresh)
 498           touchIndexFile(info);
 499
 500         return refresh ? REFRESH_NEEDED : REPO_UP_TO_DATE;
 501       }
 502       else if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 503       {
 504         RepoStatus newstatus = parser::plaindir::dirStatus(url.getPathName());
 505         bool refresh = false;
 506         if ( oldstatus.checksum() == newstatus.checksum() )
 507         {
 508           MIL << "repo has not changed" << endl;
 509           if ( policy == RefreshForced )
 510           {
 511             MIL << "refresh set to forced" << endl;
 512             refresh = true;
 513           }
 514         }
 515         else
 516         {
 517           MIL << "repo has changed, going to refresh" << endl;
 518           refresh = true;
 519         }
 520
 521         if (!refresh)
 522           touchIndexFile(info);
 523
 524         return refresh ? REFRESH_NEEDED : REPO_UP_TO_DATE;
 525       }
 526       else
 527       {
 528         ZYPP_THROW(RepoUnknownTypeException(info));
 529       }
 530     }
 531     catch ( const Exception &e )
 532     {
 533       ZYPP_CAUGHT(e);
 534       ERR << "refresh check failed for " << url << endl;
 535       ZYPP_RETHROW(e);
 536     }
 537
 538     return REFRESH_NEEDED; // default
 539   }
 540
 541   void RepoManager::refreshMetadata( const RepoInfo &info,
 542                                      RawMetadataRefreshPolicy policy,
 543                                      const ProgressData::ReceiverFnc & progress )
 544   {
 545     assert_alias(info);
 546     assert_urls(info);
 547
 548     // we will throw this later if no URL checks out fine
 549     RepoException rexception(_("Valid metadata not found at specified URL(s)"));
 550
 551     // try urls one by one
 552     for ( RepoInfo::urls_const_iterator it = info.baseUrlsBegin(); it != info.baseUrlsEnd(); ++it )
 553     {
 554       try
 555       {
 556         Url url(*it);
 557
 558         // check whether to refresh metadata
 559         // if the check fails for this url, it throws, so another url will be checked
 560         if (checkIfToRefreshMetadata(info, url, policy)!=REFRESH_NEEDED)
 561           return;
 562
 563         MIL << "Going to refresh metadata from " << url << endl;
 564
 565         repo::RepoType repokind = info.type();
 566
 567         // if the type is unknown, try probing.
 568         switch ( repokind.toEnum() )
 569         {
 570           case RepoType::NONE_e:
 571             // unknown, probe it
 572             repokind = probe(*it);
 573
 574             if (repokind.toEnum() != RepoType::NONE_e)
 575             {
 576               // Adjust the probed type in RepoInfo
 577               info.setProbedType( repokind ); // lazy init!
 578               //save probed type only for repos in system
 579               std::list<RepoInfo> repos = knownRepositories();
 580               for ( std::list<RepoInfo>::const_iterator it = repos.begin();
 581                    it != repos.end(); ++it )
 582               {
 583                 if ( info.alias() == (*it).alias() )
 584                 {
 585                   RepoInfo modifiedrepo = info;
 586                   modifiedrepo.setType(repokind);
 587                   modifyRepository(info.alias(),modifiedrepo);
 588                 }
 589               }
 590             }
 591           break;
 592           default:
 593           break;
 594         }
 595
 596         Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 597         filesystem::assert_dir(rawpath);
 598
 599         // create temp dir as sibling of rawpath
 600         filesystem::TmpDir tmpdir( filesystem::TmpDir::makeSibling( rawpath ) );
 601
 602         if ( ( repokind.toEnum() == RepoType::RPMMD_e ) ||
 603              ( repokind.toEnum() == RepoType::YAST2_e ) )
 604         {
 605           MediaSetAccess media(url);
 606           shared_ptr<repo::Downloader> downloader_ptr;
 607
 608           if ( repokind.toEnum() == RepoType::RPMMD_e )
 609             downloader_ptr.reset(new yum::Downloader(info.path()));
 610           else
 611             downloader_ptr.reset( new susetags::Downloader(info.path()));
 612
 613           /**
 614            * Given a downloader, sets the other repos raw metadata
 615            * path as cache paths for the fetcher, so if another
 616            * repo has the same file, it will not download it
 617            * but copy it from the other repository
 618            */
 619           std::list<RepoInfo> repos = knownRepositories();
 620           for ( std::list<RepoInfo>::const_iterator it = repos.begin();
 621                 it != repos.end();
 622                 ++it )
 623           {
 624             Pathname cachepath(rawcache_path_for_repoinfo( _pimpl->options, *it ));
 625             if ( PathInfo(cachepath).isExist() )
 626               downloader_ptr->addCachePath(cachepath);
 627           }
 628
 629           downloader_ptr->download( media, tmpdir.path());
 630         }
 631         else if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 632         {
 633           RepoStatus newstatus = parser::plaindir::dirStatus(url.getPathName());
 634
 635           std::ofstream file(( tmpdir.path() + "/cookie").c_str());
 636           if (!file) {
 637             ZYPP_THROW (Exception( "Can't open " + tmpdir.path().asString() + "/cookie" ) );
 638           }
 639           file << url << endl;
 640           file << newstatus.checksum() << endl;
 641
 642           file.close();
 643         }
 644         else
 645         {
 646           ZYPP_THROW(RepoUnknownTypeException());
 647         }
 648
 649         // ok we have the metadata, now exchange
 650         // the contents
 651
 652         TmpDir oldmetadata( TmpDir::makeSibling( rawpath ) );
 653         filesystem::rename( rawpath, oldmetadata.path() );
 654         // move the just downloaded there
 655         filesystem::rename( tmpdir.path(), rawpath );
 656
 657         // we are done.
 658         return;
 659       }
 660       catch ( const Exception &e )
 661       {
 662         ZYPP_CAUGHT(e);
 663         ERR << "Trying another url..." << endl;
 664
 665         // remember the exception caught for the *first URL*
 666         // if all other URLs fail, the rexception will be thrown with the
 667         // cause of the problem of the first URL remembered
 668         if (it == info.baseUrlsBegin())
 669           rexception.remember(e);
 670       }
 671     } // for every url
 672     ERR << "No more urls..." << endl;
 673     ZYPP_THROW(rexception);
 674   }
 675
 676   ////////////////////////////////////////////////////////////////////////////
 677
 678   void RepoManager::cleanMetadata( const RepoInfo &info,
 679                                    const ProgressData::ReceiverFnc & progressfnc )
 680   {
 681     ProgressData progress(100);
 682     progress.sendTo(progressfnc);
 683
 684     filesystem::recursive_rmdir(rawcache_path_for_repoinfo(_pimpl->options, info));
 685     progress.toMax();
 686   }
 687
 688   void RepoManager::cleanPackages( const RepoInfo &info,
 689                                    const ProgressData::ReceiverFnc & progressfnc )
 690   {
 691     ProgressData progress(100);
 692     progress.sendTo(progressfnc);
 693
 694     filesystem::recursive_rmdir(packagescache_path_for_repoinfo(_pimpl->options, info));
 695     progress.toMax();
 696   }
 697
 698   void RepoManager::buildCache( const RepoInfo &info,
 699                                 CacheBuildPolicy policy,
 700                                 const ProgressData::ReceiverFnc & progressrcv )
 701   {
 702     assert_alias(info);
 703     Pathname rawpath = rawcache_path_for_repoinfo(_pimpl->options, info);
 704
 705     filesystem::assert_dir(_pimpl->options.repoCachePath);
 706     RepoStatus raw_metadata_status = metadataStatus(info);
 707     if ( raw_metadata_status.empty() )
 708     {
 709        /* if there is no cache at this point, we refresh the raw
 710           in case this is the first time - if it's !autorefresh,
 711           we may still refresh */
 712       refreshMetadata(info, RefreshIfNeeded, progressrcv );
 713       raw_metadata_status = metadataStatus(info);
 714     }
 715
 716     bool needs_cleaning = false;
 717     if ( isCached( info ) )
 718     {
 719       MIL << info.alias() << " is already cached." << endl;
 720       //data::RecordId id = store.lookupRepository(info.alias());
 721       RepoStatus cache_status = cacheStatus(info);
 722
 723       if ( cache_status.checksum() == raw_metadata_status.checksum() )
 724       {
 725         MIL << info.alias() << " cache is up to date with metadata." << endl;
 726         if ( policy == BuildIfNeeded ) {
 727           return;
 728         }
 729         else {
 730           MIL << info.alias() << " cache rebuild is forced" << endl;
 731         }
 732       }
 733
 734       needs_cleaning = true;
 735     }
 736
 737     ProgressData progress(100);
 738     callback::SendReport<ProgressReport> report;
 739     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
 740     progress.name(str::form(_("Building repository '%s' cache"), info.name().c_str()));
 741     progress.toMin();
 742
 743     if (needs_cleaning)
 744     {
 745       cleanCache(info);
 746     }
 747
 748     MIL << info.alias() << " building cache..." << endl;
 749
 750     Pathname base = solv_path_for_repoinfo( _pimpl->options, info);
 751     filesystem::assert_dir(base);
 752     Pathname solvfile = base / "solv";
 753
 754     // do we have type?
 755     repo::RepoType repokind = info.type();
 756
 757     // if the type is unknown, try probing.
 758     switch ( repokind.toEnum() )
 759     {
 760       case RepoType::NONE_e:
 761         // unknown, probe the local metadata
 762         repokind = probe(rawpath.asUrl());
 763       break;
 764       default:
 765       break;
 766     }
 767
 768     MIL << "repo type is " << repokind << endl;
 769
 770     switch ( repokind.toEnum() )
 771     {
 772       case RepoType::RPMMD_e :
 773       case RepoType::YAST2_e :
 774       case RepoType::RPMPLAINDIR_e :
 775       {
 776         // Take care we unlink the solvfile on exception
 777         ManagedFile guard( solvfile, filesystem::unlink );
 778
 779         ostringstream cmd;
 780         std::string toFile( str::gsub(solvfile.asString(),"\"","\\\"") );
 781         if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 782         {
 783           cmd << str::form( "repo2solv.sh \"%s\" > \"%s\"",
 784                             str::gsub( info.baseUrlsBegin()->getPathName(),"\"","\\\"" ).c_str(),
 785                             toFile.c_str() );
 786         }
 787         else
 788         {
 789           cmd << str::form( "repo2solv.sh \"%s\" > \"%s\"",
 790                             str::gsub( rawpath.asString(),"\"","\\\"" ).c_str(),
 791                             toFile.c_str() );
 792         }
 793         MIL << "Executing: " << cmd.str() << endl;
 794         ExternalProgram prog( cmd.str(), ExternalProgram::Stderr_To_Stdout );
 795
 796         cmd << endl;
 797         for ( string output( prog.receiveLine() ); output.length(); output = prog.receiveLine() ) {
 798           WAR << "  " << output;
 799           cmd << "     " << output;
 800         }
 801
 802         int ret = prog.close();
 803         if ( ret != 0 )
 804         {
 805           RepoException ex(str::form("Failed to cache repo (%d).", ret));
 806           ex.remember( cmd.str() );
 807           ZYPP_THROW(ex);
 808         }
 809
 810         // We keep it.
 811         guard.resetDispose();
 812       }
 813       break;
 814       default:
 815         ZYPP_THROW(RepoUnknownTypeException("Unhandled repository type"));
 816       break;
 817     }
 818 #if 0
 819     switch ( repokind.toEnum() )
 820     {
 821       case RepoType::RPMMD_e :
 822       if (0)
 823       {
 824         CombinedProgressData subprogrcv( progress, 100);
 825         parser::yum::RepoParser parser(id, store, parser::yum::RepoParserOpts(), subprogrcv);
 826         parser.parse(rawpath);
 827           // no error
 828       }
 829       break;
 830       case RepoType::YAST2_e :
 831       if (0)
 832       {
 833         CombinedProgressData subprogrcv( progress, 100);
 834         parser::susetags::RepoParser parser(id, store, subprogrcv);
 835         parser.parse(rawpath);
 836         // no error
 837       }
 838       break;
 839
 840       default:
 841         ZYPP_THROW(RepoUnknownTypeException());
 842     }
 843 #endif
 844     // update timestamp and checksum
 845     //store.updateRepositoryStatus(id, raw_metadata_status);
 846     setCacheStatus(info, raw_metadata_status);
 847     MIL << "Commit cache.." << endl;
 848     //store.commit();
 849     //progress.toMax();
 850   }
 851
 852   ////////////////////////////////////////////////////////////////////////////
 853
 854   repo::RepoType RepoManager::probe( const Url &url ) const
 855   {
 856     if ( url.getScheme() == "dir" && ! PathInfo( url.getPathName() ).isDir() )
 857     {
 858       // Handle non existing local directory in advance, as
 859       // MediaSetAccess does not support it.
 860       return repo::RepoType::NONE;
 861     }
 862
 863     try
 864     {
 865       MediaSetAccess access(url);
 866       if ( access.doesFileExist("/repodata/repomd.xml") )
 867         return repo::RepoType::RPMMD;
 868       if ( access.doesFileExist("/content") )
 869         return repo::RepoType::YAST2;
 870
 871       // if it is a local url of type dir
 872       if ( (! media::MediaManager::downloads(url)) && ( url.getScheme() == "dir" ) )
 873       {
 874         Pathname path = Pathname(url.getPathName());
 875         if ( PathInfo(path).isDir() )
 876         {
 877           // allow empty dirs for now
 878           return repo::RepoType::RPMPLAINDIR;
 879         }
 880       }
 881     }
 882     catch ( const media::MediaException &e )
 883     {
 884       ZYPP_CAUGHT(e);
 885       RepoException enew("Error trying to read from " + url.asString());
 886       enew.remember(e);
 887       ZYPP_THROW(enew);
 888     }
 889     catch ( const Exception &e )
 890     {
 891       ZYPP_CAUGHT(e);
 892       Exception enew("Unknown error reading from " + url.asString());
 893       enew.remember(e);
 894       ZYPP_THROW(enew);
 895     }
 896
 897     return repo::RepoType::NONE;
 898   }
 899
 900   ////////////////////////////////////////////////////////////////////////////
 901
 902   void RepoManager::cleanCache( const RepoInfo &info,
 903                                 const ProgressData::ReceiverFnc & progressrcv )
 904   {
 905     ProgressData progress(100);
 906     progress.sendTo(progressrcv);
 907     progress.toMin();
 908
 909     filesystem::recursive_rmdir(solv_path_for_repoinfo(_pimpl->options, info));
 910
 911     progress.toMax();
 912   }
 913
 914   ////////////////////////////////////////////////////////////////////////////
 915
 916   bool RepoManager::isCached( const RepoInfo &info ) const
 917   {
 918     return PathInfo(solv_path_for_repoinfo( _pimpl->options, info ) / "solv").isExist();
 919   }
 920
 921   RepoStatus RepoManager::cacheStatus( const RepoInfo &info ) const
 922   {
 923
 924     Pathname cookiefile = solv_path_for_repoinfo(_pimpl->options, info) / "cookie";
 925
 926     return RepoStatus::fromCookieFile(cookiefile);
 927   }
 928
 929   void RepoManager::setCacheStatus( const RepoInfo &info, const RepoStatus &status )
 930   {
 931     Pathname base = solv_path_for_repoinfo(_pimpl->options, info);
 932     filesystem::assert_dir(base);
 933     Pathname cookiefile = base / "cookie";
 934
 935     status.saveToCookieFile(cookiefile);
 936   }
 937
 938   void RepoManager::loadFromCache( const RepoInfo & info,
 939                                    const ProgressData::ReceiverFnc & progressrcv )
 940   {
 941     assert_alias(info);
 942     Pathname solvfile = solv_path_for_repoinfo(_pimpl->options, info) / "solv";
 943
 944     if ( ! PathInfo(solvfile).isExist() )
 945       ZYPP_THROW(RepoNotCachedException(info));
 946
 947     try
 948     {
 949       sat::Pool::instance().addRepoSolv( solvfile, info );
 950     }
 951     catch ( const Exception & exp )
 952     {
 953       ZYPP_CAUGHT( exp );
 954       MIL << "Try to handle exception by rebuilding the solv-file" << endl;
 955       cleanCache( info, progressrcv );
 956       buildCache( info, BuildIfNeeded, progressrcv );
 957
 958       sat::Pool::instance().addRepoSolv( solvfile, info );
 959     }
 960   }
 961
 962   ////////////////////////////////////////////////////////////////////////////
 963
 964   /**
 965    * Generate a non existing filename in a directory, using a base
 966    * name. For example if a directory contains 3 files
 967    *
 968    * |-- bar
 969    * |-- foo
 970    * `-- moo
 971    *
 972    * If you try to generate a unique filename for this directory,
 973    * based on "ruu" you will get "ruu", but if you use the base
 974    * "foo" you will get "foo_1"
 975    *
 976    * \param dir Directory where the file needs to be unique
 977    * \param basefilename string to base the filename on.
 978    */
 979   static Pathname generate_non_existing_name( const Pathname &dir,
 980                                               const std::string &basefilename )
 981   {
 982     string final_filename = basefilename;
 983     int counter = 1;
 984     while ( PathInfo(dir + final_filename).isExist() )
 985     {
 986       final_filename = basefilename + "_" + str::numstring(counter);
 987       counter++;
 988     }
 989     return dir + Pathname(final_filename);
 990   }
 991
 992   ////////////////////////////////////////////////////////////////////////////
 993
 994   /**
 995    * \short Generate a related filename from a repo info
 996    *
 997    * From a repo info, it will try to use the alias as a filename
 998    * escaping it if necessary. Other fallbacks can be added to
 999    * this function in case there is no way to use the alias
1000    */
1001   static std::string generate_filename( const RepoInfo &info )
1002   {
1003     std::string fnd="/";
1004     std::string rep="_";
1005     std::string filename = info.alias();
1006     // replace slashes with underscores
1007     size_t pos = filename.find(fnd);
1008     while(pos!=string::npos)
1009     {
1010       filename.replace(pos,fnd.length(),rep);
1011       pos = filename.find(fnd,pos+rep.length());
1012     }
1013     filename = Pathname(filename).extend(".repo").asString();
1014     MIL << "generating filename for repo [" << info.alias() << "] : '" << filename << "'" << endl;
1015     return filename;
1016   }
1017
1018
1019   ////////////////////////////////////////////////////////////////////////////
1020
1021   void RepoManager::addRepository( const RepoInfo &info,
1022                                    const ProgressData::ReceiverFnc & progressrcv )
1023   {
1024     assert_alias(info);
1025
1026     ProgressData progress(100);
1027     callback::SendReport<ProgressReport> report;
1028     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
1029     progress.name(str::form(_("Adding repository '%s'"), info.name().c_str()));
1030     progress.toMin();
1031
1032     std::list<RepoInfo> repos = knownRepositories();
1033     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1034           it != repos.end();
1035           ++it )
1036     {
1037       if ( info.alias() == (*it).alias() )
1038         ZYPP_THROW(RepoAlreadyExistsException(info));
1039     }
1040
1041     RepoInfo tosave = info;
1042
1043     // check the first url for now
1044     if ( _pimpl->options.probe )
1045     {
1046       DBG << "unknown repository type, probing" << endl;
1047
1048       RepoType probedtype;
1049       probedtype = probe(*tosave.baseUrlsBegin());
1050       if ( tosave.baseUrlsSize() > 0 )
1051       {
1052         if ( probedtype == RepoType::NONE )
1053           ZYPP_THROW(RepoUnknownTypeException());
1054         else
1055           tosave.setType(probedtype);
1056       }
1057     }
1058
1059     progress.set(50);
1060
1061     // assert the directory exists
1062     filesystem::assert_dir(_pimpl->options.knownReposPath);
1063
1064     Pathname repofile = generate_non_existing_name(_pimpl->options.knownReposPath,
1065                                                     generate_filename(tosave));
1066     // now we have a filename that does not exists
1067     MIL << "Saving repo in " << repofile << endl;
1068
1069     std::ofstream file(repofile.c_str());
1070     if (!file) {
1071       ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
1072     }
1073
1074     tosave.dumpRepoOn(file);
1075     progress.toMax();
1076     MIL << "done" << endl;
1077   }
1078
1079   void RepoManager::addRepositories( const Url &url,
1080                                      const ProgressData::ReceiverFnc & progressrcv )
1081   {
1082     std::list<RepoInfo> knownrepos = knownRepositories();
1083     std::list<RepoInfo> repos = readRepoFile(url);
1084     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1085           it != repos.end();
1086           ++it )
1087     {
1088       // look if the alias is in the known repos.
1089       for ( std::list<RepoInfo>::const_iterator kit = knownrepos.begin();
1090           kit != knownrepos.end();
1091           ++kit )
1092       {
1093         if ( (*it).alias() == (*kit).alias() )
1094         {
1095           ERR << "To be added repo " << (*it).alias() << " conflicts with existing repo " << (*kit).alias() << endl;
1096           ZYPP_THROW(RepoAlreadyExistsException(*it));
1097         }
1098       }
1099     }
1100
1101     string filename = Pathname(url.getPathName()).basename();
1102
1103     if ( filename == Pathname() )
1104       ZYPP_THROW(RepoException("Invalid repo file name at " + url.asString() ));
1105
1106     // assert the directory exists
1107     filesystem::assert_dir(_pimpl->options.knownReposPath);
1108
1109     Pathname repofile = generate_non_existing_name(_pimpl->options.knownReposPath, filename);
1110     // now we have a filename that does not exists
1111     MIL << "Saving " << repos.size() << " repo" << ( repos.size() ? "s" : "" ) << " in " << repofile << endl;
1112
1113     std::ofstream file(repofile.c_str());
1114     if (!file) {
1115       ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
1116     }
1117
1118     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1119           it != repos.end();
1120           ++it )
1121     {
1122       MIL << "Saving " << (*it).alias() << endl;
1123       (*it).dumpRepoOn(file);
1124     }
1125     MIL << "done" << endl;
1126   }
1127
1128   ////////////////////////////////////////////////////////////////////////////
1129
1130   void RepoManager::removeRepository( const RepoInfo & info,
1131                                       const ProgressData::ReceiverFnc & progressrcv)
1132   {
1133     ProgressData progress;
1134     callback::SendReport<ProgressReport> report;
1135     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
1136     progress.name(str::form(_("Removing repository '%s'"), info.name().c_str()));
1137
1138     MIL << "Going to delete repo " << info.alias() << endl;
1139
1140     std::list<RepoInfo> repos = knownRepositories();
1141     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1142           it != repos.end();
1143           ++it )
1144     {
1145       // they can be the same only if the provided is empty, that means
1146       // the provided repo has no alias
1147       // then skip
1148       if ( (!info.alias().empty()) && ( info.alias() != (*it).alias() ) )
1149         continue;
1150
1151       // TODO match by url
1152
1153       // we have a matcing repository, now we need to know
1154       // where it does come from.
1155       RepoInfo todelete = *it;
1156       if (todelete.filepath().empty())
1157       {
1158         ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
1159       }
1160       else
1161       {
1162         // figure how many repos are there in the file:
1163         std::list<RepoInfo> filerepos = repositories_in_file(todelete.filepath());
1164         if ( (filerepos.size() == 1) && ( filerepos.front().alias() == todelete.alias() ) )
1165         {
1166           // easy, only this one, just delete the file
1167           if ( filesystem::unlink(todelete.filepath()) != 0 )
1168           {
1169             ZYPP_THROW(RepoException("Can't delete " + todelete.filepath().asString()));
1170           }
1171           MIL << todelete.alias() << " sucessfully deleted." << endl;
1172         }
1173         else
1174         {
1175           // there are more repos in the same file
1176           // write them back except the deleted one.
1177           //TmpFile tmp;
1178           //std::ofstream file(tmp.path().c_str());
1179
1180           // assert the directory exists
1181           filesystem::assert_dir(todelete.filepath().dirname());
1182
1183           std::ofstream file(todelete.filepath().c_str());
1184           if (!file) {
1185             //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
1186             ZYPP_THROW (Exception( "Can't open " + todelete.filepath().asString() ) );
1187           }
1188           for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
1189                 fit != filerepos.end();
1190                 ++fit )
1191           {
1192             if ( (*fit).alias() != todelete.alias() )
1193               (*fit).dumpRepoOn(file);
1194           }
1195         }
1196
1197         CombinedProgressData subprogrcv(progress, 70);
1198         CombinedProgressData cleansubprogrcv(progress, 30);
1199         // now delete it from cache
1200         if ( isCached(todelete) )
1201           cleanCache( todelete, subprogrcv);
1202         // now delete metadata (#301037)
1203         cleanMetadata( todelete, cleansubprogrcv);
1204         MIL << todelete.alias() << " sucessfully deleted." << endl;
1205         return;
1206       } // else filepath is empty
1207
1208     }
1209     // should not be reached on a sucess workflow
1210     ZYPP_THROW(RepoNotFoundException(info));
1211   }
1212
1213   ////////////////////////////////////////////////////////////////////////////
1214
1215   void RepoManager::modifyRepository( const std::string &alias,
1216                                       const RepoInfo & newinfo,
1217                                       const ProgressData::ReceiverFnc & progressrcv )
1218   {
1219     RepoInfo toedit = getRepositoryInfo(alias);
1220
1221     // check if the new alias already exists when renaming the repo
1222     if (alias != newinfo.alias())
1223     {
1224       std::list<RepoInfo> repos = knownRepositories();
1225       for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1226             it != repos.end();
1227             ++it )
1228       {
1229         if ( newinfo.alias() == (*it).alias() )
1230           ZYPP_THROW(RepoAlreadyExistsException(newinfo));
1231       }
1232     }
1233
1234     if (toedit.filepath().empty())
1235     {
1236       ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
1237     }
1238     else
1239     {
1240       // figure how many repos are there in the file:
1241       std::list<RepoInfo> filerepos = repositories_in_file(toedit.filepath());
1242
1243       // there are more repos in the same file
1244       // write them back except the deleted one.
1245       //TmpFile tmp;
1246       //std::ofstream file(tmp.path().c_str());
1247
1248       // assert the directory exists
1249       filesystem::assert_dir(toedit.filepath().dirname());
1250
1251       std::ofstream file(toedit.filepath().c_str());
1252       if (!file) {
1253         //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
1254         ZYPP_THROW (Exception( "Can't open " + toedit.filepath().asString() ) );
1255       }
1256       for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
1257             fit != filerepos.end();
1258             ++fit )
1259       {
1260           // if the alias is different, dump the original
1261           // if it is the same, dump the provided one
1262           if ( (*fit).alias() != toedit.alias() )
1263             (*fit).dumpRepoOn(file);
1264           else
1265             newinfo.dumpRepoOn(file);
1266       }
1267     }
1268   }
1269
1270   ////////////////////////////////////////////////////////////////////////////
1271
1272   RepoInfo RepoManager::getRepositoryInfo( const std::string &alias,
1273                                            const ProgressData::ReceiverFnc & progressrcv )
1274   {
1275     std::list<RepoInfo> repos = knownRepositories();
1276     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1277           it != repos.end();
1278           ++it )
1279     {
1280       if ( (*it).alias() == alias )
1281         return *it;
1282     }
1283     RepoInfo info;
1284     info.setAlias(info.alias());
1285     ZYPP_THROW(RepoNotFoundException(info));
1286   }
1287
1288   ////////////////////////////////////////////////////////////////////////////
1289
1290   RepoInfo RepoManager::getRepositoryInfo( const Url & url,
1291                                            const url::ViewOption & urlview,
1292                                            const ProgressData::ReceiverFnc & progressrcv )
1293   {
1294     std::list<RepoInfo> repos = knownRepositories();
1295     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1296           it != repos.end();
1297           ++it )
1298     {
1299       for(RepoInfo::urls_const_iterator urlit = (*it).baseUrlsBegin();
1300           urlit != (*it).baseUrlsEnd();
1301           ++urlit)
1302       {
1303         if ((*urlit).asString(urlview) == url.asString(urlview))
1304           return *it;
1305       }
1306     }
1307     RepoInfo info;
1308     info.setAlias(info.alias());
1309     info.setBaseUrl(url);
1310     ZYPP_THROW(RepoNotFoundException(info));
1311   }
1312
1313   ////////////////////////////////////////////////////////////////////////////
1314
1315   std::ostream & operator<<( std::ostream & str, const RepoManager & obj )
1316   {
1317     return str << *obj._pimpl;
1318   }
1319
1320   /////////////////////////////////////////////////////////////////
1321 } // namespace zypp
1322 ///////////////////////////////////////////////////////////////////