zypp/RepoManager.cc

   1 /*---------------------------------------------------------------------\
   2 |                          ____ _   __ __ ___                          |
   3 |                         |__  / \ / / . \ . \                         |
   4 |                           / / \ V /|  _/  _/                         |
   5 |                          / /__ | | | | | |                           |
   6 |                         /_____||_| |_| |_|                           |
   7 |                                                                      |
   8 \---------------------------------------------------------------------*/
   9 /** \file       zypp/RepoManager.cc
  10  *
  11 */
  12
  13 #include <cstdlib>
  14 #include <iostream>
  15 #include <fstream>
  16 #include <sstream>
  17 #include <list>
  18 #include <algorithm>
  19 #include "zypp/base/InputStream.h"
  20 #include "zypp/base/Logger.h"
  21 #include "zypp/base/Gettext.h"
  22 #include "zypp/base/Function.h"
  23 #include "zypp/base/Regex.h"
  24 #include "zypp/PathInfo.h"
  25 #include "zypp/TmpPath.h"
  26
  27 #include "zypp/repo/RepoException.h"
  28 #include "zypp/RepoManager.h"
  29
  30 #include "zypp/cache/SolvStore.h"
  31 #include "zypp/repo/cached/RepoImpl.h"
  32 #include "zypp/media/MediaManager.h"
  33 #include "zypp/MediaSetAccess.h"
  34
  35 #include "zypp/parser/RepoFileReader.h"
  36 #include "zypp/repo/yum/Downloader.h"
  37 #include "zypp/parser/yum/RepoParser.h"
  38 //#include "zypp/parser/plaindir/RepoParser.h"
  39 #include "zypp/repo/susetags/Downloader.h"
  40 #include "zypp/parser/susetags/RepoParser.h"
  41
  42 #include "zypp/ZYppCallbacks.h"
  43
  44 #include "satsolver/pool.h"
  45 #include "satsolver/repo.h"
  46 #include "satsolver/repo_solv.h"
  47
  48 using namespace std;
  49 using namespace zypp;
  50 using namespace zypp::repo;
  51 using namespace zypp::filesystem;
  52
  53 using namespace zypp::repo;
  54
  55 ///////////////////////////////////////////////////////////////////
  56 namespace zypp
  57 { /////////////////////////////////////////////////////////////////
  58
  59   ///////////////////////////////////////////////////////////////////
  60   //
  61   //    CLASS NAME : RepoManagerOptions
  62   //
  63   ///////////////////////////////////////////////////////////////////
  64
  65   RepoManagerOptions::RepoManagerOptions()
  66   {
  67     repoCachePath    = ZConfig::instance().repoCachePath();
  68     repoRawCachePath = ZConfig::instance().repoMetadataPath();
  69     knownReposPath   = ZConfig::instance().knownReposPath();
  70   }
  71
  72   ////////////////////////////////////////////////////////////////////////////
  73
  74   /**
  75     * \short Simple callback to collect the results
  76     *
  77     * Classes like RepoFileParser call the callback
  78     * once per each repo in a file.
  79     *
  80     * Passing this functor as callback, you can collect
  81     * all resuls at the end, without dealing with async
  82     * code.
  83     */
  84     struct RepoCollector
  85     {
  86       RepoCollector()
  87       {
  88         MIL << endl;
  89       }
  90
  91       ~RepoCollector()
  92       {
  93         MIL << endl;
  94       }
  95
  96       bool collect( const RepoInfo &repo )
  97       {
  98         //MIL << "here in collector: " << repo.alias() << endl;
  99         repos.push_back(repo);
 100         //MIL << "added: " << repo.alias() << endl;
 101         return true;
 102       }
 103
 104       RepoInfoList repos;
 105     };
 106
 107   ////////////////////////////////////////////////////////////////////////////
 108
 109    /**
 110     * \short Internal version of clean cache
 111     *
 112     * Takes an extra SolvStore reference, so we avoid internally
 113     * having 2 SolvStores writing to the same database.
 114     */
 115   static void cleanCacheInternal( cache::SolvStore &store,
 116                                   const RepoInfo &info,
 117                                   const ProgressData::ReceiverFnc & progressrcv = ProgressData::ReceiverFnc() )
 118   {
 119     ProgressData progress;
 120     callback::SendReport<ProgressReport> report;
 121     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
 122     progress.name(str::form(_("Cleaning repository '%s' cache"), info.name().c_str()));
 123
 124     if ( !store.isCached(info.alias()) )
 125       return;
 126
 127     MIL << info.alias() << " cleaning cache..." << endl;
 128     data::RecordId id = store.lookupRepository(info.alias());
 129
 130     CombinedProgressData subprogrcv(progress);
 131
 132     store.cleanRepository(id, subprogrcv);
 133   }
 134
 135   ////////////////////////////////////////////////////////////////////////////
 136
 137   /**
 138    * Reads RepoInfo's from a repo file.
 139    *
 140    * \param file pathname of the file to read.
 141    */
 142   static std::list<RepoInfo> repositories_in_file( const Pathname & file )
 143   {
 144     MIL << "repo file: " << file << endl;
 145     RepoCollector collector;
 146     parser::RepoFileReader parser( file, bind( &RepoCollector::collect, &collector, _1 ) );
 147     return collector.repos;
 148   }
 149
 150   ////////////////////////////////////////////////////////////////////////////
 151
 152   std::list<RepoInfo> readRepoFile(const Url & repo_file)
 153    {
 154      // no interface to download a specific file, using workaround:
 155      //! \todo add MediaManager::provideFile(Url file_url) to easily access any file URLs? (no need for media access id or media_nr)
 156      Url url(repo_file);
 157      Pathname path(url.getPathName());
 158      url.setPathName ("/");
 159      MediaSetAccess access(url);
 160      Pathname local = access.provideFile(path);
 161
 162      DBG << "reading repo file " << repo_file << ", local path: " << local << endl;
 163
 164      return repositories_in_file(local);
 165    }
 166
 167   ////////////////////////////////////////////////////////////////////////////
 168
 169   /**
 170    * \short List of RepoInfo's from a directory
 171    *
 172    * Goes trough every file ending with ".repo" in a directory and adds all
 173    * RepoInfo's contained in that file.
 174    *
 175    * \param dir pathname of the directory to read.
 176    */
 177   static std::list<RepoInfo> repositories_in_dir( const Pathname &dir )
 178   {
 179     MIL << "directory " << dir << endl;
 180     list<RepoInfo> repos;
 181     list<Pathname> entries;
 182     if ( filesystem::readdir( entries, Pathname(dir), false ) != 0 )
 183       ZYPP_THROW(Exception("failed to read directory"));
 184
 185     str::regex allowedRepoExt("^\\.repo(_[0-9]+)?$");
 186     for ( list<Pathname>::const_iterator it = entries.begin(); it != entries.end(); ++it )
 187     {
 188       if (str::regex_match(it->extension(), allowedRepoExt))
 189       {
 190         list<RepoInfo> tmp = repositories_in_file( *it );
 191         repos.insert( repos.end(), tmp.begin(), tmp.end() );
 192
 193         //std::copy( collector.repos.begin(), collector.repos.end(), std::back_inserter(repos));
 194         //MIL << "ok" << endl;
 195       }
 196     }
 197     return repos;
 198   }
 199
 200   ////////////////////////////////////////////////////////////////////////////
 201
 202   static void assert_alias( const RepoInfo &info )
 203   {
 204     if (info.alias().empty())
 205         ZYPP_THROW(RepoNoAliasException());
 206   }
 207
 208   ////////////////////////////////////////////////////////////////////////////
 209
 210   static void assert_urls( const RepoInfo &info )
 211   {
 212     if (info.baseUrlsEmpty())
 213         ZYPP_THROW(RepoNoUrlException());
 214   }
 215
 216   ////////////////////////////////////////////////////////////////////////////
 217
 218   /**
 219    * \short Calculates the raw cache path for a repository
 220    */
 221   static Pathname rawcache_path_for_repoinfo( const RepoManagerOptions &opt, const RepoInfo &info )
 222   {
 223     assert_alias(info);
 224     return opt.repoRawCachePath + info.alias();
 225   }
 226
 227   ///////////////////////////////////////////////////////////////////
 228   //
 229   //    CLASS NAME : RepoManager::Impl
 230   //
 231   ///////////////////////////////////////////////////////////////////
 232
 233   /**
 234    * \short RepoManager implementation.
 235    */
 236   struct RepoManager::Impl
 237   {
 238     Impl( const RepoManagerOptions &opt )
 239       : options(opt)
 240     {
 241
 242     }
 243
 244     Impl()
 245     {
 246
 247     }
 248
 249     RepoManagerOptions options;
 250
 251   public:
 252     /** Offer default Impl. */
 253     static shared_ptr<Impl> nullimpl()
 254     {
 255       static shared_ptr<Impl> _nullimpl( new Impl );
 256       return _nullimpl;
 257     }
 258
 259   private:
 260     friend Impl * rwcowClone<Impl>( const Impl * rhs );
 261     /** clone for RWCOW_pointer */
 262     Impl * clone() const
 263     { return new Impl( *this ); }
 264   };
 265
 266   ///////////////////////////////////////////////////////////////////
 267
 268   /** \relates RepoManager::Impl Stream output */
 269   inline std::ostream & operator<<( std::ostream & str, const RepoManager::Impl & obj )
 270   {
 271     return str << "RepoManager::Impl";
 272   }
 273
 274   ///////////////////////////////////////////////////////////////////
 275   //
 276   //    CLASS NAME : RepoManager
 277   //
 278   ///////////////////////////////////////////////////////////////////
 279
 280   RepoManager::RepoManager( const RepoManagerOptions &opt )
 281   : _pimpl( new Impl(opt) )
 282   {}
 283
 284   ////////////////////////////////////////////////////////////////////////////
 285
 286   RepoManager::~RepoManager()
 287   {}
 288
 289   ////////////////////////////////////////////////////////////////////////////
 290
 291   std::list<RepoInfo> RepoManager::knownRepositories() const
 292   {
 293     MIL << endl;
 294
 295     if ( PathInfo(_pimpl->options.knownReposPath).isExist() )
 296     {
 297       RepoInfoList repos = repositories_in_dir(_pimpl->options.knownReposPath);
 298       for ( RepoInfoList::iterator it = repos.begin();
 299             it != repos.end();
 300             ++it )
 301       {
 302         // set the metadata path for the repo
 303         Pathname metadata_path = rawcache_path_for_repoinfo(_pimpl->options, (*it));
 304         (*it).setMetadataPath(metadata_path);
 305       }
 306       return repos;
 307     }
 308     else
 309       return std::list<RepoInfo>();
 310
 311     MIL << endl;
 312   }
 313
 314   ////////////////////////////////////////////////////////////////////////////
 315
 316   Pathname RepoManager::metadataPath( const RepoInfo &info ) const
 317   {
 318     return rawcache_path_for_repoinfo(_pimpl->options, info );
 319   }
 320
 321   ////////////////////////////////////////////////////////////////////////////
 322
 323   RepoStatus RepoManager::metadataStatus( const RepoInfo &info ) const
 324   {
 325     Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 326     RepoType repokind = info.type();
 327     RepoStatus status;
 328
 329     switch ( repokind.toEnum() )
 330     {
 331       case RepoType::NONE_e:
 332       // unknown, probe the local metadata
 333         repokind = probe(rawpath.asUrl());
 334       break;
 335       default:
 336       break;
 337     }
 338
 339     switch ( repokind.toEnum() )
 340     {
 341       case RepoType::RPMMD_e :
 342       {
 343         status = RepoStatus( rawpath + "/repodata/repomd.xml");
 344       }
 345       break;
 346
 347       case RepoType::YAST2_e :
 348       {
 349         // the order of RepoStatus && RepoStatus matters! (#304310)
 350         status = RepoStatus( rawpath + "/content") && (RepoStatus( rawpath + "/media.1/media"));
 351       }
 352       break;
 353
 354       case RepoType::RPMPLAINDIR_e :
 355       {
 356         if ( PathInfo(Pathname(rawpath + "/cookie")).isExist() )
 357           status = RepoStatus( rawpath + "/cookie");
 358       }
 359       break;
 360
 361       case RepoType::NONE_e :
 362         // Return default RepoStatus in case of RepoType::NONE
 363         // indicating it should be created?
 364         // ZYPP_THROW(RepoUnknownTypeException());
 365         break;
 366     }
 367     return status;
 368   }
 369
 370   void RepoManager::touchIndexFile(const RepoInfo & info)
 371   {
 372     Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 373
 374     RepoType repokind = info.type();
 375     if ( repokind.toEnum() == RepoType::NONE_e )
 376       // unknown, probe the local metadata
 377       repokind = probe(rawpath.asUrl());
 378     // if still unknown, just return
 379     if (repokind == RepoType::NONE_e)
 380       return;
 381
 382     Pathname p;
 383     switch ( repokind.toEnum() )
 384     {
 385       case RepoType::RPMMD_e :
 386         p = Pathname(rawpath + "/repodata/repomd.xml");
 387         break;
 388
 389       case RepoType::YAST2_e :
 390         p = Pathname(rawpath + "/content");
 391         break;
 392
 393       case RepoType::RPMPLAINDIR_e :
 394         p = Pathname(rawpath + "/cookie");
 395         break;
 396
 397       case RepoType::NONE_e :
 398       default:
 399         break;
 400     }
 401
 402     // touch the file, ignore error (they are logged anyway)
 403     filesystem::touch(p);
 404   }
 405
 406   bool RepoManager::checkIfToRefreshMetadata( const RepoInfo &info,
 407                                               const Url &url,
 408                                               RawMetadataRefreshPolicy policy )
 409   {
 410     assert_alias(info);
 411
 412     RepoStatus oldstatus;
 413     RepoStatus newstatus;
 414
 415     try
 416     {
 417       MIL << "Going to try to check whether refresh is needed for " << url << endl;
 418
 419       repo::RepoType repokind = info.type();
 420
 421       // if the type is unknown, try probing.
 422       switch ( repokind.toEnum() )
 423       {
 424         case RepoType::NONE_e:
 425           // unknown, probe it
 426           repokind = probe(url);
 427         break;
 428         default:
 429         break;
 430       }
 431
 432       Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 433       filesystem::assert_dir(rawpath);
 434       oldstatus = metadataStatus(info);
 435
 436       // now we've got the old (cached) status, we can decide repo.refresh.delay
 437       if (policy != RefreshForced)
 438       {
 439         // difference in seconds
 440         double diff = difftime(
 441           (Date::ValueType)Date::now(),
 442           (Date::ValueType)oldstatus.timestamp()) / 60;
 443
 444         DBG << "oldstatus: " << (Date::ValueType)oldstatus.timestamp() << endl;
 445         DBG << "current time: " << (Date::ValueType)Date::now() << endl;
 446         DBG << "last refresh = " << diff << " minutes ago" << endl;
 447
 448         if (diff < ZConfig::instance().repo_refresh_delay())
 449         {
 450           MIL << "Repository '" << info.alias()
 451               << "' has been refreshed less than repo.refresh.delay ("
 452               << ZConfig::instance().repo_refresh_delay()
 453               << ") minutes ago. Advising to skip refresh" << endl;
 454           return false;
 455         }
 456       }
 457
 458       // create temp dir as sibling of rawpath
 459       filesystem::TmpDir tmpdir( filesystem::TmpDir::makeSibling( rawpath ) );
 460
 461       if ( ( repokind.toEnum() == RepoType::RPMMD_e ) ||
 462            ( repokind.toEnum() == RepoType::YAST2_e ) )
 463       {
 464         MediaSetAccess media(url);
 465         shared_ptr<repo::Downloader> downloader_ptr;
 466
 467         if ( repokind.toEnum() == RepoType::RPMMD_e )
 468           downloader_ptr.reset(new yum::Downloader(info.path()));
 469         else
 470           downloader_ptr.reset( new susetags::Downloader(info.path()));
 471
 472         RepoStatus newstatus = downloader_ptr->status(media);
 473         bool refresh = false;
 474         if ( oldstatus.checksum() == newstatus.checksum() )
 475         {
 476           MIL << "repo has not changed" << endl;
 477           if ( policy == RefreshForced )
 478           {
 479             MIL << "refresh set to forced" << endl;
 480             refresh = true;
 481           }
 482         }
 483         else
 484         {
 485           MIL << "repo has changed, going to refresh" << endl;
 486           refresh = true;
 487         }
 488
 489         if (!refresh)
 490           touchIndexFile(info);
 491
 492         return refresh;
 493       }
 494 #if 0
 495       else if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 496       {
 497         RepoStatus newstatus = parser::plaindir::dirStatus(url.getPathName());
 498         bool refresh = false;
 499         if ( oldstatus.checksum() == newstatus.checksum() )
 500         {
 501           MIL << "repo has not changed" << endl;
 502           if ( policy == RefreshForced )
 503           {
 504             MIL << "refresh set to forced" << endl;
 505             refresh = true;
 506           }
 507         }
 508         else
 509         {
 510           MIL << "repo has changed, going to refresh" << endl;
 511           refresh = true;
 512         }
 513
 514         if (!refresh)
 515           touchIndexFile(info);
 516
 517         return refresh;
 518       }
 519 #endif
 520       else
 521       {
 522         ZYPP_THROW(RepoUnknownTypeException());
 523       }
 524     }
 525     catch ( const Exception &e )
 526     {
 527       ZYPP_CAUGHT(e);
 528       ERR << "refresh check failed for " << url << endl;
 529       ZYPP_RETHROW(e);
 530     }
 531
 532     return true; // default
 533   }
 534
 535   void RepoManager::refreshMetadata( const RepoInfo &info,
 536                                      RawMetadataRefreshPolicy policy,
 537                                      const ProgressData::ReceiverFnc & progress )
 538   {
 539     assert_alias(info);
 540     assert_urls(info);
 541
 542     // we will throw this later if no URL checks out fine
 543     RepoException rexception(_("Valid metadata not found at specified URL(s)"));
 544
 545     // try urls one by one
 546     for ( RepoInfo::urls_const_iterator it = info.baseUrlsBegin(); it != info.baseUrlsEnd(); ++it )
 547     {
 548       try
 549       {
 550         Url url(*it);
 551
 552         // check whether to refresh metadata
 553         // if the check fails for this url, it throws, so another url will be checked
 554         if (!checkIfToRefreshMetadata(info, url, policy))
 555           return;
 556
 557         MIL << "Going to refresh metadata from " << url << endl;
 558
 559         repo::RepoType repokind = info.type();
 560
 561         // if the type is unknown, try probing.
 562         switch ( repokind.toEnum() )
 563         {
 564           case RepoType::NONE_e:
 565             // unknown, probe it
 566             repokind = probe(*it);
 567           break;
 568           default:
 569           break;
 570         }
 571
 572         Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 573         filesystem::assert_dir(rawpath);
 574
 575         // create temp dir as sibling of rawpath
 576         filesystem::TmpDir tmpdir( filesystem::TmpDir::makeSibling( rawpath ) );
 577
 578         if ( ( repokind.toEnum() == RepoType::RPMMD_e ) ||
 579              ( repokind.toEnum() == RepoType::YAST2_e ) )
 580         {
 581           MediaSetAccess media(url);
 582           shared_ptr<repo::Downloader> downloader_ptr;
 583
 584           if ( repokind.toEnum() == RepoType::RPMMD_e )
 585             downloader_ptr.reset(new yum::Downloader(info.path()));
 586           else
 587             downloader_ptr.reset( new susetags::Downloader(info.path()));
 588
 589           /**
 590            * Given a downloader, sets the other repos raw metadata
 591            * path as cache paths for the fetcher, so if another
 592            * repo has the same file, it will not download it
 593            * but copy it from the other repository
 594            */
 595           std::list<RepoInfo> repos = knownRepositories();
 596           for ( std::list<RepoInfo>::const_iterator it = repos.begin();
 597                 it != repos.end();
 598                 ++it )
 599           {
 600             downloader_ptr->addCachePath(rawcache_path_for_repoinfo( _pimpl->options, *it ));
 601           }
 602
 603           downloader_ptr->download( media, tmpdir.path());
 604         }
 605 #if 0
 606         else if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 607         {
 608           RepoStatus newstatus = parser::plaindir::dirStatus(url.getPathName());
 609
 610           std::ofstream file(( tmpdir.path() + "/cookie").c_str());
 611           if (!file) {
 612             ZYPP_THROW (Exception( "Can't open " + tmpdir.path().asString() + "/cookie" ) );
 613           }
 614           file << url << endl;
 615           file << newstatus.checksum() << endl;
 616
 617           file.close();
 618         }
 619 #endif
 620         else
 621         {
 622           ZYPP_THROW(RepoUnknownTypeException());
 623         }
 624
 625         // ok we have the metadata, now exchange
 626         // the contents
 627         TmpDir oldmetadata( TmpDir::makeSibling( rawpath ) );
 628         filesystem::rename( rawpath, oldmetadata.path() );
 629         // move the just downloaded there
 630         filesystem::rename( tmpdir.path(), rawpath );
 631         // we are done.
 632         return;
 633       }
 634       catch ( const Exception &e )
 635       {
 636         ZYPP_CAUGHT(e);
 637         ERR << "Trying another url..." << endl;
 638
 639         // remember the exception caught for the *first URL*
 640         // if all other URLs fail, the rexception will be thrown with the
 641         // cause of the problem of the first URL remembered
 642         if (it == info.baseUrlsBegin())
 643           rexception.remember(e);
 644       }
 645     } // for every url
 646     ERR << "No more urls..." << endl;
 647     ZYPP_THROW(rexception);
 648   }
 649
 650   ////////////////////////////////////////////////////////////////////////////
 651
 652   void RepoManager::cleanMetadata( const RepoInfo &info,
 653                                    const ProgressData::ReceiverFnc & progressfnc )
 654   {
 655     ProgressData progress(100);
 656     progress.sendTo(progressfnc);
 657
 658     filesystem::recursive_rmdir(rawcache_path_for_repoinfo(_pimpl->options, info));
 659     progress.toMax();
 660   }
 661
 662   void RepoManager::buildCache( const RepoInfo &info,
 663                                 CacheBuildPolicy policy,
 664                                 const ProgressData::ReceiverFnc & progressrcv )
 665   {
 666     assert_alias(info);
 667     Pathname rawpath = rawcache_path_for_repoinfo(_pimpl->options, info);
 668
 669     Pathname base = _pimpl->options.repoCachePath + Pathname(info.alias());
 670     Pathname solvfile = base.extend(".solv");
 671
 672     //cache::SolvStore store(_pimpl->options.repoCachePath);
 673
 674     RepoStatus raw_metadata_status = metadataStatus(info);
 675     if ( raw_metadata_status.empty() )
 676     {
 677       ZYPP_THROW(RepoMetadataException(info));
 678     }
 679
 680     bool needs_cleaning = false;
 681     if ( isCached( info ) )
 682     {
 683       MIL << info.alias() << " is already cached." << endl;
 684       //data::RecordId id = store.lookupRepository(info.alias());
 685       RepoStatus cache_status = cacheStatus(info);
 686
 687       if ( cache_status.checksum() == raw_metadata_status.checksum() )
 688       {
 689         MIL << info.alias() << " cache is up to date with metadata." << endl;
 690         if ( policy == BuildIfNeeded ) {
 691           return;
 692         }
 693         else {
 694           MIL << info.alias() << " cache rebuild is forced" << endl;
 695         }
 696       }
 697
 698       needs_cleaning = true;
 699     }
 700
 701     ProgressData progress(100);
 702     callback::SendReport<ProgressReport> report;
 703     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
 704     progress.name(str::form(_("Building repository '%s' cache"), info.name().c_str()));
 705     progress.toMin();
 706
 707     if (needs_cleaning)
 708     {
 709 //       Pathname name = _pimpl->options.repoCachePath;
 710 //       //data::RecordId id = store.lookupRepository(info.alias());
 711 //       ostringstream os;
 712 //       os << id.get();
 713 //       name += os.str() + ".solv";
 714 //       unlink (name);
 715 //       cleanCacheInternal( store, info);
 716       cleanCache(info);
 717     }
 718
 719     MIL << info.alias() << " building cache..." << endl;
 720     //data::RecordId id = store.lookupOrAppendRepository(info.alias());
 721     // do we have type?
 722     repo::RepoType repokind = info.type();
 723
 724     // if the type is unknown, try probing.
 725     switch ( repokind.toEnum() )
 726     {
 727       case RepoType::NONE_e:
 728         // unknown, probe the local metadata
 729         repokind = probe(rawpath.asUrl());
 730       break;
 731       default:
 732       break;
 733     }
 734
 735
 736     switch ( repokind.toEnum() )
 737     {
 738       case RepoType::RPMMD_e :
 739       case RepoType::YAST2_e :
 740       {
 741         string cmd = "repo2solv.sh \"";
 742         cmd += rawpath.asString() + "\" > " + solvfile.asString();
 743         int ret = system (cmd.c_str());
 744         if (WIFEXITED (ret) && WEXITSTATUS (ret) != 0)
 745           ZYPP_THROW(RepoUnknownTypeException());
 746       }
 747       break;
 748       default:
 749       break;
 750     }
 751 #if 0
 752     switch ( repokind.toEnum() )
 753     {
 754       case RepoType::RPMMD_e :
 755       if (0)
 756       {
 757         CombinedProgressData subprogrcv( progress, 100);
 758         parser::yum::RepoParser parser(id, store, parser::yum::RepoParserOpts(), subprogrcv);
 759         parser.parse(rawpath);
 760           // no error
 761       }
 762       break;
 763       case RepoType::YAST2_e :
 764       if (0)
 765       {
 766         CombinedProgressData subprogrcv( progress, 100);
 767         parser::susetags::RepoParser parser(id, store, subprogrcv);
 768         parser.parse(rawpath);
 769         // no error
 770       }
 771       break;
 772 #endif
 773 #if 0
 774       case RepoType::RPMPLAINDIR_e :
 775       {
 776         CombinedProgressData subprogrcv( progress, 100);
 777         InputStream is(rawpath + "cookie");
 778         string buffer;
 779         getline( is.stream(), buffer);
 780         Url url(buffer);
 781         parser::plaindir::RepoParser parser(id, store, subprogrcv);
 782         parser.parse(url.getPathName());
 783       }
 784       break;
 785
 786       default:
 787         ZYPP_THROW(RepoUnknownTypeException());
 788     }
 789 #endif
 790     // update timestamp and checksum
 791     //store.updateRepositoryStatus(id, raw_metadata_status);
 792     setCacheStatus(info.alias(), raw_metadata_status);
 793     MIL << "Commit cache.." << endl;
 794     //store.commit();
 795     //progress.toMax();
 796   }
 797
 798   ////////////////////////////////////////////////////////////////////////////
 799
 800   repo::RepoType RepoManager::probe( const Url &url ) const
 801   {
 802     if ( url.getScheme() == "dir" && ! PathInfo( url.getPathName() ).isDir() )
 803     {
 804       // Handle non existing local directory in advance, as
 805       // MediaSetAccess does not support it.
 806       return repo::RepoType::NONE;
 807     }
 808
 809     try
 810     {
 811       MediaSetAccess access(url);
 812       if ( access.doesFileExist("/repodata/repomd.xml") )
 813         return repo::RepoType::RPMMD;
 814       if ( access.doesFileExist("/content") )
 815         return repo::RepoType::YAST2;
 816
 817       // if it is a local url of type dir
 818       if ( (! media::MediaManager::downloads(url)) && ( url.getScheme() == "dir" ) )
 819       {
 820         Pathname path = Pathname(url.getPathName());
 821         if ( PathInfo(path).isDir() )
 822         {
 823           // allow empty dirs for now
 824           return repo::RepoType::RPMPLAINDIR;
 825         }
 826       }
 827     }
 828     catch ( const media::MediaException &e )
 829     {
 830       ZYPP_CAUGHT(e);
 831       RepoException enew("Error trying to read from " + url.asString());
 832       enew.remember(e);
 833       ZYPP_THROW(enew);
 834     }
 835     catch ( const Exception &e )
 836     {
 837       ZYPP_CAUGHT(e);
 838       Exception enew("Unknown error reading from " + url.asString());
 839       enew.remember(e);
 840       ZYPP_THROW(enew);
 841     }
 842
 843     return repo::RepoType::NONE;
 844   }
 845
 846   ////////////////////////////////////////////////////////////////////////////
 847
 848   void RepoManager::cleanCache( const RepoInfo &info,
 849                                 const ProgressData::ReceiverFnc & progressrcv )
 850   {
 851     Pathname name = _pimpl->options.repoCachePath;
 852     name += info.alias() + ".solv";
 853     unlink (name);
 854   }
 855
 856   ////////////////////////////////////////////////////////////////////////////
 857
 858   bool RepoManager::isCached( const RepoInfo &info ) const
 859   {
 860     Pathname name = _pimpl->options.repoCachePath;
 861     return PathInfo(name + Pathname(info.alias()).extend(".solv")).isExist();
 862   }
 863
 864   RepoStatus RepoManager::cacheStatus( const RepoInfo &info ) const
 865   {
 866     Pathname name = _pimpl->options.repoCachePath;
 867     RepoStatus status;
 868     Pathname base = _pimpl->options.repoCachePath + Pathname(info.alias());
 869     Pathname solvfile = base.extend(".solv");
 870     Pathname cookiefile = base.extend(".cookie");
 871
 872     std::ifstream file(cookiefile.c_str());
 873     if (!file) {
 874       ZYPP_THROW (Exception( "Can't open " + cookiefile.asString() ) );
 875     }
 876
 877     std::string buffer;
 878     while(file && !file.eof()) {
 879       getline(file, buffer);
 880     }
 881
 882     std::vector<std::string> words;
 883     if ( str::split( buffer, std::back_inserter(words) ) != 2 )
 884       ZYPP_THROW (Exception( "corrupt file " + cookiefile.asString() ) );
 885
 886     status.setTimestamp(Date(str::strtonum<time_t>(words[1])));
 887     status.setChecksum(words[0]);
 888     return status;
 889   }
 890
 891   void RepoManager::setCacheStatus( const string &alias, const RepoStatus &status )
 892   {
 893     Pathname base = _pimpl->options.repoCachePath + alias;
 894     Pathname cookiefile = base.extend(".cookie");
 895
 896     std::ofstream file(cookiefile.c_str());
 897     if (!file) {
 898       ZYPP_THROW (Exception( "Can't open " + cookiefile.asString() ) );
 899     }
 900     file << status;
 901     file.close();
 902   }
 903
 904   map<data::RecordId, Repo *> repo2solv;
 905
 906   Repository RepoManager::createFromCache( const RepoInfo &info,
 907                                            const ProgressData::ReceiverFnc & progressrcv )
 908   {
 909     callback::SendReport<ProgressReport> report;
 910     ProgressData progress;
 911     progress.sendTo(ProgressReportAdaptor( progressrcv, report ));
 912     //progress.sendTo( progressrcv );
 913     progress.name(str::form(_("Reading repository '%s' cache"), info.name().c_str()));
 914
 915     //_pimpl->options.repoCachePath
 916     if ( ! isCached( info ) )
 917       ZYPP_THROW(RepoNotCachedException());
 918
 919     MIL << "Repository " << info.alias() << " is cached" << endl;
 920
 921     //sat::Pool satpool( sat::Pool::instance() );
 922
 923     //Pathname name = _pimpl->options.repoCachePath + Pathname(info.alias()).extend(".solv");
 924
 925     try
 926     {
 927       //satpool.addRepoSolv(name);
 928     }
 929     catch ( const Exception &e )
 930     {
 931
 932     }
 933
 934     CombinedProgressData subprogrcv(progress);
 935
 936 //     repo::cached::RepoOptions opts( info, _pimpl->options.repoCachePath, id );
 937 //     opts.readingResolvablesProgress = subprogrcv;
 938 //     //opts.repo = repo;
 939 //     repo::cached::RepoImpl::Ptr repoimpl =
 940 //         new repo::cached::RepoImpl( opts );
 941 //
 942 //     repoimpl->resolvables();
 943     // read the resolvables from cache
 944     return Repository::noRepository;
 945   }
 946
 947   ////////////////////////////////////////////////////////////////////////////
 948
 949   /**
 950    * Generate a non existing filename in a directory, using a base
 951    * name. For example if a directory contains 3 files
 952    *
 953    * |-- bar
 954    * |-- foo
 955    * `-- moo
 956    *
 957    * If you try to generate a unique filename for this directory,
 958    * based on "ruu" you will get "ruu", but if you use the base
 959    * "foo" you will get "foo_1"
 960    *
 961    * \param dir Directory where the file needs to be unique
 962    * \param basefilename string to base the filename on.
 963    */
 964   static Pathname generate_non_existing_name( const Pathname &dir,
 965                                               const std::string &basefilename )
 966   {
 967     string final_filename = basefilename;
 968     int counter = 1;
 969     while ( PathInfo(dir + final_filename).isExist() )
 970     {
 971       final_filename = basefilename + "_" + str::numstring(counter);
 972       counter++;
 973     }
 974     return dir + Pathname(final_filename);
 975   }
 976
 977   ////////////////////////////////////////////////////////////////////////////
 978
 979   /**
 980    * \short Generate a related filename from a repo info
 981    *
 982    * From a repo info, it will try to use the alias as a filename
 983    * escaping it if necessary. Other fallbacks can be added to
 984    * this function in case there is no way to use the alias
 985    */
 986   static std::string generate_filename( const RepoInfo &info )
 987   {
 988     std::string fnd="/";
 989     std::string rep="_";
 990     std::string filename = info.alias();
 991     // replace slashes with underscores
 992     size_t pos = filename.find(fnd);
 993     while(pos!=string::npos)
 994     {
 995       filename.replace(pos,fnd.length(),rep);
 996       pos = filename.find(fnd,pos+rep.length());
 997     }
 998     filename = Pathname(filename).extend(".repo").asString();
 999     MIL << "generating filename for repo [" << info.alias() << "] : '" << filename << "'" << endl;
1000     return filename;
1001   }
1002
1003
1004   ////////////////////////////////////////////////////////////////////////////
1005
1006   void RepoManager::addRepository( const RepoInfo &info,
1007                                    const ProgressData::ReceiverFnc & progressrcv )
1008   {
1009     assert_alias(info);
1010
1011     ProgressData progress(100);
1012     callback::SendReport<ProgressReport> report;
1013     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
1014     progress.name(str::form(_("Adding repository '%s'"), info.name().c_str()));
1015     progress.toMin();
1016
1017     std::list<RepoInfo> repos = knownRepositories();
1018     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1019           it != repos.end();
1020           ++it )
1021     {
1022       if ( info.alias() == (*it).alias() )
1023         ZYPP_THROW(RepoAlreadyExistsException(info.alias()));
1024     }
1025
1026     RepoInfo tosave = info;
1027
1028     // check the first url for now
1029     if ( ZConfig::instance().repo_add_probe()
1030         || ( tosave.type() == RepoType::NONE && tosave.enabled()) )
1031     {
1032       DBG << "unknown repository type, probing" << endl;
1033
1034       RepoType probedtype;
1035       probedtype = probe(*tosave.baseUrlsBegin());
1036       if ( tosave.baseUrlsSize() > 0 )
1037       {
1038         if ( probedtype == RepoType::NONE )
1039           ZYPP_THROW(RepoUnknownTypeException());
1040         else
1041           tosave.setType(probedtype);
1042       }
1043     }
1044
1045     progress.set(50);
1046
1047     // assert the directory exists
1048     filesystem::assert_dir(_pimpl->options.knownReposPath);
1049
1050     Pathname repofile = generate_non_existing_name(_pimpl->options.knownReposPath,
1051                                                     generate_filename(tosave));
1052     // now we have a filename that does not exists
1053     MIL << "Saving repo in " << repofile << endl;
1054
1055     std::ofstream file(repofile.c_str());
1056     if (!file) {
1057       ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
1058     }
1059
1060     tosave.dumpRepoOn(file);
1061     progress.toMax();
1062     MIL << "done" << endl;
1063   }
1064
1065   void RepoManager::addRepositories( const Url &url,
1066                                      const ProgressData::ReceiverFnc & progressrcv )
1067   {
1068     std::list<RepoInfo> knownrepos = knownRepositories();
1069     std::list<RepoInfo> repos = readRepoFile(url);
1070     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1071           it != repos.end();
1072           ++it )
1073     {
1074       // look if the alias is in the known repos.
1075       for ( std::list<RepoInfo>::const_iterator kit = knownrepos.begin();
1076           kit != knownrepos.end();
1077           ++kit )
1078       {
1079         if ( (*it).alias() == (*kit).alias() )
1080         {
1081           ERR << "To be added repo " << (*it).alias() << " conflicts with existing repo " << (*kit).alias() << endl;
1082           ZYPP_THROW(RepoAlreadyExistsException((*it).alias()));
1083         }
1084       }
1085     }
1086
1087     string filename = Pathname(url.getPathName()).basename();
1088
1089     if ( filename == Pathname() )
1090       ZYPP_THROW(RepoException("Invalid repo file name at " + url.asString() ));
1091
1092     // assert the directory exists
1093     filesystem::assert_dir(_pimpl->options.knownReposPath);
1094
1095     Pathname repofile = generate_non_existing_name(_pimpl->options.knownReposPath, filename);
1096     // now we have a filename that does not exists
1097     MIL << "Saving " << repos.size() << " repo" << ( repos.size() ? "s" : "" ) << " in " << repofile << endl;
1098
1099     std::ofstream file(repofile.c_str());
1100     if (!file) {
1101       ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
1102     }
1103
1104     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1105           it != repos.end();
1106           ++it )
1107     {
1108       MIL << "Saving " << (*it).alias() << endl;
1109       (*it).dumpRepoOn(file);
1110     }
1111     MIL << "done" << endl;
1112   }
1113
1114   ////////////////////////////////////////////////////////////////////////////
1115
1116   void RepoManager::removeRepository( const RepoInfo & info,
1117                                       const ProgressData::ReceiverFnc & progressrcv)
1118   {
1119     ProgressData progress;
1120     callback::SendReport<ProgressReport> report;
1121     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
1122     progress.name(str::form(_("Removing repository '%s'"), info.name().c_str()));
1123
1124     MIL << "Going to delete repo " << info.alias() << endl;
1125
1126     std::list<RepoInfo> repos = knownRepositories();
1127     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1128           it != repos.end();
1129           ++it )
1130     {
1131       // they can be the same only if the provided is empty, that means
1132       // the provided repo has no alias
1133       // then skip
1134       if ( (!info.alias().empty()) && ( info.alias() != (*it).alias() ) )
1135         continue;
1136
1137       // TODO match by url
1138
1139       // we have a matcing repository, now we need to know
1140       // where it does come from.
1141       RepoInfo todelete = *it;
1142       if (todelete.filepath().empty())
1143       {
1144         ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
1145       }
1146       else
1147       {
1148         // figure how many repos are there in the file:
1149         std::list<RepoInfo> filerepos = repositories_in_file(todelete.filepath());
1150         if ( (filerepos.size() == 1) && ( filerepos.front().alias() == todelete.alias() ) )
1151         {
1152           // easy, only this one, just delete the file
1153           if ( filesystem::unlink(todelete.filepath()) != 0 )
1154           {
1155             ZYPP_THROW(RepoException("Can't delete " + todelete.filepath().asString()));
1156           }
1157           MIL << todelete.alias() << " sucessfully deleted." << endl;
1158         }
1159         else
1160         {
1161           // there are more repos in the same file
1162           // write them back except the deleted one.
1163           //TmpFile tmp;
1164           //std::ofstream file(tmp.path().c_str());
1165
1166           // assert the directory exists
1167           filesystem::assert_dir(todelete.filepath().dirname());
1168
1169           std::ofstream file(todelete.filepath().c_str());
1170           if (!file) {
1171             //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
1172             ZYPP_THROW (Exception( "Can't open " + todelete.filepath().asString() ) );
1173           }
1174           for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
1175                 fit != filerepos.end();
1176                 ++fit )
1177           {
1178             if ( (*fit).alias() != todelete.alias() )
1179               (*fit).dumpRepoOn(file);
1180           }
1181         }
1182
1183         CombinedProgressData subprogrcv(progress, 70);
1184         CombinedProgressData cleansubprogrcv(progress, 30);
1185         // now delete it from cache
1186         cleanCache( todelete, subprogrcv);
1187         // now delete metadata (#301037)
1188         cleanMetadata( todelete, cleansubprogrcv);
1189         MIL << todelete.alias() << " sucessfully deleted." << endl;
1190         return;
1191       } // else filepath is empty
1192
1193     }
1194     // should not be reached on a sucess workflow
1195     ZYPP_THROW(RepoNotFoundException(info));
1196   }
1197
1198   ////////////////////////////////////////////////////////////////////////////
1199
1200   void RepoManager::modifyRepository( const std::string &alias,
1201                                       const RepoInfo & newinfo,
1202                                       const ProgressData::ReceiverFnc & progressrcv )
1203   {
1204     RepoInfo toedit = getRepositoryInfo(alias);
1205
1206     if (toedit.filepath().empty())
1207     {
1208       ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
1209     }
1210     else
1211     {
1212       // figure how many repos are there in the file:
1213       std::list<RepoInfo> filerepos = repositories_in_file(toedit.filepath());
1214
1215       // there are more repos in the same file
1216       // write them back except the deleted one.
1217       //TmpFile tmp;
1218       //std::ofstream file(tmp.path().c_str());
1219
1220       // assert the directory exists
1221       filesystem::assert_dir(toedit.filepath().dirname());
1222
1223       std::ofstream file(toedit.filepath().c_str());
1224       if (!file) {
1225         //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
1226         ZYPP_THROW (Exception( "Can't open " + toedit.filepath().asString() ) );
1227       }
1228       for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
1229             fit != filerepos.end();
1230             ++fit )
1231       {
1232           // if the alias is different, dump the original
1233           // if it is the same, dump the provided one
1234           if ( (*fit).alias() != toedit.alias() )
1235             (*fit).dumpRepoOn(file);
1236           else
1237             newinfo.dumpRepoOn(file);
1238       }
1239     }
1240   }
1241
1242   ////////////////////////////////////////////////////////////////////////////
1243
1244   RepoInfo RepoManager::getRepositoryInfo( const std::string &alias,
1245                                            const ProgressData::ReceiverFnc & progressrcv )
1246   {
1247     std::list<RepoInfo> repos = knownRepositories();
1248     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1249           it != repos.end();
1250           ++it )
1251     {
1252       if ( (*it).alias() == alias )
1253         return *it;
1254     }
1255     RepoInfo info;
1256     info.setAlias(info.alias());
1257     ZYPP_THROW(RepoNotFoundException(info));
1258   }
1259
1260   ////////////////////////////////////////////////////////////////////////////
1261
1262   RepoInfo RepoManager::getRepositoryInfo( const Url & url,
1263                                            const url::ViewOption & urlview,
1264                                            const ProgressData::ReceiverFnc & progressrcv )
1265   {
1266     std::list<RepoInfo> repos = knownRepositories();
1267     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1268           it != repos.end();
1269           ++it )
1270     {
1271       for(RepoInfo::urls_const_iterator urlit = (*it).baseUrlsBegin();
1272           urlit != (*it).baseUrlsEnd();
1273           ++urlit)
1274       {
1275         if ((*urlit).asString(urlview) == url.asString(urlview))
1276           return *it;
1277       }
1278     }
1279     RepoInfo info;
1280     info.setAlias(info.alias());
1281     info.setBaseUrl(url);
1282     ZYPP_THROW(RepoNotFoundException(info));
1283   }
1284
1285   ////////////////////////////////////////////////////////////////////////////
1286
1287   std::ostream & operator<<( std::ostream & str, const RepoManager & obj )
1288   {
1289     return str << *obj._pimpl;
1290   }
1291
1292   /////////////////////////////////////////////////////////////////
1293 } // namespace zypp
1294 ///////////////////////////////////////////////////////////////////