zypp/RepoManager.cc

   1 /*---------------------------------------------------------------------\
   2 |                          ____ _   __ __ ___                          |
   3 |                         |__  / \ / / . \ . \                         |
   4 |                           / / \ V /|  _/  _/                         |
   5 |                          / /__ | | | | | |                           |
   6 |                         /_____||_| |_| |_|                           |
   7 |                                                                      |
   8 \---------------------------------------------------------------------*/
   9 /** \file       zypp/RepoManager.cc
  10  *
  11 */
  12
  13 #include <cstdlib>
  14 #include <iostream>
  15 #include <fstream>
  16 #include <sstream>
  17 #include <list>
  18 #include <algorithm>
  19 #include "zypp/base/InputStream.h"
  20 #include "zypp/base/Logger.h"
  21 #include "zypp/base/Gettext.h"
  22 #include "zypp/base/Function.h"
  23 #include "zypp/base/Regex.h"
  24 #include "zypp/PathInfo.h"
  25 #include "zypp/TmpPath.h"
  26
  27 #include "zypp/repo/RepoException.h"
  28 #include "zypp/RepoManager.h"
  29
  30 #include "zypp/cache/SolvStore.h"
  31 //#include "zypp/repo/cached/RepoImpl.h"
  32 #include "zypp/media/MediaManager.h"
  33 #include "zypp/MediaSetAccess.h"
  34 #include "zypp/ExternalProgram.h"
  35
  36 #include "zypp/parser/RepoFileReader.h"
  37 #include "zypp/repo/yum/Downloader.h"
  38 #include "zypp/parser/yum/RepoParser.h"
  39 //#include "zypp/parser/plaindir/RepoParser.h"
  40 #include "zypp/repo/susetags/Downloader.h"
  41 #include "zypp/parser/susetags/RepoParser.h"
  42
  43 #include "zypp/ZYppCallbacks.h"
  44
  45 #include "sat/Pool.h"
  46 #include "satsolver/pool.h"
  47 #include "satsolver/repo.h"
  48 #include "satsolver/repo_solv.h"
  49
  50 using namespace std;
  51 using namespace zypp;
  52 using namespace zypp::repo;
  53 using namespace zypp::filesystem;
  54
  55 using namespace zypp::repo;
  56
  57 ///////////////////////////////////////////////////////////////////
  58 namespace zypp
  59 { /////////////////////////////////////////////////////////////////
  60
  61   ///////////////////////////////////////////////////////////////////
  62   //
  63   //    CLASS NAME : RepoManagerOptions
  64   //
  65   ///////////////////////////////////////////////////////////////////
  66
  67   RepoManagerOptions::RepoManagerOptions()
  68   {
  69     repoCachePath    = ZConfig::instance().repoCachePath();
  70     repoRawCachePath = ZConfig::instance().repoMetadataPath();
  71     knownReposPath   = ZConfig::instance().knownReposPath();
  72   }
  73
  74   ////////////////////////////////////////////////////////////////////////////
  75
  76   /**
  77     * \short Simple callback to collect the results
  78     *
  79     * Classes like RepoFileParser call the callback
  80     * once per each repo in a file.
  81     *
  82     * Passing this functor as callback, you can collect
  83     * all resuls at the end, without dealing with async
  84     * code.
  85     */
  86     struct RepoCollector
  87     {
  88       RepoCollector()
  89       {
  90         MIL << endl;
  91       }
  92
  93       ~RepoCollector()
  94       {
  95         MIL << endl;
  96       }
  97
  98       bool collect( const RepoInfo &repo )
  99       {
 100         //MIL << "here in collector: " << repo.alias() << endl;
 101         repos.push_back(repo);
 102         //MIL << "added: " << repo.alias() << endl;
 103         return true;
 104       }
 105
 106       RepoInfoList repos;
 107     };
 108
 109   ////////////////////////////////////////////////////////////////////////////
 110
 111    /**
 112     * \short Internal version of clean cache
 113     *
 114     * Takes an extra SolvStore reference, so we avoid internally
 115     * having 2 SolvStores writing to the same database.
 116     */
 117   static void cleanCacheInternal( cache::SolvStore &store,
 118                                   const RepoInfo &info,
 119                                   const ProgressData::ReceiverFnc & progressrcv = ProgressData::ReceiverFnc() )
 120   {
 121     ProgressData progress;
 122     callback::SendReport<ProgressReport> report;
 123     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
 124     progress.name(str::form(_("Cleaning repository '%s' cache"), info.name().c_str()));
 125
 126     if ( !store.isCached(info.alias()) )
 127       return;
 128
 129     MIL << info.alias() << " cleaning cache..." << endl;
 130     data::RecordId id = store.lookupRepository(info.alias());
 131
 132     CombinedProgressData subprogrcv(progress);
 133
 134     store.cleanRepository(id, subprogrcv);
 135   }
 136
 137   ////////////////////////////////////////////////////////////////////////////
 138
 139   /**
 140    * Reads RepoInfo's from a repo file.
 141    *
 142    * \param file pathname of the file to read.
 143    */
 144   static std::list<RepoInfo> repositories_in_file( const Pathname & file )
 145   {
 146     MIL << "repo file: " << file << endl;
 147     RepoCollector collector;
 148     parser::RepoFileReader parser( file, bind( &RepoCollector::collect, &collector, _1 ) );
 149     return collector.repos;
 150   }
 151
 152   ////////////////////////////////////////////////////////////////////////////
 153
 154   std::list<RepoInfo> readRepoFile(const Url & repo_file)
 155    {
 156      // no interface to download a specific file, using workaround:
 157      //! \todo add MediaManager::provideFile(Url file_url) to easily access any file URLs? (no need for media access id or media_nr)
 158      Url url(repo_file);
 159      Pathname path(url.getPathName());
 160      url.setPathName ("/");
 161      MediaSetAccess access(url);
 162      Pathname local = access.provideFile(path);
 163
 164      DBG << "reading repo file " << repo_file << ", local path: " << local << endl;
 165
 166      return repositories_in_file(local);
 167    }
 168
 169   ////////////////////////////////////////////////////////////////////////////
 170
 171   /**
 172    * \short List of RepoInfo's from a directory
 173    *
 174    * Goes trough every file ending with ".repo" in a directory and adds all
 175    * RepoInfo's contained in that file.
 176    *
 177    * \param dir pathname of the directory to read.
 178    */
 179   static std::list<RepoInfo> repositories_in_dir( const Pathname &dir )
 180   {
 181     MIL << "directory " << dir << endl;
 182     list<RepoInfo> repos;
 183     list<Pathname> entries;
 184     if ( filesystem::readdir( entries, Pathname(dir), false ) != 0 )
 185       ZYPP_THROW(Exception("failed to read directory"));
 186
 187     str::regex allowedRepoExt("^\\.repo(_[0-9]+)?$");
 188     for ( list<Pathname>::const_iterator it = entries.begin(); it != entries.end(); ++it )
 189     {
 190       if (str::regex_match(it->extension(), allowedRepoExt))
 191       {
 192         list<RepoInfo> tmp = repositories_in_file( *it );
 193         repos.insert( repos.end(), tmp.begin(), tmp.end() );
 194
 195         //std::copy( collector.repos.begin(), collector.repos.end(), std::back_inserter(repos));
 196         //MIL << "ok" << endl;
 197       }
 198     }
 199     return repos;
 200   }
 201
 202   ////////////////////////////////////////////////////////////////////////////
 203
 204   static void assert_alias( const RepoInfo &info )
 205   {
 206     if (info.alias().empty())
 207         ZYPP_THROW(RepoNoAliasException());
 208   }
 209
 210   ////////////////////////////////////////////////////////////////////////////
 211
 212   static void assert_urls( const RepoInfo &info )
 213   {
 214     if (info.baseUrlsEmpty())
 215         ZYPP_THROW(RepoNoUrlException());
 216   }
 217
 218   ////////////////////////////////////////////////////////////////////////////
 219
 220   /**
 221    * \short Calculates the raw cache path for a repository
 222    */
 223   static Pathname rawcache_path_for_repoinfo( const RepoManagerOptions &opt, const RepoInfo &info )
 224   {
 225     assert_alias(info);
 226     return opt.repoRawCachePath + info.alias();
 227   }
 228
 229   ///////////////////////////////////////////////////////////////////
 230   //
 231   //    CLASS NAME : RepoManager::Impl
 232   //
 233   ///////////////////////////////////////////////////////////////////
 234
 235   /**
 236    * \short RepoManager implementation.
 237    */
 238   struct RepoManager::Impl
 239   {
 240     Impl( const RepoManagerOptions &opt )
 241       : options(opt)
 242     {
 243
 244     }
 245
 246     Impl()
 247     {
 248
 249     }
 250
 251     RepoManagerOptions options;
 252
 253   public:
 254     /** Offer default Impl. */
 255     static shared_ptr<Impl> nullimpl()
 256     {
 257       static shared_ptr<Impl> _nullimpl( new Impl );
 258       return _nullimpl;
 259     }
 260
 261   private:
 262     friend Impl * rwcowClone<Impl>( const Impl * rhs );
 263     /** clone for RWCOW_pointer */
 264     Impl * clone() const
 265     { return new Impl( *this ); }
 266   };
 267
 268   ///////////////////////////////////////////////////////////////////
 269
 270   /** \relates RepoManager::Impl Stream output */
 271   inline std::ostream & operator<<( std::ostream & str, const RepoManager::Impl & obj )
 272   {
 273     return str << "RepoManager::Impl";
 274   }
 275
 276   ///////////////////////////////////////////////////////////////////
 277   //
 278   //    CLASS NAME : RepoManager
 279   //
 280   ///////////////////////////////////////////////////////////////////
 281
 282   RepoManager::RepoManager( const RepoManagerOptions &opt )
 283   : _pimpl( new Impl(opt) )
 284   {}
 285
 286   ////////////////////////////////////////////////////////////////////////////
 287
 288   RepoManager::~RepoManager()
 289   {}
 290
 291   ////////////////////////////////////////////////////////////////////////////
 292
 293   std::list<RepoInfo> RepoManager::knownRepositories() const
 294   {
 295     MIL << endl;
 296
 297     if ( PathInfo(_pimpl->options.knownReposPath).isExist() )
 298     {
 299       RepoInfoList repos = repositories_in_dir(_pimpl->options.knownReposPath);
 300       for ( RepoInfoList::iterator it = repos.begin();
 301             it != repos.end();
 302             ++it )
 303       {
 304         // set the metadata path for the repo
 305         Pathname metadata_path = rawcache_path_for_repoinfo(_pimpl->options, (*it));
 306         (*it).setMetadataPath(metadata_path);
 307       }
 308       return repos;
 309     }
 310     else
 311       return std::list<RepoInfo>();
 312
 313     MIL << endl;
 314   }
 315
 316   ////////////////////////////////////////////////////////////////////////////
 317
 318   Pathname RepoManager::metadataPath( const RepoInfo &info ) const
 319   {
 320     return rawcache_path_for_repoinfo(_pimpl->options, info );
 321   }
 322
 323   ////////////////////////////////////////////////////////////////////////////
 324
 325   RepoStatus RepoManager::metadataStatus( const RepoInfo &info ) const
 326   {
 327     Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 328     RepoType repokind = info.type();
 329     RepoStatus status;
 330
 331     switch ( repokind.toEnum() )
 332     {
 333       case RepoType::NONE_e:
 334       // unknown, probe the local metadata
 335         repokind = probe(rawpath.asUrl());
 336       break;
 337       default:
 338       break;
 339     }
 340
 341     switch ( repokind.toEnum() )
 342     {
 343       case RepoType::RPMMD_e :
 344       {
 345         status = RepoStatus( rawpath + "/repodata/repomd.xml");
 346       }
 347       break;
 348
 349       case RepoType::YAST2_e :
 350       {
 351         // the order of RepoStatus && RepoStatus matters! (#304310)
 352         status = RepoStatus( rawpath + "/content") && (RepoStatus( rawpath + "/media.1/media"));
 353       }
 354       break;
 355
 356       case RepoType::RPMPLAINDIR_e :
 357       {
 358         if ( PathInfo(Pathname(rawpath + "/cookie")).isExist() )
 359           status = RepoStatus( rawpath + "/cookie");
 360       }
 361       break;
 362
 363       case RepoType::NONE_e :
 364         // Return default RepoStatus in case of RepoType::NONE
 365         // indicating it should be created?
 366         // ZYPP_THROW(RepoUnknownTypeException());
 367         break;
 368     }
 369     return status;
 370   }
 371
 372   void RepoManager::touchIndexFile(const RepoInfo & info)
 373   {
 374     Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 375
 376     RepoType repokind = info.type();
 377     if ( repokind.toEnum() == RepoType::NONE_e )
 378       // unknown, probe the local metadata
 379       repokind = probe(rawpath.asUrl());
 380     // if still unknown, just return
 381     if (repokind == RepoType::NONE_e)
 382       return;
 383
 384     Pathname p;
 385     switch ( repokind.toEnum() )
 386     {
 387       case RepoType::RPMMD_e :
 388         p = Pathname(rawpath + "/repodata/repomd.xml");
 389         break;
 390
 391       case RepoType::YAST2_e :
 392         p = Pathname(rawpath + "/content");
 393         break;
 394
 395       case RepoType::RPMPLAINDIR_e :
 396         p = Pathname(rawpath + "/cookie");
 397         break;
 398
 399       case RepoType::NONE_e :
 400       default:
 401         break;
 402     }
 403
 404     // touch the file, ignore error (they are logged anyway)
 405     filesystem::touch(p);
 406   }
 407
 408   bool RepoManager::checkIfToRefreshMetadata( const RepoInfo &info,
 409                                               const Url &url,
 410                                               RawMetadataRefreshPolicy policy )
 411   {
 412     assert_alias(info);
 413
 414     RepoStatus oldstatus;
 415     RepoStatus newstatus;
 416
 417     try
 418     {
 419       MIL << "Going to try to check whether refresh is needed for " << url << endl;
 420
 421       repo::RepoType repokind = info.type();
 422
 423       // if the type is unknown, try probing.
 424       switch ( repokind.toEnum() )
 425       {
 426         case RepoType::NONE_e:
 427           // unknown, probe it
 428           repokind = probe(url);
 429         break;
 430         default:
 431         break;
 432       }
 433
 434       Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 435       filesystem::assert_dir(rawpath);
 436       oldstatus = metadataStatus(info);
 437
 438       // now we've got the old (cached) status, we can decide repo.refresh.delay
 439       if (policy != RefreshForced)
 440       {
 441         // difference in seconds
 442         double diff = difftime(
 443           (Date::ValueType)Date::now(),
 444           (Date::ValueType)oldstatus.timestamp()) / 60;
 445
 446         DBG << "oldstatus: " << (Date::ValueType)oldstatus.timestamp() << endl;
 447         DBG << "current time: " << (Date::ValueType)Date::now() << endl;
 448         DBG << "last refresh = " << diff << " minutes ago" << endl;
 449
 450         if (diff < ZConfig::instance().repo_refresh_delay())
 451         {
 452           MIL << "Repository '" << info.alias()
 453               << "' has been refreshed less than repo.refresh.delay ("
 454               << ZConfig::instance().repo_refresh_delay()
 455               << ") minutes ago. Advising to skip refresh" << endl;
 456           return false;
 457         }
 458       }
 459
 460       // create temp dir as sibling of rawpath
 461       filesystem::TmpDir tmpdir( filesystem::TmpDir::makeSibling( rawpath ) );
 462
 463       if ( ( repokind.toEnum() == RepoType::RPMMD_e ) ||
 464            ( repokind.toEnum() == RepoType::YAST2_e ) )
 465       {
 466         MediaSetAccess media(url);
 467         shared_ptr<repo::Downloader> downloader_ptr;
 468
 469         if ( repokind.toEnum() == RepoType::RPMMD_e )
 470           downloader_ptr.reset(new yum::Downloader(info.path()));
 471         else
 472           downloader_ptr.reset( new susetags::Downloader(info.path()));
 473
 474         RepoStatus newstatus = downloader_ptr->status(media);
 475         bool refresh = false;
 476         if ( oldstatus.checksum() == newstatus.checksum() )
 477         {
 478           MIL << "repo has not changed" << endl;
 479           if ( policy == RefreshForced )
 480           {
 481             MIL << "refresh set to forced" << endl;
 482             refresh = true;
 483           }
 484         }
 485         else
 486         {
 487           MIL << "repo has changed, going to refresh" << endl;
 488           refresh = true;
 489         }
 490
 491         if (!refresh)
 492           touchIndexFile(info);
 493
 494         return refresh;
 495       }
 496 #if 0
 497       else if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 498       {
 499         RepoStatus newstatus = parser::plaindir::dirStatus(url.getPathName());
 500         bool refresh = false;
 501         if ( oldstatus.checksum() == newstatus.checksum() )
 502         {
 503           MIL << "repo has not changed" << endl;
 504           if ( policy == RefreshForced )
 505           {
 506             MIL << "refresh set to forced" << endl;
 507             refresh = true;
 508           }
 509         }
 510         else
 511         {
 512           MIL << "repo has changed, going to refresh" << endl;
 513           refresh = true;
 514         }
 515
 516         if (!refresh)
 517           touchIndexFile(info);
 518
 519         return refresh;
 520       }
 521 #endif
 522       else
 523       {
 524         ZYPP_THROW(RepoUnknownTypeException());
 525       }
 526     }
 527     catch ( const Exception &e )
 528     {
 529       ZYPP_CAUGHT(e);
 530       ERR << "refresh check failed for " << url << endl;
 531       ZYPP_RETHROW(e);
 532     }
 533
 534     return true; // default
 535   }
 536
 537   void RepoManager::refreshMetadata( const RepoInfo &info,
 538                                      RawMetadataRefreshPolicy policy,
 539                                      const ProgressData::ReceiverFnc & progress )
 540   {
 541     assert_alias(info);
 542     assert_urls(info);
 543
 544     // we will throw this later if no URL checks out fine
 545     RepoException rexception(_("Valid metadata not found at specified URL(s)"));
 546
 547     // try urls one by one
 548     for ( RepoInfo::urls_const_iterator it = info.baseUrlsBegin(); it != info.baseUrlsEnd(); ++it )
 549     {
 550       try
 551       {
 552         Url url(*it);
 553
 554         // check whether to refresh metadata
 555         // if the check fails for this url, it throws, so another url will be checked
 556         if (!checkIfToRefreshMetadata(info, url, policy))
 557           return;
 558
 559         MIL << "Going to refresh metadata from " << url << endl;
 560
 561         repo::RepoType repokind = info.type();
 562
 563         // if the type is unknown, try probing.
 564         switch ( repokind.toEnum() )
 565         {
 566           case RepoType::NONE_e:
 567             // unknown, probe it
 568             repokind = probe(*it);
 569           break;
 570           default:
 571           break;
 572         }
 573
 574         Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
 575         filesystem::assert_dir(rawpath);
 576
 577         // create temp dir as sibling of rawpath
 578         filesystem::TmpDir tmpdir( filesystem::TmpDir::makeSibling( rawpath ) );
 579
 580         if ( ( repokind.toEnum() == RepoType::RPMMD_e ) ||
 581              ( repokind.toEnum() == RepoType::YAST2_e ) )
 582         {
 583           MediaSetAccess media(url);
 584           shared_ptr<repo::Downloader> downloader_ptr;
 585
 586           if ( repokind.toEnum() == RepoType::RPMMD_e )
 587             downloader_ptr.reset(new yum::Downloader(info.path()));
 588           else
 589             downloader_ptr.reset( new susetags::Downloader(info.path()));
 590
 591           /**
 592            * Given a downloader, sets the other repos raw metadata
 593            * path as cache paths for the fetcher, so if another
 594            * repo has the same file, it will not download it
 595            * but copy it from the other repository
 596            */
 597           std::list<RepoInfo> repos = knownRepositories();
 598           for ( std::list<RepoInfo>::const_iterator it = repos.begin();
 599                 it != repos.end();
 600                 ++it )
 601           {
 602             downloader_ptr->addCachePath(rawcache_path_for_repoinfo( _pimpl->options, *it ));
 603           }
 604
 605           downloader_ptr->download( media, tmpdir.path());
 606         }
 607 #if 0
 608         else if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
 609         {
 610           RepoStatus newstatus = parser::plaindir::dirStatus(url.getPathName());
 611
 612           std::ofstream file(( tmpdir.path() + "/cookie").c_str());
 613           if (!file) {
 614             ZYPP_THROW (Exception( "Can't open " + tmpdir.path().asString() + "/cookie" ) );
 615           }
 616           file << url << endl;
 617           file << newstatus.checksum() << endl;
 618
 619           file.close();
 620         }
 621 #endif
 622         else
 623         {
 624           ZYPP_THROW(RepoUnknownTypeException());
 625         }
 626
 627         // ok we have the metadata, now exchange
 628         // the contents
 629         TmpDir oldmetadata( TmpDir::makeSibling( rawpath ) );
 630         filesystem::rename( rawpath, oldmetadata.path() );
 631         // move the just downloaded there
 632         filesystem::rename( tmpdir.path(), rawpath );
 633         // we are done.
 634         return;
 635       }
 636       catch ( const Exception &e )
 637       {
 638         ZYPP_CAUGHT(e);
 639         ERR << "Trying another url..." << endl;
 640
 641         // remember the exception caught for the *first URL*
 642         // if all other URLs fail, the rexception will be thrown with the
 643         // cause of the problem of the first URL remembered
 644         if (it == info.baseUrlsBegin())
 645           rexception.remember(e);
 646       }
 647     } // for every url
 648     ERR << "No more urls..." << endl;
 649     ZYPP_THROW(rexception);
 650   }
 651
 652   ////////////////////////////////////////////////////////////////////////////
 653
 654   void RepoManager::cleanMetadata( const RepoInfo &info,
 655                                    const ProgressData::ReceiverFnc & progressfnc )
 656   {
 657     ProgressData progress(100);
 658     progress.sendTo(progressfnc);
 659
 660     filesystem::recursive_rmdir(rawcache_path_for_repoinfo(_pimpl->options, info));
 661     progress.toMax();
 662   }
 663
 664   void RepoManager::buildCache( const RepoInfo &info,
 665                                 CacheBuildPolicy policy,
 666                                 const ProgressData::ReceiverFnc & progressrcv )
 667   {
 668     assert_alias(info);
 669     Pathname rawpath = rawcache_path_for_repoinfo(_pimpl->options, info);
 670
 671     Pathname base = _pimpl->options.repoCachePath + info.alias();
 672     Pathname solvfile = base.extend(".solv");
 673
 674     //cache::SolvStore store(_pimpl->options.repoCachePath);
 675
 676     RepoStatus raw_metadata_status = metadataStatus(info);
 677     if ( raw_metadata_status.empty() )
 678     {
 679       ZYPP_THROW(RepoMetadataException(info));
 680     }
 681
 682     bool needs_cleaning = false;
 683     if ( isCached( info ) )
 684     {
 685       MIL << info.alias() << " is already cached." << endl;
 686       //data::RecordId id = store.lookupRepository(info.alias());
 687       RepoStatus cache_status = cacheStatus(info);
 688
 689       if ( cache_status.checksum() == raw_metadata_status.checksum() )
 690       {
 691         MIL << info.alias() << " cache is up to date with metadata." << endl;
 692         if ( policy == BuildIfNeeded ) {
 693           return;
 694         }
 695         else {
 696           MIL << info.alias() << " cache rebuild is forced" << endl;
 697         }
 698       }
 699
 700       needs_cleaning = true;
 701     }
 702
 703     ProgressData progress(100);
 704     callback::SendReport<ProgressReport> report;
 705     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
 706     progress.name(str::form(_("Building repository '%s' cache"), info.name().c_str()));
 707     progress.toMin();
 708
 709     if (needs_cleaning)
 710     {
 711 //       Pathname name = _pimpl->options.repoCachePath;
 712 //       //data::RecordId id = store.lookupRepository(info.alias());
 713 //       ostringstream os;
 714 //       os << id.get();
 715 //       name += os.str() + ".solv";
 716 //       unlink (name);
 717 //       cleanCacheInternal( store, info);
 718       cleanCache(info);
 719     }
 720
 721     MIL << info.alias() << " building cache..." << endl;
 722     //data::RecordId id = store.lookupOrAppendRepository(info.alias());
 723     // do we have type?
 724     repo::RepoType repokind = info.type();
 725
 726     // if the type is unknown, try probing.
 727     switch ( repokind.toEnum() )
 728     {
 729       case RepoType::NONE_e:
 730         // unknown, probe the local metadata
 731         repokind = probe(rawpath.asUrl());
 732       break;
 733       default:
 734       break;
 735     }
 736
 737 <<<<<<< HEAD:zypp/RepoManager.cc
 738
 739 =======
 740     MIL << "repo type is " << repokind << endl;
 741
 742 >>>>>>> by now the repomanager returns an invalid repo
 743 but loads the solv file in the global pool
 744 (should this be automatic?):zypp/RepoManager.cc
 745     switch ( repokind.toEnum() )
 746     {
 747       case RepoType::RPMMD_e :
 748       case RepoType::YAST2_e :
 749       {
 750 //         string cmd = "repo2solv.sh \"";
 751 //      cmd += rawpath.asString() + "\" > " + solvfile.asString();
 752 //      int ret = system (cmd.c_str());
 753 //         if (WIFEXITED (ret) && WEXITSTATUS (ret) != 0)
 754 //        ZYPP_THROW(RepoUnknownTypeException());
 755         MIL << "Executing solv converter" << endl;
 756         string cmd( str::form( "/usr/bin/repo2solv.sh \"%s\" > %s", rawpath.asString().c_str(), solvfile.asString().c_str() ) );
 757         ExternalProgram prog( cmd, ExternalProgram::Stderr_To_Stdout );
 758         for ( string output( prog.receiveLine() ); output.length(); output = prog.receiveLine() ) {
 759           MIL << "  " << output;
 760         }
 761         int ret = prog.close();
 762       }
 763       break;
 764       default:
 765         ZYPP_THROW(Exception("Unhandled repostory type"));
 766       break;
 767     }
 768 #if 0
 769     switch ( repokind.toEnum() )
 770     {
 771       case RepoType::RPMMD_e :
 772       if (0)
 773       {
 774         CombinedProgressData subprogrcv( progress, 100);
 775         parser::yum::RepoParser parser(id, store, parser::yum::RepoParserOpts(), subprogrcv);
 776         parser.parse(rawpath);
 777           // no error
 778       }
 779       break;
 780       case RepoType::YAST2_e :
 781       if (0)
 782       {
 783         CombinedProgressData subprogrcv( progress, 100);
 784         parser::susetags::RepoParser parser(id, store, subprogrcv);
 785         parser.parse(rawpath);
 786         // no error
 787       }
 788       break;
 789 #endif
 790 #if 0
 791       case RepoType::RPMPLAINDIR_e :
 792       {
 793         CombinedProgressData subprogrcv( progress, 100);
 794         InputStream is(rawpath + "cookie");
 795         string buffer;
 796         getline( is.stream(), buffer);
 797         Url url(buffer);
 798         parser::plaindir::RepoParser parser(id, store, subprogrcv);
 799         parser.parse(url.getPathName());
 800       }
 801       break;
 802
 803       default:
 804         ZYPP_THROW(RepoUnknownTypeException());
 805     }
 806 #endif
 807     // update timestamp and checksum
 808     //store.updateRepositoryStatus(id, raw_metadata_status);
 809     setCacheStatus(info.alias(), raw_metadata_status);
 810     MIL << "Commit cache.." << endl;
 811     //store.commit();
 812     //progress.toMax();
 813   }
 814
 815   ////////////////////////////////////////////////////////////////////////////
 816
 817   repo::RepoType RepoManager::probe( const Url &url ) const
 818   {
 819     if ( url.getScheme() == "dir" && ! PathInfo( url.getPathName() ).isDir() )
 820     {
 821       // Handle non existing local directory in advance, as
 822       // MediaSetAccess does not support it.
 823       return repo::RepoType::NONE;
 824     }
 825
 826     try
 827     {
 828       MediaSetAccess access(url);
 829       if ( access.doesFileExist("/repodata/repomd.xml") )
 830         return repo::RepoType::RPMMD;
 831       if ( access.doesFileExist("/content") )
 832         return repo::RepoType::YAST2;
 833
 834       // if it is a local url of type dir
 835       if ( (! media::MediaManager::downloads(url)) && ( url.getScheme() == "dir" ) )
 836       {
 837         Pathname path = Pathname(url.getPathName());
 838         if ( PathInfo(path).isDir() )
 839         {
 840           // allow empty dirs for now
 841           return repo::RepoType::RPMPLAINDIR;
 842         }
 843       }
 844     }
 845     catch ( const media::MediaException &e )
 846     {
 847       ZYPP_CAUGHT(e);
 848       RepoException enew("Error trying to read from " + url.asString());
 849       enew.remember(e);
 850       ZYPP_THROW(enew);
 851     }
 852     catch ( const Exception &e )
 853     {
 854       ZYPP_CAUGHT(e);
 855       Exception enew("Unknown error reading from " + url.asString());
 856       enew.remember(e);
 857       ZYPP_THROW(enew);
 858     }
 859
 860     return repo::RepoType::NONE;
 861   }
 862
 863   ////////////////////////////////////////////////////////////////////////////
 864
 865   void RepoManager::cleanCache( const RepoInfo &info,
 866                                 const ProgressData::ReceiverFnc & progressrcv )
 867   {
 868     Pathname name = _pimpl->options.repoCachePath;
 869     name += info.alias() + ".solv";
 870     unlink (name);
 871   }
 872
 873   ////////////////////////////////////////////////////////////////////////////
 874
 875   bool RepoManager::isCached( const RepoInfo &info ) const
 876   {
 877     Pathname name = _pimpl->options.repoCachePath;
 878     return PathInfo(name + Pathname(info.alias()).extend(".solv")).isExist();
 879   }
 880
 881   RepoStatus RepoManager::cacheStatus( const RepoInfo &info ) const
 882   {
 883     RepoStatus status;
 884     Pathname base = _pimpl->options.repoCachePath + info.alias();
 885     Pathname solvfile = base.extend(".solv");
 886     Pathname cookiefile = base.extend(".cookie");
 887
 888     std::ifstream file(cookiefile.c_str());
 889     if (!file) {
 890       ZYPP_THROW (Exception( "Can't open " + cookiefile.asString() ) );
 891     }
 892
 893     std::string buffer;
 894     while(file && !file.eof()) {
 895       getline(file, buffer);
 896     }
 897
 898     std::vector<std::string> words;
 899     if ( str::split( buffer, std::back_inserter(words) ) != 2 )
 900       ZYPP_THROW (Exception( "corrupt file " + cookiefile.asString() ) );
 901
 902     status.setTimestamp(Date(str::strtonum<time_t>(words[1])));
 903     status.setChecksum(words[0]);
 904     return status;
 905   }
 906
 907   void RepoManager::setCacheStatus( const string &alias, const RepoStatus &status )
 908   {
 909     Pathname base = _pimpl->options.repoCachePath + alias;
 910     Pathname cookiefile = base.extend(".cookie");
 911
 912     std::ofstream file(cookiefile.c_str());
 913     if (!file) {
 914       ZYPP_THROW (Exception( "Can't open " + cookiefile.asString() ) );
 915     }
 916     file << status;
 917     file.close();
 918   }
 919
 920   map<data::RecordId, Repo *> repo2solv;
 921
 922   Repository RepoManager::createFromCache( const RepoInfo &info,
 923                                            const ProgressData::ReceiverFnc & progressrcv )
 924   {
 925     callback::SendReport<ProgressReport> report;
 926     ProgressData progress;
 927     progress.sendTo(ProgressReportAdaptor( progressrcv, report ));
 928     //progress.sendTo( progressrcv );
 929     progress.name(str::form(_("Reading repository '%s' cache"), info.name().c_str()));
 930
 931     //_pimpl->options.repoCachePath
 932     if ( ! isCached( info ) )
 933       ZYPP_THROW(RepoNotCachedException());
 934
 935     MIL << "Repository " << info.alias() << " is cached" << endl;
 936
 937     sat::Pool satpool( sat::Pool::instance() );
 938
 939 <<<<<<< HEAD:zypp/RepoManager.cc
 940     //Pathname name = _pimpl->options.repoCachePath + Pathname(info.alias()).extend(".solv");
 941
 942 =======
 943     Pathname solvfile = (_pimpl->options.repoCachePath + info.alias()).extend(".solv");
 944
 945 >>>>>>> by now the repomanager returns an invalid repo
 946 but loads the solv file in the global pool
 947 (should this be automatic?):zypp/RepoManager.cc
 948     try
 949     {
 950       satpool.addRepoSolv(solvfile, info.alias());
 951     }
 952     catch ( const Exception &e )
 953     {
 954 <<<<<<< HEAD:zypp/RepoManager.cc
 955
 956 =======
 957       ZYPP_RETHROW(e);
 958 >>>>>>> by now the repomanager returns an invalid repo
 959 but loads the solv file in the global pool
 960 (should this be automatic?):zypp/RepoManager.cc
 961     }
 962
 963     CombinedProgressData subprogrcv(progress);
 964
 965 //     repo::cached::RepoOptions opts( info, _pimpl->options.repoCachePath, id );
 966 //     opts.readingResolvablesProgress = subprogrcv;
 967 //     //opts.repo = repo;
 968 //     repo::cached::RepoImpl::Ptr repoimpl =
 969 //         new repo::cached::RepoImpl( opts );
 970 //
 971 //     repoimpl->resolvables();
 972     // read the resolvables from cache
 973     return Repository::noRepository;
 974   }
 975
 976   ////////////////////////////////////////////////////////////////////////////
 977
 978   /**
 979    * Generate a non existing filename in a directory, using a base
 980    * name. For example if a directory contains 3 files
 981    *
 982    * |-- bar
 983    * |-- foo
 984    * `-- moo
 985    *
 986    * If you try to generate a unique filename for this directory,
 987    * based on "ruu" you will get "ruu", but if you use the base
 988    * "foo" you will get "foo_1"
 989    *
 990    * \param dir Directory where the file needs to be unique
 991    * \param basefilename string to base the filename on.
 992    */
 993   static Pathname generate_non_existing_name( const Pathname &dir,
 994                                               const std::string &basefilename )
 995   {
 996     string final_filename = basefilename;
 997     int counter = 1;
 998     while ( PathInfo(dir + final_filename).isExist() )
 999     {
1000       final_filename = basefilename + "_" + str::numstring(counter);
1001       counter++;
1002     }
1003     return dir + Pathname(final_filename);
1004   }
1005
1006   ////////////////////////////////////////////////////////////////////////////
1007
1008   /**
1009    * \short Generate a related filename from a repo info
1010    *
1011    * From a repo info, it will try to use the alias as a filename
1012    * escaping it if necessary. Other fallbacks can be added to
1013    * this function in case there is no way to use the alias
1014    */
1015   static std::string generate_filename( const RepoInfo &info )
1016   {
1017     std::string fnd="/";
1018     std::string rep="_";
1019     std::string filename = info.alias();
1020     // replace slashes with underscores
1021     size_t pos = filename.find(fnd);
1022     while(pos!=string::npos)
1023     {
1024       filename.replace(pos,fnd.length(),rep);
1025       pos = filename.find(fnd,pos+rep.length());
1026     }
1027     filename = Pathname(filename).extend(".repo").asString();
1028     MIL << "generating filename for repo [" << info.alias() << "] : '" << filename << "'" << endl;
1029     return filename;
1030   }
1031
1032
1033   ////////////////////////////////////////////////////////////////////////////
1034
1035   void RepoManager::addRepository( const RepoInfo &info,
1036                                    const ProgressData::ReceiverFnc & progressrcv )
1037   {
1038     assert_alias(info);
1039
1040     ProgressData progress(100);
1041     callback::SendReport<ProgressReport> report;
1042     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
1043     progress.name(str::form(_("Adding repository '%s'"), info.name().c_str()));
1044     progress.toMin();
1045
1046     std::list<RepoInfo> repos = knownRepositories();
1047     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1048           it != repos.end();
1049           ++it )
1050     {
1051       if ( info.alias() == (*it).alias() )
1052         ZYPP_THROW(RepoAlreadyExistsException(info.alias()));
1053     }
1054
1055     RepoInfo tosave = info;
1056
1057     // check the first url for now
1058     if ( ZConfig::instance().repo_add_probe()
1059         || ( tosave.type() == RepoType::NONE && tosave.enabled()) )
1060     {
1061       DBG << "unknown repository type, probing" << endl;
1062
1063       RepoType probedtype;
1064       probedtype = probe(*tosave.baseUrlsBegin());
1065       if ( tosave.baseUrlsSize() > 0 )
1066       {
1067         if ( probedtype == RepoType::NONE )
1068           ZYPP_THROW(RepoUnknownTypeException());
1069         else
1070           tosave.setType(probedtype);
1071       }
1072     }
1073
1074     progress.set(50);
1075
1076     // assert the directory exists
1077     filesystem::assert_dir(_pimpl->options.knownReposPath);
1078
1079     Pathname repofile = generate_non_existing_name(_pimpl->options.knownReposPath,
1080                                                     generate_filename(tosave));
1081     // now we have a filename that does not exists
1082     MIL << "Saving repo in " << repofile << endl;
1083
1084     std::ofstream file(repofile.c_str());
1085     if (!file) {
1086       ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
1087     }
1088
1089     tosave.dumpRepoOn(file);
1090     progress.toMax();
1091     MIL << "done" << endl;
1092   }
1093
1094   void RepoManager::addRepositories( const Url &url,
1095                                      const ProgressData::ReceiverFnc & progressrcv )
1096   {
1097     std::list<RepoInfo> knownrepos = knownRepositories();
1098     std::list<RepoInfo> repos = readRepoFile(url);
1099     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1100           it != repos.end();
1101           ++it )
1102     {
1103       // look if the alias is in the known repos.
1104       for ( std::list<RepoInfo>::const_iterator kit = knownrepos.begin();
1105           kit != knownrepos.end();
1106           ++kit )
1107       {
1108         if ( (*it).alias() == (*kit).alias() )
1109         {
1110           ERR << "To be added repo " << (*it).alias() << " conflicts with existing repo " << (*kit).alias() << endl;
1111           ZYPP_THROW(RepoAlreadyExistsException((*it).alias()));
1112         }
1113       }
1114     }
1115
1116     string filename = Pathname(url.getPathName()).basename();
1117
1118     if ( filename == Pathname() )
1119       ZYPP_THROW(RepoException("Invalid repo file name at " + url.asString() ));
1120
1121     // assert the directory exists
1122     filesystem::assert_dir(_pimpl->options.knownReposPath);
1123
1124     Pathname repofile = generate_non_existing_name(_pimpl->options.knownReposPath, filename);
1125     // now we have a filename that does not exists
1126     MIL << "Saving " << repos.size() << " repo" << ( repos.size() ? "s" : "" ) << " in " << repofile << endl;
1127
1128     std::ofstream file(repofile.c_str());
1129     if (!file) {
1130       ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
1131     }
1132
1133     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1134           it != repos.end();
1135           ++it )
1136     {
1137       MIL << "Saving " << (*it).alias() << endl;
1138       (*it).dumpRepoOn(file);
1139     }
1140     MIL << "done" << endl;
1141   }
1142
1143   ////////////////////////////////////////////////////////////////////////////
1144
1145   void RepoManager::removeRepository( const RepoInfo & info,
1146                                       const ProgressData::ReceiverFnc & progressrcv)
1147   {
1148     ProgressData progress;
1149     callback::SendReport<ProgressReport> report;
1150     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
1151     progress.name(str::form(_("Removing repository '%s'"), info.name().c_str()));
1152
1153     MIL << "Going to delete repo " << info.alias() << endl;
1154
1155     std::list<RepoInfo> repos = knownRepositories();
1156     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1157           it != repos.end();
1158           ++it )
1159     {
1160       // they can be the same only if the provided is empty, that means
1161       // the provided repo has no alias
1162       // then skip
1163       if ( (!info.alias().empty()) && ( info.alias() != (*it).alias() ) )
1164         continue;
1165
1166       // TODO match by url
1167
1168       // we have a matcing repository, now we need to know
1169       // where it does come from.
1170       RepoInfo todelete = *it;
1171       if (todelete.filepath().empty())
1172       {
1173         ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
1174       }
1175       else
1176       {
1177         // figure how many repos are there in the file:
1178         std::list<RepoInfo> filerepos = repositories_in_file(todelete.filepath());
1179         if ( (filerepos.size() == 1) && ( filerepos.front().alias() == todelete.alias() ) )
1180         {
1181           // easy, only this one, just delete the file
1182           if ( filesystem::unlink(todelete.filepath()) != 0 )
1183           {
1184             ZYPP_THROW(RepoException("Can't delete " + todelete.filepath().asString()));
1185           }
1186           MIL << todelete.alias() << " sucessfully deleted." << endl;
1187         }
1188         else
1189         {
1190           // there are more repos in the same file
1191           // write them back except the deleted one.
1192           //TmpFile tmp;
1193           //std::ofstream file(tmp.path().c_str());
1194
1195           // assert the directory exists
1196           filesystem::assert_dir(todelete.filepath().dirname());
1197
1198           std::ofstream file(todelete.filepath().c_str());
1199           if (!file) {
1200             //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
1201             ZYPP_THROW (Exception( "Can't open " + todelete.filepath().asString() ) );
1202           }
1203           for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
1204                 fit != filerepos.end();
1205                 ++fit )
1206           {
1207             if ( (*fit).alias() != todelete.alias() )
1208               (*fit).dumpRepoOn(file);
1209           }
1210         }
1211
1212         CombinedProgressData subprogrcv(progress, 70);
1213         CombinedProgressData cleansubprogrcv(progress, 30);
1214         // now delete it from cache
1215         if ( isCached(todelete) )
1216           cleanCache( todelete, subprogrcv);
1217         // now delete metadata (#301037)
1218         cleanMetadata( todelete, cleansubprogrcv);
1219         MIL << todelete.alias() << " sucessfully deleted." << endl;
1220         return;
1221       } // else filepath is empty
1222
1223     }
1224     // should not be reached on a sucess workflow
1225     ZYPP_THROW(RepoNotFoundException(info));
1226   }
1227
1228   ////////////////////////////////////////////////////////////////////////////
1229
1230   void RepoManager::modifyRepository( const std::string &alias,
1231                                       const RepoInfo & newinfo,
1232                                       const ProgressData::ReceiverFnc & progressrcv )
1233   {
1234     RepoInfo toedit = getRepositoryInfo(alias);
1235
1236     if (toedit.filepath().empty())
1237     {
1238       ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
1239     }
1240     else
1241     {
1242       // figure how many repos are there in the file:
1243       std::list<RepoInfo> filerepos = repositories_in_file(toedit.filepath());
1244
1245       // there are more repos in the same file
1246       // write them back except the deleted one.
1247       //TmpFile tmp;
1248       //std::ofstream file(tmp.path().c_str());
1249
1250       // assert the directory exists
1251       filesystem::assert_dir(toedit.filepath().dirname());
1252
1253       std::ofstream file(toedit.filepath().c_str());
1254       if (!file) {
1255         //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
1256         ZYPP_THROW (Exception( "Can't open " + toedit.filepath().asString() ) );
1257       }
1258       for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
1259             fit != filerepos.end();
1260             ++fit )
1261       {
1262           // if the alias is different, dump the original
1263           // if it is the same, dump the provided one
1264           if ( (*fit).alias() != toedit.alias() )
1265             (*fit).dumpRepoOn(file);
1266           else
1267             newinfo.dumpRepoOn(file);
1268       }
1269     }
1270   }
1271
1272   ////////////////////////////////////////////////////////////////////////////
1273
1274   RepoInfo RepoManager::getRepositoryInfo( const std::string &alias,
1275                                            const ProgressData::ReceiverFnc & progressrcv )
1276   {
1277     std::list<RepoInfo> repos = knownRepositories();
1278     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1279           it != repos.end();
1280           ++it )
1281     {
1282       if ( (*it).alias() == alias )
1283         return *it;
1284     }
1285     RepoInfo info;
1286     info.setAlias(info.alias());
1287     ZYPP_THROW(RepoNotFoundException(info));
1288   }
1289
1290   ////////////////////////////////////////////////////////////////////////////
1291
1292   RepoInfo RepoManager::getRepositoryInfo( const Url & url,
1293                                            const url::ViewOption & urlview,
1294                                            const ProgressData::ReceiverFnc & progressrcv )
1295   {
1296     std::list<RepoInfo> repos = knownRepositories();
1297     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1298           it != repos.end();
1299           ++it )
1300     {
1301       for(RepoInfo::urls_const_iterator urlit = (*it).baseUrlsBegin();
1302           urlit != (*it).baseUrlsEnd();
1303           ++urlit)
1304       {
1305         if ((*urlit).asString(urlview) == url.asString(urlview))
1306           return *it;
1307       }
1308     }
1309     RepoInfo info;
1310     info.setAlias(info.alias());
1311     info.setBaseUrl(url);
1312     ZYPP_THROW(RepoNotFoundException(info));
1313   }
1314
1315   ////////////////////////////////////////////////////////////////////////////
1316
1317   std::ostream & operator<<( std::ostream & str, const RepoManager & obj )
1318   {
1319     return str << *obj._pimpl;
1320   }
1321
1322   /////////////////////////////////////////////////////////////////
1323 } // namespace zypp
1324 ///////////////////////////////////////////////////////////////////