uhm, for some reason I only get one resolvable in the store, will
[platform/upstream/libzypp.git] / zypp / RepoManager.cc
1 /*---------------------------------------------------------------------\
2 |                          ____ _   __ __ ___                          |
3 |                         |__  / \ / / . \ . \                         |
4 |                           / / \ V /|  _/  _/                         |
5 |                          / /__ | | | | | |                           |
6 |                         /_____||_| |_| |_|                           |
7 |                                                                      |
8 \---------------------------------------------------------------------*/
9 /** \file       zypp/RepoManager.cc
10  *
11 */
12
13 #include <cstdlib>
14 #include <iostream>
15 #include <fstream>
16 #include <sstream>
17 #include <list>
18 #include <algorithm>
19 #include "zypp/base/InputStream.h"
20 #include "zypp/base/Logger.h"
21 #include "zypp/base/Gettext.h"
22 #include "zypp/base/Function.h"
23 #include "zypp/base/Regex.h"
24 #include "zypp/PathInfo.h"
25 #include "zypp/TmpPath.h"
26
27 #include "zypp/repo/RepoException.h"
28 #include "zypp/RepoManager.h"
29
30 #include "zypp/cache/SolvStore.h"
31 #include "zypp/repo/cached/RepoImpl.h"
32 #include "zypp/media/MediaManager.h"
33 #include "zypp/MediaSetAccess.h"
34 #include "zypp/ExternalProgram.h"
35
36 #include "zypp/parser/RepoFileReader.h"
37 #include "zypp/repo/yum/Downloader.h"
38 #include "zypp/parser/yum/RepoParser.h"
39 //#include "zypp/parser/plaindir/RepoParser.h"
40 #include "zypp/repo/susetags/Downloader.h"
41 #include "zypp/parser/susetags/RepoParser.h"
42
43 #include "zypp/ZYppCallbacks.h"
44
45 #include "sat/Pool.h"
46 #include "satsolver/pool.h"
47 #include "satsolver/repo.h"
48 #include "satsolver/repo_solv.h"
49
50 using namespace std;
51 using namespace zypp;
52 using namespace zypp::repo;
53 using namespace zypp::filesystem;
54
55 using namespace zypp::repo;
56
57 ///////////////////////////////////////////////////////////////////
58 namespace zypp
59 { /////////////////////////////////////////////////////////////////
60
61   ///////////////////////////////////////////////////////////////////
62   //
63   //    CLASS NAME : RepoManagerOptions
64   //
65   ///////////////////////////////////////////////////////////////////
66
67   RepoManagerOptions::RepoManagerOptions()
68   {
69     repoCachePath    = ZConfig::instance().repoCachePath();
70     repoRawCachePath = ZConfig::instance().repoMetadataPath();
71     knownReposPath   = ZConfig::instance().knownReposPath();
72   }
73
74   ////////////////////////////////////////////////////////////////////////////
75
76   /**
77     * \short Simple callback to collect the results
78     *
79     * Classes like RepoFileParser call the callback
80     * once per each repo in a file.
81     *
82     * Passing this functor as callback, you can collect
83     * all resuls at the end, without dealing with async
84     * code.
85     */
86     struct RepoCollector
87     {
88       RepoCollector()
89       {
90         MIL << endl;
91       }
92
93       ~RepoCollector()
94       {
95         MIL << endl;
96       }
97
98       bool collect( const RepoInfo &repo )
99       {
100         //MIL << "here in collector: " << repo.alias() << endl;
101         repos.push_back(repo);
102         //MIL << "added: " << repo.alias() << endl;
103         return true;
104       }
105
106       RepoInfoList repos;
107     };
108
109   ////////////////////////////////////////////////////////////////////////////
110
111    /**
112     * \short Internal version of clean cache
113     *
114     * Takes an extra SolvStore reference, so we avoid internally
115     * having 2 SolvStores writing to the same database.
116     */
117   static void cleanCacheInternal( cache::SolvStore &store,
118                                   const RepoInfo &info,
119                                   const ProgressData::ReceiverFnc & progressrcv = ProgressData::ReceiverFnc() )
120   {
121     ProgressData progress;
122     callback::SendReport<ProgressReport> report;
123     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
124     progress.name(str::form(_("Cleaning repository '%s' cache"), info.name().c_str()));
125
126     if ( !store.isCached(info.alias()) )
127       return;
128
129     MIL << info.alias() << " cleaning cache..." << endl;
130     data::RecordId id = store.lookupRepository(info.alias());
131
132     CombinedProgressData subprogrcv(progress);
133
134     store.cleanRepository(id, subprogrcv);
135   }
136
137   ////////////////////////////////////////////////////////////////////////////
138
139   /**
140    * Reads RepoInfo's from a repo file.
141    *
142    * \param file pathname of the file to read.
143    */
144   static std::list<RepoInfo> repositories_in_file( const Pathname & file )
145   {
146     MIL << "repo file: " << file << endl;
147     RepoCollector collector;
148     parser::RepoFileReader parser( file, bind( &RepoCollector::collect, &collector, _1 ) );
149     return collector.repos;
150   }
151
152   ////////////////////////////////////////////////////////////////////////////
153
154   std::list<RepoInfo> readRepoFile(const Url & repo_file)
155    {
156      // no interface to download a specific file, using workaround:
157      //! \todo add MediaManager::provideFile(Url file_url) to easily access any file URLs? (no need for media access id or media_nr)
158      Url url(repo_file);
159      Pathname path(url.getPathName());
160      url.setPathName ("/");
161      MediaSetAccess access(url);
162      Pathname local = access.provideFile(path);
163
164      DBG << "reading repo file " << repo_file << ", local path: " << local << endl;
165
166      return repositories_in_file(local);
167    }
168
169   ////////////////////////////////////////////////////////////////////////////
170
171   /**
172    * \short List of RepoInfo's from a directory
173    *
174    * Goes trough every file ending with ".repo" in a directory and adds all
175    * RepoInfo's contained in that file.
176    *
177    * \param dir pathname of the directory to read.
178    */
179   static std::list<RepoInfo> repositories_in_dir( const Pathname &dir )
180   {
181     MIL << "directory " << dir << endl;
182     list<RepoInfo> repos;
183     list<Pathname> entries;
184     if ( filesystem::readdir( entries, Pathname(dir), false ) != 0 )
185       ZYPP_THROW(Exception("failed to read directory"));
186
187     str::regex allowedRepoExt("^\\.repo(_[0-9]+)?$");
188     for ( list<Pathname>::const_iterator it = entries.begin(); it != entries.end(); ++it )
189     {
190       if (str::regex_match(it->extension(), allowedRepoExt))
191       {
192         list<RepoInfo> tmp = repositories_in_file( *it );
193         repos.insert( repos.end(), tmp.begin(), tmp.end() );
194
195         //std::copy( collector.repos.begin(), collector.repos.end(), std::back_inserter(repos));
196         //MIL << "ok" << endl;
197       }
198     }
199     return repos;
200   }
201
202   ////////////////////////////////////////////////////////////////////////////
203
204   static void assert_alias( const RepoInfo &info )
205   {
206     if (info.alias().empty())
207         ZYPP_THROW(RepoNoAliasException());
208   }
209
210   ////////////////////////////////////////////////////////////////////////////
211
212   static void assert_urls( const RepoInfo &info )
213   {
214     if (info.baseUrlsEmpty())
215         ZYPP_THROW(RepoNoUrlException());
216   }
217
218   ////////////////////////////////////////////////////////////////////////////
219
220   /**
221    * \short Calculates the raw cache path for a repository
222    */
223   static Pathname rawcache_path_for_repoinfo( const RepoManagerOptions &opt, const RepoInfo &info )
224   {
225     assert_alias(info);
226     return opt.repoRawCachePath + info.alias();
227   }
228
229   ///////////////////////////////////////////////////////////////////
230   //
231   //    CLASS NAME : RepoManager::Impl
232   //
233   ///////////////////////////////////////////////////////////////////
234
235   /**
236    * \short RepoManager implementation.
237    */
238   struct RepoManager::Impl
239   {
240     Impl( const RepoManagerOptions &opt )
241       : options(opt)
242     {
243
244     }
245
246     Impl()
247     {
248
249     }
250
251     RepoManagerOptions options;
252
253   public:
254     /** Offer default Impl. */
255     static shared_ptr<Impl> nullimpl()
256     {
257       static shared_ptr<Impl> _nullimpl( new Impl );
258       return _nullimpl;
259     }
260
261   private:
262     friend Impl * rwcowClone<Impl>( const Impl * rhs );
263     /** clone for RWCOW_pointer */
264     Impl * clone() const
265     { return new Impl( *this ); }
266   };
267
268   ///////////////////////////////////////////////////////////////////
269
270   /** \relates RepoManager::Impl Stream output */
271   inline std::ostream & operator<<( std::ostream & str, const RepoManager::Impl & obj )
272   {
273     return str << "RepoManager::Impl";
274   }
275
276   ///////////////////////////////////////////////////////////////////
277   //
278   //    CLASS NAME : RepoManager
279   //
280   ///////////////////////////////////////////////////////////////////
281
282   RepoManager::RepoManager( const RepoManagerOptions &opt )
283   : _pimpl( new Impl(opt) )
284   {}
285
286   ////////////////////////////////////////////////////////////////////////////
287
288   RepoManager::~RepoManager()
289   {}
290
291   ////////////////////////////////////////////////////////////////////////////
292
293   std::list<RepoInfo> RepoManager::knownRepositories() const
294   {
295     MIL << endl;
296
297     if ( PathInfo(_pimpl->options.knownReposPath).isExist() )
298     {
299       RepoInfoList repos = repositories_in_dir(_pimpl->options.knownReposPath);
300       for ( RepoInfoList::iterator it = repos.begin();
301             it != repos.end();
302             ++it )
303       {
304         // set the metadata path for the repo
305         Pathname metadata_path = rawcache_path_for_repoinfo(_pimpl->options, (*it));
306         (*it).setMetadataPath(metadata_path);
307       }
308       return repos;
309     }
310     else
311       return std::list<RepoInfo>();
312
313     MIL << endl;
314   }
315
316   ////////////////////////////////////////////////////////////////////////////
317
318   Pathname RepoManager::metadataPath( const RepoInfo &info ) const
319   {
320     return rawcache_path_for_repoinfo(_pimpl->options, info );
321   }
322
323   ////////////////////////////////////////////////////////////////////////////
324
325   RepoStatus RepoManager::metadataStatus( const RepoInfo &info ) const
326   {
327     Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
328     RepoType repokind = info.type();
329     RepoStatus status;
330
331     switch ( repokind.toEnum() )
332     {
333       case RepoType::NONE_e:
334       // unknown, probe the local metadata
335         repokind = probe(rawpath.asUrl());
336       break;
337       default:
338       break;
339     }
340
341     switch ( repokind.toEnum() )
342     {
343       case RepoType::RPMMD_e :
344       {
345         status = RepoStatus( rawpath + "/repodata/repomd.xml");
346       }
347       break;
348
349       case RepoType::YAST2_e :
350       {
351         // the order of RepoStatus && RepoStatus matters! (#304310)
352         status = RepoStatus( rawpath + "/content") && (RepoStatus( rawpath + "/media.1/media"));
353       }
354       break;
355
356       case RepoType::RPMPLAINDIR_e :
357       {
358         if ( PathInfo(Pathname(rawpath + "/cookie")).isExist() )
359           status = RepoStatus( rawpath + "/cookie");
360       }
361       break;
362
363       case RepoType::NONE_e :
364         // Return default RepoStatus in case of RepoType::NONE
365         // indicating it should be created?
366         // ZYPP_THROW(RepoUnknownTypeException());
367         break;
368     }
369     return status;
370   }
371
372   void RepoManager::touchIndexFile(const RepoInfo & info)
373   {
374     Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
375
376     RepoType repokind = info.type();
377     if ( repokind.toEnum() == RepoType::NONE_e )
378       // unknown, probe the local metadata
379       repokind = probe(rawpath.asUrl());
380     // if still unknown, just return
381     if (repokind == RepoType::NONE_e)
382       return;
383
384     Pathname p;
385     switch ( repokind.toEnum() )
386     {
387       case RepoType::RPMMD_e :
388         p = Pathname(rawpath + "/repodata/repomd.xml");
389         break;
390
391       case RepoType::YAST2_e :
392         p = Pathname(rawpath + "/content");
393         break;
394
395       case RepoType::RPMPLAINDIR_e :
396         p = Pathname(rawpath + "/cookie");
397         break;
398
399       case RepoType::NONE_e :
400       default:
401         break;
402     }
403
404     // touch the file, ignore error (they are logged anyway)
405     filesystem::touch(p);
406   }
407
408   bool RepoManager::checkIfToRefreshMetadata( const RepoInfo &info,
409                                               const Url &url,
410                                               RawMetadataRefreshPolicy policy )
411   {
412     assert_alias(info);
413
414     RepoStatus oldstatus;
415     RepoStatus newstatus;
416
417     try
418     {
419       MIL << "Going to try to check whether refresh is needed for " << url << endl;
420
421       repo::RepoType repokind = info.type();
422
423       // if the type is unknown, try probing.
424       switch ( repokind.toEnum() )
425       {
426         case RepoType::NONE_e:
427           // unknown, probe it
428           repokind = probe(url);
429         break;
430         default:
431         break;
432       }
433
434       Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
435       filesystem::assert_dir(rawpath);
436       oldstatus = metadataStatus(info);
437
438       // now we've got the old (cached) status, we can decide repo.refresh.delay
439       if (policy != RefreshForced)
440       {
441         // difference in seconds
442         double diff = difftime(
443           (Date::ValueType)Date::now(),
444           (Date::ValueType)oldstatus.timestamp()) / 60;
445
446         DBG << "oldstatus: " << (Date::ValueType)oldstatus.timestamp() << endl;
447         DBG << "current time: " << (Date::ValueType)Date::now() << endl;
448         DBG << "last refresh = " << diff << " minutes ago" << endl;
449
450         if (diff < ZConfig::instance().repo_refresh_delay())
451         {
452           MIL << "Repository '" << info.alias()
453               << "' has been refreshed less than repo.refresh.delay ("
454               << ZConfig::instance().repo_refresh_delay()
455               << ") minutes ago. Advising to skip refresh" << endl;
456           return false;
457         }
458       }
459
460       // create temp dir as sibling of rawpath
461       filesystem::TmpDir tmpdir( filesystem::TmpDir::makeSibling( rawpath ) );
462
463       if ( ( repokind.toEnum() == RepoType::RPMMD_e ) ||
464            ( repokind.toEnum() == RepoType::YAST2_e ) )
465       {
466         MediaSetAccess media(url);
467         shared_ptr<repo::Downloader> downloader_ptr;
468
469         if ( repokind.toEnum() == RepoType::RPMMD_e )
470           downloader_ptr.reset(new yum::Downloader(info.path()));
471         else
472           downloader_ptr.reset( new susetags::Downloader(info.path()));
473
474         RepoStatus newstatus = downloader_ptr->status(media);
475         bool refresh = false;
476         if ( oldstatus.checksum() == newstatus.checksum() )
477         {
478           MIL << "repo has not changed" << endl;
479           if ( policy == RefreshForced )
480           {
481             MIL << "refresh set to forced" << endl;
482             refresh = true;
483           }
484         }
485         else
486         {
487           MIL << "repo has changed, going to refresh" << endl;
488           refresh = true;
489         }
490
491         if (!refresh)
492           touchIndexFile(info);
493
494         return refresh;
495       }
496 #if 0
497       else if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
498       {
499         RepoStatus newstatus = parser::plaindir::dirStatus(url.getPathName());
500         bool refresh = false;
501         if ( oldstatus.checksum() == newstatus.checksum() )
502         {
503           MIL << "repo has not changed" << endl;
504           if ( policy == RefreshForced )
505           {
506             MIL << "refresh set to forced" << endl;
507             refresh = true;
508           }
509         }
510         else
511         {
512           MIL << "repo has changed, going to refresh" << endl;
513           refresh = true;
514         }
515
516         if (!refresh)
517           touchIndexFile(info);
518
519         return refresh;
520       }
521 #endif
522       else
523       {
524         ZYPP_THROW(RepoUnknownTypeException());
525       }
526     }
527     catch ( const Exception &e )
528     {
529       ZYPP_CAUGHT(e);
530       ERR << "refresh check failed for " << url << endl;
531       ZYPP_RETHROW(e);
532     }
533
534     return true; // default
535   }
536
537   void RepoManager::refreshMetadata( const RepoInfo &info,
538                                      RawMetadataRefreshPolicy policy,
539                                      const ProgressData::ReceiverFnc & progress )
540   {
541     assert_alias(info);
542     assert_urls(info);
543
544     // we will throw this later if no URL checks out fine
545     RepoException rexception(_("Valid metadata not found at specified URL(s)"));
546
547     // try urls one by one
548     for ( RepoInfo::urls_const_iterator it = info.baseUrlsBegin(); it != info.baseUrlsEnd(); ++it )
549     {
550       try
551       {
552         Url url(*it);
553
554         // check whether to refresh metadata
555         // if the check fails for this url, it throws, so another url will be checked
556         if (!checkIfToRefreshMetadata(info, url, policy))
557           return;
558
559         MIL << "Going to refresh metadata from " << url << endl;
560
561         repo::RepoType repokind = info.type();
562
563         // if the type is unknown, try probing.
564         switch ( repokind.toEnum() )
565         {
566           case RepoType::NONE_e:
567             // unknown, probe it
568             repokind = probe(*it);
569           break;
570           default:
571           break;
572         }
573
574         Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
575         filesystem::assert_dir(rawpath);
576
577         // create temp dir as sibling of rawpath
578         filesystem::TmpDir tmpdir( filesystem::TmpDir::makeSibling( rawpath ) );
579
580         if ( ( repokind.toEnum() == RepoType::RPMMD_e ) ||
581              ( repokind.toEnum() == RepoType::YAST2_e ) )
582         {
583           MediaSetAccess media(url);
584           shared_ptr<repo::Downloader> downloader_ptr;
585
586           if ( repokind.toEnum() == RepoType::RPMMD_e )
587             downloader_ptr.reset(new yum::Downloader(info.path()));
588           else
589             downloader_ptr.reset( new susetags::Downloader(info.path()));
590
591           /**
592            * Given a downloader, sets the other repos raw metadata
593            * path as cache paths for the fetcher, so if another
594            * repo has the same file, it will not download it
595            * but copy it from the other repository
596            */
597           std::list<RepoInfo> repos = knownRepositories();
598           for ( std::list<RepoInfo>::const_iterator it = repos.begin();
599                 it != repos.end();
600                 ++it )
601           {
602             downloader_ptr->addCachePath(rawcache_path_for_repoinfo( _pimpl->options, *it ));
603           }
604
605           downloader_ptr->download( media, tmpdir.path());
606         }
607 #if 0
608         else if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
609         {
610           RepoStatus newstatus = parser::plaindir::dirStatus(url.getPathName());
611
612           std::ofstream file(( tmpdir.path() + "/cookie").c_str());
613           if (!file) {
614             ZYPP_THROW (Exception( "Can't open " + tmpdir.path().asString() + "/cookie" ) );
615           }
616           file << url << endl;
617           file << newstatus.checksum() << endl;
618
619           file.close();
620         }
621 #endif
622         else
623         {
624           ZYPP_THROW(RepoUnknownTypeException());
625         }
626
627         // ok we have the metadata, now exchange
628         // the contents
629         TmpDir oldmetadata( TmpDir::makeSibling( rawpath ) );
630         filesystem::rename( rawpath, oldmetadata.path() );
631         // move the just downloaded there
632         filesystem::rename( tmpdir.path(), rawpath );
633         // we are done.
634         return;
635       }
636       catch ( const Exception &e )
637       {
638         ZYPP_CAUGHT(e);
639         ERR << "Trying another url..." << endl;
640
641         // remember the exception caught for the *first URL*
642         // if all other URLs fail, the rexception will be thrown with the
643         // cause of the problem of the first URL remembered
644         if (it == info.baseUrlsBegin())
645           rexception.remember(e);
646       }
647     } // for every url
648     ERR << "No more urls..." << endl;
649     ZYPP_THROW(rexception);
650   }
651
652   ////////////////////////////////////////////////////////////////////////////
653
654   void RepoManager::cleanMetadata( const RepoInfo &info,
655                                    const ProgressData::ReceiverFnc & progressfnc )
656   {
657     ProgressData progress(100);
658     progress.sendTo(progressfnc);
659
660     filesystem::recursive_rmdir(rawcache_path_for_repoinfo(_pimpl->options, info));
661     progress.toMax();
662   }
663
664   void RepoManager::buildCache( const RepoInfo &info,
665                                 CacheBuildPolicy policy,
666                                 const ProgressData::ReceiverFnc & progressrcv )
667   {
668     assert_alias(info);
669     Pathname rawpath = rawcache_path_for_repoinfo(_pimpl->options, info);
670
671     Pathname base = _pimpl->options.repoCachePath + info.alias();
672     Pathname solvfile = base.extend(".solv");
673
674     //cache::SolvStore store(_pimpl->options.repoCachePath);
675
676     RepoStatus raw_metadata_status = metadataStatus(info);
677     if ( raw_metadata_status.empty() )
678     {
679       ZYPP_THROW(RepoMetadataException(info));
680     }
681
682     bool needs_cleaning = false;
683     if ( isCached( info ) )
684     {
685       MIL << info.alias() << " is already cached." << endl;
686       //data::RecordId id = store.lookupRepository(info.alias());
687       RepoStatus cache_status = cacheStatus(info);
688
689       if ( cache_status.checksum() == raw_metadata_status.checksum() )
690       {
691         MIL << info.alias() << " cache is up to date with metadata." << endl;
692         if ( policy == BuildIfNeeded ) {
693           return;
694         }
695         else {
696           MIL << info.alias() << " cache rebuild is forced" << endl;
697         }
698       }
699
700       needs_cleaning = true;
701     }
702
703     ProgressData progress(100);
704     callback::SendReport<ProgressReport> report;
705     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
706     progress.name(str::form(_("Building repository '%s' cache"), info.name().c_str()));
707     progress.toMin();
708
709     if (needs_cleaning)
710     {
711 //       Pathname name = _pimpl->options.repoCachePath;
712 //       //data::RecordId id = store.lookupRepository(info.alias());
713 //       ostringstream os;
714 //       os << id.get();
715 //       name += os.str() + ".solv";
716 //       unlink (name);
717 //       cleanCacheInternal( store, info);
718       cleanCache(info);
719     }
720
721     MIL << info.alias() << " building cache..." << endl;
722     //data::RecordId id = store.lookupOrAppendRepository(info.alias());
723     // do we have type?
724     repo::RepoType repokind = info.type();
725
726     // if the type is unknown, try probing.
727     switch ( repokind.toEnum() )
728     {
729       case RepoType::NONE_e:
730         // unknown, probe the local metadata
731         repokind = probe(rawpath.asUrl());
732       break;
733       default:
734       break;
735     }
736
737     MIL << "repo type is " << repokind << endl;
738     
739     switch ( repokind.toEnum() )
740     {
741       case RepoType::RPMMD_e :
742       case RepoType::YAST2_e :
743       {
744 //         string cmd = "repo2solv.sh \"";
745 //      cmd += rawpath.asString() + "\" > " + solvfile.asString();
746 //      int ret = system (cmd.c_str());
747 //         if (WIFEXITED (ret) && WEXITSTATUS (ret) != 0)
748 //        ZYPP_THROW(RepoUnknownTypeException());
749         MIL << "Executing solv converter" << endl;
750         string cmd( str::form( "/usr/bin/repo2solv.sh \"%s\" > %s", rawpath.asString().c_str(), solvfile.asString().c_str() ) );
751         ExternalProgram prog( cmd, ExternalProgram::Stderr_To_Stdout );
752         for ( string output( prog.receiveLine() ); output.length(); output = prog.receiveLine() ) {
753           MIL << "  " << output;
754         }
755         int ret = prog.close();
756       }
757       break;
758       default:
759         ZYPP_THROW(Exception("Unhandled repostory type"));
760       break;
761     }
762 #if 0
763     switch ( repokind.toEnum() )
764     {
765       case RepoType::RPMMD_e :
766       if (0)
767       {
768         CombinedProgressData subprogrcv( progress, 100);
769         parser::yum::RepoParser parser(id, store, parser::yum::RepoParserOpts(), subprogrcv);
770         parser.parse(rawpath);
771           // no error
772       }
773       break;
774       case RepoType::YAST2_e :
775       if (0)
776       {
777         CombinedProgressData subprogrcv( progress, 100);
778         parser::susetags::RepoParser parser(id, store, subprogrcv);
779         parser.parse(rawpath);
780         // no error
781       }
782       break;
783 #endif
784 #if 0
785       case RepoType::RPMPLAINDIR_e :
786       {
787         CombinedProgressData subprogrcv( progress, 100);
788         InputStream is(rawpath + "cookie");
789         string buffer;
790         getline( is.stream(), buffer);
791         Url url(buffer);
792         parser::plaindir::RepoParser parser(id, store, subprogrcv);
793         parser.parse(url.getPathName());
794       }
795       break;
796
797       default:
798         ZYPP_THROW(RepoUnknownTypeException());
799     }
800 #endif
801     // update timestamp and checksum
802     //store.updateRepositoryStatus(id, raw_metadata_status);
803     setCacheStatus(info.alias(), raw_metadata_status);
804     MIL << "Commit cache.." << endl;
805     //store.commit();
806     //progress.toMax();
807   }
808
809   ////////////////////////////////////////////////////////////////////////////
810
811   repo::RepoType RepoManager::probe( const Url &url ) const
812   {
813     if ( url.getScheme() == "dir" && ! PathInfo( url.getPathName() ).isDir() )
814     {
815       // Handle non existing local directory in advance, as
816       // MediaSetAccess does not support it.
817       return repo::RepoType::NONE;
818     }
819
820     try
821     {
822       MediaSetAccess access(url);
823       if ( access.doesFileExist("/repodata/repomd.xml") )
824         return repo::RepoType::RPMMD;
825       if ( access.doesFileExist("/content") )
826         return repo::RepoType::YAST2;
827
828       // if it is a local url of type dir
829       if ( (! media::MediaManager::downloads(url)) && ( url.getScheme() == "dir" ) )
830       {
831         Pathname path = Pathname(url.getPathName());
832         if ( PathInfo(path).isDir() )
833         {
834           // allow empty dirs for now
835           return repo::RepoType::RPMPLAINDIR;
836         }
837       }
838     }
839     catch ( const media::MediaException &e )
840     {
841       ZYPP_CAUGHT(e);
842       RepoException enew("Error trying to read from " + url.asString());
843       enew.remember(e);
844       ZYPP_THROW(enew);
845     }
846     catch ( const Exception &e )
847     {
848       ZYPP_CAUGHT(e);
849       Exception enew("Unknown error reading from " + url.asString());
850       enew.remember(e);
851       ZYPP_THROW(enew);
852     }
853
854     return repo::RepoType::NONE;
855   }
856
857   ////////////////////////////////////////////////////////////////////////////
858
859   void RepoManager::cleanCache( const RepoInfo &info,
860                                 const ProgressData::ReceiverFnc & progressrcv )
861   {
862     Pathname name = _pimpl->options.repoCachePath;
863     name += info.alias() + ".solv";
864     unlink (name);
865   }
866
867   ////////////////////////////////////////////////////////////////////////////
868
869   bool RepoManager::isCached( const RepoInfo &info ) const
870   {
871     Pathname name = _pimpl->options.repoCachePath;
872     return PathInfo(name + Pathname(info.alias()).extend(".solv")).isExist();
873   }
874
875   RepoStatus RepoManager::cacheStatus( const RepoInfo &info ) const
876   {
877     RepoStatus status;
878     Pathname base = _pimpl->options.repoCachePath + info.alias();
879     Pathname solvfile = base.extend(".solv");
880     Pathname cookiefile = base.extend(".cookie");
881
882     std::ifstream file(cookiefile.c_str());
883     if (!file) {
884       ZYPP_THROW (Exception( "Can't open " + cookiefile.asString() ) );
885     }
886
887     std::string buffer;
888     while(file && !file.eof()) {
889       getline(file, buffer);
890     }
891
892     std::vector<std::string> words;
893     if ( str::split( buffer, std::back_inserter(words) ) != 2 )
894       ZYPP_THROW (Exception( "corrupt file " + cookiefile.asString() ) );
895
896     status.setTimestamp(Date(str::strtonum<time_t>(words[1])));
897     status.setChecksum(words[0]);
898     return status;
899   }
900
901   void RepoManager::setCacheStatus( const string &alias, const RepoStatus &status )
902   {
903     Pathname base = _pimpl->options.repoCachePath + alias;
904     Pathname cookiefile = base.extend(".cookie");
905
906     std::ofstream file(cookiefile.c_str());
907     if (!file) {
908       ZYPP_THROW (Exception( "Can't open " + cookiefile.asString() ) );
909     }
910     file << status;
911     file.close();
912   }
913
914   map<data::RecordId, Repo *> repo2solv;
915
916   Repository RepoManager::createFromCache( const RepoInfo &info,
917                                            const ProgressData::ReceiverFnc & progressrcv )
918   {
919     callback::SendReport<ProgressReport> report;
920     ProgressData progress;
921     progress.sendTo(ProgressReportAdaptor( progressrcv, report ));
922     //progress.sendTo( progressrcv );
923     progress.name(str::form(_("Reading repository '%s' cache"), info.name().c_str()));
924
925     //_pimpl->options.repoCachePath
926     if ( ! isCached( info ) )
927       ZYPP_THROW(RepoNotCachedException());
928
929     MIL << "Repository " << info.alias() << " is cached" << endl;
930
931     CombinedProgressData subprogrcv(progress);
932
933     repo::cached::RepoOptions opts( info, _pimpl->options.repoCachePath );
934     opts.readingResolvablesProgress = subprogrcv;
935     //opts.repo = repo;
936     repo::cached::RepoImpl::Ptr repoimpl =
937          new repo::cached::RepoImpl( opts );
938
939     //repoimpl->createResolvables();
940     repoimpl->resolvables();
941     // read the resolvables from cache
942     //return Repository::noRepository;
943     return Repository(repoimpl);
944   }
945
946   ////////////////////////////////////////////////////////////////////////////
947
948   /**
949    * Generate a non existing filename in a directory, using a base
950    * name. For example if a directory contains 3 files
951    *
952    * |-- bar
953    * |-- foo
954    * `-- moo
955    *
956    * If you try to generate a unique filename for this directory,
957    * based on "ruu" you will get "ruu", but if you use the base
958    * "foo" you will get "foo_1"
959    *
960    * \param dir Directory where the file needs to be unique
961    * \param basefilename string to base the filename on.
962    */
963   static Pathname generate_non_existing_name( const Pathname &dir,
964                                               const std::string &basefilename )
965   {
966     string final_filename = basefilename;
967     int counter = 1;
968     while ( PathInfo(dir + final_filename).isExist() )
969     {
970       final_filename = basefilename + "_" + str::numstring(counter);
971       counter++;
972     }
973     return dir + Pathname(final_filename);
974   }
975
976   ////////////////////////////////////////////////////////////////////////////
977
978   /**
979    * \short Generate a related filename from a repo info
980    *
981    * From a repo info, it will try to use the alias as a filename
982    * escaping it if necessary. Other fallbacks can be added to
983    * this function in case there is no way to use the alias
984    */
985   static std::string generate_filename( const RepoInfo &info )
986   {
987     std::string fnd="/";
988     std::string rep="_";
989     std::string filename = info.alias();
990     // replace slashes with underscores
991     size_t pos = filename.find(fnd);
992     while(pos!=string::npos)
993     {
994       filename.replace(pos,fnd.length(),rep);
995       pos = filename.find(fnd,pos+rep.length());
996     }
997     filename = Pathname(filename).extend(".repo").asString();
998     MIL << "generating filename for repo [" << info.alias() << "] : '" << filename << "'" << endl;
999     return filename;
1000   }
1001
1002
1003   ////////////////////////////////////////////////////////////////////////////
1004
1005   void RepoManager::addRepository( const RepoInfo &info,
1006                                    const ProgressData::ReceiverFnc & progressrcv )
1007   {
1008     assert_alias(info);
1009
1010     ProgressData progress(100);
1011     callback::SendReport<ProgressReport> report;
1012     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
1013     progress.name(str::form(_("Adding repository '%s'"), info.name().c_str()));
1014     progress.toMin();
1015
1016     std::list<RepoInfo> repos = knownRepositories();
1017     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1018           it != repos.end();
1019           ++it )
1020     {
1021       if ( info.alias() == (*it).alias() )
1022         ZYPP_THROW(RepoAlreadyExistsException(info.alias()));
1023     }
1024
1025     RepoInfo tosave = info;
1026
1027     // check the first url for now
1028     if ( ZConfig::instance().repo_add_probe()
1029         || ( tosave.type() == RepoType::NONE && tosave.enabled()) )
1030     {
1031       DBG << "unknown repository type, probing" << endl;
1032
1033       RepoType probedtype;
1034       probedtype = probe(*tosave.baseUrlsBegin());
1035       if ( tosave.baseUrlsSize() > 0 )
1036       {
1037         if ( probedtype == RepoType::NONE )
1038           ZYPP_THROW(RepoUnknownTypeException());
1039         else
1040           tosave.setType(probedtype);
1041       }
1042     }
1043
1044     progress.set(50);
1045
1046     // assert the directory exists
1047     filesystem::assert_dir(_pimpl->options.knownReposPath);
1048
1049     Pathname repofile = generate_non_existing_name(_pimpl->options.knownReposPath,
1050                                                     generate_filename(tosave));
1051     // now we have a filename that does not exists
1052     MIL << "Saving repo in " << repofile << endl;
1053
1054     std::ofstream file(repofile.c_str());
1055     if (!file) {
1056       ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
1057     }
1058
1059     tosave.dumpRepoOn(file);
1060     progress.toMax();
1061     MIL << "done" << endl;
1062   }
1063
1064   void RepoManager::addRepositories( const Url &url,
1065                                      const ProgressData::ReceiverFnc & progressrcv )
1066   {
1067     std::list<RepoInfo> knownrepos = knownRepositories();
1068     std::list<RepoInfo> repos = readRepoFile(url);
1069     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1070           it != repos.end();
1071           ++it )
1072     {
1073       // look if the alias is in the known repos.
1074       for ( std::list<RepoInfo>::const_iterator kit = knownrepos.begin();
1075           kit != knownrepos.end();
1076           ++kit )
1077       {
1078         if ( (*it).alias() == (*kit).alias() )
1079         {
1080           ERR << "To be added repo " << (*it).alias() << " conflicts with existing repo " << (*kit).alias() << endl;
1081           ZYPP_THROW(RepoAlreadyExistsException((*it).alias()));
1082         }
1083       }
1084     }
1085
1086     string filename = Pathname(url.getPathName()).basename();
1087
1088     if ( filename == Pathname() )
1089       ZYPP_THROW(RepoException("Invalid repo file name at " + url.asString() ));
1090
1091     // assert the directory exists
1092     filesystem::assert_dir(_pimpl->options.knownReposPath);
1093
1094     Pathname repofile = generate_non_existing_name(_pimpl->options.knownReposPath, filename);
1095     // now we have a filename that does not exists
1096     MIL << "Saving " << repos.size() << " repo" << ( repos.size() ? "s" : "" ) << " in " << repofile << endl;
1097
1098     std::ofstream file(repofile.c_str());
1099     if (!file) {
1100       ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
1101     }
1102
1103     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1104           it != repos.end();
1105           ++it )
1106     {
1107       MIL << "Saving " << (*it).alias() << endl;
1108       (*it).dumpRepoOn(file);
1109     }
1110     MIL << "done" << endl;
1111   }
1112
1113   ////////////////////////////////////////////////////////////////////////////
1114
1115   void RepoManager::removeRepository( const RepoInfo & info,
1116                                       const ProgressData::ReceiverFnc & progressrcv)
1117   {
1118     ProgressData progress;
1119     callback::SendReport<ProgressReport> report;
1120     progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
1121     progress.name(str::form(_("Removing repository '%s'"), info.name().c_str()));
1122
1123     MIL << "Going to delete repo " << info.alias() << endl;
1124
1125     std::list<RepoInfo> repos = knownRepositories();
1126     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1127           it != repos.end();
1128           ++it )
1129     {
1130       // they can be the same only if the provided is empty, that means
1131       // the provided repo has no alias
1132       // then skip
1133       if ( (!info.alias().empty()) && ( info.alias() != (*it).alias() ) )
1134         continue;
1135
1136       // TODO match by url
1137
1138       // we have a matcing repository, now we need to know
1139       // where it does come from.
1140       RepoInfo todelete = *it;
1141       if (todelete.filepath().empty())
1142       {
1143         ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
1144       }
1145       else
1146       {
1147         // figure how many repos are there in the file:
1148         std::list<RepoInfo> filerepos = repositories_in_file(todelete.filepath());
1149         if ( (filerepos.size() == 1) && ( filerepos.front().alias() == todelete.alias() ) )
1150         {
1151           // easy, only this one, just delete the file
1152           if ( filesystem::unlink(todelete.filepath()) != 0 )
1153           {
1154             ZYPP_THROW(RepoException("Can't delete " + todelete.filepath().asString()));
1155           }
1156           MIL << todelete.alias() << " sucessfully deleted." << endl;
1157         }
1158         else
1159         {
1160           // there are more repos in the same file
1161           // write them back except the deleted one.
1162           //TmpFile tmp;
1163           //std::ofstream file(tmp.path().c_str());
1164
1165           // assert the directory exists
1166           filesystem::assert_dir(todelete.filepath().dirname());
1167
1168           std::ofstream file(todelete.filepath().c_str());
1169           if (!file) {
1170             //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
1171             ZYPP_THROW (Exception( "Can't open " + todelete.filepath().asString() ) );
1172           }
1173           for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
1174                 fit != filerepos.end();
1175                 ++fit )
1176           {
1177             if ( (*fit).alias() != todelete.alias() )
1178               (*fit).dumpRepoOn(file);
1179           }
1180         }
1181
1182         CombinedProgressData subprogrcv(progress, 70);
1183         CombinedProgressData cleansubprogrcv(progress, 30);
1184         // now delete it from cache
1185         if ( isCached(todelete) )
1186           cleanCache( todelete, subprogrcv);
1187         // now delete metadata (#301037)
1188         cleanMetadata( todelete, cleansubprogrcv);
1189         MIL << todelete.alias() << " sucessfully deleted." << endl;
1190         return;
1191       } // else filepath is empty
1192
1193     }
1194     // should not be reached on a sucess workflow
1195     ZYPP_THROW(RepoNotFoundException(info));
1196   }
1197
1198   ////////////////////////////////////////////////////////////////////////////
1199
1200   void RepoManager::modifyRepository( const std::string &alias,
1201                                       const RepoInfo & newinfo,
1202                                       const ProgressData::ReceiverFnc & progressrcv )
1203   {
1204     RepoInfo toedit = getRepositoryInfo(alias);
1205
1206     if (toedit.filepath().empty())
1207     {
1208       ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
1209     }
1210     else
1211     {
1212       // figure how many repos are there in the file:
1213       std::list<RepoInfo> filerepos = repositories_in_file(toedit.filepath());
1214
1215       // there are more repos in the same file
1216       // write them back except the deleted one.
1217       //TmpFile tmp;
1218       //std::ofstream file(tmp.path().c_str());
1219
1220       // assert the directory exists
1221       filesystem::assert_dir(toedit.filepath().dirname());
1222
1223       std::ofstream file(toedit.filepath().c_str());
1224       if (!file) {
1225         //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
1226         ZYPP_THROW (Exception( "Can't open " + toedit.filepath().asString() ) );
1227       }
1228       for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
1229             fit != filerepos.end();
1230             ++fit )
1231       {
1232           // if the alias is different, dump the original
1233           // if it is the same, dump the provided one
1234           if ( (*fit).alias() != toedit.alias() )
1235             (*fit).dumpRepoOn(file);
1236           else
1237             newinfo.dumpRepoOn(file);
1238       }
1239     }
1240   }
1241
1242   ////////////////////////////////////////////////////////////////////////////
1243
1244   RepoInfo RepoManager::getRepositoryInfo( const std::string &alias,
1245                                            const ProgressData::ReceiverFnc & progressrcv )
1246   {
1247     std::list<RepoInfo> repos = knownRepositories();
1248     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1249           it != repos.end();
1250           ++it )
1251     {
1252       if ( (*it).alias() == alias )
1253         return *it;
1254     }
1255     RepoInfo info;
1256     info.setAlias(info.alias());
1257     ZYPP_THROW(RepoNotFoundException(info));
1258   }
1259
1260   ////////////////////////////////////////////////////////////////////////////
1261
1262   RepoInfo RepoManager::getRepositoryInfo( const Url & url,
1263                                            const url::ViewOption & urlview,
1264                                            const ProgressData::ReceiverFnc & progressrcv )
1265   {
1266     std::list<RepoInfo> repos = knownRepositories();
1267     for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1268           it != repos.end();
1269           ++it )
1270     {
1271       for(RepoInfo::urls_const_iterator urlit = (*it).baseUrlsBegin();
1272           urlit != (*it).baseUrlsEnd();
1273           ++urlit)
1274       {
1275         if ((*urlit).asString(urlview) == url.asString(urlview))
1276           return *it;
1277       }
1278     }
1279     RepoInfo info;
1280     info.setAlias(info.alias());
1281     info.setBaseUrl(url);
1282     ZYPP_THROW(RepoNotFoundException(info));
1283   }
1284
1285   ////////////////////////////////////////////////////////////////////////////
1286
1287   std::ostream & operator<<( std::ostream & str, const RepoManager & obj )
1288   {
1289     return str << *obj._pimpl;
1290   }
1291
1292   /////////////////////////////////////////////////////////////////
1293 } // namespace zypp
1294 ///////////////////////////////////////////////////////////////////