1 /*---------------------------------------------------------------------\
3 | |__ / \ / / . \ . \ |
8 \---------------------------------------------------------------------*/
9 /** \file zypp/RepoManager.cc
20 #include "zypp/base/InputStream.h"
21 #include "zypp/base/Logger.h"
22 #include "zypp/base/Gettext.h"
23 #include "zypp/base/Function.h"
24 #include "zypp/base/Regex.h"
25 #include "zypp/PathInfo.h"
26 #include "zypp/TmpPath.h"
27 #include "zypp/ServiceInfo.h"
29 #include "zypp/repo/RepoException.h"
30 #include "zypp/RepoManager.h"
32 #include "zypp/media/MediaManager.h"
33 #include "zypp/MediaSetAccess.h"
34 #include "zypp/ExternalProgram.h"
35 #include "zypp/ManagedFile.h"
37 #include "zypp/parser/RepoFileReader.h"
38 #include "zypp/parser/ServiceFileReader.h"
39 #include "zypp/parser/RepoindexFileReader.h"
40 #include "zypp/repo/yum/Downloader.h"
41 #include "zypp/repo/susetags/Downloader.h"
42 #include "zypp/parser/plaindir/RepoParser.h"
44 #include "zypp/Target.h" // for Target::targetDistribution() for repo index services
45 #include "zypp/ZYppFactory.h" // to get the Target from ZYpp instance
47 #include "zypp/ZYppCallbacks.h"
50 #include "satsolver/pool.h"
51 #include "satsolver/repo.h"
52 #include "satsolver/repo_solv.h"
56 using namespace zypp::repo;
57 using namespace zypp::filesystem;
59 using namespace zypp::repo;
61 ///////////////////////////////////////////////////////////////////
63 { /////////////////////////////////////////////////////////////////
65 ///////////////////////////////////////////////////////////////////
67 // CLASS NAME : RepoManagerOptions
69 ///////////////////////////////////////////////////////////////////
71 RepoManagerOptions::RepoManagerOptions( const Pathname & root_r )
73 repoCachePath = Pathname::assertprefix( root_r, ZConfig::instance().repoCachePath() );
74 repoRawCachePath = Pathname::assertprefix( root_r, ZConfig::instance().repoMetadataPath() );
75 repoSolvCachePath = Pathname::assertprefix( root_r, ZConfig::instance().repoSolvfilesPath() );
76 repoPackagesCachePath = Pathname::assertprefix( root_r, ZConfig::instance().repoPackagesPath() );
77 knownReposPath = Pathname::assertprefix( root_r, ZConfig::instance().knownReposPath() );
78 knownServicesPath = Pathname::assertprefix( root_r, ZConfig::instance().knownServicesPath() );
79 probe = ZConfig::instance().repo_add_probe();
82 servicesTargetDistro = getZYpp()->target()->targetDistribution();
84 catch (const Exception & e)
86 DBG << "Target not initialized, using an empty servicesTargetDistro." << endl;
90 RepoManagerOptions RepoManagerOptions::makeTestSetup( const Pathname & root_r )
92 RepoManagerOptions ret;
93 ret.repoCachePath = root_r;
94 ret.repoRawCachePath = root_r/"raw";
95 ret.repoSolvCachePath = root_r/"solv";
96 ret.repoPackagesCachePath = root_r/"packages";
97 ret.knownReposPath = root_r/"repos.d";
98 ret.knownServicesPath = root_r/"services.d";
102 ////////////////////////////////////////////////////////////////////////////
105 * \short Simple callback to collect the results
107 * Classes like RepoFileParser call the callback
108 * once per each repo in a file.
110 * Passing this functor as callback, you can collect
111 * all results at the end, without dealing with async
114 * If targetDistro is set, all repos with non-empty RepoInfo::targetDistribution()
116 * \todo do this through a separate filter
123 RepoCollector(const string & targetDistro_)
124 : targetDistro(targetDistro_)
130 bool collect( const RepoInfo &repo )
132 // skip repositories meant for other distros than specified
133 if (!targetDistro.empty()
134 && !repo.targetDistribution().empty()
135 && repo.targetDistribution() != targetDistro)
138 << "Skipping repository meant for '" << targetDistro
139 << "' distribution (current distro is '"
140 << repo.targetDistribution() << "')." << endl;
145 repos.push_back(repo);
153 ////////////////////////////////////////////////////////////////////////////
156 * Reads RepoInfo's from a repo file.
158 * \param file pathname of the file to read.
160 static std::list<RepoInfo> repositories_in_file( const Pathname & file )
162 MIL << "repo file: " << file << endl;
163 RepoCollector collector;
164 parser::RepoFileReader parser( file, bind( &RepoCollector::collect, &collector, _1 ) );
165 return collector.repos;
168 ////////////////////////////////////////////////////////////////////////////
170 std::list<RepoInfo> readRepoFile(const Url & repo_file)
172 // no interface to download a specific file, using workaround:
173 //! \todo add MediaManager::provideFile(Url file_url) to easily access any file URLs? (no need for media access id or media_nr)
175 Pathname path(url.getPathName());
176 url.setPathName ("/");
177 MediaSetAccess access(url);
178 Pathname local = access.provideFile(path);
180 DBG << "reading repo file " << repo_file << ", local path: " << local << endl;
182 return repositories_in_file(local);
185 ////////////////////////////////////////////////////////////////////////////
188 * \short List of RepoInfo's from a directory
190 * Goes trough every file ending with ".repo" in a directory and adds all
191 * RepoInfo's contained in that file.
193 * \param dir pathname of the directory to read.
195 static std::list<RepoInfo> repositories_in_dir( const Pathname &dir )
197 MIL << "directory " << dir << endl;
198 list<RepoInfo> repos;
199 list<Pathname> entries;
200 if ( filesystem::readdir( entries, Pathname(dir), false ) != 0 )
201 ZYPP_THROW(Exception("failed to read directory"));
203 str::regex allowedRepoExt("^\\.repo(_[0-9]+)?$");
204 for ( list<Pathname>::const_iterator it = entries.begin(); it != entries.end(); ++it )
206 if (str::regex_match(it->extension(), allowedRepoExt))
208 list<RepoInfo> tmp = repositories_in_file( *it );
209 repos.insert( repos.end(), tmp.begin(), tmp.end() );
211 //std::copy( collector.repos.begin(), collector.repos.end(), std::back_inserter(repos));
212 //MIL << "ok" << endl;
218 ////////////////////////////////////////////////////////////////////////////
220 static void assert_alias( const RepoInfo &info )
222 if (info.alias().empty())
223 ZYPP_THROW(RepoNoAliasException());
226 ////////////////////////////////////////////////////////////////////////////
228 static void assert_urls( const RepoInfo &info )
230 if (info.baseUrlsEmpty())
231 ZYPP_THROW(RepoNoUrlException());
234 ////////////////////////////////////////////////////////////////////////////
237 * \short Calculates the raw cache path for a repository
239 static Pathname rawcache_path_for_repoinfo( const RepoManagerOptions &opt, const RepoInfo &info )
242 return opt.repoRawCachePath / info.escaped_alias();
246 * \short Calculates the packages cache path for a repository
248 static Pathname packagescache_path_for_repoinfo( const RepoManagerOptions &opt, const RepoInfo &info )
251 return opt.repoPackagesCachePath / info.escaped_alias();
254 static Pathname solv_path_for_repoinfo( const RepoManagerOptions &opt, const RepoInfo &info)
257 return opt.repoSolvCachePath / info.escaped_alias();
260 ///////////////////////////////////////////////////////////////////
262 // CLASS NAME : RepoManager::Impl
264 ///////////////////////////////////////////////////////////////////
267 * \short RepoManager implementation.
269 struct RepoManager::Impl
271 Impl( const RepoManagerOptions &opt )
283 RepoManagerOptions options;
290 /** Offer default Impl. */
291 static shared_ptr<Impl> nullimpl()
293 static shared_ptr<Impl> _nullimpl( new Impl );
297 void saveService( const ServiceInfo & service ) const;
299 Pathname generateNonExistingName( const Pathname &dir,
300 const std::string &basefilename ) const;
302 std::string generateFilename( const RepoInfo & info ) const;
303 std::string generateFilename( const ServiceInfo & info ) const;
305 struct ServiceCollector
307 ServiceCollector(ServiceSet & services_) : services(services_) {}
309 bool collect(ServiceInfo service) { services.insert(service); return true; }
312 ServiceSet & services;
315 void knownServices();
317 void knownRepositories();
320 friend Impl * rwcowClone<Impl>( const Impl * rhs );
321 /** clone for RWCOW_pointer */
323 { return new Impl( *this ); }
326 ///////////////////////////////////////////////////////////////////
328 /** \relates RepoManager::Impl Stream output */
329 inline std::ostream & operator<<( std::ostream & str, const RepoManager::Impl & obj )
331 return str << "RepoManager::Impl";
334 ///////////////////////////////////////////////////////////////////
336 // CLASS NAME : RepoManager
338 ///////////////////////////////////////////////////////////////////
340 RepoManager::RepoManager( const RepoManagerOptions &opt )
341 : _pimpl( new Impl(opt) )
344 ////////////////////////////////////////////////////////////////////////////
346 RepoManager::~RepoManager()
349 ////////////////////////////////////////////////////////////////////////////
351 bool RepoManager::repoEmpty() const { return _pimpl->repos.empty(); }
352 RepoManager::RepoSizeType RepoManager::repoSize() const
353 { return _pimpl->repos.size(); }
354 RepoManager::RepoConstIterator RepoManager::repoBegin() const
355 { return _pimpl->repos.begin(); }
356 RepoManager::RepoConstIterator RepoManager::repoEnd() const
357 { return _pimpl->repos.end(); }
360 std::list<RepoInfo> RepoManager::knownRepositories() const
362 return std::list<RepoInfo>(repoBegin(),repoEnd());
365 void RepoManager::Impl::knownRepositories()
367 MIL << "start construct known repos" << endl;
369 if ( PathInfo(options.knownReposPath).isExist() )
371 RepoInfoList repol = repositories_in_dir(options.knownReposPath);
372 for ( RepoInfoList::iterator it = repol.begin();
376 // set the metadata path for the repo
377 Pathname metadata_path = rawcache_path_for_repoinfo(options, (*it));
378 (*it).setMetadataPath(metadata_path);
380 // set the downloaded packages path for the repo
381 Pathname packages_path = packagescache_path_for_repoinfo(options, (*it));
382 (*it).setPackagesPath(packages_path);
388 MIL << "end construct known repos" << endl;
391 ////////////////////////////////////////////////////////////////////////////
393 Pathname RepoManager::metadataPath( const RepoInfo &info ) const
395 return rawcache_path_for_repoinfo(_pimpl->options, info );
398 Pathname RepoManager::packagesPath( const RepoInfo &info ) const
400 return packagescache_path_for_repoinfo(_pimpl->options, info );
403 ////////////////////////////////////////////////////////////////////////////
405 RepoStatus RepoManager::metadataStatus( const RepoInfo &info ) const
407 Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
408 RepoType repokind = info.type();
411 switch ( repokind.toEnum() )
413 case RepoType::NONE_e:
414 // unknown, probe the local metadata
415 repokind = probe(rawpath.asUrl());
421 switch ( repokind.toEnum() )
423 case RepoType::RPMMD_e :
425 status = RepoStatus( rawpath + "/repodata/repomd.xml");
429 case RepoType::YAST2_e :
431 status = RepoStatus( rawpath + "/content") && (RepoStatus( rawpath + "/media.1/media"));
435 case RepoType::RPMPLAINDIR_e :
437 if ( PathInfo(Pathname(rawpath + "/cookie")).isExist() )
438 status = RepoStatus( rawpath + "/cookie");
442 case RepoType::NONE_e :
443 // Return default RepoStatus in case of RepoType::NONE
444 // indicating it should be created?
445 // ZYPP_THROW(RepoUnknownTypeException());
451 void RepoManager::touchIndexFile(const RepoInfo & info)
453 Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
455 RepoType repokind = info.type();
456 if ( repokind.toEnum() == RepoType::NONE_e )
457 // unknown, probe the local metadata
458 repokind = probe(rawpath.asUrl());
459 // if still unknown, just return
460 if (repokind == RepoType::NONE_e)
464 switch ( repokind.toEnum() )
466 case RepoType::RPMMD_e :
467 p = Pathname(rawpath + "/repodata/repomd.xml");
470 case RepoType::YAST2_e :
471 p = Pathname(rawpath + "/content");
474 case RepoType::RPMPLAINDIR_e :
475 p = Pathname(rawpath + "/cookie");
478 case RepoType::NONE_e :
483 // touch the file, ignore error (they are logged anyway)
484 filesystem::touch(p);
487 RepoManager::RefreshCheckStatus RepoManager::checkIfToRefreshMetadata(
488 const RepoInfo &info,
490 RawMetadataRefreshPolicy policy )
494 RepoStatus oldstatus;
495 RepoStatus newstatus;
499 MIL << "Going to try to check whether refresh is needed for " << url << endl;
501 repo::RepoType repokind = info.type();
503 // if the type is unknown, try probing.
504 switch ( repokind.toEnum() )
506 case RepoType::NONE_e:
508 repokind = probe(url);
514 Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
515 filesystem::assert_dir(rawpath);
516 oldstatus = metadataStatus(info);
518 // now we've got the old (cached) status, we can decide repo.refresh.delay
519 if (policy != RefreshForced && policy != RefreshIfNeededIgnoreDelay)
521 // difference in seconds
522 double diff = difftime(
523 (Date::ValueType)Date::now(),
524 (Date::ValueType)oldstatus.timestamp()) / 60;
526 DBG << "oldstatus: " << (Date::ValueType)oldstatus.timestamp() << endl;
527 DBG << "current time: " << (Date::ValueType)Date::now() << endl;
528 DBG << "last refresh = " << diff << " minutes ago" << endl;
530 if (diff < ZConfig::instance().repo_refresh_delay())
532 MIL << "Repository '" << info.alias()
533 << "' has been refreshed less than repo.refresh.delay ("
534 << ZConfig::instance().repo_refresh_delay()
535 << ") minutes ago. Advising to skip refresh" << endl;
536 return REPO_CHECK_DELAYED;
540 // create temp dir as sibling of rawpath
541 filesystem::TmpDir tmpdir( filesystem::TmpDir::makeSibling( rawpath ) );
543 if ( ( repokind.toEnum() == RepoType::RPMMD_e ) ||
544 ( repokind.toEnum() == RepoType::YAST2_e ) )
546 MediaSetAccess media(url);
547 shared_ptr<repo::Downloader> downloader_ptr;
549 if ( repokind.toEnum() == RepoType::RPMMD_e )
550 downloader_ptr.reset(new yum::Downloader(info));
552 downloader_ptr.reset( new susetags::Downloader(info));
554 RepoStatus newstatus = downloader_ptr->status(media);
555 bool refresh = false;
556 if ( oldstatus.checksum() == newstatus.checksum() )
558 MIL << "repo has not changed" << endl;
559 if ( policy == RefreshForced )
561 MIL << "refresh set to forced" << endl;
567 MIL << "repo has changed, going to refresh" << endl;
572 touchIndexFile(info);
574 return refresh ? REFRESH_NEEDED : REPO_UP_TO_DATE;
576 else if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
578 RepoStatus newstatus = parser::plaindir::dirStatus(url.getPathName());
579 bool refresh = false;
580 if ( oldstatus.checksum() == newstatus.checksum() )
582 MIL << "repo has not changed" << endl;
583 if ( policy == RefreshForced )
585 MIL << "refresh set to forced" << endl;
591 MIL << "repo has changed, going to refresh" << endl;
596 touchIndexFile(info);
598 return refresh ? REFRESH_NEEDED : REPO_UP_TO_DATE;
602 ZYPP_THROW(RepoUnknownTypeException(info));
605 catch ( const Exception &e )
608 ERR << "refresh check failed for " << url << endl;
612 return REFRESH_NEEDED; // default
615 void RepoManager::refreshMetadata( const RepoInfo &info,
616 RawMetadataRefreshPolicy policy,
617 const ProgressData::ReceiverFnc & progress )
622 // we will throw this later if no URL checks out fine
623 RepoException rexception(_("Valid metadata not found at specified URL(s)"));
625 // try urls one by one
626 for ( RepoInfo::urls_const_iterator it = info.baseUrlsBegin(); it != info.baseUrlsEnd(); ++it )
632 // check whether to refresh metadata
633 // if the check fails for this url, it throws, so another url will be checked
634 if (checkIfToRefreshMetadata(info, url, policy)!=REFRESH_NEEDED)
637 MIL << "Going to refresh metadata from " << url << endl;
639 repo::RepoType repokind = info.type();
641 // if the type is unknown, try probing.
642 switch ( repokind.toEnum() )
644 case RepoType::NONE_e:
646 repokind = probe(*it);
648 if (repokind.toEnum() != RepoType::NONE_e)
650 // Adjust the probed type in RepoInfo
651 info.setProbedType( repokind ); // lazy init!
652 //save probed type only for repos in system
653 for_( it, repoBegin(), repoEnd() )
655 if ( info.alias() == (*it).alias() )
657 RepoInfo modifiedrepo = info;
658 modifiedrepo.setType(repokind);
659 modifyRepository(info.alias(),modifiedrepo);
668 Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
669 filesystem::assert_dir(rawpath);
671 // create temp dir as sibling of rawpath
672 filesystem::TmpDir tmpdir( filesystem::TmpDir::makeSibling( rawpath ) );
674 if ( ( repokind.toEnum() == RepoType::RPMMD_e ) ||
675 ( repokind.toEnum() == RepoType::YAST2_e ) )
677 MediaSetAccess media(url);
678 shared_ptr<repo::Downloader> downloader_ptr;
680 MIL << "Creating downloader for [ " << info.name() << " ]" << endl;
682 if ( repokind.toEnum() == RepoType::RPMMD_e )
683 downloader_ptr.reset(new yum::Downloader(info));
685 downloader_ptr.reset( new susetags::Downloader(info) );
688 * Given a downloader, sets the other repos raw metadata
689 * path as cache paths for the fetcher, so if another
690 * repo has the same file, it will not download it
691 * but copy it from the other repository
693 for_( it, repoBegin(), repoEnd() )
695 Pathname cachepath(rawcache_path_for_repoinfo( _pimpl->options, *it ));
696 if ( PathInfo(cachepath).isExist() )
697 downloader_ptr->addCachePath(cachepath);
700 downloader_ptr->download( media, tmpdir.path());
702 else if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
704 RepoStatus newstatus = parser::plaindir::dirStatus(url.getPathName());
706 std::ofstream file(( tmpdir.path() + "/cookie").c_str());
708 ZYPP_THROW (Exception( "Can't open " + tmpdir.path().asString() + "/cookie" ) );
711 file << newstatus.checksum() << endl;
717 ZYPP_THROW(RepoUnknownTypeException());
720 // ok we have the metadata, now exchange
723 TmpDir oldmetadata( TmpDir::makeSibling( rawpath ) );
724 filesystem::rename( rawpath, oldmetadata.path() );
725 // move the just downloaded there
726 filesystem::rename( tmpdir.path(), rawpath );
731 catch ( const Exception &e )
734 ERR << "Trying another url..." << endl;
736 // remember the exception caught for the *first URL*
737 // if all other URLs fail, the rexception will be thrown with the
738 // cause of the problem of the first URL remembered
739 if (it == info.baseUrlsBegin())
740 rexception.remember(e);
743 ERR << "No more urls..." << endl;
744 ZYPP_THROW(rexception);
747 ////////////////////////////////////////////////////////////////////////////
749 void RepoManager::cleanMetadata( const RepoInfo &info,
750 const ProgressData::ReceiverFnc & progressfnc )
752 ProgressData progress(100);
753 progress.sendTo(progressfnc);
755 filesystem::recursive_rmdir(rawcache_path_for_repoinfo(_pimpl->options, info));
759 void RepoManager::cleanPackages( const RepoInfo &info,
760 const ProgressData::ReceiverFnc & progressfnc )
762 ProgressData progress(100);
763 progress.sendTo(progressfnc);
765 filesystem::recursive_rmdir(packagescache_path_for_repoinfo(_pimpl->options, info));
769 void RepoManager::buildCache( const RepoInfo &info,
770 CacheBuildPolicy policy,
771 const ProgressData::ReceiverFnc & progressrcv )
774 Pathname rawpath = rawcache_path_for_repoinfo(_pimpl->options, info);
776 filesystem::assert_dir(_pimpl->options.repoCachePath);
777 RepoStatus raw_metadata_status = metadataStatus(info);
778 if ( raw_metadata_status.empty() )
780 /* if there is no cache at this point, we refresh the raw
781 in case this is the first time - if it's !autorefresh,
782 we may still refresh */
783 refreshMetadata(info, RefreshIfNeeded, progressrcv );
784 raw_metadata_status = metadataStatus(info);
787 bool needs_cleaning = false;
788 if ( isCached( info ) )
790 MIL << info.alias() << " is already cached." << endl;
791 RepoStatus cache_status = cacheStatus(info);
793 if ( cache_status.checksum() == raw_metadata_status.checksum() )
795 MIL << info.alias() << " cache is up to date with metadata." << endl;
796 if ( policy == BuildIfNeeded ) {
800 MIL << info.alias() << " cache rebuild is forced" << endl;
804 needs_cleaning = true;
807 ProgressData progress(100);
808 callback::SendReport<ProgressReport> report;
809 progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
810 progress.name(str::form(_("Building repository '%s' cache"), info.name().c_str()));
818 MIL << info.alias() << " building cache..." << endl;
820 Pathname base = solv_path_for_repoinfo( _pimpl->options, info);
821 filesystem::assert_dir(base);
822 Pathname solvfile = base / "solv";
825 repo::RepoType repokind = info.type();
827 // if the type is unknown, try probing.
828 switch ( repokind.toEnum() )
830 case RepoType::NONE_e:
831 // unknown, probe the local metadata
832 repokind = probe(rawpath.asUrl());
838 MIL << "repo type is " << repokind << endl;
840 switch ( repokind.toEnum() )
842 case RepoType::RPMMD_e :
843 case RepoType::YAST2_e :
844 case RepoType::RPMPLAINDIR_e :
846 // Take care we unlink the solvfile on exception
847 ManagedFile guard( solvfile, filesystem::unlink );
850 std::string toFile( str::gsub(solvfile.asString(),"\"","\\\"") );
851 if ( repokind.toEnum() == RepoType::RPMPLAINDIR_e )
853 // FIXME this does only work form dir: URLs
854 cmd << str::form( "repo2solv.sh \"%s\" > \"%s\"",
855 str::gsub( info.baseUrlsBegin()->getPathName(),"\"","\\\"" ).c_str(),
860 cmd << str::form( "repo2solv.sh \"%s\" > \"%s\"",
861 str::gsub( rawpath.asString(),"\"","\\\"" ).c_str(),
864 MIL << "Executing: " << cmd.str() << endl;
865 ExternalProgram prog( cmd.str(), ExternalProgram::Stderr_To_Stdout );
868 for ( string output( prog.receiveLine() ); output.length(); output = prog.receiveLine() ) {
869 WAR << " " << output;
870 cmd << " " << output;
873 int ret = prog.close();
876 RepoException ex(str::form("Failed to cache repo (%d).", ret));
877 ex.remember( cmd.str() );
882 guard.resetDispose();
886 ZYPP_THROW(RepoUnknownTypeException("Unhandled repository type"));
889 // update timestamp and checksum
890 setCacheStatus(info, raw_metadata_status);
891 MIL << "Commit cache.." << endl;
895 ////////////////////////////////////////////////////////////////////////////
897 repo::RepoType RepoManager::probe( const Url &url ) const
899 if ( url.getScheme() == "dir" && ! PathInfo( url.getPathName() ).isDir() )
901 // Handle non existing local directory in advance, as
902 // MediaSetAccess does not support it.
903 return repo::RepoType::NONE;
908 MediaSetAccess access(url);
909 if ( access.doesFileExist("/repodata/repomd.xml") )
910 return repo::RepoType::RPMMD;
911 if ( access.doesFileExist("/content") )
912 return repo::RepoType::YAST2;
914 // if it is a local url of type dir
915 if ( (! media::MediaManager::downloads(url)) && ( url.getScheme() == "dir" ) )
917 Pathname path = Pathname(url.getPathName());
918 if ( PathInfo(path).isDir() )
920 // allow empty dirs for now
921 return repo::RepoType::RPMPLAINDIR;
925 catch ( const media::MediaException &e )
928 RepoException enew("Error trying to read from " + url.asString());
932 catch ( const Exception &e )
935 Exception enew("Unknown error reading from " + url.asString());
940 return repo::RepoType::NONE;
943 ////////////////////////////////////////////////////////////////////////////
945 void RepoManager::cleanCache( const RepoInfo &info,
946 const ProgressData::ReceiverFnc & progressrcv )
948 ProgressData progress(100);
949 progress.sendTo(progressrcv);
952 filesystem::recursive_rmdir(solv_path_for_repoinfo(_pimpl->options, info));
957 ////////////////////////////////////////////////////////////////////////////
959 bool RepoManager::isCached( const RepoInfo &info ) const
961 return PathInfo(solv_path_for_repoinfo( _pimpl->options, info ) / "solv").isExist();
964 RepoStatus RepoManager::cacheStatus( const RepoInfo &info ) const
967 Pathname cookiefile = solv_path_for_repoinfo(_pimpl->options, info) / "cookie";
969 return RepoStatus::fromCookieFile(cookiefile);
972 void RepoManager::setCacheStatus( const RepoInfo &info, const RepoStatus &status )
974 Pathname base = solv_path_for_repoinfo(_pimpl->options, info);
975 filesystem::assert_dir(base);
976 Pathname cookiefile = base / "cookie";
978 status.saveToCookieFile(cookiefile);
981 void RepoManager::loadFromCache( const RepoInfo & info,
982 const ProgressData::ReceiverFnc & progressrcv )
985 Pathname solvfile = solv_path_for_repoinfo(_pimpl->options, info) / "solv";
987 if ( ! PathInfo(solvfile).isExist() )
988 ZYPP_THROW(RepoNotCachedException(info));
992 sat::Pool::instance().addRepoSolv( solvfile, info );
994 catch ( const Exception & exp )
997 MIL << "Try to handle exception by rebuilding the solv-file" << endl;
998 cleanCache( info, progressrcv );
999 buildCache( info, BuildIfNeeded, progressrcv );
1001 sat::Pool::instance().addRepoSolv( solvfile, info );
1005 ////////////////////////////////////////////////////////////////////////////
1008 * Generate a non existing filename in a directory, using a base
1009 * name. For example if a directory contains 3 files
1015 * If you try to generate a unique filename for this directory,
1016 * based on "ruu" you will get "ruu", but if you use the base
1017 * "foo" you will get "foo_1"
1019 * \param dir Directory where the file needs to be unique
1020 * \param basefilename string to base the filename on.
1022 Pathname RepoManager::Impl::generateNonExistingName( const Pathname &dir,
1023 const std::string &basefilename ) const
1025 string final_filename = basefilename;
1027 while ( PathInfo(dir + final_filename).isExist() )
1029 final_filename = basefilename + "_" + str::numstring(counter);
1032 return dir + Pathname(final_filename);
1035 ////////////////////////////////////////////////////////////////////////////
1038 * \short Generate a related filename from a repo info
1040 * From a repo info, it will try to use the alias as a filename
1041 * escaping it if necessary. Other fallbacks can be added to
1042 * this function in case there is no way to use the alias
1044 std::string RepoManager::Impl::generateFilename( const RepoInfo &info ) const
1046 std::string filename = info.alias();
1047 // replace slashes with underscores
1048 str::replaceAll( filename, "/", "_" );
1050 filename = Pathname(filename).extend(".repo").asString();
1051 MIL << "generating filename for repo [" << info.alias() << "] : '" << filename << "'" << endl;
1055 std::string RepoManager::Impl::generateFilename( const ServiceInfo & info ) const
1057 std::string filename = info.alias();
1058 // replace slashes with underscores
1059 str::replaceAll( filename, "/", "_" );
1061 filename = Pathname(filename).extend(".service").asString();
1062 MIL << "generating filename for service [" << info.alias() << "] : '" << filename << "'" << endl;
1066 ////////////////////////////////////////////////////////////////////////////
1068 void RepoManager::addRepository( const RepoInfo &info,
1069 const ProgressData::ReceiverFnc & progressrcv )
1073 ProgressData progress(100);
1074 callback::SendReport<ProgressReport> report;
1075 progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
1076 progress.name(str::form(_("Adding repository '%s'"), info.name().c_str()));
1079 RepoInfo tosave = info;
1080 if(_pimpl->repos.find(tosave)!= _pimpl->repos.end())
1081 ZYPP_THROW(RepoAlreadyExistsException(info));
1084 // check the first url for now
1085 if ( _pimpl->options.probe )
1087 DBG << "unknown repository type, probing" << endl;
1089 RepoType probedtype;
1090 probedtype = probe(*tosave.baseUrlsBegin());
1091 if ( tosave.baseUrlsSize() > 0 )
1093 if ( probedtype == RepoType::NONE )
1094 ZYPP_THROW(RepoUnknownTypeException());
1096 tosave.setType(probedtype);
1102 // assert the directory exists
1103 filesystem::assert_dir(_pimpl->options.knownReposPath);
1105 Pathname repofile = _pimpl->generateNonExistingName(
1106 _pimpl->options.knownReposPath, _pimpl->generateFilename(tosave));
1107 // now we have a filename that does not exists
1108 MIL << "Saving repo in " << repofile << endl;
1110 std::ofstream file(repofile.c_str());
1112 ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
1115 tosave.dumpAsIniOn(file);
1116 tosave.setFilepath(repofile);
1117 _pimpl->repos.insert(tosave);
1119 MIL << "done" << endl;
1122 void RepoManager::addRepositories( const Url &url,
1123 const ProgressData::ReceiverFnc & progressrcv )
1125 std::list<RepoInfo> repos = readRepoFile(url);
1126 for ( std::list<RepoInfo>::const_iterator it = repos.begin();
1130 // look if the alias is in the known repos.
1131 for_ ( kit, repoBegin(), repoEnd() )
1133 if ( (*it).alias() == (*kit).alias() )
1135 ERR << "To be added repo " << (*it).alias() << " conflicts with existing repo " << (*kit).alias() << endl;
1136 ZYPP_THROW(RepoAlreadyExistsException(*it));
1141 string filename = Pathname(url.getPathName()).basename();
1143 if ( filename == Pathname() )
1144 ZYPP_THROW(RepoException("Invalid repo file name at " + url.asString() ));
1146 // assert the directory exists
1147 filesystem::assert_dir(_pimpl->options.knownReposPath);
1149 Pathname repofile = _pimpl->generateNonExistingName(_pimpl->options.knownReposPath, filename);
1150 // now we have a filename that does not exists
1151 MIL << "Saving " << repos.size() << " repo" << ( repos.size() ? "s" : "" ) << " in " << repofile << endl;
1153 std::ofstream file(repofile.c_str());
1155 ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
1158 for ( std::list<RepoInfo>::iterator it = repos.begin();
1162 MIL << "Saving " << (*it).alias() << endl;
1163 it->setFilepath(repofile.asString());
1164 it->dumpAsIniOn(file);
1165 _pimpl->repos.insert(*it);
1167 MIL << "done" << endl;
1170 ////////////////////////////////////////////////////////////////////////////
1172 void RepoManager::removeRepository( const RepoInfo & info,
1173 const ProgressData::ReceiverFnc & progressrcv)
1175 ProgressData progress;
1176 callback::SendReport<ProgressReport> report;
1177 progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
1178 progress.name(str::form(_("Removing repository '%s'"), info.name().c_str()));
1180 MIL << "Going to delete repo " << info.alias() << endl;
1182 for_( it, repoBegin(), repoEnd() )
1184 // they can be the same only if the provided is empty, that means
1185 // the provided repo has no alias
1187 if ( (!info.alias().empty()) && ( info.alias() != (*it).alias() ) )
1190 // TODO match by url
1192 // we have a matcing repository, now we need to know
1193 // where it does come from.
1194 RepoInfo todelete = *it;
1195 if (todelete.filepath().empty())
1197 ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
1201 // figure how many repos are there in the file:
1202 std::list<RepoInfo> filerepos = repositories_in_file(todelete.filepath());
1203 if ( (filerepos.size() == 1) && ( filerepos.front().alias() == todelete.alias() ) )
1205 // easy, only this one, just delete the file
1206 if ( filesystem::unlink(todelete.filepath()) != 0 )
1208 ZYPP_THROW(RepoException("Can't delete " + todelete.filepath().asString()));
1210 MIL << todelete.alias() << " sucessfully deleted." << endl;
1214 // there are more repos in the same file
1215 // write them back except the deleted one.
1217 //std::ofstream file(tmp.path().c_str());
1219 // assert the directory exists
1220 filesystem::assert_dir(todelete.filepath().dirname());
1222 std::ofstream file(todelete.filepath().c_str());
1224 //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
1225 ZYPP_THROW (Exception( "Can't open " + todelete.filepath().asString() ) );
1227 for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
1228 fit != filerepos.end();
1231 if ( (*fit).alias() != todelete.alias() )
1232 (*fit).dumpAsIniOn(file);
1236 CombinedProgressData subprogrcv(progress, 70);
1237 CombinedProgressData cleansubprogrcv(progress, 30);
1238 // now delete it from cache
1239 if ( isCached(todelete) )
1240 cleanCache( todelete, subprogrcv);
1241 // now delete metadata (#301037)
1242 cleanMetadata( todelete, cleansubprogrcv);
1243 _pimpl->repos.erase(todelete);
1244 MIL << todelete.alias() << " sucessfully deleted." << endl;
1246 } // else filepath is empty
1249 // should not be reached on a sucess workflow
1250 ZYPP_THROW(RepoNotFoundException(info));
1253 ////////////////////////////////////////////////////////////////////////////
1255 void RepoManager::modifyRepository( const std::string &alias,
1256 const RepoInfo & newinfo,
1257 const ProgressData::ReceiverFnc & progressrcv )
1259 RepoInfo toedit = getRepositoryInfo(alias);
1261 // check if the new alias already exists when renaming the repo
1262 if (alias != newinfo.alias())
1264 for_( it, repoBegin(), repoEnd() )
1266 if ( newinfo.alias() == (*it).alias() )
1267 ZYPP_THROW(RepoAlreadyExistsException(newinfo));
1271 if (toedit.filepath().empty())
1273 ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
1277 // figure how many repos are there in the file:
1278 std::list<RepoInfo> filerepos = repositories_in_file(toedit.filepath());
1280 // there are more repos in the same file
1281 // write them back except the deleted one.
1283 //std::ofstream file(tmp.path().c_str());
1285 // assert the directory exists
1286 filesystem::assert_dir(toedit.filepath().dirname());
1288 std::ofstream file(toedit.filepath().c_str());
1290 //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
1291 ZYPP_THROW (Exception( "Can't open " + toedit.filepath().asString() ) );
1293 for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
1294 fit != filerepos.end();
1297 // if the alias is different, dump the original
1298 // if it is the same, dump the provided one
1299 if ( (*fit).alias() != toedit.alias() )
1300 (*fit).dumpAsIniOn(file);
1302 newinfo.dumpAsIniOn(file);
1305 _pimpl->repos.erase(toedit);
1306 _pimpl->repos.insert(newinfo);
1310 ////////////////////////////////////////////////////////////////////////////
1312 RepoInfo RepoManager::getRepositoryInfo( const std::string &alias,
1313 const ProgressData::ReceiverFnc & progressrcv )
1316 info.setAlias(alias);
1317 RepoConstIterator it = _pimpl->repos.find( info );
1318 if( it == repoEnd() )
1319 ZYPP_THROW(RepoNotFoundException(info));
1324 ////////////////////////////////////////////////////////////////////////////
1326 RepoInfo RepoManager::getRepositoryInfo( const Url & url,
1327 const url::ViewOption & urlview,
1328 const ProgressData::ReceiverFnc & progressrcv )
1330 for_( it, repoBegin(), repoEnd() )
1332 for(RepoInfo::urls_const_iterator urlit = (*it).baseUrlsBegin();
1333 urlit != (*it).baseUrlsEnd();
1336 if ((*urlit).asString(urlview) == url.asString(urlview))
1341 info.setBaseUrl(url);
1342 ZYPP_THROW(RepoNotFoundException(info));
1345 void RepoManager::addService( const std::string & alias, const Url & url )
1347 addService( ServiceInfo(alias, url) );
1351 void RepoManager::addService( const ServiceInfo & service )
1353 // check if service already exists
1354 if( _pimpl->services.find(service) != _pimpl->services.end() )
1355 return; //FIXME ZYPP_THROW(RepoAlreadyExistsException(service.name()));
1357 // this is need to save location to correct service
1358 const ServiceInfo & savedService =
1359 *(_pimpl->services.insert( service )).first;
1361 MIL << "added service " << savedService.alias() << endl;
1363 _pimpl->saveService( savedService );
1367 void RepoManager::removeService( const string & alias)
1369 MIL << "Going to delete repo " << alias << endl;
1371 const ServiceInfo & service = getService( alias );
1373 Pathname location = service.filepath();
1374 if( location.empty() )
1376 ZYPP_THROW(RepoException("Can't figure where the service is stored"));
1380 Impl::ServiceCollector collector(tmpSet);
1382 parser::ServiceFileReader reader( location,
1383 bind(&Impl::ServiceCollector::collect,collector,_1) );
1385 // only one service definition in the file
1386 if ( tmpSet.size() == 1 )
1388 if ( filesystem::unlink(location) != 0 )
1390 ZYPP_THROW(RepoException("Can't delete " + location.asString()));
1392 MIL << alias << " sucessfully deleted." << endl;
1396 filesystem::assert_dir(location.dirname());
1398 std::ofstream file(location.c_str());
1400 ZYPP_THROW(Exception("failed open file to write"));
1402 for_(it, tmpSet.begin(), tmpSet.end())
1404 if( it->alias() != alias )
1405 it->dumpAsIniOn(file);
1408 MIL << alias << " sucessfully deleted from file " << location << endl;
1411 // now remove all repositories added by this service
1412 RepoCollector rcollector;
1413 getRepositoriesInService( alias,
1414 boost::make_function_output_iterator(
1415 bind( &RepoCollector::collect, &rcollector, _1 ) ) );
1416 // cannot do this directly in getRepositoriesInService - would invalidate iterators
1417 for_(rit, rcollector.repos.begin(), rcollector.repos.end())
1418 removeRepository(*rit);
1422 void RepoManager::removeService( const ServiceInfo & service )
1423 { removeService(service.alias()); }
1426 void RepoManager::Impl::saveService( const ServiceInfo & service ) const
1428 filesystem::assert_dir( options.knownServicesPath );
1430 Pathname servfile = generateNonExistingName( options.knownServicesPath,
1431 generateFilename( service ) );
1433 MIL << "saving service in " << servfile << endl;
1435 std::ofstream file(servfile.c_str());
1437 ZYPP_THROW (Exception( "Can't open " + servfile.asString() ) );
1440 service.dumpAsIniOn( file );
1442 const_cast<ServiceInfo&>(service).setFilepath( servfile );
1443 MIL << "done" << endl;
1446 ServiceInfo RepoManager::getService( const std::string & alias ) const
1448 for_ (it, serviceBegin(), serviceEnd())
1449 if ( it->alias() == alias )
1451 return ServiceInfo::noService;
1454 bool RepoManager::serviceEmpty() const { return _pimpl->services.empty(); }
1456 RepoManager::ServiceSizeType RepoManager::serviceSize() const
1458 return _pimpl->services.size();
1461 RepoManager::ServiceConstIterator RepoManager::serviceBegin() const
1463 return _pimpl->services.begin();
1466 RepoManager::ServiceConstIterator RepoManager::serviceEnd() const
1468 return _pimpl->services.end();
1471 void RepoManager::refreshServices()
1473 // cannot user for_, because it uses const version
1474 for (ServiceConstIterator it = _pimpl->services.begin();
1475 it != _pimpl->services.end(); ++it)
1477 if ( !it->enabled() )
1480 MIL << "refresh: " << it->alias() << " with url: "<< it->url().asString() << endl;
1481 refreshService(*it);
1485 void RepoManager::refreshService( const ServiceInfo & service )
1487 //! \todo add callbacks for apps (start, end, repo removed, repo added, repo changed)
1489 repo::ServiceType type = service.type();
1490 // if the type is unknown, try probing.
1491 if ( type == repo::ServiceType::NONE )
1493 // unknown, probe it
1494 type = probeService(service.url());
1496 if (type != ServiceType::NONE)
1498 // Adjust the probed type in ServiceInfo
1499 service.setProbedType( type ); // lazy init!
1500 // save probed type only for repos in system
1501 for_( sit, serviceBegin(), serviceEnd() )
1503 if ( service.alias() == sit->alias() )
1505 ServiceInfo modifiedservice = service;
1506 modifiedservice.setType(type);
1507 modifyService(service.alias(), modifiedservice); // FIXME this causes a segfault, whe the same code from repos doesn't?
1514 // download the repo index file
1515 media::MediaManager mediamanager;
1516 //if (service.url().empty())
1517 // throw RepoNoUrlException();
1518 media::MediaAccessId mid = mediamanager.open( service.url() );
1519 mediamanager.attachDesiredMedia( mid );
1520 mediamanager.provideFile( mid, "repo/repoindex.xml" );
1521 Pathname path = mediamanager.localPath(mid, "repo/repoindex.xml" );
1524 RepoCollector collector(_pimpl->options.servicesTargetDistro);
1525 parser::RepoindexFileReader reader( path,
1526 bind( &RepoCollector::collect, &collector, _1 ) );
1527 mediamanager.release( mid );
1528 mediamanager.close( mid );
1530 // set service alias and base url for all collected repositories
1531 for_( it, collector.repos.begin(), collector.repos.end() )
1535 // if the repo url was not set by the repoindex parser, set service's url
1536 if ( it->baseUrlsEmpty() )
1537 url = service.url();
1539 // service repo can contain only one URL now, so no need to iterate
1540 url = *it->baseUrlsBegin();
1542 // libzypp currently has problem with separate url + path handling
1543 // so just append the path to the baseurl
1544 if (!it->path().empty())
1546 Pathname path(url.getPathName());
1548 url.setPathName( path.asString() );
1553 it->setBaseUrl( url );
1554 // set refrence to the parent service
1555 it->setService( service.alias() );
1558 // compare old and new repositories (hope not too much, if it change
1559 // then construct set and use set operation on it)
1560 std::list<RepoInfo> oldRepos;
1561 getRepositoriesInService(service.alias(),
1562 insert_iterator<std::list<RepoInfo> > (oldRepos, oldRepos.begin()));
1564 // find old to remove
1565 for_( it, oldRepos.begin(), oldRepos.end() )
1569 for_( it2, collector.repos.begin(), collector.repos.end() )
1570 if ( it->alias() == it2->alias() )
1577 removeRepository( *it );
1581 for_( it, collector.repos.begin(), collector.repos.end() )
1585 for_( it2, oldRepos.begin(), oldRepos.end() )
1586 if( it->alias() == it2->alias() )
1593 addRepository( *it );
1597 void RepoManager::modifyService(const std::string & oldAlias, const ServiceInfo & service)
1599 MIL << "Going to modify service " << oldAlias << endl;
1601 const ServiceInfo & oldService = getService(oldAlias);
1603 Pathname location = oldService.filepath();
1604 if( location.empty() )
1606 ZYPP_THROW(RepoException(
1607 "Cannot figure out where the service file is stored."));
1611 Impl::ServiceCollector collector(tmpSet);
1613 parser::ServiceFileReader reader( location,
1614 bind(&Impl::ServiceCollector::collect,collector,_1) );
1616 filesystem::assert_dir(location.dirname());
1618 std::ofstream file(location.c_str());
1620 for_(it, tmpSet.begin(), tmpSet.end())
1622 if( *it != oldAlias )
1623 it->dumpAsIniOn(file);
1626 service.dumpAsIniOn(file);
1630 _pimpl->services.erase(oldAlias);
1631 _pimpl->services.insert(service);
1633 // changed name, must change also repositories
1634 if( oldAlias != service.alias() )
1636 std::vector<RepoInfo> toModify;
1637 getRepositoriesInService(oldAlias,
1638 insert_iterator<std::vector<RepoInfo> >( toModify, toModify.begin() ));
1639 for_( it, toModify.begin(), toModify.end() )
1641 it->setService(service.alias());
1642 modifyRepository(it->alias(), *it);
1646 //! \todo changed enabled status
1647 if ( oldService.enabled() != service.enabled())
1652 //! \todo refresh the service automatically if url is changed?
1655 void RepoManager::Impl::knownServices()
1657 ServiceCollector collector(services);
1658 Pathname dir = options.knownServicesPath;
1659 list<Pathname> entries;
1660 if (PathInfo(dir).isExist())
1662 if ( filesystem::readdir( entries, Pathname(dir), false ) != 0 )
1663 ZYPP_THROW(Exception("failed to read directory"));
1665 //str::regex allowedServiceExt("^\\.service(_[0-9]+)?$");
1666 for_(it, entries.begin(), entries.end() )
1668 parser::ServiceFileReader reader(*it,
1669 bind(&ServiceCollector::collect, collector, _1) );
1674 repo::ServiceType RepoManager::probeService( const Url &url ) const
1678 MediaSetAccess access(url);
1679 if ( access.doesFileExist("/repo/repoindex.xml") )
1680 return repo::ServiceType::RIS;
1682 catch ( const media::MediaException &e )
1685 RepoException enew("Error trying to read from " + url.asString());
1689 catch ( const Exception &e )
1692 Exception enew("Unknown error reading from " + url.asString());
1697 return repo::ServiceType::NONE;
1700 ////////////////////////////////////////////////////////////////////////////
1702 std::ostream & operator<<( std::ostream & str, const RepoManager & obj )
1704 return str << *obj._pimpl;
1707 /////////////////////////////////////////////////////////////////
1709 ///////////////////////////////////////////////////////////////////