1 /*---------------------------------------------------------------------\
3 | |__ / \ / / . \ . \ |
8 \---------------------------------------------------------------------*/
9 /** \file zypp/RepoManager.cc
17 #include "zypp/base/InputStream.h"
18 #include "zypp/base/Logger.h"
19 #include "zypp/base/Gettext.h"
20 #include "zypp/base/Function.h"
21 #include "zypp/PathInfo.h"
22 #include "zypp/TmpPath.h"
24 #include "zypp/repo/RepoException.h"
25 #include "zypp/RepoManager.h"
27 #include "zypp/cache/CacheStore.h"
28 #include "zypp/repo/cached/RepoImpl.h"
29 #include "zypp/media/MediaManager.h"
30 #include "zypp/MediaSetAccess.h"
32 #include "zypp/parser/RepoFileReader.h"
33 #include "zypp/repo/yum/Downloader.h"
34 #include "zypp/parser/yum/RepoParser.h"
35 #include "zypp/parser/plaindir/RepoParser.h"
36 #include "zypp/repo/susetags/Downloader.h"
37 #include "zypp/parser/susetags/RepoParser.h"
39 #include "zypp/ZYppCallbacks.h"
43 using namespace zypp::repo;
44 using namespace zypp::filesystem;
46 using namespace zypp::repo;
48 ///////////////////////////////////////////////////////////////////
50 { /////////////////////////////////////////////////////////////////
52 ///////////////////////////////////////////////////////////////////
54 // CLASS NAME : RepoManagerOptions
56 ///////////////////////////////////////////////////////////////////
58 RepoManagerOptions::RepoManagerOptions()
61 repoCachePath = globalConfig.defaultRepoCachePath();
62 repoRawCachePath = globalConfig.defaultRepoRawCachePath();
63 knownReposPath = globalConfig.defaultKnownReposPath();
66 ////////////////////////////////////////////////////////////////////////////
69 * \short Simple callback to collect the results
71 * Classes like RepoFileParser call the callback
72 * once per each repo in a file.
74 * Passing this functor as callback, you can collect
75 * all resuls at the end, without dealing with async
90 bool collect( const RepoInfo &repo )
92 //MIL << "here in collector: " << repo.alias() << endl;
93 repos.push_back(repo);
94 //MIL << "added: " << repo.alias() << endl;
101 ////////////////////////////////////////////////////////////////////////////
104 * Reads RepoInfo's from a repo file.
106 * \param file pathname of the file to read.
108 static std::list<RepoInfo> repositories_in_file( const Pathname & file )
110 MIL << "repo file: " << file << endl;
111 RepoCollector collector;
112 parser::RepoFileReader parser( file, bind( &RepoCollector::collect, &collector, _1 ) );
113 return collector.repos;
116 ////////////////////////////////////////////////////////////////////////////
118 std::list<RepoInfo> readRepoFile(const Url & repo_file)
120 // no interface to download a specific file, using workaround:
121 //! \todo add MediaManager::provideFile(Url file_url) to easily access any file URLs? (no need for media access id or media_nr)
123 Pathname path(url.getPathName());
124 url.setPathName ("/");
125 MediaSetAccess access(url);
126 Pathname local = access.provideFile(path);
128 DBG << "reading repo file " << repo_file << ", local path: " << local << endl;
130 return repositories_in_file(local);
133 ////////////////////////////////////////////////////////////////////////////
136 * \short List of RepoInfo's from a directory
138 * Goes trough every file in a directory and adds all
139 * RepoInfo's contained in that file.
141 * \param dir pathname of the directory to read.
143 static std::list<RepoInfo> repositories_in_dir( const Pathname &dir )
145 MIL << "directory " << dir << endl;
146 list<RepoInfo> repos;
147 list<Pathname> entries;
148 if ( filesystem::readdir( entries, Pathname(dir), false ) != 0 )
149 ZYPP_THROW(Exception("failed to read directory"));
151 for ( list<Pathname>::const_iterator it = entries.begin(); it != entries.end(); ++it )
153 list<RepoInfo> tmp = repositories_in_file( *it );
154 repos.insert( repos.end(), tmp.begin(), tmp.end() );
156 //std::copy( collector.repos.begin(), collector.repos.end(), std::back_inserter(repos));
157 //MIL << "ok" << endl;
162 ////////////////////////////////////////////////////////////////////////////
164 static void assert_alias( const RepoInfo &info )
166 if (info.alias().empty())
167 ZYPP_THROW(RepoNoAliasException());
170 ////////////////////////////////////////////////////////////////////////////
172 static void assert_urls( const RepoInfo &info )
174 if (info.baseUrls().empty())
175 ZYPP_THROW(RepoNoUrlException());
178 ////////////////////////////////////////////////////////////////////////////
181 * \short Calculates the raw cache path for a repository
183 static Pathname rawcache_path_for_repoinfo( const RepoManagerOptions &opt, const RepoInfo &info )
186 return opt.repoRawCachePath + info.alias();
189 ///////////////////////////////////////////////////////////////////
191 // CLASS NAME : RepoManager::Impl
193 ///////////////////////////////////////////////////////////////////
196 * \short RepoManager implementation.
198 struct RepoManager::Impl
200 Impl( const RepoManagerOptions &opt )
211 RepoManagerOptions options;
214 /** Offer default Impl. */
215 static shared_ptr<Impl> nullimpl()
217 static shared_ptr<Impl> _nullimpl( new Impl );
222 friend Impl * rwcowClone<Impl>( const Impl * rhs );
223 /** clone for RWCOW_pointer */
225 { return new Impl( *this ); }
227 ///////////////////////////////////////////////////////////////////
229 /** \relates RepoManager::Impl Stream output */
230 inline std::ostream & operator<<( std::ostream & str, const RepoManager::Impl & obj )
232 return str << "RepoManager::Impl";
235 ///////////////////////////////////////////////////////////////////
237 // CLASS NAME : RepoManager
239 ///////////////////////////////////////////////////////////////////
241 RepoManager::RepoManager( const RepoManagerOptions &opt )
242 : _pimpl( new Impl(opt) )
245 ////////////////////////////////////////////////////////////////////////////
247 RepoManager::~RepoManager()
250 ////////////////////////////////////////////////////////////////////////////
252 std::list<RepoInfo> RepoManager::knownRepositories() const
256 if ( PathInfo(_pimpl->options.knownReposPath).isExist() )
257 return repositories_in_dir(_pimpl->options.knownReposPath);
259 return std::list<RepoInfo>();
264 ////////////////////////////////////////////////////////////////////////////
266 RepoStatus RepoManager::rawMetadataStatus( const RepoInfo &info )
268 Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
269 RepoType repokind = info.type();
272 switch ( repokind.toEnum() )
274 case RepoType::NONE_e:
275 // unknown, probe the local metadata
276 repokind = probe(rawpath.asUrl());
282 switch ( repokind.toEnum() )
284 case RepoType::RPMMD_e :
286 status = RepoStatus( rawpath + "/repodata/repomd.xml");
290 case RepoType::YAST2_e :
292 status = RepoStatus( rawpath + "/content");
296 case RepoType::RPMPLAINDIR_e :
298 if ( PathInfo(Pathname(rawpath + "/cookie")).isExist() )
299 status = RepoStatus( rawpath + "/cookie");
303 case RepoType::NONE_e :
304 // Return default RepoStatus in case of RepoType::NONE
305 // indicating it should be created?
306 // ZYPP_THROW(RepoUnknownTypeException());
313 void RepoManager::refreshMetadata( const RepoInfo &info,
314 RawMetadataRefreshPolicy policy,
315 const ProgressData::ReceiverFnc & progress )
320 RepoStatus oldstatus;
321 RepoStatus newstatus;
322 // try urls one by one
323 for ( RepoInfo::urls_const_iterator it = info.baseUrlsBegin(); it != info.baseUrlsEnd(); ++it )
328 filesystem::TmpDir tmpdir;
330 repo::RepoType repokind = info.type();
332 // if the type is unknown, try probing.
333 switch ( repokind.toEnum() )
335 case RepoType::NONE_e:
337 repokind = probe(*it);
343 Pathname rawpath = rawcache_path_for_repoinfo( _pimpl->options, info );
344 oldstatus = rawMetadataStatus(info);
346 switch ( repokind.toEnum() )
348 case RepoType::RPMMD_e :
350 yum::Downloader downloader( url, "/" );
352 RepoStatus newstatus = downloader.status();
353 bool refresh = false;
354 if ( oldstatus.checksum() == newstatus.checksum() )
356 MIL << "repo has not changed" << endl;
357 if ( policy == RefreshForced )
359 MIL << "refresh set to forced" << endl;
369 downloader.download(tmpdir.path());
375 case RepoType::YAST2_e :
377 susetags::Downloader downloader( url, "/" );
379 RepoStatus newstatus = downloader.status();
380 bool refresh = false;
381 if ( oldstatus.checksum() == newstatus.checksum() )
383 MIL << "repo has not changed" << endl;
384 if ( policy == RefreshForced )
386 MIL << "refresh set to forced" << endl;
396 downloader.download(tmpdir.path());
403 case RepoType::RPMPLAINDIR_e :
405 RepoStatus newstatus = parser::plaindir::dirStatus(url.getPathName());
406 bool refresh = false;
407 if ( oldstatus.checksum() == newstatus.checksum() )
409 MIL << "repo has not changed" << endl;
410 if ( policy == RefreshForced )
412 MIL << "refresh set to forced" << endl;
423 std::ofstream file(( tmpdir.path() + "/cookie").c_str());
425 ZYPP_THROW (Exception( "Can't open " + tmpdir.path().asString() + "/cookie" ) );
428 file << newstatus.checksum() << endl;
438 ZYPP_THROW(RepoUnknownTypeException());
441 // ok we have the metadata, now exchange
444 filesystem::assert_dir(rawpath);
445 filesystem::rename( rawpath, oldmetadata.path() );
446 // move the just downloaded there
447 filesystem::rename( tmpdir.path(), rawpath );
451 catch ( const Exception &e )
454 ERR << "Trying another url..." << endl;
457 ERR << "No more urls..." << endl;
458 ZYPP_THROW(RepoException("Cant refresh metadata"));
461 ////////////////////////////////////////////////////////////////////////////
463 void RepoManager::cleanMetadata( const RepoInfo &info,
464 const ProgressData::ReceiverFnc & progress )
466 filesystem::recursive_rmdir(rawcache_path_for_repoinfo(_pimpl->options, info));
469 void RepoManager::buildCache( const RepoInfo &info,
470 CacheBuildPolicy policy,
471 const ProgressData::ReceiverFnc & progressrcv )
473 ProgressData progress(100);
474 callback::SendReport<ProgressReport> report;
475 progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
476 progress.name(str::form(_("Building repository '%s' cache"), info.alias().c_str()));
480 Pathname rawpath = rawcache_path_for_repoinfo(_pimpl->options, info);
482 cache::CacheStore store(_pimpl->options.repoCachePath);
484 RepoStatus raw_metadata_status = rawMetadataStatus(info);
485 if ( store.isCached( info.alias() ) )
487 MIL << info.alias() << " is already cached." << endl;
488 data::RecordId id = store.lookupRepository(info.alias());
489 RepoStatus cache_status = store.repositoryStatus(id);
491 if ( cache_status.checksum() == raw_metadata_status.checksum() )
493 MIL << info.alias() << " cache is up to date with metadata." << endl;
494 if ( policy == BuildIfNeeded ) {
499 MIL << info.alias() << " cache rebuild is forced" << endl;
502 MIL << info.alias() << " cleaning cache..." << endl;
503 store.cleanRepository(id);
506 MIL << info.alias() << " building cache..." << endl;
507 data::RecordId id = store.lookupOrAppendRepository(info.alias());
509 repo::RepoType repokind = info.type();
511 // if the type is unknown, try probing.
512 switch ( repokind.toEnum() )
514 case RepoType::NONE_e:
515 // unknown, probe the local metadata
516 repokind = probe(rawpath.asUrl());
522 CombinedProgressData subprogrcv( progress, 100);
524 switch ( repokind.toEnum() )
526 case RepoType::RPMMD_e :
528 parser::yum::RepoParser parser(id, store, parser::yum::RepoParserOpts(), subprogrcv);
529 parser.parse(rawpath);
533 case RepoType::YAST2_e :
535 parser::susetags::RepoParser parser(id, store, subprogrcv);
536 parser.parse(rawpath);
540 case RepoType::RPMPLAINDIR_e :
542 InputStream is(rawpath + "cookie");
544 getline( is.stream(), buffer);
546 parser::plaindir::RepoParser parser(id, store, subprogrcv);
547 parser.parse(url.getPathName());
551 ZYPP_THROW(RepoUnknownTypeException());
554 // update timestamp and checksum
555 store.updateRepositoryStatus(id, raw_metadata_status);
557 MIL << "Commit cache.." << endl;
562 ////////////////////////////////////////////////////////////////////////////
564 repo::RepoType RepoManager::probe( const Url &url )
566 if ( url.getScheme() == "dir" && ! PathInfo( url.getPathName() ).isDir() )
568 // Handle non existing local directory in advance, as
569 // MediaSetAccess does not support it.
570 return repo::RepoType::NONE;
573 MediaSetAccess access(url);
574 if ( access.doesFileExist("/repodata/repomd.xml") )
575 return repo::RepoType::RPMMD;
576 if ( access.doesFileExist("/content") )
577 return repo::RepoType::YAST2;
579 // if it is a local url of type dir
580 if ( (! media::MediaManager::downloads(url)) && ( url.getScheme() == "dir" ) )
582 Pathname path = Pathname(url.getPathName());
583 if ( PathInfo(path).isDir() )
585 // allow empty dirs for now
586 return repo::RepoType::RPMPLAINDIR;
590 return repo::RepoType::NONE;
593 ////////////////////////////////////////////////////////////////////////////
595 void RepoManager::cleanCache( const RepoInfo &info,
596 const ProgressData::ReceiverFnc & progressrcv )
598 ProgressData progress(100);
599 callback::SendReport<ProgressReport> report;
600 progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
601 progress.name(str::form(_("Cleaning repository '%s' cache"), info.alias().c_str()));
604 cache::CacheStore store(_pimpl->options.repoCachePath);
606 data::RecordId id = store.lookupRepository(info.alias());
607 store.cleanRepository(id);
611 ////////////////////////////////////////////////////////////////////////////
613 bool RepoManager::isCached( const RepoInfo &info ) const
615 cache::CacheStore store(_pimpl->options.repoCachePath);
616 return store.isCached(info.alias());
619 Repository RepoManager::createFromCache( const RepoInfo &info,
620 const ProgressData::ReceiverFnc & progressrcv )
622 callback::SendReport<ProgressReport> report;
623 ProgressData progress;
624 progress.sendTo(ProgressReportAdaptor( progressrcv, report ));
625 progress.sendTo( progressrcv );
626 progress.name(str::form(_("Reading repository '%s' cache"), info.alias().c_str()));
629 cache::CacheStore store(_pimpl->options.repoCachePath);
631 if ( ! store.isCached( info.alias() ) )
632 ZYPP_THROW(RepoNotCachedException());
634 MIL << "Repository " << info.alias() << " is cached" << endl;
636 data::RecordId id = store.lookupRepository(info.alias());
638 repo::cached::RepoOptions opts( info, _pimpl->options.repoCachePath, id );
639 opts.readingResolvablesProgress = progressrcv;
640 repo::cached::RepoImpl::Ptr repoimpl =
641 new repo::cached::RepoImpl( opts );
643 repoimpl->resolvables();
644 // read the resolvables from cache
645 return Repository(repoimpl);
648 ////////////////////////////////////////////////////////////////////////////
651 * Generate a non existing filename in a directory, using a base
652 * name. For example if a directory contains 3 files
658 * If you try to generate a unique filename for this directory,
659 * based on "ruu" you will get "ruu", but if you use the base
660 * "foo" you will get "foo_1"
662 * \param dir Directory where the file needs to be unique
663 * \param basefilename string to base the filename on.
665 static Pathname generate_non_existing_name( const Pathname &dir,
666 const std::string &basefilename )
668 string final_filename = basefilename;
670 while ( PathInfo(dir + final_filename).isExist() )
672 final_filename = basefilename + "_" + str::numstring(counter);
675 return dir + Pathname(final_filename);
678 ////////////////////////////////////////////////////////////////////////////
680 void RepoManager::addRepository( const RepoInfo &info,
681 const ProgressData::ReceiverFnc & progressrcv )
685 ProgressData progress(100);
686 callback::SendReport<ProgressReport> report;
687 progress.sendTo( ProgressReportAdaptor( progressrcv, report ) );
688 progress.name(str::form(_("Adding repository '%s'"), info.alias().c_str()));
691 std::list<RepoInfo> repos = knownRepositories();
692 for ( std::list<RepoInfo>::const_iterator it = repos.begin();
696 if ( info.alias() == (*it).alias() )
697 ZYPP_THROW(RepoAlreadyExistsException(info.alias()));
702 // assert the directory exists
703 filesystem::assert_dir(_pimpl->options.knownReposPath);
705 Pathname repofile = generate_non_existing_name(_pimpl->options.knownReposPath,
706 Pathname(info.alias()).extend(".repo").asString());
707 // now we have a filename that does not exists
708 MIL << "Saving repo in " << repofile << endl;
710 std::ofstream file(repofile.c_str());
712 ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
715 info.dumpRepoOn(file);
717 MIL << "done" << endl;
720 void RepoManager::addRepositories( const Url &url,
721 const ProgressData::ReceiverFnc & progressrcv )
723 std::list<RepoInfo> knownrepos = knownRepositories();
724 std::list<RepoInfo> repos = readRepoFile(url);
725 for ( std::list<RepoInfo>::const_iterator it = repos.begin();
729 // look if the alias is in the known repos.
730 for ( std::list<RepoInfo>::const_iterator kit = knownrepos.begin();
731 kit != knownrepos.end();
734 if ( (*it).alias() == (*kit).alias() )
736 ERR << "To be added repo " << (*it).alias() << " conflicts with existing repo " << (*kit).alias() << endl;
737 ZYPP_THROW(RepoAlreadyExistsException((*it).alias()));
742 string filename = Pathname(url.getPathName()).basename();
744 if ( filename == Pathname() )
745 ZYPP_THROW(RepoException("Invalid repo file name at " + url.asString() ));
747 // assert the directory exists
748 filesystem::assert_dir(_pimpl->options.knownReposPath);
750 Pathname repofile = generate_non_existing_name(_pimpl->options.knownReposPath, filename);
751 // now we have a filename that does not exists
752 MIL << "Saving " << repos.size() << " repo" << ( repos.size() ? "s" : "" ) << " in " << repofile << endl;
754 std::ofstream file(repofile.c_str());
756 ZYPP_THROW (Exception( "Can't open " + repofile.asString() ) );
759 for ( std::list<RepoInfo>::const_iterator it = repos.begin();
763 MIL << "Saving " << (*it).alias() << endl;
764 (*it).dumpRepoOn(file);
766 MIL << "done" << endl;
769 ////////////////////////////////////////////////////////////////////////////
771 void RepoManager::removeRepository( const RepoInfo & info,
772 const ProgressData::ReceiverFnc & progressrcv)
774 MIL << "Going to delete repo " << info.alias() << endl;
776 std::list<RepoInfo> repos = knownRepositories();
777 for ( std::list<RepoInfo>::const_iterator it = repos.begin();
781 // they can be the same only if the provided is empty, that means
782 // the provided repo has no alias
784 if ( (!info.alias().empty()) && ( info.alias() != (*it).alias() ) )
789 // we have a matcing repository, now we need to know
790 // where it does come from.
791 RepoInfo todelete = *it;
792 if (todelete.filepath().empty())
794 ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
798 // figure how many repos are there in the file:
799 std::list<RepoInfo> filerepos = repositories_in_file(todelete.filepath());
800 if ( (filerepos.size() == 1) && ( filerepos.front().alias() == todelete.alias() ) )
802 // easy, only this one, just delete the file
803 if ( filesystem::unlink(todelete.filepath()) != 0 )
805 ZYPP_THROW(RepoException("Can't delete " + todelete.filepath().asString()));
807 MIL << todelete.alias() << " sucessfully deleted." << endl;
812 // there are more repos in the same file
813 // write them back except the deleted one.
815 //std::ofstream file(tmp.path().c_str());
817 // assert the directory exists
818 filesystem::assert_dir(todelete.filepath().dirname());
820 std::ofstream file(todelete.filepath().c_str());
822 //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
823 ZYPP_THROW (Exception( "Can't open " + todelete.filepath().asString() ) );
825 for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
826 fit != filerepos.end();
829 if ( (*fit).alias() != todelete.alias() )
830 (*fit).dumpRepoOn(file);
833 cache::CacheStore store(_pimpl->options.repoCachePath);
835 if ( store.isCached( todelete.alias() ) ) {
836 MIL << "repository was cached. cleaning cache" << endl;
837 store.cleanRepository(todelete.alias());
840 MIL << todelete.alias() << " sucessfully deleted." << endl;
843 } // else filepath is empty
846 // should not be reached on a sucess workflow
847 ZYPP_THROW(RepoNotFoundException(info));
850 ////////////////////////////////////////////////////////////////////////////
852 void RepoManager::modifyRepository( const std::string &alias,
853 const RepoInfo & newinfo,
854 const ProgressData::ReceiverFnc & progressrcv )
856 RepoInfo toedit = getRepositoryInfo(alias);
858 if (toedit.filepath().empty())
860 ZYPP_THROW(RepoException("Can't figure where the repo is stored"));
864 // figure how many repos are there in the file:
865 std::list<RepoInfo> filerepos = repositories_in_file(toedit.filepath());
867 // there are more repos in the same file
868 // write them back except the deleted one.
870 //std::ofstream file(tmp.path().c_str());
872 // assert the directory exists
873 filesystem::assert_dir(toedit.filepath().dirname());
875 std::ofstream file(toedit.filepath().c_str());
877 //ZYPP_THROW (Exception( "Can't open " + tmp.path().asString() ) );
878 ZYPP_THROW (Exception( "Can't open " + toedit.filepath().asString() ) );
880 for ( std::list<RepoInfo>::const_iterator fit = filerepos.begin();
881 fit != filerepos.end();
884 // if the alias is different, dump the original
885 // if it is the same, dump the provided one
886 if ( (*fit).alias() != toedit.alias() )
887 (*fit).dumpRepoOn(file);
889 newinfo.dumpRepoOn(file);
894 ////////////////////////////////////////////////////////////////////////////
896 RepoInfo RepoManager::getRepositoryInfo( const std::string &alias,
897 const ProgressData::ReceiverFnc & progressrcv )
899 std::list<RepoInfo> repos = knownRepositories();
900 for ( std::list<RepoInfo>::const_iterator it = repos.begin();
904 if ( (*it).alias() == alias )
908 info.setAlias(info.alias());
909 ZYPP_THROW(RepoNotFoundException(info));
912 ////////////////////////////////////////////////////////////////////////////
914 RepoInfo RepoManager::getRepositoryInfo( const Url & url,
915 const url::ViewOption & urlview,
916 const ProgressData::ReceiverFnc & progressrcv )
918 std::list<RepoInfo> repos = knownRepositories();
919 for ( std::list<RepoInfo>::const_iterator it = repos.begin();
923 for(RepoInfo::urls_const_iterator urlit = (*it).baseUrlsBegin();
924 urlit != (*it).baseUrlsEnd();
927 if ((*urlit).asString(urlview) == url.asString(urlview))
932 info.setAlias(info.alias());
933 info.setBaseUrl(url);
934 ZYPP_THROW(RepoNotFoundException(info));
937 ////////////////////////////////////////////////////////////////////////////
939 std::ostream & operator<<( std::ostream & str, const RepoManager & obj )
941 return str << *obj._pimpl;
944 /////////////////////////////////////////////////////////////////
946 ///////////////////////////////////////////////////////////////////