X-Git-Url: http://review.tizen.org/git/?a=blobdiff_plain;f=zypp%2FFetcher.cc;h=ce2f6140e55d6ce39b72810f6056c25b791f8417;hb=760e580b871de286a0397363be88ee50362ef887;hp=7f640a0e04c6a18fd791d3ea5127c4ec081257a6;hpb=e52c720eecf36a67a09d5cd863b590cfe0838ce0;p=platform%2Fupstream%2Flibzypp.git diff --git a/zypp/Fetcher.cc b/zypp/Fetcher.cc index 7f640a0..ce2f614 100644 --- a/zypp/Fetcher.cc +++ b/zypp/Fetcher.cc @@ -15,17 +15,20 @@ #include #include "zypp/base/Easy.h" -#include "zypp/base/Logger.h" +#include "zypp/base/LogControl.h" +#include "zypp/base/LogTools.h" #include "zypp/base/PtrTypes.h" #include "zypp/base/DefaultIntegral.h" #include "zypp/base/String.h" #include "zypp/Fetcher.h" +#include "zypp/ZYppFactory.h" #include "zypp/CheckSum.h" #include "zypp/base/UserRequestException.h" #include "zypp/parser/susetags/ContentFileReader.h" #include "zypp/parser/susetags/RepoIndex.h" -using namespace std; +#undef ZYPP_BASE_LOGGER_LOGGROUP +#define ZYPP_BASE_LOGGER_LOGGROUP "zypp:fetcher" /////////////////////////////////////////////////////////////////// namespace zypp @@ -40,13 +43,34 @@ namespace zypp { FetcherIndex( const OnMediaLocation &loc ) : location(loc) + {} + /** Index localtion. */ + OnMediaLocation location; + /** Whether we read this index. */ + DefaultIntegral read; + }; + + typedef shared_ptr FetcherIndex_Ptr; + + /** std::set ordering (less semantic) */ + struct SameFetcherIndex + { + bool operator()( const FetcherIndex_Ptr & lhs, const FetcherIndex_Ptr & rhs ) const { + if ( lhs == rhs ) + return false; // incl. NULL == NULL + if ( ! lhs ) + return true; // NULL < nonNULL + if ( ! rhs ) + return false; // nonNULL > NULL + // both nonNULL ==> compare medianr and path + if ( lhs->location.medianr() == rhs->location.medianr() ) + return lhs->location.filename() < rhs->location.filename(); + //else + return lhs->location.medianr() < rhs->location.medianr(); } - - OnMediaLocation location; }; - typedef shared_ptr FetcherIndex_Ptr; - + /** * Class to encapsulate the \ref OnMediaLocation object * and the \ref FileChecker together @@ -55,7 +79,7 @@ namespace zypp { enum Flag { - None = 0x0000, + None = 0x0000, Directory = 0x0001, Recursive = 0x0002, RecursiveDirectory = Directory | Recursive, @@ -64,26 +88,29 @@ namespace zypp AlwaysVerifyChecksum = 0x0004, }; ZYPP_DECLARE_FLAGS(Flags, Flag); - - FetcherJob( const OnMediaLocation &loc ) + + FetcherJob( const OnMediaLocation &loc, const Pathname dfile = Pathname()) : location(loc) + , deltafile(dfile) , flags(None) { //MIL << location << endl; } - + ~FetcherJob() { //MIL << location << " | * " << checkers.size() << endl; } OnMediaLocation location; + Pathname deltafile; //CompositeFileChecker checkers; - list checkers; + std::list checkers; Flags flags; }; - ZYPP_DECLARE_OPERATORS_FOR_FLAGS(FetcherJob::Flags); + + ZYPP_DECLARE_OPERATORS_FOR_FLAGS(FetcherJob::Flags); typedef shared_ptr FetcherJob_Ptr; std::ostream & operator<<( std::ostream & str, const FetcherJob_Ptr & obj ) @@ -91,7 +118,6 @@ namespace zypp return str << obj->location; } - /////////////////////////////////////////////////////////////////// // // CLASS NAME : Fetcher::Impl @@ -100,23 +126,22 @@ namespace zypp class Fetcher::Impl { friend std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj ); + public: - Impl(); - - ~Impl() { - MIL << endl; - } - - void setOptions( Fetcher::Options options ); - Fetcher::Options options() const; - - void addIndex( const OnMediaLocation &resource ); - - void enqueueDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() ); - void enqueueDigestedDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() ); - - void enqueue( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() ); - void enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() ); + Impl(); + + ~Impl() {} + + void setOptions( Fetcher::Options options ); + Fetcher::Options options() const; + + void addIndex( const OnMediaLocation &resource ); + + void enqueueDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() ); + void enqueueDigestedDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() ); + + void enqueue( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() ); + void enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker = FileChecker(), const Pathname &deltafile = Pathname() ); void addCachePath( const Pathname &cache_dir ); void reset(); void start( const Pathname &dest_dir, @@ -149,27 +174,26 @@ namespace zypp */ void readIndex( const Pathname &index, const Pathname &basedir ); - /** specific version of \ref readIndex for SHA1SUMS file */ - void readSha1sumsIndex( const Pathname &index, const Pathname &basedir ); - - /** specific version of \ref readIndex for SHA1SUMS file */ + /** specific version of \ref readIndex for CHECKSUMS file */ + void readChecksumsIndex( const Pathname &index, const Pathname &basedir ); + + /** specific version of \ref readIndex for content file */ void readContentFileIndex( const Pathname &index, const Pathname &basedir ); - + /** reads the content of a directory but keeps a cache **/ void getDirectoryContent( MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content ); - + /** - * tries to provide the file represented by job into dest_dir by - * looking at the cache. If success, returns true, and the desired - * file should be available on dest_dir + * Tries to locate the file represented by job by looking at + * the cache (matching checksum is mandatory). Returns the + * location of the cached file or an empty \ref Pathname. */ - bool provideFromCache( const OnMediaLocation &resource, const Pathname &dest_dir ); + Pathname locateInCache( const OnMediaLocation & resource_r, const Pathname & destDir_r ); /** - * Validates the job against is checkers, by using the file instance - * on dest_dir + * Validates the provided file against its checkers. * \throws Exception */ - void validate( const OnMediaLocation &resource, const Pathname &dest_dir, const list &checkers ); + void validate( const Pathname & localfile_r, const std::list & checkers_r ); /** * scan the directory and adds the individual jobs @@ -187,7 +211,7 @@ namespace zypp /** * Provide the resource to \ref dest_dir */ - void provideToDest( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir ); + void provideToDest( MediaSetAccess & media_r, const Pathname & destDir_r , const FetcherJob_Ptr & jobp_r ); private: friend Impl * rwcowClone( const Impl * rhs ); @@ -195,37 +219,37 @@ namespace zypp Impl * clone() const { return new Impl( *this ); } - list _resources; - list _indexes; - list _caches; + std::list _resources; + std::set _indexes; + std::set _caches; // checksums read from the indexes - map _checksums; + std::map _checksums; // cache of dir contents - map _dircontent; - - Fetcher::Options _options; + std::map _dircontent; + + Fetcher::Options _options; }; /////////////////////////////////////////////////////////////////// - void Fetcher::Impl::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker ) + void Fetcher::Impl::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker, const Pathname &deltafile ) { FetcherJob_Ptr job; - job.reset(new FetcherJob(resource)); + job.reset(new FetcherJob(resource, deltafile)); job->flags |= FetcherJob:: AlwaysVerifyChecksum; _resources.push_back(job); } - + Fetcher::Impl::Impl() : _options(0) { } - + void Fetcher::Impl::setOptions( Fetcher::Options options ) { _options = options; } - + Fetcher::Options Fetcher::Impl::options() const { return _options; } - + void Fetcher::Impl::enqueueDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker ) @@ -269,10 +293,8 @@ namespace zypp void Fetcher::Impl::addIndex( const OnMediaLocation &resource ) { - MIL << "adding index " << resource << endl; - FetcherIndex_Ptr index; - index.reset(new FetcherIndex(resource)); - _indexes.push_back(index); + MIL << "adding index " << resource << endl; + _indexes.insert(FetcherIndex_Ptr(new FetcherIndex(resource))); } @@ -292,7 +314,7 @@ namespace zypp if ( info.isDir() ) { DBG << "Adding fetcher cache: '" << cache_dir << "'." << endl; - _caches.push_back(cache_dir); + _caches.insert(cache_dir); } else { @@ -307,82 +329,48 @@ namespace zypp } - // tries to provide resource to dest_dir from any of the configured additional - // cache paths where the file may already be present. returns true if the - // file was provided from the cache. - bool Fetcher::Impl::provideFromCache( const OnMediaLocation &resource, const Pathname &dest_dir ) + Pathname Fetcher::Impl::locateInCache( const OnMediaLocation & resource_r, const Pathname & destDir_r ) { - Pathname dest_full_path = dest_dir + resource.filename(); + Pathname ret; + // No checksum - no match + if ( resource_r.checksum().empty() ) + return ret; // first check in the destination directory - if ( PathInfo(dest_full_path).isExist() ) + Pathname cacheLocation = destDir_r / resource_r.filename(); + if ( PathInfo(cacheLocation).isExist() && is_checksum( cacheLocation, resource_r.checksum() ) ) { - if ( is_checksum( dest_full_path, resource.checksum() ) - && (! resource.checksum().empty() ) ) - return true; + swap( ret, cacheLocation ); + return ret; } MIL << "start fetcher with " << _caches.size() << " cache directories." << endl; - for_ ( it_cache, _caches.begin(), _caches.end() ) + for( const Pathname & cacheDir : _caches ) { - // does the current file exists in the current cache? - Pathname cached_file = *it_cache + resource.filename(); - if ( PathInfo( cached_file ).isExist() ) + cacheLocation = cacheDir / resource_r.filename(); + if ( PathInfo(cacheLocation).isExist() && is_checksum( cacheLocation, resource_r.checksum() ) ) { - DBG << "File '" << cached_file << "' exist, testing checksum " << resource.checksum() << endl; - // check the checksum - if ( is_checksum( cached_file, resource.checksum() ) && (! resource.checksum().empty() ) ) - { - // cached - MIL << "file " << resource.filename() << " found in previous cache. Using cached copy." << endl; - // checksum is already checked. - // we could later implement double failover and try to download if file copy fails. - // replicate the complete path in the target directory - if( dest_full_path != cached_file ) - { - if ( assert_dir( dest_full_path.dirname() ) != 0 ) - ZYPP_THROW( Exception("Can't create " + dest_full_path.dirname().asString())); - - if ( filesystem::hardlink(cached_file, dest_full_path ) != 0 ) - { - WAR << "Can't hardlink '" << cached_file << "' to '" << dest_dir << "'. Trying copying." << endl; - if ( filesystem::copy(cached_file, dest_full_path ) != 0 ) - { - ERR << "Can't copy " << cached_file + " to " + dest_dir << endl; - // try next cache - continue; - } - } - } - // found in cache - return true; - } + MIL << "file " << resource_r.filename() << " found in cache " << cacheDir << endl; + swap( ret, cacheLocation ); + return ret; } - } // iterate over caches - return false; + } + + return ret; } - - void Fetcher::Impl::validate( const OnMediaLocation &resource, const Pathname &dest_dir, const list &checkers ) + + void Fetcher::Impl::validate( const Pathname & localfile_r, const std::list & checkers_r ) { - // no matter where did we got the file, try to validate it: - Pathname localfile = dest_dir + resource.filename(); - // call the checker function try { - MIL << "Checking job [" << localfile << "] (" << checkers.size() << " checkers )" << endl; + MIL << "Checking job [" << localfile_r << "] (" << checkers_r.size() << " checkers )" << endl; - for ( list::const_iterator it = checkers.begin(); - it != checkers.end(); - ++it ) + for ( const FileChecker & chkfnc : checkers_r ) { - if (*it) - { - (*it)(localfile); - } + if ( chkfnc ) + chkfnc( localfile_r ); else - { - ERR << "Invalid checker for '" << localfile << "'" << endl; - } + ERR << "Invalid checker for '" << localfile_r << "'" << endl; } } @@ -396,7 +384,7 @@ namespace zypp } catch (...) { - ZYPP_THROW(Exception("Unknown error while validating " + resource.filename().asString())); + ZYPP_THROW(Exception("Unknown error while validating " + localfile_r.asString())); } } @@ -405,35 +393,26 @@ namespace zypp const OnMediaLocation &resource, const Pathname &dest_dir ) { - if ( _options & AutoAddSha1sumsIndexes ) + auto fnc_addIfInContent( [&]( const std::string & index_r ) -> bool { - // only try to add an index if it exists - filesystem::DirEntry shafile; - shafile.name = "SHA1SUMS"; shafile.type = filesystem::FT_FILE; - if ( find( content.begin(), content.end(), shafile ) != content.end() ) - { - // add the index of this directory - OnMediaLocation indexloc(resource); - indexloc.changeFilename(resource.filename() + "SHA1SUMS"); - addIndex(indexloc); - // we need to read it now - downloadAndReadIndexList(media, dest_dir); - } + if ( find( content.begin(), content.end(), filesystem::DirEntry(index_r,filesystem::FT_FILE) ) == content.end() ) + return false; + // add the index of this directory + OnMediaLocation indexloc( resource ); + indexloc.changeFilename( resource.filename() + index_r ); + addIndex( indexloc ); + // we need to read it now + downloadAndReadIndexList( media, dest_dir ); + return true; + } ); + + if ( _options & AutoAddChecksumsIndexes ) + { + fnc_addIfInContent( "CHECKSUMS" ) || fnc_addIfInContent( "SHA1SUMS" ); } if ( _options & AutoAddContentFileIndexes ) { - // only try to add an index if it exists - filesystem::DirEntry contentfile; - contentfile.name = "content"; contentfile.type = filesystem::FT_FILE; - if ( find( content.begin(), content.end(), contentfile ) != content.end() ) - { - // add the index of this directory - OnMediaLocation indexloc(resource); - indexloc.changeFilename(resource.filename() + "content"); - addIndex(indexloc); - // we need to read it now - downloadAndReadIndexList(media, dest_dir); - } + fnc_addIfInContent( "content" ); } } @@ -441,11 +420,11 @@ namespace zypp const OnMediaLocation &resource, filesystem::DirContent &content ) { - if ( _dircontent.find(resource.filename().asString()) + if ( _dircontent.find(resource.filename().asString()) != _dircontent.end() ) { filesystem::DirContent filled(_dircontent[resource.filename().asString()]); - + std::copy(filled.begin(), filled.end(), std::back_inserter(content)); } else @@ -459,7 +438,7 @@ namespace zypp _dircontent[resource.filename().asString()] = tofill; } } - + void Fetcher::Impl::addDirJobs( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir, FetcherJob::Flags flags ) @@ -468,18 +447,26 @@ namespace zypp // individual transfer jobs MIL << "Adding directory " << resource.filename() << endl; filesystem::DirContent content; - getDirectoryContent(media, resource, content); - + try { + getDirectoryContent(media, resource, content); + } + catch ( media::MediaFileNotFoundException & exception ) + { + ZYPP_CAUGHT( exception ); + WAR << "Skiping subtree hidden at " << resource.filename() << endl; + return; + } + // this method test for the option flags so indexes are added // only if the options are enabled autoaddIndexes(content, media, resource, dest_dir); - + for ( filesystem::DirContent::const_iterator it = content.begin(); it != content.end(); ++it ) { - // skip SHA1SUMS* as they were already retrieved - if ( str::hasPrefix(it->name, "SHA1SUMS") ) + // skip CHECKSUMS* as they were already retrieved + if ( str::hasPrefix(it->name, "CHECKSUMS") || str::hasPrefix(it->name, "SHA1SUMS") ) continue; Pathname filename = resource.filename() + it->name; @@ -489,7 +476,7 @@ namespace zypp case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all case filesystem::FT_FILE: { - CheckSum chksm(resource.checksum()); + CheckSum chksm(resource.checksum()); if ( _checksums.find(filename.asString()) != _checksums.end() ) { // the checksum can be replaced with the one in the index. @@ -516,58 +503,58 @@ namespace zypp } } - void Fetcher::Impl::provideToDest( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir ) + void Fetcher::Impl::provideToDest( MediaSetAccess & media_r, const Pathname & destDir_r , const FetcherJob_Ptr & jobp_r ) { - bool got_from_cache = false; + const OnMediaLocation & resource( jobp_r->location ); - // start look in cache - got_from_cache = provideFromCache(resource, dest_dir); - - if ( ! got_from_cache ) + try { - MIL << "Not found in cache, downloading" << endl; + scoped_ptr releaseFileGuard; // will take care provided files get released - // try to get the file from the net - try + // get cached file (by checksum) or provide from media + Pathname tmpFile = locateInCache( resource, destDir_r ); + if ( tmpFile.empty() ) { - Pathname tmp_file = media.provideFile(resource); - Pathname dest_full_path = dest_dir + resource.filename(); - if ( assert_dir( dest_full_path.dirname() ) != 0 ) - ZYPP_THROW( Exception("Can't create " + dest_full_path.dirname().asString())); - if ( filesystem::copy(tmp_file, dest_full_path ) != 0 ) - { - if ( ! PathInfo(tmp_file).isExist() ) - ERR << tmp_file << " does not exist" << endl; - if ( ! PathInfo(dest_full_path.dirname()).isExist() ) - ERR << dest_full_path.dirname() << " does not exist" << endl; + MIL << "Not found in cache, retrieving..." << endl; + tmpFile = media_r.provideFile( resource, resource.optional() ? MediaSetAccess::PROVIDE_NON_INTERACTIVE : MediaSetAccess::PROVIDE_DEFAULT, jobp_r->deltafile ); + releaseFileGuard.reset( new MediaSetAccess::ReleaseFileGuard( media_r, resource ) ); // release it when we leave the block + } - media.releaseFile(resource); //not needed anymore, only eat space - ZYPP_THROW( Exception("Can't copy " + tmp_file.asString() + " to " + dest_dir.asString())); - } + // The final destination: locateInCache also checks destFullPath! + // If we find a cache match (by checksum) at destFullPath, take + // care it gets deleted, in case the validation fails. + ManagedFile destFullPath( destDir_r / resource.filename() ); + if ( tmpFile == destFullPath ) + destFullPath.setDispose( filesystem::unlink ); - media.releaseFile(resource); //not needed anymore, only eat space - } - catch (Exception & excpt_r) + // validate the file (throws if not valid) + validate( tmpFile, jobp_r->checkers ); + + // move it to the final destination + if ( tmpFile == destFullPath ) + destFullPath.resetDispose(); // keep it! + else { - ZYPP_CAUGHT(excpt_r); - excpt_r.remember("Can't provide " + resource.filename().asString() + " : " + excpt_r.msg()); + if ( assert_dir( destFullPath->dirname() ) != 0 ) + ZYPP_THROW( Exception( "Can't create " + destFullPath->dirname().asString() ) ); - if ( resource.optional() ) - { - WAR << "optional resource " << resource << " could not be transfered" << endl; - return; - } - else - { - ZYPP_RETHROW(excpt_r); - } + if ( filesystem::hardlinkCopy( tmpFile, destFullPath ) != 0 ) + ZYPP_THROW( Exception( "Can't hardlink/copy " + tmpFile.asString() + " to " + destDir_r.asString() ) ); } } - else + catch ( Exception & excpt ) { - // We got the file from cache - // continue with next file - return; + if ( resource.optional() ) + { + ZYPP_CAUGHT( excpt ); + WAR << "optional resource " << resource << " could not be transferred." << endl; + return; + } + else + { + excpt.remember( "Can't provide " + resource.filename().asString() ); + ZYPP_RETHROW( excpt ); + } } } @@ -578,7 +565,7 @@ namespace zypp { setRepoIndexConsumer( bind( &ContentReaderHelper::consumeIndex, this, _1 ) ); } - + void consumeIndex( const parser::susetags::RepoIndex_Ptr & data_r ) { _repoindex = data_r; } @@ -588,8 +575,8 @@ namespace zypp // generic function for reading indexes void Fetcher::Impl::readIndex( const Pathname &index, const Pathname &basedir ) { - if ( index.basename() == "SHA1SUMS" ) - readSha1sumsIndex(index, basedir); + if ( index.basename() == "CHECKSUMS" || index.basename() == "SHA1SUMS" ) + readChecksumsIndex(index, basedir); else if ( index.basename() == "content" ) readContentFileIndex(index, basedir); else @@ -609,39 +596,46 @@ namespace zypp } } - // reads a SHA1SUMS file index - void Fetcher::Impl::readSha1sumsIndex( const Pathname &index, const Pathname &basedir ) + // reads a CHECKSUMS (old SHA1SUMS) file index + void Fetcher::Impl::readChecksumsIndex( const Pathname &index, const Pathname &basedir ) { std::ifstream in( index.c_str() ); - string buffer; if ( ! in.fail() ) { - while ( getline(in, buffer) ) + std::string buffer; + while ( getline( in, buffer ) ) { - vector words; - str::split( buffer, back_inserter(words) ); - if ( words.size() != 2 ) - ZYPP_THROW(Exception("Wrong format for SHA1SUMS file")); - //MIL << "check: '" << words[0] << "' | '" << words[1] << "'" << endl; - if ( ! words[1].empty() ) - _checksums[(basedir + words[1]).asString()] = CheckSum::sha1(words[0]); + + if ( buffer[0] == '#' ) + continue; // simple comment + + CheckSum checksum( str::stripFirstWord( buffer, /*ltrim before strip*/true ) ); + if ( checksum.empty() ) + continue; // empty line | unknown cheksum format + + if ( buffer.empty() ) + { + WAR << "Missing filename in CHECKSUMS file: " << index.asString() << " (" << checksum << ")" << endl; + continue; + } + + _checksums[(basedir/buffer).asString()] = checksum; } } else - ZYPP_THROW(Exception("Can't open SHA1SUMS file: " + index.asString())); + ZYPP_THROW(Exception("Can't open CHECKSUMS file: " + index.asString())); } void Fetcher::Impl::downloadIndex( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir) { MIL << "downloading index " << resource << endl; - // create a new fetcher with a different state to transfer the // file containing checksums and its signature Fetcher fetcher; // signature checker for index. We havent got the signature from // the nextwork yet. SignatureFileChecker sigchecker; - + // build the name of the index and the signature OnMediaLocation idxloc(resource); OnMediaLocation sigloc(resource); @@ -651,27 +645,33 @@ namespace zypp // the checking will warn later sigloc.setOptional(true); keyloc.setOptional(true); - + // calculate signature and key name sigloc.changeFilename( sigloc.filename().extend(".asc") ); keyloc.changeFilename( keyloc.filename().extend(".key") ); - + //assert_dir(dest_dir + idxloc.filename().dirname()); - + // transfer the signature fetcher.enqueue(sigloc); fetcher.start( dest_dir, media ); // if we get the signature, update the checker if ( PathInfo(dest_dir + sigloc.filename()).isExist() ) sigchecker = SignatureFileChecker(dest_dir + sigloc.filename()); - + fetcher.reset(); - + // now the key fetcher.enqueue(keyloc); fetcher.start( dest_dir, media ); fetcher.reset(); - + + // try to import the key + if ( PathInfo(dest_dir + keyloc.filename()).isExist() ) + getZYpp()->keyRing()->importKey(PublicKey(dest_dir + keyloc.filename()), false); + else + WAR << "No public key specified by user for index '" << keyloc.filename() << "'"<< endl; + // now the index itself fetcher.enqueue( idxloc, FileChecker(sigchecker) ); fetcher.start( dest_dir, media ); @@ -691,16 +691,26 @@ namespace zypp return; } - for ( list::const_iterator it_idx = _indexes.begin(); - it_idx != _indexes.end(); ++it_idx ) + for_( it_idx, _indexes.begin(), _indexes.end() ) { + if ( (*it_idx)->read ) + { + DBG << "Already read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl; + } + else + { + // base::LogControl::TmpLineWriter shutUp; downloadIndex( media, (*it_idx)->location, dest_dir ); // now we have the indexes in dest_dir readIndex( dest_dir + (*it_idx)->location.filename(), (*it_idx)->location.filename().dirname() ); + // Take care we don't process it again + MIL << "Remember read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl; + (*it_idx)->read = true; + } } MIL << "done reading indexes" << endl; } - + // start processing all fetcher jobs. // it processes any user pointed index first void Fetcher::Impl::start( const Pathname &dest_dir, @@ -712,34 +722,33 @@ namespace zypp downloadAndReadIndexList(media, dest_dir); - for ( list::const_iterator it_res = _resources.begin(); it_res != _resources.end(); ++it_res ) + for ( const FetcherJob_Ptr & jobp : _resources ) { - - if ( (*it_res)->flags & FetcherJob::Directory ) + if ( jobp->flags & FetcherJob::Directory ) { - const OnMediaLocation location((*it_res)->location); - addDirJobs(media, location, dest_dir, (*it_res)->flags); + const OnMediaLocation location(jobp->location); + addDirJobs(media, location, dest_dir, jobp->flags); continue; } // may be this code can be factored out // together with the autodiscovery of indexes // of addDirJobs - if ( ( _options & AutoAddSha1sumsIndexes ) || + if ( ( _options & AutoAddChecksumsIndexes ) || ( _options & AutoAddContentFileIndexes ) ) { // if auto indexing is enabled, then we need to read the // index for each file. We look only in the directory // where the file is. this is expensive of course. filesystem::DirContent content; - getDirectoryContent(media, (*it_res)->location.filename().dirname(), content); + getDirectoryContent(media, jobp->location.filename().dirname(), content); // this method test for the option flags so indexes are added // only if the options are enabled MIL << "Autodiscovering signed indexes on '" - << (*it_res)->location.filename().dirname() << "' for '" - << (*it_res)->location.filename() << "'" << endl; - - autoaddIndexes(content, media, (*it_res)->location.filename().dirname(), dest_dir); + << jobp->location.filename().dirname() << "' for '" + << jobp->location.filename() << "'" << endl; + + autoaddIndexes(content, media, jobp->location.filename().dirname(), dest_dir); // also look in the root of the media content.clear(); @@ -748,50 +757,44 @@ namespace zypp // only if the options are enabled MIL << "Autodiscovering signed indexes on '" << "/" << "' for '" - << (*it_res)->location.filename() << "'" << endl; - - autoaddIndexes(content, media, Pathname("/"), dest_dir); - } + << jobp->location.filename() << "'" << endl; - provideToDest(media, (*it_res)->location, dest_dir); - - // if the file was not transfered, and no exception, just - // return, as it was an optional file - if ( ! PathInfo(dest_dir + (*it_res)->location.filename()).isExist() ) - return; + autoaddIndexes(content, media, Pathname("/"), dest_dir); + } // if the checksum is empty, but the checksum is in one of the // indexes checksum, then add a checker - if ( (*it_res)->location.checksum().empty() ) + if ( jobp->location.checksum().empty() ) { - if ( _checksums.find((*it_res)->location.filename().asString()) + if ( _checksums.find(jobp->location.filename().asString()) != _checksums.end() ) { - CheckSum chksm = _checksums[(*it_res)->location.filename().asString()]; - ChecksumFileChecker digest_check(chksm); - (*it_res)->checkers.push_back(digest_check); + CheckSum chksm = _checksums[jobp->location.filename().asString()]; + ChecksumFileChecker digest_check(chksm); + jobp->checkers.push_back(digest_check); } else { // if the index checksum is empty too, we only add the checker // if the AlwaysVerifyChecksum option is set on - if ( (*it_res)->flags & FetcherJob::AlwaysVerifyChecksum ) + if ( jobp->flags & FetcherJob::AlwaysVerifyChecksum ) { // add the checker with the empty checksum - ChecksumFileChecker digest_check((*it_res)->location.checksum()); - (*it_res)->checkers.push_back(digest_check); + ChecksumFileChecker digest_check(jobp->location.checksum()); + jobp->checkers.push_back(digest_check); } } } else { // checksum is not empty, so add a checksum checker - ChecksumFileChecker digest_check((*it_res)->location.checksum()); - (*it_res)->checkers.push_back(digest_check); + ChecksumFileChecker digest_check(jobp->location.checksum()); + jobp->checkers.push_back(digest_check); } - - // validate job, this throws if not valid - validate((*it_res)->location, dest_dir, (*it_res)->checkers); + + // Provide and validate the file. If the file was not transferred + // and no exception was thrown, it was an optional file. + provideToDest( media, dest_dir, jobp ); if ( ! progress.incr() ) ZYPP_THROW(AbortRequestException()); @@ -801,7 +804,7 @@ namespace zypp /** \relates Fetcher::Impl Stream output */ inline std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj ) { - for ( list::const_iterator it_res = obj._resources.begin(); it_res != obj._resources.end(); ++it_res ) + for ( std::list::const_iterator it_res = obj._resources.begin(); it_res != obj._resources.end(); ++it_res ) { str << *it_res; } @@ -819,15 +822,15 @@ namespace zypp { _pimpl->setOptions(options); } - + Fetcher::Options Fetcher::options() const { return _pimpl->options(); } - void Fetcher::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker ) + void Fetcher::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker, const Pathname &deltafile ) { - _pimpl->enqueueDigested(resource, checker); + _pimpl->enqueueDigested(resource, checker, deltafile); } void Fetcher::enqueueDir( const OnMediaLocation &resource,