1 /*---------------------------------------------------------------------\
3 | |__ / \ / / . \ . \ |
8 \---------------------------------------------------------------------*/
9 /** \file zypp/Fetcher.cc
17 #include "zypp/base/Easy.h"
18 #include "zypp/base/Logger.h"
19 #include "zypp/base/PtrTypes.h"
20 #include "zypp/base/DefaultIntegral.h"
21 #include "zypp/base/String.h"
22 #include "zypp/Fetcher.h"
23 #include "zypp/CheckSum.h"
24 #include "zypp/base/UserRequestException.h"
25 #include "zypp/parser/susetags/ContentFileReader.h"
26 #include "zypp/parser/susetags/RepoIndex.h"
30 ///////////////////////////////////////////////////////////////////
32 { /////////////////////////////////////////////////////////////////
35 * class that represents indexes which add metadata
36 * to fetcher jobs and therefore need to be retrieved
41 FetcherIndex( const OnMediaLocation &loc )
46 OnMediaLocation location;
48 typedef shared_ptr<FetcherIndex> FetcherIndex_Ptr;
51 * Class to encapsulate the \ref OnMediaLocation object
52 * and the \ref FileChecker together
56 FetcherJob( const OnMediaLocation &loc )
61 //MIL << location << endl;
66 //MIL << location << " | * " << checkers.size() << endl;
69 OnMediaLocation location;
70 //CompositeFileChecker checkers;
71 list<FileChecker> checkers;
76 typedef shared_ptr<FetcherJob> FetcherJob_Ptr;
78 std::ostream & operator<<( std::ostream & str, const FetcherJob_Ptr & obj )
80 return str << obj->location;
84 ///////////////////////////////////////////////////////////////////
86 // CLASS NAME : Fetcher::Impl
88 /** Fetcher implementation. */
91 friend std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj );
99 void setOptions( Fetcher::Options options );
100 Fetcher::Options options() const;
102 void addIndex( const OnMediaLocation &resource );
103 void enqueue( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() );
104 void enqueueDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
105 void enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() );
106 void addCachePath( const Pathname &cache_dir );
108 void start( const Pathname &dest_dir,
109 MediaSetAccess &media,
110 const ProgressData::ReceiverFnc & progress_receiver );
112 /** Offer default Impl. */
113 static shared_ptr<Impl> nullimpl()
115 static shared_ptr<Impl> _nullimpl( new Impl );
120 * download the indexes and reads them
122 void readIndexes( MediaSetAccess &media, const Pathname &dest_dir);
125 * reads a downloaded index file and updates internal
128 * The index lists files relative to a directory, which is
129 * normally the same as the index file is located.
131 void readIndex( const Pathname &index, const Pathname &basedir );
133 /** specific version of \ref readIndex for SHA1SUMS file */
134 void readSha1sumsIndex( const Pathname &index, const Pathname &basedir );
136 /** specific version of \ref readIndex for SHA1SUMS file */
137 void readContentFileIndex( const Pathname &index, const Pathname &basedir );
140 * tries to provide the file represented by job into dest_dir by
141 * looking at the cache. If success, returns true, and the desired
142 * file should be available on dest_dir
144 bool provideFromCache( const OnMediaLocation &resource, const Pathname &dest_dir );
146 * Validates the job against is checkers, by using the file instance
150 void validate( const OnMediaLocation &resource, const Pathname &dest_dir, const list<FileChecker> &checkers );
153 * scan the directory and adds the individual jobs
155 void addDirJobs( MediaSetAccess &media, const OnMediaLocation &resource,
156 const Pathname &dest_dir, bool recursive );
158 * Provide the resource to \ref dest_dir
160 void provideToDest( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir );
163 friend Impl * rwcowClone<Impl>( const Impl * rhs );
164 /** clone for RWCOW_pointer */
166 { return new Impl( *this ); }
168 list<FetcherJob_Ptr> _resources;
169 list<FetcherIndex_Ptr> _indexes;
170 list<Pathname> _caches;
171 // checksums read from the indexes
172 map<string, CheckSum> _checksums;
173 Fetcher::Options _options;
175 ///////////////////////////////////////////////////////////////////
177 void Fetcher::Impl::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker )
180 job.reset(new FetcherJob(resource));
181 ChecksumFileChecker digest_check(resource.checksum());
182 job->checkers.push_back(digest_check);
184 job->checkers.push_back(checker);
185 _resources.push_back(job);
188 Fetcher::Impl::Impl()
193 void Fetcher::Impl::setOptions( Fetcher::Options options )
194 { _options = options; }
196 Fetcher::Options Fetcher::Impl::options() const
199 void Fetcher::Impl::enqueueDir( const OnMediaLocation &resource,
201 const FileChecker &checker )
204 job.reset(new FetcherJob(resource));
206 job->checkers.push_back(checker);
207 job->directory = true;
208 job->recursive = recursive;
209 _resources.push_back(job);
212 void Fetcher::Impl::enqueue( const OnMediaLocation &resource, const FileChecker &checker )
215 job.reset(new FetcherJob(resource));
217 job->checkers.push_back(checker);
218 _resources.push_back(job);
221 void Fetcher::Impl::addIndex( const OnMediaLocation &resource )
223 MIL << "adding index " << resource << endl;
224 FetcherIndex_Ptr index;
225 index.reset(new FetcherIndex(resource));
226 _indexes.push_back(index);
230 void Fetcher::Impl::reset()
236 void Fetcher::Impl::addCachePath( const Pathname &cache_dir )
238 PathInfo info(cache_dir);
239 if ( info.isExist() )
243 DBG << "Adding fetcher cache: '" << cache_dir << "'." << endl;
244 _caches.push_back(cache_dir);
248 // don't add bad cache directory, just log the error
249 ERR << "Not adding cache: '" << cache_dir << "'. Not a directory." << endl;
254 ERR << "Not adding cache '" << cache_dir << "'. Path does not exists." << endl;
259 bool Fetcher::Impl::provideFromCache( const OnMediaLocation &resource, const Pathname &dest_dir )
261 Pathname dest_full_path = dest_dir + resource.filename();
263 // first check in the destination directory
264 if ( PathInfo(dest_full_path).isExist() )
266 if ( is_checksum( dest_full_path, resource.checksum() )
267 && (! resource.checksum().empty() ) )
271 MIL << "start fetcher with " << _caches.size() << " cache directories." << endl;
272 for_ ( it_cache, _caches.begin(), _caches.end() )
274 // does the current file exists in the current cache?
275 Pathname cached_file = *it_cache + resource.filename();
276 if ( PathInfo( cached_file ).isExist() )
278 DBG << "File '" << cached_file << "' exist, testing checksum " << resource.checksum() << endl;
279 // check the checksum
280 if ( is_checksum( cached_file, resource.checksum() ) && (! resource.checksum().empty() ) )
283 MIL << "file " << resource.filename() << " found in previous cache. Using cached copy." << endl;
284 // checksum is already checked.
285 // we could later implement double failover and try to download if file copy fails.
286 // replicate the complete path in the target directory
287 if( dest_full_path != cached_file )
289 if ( assert_dir( dest_full_path.dirname() ) != 0 )
290 ZYPP_THROW( Exception("Can't create " + dest_full_path.dirname().asString()));
292 if ( filesystem::hardlink(cached_file, dest_full_path ) != 0 )
294 WAR << "Can't hardlink '" << cached_file << "' to '" << dest_dir << "'. Trying copying." << endl;
295 if ( filesystem::copy(cached_file, dest_full_path ) != 0 )
297 ERR << "Can't copy " << cached_file + " to " + dest_dir << endl;
307 } // iterate over caches
311 void Fetcher::Impl::validate( const OnMediaLocation &resource, const Pathname &dest_dir, const list<FileChecker> &checkers )
313 // no matter where did we got the file, try to validate it:
314 Pathname localfile = dest_dir + resource.filename();
315 // call the checker function
318 MIL << "Checking job [" << localfile << "] (" << checkers.size() << " checkers )" << endl;
320 for ( list<FileChecker>::const_iterator it = checkers.begin();
321 it != checkers.end();
330 ERR << "Invalid checker for '" << localfile << "'" << endl;
335 catch ( const FileCheckException &e )
339 catch ( const Exception &e )
345 ZYPP_THROW(Exception("Unknown error while validating " + resource.filename().asString()));
349 void Fetcher::Impl::addDirJobs( MediaSetAccess &media,
350 const OnMediaLocation &resource,
351 const Pathname &dest_dir, bool recursive )
353 // first get the content of the directory so we can add
354 // individual transfer jobs
355 MIL << "Adding directory " << resource.filename() << endl;
356 filesystem::DirContent content;
357 media.dirInfo( content, resource.filename(), false /* dots */, resource.medianr());
359 // only try to add an index if it exists
360 filesystem::DirEntry shafile;
361 shafile.name = "SHA1SUMS"; shafile.type = filesystem::FT_FILE;
362 if ( find( content.begin(), content.end(), shafile ) != content.end() )
364 // add the index of this directory
365 OnMediaLocation indexloc(resource);
366 indexloc.changeFilename(resource.filename() + "SHA1SUMS");
368 // we need to read it now
369 readIndexes(media, dest_dir);
372 for ( filesystem::DirContent::const_iterator it = content.begin();
376 // skip SHA1SUMS* as they were already retrieved
377 if ( str::hasPrefix(it->name, "SHA1SUMS") )
380 Pathname filename = resource.filename() + it->name;
384 case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
385 case filesystem::FT_FILE:
387 CheckSum chksm(resource.checksum());
388 if ( _checksums.find(filename.asString()) != _checksums.end() )
390 // the checksum can be replaced with the one in the index.
391 chksm = _checksums[filename.asString()];
392 //MIL << "resource " << filename << " has checksum in the index file." << endl;
395 WAR << "Resource " << filename << " has no checksum in the index either." << endl;
397 enqueueDigested(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm));
400 case filesystem::FT_DIR: // newer directory.yast contain at least directory info
402 addDirJobs(media, filename, dest_dir, recursive);
405 // don't provide devices, sockets, etc.
411 void Fetcher::Impl::provideToDest( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir )
413 bool got_from_cache = false;
415 // start look in cache
416 got_from_cache = provideFromCache(resource, dest_dir);
418 if ( ! got_from_cache )
420 MIL << "Not found in cache, downloading" << endl;
422 // try to get the file from the net
425 Pathname tmp_file = media.provideFile(resource);
426 Pathname dest_full_path = dest_dir + resource.filename();
427 if ( assert_dir( dest_full_path.dirname() ) != 0 )
428 ZYPP_THROW( Exception("Can't create " + dest_full_path.dirname().asString()));
429 if ( filesystem::copy(tmp_file, dest_full_path ) != 0 )
431 ZYPP_THROW( Exception("Can't copy " + tmp_file.asString() + " to " + dest_dir.asString()));
434 media.releaseFile(resource); //not needed anymore, only eat space
436 catch (Exception & excpt_r)
438 ZYPP_CAUGHT(excpt_r);
439 excpt_r.remember("Can't provide " + resource.filename().asString() + " : " + excpt_r.msg());
440 ZYPP_RETHROW(excpt_r);
445 // We got the file from cache
446 // continue with next file
451 // helper class to consume a content file
452 struct ContentReaderHelper : public parser::susetags::ContentFileReader
454 ContentReaderHelper()
456 setRepoIndexConsumer( bind( &ContentReaderHelper::consumeIndex, this, _1 ) );
459 void consumeIndex( const parser::susetags::RepoIndex_Ptr & data_r )
460 { _repoindex = data_r; }
462 parser::susetags::RepoIndex_Ptr _repoindex;
465 // generic function for reading indexes
466 void Fetcher::Impl::readIndex( const Pathname &index, const Pathname &basedir )
468 if ( index.basename() == "SHA1SUMS" )
469 readSha1sumsIndex(index, basedir);
470 else if ( index.basename() == "content" )
471 readContentFileIndex(index, basedir);
473 WAR << index << ": index file format not known" << endl;
476 // reads a content file index
477 void Fetcher::Impl::readContentFileIndex( const Pathname &index, const Pathname &basedir )
479 ContentReaderHelper reader;
481 MIL << index << " contains " << reader._repoindex->mediaFileChecksums.size() << " checksums." << endl;
482 for_( it, reader._repoindex->mediaFileChecksums.begin(), reader._repoindex->mediaFileChecksums.end() )
484 // content file entries don't start with /
485 _checksums[(basedir + it->first).asString()] = it->second;
489 // reads a SHA1SUMS file index
490 void Fetcher::Impl::readSha1sumsIndex( const Pathname &index, const Pathname &basedir )
492 std::ifstream in( index.c_str() );
496 while ( getline(in, buffer) )
498 vector<string> words;
499 str::split( buffer, back_inserter(words) );
500 if ( words.size() != 2 )
501 ZYPP_THROW(Exception("Wrong format for SHA1SUMS file"));
502 //MIL << "check: '" << words[0] << "' | '" << words[1] << "'" << endl;
503 if ( ! words[1].empty() )
504 _checksums[(basedir + words[1]).asString()] = CheckSum::sha1(words[0]);
508 ZYPP_THROW(Exception("Can't open SHA1SUMS file: " + index.asString()));
511 void Fetcher::Impl::readIndexes( MediaSetAccess &media, const Pathname &dest_dir)
513 // if there is no indexes, then just return to avoid
514 // the directory listing
515 if ( _indexes.empty() )
517 MIL << "No indexes to read." << endl;
521 // create a new fetcher with a different state to transfer the
522 // file containing checksums and its signature
524 // signature checker for index. We havent got the signature from
526 SignatureFileChecker sigchecker;
528 for ( list<FetcherIndex_Ptr>::const_iterator it_idx = _indexes.begin();
529 it_idx != _indexes.end(); ++it_idx )
531 MIL << "reading index " << (*it_idx)->location << endl;
532 // build the name of the index and the signature
533 OnMediaLocation idxloc((*it_idx)->location);
534 OnMediaLocation sigloc((*it_idx)->location.setOptional(true));
535 OnMediaLocation keyloc((*it_idx)->location.setOptional(true));
537 // calculate signature and key name
538 sigloc.changeFilename( sigloc.filename().extend(".asc") );
539 keyloc.changeFilename( keyloc.filename().extend(".key") );
541 //assert_dir(dest_dir + idxloc.filename().dirname());
543 // transfer the signature
544 fetcher.enqueue(sigloc);
545 fetcher.start( dest_dir, media );
546 // if we get the signature, update the checker
547 sigchecker = SignatureFileChecker(dest_dir + sigloc.filename());
551 fetcher.enqueue(keyloc);
552 fetcher.start( dest_dir, media );
555 // now the index itself
556 fetcher.enqueue( idxloc, FileChecker(sigchecker) );
557 fetcher.start( dest_dir, media );
560 // now we have the indexes in dest_dir
561 readIndex( dest_dir + idxloc.filename(), idxloc.filename().dirname() );
563 MIL << "done reading indexes" << endl;
566 void Fetcher::Impl::start( const Pathname &dest_dir,
567 MediaSetAccess &media,
568 const ProgressData::ReceiverFnc & progress_receiver )
570 ProgressData progress(_resources.size());
571 progress.sendTo(progress_receiver);
573 readIndexes(media, dest_dir);
575 for ( list<FetcherJob_Ptr>::const_iterator it_res = _resources.begin(); it_res != _resources.end(); ++it_res )
578 if ( (*it_res)->directory )
580 const OnMediaLocation location((*it_res)->location);
581 addDirJobs(media, location, dest_dir, true);
585 provideToDest(media, (*it_res)->location, dest_dir);
587 // if the checksum is empty, but the checksum is in one of the
588 // indexes checksum, then add a checker
589 if ( (*it_res)->location.checksum().empty() )
591 if ( _checksums.find((*it_res)->location.filename().asString())
592 != _checksums.end() )
594 // the checksum can be replaced with the one in the index.
595 CheckSum chksm = _checksums[(*it_res)->location.filename().asString()];
596 ChecksumFileChecker digest_check(chksm);
597 (*it_res)->checkers.push_back(digest_check);
601 // validate job, this throws if not valid
602 validate((*it_res)->location, dest_dir, (*it_res)->checkers);
604 if ( ! progress.incr() )
605 ZYPP_THROW(AbortRequestException());
609 /** \relates Fetcher::Impl Stream output */
610 inline std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj )
612 for ( list<FetcherJob_Ptr>::const_iterator it_res = obj._resources.begin(); it_res != obj._resources.end(); ++it_res )
619 ///////////////////////////////////////////////////////////////////
621 // CLASS NAME : Fetcher
623 ///////////////////////////////////////////////////////////////////
625 ///////////////////////////////////////////////////////////////////
627 // METHOD NAME : Fetcher::Fetcher
628 // METHOD TYPE : Ctor
631 : _pimpl( new Impl() )
634 ///////////////////////////////////////////////////////////////////
636 // METHOD NAME : Fetcher::~Fetcher
637 // METHOD TYPE : Dtor
642 void Fetcher::setOptions( Fetcher::Options options )
644 _pimpl->setOptions(options);
647 Fetcher::Options Fetcher::options() const
649 return _pimpl->options();
652 void Fetcher::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker )
654 _pimpl->enqueueDigested(resource, checker);
657 void Fetcher::enqueueDir( const OnMediaLocation &resource,
659 const FileChecker &checker )
661 _pimpl->enqueueDir(resource, recursive, checker);
664 void Fetcher::addIndex( const OnMediaLocation &resource )
666 _pimpl->addIndex(resource);
670 void Fetcher::enqueue( const OnMediaLocation &resource, const FileChecker &checker )
672 _pimpl->enqueue(resource, checker);
675 void Fetcher::addCachePath( const Pathname &cache_dir )
677 _pimpl->addCachePath(cache_dir);
680 void Fetcher::reset()
685 void Fetcher::start( const Pathname &dest_dir,
686 MediaSetAccess &media,
687 const ProgressData::ReceiverFnc & progress_receiver )
689 _pimpl->start(dest_dir, media, progress_receiver);
693 /******************************************************************
695 ** FUNCTION NAME : operator<<
696 ** FUNCTION TYPE : std::ostream &
698 std::ostream & operator<<( std::ostream & str, const Fetcher & obj )
700 return str << *obj._pimpl;
703 /////////////////////////////////////////////////////////////////
705 ///////////////////////////////////////////////////////////////////