1 /*---------------------------------------------------------------------\
3 | |__ / \ / / . \ . \ |
8 \---------------------------------------------------------------------*/
9 /** \file zypp/Fetcher.cc
17 #include "zypp/base/Flags.h"
18 #include "zypp/base/Easy.h"
19 #include "zypp/base/Logger.h"
20 #include "zypp/base/PtrTypes.h"
21 #include "zypp/base/DefaultIntegral.h"
22 #include "zypp/base/String.h"
23 #include "zypp/Fetcher.h"
24 #include "zypp/CheckSum.h"
25 #include "zypp/base/UserRequestException.h"
29 ///////////////////////////////////////////////////////////////////
31 { /////////////////////////////////////////////////////////////////
34 * class that represents indexes which add metadata
35 * to fetcher jobs and therefore need to be retrieved
40 FetcherIndex( const OnMediaLocation &loc )
45 OnMediaLocation location;
47 typedef shared_ptr<FetcherIndex> FetcherIndex_Ptr;
50 * Class to encapsulate the \ref OnMediaLocation object
51 * and the \ref FileChecker together
55 FetcherJob( const OnMediaLocation &loc )
60 //MIL << location << endl;
65 //MIL << location << " | * " << checkers.size() << endl;
68 OnMediaLocation location;
69 //CompositeFileChecker checkers;
70 list<FileChecker> checkers;
75 typedef shared_ptr<FetcherJob> FetcherJob_Ptr;
77 std::ostream & operator<<( std::ostream & str, const FetcherJob_Ptr & obj )
79 return str << obj->location;
83 ///////////////////////////////////////////////////////////////////
85 // CLASS NAME : Fetcher::Impl
87 /** Fetcher implementation. */
90 friend std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj );
97 void addIndex( const OnMediaLocation &resource );
98 void enqueue( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() );
99 void enqueueDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
100 void enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() );
101 void addCachePath( const Pathname &cache_dir );
103 void start( const Pathname &dest_dir,
104 MediaSetAccess &media,
105 const ProgressData::ReceiverFnc & progress_receiver );
107 /** Offer default Impl. */
108 static shared_ptr<Impl> nullimpl()
110 static shared_ptr<Impl> _nullimpl( new Impl );
115 * download the indexes and reads them
117 void readIndexes( MediaSetAccess &media, const Pathname &dest_dir);
120 * reads a downloaded index file and updates internal
123 * The index lists files relative to a directory, which is
124 * normally the same as the index file is located.
126 void readIndex( const Pathname &index, const Pathname &basedir );
129 * tries to provide the file represented by job into dest_dir by
130 * looking at the cache. If success, returns true, and the desired
131 * file should be available on dest_dir
133 bool provideFromCache( const OnMediaLocation &resource, const Pathname &dest_dir );
135 * Validates the job against is checkers, by using the file instance
139 void validate( const OnMediaLocation &resource, const Pathname &dest_dir, const list<FileChecker> &checkers );
142 * scan the directory and adds the individual jobs
144 void addDirJobs( MediaSetAccess &media, const OnMediaLocation &resource,
145 const Pathname &dest_dir, bool recursive );
147 * Provide the resource to \ref dest_dir
149 void provideToDest( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir );
152 friend Impl * rwcowClone<Impl>( const Impl * rhs );
153 /** clone for RWCOW_pointer */
155 { return new Impl( *this ); }
157 list<FetcherJob_Ptr> _resources;
158 list<FetcherIndex_Ptr> _indexes;
159 list<Pathname> _caches;
160 // checksums read from the indexes
161 map<string, CheckSum> _checksums;
163 ///////////////////////////////////////////////////////////////////
165 void Fetcher::Impl::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker )
168 job.reset(new FetcherJob(resource));
169 ChecksumFileChecker digest_check(resource.checksum());
170 job->checkers.push_back(digest_check);
172 job->checkers.push_back(checker);
173 _resources.push_back(job);
176 void Fetcher::Impl::enqueueDir( const OnMediaLocation &resource,
178 const FileChecker &checker )
181 job.reset(new FetcherJob(resource));
183 job->checkers.push_back(checker);
184 job->directory = true;
185 job->recursive = recursive;
186 _resources.push_back(job);
189 void Fetcher::Impl::enqueue( const OnMediaLocation &resource, const FileChecker &checker )
192 job.reset(new FetcherJob(resource));
194 job->checkers.push_back(checker);
195 _resources.push_back(job);
198 void Fetcher::Impl::addIndex( const OnMediaLocation &resource )
200 MIL << "adding index " << resource << endl;
201 FetcherIndex_Ptr index;
202 index.reset(new FetcherIndex(resource));
203 _indexes.push_back(index);
207 void Fetcher::Impl::reset()
213 void Fetcher::Impl::addCachePath( const Pathname &cache_dir )
215 PathInfo info(cache_dir);
216 if ( info.isExist() )
220 DBG << "Adding fetcher cache: '" << cache_dir << "'." << endl;
221 _caches.push_back(cache_dir);
225 // don't add bad cache directory, just log the error
226 ERR << "Not adding cache: '" << cache_dir << "'. Not a directory." << endl;
231 ERR << "Not adding cache '" << cache_dir << "'. Path does not exists." << endl;
236 bool Fetcher::Impl::provideFromCache( const OnMediaLocation &resource, const Pathname &dest_dir )
238 Pathname dest_full_path = dest_dir + resource.filename();
240 // first check in the destination directory
241 if ( PathInfo(dest_full_path).isExist() )
243 if ( is_checksum( dest_full_path, resource.checksum() )
244 && (! resource.checksum().empty() ) )
248 MIL << "start fetcher with " << _caches.size() << " cache directories." << endl;
249 for_ ( it_cache, _caches.begin(), _caches.end() )
251 // does the current file exists in the current cache?
252 Pathname cached_file = *it_cache + resource.filename();
253 if ( PathInfo( cached_file ).isExist() )
255 DBG << "File '" << cached_file << "' exist, testing checksum " << resource.checksum() << endl;
256 // check the checksum
257 if ( is_checksum( cached_file, resource.checksum() ) && (! resource.checksum().empty() ) )
260 MIL << "file " << resource.filename() << " found in previous cache. Using cached copy." << endl;
261 // checksum is already checked.
262 // we could later implement double failover and try to download if file copy fails.
263 // replicate the complete path in the target directory
264 if( dest_full_path != cached_file )
266 if ( assert_dir( dest_full_path.dirname() ) != 0 )
267 ZYPP_THROW( Exception("Can't create " + dest_full_path.dirname().asString()));
269 if ( filesystem::hardlink(cached_file, dest_full_path ) != 0 )
271 WAR << "Can't hardlink '" << cached_file << "' to '" << dest_dir << "'. Trying copying." << endl;
272 if ( filesystem::copy(cached_file, dest_full_path ) != 0 )
274 ERR << "Can't copy " << cached_file + " to " + dest_dir << endl;
284 } // iterate over caches
288 void Fetcher::Impl::validate( const OnMediaLocation &resource, const Pathname &dest_dir, const list<FileChecker> &checkers )
290 // no matter where did we got the file, try to validate it:
291 Pathname localfile = dest_dir + resource.filename();
292 // call the checker function
295 MIL << "Checking job [" << localfile << "] (" << checkers.size() << " checkers )" << endl;
297 for ( list<FileChecker>::const_iterator it = checkers.begin();
298 it != checkers.end();
307 ERR << "Invalid checker for '" << localfile << "'" << endl;
312 catch ( const FileCheckException &e )
316 catch ( const Exception &e )
322 ZYPP_THROW(Exception("Unknown error while validating " + resource.filename().asString()));
326 void Fetcher::Impl::addDirJobs( MediaSetAccess &media,
327 const OnMediaLocation &resource,
328 const Pathname &dest_dir, bool recursive )
330 // first get the content of the directory so we can add
331 // individual transfer jobs
332 MIL << "Adding directory " << resource.filename() << endl;
333 filesystem::DirContent content;
334 media.dirInfo( content, resource.filename(), false /* dots */, resource.medianr());
336 // only try to add an index if it exists
337 filesystem::DirEntry shafile;
338 shafile.name = "SHA1SUMS"; shafile.type = filesystem::FT_FILE;
339 if ( find( content.begin(), content.end(), shafile ) != content.end() )
341 // add the index of this directory
342 OnMediaLocation indexloc(resource);
343 indexloc.changeFilename(resource.filename() + "SHA1SUMS");
345 // we need to read it now
346 readIndexes(media, dest_dir);
349 for ( filesystem::DirContent::const_iterator it = content.begin();
353 // skip SHA1SUMS* as they were already retrieved
354 if ( str::hasPrefix(it->name, "SHA1SUMS") )
357 Pathname filename = resource.filename() + it->name;
361 case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
362 case filesystem::FT_FILE:
364 CheckSum chksm(resource.checksum());
365 if ( _checksums.find(filename.asString()) != _checksums.end() )
367 // the checksum can be replaced with the one in the index.
368 chksm = _checksums[filename.asString()];
369 //MIL << "resource " << filename << " has checksum in the index file." << endl;
372 WAR << "Resource " << filename << " has no checksum in the index either." << endl;
374 enqueueDigested(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm));
377 case filesystem::FT_DIR: // newer directory.yast contain at least directory info
379 addDirJobs(media, filename, dest_dir, recursive);
382 // don't provide devices, sockets, etc.
388 void Fetcher::Impl::provideToDest( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir )
390 bool got_from_cache = false;
392 // start look in cache
393 got_from_cache = provideFromCache(resource, dest_dir);
395 if ( ! got_from_cache )
397 MIL << "Not found in cache, downloading" << endl;
399 // try to get the file from the net
402 Pathname tmp_file = media.provideFile(resource);
403 Pathname dest_full_path = dest_dir + resource.filename();
404 if ( assert_dir( dest_full_path.dirname() ) != 0 )
405 ZYPP_THROW( Exception("Can't create " + dest_full_path.dirname().asString()));
406 if ( filesystem::copy(tmp_file, dest_full_path ) != 0 )
408 ZYPP_THROW( Exception("Can't copy " + tmp_file.asString() + " to " + dest_dir.asString()));
411 media.releaseFile(resource); //not needed anymore, only eat space
413 catch (Exception & excpt_r)
415 ZYPP_CAUGHT(excpt_r);
416 excpt_r.remember("Can't provide " + resource.filename().asString() + " : " + excpt_r.msg());
417 ZYPP_RETHROW(excpt_r);
422 // We got the file from cache
423 // continue with next file
428 void Fetcher::Impl::readIndex( const Pathname &index, const Pathname &basedir )
430 std::ifstream in( index.c_str() );
434 while ( getline(in, buffer) )
436 vector<string> words;
437 str::split( buffer, back_inserter(words) );
438 if ( words.size() != 2 )
439 ZYPP_THROW(Exception("Wrong format for SHA1SUMS file"));
440 //MIL << "check: '" << words[0] << "' | '" << words[1] << "'" << endl;
441 if ( ! words[1].empty() )
442 _checksums[(basedir + words[1]).asString()] = CheckSum::sha1(words[0]);
446 ZYPP_THROW(Exception("Can't open SHA1SUMS file: " + index.asString()));
449 void Fetcher::Impl::readIndexes( MediaSetAccess &media, const Pathname &dest_dir)
451 // if there is no indexes, then just return to avoid
452 // the directory listing
453 if ( _indexes.empty() )
455 MIL << "No indexes to read." << endl;
459 // create a new fetcher with a different state to transfer the
460 // file containing checksums and its signature
462 // signature checker for index. We havent got the signature from
464 SignatureFileChecker sigchecker;
466 for ( list<FetcherIndex_Ptr>::const_iterator it_idx = _indexes.begin();
467 it_idx != _indexes.end(); ++it_idx )
469 MIL << "reading index " << (*it_idx)->location << endl;
470 // build the name of the index and the signature
471 OnMediaLocation idxloc((*it_idx)->location);
472 OnMediaLocation sigloc((*it_idx)->location.setOptional(true));
473 OnMediaLocation keyloc((*it_idx)->location.setOptional(true));
475 // calculate signature and key name
476 sigloc.changeFilename( sigloc.filename().extend(".asc") );
477 keyloc.changeFilename( keyloc.filename().extend(".key") );
479 //assert_dir(dest_dir + idxloc.filename().dirname());
481 // transfer the signature
482 fetcher.enqueue(sigloc);
483 fetcher.start( dest_dir, media );
484 // if we get the signature, update the checker
485 sigchecker = SignatureFileChecker(dest_dir + sigloc.filename());
489 fetcher.enqueue(keyloc);
490 fetcher.start( dest_dir, media );
493 // now the index itself
494 fetcher.enqueue( idxloc, FileChecker(sigchecker) );
495 fetcher.start( dest_dir, media );
498 // now we have the indexes in dest_dir
499 readIndex( dest_dir + idxloc.filename(), idxloc.filename().dirname() );
501 MIL << "done reading indexes" << endl;
504 void Fetcher::Impl::start( const Pathname &dest_dir,
505 MediaSetAccess &media,
506 const ProgressData::ReceiverFnc & progress_receiver )
508 ProgressData progress(_resources.size());
509 progress.sendTo(progress_receiver);
511 readIndexes(media, dest_dir);
513 for ( list<FetcherJob_Ptr>::const_iterator it_res = _resources.begin(); it_res != _resources.end(); ++it_res )
516 if ( (*it_res)->directory )
518 const OnMediaLocation location((*it_res)->location);
519 addDirJobs(media, location, dest_dir, true);
523 provideToDest(media, (*it_res)->location, dest_dir);
525 // validate job, this throws if not valid
526 validate((*it_res)->location, dest_dir, (*it_res)->checkers);
528 if ( ! progress.incr() )
529 ZYPP_THROW(AbortRequestException());
533 /** \relates Fetcher::Impl Stream output */
534 inline std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj )
536 for ( list<FetcherJob_Ptr>::const_iterator it_res = obj._resources.begin(); it_res != obj._resources.end(); ++it_res )
543 ///////////////////////////////////////////////////////////////////
545 // CLASS NAME : Fetcher
547 ///////////////////////////////////////////////////////////////////
549 ///////////////////////////////////////////////////////////////////
551 // METHOD NAME : Fetcher::Fetcher
552 // METHOD TYPE : Ctor
555 : _pimpl( new Impl() )
558 ///////////////////////////////////////////////////////////////////
560 // METHOD NAME : Fetcher::~Fetcher
561 // METHOD TYPE : Dtor
566 void Fetcher::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker )
568 _pimpl->enqueueDigested(resource, checker);
571 void Fetcher::enqueueDir( const OnMediaLocation &resource,
573 const FileChecker &checker )
575 _pimpl->enqueueDir(resource, recursive, checker);
578 void Fetcher::addIndex( const OnMediaLocation &resource )
580 _pimpl->addIndex(resource);
584 void Fetcher::enqueue( const OnMediaLocation &resource, const FileChecker &checker )
586 _pimpl->enqueue(resource, checker);
589 void Fetcher::addCachePath( const Pathname &cache_dir )
591 _pimpl->addCachePath(cache_dir);
594 void Fetcher::reset()
599 void Fetcher::start( const Pathname &dest_dir,
600 MediaSetAccess &media,
601 const ProgressData::ReceiverFnc & progress_receiver )
603 _pimpl->start(dest_dir, media, progress_receiver);
607 /******************************************************************
609 ** FUNCTION NAME : operator<<
610 ** FUNCTION TYPE : std::ostream &
612 std::ostream & operator<<( std::ostream & str, const Fetcher & obj )
614 return str << *obj._pimpl;
617 /////////////////////////////////////////////////////////////////
619 ///////////////////////////////////////////////////////////////////