#include <map>
#include "zypp/base/Easy.h"
-#include "zypp/base/Logger.h"
+#include "zypp/base/LogControl.h"
+#include "zypp/base/LogTools.h"
#include "zypp/base/PtrTypes.h"
#include "zypp/base/DefaultIntegral.h"
#include "zypp/base/String.h"
#include "zypp/Fetcher.h"
+#include "zypp/ZYppFactory.h"
#include "zypp/CheckSum.h"
#include "zypp/base/UserRequestException.h"
#include "zypp/parser/susetags/ContentFileReader.h"
using namespace std;
+#undef ZYPP_BASE_LOGGER_LOGGROUP
+#define ZYPP_BASE_LOGGER_LOGGROUP "zypp:fetcher"
+
///////////////////////////////////////////////////////////////////
namespace zypp
{ /////////////////////////////////////////////////////////////////
{
FetcherIndex( const OnMediaLocation &loc )
: location(loc)
+ {}
+ /** Index localtion. */
+ OnMediaLocation location;
+ /** Whether we read this index. */
+ DefaultIntegral<bool,false> read;
+ };
+
+ typedef shared_ptr<FetcherIndex> FetcherIndex_Ptr;
+
+ /** std::set ordering (less semantic) */
+ struct SameFetcherIndex
+ {
+ bool operator()( const FetcherIndex_Ptr & lhs, const FetcherIndex_Ptr & rhs )
{
+ if ( lhs == rhs )
+ return false; // incl. NULL == NULL
+ if ( ! lhs )
+ return true; // NULL < nonNULL
+ if ( ! rhs )
+ return false; // nonNULL > NULL
+ // both nonNULL ==> compare medianr and path
+ if ( lhs->location.medianr() == rhs->location.medianr() )
+ return lhs->location.filename() < rhs->location.filename();
+ //else
+ return lhs->location.medianr() < rhs->location.medianr();
}
-
- OnMediaLocation location;
};
- typedef shared_ptr<FetcherIndex> FetcherIndex_Ptr;
-
+
/**
* Class to encapsulate the \ref OnMediaLocation object
* and the \ref FileChecker together
{
enum Flag
{
- None = 0x0000,
+ None = 0x0000,
Directory = 0x0001,
Recursive = 0x0002,
RecursiveDirectory = Directory | Recursive,
AlwaysVerifyChecksum = 0x0004,
};
ZYPP_DECLARE_FLAGS(Flags, Flag);
-
- FetcherJob( const OnMediaLocation &loc )
+
+ FetcherJob( const OnMediaLocation &loc, const Pathname dfile = Pathname())
: location(loc)
+ , deltafile(dfile)
, flags(None)
{
//MIL << location << endl;
}
-
+
~FetcherJob()
{
//MIL << location << " | * " << checkers.size() << endl;
}
OnMediaLocation location;
+ Pathname deltafile;
//CompositeFileChecker checkers;
list<FileChecker> checkers;
Flags flags;
};
- ZYPP_DECLARE_OPERATORS_FOR_FLAGS(FetcherJob::Flags);
+
+ ZYPP_DECLARE_OPERATORS_FOR_FLAGS(FetcherJob::Flags);
typedef shared_ptr<FetcherJob> FetcherJob_Ptr;
std::ostream & operator<<( std::ostream & str, const FetcherJob_Ptr & obj )
return str << obj->location;
}
-
///////////////////////////////////////////////////////////////////
//
// CLASS NAME : Fetcher::Impl
class Fetcher::Impl
{
friend std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj );
+
public:
- Impl();
-
- ~Impl() {
- MIL << endl;
- }
-
- void setOptions( Fetcher::Options options );
- Fetcher::Options options() const;
-
- void addIndex( const OnMediaLocation &resource );
-
- void enqueueDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
- void enqueueDigestedDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
-
- void enqueue( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() );
- void enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() );
+ Impl();
+
+ ~Impl() {}
+
+ void setOptions( Fetcher::Options options );
+ Fetcher::Options options() const;
+
+ void addIndex( const OnMediaLocation &resource );
+
+ void enqueueDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
+ void enqueueDigestedDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
+
+ void enqueue( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() );
+ void enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker = FileChecker(), const Pathname &deltafile = Pathname() );
void addCachePath( const Pathname &cache_dir );
void reset();
void start( const Pathname &dest_dir,
*/
void readIndex( const Pathname &index, const Pathname &basedir );
- /** specific version of \ref readIndex for SHA1SUMS file */
- void readSha1sumsIndex( const Pathname &index, const Pathname &basedir );
-
- /** specific version of \ref readIndex for SHA1SUMS file */
+ /** specific version of \ref readIndex for CHECKSUMS file */
+ void readChecksumsIndex( const Pathname &index, const Pathname &basedir );
+
+ /** specific version of \ref readIndex for content file */
void readContentFileIndex( const Pathname &index, const Pathname &basedir );
-
+
/** reads the content of a directory but keeps a cache **/
void getDirectoryContent( MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content );
-
+
/**
* tries to provide the file represented by job into dest_dir by
* looking at the cache. If success, returns true, and the desired
/**
* Provide the resource to \ref dest_dir
*/
- void provideToDest( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir );
+ void provideToDest( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir , const Pathname &deltafile);
private:
friend Impl * rwcowClone<Impl>( const Impl * rhs );
Impl * clone() const
{ return new Impl( *this ); }
- list<FetcherJob_Ptr> _resources;
- list<FetcherIndex_Ptr> _indexes;
-
list<Pathname> _caches;
+ list<FetcherJob_Ptr> _resources;
+ std::set<FetcherIndex_Ptr,SameFetcherIndex> _indexes;
+
std::set<Pathname> _caches;
// checksums read from the indexes
map<string, CheckSum> _checksums;
// cache of dir contents
map<string, filesystem::DirContent> _dircontent;
-
- Fetcher::Options _options;
+
+ Fetcher::Options _options;
};
///////////////////////////////////////////////////////////////////
- void Fetcher::Impl::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker )
+ void Fetcher::Impl::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker, const Pathname &deltafile )
{
FetcherJob_Ptr job;
- job.reset(new FetcherJob(resource));
+ job.reset(new FetcherJob(resource, deltafile));
job->flags |= FetcherJob:: AlwaysVerifyChecksum;
_resources.push_back(job);
}
-
+
Fetcher::Impl::Impl()
: _options(0)
{
}
-
+
void Fetcher::Impl::setOptions( Fetcher::Options options )
{ _options = options; }
-
+
Fetcher::Options Fetcher::Impl::options() const
{ return _options; }
-
+
void Fetcher::Impl::enqueueDir( const OnMediaLocation &resource,
bool recursive,
const FileChecker &checker )
void Fetcher::Impl::addIndex( const OnMediaLocation &resource )
{
- MIL << "adding index " << resource << endl;
- FetcherIndex_Ptr index;
- index.reset(new FetcherIndex(resource));
- _indexes.push_back(index);
+ MIL << "adding index " << resource << endl;
+ _indexes.insert(FetcherIndex_Ptr(new FetcherIndex(resource)));
}
if ( info.isDir() )
{
DBG << "Adding fetcher cache: '" << cache_dir << "'." << endl;
- _caches.push_back(cache_dir);
+ _caches.insert(cache_dir);
}
else
{
if ( assert_dir( dest_full_path.dirname() ) != 0 )
ZYPP_THROW( Exception("Can't create " + dest_full_path.dirname().asString()));
- if ( filesystem::hardlink(cached_file, dest_full_path ) != 0 )
+ if ( filesystem::hardlinkCopy(cached_file, dest_full_path ) != 0 )
{
- WAR << "Can't hardlink '" << cached_file << "' to '" << dest_dir << "'. Trying copying." << endl;
- if ( filesystem::copy(cached_file, dest_full_path ) != 0 )
- {
- ERR << "Can't copy " << cached_file + " to " + dest_dir << endl;
- // try next cache
- continue;
- }
+ ERR << "Can't hardlink/copy " << cached_file + " to " + dest_dir << endl;
+ continue;
}
}
// found in cache
} // iterate over caches
return false;
}
-
+
void Fetcher::Impl::validate( const OnMediaLocation &resource, const Pathname &dest_dir, const list<FileChecker> &checkers )
{
// no matter where did we got the file, try to validate it:
const OnMediaLocation &resource,
const Pathname &dest_dir )
{
- if ( _options & AutoAddSha1sumsIndexes )
+ auto fnc_addIfInContent( [&]( const std::string & index_r ) -> bool
{
- // only try to add an index if it exists
- filesystem::DirEntry shafile;
- shafile.name = "SHA1SUMS"; shafile.type = filesystem::FT_FILE;
- if ( find( content.begin(), content.end(), shafile ) != content.end() )
- {
- // add the index of this directory
- OnMediaLocation indexloc(resource);
- indexloc.changeFilename(resource.filename() + "SHA1SUMS");
- addIndex(indexloc);
- // we need to read it now
- downloadAndReadIndexList(media, dest_dir);
- }
+ if ( find( content.begin(), content.end(), filesystem::DirEntry(index_r,filesystem::FT_FILE) ) == content.end() )
+ return false;
+ // add the index of this directory
+ OnMediaLocation indexloc( resource );
+ indexloc.changeFilename( resource.filename() + index_r );
+ addIndex( indexloc );
+ // we need to read it now
+ downloadAndReadIndexList( media, dest_dir );
+ return true;
+ } );
+
+ if ( _options & AutoAddChecksumsIndexes )
+ {
+ fnc_addIfInContent( "CHECKSUMS" ) || fnc_addIfInContent( "SHA1SUMS" );
}
if ( _options & AutoAddContentFileIndexes )
{
- // only try to add an index if it exists
- filesystem::DirEntry contentfile;
- contentfile.name = "content"; contentfile.type = filesystem::FT_FILE;
- if ( find( content.begin(), content.end(), contentfile ) != content.end() )
- {
- // add the index of this directory
- OnMediaLocation indexloc(resource);
- indexloc.changeFilename(resource.filename() + "content");
- addIndex(indexloc);
- // we need to read it now
- downloadAndReadIndexList(media, dest_dir);
- }
+ fnc_addIfInContent( "content" );
}
}
const OnMediaLocation &resource,
filesystem::DirContent &content )
{
- if ( _dircontent.find(resource.filename().asString())
+ if ( _dircontent.find(resource.filename().asString())
!= _dircontent.end() )
{
filesystem::DirContent filled(_dircontent[resource.filename().asString()]);
-
+
std::copy(filled.begin(), filled.end(), std::back_inserter(content));
}
else
_dircontent[resource.filename().asString()] = tofill;
}
}
-
+
void Fetcher::Impl::addDirJobs( MediaSetAccess &media,
const OnMediaLocation &resource,
const Pathname &dest_dir, FetcherJob::Flags flags )
// individual transfer jobs
MIL << "Adding directory " << resource.filename() << endl;
filesystem::DirContent content;
- getDirectoryContent(media, resource, content);
-
+ try {
+ getDirectoryContent(media, resource, content);
+ }
+ catch ( media::MediaFileNotFoundException & exception )
+ {
+ ZYPP_CAUGHT( exception );
+ WAR << "Skiping subtree hidden at " << resource.filename() << endl;
+ return;
+ }
+
// this method test for the option flags so indexes are added
// only if the options are enabled
autoaddIndexes(content, media, resource, dest_dir);
-
+
for ( filesystem::DirContent::const_iterator it = content.begin();
it != content.end();
++it )
{
- // skip SHA1SUMS* as they were already retrieved
- if ( str::hasPrefix(it->name, "SHA1SUMS") )
+ // skip CHECKSUMS* as they were already retrieved
+ if ( str::hasPrefix(it->name, "CHECKSUMS") || str::hasPrefix(it->name, "SHA1SUMS") )
continue;
Pathname filename = resource.filename() + it->name;
case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
case filesystem::FT_FILE:
{
- CheckSum chksm(resource.checksum());
+ CheckSum chksm(resource.checksum());
if ( _checksums.find(filename.asString()) != _checksums.end() )
{
// the checksum can be replaced with the one in the index.
}
}
- void Fetcher::Impl::provideToDest( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir )
+ void Fetcher::Impl::provideToDest( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir, const Pathname &deltafile )
{
bool got_from_cache = false;
// try to get the file from the net
try
{
- Pathname tmp_file = media.provideFile(resource);
+ Pathname tmp_file = media.provideFile(resource, resource.optional() ? MediaSetAccess::PROVIDE_NON_INTERACTIVE : MediaSetAccess::PROVIDE_DEFAULT, deltafile );
+
Pathname dest_full_path = dest_dir + resource.filename();
+
if ( assert_dir( dest_full_path.dirname() ) != 0 )
ZYPP_THROW( Exception("Can't create " + dest_full_path.dirname().asString()));
- if ( filesystem::copy(tmp_file, dest_full_path ) != 0 )
+ if ( filesystem::hardlinkCopy( tmp_file, dest_full_path ) != 0 )
{
if ( ! PathInfo(tmp_file).isExist() )
ERR << tmp_file << " does not exist" << endl;
ERR << dest_full_path.dirname() << " does not exist" << endl;
media.releaseFile(resource); //not needed anymore, only eat space
- ZYPP_THROW( Exception("Can't copy " + tmp_file.asString() + " to " + dest_dir.asString()));
+ ZYPP_THROW( Exception("Can't hardlink/copy " + tmp_file.asString() + " to " + dest_dir.asString()));
}
media.releaseFile(resource); //not needed anymore, only eat space
}
catch (Exception & excpt_r)
{
- ZYPP_CAUGHT(excpt_r);
- excpt_r.remember("Can't provide " + resource.filename().asString() + " : " + excpt_r.msg());
-
if ( resource.optional() )
{
- WAR << "optional resource " << resource << " could not be transfered" << endl;
+ ZYPP_CAUGHT(excpt_r);
+ WAR << "optional resource " << resource << " could not be transferred" << endl;
return;
}
else
- {
+ {
+ excpt_r.remember("Can't provide " + resource.filename().asString() );
ZYPP_RETHROW(excpt_r);
}
}
{
setRepoIndexConsumer( bind( &ContentReaderHelper::consumeIndex, this, _1 ) );
}
-
+
void consumeIndex( const parser::susetags::RepoIndex_Ptr & data_r )
{ _repoindex = data_r; }
// generic function for reading indexes
void Fetcher::Impl::readIndex( const Pathname &index, const Pathname &basedir )
{
- if ( index.basename() == "SHA1SUMS" )
- readSha1sumsIndex(index, basedir);
+ if ( index.basename() == "CHECKSUMS" || index.basename() == "SHA1SUMS" )
+ readChecksumsIndex(index, basedir);
else if ( index.basename() == "content" )
readContentFileIndex(index, basedir);
else
}
}
- // reads a SHA1SUMS file index
- void Fetcher::Impl::readSha1sumsIndex( const Pathname &index, const Pathname &basedir )
+ // reads a CHECKSUMS (old SHA1SUMS) file index
+ void Fetcher::Impl::readChecksumsIndex( const Pathname &index, const Pathname &basedir )
{
std::ifstream in( index.c_str() );
- string buffer;
if ( ! in.fail() )
{
- while ( getline(in, buffer) )
+ std::string buffer;
+ while ( getline( in, buffer ) )
{
- vector<string> words;
- str::split( buffer, back_inserter(words) );
- if ( words.size() != 2 )
- ZYPP_THROW(Exception("Wrong format for SHA1SUMS file"));
- //MIL << "check: '" << words[0] << "' | '" << words[1] << "'" << endl;
- if ( ! words[1].empty() )
- _checksums[(basedir + words[1]).asString()] = CheckSum::sha1(words[0]);
+
+ if ( buffer[0] == '#' )
+ continue; // simple comment
+
+ CheckSum checksum( str::stripFirstWord( buffer, /*ltrim before strip*/true ) );
+ if ( checksum.empty() )
+ continue; // empty line | unknown cheksum format
+
+ if ( buffer.empty() )
+ {
+ WAR << "Missing filename in CHECKSUMS file: " << index.asString() << " (" << checksum << ")" << endl;
+ continue;
+ }
+
+ _checksums[(basedir/buffer).asString()] = checksum;
}
}
else
- ZYPP_THROW(Exception("Can't open SHA1SUMS file: " + index.asString()));
+ ZYPP_THROW(Exception("Can't open CHECKSUMS file: " + index.asString()));
}
void Fetcher::Impl::downloadIndex( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
{
MIL << "downloading index " << resource << endl;
-
// create a new fetcher with a different state to transfer the
// file containing checksums and its signature
Fetcher fetcher;
// signature checker for index. We havent got the signature from
// the nextwork yet.
SignatureFileChecker sigchecker;
-
+
// build the name of the index and the signature
OnMediaLocation idxloc(resource);
OnMediaLocation sigloc(resource);
// the checking will warn later
sigloc.setOptional(true);
keyloc.setOptional(true);
-
+
// calculate signature and key name
sigloc.changeFilename( sigloc.filename().extend(".asc") );
keyloc.changeFilename( keyloc.filename().extend(".key") );
-
+
//assert_dir(dest_dir + idxloc.filename().dirname());
-
+
// transfer the signature
fetcher.enqueue(sigloc);
fetcher.start( dest_dir, media );
// if we get the signature, update the checker
if ( PathInfo(dest_dir + sigloc.filename()).isExist() )
sigchecker = SignatureFileChecker(dest_dir + sigloc.filename());
-
+
fetcher.reset();
-
+
// now the key
fetcher.enqueue(keyloc);
fetcher.start( dest_dir, media );
fetcher.reset();
-
+
+ // try to import the key
+ if ( PathInfo(dest_dir + keyloc.filename()).isExist() )
+ getZYpp()->keyRing()->importKey(PublicKey(dest_dir + keyloc.filename()), false);
+ else
+ WAR << "No public key specified by user for index '" << keyloc.filename() << "'"<< endl;
+
// now the index itself
fetcher.enqueue( idxloc, FileChecker(sigchecker) );
fetcher.start( dest_dir, media );
return;
}
- for ( list<FetcherIndex_Ptr>::const_iterator it_idx = _indexes.begin();
- it_idx != _indexes.end(); ++it_idx )
+ for_( it_idx, _indexes.begin(), _indexes.end() )
{
+ if ( (*it_idx)->read )
+ {
+ DBG << "Already read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
+ }
+ else
+ {
+ // base::LogControl::TmpLineWriter shutUp;
downloadIndex( media, (*it_idx)->location, dest_dir );
// now we have the indexes in dest_dir
readIndex( dest_dir + (*it_idx)->location.filename(), (*it_idx)->location.filename().dirname() );
+ // Take care we don't process it again
+ MIL << "Remember read index " << PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
+ (*it_idx)->read = true;
+ }
}
MIL << "done reading indexes" << endl;
}
-
+
// start processing all fetcher jobs.
// it processes any user pointed index first
void Fetcher::Impl::start( const Pathname &dest_dir,
// may be this code can be factored out
// together with the autodiscovery of indexes
// of addDirJobs
- if ( ( _options & AutoAddSha1sumsIndexes ) ||
+ if ( ( _options & AutoAddChecksumsIndexes ) ||
( _options & AutoAddContentFileIndexes ) )
{
// if auto indexing is enabled, then we need to read the
MIL << "Autodiscovering signed indexes on '"
<< (*it_res)->location.filename().dirname() << "' for '"
<< (*it_res)->location.filename() << "'" << endl;
-
+
autoaddIndexes(content, media, (*it_res)->location.filename().dirname(), dest_dir);
// also look in the root of the media
MIL << "Autodiscovering signed indexes on '"
<< "/" << "' for '"
<< (*it_res)->location.filename() << "'" << endl;
-
+
autoaddIndexes(content, media, Pathname("/"), dest_dir);
- }
+ }
- provideToDest(media, (*it_res)->location, dest_dir);
+ provideToDest(media, (*it_res)->location, dest_dir, (*it_res)->deltafile);
- // if the file was not transfered, and no exception, just
+ // if the file was not transferred, and no exception, just
// return, as it was an optional file
if ( ! PathInfo(dest_dir + (*it_res)->location.filename()).isExist() )
return;
// indexes checksum, then add a checker
if ( (*it_res)->location.checksum().empty() )
{
- if ( _checksums.find((*it_res)->location.filename().asString())
+ if ( _checksums.find((*it_res)->location.filename().asString())
!= _checksums.end() )
{
CheckSum chksm = _checksums[(*it_res)->location.filename().asString()];
- ChecksumFileChecker digest_check(chksm);
+ ChecksumFileChecker digest_check(chksm);
(*it_res)->checkers.push_back(digest_check);
}
else
{
// if the index checksum is empty too, we only add the checker
// if the AlwaysVerifyChecksum option is set on
- if ( (*it_res)->flags & FetcherJob::AlwaysVerifyChecksum )
+ if ( (*it_res)->flags & FetcherJob::AlwaysVerifyChecksum )
{
// add the checker with the empty checksum
- ChecksumFileChecker digest_check((*it_res)->location.checksum());
+ ChecksumFileChecker digest_check((*it_res)->location.checksum());
(*it_res)->checkers.push_back(digest_check);
}
}
else
{
// checksum is not empty, so add a checksum checker
- ChecksumFileChecker digest_check((*it_res)->location.checksum());
+ ChecksumFileChecker digest_check((*it_res)->location.checksum());
(*it_res)->checkers.push_back(digest_check);
}
-
+
// validate job, this throws if not valid
validate((*it_res)->location, dest_dir, (*it_res)->checkers);
{
_pimpl->setOptions(options);
}
-
+
Fetcher::Options Fetcher::options() const
{
return _pimpl->options();
}
- void Fetcher::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker )
+ void Fetcher::enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker, const Pathname &deltafile )
{
- _pimpl->enqueueDigested(resource, checker);
+ _pimpl->enqueueDigested(resource, checker, deltafile);
}
void Fetcher::enqueueDir( const OnMediaLocation &resource,
projects / platform / upstream / libzypp.git / blobdiff