## Process this file with automake to produce Makefile.in
## ##################################################
-noinst_PROGRAMS = test lock testbed aj yumparser
+noinst_PROGRAMS = test lock testbed aj yumparser
## ##################################################
int main()
{
-
- NVRAD a("hola", Edition("2.0-3"));
- NVRAD b("hola", Edition("2.0-3"));
-
- MIL << "a==b : " << (a==b) << std::endl;
+ std::ofstream file;
+ file.open("lala.txt", ios::out);
+ //file << xml;
+ file.close();
}
-------------------------------------------------------------------
Mon Jun 19 13:52:14 CEST 2006 - dmacvicar@suse.de
+- merge download algorithm and refactoring from branch
+ (#181204)
+
+-------------------------------------------------------------------
+Mon Jun 12 15:37:10 CEST 2006 - dmacvicar@suse.de
+
- right fix for tmpdir initialized in static constructor
- catch around provideJustFile in providePackage
-rev 3654
- revision 3517
-------------------------------------------------------------------
+Wed Jun 7 01:00:05 CEST 2006 - dmacvicar@suse.de
+
+- Fixes unneeded file download, and add download callbacks
+ (still need yast side) #181204 and #160206
+- Fix stalle tmpdir due to cyclic references, using a master
+ TmpDir for zypp. # 178292
+
+-------------------------------------------------------------------
Fri Jun 2 16:09:03 CEST 2006 - schubi@suse.de
-latest fi translation added
//
std::string sha1sum( const Pathname & file )
{
+ return checksum(file, "SHA1");
+ }
+
+ ///////////////////////////////////////////////////////////////////
+ //
+ // METHOD NAME : checksum
+ // METHOD TYPE : std::string
+ //
+ std::string checksum( const Pathname & file, const std::string &algorithm )
+ {
if ( ! PathInfo( file ).isFile() ) {
return string();
}
if ( ! istr ) {
return string();
}
- return Digest::digest( "SHA1", istr );
+ return Digest::digest( algorithm, istr );
+ }
+
+ bool is_checksum( const Pathname & file, const CheckSum &checksum )
+ {
+ return ( filesystem::checksum(file, checksum.type()) == checksum.checksum() );
}
///////////////////////////////////////////////////////////////////
#include <map>
#include "zypp/Pathname.h"
+#include "zypp/CheckSum.h"
///////////////////////////////////////////////////////////////////
namespace zypp
std::string sha1sum( const Pathname & file );
//@}
+ /**
+ * Compute a files checksum
+ *
+ * @return the files checksum on success, otherwise an empty string..
+ **/
+ std::string checksum( const Pathname & file, const std::string &algorithm );
+
+ /**
+ * check files checksum
+ *
+ * @return true if the checksum matchs
+ **/
+ bool is_checksum( const Pathname & file, const CheckSum &checksum );
+
///////////////////////////////////////////////////////////////////
/** \name Changing permissions. */
//@{
bool Target::setInstallationLogfile(const Pathname & path_r)
{ return _pimpl->setInstallationLogfile(path_r); }
+ Date Target::timestamp() const
+ { return _pimpl->timestamp(); }
+
/////////////////////////////////////////////////////////////////
} // namespace zypp
///////////////////////////////////////////////////////////////////
/** Return the root set for this target */
Pathname root() const;
+ /** return the last modification date of the target */
+ Date timestamp() const;
public:
/** Ctor */
explicit
return file;
}
+ const Pathname SourceImpl::downloadMetadataFile( const Pathname &file_to_download )
+ {
+
+ Pathname downloaded_file;
+
+ bool retry = true;
+ callback::SendReport<source::DownloadFileReport> report;
+ report->start( selfSourceRef(), url() );
+ while (retry)
+ {
+ try
+ {
+ downloaded_file = provideFile(file_to_download);
+ report->finish( url(), DownloadFileReport::NO_ERROR, file_to_download.asString() + " downloaded " + url().asString() );
+ retry = false;
+ }
+ catch (const Exception &e)
+ {
+ if ( report->problem(url(), DownloadFileReport::IO, "Can't provide " + file_to_download.asString() + " from " + url().asString()) != DownloadFileReport::RETRY )
+ {
+ report->finish( url(), DownloadFileReport::IO, "Can't provide " + file_to_download.asString() + " from " + url().asString() );
+ ZYPP_THROW(Exception("Can't provide " + file_to_download.asString() + " from " + url().asString() ));
+ }
+ }
+ }
+ return downloaded_file;
+ }
+
+ void SourceImpl::getPossiblyCachedMetadataFile( const Pathname &file_to_download, const Pathname &destination, const Pathname &cached_file, const CheckSum &checksum )
+ {
+ Url file_url( url().asString() + file_to_download.asString() );
+ // if we have a cached file and its the same
+ if ( PathInfo(cached_file).isExist() && (! checksum.empty()) && is_checksum( cached_file, checksum ) )
+ {
+ MIL << "file " << file_url << " found in previous cache. Using cached copy." << std::endl;
+ // checksum is already checked.
+ // we could later implement double failover and try to download if file copy fails.
+ if ( filesystem::copy(cached_file, destination) != 0 )
+ ZYPP_THROW(Exception("Can't copy " + cached_file.asString() + " to " + destination.asString()));
+ }
+ else
+ {
+ // we dont have it or its not the same, download it.
+ Pathname downloaded_file = downloadMetadataFile( file_to_download);
+
+ if ( filesystem::copy(downloaded_file, destination) != 0 )
+ ZYPP_THROW(Exception("Can't copy " + downloaded_file.asString() + " to " + destination.asString()));
+
+ callback::SendReport<DigestReport> report;
+ if ( checksum.empty() )
+ {
+ MIL << "File " << file_url << " has no checksum available." << std::endl;
+ if ( report->askUserToAcceptNoDigest(file_to_download) )
+ {
+ MIL << "User accepted " << file_url << " with no checksum." << std::endl;
+ return;
+ }
+ else
+ {
+ ZYPP_THROW(Exception( file_url.asString() + " " + N_(" miss checksum.") ));
+ }
+ }
+ else
+ {
+ if (! is_checksum( destination, checksum))
+ ZYPP_THROW(Exception( file_url.asString() + " " + N_(" fails checksum verification.") ));
+ }
+
+ }
+ }
+
void SourceImpl::resetMediaVerifier()
{
try
#include "zypp/Source.h"
#include "zypp/ResStore.h"
#include "zypp/Pathname.h"
+#include "zypp/CheckSum.h"
#include "zypp/media/MediaManager.h"
#include "zypp/source/MediaSet.h"
#include "zypp/TmpPath.h"
{ /////////////////////////////////////////////////////////////////
DEFINE_PTR_TYPE(SourceImpl);
-
///////////////////////////////////////////////////////////////////
//
// CLASS NAME : SourceImpl
void copyLocalMetadata(const Pathname &src, const Pathname &dst) const;
+ /**
+ * function that creates the tmp metadata dir if it was not created.
+ * this directory is used when cache_dir is not set (design flaw FIXME)
+ */
+ Pathname tmpMetadataDir() const;
+
+ /**
+ * wrapper around provideFile
+ * downloads a single file, providing download information callbacks
+ * \throw EXCEPTION on download failure and user abort
+ */
+ const Pathname downloadMetadataFile( const Pathname &file_to_download );
+
/**
* reset the media verifier to no verifier
*/
void resetMediaVerifier();
- /**
- * function that creates the tmp metadata dir if it was not created.
- * this directory is used when cache_dir is not set (design flaw FIXME)
- */
- Pathname tmpMetadataDir() const;
+ /**
+ * checks if a file exists in cache
+ * if no, downloads it, copies it in given destination, and check matching checksum
+ * if yes, compares checksum and copies it to destination locally
+ * \throw EXCEPTION on download/copy failure and user abort
+ */
+ void getPossiblyCachedMetadataFile( const Pathname &file_to_download, const Pathname &destination, const Pathname &cached_file, const CheckSum &checksum );
protected:
/** All resolvables provided by this source. */
bool SuseTagsImpl::downloadNeeded(const Pathname & localdir)
{
- Pathname new_media_file = provideFile("media.1/media");
+ Pathname new_media_file;
+ try {
+ new_media_file = tryToProvideFile("media.1/media");
+ }
+ catch( const Exception &e )
+ {
+ MIL << "media file used to determine if source changed not found. Assuming refresh needed." << std::endl;
+ return true;
+ }
+
// before really download all the data and init the cache, check
// if the source has really changed, otherwise, make it quick
Pathname cached_media_file = localdir + "/MEDIA/media.1/media";
// now we have the content file copied, we need to init data and descrdir from the product
readContentFile(local_dir + "/DATA/content");
- try {
- descr_src = provideDirTree(mediaDescrDir());
+ // make sure a local descr dir exists
+ if ( assert_dir( local_dir + "/DATA/descr") != 0 )
+ ZYPP_THROW (Exception( "Error. Can't create local descr directory. "));
+
+ // we can get the list of files in description dir in 2 ways
+ // from the checksum list, or ls'ing the dir via directory.yast
+ if ( ! _prodImpl->_descr_files_checksums.empty() )
+ {
+ // iterate through all available checksums
+ for ( std::map<std::string, CheckSum>::const_iterator it = _prodImpl->_descr_files_checksums.begin(); it != _prodImpl->_descr_files_checksums.end(); ++it)
+ {
+ std::string key = it->first;
+ getPossiblyCachedMetadataFile( mediaDescrDir() + key, local_dir + "/DATA/descr" + key, _cache_dir + "/DATA/descr" + key, _prodImpl->_descr_files_checksums[key] );
+ }
}
- catch(Exception &e) {
- ZYPP_THROW(Exception("Can't provide " + _path.asString() + " " + mediaDescrDir().asString() + " from " + url().asString() ));
+ else
+ {
+ // in case we dont have list of valid files in content file, we just glob for them
+ std::list<std::string> descr_dir_file_list;
+ try {
+ dirInfo( 1, descr_dir_file_list, _path);
+ }
+ catch(Exception &e) {
+ ZYPP_THROW(Exception("Can't list description directory content from " + url().asString() ));
+ }
+
+ for( std::list<std::string>::const_iterator it = descr_dir_file_list.begin(); it != descr_dir_file_list.end(); ++it)
+ {
+ std::string filename = *it;
+ getPossiblyCachedMetadataFile( mediaDescrDir() + filename, local_dir + "/DATA/descr" + filename, _cache_dir + "/DATA/descr" + filename, CheckSum() );
+ }
}
-
- if ( filesystem::copy_dir(descr_src, local_dir + "DATA") != 0 )
- ZYPP_THROW(Exception("Unable to copy the descr dir to " + (local_dir + "DATA").asString()));
-
- // now we have descr dir cached and keys, lets validate checksums
- checkMetadataChecksums(local_dir);
+
return tmpdir;
}
if ( need_to_refresh )
{
- MIL << "SuseTags source " << alias() << "has changed since last download. Re-reading metadata into " << dir_r << endl;
+ MIL << "SuseTags source " << alias() << " has changed since last download. Re-reading metadata into " << dir_r << endl;
}
else
{
- MIL << "SUSEtags source " << alias() << "has not changed. Refresh completed. timestamp of media file is the same." << std::endl;
+ MIL << "SUSEtags source " << alias() << " has not changed. Refresh completed. timestamp of media file is the same." << std::endl;
return;
}
return exists;
}
- bool SuseTagsImpl::verifyChecksumsMode() const
- {
- return ! _prodImpl->_descr_files_checksums.empty();
- }
-
void SuseTagsImpl::factoryInit()
{
bool cache = cacheExists();
store.insert( _product );
}
- void SuseTagsImpl::checkMetadataChecksums(const Pathname &p) const
- {
- // iterate through all available checksums
- for ( std::map<std::string, CheckSum>::const_iterator it = _prodImpl->_descr_files_checksums.begin(); it != _prodImpl->_descr_files_checksums.end(); ++it)
- {
- std::string key = it->first;
- verifyFile( p + "/DATA/descr" + key, key);
- }
-
- // FIXME add and check the gpg keys
- }
-
- void SuseTagsImpl::verifyFile( const Pathname &path, const std::string &key) const
- {
- // for old products, we dont check anything.
- if ( verifyChecksumsMode() )
- {
- MIL << "Going to check " << path << " file checksum" << std::endl;
- std::ifstream file(path.asString().c_str());
- if (!file) {
- ZYPP_THROW (Exception( "Can't open " + path.asString() ) );
- }
-
- // get the checksum for that file.
- CheckSum checksum = _prodImpl->_descr_files_checksums[key];
- if (checksum.empty())
- {
- callback::SendReport<DigestReport> report;
-
- if ( report->askUserToAcceptNoDigest(path) )
- {
- MIL << path << " user accepted file without a checksum " << endl;
- return;
- }
-
- ZYPP_THROW (Exception( "Error, Missing checksum for " + path.asString() ) );
- }
- else
- {
- std::string found_checksum = Digest::digest( checksum.type(), file);
- if ( found_checksum != checksum.checksum() )
- {
- ZYPP_THROW (Exception( "Corrupt source, Expected " + checksum.type() + " " + checksum.checksum() + ", got " + found_checksum + " for " + path.asString() ) );
- }
- else
- {
- MIL << path << " checksum ok (" << checksum.type() << " " << checksum.checksum() << ")" << std::endl;
- return;
- }
- }
- }
- }
-
void SuseTagsImpl::providePackages(Source_Ref source_r, ResStore &store)
{
Pathname p = descrDir() + "packages";
* but it does not add it to the store yet
*/
void readContentFile(const Pathname &p);
-
- /**
- * checks metadata
- * against it checksums
- * requires reading content file first
- * \param dir Download directory
- * \throw EXCEPTION on fail
- **/
- void checkMetadataChecksums(const Pathname &dir) const;
/**
* reads a media file and installs
void provideSelection(Source_Ref source_r, ResStore& store);
void providePatterns(Source_Ref source_r, ResStore& store);
- /**
- * verify media mode (use the new META tags)
- */
- bool verifyChecksumsMode() const;
-
- /**
- * Verify file checksum
- * \throw EXCEPTION on verification file
- */
- void verifyFile( const Pathname &path, const std::string &key) const;
-
unsigned _media_count;
// data dir we are using
else if (_do_location != "" && _do_location != "/")
{
Pathname script = source().provideFile(_do_location, _do_media);
- if (! YUMSourceImpl::checkCheckSum(script, _do_checksum.type(), _do_checksum.checksum()))
+ if (! filesystem::is_checksum(script, _do_checksum))
{
ZYPP_THROW(Exception(N_("Failed check for the script file check sum")));
}
else if (_undo_location != "" && _undo_location != "/")
{
Pathname script = source().provideFile(_undo_location, _undo_media);
- if (! YUMSourceImpl::checkCheckSum(script, _undo_checksum.type(), _undo_checksum.checksum()))
+ if (! filesystem::is_checksum(script, _undo_checksum) )
{
ZYPP_THROW(Exception(N_("Failed check for the script file check sum")));
}
if ((*repomd)->type == "other") // don't parse 'other.xml' (#159316)
continue;
- Pathname src;
- try
- {
- src = provideFile(_path + (*repomd)->location);
- }
- catch (const Exception &e)
- {
- ZYPP_THROW(Exception("Can't provide " + _path.asString() + (*repomd)->location + " from " + url().asString() ));
- }
-
- Pathname dst = local_dir + (*repomd)->location;
-
- //if (0 != assert_dir(dst, 0755))
- // ZYPP_THROW(Exception("Cannot create directory: " + dst.asString()));
-
- if ( filesystem::copy(src, dst) != 0 )
- ZYPP_THROW(Exception("Can't copy " + src.asString() + " to " + dst.asString()));
-
- if (! checkCheckSum( dst, (*repomd)->checksumType, (*repomd)->checksum))
- ZYPP_THROW(Exception( (*repomd)->location + " " + N_(" fails checksum verification.") ));
-
+ getPossiblyCachedMetadataFile( _path + (*repomd)->location, local_dir + (*repomd)->location, _cache_dir + (*repomd)->location, CheckSum((*repomd)->checksumType, (*repomd)->checksum) );
// if it is a patch, we read the patches individually
if ((*repomd)->type == "patches")
{
// use the local copy now
- Pathname patches_list = dst;
+ Pathname patches_list = local_dir + (*repomd)->location;
MIL << "Reading patches file " << patches_list << std::endl;
ifgzstream st ( patches_list.asString().c_str() );
YUMPatchesParser patch(st, "");
for (; !patch.atEnd(); ++patch)
{
- string filename = (*patch)->location;
- Pathname patch_src;
- Pathname patch_dst;
- try
- {
- patch_src = provideFile(_path + filename);
- }
- catch (const Exception &e)
- {
- ZYPP_CAUGHT(e);
- ZYPP_THROW(Exception("Can't provide patch " + _path.asString() + (*repomd)->location + " from " + url().asString()));
- }
-
- patch_dst = local_dir + filename;
- if ( filesystem::copy(patch_src, patch_dst) != 0 )
- ZYPP_THROW(Exception("Can't copy patch file " + patch_src.asString() + " to " + patch_dst.asString()));
-
- // check patch checksum
- if (! checkCheckSum( patch_dst, (*patch)->checksumType, (*patch)->checksum))
- ZYPP_THROW(Exception( (*repomd)->location + " " + N_(" fails checksum verification.") ));
+ getPossiblyCachedMetadataFile( _path + (*patch)->location, local_dir + (*patch)->location, _cache_dir + (*patch)->location, CheckSum((*patch)->checksumType, (*patch)->checksum) );
} // end of single patch parsing
}// end of patches file parsing
} // end of copying
else
{
Pathname file_to_check = metadataRoot() + _path + (*repomd)->location;
- if (! checkCheckSum( file_to_check, (*repomd)->checksumType, (*repomd)->checksum))
+ if (! filesystem::is_checksum( file_to_check, CheckSum((*repomd)->checksumType, (*repomd)->checksum)))
{
ZYPP_THROW(Exception( (*repomd)->location + " " + N_("fails checksum verification.") ));
}
for (; !patch.atEnd(); ++patch)
{
Pathname patch_filename = metadataRoot() + _path + (*patch)->location;
- if (! checkCheckSum(patch_filename, (*patch)->checksumType, (*patch)->checksum))
+ if (! filesystem::is_checksum(patch_filename, CheckSum((*patch)->checksumType, (*patch)->checksum)))
{
ZYPP_THROW(Exception( (*patch)->location + " " + N_("fails checksum verification.") ));
}
return cap;
}
-
-
-
- bool YUMSourceImpl::checkCheckSum (const Pathname & filename, std::string csum_type, const std::string & csum)
- {
- MIL << "Checking checksum for " << filename << " as type: " << csum_type << "; value: " << csum << endl;
- if (str::toLower(csum_type) == "sha")
- {
- if (csum.size() == 40)
- csum_type = "sha1";
- else if (csum.size() == 64)
- csum_type = "sha256";
- DBG << "Checksum size is " << csum.size() << ", checksum type set to " << csum_type << endl;
- }
- ifstream st(filename.asString().c_str());
- std::string dig = Digest::digest (csum_type, st, 4096);
- if (dig == "")
- {
- ERR << "Cannot compute the checksum" << endl;
- return false;
- }
- dig = str::toLower (dig);
- bool ret = (dig == str::toLower(csum));
- if (ret)
- {
- MIL << "Checksums are the same" << endl;
- return true;
- }
- else
- {
- WAR << "Checksum missmatch: metadata: " << csum << "; real: " << dig << endl;
- return false;
- }
- return false;
- }
-
} // namespace yum
/////////////////////////////////////////////////////////////////
} // namespace source
const Pathname metadataRoot() const;
bool cacheExists();
+
const TmpDir downloadMetadata();
void saveMetadataTo(const Pathname & dir_r);
const Pathname repomdFile() const;
typedef std::map<zypp::NVRA, ImplAndPackage> PackageImplMapT;
PackageImplMapT _package_impl;
- public:
- static bool checkCheckSum (const Pathname & filename, std::string csum_type, const std::string & csum);
-
};
///////////////////////////////////////////////////////////////////
srclist_r.swap( collect._toSrcinstall );
}
+ Date TargetImpl::timestamp() const
+ {
+ Date ts_rpm;
+ Date ts_store;
+
+ PathInfo rpmdb_info(root() + "/var/lib/rpm/Packages");
+ if ( rpmdb_info.isExist() )
+ ts_rpm = rpmdb_info.mtime();
+
+ if ( isStorageEnabled() )
+ ts_store = _storage.timestamp();
+
+ if ( ts_rpm > ts_store )
+ {
+ return ts_rpm;
+ }
+ else if (ts_rpm < ts_store)
+ {
+ return ts_store;
+ }
+ else
+ {
+ // they are the same
+ if ( ts_rpm != 0 )
+ return ts_rpm;
+ else
+ return Date::now();
+ }
+ }
+
/////////////////////////////////////////////////////////////////
} // namespace target
///////////////////////////////////////////////////////////////////
/** Set the log file for target */
bool setInstallationLogfile(const Pathname & path_r);
+ /** return the last modification date of the target */
+ Date timestamp() const;
+
protected:
/** All resolvables provided by the target. */
ResStore _store;
virtual void initBackend() = 0;
/**
+ * timestamp of last modification
+ */
+ virtual Date timestamp() const = 0;
+
+ /**
* Stores a Resolvable in the active backend.
*/
virtual void storeObject( ResObject::constPtr resolvable ) = 0;
d->backend->doTest();
}
+Date
+PersistentStorage::timestamp() const
+{
+ return d->backend->timestamp();
+}
+
void
PersistentStorage::storeObject( ResObject::constPtr resolvable )
{
#include "zypp/base/PtrTypes.h"
#include <zypp/Pathname.h>
#include <zypp/Url.h>
+#include <zypp/Date.h>
#include <zypp/Patch.h>
///////////////////////////////////////////////////////////////////
*/
bool isInitialized() const;
+ /**
+ * last modification
+ */
+ Date timestamp() const;
+
/**
* Stores a Resolvable in the active backend.
*/
}
}
+Date XMLFilesBackend::timestamp() const
+{
+ PathInfo ts_info = PathInfo( d->root + Pathname(ZYPP_DB_DIR) + "timestamp" );
+ if ( ts_info.isExist() )
+ {
+ return Date(ts_info.mtime());
+ }
+ else
+ {
+ updateTimestamp();
+ return Date::now();
+ }
+}
+
// Taken from KApplication
int XMLFilesBackend::random() const
{
}
catch( std::exception &e )
{
- //ZYPP_RETHROW(e);
+ ZYPP_THROW (Exception( "Can't write flags to store") );
}
+ updateTimestamp();
}
std::set<std::string>
return _flags;
}
+void
+XMLFilesBackend::updateTimestamp() const
+{
+ Pathname filename = d->root + Pathname(ZYPP_DB_DIR) + "timestamp";
+ std::ofstream file(filename.asString().c_str(), std::ios::out);
+ if (!file)
+ {
+ ZYPP_THROW (Exception( "Can't open timestamp file " + filename.asString() ) );
+ }
+ file.close();
+}
+
/////////////////////////////////////////////////////////
// Resolvables storage
////////////////////////////////////////////////////////
ERR << "Error saving resolvable " << resolvable << std::endl;
ZYPP_THROW(Exception(e.what()));
}
+ updateTimestamp();
}
void
ERR << "Error removing resolvable " << resolvable << std::endl;
ZYPP_THROW(Exception("Error deleting " + filename));
}
+ updateTimestamp();
}
catch(std::exception &e)
{
ERR << "Error saving source " << data.alias << " in the cache" << std::endl;
ZYPP_THROW(Exception(e.what()));
}
+ updateTimestamp();
}
void
ERR << "Error deleting source " << alias << " in the cache" << std::endl;
ZYPP_THROW(Exception(e.what()));
}
+ updateTimestamp();
}
/////////////////////////////////////////////////////////////////
* initialize the storage backend
*/
virtual void initBackend();
+
+ virtual Date timestamp() const;
+
/**
* Stores a Resolvable in the active backend.
*/
void writeFlagsInFile( const std::string &filename, const std::set<std::string> &pflags );
std::set<std::string> flagsFromFile( const std::string &filename ) const;
+
+ void updateTimestamp() const;
/////////////////////////////////////////////////////////
// SOURCES API
////////////////////////////////////////////////////////
projects / platform / upstream / libzypp.git / commitdiff