From fda8b348618f47f9da3743bbde02ce9c64d39417 Mon Sep 17 00:00:00 2001 From: Michael Andres Date: Fri, 3 Feb 2012 13:17:32 +0100 Subject: [PATCH] Add Fetcher support for CHECKSUMS file (replaceing SHA1SUMS) (bnc#744302) --- tools/ProvideSignedDirectory.cc | 55 +++++++++++++++++++++++ zypp/Fetcher.cc | 99 ++++++++++++++++++++++------------------- zypp/Fetcher.h | 55 +++++++++++++---------- 3 files changed, 139 insertions(+), 70 deletions(-) create mode 100644 tools/ProvideSignedDirectory.cc diff --git a/tools/ProvideSignedDirectory.cc b/tools/ProvideSignedDirectory.cc new file mode 100644 index 0000000..119220d --- /dev/null +++ b/tools/ProvideSignedDirectory.cc @@ -0,0 +1,55 @@ +#include + +#include +#include +#include + +#include +#include +#include +#include + + +using std::endl; + +int main ( int argc, const char * argv[] ) +try { + --argc; + ++argv; + //zypp::base::LogControl::instance().logToStdErr(); + INT << "===[START]==========================================" << endl; + /////////////////////////////////////////////////////////////////// + + zypp::Url oRemoteUrl( argv[0] ); //"http://download.opensuse.org/distribution/openSUSE-current/repo/oss" ); + std::string oRemoteDir( argv[1] ); //"/suse/setup/slide" ); + const bool oRecursive( true ); + zypp::Pathname oLocalDir( "" ); + + zypp::scoped_ptr tmpdir; + if ( oLocalDir.empty() ) + { + tmpdir.reset( new zypp::filesystem::TmpDir ); + oLocalDir = tmpdir->path(); + } + + zypp::Fetcher fetcher; + fetcher.setOptions( zypp::Fetcher::AutoAddIndexes ); + fetcher.enqueueDir( zypp::OnMediaLocation( oRemoteDir ), oRecursive ); + + zypp::KeyRing::setDefaultAccept( zypp::KeyRing::TRUST_KEY_TEMPORARILY ); + zypp::MediaSetAccess media( oRemoteUrl, "/" ); + fetcher.start( oLocalDir, media ); + + zypp::ExternalProgram( "find "+(oLocalDir/oRemoteDir).asString()+" -ls" ) >> std::cout; + + /////////////////////////////////////////////////////////////////// + INT << "===[END]============================================" << endl << endl; + zypp::base::LogControl::instance().logNothing(); + return 0; +} +catch ( const zypp::Exception & exp ) +{ + INT << exp << endl << exp.historyAsString(); +} +catch (...) +{} diff --git a/zypp/Fetcher.cc b/zypp/Fetcher.cc index beebbcc..5f16dbf 100644 --- a/zypp/Fetcher.cc +++ b/zypp/Fetcher.cc @@ -176,10 +176,10 @@ namespace zypp */ void readIndex( const Pathname &index, const Pathname &basedir ); - /** specific version of \ref readIndex for SHA1SUMS file */ - void readSha1sumsIndex( const Pathname &index, const Pathname &basedir ); + /** specific version of \ref readIndex for CHECKSUMS file */ + void readChecksumsIndex( const Pathname &index, const Pathname &basedir ); - /** specific version of \ref readIndex for SHA1SUMS file */ + /** specific version of \ref readIndex for content file */ void readContentFileIndex( const Pathname &index, const Pathname &basedir ); /** reads the content of a directory but keeps a cache **/ @@ -425,35 +425,26 @@ namespace zypp const OnMediaLocation &resource, const Pathname &dest_dir ) { - if ( _options & AutoAddSha1sumsIndexes ) + auto fnc_addIfInContent( [&]( const std::string & index_r ) -> bool { - // only try to add an index if it exists - filesystem::DirEntry shafile; - shafile.name = "SHA1SUMS"; shafile.type = filesystem::FT_FILE; - if ( find( content.begin(), content.end(), shafile ) != content.end() ) - { - // add the index of this directory - OnMediaLocation indexloc(resource); - indexloc.changeFilename(resource.filename() + "SHA1SUMS"); - addIndex(indexloc); - // we need to read it now - downloadAndReadIndexList(media, dest_dir); - } + if ( find( content.begin(), content.end(), filesystem::DirEntry(index_r,filesystem::FT_FILE) ) == content.end() ) + return false; + // add the index of this directory + OnMediaLocation indexloc( resource ); + indexloc.changeFilename( resource.filename() + index_r ); + addIndex( indexloc ); + // we need to read it now + downloadAndReadIndexList( media, dest_dir ); + return true; + } ); + + if ( _options & AutoAddChecksumsIndexes ) + { + fnc_addIfInContent( "CHECKSUMS" ) || fnc_addIfInContent( "SHA1SUMS" ); } if ( _options & AutoAddContentFileIndexes ) { - // only try to add an index if it exists - filesystem::DirEntry contentfile; - contentfile.name = "content"; contentfile.type = filesystem::FT_FILE; - if ( find( content.begin(), content.end(), contentfile ) != content.end() ) - { - // add the index of this directory - OnMediaLocation indexloc(resource); - indexloc.changeFilename(resource.filename() + "content"); - addIndex(indexloc); - // we need to read it now - downloadAndReadIndexList(media, dest_dir); - } + fnc_addIfInContent( "content" ); } } @@ -488,7 +479,15 @@ namespace zypp // individual transfer jobs MIL << "Adding directory " << resource.filename() << endl; filesystem::DirContent content; - getDirectoryContent(media, resource, content); + try { + getDirectoryContent(media, resource, content); + } + catch ( media::MediaFileNotFoundException & exception ) + { + ZYPP_CAUGHT( exception ); + WAR << "Skiping subtree hidden at " << resource.filename() << endl; + return; + } // this method test for the option flags so indexes are added // only if the options are enabled @@ -498,8 +497,8 @@ namespace zypp it != content.end(); ++it ) { - // skip SHA1SUMS* as they were already retrieved - if ( str::hasPrefix(it->name, "SHA1SUMS") ) + // skip CHECKSUMS* as they were already retrieved + if ( str::hasPrefix(it->name, "CHECKSUMS") || str::hasPrefix(it->name, "SHA1SUMS") ) continue; Pathname filename = resource.filename() + it->name; @@ -609,8 +608,8 @@ namespace zypp // generic function for reading indexes void Fetcher::Impl::readIndex( const Pathname &index, const Pathname &basedir ) { - if ( index.basename() == "SHA1SUMS" ) - readSha1sumsIndex(index, basedir); + if ( index.basename() == "CHECKSUMS" || index.basename() == "SHA1SUMS" ) + readChecksumsIndex(index, basedir); else if ( index.basename() == "content" ) readContentFileIndex(index, basedir); else @@ -630,26 +629,34 @@ namespace zypp } } - // reads a SHA1SUMS file index - void Fetcher::Impl::readSha1sumsIndex( const Pathname &index, const Pathname &basedir ) + // reads a CHECKSUMS (old SHA1SUMS) file index + void Fetcher::Impl::readChecksumsIndex( const Pathname &index, const Pathname &basedir ) { std::ifstream in( index.c_str() ); - string buffer; if ( ! in.fail() ) { - while ( getline(in, buffer) ) + std::string buffer; + while ( getline( in, buffer ) ) { - vector words; - str::split( buffer, back_inserter(words) ); - if ( words.size() != 2 ) - ZYPP_THROW(Exception("Wrong format for SHA1SUMS file")); - //MIL << "check: '" << words[0] << "' | '" << words[1] << "'" << endl; - if ( ! words[1].empty() ) - _checksums[(basedir + words[1]).asString()] = CheckSum::sha1(words[0]); + + if ( buffer[0] == '#' ) + continue; // simple comment + + CheckSum checksum( str::stripFirstWord( buffer, /*ltrim before strip*/true ) ); + if ( checksum.empty() ) + continue; // empty line | unknown cheksum format + + if ( buffer.empty() ) + { + WAR << "Missing filename in CHECKSUMS file: " << index.asString() << " (" << checksum << ")" << endl; + continue; + } + + _checksums[(basedir/buffer).asString()] = checksum; } } else - ZYPP_THROW(Exception("Can't open SHA1SUMS file: " + index.asString())); + ZYPP_THROW(Exception("Can't open CHECKSUMS file: " + index.asString())); } void Fetcher::Impl::downloadIndex( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir) @@ -761,7 +768,7 @@ namespace zypp // may be this code can be factored out // together with the autodiscovery of indexes // of addDirJobs - if ( ( _options & AutoAddSha1sumsIndexes ) || + if ( ( _options & AutoAddChecksumsIndexes ) || ( _options & AutoAddContentFileIndexes ) ) { // if auto indexing is enabled, then we need to read the diff --git a/zypp/Fetcher.h b/zypp/Fetcher.h index 4cb8c54..0267967 100644 --- a/zypp/Fetcher.h +++ b/zypp/Fetcher.h @@ -88,12 +88,19 @@ namespace zypp * fetcher.reset(); * \endcode * - * Indexes are supported in SHA1SUMS format (simple text file) - * with sha1sums and file name, or content file, whith + * Indexes are supported in CHECKSUMS format (simple text file) + * with checksum and file name, or content file, whith * HASH SHA1 line entries. - * + * * \note The indexes file names are relative to the directory * where the index is. + * + * \note libzypp-11.x: Introduction of sha256 lead to the insight + * that using SHA1SUMS as filename was a bad choice. New media store + * the checksums in a CHECKSUMS file. If a CHECKSUMS file is not + * present, we fall back looking for a SHA1SUMS file. The checksum + * type (md5,sha1,sha256) is auto detected by looking at the cheksums + * length. No need to somehow encode it in the filename. */ class Fetcher { @@ -115,15 +122,15 @@ namespace zypp */ AutoAddContentFileIndexes = 0x0001, /** - * If a SHA1SUMS file is found, it is + * If a CHECKSUMS file is found, it is * downloaded and read. */ - AutoAddSha1sumsIndexes = 0x0002, + AutoAddChecksumsIndexes = 0x0002, /** - * If a content or SHA1SUMS file is found, + * If a content or CHECKSUMS file is found, * it is downloaded and read. */ - AutoAddIndexes = 0x0001 | 0x0002, + AutoAddIndexes = AutoAddContentFileIndexes | AutoAddChecksumsIndexes, }; ZYPP_DECLARE_FLAGS(Options, Option); @@ -171,7 +178,7 @@ namespace zypp * */ void addIndex( const OnMediaLocation &resource ); - + /** * Enqueue a object for transferal, they will not * be transfered until \ref start() is called @@ -179,7 +186,7 @@ namespace zypp */ void enqueue( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() ); - + /** * Enqueue a object for transferal, they will not * be transfered until \ref start() is called @@ -214,14 +221,14 @@ namespace zypp * using \ref AutoAddIndexes flag. * * Files are checked by providing a - * SHA1SUMS or content file listing + * CHECKSUMS or content file listing * filename - * and a respective SHA1SUMS.asc/content.asc which has + * and a respective CHECKSUMS.asc/content.asc which has * the signature for the checksums. * * If you expect the user to not have the key of * the signature either in the trusted or untrusted - * keyring, you can offer it as SHA1SUMS.key (or content.key) + * keyring, you can offer it as CHECKSUMS.key (or content.key) * * \param recursive True if the complete tree should * be enqueued. @@ -232,11 +239,11 @@ namespace zypp * so make sure you don't add another one or * the user could be asked twice. * - * \note The format of the file SHA1SUMS is the output of: - * ls | grep -v SHA1SUMS | xargs sha1sum > SHA1SUMS + * \note The format of the file CHECKSUMS is the output of: + * ls | grep -v CHECKSUMS | xargs sha256sum > CHECKSUMS * in each subdirectory. * - * \note Every file SHA1SUMS.* except of SHA1SUMS.(asc|key|(void)) will + * \note Every file CHECKSUMS.* except of CHECKSUMS.(asc|key|(void)) will * not be transfered and will be ignored. * */ @@ -259,14 +266,14 @@ namespace zypp * using \ref AutoAddIndexes flag. * * Files are checked by providing a - * SHA1SUMS or content file listing + * CHECKSUMS or content file listing * filename - * and a respective SHA1SUMS.asc/content.asc which has + * and a respective CHECKSUMS.asc/content.asc which has * the signature for the checksums. * * If you expect the user to not have the key of * the signature either in the trusted or untrusted - * keyring, you can offer it as SHA1SUMS.key (or content.key) + * keyring, you can offer it as CHECKSUMS.key (or content.key) * * \param recursive True if the complete tree should * be enqueued. @@ -276,30 +283,30 @@ namespace zypp * transfer job, so make sure you don't add another one or * the user could be asked twice. * - * \note The format of the file SHA1SUMS is the output of: - * ls | grep -v SHA1SUMS | xargs sha1sum > SHA1SUMS + * \note The format of the file CHECKSUMS is the output of: + * ls | grep -v CHECKSUMS | xargs sha256sum > CHECKSUMS * in each subdirectory. * - * \note Every file SHA1SUMS.* except of SHA1SUMS.(asc|key|(void)) will + * \note Every file CHECKSUMS.* except of CHECKSUMS.(asc|key|(void)) will * not be transfered and will be ignored. * */ void enqueueDigestedDir( const OnMediaLocation &resource, bool recursive = false, const FileChecker &checker = FileChecker() ); - + /** * adds a directory to the list of directories * where to look for cached files */ void addCachePath( const Pathname &cache_dir ); - + /** * Reset the transfer (jobs) list * \note It does not reset the cache directory list */ void reset(); - + /** * start the transfer to a destination directory * \a dest_dir -- 2.7.4