Do autoindexing for simple files to if enabled.

Warning, this is expensive, and I already wait for someone
to use it in the wrong way and then complain that it is not
efficient. If you want to transfer lot of files with one index
use enqueueDir, not enqueue

diff --git a/tests/zypp/Fetcher_test.cc b/tests/zypp/Fetcher_test.cc
index 69bd1c4..d3330c3 100644 (file)
--- a/tests/zypp/Fetcher_test.cc
+++ b/tests/zypp/Fetcher_test.cc
@@ -1,5 +1,6 @@
 #include <stdio.h>
 #include <iostream>
+#define BOOST_TEST_MODULE fetcher_test
 #include <boost/test/auto_unit_test.hpp>
 #include <boost/test/parameterized_test.hpp>
 #include <boost/test/unit_test_log.hpp>
@@ -23,6 +24,8 @@ using namespace boost::unit_test;
 
 #define DATADIR (Pathname(TESTS_SRC_DIR) + "/zypp/data/Fetcher/remote-site")
 
+BOOST_AUTO_TEST_SUITE( fetcher_test );
+
 BOOST_AUTO_TEST_CASE(fetcher_enqueuedir_noindex)
 {
   MediaSetAccess media( ( DATADIR).asUrl(), "/" );
@@ -121,7 +124,7 @@ BOOST_AUTO_TEST_CASE(fetcher_enqueuebrokendir_index)
 }
 
 
-BOOST_AUTO_TEST_CASE(fetcher_enqueue_digested_brokendir_with_index)
+BOOST_AUTO_TEST_CASE(fetcher_enqueue_digesteddir_brokendir_with_index)
 {
   MediaSetAccess media( ( DATADIR).asUrl(), "/" );
   // do the test by trusting the SHA1SUMS file signature key but with a broken file
@@ -137,6 +140,39 @@ BOOST_AUTO_TEST_CASE(fetcher_enqueue_digested_brokendir_with_index)
   }
 }
 
+BOOST_AUTO_TEST_CASE(fetcher_enqueue_digested_broken_with_autoindex)
+{
+  MediaSetAccess media( ( DATADIR).asUrl(), "/" );
+  // do the test by trusting the SHA1SUMS file signature key but with a broken file
+  {
+      filesystem::TmpDir dest;
+      Fetcher fetcher;
+      // add the key as trusted
+      getZYpp()->keyRing()->importKey(PublicKey(DATADIR + "/complexdir-broken/subdir1/SHA1SUMS.key"), true);
+      fetcher.setOptions( Fetcher::AutoAddIndexes );
+      fetcher.enqueueDigested(OnMediaLocation("/complexdir-broken/subdir1/subdir1-file1.txt"));
+      BOOST_CHECK_THROW( fetcher.start( dest.path(), media ), Exception);
+      fetcher.reset();
+  }
+}
+
+BOOST_AUTO_TEST_CASE(fetcher_enqueue_digested_with_autoindex)
+{
+  MediaSetAccess media( ( DATADIR).asUrl(), "/" );
+  // do the test by trusting the SHA1SUMS file signature key with a good file
+  // checksum in auto discovered index
+  {
+      filesystem::TmpDir dest;
+      Fetcher fetcher;
+      // add the key as trusted
+      getZYpp()->keyRing()->importKey(PublicKey(DATADIR + "/complexdir/subdir1/SHA1SUMS.key"), true);
+      fetcher.setOptions( Fetcher::AutoAddIndexes );
+      fetcher.enqueueDigested(OnMediaLocation("/complexdir/subdir1/subdir1-file1.txt"));
+      fetcher.start( dest.path(), media );
+      fetcher.reset();
+  }
+}
+
 
 BOOST_AUTO_TEST_CASE(fetcher_enqueuefile_noindex)
 {
@@ -197,9 +233,8 @@ BOOST_AUTO_TEST_CASE(enqueue_broken_content_index)
 {
   MediaSetAccess media( ( DATADIR).asUrl(), "/" );
   Fetcher fetcher;
-
+  filesystem::TmpDir dest;
   {
-        filesystem::TmpDir dest;
         OnMediaLocation loc("/contentindex-broken-digest/subdir1/subdir1-file1.txt",1);
         // key was already imported as trusted
         fetcher.addIndex(OnMediaLocation("/contentindex-broken-digest/content", 1));
@@ -210,28 +245,8 @@ BOOST_AUTO_TEST_CASE(enqueue_broken_content_index)
 
         // now retrieve a file that is modified, so the checksum has to fail
         loc = OnMediaLocation("/contentindex-broken-digest/subdir1/subdir1-file2.txt",1);
-        fetcher.enqueue(loc);
-        BOOST_CHECK_THROW( fetcher.start( dest.path(), media ), Exception);
-        fetcher.reset();
-  }
-}
-
-BOOST_AUTO_TEST_CASE(enqueue_digested_broken_content_index)
-{
-  MediaSetAccess media( ( DATADIR).asUrl(), "/" );
-  Fetcher fetcher;
-
-  {
-        filesystem::TmpDir dest;
-        OnMediaLocation loc("/contentindex-broken-digest/subdir1/subdir1-file1.txt",1);
-        // key was already imported as trusted
         fetcher.addIndex(OnMediaLocation("/contentindex-broken-digest/content", 1));
-        fetcher.enqueueDigested(loc);
-        fetcher.start(dest.path(), media);
-        fetcher.reset();
-        // now retrieve a file that is modified, so the checksum has to fail
-        loc = OnMediaLocation("/contentindex-broken-digest/subdir1/subdir1-file2.txt",1);
-        fetcher.enqueueDigested(loc);
+        fetcher.enqueue(loc);
         BOOST_CHECK_THROW( fetcher.start( dest.path(), media ), Exception);
         fetcher.reset();
   }
@@ -296,5 +311,6 @@ BOOST_AUTO_TEST_CASE(fetcher_remove)
   }
 }
 
+BOOST_AUTO_TEST_SUITE_END();
 
 // vim: set ts=2 sts=2 sw=2 ai et:

diff --git a/zypp/Fetcher.cc b/zypp/Fetcher.cc
index ed2c4d5..98e1be0 100644 (file)
--- a/zypp/Fetcher.cc
+++ b/zypp/Fetcher.cc
@@ -133,7 +133,7 @@ namespace zypp
       /**
        * download the indexes and reads them
        */
-      void readIndexes( MediaSetAccess &media, const Pathname &dest_dir);
+      void downloadAndReadIndexList( MediaSetAccess &media, const Pathname &dest_dir);
 
       /**
        * reads a downloaded index file and updates internal
@@ -150,6 +150,9 @@ namespace zypp
       /** specific version of \ref readIndex for SHA1SUMS file */
       void readContentFileIndex( const Pathname &index, const Pathname &basedir );
       
+      /** reads the content of a directory but keeps a cache **/
+      void getDirectoryContent( MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content );
+      
       /**
        * tries to provide the file represented by job into dest_dir by
        * looking at the cache. If success, returns true, and the desired
@@ -192,6 +195,9 @@ namespace zypp
     list<Pathname> _caches;
     // checksums read from the indexes
     map<string, CheckSum> _checksums;
+    // cache of dir contents
+    map<string, filesystem::DirContent> _dircontent;
+      
     Fetcher::Options _options;      
   };
   ///////////////////////////////////////////////////////////////////
@@ -269,6 +275,8 @@ namespace zypp
   {
     _resources.clear();
     _indexes.clear();
+    _checksums.clear();
+    _dircontent.clear();
   }
 
   void Fetcher::Impl::addCachePath( const Pathname &cache_dir )
@@ -404,7 +412,7 @@ namespace zypp
               indexloc.changeFilename(resource.filename() + "SHA1SUMS");
               addIndex(indexloc);
               // we need to read it now
-              readIndexes(media, dest_dir);
+              downloadAndReadIndexList(media, dest_dir);
           }
       }
       if ( _options & AutoAddContentFileIndexes )
@@ -419,10 +427,33 @@ namespace zypp
               indexloc.changeFilename(resource.filename() + "content");
               addIndex(indexloc);
               // we need to read it now
-              readIndexes(media, dest_dir);
+              downloadAndReadIndexList(media, dest_dir);
           }
       }
   }
+
+  void Fetcher::Impl::getDirectoryContent( MediaSetAccess &media,
+                                           const OnMediaLocation &resource,
+                                           filesystem::DirContent &content )
+  {
+      if ( _dircontent.find(resource.filename().asString()) 
+           != _dircontent.end() )
+      {
+          filesystem::DirContent filled(_dircontent[resource.filename().asString()]);
+          
+          std::copy(filled.begin(), filled.end(), std::back_inserter(content));
+      }
+      else
+      {
+          filesystem::DirContent tofill;
+          media.dirInfo( tofill,
+                         resource.filename(),
+                         false /* dots */,
+                         resource.medianr());
+          std::copy(tofill.begin(), tofill.end(), std::back_inserter(content));
+          _dircontent[resource.filename().asString()] = tofill;
+      }
+  }
     
   void Fetcher::Impl::addDirJobs( MediaSetAccess &media,
                                   const OnMediaLocation &resource,
@@ -432,7 +463,7 @@ namespace zypp
       // individual transfer jobs
       MIL << "Adding directory " << resource.filename() << endl;
       filesystem::DirContent content;
-      media.dirInfo( content, resource.filename(), false /* dots */, resource.medianr());
+      getDirectoryContent(media, resource, content);
       
       // this method test for the option flags so indexes are added
       // only if the options are enabled
@@ -462,6 +493,7 @@ namespace zypp
               }
               else
                   WAR << "Resource " << filename << " has no checksum in the index either." << endl;
+
               if ( flags & FetcherJob::AlwaysVerifyChecksum )
                   enqueueDigested(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm));
               else
@@ -499,6 +531,12 @@ namespace zypp
               ZYPP_THROW( Exception("Can't create " + dest_full_path.dirname().asString()));
         if ( filesystem::copy(tmp_file, dest_full_path ) != 0 )
         {
+          if ( ! PathInfo(tmp_file).isExist() )
+              ERR << tmp_file << " does not exist" << endl;
+          if ( ! PathInfo(dest_full_path.dirname()).isExist() )
+              ERR << dest_full_path.dirname() << " does not exist" << endl;
+
+          media.releaseFile(resource); //not needed anymore, only eat space
           ZYPP_THROW( Exception("Can't copy " + tmp_file.asString() + " to " + dest_dir.asString()));
         }
 
@@ -582,7 +620,7 @@ namespace zypp
   // this method takes all the user pointed indexes, gets them and also tries to
   // download their signature, and verify them. After that, its parses each one
   // to fill the checksum cache.
-  void Fetcher::Impl::readIndexes( MediaSetAccess &media, const Pathname &dest_dir)
+  void Fetcher::Impl::downloadAndReadIndexList( MediaSetAccess &media, const Pathname &dest_dir)
   {
       // if there is no indexes, then just return to avoid
       // the directory listing
@@ -646,7 +684,7 @@ namespace zypp
     ProgressData progress(_resources.size());
     progress.sendTo(progress_receiver);
 
-    readIndexes(media, dest_dir);
+    downloadAndReadIndexList(media, dest_dir);
 
     for ( list<FetcherJob_Ptr>::const_iterator it_res = _resources.begin(); it_res != _resources.end(); ++it_res )
     {
@@ -658,6 +696,26 @@ namespace zypp
           continue;
       }
 
+      // may be this code can be factored out
+      // together with the autodiscovery of indexes
+      // of addDirJobs
+      if ( ( _options & AutoAddSha1sumsIndexes ) ||
+           ( _options & AutoAddContentFileIndexes ) )
+      {
+          // if auto indexing is enabled, then we need to read the
+          // index for each file. We look only in the directory
+          // where the file is. this is expensive of course.
+          filesystem::DirContent content;
+          getDirectoryContent(media, (*it_res)->location.filename().dirname(), content);
+          // this method test for the option flags so indexes are added
+          // only if the options are enabled
+          MIL << "Autodiscovering signed indexes on '"
+              << (*it_res)->location.filename().dirname() << "' for '"
+              << (*it_res)->location.filename() << "'" << endl;
+          
+          autoaddIndexes(content, media, (*it_res)->location.filename().dirname(), dest_dir);
+      }        
+
       provideToDest(media, (*it_res)->location, dest_dir);
 
       // if the checksum is empty, but the checksum is in one of the