1 #include <zypp/zyppng/base/EventDispatcher>
2 #include <zypp/zyppng/media/network/downloader.h>
3 #include <zypp/zyppng/media/network/networkrequesterror.h>
4 #include <zypp/zyppng/media/network/networkrequestdispatcher.h>
5 #include <zypp/zyppng/media/network/request.h>
6 #include <zypp/TmpPath.h>
7 #include <zypp/PathInfo.h>
8 #include <zypp/ZConfig.h>
12 #include "WebServer.h"
14 #include <boost/test/unit_test.hpp>
15 #include <boost/test/data/test_case.hpp>
17 #define BOOST_TEST_REQ_SUCCESS(REQ) \
19 BOOST_REQUIRE_MESSAGE( REQ->state() == zyppng::Download::Success, zypp::str::Format(" %1% != zyppng::Download::Success (%2%)") % REQ->state() % REQ->errorString() ); \
20 BOOST_REQUIRE_EQUAL( REQ->lastRequestError().type(), zyppng::NetworkRequestError::NoError ); \
21 BOOST_REQUIRE( REQ->errorString().empty() ); \
24 #define BOOST_TEST_REQ_FAILED(REQ) \
26 BOOST_REQUIRE_EQUAL( REQ->state(), zyppng::Download::Failed ); \
27 BOOST_REQUIRE_NE( REQ->lastRequestError().type(), zyppng::NetworkRequestError::NoError ); \
28 BOOST_REQUIRE( !REQ->errorString().empty() ); \
31 namespace bdata = boost::unit_test::data;
33 bool withSSL[] = {true, false};
35 //read all contents of a file into a string)
36 std::string readFile ( const zypp::Pathname &file )
38 if ( ! zypp::PathInfo( file ).isFile() ) {
41 std::ifstream istr( file.asString().c_str() );
46 std::string str((std::istreambuf_iterator<char>(istr)),
47 std::istreambuf_iterator<char>());
52 BOOST_DATA_TEST_CASE( dltest_basic, bdata::make( withSSL ), withSSL)
54 auto ev = zyppng::EventDispatcher::createMain();
56 zyppng::Downloader downloader;
58 //make sure the data here is big enough to cross the threshold of 256 bytes so we get a progress signal emitted and not only the alive signal.
59 std::string dummyContent = "This is just some dummy content,\nto test downloading and signals.\n"
60 "This is just some dummy content,\nto test downloading and signals.\n"
61 "This is just some dummy content,\nto test downloading and signals.\n"
62 "This is just some dummy content,\nto test downloading and signals.\n";
64 WebServer web((zypp::Pathname(TESTS_SRC_DIR)/"data"/"dummywebroot").c_str(), 10001, withSSL );
65 web.addRequestHandler("getData", WebServer::makeResponse("200", dummyContent ) );
66 BOOST_REQUIRE( web.start() );
68 zypp::filesystem::TmpFile targetFile;
69 zyppng::Url weburl (web.url());
70 weburl.setPathName("/handler/getData");
71 zyppng::TransferSettings set = web.transferSettings();
73 bool gotStarted = false;
74 bool gotFinished = false;
75 bool gotProgress = false;
76 bool gotAlive = false;
77 off_t lastProgress = 0;
80 std::vector<zyppng::Download::State> allStates;
83 zyppng::Download::Ptr dl = downloader.downloadFile( weburl, targetFile.path(), dummyContent.length() );
86 dl->sigFinished().connect([&]( zyppng::Download & ){
91 dl->sigStarted().connect([&]( zyppng::Download & ){
95 dl->sigAlive().connect([&]( zyppng::Download &, off_t ){
99 dl->sigStateChanged().connect([&]( zyppng::Download &, zyppng::Download::State state ){
100 allStates.push_back( state );
103 dl->sigProgress().connect([&]( zyppng::Download &, off_t dltotal, off_t dlnow ){
105 lastProgress = dlnow;
111 BOOST_TEST_REQ_SUCCESS( dl );
112 BOOST_REQUIRE( gotStarted );
113 BOOST_REQUIRE( gotFinished );
114 BOOST_REQUIRE( gotProgress );
115 BOOST_REQUIRE( gotAlive );
116 BOOST_REQUIRE_EQUAL( totalDL, dummyContent.length() );
117 BOOST_REQUIRE_EQUAL( lastProgress, dummyContent.length() );
118 BOOST_REQUIRE ( allStates == std::vector<zyppng::Download::State>({zyppng::Download::Initializing,zyppng::Download::Running, zyppng::Download::Success}) );
124 std::string name; //<dataset name, used only in debug output if the test fails
125 std::string handlerPath; //< the webhandler path used to query the resource
126 std::vector< std::pair<int, std::string> > mirrors; //all mirrors injected into the metalink file
127 int expectedFileDownloads; //< how many downloads are direct file downloads
128 int expectedHandlerDownloads; //< how many started downloads are handler requests
129 std::vector<zyppng::Download::State> expectedStates;
130 bool expectSuccess; //< should the download work out?
132 std::ostream & operator<<( std::ostream & str ) const {
133 str << "MirrorSet{ " << name << " }";
138 namespace boost{ namespace test_tools{ namespace tt_detail{
140 struct print_log_value< MirrorSet > {
141 void operator()( std::ostream& str, MirrorSet const& set) {
142 set.operator<<( str );
147 std::vector< MirrorSet > generateMirr ()
149 std::vector< MirrorSet > res;
152 res.push_back( MirrorSet() );
153 res.back().name = "All good mirrors";
154 res.back().expectSuccess = true;
155 res.back().expectedHandlerDownloads = 1;
156 res.back().expectedFileDownloads = 9;
157 res.back().expectedStates = {zyppng::Download::Initializing, zyppng::Download::RunningMulti, zyppng::Download::Success};
158 for ( int i = 100 ; i >= 10; i -= 10 )
159 res.back().mirrors.push_back( std::make_pair( i, "/test.txt") );
162 res.push_back( MirrorSet() );
163 res.back().name = "Empty mirrors";
164 res.back().expectSuccess = true;
165 res.back().expectedHandlerDownloads = 10;
166 res.back().expectedFileDownloads = 0;
167 res.back().expectedStates = {zyppng::Download::Initializing, zyppng::Download::RunningMulti, zyppng::Download::Success};
169 //only broken mirrors:
170 res.push_back( MirrorSet() );
171 res.back().name = "All broken mirrors";
172 res.back().expectSuccess = true;
173 res.back().expectedHandlerDownloads = 2; //has to fall back to url handler download
174 res.back().expectedFileDownloads = 10; //should try all mirrors and fail
175 res.back().expectedStates = {zyppng::Download::Initializing, zyppng::Download::RunningMulti, zyppng::Download::Running, zyppng::Download::Success};
176 for ( int i = 100 ; i >= 10; i -= 10 )
177 res.back().mirrors.push_back( std::make_pair( i, "/doesnotexist.txt") );
179 //some broken mirrors:
180 res.push_back( MirrorSet() );
181 res.back().name = "Some broken mirrors less URLs than blocks";
182 res.back().expectSuccess = true;
183 res.back().expectedHandlerDownloads = 1;
184 res.back().expectedFileDownloads = 9 + 3; // 3 should fail due to broken mirrors
185 res.back().expectedStates = {zyppng::Download::Initializing, zyppng::Download::RunningMulti, zyppng::Download::Success};
186 for ( int i = 10 ; i >= 5; i-- ) {
188 res.back().mirrors.push_back( std::make_pair( i*10, "/doesnotexist.txt") );
190 res.back().mirrors.push_back( std::make_pair( i*10, "/test.txt") );
194 //some broken mirrors with more URLs than blocks:
195 res.push_back( MirrorSet() );
196 res.back().name = "Some broken mirrors more URLs than blocks";
197 res.back().expectSuccess = true;
198 res.back().expectedHandlerDownloads = 1;
199 //its not really possible to know how many times the downloads will fail, there are
200 //5 broken mirrors in the set, but if a working mirror is done before the last broken
201 //URL is picked from the dataset not all broken URLs will be used
202 res.back().expectedFileDownloads = -1;
203 res.back().expectedStates = {zyppng::Download::Initializing, zyppng::Download::RunningMulti, zyppng::Download::Success};
204 for ( int i = 10 ; i >= 1; i-- ) {
206 res.back().mirrors.push_back( std::make_pair( i*10, "/doesnotexist.txt") );
208 res.back().mirrors.push_back( std::make_pair( i*10, "/test.txt") );
212 //mirrors where some return a invalid block
213 res.push_back( MirrorSet() );
214 res.back().name = "Some mirrors return broken blocks";
215 res.back().expectSuccess = true;
216 res.back().expectedHandlerDownloads = 1;
217 //its not really possible to know how many times the downloads will fail, there are
218 //5 broken mirrors in the set, but if a working mirror is done before the last broken
219 //URL is picked from the dataset not all broken URLs will be used
220 res.back().expectedFileDownloads = -1;
221 res.back().expectedStates = {zyppng::Download::Initializing, zyppng::Download::RunningMulti, zyppng::Download::Success};
222 for ( int i = 10 ; i >= 1; i-- ) {
224 res.back().mirrors.push_back( std::make_pair( i*10, "/handler/random") );
226 res.back().mirrors.push_back( std::make_pair( i*10, "/test.txt") );
233 //create one URL line for a metalink template file
234 std::string makeUrl ( int pref, const zyppng::Url &url )
236 return ( zypp::str::Format( "<url preference=\"%1%\" location=\"de\" type=\"%2%\">%3%</url>" ) % pref % url.getScheme() % url );
240 static bool requestWantsMetaLink ( const WebServer::Request &req )
242 auto it = req.params.find( "HTTP_ACCEPT" );
243 if ( it != req.params.end() ) {
244 if ( (*it).second.find("application/metalink+xml") != std::string::npos ||
245 (*it).second.find("application/metalink4+xml") != std::string::npos ) {
252 //creates a request handler for the Mock WebServer that returns the metalink data
253 //specified in \a data if the request has the metalink accept handler
254 WebServer::RequestHandler makeMetaFileHandler ( const std::string *data )
256 return [ data ]( WebServer::Request &req ){
257 if ( requestWantsMetaLink( req ) ) {
258 req.rout << WebServer::makeResponseString( "200", { "Content-Type: application/metalink+xml; charset=utf-8\r\n" }, *data );
261 req.rout << "Location: /test.txt\r\n\r\n";
266 //creates a request handler for the Mock WebServer that returns a junk block of
267 //data for a range request, otherwise relocates the request
268 WebServer::RequestHandler makeJunkBlockHandler ( )
270 return [ ]( WebServer::Request &req ){
271 auto it = req.params.find( "HTTP_RANGE" );
272 if ( it != req.params.end() && zypp::str::startsWith( it->second, "bytes=" ) ) {
273 //bytes=786432-1048575
274 std::string range = it->second.substr( 6 ); //remove bytes=
275 size_t dash = range.find_first_of( "-" );
278 if ( dash != std::string::npos ) {
279 start = std::stoll( range.substr( 0, dash ) );
280 end = std::stoll( range.substr( dash+1 ) );
283 if ( start != -1 && end != -1 ) {
285 for ( off_t curr = 0; curr < ( end - start); curr++ ) {
288 req.rout << "Status: 206 Partial Content\r\n"
289 << "Accept-Ranges: bytes\r\n"
290 << "Content-Length: "<<( end - start)<<"\r\n\r\n"
295 req.rout << "Location: /test.txt\r\n\r\n";
300 int maxConcurrentDLs[] = { 1, 2, 4, 8, 10, 15 };
302 BOOST_DATA_TEST_CASE( test1, bdata::make( generateMirr() ) * bdata::make( withSSL ) * bdata::make( maxConcurrentDLs ) , elem, withSSL, maxDLs )
304 //each URL in the metalink file has a preference , a schema and of course the URL, we need to adapt those to our test setup
305 //so we generate the file on the fly from a template in the test data
306 std::string metaTempl = readFile ( zypp::Pathname(TESTS_SRC_DIR)/"/zyppng/data/downloader/test.txt.meta" );
307 BOOST_REQUIRE( !metaTempl.empty() );
309 auto ev = zyppng::EventDispatcher::createMain();
311 WebServer web((zypp::Pathname(TESTS_SRC_DIR)/"/zyppng/data/downloader").c_str(), 10001, withSSL );
312 BOOST_REQUIRE( web.start() );
314 zypp::filesystem::TmpFile targetFile;
315 zyppng::Downloader downloader;
316 downloader.requestDispatcher()->setMaximumConcurrentConnections( maxDLs );
318 //first metalink download, generate a fully valid one
319 zyppng::Url weburl (web.url());
320 weburl.setPathName("/handler/test.txt");
323 if ( elem.mirrors.size() ) {
324 for ( const auto &mirr : elem.mirrors ) {
325 zyppng::Url mirrUrl (web.url());
326 mirrUrl.setPathName( mirr.second );
327 urls += makeUrl( mirr.first, mirrUrl ) + "\n";
331 std::string metaFile = zypp::str::Format( metaTempl ) % urls;
332 web.addRequestHandler("test.txt", makeMetaFileHandler( &metaFile ) );
333 web.addRequestHandler("random", makeJunkBlockHandler( ) );
335 int expectedDownloads = elem.expectedHandlerDownloads + elem.expectedFileDownloads;
336 int startedDownloads = 0;
337 int finishedDownloads = 0;
338 bool downloadHadError = false;
339 bool gotProgress = false;
340 bool gotAlive = false;
341 bool gotMultiDLState = false;
342 std::vector<zyppng::Download::State> allStates;
344 off_t lastProgress = 0;
346 int countHandlerReq = 0; //the requests made to the handler slot
347 int countFileReq = 0; //the requests made to the file directly, e.g. a mirror read from the metalink file
349 auto dl = downloader.downloadFile( weburl, targetFile );
350 dl->settings() = web.transferSettings();
352 dl->dispatcher().sigDownloadStarted().connect( [&]( zyppng::NetworkRequestDispatcher &, zyppng::NetworkRequest &req){
354 if ( req.url() == weburl )
360 dl->dispatcher().sigDownloadFinished().connect( [&]( zyppng::NetworkRequestDispatcher &, zyppng::NetworkRequest &req ){
363 if ( !downloadHadError )
364 downloadHadError = req.hasError();
367 dl->sigFinished().connect([&]( zyppng::Download & ){
371 dl->sigAlive().connect([&]( zyppng::Download &, off_t dlnow ){
373 lastProgress = dlnow;
376 dl->sigProgress().connect([&]( zyppng::Download &, off_t dltotal, off_t dlnow ){
379 lastProgress = dlnow;
382 dl->sigStateChanged().connect([&]( zyppng::Download &, zyppng::Download::State state ){
383 if ( state == zyppng::Download::RunningMulti )
384 gotMultiDLState = true;
387 dl->sigStateChanged().connect([&]( zyppng::Download &, zyppng::Download::State state ){
388 allStates.push_back( state );
394 //std::cout << dl->errorString() << std::endl;
396 if ( elem.expectSuccess )
397 BOOST_TEST_REQ_SUCCESS( dl );
399 BOOST_TEST_REQ_FAILED ( dl );
401 if ( elem.expectedHandlerDownloads > -1 && elem.expectedFileDownloads > -1 )
402 BOOST_REQUIRE_EQUAL( startedDownloads, expectedDownloads );
404 BOOST_REQUIRE_EQUAL( startedDownloads, finishedDownloads );
405 BOOST_REQUIRE( gotAlive );
406 BOOST_REQUIRE( gotProgress );
407 BOOST_REQUIRE( gotMultiDLState );
408 BOOST_REQUIRE_EQUAL( lastProgress, 2148018 );
409 BOOST_REQUIRE_EQUAL( lastProgress, gotTotal );
411 if ( elem.expectedHandlerDownloads > -1 )
412 BOOST_REQUIRE_EQUAL( countHandlerReq, elem.expectedHandlerDownloads );
414 if ( elem.expectedFileDownloads > -1 )
415 BOOST_REQUIRE_EQUAL( countFileReq, elem.expectedFileDownloads );
417 if ( !elem.expectedStates.empty() )
418 BOOST_REQUIRE( elem.expectedStates == allStates );
423 // - correct expected filesize
424 // - invalid filesize
425 // - password handling and propagation
428 //creates a request handler that requires a authentication to work
429 WebServer::RequestHandler createAuthHandler ( )
431 return [ ]( WebServer::Request &req ){
433 auto it = req.params.find( "HTTP_AUTHORIZATION" );
434 bool authorized = false;
435 if ( it != req.params.end() )
436 authorized = ( it->second == "Basic dGVzdDp0ZXN0" );
439 req.rout << "Status: 401 Unauthorized\r\n"
440 "Content-Type: text/html; charset=utf-8\r\n"
441 "WWW-Authenticate: Basic realm=\"User Visible Realm\", charset=\"UTF-8\" \r\n"
443 "Sorry you are not authorized.";
447 if ( requestWantsMetaLink( req ) ) {
448 it = req.params.find( "HTTPS" );
449 if ( it != req.params.end() && it->second == "on" ) {
450 req.rout << "Status: 307 Temporary Redirect\r\n"
451 << "Cache-Control: no-cache\r\n"
452 << "Location: /auth-https.meta\r\n\r\n";
454 req.rout << "Status: 307 Temporary Redirect\r\n"
455 << "Cache-Control: no-cache\r\n"
456 << "Location: /auth-http.meta\r\n\r\n";
460 req.rout << "Status: 307 Temporary Redirect\r\n"
461 "Location: /test.txt\r\n\r\n";
466 BOOST_DATA_TEST_CASE( dltest_auth, bdata::make( withSSL ), withSSL )
468 //don't write or read creds from real settings dir
469 zypp::filesystem::TmpDir repoManagerRoot;
470 zypp::ZConfig::instance().setRepoManagerRoot( repoManagerRoot.path() );
472 auto ev = zyppng::EventDispatcher::createMain();
474 zyppng::Downloader downloader;
476 WebServer web((zypp::Pathname(TESTS_SRC_DIR)/"/zyppng/data/downloader").c_str(), 10001, withSSL );
477 BOOST_REQUIRE( web.start() );
479 zypp::filesystem::TmpFile targetFile;
480 zyppng::Url weburl (web.url());
481 weburl.setPathName("/handler/test.txt");
482 zyppng::TransferSettings set = web.transferSettings();
484 web.addRequestHandler( "test.txt", createAuthHandler() );
485 web.addRequestHandler( "quit", [ &ev ]( WebServer::Request & ){ ev->quit();} );
488 auto dl = downloader.downloadFile( weburl, targetFile.path() );
489 dl->settings() = set;
491 dl->sigFinished( ).connect([ &ev ]( zyppng::Download & ){
497 BOOST_TEST_REQ_FAILED( dl );
498 BOOST_REQUIRE_EQUAL( dl->lastRequestError().type(), zyppng::NetworkRequestError::Unauthorized );
502 auto dl = downloader.downloadFile( weburl, targetFile.path() );
503 dl->settings() = set;
505 int gotAuthRequest = 0;
507 dl->sigFinished( ).connect([ &ev ]( zyppng::Download & ){
511 dl->sigAuthRequired().connect( [&]( zyppng::Download &, zyppng::NetworkAuthData &auth, const std::string &availAuth ){
513 if ( gotAuthRequest >= 2 )
515 auth.setUsername("wrong");
516 auth.setPassword("credentials");
521 BOOST_TEST_REQ_FAILED( dl );
522 BOOST_REQUIRE_EQUAL( gotAuthRequest, 2 );
523 BOOST_REQUIRE_EQUAL( dl->lastRequestError().type(), zyppng::NetworkRequestError::AuthFailed );
527 int gotAuthRequest = 0;
528 std::vector<zyppng::Download::State> allStates;
529 auto dl = downloader.downloadFile( weburl, targetFile.path() );
530 dl->settings() = set;
532 dl->sigFinished( ).connect([ &ev ]( zyppng::Download & ){
536 dl->sigAuthRequired().connect( [&]( zyppng::Download &, zyppng::NetworkAuthData &auth, const std::string &availAuth ){
538 auth.setUsername("test");
539 auth.setPassword("test");
542 dl->sigStateChanged().connect([&]( zyppng::Download &, zyppng::Download::State state ){
543 allStates.push_back( state );
549 BOOST_TEST_REQ_SUCCESS( dl );
550 BOOST_REQUIRE_EQUAL( gotAuthRequest, 1 );
551 BOOST_REQUIRE ( allStates == std::vector<zyppng::Download::State>({zyppng::Download::Initializing, zyppng::Download::RunningMulti, zyppng::Download::Success}) );
555 //the creds should be in the credential manager now, we should not need to specify them again in the slot
556 bool gotAuthRequest = false;
557 auto dl = downloader.downloadFile( weburl, targetFile.path() );
558 dl->settings() = set;
560 dl->sigFinished( ).connect([ &ev ]( zyppng::Download & ){
564 dl->sigAuthRequired().connect( [&]( zyppng::Download &, zyppng::NetworkAuthData &auth, const std::string &availAuth ){
565 gotAuthRequest = true;
570 BOOST_TEST_REQ_SUCCESS( dl );
571 BOOST_REQUIRE( !gotAuthRequest );