1 /*---------------------------------------------------------------------\
3 | |__ / \ / / . \ . \ |
8 \---------------------------------------------------------------------*/
9 /** \file zypp/media/MediaAria2c.cc
17 #include <boost/lexical_cast.hpp>
19 #include "zypp/base/Logger.h"
20 #include "zypp/ExternalProgram.h"
21 #include "zypp/ProgressData.h"
22 #include "zypp/base/String.h"
23 #include "zypp/base/Gettext.h"
24 #include "zypp/base/Sysconfig.h"
25 #include "zypp/base/Gettext.h"
26 #include "zypp/ZYppCallbacks.h"
28 #include "zypp/Edition.h"
29 #include "zypp/Target.h"
30 #include "zypp/ZYppFactory.h"
31 #include "zypp/ZConfig.h"
33 #include "zypp/media/MediaAria2c.h"
34 #include "zypp/media/proxyinfo/ProxyInfos.h"
35 #include "zypp/media/ProxyInfo.h"
36 #include "zypp/media/MediaUserAuth.h"
37 #include "zypp/media/MediaCurl.h"
38 #include "zypp/thread/Once.h"
40 #include <sys/types.h>
42 #include <sys/mount.h>
46 #include <boost/format.hpp>
48 #define DETECT_DIR_INDEX 0
49 #define CONNECT_TIMEOUT 60
50 #define TRANSFER_TIMEOUT 60 * 3
51 #define TRANSFER_TIMEOUT_MAX 60 * 60
53 #define ARIA_BINARY "aria2c"
56 using namespace zypp::base;
63 Pathname MediaAria2c::_cookieFile = "/var/lib/YaST2/cookies";
64 std::string MediaAria2c::_aria2cVersion = "WE DON'T KNOW ARIA2C VERSION";
66 //check if aria2c is present in the system
68 MediaAria2c::existsAria2cmd()
70 static const char* argv[] =
76 ExternalProgram aria( argv, ExternalProgram::Stderr_To_Stdout );
77 return( aria.close() == 0 );
81 * comannd line for aria.
82 * The argument list gets passed as reference
85 void fillAriaCmdLine( const string &ariaver,
86 const TransferSettings &s,
88 const Pathname &destination,
89 ExternalProgram::Arguments &args )
91 args.push_back(ARIA_BINARY);
92 args.push_back(str::form("--user-agent=%s", s.userAgentString().c_str()));
93 args.push_back("--summary-interval=1");
94 args.push_back("--follow-metalink=mem");
95 args.push_back("--check-integrity=true");
96 args.push_back("--file-allocation=none");
98 // save the stats of the mirrors and use them as input later
99 Pathname statsFile = ZConfig::instance().repoCachePath() / "aria2.stats";
100 args.push_back(str::form("--server-stat-of=%s", statsFile.c_str()));
101 args.push_back(str::form("--server-stat-if=%s", statsFile.c_str()));
102 args.push_back("--uri-selector=adaptive");
104 // only present in recent aria lets find out the aria version
105 vector<string> fields;
106 // "aria2c version x.x"
107 str::split( ariaver, std::back_inserter(fields));
108 if ( fields.size() == 3 )
110 if ( Edition(fields[2]) >= Edition("1.1.2") )
111 args.push_back( "--use-head=false");
114 if ( s.maxDownloadSpeed() > 0 )
115 args.push_back(str::form("--max-download-limit=%ld", s.maxDownloadSpeed()));
116 if ( s.minDownloadSpeed() > 0 )
117 args.push_back(str::form("--lowest-speed-limit=%ld", s.minDownloadSpeed()));
119 args.push_back(str::form("--max-tries=%ld", s.maxSilentTries()));
121 if ( Edition(fields[2]) < Edition("1.2.0") )
122 WAR << "aria2c is older than 1.2.0, some features may be disabled" << endl;
124 // TODO make this one configurable
125 args.push_back(str::form("--max-concurrent-downloads=%ld", s.maxConcurrentConnections()));
127 // add the anonymous id.
128 for ( TransferSettings::Headers::const_iterator it = s.headersBegin();
129 it != s.headersEnd();
131 args.push_back(str::form("--header=%s", it->c_str() ));
133 args.push_back( str::form("--connect-timeout=%ld", s.timeout()));
135 if ( s.username().empty() )
137 if ( url.getScheme() == "ftp" )
140 args.push_back(str::form("--ftp-user=%s", "suseuser" ));
141 args.push_back(str::form("--ftp-passwd=%s", VERSION ));
145 DBG << "Anonymous FTP identification: '" << id << "'" << endl;
150 if ( url.getScheme() == "ftp" )
151 args.push_back(str::form("--ftp-user=%s", s.username().c_str() ));
152 else if ( url.getScheme() == "http" ||
153 url.getScheme() == "https" )
154 args.push_back(str::form("--http-user=%s", s.username().c_str() ));
156 if ( s.password().size() )
158 if ( url.getScheme() == "ftp" )
159 args.push_back(str::form("--ftp-passwd=%s", s.password().c_str() ));
160 else if ( url.getScheme() == "http" ||
161 url.getScheme() == "https" )
162 args.push_back(str::form("--http-passwd=%s", s.password().c_str() ));
166 if ( s.proxyEnabled() )
168 args.push_back(str::form("--http-proxy=%s", s.proxy().c_str() ));
169 if ( ! s.proxyUsername().empty() )
171 args.push_back(str::form("--http-proxy-user=%s", s.proxyUsername().c_str() ));
172 if ( ! s.proxyPassword().empty() )
173 args.push_back(str::form("--http-proxy-passwd=%s", s.proxyPassword().c_str() ));
177 if ( ! destination.empty() )
178 args.push_back(str::form("--dir=%s", destination.c_str()));
180 args.push_back(url.asString().c_str());
183 const char *const MediaAria2c::agentString()
185 // we need to add the release and identifier to the
187 // The target could be not initialized, and then this information
189 Target_Ptr target = zypp::getZYpp()->getTarget();
191 static const std::string _value(
195 , MediaAria2c::_aria2cVersion.c_str()
196 , Target::targetDistribution( Pathname()/*guess root*/ ).c_str()
199 return _value.c_str();
204 MediaAria2c::MediaAria2c( const Url & url_r,
205 const Pathname & attach_point_hint_r )
206 : MediaCurl( url_r, attach_point_hint_r )
208 MIL << "MediaAria2c::MediaAria2c(" << url_r << ", " << attach_point_hint_r << ")" << endl;
210 _aria2cVersion = getAria2cVersion();
213 void MediaAria2c::attachTo (bool next)
215 MediaCurl::attachTo(next);
216 _settings.setUserAgentString(agentString());
220 MediaAria2c::checkAttachPoint(const Pathname &apoint) const
222 return MediaCurl::checkAttachPoint( apoint );
225 void MediaAria2c::disconnectFrom()
227 MediaCurl::disconnectFrom();
230 void MediaAria2c::releaseFrom( const std::string & ejectDev )
232 MediaCurl::releaseFrom(ejectDev);
235 static Url getFileUrl(const Url & url, const Pathname & filename)
238 string path = url.getPathName();
239 if ( !path.empty() && path != "/" && *path.rbegin() == '/' &&
240 filename.absolute() )
242 // If url has a path with trailing slash, remove the leading slash from
243 // the absolute file name
244 path += filename.asString().substr( 1, filename.asString().size() - 1 );
246 else if ( filename.relative() )
248 // Add trailing slash to path, if not already there
249 if (path.empty()) path = "/";
250 else if (*path.rbegin() != '/' ) path += "/";
251 // Remove "./" from begin of relative file name
252 path += filename.asString().substr( 2, filename.asString().size() - 2 );
256 path += filename.asString();
259 newurl.setPathName(path);
263 void MediaAria2c::getFile( const Pathname & filename ) const
265 // Use absolute file name to prevent access of files outside of the
266 // hierarchy below the attach point.
267 getFileCopy(filename, localPath(filename).absolutename());
270 void MediaAria2c::getFileCopy( const Pathname & filename , const Pathname & target) const
272 callback::SendReport<DownloadProgressReport> report;
274 Url fileurl(getFileUrl(_url, filename));
278 ExternalProgram::Arguments args;
280 fillAriaCmdLine(_aria2cVersion, _settings, fileurl, target.dirname(), args);
286 report->start(fileurl, target.asString() );
288 ExternalProgram aria(args, ExternalProgram::Stderr_To_Stdout);
290 // progress line like: [#1 SIZE:8.3MiB/10.1MiB(82%) CN:5 SPD:6899.88KiB/s]
291 // but since 1.4.0: [#1 SIZE:8.3MiB/10.1MiB(82%) CN:5 SPD:899.8KiBs]
292 // (bnc #513944) [#1 SIZE:8.3MiB/10.1MiB(82%) CN:5 SPD:3.8MiBs]
293 // we save it until we find a string with FILE: later
296 // file line, which tell which file is the previous progress
297 // ie: FILE: ./packages.FL.gz
298 double average_speed = 0;
299 long average_speed_count = 0;
301 // here we capture aria output exceptions
302 vector<string> ariaExceptions;
304 // TODO: Detect partial downloads!
305 bool partialDownload = false; // Whether it makes sense to retry with --continue!
308 for(std::string ariaResponse( aria.receiveLine());
309 ariaResponse.length();
310 ariaResponse = aria.receiveLine())
312 //cout << ariaResponse;
313 string line = str::trim(ariaResponse);
315 // look for the progress line and save it for later
316 if ( str::hasPrefix(line, "[#") )
320 // save error messages for later
321 else if ( str::hasPrefix(line, "Exception: ") )
323 // for auth exception, we throw
324 if (!line.substr(0,31).compare("Exception: Authorization failed") )
326 ZYPP_THROW(MediaUnauthorizedException(
327 _url, "Login failed.", "Login failed", "auth hint"
330 // otherwise, remember the error
331 string excpMsg = line.substr(10, line.size());
332 DBG << "aria2c reported: '" << excpMsg << "'" << endl;
333 ariaExceptions.push_back(excpMsg);
335 else if ( str::hasPrefix(line, "FILE: ") )
338 Pathname theFile(line.substr(6, line.size()));
339 // is the report about the filename we are downloading?
340 // aria may report progress about metalinks, torrent and
341 // other stuff which is not the main transfer
342 if ( theFile == target )
344 // once we find the FILE: line, progress has to be
346 if ( ! progressLine.empty() )
348 // get the percentage (progress) data
350 size_t left_bound = progressLine.find('(',0) + 1;
351 size_t count = progressLine.find('%',left_bound) - left_bound;
352 string progressStr = progressLine.substr(left_bound, count);
354 if ( count != string::npos )
355 progress = std::atoi(progressStr.c_str());
357 ERR << "Can't parse progress from '" << progressStr << "'" << endl;
360 double current_speed = 0;
361 left_bound = progressLine.find("SPD:",0) + 4;
362 count = progressLine.find("KiB",left_bound);
363 bool kibs = true; // KiBs ? (MiBs if false)
364 if ( count == string::npos ) // try MiBs
366 count = progressLine.find("MiBs",left_bound);
369 if ( count != string::npos )
370 { // convert the string to a double
372 string speedStr = progressLine.substr(left_bound, count);
374 current_speed = boost::lexical_cast<double>(speedStr);
376 catch (const std::exception&) {
377 ERR << "Can't parse speed from '" << speedStr << "'" << endl;
382 // we have a new average speed
383 average_speed_count++;
385 // this is basically A: average
386 // ((n-1)A(n-1) + Xn)/n = A(n)
388 (((average_speed_count - 1 )*average_speed) + current_speed)
389 / average_speed_count;
391 // note that aria report speed in kBps or MBps, while the report takes Bps
392 report->progress ( progress, fileurl,
393 average_speed * (kibs ? 0x400 : 0x10000),
394 current_speed * (kibs ? 0x400 : 0x10000));
395 // clear the progress line to detect mismatches between
396 // [# and FILE: lines
397 progressLine.clear();
401 WAR << "aria2c reported a file, but no progress data available" << endl;
407 DBG << "Progress is not about '" << target << "' but '" << theFile << "'" << endl;
412 // other line type, just ignore it.
416 int code = aria.close();
425 MediaTimeoutException e(_url);
426 for_(it, ariaExceptions.begin(), ariaExceptions.end())
431 case 4: // max notfound reached
433 MediaFileNotFoundException e(_url, filename);
434 for_(it, ariaExceptions.begin(), ariaExceptions.end())
439 case 6: // network problem
440 case 7: // unfinished downloads (ctr-c)
444 if ( partialDownload )
446 // Ask for retry on partial downloads, when it makes sense to retry with --continue!
447 // Other errors are handled by the layers above.
448 MediaException e(str::form(_("Download interrupted at %d%%"), progress ));
449 for_(it, ariaExceptions.begin(), ariaExceptions.end())
452 DownloadProgressReport::Action action = report->problem( _url, DownloadProgressReport::ERROR, e.asUserHistory() );
453 if ( action == DownloadProgressReport::RETRY )
460 // TranslatorExplanation: Failed to download <FILENAME> from <SERVERURL>.
461 MediaException e(str::form(_("Failed to download %s from %s"), filename.c_str(), _url.asString().c_str()));
462 for_(it, ariaExceptions.begin(), ariaExceptions.end())
471 // retry with proper authentication data
472 catch (MediaUnauthorizedException & ex_r)
474 if(authenticate(ex_r.hint(), !retry))
478 report->finish(fileurl, zypp::media::DownloadProgressReport::ACCESS_DENIED, ex_r.asUserHistory());
483 // unexpected exception
484 catch (MediaException & excpt_r)
486 // FIXME: error number fix
487 report->finish(fileurl, zypp::media::DownloadProgressReport::ERROR, excpt_r.asUserHistory());
488 ZYPP_RETHROW(excpt_r);
493 report->finish(fileurl, zypp::media::DownloadProgressReport::NO_ERROR, "");
496 bool MediaAria2c::getDoesFileExist( const Pathname & filename ) const
498 return MediaCurl::getDoesFileExist(filename);
501 bool MediaAria2c::doGetDoesFileExist( const Pathname & filename ) const
503 return MediaCurl::doGetDoesFileExist(filename);
506 void MediaAria2c::getDir( const Pathname & dirname, bool recurse_r ) const
508 MediaCurl::getDir(dirname, recurse_r);
511 bool MediaAria2c::authenticate(const std::string & availAuthTypes, bool firstTry) const
517 void MediaAria2c::getDirInfo( std::list<std::string> & retlist,
518 const Pathname & dirname, bool dots ) const
520 getDirectoryYast( retlist, dirname, dots );
523 void MediaAria2c::getDirInfo( filesystem::DirContent & retlist,
524 const Pathname & dirname, bool dots ) const
526 getDirectoryYast( retlist, dirname, dots );
529 std::string MediaAria2c::getAria2cVersion()
531 static const char* argv[] =
537 ExternalProgram aria(argv, ExternalProgram::Stderr_To_Stdout);
538 std::string vResponse( str::trim( aria.receiveLine() ) );