#include <iostream>
#include <list>
-
+#include <vector>
+#include <fstream>
#include <boost/lexical_cast.hpp>
#include "zypp/base/Logger.h"
#include "zypp/Edition.h"
#include "zypp/Target.h"
#include "zypp/ZYppFactory.h"
+#include "zypp/ZConfig.h"
+
+#include "zypp/TmpPath.h"
#include "zypp/media/MediaAria2c.h"
#include "zypp/media/proxyinfo/ProxyInfos.h"
#define TRANSFER_TIMEOUT 60 * 3
#define TRANSFER_TIMEOUT_MAX 60 * 60
+#define ARIA_BINARY "aria2c"
using namespace std;
using namespace zypp::base;
{
Pathname MediaAria2c::_cookieFile = "/var/lib/YaST2/cookies";
-Pathname MediaAria2c::_aria2cPath = "/usr/local/bin/aria2c";
std::string MediaAria2c::_aria2cVersion = "WE DON'T KNOW ARIA2C VERSION";
//check if aria2c is present in the system
bool
MediaAria2c::existsAria2cmd()
{
- const char* argv[] =
- {
- "which",
- "aria2c",
- NULL
- };
-
- ExternalProgram aria(argv, ExternalProgram::Stderr_To_Stdout);
-
- for(std::string ariaResponse( aria.receiveLine());
- ariaResponse.length();
- ariaResponse = aria.receiveLine())
- {
- // nothing
- }
-
- return ( aria.close() == 0 );
+ static const char* argv[] =
+ {
+ ARIA_BINARY,
+ "--version",
+ NULL
+ };
+ ExternalProgram aria( argv, ExternalProgram::Stderr_To_Stdout );
+ return( aria.close() == 0 );
}
/**
* The argument list gets passed as reference
* and it is filled.
*/
-void fillAriaCmdLine( const Pathname &ariapath,
- const string &ariaver,
+void fillAriaCmdLine( const string &ariaver,
const TransferSettings &s,
+ filesystem::TmpPath &credentials,
const Url &url,
const Pathname &destination,
ExternalProgram::Arguments &args )
{
- args.push_back(ariapath.c_str());
+
+ // options that are not passed in the command line
+ // like credentials, every string is in the
+ // opt=val format
+ list<string> file_options;
+
+ args.push_back(ARIA_BINARY);
args.push_back(str::form("--user-agent=%s", s.userAgentString().c_str()));
args.push_back("--summary-interval=1");
args.push_back("--follow-metalink=mem");
args.push_back("--check-integrity=true");
args.push_back("--file-allocation=none");
+ // save the stats of the mirrors and use them as input later
+ Pathname statsFile = ZConfig::instance().repoCachePath() / "aria2.stats";
+ args.push_back(str::form("--server-stat-of=%s", statsFile.c_str()));
+ args.push_back(str::form("--server-stat-if=%s", statsFile.c_str()));
+ args.push_back("--uri-selector=adaptive");
+
// only present in recent aria lets find out the aria version
- vector<string> fields;
+ vector<string> fields;
// "aria2c version x.x"
str::split( ariaver, std::back_inserter(fields));
if ( fields.size() == 3 )
if ( Edition(fields[2]) >= Edition("1.1.2") )
args.push_back( "--use-head=false");
}
-
+
if ( s.maxDownloadSpeed() > 0 )
args.push_back(str::form("--max-download-limit=%ld", s.maxDownloadSpeed()));
if ( s.minDownloadSpeed() > 0 )
if ( Edition(fields[2]) < Edition("1.2.0") )
WAR << "aria2c is older than 1.2.0, some features may be disabled" << endl;
-
+
// TODO make this one configurable
args.push_back(str::form("--max-concurrent-downloads=%ld", s.maxConcurrentConnections()));
it != s.headersEnd();
++it )
args.push_back(str::form("--header=%s", it->c_str() ));
-
+
args.push_back( str::form("--connect-timeout=%ld", s.timeout()));
if ( s.username().empty() )
else
{
if ( url.getScheme() == "ftp" )
- args.push_back(str::form("--ftp-user=%s", s.username().c_str() ));
+ file_options.push_back(str::form("ftp-user=%s", s.username().c_str() ));
else if ( url.getScheme() == "http" ||
url.getScheme() == "https" )
- args.push_back(str::form("--http-user=%s", s.username().c_str() ));
-
+ file_options.push_back(str::form("http-user=%s", s.username().c_str() ));
+
if ( s.password().size() )
{
if ( url.getScheme() == "ftp" )
- args.push_back(str::form("--ftp-passwd=%s", s.password().c_str() ));
+ file_options.push_back(str::form("ftp-passwd=%s", s.password().c_str() ));
else if ( url.getScheme() == "http" ||
url.getScheme() == "https" )
- args.push_back(str::form("--http-passwd=%s", s.password().c_str() ));
+ file_options.push_back(str::form("http-passwd=%s", s.password().c_str() ));
}
}
-
+
if ( s.proxyEnabled() )
{
args.push_back(str::form("--http-proxy=%s", s.proxy().c_str() ));
if ( ! s.proxyUsername().empty() )
{
- args.push_back(str::form("--http-proxy-user=%s", s.proxyUsername().c_str() ));
+ file_options.push_back(str::form("http-proxy-user=%s", s.proxyUsername().c_str() ));
if ( ! s.proxyPassword().empty() )
- args.push_back(str::form("--http-proxy-passwd=%s", s.proxyPassword().c_str() ));
+ file_options.push_back(str::form("http-proxy-passwd=%s", s.proxyPassword().c_str() ));
}
}
if ( ! destination.empty() )
args.push_back(str::form("--dir=%s", destination.c_str()));
+ // now append the file if there are hidden options
+ if ( ! file_options.empty() )
+ {
+ filesystem::TmpFile tmp;
+ ofstream outs( tmp.path().c_str() );
+ for_( it, file_options.begin(), file_options.end() )
+ outs << *it << endl;
+ outs.close();
+
+ credentials = tmp;
+ args.push_back(str::form("--conf-path=%s", credentials.path().c_str()));
+ }
+
args.push_back(url.asString().c_str());
}
"ZYpp %s (%s) %s"
, VERSION
, MediaAria2c::_aria2cVersion.c_str()
- , target ? target->targetDistribution().c_str() : ""
+ , Target::targetDistribution( Pathname()/*guess root*/ ).c_str()
)
);
return _value.c_str();
: MediaCurl( url_r, attach_point_hint_r )
{
MIL << "MediaAria2c::MediaAria2c(" << url_r << ", " << attach_point_hint_r << ")" << endl;
-
- //At this point, we initialize aria2c path
- _aria2cPath = Pathname( whereisAria2c().asString() );
-
- //Get aria2c version
- _aria2cVersion = getAria2cVersion();
+ //Get aria2c version
+ _aria2cVersion = getAria2cVersion();
}
void MediaAria2c::attachTo (bool next)
ExternalProgram::Arguments args;
- fillAriaCmdLine(_aria2cPath, _aria2cVersion, _settings, fileurl, target.dirname(), args);
-
+ filesystem::TmpPath credentials;
+ fillAriaCmdLine(_aria2cVersion, _settings, credentials, fileurl, target.dirname(), args);
+
do
{
try
{
- report->start(_url, target.asString() );
+ report->start(fileurl, target.asString() );
ExternalProgram aria(args, ExternalProgram::Stderr_To_Stdout);
// progress line like: [#1 SIZE:8.3MiB/10.1MiB(82%) CN:5 SPD:6899.88KiB/s]
+ // but since 1.4.0: [#1 SIZE:8.3MiB/10.1MiB(82%) CN:5 SPD:899.8KiBs]
+ // (bnc #513944) [#1 SIZE:8.3MiB/10.1MiB(82%) CN:5 SPD:3.8MiBs]
// we save it until we find a string with FILE: later
string progressLine;
+ int progress = 0;
// file line, which tell which file is the previous progress
// ie: FILE: ./packages.FL.gz
double average_speed = 0;
long average_speed_count = 0;
-
+
+ // here we capture aria output exceptions
+ vector<string> ariaExceptions;
+
+ // TODO: Detect partial downloads!
+ bool partialDownload = false; // Whether it makes sense to retry with --continue!
+
//Process response
for(std::string ariaResponse( aria.receiveLine());
ariaResponse.length();
{
//cout << ariaResponse;
string line = str::trim(ariaResponse);
-
- if (!line.substr(0,31).compare("Exception: Authorization failed") )
+
+ // look for the progress line and save it for later
+ if ( str::hasPrefix(line, "[#") )
{
- ZYPP_THROW(MediaUnauthorizedException(
- _url, "Login failed.", "Login failed", "auth hint"
- ));
+ progressLine = line;
}
- if (!line.substr(0,29).compare("Exception: Resource not found") )
+ // save error messages for later
+ else if ( str::hasPrefix(line, "Exception: ") )
{
- ZYPP_THROW(MediaFileNotFoundException(_url, filename));
+ // for auth exception, we throw
+ if (!line.substr(0,31).compare("Exception: Authorization failed") )
+ {
+ ZYPP_THROW(MediaUnauthorizedException(
+ _url, "Login failed.", "Login failed", "auth hint"
+ ));
+ }
+ // otherwise, remember the error
+ string excpMsg = line.substr(10, line.size());
+ DBG << "aria2c reported: '" << excpMsg << "'" << endl;
+ ariaExceptions.push_back(excpMsg);
}
-
- // look for the progress line and save it for later
- if ( str::hasPrefix(line, "[#") )
- progressLine = line;
-
- if ( str::hasPrefix(line, "FILE: ") )
+ else if ( str::hasPrefix(line, "FILE: ") )
{
- // get the FILE name
- Pathname theFile(line.substr(6, line.size()));
- // is the report about the filename we are downloading?
- // aria may report progress about metalinks, torrent and
- // other stuff which is not the main transfer
- if ( theFile == target )
+ // get the FILE name
+ string theFile(line.substr(6, line.size()));
+ // is the report about the filename we are downloading?
+ // aria may report progress about metalinks, torrent and
+ // other stuff which is not the main transfer
+ // the reported file is the url before the server emits a response
+ // and then is reported as the target file
+ if ( Pathname(theFile) == target || theFile == fileurl.asCompleteString() )
+ {
+ // once we find the FILE: line, progress has to be
+ // non empty
+ if ( ! progressLine.empty() )
{
- // once we find the FILE: line, progress has to be
- // non empty
- if ( ! progressLine.empty() )
- {
- // get the percentage (progress) data
- int progress = 0;
- size_t left_bound = progressLine.find('(',0) + 1;
- size_t count = progressLine.find('%',left_bound) - left_bound;
- string progressStr = progressLine.substr(left_bound, count);
-
- if ( count != string::npos )
- progress = std::atoi(progressStr.c_str());
- else
- ERR << "Can't parse progress from '" << progressStr << "'" << endl;
- // get the speed
- double current_speed = 0;
- left_bound = progressLine.find("SPD:",0) + 4;
- count = progressLine.find("KiB/s",left_bound) - left_bound;
- if ( count != string::npos )
- { // convert the string to a double
- string speedStr = progressLine.substr(left_bound, count);
- try {
- current_speed = boost::lexical_cast<double>(speedStr);
- }
- catch (const std::exception&) {
- ERR << "Can't parse speed from '" << speedStr << "'" << endl;
- current_speed = 0;
- }
- }
-
- // we have a new average speed
- average_speed_count++;
-
- // this is basically A: average
- // ((n-1)A(n-1) + Xn)/n = A(n)
- average_speed = (((average_speed_count - 1 )*average_speed) + current_speed)/average_speed_count;
-
- report->progress ( progress, _url, average_speed, current_speed );
- // clear the progress line to detect mismatches between
- // [# and FILE: lines
- progressLine.clear();
+ // get the percentage (progress) data
+ progress = 0;
+ size_t left_bound = progressLine.find('(',0) + 1;
+ size_t count = progressLine.find('%',left_bound) - left_bound;
+ string progressStr = progressLine.substr(left_bound, count);
+
+ if ( count != string::npos )
+ progress = std::atoi(progressStr.c_str());
+ else
+ ERR << "Can't parse progress from '" << progressStr << "'" << endl;
+
+ // get the speed
+ double current_speed = 0;
+ left_bound = progressLine.find("SPD:",0) + 4;
+ count = progressLine.find("KiB",left_bound);
+ bool kibs = true; // KiBs ? (MiBs if false)
+ if ( count == string::npos ) // try MiBs
+ {
+ count = progressLine.find("MiBs",left_bound);
+ kibs = false;
+ }
+ if ( count != string::npos )
+ { // convert the string to a double
+ count -= left_bound;
+ string speedStr = progressLine.substr(left_bound, count);
+ try {
+ current_speed = boost::lexical_cast<double>(speedStr);
}
- else
- {
- WAR << "aria2c reported a file, but no progress data available" << endl;
+ catch (const std::exception&) {
+ ERR << "Can't parse speed from '" << speedStr << "'" << endl;
+ current_speed = 0;
}
-
+ }
+
+ // we have a new average speed
+ average_speed_count++;
+
+ // this is basically A: average
+ // ((n-1)A(n-1) + Xn)/n = A(n)
+ average_speed =
+ (((average_speed_count - 1 )*average_speed) + current_speed)
+ / average_speed_count;
+
+ // note that aria report speed in kBps or MBps, while the report takes Bps
+ report->progress ( progress, fileurl,
+ average_speed * (kibs ? 0x400 : 0x10000),
+ current_speed * (kibs ? 0x400 : 0x10000));
+ // clear the progress line to detect mismatches between
+ // [# and FILE: lines
+ progressLine.clear();
}
else
{
- DBG << "Progress is not about '" << target << "' but '" << theFile << "'" << endl;
- }
+ WAR << "aria2c reported a file, but no progress data available" << endl;
+ }
+
+ }
+ else
+ {
+ DBG << "Progress is not about '" << target << "' but '" << theFile << "'" << endl;
+ }
+ }
+ else
+ {
+ // other line type, just ignore it.
}
}
- aria.close();
+ int code = aria.close();
+
+ switch ( code )
+ {
+ // success
+ case 0: // success
+ break;
+ case 2: // timeout
+ {
+ MediaTimeoutException e(_url);
+ for_(it, ariaExceptions.begin(), ariaExceptions.end())
+ e.addHistory(*it);
+ ZYPP_THROW(e);
+ }
+ case 3: // not found
+ case 4: // max notfound reached
+ {
+ MediaFileNotFoundException e(_url, filename);
+ for_(it, ariaExceptions.begin(), ariaExceptions.end())
+ e.addHistory(*it);
+ ZYPP_THROW(e);
+ }
+ case 5: // too slow
+ case 6: // network problem
+ case 7: // unfinished downloads (ctr-c)
+ case 1: // unknown
+ default:
+ {
+ if ( partialDownload )
+ {
+ // Ask for retry on partial downloads, when it makes sense to retry with --continue!
+ // Other errors are handled by the layers above.
+ MediaException e(str::form(_("Download interrupted at %d%%"), progress ));
+ for_(it, ariaExceptions.begin(), ariaExceptions.end())
+ e.addHistory(*it);
+
+ DownloadProgressReport::Action action = report->problem( _url, DownloadProgressReport::ERROR, e.asUserHistory() );
+ if ( action == DownloadProgressReport::RETRY )
+ {
+ retry = true;
+ continue;
+ }
+ }
+
+ // TranslatorExplanation: Failed to download <FILENAME> from <SERVERURL>.
+ MediaException e(str::form(_("Failed to download %s from %s"), filename.c_str(), _url.asString().c_str()));
+ for_(it, ariaExceptions.begin(), ariaExceptions.end())
+ e.addHistory(*it);
+
+ ZYPP_THROW(e);
+ }
+ }
- report->finish( _url , zypp::media::DownloadProgressReport::NO_ERROR, "");
retry = false;
}
-
// retry with proper authentication data
catch (MediaUnauthorizedException & ex_r)
{
{
return MediaCurl::doGetDoesFileExist(filename);
}
-
+
void MediaAria2c::getDir( const Pathname & dirname, bool recurse_r ) const
{
MediaCurl::getDir(dirname, recurse_r);
return false;
}
-
void MediaAria2c::getDirInfo( std::list<std::string> & retlist,
const Pathname & dirname, bool dots ) const
{
std::string MediaAria2c::getAria2cVersion()
{
- const char* argv[] =
+ static const char* argv[] =
{
- _aria2cPath.c_str(),
+ ARIA_BINARY,
"--version",
NULL
};
-
ExternalProgram aria(argv, ExternalProgram::Stderr_To_Stdout);
-
- std::string vResponse = aria.receiveLine();
+ std::string vResponse( str::trim( aria.receiveLine() ) );
aria.close();
- return str::trim(vResponse);
+ return vResponse;
}
-
-#define ARIA_DEFAULT_BINARY "/usr/bin/aria2c"
-
-Pathname MediaAria2c::whereisAria2c()
-{
- Pathname aria2cPathr(ARIA_DEFAULT_BINARY);
-
- const char* argv[] =
- {
- "which",
- "aria2c",
- NULL
- };
-
- ExternalProgram aria(argv, ExternalProgram::Stderr_To_Stdout);
-
- std::string ariaResponse( aria.receiveLine());
- int code = aria.close();
-
- if( code == 0 )
- {
- aria2cPathr = str::trim(ariaResponse);
- MIL << "We will use aria2c located here: " << aria2cPathr << endl;
- }
- else
- {
- MIL << "We don't know were is ari2ac binary. We will use aria2c located here: " << aria2cPathr << endl;
- }
-
- return aria2cPathr;
-}
-
} // namespace media
} // namespace zypp
//
projects / platform / upstream / libzypp.git / blobdiff