#include <iostream>
#include <list>
#include <vector>
-
+#include <fstream>
#include <boost/lexical_cast.hpp>
#include "zypp/base/Logger.h"
#include "zypp/ZYppFactory.h"
#include "zypp/ZConfig.h"
+#include "zypp/TmpPath.h"
+
#include "zypp/media/MediaAria2c.h"
#include "zypp/media/proxyinfo/ProxyInfos.h"
#include "zypp/media/ProxyInfo.h"
#define TRANSFER_TIMEOUT 60 * 3
#define TRANSFER_TIMEOUT_MAX 60 * 60
+#define ARIA_BINARY "aria2c"
using namespace std;
using namespace zypp::base;
{
Pathname MediaAria2c::_cookieFile = "/var/lib/YaST2/cookies";
-Pathname MediaAria2c::_aria2cPath = "/usr/local/bin/aria2c";
std::string MediaAria2c::_aria2cVersion = "WE DON'T KNOW ARIA2C VERSION";
//check if aria2c is present in the system
bool
MediaAria2c::existsAria2cmd()
{
- const char* argv[] =
- {
- "which",
- "aria2c",
- NULL
- };
-
- ExternalProgram aria(argv, ExternalProgram::Stderr_To_Stdout);
-
- for(std::string ariaResponse( aria.receiveLine());
- ariaResponse.length();
- ariaResponse = aria.receiveLine())
- {
- // nothing
- }
-
- return ( aria.close() == 0 );
+ static const char* argv[] =
+ {
+ ARIA_BINARY,
+ "--version",
+ NULL
+ };
+ ExternalProgram aria( argv, ExternalProgram::Stderr_To_Stdout );
+ return( aria.close() == 0 );
}
/**
* The argument list gets passed as reference
* and it is filled.
*/
-void fillAriaCmdLine( const Pathname &ariapath,
- const string &ariaver,
+void fillAriaCmdLine( const string &ariaver,
const TransferSettings &s,
+ filesystem::TmpPath &credentials,
const Url &url,
const Pathname &destination,
ExternalProgram::Arguments &args )
{
- args.push_back(ariapath.c_str());
+
+ // options that are not passed in the command line
+ // like credentials, every string is in the
+ // opt=val format
+ list<string> file_options;
+
+ args.push_back(ARIA_BINARY);
args.push_back(str::form("--user-agent=%s", s.userAgentString().c_str()));
args.push_back("--summary-interval=1");
args.push_back("--follow-metalink=mem");
args.push_back("--uri-selector=adaptive");
// only present in recent aria lets find out the aria version
- vector<string> fields;
+ vector<string> fields;
// "aria2c version x.x"
str::split( ariaver, std::back_inserter(fields));
if ( fields.size() == 3 )
if ( Edition(fields[2]) >= Edition("1.1.2") )
args.push_back( "--use-head=false");
}
-
+
if ( s.maxDownloadSpeed() > 0 )
args.push_back(str::form("--max-download-limit=%ld", s.maxDownloadSpeed()));
if ( s.minDownloadSpeed() > 0 )
if ( Edition(fields[2]) < Edition("1.2.0") )
WAR << "aria2c is older than 1.2.0, some features may be disabled" << endl;
-
+
// TODO make this one configurable
args.push_back(str::form("--max-concurrent-downloads=%ld", s.maxConcurrentConnections()));
it != s.headersEnd();
++it )
args.push_back(str::form("--header=%s", it->c_str() ));
-
+
args.push_back( str::form("--connect-timeout=%ld", s.timeout()));
if ( s.username().empty() )
else
{
if ( url.getScheme() == "ftp" )
- args.push_back(str::form("--ftp-user=%s", s.username().c_str() ));
+ file_options.push_back(str::form("ftp-user=%s", s.username().c_str() ));
else if ( url.getScheme() == "http" ||
url.getScheme() == "https" )
- args.push_back(str::form("--http-user=%s", s.username().c_str() ));
-
+ file_options.push_back(str::form("http-user=%s", s.username().c_str() ));
+
if ( s.password().size() )
{
if ( url.getScheme() == "ftp" )
- args.push_back(str::form("--ftp-passwd=%s", s.password().c_str() ));
+ file_options.push_back(str::form("ftp-passwd=%s", s.password().c_str() ));
else if ( url.getScheme() == "http" ||
url.getScheme() == "https" )
- args.push_back(str::form("--http-passwd=%s", s.password().c_str() ));
+ file_options.push_back(str::form("http-passwd=%s", s.password().c_str() ));
}
}
-
+
if ( s.proxyEnabled() )
{
args.push_back(str::form("--http-proxy=%s", s.proxy().c_str() ));
if ( ! s.proxyUsername().empty() )
{
- args.push_back(str::form("--http-proxy-user=%s", s.proxyUsername().c_str() ));
+ file_options.push_back(str::form("http-proxy-user=%s", s.proxyUsername().c_str() ));
if ( ! s.proxyPassword().empty() )
- args.push_back(str::form("--http-proxy-passwd=%s", s.proxyPassword().c_str() ));
+ file_options.push_back(str::form("http-proxy-passwd=%s", s.proxyPassword().c_str() ));
}
}
if ( ! destination.empty() )
args.push_back(str::form("--dir=%s", destination.c_str()));
+ // now append the file if there are hidden options
+ if ( ! file_options.empty() )
+ {
+ filesystem::TmpFile tmp;
+ ofstream outs( tmp.path().c_str() );
+ for_( it, file_options.begin(), file_options.end() )
+ outs << *it << endl;
+ outs.close();
+
+ credentials = tmp;
+ args.push_back(str::form("--conf-path=%s", credentials.path().c_str()));
+ }
+
args.push_back(url.asString().c_str());
}
"ZYpp %s (%s) %s"
, VERSION
, MediaAria2c::_aria2cVersion.c_str()
- , target ? target->targetDistribution().c_str() : ""
+ , Target::targetDistribution( Pathname()/*guess root*/ ).c_str()
)
);
return _value.c_str();
: MediaCurl( url_r, attach_point_hint_r )
{
MIL << "MediaAria2c::MediaAria2c(" << url_r << ", " << attach_point_hint_r << ")" << endl;
-
- //At this point, we initialize aria2c path
- _aria2cPath = Pathname( whereisAria2c().asString() );
-
- //Get aria2c version
- _aria2cVersion = getAria2cVersion();
+ //Get aria2c version
+ _aria2cVersion = getAria2cVersion();
}
void MediaAria2c::attachTo (bool next)
ExternalProgram::Arguments args;
- fillAriaCmdLine(_aria2cPath, _aria2cVersion, _settings, fileurl, target.dirname(), args);
-
+ filesystem::TmpPath credentials;
+ fillAriaCmdLine(_aria2cVersion, _settings, credentials, fileurl, target.dirname(), args);
+
do
{
try
{
- report->start(_url, target.asString() );
+ report->start(fileurl, target.asString() );
ExternalProgram aria(args, ExternalProgram::Stderr_To_Stdout);
// progress line like: [#1 SIZE:8.3MiB/10.1MiB(82%) CN:5 SPD:6899.88KiB/s]
+ // but since 1.4.0: [#1 SIZE:8.3MiB/10.1MiB(82%) CN:5 SPD:899.8KiBs]
+ // (bnc #513944) [#1 SIZE:8.3MiB/10.1MiB(82%) CN:5 SPD:3.8MiBs]
// we save it until we find a string with FILE: later
string progressLine;
+ int progress = 0;
// file line, which tell which file is the previous progress
// ie: FILE: ./packages.FL.gz
double average_speed = 0;
// here we capture aria output exceptions
vector<string> ariaExceptions;
-
+
+ // TODO: Detect partial downloads!
+ bool partialDownload = false; // Whether it makes sense to retry with --continue!
+
//Process response
for(std::string ariaResponse( aria.receiveLine());
ariaResponse.length();
{
//cout << ariaResponse;
string line = str::trim(ariaResponse);
-
+
// look for the progress line and save it for later
if ( str::hasPrefix(line, "[#") )
{
}
// otherwise, remember the error
string excpMsg = line.substr(10, line.size());
- DBG << "aria2c reported: '" << excpMsg << "'" << endl;
+ DBG << "aria2c reported: '" << excpMsg << "'" << endl;
ariaExceptions.push_back(excpMsg);
}
else if ( str::hasPrefix(line, "FILE: ") )
{
// get the FILE name
- Pathname theFile(line.substr(6, line.size()));
+ string theFile(line.substr(6, line.size()));
// is the report about the filename we are downloading?
// aria may report progress about metalinks, torrent and
// other stuff which is not the main transfer
- if ( theFile == target )
+ // the reported file is the url before the server emits a response
+ // and then is reported as the target file
+ if ( Pathname(theFile) == target || theFile == fileurl.asCompleteString() )
{
// once we find the FILE: line, progress has to be
// non empty
if ( ! progressLine.empty() )
{
// get the percentage (progress) data
- int progress = 0;
+ progress = 0;
size_t left_bound = progressLine.find('(',0) + 1;
size_t count = progressLine.find('%',left_bound) - left_bound;
string progressStr = progressLine.substr(left_bound, count);
-
+
if ( count != string::npos )
progress = std::atoi(progressStr.c_str());
else
ERR << "Can't parse progress from '" << progressStr << "'" << endl;
+
// get the speed
double current_speed = 0;
left_bound = progressLine.find("SPD:",0) + 4;
- count = progressLine.find("KiB/s",left_bound) - left_bound;
+ count = progressLine.find("KiB",left_bound);
+ bool kibs = true; // KiBs ? (MiBs if false)
+ if ( count == string::npos ) // try MiBs
+ {
+ count = progressLine.find("MiBs",left_bound);
+ kibs = false;
+ }
if ( count != string::npos )
{ // convert the string to a double
+ count -= left_bound;
string speedStr = progressLine.substr(left_bound, count);
try {
current_speed = boost::lexical_cast<double>(speedStr);
current_speed = 0;
}
}
-
+
// we have a new average speed
average_speed_count++;
-
+
// this is basically A: average
// ((n-1)A(n-1) + Xn)/n = A(n)
- average_speed = (((average_speed_count - 1 )*average_speed) + current_speed)/average_speed_count;
-
- // note that aria report speed in kbpss, while the report takes bps
- report->progress ( progress, _url, average_speed * 1024, current_speed * 1024 );
+ average_speed =
+ (((average_speed_count - 1 )*average_speed) + current_speed)
+ / average_speed_count;
+
+ // note that aria report speed in kBps or MBps, while the report takes Bps
+ report->progress ( progress, fileurl,
+ average_speed * (kibs ? 0x400 : 0x10000),
+ current_speed * (kibs ? 0x400 : 0x10000));
// clear the progress line to detect mismatches between
// [# and FILE: lines
progressLine.clear();
else
{
DBG << "Progress is not about '" << target << "' but '" << theFile << "'" << endl;
- }
+ }
}
else
{
case 1: // unknown
default:
{
- MediaException e(str::form("Failed to download %s from %s", filename.c_str(), _url.asString().c_str()));
+ if ( partialDownload )
+ {
+ // Ask for retry on partial downloads, when it makes sense to retry with --continue!
+ // Other errors are handled by the layers above.
+ MediaException e(str::form(_("Download interrupted at %d%%"), progress ));
+ for_(it, ariaExceptions.begin(), ariaExceptions.end())
+ e.addHistory(*it);
+
+ DownloadProgressReport::Action action = report->problem( _url, DownloadProgressReport::ERROR, e.asUserHistory() );
+ if ( action == DownloadProgressReport::RETRY )
+ {
+ retry = true;
+ continue;
+ }
+ }
+
+ // TranslatorExplanation: Failed to download <FILENAME> from <SERVERURL>.
+ MediaException e(str::form(_("Failed to download %s from %s"), filename.c_str(), _url.asString().c_str()));
for_(it, ariaExceptions.begin(), ariaExceptions.end())
e.addHistory(*it);
-
+
ZYPP_THROW(e);
}
}
-
- report->finish( _url , zypp::media::DownloadProgressReport::NO_ERROR, "");
+
retry = false;
}
// retry with proper authentication data
{
return MediaCurl::doGetDoesFileExist(filename);
}
-
+
void MediaAria2c::getDir( const Pathname & dirname, bool recurse_r ) const
{
MediaCurl::getDir(dirname, recurse_r);
return false;
}
-
void MediaAria2c::getDirInfo( std::list<std::string> & retlist,
const Pathname & dirname, bool dots ) const
{
std::string MediaAria2c::getAria2cVersion()
{
- const char* argv[] =
+ static const char* argv[] =
{
- _aria2cPath.c_str(),
+ ARIA_BINARY,
"--version",
NULL
};
-
ExternalProgram aria(argv, ExternalProgram::Stderr_To_Stdout);
-
- std::string vResponse = aria.receiveLine();
+ std::string vResponse( str::trim( aria.receiveLine() ) );
aria.close();
- return str::trim(vResponse);
+ return vResponse;
}
-
-#define ARIA_DEFAULT_BINARY "/usr/bin/aria2c"
-
-Pathname MediaAria2c::whereisAria2c()
-{
- Pathname aria2cPathr(ARIA_DEFAULT_BINARY);
-
- const char* argv[] =
- {
- "which",
- "aria2c",
- NULL
- };
-
- ExternalProgram aria(argv, ExternalProgram::Stderr_To_Stdout);
-
- std::string ariaResponse( aria.receiveLine());
- int code = aria.close();
-
- if( code == 0 )
- {
- aria2cPathr = str::trim(ariaResponse);
- MIL << "We will use aria2c located here: " << aria2cPathr << endl;
- }
- else
- {
- MIL << "We don't know were is ari2ac binary. We will use aria2c located here: " << aria2cPathr << endl;
- }
-
- return aria2cPathr;
-}
-
} // namespace media
} // namespace zypp
//