#include <iostream>
#include <list>
+#include <vector>
+#include <fstream>
+#include <boost/lexical_cast.hpp>
#include "zypp/base/Logger.h"
#include "zypp/ExternalProgram.h"
#include "zypp/Edition.h"
#include "zypp/Target.h"
#include "zypp/ZYppFactory.h"
+#include "zypp/ZConfig.h"
+
+#include "zypp/TmpPath.h"
-#include "zypp/media/TransferProgram.h"
#include "zypp/media/MediaAria2c.h"
#include "zypp/media/proxyinfo/ProxyInfos.h"
#include "zypp/media/ProxyInfo.h"
#define TRANSFER_TIMEOUT 60 * 3
#define TRANSFER_TIMEOUT_MAX 60 * 60
+#define ARIA_BINARY "aria2c"
using namespace std;
using namespace zypp::base;
{
Pathname MediaAria2c::_cookieFile = "/var/lib/YaST2/cookies";
-Pathname MediaAria2c::_aria2cPath = "/usr/local/bin/aria2c";
std::string MediaAria2c::_aria2cVersion = "WE DON'T KNOW ARIA2C VERSION";
//check if aria2c is present in the system
bool
MediaAria2c::existsAria2cmd()
{
- const char* argv[] =
- {
- "which",
- "aria2c",
- NULL
- };
-
- ExternalProgram aria(argv, ExternalProgram::Stderr_To_Stdout);
- return ( aria.close() == 0 );
+ static const char* argv[] =
+ {
+ ARIA_BINARY,
+ "--version",
+ NULL
+ };
+ ExternalProgram aria( argv, ExternalProgram::Stderr_To_Stdout );
+ return( aria.close() == 0 );
}
-void fillSettingsFromUrl( const Url &url, TransferSettings &s )
-{
- std::string param(url.getQueryParam("timeout"));
- if( !param.empty())
- {
- long num = str::strtonum<long>(param);
- if( num >= 0 && num <= TRANSFER_TIMEOUT_MAX)
- s.setTimeout(num);
- }
-
- if ( ! url.getUsername().empty() )
- {
- s.setUsername(url.getUsername());
- if ( url.getPassword().size() )
- {
- s.setPassword(url.getPassword());
- }
- }
-
- string proxy = url.getQueryParam( "proxy" );
-
- if ( ! proxy.empty() )
- {
- string proxyport( url.getQueryParam( "proxyport" ) );
- if ( ! proxyport.empty() ) {
- proxy += ":" + proxyport;
- }
- s.setProxy(proxy);
- s.setProxyEnabled(true);
- }
-}
-
-void fillSettingsSystemProxy( const Url&url, TransferSettings &s )
-{
- ProxyInfo proxy_info (ProxyInfo::ImplPtr(new ProxyInfoSysconfig("proxy")));
-
- if ( proxy_info.enabled())
- {
- s.setProxyEnabled(true);
- std::list<std::string> nope = proxy_info.noProxy();
- for (ProxyInfo::NoProxyIterator it = proxy_info.noProxyBegin();
- it != proxy_info.noProxyEnd();
- it++)
- {
- std::string host( str::toLower(url.getHost()));
- std::string temp( str::toLower(*it));
-
- // no proxy if it points to a suffix
- // preceeded by a '.', that maches
- // the trailing portion of the host.
- if( temp.size() > 1 && temp.at(0) == '.')
- {
- if(host.size() > temp.size() &&
- host.compare(host.size() - temp.size(), temp.size(), temp) == 0)
- {
- DBG << "NO_PROXY: '" << *it << "' matches host '"
- << host << "'" << endl;
- s.setProxyEnabled(false);
- break;
- }
- }
- else
- // no proxy if we have an exact match
- if( host == temp)
- {
- DBG << "NO_PROXY: '" << *it << "' matches host '"
- << host << "'" << endl;
- s.setProxyEnabled(false);
- break;
- }
- }
-
- if ( s.proxyEnabled() )
- s.setProxy(proxy_info.proxy(url.getScheme()));
- }
-
-}
-
/**
* comannd line for aria.
* The argument list gets passed as reference
* and it is filled.
*/
-void fillAriaCmdLine( const Pathname &ariapath,
- const string &ariaver,
+void fillAriaCmdLine( const string &ariaver,
const TransferSettings &s,
+ filesystem::TmpPath &credentials,
const Url &url,
const Pathname &destination,
ExternalProgram::Arguments &args )
{
- args.push_back(ariapath.c_str());
+
+ // options that are not passed in the command line
+ // like credentials, every string is in the
+ // opt=val format
+ list<string> file_options;
+
+ args.push_back(ARIA_BINARY);
args.push_back(str::form("--user-agent=%s", s.userAgentString().c_str()));
args.push_back("--summary-interval=1");
args.push_back("--follow-metalink=mem");
args.push_back("--check-integrity=true");
+ args.push_back("--file-allocation=none");
+
+ // save the stats of the mirrors and use them as input later
+ Pathname statsFile = ZConfig::instance().repoCachePath() / "aria2.stats";
+ args.push_back(str::form("--server-stat-of=%s", statsFile.c_str()));
+ args.push_back(str::form("--server-stat-if=%s", statsFile.c_str()));
+ args.push_back("--uri-selector=adaptive");
// only present in recent aria lets find out the aria version
- vector<string> fields;
+ vector<string> fields;
// "aria2c version x.x"
str::split( ariaver, std::back_inserter(fields));
if ( fields.size() == 3 )
if ( Edition(fields[2]) >= Edition("1.1.2") )
args.push_back( "--use-head=false");
}
-
+
if ( s.maxDownloadSpeed() > 0 )
args.push_back(str::form("--max-download-limit=%ld", s.maxDownloadSpeed()));
if ( s.minDownloadSpeed() > 0 )
if ( Edition(fields[2]) < Edition("1.2.0") )
WAR << "aria2c is older than 1.2.0, some features may be disabled" << endl;
-
+
// TODO make this one configurable
args.push_back(str::form("--max-concurrent-downloads=%ld", s.maxConcurrentConnections()));
it != s.headersEnd();
++it )
args.push_back(str::form("--header=%s", it->c_str() ));
-
+
args.push_back( str::form("--connect-timeout=%ld", s.timeout()));
if ( s.username().empty() )
else
{
if ( url.getScheme() == "ftp" )
- args.push_back(str::form("--ftp-user=%s", s.username().c_str() ));
+ file_options.push_back(str::form("ftp-user=%s", s.username().c_str() ));
else if ( url.getScheme() == "http" ||
url.getScheme() == "https" )
- args.push_back(str::form("--http-user=%s", s.username().c_str() ));
-
+ file_options.push_back(str::form("http-user=%s", s.username().c_str() ));
+
if ( s.password().size() )
{
if ( url.getScheme() == "ftp" )
- args.push_back(str::form("--ftp-passwd=%s", s.password().c_str() ));
+ file_options.push_back(str::form("ftp-passwd=%s", s.password().c_str() ));
else if ( url.getScheme() == "http" ||
url.getScheme() == "https" )
- args.push_back(str::form("--http-passwd=%s", s.password().c_str() ));
+ file_options.push_back(str::form("http-passwd=%s", s.password().c_str() ));
}
}
-
+
if ( s.proxyEnabled() )
{
args.push_back(str::form("--http-proxy=%s", s.proxy().c_str() ));
if ( ! s.proxyUsername().empty() )
{
- args.push_back(str::form("--http-proxy-user=%s", s.proxyUsername().c_str() ));
+ file_options.push_back(str::form("http-proxy-user=%s", s.proxyUsername().c_str() ));
if ( ! s.proxyPassword().empty() )
- args.push_back(str::form("--http-proxy-passwd=%s", s.proxyPassword().c_str() ));
+ file_options.push_back(str::form("http-proxy-passwd=%s", s.proxyPassword().c_str() ));
}
}
if ( ! destination.empty() )
args.push_back(str::form("--dir=%s", destination.c_str()));
+ // now append the file if there are hidden options
+ if ( ! file_options.empty() )
+ {
+ filesystem::TmpFile tmp;
+ ofstream outs( tmp.path().c_str() );
+ for_( it, file_options.begin(), file_options.end() )
+ outs << *it << endl;
+ outs.close();
+
+ credentials = tmp;
+ args.push_back(str::form("--conf-path=%s", credentials.path().c_str()));
+ }
+
args.push_back(url.asString().c_str());
}
-static const char *const anonymousIdHeader()
-{
- // we need to add the release and identifier to the
- // agent string.
- // The target could be not initialized, and then this information
- // is not available.
- Target_Ptr target = zypp::getZYpp()->getTarget();
-
- static const std::string _value(
- str::form(
- "X-Zypp-AnonymousId: %s",
- target ? target->anonymousUniqueId().c_str() : "" )
- );
- return _value.c_str();
-}
-
-static const char *const distributionFlavorHeader()
-{
- // we need to add the release and identifier to the
- // agent string.
- // The target could be not initialized, and then this information
- // is not available.
- Target_Ptr target = zypp::getZYpp()->getTarget();
-
- static const std::string _value(
- str::trim( str::form(
- "X-ZYpp-DistributionFlavor: %s",
- target ? target->distributionFlavor().c_str() : "" ) )
- );
- return _value.c_str();
-}
-
const char *const MediaAria2c::agentString()
{
// we need to add the release and identifier to the
"ZYpp %s (%s) %s"
, VERSION
, MediaAria2c::_aria2cVersion.c_str()
- , target ? target->targetDistribution().c_str() : ""
+ , Target::targetDistribution( Pathname()/*guess root*/ ).c_str()
)
);
return _value.c_str();
: MediaCurl( url_r, attach_point_hint_r )
{
MIL << "MediaAria2c::MediaAria2c(" << url_r << ", " << attach_point_hint_r << ")" << endl;
-
- /*
- if( !attachPoint().empty())
- {
- PathInfo ainfo(attachPoint());
- Pathname apath(attachPoint() + "XXXXXX");
- char *atemp = ::strdup( apath.asString().c_str());
- char *atest = NULL;
- if( !ainfo.isDir() || !ainfo.userMayRWX() ||
- atemp == NULL || (atest=::mkdtemp(atemp)) == NULL)
- {
- WAR << "attach point " << ainfo.path()
- << " is not useable for " << url_r.getScheme() << endl;
- setAttachPoint("", true);
- }
- else if( atest != NULL)
- ::rmdir(atest);
-
- if( atemp != NULL)
- ::free(atemp);
- }
- */
-
- //At this point, we initialize aria2c path
- _aria2cPath = Pathname( whereisAria2c().asString() );
-
- //Get aria2c version
- _aria2cVersion = getAria2cVersion();
+ //Get aria2c version
+ _aria2cVersion = getAria2cVersion();
}
void MediaAria2c::attachTo (bool next)
{
MediaCurl::attachTo(next);
-
_settings.setUserAgentString(agentString());
- _settings.addHeader(anonymousIdHeader());
- _settings.addHeader(distributionFlavorHeader());
-
- _settings.setTimeout(TRANSFER_TIMEOUT);
- _settings.setConnectTimeout(CONNECT_TIMEOUT);
-
- // fill some settings from url query parameters
- fillSettingsFromUrl(_url, _settings);
-
- // if the proxy was not set by url, then look
- if ( _settings.proxy().empty() )
- {
- // at the system proxy settings
- fillSettingsSystemProxy(_url, _settings);
- }
-
- DBG << "Proxy: " << (_settings.proxy().empty() ? "-none-" : _settings.proxy()) << endl;
}
bool
void MediaAria2c::disconnectFrom()
{
MediaCurl::disconnectFrom();
-
}
void MediaAria2c::releaseFrom( const std::string & ejectDev )
ExternalProgram::Arguments args;
- fillAriaCmdLine(_aria2cPath, _aria2cVersion, _settings, fileurl, target.dirname(), args);
-
+ filesystem::TmpPath credentials;
+ fillAriaCmdLine(_aria2cVersion, _settings, credentials, fileurl, target.dirname(), args);
+
do
{
try
{
- report->start(_url, target.asString() );
+ report->start(fileurl, target.asString() );
ExternalProgram aria(args, ExternalProgram::Stderr_To_Stdout);
- int nLine = 0;
+
+ // progress line like: [#1 SIZE:8.3MiB/10.1MiB(82%) CN:5 SPD:6899.88KiB/s]
+ // but since 1.4.0: [#1 SIZE:8.3MiB/10.1MiB(82%) CN:5 SPD:899.8KiBs]
+ // (bnc #513944) [#1 SIZE:8.3MiB/10.1MiB(82%) CN:5 SPD:3.8MiBs]
+ // we save it until we find a string with FILE: later
+ string progressLine;
+ int progress = 0;
+ // file line, which tell which file is the previous progress
+ // ie: FILE: ./packages.FL.gz
+ double average_speed = 0;
+ long average_speed_count = 0;
+
+ // here we capture aria output exceptions
+ vector<string> ariaExceptions;
+
+ // TODO: Detect partial downloads!
+ bool partialDownload = false; // Whether it makes sense to retry with --continue!
//Process response
for(std::string ariaResponse( aria.receiveLine());
ariaResponse = aria.receiveLine())
{
//cout << ariaResponse;
+ string line = str::trim(ariaResponse);
- if (!ariaResponse.substr(0,31).compare("Exception: Authorization failed") )
+ // look for the progress line and save it for later
+ if ( str::hasPrefix(line, "[#") )
{
- ZYPP_THROW(MediaUnauthorizedException(
- _url, "Login failed.", "Login failed", "auth hint"
- ));
+ progressLine = line;
}
- if (!ariaResponse.substr(0,29).compare("Exception: Resource not found") )
+ // save error messages for later
+ else if ( str::hasPrefix(line, "Exception: ") )
{
- ZYPP_THROW(MediaFileNotFoundException(_url, filename));
+ // for auth exception, we throw
+ if (!line.substr(0,31).compare("Exception: Authorization failed") )
+ {
+ ZYPP_THROW(MediaUnauthorizedException(
+ _url, "Login failed.", "Login failed", "auth hint"
+ ));
+ }
+ // otherwise, remember the error
+ string excpMsg = line.substr(10, line.size());
+ DBG << "aria2c reported: '" << excpMsg << "'" << endl;
+ ariaExceptions.push_back(excpMsg);
}
-
- if (!ariaResponse.substr(0,9).compare("[#2 SIZE:"))
+ else if ( str::hasPrefix(line, "FILE: ") )
{
- if (!nLine)
+ // get the FILE name
+ string theFile(line.substr(6, line.size()));
+ // is the report about the filename we are downloading?
+ // aria may report progress about metalinks, torrent and
+ // other stuff which is not the main transfer
+ // the reported file is the url before the server emits a response
+ // and then is reported as the target file
+ if ( Pathname(theFile) == target || theFile == fileurl.asCompleteString() )
{
- size_t left_bound = ariaResponse.find('(',0) + 1;
- size_t count = ariaResponse.find('%',left_bound) - left_bound;
- //cout << ariaResponse.substr(left_bound, count) << endl;
- //progressData.toMax();
- report->progress ( std::atoi(ariaResponse.substr(left_bound, count).c_str()), _url, -1, -1 );
- nLine = 1;
+ // once we find the FILE: line, progress has to be
+ // non empty
+ if ( ! progressLine.empty() )
+ {
+ // get the percentage (progress) data
+ progress = 0;
+ size_t left_bound = progressLine.find('(',0) + 1;
+ size_t count = progressLine.find('%',left_bound) - left_bound;
+ string progressStr = progressLine.substr(left_bound, count);
+
+ if ( count != string::npos )
+ progress = std::atoi(progressStr.c_str());
+ else
+ ERR << "Can't parse progress from '" << progressStr << "'" << endl;
+
+ // get the speed
+ double current_speed = 0;
+ left_bound = progressLine.find("SPD:",0) + 4;
+ count = progressLine.find("KiB",left_bound);
+ bool kibs = true; // KiBs ? (MiBs if false)
+ if ( count == string::npos ) // try MiBs
+ {
+ count = progressLine.find("MiBs",left_bound);
+ kibs = false;
+ }
+ if ( count != string::npos )
+ { // convert the string to a double
+ count -= left_bound;
+ string speedStr = progressLine.substr(left_bound, count);
+ try {
+ current_speed = boost::lexical_cast<double>(speedStr);
+ }
+ catch (const std::exception&) {
+ ERR << "Can't parse speed from '" << speedStr << "'" << endl;
+ current_speed = 0;
+ }
+ }
+
+ // we have a new average speed
+ average_speed_count++;
+
+ // this is basically A: average
+ // ((n-1)A(n-1) + Xn)/n = A(n)
+ average_speed =
+ (((average_speed_count - 1 )*average_speed) + current_speed)
+ / average_speed_count;
+
+ // note that aria report speed in kBps or MBps, while the report takes Bps
+ report->progress ( progress, fileurl,
+ average_speed * (kibs ? 0x400 : 0x10000),
+ current_speed * (kibs ? 0x400 : 0x10000));
+ // clear the progress line to detect mismatches between
+ // [# and FILE: lines
+ progressLine.clear();
+ }
+ else
+ {
+ WAR << "aria2c reported a file, but no progress data available" << endl;
+ }
+
}
else
{
- nLine = 0;
+ DBG << "Progress is not about '" << target << "' but '" << theFile << "'" << endl;
}
}
+ else
+ {
+ // other line type, just ignore it.
+ }
}
- aria.close();
+ int code = aria.close();
+
+ switch ( code )
+ {
+ // success
+ case 0: // success
+ break;
+ case 2: // timeout
+ {
+ MediaTimeoutException e(_url);
+ for_(it, ariaExceptions.begin(), ariaExceptions.end())
+ e.addHistory(*it);
+ ZYPP_THROW(e);
+ }
+ case 3: // not found
+ case 4: // max notfound reached
+ {
+ MediaFileNotFoundException e(_url, filename);
+ for_(it, ariaExceptions.begin(), ariaExceptions.end())
+ e.addHistory(*it);
+ ZYPP_THROW(e);
+ }
+ case 5: // too slow
+ case 6: // network problem
+ case 7: // unfinished downloads (ctr-c)
+ case 1: // unknown
+ default:
+ {
+ if ( partialDownload )
+ {
+ // Ask for retry on partial downloads, when it makes sense to retry with --continue!
+ // Other errors are handled by the layers above.
+ MediaException e(str::form(_("Download interrupted at %d%%"), progress ));
+ for_(it, ariaExceptions.begin(), ariaExceptions.end())
+ e.addHistory(*it);
+
+ DownloadProgressReport::Action action = report->problem( _url, DownloadProgressReport::ERROR, e.asUserHistory() );
+ if ( action == DownloadProgressReport::RETRY )
+ {
+ retry = true;
+ continue;
+ }
+ }
+
+ // TranslatorExplanation: Failed to download <FILENAME> from <SERVERURL>.
+ MediaException e(str::form(_("Failed to download %s from %s"), filename.c_str(), _url.asString().c_str()));
+ for_(it, ariaExceptions.begin(), ariaExceptions.end())
+ e.addHistory(*it);
+
+ ZYPP_THROW(e);
+ }
+ }
- report->finish( _url , zypp::media::DownloadProgressReport::NO_ERROR, "");
retry = false;
}
-
// retry with proper authentication data
catch (MediaUnauthorizedException & ex_r)
{
{
return MediaCurl::doGetDoesFileExist(filename);
}
-
+
void MediaAria2c::getDir( const Pathname & dirname, bool recurse_r ) const
{
MediaCurl::getDir(dirname, recurse_r);
return false;
}
-
void MediaAria2c::getDirInfo( std::list<std::string> & retlist,
const Pathname & dirname, bool dots ) const
{
std::string MediaAria2c::getAria2cVersion()
{
- const char* argv[] =
+ static const char* argv[] =
{
- _aria2cPath.c_str(),
+ ARIA_BINARY,
"--version",
NULL
};
-
ExternalProgram aria(argv, ExternalProgram::Stderr_To_Stdout);
-
- std::string vResponse = aria.receiveLine();
+ std::string vResponse( str::trim( aria.receiveLine() ) );
aria.close();
- return str::trim(vResponse);
-}
-
-#define ARIA_DEFAULT_BINARY "/usr/bin/aria2c"
-
-Pathname MediaAria2c::whereisAria2c()
-{
- Pathname aria2cPathr(ARIA_DEFAULT_BINARY);
-
- const char* argv[] =
- {
- "which",
- "aria2c",
- NULL
- };
-
- ExternalProgram aria(argv, ExternalProgram::Stderr_To_Stdout);
-
- std::string ariaResponse( aria.receiveLine());
- int code = aria.close();
-
- if( code == 0 )
- {
- aria2cPathr = str::trim(ariaResponse);
- MIL << "We will use aria2c located here: " << aria2cPathr << endl;
- }
- else
- {
- MIL << "We don't know were is ari2ac binary. We will use aria2c located here: " << aria2cPathr << endl;
- }
-
- return aria2cPathr;
+ return vResponse;
}
-
} // namespace media
} // namespace zypp
//