1 /*---------------------------------------------------------------------\
3 | |__ / \ / / . \ . \ |
8 \---------------------------------------------------------------------*/
9 /** \file zypp/media/MediaAria2c.cc
16 #include "zypp/base/Logger.h"
17 #include "zypp/ExternalProgram.h"
18 #include "zypp/ProgressData.h"
19 #include "zypp/base/String.h"
20 #include "zypp/base/Gettext.h"
21 #include "zypp/base/Sysconfig.h"
22 #include "zypp/base/Gettext.h"
23 #include "zypp/ZYppCallbacks.h"
25 #include "zypp/Edition.h"
26 #include "zypp/Target.h"
27 #include "zypp/ZYppFactory.h"
29 #include "zypp/media/MediaAria2c.h"
30 #include "zypp/media/proxyinfo/ProxyInfos.h"
31 #include "zypp/media/ProxyInfo.h"
32 #include "zypp/media/MediaUserAuth.h"
33 #include "zypp/thread/Once.h"
35 #include <sys/types.h>
37 #include <sys/mount.h>
41 #include <boost/format.hpp>
43 #define DETECT_DIR_INDEX 0
44 #define CONNECT_TIMEOUT 60
45 #define TRANSFER_TIMEOUT 60 * 3
46 #define TRANSFER_TIMEOUT_MAX 60 * 60
50 using namespace zypp::base;
57 Pathname MediaAria2c::_cookieFile = "/var/lib/YaST2/cookies";
58 Pathname MediaAria2c::_aria2cPath = "/usr/local/bin/aria2c";
59 std::string MediaAria2c::_aria2cVersion = "WE DON'T KNOW ARIA2C VERSION";
61 //check if aria2c is present in the system
63 MediaAria2c::existsAria2cmd()
72 ExternalProgram aria(argv, ExternalProgram::Stderr_To_Stdout);
73 return ( aria.close() == 0 );
76 void fillSettingsFromUrl( const Url &url, TransferSettings &s )
78 std::string param(url.getQueryParam("timeout"));
81 long num = str::strtonum<long>(param);
82 if( num >= 0 && num <= TRANSFER_TIMEOUT_MAX)
86 if ( ! url.getUsername().empty() )
88 s.setUsername(url.getUsername());
89 if ( url.getPassword().size() )
91 s.setPassword(url.getPassword());
95 string proxy = url.getQueryParam( "proxy" );
97 if ( ! proxy.empty() )
99 string proxyport( url.getQueryParam( "proxyport" ) );
100 if ( ! proxyport.empty() ) {
101 proxy += ":" + proxyport;
104 s.setProxyEnabled(true);
108 void fillSettingsSystemProxy( const Url&url, TransferSettings &s )
110 ProxyInfo proxy_info (ProxyInfo::ImplPtr(new ProxyInfoSysconfig("proxy")));
112 if ( proxy_info.enabled())
114 s.setProxyEnabled(true);
115 std::list<std::string> nope = proxy_info.noProxy();
116 for (ProxyInfo::NoProxyIterator it = proxy_info.noProxyBegin();
117 it != proxy_info.noProxyEnd();
120 std::string host( str::toLower(url.getHost()));
121 std::string temp( str::toLower(*it));
123 // no proxy if it points to a suffix
124 // preceeded by a '.', that maches
125 // the trailing portion of the host.
126 if( temp.size() > 1 && temp.at(0) == '.')
128 if(host.size() > temp.size() &&
129 host.compare(host.size() - temp.size(), temp.size(), temp) == 0)
131 DBG << "NO_PROXY: '" << *it << "' matches host '"
132 << host << "'" << endl;
133 s.setProxyEnabled(false);
138 // no proxy if we have an exact match
141 DBG << "NO_PROXY: '" << *it << "' matches host '"
142 << host << "'" << endl;
143 s.setProxyEnabled(false);
148 if ( s.proxyEnabled() )
149 s.setProxy(proxy_info.proxy(url.getScheme()));
155 * comannd line for aria.
156 * The argument list gets passed as reference
159 void fillAriaCmdLine( const Pathname &ariapath,
160 const string &ariaver,
161 const TransferSettings &s,
163 const Pathname &destination,
164 ExternalProgram::Arguments &args )
166 args.push_back(ariapath.c_str());
167 args.push_back(str::form("--user-agent=%s", s.userAgentString().c_str()));
168 args.push_back("--summary-interval=1");
169 args.push_back("--follow-metalink=mem");
170 args.push_back("--check-integrity=true");
172 // only present in recent aria
173 if ( Edition(ariaver) >= Edition("1.20") )
174 args.push_back( "--use-head=false");
176 // TODO make this one configurable
177 args.push_back( "--max-concurrent-downloads=2");
179 // add the anonymous id.
180 for ( TransferSettings::Headers::const_iterator it = s.headersBegin();
181 it != s.headersEnd();
183 args.push_back(str::form("--header=%s", it->c_str() ));
185 args.push_back( str::form("--connect-timeout=%ld", s.timeout()));
187 if ( s.username().empty() )
189 if ( url.getScheme() == "ftp" )
192 args.push_back(str::form("--ftp-user=%s", "suseuser" ));
193 args.push_back(str::form("--ftp-passwd=%s", VERSION ));
197 DBG << "Anonymous FTP identification: '" << id << "'" << endl;
202 if ( url.getScheme() == "ftp" )
203 args.push_back(str::form("--ftp-user=%s", s.username().c_str() ));
204 else if ( url.getScheme() == "http" ||
205 url.getScheme() == "https" )
206 args.push_back(str::form("--http-user=%s", s.username().c_str() ));
208 if ( s.password().size() )
210 if ( url.getScheme() == "ftp" )
211 args.push_back(str::form("--ftp-passwd=%s", s.password().c_str() ));
212 else if ( url.getScheme() == "http" ||
213 url.getScheme() == "https" )
214 args.push_back(str::form("--http-passwd=%s", s.password().c_str() ));
218 if ( s.proxyEnabled() )
220 args.push_back(str::form("--http-proxy=%s", s.proxy().c_str() ));
221 if ( ! s.proxyUsername().empty() )
223 args.push_back(str::form("--http-proxy-user=%s", s.proxyUsername().c_str() ));
224 if ( ! s.proxyPassword().empty() )
225 args.push_back(str::form("--http-proxy-passwd=%s", s.proxyPassword().c_str() ));
229 if ( ! destination.empty() )
230 args.push_back(str::form("--dir=%s", destination.c_str()));
232 args.push_back(url.asString().c_str());
236 * comannd line for curl.
237 * The argument list gets passed as reference
240 void fillCurlCmdLine( const Pathname &curlpath,
241 const TransferSettings &s,
243 ExternalProgram::Arguments &args )
245 args.push_back(curlpath.c_str());
246 // only do a head request
247 args.push_back("-I");
248 args.push_back("-A"); args.push_back(s.userAgentString());
251 for ( TransferSettings::Headers::const_iterator it = s.headersBegin();
252 it != s.headersEnd();
255 args.push_back("-H");
256 args.push_back(it->c_str());
259 args.push_back("--connect-timeout");
260 args.push_back(str::numstring(s.timeout()));
262 if ( s.username().empty() )
264 if ( url.getScheme() == "ftp" )
266 string id = "yast2:";
268 args.push_back("--user");
270 DBG << "Anonymous FTP identification: '" << id << "'" << endl;
275 string userpass = s.username();
277 if ( s.password().size() )
278 userpass += (":" + s.password());
279 args.push_back("--user");
280 args.push_back(userpass);
283 if ( s.proxyEnabled() )
285 args.push_back("--proxy");
286 args.push_back(s.proxy());
287 if ( ! s.proxyUsername().empty() )
289 string userpass = s.proxyUsername();
291 if ( s.proxyPassword().size() )
292 userpass += (":" + s.proxyPassword());
293 args.push_back("--proxy-user");
294 args.push_back(userpass);
298 args.push_back("--url");
299 args.push_back(url.asString().c_str());
303 static const char *const anonymousIdHeader()
305 // we need to add the release and identifier to the
307 // The target could be not initialized, and then this information
309 Target_Ptr target = zypp::getZYpp()->getTarget();
311 static const std::string _value(
313 "X-Zypp-AnonymousId: %s",
314 target ? target->anonymousUniqueId().c_str() : "" )
316 return _value.c_str();
319 static const char *const distributionFlavorHeader()
321 // we need to add the release and identifier to the
323 // The target could be not initialized, and then this information
325 Target_Ptr target = zypp::getZYpp()->getTarget();
327 static const std::string _value(
328 str::trim( str::form(
329 "X-ZYpp-DistributionFlavor: %s",
330 target ? target->distributionFlavor().c_str() : "" ) )
332 return _value.c_str();
335 const char *const MediaAria2c::agentString()
337 // we need to add the release and identifier to the
339 // The target could be not initialized, and then this information
341 Target_Ptr target = zypp::getZYpp()->getTarget();
343 static const std::string _value(
347 , MediaAria2c::_aria2cVersion.c_str()
348 , target ? target->targetDistribution().c_str() : ""
351 return _value.c_str();
356 MediaAria2c::MediaAria2c( const Url & url_r,
357 const Pathname & attach_point_hint_r )
358 : MediaHandler( url_r, attach_point_hint_r,
359 "/", // urlpath at attachpoint
360 true ) // does_download
362 MIL << "MediaAria2c::MediaAria2c(" << url_r << ", " << attach_point_hint_r << ")" << endl;
364 if( !attachPoint().empty())
366 PathInfo ainfo(attachPoint());
367 Pathname apath(attachPoint() + "XXXXXX");
368 char *atemp = ::strdup( apath.asString().c_str());
370 if( !ainfo.isDir() || !ainfo.userMayRWX() ||
371 atemp == NULL || (atest=::mkdtemp(atemp)) == NULL)
373 WAR << "attach point " << ainfo.path()
374 << " is not useable for " << url_r.getScheme() << endl;
375 setAttachPoint("", true);
377 else if( atest != NULL)
384 //At this point, we initialize aria2c path
385 _aria2cPath = Pathname( whereisAria2c().asString() );
388 _aria2cVersion = getAria2cVersion();
391 void MediaAria2c::attachTo (bool next)
393 // clear last arguments
395 ZYPP_THROW(MediaNotSupportedException(_url));
397 if ( !_url.isValid() )
398 ZYPP_THROW(MediaBadUrlException(_url));
400 if( !isUseableAttachPoint(attachPoint()))
402 std::string mountpoint = createAttachPoint().asString();
404 if( mountpoint.empty())
405 ZYPP_THROW( MediaBadAttachPointException(url()));
407 setAttachPoint( mountpoint, true);
412 _settings.setUserAgentString(agentString());
413 _settings.addHeader(anonymousIdHeader());
414 _settings.addHeader(distributionFlavorHeader());
416 _settings.setTimeout(TRANSFER_TIMEOUT);
417 _settings.setConnectTimeout(CONNECT_TIMEOUT);
419 // fill some settings from url query parameters
420 fillSettingsFromUrl(_url, _settings);
422 // if the proxy was not set by url, then look
423 if ( _settings.proxy().empty() )
425 // at the system proxy settings
426 fillSettingsSystemProxy(_url, _settings);
429 DBG << "Proxy: " << (_settings.proxy().empty() ? "-none-" : _settings.proxy()) << endl;
431 MediaSourceRef media( new MediaSource(_url.getScheme(), _url.asString()));
432 setMediaSource(media);
437 MediaAria2c::checkAttachPoint(const Pathname &apoint) const
439 return MediaHandler::checkAttachPoint( apoint, true, true);
442 void MediaAria2c::disconnectFrom()
446 void MediaAria2c::releaseFrom( const std::string & ejectDev )
451 static Url getFileUrl(const Url & url, const Pathname & filename)
454 string path = url.getPathName();
455 if ( !path.empty() && path != "/" && *path.rbegin() == '/' &&
456 filename.absolute() )
458 // If url has a path with trailing slash, remove the leading slash from
459 // the absolute file name
460 path += filename.asString().substr( 1, filename.asString().size() - 1 );
462 else if ( filename.relative() )
464 // Add trailing slash to path, if not already there
465 if (path.empty()) path = "/";
466 else if (*path.rbegin() != '/' ) path += "/";
467 // Remove "./" from begin of relative file name
468 path += filename.asString().substr( 2, filename.asString().size() - 2 );
472 path += filename.asString();
475 newurl.setPathName(path);
479 void MediaAria2c::getFile( const Pathname & filename ) const
481 // Use absolute file name to prevent access of files outside of the
482 // hierarchy below the attach point.
483 getFileCopy(filename, localPath(filename).absolutename());
486 void MediaAria2c::getFileCopy( const Pathname & filename , const Pathname & target) const
488 callback::SendReport<DownloadProgressReport> report;
490 Url fileurl(getFileUrl(_url, filename));
494 ExternalProgram::Arguments args;
496 fillAriaCmdLine(_aria2cPath, _aria2cVersion, _settings, fileurl, target.dirname(), args);
502 report->start(_url, target.asString() );
504 ExternalProgram aria(args, ExternalProgram::Stderr_To_Stdout);
508 for(std::string ariaResponse( aria.receiveLine());
509 ariaResponse.length();
510 ariaResponse = aria.receiveLine())
512 //cout << ariaResponse;
514 if (!ariaResponse.substr(0,31).compare("Exception: Authorization failed") )
516 ZYPP_THROW(MediaUnauthorizedException(
517 _url, "Login failed.", "Login failed", "auth hint"
520 if (!ariaResponse.substr(0,29).compare("Exception: Resource not found") )
522 ZYPP_THROW(MediaFileNotFoundException(_url, filename));
525 if (!ariaResponse.substr(0,9).compare("[#2 SIZE:"))
529 size_t left_bound = ariaResponse.find('(',0) + 1;
530 size_t count = ariaResponse.find('%',left_bound) - left_bound;
531 //cout << ariaResponse.substr(left_bound, count) << endl;
532 //progressData.toMax();
533 report->progress ( std::atoi(ariaResponse.substr(left_bound, count).c_str()), _url, -1, -1 );
545 report->finish( _url , zypp::media::DownloadProgressReport::NO_ERROR, "");
549 // retry with proper authentication data
550 catch (MediaUnauthorizedException & ex_r)
552 if(authenticate(ex_r.hint(), !retry))
556 report->finish(fileurl, zypp::media::DownloadProgressReport::ACCESS_DENIED, ex_r.asUserHistory());
561 // unexpected exception
562 catch (MediaException & excpt_r)
564 // FIXME: error number fix
565 report->finish(fileurl, zypp::media::DownloadProgressReport::ERROR, excpt_r.asUserHistory());
566 ZYPP_RETHROW(excpt_r);
571 report->finish(fileurl, zypp::media::DownloadProgressReport::NO_ERROR, "");
574 bool MediaAria2c::getDoesFileExist( const Pathname & filename ) const
583 return doGetDoesFileExist( filename );
585 // authentication problem, retry with proper authentication data
586 catch (MediaUnauthorizedException & ex_r)
588 if(authenticate(ex_r.hint(), !retry))
593 // unexpected exception
594 catch (MediaException & excpt_r)
596 ZYPP_RETHROW(excpt_r);
604 bool MediaAria2c::doGetDoesFileExist( const Pathname & filename ) const
606 DBG << filename.asString() << endl;
607 callback::SendReport<DownloadProgressReport> report;
609 Url fileurl(getFileUrl(_url, filename));
612 ExternalProgram::Arguments args;
614 fillCurlCmdLine("/usr/bin/curl", _settings, fileurl, args);
620 report->start(_url, fileurl.asString() );
622 ExternalProgram curl(args, ExternalProgram::Stderr_To_Stdout);
624 for(std::string curlResponse( curl.receiveLine());
625 curlResponse.length();
626 curlResponse = curl.receiveLine())
629 if ( str::contains(curlResponse, "401 Authorization Required") )
631 ZYPP_THROW(MediaUnauthorizedException(
632 _url, "Login failed.", "Login failed", "auth hint"
636 if ( str::contains(curlResponse, "404 Not Found") )
639 if ( str::contains(curlResponse, "200 OK") )
643 int code = curl.close();
648 // connection problems
655 ZYPP_THROW(MediaException(_url.asString()));
659 report->finish( _url , zypp::media::DownloadProgressReport::NO_ERROR, "");
662 // retry with proper authentication data
663 catch (MediaUnauthorizedException & ex_r)
665 if(authenticate(ex_r.hint(), !retry))
669 report->finish(fileurl, zypp::media::DownloadProgressReport::ACCESS_DENIED, ex_r.asUserHistory());
674 // unexpected exception
675 catch (MediaException & excpt_r)
677 // FIXME: error number fix
678 report->finish(fileurl, zypp::media::DownloadProgressReport::ERROR, excpt_r.asUserHistory());
679 ZYPP_RETHROW(excpt_r);
684 report->finish(fileurl, zypp::media::DownloadProgressReport::NO_ERROR, "");
688 void MediaAria2c::getDir( const Pathname & dirname, bool recurse_r ) const
690 filesystem::DirContent content;
691 getDirInfo( content, dirname, /*dots*/false );
693 for ( filesystem::DirContent::const_iterator it = content.begin(); it != content.end(); ++it ) {
694 Pathname filename = dirname + it->name;
697 switch ( it->type ) {
698 case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
699 case filesystem::FT_FILE:
702 case filesystem::FT_DIR: // newer directory.yast contain at least directory info
704 getDir( filename, recurse_r );
706 res = assert_dir( localPath( filename ) );
708 WAR << "Ignore error (" << res << ") on creating local directory '" << localPath( filename ) << "'" << endl;
713 // don't provide devices, sockets, etc.
719 bool MediaAria2c::authenticate(const std::string & availAuthTypes, bool firstTry) const
725 void MediaAria2c::getDirInfo( std::list<std::string> & retlist,
726 const Pathname & dirname, bool dots ) const
728 getDirectoryYast( retlist, dirname, dots );
731 void MediaAria2c::getDirInfo( filesystem::DirContent & retlist,
732 const Pathname & dirname, bool dots ) const
734 getDirectoryYast( retlist, dirname, dots );
737 std::string MediaAria2c::getAria2cVersion()
746 ExternalProgram aria(argv, ExternalProgram::Stderr_To_Stdout);
748 std::string vResponse = aria.receiveLine();
750 return str::trim(vResponse);
753 #define ARIA_DEFAULT_BINARY "/usr/bin/aria2c"
755 Pathname MediaAria2c::whereisAria2c()
757 Pathname aria2cPathr(ARIA_DEFAULT_BINARY);
766 ExternalProgram aria(argv, ExternalProgram::Stderr_To_Stdout);
768 std::string ariaResponse( aria.receiveLine());
769 int code = aria.close();
773 aria2cPathr = str::trim(ariaResponse);
774 MIL << "We will use aria2c located here: " << aria2cPathr << endl;
778 MIL << "We don't know were is ari2ac binary. We will use aria2c located here: " << aria2cPathr << endl;