1 /*---------------------------------------------------------------------\
3 | |__ / \ / / . \ . \ |
8 \---------------------------------------------------------------------*/
9 /** \file zypp/media/MediaAria2c.cc
16 #include "zypp/base/Logger.h"
17 #include "zypp/ExternalProgram.h"
18 #include "zypp/ProgressData.h"
19 #include "zypp/base/String.h"
20 #include "zypp/base/Gettext.h"
21 #include "zypp/base/Sysconfig.h"
22 #include "zypp/base/Gettext.h"
23 #include "zypp/ZYppCallbacks.h"
25 #include "zypp/Target.h"
26 #include "zypp/ZYppFactory.h"
28 #include "zypp/media/MediaAria2c.h"
29 #include "zypp/media/proxyinfo/ProxyInfos.h"
30 #include "zypp/media/ProxyInfo.h"
31 #include "zypp/media/MediaUserAuth.h"
32 #include "zypp/thread/Once.h"
34 #include <sys/types.h>
36 #include <sys/mount.h>
40 #include <boost/format.hpp>
42 #define DETECT_DIR_INDEX 0
43 #define CONNECT_TIMEOUT 60
44 #define TRANSFER_TIMEOUT 60 * 3
45 #define TRANSFER_TIMEOUT_MAX 60 * 60
49 using namespace zypp::base;
56 Pathname MediaAria2c::_cookieFile = "/var/lib/YaST2/cookies";
57 Pathname MediaAria2c::_aria2cPath = "/usr/local/bin/aria2c";
58 std::string MediaAria2c::_aria2cVersion = "WE DON'T KNOW ARIA2C VERSION";
60 //check if aria2c is present in the system
62 MediaAria2c::existsAria2cmd()
71 ExternalProgram aria(argv, ExternalProgram::Stderr_To_Stdout);
72 return ( aria.close() == 0 );
75 void fillSettingsFromUrl( const Url &url, TransferSettings &s )
77 std::string param(url.getQueryParam("timeout"));
80 long num = str::strtonum<long>(param);
81 if( num >= 0 && num <= TRANSFER_TIMEOUT_MAX)
85 if ( ! url.getUsername().empty() )
87 s.setUsername(url.getUsername());
88 if ( url.getPassword().size() )
90 s.setPassword(url.getPassword());
94 string proxy = url.getQueryParam( "proxy" );
96 if ( ! proxy.empty() )
98 string proxyport( url.getQueryParam( "proxyport" ) );
99 if ( ! proxyport.empty() ) {
100 proxy += ":" + proxyport;
103 s.setProxyEnabled(true);
107 void fillSettingsSystemProxy( const Url&url, TransferSettings &s )
109 ProxyInfo proxy_info (ProxyInfo::ImplPtr(new ProxyInfoSysconfig("proxy")));
111 if ( proxy_info.enabled())
113 s.setProxyEnabled(true);
114 std::list<std::string> nope = proxy_info.noProxy();
115 for (ProxyInfo::NoProxyIterator it = proxy_info.noProxyBegin();
116 it != proxy_info.noProxyEnd();
119 std::string host( str::toLower(url.getHost()));
120 std::string temp( str::toLower(*it));
122 // no proxy if it points to a suffix
123 // preceeded by a '.', that maches
124 // the trailing portion of the host.
125 if( temp.size() > 1 && temp.at(0) == '.')
127 if(host.size() > temp.size() &&
128 host.compare(host.size() - temp.size(), temp.size(), temp) == 0)
130 DBG << "NO_PROXY: '" << *it << "' matches host '"
131 << host << "'" << endl;
132 s.setProxyEnabled(false);
137 // no proxy if we have an exact match
140 DBG << "NO_PROXY: '" << *it << "' matches host '"
141 << host << "'" << endl;
142 s.setProxyEnabled(false);
147 if ( s.proxyEnabled() )
148 s.setProxy(proxy_info.proxy(url.getScheme()));
154 * comannd line for aria.
155 * The argument list gets passed as reference
158 void fillAriaCmdLine( const Pathname &ariapath,
159 const TransferSettings &s,
161 const Pathname &destination,
162 ExternalProgram::Arguments &args )
164 args.push_back(ariapath.c_str());
165 args.push_back(str::form("--user-agent=%s", s.userAgentString().c_str()));
166 args.push_back("--summary-interval=1");
167 args.push_back("--follow-metalink=mem");
168 args.push_back("--check-integrity=true");
170 // add the anonymous id.
171 for ( TransferSettings::Headers::const_iterator it = s.headersBegin();
172 it != s.headersEnd();
174 args.push_back(str::form("--header=%s", it->c_str() ));
176 args.push_back( str::form("--connect-timeout=%ld", s.timeout()));
178 if ( s.username().empty() )
180 if ( url.getScheme() == "ftp" )
183 args.push_back(str::form("--ftp-user=%s", "suseuser" ));
184 args.push_back(str::form("--ftp-passwd=%s", VERSION ));
188 DBG << "Anonymous FTP identification: '" << id << "'" << endl;
193 if ( url.getScheme() == "ftp" )
194 args.push_back(str::form("--ftp-user=%s", s.username().c_str() ));
195 else if ( url.getScheme() == "http" ||
196 url.getScheme() == "https" )
197 args.push_back(str::form("--http-user=%s", s.username().c_str() ));
199 if ( s.password().size() )
201 if ( url.getScheme() == "ftp" )
202 args.push_back(str::form("--ftp-passwd=%s", s.password().c_str() ));
203 else if ( url.getScheme() == "http" ||
204 url.getScheme() == "https" )
205 args.push_back(str::form("--http-passwd=%s", s.password().c_str() ));
209 if ( s.proxyEnabled() )
211 args.push_back(str::form("--http-proxy=%s", s.proxy().c_str() ));
212 if ( ! s.proxyUsername().empty() )
214 args.push_back(str::form("--http-proxy-user=%s", s.proxyUsername().c_str() ));
215 if ( ! s.proxyPassword().empty() )
216 args.push_back(str::form("--http-proxy-passwd=%s", s.proxyPassword().c_str() ));
220 if ( ! destination.empty() )
221 args.push_back(str::form("--dir=%s", destination.c_str()));
223 args.push_back(url.asString().c_str());
227 * comannd line for curl.
228 * The argument list gets passed as reference
231 void fillCurlCmdLine( const Pathname &curlpath,
232 const TransferSettings &s,
234 ExternalProgram::Arguments &args )
236 args.push_back(curlpath.c_str());
237 // only do a head request
238 args.push_back("-I");
239 args.push_back("-A"); args.push_back(s.userAgentString());
242 for ( TransferSettings::Headers::const_iterator it = s.headersBegin();
243 it != s.headersEnd();
246 args.push_back("-H");
247 args.push_back(it->c_str());
250 args.push_back("--connect-timeout");
251 args.push_back(str::numstring(s.timeout()));
253 if ( s.username().empty() )
255 if ( url.getScheme() == "ftp" )
257 string id = "yast2:";
259 args.push_back("--user");
261 DBG << "Anonymous FTP identification: '" << id << "'" << endl;
266 string userpass = s.username();
268 if ( s.password().size() )
269 userpass += (":" + s.password());
270 args.push_back("--user");
271 args.push_back(userpass);
274 if ( s.proxyEnabled() )
276 args.push_back("--proxy");
277 args.push_back(s.proxy());
278 if ( ! s.proxyUsername().empty() )
280 string userpass = s.proxyUsername();
282 if ( s.proxyPassword().size() )
283 userpass += (":" + s.proxyPassword());
284 args.push_back("--proxy-user");
285 args.push_back(userpass);
289 args.push_back("--url");
290 args.push_back(url.asString().c_str());
294 static const char *const anonymousIdHeader()
296 // we need to add the release and identifier to the
298 // The target could be not initialized, and then this information
300 Target_Ptr target = zypp::getZYpp()->getTarget();
302 static const std::string _value(
304 "X-Zypp-AnonymousId: %s",
305 target ? target->anonymousUniqueId().c_str() : "" )
307 return _value.c_str();
310 static const char *const distributionFlavorHeader()
312 // we need to add the release and identifier to the
314 // The target could be not initialized, and then this information
316 Target_Ptr target = zypp::getZYpp()->getTarget();
318 static const std::string _value(
319 str::trim( str::form(
320 "X-ZYpp-DistributionFlavor: %s",
321 target ? target->distributionFlavor().c_str() : "" ) )
323 return _value.c_str();
326 const char *const MediaAria2c::agentString()
328 // we need to add the release and identifier to the
330 // The target could be not initialized, and then this information
332 Target_Ptr target = zypp::getZYpp()->getTarget();
334 static const std::string _value(
338 , MediaAria2c::_aria2cVersion.c_str()
339 , target ? target->targetDistribution().c_str() : ""
342 return _value.c_str();
347 MediaAria2c::MediaAria2c( const Url & url_r,
348 const Pathname & attach_point_hint_r )
349 : MediaHandler( url_r, attach_point_hint_r,
350 "/", // urlpath at attachpoint
351 true ) // does_download
353 MIL << "MediaAria2c::MediaAria2c(" << url_r << ", " << attach_point_hint_r << ")" << endl;
355 if( !attachPoint().empty())
357 PathInfo ainfo(attachPoint());
358 Pathname apath(attachPoint() + "XXXXXX");
359 char *atemp = ::strdup( apath.asString().c_str());
361 if( !ainfo.isDir() || !ainfo.userMayRWX() ||
362 atemp == NULL || (atest=::mkdtemp(atemp)) == NULL)
364 WAR << "attach point " << ainfo.path()
365 << " is not useable for " << url_r.getScheme() << endl;
366 setAttachPoint("", true);
368 else if( atest != NULL)
375 //At this point, we initialize aria2c path
376 _aria2cPath = Pathname( whereisAria2c().asString() );
379 _aria2cVersion = getAria2cVersion();
382 void MediaAria2c::attachTo (bool next)
384 // clear last arguments
386 ZYPP_THROW(MediaNotSupportedException(_url));
388 if ( !_url.isValid() )
389 ZYPP_THROW(MediaBadUrlException(_url));
391 if( !isUseableAttachPoint(attachPoint()))
393 std::string mountpoint = createAttachPoint().asString();
395 if( mountpoint.empty())
396 ZYPP_THROW( MediaBadAttachPointException(url()));
398 setAttachPoint( mountpoint, true);
403 _settings.setUserAgentString(agentString());
404 _settings.addHeader(anonymousIdHeader());
405 _settings.addHeader(distributionFlavorHeader());
407 _settings.setTimeout(TRANSFER_TIMEOUT);
408 _settings.setConnectTimeout(CONNECT_TIMEOUT);
410 // fill some settings from url query parameters
411 fillSettingsFromUrl(_url, _settings);
413 // if the proxy was not set by url, then look
414 if ( _settings.proxy().empty() )
416 // at the system proxy settings
417 fillSettingsSystemProxy(_url, _settings);
420 DBG << "Proxy: " << (_settings.proxy().empty() ? "-none-" : _settings.proxy()) << endl;
422 MediaSourceRef media( new MediaSource(_url.getScheme(), _url.asString()));
423 setMediaSource(media);
428 MediaAria2c::checkAttachPoint(const Pathname &apoint) const
430 return MediaHandler::checkAttachPoint( apoint, true, true);
433 void MediaAria2c::disconnectFrom()
437 void MediaAria2c::releaseFrom( const std::string & ejectDev )
442 static Url getFileUrl(const Url & url, const Pathname & filename)
445 string path = url.getPathName();
446 if ( !path.empty() && path != "/" && *path.rbegin() == '/' &&
447 filename.absolute() )
449 // If url has a path with trailing slash, remove the leading slash from
450 // the absolute file name
451 path += filename.asString().substr( 1, filename.asString().size() - 1 );
453 else if ( filename.relative() )
455 // Add trailing slash to path, if not already there
456 if (path.empty()) path = "/";
457 else if (*path.rbegin() != '/' ) path += "/";
458 // Remove "./" from begin of relative file name
459 path += filename.asString().substr( 2, filename.asString().size() - 2 );
463 path += filename.asString();
466 newurl.setPathName(path);
470 void MediaAria2c::getFile( const Pathname & filename ) const
472 // Use absolute file name to prevent access of files outside of the
473 // hierarchy below the attach point.
474 getFileCopy(filename, localPath(filename).absolutename());
477 void MediaAria2c::getFileCopy( const Pathname & filename , const Pathname & target) const
479 callback::SendReport<DownloadProgressReport> report;
481 Url fileurl(getFileUrl(_url, filename));
485 ExternalProgram::Arguments args;
487 fillAriaCmdLine(_aria2cPath, _settings, fileurl, target.dirname(), args);
493 report->start(_url, target.asString() );
495 ExternalProgram aria(args, ExternalProgram::Stderr_To_Stdout);
499 for(std::string ariaResponse( aria.receiveLine());
500 ariaResponse.length();
501 ariaResponse = aria.receiveLine())
503 //cout << ariaResponse;
505 if (!ariaResponse.substr(0,31).compare("Exception: Authorization failed") )
507 ZYPP_THROW(MediaUnauthorizedException(
508 _url, "Login failed.", "Login failed", "auth hint"
511 if (!ariaResponse.substr(0,29).compare("Exception: Resource not found") )
513 ZYPP_THROW(MediaFileNotFoundException(_url, filename));
516 if (!ariaResponse.substr(0,9).compare("[#2 SIZE:"))
520 size_t left_bound = ariaResponse.find('(',0) + 1;
521 size_t count = ariaResponse.find('%',left_bound) - left_bound;
522 //cout << ariaResponse.substr(left_bound, count) << endl;
523 //progressData.toMax();
524 report->progress ( std::atoi(ariaResponse.substr(left_bound, count).c_str()), _url, -1, -1 );
536 report->finish( _url , zypp::media::DownloadProgressReport::NO_ERROR, "");
540 // retry with proper authentication data
541 catch (MediaUnauthorizedException & ex_r)
543 if(authenticate(ex_r.hint(), !retry))
547 report->finish(fileurl, zypp::media::DownloadProgressReport::ACCESS_DENIED, ex_r.asUserHistory());
552 // unexpected exception
553 catch (MediaException & excpt_r)
555 // FIXME: error number fix
556 report->finish(fileurl, zypp::media::DownloadProgressReport::ERROR, excpt_r.asUserHistory());
557 ZYPP_RETHROW(excpt_r);
562 report->finish(fileurl, zypp::media::DownloadProgressReport::NO_ERROR, "");
565 bool MediaAria2c::getDoesFileExist( const Pathname & filename ) const
574 return doGetDoesFileExist( filename );
576 // authentication problem, retry with proper authentication data
577 catch (MediaUnauthorizedException & ex_r)
579 if(authenticate(ex_r.hint(), !retry))
584 // unexpected exception
585 catch (MediaException & excpt_r)
587 ZYPP_RETHROW(excpt_r);
595 bool MediaAria2c::doGetDoesFileExist( const Pathname & filename ) const
597 DBG << filename.asString() << endl;
598 callback::SendReport<DownloadProgressReport> report;
600 Url fileurl(getFileUrl(_url, filename));
603 ExternalProgram::Arguments args;
605 fillCurlCmdLine("/usr/bin/curl", _settings, fileurl, args);
611 report->start(_url, fileurl.asString() );
613 ExternalProgram curl(args, ExternalProgram::Stderr_To_Stdout);
615 for(std::string curlResponse( curl.receiveLine());
616 curlResponse.length();
617 curlResponse = curl.receiveLine())
620 if ( str::contains(curlResponse, "401 Authorization Required") )
622 ZYPP_THROW(MediaUnauthorizedException(
623 _url, "Login failed.", "Login failed", "auth hint"
627 if ( str::contains(curlResponse, "404 Not Found") )
630 if ( str::contains(curlResponse, "200 OK") )
634 int code = curl.close();
639 // connection problems
646 ZYPP_THROW(MediaException(_url.asString()));
650 report->finish( _url , zypp::media::DownloadProgressReport::NO_ERROR, "");
653 // retry with proper authentication data
654 catch (MediaUnauthorizedException & ex_r)
656 if(authenticate(ex_r.hint(), !retry))
660 report->finish(fileurl, zypp::media::DownloadProgressReport::ACCESS_DENIED, ex_r.asUserHistory());
665 // unexpected exception
666 catch (MediaException & excpt_r)
668 // FIXME: error number fix
669 report->finish(fileurl, zypp::media::DownloadProgressReport::ERROR, excpt_r.asUserHistory());
670 ZYPP_RETHROW(excpt_r);
675 report->finish(fileurl, zypp::media::DownloadProgressReport::NO_ERROR, "");
679 void MediaAria2c::getDir( const Pathname & dirname, bool recurse_r ) const
681 filesystem::DirContent content;
682 getDirInfo( content, dirname, /*dots*/false );
684 for ( filesystem::DirContent::const_iterator it = content.begin(); it != content.end(); ++it ) {
685 Pathname filename = dirname + it->name;
688 switch ( it->type ) {
689 case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
690 case filesystem::FT_FILE:
693 case filesystem::FT_DIR: // newer directory.yast contain at least directory info
695 getDir( filename, recurse_r );
697 res = assert_dir( localPath( filename ) );
699 WAR << "Ignore error (" << res << ") on creating local directory '" << localPath( filename ) << "'" << endl;
704 // don't provide devices, sockets, etc.
710 bool MediaAria2c::authenticate(const std::string & availAuthTypes, bool firstTry) const
716 void MediaAria2c::getDirInfo( std::list<std::string> & retlist,
717 const Pathname & dirname, bool dots ) const
719 getDirectoryYast( retlist, dirname, dots );
722 void MediaAria2c::getDirInfo( filesystem::DirContent & retlist,
723 const Pathname & dirname, bool dots ) const
725 getDirectoryYast( retlist, dirname, dots );
728 std::string MediaAria2c::getAria2cVersion()
737 ExternalProgram aria(argv, ExternalProgram::Stderr_To_Stdout);
739 std::string vResponse = aria.receiveLine();
741 return str::trim(vResponse);
744 #define ARIA_DEFAULT_BINARY "/usr/bin/aria2c"
746 Pathname MediaAria2c::whereisAria2c()
748 Pathname aria2cPathr(ARIA_DEFAULT_BINARY);
757 ExternalProgram aria(argv, ExternalProgram::Stderr_To_Stdout);
759 std::string ariaResponse( aria.receiveLine());
760 int code = aria.close();
764 aria2cPathr = str::trim(ariaResponse);
765 MIL << "We will use aria2c located here: " << aria2cPathr << endl;
769 MIL << "We don't know were is ari2ac binary. We will use aria2c located here: " << aria2cPathr << endl;