1 /*---------------------------------------------------------------------\
3 | |__ / \ / / . \ . \ |
8 \---------------------------------------------------------------------*/
9 /** \file zypp/media/MediaCurl.cc
16 #include "zypp/base/Logger.h"
17 #include "zypp/ExternalProgram.h"
18 #include "zypp/base/String.h"
19 #include "zypp/base/Gettext.h"
20 #include "zypp/base/Sysconfig.h"
21 #include "zypp/base/Gettext.h"
23 #include "zypp/media/MediaCurl.h"
24 #include "zypp/media/proxyinfo/ProxyInfos.h"
25 #include "zypp/media/ProxyInfo.h"
26 #include "zypp/media/MediaUserAuth.h"
27 #include "zypp/media/CredentialManager.h"
28 #include "zypp/media/CurlConfig.h"
29 #include "zypp/thread/Once.h"
30 #include "zypp/Target.h"
31 #include "zypp/ZYppFactory.h"
34 #include <sys/types.h>
36 #include <sys/mount.h>
40 #include <boost/format.hpp>
42 #define DETECT_DIR_INDEX 0
43 #define CONNECT_TIMEOUT 60
44 #define TRANSFER_TIMEOUT 60 * 3
45 #define TRANSFER_TIMEOUT_MAX 60 * 60
49 using namespace zypp::base;
53 zypp::thread::OnceFlag g_InitOnceFlag = PTHREAD_ONCE_INIT;
54 zypp::thread::OnceFlag g_FreeOnceFlag = PTHREAD_ONCE_INIT;
56 extern "C" void _do_free_once()
58 curl_global_cleanup();
61 extern "C" void globalFreeOnce()
63 zypp::thread::callOnce(g_FreeOnceFlag, _do_free_once);
66 extern "C" void _do_init_once()
68 CURLcode ret = curl_global_init( CURL_GLOBAL_ALL );
71 WAR << "curl global init failed" << endl;
75 // register at exit handler ?
76 // this may cause trouble, because we can protect it
77 // against ourself only.
78 // if the app sets an atexit handler as well, it will
79 // cause a double free while the second of them runs.
81 //std::atexit( globalFreeOnce);
84 inline void globalInitOnce()
86 zypp::thread::callOnce(g_InitOnceFlag, _do_init_once);
89 int log_curl(CURL *curl, curl_infotype info,
90 char *ptr, size_t len, void *max_lvl)
96 case CURLINFO_TEXT: lvl = 1; pfx = "*"; break;
97 case CURLINFO_HEADER_IN: lvl = 2; pfx = "<"; break;
98 case CURLINFO_HEADER_OUT: lvl = 2; pfx = ">"; break;
101 if( lvl > 0 && max_lvl != NULL && lvl <= *((long *)max_lvl))
103 std::string msg(ptr, len);
104 std::list<std::string> lines;
105 std::list<std::string>::const_iterator line;
106 zypp::str::split(msg, std::back_inserter(lines), "\r\n");
107 for(line = lines.begin(); line != lines.end(); ++line)
109 DBG << pfx << " " << *line << endl;
122 ProgressData(const long _timeout, const zypp::Url &_url = zypp::Url(),
123 callback::SendReport<DownloadProgressReport> *_report=NULL)
138 callback::SendReport<DownloadProgressReport> *report;
139 // download rate of the last period (cca 1 sec)
141 // bytes downloaded at the start of the last period
143 // seconds from the start of the download
145 // average download rate
147 // last time the progress was reported
149 // bytes downloaded at the moment the progress was last reported
151 // bytes uploaded at the moment the progress was last reported
156 ///////////////////////////////////////////////////////////////////
158 inline void escape( string & str_r,
159 const char char_r, const string & escaped_r ) {
160 for ( string::size_type pos = str_r.find( char_r );
161 pos != string::npos; pos = str_r.find( char_r, pos ) ) {
162 str_r.replace( pos, 1, escaped_r );
166 inline string escapedPath( string path_r ) {
167 escape( path_r, ' ', "%20" );
171 inline string unEscape( string text_r ) {
172 char * tmp = curl_unescape( text_r.c_str(), 0 );
180 ///////////////////////////////////////////////////////////////////
182 // CLASS NAME : MediaCurl
184 ///////////////////////////////////////////////////////////////////
186 Pathname MediaCurl::_cookieFile = "/var/lib/YaST2/cookies";
188 const char *const MediaCurl::anonymousIdHeader()
190 // we need to add the release and identifier to the
192 // The target could be not initialized, and then this information
195 // FIXME this has to go away as soon as the target
196 // does not throw when not initialized.
198 target = zypp::getZYpp()->target();
200 catch ( const Exception &e )
205 static const std::string _value(
207 "X-ZYpp-AnonymousUniqueId: %s",
208 target ? target->anonymousUniqueId().c_str() : "" )
210 return _value.c_str();
213 const char *const MediaCurl::agentString()
215 // we need to add the release and identifier to the
217 // The target could be not initialized, and then this information
220 // FIXME this has to go away as soon as the target
221 // does not throw when not initialized.
223 target = zypp::getZYpp()->target();
225 catch ( const Exception &e )
230 static const std::string _value(
232 "ZYpp %s (curl %s) %s"
234 , curl_version_info(CURLVERSION_NOW)->version
235 , target ? target->targetDistribution().c_str() : ""
238 return _value.c_str();
242 MediaCurl::MediaCurl( const Url & url_r,
243 const Pathname & attach_point_hint_r )
244 : MediaHandler( url_r, attach_point_hint_r,
245 "/", // urlpath at attachpoint
246 true ), // does_download
249 _curlError[0] = '\0';
252 MIL << "MediaCurl::MediaCurl(" << url_r << ", " << attach_point_hint_r << ")" << endl;
256 if( !attachPoint().empty())
258 PathInfo ainfo(attachPoint());
259 Pathname apath(attachPoint() + "XXXXXX");
260 char *atemp = ::strdup( apath.asString().c_str());
262 if( !ainfo.isDir() || !ainfo.userMayRWX() ||
263 atemp == NULL || (atest=::mkdtemp(atemp)) == NULL)
265 WAR << "attach point " << ainfo.path()
266 << " is not useable for " << url_r.getScheme() << endl;
267 setAttachPoint("", true);
269 else if( atest != NULL)
277 void MediaCurl::setCookieFile( const Pathname &fileName )
279 _cookieFile = fileName;
282 ///////////////////////////////////////////////////////////////////
285 // METHOD NAME : MediaCurl::attachTo
286 // METHOD TYPE : PMError
288 // DESCRIPTION : Asserted that not already attached, and attachPoint is a directory.
290 void MediaCurl::attachTo (bool next)
293 ZYPP_THROW(MediaNotSupportedException(_url));
295 if ( !_url.isValid() )
296 ZYPP_THROW(MediaBadUrlException(_url));
299 CurlConfig::parseConfig(curlconf); // parse ~/.curlrc
301 curl_version_info_data *curl_info = NULL;
302 curl_info = curl_version_info(CURLVERSION_NOW);
303 // curl_info does not need any free (is static)
304 if (curl_info->protocols)
306 const char * const *proto;
307 std::string scheme( _url.getScheme());
309 for(proto=curl_info->protocols; !found && *proto; ++proto)
311 if( scheme == std::string((const char *)*proto))
316 std::string msg("Unsupported protocol '");
319 ZYPP_THROW(MediaBadUrlException(_url, msg));
323 if( !isUseableAttachPoint(attachPoint()))
325 std::string mountpoint = createAttachPoint().asString();
327 if( mountpoint.empty())
328 ZYPP_THROW( MediaBadAttachPointException(url()));
330 setAttachPoint( mountpoint, true);
333 disconnectFrom(); // clean _curl if needed
334 _curl = curl_easy_init();
336 ZYPP_THROW(MediaCurlInitException(_url));
340 char *ptr = getenv("ZYPP_MEDIA_CURL_DEBUG");
341 _curlDebug = (ptr && *ptr) ? str::strtonum<long>( ptr) : 0L;
344 curl_easy_setopt( _curl, CURLOPT_VERBOSE, 1);
345 curl_easy_setopt( _curl, CURLOPT_DEBUGFUNCTION, log_curl);
346 curl_easy_setopt( _curl, CURLOPT_DEBUGDATA, &_curlDebug);
350 CURLcode ret = curl_easy_setopt( _curl, CURLOPT_ERRORBUFFER, _curlError );
353 ZYPP_THROW(MediaCurlSetOptException(_url, "Error setting error buffer"));
356 ret = curl_easy_setopt( _curl, CURLOPT_FAILONERROR, true );
359 ZYPP_THROW(MediaCurlSetOptException(_url, _curlError));
362 ret = curl_easy_setopt( _curl, CURLOPT_NOSIGNAL, 1 );
365 ZYPP_THROW(MediaCurlSetOptException(_url, _curlError));
372 _xfer_timeout = TRANSFER_TIMEOUT;
374 std::string param(_url.getQueryParam("timeout"));
377 long num = str::strtonum<long>( param);
378 if( num >= 0 && num <= TRANSFER_TIMEOUT_MAX)
386 ret = curl_easy_setopt( _curl, CURLOPT_CONNECTTIMEOUT, CONNECT_TIMEOUT);
389 ZYPP_THROW(MediaCurlSetOptException(_url, _curlError));
392 if ( _url.getScheme() == "http" ) {
393 // follow any Location: header that the server sends as part of
394 // an HTTP header (#113275)
395 ret = curl_easy_setopt ( _curl, CURLOPT_FOLLOWLOCATION, true );
398 ZYPP_THROW(MediaCurlSetOptException(_url, _curlError));
400 ret = curl_easy_setopt ( _curl, CURLOPT_MAXREDIRS, 3L );
403 ZYPP_THROW(MediaCurlSetOptException(_url, _curlError));
406 ret = curl_easy_setopt ( _curl, CURLOPT_USERAGENT, agentString() );
411 ZYPP_THROW(MediaCurlSetOptException(_url, _curlError));
415 if ( _url.getScheme() == "https" )
417 bool verify_peer = false;
418 bool verify_host = false;
420 std::string verify( _url.getQueryParam("ssl_verify"));
421 if( verify.empty() ||
435 std::vector<std::string> flags;
436 std::vector<std::string>::const_iterator flag;
437 str::split( verify, std::back_inserter(flags), ",");
438 for(flag = flags.begin(); flag != flags.end(); ++flag)
452 ZYPP_THROW(MediaBadUrlException(_url, "Unknown ssl_verify flag"));
457 _ca_path = Pathname(_url.getQueryParam("ssl_capath")).asString();
458 if( _ca_path.empty())
460 _ca_path = "/etc/ssl/certs/";
463 if( !PathInfo(_ca_path).isDir() || !Pathname(_ca_path).absolute())
466 ZYPP_THROW(MediaBadUrlException(_url, "Invalid ssl_capath path"));
469 if( verify_peer || verify_host)
471 ret = curl_easy_setopt( _curl, CURLOPT_CAPATH, _ca_path.c_str());
474 ZYPP_THROW(MediaCurlSetOptException(_url, _curlError));
478 ret = curl_easy_setopt( _curl, CURLOPT_SSL_VERIFYPEER, verify_peer ? 1L : 0L);
481 ZYPP_THROW(MediaCurlSetOptException(_url, _curlError));
483 ret = curl_easy_setopt( _curl, CURLOPT_SSL_VERIFYHOST, verify_host ? 2L : 0L);
486 ZYPP_THROW(MediaCurlSetOptException(_url, _curlError));
489 ret = curl_easy_setopt ( _curl, CURLOPT_USERAGENT, agentString() );
492 ZYPP_THROW(MediaCurlSetOptException(_url, _curlError));
495 // now add the anonymous id header
496 curl_slist *chunk = NULL;
497 chunk = curl_slist_append(chunk, anonymousIdHeader());
498 ret = curl_easy_setopt ( _curl, CURLOPT_HTTPHEADER, chunk );
499 curl_slist_free_all(chunk);
503 ZYPP_THROW(MediaCurlSetOptException(_url, _curlError));
509 /*---------------------------------------------------------------*
510 CURLOPT_USERPWD: [user name]:[password]
512 Url::username/password -> CURLOPT_USERPWD
513 If not provided, anonymous FTP identification
514 *---------------------------------------------------------------*/
516 if ( _url.getUsername().empty() ) {
517 if ( _url.getScheme() == "ftp" ) {
518 string id = "yast2@";
520 DBG << "Anonymous FTP identification: '" << id << "'" << endl;
521 _userpwd = "anonymous:" + id;
524 _userpwd = _url.getUsername();
525 if ( _url.getPassword().size() ) {
526 _userpwd += ":" + _url.getPassword();
530 if ( _userpwd.size() ) {
531 _userpwd = unEscape( _userpwd );
532 ret = curl_easy_setopt( _curl, CURLOPT_USERPWD, _userpwd.c_str() );
535 ZYPP_THROW(MediaCurlSetOptException(_url, _curlError));
538 // HTTP authentication type
539 if(_url.getScheme() == "http" || _url.getScheme() == "https")
541 string use_auth = _url.getQueryParam("auth");
542 if( use_auth.empty())
543 use_auth = "digest,basic";
547 long auth = CurlAuthData::auth_type_str2long(use_auth);
548 if( auth != CURLAUTH_NONE)
550 DBG << "Enabling HTTP authentication methods: " << use_auth
551 << " (CURLOPT_HTTPAUTH=" << auth << ")" << std::endl;
553 ret = curl_easy_setopt( _curl, CURLOPT_HTTPAUTH, auth);
556 ZYPP_THROW(MediaCurlSetOptException(_url, _curlError));
560 catch (MediaException & ex_r)
562 string auth_hint = getAuthHint();
564 DBG << "Rethrowing as MediaUnauthorizedException. auth hint: '"
565 << auth_hint << "'" << endl;
567 ZYPP_THROW(MediaUnauthorizedException(
568 _url, ex_r.msg(), _curlError, auth_hint
574 /*---------------------------------------------------------------*
575 CURLOPT_PROXY: host[:port]
577 Url::option(proxy and proxyport) -> CURLOPT_PROXY
578 If not provided, /etc/sysconfig/proxy is evaluated
579 *---------------------------------------------------------------*/
581 _proxy = _url.getQueryParam( "proxy" );
583 if ( ! _proxy.empty() ) {
584 string proxyport( _url.getQueryParam( "proxyport" ) );
585 if ( ! proxyport.empty() ) {
586 _proxy += ":" + proxyport;
590 ProxyInfo proxy_info (ProxyInfo::ImplPtr(new ProxyInfoSysconfig("proxy")));
592 if ( proxy_info.enabled())
594 bool useproxy = true;
596 std::list<std::string> nope = proxy_info.noProxy();
597 for (ProxyInfo::NoProxyIterator it = proxy_info.noProxyBegin();
598 it != proxy_info.noProxyEnd();
601 std::string host( str::toLower(_url.getHost()));
602 std::string temp( str::toLower(*it));
604 // no proxy if it points to a suffix
605 // preceeded by a '.', that maches
606 // the trailing portion of the host.
607 if( temp.size() > 1 && temp.at(0) == '.')
609 if(host.size() > temp.size() &&
610 host.compare(host.size() - temp.size(), temp.size(), temp) == 0)
612 DBG << "NO_PROXY: '" << *it << "' matches host '"
613 << host << "'" << endl;
619 // no proxy if we have an exact match
622 DBG << "NO_PROXY: '" << *it << "' matches host '"
623 << host << "'" << endl;
630 _proxy = proxy_info.proxy(_url.getScheme());
636 DBG << "Proxy: " << (_proxy.empty() ? "-none-" : _proxy) << endl;
638 if ( ! _proxy.empty() ) {
640 ret = curl_easy_setopt( _curl, CURLOPT_PROXY, _proxy.c_str() );
643 ZYPP_THROW(MediaCurlSetOptException(_url, _curlError));
646 /*---------------------------------------------------------------*
647 CURLOPT_PROXYUSERPWD: [user name]:[password]
649 Url::option(proxyuser and proxypassword) -> CURLOPT_PROXYUSERPWD
650 If not provided, $HOME/.curlrc is evaluated
651 *---------------------------------------------------------------*/
653 _proxyuserpwd = _url.getQueryParam( "proxyuser" );
655 if ( ! _proxyuserpwd.empty() ) {
656 string proxypassword( _url.getQueryParam( "proxypassword" ) );
657 if ( ! proxypassword.empty() ) {
658 _proxyuserpwd += ":" + proxypassword;
661 if (curlconf.proxyuserpwd.empty())
662 DBG << "~/.curlrc does not contain the proxy-user option" << endl;
665 _proxyuserpwd = curlconf.proxyuserpwd;
666 DBG << "using proxy-user from ~/.curlrc" << endl;
670 _proxyuserpwd = unEscape( _proxyuserpwd );
671 ret = curl_easy_setopt( _curl, CURLOPT_PROXYUSERPWD, _proxyuserpwd.c_str() );
674 ZYPP_THROW(MediaCurlSetOptException(_url, _curlError));
678 /*---------------------------------------------------------------*
679 *---------------------------------------------------------------*/
681 _currentCookieFile = _cookieFile.asString();
683 ret = curl_easy_setopt( _curl, CURLOPT_COOKIEFILE,
684 _currentCookieFile.c_str() );
687 ZYPP_THROW(MediaCurlSetOptException(_url, _curlError));
690 ret = curl_easy_setopt( _curl, CURLOPT_COOKIEJAR,
691 _currentCookieFile.c_str() );
694 ZYPP_THROW(MediaCurlSetOptException(_url, _curlError));
697 ret = curl_easy_setopt( _curl, CURLOPT_PROGRESSFUNCTION,
701 ZYPP_THROW(MediaCurlSetOptException(_url, _curlError));
704 ret = curl_easy_setopt( _curl, CURLOPT_NOPROGRESS, false );
707 ZYPP_THROW(MediaCurlSetOptException(_url, _curlError));
711 ret = curl_easy_setopt( _curl, CURLOPT_PROXY_TRANSFER_MODE, 1 );
714 ZYPP_THROW(MediaCurlSetOptException(_url, _curlError));
718 // FIXME: need a derived class to propelly compare url's
719 MediaSourceRef media( new MediaSource(_url.getScheme(), _url.asString()));
720 setMediaSource(media);
724 MediaCurl::checkAttachPoint(const Pathname &apoint) const
726 return MediaHandler::checkAttachPoint( apoint, true, true);
729 ///////////////////////////////////////////////////////////////////
732 // METHOD NAME : MediaCurl::disconnectFrom
733 // METHOD TYPE : PMError
735 void MediaCurl::disconnectFrom()
739 curl_easy_cleanup( _curl );
745 ///////////////////////////////////////////////////////////////////
748 // METHOD NAME : MediaCurl::releaseFrom
749 // METHOD TYPE : void
751 // DESCRIPTION : Asserted that media is attached.
753 void MediaCurl::releaseFrom( const std::string & ejectDev )
758 static Url getFileUrl(const Url & url, const Pathname & filename)
761 string path = url.getPathName();
762 if ( !path.empty() && path != "/" && *path.rbegin() == '/' &&
763 filename.absolute() )
765 // If url has a path with trailing slash, remove the leading slash from
766 // the absolute file name
767 path += filename.asString().substr( 1, filename.asString().size() - 1 );
769 else if ( filename.relative() )
771 // Add trailing slash to path, if not already there
772 if (path.empty()) path = "/";
773 else if (*path.rbegin() != '/' ) path += "/";
774 // Remove "./" from begin of relative file name
775 path += filename.asString().substr( 2, filename.asString().size() - 2 );
779 path += filename.asString();
782 newurl.setPathName(path);
787 ///////////////////////////////////////////////////////////////////
789 // METHOD NAME : MediaCurl::getFile
790 // METHOD TYPE : void
792 void MediaCurl::getFile( const Pathname & filename ) const
794 // Use absolute file name to prevent access of files outside of the
795 // hierarchy below the attach point.
796 getFileCopy(filename, localPath(filename).absolutename());
799 ///////////////////////////////////////////////////////////////////
801 // METHOD NAME : MediaCurl::getFileCopy
802 // METHOD TYPE : void
804 void MediaCurl::getFileCopy( const Pathname & filename , const Pathname & target) const
806 callback::SendReport<DownloadProgressReport> report;
808 Url fileurl(getFileUrl(_url, filename));
816 doGetFileCopy(filename, target, report);
819 // retry with proper authentication data
820 catch (MediaUnauthorizedException & ex_r)
822 if(authenticate(ex_r.hint(), !retry))
826 report->finish(fileurl, zypp::media::DownloadProgressReport::ACCESS_DENIED, ex_r.asUserString());
830 // unexpected exception
831 catch (MediaException & excpt_r)
833 // FIXME: error number fix
834 report->finish(fileurl, zypp::media::DownloadProgressReport::ERROR, excpt_r.asUserString());
835 ZYPP_RETHROW(excpt_r);
840 report->finish(fileurl, zypp::media::DownloadProgressReport::NO_ERROR, "");
844 bool MediaCurl::getDoesFileExist( const Pathname & filename ) const
852 return doGetDoesFileExist( filename );
854 // authentication problem, retry with proper authentication data
855 catch (MediaUnauthorizedException & ex_r)
857 if(authenticate(ex_r.hint(), !retry))
862 // unexpected exception
863 catch (MediaException & excpt_r)
865 ZYPP_RETHROW(excpt_r);
873 bool MediaCurl::doGetDoesFileExist( const Pathname & filename ) const
875 DBG << filename.asString() << endl;
878 ZYPP_THROW(MediaBadUrlException(_url));
880 if(_url.getHost().empty())
881 ZYPP_THROW(MediaBadUrlEmptyHostException(_url));
883 string path = _url.getPathName();
884 if ( !path.empty() && path != "/" && *path.rbegin() == '/' &&
885 filename.absolute() ) {
886 // If url has a path with trailing slash, remove the leading slash from
887 // the absolute file name
888 path += filename.asString().substr( 1, filename.asString().size() - 1 );
889 } else if ( filename.relative() ) {
890 // Add trailing slash to path, if not already there
891 if ( !path.empty() && *path.rbegin() != '/' ) path += "/";
892 // Remove "./" from begin of relative file name
893 path += filename.asString().substr( 2, filename.asString().size() - 2 );
895 path += filename.asString();
899 url.setPathName( path );
901 DBG << "URL: " << url.asString() << endl;
902 // Use URL without options and without username and passwd
903 // (some proxies dislike them in the URL).
904 // Curl seems to need the just scheme, hostname and a path;
905 // the rest was already passed as curl options (in attachTo).
908 // Use asString + url::ViewOptions instead?
909 curlUrl.setUsername( "" );
910 curlUrl.setPassword( "" );
911 curlUrl.setPathParams( "" );
912 curlUrl.setQueryString( "" );
913 curlUrl.setFragment( "" );
916 // See also Bug #154197 and ftp url definition in RFC 1738:
917 // The url "ftp://user@host/foo/bar/file" contains a path,
918 // that is relative to the user's home.
919 // The url "ftp://user@host//foo/bar/file" (or also with
920 // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
921 // contains an absolute path.
923 string urlBuffer( curlUrl.asString());
924 CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
927 ZYPP_THROW(MediaCurlSetOptException(url, _curlError));
930 // instead of returning no data with NOBODY, we return
931 // little data, that works with broken servers, and
932 // works for ftp as well, because retrieving only headers
933 // ftp will return always OK code ?
934 if ( _url.getScheme() == "http" || _url.getScheme() == "https" )
935 ret = curl_easy_setopt( _curl, CURLOPT_NOBODY, 1 );
937 ret = curl_easy_setopt( _curl, CURLOPT_RANGE, "0-1" );
940 curl_easy_setopt( _curl, CURLOPT_NOBODY, NULL );
941 curl_easy_setopt( _curl, CURLOPT_RANGE, NULL );
942 /* yes, this is why we never got to get NOBODY working before,
943 because setting it changes this option too, and we also
945 See: http://curl.haxx.se/mail/archive-2005-07/0073.html
947 curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1 );
948 ZYPP_THROW(MediaCurlSetOptException(url, _curlError));
952 FILE *file = ::fopen( "/dev/null", "w" );
955 ERR << "fopen failed for /dev/null" << endl;
956 curl_easy_setopt( _curl, CURLOPT_NOBODY, NULL );
957 curl_easy_setopt( _curl, CURLOPT_RANGE, NULL );
958 /* yes, this is why we never got to get NOBODY working before,
959 because setting it changes this option too, and we also
961 See: http://curl.haxx.se/mail/archive-2005-07/0073.html
963 curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1 );
965 ZYPP_THROW(MediaCurlSetOptException(url, _curlError));
967 ZYPP_THROW(MediaWriteException("/dev/null"));
970 ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, file );
973 std::string err( _curlError);
974 curl_easy_setopt( _curl, CURLOPT_RANGE, NULL );
975 curl_easy_setopt( _curl, CURLOPT_NOBODY, NULL );
976 /* yes, this is why we never got to get NOBODY working before,
977 because setting it changes this option too, and we also
979 See: http://curl.haxx.se/mail/archive-2005-07/0073.html
981 curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1 );
983 ZYPP_THROW(MediaCurlSetOptException(url, _curlError));
985 ZYPP_THROW(MediaCurlSetOptException(url, err));
987 // Set callback and perform.
988 //ProgressData progressData(_xfer_timeout, url, &report);
989 //report->start(url, dest);
990 //if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, &progressData ) != 0 ) {
991 // WAR << "Can't set CURLOPT_PROGRESSDATA: " << _curlError << endl;;
994 CURLcode ok = curl_easy_perform( _curl );
995 MIL << "perform code: " << ok << " [ " << curl_easy_strerror(ok) << " ]" << endl;
997 // reset curl settings
998 if ( _url.getScheme() == "http" || _url.getScheme() == "https" )
1000 ret = curl_easy_setopt( _curl, CURLOPT_NOBODY, NULL );
1001 /* yes, this is why we never got to get NOBODY working before,
1002 because setting it changes this option too, and we also
1004 See: http://curl.haxx.se/mail/archive-2005-07/0073.html
1006 ret = curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1 );
1009 ret = curl_easy_setopt( _curl, CURLOPT_RANGE, NULL );
1013 ZYPP_THROW(MediaCurlSetOptException(url, _curlError));
1023 bool err_file_not_found = false;
1026 case CURLE_FTP_COULDNT_RETR_FILE:
1027 case CURLE_FTP_ACCESS_DENIED:
1028 err_file_not_found = true;
1030 case CURLE_LOGIN_DENIED:
1032 MediaUnauthorizedException(url, "Login failed.", _curlError, ""));
1034 case CURLE_HTTP_RETURNED_ERROR:
1036 long httpReturnCode = 0;
1037 CURLcode infoRet = curl_easy_getinfo( _curl,
1038 CURLINFO_RESPONSE_CODE,
1040 if ( infoRet == CURLE_OK )
1042 string msg = "HTTP response: " +
1043 str::numstring( httpReturnCode );
1044 if ( httpReturnCode == 401 ) // authorization required
1046 std::string auth_hint = getAuthHint();
1048 DBG << msg << " Login failed (URL: " << url.asString() << ")" << std::endl;
1049 DBG << "MediaUnauthorizedException auth hint: '" << auth_hint << "'" << std::endl;
1051 ZYPP_THROW(MediaUnauthorizedException(
1052 url, "Login failed.", _curlError, auth_hint
1056 if ( httpReturnCode == 403) // access denied
1058 ZYPP_THROW(MediaForbiddenException(url));
1061 if ( httpReturnCode == 404) // not found
1063 err_file_not_found = true;
1068 DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
1069 ZYPP_THROW(MediaCurlException(url, msg, _curlError));
1073 string msg = "Unable to retrieve HTTP response:";
1075 DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
1076 ZYPP_THROW(MediaCurlException(url, msg, _curlError));
1080 case CURLE_UNSUPPORTED_PROTOCOL:
1081 case CURLE_URL_MALFORMAT:
1082 case CURLE_URL_MALFORMAT_USER:
1083 case CURLE_BAD_PASSWORD_ENTERED:
1084 case CURLE_FTP_USER_PASSWORD_INCORRECT:
1085 err = "Login failed";
1087 case CURLE_COULDNT_RESOLVE_PROXY:
1088 case CURLE_COULDNT_RESOLVE_HOST:
1089 case CURLE_COULDNT_CONNECT:
1090 case CURLE_FTP_CANT_GET_HOST:
1091 err = "Connection failed";
1093 case CURLE_WRITE_ERROR:
1094 err = "Write error";
1096 case CURLE_ABORTED_BY_CALLBACK:
1097 case CURLE_OPERATION_TIMEOUTED:
1098 err = "Timeout reached";
1099 ZYPP_THROW(MediaTimeoutException(url));
1101 case CURLE_SSL_CACERT:
1102 ZYPP_THROW(MediaBadCAException(url,_curlError));
1103 case CURLE_SSL_PEER_CERTIFICATE:
1105 err = curl_easy_strerror(ok);
1107 err = "Unrecognized error";
1111 if( err_file_not_found)
1113 // file does not exists
1118 // there was an error
1119 ZYPP_THROW(MediaCurlException(url, string(), _curlError));
1122 catch (const MediaException & excpt_r)
1124 ZYPP_RETHROW(excpt_r);
1129 return ( ok == CURLE_OK );
1130 //if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, NULL ) != 0 ) {
1131 // WAR << "Can't unset CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1136 void MediaCurl::doGetFileCopy( const Pathname & filename , const Pathname & target, callback::SendReport<DownloadProgressReport> & report) const
1138 DBG << filename.asString() << endl;
1141 ZYPP_THROW(MediaBadUrlException(_url));
1143 if(_url.getHost().empty())
1144 ZYPP_THROW(MediaBadUrlEmptyHostException(_url));
1146 Url url(getFileUrl(_url, filename));
1148 Pathname dest = target.absolutename();
1149 if( assert_dir( dest.dirname() ) )
1151 DBG << "assert_dir " << dest.dirname() << " failed" << endl;
1152 ZYPP_THROW( MediaSystemException(url, "System error on " + dest.dirname().asString()) );
1155 DBG << "URL: " << url.asString() << endl;
1156 // Use URL without options and without username and passwd
1157 // (some proxies dislike them in the URL).
1158 // Curl seems to need the just scheme, hostname and a path;
1159 // the rest was already passed as curl options (in attachTo).
1162 // Use asString + url::ViewOptions instead?
1163 curlUrl.setUsername( "" );
1164 curlUrl.setPassword( "" );
1165 curlUrl.setPathParams( "" );
1166 curlUrl.setQueryString( "" );
1167 curlUrl.setFragment( "" );
1170 // See also Bug #154197 and ftp url definition in RFC 1738:
1171 // The url "ftp://user@host/foo/bar/file" contains a path,
1172 // that is relative to the user's home.
1173 // The url "ftp://user@host//foo/bar/file" (or also with
1174 // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
1175 // contains an absolute path.
1177 string urlBuffer( curlUrl.asString());
1178 CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
1179 urlBuffer.c_str() );
1181 ZYPP_THROW(MediaCurlSetOptException(url, _curlError));
1184 // set IFMODSINCE time condition (no download if not modified)
1185 curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
1186 curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, PathInfo(target).mtime());
1188 string destNew = target.asString() + ".new.zypp.XXXXXX";
1189 char *buf = ::strdup( destNew.c_str());
1192 ERR << "out of memory for temp file name" << endl;
1193 ZYPP_THROW(MediaSystemException(
1194 url, "out of memory for temp file name"
1198 int tmp_fd = ::mkstemp( buf );
1202 ERR << "mkstemp failed for file '" << destNew << "'" << endl;
1203 ZYPP_THROW(MediaWriteException(destNew));
1208 FILE *file = ::fdopen( tmp_fd, "w" );
1211 filesystem::unlink( destNew );
1212 ERR << "fopen failed for file '" << destNew << "'" << endl;
1213 ZYPP_THROW(MediaWriteException(destNew));
1216 DBG << "dest: " << dest << endl;
1217 DBG << "temp: " << destNew << endl;
1219 ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, file );
1222 filesystem::unlink( destNew );
1223 ZYPP_THROW(MediaCurlSetOptException(url, _curlError));
1226 // Set callback and perform.
1227 ProgressData progressData(_xfer_timeout, url, &report);
1228 report->start(url, dest);
1229 if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, &progressData ) != 0 ) {
1230 WAR << "Can't set CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1233 ret = curl_easy_perform( _curl );
1235 if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, NULL ) != 0 ) {
1236 WAR << "Can't unset CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1240 ERR << "curl error: " << ret << ": " << _curlError
1241 << ", temp file size " << PathInfo(destNew).size()
1242 << " byte." << endl;
1245 filesystem::unlink( destNew );
1249 bool err_file_not_found = false;
1251 case CURLE_UNSUPPORTED_PROTOCOL:
1252 case CURLE_URL_MALFORMAT:
1253 case CURLE_URL_MALFORMAT_USER:
1255 case CURLE_LOGIN_DENIED:
1257 MediaUnauthorizedException(url, "Login failed.", _curlError, ""));
1259 case CURLE_HTTP_RETURNED_ERROR:
1261 long httpReturnCode = 0;
1262 CURLcode infoRet = curl_easy_getinfo( _curl,
1263 CURLINFO_RESPONSE_CODE,
1265 if ( infoRet == CURLE_OK ) {
1266 string msg = "HTTP response: " +
1267 str::numstring( httpReturnCode );
1268 if ( httpReturnCode == 401 )
1270 std::string auth_hint = getAuthHint();
1272 DBG << msg << " Login failed (URL: " << url.asString() << ")" << std::endl;
1273 DBG << "MediaUnauthorizedException auth hint: '" << auth_hint << "'" << std::endl;
1275 ZYPP_THROW(MediaUnauthorizedException(
1276 url, "Login failed.", _curlError, auth_hint
1280 if ( httpReturnCode == 403)
1282 ZYPP_THROW(MediaForbiddenException(url));
1285 if ( httpReturnCode == 404)
1287 ZYPP_THROW(MediaFileNotFoundException(_url, filename));
1291 DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
1292 ZYPP_THROW(MediaCurlException(url, msg, _curlError));
1296 string msg = "Unable to retrieve HTTP response:";
1298 DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
1299 ZYPP_THROW(MediaCurlException(url, msg, _curlError));
1303 case CURLE_FTP_COULDNT_RETR_FILE:
1304 case CURLE_FTP_ACCESS_DENIED:
1305 err = "File not found";
1306 err_file_not_found = true;
1308 case CURLE_BAD_PASSWORD_ENTERED:
1309 case CURLE_FTP_USER_PASSWORD_INCORRECT:
1310 err = "Login failed";
1312 case CURLE_COULDNT_RESOLVE_PROXY:
1313 case CURLE_COULDNT_RESOLVE_HOST:
1314 case CURLE_COULDNT_CONNECT:
1315 case CURLE_FTP_CANT_GET_HOST:
1316 err = "Connection failed";
1318 case CURLE_WRITE_ERROR:
1319 err = "Write error";
1321 case CURLE_ABORTED_BY_CALLBACK:
1322 case CURLE_OPERATION_TIMEDOUT:
1323 if( progressData.reached)
1325 err = "Timeout reached";
1326 ZYPP_THROW(MediaTimeoutException(url));
1333 case CURLE_SSL_PEER_CERTIFICATE:
1335 err = "Unrecognized error";
1338 if( err_file_not_found)
1340 ZYPP_THROW(MediaFileNotFoundException(_url, filename));
1344 ZYPP_THROW(MediaCurlException(url, err, _curlError));
1347 catch (const MediaException & excpt_r)
1349 ZYPP_RETHROW(excpt_r);
1352 #if DETECT_DIR_INDEX
1354 if(curlUrl.getScheme() == "http" ||
1355 curlUrl.getScheme() == "https")
1358 // try to check the effective url and set the not_a_file flag
1359 // if the url path ends with a "/", what usually means, that
1360 // we've received a directory index (index.html content).
1362 // Note: This may be dangerous and break file retrieving in
1363 // case of some server redirections ... ?
1365 bool not_a_file = false;
1367 CURLcode ret = curl_easy_getinfo( _curl,
1368 CURLINFO_EFFECTIVE_URL,
1370 if ( ret == CURLE_OK && ptr != NULL)
1375 std::string path( eurl.getPathName());
1376 if( !path.empty() && path != "/" && *path.rbegin() == '/')
1378 DBG << "Effective url ("
1380 << ") seems to provide the index of a directory"
1392 filesystem::unlink( destNew );
1393 ZYPP_THROW(MediaNotAFileException(_url, filename));
1396 #endif // DETECT_DIR_INDEX
1398 long httpReturnCode = 0;
1399 CURLcode infoRet = curl_easy_getinfo(_curl,
1400 CURLINFO_RESPONSE_CODE,
1402 bool modified = true;
1403 if (infoRet == CURLE_OK)
1405 DBG << "HTTP response: " + str::numstring(httpReturnCode);
1406 if ( httpReturnCode == 304 ) // not modified
1408 DBG << " Not modified.";
1415 WAR << "Could not get the reponse code." << endl;
1418 if (modified || infoRet != CURLE_OK)
1421 if ( ::fchmod( ::fileno(file), filesystem::applyUmaskTo( 0644 ) ) )
1423 ERR << "Failed to chmod file " << destNew << endl;
1427 // move the temp file into dest
1428 if ( rename( destNew, dest ) != 0 ) {
1429 ERR << "Rename failed" << endl;
1430 ZYPP_THROW(MediaWriteException(dest));
1435 // close and remove the temp file
1437 filesystem::unlink( destNew );
1440 DBG << "done: " << PathInfo(dest) << endl;
1444 ///////////////////////////////////////////////////////////////////
1447 // METHOD NAME : MediaCurl::getDir
1448 // METHOD TYPE : PMError
1450 // DESCRIPTION : Asserted that media is attached
1452 void MediaCurl::getDir( const Pathname & dirname, bool recurse_r ) const
1454 filesystem::DirContent content;
1455 getDirInfo( content, dirname, /*dots*/false );
1457 for ( filesystem::DirContent::const_iterator it = content.begin(); it != content.end(); ++it ) {
1458 Pathname filename = dirname + it->name;
1461 switch ( it->type ) {
1462 case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
1463 case filesystem::FT_FILE:
1464 getFile( filename );
1466 case filesystem::FT_DIR: // newer directory.yast contain at least directory info
1468 getDir( filename, recurse_r );
1470 res = assert_dir( localPath( filename ) );
1472 WAR << "Ignore error (" << res << ") on creating local directory '" << localPath( filename ) << "'" << endl;
1477 // don't provide devices, sockets, etc.
1483 ///////////////////////////////////////////////////////////////////
1486 // METHOD NAME : MediaCurl::getDirInfo
1487 // METHOD TYPE : PMError
1489 // DESCRIPTION : Asserted that media is attached and retlist is empty.
1491 void MediaCurl::getDirInfo( std::list<std::string> & retlist,
1492 const Pathname & dirname, bool dots ) const
1494 getDirectoryYast( retlist, dirname, dots );
1497 ///////////////////////////////////////////////////////////////////
1500 // METHOD NAME : MediaCurl::getDirInfo
1501 // METHOD TYPE : PMError
1503 // DESCRIPTION : Asserted that media is attached and retlist is empty.
1505 void MediaCurl::getDirInfo( filesystem::DirContent & retlist,
1506 const Pathname & dirname, bool dots ) const
1508 getDirectoryYast( retlist, dirname, dots );
1511 ///////////////////////////////////////////////////////////////////
1514 // METHOD NAME : MediaCurl::progressCallback
1515 // METHOD TYPE : int
1517 // DESCRIPTION : Progress callback triggered from MediaCurl::getFile
1519 int MediaCurl::progressCallback( void *clientp,
1520 double dltotal, double dlnow,
1521 double ultotal, double ulnow)
1523 ProgressData *pdata = reinterpret_cast<ProgressData *>(clientp);
1526 time_t now = time(NULL);
1529 // reset time of last change in case initial time()
1530 // failed or the time was adjusted (goes backward)
1531 if( pdata->ltime <= 0 || pdata->ltime > now)
1536 // start time counting as soon as first data arrives
1537 // (skip the connection / redirection time at begin)
1539 if (dlnow > 0 || ulnow > 0)
1541 dif = (now - pdata->ltime);
1542 dif = dif > 0 ? dif : 0;
1547 // update the drate_avg and drate_period only after a second has passed
1548 // (this callback is called much more often than a second)
1549 // otherwise the values would be far from accurate when measuring
1550 // the time in seconds
1551 //! \todo more accurate download rate computationn, e.g. compute average value from last 5 seconds, or work with milliseconds instead of seconds
1553 if ( pdata->secs > 1 && (dif > 0 || dlnow == dltotal ))
1554 pdata->drate_avg = (dlnow / pdata->secs);
1558 pdata->drate_period = ((dlnow - pdata->dload_period) / dif);
1559 pdata->dload_period = dlnow;
1563 // send progress report first, abort transfer if requested
1566 if (!(*(pdata->report))->progress(int( dlnow * 100 / dltotal ),
1569 pdata->drate_period))
1571 return 1; // abort transfer
1575 // check if we there is a timeout set
1576 if( pdata->timeout > 0)
1580 bool progress = false;
1582 // update download data if changed, mark progress
1583 if( dlnow != pdata->dload)
1586 pdata->dload = dlnow;
1589 // update upload data if changed, mark progress
1590 if( ulnow != pdata->uload)
1593 pdata->uload = ulnow;
1597 if( !progress && (now >= (pdata->ltime + pdata->timeout)))
1599 pdata->reached = true;
1600 return 1; // aborts transfer
1609 string MediaCurl::getAuthHint() const
1611 long auth_info = CURLAUTH_NONE;
1614 curl_easy_getinfo(_curl, CURLINFO_HTTPAUTH_AVAIL, &auth_info);
1616 if(infoRet == CURLE_OK)
1618 return CurlAuthData::auth_type_long2str(auth_info);
1625 bool MediaCurl::authenticate(const string & availAuthTypes, bool firstTry) const
1627 //! \todo need a way to pass different CredManagerOptions here
1628 Target_Ptr target = zypp::getZYpp()->getTarget();
1629 CredentialManager cm(CredManagerOptions(target ? target->root() : ""));
1630 CurlAuthData_Ptr credentials;
1632 // get stored credentials
1633 AuthData_Ptr cmcred = cm.getCred(_url);
1635 if (cmcred && firstTry)
1637 credentials.reset(new CurlAuthData(*cmcred));
1638 DBG << "got stored credentials:" << endl << *credentials << endl;
1640 // if not found, ask user
1644 CurlAuthData_Ptr curlcred;
1645 curlcred.reset(new CurlAuthData());
1646 callback::SendReport<AuthenticationReport> auth_report;
1648 // preset the username if present in current url
1649 if (!_url.getUsername().empty() && firstTry)
1650 curlcred->setUsername(_url.getUsername());
1651 // if CM has found some credentials, preset the username from there
1653 curlcred->setUsername(cmcred->username());
1655 // indicate we have no good credentials from CM
1658 string prompt_msg = boost::str(boost::format(
1659 //!\todo add comma to the message for the next release
1660 _("Authentication required for '%s'")) % _url.asString());
1662 // set available authentication types from the exception
1663 // might be needed in prompt
1664 curlcred->setAuthType(availAuthTypes);
1667 if (auth_report->prompt(_url, prompt_msg, *curlcred))
1669 DBG << "callback answer: retry" << endl
1670 << "CurlAuthData: " << *curlcred << endl;
1672 if (curlcred->valid())
1674 credentials = curlcred;
1675 // if (credentials->username() != _url.getUsername())
1676 // _url.setUsername(credentials->username());
1678 * \todo find a way to save the url with changed username
1679 * back to repoinfo or dont store urls with username
1680 * (and either forbid more repos with the same url and different
1681 * user, or return a set of credentials from CM and try them one
1688 DBG << "callback answer: cancel" << endl;
1692 // set username and password
1695 _userpwd = credentials->getUserPwd();
1697 // set username and password
1698 CURLcode ret = curl_easy_setopt(_curl, CURLOPT_USERPWD, _userpwd.c_str());
1699 if ( ret != 0 ) ZYPP_THROW(MediaCurlSetOptException(_url, _curlError));
1701 // set available authentication types from the exception
1702 if (credentials->authType() == CURLAUTH_NONE)
1703 credentials->setAuthType(availAuthTypes);
1705 // set auth type (seems this must be set _after_ setting the userpwd
1706 if (credentials->authType() != CURLAUTH_NONE)
1708 ret = curl_easy_setopt(_curl, CURLOPT_HTTPAUTH, credentials->authType());
1709 if ( ret != 0 ) ZYPP_THROW(MediaCurlSetOptException(_url, _curlError));
1714 credentials->setUrl(_url);
1715 cm.addCred(*credentials);
1726 } // namespace media