1 #include <zypp/zyppng/media/network/private/request_p.h>
2 #include <zypp/zyppng/media/network/private/networkrequesterror_p.h>
3 #include <zypp/zyppng/media/network/networkrequestdispatcher.h>
4 #include <zypp/media/CurlHelper.h>
5 #include <zypp/media/CurlConfig.h>
6 #include <zypp/media/MediaUserAuth.h>
7 #include <zypp/ZConfig.h>
8 #include <zypp/base/Logger.h>
9 #include <zypp/base/String.h>
10 #include <zypp/Pathname.h>
18 std::vector<char> peek_data_fd( FILE *fd, off_t offset, size_t count )
25 std::vector<char> data( count + 1 , '\0' );
28 while ((l = pread( fileno( fd ), data.data(), count, offset ) ) == -1 && errno == EINTR)
36 NetworkRequestPrivate::NetworkRequestPrivate(Url &&url, zypp::Pathname &&targetFile, off_t &&start, off_t &&len, NetworkRequest::FileMode fMode )
37 : _url ( std::move(url) )
38 , _targetFile ( std::move( targetFile) )
39 , _start ( std::move(start) )
40 , _len ( std::move(len) )
41 , _fMode ( std::move(fMode) )
42 , _activityTimer ( Timer::create() )
43 , _headers( std::unique_ptr< curl_slist, decltype (&curl_slist_free_all) >( nullptr, &curl_slist_free_all ) )
45 _activityTimer->sigExpired().connect( sigc::mem_fun( this, &NetworkRequestPrivate::onActivityTimeout ));
48 NetworkRequestPrivate::~NetworkRequestPrivate()
51 //clean up for now, later we might reuse handles
52 curl_easy_cleanup( _easyHandle );
53 //reset in request but make sure the request was not enqueued again and got a new handle
54 _easyHandle = nullptr;
58 bool NetworkRequestPrivate::initialize( std::string &errBuf )
63 //will reset to defaults but keep live connections, session ID and DNS caches
64 curl_easy_reset( _easyHandle );
66 _easyHandle = curl_easy_init();
68 _errorBuf.fill( '\0' );
69 curl_easy_setopt( _easyHandle, CURLOPT_ERRORBUFFER, this->_errorBuf.data() );
73 setCurlOption( CURLOPT_PRIVATE, this );
74 setCurlOption( CURLOPT_XFERINFOFUNCTION, NetworkRequestPrivate::curlProgressCallback );
75 setCurlOption( CURLOPT_XFERINFODATA, this );
76 setCurlOption( CURLOPT_NOPROGRESS, 0L);
77 setCurlOption( CURLOPT_FAILONERROR, 1L);
78 setCurlOption( CURLOPT_NOSIGNAL, 1L);
80 std::string urlBuffer( _url.asString() );
81 setCurlOption( CURLOPT_URL, urlBuffer.c_str() );
83 setCurlOption( CURLOPT_WRITEFUNCTION, NetworkRequestPrivate::writeCallback );
84 setCurlOption( CURLOPT_WRITEDATA, this );
86 if ( _options & NetworkRequest::HeadRequest ) {
87 // instead of returning no data with NOBODY, we return
88 // little data, that works with broken servers, and
89 // works for ftp as well, because retrieving only headers
90 // ftp will return always OK code ?
91 // See http://curl.haxx.se/docs/knownbugs.html #58
92 if ( (_url.getScheme() == "http" || _url.getScheme() == "https") && _settings.headRequestsAllowed() )
93 setCurlOption( CURLOPT_NOBODY, 1L );
95 setCurlOption( CURLOPT_RANGE, "0-1" );
97 std::string rangeDesc;
99 _expectRangeStatus = true;
100 rangeDesc = zypp::str::form("%llu-", static_cast<unsigned long long>( _start ));
102 rangeDesc.append( zypp::str::form( "%llu", static_cast<unsigned long long>(_start + _len - 1) ) );
104 if ( setCurlOption( CURLOPT_RANGE, rangeDesc.c_str() ), CURLE_OK ) {
105 strncpy( _errorBuf.data(), "curl_easy_setopt range failed", CURL_ERROR_SIZE);
109 _expectRangeStatus = false;
114 //make a local copy of the settings, so headers are not added multiple times
115 TransferSettings locSet = _settings;
117 // add custom headers for download.opensuse.org (bsc#955801)
118 if ( _url.getHost() == "download.opensuse.org" )
120 locSet.addHeader( internal::anonymousIdHeader() );
121 locSet.addHeader( internal::distributionFlavorHeader() );
124 locSet.addHeader("Pragma:");
126 locSet.setTimeout( zypp::ZConfig::instance().download_transfer_timeout() );
127 locSet.setConnectTimeout( CONNECT_TIMEOUT );
129 locSet.setUserAgentString( internal::agentString() );
132 char *ptr = getenv("ZYPP_MEDIA_CURL_DEBUG");
133 _curlDebug = (ptr && *ptr) ? zypp::str::strtonum<long>( ptr) : 0L;
136 setCurlOption( CURLOPT_VERBOSE, 1L);
137 setCurlOption( CURLOPT_DEBUGFUNCTION, internal::log_curl);
138 setCurlOption( CURLOPT_DEBUGDATA, &_curlDebug);
143 switch ( internal::env::ZYPP_MEDIA_CURL_IPRESOLVE() )
145 case 4: setCurlOption( CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4 ); break;
146 case 6: setCurlOption( CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6 ); break;
150 setCurlOption( CURLOPT_HEADERFUNCTION, internal::log_redirects_curl );
151 setCurlOption( CURLOPT_HEADERDATA, &_lastRedirect );
156 setCurlOption( CURLOPT_CONNECTTIMEOUT, locSet.connectTimeout() );
157 // If a transfer timeout is set, also set CURLOPT_TIMEOUT to an upper limit
158 // just in case curl does not trigger its progress callback frequently
160 if ( locSet.timeout() )
162 setCurlOption( CURLOPT_TIMEOUT, 3600L );
165 if ( _url.getScheme() == "https" )
167 #if CURLVERSION_AT_LEAST(7,19,4)
168 // restrict following of redirections from https to https only
169 setCurlOption( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTPS );
172 #if CURLVERSION_AT_LEAST(7,60,0) // SLE15+
173 setCurlOption( CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS );
176 if( locSet.verifyPeerEnabled() ||
177 locSet.verifyHostEnabled() )
179 setCurlOption(CURLOPT_CAPATH, locSet.certificateAuthoritiesPath().c_str());
182 if( ! locSet.clientCertificatePath().empty() )
184 setCurlOption(CURLOPT_SSLCERT, locSet.clientCertificatePath().c_str());
186 if( ! locSet.clientKeyPath().empty() )
188 setCurlOption(CURLOPT_SSLKEY, locSet.clientKeyPath().c_str());
191 #ifdef CURLSSLOPT_ALLOW_BEAST
193 setCurlOption( CURLOPT_SSL_OPTIONS, CURLSSLOPT_ALLOW_BEAST );
195 setCurlOption(CURLOPT_SSL_VERIFYPEER, locSet.verifyPeerEnabled() ? 1L : 0L);
196 setCurlOption(CURLOPT_SSL_VERIFYHOST, locSet.verifyHostEnabled() ? 2L : 0L);
197 // bnc#903405 - POODLE: libzypp should only talk TLS
198 setCurlOption(CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
201 // follow any Location: header that the server sends as part of
202 // an HTTP header (#113275)
203 setCurlOption( CURLOPT_FOLLOWLOCATION, 1L);
204 // 3 redirects seem to be too few in some cases (bnc #465532)
205 setCurlOption( CURLOPT_MAXREDIRS, 6L );
208 setCurlOption(CURLOPT_USERAGENT, locSet.userAgentString().c_str() );
211 /*---------------------------------------------------------------*
212 CURLOPT_USERPWD: [user name]:[password]
213 Url::username/password -> CURLOPT_USERPWD
214 If not provided, anonymous FTP identification
215 *---------------------------------------------------------------*/
216 if ( locSet.userPassword().size() )
218 setCurlOption(CURLOPT_USERPWD, locSet.userPassword().c_str());
219 std::string use_auth = _settings.authType();
220 if (use_auth.empty())
221 use_auth = "digest,basic"; // our default
222 long auth = zypp::media::CurlAuthData::auth_type_str2long(use_auth);
223 if( auth != CURLAUTH_NONE)
225 DBG << "Enabling HTTP authentication methods: " << use_auth
226 << " (CURLOPT_HTTPAUTH=" << auth << ")" << std::endl;
227 setCurlOption(CURLOPT_HTTPAUTH, auth);
231 if ( locSet.proxyEnabled() && ! locSet.proxy().empty() )
233 DBG << "Proxy: '" << locSet.proxy() << "'" << std::endl;
234 setCurlOption(CURLOPT_PROXY, locSet.proxy().c_str());
235 setCurlOption(CURLOPT_PROXYAUTH, CURLAUTH_BASIC|CURLAUTH_DIGEST|CURLAUTH_NTLM );
237 /*---------------------------------------------------------------*
238 * CURLOPT_PROXYUSERPWD: [user name]:[password]
240 * Url::option(proxyuser and proxypassword) -> CURLOPT_PROXYUSERPWD
241 * If not provided, $HOME/.curlrc is evaluated
242 *---------------------------------------------------------------*/
244 std::string proxyuserpwd = locSet.proxyUserPassword();
246 if ( proxyuserpwd.empty() )
248 zypp::media::CurlConfig curlconf;
249 zypp::media::CurlConfig::parseConfig(curlconf); // parse ~/.curlrc
250 if ( curlconf.proxyuserpwd.empty() )
251 DBG << "Proxy: ~/.curlrc does not contain the proxy-user option" << std::endl;
254 proxyuserpwd = curlconf.proxyuserpwd;
255 DBG << "Proxy: using proxy-user from ~/.curlrc" << std::endl;
260 DBG << "Proxy: using provided proxy-user '" << _settings.proxyUsername() << "'" << std::endl;
263 if ( ! proxyuserpwd.empty() )
265 setCurlOption(CURLOPT_PROXYUSERPWD, internal::curlUnEscape( proxyuserpwd ).c_str());
268 #if CURLVERSION_AT_LEAST(7,19,4)
269 else if ( locSet.proxy() == EXPLICITLY_NO_PROXY )
271 // Explicitly disabled in URL (see fillSettingsFromUrl()).
272 // This should also prevent libcurl from looking into the environment.
273 DBG << "Proxy: explicitly NOPROXY" << std::endl;
274 setCurlOption(CURLOPT_NOPROXY, "*");
280 DBG << "Proxy: not explicitly set" << std::endl;
281 DBG << "Proxy: libcurl may look into the environment" << std::endl;
285 if ( locSet.minDownloadSpeed() != 0 )
287 setCurlOption(CURLOPT_LOW_SPEED_LIMIT, locSet.minDownloadSpeed());
288 // default to 10 seconds at low speed
289 setCurlOption(CURLOPT_LOW_SPEED_TIME, 60L);
292 #if CURLVERSION_AT_LEAST(7,15,5)
293 if ( locSet.maxDownloadSpeed() != 0 )
294 setCurlOption(CURLOPT_MAX_RECV_SPEED_LARGE, locSet.maxDownloadSpeed());
297 if ( zypp::str::strToBool( _url.getQueryParam( "cookies" ), true ) )
298 setCurlOption( CURLOPT_COOKIEFILE, _currentCookieFile.c_str() );
300 MIL << "No cookies requested" << std::endl;
301 setCurlOption(CURLOPT_COOKIEJAR, _currentCookieFile.c_str() );
303 #if CURLVERSION_AT_LEAST(7,18,0)
305 setCurlOption(CURLOPT_PROXY_TRANSFER_MODE, 1L );
308 // append settings custom headers to curl
309 for ( TransferSettings::Headers::const_iterator it = locSet.headersBegin();
310 it != locSet.headersEnd();
312 if ( !z_func()->addRequestHeader( it->c_str() ) )
313 ZYPP_THROW(zypp::media::MediaCurlInitException(_url));
317 setCurlOption( CURLOPT_HTTPHEADER, _headers.get() );
321 } catch ( const zypp::Exception &excp ) {
323 errBuf = excp.asString();
328 void NetworkRequestPrivate::aboutToStart()
330 if ( _activityTimer )
331 _activityTimer->start( static_cast<uint64_t>( _settings.timeout() ) * 1000 );
333 _state = NetworkRequest::Running;
334 _sigStarted.emit( *z_func() );
337 void NetworkRequestPrivate::setResult( NetworkRequestError &&err )
343 _result = std::move(err);
345 if ( _activityTimer )
346 _activityTimer->stop();
348 if ( _result.type() == NetworkRequestError::NoError ) {
349 //we have a successful download, lets see if the checksum is fine IF we have one
350 _state = NetworkRequest::Finished;
351 if ( _expectedChecksum.size() && _digest ) {
352 if ( _digest->digestVector() != _expectedChecksum ) {
353 _state = NetworkRequest::Error;
355 auto hexToStr = []( const std::vector<u_char> &hex ) {
357 for (std::vector<u_char>::size_type j = 0; j < hex.size(); j++)
358 res += zypp::str::form("%02hhx", hex[j]);
362 _result = NetworkRequestErrorPrivate::customError( NetworkRequestError::InvalidChecksum, (zypp::str::Format("Invalid checksum %1%, expected checksum %2%") % _digest->digest() % hexToStr( _expectedChecksum ) ) );
366 _state = NetworkRequest::Error;
368 _sigFinished.emit( *z_func(), _result );
371 void NetworkRequestPrivate::reset()
380 _easyHandle = nullptr;
381 _result = NetworkRequestError();
382 _state = NetworkRequest::Pending;
386 _headers.reset( nullptr );
389 void NetworkRequestPrivate::onActivityTimeout( Timer & )
391 std::map<std::string, boost::any> extraInfo;
392 extraInfo.insert( {"requestUrl", _url } );
393 extraInfo.insert( {"filepath", _targetFile } );
394 _dispatcher->cancel( *z_func(), NetworkRequestErrorPrivate::customError( NetworkRequestError::Timeout, "Download timed out", std::move(extraInfo) ) );
397 int NetworkRequestPrivate::curlProgressCallback( void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow )
401 NetworkRequestPrivate *that = reinterpret_cast<NetworkRequestPrivate *>( clientp );
404 if ( that->_activityTimer && that->_activityTimer->isRunning() )
405 that->_activityTimer->start();
407 //keep signals to a minimum
408 if ( that->_downloaded == dlnow )
411 that->_downloaded = dlnow;
412 that->_reportedSize = dltotal;
414 if ( that->_len > 0 && that->_len < dlnow ) {
415 that->_dispatcher->cancel( *that->z_func(), NetworkRequestErrorPrivate::customError( NetworkRequestError::ExceededMaxLen ) );
419 that->_sigProgress.emit( *that->z_func(), dltotal, dlnow, ultotal, ulnow );
423 size_t NetworkRequestPrivate::writeCallback( char *ptr, size_t size, size_t nmemb, void *userdata )
428 //it is valid to call this function with no data to write, just return OK
429 if ( size * nmemb == 0)
432 NetworkRequestPrivate *that = reinterpret_cast<NetworkRequestPrivate *>( userdata );
434 //in case of a HEAD request, we do not write anything
435 if ( that->_options & NetworkRequest::HeadRequest ) {
436 return ( size * nmemb );
439 //If we expect a file range we better double check that we got the status code for it
440 if ( that->_expectRangeStatus ) {
442 (void)curl_easy_getinfo( that->_easyHandle, CURLINFO_EFFECTIVE_URL, &effurl);
443 if (effurl && !strncasecmp(effurl, "http", 4))
446 (void)curl_easy_getinfo( that->_easyHandle, CURLINFO_RESPONSE_CODE, &statuscode);
447 if (statuscode != 206) {
448 strncpy( that->_errorBuf.data(), "Expected range status code 206, but got none.", CURL_ERROR_SIZE);
454 if ( !that->_outFile ) {
455 std::string openMode = "w+b";
456 if ( that->_fMode == NetworkRequest::WriteShared )
459 that->_outFile = fopen( that->_targetFile.asString().c_str() , openMode.c_str() );
461 //if the file does not exist create a new one
462 if ( !that->_outFile && that->_fMode == NetworkRequest::WriteShared ) {
463 that->_outFile = fopen( that->_targetFile.asString().c_str() , "w+b" );
466 if ( !that->_outFile ) {
467 strncpy( that->_errorBuf.data(), "Unable to open target file.", CURL_ERROR_SIZE);
471 if ( that->_start > 0 )
472 if ( fseek( that->_outFile, that->_start, SEEK_SET ) != 0 ) {
473 strncpy( that->_errorBuf.data(), "Unable to set output file pointer.", CURL_ERROR_SIZE);
478 size_t written = fwrite( ptr, size, nmemb, that->_outFile );
479 if ( that->_digest ) {
480 that->_digest->update( ptr, written );
486 NetworkRequest::NetworkRequest(zyppng::Url url, zypp::filesystem::Pathname targetFile, off_t start, off_t len, zyppng::NetworkRequest::FileMode fMode)
487 : Base ( *new NetworkRequestPrivate( std::move(url), std::move(targetFile), std::move(start), std::move(len), std::move(fMode) ) )
491 NetworkRequest::~NetworkRequest()
495 if ( d->_dispatcher )
496 d->_dispatcher->cancel( *this, "Request destroyed while still running" );
499 fclose( d->_outFile );
502 void NetworkRequest::setPriority(NetworkRequest::Priority prio)
504 d_func()->_priority = prio;
507 NetworkRequest::Priority NetworkRequest::priority() const
509 return d_func()->_priority;
512 void NetworkRequest::setOptions( Options opt )
514 d_func()->_options = opt;
517 NetworkRequest::Options NetworkRequest::options() const
519 return d_func()->_options;
522 void NetworkRequest::setRequestRange(off_t start, off_t len)
525 if ( d->_state == Running )
531 const std::string &NetworkRequest::lastRedirectInfo() const
533 return d_func()->_lastRedirect;
536 void *NetworkRequest::nativeHandle() const
538 return d_func()->_easyHandle;
541 std::vector<char> NetworkRequest::peekData( off_t offset, size_t count ) const
544 return peek_data_fd( d->_outFile, offset, count );
547 Url NetworkRequest::url() const
549 return d_func()->_url;
552 void NetworkRequest::setUrl(const Url &url)
555 if ( d->_state == NetworkRequest::Running )
561 const zypp::filesystem::Pathname &NetworkRequest::targetFilePath() const
563 return d_func()->_targetFile;
566 std::string NetworkRequest::contentType() const
569 if ( curl_easy_getinfo( d_func()->_easyHandle, CURLINFO_CONTENT_TYPE, &ptr ) == CURLE_OK && ptr )
570 return std::string(ptr);
571 return std::string();
574 off_t NetworkRequest::downloadOffset() const
576 return d_func()->_start;
579 off_t NetworkRequest::reportedByteCount() const
581 return d_func()->_reportedSize;
584 off_t NetworkRequest::expectedByteCount() const
586 return d_func()->_len;
589 off_t NetworkRequest::downloadedByteCount() const
592 if ( d->_downloaded == -1 )
594 return d->_downloaded;
597 void NetworkRequest::setDigest( std::shared_ptr<zypp::Digest> dig )
599 d_func()->_digest = dig;
602 void NetworkRequest::setExpectedChecksum(std::vector<unsigned char> checksum )
604 d_func()->_expectedChecksum = std::move(checksum);
607 TransferSettings &NetworkRequest::transferSettings()
609 return d_func()->_settings;
612 std::shared_ptr<zypp::Digest> NetworkRequest::digest( ) const
614 return d_func()->_digest;
617 NetworkRequest::State NetworkRequest::state() const
619 return d_func()->_state;
622 const NetworkRequestError &NetworkRequest::error() const
624 return d_func()->_result;
627 std::string NetworkRequest::extendedErrorString() const
631 return std::string();
633 return d->_result.nativeErrorString();
636 bool NetworkRequest::hasError() const
638 return error().isError();
641 bool NetworkRequest::addRequestHeader( const std::string &header )
645 curl_slist *res = curl_slist_append( d->_headers ? d->_headers.get() : nullptr, header.c_str() );
650 d->_headers = std::unique_ptr< curl_slist, decltype (&curl_slist_free_all) >( res, &curl_slist_free_all );
655 SignalProxy<void (NetworkRequest &req)> NetworkRequest::sigStarted()
657 return d_func()->_sigStarted;
660 SignalProxy<void (NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow)> NetworkRequest::sigProgress()
662 return d_func()->_sigProgress;
665 SignalProxy<void (zyppng::NetworkRequest &req, const zyppng::NetworkRequestError &err)> NetworkRequest::sigFinished()
667 return d_func()->_sigFinished;