#include "zypp/base/Logger.h"
#include "zypp/media/MediaMultiCurl.h"
#include "zypp/media/MetaLinkParser.h"
+#include "zypp/ManagedFile.h"
using namespace std;
using namespace zypp::base;
string line(p + 9, l - 9);
if (line[l - 10] == '\r')
line.erase(l - 10, 1);
- DBG << "#" << _workerno << ": redirecting to" << line << endl;
+ XXX << "#" << _workerno << ": redirecting to" << line << endl;
return size;
}
if (l <= 14 || l >= 128 || strncasecmp(p, "Content-Range:", 14) != 0)
}
if (_request->_filesize != (off_t)filesize)
{
- DBG << "#" << _workerno << ": filesize mismatch" << endl;
+ XXX << "#" << _workerno << ": filesize mismatch" << endl;
_state = WORKER_BROKEN;
strncpy(_curlError, "filesize mismatch", CURL_ERROR_SIZE);
}
_urlbuf = curlUrl.asString();
_curl = _request->_context->fromEasyPool(_url.getHost());
if (_curl)
- DBG << "reused worker from pool" << endl;
+ XXX << "reused worker from pool" << endl;
if (!_curl && !(_curl = curl_easy_init()))
{
_state = WORKER_BROKEN;
long auth = CurlAuthData::auth_type_str2long(use_auth);
if( auth != CURLAUTH_NONE)
{
- DBG << "#" << _workerno << ": Enabling HTTP authentication methods: " << use_auth
+ XXX << "#" << _workerno << ": Enabling HTTP authentication methods: " << use_auth
<< " (CURLOPT_HTTPAUTH=" << auth << ")" << std::endl;
curl_easy_setopt(_curl, CURLOPT_HTTPAUTH, auth);
}
return;
}
- DBG << "checking DNS lookup of " << host << endl;
+ XXX << "checking DNS lookup of " << host << endl;
int pipefds[2];
if (pipe(pipefds))
{
struct addrinfo *ai, aihints;
memset(&aihints, 0, sizeof(aihints));
aihints.ai_family = PF_UNSPEC;
- int tstsock = socket(PF_INET6, SOCK_DGRAM, 0);
+ int tstsock = socket(PF_INET6, SOCK_DGRAM | SOCK_CLOEXEC, 0);
if (tstsock == -1)
aihints.ai_family = PF_INET;
else
return;
}
int exitcode = WEXITSTATUS(status);
- DBG << "#" << _workerno << ": DNS lookup returned " << exitcode << endl;
+ XXX << "#" << _workerno << ": DNS lookup returned " << exitcode << endl;
if (exitcode != 0)
{
_state = WORKER_BROKEN;
bool
multifetchworker::checkChecksum()
{
- // DBG << "checkChecksum block " << _blkno << endl;
+ // XXX << "checkChecksum block " << _blkno << endl;
if (!_blksize || !_request->_blklist)
return true;
return _request->_blklist->verifyDigest(_blkno, _dig);
bool
multifetchworker::recheckChecksum()
{
- // DBG << "recheckChecksum block " << _blkno << endl;
+ // XXX << "recheckChecksum block " << _blkno << endl;
if (!_request->_fp || !_blksize || !_request->_blklist)
return true;
if (fseeko(_request->_fp, _blkstart, SEEK_SET))
{
if (!_request->_stealing)
{
- DBG << "start stealing!" << endl;
+ XXX << "start stealing!" << endl;
_request->_stealing = true;
}
multifetchworker *best = 0;
}
// lets see if we should sleep a bit
- DBG << "me #" << _workerno << ": " << _avgspeed << ", size " << best->_blksize << endl;
- DBG << "best #" << best->_workerno << ": " << best->_avgspeed << ", size " << (best->_blksize - best->_blkreceived) << endl;
+ XXX << "me #" << _workerno << ": " << _avgspeed << ", size " << best->_blksize << endl;
+ XXX << "best #" << best->_workerno << ": " << best->_avgspeed << ", size " << (best->_blksize - best->_blkreceived) << endl;
if (_avgspeed && best->_avgspeed && best->_blksize - best->_blkreceived > 0 &&
(best->_blksize - best->_blkreceived) * _avgspeed < best->_blksize * best->_avgspeed)
{
double sl = (best->_blksize - best->_blkreceived) / best->_avgspeed * 2;
if (sl > 1)
sl = 1;
- DBG << "#" << _workerno << ": going to sleep for " << sl * 1000 << " ms" << endl;
+ XXX << "#" << _workerno << ": going to sleep for " << sl * 1000 << " ms" << endl;
_sleepuntil = now + sl;
_state = WORKER_SLEEP;
_request->_sleepworkers++;
else
{
MediaBlock blk = blklist->getBlock(_request->_blkno);
- while (_request->_blkoff >= blk.off + blk.size)
+ while (_request->_blkoff >= (off_t)(blk.off + blk.size))
{
if (++_request->_blkno == blklist->numBlocks())
{
sprintf(rangebuf, "%llu-", (unsigned long long)_blkstart);
else
sprintf(rangebuf, "%llu-%llu", (unsigned long long)_blkstart, (unsigned long long)_blkstart + _blksize - 1);
- DBG << "#" << _workerno << ": BLK " << _blkno << ":" << rangebuf << " " << _url << endl;
+ XXX << "#" << _workerno << ": BLK " << _blkno << ":" << rangebuf << " " << _url << endl;
if (curl_easy_setopt(_curl, CURLOPT_RANGE, !_noendrange || _blkstart != 0 ? rangebuf : (char *)0) != CURLE_OK)
{
_request->_activeworkers--;
if (_finished)
{
- DBG << "finished!" << endl;
+ XXX << "finished!" << endl;
break;
}
- if (_activeworkers < _maxworkers && urliter != urllist.end() && _workers.size() < MAXURLS)
+ if ((int)_activeworkers < _maxworkers && urliter != urllist.end() && _workers.size() < MAXURLS)
{
// spawn another worker!
multifetchworker *worker = new multifetchworker(workerno++, *this, *urliter);
continue;
if (_minsleepuntil == worker->_sleepuntil)
_minsleepuntil = 0;
- DBG << "#" << worker->_workerno << ": sleep done, wake up" << endl;
+ XXX << "#" << worker->_workerno << ": sleep done, wake up" << endl;
_sleepworkers--;
// nextjob chnages the state
worker->nextjob();
else
worker->_avgspeed = worker->_blkreceived / (now - worker->_blkstarttime);
}
- DBG << "#" << worker->_workerno << ": BLK " << worker->_blkno << " done code " << cc << " speed " << worker->_avgspeed << endl;
+ XXX << "#" << worker->_workerno << ": BLK " << worker->_blkno << " done code " << cc << " speed " << worker->_avgspeed << endl;
curl_multi_remove_handle(_multi, easy);
if (cc == CURLE_HTTP_RETURNED_ERROR)
{
long statuscode = 0;
(void)curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &statuscode);
- DBG << "HTTP status " << statuscode << endl;
+ XXX << "HTTP status " << statuscode << endl;
if (statuscode == 416 && !_blklist) /* Range error */
{
if (_filesize == off_t(-1))
{
if (!worker->_noendrange)
{
- DBG << "#" << worker->_workerno << ": retrying with no end range" << endl;
+ XXX << "#" << worker->_workerno << ": retrying with no end range" << endl;
worker->_noendrange = true;
worker->run();
continue;
// with something broken. Thus we have to re-check the block.
if (!worker->recheckChecksum())
{
- DBG << "#" << worker->_workerno << ": recheck checksum error, refetch block" << endl;
+ XXX << "#" << worker->_workerno << ": recheck checksum error, refetch block" << endl;
// re-fetch! No need to worry about the bad workers,
// they will now be set to DISCARD. At the end of their block
// they will notice that they wrote bad data and go into BROKEN.
ratio = ratio * ratio;
if (ratio > .01)
{
- DBG << "#" << worker->_workerno << ": too slow ("<< ratio << ", " << worker->_avgspeed << ", #" << maxworkerno << ": " << maxavg << "), going to sleep for " << ratio * 1000 << " ms" << endl;
+ XXX << "#" << worker->_workerno << ": too slow ("<< ratio << ", " << worker->_avgspeed << ", #" << maxworkerno << ": " << maxavg << "), going to sleep for " << ratio * 1000 << " ms" << endl;
worker->_sleepuntil = now + ratio;
worker->_state = WORKER_SLEEP;
_sleepworkers++;
worker->evaluateCurlCode(Pathname(), cc, false);
}
}
+
+ if ( _filesize > 0 && _fetchedgoodsize > _filesize ) {
+ ZYPP_THROW(MediaFileSizeExceededException(_baseurl, _filesize));
+ }
}
// send report
if (_report)
{
int percent = _totalsize ? (100 * (_fetchedgoodsize + _fetchedsize)) / (_totalsize + _fetchedsize) : 0;
+
double avg = 0;
if (now > _starttime)
avg = _fetchedsize / (now - _starttime);
_customHeadersMetalink = curl_slist_append(_customHeadersMetalink, "Accept: */*, application/metalink+xml, application/metalink4+xml");
}
-static bool looks_like_metalink(const Pathname & file)
+static bool looks_like_metalink_fd(int fd)
{
char buf[256], *p;
- int fd, l;
- if ((fd = open(file.asString().c_str(), O_RDONLY)) == -1)
- return false;
- while ((l = read(fd, buf, sizeof(buf) - 1)) == -1 && errno == EINTR)
+ int l;
+ while ((l = pread(fd, buf, sizeof(buf) - 1, (off_t)0)) == -1 && errno == EINTR)
;
- close(fd);
if (l == -1)
return 0;
buf[l] = 0;
p++;
}
bool ret = !strncasecmp(p, "<metalink", 9) ? true : false;
+ return ret;
+}
+
+static bool looks_like_metalink(const Pathname & file)
+{
+ int fd;
+ if ((fd = open(file.asString().c_str(), O_RDONLY|O_CLOEXEC)) == -1)
+ return false;
+ bool ret = looks_like_metalink_fd(fd);
+ close(fd);
DBG << "looks_like_metalink(" << file << "): " << ret << endl;
return ret;
}
-void MediaMultiCurl::doGetFileCopy( const Pathname & filename , const Pathname & target, callback::SendReport<DownloadProgressReport> & report, RequestOptions options ) const
+// here we try to suppress all progress coming from a metalink download
+// bsc#1021291: Nevertheless send alive trigger (without stats), so UIs
+// are able to abort a hanging metalink download via callback response.
+int MediaMultiCurl::progressCallback( void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
+{
+ CURL *_curl = MediaCurl::progressCallback_getcurl(clientp);
+ if (!_curl)
+ return MediaCurl::aliveCallback(clientp, dltotal, dlnow, ultotal, ulnow);
+
+ // bsc#408814: Don't report any sizes before we don't have data on disk. Data reported
+ // due to redirection etc. are not interesting, but may disturb filesize checks.
+ FILE *fp = 0;
+ if ( curl_easy_getinfo( _curl, CURLINFO_PRIVATE, &fp ) != CURLE_OK || !fp )
+ return MediaCurl::aliveCallback( clientp, dltotal, dlnow, ultotal, ulnow );
+ if ( ftell( fp ) == 0 )
+ return MediaCurl::aliveCallback( clientp, dltotal, 0.0, ultotal, ulnow );
+
+ // (no longer needed due to the filesize check above?)
+ // work around curl bug that gives us old data
+ long httpReturnCode = 0;
+ if (curl_easy_getinfo(_curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) != CURLE_OK || httpReturnCode == 0)
+ return MediaCurl::aliveCallback(clientp, dltotal, dlnow, ultotal, ulnow);
+
+ char *ptr = NULL;
+ bool ismetalink = false;
+ if (curl_easy_getinfo(_curl, CURLINFO_CONTENT_TYPE, &ptr) == CURLE_OK && ptr)
+ {
+ string ct = string(ptr);
+ if (ct.find("application/metalink+xml") == 0 || ct.find("application/metalink4+xml") == 0)
+ ismetalink = true;
+ }
+ if (!ismetalink && dlnow < 256)
+ {
+ // can't tell yet, ...
+ return MediaCurl::aliveCallback(clientp, dltotal, dlnow, ultotal, ulnow);
+ }
+ if (!ismetalink)
+ {
+ fflush(fp);
+ ismetalink = looks_like_metalink_fd(fileno(fp));
+ DBG << "looks_like_metalink_fd: " << ismetalink << endl;
+ }
+ if (ismetalink)
+ {
+ // this is a metalink file change the expected filesize
+ MediaCurl::resetExpectedFileSize( clientp, ByteCount( 2, ByteCount::MB) );
+ // we're downloading the metalink file. Just trigger aliveCallbacks
+ curl_easy_setopt(_curl, CURLOPT_PROGRESSFUNCTION, &MediaCurl::aliveCallback);
+ return MediaCurl::aliveCallback(clientp, dltotal, dlnow, ultotal, ulnow);
+ }
+ curl_easy_setopt(_curl, CURLOPT_PROGRESSFUNCTION, &MediaCurl::progressCallback);
+ return MediaCurl::progressCallback(clientp, dltotal, dlnow, ultotal, ulnow);
+}
+
+void MediaMultiCurl::doGetFileCopy( const Pathname & filename , const Pathname & target, callback::SendReport<DownloadProgressReport> & report, const ByteCount &expectedFileSize_r, RequestOptions options ) const
{
Pathname dest = target.absolutename();
if( assert_dir( dest.dirname() ) )
{
DBG << "assert_dir " << dest.dirname() << " failed" << endl;
- Url url(getFileUrl(filename));
- ZYPP_THROW( MediaSystemException(url, "System error on " + dest.dirname().asString()) );
- }
- string destNew = target.asString() + ".new.zypp.XXXXXX";
- char *buf = ::strdup( destNew.c_str());
- if( !buf)
- {
- ERR << "out of memory for temp file name" << endl;
- Url url(getFileUrl(filename));
- ZYPP_THROW(MediaSystemException(url, "out of memory for temp file name"));
+ ZYPP_THROW( MediaSystemException(getFileUrl(filename), "System error on " + dest.dirname().asString()) );
}
- int tmp_fd = ::mkstemp( buf );
- if( tmp_fd == -1)
+ ManagedFile destNew { target.extend( ".new.zypp.XXXXXX" ) };
+ AutoFILE file;
{
- free( buf);
- ERR << "mkstemp failed for file '" << destNew << "'" << endl;
- ZYPP_THROW(MediaWriteException(destNew));
- }
- destNew = buf;
- free( buf);
-
- FILE *file = ::fdopen( tmp_fd, "w" );
- if ( !file ) {
- ::close( tmp_fd);
- filesystem::unlink( destNew );
- ERR << "fopen failed for file '" << destNew << "'" << endl;
- ZYPP_THROW(MediaWriteException(destNew));
+ AutoFREE<char> buf { ::strdup( (*destNew).c_str() ) };
+ if( ! buf )
+ {
+ ERR << "out of memory for temp file name" << endl;
+ ZYPP_THROW(MediaSystemException(getFileUrl(filename), "out of memory for temp file name"));
+ }
+
+ AutoFD tmp_fd { ::mkostemp( buf, O_CLOEXEC ) };
+ if( tmp_fd == -1 )
+ {
+ ERR << "mkstemp failed for file '" << destNew << "'" << endl;
+ ZYPP_THROW(MediaWriteException(destNew));
+ }
+ destNew = ManagedFile( (*buf), filesystem::unlink );
+
+ file = ::fdopen( tmp_fd, "we" );
+ if ( ! file )
+ {
+ ERR << "fopen failed for file '" << destNew << "'" << endl;
+ ZYPP_THROW(MediaWriteException(destNew));
+ }
+ tmp_fd.resetDispose(); // don't close it here! ::fdopen moved ownership to file
}
+
DBG << "dest: " << dest << endl;
DBG << "temp: " << destNew << endl;
}
// change header to include Accept: metalink
curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, _customHeadersMetalink);
+ // change to our own progress funcion
+ curl_easy_setopt(_curl, CURLOPT_PROGRESSFUNCTION, &progressCallback);
+ curl_easy_setopt(_curl, CURLOPT_PRIVATE, (*file) ); // important to pass the FILE* explicitly (passing through varargs)
try
{
- MediaCurl::doGetFileCopyFile(filename, dest, file, report, options);
+ MediaCurl::doGetFileCopyFile(filename, dest, file, report, expectedFileSize_r, options);
}
catch (Exception &ex)
{
- ::fclose(file);
- filesystem::unlink(destNew);
curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, _customHeaders);
+ curl_easy_setopt(_curl, CURLOPT_PRIVATE, (void *)0);
ZYPP_RETHROW(ex);
}
curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, _customHeaders);
+ curl_easy_setopt(_curl, CURLOPT_PRIVATE, (void *)0);
long httpReturnCode = 0;
CURLcode infoRet = curl_easy_getinfo(_curl, CURLINFO_RESPONSE_CODE, &httpReturnCode);
if (infoRet == CURLE_OK)
// some proxies do not store the content type, so also look at the file to find
// out if we received a metalink (bnc#649925)
fflush(file);
- if (looks_like_metalink(Pathname(destNew)))
+ if (looks_like_metalink(destNew))
ismetalink = true;
}
if (ismetalink)
{
bool userabort = false;
- fclose(file);
- file = NULL;
Pathname failedFile = ZConfig::instance().repoCachePath() / "MultiCurl.failed";
+ file = nullptr; // explicitly close destNew before the parser reads it.
try
{
MetaLinkParser mlp;
- mlp.parse(Pathname(destNew));
+ mlp.parse(destNew);
MediaBlockList bl = mlp.getBlockList();
vector<Url> urls = mlp.getUrls();
- DBG << bl << endl;
- file = fopen(destNew.c_str(), "w+");
+ XXX << bl << endl;
+ file = fopen((*destNew).c_str(), "w+e");
if (!file)
ZYPP_THROW(MediaWriteException(destNew));
if (PathInfo(target).isExist())
{
- DBG << "reusing blocks from file " << target << endl;
+ XXX << "reusing blocks from file " << target << endl;
bl.reuseBlocks(file, target.asString());
- DBG << bl << endl;
+ XXX << bl << endl;
}
if (bl.haveChecksum(1) && PathInfo(failedFile).isExist())
{
- DBG << "reusing blocks from file " << failedFile << endl;
+ XXX << "reusing blocks from file " << failedFile << endl;
bl.reuseBlocks(file, failedFile.asString());
- DBG << bl << endl;
+ XXX << bl << endl;
filesystem::unlink(failedFile);
}
Pathname df = deltafile();
if (!df.empty())
{
- DBG << "reusing blocks from file " << df << endl;
+ XXX << "reusing blocks from file " << df << endl;
bl.reuseBlocks(file, df.asString());
- DBG << bl << endl;
+ XXX << bl << endl;
}
try
{
- multifetch(filename, file, &urls, &report, &bl);
+ multifetch(filename, file, &urls, &report, &bl, expectedFileSize_r);
}
catch (MediaCurlException &ex)
{
ZYPP_RETHROW(ex);
}
}
+ catch (MediaFileSizeExceededException &ex) {
+ ZYPP_RETHROW(ex);
+ }
catch (Exception &ex)
{
// something went wrong. fall back to normal download
- if (file)
- fclose(file);
- file = NULL;
+ file = nullptr; // explicitly close destNew before moving it
if (PathInfo(destNew).size() >= 63336)
{
::unlink(failedFile.asString().c_str());
}
if (userabort)
{
- filesystem::unlink(destNew);
ZYPP_RETHROW(ex);
}
- file = fopen(destNew.c_str(), "w+");
+ file = fopen((*destNew).c_str(), "w+e");
if (!file)
ZYPP_THROW(MediaWriteException(destNew));
- MediaCurl::doGetFileCopyFile(filename, dest, file, report, options | OPTION_NO_REPORT_START);
+ MediaCurl::doGetFileCopyFile(filename, dest, file, report, expectedFileSize_r, options | OPTION_NO_REPORT_START);
}
}
{
ERR << "Failed to chmod file " << destNew << endl;
}
+
+ file.resetDispose(); // we're going to close it manually here
if (::fclose(file))
{
filesystem::unlink(destNew);
ERR << "Fclose failed for file '" << destNew << "'" << endl;
ZYPP_THROW(MediaWriteException(destNew));
}
+
if ( rename( destNew, dest ) != 0 )
{
ERR << "Rename failed" << endl;
ZYPP_THROW(MediaWriteException(dest));
}
+ destNew.resetDispose(); // no more need to unlink it
+
DBG << "done: " << PathInfo(dest) << endl;
}
+///////////////////////////////////////////////////////////////////
+namespace {
+ // bsc#933839: propagate proxy settings passed in the repo URL
+ inline Url propagateQueryParams( Url url_r, const Url & template_r )
+ {
+ for ( std::string param : { "proxy", "proxyport", "proxyuser", "proxypass"} )
+ {
+ const std::string & value( template_r.getQueryParam( param ) );
+ if ( ! value.empty() )
+ url_r.setQueryParam( param, value );
+ }
+ return url_r;
+ }
+}
+///////////////////////////////////////////////////////////////////
+
void MediaMultiCurl::multifetch(const Pathname & filename, FILE *fp, std::vector<Url> *urllist, callback::SendReport<DownloadProgressReport> *report, MediaBlockList *blklist, off_t filesize) const
{
Url baseurl(getFileUrl(filename));
if (!_multi)
ZYPP_THROW(MediaCurlInitException(baseurl));
}
+
multifetchrequest req(this, filename, baseurl, _multi, fp, report, blklist, filesize);
req._timeout = _settings.timeout();
req._connect_timeout = _settings.connectTimeout();
try
{
string scheme = urliter->getScheme();
- if (scheme == "http" || scheme == "https" || scheme == "ftp")
+ if (scheme == "http" || scheme == "https" || scheme == "ftp" || scheme == "tftp")
{
checkProtocol(*urliter);
- myurllist.push_back(*urliter);
+ myurllist.push_back(propagateQueryParams(*urliter, _url));
}
}
catch (...)
projects / platform / upstream / libzypp.git / blobdiff