SpacesInParentheses: true
SpacesInSquareBrackets: true
Standard: Cpp11
-TabWidth: 8
+TabWidth: 4
UseTab: Never
...
OPTION (EXPORT_NG_API "Export experimental libzypp API" OFF)
# This option will reroute all tool binaries to the libzypp build dir instead of taking those installed in the default directories.
OPTION (ENABLE_DEVEL_BUILD "Developer build, use zypp tools directly from build dir rather than the default locations" OFF)
+OPTION (INSTALL_NG_BINARIES "Installs the NG binaries, disabled for now since we are not actively using them." OFF)
# Distros using just zypper may want to enable this as default earlier
OPTION (ENABLE_PREVIEW_SINGLE_RPMTRANS_AS_DEFAULT_FOR_ZYPPER "[preview] Force zypper into using a single rpm transaction to install" OFF)
#--------------------------------------------------------------------------------
IF ( ENABLE_DEVEL_BUILD )
MESSAGE( WARNING "Zypp devel build enabled, do not do this in production" )
SET( ZYPP_RPM_BINARY "${CMAKE_BINARY_DIR}/tools/zypp-rpm/zypp-rpm")
+ SET( ZYPP_WORKER_PATH "${CMAKE_BINARY_DIR}/tools/workers" )
ELSE()
SET( ZYPP_RPM_BINARY "${ZYPP_LIBEXEC_INSTALL_DIR}/zypp-rpm")
+ SET( ZYPP_WORKER_PATH "${ZYPP_LIBEXEC_INSTALL_DIR}/workers" )
ENDIF( ENABLE_DEVEL_BUILD )
message ( "Using zypp-rpm from path: ${ZYPP_RPM_BINARY}" )
-add_definitions( -DZYPP_RPM_BINARY="${ZYPP_RPM_BINARY}")
+add_definitions( -DZYPP_RPM_BINARY="${ZYPP_RPM_BINARY}" )
+add_definitions( -DZYPP_WORKER_PATH="${ZYPP_WORKER_PATH}")
IF (${have_system} STREQUAL x)
MESSAGE (STATUS "Building for SUSE")
#
SET(LIBZYPP_MAJOR "17")
SET(LIBZYPP_COMPATMINOR "22")
-SET(LIBZYPP_MINOR "30")
-SET(LIBZYPP_PATCH "3")
+SET(LIBZYPP_MINOR "31")
+SET(LIBZYPP_PATCH "0")
#
-# LAST RELEASED: 17.30.3 (22)
+# LAST RELEASED: 17.31.0 (22)
# (The number in parenthesis is LIBZYPP_COMPATMINOR)
#=======
-------------------------------------------------------------------
+Tue Jul 19 16:03:01 CEST 2022 - ma@suse.de
+
+- Add PoolItem::statusReinit to reset the status it's initial
+ state in the ResPool (might help bsc#1199895)
+ This may either be 'KEEP_STATE bySOLVER' or 'LOCKED byUSER' if
+ the PoolItem matched a hard lock defined in /etc/zypp/locks.
+- Fix building with GCC 13 on i586 (fixes #407, fixes #396)
+- Be prepared to receive exceptions from curl_easy_cleanup
+ (bsc#1201092)
+- Don't auto-flag kernel-firmware as 'reboot-needed' (bsc#1200993)
+- Remove Medianetwork and dependend code.
+ This commit removes the MediaNetwork tech preview and all related
+ code. First reason for this is that MediaNetwork was just meant
+ as a way to test the new CURL based downloader and second: since
+ the Provide API is going to completely replace the current media
+ backend it would be extra work to ensure that changes on the
+ Downloader do not break MediaNetwork.
+- version 17.31.0 (22)
+
+-------------------------------------------------------------------
Tue Jul 5 10:29:19 CEST 2022 - ma@suse.de
- Fix building with GCC 12.x release (#396)
"Project-Id-Version: YaST (@memory@)\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2022-03-14 15:21+0100\n"
-"PO-Revision-Date: 2022-07-02 23:12+0000\n"
+"PO-Revision-Date: 2022-07-12 07:12+0000\n"
"Last-Translator: Kristijan Fremen Velkovski <me@krisfremen.com>\n"
"Language-Team: Macedonian <https://l10n.opensuse.org/projects/libzypp/master/"
"mk/>\n"
#. :HKG:344:
#: zypp/CountryCode.cc:252
msgid "Heard Island and McDonald Islands"
-msgstr ""
+msgstr "Остров Херд и Острови МекДоналд"
# HN
#. :HMD:334:
#. :KIR:296:
#: zypp/CountryCode.cc:275
msgid "Comoros"
-msgstr ""
+msgstr "Комори"
#. :COM:174:
#: zypp/CountryCode.cc:276
msgid "Saint Kitts and Nevis"
-msgstr ""
+msgstr "Свети Кристофер и Невис"
#. :KNA:659:
#: zypp/CountryCode.cc:277
msgid "North Korea"
-msgstr ""
+msgstr "Северна Кореја"
# ZA
#. :PRK:408:
#. :KOR:410:
#: zypp/CountryCode.cc:279
msgid "Kuwait"
-msgstr ""
+msgstr "Кувајт"
# FO
# fuzzy
#. :KWT:414:
#: zypp/CountryCode.cc:280
-#, fuzzy
msgid "Cayman Islands"
-msgstr "Ð\98Ñ\80Ñ\81ка"
+msgstr "Ð\9aаÑ\98манÑ\81ки Ð\9eÑ\81Ñ\82Ñ\80ови"
# KZ
# fuzzy
#. :KAZ:398:
#: zypp/CountryCode.cc:282
msgid "Lao People's Democratic Republic"
-msgstr ""
+msgstr "Лаоска Народна Демократска Република"
#. :LAO:418:
#: zypp/CountryCode.cc:283
msgid "Lebanon"
-msgstr ""
+msgstr "Либан"
#. :LBN:422:
#: zypp/CountryCode.cc:284
msgid "Saint Lucia"
-msgstr ""
+msgstr "Света Луција"
#. :LCA:662:
#: zypp/CountryCode.cc:285
msgid "Liechtenstein"
-msgstr ""
+msgstr "Лихтенштајн"
#. :LIE:438:
#: zypp/CountryCode.cc:286
msgid "Sri Lanka"
-msgstr ""
+msgstr "Шри Ланка"
# LR
# fuzzy
#. :MOZ:508:
#: zypp/CountryCode.cc:316
msgid "Namibia"
-msgstr ""
+msgstr "Намибија"
# NC
# fuzzy
# fuzzy
#. :NCL:540:
#: zypp/CountryCode.cc:318
-#, fuzzy
msgid "Niger"
-msgstr "СÑ\80биÑ\98а"
+msgstr "Ð\9dигеÑ\80"
# FO
# fuzzy
#. :NER:562:
#: zypp/CountryCode.cc:319
-#, fuzzy
msgid "Norfolk Island"
-msgstr "Ð\98Ñ\80Ñ\81ка"
+msgstr "Ð\9dоÑ\80Ñ\84олÑ\88ки Ð\9eÑ\81Ñ\82Ñ\80ов"
# NG
# fuzzy
#. :NOR:578:
#: zypp/CountryCode.cc:324
msgid "Nepal"
-msgstr ""
+msgstr "Непал"
#. :NPL:524:
#. language code: nau na
#: zypp/CountryCode.cc:325 zypp/LanguageCode.cc:781
msgid "Nauru"
-msgstr ""
+msgstr "Науру"
#. :NRU:520:
#: zypp/CountryCode.cc:326
return pref;
}
-WebServer::Request::Request( std::istream::__streambuf_type *rinbuf, std::ostream::__streambuf_type *routbuf, std::ostream::__streambuf_type *rerrbuf )
+WebServer::Request::Request(std::streambuf *rinbuf, std::streambuf *routbuf, std::streambuf *rerrbuf )
: rin ( rinbuf )
, rout ( routbuf )
, rerr ( rerrbuf )
//remove "/handler/" prefix
std::string key = req.params.at("SCRIPT_NAME").substr( handlerPrefix().length() );
- auto it = _handlers.find( key );
+ const auto &it = _handlers.find( key );
if ( it == _handlers.end() ) {
- req.rerr << "Status: 404 Not Found\r\n"
- << "Content-Type: text/html\r\n"
- << "\r\n"
- << "Request handler was not found";
+ if ( !_defaultHandler ) {
+ _defaultHandler = makeDefaultHandler();
+ }
+ _defaultHandler( req );
FCGX_Finish_r(&request);
continue;
}
zypp::Pathname _docroot;
zypp::shared_ptr<std::thread> _thrd;
std::map<std::string, WebServer::RequestHandler> _handlers;
+ WebServer::RequestHandler _defaultHandler;
unsigned int _port;
int _wakeupPipe[2];
_pimpl->_handlers[path] = std::move(handler);
}
+void WebServer::setDefaultHandler ( RequestHandler &&handler )
+{
+ std::lock_guard<std::mutex> lock( _pimpl->_mut );
+ _pimpl->_defaultHandler = std::move(handler);
+}
+
void WebServer::removeRequestHandler(const std::string &path)
{
std::lock_guard<std::mutex> lock( _pimpl->_mut );
};
};
+WebServer::RequestHandler WebServer::makeDefaultHandler( )
+{
+ return []( WebServer::Request & req ){
+ req.rerr << "Status: 404 Not Found\r\n"
+ << "Content-Type: text/html\r\n"
+ << "\r\n"
+ << "Request handler was not found";
+ };
+}
+
WebServer::RequestHandler WebServer::makeResponse( std::string status, std::string content ) {
return makeResponse( status , std::vector<std::string>(), content );
};
public:
struct Request {
- Request ( std::istream::__streambuf_type *rinbuf,
- std::ostream::__streambuf_type *routbuf,
- std::ostream::__streambuf_type *rerrbuf );
+ Request ( std::streambuf *rinbuf,
+ std::streambuf *routbuf,
+ std::streambuf *rerrbuf );
std::string path;
std::map< std::string, std::string > params;
std::istream rin;
*/
void addRequestHandler ( const std::string &path, RequestHandler &&handler );
+ /**
+ * Sets the default request handler callback as a fallback if no specific handler was registered for
+ * a given path. The path from the HTTP request can be queried as the "REQUEST_URI" param from the \ref Request structure passed to the callback.
+ *
+ * \note a request handler path is always prefixed with /handler so to call a handler named test, the
+ * URL is "http://localhost/handler/test"
+ */
+ void setDefaultHandler ( RequestHandler &&handler );
+
/*!
* Removes a registered request hander, can be called at any time
*/
static RequestHandler makeResponse(std::string resp);
/*!
+ * Creates a request handler that simply returns 404
+ */
+ static RequestHandler makeDefaultHandler( );
+
+ /*!
* Creates a request handler that sends a HTTP response with \a status and \a content
* in the respose
*/
// fill buffer with an arithmetic progression
for (int i = 1; i < 256; ++i) {
zyppng::ByteArray ba(i, char(i));
- char *ringPos = buf.reserve(i);
+ char *ringPos = buf.reserve( i );
BOOST_REQUIRE(ringPos);
memcpy(ringPos, ba.data(), i);
}
--- /dev/null
+openSUSE - openSUSE-15.0-x86_64-Build267.2-Media
+openSUSE-15.0-x86_64-Build267.2
+1
--- /dev/null
+Medium File 1
--- /dev/null
+openSUSE - Testmedium
+openSUSE-15.0-x86_64-Build267.2
+4
--- /dev/null
+Medium File 2
--- /dev/null
+openSUSE - Testmedium
+openSUSE-15.0-x86_64-Build267.2
+4
--- /dev/null
+Medium File 3
--- /dev/null
+openSUSE - Testmedium
+openSUSE-15.0-x86_64-Build267.2
+4
--- /dev/null
+Medium File 4
--- /dev/null
+openSUSE - Testmedium
+openSUSE-15.0-x86_64-Build267.2
+4
#include <boost/test/unit_test.hpp>
#include <zypp-core/zyppng/base/EventLoop>
+#include <zypp-core/zyppng/base/EventDispatcher>
#include <zypp-core/zyppng/base/Timer>
#include <zypp-core/zyppng/io/AsyncDataSource>
#include <thread>
#include <string_view>
#include <iostream>
#include <glib-unix.h>
+#include <fstream>
+
+#include <mutex>
+#include <condition_variable>
BOOST_AUTO_TEST_CASE ( pipe_read_close )
bool gotClosed = false;
zypp::ByteArray readData;
- BOOST_REQUIRE( dataSource->open( pipeFds[0] ) );
+ BOOST_REQUIRE( dataSource->openFds( { pipeFds[0] } ) );
BOOST_REQUIRE( dataSource->canRead() );
BOOST_REQUIRE( dataSource->readFdOpen() );
- dataSource->connectFunc( &zyppng::AsyncDataSource::sigReadyRead, [&](){
- std::cout <<"Got read"<<std::endl;
+ const auto &readAllData = [&](){
zypp::ByteArray d = dataSource->readAll();
readData.insert( readData.end(), d.begin(), d.end() );
- } );
+ };
- dataSource->connectFunc( &zyppng::AsyncDataSource::sigReadFdClosed, [&]( auto ){
+ dataSource->connectFunc( &zyppng::AsyncDataSource::sigReadyRead, readAllData );
+ dataSource->connectFunc( &zyppng::AsyncDataSource::sigReadFdClosed, [&]( auto, auto ){
gotClosed = true;
+ readAllData();
loop->quit();
});
BOOST_REQUIRE ( gotClosed );
}
+// basically the same as above, but we break the receiving socket by closing the read end
+// during a readAll call.
+BOOST_AUTO_TEST_CASE ( pipe_read_close2 )
+{
+ std::string_view text ("Hello again");
+ int pipeFds[2] { -1, -1 };
+ BOOST_REQUIRE( g_unix_open_pipe( pipeFds, FD_CLOEXEC, nullptr ) );
+
+ std::mutex lock;
+ std::condition_variable cv;
+ bool written = false;
+
+ auto loop = zyppng::EventLoop::create();
+ auto dataSource = zyppng::AsyncDataSource::create();
+
+ // make sure we are not stuck
+ auto timer = zyppng::Timer::create();
+ timer->start( 1000 );
+ timer->connectFunc( &zyppng::Timer::sigExpired, [&]( auto & ){
+ loop->quit();
+ });
+
+ bool gotClosed = false;
+ zypp::ByteArray readData;
+
+ BOOST_REQUIRE( dataSource->openFds( { pipeFds[0] } ) );
+ BOOST_REQUIRE( dataSource->canRead() );
+
+ const auto &readAllData = [&](){
+ zypp::ByteArray d = dataSource->readAll();
+ readData.insert( readData.end(), d.begin(), d.end() );
+ };
+
+ dataSource->connectFunc( &zyppng::AsyncDataSource::sigReadyRead, [&]() {
+ std::unique_lock l(lock);
+ if ( !written ){
+ cv.wait(l, [&](){ return written; });
+ }
+
+ // close the pipe here, breaking the receiving socket
+ ::close( pipeFds[1] );
+ readAllData();
+ } );
+ dataSource->connectFunc( &zyppng::AsyncDataSource::sigReadFdClosed, [&]( auto, auto ){
+ gotClosed = true;
+ readAllData();
+ loop->quit();
+
+ });
+
+ std::thread writer( [&lock, &written, &cv]( int writeFd, std::string_view text ){
+ {
+ std::unique_lock l(lock);
+ ::write( writeFd, text.data(), text.length() );
+ written = true;
+ }
+ cv.notify_all();
+ }, pipeFds[1], text );
+
+ loop->run();
+ writer.join();
+
+ ::close( pipeFds[0] );
+ BOOST_REQUIRE_EQUAL( std::string_view( readData.data(), readData.size() ), text );
+ BOOST_REQUIRE ( gotClosed );
+}
+
+
BOOST_AUTO_TEST_CASE ( pipe_write_close )
{
std::string_view text ("Hello from main");
loop->quit();
});
- BOOST_REQUIRE( dataSink->open( -1, pipeFds[1] ) );
+ BOOST_REQUIRE( dataSink->openFds( {}, pipeFds[1] ) );
BOOST_REQUIRE( dataSink->canWrite() );
- std::size_t bytesWritten = 0;
- dataSink->connectFunc( &zyppng::AsyncDataSource::sigBytesWritten, [&]( std::size_t bytes ){
+ int64_t bytesWritten = 0;
+ dataSink->connectFunc( &zyppng::AsyncDataSource::sigBytesWritten, [&]( int64_t bytes ){
bytesWritten += bytes;
- if ( bytesWritten == text.size() )
+ if ( std::size_t(bytesWritten) == text.size() )
loop->quit();
});
auto loop = zyppng::EventLoop::create();
auto dataSource = zyppng::AsyncDataSource::create();
- if ( !dataSource->open( readFd ) )
+ if ( !dataSource->openFds( { readFd } ) )
return;
// make sure we are not stuck
});
loop->run();
-
- std::cout << "Thread did read: " << readData.data() << std::endl;
-
::close( readFd );
}, pipeFds[0], text );
BOOST_REQUIRE( g_unix_open_pipe( pipeFds, FD_CLOEXEC, nullptr ) );
auto loop = zyppng::EventLoop::create();
- auto dataSink = zyppng::AsyncDataSource::create();
+ auto dataSink = zyppng::AsyncDataSource::create();
// make sure we are not stuck
auto timer = zyppng::Timer::create();
bool gotClosed = false;
- BOOST_REQUIRE( dataSink->open( -1, pipeFds[1] ) );
+ BOOST_REQUIRE( dataSink->openFds( {}, pipeFds[1] ) );
BOOST_REQUIRE( dataSink->canWrite() );
- std::size_t bytesWritten = 0;
- dataSink->connectFunc( &zyppng::AsyncDataSource::sigBytesWritten, [&]( std::size_t bytes ){
+ int64_t bytesWritten = 0;
+ dataSink->connectFunc( &zyppng::AsyncDataSource::sigBytesWritten, [&]( int64_t bytes ){
bytesWritten += bytes;
});
BOOST_REQUIRE ( gotClosed );
}
+BOOST_AUTO_TEST_CASE ( multichannel )
+{
+ std::string_view text[] = { "Hello to channel1", "Hello to channel2 with more bytes" };
+ int pipeFds[2] { -1, -1 };
+ BOOST_REQUIRE( g_unix_open_pipe( pipeFds, FD_CLOEXEC, nullptr ) );
+
+ int pipe2Fds[2] { -1, -1 };
+ BOOST_REQUIRE( g_unix_open_pipe( pipe2Fds, FD_CLOEXEC, nullptr ) );
+
+ auto loop = zyppng::EventLoop::create();
+ auto dataSink = zyppng::AsyncDataSource::create();
+ auto dataSink2 = zyppng::AsyncDataSource::create();
+
+ // make sure we are not stuck
+ auto timer = zyppng::Timer::create();
+ timer->start( 10000 );
+
+ bool timeout = false;
+ timer->connectFunc( &zyppng::Timer::sigExpired, [&]( auto & ){
+ timeout = true;
+ loop->quit();
+ });
+
+ BOOST_REQUIRE( dataSink->openFds( {}, pipeFds[1] ) );
+ BOOST_REQUIRE( dataSink->canWrite() );
+
+ BOOST_REQUIRE( dataSink2->openFds( {}, pipe2Fds[1] ) );
+ BOOST_REQUIRE( dataSink2->canWrite() );
+
+ int64_t bytesWritten = 0;
+ dataSink->connectFunc( &zyppng::AsyncDataSource::sigBytesWritten, [&]( int64_t bytes ){
+ bytesWritten += bytes;
+ });
+
+ int64_t bytesWritten2 = 0;
+ dataSink2->connectFunc( &zyppng::AsyncDataSource::sigBytesWritten, [&]( int64_t bytes ){
+ bytesWritten2 += bytes;
+ });
+
+ bool dataSink1ABW = false;
+ dataSink->connectFunc( &zyppng::AsyncDataSource::sigAllBytesWritten, [&]( ){
+ dataSink1ABW = true;
+ });
+
+ bool dataSink2ABW = false;
+ dataSink2->connectFunc( &zyppng::AsyncDataSource::sigAllBytesWritten, [&]( ){
+ dataSink2ABW = true;
+ });
+
+ auto dataReceiver = zyppng::AsyncDataSource::create();
+ BOOST_REQUIRE( dataReceiver->openFds( { pipeFds[0], pipe2Fds[0] }, -1 ) );
+ BOOST_REQUIRE( dataReceiver->canRead() );
+ BOOST_REQUIRE_EQUAL( dataReceiver->readChannelCount(), 2 );
+ BOOST_REQUIRE_EQUAL( dataReceiver->currentReadChannel(), 0 );
+
+ zypp::ByteArray dataRecv[2];
+
+ dataReceiver->connectFunc( &zyppng::AsyncDataSource::sigChannelReadyRead, [&]( auto channel ) {
+ dataReceiver->setReadChannel( channel );
+ BOOST_REQUIRE_EQUAL( dataReceiver->currentReadChannel(), channel );
+ zypp::ByteArray d = dataReceiver->readAll();
+ dataRecv[channel].insert( dataRecv[channel].end(), d.begin(), d.end() );
+ if ( dataRecv[0].size() > 0 && dataRecv[0].size() == text[0].size()
+ && dataRecv[1].size() > 0 && dataRecv[1].size() == text[1].size() )
+ loop->quit();
+ });
+
+ bool bytesPushed = false;
+ loop->eventDispatcher()->invokeOnIdle( [ & ](){
+ dataSink->write( text[0].data(), text[0].size() );
+ dataSink2->write( text[1].data(), text[1].size() );
+ bytesPushed = true;
+ return false;
+ });
+
+ loop->run();
+
+ BOOST_REQUIRE( !timeout );
+ BOOST_REQUIRE( dataSink1ABW );
+ BOOST_REQUIRE( dataSink2ABW );
+ BOOST_REQUIRE_EQUAL( bytesWritten, text[0].size() );
+ BOOST_REQUIRE_EQUAL( bytesWritten2, text[1].size() );
+ BOOST_REQUIRE_EQUAL( text[0], std::string_view( dataRecv[0].data(), dataRecv[0].size() ) );
+ BOOST_REQUIRE_EQUAL( text[1], std::string_view( dataRecv[1].data(), dataRecv[1].size() ) );
+}
+
+BOOST_AUTO_TEST_CASE ( readl )
+{
+ std::string_view text[] = { "Hello\n", "World\n", "123456" };
+ int pipeFds[2] { -1, -1 };
+ BOOST_REQUIRE( g_unix_open_pipe( pipeFds, FD_CLOEXEC, nullptr ) );
+
+ auto loop = zyppng::EventLoop::create();
+ auto dataSource = zyppng::AsyncDataSource::create();
+
+ // make sure we are not stuck
+ auto timer = zyppng::Timer::create();
+ timer->start( 3000 );
+
+ bool timeout = false;
+ timer->connectFunc( &zyppng::Timer::sigExpired, [&]( auto & ){
+ loop->quit();
+ timeout = true;
+ });
+
+ BOOST_REQUIRE( dataSource->openFds( { pipeFds[0] } ) );
+ BOOST_REQUIRE( dataSource->canRead() );
+
+ dataSource->setReadChannel( 0 );
+ dataSource->sigReadyRead ().connect([&](){
+ // now write more data without returning to the ev loop, and check if we get it all
+ ::write( pipeFds[1], text[1].data(), text[1].size() );
+ ::write( pipeFds[1], text[2].data(), text[2].size() );
+
+ auto ba = dataSource->readLine();
+ BOOST_REQUIRE_EQUAL( text[0], std::string_view( ba.data(), ba.size() ) );
+ ba = dataSource->readLine( text[1].size () + 1 ); // trigger code path with fixed size
+ BOOST_REQUIRE_EQUAL( text[1], std::string_view( ba.data(), ba.size() ) );
+ ba = dataSource->readLine();
+ BOOST_REQUIRE_EQUAL( text[2], std::string_view( ba.data(), ba.size() ) );
+ loop->quit();
+ });
+
+ //write the first line
+ ::write( pipeFds[1], text[0].data(), text[0].size() );
+
+ loop->run();
+
+ BOOST_REQUIRE( !timeout );
+
+}
BOOST_REQUIRE_EQUAL( exitCode, 123 );
BOOST_REQUIRE( !gotFailedToStart );
- BOOST_REQUIRE( proc->stdoutDevice()->bytesAvailable() );
+ proc->setReadChannel( zyppng::Process::StdOut );
+ BOOST_REQUIRE( proc->bytesAvailable( ) );
- auto data = proc->stdoutDevice()->readLine();
+ auto data = proc->readLine();
BOOST_REQUIRE_EQUAL( data.asStringView(), std::string_view("Hello\n") );
- data = proc->stdoutDevice()->readLine();
+ data = proc->readLine();
BOOST_REQUIRE_EQUAL( data.asStringView(), std::string_view("I'm the second line\n") );
BOOST_REQUIRE( !proc->isRunning() );
proc->addFd( pipeB->writeFd );
auto dataSource = zyppng::AsyncDataSource::create();
- BOOST_REQUIRE( dataSource->open( pipeB->readFd, pipeA->writeFd ) );
+ BOOST_REQUIRE( dataSource->openFds( { pipeB->readFd }, pipeA->writeFd ) );
BOOST_REQUIRE( dataSource->canRead() );
BOOST_REQUIRE( dataSource->canWrite( ) );
auto listeningSock = zyppng::Socket::create( AF_UNIX, SOCK_STREAM, 0 );
auto error = zyppng::Socket::NoError;
- size_t bytesWritten = 0;
+ int64_t bytesWritten = 0;
zyppng::ByteArray dataReceived; //< This is where the thread will write the data
auto addr = std::make_shared<zyppng::UnixSockAddr>( "socktest", true );
auto error = zyppng::Socket::NoError;
const int iterations = 1000000;
- size_t bytesShouldBeWritten = 0;
- size_t bytesWritten = 0;
- size_t bytesWrittenSignaled = 0;
- size_t bytesReceived = 0;
+ int64_t bytesShouldBeWritten = 0;
+ int64_t bytesWritten = 0;
+ int64_t bytesWrittenSignaled = 0;
+ int64_t bytesReceived = 0;
auto addr = std::make_shared<zyppng::UnixSockAddr>( "socktest", true );
BOOST_REQUIRE( listeningSock->bind( addr ) );
ADD_TESTS(
NetworkRequestDispatcher
EvDownloader
+ Provider
)
ENDIF()
auto ev = zyppng::EventLoop::create();
zyppng::Downloader::Ptr downloader = std::make_shared<zyppng::Downloader>();
- downloader->setCredManagerOptions( zypp::media::CredManagerOptions( zypp::ZConfig::instance().repoManagerRoot()) );
//make sure the data here is big enough to cross the threshold of 256 bytes so we get a progress signal emitted and not only the alive signal.
std::string dummyContent = "This is just some dummy content,\nto test downloading and signals.\n"
zypp::filesystem::TmpFile targetFile;
std::shared_ptr<zyppng::Downloader> downloader = std::make_shared<zyppng::Downloader>();
- downloader->setCredManagerOptions( zypp::media::CredManagerOptions( zypp::ZConfig::instance().repoManagerRoot()) );
downloader->requestDispatcher()->setMaximumConcurrentConnections( maxDLs );
//first metalink download, generate a fully valid one
auto ev = zyppng::EventLoop::create();
std::shared_ptr<zyppng::Downloader> downloader = std::make_shared<zyppng::Downloader>();
- downloader->setCredManagerOptions( zypp::media::CredManagerOptions( zypp::ZConfig::instance().repoManagerRoot()) );
WebServer web((zypp::Pathname(TESTS_SRC_DIR)/"/zyppng/data/downloader").c_str(), 10001, withSSL );
BOOST_REQUIRE( web.start() );
dl->sigAuthRequired().connect( [&]( zyppng::Download &, zyppng::NetworkAuthData &auth, const std::string &availAuth ){
gotAuthRequest++;
+ auth.setUrl( web.url() );
auth.setUsername("test");
auth.setPassword("test");
+ auth.setLastDatabaseUpdate( time( nullptr ) );
});
dl->sigStateChanged().connect([&]( zyppng::Download &, zyppng::Download::State state ){
BOOST_REQUIRE_EQUAL( gotAuthRequest, 1 );
BOOST_REQUIRE ( allStates == std::vector<zyppng::Download::State>({ zyppng::Download::InitialState, zyppng::Download::DlMetaLinkInfo, zyppng::Download::PrepareMulti, zyppng::Download::DlMetalink, zyppng::Download::Finished}) );
}
-
- {
- //the creds should be in the credential manager now, we should not need to specify them again in the slot
- bool gotAuthRequest = false;
- auto dl = downloader->downloadFile( zyppng::DownloadSpec(weburl, targetFile.path()) );
- dl->spec().setTransferSettings(set);
-
- dl->sigFinished( ).connect([ &ev ]( zyppng::Download & ){
- ev->quit();
- });
-
- dl->sigAuthRequired().connect( [&]( zyppng::Download &, zyppng::NetworkAuthData &auth, const std::string &availAuth ){
- gotAuthRequest = true;
- });
-
- dl->start();
- ev->run();
- BOOST_TEST_REQ_SUCCESS( dl );
- BOOST_REQUIRE( !gotAuthRequest );
- }
-}
-
-/**
- * Test for bsc#1174011 auth=basic ignored in some cases
- *
- * If the URL specifes ?auth=basic libzypp should proactively send credentials we have available in the cred store
- */
-BOOST_DATA_TEST_CASE( dltest_auth_basic, bdata::make( withSSL ), withSSL )
-{
- //don't write or read creds from real settings dir
- zypp::filesystem::TmpDir repoManagerRoot;
- zypp::ZConfig::instance().setRepoManagerRoot( repoManagerRoot.path() );
-
- auto ev = zyppng::EventLoop::create();
-
- std::shared_ptr<zyppng::Downloader> downloader = std::make_shared<zyppng::Downloader>();
- downloader->setCredManagerOptions( zypp::media::CredManagerOptions( zypp::ZConfig::instance().repoManagerRoot()) );
-
- WebServer web((zypp::Pathname(TESTS_SRC_DIR)/"/zyppng/data/downloader").c_str(), 10001, withSSL );
- BOOST_REQUIRE( web.start() );
-
- zypp::filesystem::TmpFile targetFile;
- zyppng::Url weburl (web.url());
- weburl.setPathName("/handler/test.txt");
- weburl.setQueryParam("auth", "basic");
- weburl.setUsername("test");
-
- // make sure the creds are already available
- zypp::media::CredentialManager cm( repoManagerRoot.path() );
- zypp::media::AuthData data ("test", "test");
- data.setUrl( weburl );
- cm.addCred( data );
- cm.save();
-
- zyppng::TransferSettings set = web.transferSettings();
-
- web.addRequestHandler( "test.txt", createAuthHandler() );
- web.addRequestHandler( "quit", [ &ev ]( WebServer::Request & ){ ev->quit();} );
-
- {
- // simply check by request count if the test was successful:
- // if the proactive code adding the credentials to the first request is not executed we will
- // have more than 1 request.
- int reqCount = 0;
- auto dispatcher = downloader->requestDispatcher();
- dispatcher->sigDownloadStarted().connect([&]( zyppng::NetworkRequestDispatcher &, zyppng::NetworkRequest & ){
- reqCount++;
- });
-
-
- auto dl = downloader->downloadFile( zyppng::DownloadSpec(weburl, targetFile.path())
- .setMetalinkEnabled( false )
- .setTransferSettings( set )
- );
-
- dl->sigFinished( ).connect([ &ev ]( zyppng::Download & ){
- ev->quit();
- });
-
- dl->start();
- ev->run();
- BOOST_TEST_REQ_SUCCESS( dl );
- BOOST_REQUIRE_EQUAL( reqCount, 1 );
- }
}
--- /dev/null
+#include <zypp-media/ng/Provide>
+#include <zypp-media/ng/ProvideSpec>
+#include <zypp-media/MediaException>
+#include <zypp-media/auth/AuthData>
+#include <zypp-media/auth/CredentialManager>
+#include <zypp-core/OnMediaLocation>
+#include <zypp-core/zyppng/base/EventLoop>
+#include <zypp-core/Pathname.h>
+#include <zypp-core/Url.h>
+#include <zypp-core/base/UserRequestException>
+#include <zypp/ZConfig.h>
+#include <zypp/TmpPath.h>
+#include <zypp-core/zyppng/pipelines/Wait>
+#include <zypp-core/zyppng/rpc/zerocopystreams.h>
+#include <zypp-core/zyppng/base/private/linuxhelpers_p.h>
+
+#include <zypp-proto/tvm.pb.h>
+#include <iostream>
+#include <fstream>
+
+#include "WebServer.h"
+#include "TestTools.h"
+
+#include <boost/test/unit_test.hpp>
+#include <boost/test/data/test_case.hpp>
+
+#ifndef TESTS_BUILD_DIR
+#error "TESTS_BUILD_DIR not defined"
+#endif
+
+#define ZYPP_REQUIRE_THROW( statement, exception ) \
+ try { BOOST_REQUIRE_THROW( statement, exception ); } catch( ... ) { BOOST_FAIL( #statement" throws unexpected exception"); }
+
+namespace bdata = boost::unit_test::data;
+
+BOOST_AUTO_TEST_CASE( http_prov )
+{
+ using namespace zyppng::operators;
+
+ auto ev = zyppng::EventLoop::create ();
+
+ const auto &workerPath = zypp::Pathname ( TESTS_BUILD_DIR ).dirname() / "tools" / "workers";
+ const auto &webRoot = zypp::Pathname ( TESTS_SRC_DIR ) / "zyppng" / "data" / "downloader";
+
+ zypp::filesystem::TmpDir provideRoot;
+
+ auto prov = zyppng::Provide::create ( provideRoot );
+ prov->setWorkerPath ( workerPath );
+ prov->start();
+
+ WebServer web( webRoot.c_str(), 10001, false );
+ BOOST_REQUIRE( web.start() );
+
+ auto fileUrl = web.url();
+ fileUrl.setPathName( "/media.1/media" );
+ auto op = prov->provide( fileUrl, zyppng::ProvideFileSpec() );
+
+ std::exception_ptr err;
+ std::optional<zyppng::ProvideRes> resOpt;
+ op->onReady([&]( zyppng::expected<zyppng::ProvideRes> &&res ){
+ ev->quit();
+ if ( !res )
+ err = res.error();
+ else
+ resOpt = *res;
+ });
+
+ BOOST_REQUIRE( !op->isReady() );
+
+ if ( !op->isReady() )
+ ev->run();
+
+ BOOST_REQUIRE( !err );
+ BOOST_REQUIRE( resOpt.has_value() );
+ zypp::PathInfo pi( resOpt->file() );
+ BOOST_REQUIRE( pi.isExist() );
+ BOOST_REQUIRE( pi.isFile() );
+
+ std::ifstream in( resOpt->file().asString(), std::ios::binary );
+ auto sum = zypp::CheckSum::md5( in );
+ BOOST_REQUIRE_EQUAL( sum, std::string("63b4a45ec881d90b83c2e6af7bcbfa78") );
+}
+
+BOOST_AUTO_TEST_CASE( http_attach )
+{
+ using namespace zyppng::operators;
+
+ auto ev = zyppng::EventLoop::create ();
+
+ const auto &workerPath = zypp::Pathname ( TESTS_BUILD_DIR ).dirname() / "tools" / "workers";
+ const auto &webRoot = zypp::Pathname (TESTS_SRC_DIR)/"/zyppng/data/downloader";
+ zypp::filesystem::TmpDir provideRoot;
+
+ auto prov = zyppng::Provide::create ( provideRoot );
+ prov->setWorkerPath ( workerPath );
+ prov->start();
+
+ WebServer web( webRoot.c_str(), 10001, false );
+ BOOST_REQUIRE( web.start() );
+
+ auto op = prov->attachMedia( web.url(), zyppng::ProvideMediaSpec( "OnlineMedia" )
+ .setMediaFile( webRoot / "media.1" / "media" )
+ .setMedianr(1) );
+
+ op->onReady([&]( zyppng::expected<zyppng::Provide::MediaHandle> &&res ){
+ ev->quit();
+ BOOST_REQUIRE( res.is_valid() );
+ BOOST_REQUIRE( !res->handle().empty() );
+ });
+
+ BOOST_REQUIRE( !op->isReady() );
+
+ if ( !op->isReady() )
+ ev->run();
+}
+
+BOOST_AUTO_TEST_CASE( http_attach_prov )
+{
+ using namespace zyppng::operators;
+
+ auto ev = zyppng::EventLoop::create ();
+
+ const auto &workerPath = zypp::Pathname ( TESTS_BUILD_DIR ).dirname() / "tools" / "workers";
+ const auto &webRoot = zypp::Pathname ( TESTS_SRC_DIR ) / "zyppng" / "data" / "downloader";
+
+ zypp::filesystem::TmpDir provideRoot;
+
+ auto prov = zyppng::Provide::create ( provideRoot );
+ prov->setWorkerPath ( workerPath );
+ prov->start();
+
+ WebServer web( webRoot.c_str(), 10001, false );
+ BOOST_REQUIRE( web.start() );
+
+ zyppng::Provide::MediaHandle media;
+
+ auto op = prov->attachMedia( web.url(), zyppng::ProvideMediaSpec( "OnlineMedia" )
+ .setMediaFile( webRoot / "media.1" / "media" )
+ .setMedianr(1) )
+ | mbind ( [&]( zyppng::Provide::MediaHandle &&res ){
+ media = std::move(res);
+ return prov->provide( media, "/test.txt", zyppng::ProvideFileSpec() );
+ });
+
+ std::optional<zyppng::ProvideRes> fileRes;
+ op->onReady([&]( zyppng::expected<zyppng::ProvideRes> &&res ){
+ if ( res )
+ fileRes = std::move(*res);
+ ev->quit();
+ });
+
+ BOOST_REQUIRE( !op->isReady() );
+
+ if ( !op->isReady() )
+ ev->run();
+
+ BOOST_REQUIRE( fileRes.has_value() );
+ zypp::PathInfo pi ( fileRes->file() );
+ BOOST_REQUIRE( pi.isExist() && pi.isFile() );
+
+ std::ifstream in( fileRes->file().asString(), std::ios::binary );
+ auto sum = zypp::CheckSum::md5( in );
+ BOOST_REQUIRE_EQUAL( sum, std::string("7e562d52c100b68e9d6a561fa8519575") );
+}
+
+BOOST_AUTO_TEST_CASE( http_attach_prov_404 )
+{
+ using namespace zyppng::operators;
+
+ auto ev = zyppng::EventLoop::create ();
+
+ const auto &workerPath = zypp::Pathname ( TESTS_BUILD_DIR ).dirname() / "tools" / "workers";
+ const auto &webRoot = zypp::Pathname ( TESTS_SRC_DIR ) / "zyppng" / "data" / "downloader";
+
+ zypp::filesystem::TmpDir provideRoot;
+
+ auto prov = zyppng::Provide::create ( provideRoot );
+ prov->setWorkerPath ( workerPath );
+ prov->start();
+
+ WebServer web( webRoot.c_str(), 10001, false );
+ BOOST_REQUIRE( web.start() );
+
+ zyppng::Provide::MediaHandle media;
+
+ auto op = prov->attachMedia( web.url(), zyppng::ProvideMediaSpec( "OnlineMedia" )
+ .setMediaFile( webRoot / "media.1" / "media" )
+ .setMedianr(1) )
+ | mbind ( [&]( zyppng::Provide::MediaHandle &&res ){
+ media = std::move(res);
+ return prov->provide( media, "/doesnotexist", zyppng::ProvideFileSpec() );
+ });
+
+ std::optional<zyppng::ProvideRes> fileRes;
+ op->onReady([&]( zyppng::expected<zyppng::ProvideRes> &&res ){
+ ev->quit();
+
+ BOOST_REQUIRE(!res);
+ if ( !res ) {
+ ZYPP_REQUIRE_THROW( std::rethrow_exception( res.error() ), zypp::media::MediaFileNotFoundException );
+ }
+
+ });
+
+ BOOST_REQUIRE( !op->isReady() );
+
+ if ( !op->isReady() )
+ ev->run();
+}
+
+
+// special case where we can detect that something is a directory because we already provided a file from the
+// directory that's required as a file.
+BOOST_AUTO_TEST_CASE( http_attach_prov_notafile )
+{
+ using namespace zyppng::operators;
+
+ auto ev = zyppng::EventLoop::create ();
+
+ const auto &workerPath = zypp::Pathname ( TESTS_BUILD_DIR ).dirname() / "tools" / "workers";
+ const auto &webRoot = zypp::Pathname ( TESTS_SRC_DIR ) / "zyppng" / "data" / "downloader";
+
+ zypp::filesystem::TmpDir provideRoot;
+
+ auto prov = zyppng::Provide::create ( provideRoot );
+ prov->setWorkerPath ( workerPath );
+ prov->start();
+
+ WebServer web( webRoot.c_str(), 10001, false );
+ BOOST_REQUIRE( web.start() );
+
+ zyppng::Provide::MediaHandle media;
+ std::exception_ptr err;
+
+ auto op = prov->attachMedia( web.url(), zyppng::ProvideMediaSpec( "OnlineMedia" )
+ .setMediaFile( webRoot / "media.1" / "media" )
+ .setMedianr(1) )
+ | mbind ( [&]( zyppng::Provide::MediaHandle &&res ){
+ media = std::move(res);
+ return prov->provide( media, "/media.1", zyppng::ProvideFileSpec() );
+ });
+
+ op->onReady([&]( zyppng::expected<zyppng::ProvideRes> &&res ){
+ ev->quit();
+ if ( !res )
+ err = res.error();
+ });
+
+ BOOST_REQUIRE( !op->isReady() );
+
+ if ( !op->isReady() )
+ ev->run();
+
+ BOOST_REQUIRE( err );
+ ZYPP_REQUIRE_THROW( std::rethrow_exception( err ), zypp::media::MediaNotAFileException );
+}
+
+//creates a request handler that requires a authentication to work
+WebServer::RequestHandler createAuthHandler ( const zypp::Pathname &webroot = "/" )
+{
+ return [ webroot ]( WebServer::Request &req ){
+ //Basic dGVzdDp0ZXN0
+ auto it = req.params.find( "HTTP_AUTHORIZATION" );
+ bool authorized = false;
+ if ( it != req.params.end() )
+ authorized = ( it->second == "Basic dGVzdDp0ZXN0" );
+
+ if ( !authorized ) {
+ req.rout << "Status: 401 Unauthorized\r\n"
+ "Content-Type: text/html; charset=utf-8\r\n"
+ "WWW-Authenticate: Basic realm=\"User Visible Realm\", charset=\"UTF-8\" \r\n"
+ "\r\n"
+ "Sorry you are not authorized.";
+ return;
+ }
+
+ //remove "/handler/" prefix
+ std::string key = req.params.at("SCRIPT_NAME").substr( std::string_view("/handler").length() );
+
+ req.rout << "Status: 307 Temporary Redirect\r\n"
+ "Location: "<< webroot / key <<"\r\n\r\n";
+ return;
+ };
+};
+
+BOOST_AUTO_TEST_CASE( http_prov_auth )
+{
+ using namespace zyppng::operators;
+
+ auto ev = zyppng::EventLoop::create ();
+
+ const auto &workerPath = zypp::Pathname ( TESTS_BUILD_DIR ).dirname() / "tools" / "workers";
+ const auto &webRoot = zypp::Pathname ( TESTS_SRC_DIR ) / "zyppng" / "data" / "downloader";
+
+ zypp::filesystem::TmpDir provideRoot;
+
+ auto prov = zyppng::Provide::create ( provideRoot );
+ prov->setWorkerPath ( workerPath );
+ prov->start();
+
+ //don't write or read creds from real settings dir
+ zypp::filesystem::TmpDir globCredPath;
+ zypp::filesystem::TmpDir userCredPath;
+ zypp::media::CredManagerOptions opts;
+ opts.globalCredFilePath = globCredPath.path() / "credentials.cat";
+ opts.userCredFilePath = userCredPath.path() / "credentials.cat";
+ prov->setCredManagerOptions( opts );
+
+ bool gotSigAuthRequired = false;
+ prov->sigAuthRequired().connect( [&]( const zypp::Url &reqUrl, const std::string &triedUsername, const std::map<std::string, std::string> &extraValues ) -> std::optional<zypp::media::AuthData> {
+ gotSigAuthRequired = true;
+ zypp::media::AuthData auth( reqUrl );
+ auth.setUsername( "test" );
+ auth.setPassword( "test" );
+ auth.setLastDatabaseUpdate( time( nullptr ) );
+ return auth;
+ });
+
+ WebServer web( webRoot.c_str(), 10001, false );
+ BOOST_REQUIRE( web.start() );
+
+ web.setDefaultHandler( createAuthHandler() );
+
+ auto fileUrl = web.url();
+ fileUrl.setPathName( "/handler/test.txt" );
+ auto op = prov->provide( fileUrl, zyppng::ProvideFileSpec() );
+
+ std::exception_ptr err;
+ std::optional<zyppng::ProvideRes> resOpt;
+ op->onReady([&]( zyppng::expected<zyppng::ProvideRes> &&res ){
+ ev->quit();
+ if ( !res )
+ err = res.error();
+ else
+ resOpt = *res;
+ });
+
+ BOOST_REQUIRE( !op->isReady() );
+
+ if ( !op->isReady() )
+ ev->run();
+
+ //BOOST_REQUIRE( gotSigAuthRequired );
+ BOOST_REQUIRE( !err );
+ BOOST_REQUIRE( resOpt.has_value() );
+ zypp::PathInfo pi( resOpt->file() );
+ BOOST_REQUIRE( pi.isExist() );
+ BOOST_REQUIRE( pi.isFile() );
+
+ std::ifstream in( resOpt->file().asString(), std::ios::binary );
+ auto sum = zypp::CheckSum::md5( in );
+ BOOST_REQUIRE_EQUAL( sum, std::string("7e562d52c100b68e9d6a561fa8519575") );
+}
+
+BOOST_AUTO_TEST_CASE( http_prov_auth_nouserresponse )
+{
+ using namespace zyppng::operators;
+
+ //don't write or read creds from real settings dir
+ zypp::filesystem::TmpDir repoManagerRoot;
+ zypp::ZConfig::instance().setRepoManagerRoot( repoManagerRoot.path() );
+
+ auto ev = zyppng::EventLoop::create ();
+
+ const auto &workerPath = zypp::Pathname ( TESTS_BUILD_DIR ).dirname() / "tools" / "workers";
+ const auto &webRoot = zypp::Pathname ( TESTS_SRC_DIR ) / "zyppng" / "data" / "downloader";
+
+ zypp::filesystem::TmpDir provideRoot;
+
+ auto prov = zyppng::Provide::create ( provideRoot );
+ prov->setWorkerPath ( workerPath );
+ prov->start();
+
+ //don't write or read creds from real settings dir
+ zypp::filesystem::TmpDir globCredPath;
+ zypp::filesystem::TmpDir userCredPath;
+ zypp::media::CredManagerOptions opts;
+ opts.globalCredFilePath = globCredPath.path() / "credentials.cat";
+ opts.userCredFilePath = userCredPath.path() / "credentials.cat";
+ prov->setCredManagerOptions( opts );
+
+ bool gotSigAuthRequired = false;
+ prov->sigAuthRequired().connect( [&]( const zypp::Url &, const std::string &, const std::map<std::string, std::string> & ) -> std::optional<zypp::media::AuthData> {
+ gotSigAuthRequired = true;
+ return {};
+ });
+
+ WebServer web( webRoot.c_str(), 10001, false );
+ BOOST_REQUIRE( web.start() );
+
+ web.setDefaultHandler( createAuthHandler() );
+
+ auto fileUrl = web.url();
+ fileUrl.setPathName( "/handler/test.txt" );
+ auto op = prov->provide( fileUrl, zyppng::ProvideFileSpec() );
+
+ std::exception_ptr err;
+ std::optional<zyppng::ProvideRes> resOpt;
+ op->onReady([&]( zyppng::expected<zyppng::ProvideRes> &&res ){
+ ev->quit();
+ if ( !res )
+ err = res.error();
+ else
+ resOpt = *res;
+ });
+
+ BOOST_REQUIRE( !op->isReady() );
+
+ if ( !op->isReady() )
+ ev->run();
+
+ BOOST_REQUIRE( gotSigAuthRequired );
+ BOOST_REQUIRE( err );
+ BOOST_REQUIRE( !resOpt.has_value() );
+ ZYPP_REQUIRE_THROW( std::rethrow_exception( err ), zypp::media::MediaUnauthorizedException );
+}
+
+BOOST_AUTO_TEST_CASE( http_prov_auth_wrongpw )
+{
+ using namespace zyppng::operators;
+
+ //don't write or read creds from real settings dir
+ zypp::filesystem::TmpDir repoManagerRoot;
+ zypp::ZConfig::instance().setRepoManagerRoot( repoManagerRoot.path() );
+
+ auto ev = zyppng::EventLoop::create ();
+
+ const auto &workerPath = zypp::Pathname ( TESTS_BUILD_DIR ).dirname() / "tools" / "workers";
+ const auto &webRoot = zypp::Pathname ( TESTS_SRC_DIR ) / "zyppng" / "data" / "downloader";
+
+ zypp::filesystem::TmpDir provideRoot;
+
+ auto prov = zyppng::Provide::create ( provideRoot );
+ prov->setWorkerPath ( workerPath );
+ prov->start();
+
+ //don't write or read creds from real settings dir
+ zypp::filesystem::TmpDir globCredPath;
+ zypp::filesystem::TmpDir userCredPath;
+ zypp::media::CredManagerOptions opts;
+ opts.globalCredFilePath = globCredPath.path() / "credentials.cat";
+ opts.userCredFilePath = userCredPath.path() / "credentials.cat";
+ prov->setCredManagerOptions( opts );
+
+ int cntAuthRequired = 0;
+ prov->sigAuthRequired().connect( [&]( const zypp::Url &reqUrl, const std::string &, const std::map<std::string, std::string> & ) -> std::optional<zypp::media::AuthData> {
+ cntAuthRequired++;
+ zypp::media::AuthData auth( reqUrl );
+ auth.setLastDatabaseUpdate( time( nullptr ) );
+ if ( cntAuthRequired >= 2 ) {
+ auth.setUsername( "test" );
+ auth.setPassword( "test" );
+ } else {
+ auth.setUsername( "test" );
+ auth.setPassword( "wrong" );
+ }
+
+ return auth;
+ });
+
+ WebServer web( webRoot.c_str(), 10001, false );
+ BOOST_REQUIRE( web.start() );
+
+ web.setDefaultHandler( createAuthHandler() );
+
+ auto fileUrl = web.url();
+ fileUrl.setPathName( "/handler/test.txt" );
+ auto op = prov->provide( fileUrl, zyppng::ProvideFileSpec() );
+
+ std::exception_ptr err;
+ std::optional<zyppng::ProvideRes> resOpt;
+ op->onReady([&]( zyppng::expected<zyppng::ProvideRes> &&res ){
+ ev->quit();
+ if ( !res )
+ err = res.error();
+ else
+ resOpt = *res;
+ });
+
+ BOOST_REQUIRE( !op->isReady() );
+
+ if ( !op->isReady() )
+ ev->run();
+
+ BOOST_REQUIRE_EQUAL( cntAuthRequired, 2 );
+ BOOST_REQUIRE( !err );
+ BOOST_REQUIRE( resOpt.has_value() );
+ zypp::PathInfo pi( resOpt->file() );
+ BOOST_REQUIRE( pi.isExist() );
+ BOOST_REQUIRE( pi.isFile() );
+
+ std::ifstream in( resOpt->file().asString(), std::ios::binary );
+ auto sum = zypp::CheckSum::md5( in );
+ BOOST_REQUIRE_EQUAL( sum, std::string("7e562d52c100b68e9d6a561fa8519575") );
+}
+
+BOOST_AUTO_TEST_CASE( http_attach_prov_auth )
+{
+ using namespace zyppng::operators;
+
+ auto ev = zyppng::EventLoop::create ();
+
+ const auto &workerPath = zypp::Pathname ( TESTS_BUILD_DIR ).dirname() / "tools" / "workers";
+ const auto &webRoot = zypp::Pathname ( TESTS_SRC_DIR ) / "zyppng" / "data" / "downloader";
+
+ zypp::filesystem::TmpDir provideRoot;
+
+ auto prov = zyppng::Provide::create ( provideRoot );
+ prov->setWorkerPath ( workerPath );
+
+ //don't write or read creds from real settings dir
+ zypp::filesystem::TmpDir globCredPath;
+ zypp::filesystem::TmpDir userCredPath;
+ zypp::media::CredManagerOptions opts;
+ opts.globalCredFilePath = globCredPath.path() / "credentials.cat";
+ opts.userCredFilePath = userCredPath.path() / "credentials.cat";
+ prov->setCredManagerOptions( opts );
+
+ int cntAuthRequired = 0;
+ prov->sigAuthRequired().connect( [&]( const zypp::Url &reqUrl, const std::string &, const std::map<std::string, std::string> & ) -> std::optional<zypp::media::AuthData> {
+ cntAuthRequired++;
+ zypp::media::AuthData auth( reqUrl );
+ auth.setLastDatabaseUpdate( time( nullptr ) );
+ auth.setUsername( "test" );
+ auth.setPassword( "test" );
+ return auth;
+ });
+
+ prov->start();
+
+ WebServer web( webRoot.c_str(), 10001, false );
+ web.setDefaultHandler( createAuthHandler() );
+ BOOST_REQUIRE( web.start() );
+
+ zyppng::Provide::MediaHandle media;
+
+ auto baseUrl = web.url();
+ baseUrl.setPathName("/handler");
+
+ auto op = prov->attachMedia( baseUrl, zyppng::ProvideMediaSpec( "OnlineMedia" )
+ .setMediaFile( webRoot / "media.1" / "media" )
+ .setMedianr(1) )
+ | mbind ( [&]( zyppng::Provide::MediaHandle &&res ){
+ media = std::move(res);
+ return prov->provide( media, "/test.txt", zyppng::ProvideFileSpec() );
+ });
+
+ std::optional<zyppng::ProvideRes> fileRes;
+ op->onReady([&]( zyppng::expected<zyppng::ProvideRes> &&res ){
+ if ( res )
+ fileRes = std::move(*res);
+ ev->quit();
+ });
+
+ BOOST_REQUIRE( !op->isReady() );
+
+ if ( !op->isReady() )
+ ev->run();
+
+ BOOST_REQUIRE_EQUAL( cntAuthRequired, 1 ); // after the first auth request all others should work
+ BOOST_REQUIRE( fileRes.has_value() );
+ zypp::PathInfo pi ( fileRes->file() );
+ BOOST_REQUIRE( pi.isExist() && pi.isFile() );
+
+ std::ifstream in( fileRes->file().asString(), std::ios::binary );
+ auto sum = zypp::CheckSum::md5( in );
+ BOOST_REQUIRE_EQUAL( sum, std::string("7e562d52c100b68e9d6a561fa8519575") );
+}
+
+static void writeTVMConfig ( const zypp::Pathname &file, const zypp::proto::test::TVMSettings &set )
+{
+ const auto &fname = file/"tvm.conf";
+ int fd = open( fname.asString().data(), O_WRONLY | O_CREAT | O_TRUNC, 0666 );
+ if ( fd < 0 ) {
+ ERR << "Failed to open/create file " << fname << " error: "<<zyppng::strerr_cxx(errno)<< " " << errno << std::endl;
+ return;
+ }
+ zyppng::FileOutputStream out( fd );
+ out.SetCloseOnDelete( true );
+ if ( !set.SerializeToZeroCopyStream( &out ) )
+ ERR << "Failed to serialize settings" << std::endl;
+ MIL << "TVM config serialized to: " << fname << std::endl;
+}
+
+static auto makeDVDProv ( zyppng::ProvideRef &prov, const zypp::filesystem::Pathname &devRoot, int mediaNr, const std::string &fName )
+{
+ using namespace zyppng::operators;
+
+ return prov->attachMedia( zypp::Url("tvm:/"), zyppng::ProvideMediaSpec("CD Test Set")
+ .setMediaFile( devRoot / (std::string("cd") + zypp::str::numstring(mediaNr)) / (std::string("media.") + zypp::str::numstring(mediaNr)) / "media" )
+ .setMedianr(mediaNr))
+ | [&prov, mediaNr, fName]( zyppng::expected<zyppng::Provide::MediaHandle> &&res ){
+ if ( res ) {
+ std::cout << "Attached " << mediaNr << " as: " << res->handle() << std::endl;
+ return prov->provide ( *res, fName , zyppng::ProvideFileSpec() ) | [ attachId = *res, &prov ]( auto &&res ) {
+ if ( !res ) {
+ try {
+ std::rethrow_exception(res.error());
+ } catch ( const zypp::Exception &e ) {
+ std::cout << "Provide failed with " << e <<std::endl;
+ }
+
+ } else
+ std::cout << "File provided to : " << res->file () <<std::endl;
+
+ std::cout << "DETACHING " << attachId.handle() << std::endl;
+ return res;
+ };
+ } else {
+ std::cout << "Failed to attach media" << std::endl;
+ return zyppng::makeReadyResult( zyppng::expected<zyppng::ProvideRes>::error(res.error()) );
+ }
+ } | []( zyppng::expected<zyppng::ProvideRes> &&res ) {
+ if ( !res ) {
+ try {
+ std::rethrow_exception(res.error());
+ } catch ( const zypp::Exception &e ) {
+ std::cout << "Provide failed with " << e <<std::endl;
+ return zyppng::expected<void>::error(res.error());
+ } catch ( ... ) {
+ std::cout << "Provide failed with exception " << std::endl;
+ return zyppng::expected<void>::error(res.error());
+ }
+ } else {
+ return zyppng::expected<void>::success();
+ }
+ };
+ }
+
+BOOST_AUTO_TEST_CASE( tvm_basic )
+{
+ using namespace zyppng::operators;
+
+ auto ev = zyppng::EventLoop::create ();
+
+ const auto &workerPath = zypp::Pathname ( TESTS_BUILD_DIR ).dirname() / "tools" / "workers";
+ const auto &devRoot = zypp::Pathname ( TESTS_SRC_DIR ) / "zyppng" / "data" / "provide";
+
+ zypp::filesystem::TmpDir provideRoot;
+
+ auto prov = zyppng::Provide::create ( provideRoot );
+ prov->setWorkerPath ( workerPath );
+
+ zypp::proto::test::TVMSettings devSet;
+ auto dev = devSet.add_devices();
+ dev->set_name("/fakedev/tvm/slot1");
+ dev->set_insertedpath( (devRoot/"cd1").asString() );
+ dev = devSet.add_devices();
+ dev->set_name("/fakedev/tvm/slot2");
+ dev->set_insertedpath( (devRoot/"cd2").asString() );
+ dev = devSet.add_devices();
+ dev->set_name("/fakedev/tvm/slot3");
+ dev->set_insertedpath( (devRoot/"cd3").asString() );
+ writeTVMConfig( provideRoot, devSet );
+
+ prov->start();
+
+ std::vector< zyppng::AsyncOpRef<zyppng::expected<void>>> ops;
+
+ ops.push_back( makeDVDProv( prov, devRoot, 1, "/file1") );
+ ops.push_back( makeDVDProv( prov, devRoot, 2, "/file2") );
+ ops.push_back( makeDVDProv( prov, devRoot, 3, "/file3") );
+
+ auto r = std::move(ops) | zyppng::waitFor<zyppng::expected<void>>();
+ r->sigReady().connect([&](){
+ ev->quit();
+ });
+
+ BOOST_REQUIRE( !r->isReady() );
+
+ if ( !r->isReady() )
+ ev->run();
+
+ for ( const auto &res : r->get() ) {
+ BOOST_REQUIRE(res);
+ }
+}
+
+BOOST_AUTO_TEST_CASE( tvm_medchange )
+{
+ using namespace zyppng::operators;
+
+ auto ev = zyppng::EventLoop::create ();
+
+ const auto &workerPath = zypp::Pathname ( TESTS_BUILD_DIR ).dirname() / "tools" / "workers";
+ const auto &devRoot = zypp::Pathname ( TESTS_SRC_DIR ) / "zyppng" / "data" / "provide";
+
+ zypp::filesystem::TmpDir provideRoot;
+
+ auto prov = zyppng::Provide::create ( provideRoot );
+ prov->setWorkerPath ( workerPath );
+
+ zypp::proto::test::TVMSettings devSet;
+ auto dev = devSet.add_devices();
+ dev->set_name("/fakedev/tvm/slot1");
+ dev->set_insertedpath( (devRoot/"cd1").asString() );
+ dev = devSet.add_devices();
+ dev->set_name("/fakedev/tvm/slot2");
+ dev = devSet.add_devices();
+ dev->set_name("/fakedev/tvm/slot3");
+ writeTVMConfig( provideRoot, devSet );
+
+ prov->start();
+
+ bool gotMediaChange = false;
+ std::vector<std::string> freeDevices;
+ int32_t mediaNrAsked = -1;
+ std::string labelAsked;
+ prov->sigMediaChangeRequested().connect([&]( const std::string &ref, const std::string &label, const int32_t mediaNr, const std::vector<std::string> &devices, const std::optional<std::string> &desc ){
+ gotMediaChange = true;
+ freeDevices = devices;
+ labelAsked = label;
+ mediaNrAsked = mediaNr;
+ devSet.mutable_devices(1)->set_insertedpath( (devRoot/"cd2").asString() );
+ writeTVMConfig( provideRoot, devSet );
+ return zyppng::Provide::RETRY;
+ });
+
+ std::vector< zyppng::AsyncOpRef<zyppng::expected<void>>> ops;
+
+ ops.push_back( makeDVDProv( prov, devRoot, 1, "/file1") );
+ ops.push_back( makeDVDProv( prov, devRoot, 2, "/file2") );
+
+ auto r = std::move(ops) | zyppng::waitFor<zyppng::expected<void>>();
+ r->sigReady().connect([&](){
+ ev->quit();
+ });
+
+ BOOST_REQUIRE( !r->isReady() );
+
+ if ( !r->isReady() )
+ ev->run();
+
+ BOOST_REQUIRE( gotMediaChange );
+ BOOST_REQUIRE_EQUAL( labelAsked, std::string("CD Test Set") );
+ BOOST_REQUIRE_EQUAL( mediaNrAsked, 2 );
+ BOOST_REQUIRE( std::find( freeDevices.begin(), freeDevices.end(), "/fakedev/tvm/slot2" ) != freeDevices.end() );
+ BOOST_REQUIRE( std::find( freeDevices.begin(), freeDevices.end(), "/fakedev/tvm/slot3" ) != freeDevices.end() );
+ for ( const auto &res : r->get() ) {
+ BOOST_REQUIRE(res);
+ }
+}
+
+zyppng::Provide::Action cancelOps[] = { zyppng::Provide::ABORT, zyppng::Provide::SKIP };
+
+BOOST_DATA_TEST_CASE( tvm_medchange_abort, bdata::make( cancelOps ), cancelOp )
+{
+ using namespace zyppng::operators;
+
+ auto ev = zyppng::EventLoop::create ();
+
+ const auto &workerPath = zypp::Pathname ( TESTS_BUILD_DIR ).dirname() / "tools" / "workers";
+ const auto &devRoot = zypp::Pathname ( TESTS_SRC_DIR ) / "zyppng" / "data" / "provide";
+
+ zypp::filesystem::TmpDir provideRoot;
+
+ auto prov = zyppng::Provide::create ( provideRoot );
+ prov->setWorkerPath ( workerPath );
+
+ zypp::proto::test::TVMSettings devSet;
+ auto dev = devSet.add_devices();
+ dev->set_name("/fakedev/tvm/slot1");
+ dev->set_insertedpath( (devRoot/"cd1").asString() );
+ dev = devSet.add_devices();
+ dev->set_name("/fakedev/tvm/slot2");
+ dev = devSet.add_devices();
+ dev->set_name("/fakedev/tvm/slot3");
+ writeTVMConfig( provideRoot, devSet );
+
+ prov->start();
+
+ bool gotMediaChange = false;
+ std::vector<std::string> freeDevices;
+ int32_t mediaNrAsked = -1;
+ std::string labelAsked;
+ prov->sigMediaChangeRequested().connect([&]( const std::string &ref, const std::string &label, const int32_t mediaNr, const std::vector<std::string> &devices, const std::optional<std::string> &desc ){
+ gotMediaChange = true;
+ freeDevices = devices;
+ labelAsked = label;
+ mediaNrAsked = mediaNr;
+ return cancelOp;
+ });
+
+ auto op1 = makeDVDProv( prov, devRoot, 1, "/file1");
+ auto op2 = makeDVDProv( prov, devRoot, 2, "/file2");
+
+ const auto &readyCB = [&](){
+ if ( op1->isReady() && op2->isReady() )
+ ev->quit();
+ };
+
+ op1->sigReady().connect( readyCB );
+ op2->sigReady().connect( readyCB );
+
+ BOOST_REQUIRE( !op1->isReady() );
+ BOOST_REQUIRE( !op2->isReady() );
+
+
+ if ( !op1->isReady() && !op2->isReady() )
+ ev->run();
+
+ BOOST_REQUIRE( gotMediaChange );
+ BOOST_REQUIRE_EQUAL( labelAsked, std::string("CD Test Set") );
+ BOOST_REQUIRE_EQUAL( mediaNrAsked, 2 );
+ BOOST_REQUIRE( std::find( freeDevices.begin(), freeDevices.end(), "/fakedev/tvm/slot2" ) != freeDevices.end() );
+ BOOST_REQUIRE( std::find( freeDevices.begin(), freeDevices.end(), "/fakedev/tvm/slot3" ) != freeDevices.end() );
+ BOOST_REQUIRE( op1->get().is_valid() );
+ BOOST_REQUIRE( !op2->get().is_valid() );
+ if ( cancelOp == zyppng::Provide::ABORT ) {
+ ZYPP_REQUIRE_THROW( std::rethrow_exception( op2->get().error() ), zypp::AbortRequestException );
+ } else {
+ ZYPP_REQUIRE_THROW( std::rethrow_exception( op2->get().error() ), zypp::SkipRequestException );
+ }
+}
+
+BOOST_AUTO_TEST_CASE( tvm_jammed )
+{
+ using namespace zyppng::operators;
+
+ auto ev = zyppng::EventLoop::create ();
+
+ const auto &workerPath = zypp::Pathname ( TESTS_BUILD_DIR ).dirname() / "tools" / "workers";
+ const auto &devRoot = zypp::Pathname ( TESTS_SRC_DIR ) / "zyppng" / "data" / "provide";
+
+ zypp::filesystem::TmpDir provideRoot;
+
+ auto prov = zyppng::Provide::create ( provideRoot );
+ prov->setWorkerPath ( workerPath );
+
+ bool gotMediaChange = false;
+ prov->sigMediaChangeRequested().connect([&]( const std::string &ref, const std::string &label, const int32_t mediaNr, const std::vector<std::string> &devices, const std::optional<std::string> &desc ){
+ gotMediaChange = true;
+ return zyppng::Provide::ABORT;
+ });
+
+ zypp::proto::test::TVMSettings devSet;
+ auto dev = devSet.add_devices();
+ dev->set_name("/fakedev/tvm/slot1");
+ dev->set_insertedpath( (devRoot/"cd1").asString() );
+ writeTVMConfig( provideRoot, devSet );
+
+ prov->start();
+
+ auto op1 = prov->attachMedia( zypp::Url("tvm:/"), zyppng::ProvideMediaSpec("CD Test Set")
+ .setMediaFile( devRoot / (std::string("cd") + zypp::str::numstring(1)) / (std::string("media.") + zypp::str::numstring(1)) / "media" )
+ .setMedianr(1));
+ auto op2 = prov->attachMedia( zypp::Url("tvm:/"), zyppng::ProvideMediaSpec("CD Test Set")
+ .setMediaFile( devRoot / (std::string("cd") + zypp::str::numstring(2)) / (std::string("media.") + zypp::str::numstring(2)) / "media" )
+ .setMedianr(2));
+
+
+
+ const auto &readyCB = [&](){
+ if ( op1->isReady() && op2->isReady() )
+ ev->quit();
+ };
+
+ op1->sigReady().connect( readyCB );
+ op2->sigReady().connect( readyCB );
+
+
+ BOOST_REQUIRE( !op1->isReady() );
+ BOOST_REQUIRE( !op2->isReady() );
+
+
+ if ( !op1->isReady() && !op2->isReady() )
+ ev->run();
+
+ //we have only one drive which is in use, so we can not be asked for media change
+ BOOST_REQUIRE( !gotMediaChange );
+ BOOST_REQUIRE( op1->get().is_valid() );
+ BOOST_REQUIRE( !op2->get().is_valid() );
+ ZYPP_REQUIRE_THROW( std::rethrow_exception( op2->get().error() ), zypp::media::MediaJammedException );
+}
+
+// test if we can release a medium and immediately use the resulting free space for new requests
+BOOST_AUTO_TEST_CASE( tvm_jammed_release )
+{
+ using namespace zyppng::operators;
+
+ auto ev = zyppng::EventLoop::create ();
+
+ const auto &workerPath = zypp::Pathname ( TESTS_BUILD_DIR ).dirname() / "tools" / "workers";
+ const auto &devRoot = zypp::Pathname ( TESTS_SRC_DIR ) / "zyppng" / "data" / "provide";
+
+ zypp::filesystem::TmpDir provideRoot;
+
+ auto prov = zyppng::Provide::create ( provideRoot );
+ prov->setWorkerPath ( workerPath );
+
+ zypp::proto::test::TVMSettings devSet;
+
+ bool mediaChangeAllowed = false;
+ bool gotInvalidMediaChange = false;
+ prov->sigMediaChangeRequested().connect([&]( const std::string &ref, const std::string &label, const int32_t mediaNr, const std::vector<std::string> &devices, const std::optional<std::string> &desc ){
+ if ( ! mediaChangeAllowed ) {
+ gotInvalidMediaChange = true;
+ return zyppng::Provide::ABORT;
+ }
+
+ devSet.mutable_devices(0)->set_insertedpath( (devRoot/"cd2").asString() );
+ writeTVMConfig( provideRoot, devSet );
+ return zyppng::Provide::RETRY;
+ });
+
+ auto dev = devSet.add_devices();
+ dev->set_name("/fakedev/tvm/slot1");
+ dev->set_insertedpath( (devRoot/"cd1").asString() );
+ writeTVMConfig( provideRoot, devSet );
+
+ prov->start();
+
+ auto op1 = prov->attachMedia( zypp::Url("tvm:/"), zyppng::ProvideMediaSpec("CD Test Set")
+ .setMediaFile( devRoot / (std::string("cd") + zypp::str::numstring(1)) / (std::string("media.") + zypp::str::numstring(1)) / "media" )
+ .setMedianr(1));
+
+ zyppng::AsyncOpRef<zyppng::expected<zyppng::Provide::MediaHandle>> op2;
+ zyppng::AsyncOpRef<zyppng::expected<zyppng::Provide::MediaHandle>> op3;
+
+ bool attach1Success = false;
+ bool attach3Success = false;
+ std::exception_ptr op2Error;
+ op1->sigReady().connect( [&]() {
+ if ( !op1->get() ) {
+ attach1Success = false;
+ ev->quit();
+ return;
+ }
+
+ attach1Success = true;
+ op2 = prov->attachMedia( zypp::Url("tvm:/"), zyppng::ProvideMediaSpec("CD Test Set")
+ .setMediaFile( devRoot / (std::string("cd") + zypp::str::numstring(2)) / (std::string("media.") + zypp::str::numstring(2)) / "media" )
+ .setMedianr(2));
+
+ op2->sigReady().connect([&](){
+ if ( !op2->get().is_valid() ) {
+ op2Error = op2->get().error();
+
+ op1.reset(); // kill the first media handle
+ mediaChangeAllowed = true;
+
+ op3 = prov->attachMedia( zypp::Url("tvm:/"), zyppng::ProvideMediaSpec("CD Test Set")
+ .setMediaFile( devRoot / (std::string("cd") + zypp::str::numstring(2)) / (std::string("media.") + zypp::str::numstring(2)) / "media" )
+ .setMedianr(2));
+ op3->sigReady().connect( [&]() {
+ ev->quit();
+ if ( !op3->get() )
+ attach3Success = false;
+ else
+ attach3Success = true;
+ });
+ return;
+ }
+ ev->quit();
+ });
+ });
+
+ BOOST_REQUIRE( !op1->isReady() );
+ BOOST_REQUIRE( !op2 || !op2->isReady() );
+ BOOST_REQUIRE( !op3 || !op3->isReady() );
+
+ if ( !op1->isReady() )
+ ev->run();
+
+ //we have only one drive which is in use, so we can not be asked for media change
+ BOOST_REQUIRE( !gotInvalidMediaChange );
+ BOOST_REQUIRE( attach1Success );
+ BOOST_REQUIRE( op2Error != nullptr );
+ ZYPP_REQUIRE_THROW( std::rethrow_exception( op2Error ), zypp::media::MediaJammedException );
+ BOOST_REQUIRE( attach3Success );
+ BOOST_REQUIRE( !op2->get().is_valid() );
+}
+
+
+#if 0
+BOOST_AUTO_TEST_CASE( dltest_basic )
+{
+ using namespace zyppng::operators;
+
+ auto ev = zyppng::EventLoop::create ();
+ auto prov = zyppng::Provide::create ( "/tmp/provTest" );
+ prov->setWorkerPath ( "/home/zbenjamin/build/build-libzypp-Desktop_GCC-Debug/tools/workers" );
+ prov->start();
+
+ std::string baseURL = "http://download.opensuse.org/distribution/leap/15.0/repo/oss";
+ std::vector<std::string> downloads {
+ "11822f1421ae50fb1a07f72220b79000", "/x86_64/0ad-0.0.22-lp150.2.10.x86_64.rpm",
+ "b0aaaca4c3763792a495de293c8431c5", "/x86_64/alsa-1.1.5-lp150.4.3.x86_64.rpm",
+ "a6eb92351c03bcf603a09a2e8eddcead", "/x86_64/amarok-2.9.0-lp150.2.1.x86_64.rpm",
+ "a2fd84f6d0530abbfe6d5a3da3940d70", "/x86_64/aspell-0.60.6.1-lp150.1.15.x86_64.rpm",
+ "29b5eab4a9620f158331106df4603866", "/x86_64/atk-devel-2.26.1-lp150.2.4.x86_64.rpm",
+ "a795874986018674c37af85a62c8f28a", "/x86_64/bing-1.0.5-lp150.2.1.x86_64.rpm",
+ "ce09cb1af156203c89312f9faffce219", "/x86_64/cdrtools-3.02~a09-lp150.2.11.x86_64.rpm",
+ "3f57113bd0dea2b5d748b7a343a7fb31", "/x86_64/cfengine-3.11.0-lp150.2.3.x86_64.rpm",
+ "582cae086f67382e71bf02ff0db554cd", "/x86_64/cgit-1.1-lp150.1.5.x86_64.rpm",
+ "f10cfc37a20c13d59266241ecc30b152", "/x86_64/ck-devel-0.6.0-lp150.1.3.x86_64.rpm",
+ "4a19708dc8d58129f8832d934c47888b", "/x86_64/cloud-init-18.2-lp150.1.1.x86_64.rpm",
+ "7b535587d9bfd8b88edf57b6df5c4d99", "/x86_64/collectd-plugin-pinba-5.7.2-lp150.1.4.x86_64.rpm",
+ "d86c1d65039645a1895f458f38d9d9e7", "/x86_64/compface-1.5.2-lp150.1.4.x86_64.rpm",
+ "33a0e878c92b5b298cd6aaec44c0aa46", "/x86_64/compositeproto-devel-0.4.2-lp150.1.6.x86_64.rpm",
+ "646c6cc180caf27f56bb9ec5b4d50f5b", "/x86_64/corosync-testagents-2.4.4-lp150.3.1.x86_64.rpm",
+ "10685e733abf77e7439e33471b23612c", "/x86_64/cpupower-bench-4.15-lp150.1.4.x86_64.rpm"
+ };
+
+ prov->sigMediaChangeRequested().connect([&]( const std::string &ref, const std::string &label, const int32_t mediaNr, const std::vector<std::string> &devices, const std::optional<std::string> &desc ){
+
+ std::cout << "Please insert medium: " << mediaNr << " of media set " << label << " into one of the free devices: " << std::endl;
+ if ( desc )
+ std::cout << "Media desc: " << *desc << std::endl;
+
+ std::vector<std::string> choices;
+ for ( uint i = 0 ; i < devices.size (); i++ ) {
+ std::cout << "(" << i << ") Free device: " << devices.at(i) << std::endl;
+ choices.push_back( zypp::str::numstring(i));
+ }
+
+ while ( true ) {
+ std::string choice;
+ std::cout << "Select the device you want to use, or a for abort" << std::endl;
+ std::getline( std::cin, choice );
+ if ( choice == "a" )
+ return zyppng::Provide::ABORT;
+
+ auto i = std::find( choices.begin(), choices.end(), choice );
+ if ( i == choices.end() ) {
+ std::cout << "Invalid answer, please select a device from the list" << std::endl;
+ continue;
+ }
+
+ auto sel = std::distance( choices.begin(), i );
+ prov->ejectDevice( ref, devices.at(sel) );
+ break;
+ }
+
+ std::string dummy;
+ std::cout << "Please insert the medium and press a key to continue" << std::endl;
+ std::getline( std::cin, dummy );
+
+ return zyppng::Provide::RETRY;
+ });
+
+ prov->sigAuthRequired().connect( [&]( const zypp::Url &reqUrl, const std::string &triedUsername, const std::map<std::string, std::string> &extraValues ) -> std::optional<zypp::media::AuthData> {
+ std::cout << reqUrl << " requires authentication, previously tried username was: " << triedUsername << std::endl;
+ std::cout << "Please enter the username: " << std::endl;
+ std::string username;
+ std::getline ( std::cin, username );
+ std::cout << "Please enter the password: " << std::endl;
+ std::string pass;
+ std::getline ( std::cin, pass );
+ zypp::media::AuthData d( username, pass );
+ return d;
+ });
+
+ std::vector< zyppng::AsyncOpRef<zyppng::expected<void>>> ops;
+#if 0
+ ops.push_back( prov->attachMedia ( baseURL, zyppng::ProvideMediaSpec( "Label1" ).setMediaFile( "/tmp/provTest/media" ).setMedianr(1) )
+ | [&]( zyppng::expected<std::string> &&res ){
+ if ( res ) {
+ std::cout << "Attached as: " << *res << std::endl;
+
+ std::vector<zyppng::AsyncOpRef<zyppng::expected<zyppng::ProvideRes>>> provs;
+ for ( std::size_t i = 0; i < downloads.size(); i+=2 ) {
+ const auto &pName = zypp::Pathname(downloads[i+1]);
+ std::cout << "Asking to provide file: " << pName << std::endl;
+ provs.push_back( prov->provide ( *res, pName, zyppng::ProvideFileSpec() ) | []( auto &&res ) {
+ if ( !res ) {
+ try {
+ std::rethrow_exception(res.error());
+ } catch ( const zypp::Exception &e ) {
+ std::cout << "Provide failed with " << e <<std::endl;
+ }
+
+ } else
+ std::cout << "File provided to : " << res->file () <<std::endl;
+ return res;
+ });
+ }
+ return std::move(provs) | zyppng::waitFor<zyppng::expected<zyppng::ProvideRes>>();
+ }
+
+ std::cout << "Failed to attach media" << std::endl;
+ return zyppng::makeReadyResult( std::vector<zyppng::expected<zyppng::ProvideRes>> { zyppng::expected<zyppng::ProvideRes>::error(res.error()) } );
+ } | []( std::vector<zyppng::expected<zyppng::ProvideRes>> &&results ){
+ for ( const auto &r : results ) {
+ if ( !r )
+ return zyppng::expected<void>::error( r.error() );
+ }
+ return zyppng::expected<void>::success();
+ });
+#endif
+
+ const auto &makeDVDProv = [&]( int mediaNr, const std::string &fName ){
+ return prov->attachMedia( zypp::Url("dvd:/"), zyppng::ProvideMediaSpec("CD Test Set")
+ .setMediaFile("/tmp/provTest/mediacd")
+ .setMedianr(mediaNr)
+ .addCustomHeaderValue ("device", "/dev/sr1")
+ .addCustomHeaderValue ("device", "/dev/sr2"))
+ | [&]( zyppng::expected<zyppng::Provide::MediaHandle> &&res ){
+ if ( res ) {
+ std::cout << "Attached as: " << res->handle() << std::endl;
+ return prov->provide ( *res, fName , zyppng::ProvideFileSpec() ) | [ attachId = *res, &prov ]( auto &&res ) {
+ if ( !res ) {
+ try {
+ std::rethrow_exception(res.error());
+ } catch ( const zypp::Exception &e ) {
+ std::cout << "Provide failed with " << e <<std::endl;
+ }
+
+ } else
+ std::cout << "File provided to : " << res->file () <<std::endl;
+
+ std::cout << "DETACHING " << attachId.handle() << std::endl;
+ return res;
+ };
+ } else {
+ std::cout << "Failed to attach media" << std::endl;
+ return zyppng::makeReadyResult( zyppng::expected<zyppng::ProvideRes>::error(res.error()) );
+ }
+ } | []( zyppng::expected<zyppng::ProvideRes> &&res ) {
+ if ( !res ) {
+ try {
+ std::rethrow_exception(res.error());
+ } catch ( const zypp::Exception &e ) {
+ std::cout << "Provide failed with " << e <<std::endl;
+ return zyppng::expected<void>::error(res.error());
+ } catch ( ... ) {
+ std::cout << "Provide failed with exception " << std::endl;
+ return zyppng::expected<void>::error(res.error());
+ }
+ } else {
+ return zyppng::expected<void>::success();
+ }
+ };
+ };
+
+ ops.push_back( makeDVDProv(1, "/file1") );
+ ops.push_back( makeDVDProv(1, "/file1") );
+ ops.push_back( makeDVDProv(1, "/file1") );
+
+ auto r = std::move(ops) | zyppng::waitFor<zyppng::expected<void>>();
+ r->onReady( [&]( std::vector<zyppng::expected<void>> &&results ){
+ for ( const auto &res : results ) {
+ if ( res ) {
+ std::cout << "Request finished successfull" << std::endl;
+ } else {
+ try {
+ std::rethrow_exception (res.error());
+ } catch ( const zypp::Exception &e ) {
+ std::cout << "Request failed with " << e <<std::endl;
+ } catch ( ... ) {
+ std::cout << "Request failed with exception " << std::endl;
+ }
+ }
+ }
+ std::cout << "All done "<< results.size() << " YAY" << std::endl;
+ ev->quit();
+ });
+
+ ev->run ();
+ return;
+}
+#endif
INSTALL(TARGETS zypp-CheckAccessDeleted DESTINATION "${CMAKE_INSTALL_PREFIX}/bin")
INSTALL(TARGETS zypp-NameReqPrv DESTINATION "${CMAKE_INSTALL_PREFIX}/bin")
+
+SET( ZYPP_TOOLS_DIR ${CMAKE_CURRENT_SOURCE_DIR} )
+
ADD_SUBDIRECTORY( zypp-rpm )
+ADD_SUBDIRECTORY( zypp-media-http )
+ADD_SUBDIRECTORY( zypp-media-disc )
+ADD_SUBDIRECTORY( zypp-media-nfs )
+ADD_SUBDIRECTORY( zypp-media-smb )
+ADD_SUBDIRECTORY( zypp-media-dir )
+ADD_SUBDIRECTORY( zypp-media-disk )
+ADD_SUBDIRECTORY( zypp-media-iso )
+ADD_SUBDIRECTORY( zypp-media-copy )
+ADD_SUBDIRECTORY( zypp-media-tvm )
+ADD_SUBDIRECTORY( zypp-media-chksum )
+
+FIND_PACKAGE( Notcurses++ )
+if ( Notcurses++_FOUND)
+ message("Found Notcurses++, enabling repomirror build")
+ ADD_SUBDIRECTORY( repomirror )
+endif()
--- /dev/null
+PROJECT( repomirror C CXX )
+
+set (CMAKE_CXX_STANDARD 17)
+SET (CMAKE_CXX_EXTENSIONS OFF)
+
+SET( SOURCES
+ output.h
+ main.cc
+)
+
+ADD_DEFINITIONS( -DZYPP_BUILD_DIR="${CMAKE_CURRENT_BINARY_DIR}" )
+
+message(${Notcurses++_LIBRARIES})
+
+add_executable( ${PROJECT_NAME} ${SOURCES} )
+target_link_libraries( ${PROJECT_NAME} zypp ${Notcurses++_LIBRARIES} -lnotcurses-core )
--- /dev/null
+#define INCLUDE_TESTSETUP_WITHOUT_BOOST 1
+#include "../../tests/lib/TestSetup.h"
+
+#include <zypp-media/ng/Provide>
+#include <zypp-media/ng/ProvideSpec>
+#include <zypp-media/FileCheckException>
+#include <zypp-core/zyppng/base/EventLoop>
+#include <zypp-core/zyppng/base/Timer>
+#include <zypp-core/zyppng/pipelines/Transform>
+#include <zypp-core/zyppng/pipelines/Lift>
+#include <zypp-core/zyppng/pipelines/Wait>
+#include <boost/container/vector.hpp>
+#include <solv/repo.h>
+#include <solv/solvable.h>
+
+#include <cstdlib>
+#include <clocale>
+#include <iostream>
+#include <numeric>
+
+#include "output.h"
+
+class DlSkippedException : public zypp::Exception
+{
+ public:
+ DlSkippedException() : zypp::Exception("Download was skipped" ) {}
+};
+
+class MyDLStatus : public zyppng::ProvideStatus
+{
+ public:
+ MyDLStatus ( OutputView &out, zyppng::ProvideRef parent ) : ProvideStatus( parent ), _out(out) {}
+
+ virtual void pulse ( ) {
+ zyppng::ProvideStatus::pulse();
+
+ const auto &currStats = stats();
+
+ if ( currStats._expectedBytes == 0 )
+ return;
+
+ auto perc = (double)(currStats._finishedBytes + currStats._partialBytes) / (double)currStats._expectedBytes;
+ std::string txt = zypp::str::Str() << "Downloading " << zypp::ByteCount((currStats._finishedBytes + currStats._partialBytes)) << "/" << currStats._expectedBytes << " (" << currStats._perSecond<<"/s)";
+ _out.updateProgress( txt, perc );
+ }
+
+ private:
+ OutputView &_out;
+};
+
+/*!
+ * Downloads a solvable via the given handle.
+ * Returns the AsyncResult
+ */
+zyppng::AsyncOpRef<zyppng::expected<zyppng::ProvideRes>> provideSolvable ( std::shared_ptr<OutputView> output, zyppng::ProvideMediaHandle handle, const zypp::sat::Solvable &s )
+{
+ using namespace zyppng::operators;
+
+ auto prov = handle.parent();
+ if ( !prov )
+ return zyppng::makeReadyResult( zyppng::expected<zyppng::ProvideRes>::error( std::make_exception_ptr( zypp::Exception("Handle without parent!"))) );
+
+ zypp::PoolItem pi(s);
+ if ( !pi->isKind<zypp::Package>() ) {
+ //output->putMsgErr( zypp::str::Str() << "Skipping 1: " << pi.asUserString() << "\n" );
+ return zyppng::makeReadyResult( zyppng::expected<zyppng::ProvideRes>::error( std::make_exception_ptr(DlSkippedException())) );
+ }
+
+ auto oml = pi.lookupLocation();
+ if ( oml.filename().empty() ) {
+ output->putMsgErr( zypp::str::Str() << "Skipping: " << pi.asUserString() << "\n" );
+ return zyppng::makeReadyResult( zyppng::expected<zyppng::ProvideRes>::error( std::make_exception_ptr(DlSkippedException())) );
+ }
+
+ // enable for more output, but sloooooows the startup time down significantly
+ //output->putMsgTxt( zypp::str::Str() << "Downloading " << pi.asUserString() << " size is: " << pi.downloadSize() << "\n" );
+
+ return prov->provide( handle, oml.filename(), zyppng::ProvideFileSpec( oml ) )
+ | [output]( zyppng::expected<zyppng::ProvideRes> &&res ) {
+ if ( res ) {
+ output->putMsgTxt( zypp::str::Str() << "File downloaded: " << res->file() << "\n" );
+ } else {
+ output->putMsgErr( zypp::str::Str() << "Failed to download file\n" );
+ try
+ {
+ std::rethrow_exception(res.error());
+ }
+ catch(const zypp::Exception& e)
+ {
+ output->putMsgErr( zypp::str::Str() << e.asUserHistory() << '\n' );
+ }
+ catch(const std::exception& e)
+ {
+ output->putMsgErr( zypp::str::Str() << e.what() << '\n' );
+ }
+ }
+ return res;
+ };
+}
+
+/*!
+ * Helper function to copy a ProvideRes underlying file to a target destination, keeping the ProvideRes alive until the copy operation has finished
+ */
+zyppng::AsyncOpRef<zyppng::expected<zypp::ManagedFile>> copyProvideResToFile ( std::shared_ptr<OutputView> output, zyppng::ProvideRef prov, zyppng::ProvideRes &&res, const zypp::Pathname &targetDir )
+{
+ using namespace zyppng::operators;
+
+ auto fName = res.file();
+ return prov->copyFile( fName, targetDir/fName.basename() )
+ | [ resSave = std::move(res) ] ( auto &&result ) {
+ // callback lambda to keep the ProvideRes reference around until the op is finished,
+ // if the op fails the callback will be cleaned up and so the reference
+ return result;
+ };
+}
+
+/*!
+ * Helper function to calculate a checksum on a ProvideRes result file, keeping the ProvideRes alive during the operation and returning it again after the test was successful,
+ * here we could also ask the user if a invalid checksum should be accepted and ignore the result if the user wants to
+ */
+zyppng::AsyncOpRef<zyppng::expected<zyppng::ProvideRes>> checksumIfRequired ( std::shared_ptr<OutputView> output, zyppng::ProvideRef prov, zypp::sat::Solvable solvable, zyppng::ProvideRes &&res )
+{
+ using namespace zyppng::operators;
+
+ auto fName = res.file();
+
+ const auto oml = solvable.lookupLocation();
+ if ( !oml.checksum().empty() ) {
+ output->putMsgTxt( zypp::str::Str()<<" calculating checksum for file: " << fName << "\n");
+
+ return prov->checksumForFile( fName, oml.checksum().type() )
+ | mbind([ output, expectedSum = oml.checksum(), saveRes = std::move(res) ]( auto &&checksumRes ) {
+ output->putMsgTxt( zypp::str::Str()<<" Got checksum for file: " << saveRes.file() << " is: " << checksumRes << "\n" );
+ try {
+ zypp::CheckSum s( expectedSum.type(), checksumRes );
+ if ( expectedSum == s )
+ return zyppng::expected<zyppng::ProvideRes>::success(std::move(saveRes));
+ else
+ return zyppng::expected<zyppng::ProvideRes>::error( ZYPP_EXCPT_PTR( zypp::CheckSumCheckException( zypp::str::Str() << saveRes.file() << " has wrong checksum" ) ) );
+ } catch ( const zypp::Exception &e ) {
+ ZYPP_CAUGHT(e);
+ return zyppng::expected<zyppng::ProvideRes>::error( ZYPP_EXCPT_PTR(e) );
+ }
+
+ });
+ }
+
+ return zyppng::makeReadyResult( zyppng::expected<zyppng::ProvideRes>::success(std::move(res)) );
+}
+
+
+
+int main ( int argc, char *argv[] )
+{
+ using namespace zyppng::operators;
+
+ auto ev = zyppng::EventLoop::create();
+
+ auto output = OutputView::create();
+ if ( !output ) {
+ std::cerr << "Failed to initialize Output view" << std::endl;
+ return 1;
+ }
+
+ output->putMsgTxt("Loading System\n");
+ output->putMsgErr("Errors go here\n");
+
+#if 0
+ {
+ auto prov = zyppng::Provide::create();
+ prov->setStatusTracker( std::make_shared<MyDLStatus>( *output, prov) );
+
+ zypp::Url test("copy://");
+ test.setPathName("/tmp/test/test1");
+
+ auto rootOp = prov->provide( test, zyppng::ProvideFileSpec().setDestFilenameHint("/tmp/test/test2") )
+ | [&]( zyppng::expected<zyppng::ProvideRes> &&res ) {
+ if ( !res ) {
+ output->putMsgErr( zypp::str::Str() << "Failed to copy file: ", false );
+ try {
+ std::rethrow_exception(res.error());
+
+ } catch(const zypp::Exception& e) {
+ output->putMsgErr( zypp::str::Str() << e.asUserHistory() << '\n' );
+
+ } catch(const std::exception& e) {
+ output->putMsgErr( zypp::str::Str() << e.what() << '\n' );
+
+ } catch(...) {
+ output->putMsgErr( zypp::str::Str() << "Unknown exception\n" );
+ }
+ } else {
+ output->putMsgTxt( zypp::str::Str() << "Yay copy worked\n" );
+ }
+ return 0;
+ };
+
+ prov->start();
+ if ( !rootOp->isReady() )
+ ev->run();
+ output->putMsgTxt( zypp::str::Str() << "Waiting, press Return to exit\n" );
+ output->waitForKeys( { NCKEY_ENTER } );
+ return 0;
+ }
+#endif
+
+ TestSetup t;
+ try {
+ t.LoadSystemAt( "/" );
+ } catch ( const zypp::Exception &e ) {
+ output.reset(); //disable curses
+ std::cerr << "Failed to load repo info from system." << std::endl;
+ std::cerr << "Got exception: " << e << std::endl;
+ return 1;
+ }
+
+ zypp::RepoManager rManager( zypp::RepoManagerOptions("/") );
+ auto pool = t.pool();
+ auto satPool = t.satpool();
+
+ output->putMsgTxt("Loading repositories\n");
+
+ std::vector<zypp::Repository> myRepos;
+ for ( const auto &r : t.satpool().repos() ) {
+ if ( r.isSystemRepo() )
+ continue;
+ myRepos.push_back(r);
+ }
+
+ if ( !myRepos.size() ) {
+ output->putMsgTxt( zypp::str::Str() << "No repos found, press any key to exit\n" );
+ output->waitForKeys();
+ return 0;
+ }
+
+ std::vector<int> reposToDl = output->promptMultiSelect( "Select the repos you want to download", "", myRepos );
+ output->renderNow();
+
+ if( !reposToDl.size() ) {
+ output->putMsgTxt( zypp::str::Str() << "No repos selected, press any key to exit\n" );
+ output->waitForKeys();
+ return 0;
+ }
+
+ const auto &workerPath = zypp::Pathname ( ZYPP_BUILD_DIR ).dirname() / "tools" / "workers";
+ auto prov = zyppng::Provide::create();
+ prov->setStatusTracker( std::make_shared<MyDLStatus>( *output, prov) );
+ prov->sigAuthRequired().connect( [&](const zypp::Url &reqUrl, const std::string &triedUsername, const std::map<std::string, std::string> &extraValues) -> std::optional<zypp::media::AuthData> {
+
+ auto user = output->promptUser(
+ zypp::str::Str() << "Auth request for URL: " << reqUrl << "\n"
+ << "Last tried username was: " << triedUsername << "\n",
+ "Username" );
+ if ( !user )
+ return {};
+
+ zypp::media::AuthData d;
+ d.setUrl( reqUrl );
+ d.setUsername( *user );
+
+ auto pass = output->promptUser("", "Password");
+ if ( pass )
+ d.setPassword( *pass );
+
+ return d;
+ });
+
+ std::vector< zyppng::AsyncOpRef< std::vector<zyppng::expected<zypp::ManagedFile>>> > mop;
+
+ // we need a way to remember the data for the full transaction, there is probably a better way for a real application
+ std::unordered_map<int, std::unordered_map<int, std::vector<zypp::sat::Solvable>>> repoToMediaToSolvables;
+
+ zypp::Pathname workPath = zypp::Pathname(".").realpath() / "allrpms";
+ zypp::filesystem::assert_dir( workPath );
+
+ for ( const auto repoToDl : reposToDl ) {
+ const auto &r = myRepos[repoToDl];
+ output->putMsgTxt( zypp::str::Str() << "Repo selected to download: " << r.asUserString() << "\n" );
+
+ auto urlSet = r.info().baseUrls();
+ if ( urlSet.empty() ) {
+ output->putMsgTxt( zypp::str::Str() << "Repo has no mirrors, ignoring!\n" );
+ continue;
+ }
+
+ std::vector<zypp::Url> urlsWithPath;
+ std::transform( urlSet.begin(), urlSet.end(), std::back_inserter(urlsWithPath), [&]( const zypp::Url &bUrl ){
+ zypp::Url withPath(bUrl);
+ withPath.setPathName( r.info().path() / bUrl.getPathName() );
+ return withPath;
+ });
+
+ zypp::ByteCount bc;
+ output->putMsgTxt( zypp::str::Str() << "Adding Solvables for repo: " << r << "\n" );
+ std::unordered_map<int, std::vector<zypp::sat::Solvable>> &mediaToSolvables = repoToMediaToSolvables[repoToDl];
+ std::for_each( satPool.solvablesBegin(), satPool.solvablesEnd(), [&]( const zypp::sat::Solvable &s ) {
+ if ( s.repository() != r )
+ return;
+ bc += s.downloadSize();
+ const auto mediaNr = s.lookupLocation().medianr();
+ mediaToSolvables[mediaNr].push_back(s);
+ });
+ output->putMsgTxt( zypp::str::Str() << "Download size for repo: " << r << ":" << bc << "\n" );
+
+ std::transform( mediaToSolvables.begin(), mediaToSolvables.end(), std::back_inserter(mop), [&]( const auto &s ) {
+
+ auto spec = zyppng::ProvideMediaSpec( r.info().name() );
+ zypp::Pathname mediafile = r.info().metadataPath();
+ if ( !mediafile.empty() ) {
+ mediafile += "/media.1/media";
+ if ( zypp::PathInfo(mediafile).isExist() ) {
+ spec.setMediaFile( mediafile );
+ spec.setMedianr( s.first );
+ }
+ }
+
+ return prov->attachMedia( urlsWithPath, spec )
+ | [ &, &solvables = s.second ]( zyppng::expected<zyppng::Provide::MediaHandle> &&hdl ) {
+
+ if ( !hdl ) {
+ output->putMsgErr( zypp::str::Str() << "Failed to attach medium: ", false );
+ try {
+ std::rethrow_exception(hdl.error());
+
+ } catch(const zypp::Exception& e) {
+ output->putMsgErr( zypp::str::Str() << e.asUserHistory() << '\n' );
+
+ } catch(const std::exception& e) {
+ output->putMsgErr( zypp::str::Str() << e.what() << '\n' );
+
+ } catch(...) {
+ output->putMsgErr( zypp::str::Str() << "Unknown exception\n" );
+
+ }
+ return std::vector<zyppng::AsyncOpRef<zyppng::expected<zypp::ManagedFile>>>{ zyppng::makeReadyResult( zyppng::expected<zypp::ManagedFile>::error( hdl.error() ) ) };
+ }
+
+ return zyppng::transform( solvables, [ &, hdl=hdl.get() ]( const zypp::sat::Solvable &s ) {
+ return s | (std::bind( &provideSolvable, output, hdl, std::placeholders::_1 ))
+ | mbind (std::bind( &checksumIfRequired, output, prov, sat::Solvable(s), std::placeholders::_1 ))
+ | mbind (std::bind( ©ProvideResToFile, output, prov, std::placeholders::_1, workPath ) ) ;
+ });
+ }
+ | zyppng::waitFor<zyppng::expected<zypp::ManagedFile>>()
+ | [ &r, &output ]( const auto &&res ){
+ output->putMsgTxt( zypp::str::Str() << "Finished with rpo: " << r.info().name() << "\n" );
+ return res;
+ };
+ });
+ }
+
+ std::vector<zyppng::expected<zypp::ManagedFile>> finalResults;
+
+ auto rootOp = std::move( mop )
+ | zyppng::waitFor< std::vector<zyppng::expected<zypp::ManagedFile>> >()
+ | [&]( std::vector<std::vector<zyppng::expected<zypp::ManagedFile>>> &&allResults ) {
+ // reduce to one vector of results
+ for ( auto &innerVec : allResults ) {
+ std::accumulate( std::make_move_iterator(innerVec.begin()), std::make_move_iterator(innerVec.end()), &finalResults, []( std::vector<zyppng::expected<zypp::ManagedFile>>* vec, auto &&bla ){
+ vec->push_back(std::move(bla));
+ return vec;
+ });
+ }
+ ev->quit();
+ return 0;
+ };
+
+ prov->start();
+ if ( !rootOp->isReady() )
+ ev->run();
+
+ int succ = 0;
+ int skip = 0;
+ int err = 0;
+ for ( const auto &res : finalResults ) {
+ if ( res )
+ succ++;
+ else {
+ try {
+ std::rethrow_exception( res.error() );
+ } catch(const DlSkippedException& e) {
+ skip++;
+ } catch( const zypp::Exception &e ) {
+ output->putMsgErr( zypp::str::Str()<<"Request failed with: " << e.asUserString() << "\n");
+ err++;
+ } catch( const std::exception &e ) {
+ output->putMsgErr( zypp::str::Str()<<"Request failed with: " << e.what() << "\n");
+ err++;
+ } catch( ... ) {
+ output->putMsgErr( zypp::str::Str()<<"Request failed with an unknown error.\n" );
+ err++;
+ }
+ }
+ }
+ #if 0
+ for ( const auto &outer : finalResults ) {
+
+ }
+ #endif
+
+ output->putMsgTxt( zypp::str::Str() << "All done:\nSuccess:" << succ << "\nErrors: "<< err << "\nSkipped:"<<skip<<"\n", false );
+ output->putMsgTxt( zypp::str::Str() << "Waiting, press Return to exit\n" );
+ output->waitForKeys( { NCKEY_ENTER } );
+ return 0;
+}
--- /dev/null
+#ifndef OUTPUT_H
+#define OUTPUT_H
+
+#include <zypp-core/zyppng/base/Base>
+#include <ncpp/NotCurses.hh>
+#include <ncpp/Plane.hh>
+#include <ncpp/MultiSelector.hh>
+#include <ncpp/Reader.hh>
+#include <memory>
+
+
+static void zypp_release_ncplane ( struct ncplane *ptr ) { if ( ptr ) ncplane_destroy(ptr); }
+static void zypp_release_progbar ( struct ncprogbar *ptr ) { if ( ptr ) ncprogbar_destroy(ptr); }
+
+class OutputView : public zyppng::Base
+{
+ public:
+ ~OutputView() {
+ }
+ static std::shared_ptr<OutputView> create( ) {
+
+ auto ptr = std::shared_ptr<OutputView>( new OutputView() );
+ ptr->_nc = std::make_unique<ncpp::NotCurses>();
+ if ( !ptr->_nc )
+ return nullptr;
+
+ ptr->_stdplane = std::unique_ptr<ncpp::Plane>( ptr->_nc->get_stdplane() );
+ if ( !ptr->_stdplane )
+ return nullptr;
+
+ int rows = 0;
+ int cols = 0;
+ ptr->_stdplane->get_dim( rows, cols );
+
+ int top_rows = ( rows - 2 );
+ if ( top_rows <= 0 ){
+ return nullptr;
+ }
+
+ int bottom_rows = rows - top_rows;
+ if ( bottom_rows <= 0 ) {
+ return nullptr;
+ }
+
+ // the outer plane this is just used to draw a nice border
+ ptr->_topOuterPlane = std::make_unique<ncpp::Plane>( *ptr->_stdplane, top_rows, cols, 0, 0, nullptr );
+ ptr->_topOuterPlane->rounded_box_sized( 0, 0, top_rows, cols, 0 );
+
+ // the inner plane, here we put the text
+ int r,c;
+ ptr->_topOuterPlane->get_dim( r, c );
+ ptr->_topPlaneLeft = std::make_unique<ncpp::Plane>( *ptr->_topOuterPlane, r-2 , (c-1) / 2, 1, 1 );
+ ptr->_topPlaneLeft->set_scrolling( true );
+
+ ptr->_topPlaneRight = std::make_unique<ncpp::Plane>( *ptr->_topOuterPlane, r-2 , c - ptr->_topPlaneLeft->get_dim_x() - 1 , 1, ptr->_topPlaneLeft->get_dim_x() + 1 );
+ ptr->_topPlaneRight->set_scrolling( true );
+
+ // the plane where we show the progressbar text
+ ptr->_progBarTextPlane = std::make_unique<ncpp::Plane>( *ptr->_stdplane, 1 , cols, top_rows, 0 );
+
+ // to create the progressbar we need to fall back to the C API due to some bugs in the C++ progbar implementation
+ ncplane_options nopts = {
+ .y = top_rows+1,
+ .x = 0,
+ .rows = bottom_rows-1,
+ .cols = cols,
+ .userptr = nullptr,
+ .name = nullptr,
+ .resizecb = nullptr,
+ .flags = 0,
+ .margin_b = 0,
+ .margin_r = 0,
+ };
+
+ std::unique_ptr<struct ncplane, void(*)(struct ncplane*)> bottomPlane( ncplane_create( *ptr->_stdplane, &nopts ), &zypp_release_ncplane );
+ if ( !bottomPlane ) {
+ return nullptr;
+ }
+
+ struct ncprogbar_options popts = {
+ .ulchannel = 0,
+ .urchannel = 0,
+ .blchannel = 0,
+ .brchannel = 0,
+ .flags = 0,
+ };
+ ptr->_progbar = std::unique_ptr<struct ncprogbar, void(*)(struct ncprogbar*)> ( ncprogbar_create( bottomPlane.release(), &popts ), &zypp_release_progbar );
+ if ( !ptr->_progbar ) {
+ return nullptr;
+ }
+ return ptr;
+ }
+
+ void updateProgress ( const std::string &txt, double prog, bool autoRender = true ) {
+ if ( !_progbar )
+ return;
+
+ _progBarTextPlane->erase();
+ _progBarTextPlane->putstr( 0, ncpp::NCAlign::Center, txt.data() );
+ ncprogbar_set_progress( _progbar.get(), prog );
+ if ( autoRender )
+ renderNow();
+ }
+
+ void putMsgTxt ( const std::string &txt, bool doAutoRender = true ) {
+ if ( !_topPlaneLeft )
+ return;
+
+ ncplane_puttext( *_topPlaneLeft, -1, NCALIGN_LEFT, txt.data(), nullptr );
+ if ( doAutoRender )
+ renderNow();
+ }
+
+ void putMsgErr ( const std::string &txt, bool doAutoRender = true ) {
+ if ( !_topPlaneRight )
+ return;
+
+ _topPlaneRight->set_fg_rgb( 0xff5349 );
+ ncplane_puttext( *_topPlaneRight, -1, NCALIGN_LEFT, txt.data(), nullptr );
+ _topPlaneRight->set_fg_default();
+ if ( doAutoRender )
+ renderNow();
+ }
+
+ std::optional<std::string> promptUser ( const std::string &desc, const std::string &label )
+ {
+ if ( desc.size() )
+ putMsgTxt( desc, false );
+ putMsgTxt( label + ": " );
+
+ int curX, curY;
+ _topPlaneLeft->get_cursor_yx( curY, curX );
+ ncplane_options nopts = {
+ .y = curY,
+ .x = curX,
+ .rows = 1,
+ .cols = _topPlaneLeft->get_dim_x() - curX,
+ .userptr = nullptr,
+ .name = nullptr,
+ .resizecb = nullptr,
+ .flags = 0,
+ .margin_b = 0,
+ .margin_r = 0,
+ };
+
+ std::unique_ptr<struct ncplane, void(*)(struct ncplane*)> inputPlane( ncplane_create( *_topPlaneLeft, &nopts ), &zypp_release_ncplane );
+ if ( !inputPlane ) {
+ return {};
+ }
+
+ auto readOpts = ncreader_options {
+ .tchannels = 0,
+ .tattrword = 0,
+ .flags = NCREADER_OPTION_CURSOR | NCREADER_OPTION_NOCMDKEYS
+ };
+ std::unique_ptr<struct ncreader, void(*)(struct ncreader*)> reader( ncreader_create( inputPlane.release(), &readOpts ), []( struct ncreader *relme){ ncreader_destroy(relme, nullptr);} );
+ if ( !reader )
+ return {};
+
+ while ( true ) {
+ renderNow();
+ ncinput ni;
+ _nc->get( true, &ni );
+ if ( ni.id == NCKEY_ENTER ) {
+ break;
+ }
+ ncreader_offer_input( reader.get(), &ni );
+ }
+
+ std::unique_ptr<char, void(*)(void *)> data( ncreader_contents( reader.get() ), free );
+ std::string_view cppData( data.get() );
+ if ( cppData.empty() )
+ return {};
+
+ return std::string(cppData);
+ }
+
+ template <typename T>
+ std::vector<int> promptMultiSelect( const std::string &title, const std::string &secondary, const std::vector<T> &data ) {
+
+ // use a boost container here, because the stdlib decided to completely break vector<bool> by prematurely optimizing it
+ boost::container::vector<bool> selected( data.size(), false );
+ std::vector<ncmselector_item> items;
+ std::vector<std::pair<std::string, std::string>> stringsStorage;
+ items.reserve( data.size() );
+ stringsStorage.reserve( data.size() );
+
+ for ( const T& entry : data ) {
+ stringsStorage.push_back( std::make_pair( entry.asUserString(), std::string("") ) );
+ items.push_back( ncmselector_item{
+ .option = stringsStorage.back().first.data(),
+ .desc = stringsStorage.back().second.data(),
+ .selected = false
+ });
+ }
+ items.push_back(ncmselector_item{
+ .option = nullptr,
+ .desc = nullptr,
+ .selected = false
+ });
+
+ std::string_view footer("Press Enter to accept or ESC to cancel.");
+
+ auto selOpts = ncmultiselector_options{
+ .title = title.data(),
+ .secondary = secondary.data(),
+ .footer = footer.data(),
+ .items = items.data(),
+ .maxdisplay = 0,
+ .opchannels = NCCHANNELS_INITIALIZER(0xe0, 0x80, 0x40, 0, 0, 0),
+ .descchannels = NCCHANNELS_INITIALIZER(0x80, 0xe0, 0x40, 0, 0, 0),
+ .titlechannels = NCCHANNELS_INITIALIZER(0x20, 0xff, 0xff, 0, 0, 0x20),
+ .footchannels = NCCHANNELS_INITIALIZER(0xe0, 0, 0x40, 0x20, 0x20, 0),
+ .boxchannels = NCCHANNELS_INITIALIZER(0x20, 0xe0, 0xe0, 0x20, 0, 0),
+ .flags = 0
+ };
+
+ auto tmpPlane = ncpp::Plane( *_stdplane, _stdplane->get_dim_y()-1, _stdplane->get_dim_x()-1, 1, 1 );
+ ncpp::MultiSelector selector( tmpPlane, &selOpts);
+
+ while ( true ) {
+ renderNow();
+ ncinput ni;
+ _nc->get( true, &ni );
+ if ( ni.id == NCKEY_ESC ) {
+ return {};
+ } else if ( ni.id == NCKEY_ENTER ) {
+ break;
+ }
+ selector.offer_input( &ni );
+ }
+
+ selector.get_selected( selected.data(), selected.size() );
+ std::vector<int> selIndices;
+ for ( int i = 0; i < selected.size(); i++ ) {
+ if ( selected[i])
+ selIndices.push_back(i);
+ }
+
+ return selIndices;
+ }
+
+ void renderNow () {
+ if ( _nc ) _nc->render();
+ }
+
+ uint32_t waitForKeys ( std::vector<uint32_t> keys = {} ) {
+ ncinput ni;
+ while ( true ) {
+ _nc->get( true, &ni );
+ if ( keys.empty() || std::find( keys.begin(), keys.end(), ni.id ) != keys.end() )
+ return ni.id;
+ }
+ }
+
+ private:
+ OutputView() : _topOuterPlane( nullptr ), _topPlaneLeft( nullptr ), _topPlaneRight( nullptr ), _progbar( nullptr, &zypp_release_progbar ) {}
+
+ std::unique_ptr<ncpp::NotCurses> _nc;
+ std::unique_ptr<ncpp::Plane> _stdplane;
+ std::unique_ptr<ncpp::Plane> _topOuterPlane;
+ std::unique_ptr<ncpp::Plane> _topPlaneLeft;
+ std::unique_ptr<ncpp::Plane> _topPlaneRight;
+ std::unique_ptr<ncpp::Plane> _progBarTextPlane;
+ std::unique_ptr<struct ncprogbar, void(*)(struct ncprogbar*)> _progbar;
+};
+
+
+#endif // OUTPUT_H
--- /dev/null
+PROJECT( zypp-media-chksum C CXX )
+
+set (CMAKE_CXX_STANDARD 17)
+SET (CMAKE_CXX_EXTENSIONS OFF)
+
+FIND_PACKAGE(Protobuf REQUIRED)
+
+SET( SOURCES
+ main.cc
+)
+
+add_executable( ${PROJECT_NAME} ${SOURCES} )
+target_link_libraries( ${PROJECT_NAME} zypp-media )
+target_link_libraries( ${PROJECT_NAME} zypp-core )
+target_link_libraries( ${PROJECT_NAME} zypp-protobuf )
+target_link_libraries( ${PROJECT_NAME} ${PROTOBUF_LITE_LIBRARIES} )
+
+IF ( INSTALL_NG_BINARIES )
+ INSTALL( TARGETS ${PROJECT_NAME} DESTINATION "${ZYPP_LIBEXEC_INSTALL_DIR}/workers" )
+ENDIF()
+
+set_target_properties( ${PROJECT_NAME}
+ PROPERTIES
+ ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+ LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+ RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+)
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+
+#include <csignal>
+#include <zypp-core/zyppng/base/private/linuxhelpers_p.h>
+#include <zypp-core/fs/PathInfo.h>
+#include <zypp-media/ng/worker/ProvideWorker>
+#include <zypp-media/ng/private/providemessage_p.h>
+#include <zypp-core/Digest.h>
+#include <zypp-core/zyppng/base/Signals>
+
+class ChksumProvider : public zyppng::worker::ProvideWorker
+{
+ public:
+ ChksumProvider( std::string_view workerName ) : ProvideWorker( workerName ) { }
+
+ void immediateShutdown() override { }
+
+ protected:
+ // ProvideWorker interface
+ zyppng::expected<zyppng::worker::WorkerCaps> initialize(const zyppng::worker::Configuration &conf) override {
+
+ zyppng::worker::WorkerCaps caps;
+ caps.set_worker_type ( zyppng::worker::WorkerCaps::CPUBound );
+ caps.set_cfg_flags(
+ zyppng::worker::WorkerCaps::Flags (
+ zyppng::worker::WorkerCaps::Pipeline
+ | zyppng::worker::WorkerCaps::ZyppLogFormat
+ )
+ );
+
+ return zyppng::expected<zyppng::worker::WorkerCaps>::success(caps);
+ }
+
+ void provide() override {
+ auto &queue = requestQueue();
+
+ if ( !queue.size() )
+ return;
+
+
+ auto req = queue.front();
+ queue.pop_front();
+
+ // here we only receive request codes, we only support Provide messages, all others are rejected
+ // Cancel is never to be received here
+ if ( req->_spec.code() != zyppng::ProvideMessage::Code::Provide ) {
+ req->_state = zyppng::worker::ProvideWorkerItem::Finished;
+ provideFailed( req->_spec.requestId()
+ , zyppng::ProvideMessage::Code::BadRequest
+ , "Request type not implemented"
+ , false
+ , {} );
+ return;
+ }
+
+ zypp::Url url;
+ const auto &urlVal = req->_spec.value( zyppng::ProvideMsgFields::Url );
+ try {
+ url = zypp::Url( urlVal.asString() );
+ } catch ( const zypp::Exception &excp ) {
+ ZYPP_CAUGHT(excp);
+
+ std::string err = zypp::str::Str() << "Invalid URL in request: " << urlVal.asString();
+ ERR << err << std::endl;
+
+ req->_state = zyppng::worker::ProvideWorkerItem::Finished;
+ provideFailed( req->_spec.requestId()
+ , zyppng::ProvideMessage::Code::BadRequest
+ , err
+ , false
+ , {} );
+
+ return;
+ }
+
+ std::string chksumType;
+ try {
+ chksumType = req->_spec.value( std::string_view("chksumType") ).asString();
+ } catch ( const zypp::Exception &excp ) {
+ ZYPP_CAUGHT(excp);
+
+ std::string err = zypp::str::Str() << "No or invalid chksumType in request";
+ ERR << err << std::endl;
+
+ req->_state = zyppng::worker::ProvideWorkerItem::Finished;
+ provideFailed( req->_spec.requestId()
+ , zyppng::ProvideMessage::Code::BadRequest
+ , err
+ , false
+ , {} );
+
+ return;
+ }
+
+ zypp::Pathname file = url.getPathName();
+ if ( ! zypp::PathInfo( file ).isFile() ) {
+ req->_state = zyppng::worker::ProvideWorkerItem::Finished;
+ provideFailed( req->_spec.requestId()
+ , zyppng::ProvideMessage::Code::NotFound
+ , zypp::str::Str() << "File " << file << " not found."
+ , false
+ , {} );
+ return;
+ }
+
+ std::string filesum = zypp::filesystem::checksum( file, chksumType );
+ DBG << "Calculated checksum for : " << file << " with type: " << chksumType << " is " << filesum << std::endl;
+ provideSuccess( req->_spec.requestId(), false, file, {{chksumType, {filesum}}} );
+ }
+
+ void cancel(const std::deque<zyppng::worker::ProvideWorkerItemRef>::iterator &i ) override {
+ ERR << "Bug, cancel should never be called for running items" << std::endl;
+ }
+
+};
+
+int main( int argc, char *argv[] )
+{
+ // lets ignore those, SIGPIPE is better handled via the EPIPE error, and we do not want anyone
+ // to CTRL+C us
+ zyppng::blockSignalsForCurrentThread( { SIGPIPE, SIGINT } );
+
+ auto provider = std::make_shared<ChksumProvider>("zypp-media-chksum");
+ auto res = provider->run (STDIN_FILENO, STDOUT_FILENO);
+ provider->immediateShutdown();
+ if ( res )
+ return 0;
+
+ //@TODO print error
+ return 1;
+}
--- /dev/null
+PROJECT( zypp-media-copy C CXX )
+
+set (CMAKE_CXX_STANDARD 17)
+SET (CMAKE_CXX_EXTENSIONS OFF)
+
+FIND_PACKAGE(Protobuf REQUIRED)
+
+SET( SOURCES
+ main.cc
+)
+
+add_executable( ${PROJECT_NAME} ${SOURCES} )
+target_link_libraries( ${PROJECT_NAME} zypp-media )
+target_link_libraries( ${PROJECT_NAME} zypp-core )
+target_link_libraries( ${PROJECT_NAME} zypp-protobuf )
+target_link_libraries( ${PROJECT_NAME} ${PROTOBUF_LITE_LIBRARIES} )
+
+IF ( INSTALL_NG_BINARIES )
+ INSTALL( TARGETS ${PROJECT_NAME} DESTINATION "${ZYPP_LIBEXEC_INSTALL_DIR}/workers" )
+ENDIF()
+
+set_target_properties( ${PROJECT_NAME}
+ PROPERTIES
+ ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+ LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+ RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+)
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+
+#include <csignal>
+#include <zypp-core/zyppng/base/private/linuxhelpers_p.h>
+#include <zypp-core/fs/PathInfo.h>
+#include <zypp-media/ng/worker/ProvideWorker>
+#include <zypp-media/ng/private/providemessage_p.h>
+#include <zypp-core/zyppng/base/Signals>
+
+class CopyProvider : public zyppng::worker::ProvideWorker
+{
+ public:
+ CopyProvider( std::string_view workerName ) : ProvideWorker( workerName ) { }
+
+ void immediateShutdown() override { }
+
+ protected:
+ // ProvideWorker interface
+ zyppng::expected<zyppng::worker::WorkerCaps> initialize(const zyppng::worker::Configuration &conf) override {
+
+ zyppng::worker::WorkerCaps caps;
+ caps.set_worker_type ( zyppng::worker::WorkerCaps::CPUBound );
+ caps.set_cfg_flags(
+ zyppng::worker::WorkerCaps::Flags (
+ zyppng::worker::WorkerCaps::Pipeline
+ | zyppng::worker::WorkerCaps::ZyppLogFormat
+ | zyppng::worker::WorkerCaps::FileArtifacts
+ )
+ );
+
+ return zyppng::expected<zyppng::worker::WorkerCaps>::success(caps);
+ }
+
+ void provide() override {
+ auto &queue = requestQueue();
+
+ if ( !queue.size() )
+ return;
+
+
+ auto req = queue.front();
+ queue.pop_front();
+
+ // here we only receive request codes, we only support Provide messages, all others are rejected
+ // Cancel is never to be received here
+ if ( req->_spec.code() != zyppng::ProvideMessage::Code::Provide ) {
+ req->_state = zyppng::worker::ProvideWorkerItem::Finished;
+ provideFailed( req->_spec.requestId()
+ , zyppng::ProvideMessage::Code::BadRequest
+ , "Request type not implemented"
+ , false
+ , {} );
+ return;
+ }
+
+ zypp::Url url;
+ const auto &urlVal = req->_spec.value( zyppng::ProvideMsgFields::Url );
+ try {
+ url = zypp::Url( urlVal.asString() );
+ } catch ( const zypp::Exception &excp ) {
+ ZYPP_CAUGHT(excp);
+
+ std::string err = zypp::str::Str() << "Invalid URL in request: " << urlVal.asString();
+ ERR << err << std::endl;
+
+ req->_state = zyppng::worker::ProvideWorkerItem::Finished;
+ provideFailed( req->_spec.requestId()
+ , zyppng::ProvideMessage::Code::BadRequest
+ , err
+ , false
+ , {} );
+
+ return;
+ }
+
+ auto targetFileName = req->_spec.value( zyppng::ProvideMsgFields::Filename );
+ if ( !targetFileName.valid() ) {
+ req->_state = zyppng::worker::ProvideWorkerItem::Finished;
+ provideFailed( req->_spec.requestId()
+ , zyppng::ProvideMessage::Code::BadRequest
+ , "Copy worker requires a target filename hint"
+ , false
+ , {} );
+ return;
+ }
+ zypp::Pathname targetFilePath( targetFileName.asString() );
+
+ zypp::PathInfo pi( url.getPathName() );
+ if ( !pi.isExist() ) {
+ req->_state = zyppng::worker::ProvideWorkerItem::Finished;
+ provideFailed( req->_spec.requestId()
+ , zyppng::ProvideMessage::Code::NotFound
+ , zypp::str::Str() << "File " << pi.path() << " not found."
+ , false
+ , {} );
+ return;
+ }
+
+ if ( !pi.isFile() ) {
+ provideFailed( req->_spec.requestId()
+ , zyppng::ProvideMessage::Code::NotAFile
+ , zypp::str::Str() << "Path " << pi.path() << " exists, but its not a file"
+ , false
+ , {} );
+ return;
+ }
+
+ auto res = zypp::filesystem::hardlinkCopy( pi.path(), targetFileName.asString() );
+ if ( res == 0 ) {
+ provideSuccess( req->_spec.requestId(), false, targetFilePath );
+ } else {
+ provideFailed( req->_spec.requestId()
+ , zyppng::ProvideMessage::Code::BadRequest
+ , zypp::str::Str() << "Failed to create file " << targetFilePath
+ , false
+ , {} );
+ }
+ }
+
+ void cancel(const std::deque<zyppng::worker::ProvideWorkerItemRef>::iterator &i ) override {
+ ERR << "Bug, cancel should never be called for running items" << std::endl;
+ }
+
+};
+
+int main( int argc, char *argv[] )
+{
+ // lets ignore those, SIGPIPE is better handled via the EPIPE error, and we do not want anyone
+ // to CTRL+C us
+ zyppng::blockSignalsForCurrentThread( { SIGPIPE, SIGINT } );
+
+ auto provider = std::make_shared<CopyProvider>("zypp-media-copy");
+ auto res = provider->run (STDIN_FILENO, STDOUT_FILENO);
+ provider->immediateShutdown();
+ if ( res )
+ return 0;
+
+ //@TODO print error
+ return 1;
+}
--- /dev/null
+PROJECT( zypp-media-dir C CXX )
+
+set (CMAKE_CXX_STANDARD 17)
+SET (CMAKE_CXX_EXTENSIONS OFF)
+
+FIND_PACKAGE(Protobuf REQUIRED)
+
+SET( SOURCES
+ main.cc
+ dirprovider.cc
+ dirprovider.h
+)
+
+add_executable( ${PROJECT_NAME} ${SOURCES} )
+target_link_libraries( ${PROJECT_NAME} zypp-media )
+target_link_libraries( ${PROJECT_NAME} zypp-core )
+target_link_libraries( ${PROJECT_NAME} zypp-protobuf )
+target_link_libraries( ${PROJECT_NAME} ${PROTOBUF_LITE_LIBRARIES} )
+
+IF ( INSTALL_NG_BINARIES )
+ INSTALL( TARGETS ${PROJECT_NAME} DESTINATION "${ZYPP_LIBEXEC_INSTALL_DIR}/workers" )
+ENDIF()
+
+set_target_properties( ${PROJECT_NAME}
+ PROPERTIES
+ ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+ LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+ RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+)
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+#include "dirprovider.h"
+#include <zypp-media/ng/private/providedbg_p.h>
+#include <zypp-media/ng/private/providemessage_p.h>
+#include <zypp-media/ng/MediaVerifier>
+
+#include <zypp-core/Url.h>
+#include <zypp-core/fs/TmpPath.h>
+#include <zypp-core/fs/PathInfo.h>
+
+#include <iostream>
+#include <fstream>
+
+#undef ZYPP_BASE_LOGGER_LOGGROUP
+#define ZYPP_BASE_LOGGER_LOGGROUP "DirProvider"
+
+
+DirProvider::DirProvider()
+ : DeviceDriver( zyppng::worker::WorkerCaps::SimpleMount )
+{ }
+
+DirProvider::~DirProvider()
+{ }
+
+zyppng::worker::AttachResult DirProvider::mountDevice ( const uint32_t id, const zypp::Url &attachUrl, const std::string &attachId, const std::string &label, const zyppng::HeaderValueMap &extras )
+{
+ try
+ {
+ if ( !attachUrl.getHost().empty() ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , "Host must be empty in dir:// and file:// URLs"
+ , false );
+ }
+
+ // set up the verifier
+ zyppng::MediaDataVerifierRef verifier;
+ if ( extras.contains(zyppng::AttachMsgFields::VerifyType) ) {
+ verifier = zyppng::MediaDataVerifier::createVerifier( extras[zyppng::AttachMsgFields::VerifyType].asString() );
+ if ( !verifier ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , "Invalid verifier type"
+ , false );
+ }
+
+ if ( !verifier->load( extras[zyppng::AttachMsgFields::VerifyData].asString() ) ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , "Failed to create verifier from file"
+ , false );
+ }
+ }
+ const auto &devs = knownDevices();
+
+ // we simulate a device by simply using the pathname
+ zypp::Pathname path = zypp::Pathname( attachUrl.getPathName() ).realpath();
+ const auto &pathStr = path.asString();
+
+ zypp::PathInfo adir( path );
+ if( !adir.isDir()) {
+ // URl did not point to a directory
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , zypp::str::Str()<< "Specified path '" << attachUrl << "' is not a directory"
+ , false
+ );
+ }
+
+ // lets check if the path is what we want
+ auto res = isDesiredMedium( attachUrl, path, verifier, extras.value( zyppng::AttachMsgFields::MediaNr, 1 ).asInt() );
+ if ( !res ) {
+ try {
+ std::rethrow_exception( res.error() );
+ } catch( const zypp::Exception& e ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MediumNotDesired
+ , false
+ , e );
+ } catch ( ... ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MediumNotDesired
+ , "Checking the medium failed with an uknown error"
+ , false );
+ }
+ }
+
+ // first check if we have that device already
+ auto i = std::find_if( devs.begin(), devs.end(), [&]( const auto &d ) { return d->_name == pathStr; } );
+ if ( i != devs.end() ) {
+ attachedMedia().insert( { attachId, zyppng::worker::AttachedMedia{ ._dev = *i, ._attachRoot = "/" } } );
+ return zyppng::worker::AttachResult::success();
+ }
+
+ // we did not find a existing device, well lets make a new one
+ MIL << "New device " << path << " mounted on " << path << std::endl;
+ auto newDev = std::make_shared<zyppng::worker::Device>( zyppng::worker::Device{
+ ._name = pathStr,
+ ._maj_nr = 0,
+ ._min_nr = 0,
+ ._mountPoint = path,
+ ._ephemeral = true, // device should be removed after the last attachment was released
+ ._properties = {}
+ });
+ attachedMedia().insert( { attachId, zyppng::worker::AttachedMedia{ ._dev = newDev, ._attachRoot = "/" } } );
+ return zyppng::worker::AttachResult::success();
+
+ } catch ( const zypp::Exception &e ) {
+ return zyppng::worker::AttachResult::error (
+ zyppng::ProvideMessage::Code::BadRequest
+ , false
+ , e );
+ } catch ( const std::exception &e ) {
+ return zyppng::worker::AttachResult::error (
+ zyppng::ProvideMessage::Code::BadRequest
+ , e.what()
+ , false );
+ } catch ( ... ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::BadRequest
+ , "Unknown exception"
+ , false);
+ }
+}
+
+void DirProvider::unmountDevice ( zyppng::worker::Device &dev ) {
+ // do nothing , this is just a local dir
+}
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+#ifndef ZYPP_NG_TOOLS_DIRPROVIDER_H_INCLUDED
+#define ZYPP_NG_TOOLS_DIRPROVIDER_H_INCLUDED
+
+#include <zypp-media/ng/worker/DeviceDriver>
+
+class DirProvider : public zyppng::worker::DeviceDriver
+{
+ public:
+ DirProvider( );
+ ~DirProvider();
+
+ zyppng::worker::AttachResult mountDevice ( const uint32_t id, const zypp::Url &attachUrl, const std::string &attachId, const std::string &label, const zyppng::HeaderValueMap &extras ) override;
+
+ protected:
+ void unmountDevice ( zyppng::worker::Device &dev ) override;
+
+};
+
+#endif
--- /dev/null
+#include "dirprovider.h"
+
+#include <csignal>
+#include <zypp-media/ng/worker/MountingWorker>
+#include <zypp-core/zyppng/base/private/linuxhelpers_p.h>
+
+
+int main( int , char *[] )
+{
+ // lets ignore those, SIGPIPE is better handled via the EPIPE error, and we do not want anyone
+ // to CTRL+C us
+ zyppng::blockSignalsForCurrentThread( { SIGPIPE, SIGINT } );
+
+ auto driver = std::make_shared<DirProvider>();
+ auto provider = std::make_shared<zyppng::worker::MountingWorker>( "zypp-media-dir", driver );
+ driver->setProvider( provider );
+
+ auto res = provider->run (STDIN_FILENO, STDOUT_FILENO);
+
+ MIL << "DIR Worker shutting down" << std::endl;
+
+ provider->immediateShutdown();
+ if ( res )
+ return 0;
+
+ //@TODO print error
+ return 1;
+}
--- /dev/null
+PROJECT( zypp-media-disc C CXX )
+
+set (CMAKE_CXX_STANDARD 17)
+SET (CMAKE_CXX_EXTENSIONS OFF)
+
+FIND_PACKAGE(Protobuf REQUIRED)
+
+SET( SOURCES
+ main.cc
+ discprovider.cc
+ discprovider.h
+)
+
+add_executable( ${PROJECT_NAME} ${SOURCES} )
+target_link_libraries( ${PROJECT_NAME} zypp-media )
+target_link_libraries( ${PROJECT_NAME} zypp-core )
+target_link_libraries( ${PROJECT_NAME} zypp-protobuf )
+target_link_libraries( ${PROJECT_NAME} ${PROTOBUF_LITE_LIBRARIES} )
+
+IF ( INSTALL_NG_BINARIES )
+ INSTALL( TARGETS ${PROJECT_NAME} DESTINATION "${ZYPP_LIBEXEC_INSTALL_DIR}/workers" )
+ENDIF()
+
+set_target_properties( ${PROJECT_NAME}
+ PROPERTIES
+ ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+ LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+ RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+)
+
+IF ( UDEV_FOUND )
+ TARGET_LINK_LIBRARIES( ${PROJECT_NAME} ${UDEV_LIBRARY} )
+ELSE ( UDEV_FOUND )
+ IF ( HAL_FOUND )
+ TARGET_LINK_LIBRARIES( ${PROJECT_NAME} ${HAL_LIBRARY} ${HAL_STORAGE_LIBRARY} ${DBUS_LIBRARY} )
+ ENDIF ( HAL_FOUND )
+ENDIF ( UDEV_FOUND )
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+#include "discprovider.h"
+#include <zypp-media/ng/private/providedbg_p.h>
+#include <zypp-media/ng/worker/ProvideWorker>
+#include <zypp-media/ng/MediaVerifier>
+#include <zypp-core/fs/PathInfo.h>
+#include <zypp-core/AutoDispose.h>
+#include <zypp-core/base/StringV.h>
+#include <zypp-media/Mount>
+#include <zypp-media/CDTools>
+
+#undef ZYPP_BASE_LOGGER_LOGGROUP
+#define ZYPP_BASE_LOGGER_LOGGROUP "DiscProvider"
+
+extern "C"
+{
+#include <sys/ioctl.h>
+#include <linux/cdrom.h>
+#if HAVE_UDEV
+#include <libudev.h>
+#endif
+}
+
+using namespace std::literals;
+
+DiscProvider::DiscProvider( ) : zyppng::worker::DeviceDriver( zyppng::worker::WorkerCaps::VolatileMount )
+{ }
+
+DiscProvider::~DiscProvider()
+{}
+
+zyppng::worker::AttachResult DiscProvider::mountDevice ( const uint32_t id, const zypp::Url &attachUrl, const std::string &attachId, const std::string &label, const zyppng::HeaderValueMap &extras )
+{
+ std::vector<std::string> devs;
+ if ( extras.contains( zyppng::AttachMsgFields::Device ) ) {
+ for ( const auto &d :extras.values( zyppng::AttachMsgFields::Device ) ) {
+ if ( d.isString() ) {
+ MIL << "Got device from user we are allowed to use: " << d.asString() << std::endl;
+ devs.push_back(d.asString());
+ }
+ }
+ }
+
+ auto &sysDevs = knownDevices();
+
+ std::vector<std::shared_ptr<zyppng::worker::Device>> possibleDevs;
+ if ( devs.size () ) {
+ for ( const auto &d : devs ) {
+ zypp::PathInfo p(d);
+ if ( !p.isBlk () )
+ continue;
+
+ std::shared_ptr<zyppng::worker::Device> device;
+ auto i = std::find_if( sysDevs.begin(), sysDevs.end(), [&p]( const auto &sysDev ){ return ( sysDev->_maj_nr == p.devMajor() && sysDev->_min_nr == p.devMinor() ); } );
+ if ( i == sysDevs.end() ) {
+ // this is a device we did not detect before lets see if its a CDROM
+#ifdef HAVE_UDEV
+ zypp::AutoDispose<struct udev *> udev( ::udev_new(), ::udev_unref );
+ if ( !udev ) {
+ ERR << "Can't create udev context." << std::endl;
+ continue;
+ }
+
+ auto devnum = makedev( p.devMajor(), p.devMinor() );
+ zypp::AutoDispose<struct udev_device *> udevDevice( ::udev_device_new_from_devnum ( udev, 'b', devnum ), ::udev_device_unref );
+ if ( !udevDevice ) {
+ ERR << "Can't create udev device from devnum, ignoring." << std::endl;
+ continue;
+ }
+
+ if ( zypp::strv::asStringView( ::udev_device_get_property_value( udevDevice, "ID_CDROM" ) ) != "1" ) {
+ ERR << "Device " << d << " from Attach request is not a CDROM/DVD ignoring!" << std::endl;
+ continue;
+ }
+
+ auto dev = std::make_shared<zyppng::worker::Device>( zyppng::worker::Device {
+ ._name = p.path().asString(),
+ ._maj_nr = p.devMajor(),
+ ._min_nr = p.devMinor() } );
+ DBG << "Registering device (REQUEST): " << dev->_name << " " << dev->_maj_nr << ":" << dev->_min_nr << std::endl;
+ sysDevs.push_back ( dev );
+ device = dev;
+#endif
+ } else {
+ device = *i;
+ }
+
+ if ( device )
+ possibleDevs.push_back(device);
+ }
+ } else {
+ possibleDevs = sysDevs;
+ }
+
+ // if we have no devices at this point controller gets a error
+ if ( possibleDevs.empty () ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , "No useable device found"
+ , false
+ );
+ }
+
+ const auto attachRoot = zypp::Pathname( attachUrl.getPathName() );
+ const auto attachMediaNr = extras.value( zyppng::AttachMsgFields::MediaNr, 1 ).asInt();
+
+ // set up the verifier
+ zyppng::MediaDataVerifierRef verifier;
+ if ( extras.contains( zyppng::AttachMsgFields::VerifyType ) ) {
+ verifier = zyppng::MediaDataVerifier::createVerifier( extras.value(zyppng::AttachMsgFields::VerifyType).asString() );
+ if ( !verifier ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , "Invalid verifier type"
+ , false
+ );
+ }
+
+ if ( !verifier->load( extras.value(zyppng::AttachMsgFields::VerifyData).asString() ) ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , "Failed to create verifier from file"
+ , false
+ );
+ }
+ }
+
+ //first check if any of the mounted devices are what we want
+ for( const auto &dev : possibleDevs ) {
+ if ( dev->_mountPoint.empty() )
+ continue;
+
+ auto res = isDesiredMedium( attachUrl, dev->_mountPoint / attachRoot, verifier, extras.value( zyppng::AttachMsgFields::MediaNr, 1 ).asInt() );
+ if ( !res )
+ continue;
+
+ MIL << "Found requested medium in dev " << dev->_name << std::endl;
+
+ // we found the device we want!
+ attachedMedia().insert( std::make_pair( attachId, zyppng::worker::AttachedMedia{ dev, attachRoot } ) );
+ return zyppng::worker::AttachResult::success();
+ }
+
+ std::list<std::string> filesystems;
+
+ filesystems.push_back("iso9660");
+
+ // if DVD, try UDF filesystem after iso9660
+ if ( attachUrl.getScheme() == "dvd" )
+ filesystems.push_back("udf");
+
+ std::string options;
+ auto optVal = extras.value( "mountoptions"sv );
+ if ( optVal.valid() && optVal.isString() )
+ options = optVal.asString();
+
+ if ( options.empty() ) {
+ options="ro";
+ }
+
+ while ( true ) {
+
+ // remember how many devices we were able to test
+ uint devicesTested = 0;
+
+ // none of the already mounted devices matched, lets try what we have left
+ for( const auto &dev : possibleDevs ) {
+ if ( !dev->_mountPoint.empty() )
+ continue;
+
+ MIL << "Trying to mount dev " << dev->_name << std::endl;
+
+ // make sure the tray is closed
+ zypp::media::CDTools::closeTray( dev->_name );
+
+ devicesTested++;
+ zypp::media::Mount mount;
+ bool mountsucceeded = false;
+ std::exception_ptr lastErr;
+ for( auto fsit = filesystems.begin() ; !mountsucceeded && fsit != filesystems.end() ; ++fsit) {
+
+ zypp::Pathname newAp;
+ try {
+ newAp = createAttachPoint( this->attachRoot() );
+ if ( newAp.empty() ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , "Failed to create mount directory."
+ , false
+ );
+ }
+
+ mount.mount( dev->_name, newAp.asString(), *fsit, options);
+
+ // wait for /etc/mtab update ...
+ // (shouldn't be needed)
+ int limit = 2;
+ while( !(mountsucceeded=checkAttached( newAp, devicePredicate( dev->_maj_nr, dev->_min_nr ))) && --limit)
+ {
+ WAR << "Wait for /proc/mounts update and retry...." << std::endl;
+ sleep(1);
+ }
+
+ if( !mountsucceeded) {
+ try {
+ mount.umount( newAp.asString() );
+ } catch (const zypp::media::MediaException & excpt_r) {
+ ZYPP_CAUGHT(excpt_r);
+ }
+ ZYPP_THROW( zypp::media::MediaMountException(
+ "Unable to verify that the media was mounted",
+ dev->_name, newAp.asString()
+ ));
+ } else {
+ dev->_mountPoint = newAp;
+ break;
+ }
+ }
+ catch (const zypp::media::MediaMountException &e)
+ {
+ lastErr = std::current_exception ();
+ removeAttachPoint(newAp);
+ ZYPP_CAUGHT(e);
+ }
+ catch (const zypp::media::MediaException & excpt_r)
+ {
+ removeAttachPoint(newAp);
+ ZYPP_CAUGHT(excpt_r);
+ }
+ } // for filesystems
+
+ if ( mountsucceeded ) {
+ // lets check if we have the correct medium
+ bool canUseDevice = false;
+
+ auto res = isDesiredMedium( attachUrl, dev->_mountPoint / attachRoot, verifier, extras.value( zyppng::AttachMsgFields::MediaNr, 1 ).asInt() );
+ if ( !res ) {
+ unmountDevice( *dev );
+ continue;
+ }
+
+ MIL << "Found requested medium in dev " << dev->_name << std::endl;
+ attachedMedia().insert( std::make_pair( attachId, zyppng::worker::AttachedMedia{ dev, attachRoot } ) );
+ return zyppng::worker::AttachResult::success();
+ }
+ } // for each device
+
+ // we did go through all devices, but didn't find any.
+ if ( devicesTested == 0 ) {
+ // no devices are free
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::Jammed
+ , "No free ressources available"
+ , false
+ );
+ } else {
+
+ const auto &mountedDevs = zypp::media::Mount::getEntries();
+ // k, we need to ask the user to give us the medium we need
+ // first find the devices that are free
+ std::vector<std::string> freeDevs;
+ for( const auto &dev : possibleDevs ) {
+ if ( !dev->_mountPoint.empty() )
+ continue;
+
+ auto i = std::find_if( mountedDevs.begin (), mountedDevs.end(), [&dev]( const zypp::media::MountEntry &e ) {
+ zypp::PathInfo pi( e.src );
+ return ( pi.isBlk() && pi.devMajor() == dev->_maj_nr && pi.devMinor() == dev->_min_nr );
+ });
+
+ if ( i == mountedDevs.end() ) {
+ MIL << "Adding " << dev->_name << " to list of free devs" << std::endl;
+ freeDevs.push_back( dev->_name );
+ }
+ }
+
+ // if there are no devices free, we are jammed
+ if ( freeDevs.empty() ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::Jammed
+ , "No free ressources available"
+ , false
+ );
+ }
+
+ auto changeRes = zyppng::worker::ProvideWorker::ABORT;
+ auto worker = parentWorker();
+ if ( worker ) {
+ changeRes = worker->requestMediaChange ( id, label, attachMediaNr, freeDevs );
+ }
+
+ if ( changeRes != zyppng::worker::ProvideWorker::SUCCESS ) {
+ if ( changeRes == zyppng::worker::ProvideWorker::SKIP)
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MediaChangeSkip
+ , "User asked to skip the media change."
+ , false);
+ else
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MediaChangeAbort
+ , "User asked to abort the media change."
+ , false );
+ }
+ }
+ }
+}
+
+void DiscProvider::detectDevices( )
+{
+
+ // helper lambda to add a new device to the internal registry, returning it if its either created or already known
+ const auto registerDevice = [ this ]( const zypp::PathInfo &devnode, std::string_view reason ) {
+ auto &sysDevs = knownDevices();
+ if ( devnode.isBlk() ) {
+ auto i = std::find_if( sysDevs.begin(), sysDevs.end(), [&]( const auto &dev ){
+ return ( dev->_maj_nr == devnode.devMajor() && dev->_min_nr == devnode.devMinor () );
+ });
+
+ // we already know the device
+ if ( i != sysDevs.end() )
+ return i;
+
+ auto dev = std::make_shared<zyppng::worker::Device>( zyppng::worker::Device{
+ ._name = devnode.path().asString(),
+ ._maj_nr = devnode.devMajor(),
+ ._min_nr = devnode.devMinor() } );
+ DBG << "Found (" << reason << "): " << dev->_name << " " << dev->_maj_nr << ":" << dev->_min_nr << std::endl;
+
+ sysDevs.push_back(dev);
+ return ( --sysDevs.end() );
+ }
+ return sysDevs.end();
+ };
+
+
+ auto &sysDevs = knownDevices();
+
+ // detect devices available in the system if we did not do it before:
+ if ( !_devicesDetected ) {
+ _devicesDetected = true;
+
+#ifdef HAVE_UDEV
+ // http://www.kernel.org/pub/linux/utils/kernel/hotplug/libudev/index.html
+ zypp::AutoDispose<struct udev *> udev( ::udev_new(), ::udev_unref );
+ if ( ! udev ) {
+ ERR << "Can't create udev context." << std::endl;
+ } else {
+ zypp::AutoDispose<struct udev_enumerate *> enumerate( ::udev_enumerate_new(udev), ::udev_enumerate_unref );
+ if ( ! enumerate ) {
+ ERR << "Can't create udev list entry." << std::endl;
+ } else {
+ ::udev_enumerate_add_match_subsystem( enumerate, "block" );
+ ::udev_enumerate_add_match_property( enumerate, "ID_CDROM", "1" );
+ ::udev_enumerate_scan_devices( enumerate );
+
+ struct udev_list_entry * entry = 0;
+ udev_list_entry_foreach( entry, ::udev_enumerate_get_list_entry( enumerate ) )
+ {
+
+ zypp::AutoDispose<struct udev_device *> device( ::udev_device_new_from_syspath( ::udev_enumerate_get_udev( enumerate ),
+ ::udev_list_entry_get_name( entry ) ),
+ ::udev_device_unref );
+ if ( ! device )
+ {
+ ERR << "Can't create udev device." << std::endl;
+ continue;
+ }
+
+ const char * devnodePtr( ::udev_device_get_devnode( device ) );
+ if ( ! devnodePtr ) {
+ ERR << "Got NULL devicenode." << std::endl;
+ continue;
+ }
+
+ // In case we need it someday:
+ //const char * mountpath = ::udev_device_get_property_value( device, "FSTAB_DIR" );
+ auto dev = registerDevice( zypp::PathInfo ( devnodePtr ), "udev" );
+ if ( dev != sysDevs.end() ) {
+ if ( ::udev_device_get_property_value( device, "ID_CDROM_DVD" ) ) {
+ (*dev)->_properties["DVD"] = true;
+ } else {
+ (*dev)->_properties["DVD"] = false;
+ }
+ }
+ }
+ }
+ }
+#endif
+ if ( sysDevs.empty() )
+ {
+ WAR << "CD/DVD drive detection with UDEV failed! Guessing..." << std::endl;
+ auto dev = registerDevice( zypp::PathInfo ( "/dev/dvd" ), "GUESS" );
+ if ( dev != sysDevs.end() && (*dev)->_properties.count("DVD") == 0 )
+ (*dev)->_properties["DVD"] = true;
+ dev = registerDevice( zypp::PathInfo ( "/dev/cdrom" ), "GUESS" );
+ if ( dev != sysDevs.end() && (*dev)->_properties.count("DVD") == 0 )
+ (*dev)->_properties["DVD"] = false;
+ }
+ }
+}
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+#ifndef ZYPP_NG_TOOLS_DISCPROVIDER_H_INCLUDED
+#define ZYPP_NG_TOOLS_DISCPROVIDER_H_INCLUDED
+
+#include <zypp-media/ng/worker/DeviceDriver>
+#include <zypp-core/zyppng/base/Signals>
+#include <any>
+#include <unordered_map>
+
+class DiscProvider : public zyppng::worker::DeviceDriver
+{
+public:
+ DiscProvider();
+ ~DiscProvider();
+
+ // DeviceDriver interface
+ zyppng::worker::AttachResult mountDevice ( const uint32_t id, const zypp::Url &attachUrl, const std::string &attachId, const std::string &label, const zyppng::HeaderValueMap &extras ) override;
+ void detectDevices() override;
+
+private:
+ bool _devicesDetected = false; //< We delay device detection to the first attach request, to avoid doing it without needing it
+};
+
+#endif
--- /dev/null
+
+#include "discprovider.h"
+
+#include <zypp-core/zyppng/base/private/linuxhelpers_p.h>
+#include <zypp-media/ng/worker/MountingWorker>
+
+#include <csignal>
+
+
+int main( int , char *[] )
+{
+ // lets ignore those, SIGPIPE is better handled via the EPIPE error, and we do not want anyone
+ // to CTRL+C us
+ zyppng::blockSignalsForCurrentThread( { SIGPIPE, SIGINT } );
+
+ auto driver = std::make_shared<DiscProvider>();
+ auto provider = std::make_shared<zyppng::worker::MountingWorker>( "zypp-media-disc", driver );
+ driver->setProvider( provider );
+
+ auto res = provider->run (STDIN_FILENO, STDOUT_FILENO);
+ provider->immediateShutdown();
+ if ( res )
+ return 0;
+
+ //@TODO print error
+ return 1;
+}
--- /dev/null
+PROJECT( zypp-media-disk C CXX )
+
+set (CMAKE_CXX_STANDARD 17)
+SET (CMAKE_CXX_EXTENSIONS OFF)
+
+FIND_PACKAGE(Protobuf REQUIRED)
+
+SET( SOURCES
+ main.cc
+ diskprovider.cc
+ diskprovider.h
+)
+
+add_executable( ${PROJECT_NAME} ${SOURCES} )
+target_link_libraries( ${PROJECT_NAME} zypp-media )
+target_link_libraries( ${PROJECT_NAME} zypp-core )
+target_link_libraries( ${PROJECT_NAME} zypp-protobuf )
+target_link_libraries( ${PROJECT_NAME} ${PROTOBUF_LITE_LIBRARIES} )
+
+IF ( INSTALL_NG_BINARIES )
+ INSTALL( TARGETS ${PROJECT_NAME} DESTINATION "${ZYPP_LIBEXEC_INSTALL_DIR}/workers" )
+ENDIF()
+
+set_target_properties( ${PROJECT_NAME}
+ PROPERTIES
+ ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+ LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+ RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+)
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+#include "diskprovider.h"
+#include <zypp-media/ng/private/providedbg_p.h>
+#include <zypp-media/ng/MediaVerifier>
+#include <zypp-media/Mount>
+
+#include <zypp-core/fs/TmpPath.h>
+#include <zypp-core/fs/PathInfo.h>
+
+#include <iostream>
+#include <fstream>
+
+#undef ZYPP_BASE_LOGGER_LOGGROUP
+#define ZYPP_BASE_LOGGER_LOGGROUP "DiskProvider"
+
+
+/*!
+ * Check if specified device file name is
+ * a disk volume device or throw an error.
+ */
+bool verifyIfDiskVolume( const zypp::Pathname &dev_name )
+{
+ if( dev_name.empty() ||
+ dev_name.asString().compare(0, sizeof("/dev/")-1, "/dev/"))
+ {
+ ERR << "Specified device name " << dev_name
+ << " is not allowed" << std::endl;
+ return false;
+ }
+
+ zypp::PathInfo dev_info(dev_name);
+ if( !dev_info.isBlk())
+ {
+ ERR << "Specified device name " << dev_name
+ << " is not a block device" << std::endl;
+ return false;
+ }
+
+ // check if a volume using /dev/disk/by-uuid links first
+ {
+ zypp::Pathname dpath("/dev/disk/by-uuid");
+ std::list<zypp::Pathname> dlist;
+ if( zypp::filesystem::readdir(dlist, dpath) == 0)
+ {
+ std::list<zypp::Pathname>::const_iterator it;
+ for(it = dlist.begin(); it != dlist.end(); ++it)
+ {
+ zypp::PathInfo vol_info(*it);
+ if( vol_info.isBlk() && vol_info.devMajor() == dev_info.devMajor() &&
+ vol_info.devMinor() == dev_info.devMinor())
+ {
+ DBG << "Specified device name " << dev_name
+ << " is a volume (disk/by-uuid link "
+ << vol_info.path() << ")"
+ << std::endl;
+ return true;
+ }
+ }
+ }
+ }
+
+ // check if a volume using /dev/disk/by-label links
+ // (e.g. vbd mapped volumes in a XEN vm)
+ {
+ zypp::Pathname dpath("/dev/disk/by-label");
+ std::list<zypp::Pathname> dlist;
+ if( zypp::filesystem::readdir(dlist, dpath) == 0)
+ {
+ std::list<zypp::Pathname>::const_iterator it;
+ for(it = dlist.begin(); it != dlist.end(); ++it)
+ {
+ zypp::PathInfo vol_info(*it);
+ if( vol_info.isBlk() && vol_info.devMajor() == dev_info.devMajor() &&
+ vol_info.devMinor() == dev_info.devMinor())
+ {
+ DBG << "Specified device name " << dev_name
+ << " is a volume (disk/by-label link "
+ << vol_info.path() << ")"
+ << std::endl;
+ return true;
+ }
+ }
+ }
+ }
+
+ // check if a filesystem volume using the 'blkid' tool
+ // (there is no /dev/disk link for some of them)
+ zypp::ExternalProgram::Arguments args;
+ args.push_back( "blkid" );
+ args.push_back( "-p" );
+ args.push_back( dev_name.asString() );
+
+ zypp::ExternalProgram cmd( args, zypp::ExternalProgram::Stderr_To_Stdout );
+ cmd >> DBG;
+ if ( cmd.close() != 0 )
+ {
+ ERR << cmd.execError()
+ << "\nSpecified device name " << dev_name
+ << " is not a usable disk volume"
+ << std::endl;
+ return false;
+ }
+ return true;
+}
+
+
+
+DiskProvider::DiskProvider()
+ : DeviceDriver( zyppng::worker::WorkerCaps::SimpleMount )
+{ }
+
+DiskProvider::~DiskProvider()
+{ }
+
+zyppng::worker::AttachResult DiskProvider::mountDevice ( const uint32_t id, const zypp::Url &attachUrl, const std::string &attachId, const std::string &label, const zyppng::HeaderValueMap &extras )
+{
+ try
+ {
+ if ( !attachUrl.getHost().empty() ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , "Host must be empty in dir:// and file:// URLs"
+ , false
+ );
+ }
+
+ const std::string &device = zypp::Pathname(attachUrl.getQueryParam("device")).asString();
+ if ( device.empty() ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , "Media url does not contain a device specification"
+ , false
+ );
+ }
+
+ std::string filesystem = attachUrl.getQueryParam("filesystem");
+ if( filesystem.empty() )
+ filesystem="auto";
+
+ zypp::PathInfo dev_info( device );
+ if(!dev_info.isBlk()) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , "Media url does not specify a valid block device"
+ , false
+ );
+ }
+
+ DBG << "Verifying " << device << " ..." << std::endl;
+ if( !verifyIfDiskVolume( device)) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , "Could not verify URL points to a disk volume!"
+ , false
+ );
+ }
+
+ // disks can have a attach root ( path relative to the device root )
+ const auto relAttachRoot = zypp::Pathname( attachUrl.getPathName() );
+
+ // set up the verifier
+ zyppng::MediaDataVerifierRef verifier;
+ if ( extras.value( zyppng::AttachMsgFields::VerifyType ).valid() ) {
+ verifier = zyppng::MediaDataVerifier::createVerifier( extras.value(zyppng::AttachMsgFields::VerifyType).asString() );
+ if ( !verifier ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , "Invalid verifier type"
+ , false
+ );
+ }
+
+ if ( !verifier->load( extras.value(zyppng::AttachMsgFields::VerifyData).asString() ) ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , "Failed to create verifier from file"
+ , false
+ );
+ }
+ }
+ const auto &devs = knownDevices();
+
+ // first check if we have that device already
+ auto i = std::find_if( devs.begin(), devs.end(), [&]( const auto &d ) {
+ return d->_maj_nr == dev_info.devMajor()
+ && d->_min_nr == dev_info.devMinor();
+ });
+ if ( i != devs.end() ) {
+ auto res = isDesiredMedium( attachUrl, (*i)->_mountPoint / relAttachRoot, verifier, extras.value( zyppng::AttachMsgFields::MediaNr, 1 ).asInt() );
+ if ( !res ) {
+ try {
+ std::rethrow_exception( res.error() );
+ } catch( const zypp::Exception& e ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MediumNotDesired
+ , false
+ , e
+ );
+ } catch ( ... ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MediumNotDesired
+ , "Checking the medium failed with an uknown error"
+ , false
+ );
+ }
+ } else {
+ attachedMedia().insert( std::make_pair( attachId, zyppng::worker::AttachedMedia{ *i, relAttachRoot } ) );
+ return zyppng::worker::AttachResult::success();
+ }
+ }
+
+ // we did not find a existing mount, well lets make a new one ...
+ std::optional<zypp::Pathname> bindSource;
+ auto devPtr = std::make_shared<zyppng::worker::Device>( zyppng::worker::Device{
+ ._name = dev_info.path().asString(),
+ ._maj_nr = dev_info.devMajor(),
+ ._min_nr = dev_info.devMinor(),
+ ._mountPoint = {},
+ ._ephemeral = true, // forget about the device after we are finished with it
+ ._properties = {}
+ });
+
+ // since the kernel will not let us remount a disc ro if it was already mounted in the system as rw we need to
+ // go over the existing mounts and revert to a bind mount if we find that the device has a mountpoint already (#163486).
+ zypp::media::MountEntries entries( zypp::media::Mount::getEntries() );
+ for( auto e = entries.cbegin(); e != entries.cend(); ++e)
+ {
+ bool is_device = false;
+ std::string dev_path(zypp::Pathname(e->src).asString());
+ zypp::PathInfo dev_info;
+
+ if( dev_path.compare(0, sizeof("/dev/")-1, "/dev/") == 0 &&
+ dev_info(e->src) && dev_info.isBlk()) {
+ is_device = true;
+ }
+
+ if( is_device && devPtr->_maj_nr == dev_info.devMajor() &&
+ devPtr->_min_nr == dev_info.devMinor())
+ {
+ DBG << "Device " << devPtr->_name << " is already mounted, using bind mount!" << std::endl;
+ bindSource = e->dir;
+ break;
+ }
+ }
+
+ zypp::media::Mount mount;
+ zypp::Pathname newAp;
+ try {
+ newAp = createAttachPoint( attachRoot() );
+ if ( newAp.empty() ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , "Failed to create mount directory."
+ , false
+ );
+ }
+
+ std::string options = attachUrl.getQueryParam("mountoptions");
+ if(options.empty()) {
+ options = "ro";
+ }
+
+ if( bindSource ) {
+ options += ",bind";
+ mount.mount( bindSource->asString(), newAp.asString(), "none", options);
+ } else {
+ mount.mount( dev_info.path().asString(), newAp.asString(), filesystem, options);
+ }
+
+ // wait for /etc/mtab update ...
+ // (shouldn't be needed)
+ int limit = 3;
+ bool mountsucceeded = false;
+
+ const auto &checkAttachedDisk = [&]( const zypp::media::MountEntry &e ) {
+ if ( bindSource) {
+ if ( *bindSource == e.src ) {
+ DBG << "Found bound media "
+ << devPtr->_name
+ << " in the mount table as " << e.src << std::endl;
+ return true;
+ }
+ }
+ return DeviceDriver::devicePredicate( devPtr->_maj_nr, devPtr->_min_nr ) ( e );
+ };
+
+ // mount command came back OK, lets wait for mtab to update
+ while( !(mountsucceeded=checkAttached( newAp, checkAttachedDisk )) && --limit) {
+ MIL << "Mount did not appear yet, sleeping for 1s" << std::endl;
+ sleep(1);
+ }
+
+ // mount didn't work after all, bail out
+ if ( !mountsucceeded ) {
+ try {
+ mount.umount( newAp.asString() );
+ } catch (const zypp::media::MediaException & excpt_r) {
+ ZYPP_CAUGHT(excpt_r);
+ }
+ ZYPP_THROW( zypp::media::MediaMountException(
+ "Unable to verify that the media was mounted",
+ devPtr->_name, newAp.asString()
+ ));
+ }
+
+ // if we reach this place, mount worked -> YAY, lets see if that is the desired medium!
+ auto isDesired = isDesiredMedium( attachUrl, newAp / relAttachRoot, verifier, extras.value( zyppng::AttachMsgFields::MediaNr, 1 ).asInt() );
+ if ( !isDesired ) {
+ try {
+ mount.umount( newAp.asString() );
+ } catch (const zypp::media::MediaException & excpt_r) {
+ ZYPP_CAUGHT(excpt_r);
+ }
+ ZYPP_THROW( zypp::media::MediaNotDesiredException( attachUrl ) );
+ }
+ }
+ catch ( const zypp::Exception &e ) {
+ removeAttachPoint(newAp);
+ ZYPP_CAUGHT(e);
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , false
+ , e
+ );
+ }
+ catch ( ... ) {
+ removeAttachPoint(newAp);
+ std::rethrow_exception( std::current_exception() );
+ }
+
+ // mount worked ! YAY
+ devPtr->_mountPoint = newAp;
+ knownDevices().push_back( devPtr );
+ attachedMedia().insert( std::make_pair( attachId, zyppng::worker::AttachedMedia{ devPtr, relAttachRoot } ) );
+ return zyppng::worker::AttachResult::success();
+
+ } catch ( const zypp::Exception &e ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::BadRequest
+ , false
+ , e
+ );
+ } catch ( const std::exception &e ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::BadRequest
+ , e.what()
+ , false
+ );
+ } catch ( ... ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::BadRequest
+ , "Unknown exception"
+ , false
+ );
+ }
+}
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+#ifndef ZYPP_NG_TOOLS_DISKPROVIDER_H_INCLUDED
+#define ZYPP_NG_TOOLS_DISKPROVIDER_H_INCLUDED
+
+#include <zypp-media/ng/worker/DeviceDriver>
+
+class DiskProvider : public zyppng::worker::DeviceDriver
+{
+ public:
+ DiskProvider();
+ ~DiskProvider();
+
+ // DeviceDriver interface
+ zyppng::worker::AttachResult mountDevice ( const uint32_t id, const zypp::Url &attachUrl, const std::string &attachId, const std::string &label, const zyppng::HeaderValueMap &extras ) override;
+
+};
+
+#endif
--- /dev/null
+#include "diskprovider.h"
+
+#include <csignal>
+#include <zypp-core/zyppng/base/private/linuxhelpers_p.h>
+#include <zypp-media/ng/worker/MountingWorker>
+
+int main( int , char *[] )
+{
+ // lets ignore those, SIGPIPE is better handled via the EPIPE error, and we do not want anyone
+ // to CTRL+C us
+ zyppng::blockSignalsForCurrentThread( { SIGPIPE, SIGINT } );
+
+ auto driver = std::make_shared<DiskProvider>();
+ auto worker = std::make_shared<zyppng::worker::MountingWorker>( "zypp-media-disk", driver );
+ driver->setProvider( worker );
+
+ auto res = worker->run (STDIN_FILENO, STDOUT_FILENO);
+
+ MIL << "DISK Worker shutting down" << std::endl;
+
+ if ( res )
+ return 0;
+
+ //@TODO print error
+ return 1;
+}
--- /dev/null
+PROJECT( zypp-media-http C CXX )
+
+set (CMAKE_CXX_STANDARD 17)
+SET (CMAKE_CXX_EXTENSIONS OFF)
+
+FIND_PACKAGE(Protobuf REQUIRED)
+
+SET( SOURCES
+ main.cc
+ networkprovider.cc
+ networkprovider.h
+)
+
+add_executable( ${PROJECT_NAME} ${SOURCES} )
+target_link_libraries( ${PROJECT_NAME} zypp-curl )
+target_link_libraries( ${PROJECT_NAME} zypp-media )
+target_link_libraries( ${PROJECT_NAME} zypp-core )
+target_link_libraries( ${PROJECT_NAME} zypp-protobuf )
+target_link_libraries( ${PROJECT_NAME} ${PROTOBUF_LITE_LIBRARIES} )
+
+IF ( INSTALL_NG_BINARIES )
+ INSTALL( TARGETS ${PROJECT_NAME} DESTINATION "${ZYPP_LIBEXEC_INSTALL_DIR}/workers" )
+ENDIF()
+
+set_target_properties( ${PROJECT_NAME}
+ PROPERTIES
+ ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+ LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+ RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+)
--- /dev/null
+#include "networkprovider.h"
+
+#include <csignal>
+#include <zypp-core/zyppng/base/private/linuxhelpers_p.h>
+
+int main( int , char *[] )
+{
+ // lets ignore those, SIGPIPE is better handled via the EPIPE error, and we do not want anyone
+ // to CTRL+C us
+ zyppng::blockSignalsForCurrentThread( { SIGPIPE, SIGINT } );
+
+ auto provider = std::make_shared<NetworkProvider>("zypp-media-http");
+ auto res = provider->run (STDIN_FILENO, STDOUT_FILENO);
+ provider->immediateShutdown();
+ if ( res )
+ return 0;
+
+ //@TODO print error
+ return 1;
+ //@TODO @WARNING @BUG Initialize the requestmanager with user agent and anon headers \sa anonymousIdHeader, distributionFlavorHeader, agentString
+}
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+#include "networkprovider.h"
+
+#include <zypp-curl/ng/network/Downloader>
+#include <zypp-curl/ng/network/NetworkRequestDispatcher>
+#include <zypp-curl/ng/network/DownloadSpec>
+#include <zypp-curl/parser/MetaLinkParser>
+#include <zypp-core/fs/PathInfo.h>
+#include <zypp-core/CheckSum.h>
+#include <zypp-media/ng/private/providedbg_p.h>
+
+
+#undef ZYPP_BASE_LOGGER_LOGGROUP
+#define ZYPP_BASE_LOGGER_LOGGROUP "NetworkProvider"
+
+
+NetworkProvideItem::NetworkProvideItem(NetworkProvider &parent, zyppng::ProvideMessage &&spec)
+ : zyppng::worker::ProvideWorkerItem( std::move(spec) )
+ , _parent(parent)
+{}
+
+NetworkProvideItem::~NetworkProvideItem()
+{
+ clearConnections();
+ if ( _dl ) {
+ _dl->cancel();
+ }
+}
+
+void NetworkProvideItem::startDownload( std::shared_ptr<zyppng::Download> &&dl )
+{
+ if ( _state == ProvideWorkerItem::Running || _dl )
+ throw std::runtime_error("Can not start a already running Download");
+
+ MIL_PRV << "Starting download of : " << dl->spec().url() << "with target path: " << dl->spec().targetPath() << std::endl;
+
+ _connections.clear ();
+ // we set the state to running immediately but only send the message to controller once the item actually started
+ _state = ProvideWorkerItem::Running;
+ _dl = std::move(dl);
+ _connections.emplace_back( connect( *_dl, &zyppng::Download::sigStarted, *this, &NetworkProvideItem::onStarted ) );
+ _connections.emplace_back( connect( *_dl, &zyppng::Download::sigFinished, *this, &NetworkProvideItem::onFinished ) );
+ _connections.emplace_back( connect( *_dl, &zyppng::Download::sigAuthRequired, *this, &NetworkProvideItem::onAuthRequired ) );
+
+ _dl->setStopOnMetalink( true );
+ _dl->start();
+}
+
+void NetworkProvideItem::cancelDownload()
+{
+ _state = zyppng::worker::ProvideWorkerItem::Finished;
+ if ( _dl ) {
+ _dl->cancel();
+ clearConnections();
+ _dl.reset();
+ }
+}
+
+void NetworkProvideItem::clearConnections()
+{
+ for ( auto c : _connections)
+ c.disconnect();
+ _connections.clear();
+}
+
+void NetworkProvideItem::onStarted( zyppng::Download & )
+{
+ _parent.itemStarted( shared_this<NetworkProvideItem>() );
+}
+
+void NetworkProvideItem::onFinished( zyppng::Download & )
+{
+ _parent.itemFinished( shared_this<NetworkProvideItem>() );
+}
+
+void NetworkProvideItem::onAuthRequired( zyppng::Download &, zyppng::NetworkAuthData &auth, const std::string &availAuth )
+{
+ _parent.itemAuthRequired( shared_this<NetworkProvideItem>(), auth, availAuth );
+}
+
+
+NetworkProvider::NetworkProvider( std::string_view workerName )
+ : zyppng::worker::ProvideWorker( workerName )
+ , _dlManager( std::make_shared<zyppng::Downloader>() )
+{
+ // we only want to hear about new provides
+ setProvNotificationMode( ProvideWorker::ONLY_NEW_PROVIDES );
+}
+
+zyppng::expected<zyppng::worker::WorkerCaps> NetworkProvider::initialize( const zyppng::worker::Configuration &conf )
+{
+ const auto &values = conf.values();
+ const auto iEnd = values.end();
+ if ( const auto &i = values.find( std::string(zyppng::AGENT_STRING_CONF) ); i != iEnd ) {
+ const auto &val = i->second;
+ MIL << "Setting agent string to: " << val << std::endl;
+ _dlManager->requestDispatcher()->setAgentString( val );
+ }
+ if ( const auto &i = values.find( std::string(zyppng::DISTRO_FLAV_CONF) ); i != iEnd ) {
+ const auto &val = i->second;
+ MIL << "Setting distro flavor header to: " << val << std::endl;
+ _dlManager->requestDispatcher()->setHostSpecificHeader("download.opensuse.org", "X-ZYpp-DistributionFlavor", val );
+ }
+ if ( const auto &i = values.find( std::string(zyppng::ANON_ID_CONF) ); i != iEnd ) {
+ const auto &val = i->second;
+ MIL << "Got anonymous ID setting from controller" << std::endl;
+ _dlManager->requestDispatcher()->setHostSpecificHeader("download.opensuse.org", "X-ZYpp-AnonymousId", val );
+ }
+ if ( const auto &i = values.find( std::string(zyppng::ATTACH_POINT) ); i != iEnd ) {
+ const auto &val = i->second;
+ MIL << "Got attachpoint from controller: " << val << std::endl;
+ _attachPoint = val;
+ } else {
+ return zyppng::expected<zyppng::worker::WorkerCaps>::error(ZYPP_EXCPT_PTR( zypp::Exception("Attach point required to work.") ));
+ }
+
+ zyppng::worker::WorkerCaps caps;
+ caps.set_worker_type ( zyppng::worker::WorkerCaps::Downloading );
+ caps.set_cfg_flags(
+ zyppng::worker::WorkerCaps::Flags (
+ zyppng::worker::WorkerCaps::Pipeline
+ | zyppng::worker::WorkerCaps::ZyppLogFormat
+ )
+ );
+
+ return zyppng::expected<zyppng::worker::WorkerCaps>::success(caps);
+}
+
+void NetworkProvider::provide()
+{
+ auto &queue = requestQueue();
+
+ if ( !queue.size() )
+ return;
+
+ const auto now = std::chrono::steady_clock::now();
+ auto &io = controlIO();
+
+ while ( io.readFdOpen() ) {
+ // find next pending item
+ auto i = std::find_if( queue.begin(), queue.end(), [&now]( const zyppng::worker::ProvideWorkerItemRef &req ){
+ NetworkProvideItemRef nReq = std::static_pointer_cast<NetworkProvideItem>(req);
+ return nReq && nReq->_state == zyppng::worker::ProvideWorkerItem::Pending && nReq->_scheduleAfter < now;
+ });
+
+ // none left, bail out
+ if ( i == queue.end () )
+ break;
+
+ auto req = std::static_pointer_cast<NetworkProvideItem>( *i );
+ if ( req->_state != zyppng::worker::ProvideWorkerItem::Pending )
+ return;
+
+ MIL_PRV << "About to schedule: " << req->_spec.code() << " with ID: "<< req->_spec.requestId() << std::endl;
+
+ // here we only receive request codes, we only support Provide messages, all others are rejected
+ // Cancel is never to be received here
+ if ( req->_spec.code() != zyppng::ProvideMessage::Code::Provide ) {
+ req->_state = zyppng::worker::ProvideWorkerItem::Finished;
+ provideFailed( req->_spec.requestId()
+ , zyppng::ProvideMessage::Code::BadRequest
+ , "Request type not implemented"
+ , false
+ , {} );
+
+ queue.erase(i);
+ continue;
+ }
+
+ zypp::Url url;
+ const auto &urlVal = req->_spec.value( zyppng::ProvideMsgFields::Url );
+ try {
+ url = zypp::Url( urlVal.asString() );
+ } catch ( const zypp::Exception &excp ) {
+ ZYPP_CAUGHT(excp);
+
+ std::string err = zypp::str::Str() << "Invalid URL in request: " << urlVal.asString();
+ ERR << err << std::endl;
+
+ req->_state = zyppng::worker::ProvideWorkerItem::Finished;
+ provideFailed( req->_spec.requestId()
+ , zyppng::ProvideMessage::Code::BadRequest
+ , err
+ , false
+ , {} );
+
+ queue.erase(i);
+ continue;
+ }
+
+ const auto &fPath = zypp::Pathname( url.getPathName() );
+
+ /*
+ * We download into the staging directory first, using the md5sum of the full path as filename.
+ * Once the download is finished its transferred into the cache directory. This way we can clearly
+ * distinguish between cancelled downloads and downloads we have completely finished.
+ */
+ zypp::Pathname localPath = zypp::Pathname( _attachPoint ) / "cache" / fPath;
+ zypp::Pathname stagingPath = ( zypp::Pathname( _attachPoint ) / "staging" / zypp::CheckSum::md5FromString( fPath.asString() ).asString() ).extend(".part");
+
+ MIL_PRV << "Providing " << url << " to local: " << localPath << " staging: " << stagingPath << std::endl;
+
+ // Cache hit?
+ zypp::PathInfo cacheFile(localPath);
+ if ( cacheFile.isExist () ) {
+
+ // could be a directory
+ if ( !cacheFile.isFile() ) {
+ MIL_PRV << "Path " << url << " exists in Cache but its not a file!" << std::endl;
+ req->_state = zyppng::worker::ProvideWorkerItem::Finished;
+ provideFailed( req->_spec.requestId()
+ , zyppng::ProvideMessage::Code::NotAFile
+ , "Not a file"
+ , false
+ , {} );
+
+ queue.erase(i);
+ continue;
+ }
+
+ MIL_PRV << "Providing " << url << "Cache HIT" << std::endl;
+ req->_state = zyppng::worker::ProvideWorkerItem::Finished;
+ provideSuccess ( req->_spec.requestId(), true, localPath );
+ queue.erase(i);
+ continue;
+ }
+
+ if ( zypp::PathInfo(stagingPath).isExist () ) {
+ // check if there is another request running with this staging file name
+ auto runningIt = std::find_if( queue.begin (), queue.end(), [&]( const zyppng::worker::ProvideWorkerItemRef &queueItem ){
+ if ( req == queueItem || queueItem->_state != zyppng::worker::ProvideWorkerItem::Running )
+ return false;
+ return ( static_cast<NetworkProvideItem*>( queueItem.get() )->_stagingFileName == stagingPath );
+ });
+
+ // we schedule that later again which should result in a immediate cache hit
+ if ( runningIt != queue.end() ) {
+ MIL << "Found a existing request that results in the same staging file, postponing the request for later" << std::endl;
+ req->_scheduleAfter = now;
+ continue;
+ }
+
+ MIL<< "Removing broken staging file" << stagingPath << std::endl;
+
+ // here this is most likely a broken download
+ zypp::filesystem::unlink( stagingPath );
+ }
+
+ auto errCode = zypp::filesystem::assert_dir( localPath.dirname() );
+ if( errCode ) {
+ std::string err = zypp::str::Str() << "assert_dir " << localPath.dirname() << " failed";
+ DBG << err << std::endl;
+
+ req->_state = zyppng::worker::ProvideWorkerItem::Finished;
+ provideFailed( req->_spec.requestId()
+ , zyppng::ProvideMessage::Code::InternalError
+ , err
+ , false
+ , {} );
+ queue.erase(i);
+ continue;
+ }
+
+ errCode = zypp::filesystem::assert_dir( stagingPath.dirname() );
+ if( errCode ) {
+ std::string err = zypp::str::Str() << "assert_dir " << stagingPath.dirname() << " failed";
+ DBG << err << std::endl;
+
+ req->_state = zyppng::worker::ProvideWorkerItem::Finished;
+ provideFailed( req->_spec.requestId()
+ , zyppng::ProvideMessage::Code::InternalError
+ , err
+ , false
+ , {} );
+ queue.erase(i);
+ continue;
+ }
+
+ bool doMetalink = true;
+ const auto &metalinkVal = req->_spec.value( zyppng::NETWORK_METALINK_ENABLED );
+ if ( metalinkVal.valid() && metalinkVal.isBool () ) {
+ doMetalink = metalinkVal.asBool();
+ MIL << "Explicity setting metalink " << ( doMetalink ? "enabled" : "disabled" ) << std::endl;
+ }
+
+ req->_targetFileName = localPath;
+ req->_stagingFileName = stagingPath;
+
+ const auto &expFilesize = req->_spec.value( zyppng::ProvideMsgFields::ExpectedFilesize );
+ const auto &checkExistsOnly = req->_spec.value( zyppng::ProvideMsgFields::CheckExistOnly );
+ const auto &deltaFile = req->_spec.value( zyppng::ProvideMsgFields::DeltaFile );
+
+ zyppng::DownloadSpec spec(
+ url
+ , stagingPath
+ , expFilesize.valid() ? zypp::ByteCount( expFilesize.asInt64() ) : zypp::ByteCount()
+ );
+ spec
+ .setCheckExistsOnly( checkExistsOnly.valid() ? checkExistsOnly.asBool() : false )
+ .setDeltaFile ( deltaFile.valid() ? deltaFile.asString() : zypp::Pathname() )
+ .setMetalinkEnabled ( doMetalink );
+
+ req->startDownload( _dlManager->downloadFile ( spec ) );
+ }
+}
+
+void NetworkProvider::cancel( const std::deque<zyppng::worker::ProvideWorkerItemRef>::iterator &i )
+{
+ auto &queue = requestQueue ();
+ if ( i == queue.end() ) {
+ ERR << "Unknown request ID, ignoring." << std::endl;
+ return;
+ }
+
+ const auto &req = std::static_pointer_cast<NetworkProvideItem>(*i);
+ req->cancelDownload();
+ queue.erase(i);
+}
+
+void NetworkProvider::immediateShutdown()
+{
+ for ( const auto &pItem : requestQueue () ) {
+ auto ref = std::static_pointer_cast<NetworkProvideItem>( pItem );
+ ref->cancelDownload();
+ }
+}
+
+zyppng::worker::ProvideWorkerItemRef NetworkProvider::makeItem(zyppng::ProvideMessage &&spec )
+{
+ return std::make_shared<NetworkProvideItem>( *this, std::move(spec) );
+}
+
+void NetworkProvider::itemStarted( NetworkProvideItemRef item )
+{
+ provideStart( item->_spec.requestId(), item->_dl->spec().url().asCompleteString(), item->_targetFileName.asString(), item->_stagingFileName.asString() );
+}
+
+void NetworkProvider::itemFinished( NetworkProvideItemRef item )
+{
+ auto &queue = requestQueue ();
+ auto i = std::find( queue.begin(), queue.end(), item );
+ if ( i == queue.end() ) {
+ ERR << "Unknown request finished. This is a BUG!" << std::endl;
+ return;
+ }
+
+ item->_state = NetworkProvideItem::Finished;
+ if ( !item->_dl->hasError() ) {
+
+ if ( item->_dl->stoppedOnMetalink () ) {
+ // need to read the metalink info and send back a redirect
+ try {
+ zypp::media::MetaLinkParser pars;
+ pars.parse( item->_stagingFileName );
+
+ std::vector<zypp::Url> urls;
+
+ for ( const auto &mirr : pars.getMirrors () ) {
+ try {
+ zypp::Url url( mirr.url.asCompleteString () );
+ urls.push_back(url);
+
+ if ( urls.size() == 10 )
+ break;
+
+ } catch ( const zypp::Exception &e ) {
+ ZYPP_CAUGHT (e);
+ }
+ }
+
+ // @TODO , restart the download without metalink?
+ if ( urls.size() == 0 )
+ throw zypp::Exception("No usable mirrors in Mirrorlink file");
+
+ if ( !messageStream()->sendMessage( zyppng::ProvideMessage::createMetalinkRedir( item->_spec.requestId(), urls ).impl() ) ) {
+ ERR << "Failed to send ProvideSuccess message" << std::endl;
+ }
+
+ } catch ( const zypp::Exception &exp ) {
+ provideFailed( item->_spec.requestId(), zyppng::ProvideMessage::Code::InternalError, exp.asUserString (), false );
+ }
+
+ zypp::filesystem::unlink( item->_stagingFileName );
+
+ } else {
+ const auto errCode = zypp::filesystem::rename( item->_stagingFileName, item->_targetFileName );
+ if( errCode ) {
+
+ zypp::filesystem::unlink( item->_stagingFileName );
+
+ std::string err = zypp::str::Str() << "Renaming " << item->_stagingFileName << " to " << item->_targetFileName << " failed!";
+ DBG << err << std::endl;
+
+ provideFailed( item->_spec.requestId()
+ , zyppng::ProvideMessage::Code::InternalError
+ , err
+ , false
+ , {} );
+
+ } else {
+ provideSuccess( item->_spec.requestId(), false, item->_targetFileName );
+ }
+ }
+ } else {
+ zyppng::HeaderValueMap extra;
+ bool isTransient = false;
+ auto errCode = zyppng::ProvideMessage::Code::InternalError;
+
+ const auto &error = item->_dl->lastRequestError();
+ switch( error.type() ) {
+ case zyppng::NetworkRequestError::NoError: { throw std::runtime_error("DownloadError info broken"); break;}
+ case zyppng::NetworkRequestError::InternalError: { errCode = zyppng::ProvideMessage::Code::InternalError; break;}
+ case zyppng::NetworkRequestError::Cancelled: { errCode = zyppng::ProvideMessage::Code::Cancelled; break;}
+ case zyppng::NetworkRequestError::PeerCertificateInvalid: { errCode = zyppng::ProvideMessage::Code::PeerCertificateInvalid; break;}
+ case zyppng::NetworkRequestError::ConnectionFailed: { errCode = zyppng::ProvideMessage::Code::ConnectionFailed; break;}
+ case zyppng::NetworkRequestError::ExceededMaxLen: { errCode = zyppng::ProvideMessage::Code::ExpectedSizeExceeded; break;}
+ case zyppng::NetworkRequestError::InvalidChecksum: { errCode = zyppng::ProvideMessage::Code::InvalidChecksum; break;}
+ case zyppng::NetworkRequestError::UnsupportedProtocol: { errCode = zyppng::ProvideMessage::Code::BadRequest; break; }
+ case zyppng::NetworkRequestError::MalformedURL: { errCode = zyppng::ProvideMessage::Code::BadRequest; break; }
+ case zyppng::NetworkRequestError::TemporaryProblem: { errCode = zyppng::ProvideMessage::Code::InternalError; isTransient = true; break;}
+ case zyppng::NetworkRequestError::Timeout: { errCode = zyppng::ProvideMessage::Code::Timeout; break;}
+ case zyppng::NetworkRequestError::Forbidden: { errCode = zyppng::ProvideMessage::Code::Forbidden; break;}
+ case zyppng::NetworkRequestError::NotFound: { errCode = zyppng::ProvideMessage::Code::NotFound; break;}
+ case zyppng::NetworkRequestError::Unauthorized: {
+ errCode = zyppng::ProvideMessage::Code::Unauthorized;
+ const auto &authHint = error.extraInfoValue("authHint", std::string());
+ if ( authHint.size () )
+ extra.set ( "authhint", authHint );
+ break;
+ }
+ case zyppng::NetworkRequestError::AuthFailed: {
+ errCode = zyppng::ProvideMessage::Code::NoAuthData;
+ const auto &authHint = error.extraInfoValue("authHint", std::string());
+ if ( authHint.size () )
+ extra.set ( "authhint", authHint );
+ break;
+ }
+ case zyppng::NetworkRequestError::ServerReturnedError: { errCode = zyppng::ProvideMessage::Code::InternalError; break; }
+ case zyppng::NetworkRequestError::MissingData: { errCode = zyppng::ProvideMessage::Code::BadRequest; break; }
+ }
+
+ provideFailed( item->_spec.requestId(), errCode, item->_dl->errorString(), isTransient, extra );
+ }
+ queue.erase(i);
+
+ // pick the next request
+ provide();
+}
+
+void NetworkProvider::itemAuthRequired( NetworkProvideItemRef item, zyppng::NetworkAuthData &auth, const std::string & availAuth )
+{
+ auto res = requireAuthorization( item->_spec.requestId(), auth.url(), auth.username(), auth.lastDatabaseUpdate(), { { std::string( zyppng::AuthDataRequestMsgFields::AuthHint ), availAuth } } );
+ if ( !res ) {
+ auth = zyppng::NetworkAuthData();
+ return;
+ }
+
+ auth.setUrl ( item->_dl->spec().url() );
+ auth.setUsername ( res->username );
+ auth.setPassword ( res->password );
+ auth.setLastDatabaseUpdate ( res->last_auth_timestamp );
+ auth.extraValues() = res->extraKeys;
+
+ const auto &keyStr = std::string(zyppng::AuthInfoMsgFields::AuthType);
+ if ( res->extraKeys.count( keyStr ) ) {
+ auth.setAuthType( res->extraKeys[keyStr] );
+ }
+
+ //@todo migrate away from using NetworkAuthData, AuthData now has a map of extra values that can carry the AuthType
+ const auto &authTypeKey = std::string( zyppng::AuthInfoMsgFields::AuthType );
+ if ( res->extraKeys.count( authTypeKey ) )
+ auth.setAuthType( res->extraKeys[ authTypeKey ] );
+}
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+#ifndef ZYPP_NG_TOOLS_NETWORKPROVIDER_H_INCLUDED
+#define ZYPP_NG_TOOLS_NETWORKPROVIDER_H_INCLUDED
+
+#include <zypp-media/ng/worker/ProvideWorker>
+#include <zypp-core/zyppng/base/Signals>
+#include <zypp-core/zyppng/base/AutoDisconnect>
+#include <zypp-curl/ng/network/AuthData>
+#include <chrono>
+
+namespace zyppng {
+ class Downloader;
+ class Download;
+}
+
+class NetworkProvider;
+
+struct NetworkProvideItem : public zyppng::worker::ProvideWorkerItem
+{
+public:
+ NetworkProvideItem( NetworkProvider &parent, zyppng::ProvideMessage &&spec );
+ ~NetworkProvideItem();
+
+ void startDownload( std::shared_ptr<zyppng::Download> &&dl );
+ void cancelDownload ();
+
+ std::shared_ptr<zyppng::Download> _dl;
+ zypp::Pathname _targetFileName;
+ zypp::Pathname _stagingFileName;
+
+ std::chrono::steady_clock::time_point _scheduleAfter = std::chrono::steady_clock::time_point::min();
+
+private:
+ void clearConnections ();
+ void onStarted ( zyppng::Download & );
+ void onFinished ( zyppng::Download & );
+ void onAuthRequired ( zyppng::Download &, zyppng::NetworkAuthData &auth, const std::string &availAuth );
+
+private:
+ std::vector<zyppng::connection> _connections;
+ NetworkProvider &_parent;
+};
+
+using NetworkProvideItemRef = std::shared_ptr<NetworkProvideItem>;
+
+class NetworkProvider : public zyppng::worker::ProvideWorker
+{
+public:
+ NetworkProvider( std::string_view workerName );
+ void immediateShutdown() override;
+
+protected:
+ // ProvideWorker interface
+ zyppng::expected<zyppng::worker::WorkerCaps> initialize(const zyppng::worker::Configuration &conf) override;
+ void provide() override;
+ void cancel(const std::deque<zyppng::worker::ProvideWorkerItemRef>::iterator &i ) override;
+ zyppng::worker::ProvideWorkerItemRef makeItem(zyppng::ProvideMessage &&spec) override;
+
+ friend struct NetworkProvideItem;
+ void itemStarted ( NetworkProvideItemRef item );
+ void itemFinished ( NetworkProvideItemRef item );
+ void itemAuthRequired (NetworkProvideItemRef item, zyppng::NetworkAuthData &auth, const std::string &);
+
+private:
+ std::shared_ptr<zyppng::Downloader> _dlManager;
+ zypp::Pathname _attachPoint;
+};
+
+
+
+
+
+#endif
--- /dev/null
+PROJECT( zypp-media-iso C CXX )
+
+set (CMAKE_CXX_STANDARD 17)
+SET (CMAKE_CXX_EXTENSIONS OFF)
+
+FIND_PACKAGE(Protobuf REQUIRED)
+
+SET( SOURCES
+ main.cc
+ isoprovider.cc
+ isoprovider.h
+ ${ZYPP_TOOLS_DIR}/zypp-media-dir/dirprovider.h
+ ${ZYPP_TOOLS_DIR}/zypp-media-dir/dirprovider.cc
+ ${ZYPP_TOOLS_DIR}/zypp-media-disk/diskprovider.h
+ ${ZYPP_TOOLS_DIR}/zypp-media-disk/diskprovider.cc
+ ${ZYPP_TOOLS_DIR}/zypp-media-nfs/nfsprovider.h
+ ${ZYPP_TOOLS_DIR}/zypp-media-nfs/nfsprovider.cc
+ ${ZYPP_TOOLS_DIR}/zypp-media-smb/smbprovider.h
+ ${ZYPP_TOOLS_DIR}/zypp-media-smb/smbprovider.cc
+)
+
+include_directories(
+ ${ZYPP_TOOLS_DIR}/zypp-media-dir
+ ${ZYPP_TOOLS_DIR}/zypp-media-disk
+ ${ZYPP_TOOLS_DIR}/zypp-media-nfs
+ ${ZYPP_TOOLS_DIR}/zypp-media-smb
+)
+
+add_executable( ${PROJECT_NAME} ${SOURCES} )
+target_link_libraries( ${PROJECT_NAME} zypp-media )
+target_link_libraries( ${PROJECT_NAME} zypp-core )
+target_link_libraries( ${PROJECT_NAME} zypp-protobuf )
+target_link_libraries( ${PROJECT_NAME} ${PROTOBUF_LITE_LIBRARIES} )
+
+IF ( INSTALL_NG_BINARIES )
+ INSTALL( TARGETS ${PROJECT_NAME} DESTINATION "${ZYPP_LIBEXEC_INSTALL_DIR}/workers" )
+ENDIF()
+
+set_target_properties( ${PROJECT_NAME}
+ PROPERTIES
+ ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+ LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+ RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+)
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+#include "isoprovider.h"
+#include <zypp-media/ng/private/providedbg_p.h>
+#include <zypp-media/ng/MediaVerifier>
+
+#include <zypp-core/fs/TmpPath.h>
+#include <zypp-core/fs/PathInfo.h>
+
+#include <iostream>
+#include <fstream>
+
+#undef ZYPP_BASE_LOGGER_LOGGROUP
+#define ZYPP_BASE_LOGGER_LOGGROUP "IsoProvider"
+
+RaiiHelper::~RaiiHelper()
+{
+ _backingDriver->detachMedia( id );
+ _backingDriver->releaseIdleDevices();
+}
+
+IsoProvider::IsoProvider( )
+ : DeviceDriver( zyppng::worker::WorkerCaps::SimpleMount )
+{ }
+
+IsoProvider::~IsoProvider()
+{ }
+
+zyppng::worker::AttachResult IsoProvider::mountDevice ( const uint32_t id, const zypp::Url &attachUrl, const std::string &attachId, const std::string &label, const zyppng::HeaderValueMap &extras )
+{
+ try
+ {
+ zypp::Pathname isofile = attachUrl.getQueryParam("iso");
+ if( isofile.empty()) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , zypp::str::Str()<<"Media url: "<<attachUrl<<" does not contain iso filename"
+ , false );
+ }
+
+ std::string filesystem = attachUrl.getQueryParam("filesystem");
+ if( filesystem.empty())
+ filesystem = "auto";
+
+ zypp::Url src;
+ {
+ const std::string & arg { attachUrl.getQueryParam("url") };
+ if ( arg.empty() ) {
+ src = "dir:/";
+ src.setPathName( isofile.dirname() );
+ isofile = isofile.basename();
+ }
+ else try {
+ src = arg;
+ }
+ catch( const zypp::url::UrlException & e )
+ {
+ ZYPP_CAUGHT(e);
+ const std::string &err = zypp::str::Str()<<"Unable to parse iso filename source media url: "<<attachUrl;
+ ERR << err << std::endl;
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , err
+ , false );
+ }
+ }
+
+ // the full URL to the ISO file, we will use that to identify the device
+ auto fullIsoUrl = src;
+ fullIsoUrl.appendPathName( isofile );
+
+ // the root of the medium inside the ISO file
+ zypp::Pathname relAttachRoot = attachUrl.getPathName();
+ if ( relAttachRoot.empty() )
+ relAttachRoot = "/";
+
+ // set up the verifier
+ zyppng::MediaDataVerifierRef verifier;
+ if ( extras.value( zyppng::AttachMsgFields::VerifyType ).valid() ) {
+ verifier = zyppng::MediaDataVerifier::createVerifier( extras.value(zyppng::AttachMsgFields::VerifyType).asString() );
+ if ( !verifier ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , "Invalid verifier type"
+ , false
+ );
+ }
+
+ if ( !verifier->load( extras.value(zyppng::AttachMsgFields::VerifyData).asString() ) ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , "Failed to create verifier from file"
+ , false
+ );
+ }
+ }
+
+ // first check if we have that device already
+ const auto &devs = knownDevices();
+ auto i = std::find_if( devs.begin(), devs.end(), [&]( const auto &d ) {
+ return d->_name == fullIsoUrl.asString();
+ });
+ if ( i != devs.end() ) {
+ // if we can find the device, isDesiredMedium needs to return true,
+ // otherwise URL and verifier do not match and its not the desired medium
+ auto res = isDesiredMedium( attachUrl, (*i)->_mountPoint / relAttachRoot, verifier, extras.value( zyppng::AttachMsgFields::MediaNr, 1 ).asInt() );
+ if ( !res ) {
+ try {
+ std::rethrow_exception( res.error() );
+ } catch( const zypp::Exception& e ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MediumNotDesired
+ , false
+ , e
+ );
+ } catch ( ... ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MediumNotDesired
+ , "Checking the medium failed with an uknown error"
+ , false
+ );
+ }
+ } else {
+ attachedMedia().insert( std::make_pair( attachId, zyppng::worker::AttachedMedia{ *i, relAttachRoot } ) );
+ return zyppng::worker::AttachResult::success();
+ }
+ }
+
+ // helper lambda to invoke the deviceDriver
+ const auto &attachViaHelper = [&]( auto &driver, const std::string &type ) {
+
+ auto mountRes = driver->mountDevice( id, src, attachId, "", {} );
+ if ( !mountRes ) {
+ return mountRes;
+ }
+
+ // make sure the media is released as soon as its not needed anymore
+ auto hlpr = std::make_shared<RaiiHelper>();
+ hlpr->_backingDriver = driver;
+ hlpr->id = attachId;
+
+ // k, get the mountpoint from the backing driver
+ const auto &attMedias = driver->attachedMedia();
+ auto i = attMedias.find( attachId );
+ if ( i == attMedias.end() ) {
+ const std::string &err = zypp::str::Str()<< "Failed to query mounted supporting device: " << src.asString();
+ ERR << err << std::endl;
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , err
+ , false );
+ }
+
+ // full path to the ISO in the filesystem
+ auto isopath = zypp::filesystem::expandlink( i->second._dev->_mountPoint / isofile );
+ if( isopath.empty() || !zypp::PathInfo(isopath).isFile()) {
+ const std::string &err = zypp::str::Str()<< "No ISO file found on given source location: " << src.asString();
+ ERR << err << std::endl;
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , err
+ , false );
+ }
+
+ // we have everything we need, let's try to mount it
+ zypp::media::Mount mount;
+ zypp::Pathname newAp;
+ try {
+ newAp = createAttachPoint( attachRoot() );
+ if ( newAp.empty() ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , "Failed to create mount directory."
+ , false
+ );
+ }
+
+ std::string mountpoint( newAp.asString() );
+ std::string mountopts("ro,loop");
+
+ mount.mount( isopath.asString(), mountpoint,
+ filesystem, mountopts );
+
+ // wait for /etc/mtab update ...
+ // (shouldn't be needed)
+ int limit = 3;
+ bool mountsucceeded = false;
+
+ // Type ISO: Since 11.1 mtab might contain the name of
+ // the loop device instead of the iso file:
+ const auto &checkAttachedISO = [&]( const zypp::media::MountEntry &e ) {
+ if ( zypp::str::hasPrefix( zypp::Pathname(e.src).asString(), "/dev/loop" )
+ && mountpoint == zypp::Pathname(e.dir) ) {
+ DBG << "Found bound media "
+ << isopath
+ << " in the mount table as " << e.src << std::endl;
+ return true;
+ }
+ return false;
+ };
+
+ // mount command came back OK, lets wait for mtab to update
+ while( !(mountsucceeded=checkAttached( newAp, checkAttachedISO )) && --limit) {
+ MIL << "Mount did not appear yet, sleeping for 1s" << std::endl;
+ sleep(1);
+ }
+
+ // mount didn't work after all, bail out
+ if ( !mountsucceeded ) {
+ try {
+ mount.umount( newAp.asString() );
+ } catch (const zypp::media::MediaException & excpt_r) {
+ ZYPP_CAUGHT(excpt_r);
+ }
+ ZYPP_THROW( zypp::media::MediaMountException(
+ "Unable to verify that the media was mounted",
+ isopath.asString(), newAp.asString()
+ ));
+ }
+
+ // if we reach this place, mount worked -> YAY, lets see if that is the desired medium!
+ auto isDesired = isDesiredMedium( attachUrl, newAp / relAttachRoot, verifier, extras.value( zyppng::AttachMsgFields::MediaNr, 1 ).asInt() );
+ if ( !isDesired ) {
+ try {
+ mount.umount( newAp.asString() );
+ } catch (const zypp::media::MediaException & excpt_r) {
+ ZYPP_CAUGHT(excpt_r);
+ }
+ ZYPP_THROW( zypp::media::MediaNotDesiredException( attachUrl ) );
+ }
+
+ } catch ( const zypp::Exception &e ) {
+ removeAttachPoint(newAp);
+ ZYPP_CAUGHT(e);
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , false
+ , e
+ );
+ }
+
+ // here we have our device AND it is verified , lets register it and return it to the user
+ auto devPtr = std::make_shared<zyppng::worker::Device>( zyppng::worker::Device{
+ ._name = fullIsoUrl.asString(),
+ ._maj_nr = 0,
+ ._min_nr = 0,
+ ._mountPoint = newAp,
+ ._ephemeral = true, // forget about the device after we are finished with it
+ ._properties = {}
+ });
+ devPtr->_properties["raiiHelper"] = hlpr;
+
+
+ knownDevices().push_back( devPtr );
+ attachedMedia().insert( std::make_pair( attachId, zyppng::worker::AttachedMedia{ devPtr, relAttachRoot } ) );
+ return zyppng::worker::AttachResult::success();
+
+ };
+
+ if( !src.isValid()) {
+ const std::string &err = zypp::str::Str()<<"Invalid iso filename source media url: "<<attachUrl;
+ ERR << err << std::endl;
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , err
+ , false );
+ }
+ if( src.getScheme() == "iso") {
+ const std::string &err = zypp::str::Str()<< "ISO filename source media url with iso scheme (nested iso): " << src.asString();
+ ERR << err << std::endl;
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , err
+ , false );
+
+ } else if ( src.getScheme() == "hd" ) {
+
+ if ( !_diskWorker ) {
+ _diskWorker = std::make_shared<DiskProvider>();
+
+ auto subAttachRoot = ( attachRoot().realpath()/"disk" );
+ zypp::filesystem::assert_dir( subAttachRoot );
+
+ auto confCopy = config();
+ (*confCopy.mutable_values())[std::string(zyppng::ATTACH_POINT)] = subAttachRoot.asString();
+ const auto &res = _diskWorker->initialize( confCopy );
+ if ( !res ) {
+ const std::string &err = zypp::str::Str()<< "Failed to initialize worker for: " << src.asString();
+ ERR << err << std::endl;
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , err
+ , false );
+ }
+ }
+
+ return attachViaHelper( _diskWorker, "disk" );
+
+ } else if ( src.getScheme() == "dir" || src.getScheme() == "file" ) {
+
+ if ( !_dirWorker ) {
+ _dirWorker = std::make_shared<DirProvider>();
+
+ auto subAttachRoot = ( attachRoot().realpath()/"dir" );
+ zypp::filesystem::assert_dir( subAttachRoot );
+
+ auto confCopy = config();
+ (*confCopy.mutable_values())[std::string(zyppng::ATTACH_POINT)] = subAttachRoot.asString();
+ const auto &res = _dirWorker->initialize( confCopy );
+ if ( !res ) {
+ const std::string &err = zypp::str::Str()<< "Failed to initialize worker for: " << src.asString();
+ ERR << err << std::endl;
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , err
+ , false );
+ }
+ }
+
+ return attachViaHelper( _dirWorker, "dir" );
+
+ } else if ( src.getScheme() == "nfs" || src.getScheme() == "nfs4" ) {
+
+ if ( !_nfsWorker ) {
+ _nfsWorker = std::make_shared<NfsProvider>();
+
+ auto subAttachRoot = ( attachRoot().realpath()/"nfs" );
+ zypp::filesystem::assert_dir( subAttachRoot );
+
+ auto confCopy = config();
+ (*confCopy.mutable_values())[std::string(zyppng::ATTACH_POINT)] = subAttachRoot.asString();
+ const auto &res = _nfsWorker->initialize( confCopy );
+ MIL << "AFTER INIT " << _nfsWorker->attachRoot() << std::endl;
+ if ( !res ) {
+ const std::string &err = zypp::str::Str()<< "Failed to initialize worker for: " << src.asString();
+ ERR << err << std::endl;
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , err
+ , false );
+ }
+ }
+
+ return attachViaHelper( _nfsWorker, "nfs" );
+
+ } else if ( src.getScheme() == "smb" || src.getScheme() == "cifs" ) {
+
+ if ( !_smbWorker ) {
+ _smbWorker = std::make_shared<SmbProvider>();
+
+ auto subAttachRoot = ( attachRoot().realpath()/"smb" );
+ zypp::filesystem::assert_dir( subAttachRoot );
+
+ auto confCopy = config();
+ (*confCopy.mutable_values())[std::string(zyppng::ATTACH_POINT)] = subAttachRoot.asString();
+ const auto &res = _smbWorker->initialize( confCopy );
+ if ( !res ) {
+ const std::string &err = zypp::str::Str()<< "Failed to initialize worker for: " << src.asString();
+ ERR << err << std::endl;
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , err
+ , false );
+ }
+ }
+
+ return attachViaHelper( _smbWorker, "smb" );
+
+ } else {
+ const std::string &err = zypp::str::Str()<< "ISO filename source media url scheme is not supported: " << src.asString();
+ ERR << err << std::endl;
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , err
+ , false );
+ }
+ } catch ( const zypp::Exception &e ) {
+ return zyppng::worker::AttachResult::error (
+ zyppng::ProvideMessage::Code::BadRequest
+ , false
+ , e );
+ } catch ( const std::exception &e ) {
+ return zyppng::worker::AttachResult::error (
+ zyppng::ProvideMessage::Code::BadRequest
+ , e.what()
+ , false );
+ } catch ( ... ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::BadRequest
+ , "Unknown exception"
+ , false);
+ }
+}
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+#ifndef ZYPP_NG_TOOLS_ISOPROVIDER_H_INCLUDED
+#define ZYPP_NG_TOOLS_ISOPROVIDER_H_INCLUDED
+
+#include <zypp-media/ng/worker/DeviceDriver>
+#include "dirprovider.h"
+#include "smbprovider.h"
+#include "diskprovider.h"
+#include "nfsprovider.h"
+
+
+struct RaiiHelper
+{
+ ~RaiiHelper();
+ zyppng::worker::DeviceDriverRef _backingDriver;
+ std::string id;
+};
+
+class IsoProvider : public zyppng::worker::DeviceDriver
+{
+ public:
+ IsoProvider( );
+ ~IsoProvider();
+
+ zyppng::worker::AttachResult mountDevice ( const uint32_t id, const zypp::Url &attachUrl, const std::string &attachId, const std::string &label, const zyppng::HeaderValueMap &extras ) override;
+
+ private:
+ std::shared_ptr<DirProvider> _dirWorker;
+ std::shared_ptr<DiskProvider> _diskWorker;
+ std::shared_ptr<NfsProvider> _nfsWorker;
+ std::shared_ptr<SmbProvider> _smbWorker;
+
+};
+
+#endif
--- /dev/null
+#include "isoprovider.h"
+
+#include <csignal>
+#include <zypp-core/zyppng/base/private/linuxhelpers_p.h>
+#include <zypp-media/ng/worker/MountingWorker>
+
+int main( int , char *[] )
+{
+ // lets ignore those, SIGPIPE is better handled via the EPIPE error, and we do not want anyone
+ // to CTRL+C us
+ zyppng::blockSignalsForCurrentThread( { SIGPIPE, SIGINT } );
+
+ auto driver = std::make_shared<IsoProvider>();
+ auto provider = std::make_shared<zyppng::worker::MountingWorker>( "zypp-media-iso", driver );
+ driver->setProvider( provider );
+
+ auto res = provider->run (STDIN_FILENO, STDOUT_FILENO);
+
+ MIL << "ISO Worker shutting down" << std::endl;
+
+ provider->immediateShutdown();
+ if ( res )
+ return 0;
+
+ //@TODO print error
+ return 1;
+}
--- /dev/null
+PROJECT( zypp-media-nfs C CXX )
+
+set (CMAKE_CXX_STANDARD 17)
+SET (CMAKE_CXX_EXTENSIONS OFF)
+
+FIND_PACKAGE(Protobuf REQUIRED)
+
+SET( SOURCES
+ main.cc
+ nfsprovider.cc
+ nfsprovider.h
+)
+
+add_executable( ${PROJECT_NAME} ${SOURCES} )
+target_link_libraries( ${PROJECT_NAME} zypp-media )
+target_link_libraries( ${PROJECT_NAME} zypp-core )
+target_link_libraries( ${PROJECT_NAME} zypp-protobuf )
+target_link_libraries( ${PROJECT_NAME} ${PROTOBUF_LITE_LIBRARIES} )
+
+IF ( INSTALL_NG_BINARIES )
+ INSTALL( TARGETS ${PROJECT_NAME} DESTINATION "${ZYPP_LIBEXEC_INSTALL_DIR}/workers" )
+ENDIF()
+
+set_target_properties( ${PROJECT_NAME}
+ PROPERTIES
+ ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+ LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+ RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+)
--- /dev/null
+#include "nfsprovider.h"
+
+#include <csignal>
+#include <zypp-core/zyppng/base/private/linuxhelpers_p.h>
+#include <zypp-media/ng/worker/MountingWorker>
+
+int main( int , char *[] )
+{
+ // lets ignore those, SIGPIPE is better handled via the EPIPE error, and we do not want anyone
+ // to CTRL+C us
+ zyppng::blockSignalsForCurrentThread( { SIGPIPE, SIGINT } );
+
+ auto driver = std::make_shared<NfsProvider>();
+ auto worker = std::make_shared<zyppng::worker::MountingWorker>( "zypp-media-nfs", driver );
+ driver->setProvider( worker );
+
+ auto res = worker->run (STDIN_FILENO, STDOUT_FILENO);
+ if ( res )
+ return 0;
+
+ //@TODO print error
+ return 1;
+}
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+#include "nfsprovider.h"
+#include <zypp-media/ng/private/providedbg_p.h>
+#include <zypp-media/Mount>
+#include <zypp-media/ng/MediaVerifier>
+
+#undef ZYPP_BASE_LOGGER_LOGGROUP
+#define ZYPP_BASE_LOGGER_LOGGROUP "NfsProvider"
+
+using namespace std::literals::string_view_literals;
+
+/**
+ * Value of NFS mount minor timeout (passed to <code>timeo</code> option
+ * of the NFS mount) in tenths of a second.
+ *
+ * The value of 300 should give a major timeout after 3.5 minutes
+ * for UDP and 1.5 minutes for TCP. (#350309)
+ */
+constexpr auto NFS_MOUNT_TIMEOUT = 300;
+constexpr std::array<std::string_view, 2> NFS_MOUNT_FSTYPES = { "nfs"sv, "nfs4"sv };
+
+NfsProvider::NfsProvider( )
+ : DeviceDriver( zyppng::worker::WorkerCaps::SimpleMount )
+{ }
+
+NfsProvider::~NfsProvider()
+{ }
+
+zyppng::worker::AttachResult NfsProvider::mountDevice ( const uint32_t id, const zypp::Url &attachUrl, const std::string &attachId, const std::string &label, const zyppng::HeaderValueMap &extras )
+{
+ try
+ {
+ if ( attachUrl.getHost().empty() ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , "Host can not be empty"
+ , false
+ );
+ }
+
+ // set up the verifier
+ zyppng::MediaDataVerifierRef verifier;
+ zyppng::MediaDataVerifierRef devVerifier;
+ if ( extras.value( zyppng::AttachMsgFields::VerifyType ).valid() ) {
+ verifier = zyppng::MediaDataVerifier::createVerifier( extras.value(zyppng::AttachMsgFields::VerifyType).asString() );
+ if ( !verifier ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , "Invalid verifier type"
+ , false
+ );
+ }
+
+ if ( !verifier->load( extras.value(zyppng::AttachMsgFields::VerifyData).asString() ) ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , "Failed to create verifier from file"
+ , false
+ );
+ }
+ }
+ const auto &devs = knownDevices();
+
+ // build a device name the same way the old media backend did
+ std::string path = attachUrl.getHost();
+ path += ':';
+ path += zypp::Pathname(attachUrl.getPathName()).asString();
+
+ // first check if we have that device already
+ auto i = std::find_if( devs.begin(), devs.end(), [&]( const auto &d ) { return d->_name == path; } );
+ if ( i != devs.end() ) {
+ auto res = isDesiredMedium( attachUrl, (*i)->_mountPoint, verifier, extras.value( zyppng::AttachMsgFields::MediaNr, 1 ).asInt() );
+ if ( !res ) {
+ try {
+ std::rethrow_exception( res.error() );
+ } catch( const zypp::Exception& e ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MediumNotDesired
+ , false
+ , e );
+ } catch ( ... ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MediumNotDesired
+ , "Checking the medium failed with an uknown error"
+ , false
+ );
+ }
+ } else {
+ attachedMedia().insert( std::make_pair( attachId, zyppng::worker::AttachedMedia{ *i, "/" } ) );
+ return zyppng::worker::AttachResult::success();
+ }
+ }
+
+ // we did not find a existing mount, well lets make a new one
+ // start with some legacy code:
+ std::string filesystem( attachUrl.getScheme() );
+ if ( filesystem != "nfs4" && attachUrl.getQueryParam("type") == "nfs4" ) {
+ filesystem = "nfs4";
+ }
+
+ std::string options = attachUrl.getQueryParam("mountoptions");
+ if(options.empty()) {
+ options="ro";
+ }
+
+ std::vector<std::string> optionList;
+ zypp::str::split( options, std::back_inserter(optionList), "," );
+ std::vector<std::string>::const_iterator it;
+ bool contains_lock = false, contains_soft = false,
+ contains_timeo = false, contains_hard = false;
+
+ for( it = optionList.begin(); it != optionList.end(); ++it ) {
+ if ( *it == "lock" || *it == "nolock" ) contains_lock = true;
+ else if ( *it == "soft") contains_soft = true;
+ else if ( *it == "hard") contains_hard = true;
+ else if ( it->find("timeo") != std::string::npos ) contains_timeo = true;
+ }
+
+ if ( !(contains_lock && contains_soft) ) {
+ // Add option "nolock", unless option "lock" or "unlock" is already set.
+ // This should prevent the mount command hanging when the portmapper isn't
+ // running.
+ if ( !contains_lock ) {
+ optionList.push_back( "nolock" );
+ }
+ // Add options "soft,timeo=NFS_MOUNT_TIMEOUT", unless they are set
+ // already or "hard" option is explicitly specified. This prevent
+ // the mount command from hanging when the nfs server is not responding
+ // and file transactions from an unresponsive to throw an error after
+ // a short time instead of hanging forever
+ if ( !(contains_soft || contains_hard) ) {
+ optionList.push_back( "soft" );
+ if ( !contains_timeo ) {
+ std::ostringstream s;
+ s << "timeo=" << NFS_MOUNT_TIMEOUT;
+ optionList.push_back( s.str() );
+ }
+ }
+ options = zypp::str::join( optionList, "," );
+ }
+
+ zypp::Pathname newAp;
+ zypp::media::Mount mount;
+ try
+ {
+ newAp = createAttachPoint( attachRoot() );
+ if ( newAp.empty() ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , "Failed to create mount directory."
+ , false
+ );
+ }
+
+ mount.mount( path, newAp.asString(), filesystem, options );
+
+ // wait for /etc/mtab update ...
+ // (shouldn't be needed)
+ int limit = 3;
+ bool mountsucceeded = false;
+
+ // we only check if our attachpoint is in mounts and of type nfs/nfs4, everything else is not reliable anyways
+ constexpr auto checkFsTypePredicate = []( const zypp::media::MountEntry &entry ){
+ return ( std::find( NFS_MOUNT_FSTYPES.begin(), NFS_MOUNT_FSTYPES.end(), entry.type ) != NFS_MOUNT_FSTYPES.end() );
+ };
+ while( !(mountsucceeded=checkAttached( newAp, checkFsTypePredicate )) && --limit) {
+ MIL << "Mount did not appear yet, sleeping for 1s" << std::endl;
+ sleep(1);
+ }
+
+ if( !mountsucceeded) {
+ try {
+ mount.umount( newAp.asString() );
+ } catch (const zypp::media::MediaException & excpt_r) {
+ ZYPP_CAUGHT(excpt_r);
+ }
+ ZYPP_THROW( zypp::media::MediaMountException(
+ "Unable to verify that the media was mounted",
+ path, newAp.asString()
+ ));
+ }
+
+ // if we reach this place, mount worked -> YAY, lets see if that is the desired medium!
+ const auto &isDesired = isDesiredMedium( attachUrl, newAp, verifier, extras.value( zyppng::AttachMsgFields::MediaNr, 1 ).asInt() );
+ if ( !isDesired ) {
+ try {
+ mount.umount( newAp.asString() );
+ } catch (const zypp::media::MediaException & excpt_r) {
+ ZYPP_CAUGHT(excpt_r);
+ }
+ ZYPP_THROW( zypp::media::MediaNotDesiredException( attachUrl ) );
+ }
+
+ MIL << "New device " << path << " mounted on " << newAp << std::endl;
+ auto newDev = std::shared_ptr<zyppng::worker::Device>( new zyppng::worker::Device{
+ ._name = path,
+ ._maj_nr = 0,
+ ._min_nr = 0,
+ ._mountPoint = newAp,
+ ._ephemeral = true, // device should be removed after the last attachment was released
+ ._properties = {}
+ });
+ knownDevices().push_back( newDev );
+ attachedMedia().insert( std::make_pair( attachId, zyppng::worker::AttachedMedia{ newDev, "/" } ) );
+ return zyppng::worker::AttachResult::success();
+
+ }
+ catch (const zypp::media::MediaMountException &e )
+ {
+ removeAttachPoint(newAp);
+ ZYPP_CAUGHT(e);
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , false
+ , e
+ );
+ }
+ catch (const zypp::media::MediaException & e)
+ {
+ removeAttachPoint(newAp);
+ ZYPP_CAUGHT(e);
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , false
+ , e
+ );
+ }
+
+ // We should never reach this place, but if we do fail the attach request
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , zypp::str::Str()<<"Mounting the medium " << attachUrl << " failed for an unknown reason"
+ , false
+ );
+
+ } catch ( const zypp::Exception &e ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::BadRequest
+ , false
+ , e
+ );
+ } catch ( const std::exception &e ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::BadRequest
+ , e.what()
+ , false
+ );
+ } catch ( ... ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::BadRequest
+ , "Unknown exception"
+ , false
+ );
+ }
+}
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+#ifndef ZYPP_NG_TOOLS_NFSPROVIDER_H_INCLUDED
+#define ZYPP_NG_TOOLS_NFSPROVIDER_H_INCLUDED
+
+#include <zypp-media/ng/worker/DeviceDriver>
+
+class NfsProvider : public zyppng::worker::DeviceDriver
+{
+ public:
+ NfsProvider();
+ ~NfsProvider();
+
+ // DeviceDriver interface
+ zyppng::worker::AttachResult mountDevice ( const uint32_t id, const zypp::Url &attachUrl, const std::string &attachId, const std::string &label, const zyppng::HeaderValueMap &extras ) override;
+
+};
+
+#endif
--- /dev/null
+PROJECT( zypp-media-smb C CXX )
+
+set (CMAKE_CXX_STANDARD 17)
+SET (CMAKE_CXX_EXTENSIONS OFF)
+
+FIND_PACKAGE(Protobuf REQUIRED)
+
+SET( SOURCES
+ main.cc
+ smbprovider.cc
+ smbprovider.h
+)
+
+add_executable( ${PROJECT_NAME} ${SOURCES} )
+target_link_libraries( ${PROJECT_NAME} zypp-media )
+target_link_libraries( ${PROJECT_NAME} zypp-core )
+target_link_libraries( ${PROJECT_NAME} zypp-protobuf )
+target_link_libraries( ${PROJECT_NAME} ${PROTOBUF_LITE_LIBRARIES} )
+
+IF ( INSTALL_NG_BINARIES )
+ INSTALL( TARGETS ${PROJECT_NAME} DESTINATION "${ZYPP_LIBEXEC_INSTALL_DIR}/workers" )
+ENDIF()
+
+set_target_properties( ${PROJECT_NAME}
+ PROPERTIES
+ ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+ LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+ RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+)
--- /dev/null
+#include "smbprovider.h"
+
+#include <csignal>
+#include <zypp-core/zyppng/base/private/linuxhelpers_p.h>
+#include <zypp-media/ng/worker/MountingWorker>
+
+int main( int , char *[] )
+{
+ // lets ignore those, SIGPIPE is better handled via the EPIPE error, and we do not want anyone
+ // to CTRL+C us
+ zyppng::blockSignalsForCurrentThread( { SIGPIPE, SIGINT } );
+
+ auto driver = std::make_shared<SmbProvider>();
+ auto worker = std::make_shared<zyppng::worker::MountingWorker>( "zypp-media-smb", driver );
+ driver->setProvider( worker );
+
+ auto res = worker->run (STDIN_FILENO, STDOUT_FILENO);
+
+ MIL << "SMB Worker shutting down" << std::endl;
+
+ if ( res )
+ return 0;
+
+ //@TODO print error
+ return 1;
+}
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+#include "smbprovider.h"
+#include <zypp-media/ng/private/providedbg_p.h>
+#include <zypp-media/Mount>
+#include <zypp-media/ng/MediaVerifier>
+
+#include <zypp-core/fs/TmpPath.h>
+
+#include <iostream>
+#include <fstream>
+
+#undef ZYPP_BASE_LOGGER_LOGGROUP
+#define ZYPP_BASE_LOGGER_LOGGROUP "SmbProvider"
+
+using namespace std::literals::string_view_literals;
+
+
+/*!
+ * Get the 1st path component (CIFS share name).
+ */
+inline std::string getShare( zypp::Pathname spath_r )
+{
+ if ( spath_r.empty() )
+ return std::string();
+
+ std::string share( spath_r.absolutename().asString() );
+ std::string::size_type sep = share.find( "/", 1 );
+ if ( sep == std::string::npos )
+ share = share.erase( 0, 1 ); // nothing but the share name in spath_r
+ else
+ share = share.substr( 1, sep-1 );
+
+ // deescape %2f in sharename
+ while ( (sep = share.find( "%2f" )) != std::string::npos ) {
+ share.replace( sep, 3, "/" );
+ }
+
+ return share;
+}
+
+/*!
+ * Strip off the 1st path component (CIFS share name).
+ */
+inline zypp::Pathname stripShare( zypp::Pathname spath_r )
+{
+ if ( spath_r.empty() )
+ return zypp::Pathname();
+
+ std::string striped( spath_r.absolutename().asString() );
+ std::string::size_type sep = striped.find( "/", 1 );
+ if ( sep == std::string::npos )
+ return "/"; // nothing but the share name in spath_r
+
+ return striped.substr( sep );
+}
+
+
+SmbProvider::SmbProvider( )
+ : DeviceDriver( zyppng::worker::WorkerCaps::SimpleMount )
+{ }
+
+SmbProvider::~SmbProvider()
+{ }
+
+zyppng::worker::AttachResult SmbProvider::mountDevice ( const uint32_t id, const zypp::Url &attachUrl, const std::string &attachId, const std::string &label, const zyppng::HeaderValueMap &extras )
+{
+ try
+ {
+ if ( attachUrl.getHost().empty() ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , "Host can not be empty"
+ , false
+ );
+ }
+
+ // set up the verifier
+ zyppng::MediaDataVerifierRef verifier;
+ if ( extras.value( zyppng::AttachMsgFields::VerifyType ).valid() ) {
+ verifier = zyppng::MediaDataVerifier::createVerifier( extras.value(zyppng::AttachMsgFields::VerifyType).asString() );
+ if ( !verifier ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , "Invalid verifier type"
+ , false
+ );
+ }
+
+ if ( !verifier->load( extras.value(zyppng::AttachMsgFields::VerifyData).asString() ) ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , "Failed to create verifier from file"
+ , false
+ );
+ }
+ }
+ const auto &devs = knownDevices();
+
+ // build a device name the same way the old media backend did
+ std::string path = "//";
+ path += attachUrl.getHost() + "/" + getShare( attachUrl.getPathName() );
+
+ // first check if we have that device already
+ auto i = std::find_if( devs.begin(), devs.end(), [&]( const auto &d ) { return d->_name == path; } );
+ if ( i != devs.end() ) {
+ auto res = isDesiredMedium( attachUrl, (*i)->_mountPoint, verifier, extras.value( zyppng::AttachMsgFields::MediaNr, 1 ).asInt() );
+ if ( !res ) {
+ try {
+ std::rethrow_exception( res.error() );
+ } catch( const zypp::Exception& e ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MediumNotDesired
+ , false
+ , e
+ );
+ } catch ( ... ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MediumNotDesired
+ , "Checking the medium failed with an uknown error"
+ , false
+ );
+ }
+ } else {
+ attachedMedia().insert( std::make_pair( attachId, zyppng::worker::AttachedMedia{ *i, "/" } ) );
+ return zyppng::worker::AttachResult::success();
+ }
+ }
+
+ // we did not find a existing mount, well lets make a new one
+ // start with some legacy code:
+ zypp::media::Mount::Options options( attachUrl.getQueryParam("mountoptions") );
+
+ int64_t lastTimestamp = -1;
+ std::string username = attachUrl.getUsername();
+ std::string password = attachUrl.getPassword();
+
+ if ( ! options.has( "rw" ) ) {
+ options["ro"];
+ }
+
+ // look for a workgroup
+ std::string workgroup = attachUrl.getQueryParam("workgroup");
+ if ( workgroup.empty() )
+ workgroup = attachUrl.getQueryParam("domain");
+ if ( !workgroup.empty() )
+ options["domain"] = workgroup;
+
+ // extract 'username', do not overwrite any _url.username
+
+ zypp::media::Mount::Options::iterator toEnv;
+ toEnv = options.find("username");
+ if ( toEnv != options.end() ) {
+ if ( username.empty() )
+ username = toEnv->second;
+ options.erase( toEnv );
+ }
+
+ toEnv = options.find("user"); // actually cifs specific
+ if ( toEnv != options.end() ) {
+ if ( username.empty() )
+ username = toEnv->second;
+ options.erase( toEnv );
+ }
+
+ // extract 'password', do not overwrite any _url.password
+
+ toEnv = options.find("password");
+ if ( toEnv != options.end() ) {
+ if ( password.empty() )
+ password = toEnv->second;
+ options.erase( toEnv );
+ }
+
+ toEnv = options.find("pass"); // actually cifs specific
+ if ( toEnv != options.end() ) {
+ if ( password.empty() )
+ password = toEnv->second;
+ options.erase( toEnv );
+ }
+
+ bool canContinue = true;
+ while ( canContinue ) {
+
+ canContinue = false;
+
+ // pass 'username' and 'password' via environment
+ zypp::media::Mount::Environment environment;
+ if ( !username.empty() )
+ environment["USER"] = username;
+ if ( !password.empty() )
+ environment["PASSWD"] = password;
+
+ // In case we need a tmpfile, credentials will remove
+ // it in its destructor after the mount call below.
+ zypp::filesystem::TmpPath credentials;
+ if ( !username.empty() || !password.empty() )
+ {
+ zypp::filesystem::TmpFile tmp;
+ std::ofstream outs( tmp.path().asString().c_str() );
+ outs << "username=" << username << std::endl;
+ outs << "password=" << password << std::endl;
+ outs.close();
+
+ credentials = tmp;
+ options["credentials"] = credentials.path().asString();
+
+ } else {
+ // Use 'guest' option unless explicitly disabled (bnc #547354)
+ if ( options.has( "noguest" ) )
+ options.erase( "noguest" );
+ else
+ // prevent smbmount from asking for password
+ // only add this option if 'credentials' is not used (bnc #560496)
+ options["guest"];
+ }
+
+ zypp::Pathname newAp;
+ zypp::media::Mount mount;
+ try
+ {
+ newAp = createAttachPoint( attachRoot() );
+ if ( newAp.empty() ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , "Failed to create mount directory."
+ , false
+ );
+ }
+
+ mount.mount( path, newAp.asString(), "cifs", options.asString(), environment );
+
+ // wait for /etc/mtab update ...
+ // (shouldn't be needed)
+ int limit = 3;
+ bool mountsucceeded = false;
+
+ while( !(mountsucceeded=checkAttached( newAp, DeviceDriver::fstypePredicate( path, {"smb", "cifs"} ) )) && --limit) {
+ MIL << "Mount did not appear yet, sleeping for 1s" << std::endl;
+ sleep(1);
+ }
+
+ if ( !mountsucceeded ) {
+ try {
+ mount.umount( newAp.asString() );
+ } catch (const zypp::media::MediaException & excpt_r) {
+ ZYPP_CAUGHT(excpt_r);
+ }
+ ZYPP_THROW( zypp::media::MediaMountException(
+ "Unable to verify that the media was mounted",
+ path, newAp.asString()
+ ));
+ }
+
+ // if we reach this place, mount worked -> YAY, lets see if that is the desired medium!
+ const auto &isDesired = isDesiredMedium( attachUrl, newAp, verifier, extras.value( zyppng::AttachMsgFields::MediaNr, 1 ).asInt() );
+ if ( !isDesired ) {
+ try {
+ mount.umount( newAp.asString() );
+ } catch (const zypp::media::MediaException & excpt_r) {
+ ZYPP_CAUGHT(excpt_r);
+ }
+ ZYPP_THROW( zypp::media::MediaNotDesiredException( attachUrl ) );
+ }
+
+ MIL << "New device " << path << " mounted on " << newAp << std::endl;
+ auto newDev = std::shared_ptr<zyppng::worker::Device>( new zyppng::worker::Device{
+ ._name = path,
+ ._maj_nr = 0,
+ ._min_nr = 0,
+ ._mountPoint = newAp,
+ ._ephemeral = true, // device should be removed after the last attachment was released
+ ._properties = {}
+ });
+ knownDevices().push_back( newDev );
+ attachedMedia().insert( std::make_pair( attachId, zyppng::worker::AttachedMedia{ newDev, "/" } ) );
+ return zyppng::worker::AttachResult::success();
+
+ } catch (const zypp::media::MediaMountException & e) {
+ ZYPP_CAUGHT( e );
+
+ if ( e.mountError() == "Permission denied" ) {
+ auto parent = parentWorker();
+ if ( parent ) {
+ auto res = parent->requireAuthorization( id, attachUrl, username, lastTimestamp );
+ if ( res ) {
+ canContinue = true;
+ username = res->username;
+ password = res->password;
+ lastTimestamp = res->last_auth_timestamp;
+ }
+ }
+ if ( !canContinue ) {
+ removeAttachPoint(newAp);
+ return zyppng::worker::AttachResult::error( zyppng::ProvideMessage::Code::Forbidden, "No authentication data", false );
+ }
+ } else {
+ removeAttachPoint(newAp);
+ return zyppng::worker::AttachResult::error( zyppng::ProvideMessage::Code::MountFailed, false, e );
+ }
+ } catch (const zypp::media::MediaException & e) {
+ removeAttachPoint(newAp);
+ ZYPP_CAUGHT(e);
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , false
+ , e );
+ }
+ }
+
+ // We should never reach this place, but if we do fail the attach request
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , zypp::str::Str()<<"Mounting the medium " << attachUrl << " failed for an unknown reason"
+ , false
+ );
+
+ } catch ( const zypp::Exception &e ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::BadRequest
+ , false
+ , e );;
+ } catch ( const std::exception &e ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::BadRequest
+ , e.what()
+ , false
+ );
+ } catch ( ... ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::BadRequest
+ , "Unknown exception"
+ , false
+ );
+ }
+}
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+#ifndef ZYPP_NG_TOOLS_SMBPROVIDER_H_INCLUDED
+#define ZYPP_NG_TOOLS_SMBPROVIDER_H_INCLUDED
+
+#include <zypp-media/ng/worker/DeviceDriver>
+
+class SmbProvider : public zyppng::worker::DeviceDriver
+{
+ public:
+ SmbProvider();
+ ~SmbProvider();
+
+ // DeviceDriver interface
+ zyppng::worker::AttachResult mountDevice ( const uint32_t id, const zypp::Url &attachUrl, const std::string &attachId, const std::string &label, const zyppng::HeaderValueMap &extras ) override;
+};
+
+#endif
--- /dev/null
+PROJECT( zypp-media-tvm C CXX )
+
+set (CMAKE_CXX_STANDARD 17)
+SET (CMAKE_CXX_EXTENSIONS OFF)
+
+FIND_PACKAGE(Protobuf REQUIRED)
+
+SET( SOURCES
+ main.cc
+ testvmprovider.cc
+ testvmprovider.h
+)
+
+add_executable( ${PROJECT_NAME} ${SOURCES} )
+target_link_libraries( ${PROJECT_NAME} zypp-media )
+target_link_libraries( ${PROJECT_NAME} zypp-core )
+target_link_libraries( ${PROJECT_NAME} zypp-protobuf )
+target_link_libraries( ${PROJECT_NAME} ${PROTOBUF_LITE_LIBRARIES} )
+
+set_target_properties( ${PROJECT_NAME}
+ PROPERTIES
+ ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+ LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+ RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers"
+)
--- /dev/null
+#include "testvmprovider.h"
+
+#include <csignal>
+#include <zypp-core/zyppng/base/private/linuxhelpers_p.h>
+#include <zypp-media/ng/worker/MountingWorker>
+
+int main( int , char *[] )
+{
+ // lets ignore those, SIGPIPE is better handled via the EPIPE error, and we do not want anyone
+ // to CTRL+C us
+ zyppng::blockSignalsForCurrentThread( { SIGPIPE, SIGINT } );
+
+ auto driver = std::make_shared<TestVMProvider>();
+ auto worker = std::make_shared<zyppng::worker::MountingWorker>( "zypp-media-tvm", driver );
+ driver->setProvider(worker);
+
+ auto res = worker->run (STDIN_FILENO, STDOUT_FILENO);
+ if ( res )
+ return 0;
+
+ //@TODO print error
+ return 1;
+}
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+#include "testvmprovider.h"
+#include <zypp-media/ng/private/providedbg_p.h>
+#include <zypp-media/ng/worker/ProvideWorker>
+#include <zypp-core/fs/PathInfo.h>
+#include <zypp-core/AutoDispose.h>
+#include <zypp-core/base/StringV.h>
+#include <zypp-media/ng/MediaVerifier>
+#include <zypp-proto/tvm.pb.h>
+#include <zypp-core/zyppng/rpc/zerocopystreams.h>
+#include <zypp-core/zyppng/base/private/linuxhelpers_p.h>
+
+
+#undef ZYPP_BASE_LOGGER_LOGGROUP
+#define ZYPP_BASE_LOGGER_LOGGROUP "TestVMProvider"
+
+#include <iostream>
+#include <fstream>
+
+extern "C"
+{
+#include <sys/ioctl.h>
+#include <linux/cdrom.h>
+#if HAVE_UDEV
+#include <libudev.h>
+#endif
+}
+
+using namespace std::literals;
+
+constexpr std::string_view CONTENTDIR_PROP = "contentDir";
+
+TestVMProvider::TestVMProvider( ) : zyppng::worker::DeviceDriver( zyppng::worker::WorkerCaps::VolatileMount )
+{ }
+
+TestVMProvider::~TestVMProvider()
+{ }
+
+zyppng::expected<zyppng::worker::WorkerCaps> TestVMProvider::initialize(const zyppng::worker::Configuration &conf)
+{
+ const auto &values = conf.values();
+
+ if ( const auto &i = values.find( std::string(zyppng::PROVIDER_ROOT) ); i != values.end() ) {
+ const auto &val = i->second;
+ MIL << "Got provider root from controller: " << val << std::endl;
+ _provRoot = val;
+ } else {
+ return zyppng::expected<zyppng::worker::WorkerCaps>::error(ZYPP_EXCPT_PTR( zypp::Exception("Provider root required to work.") ));
+ }
+
+ return DeviceDriver::initialize( conf );
+}
+
+zyppng::worker::AttachResult TestVMProvider::mountDevice ( const uint32_t id, const zypp::Url &attachUrl, const std::string &attachId, const std::string &label, const zyppng::HeaderValueMap &extras )
+{
+ std::vector<std::string> devs;
+ if ( extras.contains( zyppng::AttachMsgFields::Device ) ) {
+ for ( const auto &d : extras.values( zyppng::AttachMsgFields::Device ) ) {
+ if ( d.isString() ) {
+ MIL << "Got device from user we are allowed to use: " << d.asString() << std::endl;
+ devs.push_back(d.asString());
+ }
+ }
+ }
+
+ auto &sysDevs = knownDevices();
+ std::vector<std::shared_ptr<zyppng::worker::Device>> possibleDevs;
+ if ( devs.size () ) {
+ for ( const auto &d : devs ) {
+ std::shared_ptr<zyppng::worker::Device> device;
+ auto i = std::find_if( sysDevs.begin(), sysDevs.end(), [&]( const auto &sysDev ){ return ( sysDev->_name == d ); } );
+ if ( i == sysDevs.end() ) {
+ ERR << "Device " << d << " from Attach request is not known ignoring!" << std::endl;
+ continue;
+ } else {
+ device = *i;
+ }
+
+ if ( device )
+ possibleDevs.push_back(device);
+ }
+ } else {
+ possibleDevs = sysDevs;
+ }
+
+ // if we have no devices at this point controller gets a error
+ if ( possibleDevs.empty () ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , "No useable device found"
+ , false
+ );
+ }
+
+ const auto attachRoot = zypp::Pathname( attachUrl.getPathName() );
+ const auto attachMediaNr = extras.value( zyppng::AttachMsgFields::MediaNr, 1 ).asInt();
+
+ // set up the verifier
+ zyppng::MediaDataVerifierRef verifier;
+ if ( extras.contains( zyppng::AttachMsgFields::VerifyType ) ) {
+ verifier = zyppng::MediaDataVerifier::createVerifier( extras.value(zyppng::AttachMsgFields::VerifyType).asString() );
+ if ( !verifier ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , "Invalid verifier type"
+ , false
+ );
+ }
+
+ if ( !verifier->load( extras.value(zyppng::AttachMsgFields::VerifyData).asString() ) ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , "Failed to create verifier from file"
+ , false
+ );
+ }
+ }
+
+ //first check if any of the mounted devices are what we want
+ for( const auto &dev : possibleDevs ) {
+ if ( dev->_mountPoint.empty() )
+ continue;
+
+ MIL << "Found mounted device: " << dev->_name << " on mountpoint: " << dev->_mountPoint << std::endl;
+
+ auto res = isDesiredMedium( attachUrl, dev->_mountPoint / attachRoot, verifier, attachMediaNr );
+ if ( !res )
+ continue;
+
+ MIL << "Found requested medium in dev " << dev->_name << std::endl;
+
+ // we found the device we want!
+ attachedMedia().insert( std::make_pair( attachId, zyppng::worker::AttachedMedia{ dev, attachRoot } ) );
+ return zyppng::worker::AttachResult::success();
+ }
+
+ while ( true ) {
+
+ // pick up changes from the test
+ detectDevices();
+
+ // remember how many devices we were able to test
+ uint devicesTested = 0;
+
+ // none of the already mounted devices matched, lets try what we have left
+ for( const auto &dev : possibleDevs ) {
+ if ( !dev->_mountPoint.empty() )
+ continue;
+
+ MIL << "Trying to mount dev " << dev->_name << std::endl;
+
+ devicesTested++;
+
+ bool mountsucceeded = false;
+
+ // we simply create a symbolic link in the attach dir to simulate mounting
+ zypp::Pathname newAp;
+ try {
+ newAp = createAttachPoint( this->attachRoot() );
+ if ( newAp.empty() ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MountFailed
+ , "Failed to create mount directory."
+ , false
+ );
+ }
+
+ newAp = newAp/"mount";
+
+ // simulate failed mount
+ const auto &contDir = std::any_cast<zypp::Pathname>(dev->_properties[std::string(CONTENTDIR_PROP)]);
+ if ( contDir.empty() ) {
+ ZYPP_THROW( zypp::media::MediaMountException(
+ "Failed to mount device",
+ dev->_name, newAp.asString()
+ ));
+ }
+
+ const auto res = zypp::filesystem::symlink( contDir, newAp );
+ mountsucceeded = ( res == 0 );
+
+ if( !mountsucceeded) {
+ ZYPP_THROW( zypp::media::MediaMountException(
+ "Failed to mount device",
+ dev->_name, newAp.asString()
+ ));
+ } else {
+ MIL << "Device mounted on " << newAp << std::endl;
+ dev->_mountPoint = newAp;
+ }
+ } catch (const zypp::media::MediaException & excpt_r) {
+ removeAttachPoint(newAp.dirname());
+ ZYPP_CAUGHT(excpt_r);
+ }
+
+ if ( mountsucceeded ) {
+ // lets check if we have the correct medium
+ auto res = isDesiredMedium( attachUrl, dev->_mountPoint / attachRoot, verifier, attachMediaNr );
+ if ( !res ) {
+ unmountDevice( *dev );
+ continue;
+ }
+
+ MIL << "Found requested medium in dev " << dev->_name << std::endl;
+ attachedMedia().insert( std::make_pair( attachId, zyppng::worker::AttachedMedia{ dev, attachRoot } ) );
+ return zyppng::worker::AttachResult::success();
+ }
+ } // for each device
+
+ // we did go through all devices, but didn't find any.
+ if ( devicesTested == 0 ) {
+ // no devices are free
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::Jammed
+ , "No free ressources available"
+ , false
+ );
+ } else {
+
+ // k, we need to ask the user to give us the medium we need
+ // first find the devices that are free
+ std::vector<std::string> freeDevs;
+
+ for( const auto &dev : possibleDevs ) {
+ if ( !dev->_mountPoint.empty() )
+ continue;
+
+ MIL << "Adding " << dev->_name << " to list of free devs" << std::endl;
+ freeDevs.push_back( dev->_name );
+ }
+
+ // if there are no devices free, we are jammed
+ if ( freeDevs.empty() ) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::Jammed
+ , "No free ressources available"
+ , false
+ );
+ }
+
+ auto changeRes = zyppng::worker::ProvideWorker::ABORT;
+ auto parent = parentWorker();
+ if ( parent )
+ changeRes = parent->requestMediaChange ( id, label, attachMediaNr, freeDevs );
+ if ( changeRes != zyppng::worker::ProvideWorker::SUCCESS ) {
+ if ( changeRes == zyppng::worker::ProvideWorker::SKIP) {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MediaChangeSkip
+ , "User asked to skip the media change."
+ , false
+ );
+ } else {
+ return zyppng::worker::AttachResult::error(
+ zyppng::ProvideMessage::Code::MediaChangeAbort
+ , "User asked to abort the media change."
+ , false
+ );
+ }
+ }
+ }
+ }
+}
+
+void TestVMProvider::unmountDevice( zyppng::worker::Device &dev )
+{
+ if ( dev._mountPoint.empty () )
+ return;
+ try {
+ zypp::filesystem::unlink( dev._mountPoint );
+ removeAttachPoint( dev._mountPoint.dirname() );
+ } catch (const zypp::media::MediaException & excpt_r) {
+ ERR << "Failed to unmount device: " << dev._name << std::endl;
+ ZYPP_CAUGHT(excpt_r);
+ }
+ dev._mountPoint = zypp::Pathname();
+}
+
+void TestVMProvider::detectDevices( )
+{
+ const auto &fname = _provRoot/"tvm.conf";
+ int fd = open( fname.asString().data(), O_RDONLY );
+ if ( fd < 0 ) {
+ ERR << "Failed to open file " << fname << " error: "<<"("<<errno<<")"<<zyppng::strerr_cxx(errno)<< std::endl;
+ return;
+ }
+
+ zyppng::FileInputStream in( fd );
+ in.SetCloseOnDelete(true);
+
+ zypp::proto::test::TVMSettings set;
+ if ( !set.ParseFromZeroCopyStream( &in ) ) {
+ MIL << "No devices configured!" << std::endl;
+ return;
+ }
+
+ auto &sysDevs = knownDevices();
+
+ if ( sysDevs.size() ) {
+ for ( int i = 0; i < set.devices_size(); i++ ) {
+ const auto &dev = set.devices(i);
+ auto iDev = std::find_if( sysDevs.begin(), sysDevs.end(), [&]( const auto &d ){
+ return d->_name == dev.name();
+ });
+ if ( iDev == sysDevs.end() ) {
+ MIL << "Previously unknown device detected" << std::endl;
+ auto d = std::make_shared<zyppng::worker::Device>( zyppng::worker::Device{
+ ._name = dev.name()
+ });
+ if ( dev.insertedpath().size() ) {
+ d->_properties[std::string(CONTENTDIR_PROP)] = zypp::Pathname(dev.insertedpath());
+ }
+ sysDevs.push_back(d);
+ continue;
+ }
+
+ // mounted devices are never updated
+ if ( !(*iDev)->_mountPoint.empty() ) {
+ continue;
+ }
+
+ (*iDev)->_properties[std::string(CONTENTDIR_PROP)] = zypp::Pathname(dev.insertedpath());
+ }
+ } else {
+ for ( int i = 0; i < set.devices_size(); i++ ) {
+ const auto &dev = set.devices(i);
+ MIL << "Found device: " << dev.name() << std::endl;
+ auto d = std::make_shared<zyppng::worker::Device>( zyppng::worker::Device{
+ ._name = dev.name()
+ });
+ if ( dev.insertedpath().size() ) {
+ d->_properties[std::string(CONTENTDIR_PROP)] = zypp::Pathname(dev.insertedpath());
+ }
+ sysDevs.push_back(d);
+ }
+ }
+}
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+#ifndef ZYPP_NG_TOOLS_DISCPROVIDER_H_INCLUDED
+#define ZYPP_NG_TOOLS_DISCPROVIDER_H_INCLUDED
+
+#include <zypp-media/ng/worker/DeviceDriver>
+#include <zypp-core/zyppng/base/Signals>
+#include <any>
+#include <unordered_map>
+
+/*!
+ * \file Contains a testprovider to simulate volatile mounting devices.
+ * This is only used in the zypp testsuite.
+ */
+
+class TestVMProvider : public zyppng::worker::DeviceDriver
+{
+public:
+
+ TestVMProvider();
+ ~TestVMProvider();
+
+ // DeviceDriver interface
+ zyppng::expected<zyppng::worker::WorkerCaps> initialize(const zyppng::worker::Configuration &conf) override;
+ zyppng::worker::AttachResult mountDevice ( const uint32_t id, const zypp::Url &attachUrl, const std::string &attachId, const std::string &label, const zyppng::HeaderValueMap &extras ) override;
+ void detectDevices();
+
+protected:
+ void unmountDevice ( zyppng::worker::Device &dev ) override;
+
+private:
+ zypp::Pathname _provRoot;
+};
+
+#endif
void swap( AutoDispose & rhs )
{ _pimpl.swap( rhs._pimpl ); }
+ /** Returns true if this is the only AutoDispose instance managing the current data object */
+ bool unique () const
+ { return _pimpl.unique(); }
+
public:
/** Return the current dispose function. */
const Dispose & getDispose() const
return std::string_view( data(), size() );
}
#endif
+
+ static std::size_t maxSize () {
+ static const auto size = ByteArray().max_size();
+ return size;
+ }
+
};
class UByteArray : public std::vector<unsigned char>
public:
using vector<unsigned char>::vector;
explicit UByteArray ( const char *data, const int len = -1 ) : UByteArray( data, data + (len == -1 ? strlen(data) : len) ) { }
+
+ static std::size_t maxSize () {
+ static const auto size = UByteArray().max_size();
+ return size;
+ }
};
}
)
SET( zyppng_io_private_HEADERS
+ zyppng/io/private/asyncdatasource_p.h
zyppng/io/private/iobuffer_p.h
zyppng/io/private/iodevice_p.h
zyppng/io/private/socket_p.h
SET( zyppng_rpc_HEADERS
zyppng/rpc/rpc.h
+ zyppng/rpc/MessageStream
+ zyppng/rpc/messagestream.h
zyppng/rpc/zerocopystreams.h
)
SET( zyppng_rpc_SRCS
zyppng/rpc/rpc.cc
+ zyppng/rpc/messagestream.cc
zyppng/rpc/zerocopystreams.cc
)
# Default loggroup for all files
SET_LOGGROUP( "zypp-core" ${zypp_SRCS} )
+# System libraries
+SET(UTIL_LIBRARY util)
+
ADD_LIBRARY( zypp-core STATIC ${zypp_SRCS} ${zypp_HEADERS} )
#we include generated headers, so we need to wait for zypp-protobuf to be ready
add_dependencies( zypp-core zypp-protobuf )
target_link_libraries( zypp-core ${CRYPTO_LIBRARIES} )
target_link_libraries( zypp-core pthread )
target_link_libraries( zypp-core ${SIGCPP_LIBRARIES} )
+TARGET_LINK_LIBRARIES( zypp-core ${ZLIB_LIBRARY} )
+message("URIL LIB ${UTIL_LIBRARY}")
+TARGET_LINK_LIBRARIES( zypp-core ${UTIL_LIBRARY} )
+
+IF (ENABLE_ZSTD_COMPRESSION)
+ TARGET_LINK_LIBRARIES( zypp-core ${ZSTD_LIBRARY})
+ENDIF (ENABLE_ZSTD_COMPRESSION)
+
+IF (ENABLE_ZCHUNK_COMPRESSION)
+ TARGET_LINK_LIBRARIES( zypp-core ${ZCHUNK_LDFLAGS})
+ENDIF(ENABLE_ZCHUNK_COMPRESSION)
return base.substr( pos );
}
+ Pathname Pathname::realpath() const
+ {
+ std::string real;
+ if( !empty())
+ {
+ #if __GNUC__ > 2
+ /** GNU extension */
+ char *ptr = ::realpath(_name.c_str(), NULL);
+ if( ptr != NULL)
+ {
+ real = ptr;
+ free( ptr);
+ }
+ else
+ /** the SUSv2 way */
+ if( EINVAL == errno)
+ {
+ char buff[PATH_MAX + 2];
+ memset(buff, '\0', sizeof(buff));
+ if( ::realpath(_name.c_str(), buff) != NULL)
+ {
+ real = buff;
+ }
+ }
+ #else
+ char buff[PATH_MAX + 2];
+ memset(buff, '\0', sizeof(buff));
+ if( ::realpath(_name.c_str(), buff) != NULL)
+ {
+ real = buff;
+ }
+ #endif
+ }
+ return zypp::Pathname(real);
+ }
+
///////////////////////////////////////////////////////////////////
//
// METHOD NAME : Pathname::assertprefix
static Pathname relativename( const Pathname & name_r )
{ return name_r.absolute() ? cat( ".", name_r ) : name_r; }
+ /** Returns this path as the absolute canonical pathname */
+ Pathname realpath() const;
+
/** Return \c path_r prefixed with \c root_r, unless it is already prefixed. */
static Pathname assertprefix( const Pathname & root_r, const Pathname & path_r );
std::exception_ptr do_ZYPP_EXCPT_PTR( const TExcpt & excpt_r, const CodeLocation & where_r )
{
excpt_r.relocate( where_r );
- Exception::log( excpt_r, where_r, "THROW: " );
+ Exception::log( excpt_r, where_r, "EXCPTR: " );
return std::make_exception_ptr( excpt_r );
}
template<class TExcpt, EnableIfNotException<TExcpt>>
std::exception_ptr do_ZYPP_EXCPT_PTR( const TExcpt & excpt_r, const CodeLocation & where_r )
{
- Exception::log( typeid(excpt_r).name(), where_r, "THROW: " );
+ Exception::log( typeid(excpt_r).name(), where_r, "EXCPTR: " );
return std::make_exception_ptr( excpt_r );
}
auto writer = getLineWriter();
if ( writer ) {
for ( auto &sock : clients ){
- while ( sock->canReadLine() ) {
- auto br = sock->readLine();
- writer->writeOut( std::string( br.data(), br.size() - 1 ) );
+ auto br = sock->readLine();
+ while ( !br.empty() ) {
+ if ( br.back () == '\n' )
+ writer->writeOut( std::string( br.data(), br.size() - 1 ) );
+ else
+ writer->writeOut( std::string( br.data(), br.size() ) );
+
+ br = sock->readLine();
}
}
}
/** \overload Setter */
void hideThreadName( bool onOff_r )
{ _hideThreadName = onOff_r; }
+
/** \overload Static getter */
static bool instanceHideThreadName()
{
if ( impl ) impl->hideThreadName( onOff_r );
}
+ /** Hint for formatter wether we forward all logs to a parents log */
+ static bool instanceLogToPPID( )
+ {
+ auto impl = LogControlImpl::instance();
+ return impl ? impl->_logToPPIDMode : false;
+ }
+
+ /** \overload Static setter */
+ static void instanceSetLogToPPID( bool onOff_r )
+ {
+ auto impl = LogControlImpl::instance();
+ if ( impl )
+ impl->_logToPPIDMode = onOff_r;
+ }
/** NULL _lineWriter indicates no loggin. */
void setLineWriter( const shared_ptr<LogControl::LineWriter> & writer_r )
LogClient _logClient;
std::ostream _no_stream;
bool _excessive;
+ bool _logToPPIDMode = false; ///< Hint for formatter to use the PPID and always show the thread name
mutable TriBool _hideThreadName = indeterminate; ///< Hint for Formater whether to hide the thread name.
shared_ptr<LogControl::LineFormater> _lineFormater;
return ret;
}
+ void putRawLine ( std::string &&line ) {
+ _logClient.pushMessage( std::move(line) );
+ }
+
/** Format and write out a logline from Loglinebuf. */
void putStream( const std::string & group_r,
LogLevel level_r,
static char nohostname[] = "unknown";
std::string now( Date::now().form( "%Y-%m-%d %H:%M:%S" ) );
std::string ret;
- if ( LogControlImpl::instanceHideThreadName() )
+
+ const bool logToPPID = LogControlImpl::instanceLogToPPID();
+ if ( !logToPPID && LogControlImpl::instanceHideThreadName() )
ret = str::form( "%s <%d> %s(%d) [%s] %s(%s):%d %s",
now.c_str(), level_r,
( gethostname( hostname, 1024 ) ? nohostname : hostname ),
ret = str::form( "%s <%d> %s(%d) [%s] %s(%s):%d {T:%s} %s",
now.c_str(), level_r,
( gethostname( hostname, 1024 ) ? nohostname : hostname ),
- getpid(),
+ logToPPID ? getppid() : getpid(),
group_r.c_str(),
file_r, func_r, line_r,
zyppng::ThreadData::current().name().c_str(),
impl->setLineFormater( formater_r );
}
+ void LogControl::enableLogForwardingMode(bool enable)
+ {
+ LogControlImpl::instanceSetLogToPPID ( enable );
+ }
+
void LogControl::logNothing()
{
auto impl = LogControlImpl::instance();
logger::logControlValidFlag () = 0;
}
+ void LogControl::logRawLine ( std::string &&line )
+ {
+ LogControlImpl::instance ()->putRawLine ( std::move(line) );
+ }
+
///////////////////////////////////////////////////////////////////
//
// LogControl::TmpExcessive
*/
void setLineFormater( const shared_ptr<LineFormater> & formater_r );
+ /*!
+ * Sets a special log mode that uses the ppid in the log output and always shows the
+ * thread name (if the default formatter is used). This is only used by internal zypp tools that directly want to log into the
+ * parents logfile.
+ */
+ void enableLogForwardingMode ( bool enable = true );
+
public:
/** Set path for the logfile.
* Permission for logfiles is set to 0640 unless an explicit mode_t
*/
static void notifyFork();
+ /** will push a line to the logthread without formatting it */
+ void logRawLine ( std::string &&line );
+
public:
/** Get the current LineWriter */
shared_ptr<LineWriter> getLineWriter() const;
#define L_INT(GROUP) ZYPP_BASE_LOGGER_LOG( GROUP, zypp::base::logger::E_INT )
#define L_USR(GROUP) ZYPP_BASE_LOGGER_LOG( GROUP, zypp::base::logger::E_USR )
+
#define L_ENV_CONSTR_DEFINE_FUNC(ENV) \
- const char *empty_or_group_if_##ENV ( const char *group ) \
- { \
- static bool has_##ENV = (::getenv(#ENV) != NULL); \
- return has_##ENV ? group : nullptr; \
+ namespace zypp::log { \
+ bool has_env_constr_##ENV () \
+ { \
+ static bool has_##ENV = (::getenv(#ENV) != NULL); \
+ return has_##ENV; \
+ } \
+ const char *empty_or_group_if_##ENV ( const char *group ) \
+ { \
+ return has_env_constr_##ENV() ? group : nullptr; \
+ } \
}
-#define L_ENV_CONSTR_FWD_DECLARE_FUNC(ENV) const char *empty_or_group_if_##ENV ( const char *group )
-#define L_ENV_CONSTR(ENV,GROUP,LEVEL) ZYPP_BASE_LOGGER_LOG( zypp::empty_or_group_if_##ENV( #GROUP ), LEVEL )
+#define L_ENV_CONSTR_FWD_DECLARE_FUNC(ENV) namespace zypp::log { bool has_env_constr_##ENV (); const char *empty_or_group_if_##ENV ( const char *group ); }
+#define L_ENV_CONSTR(ENV,GROUP,LEVEL) ZYPP_BASE_LOGGER_LOG( zypp::log::empty_or_group_if_##ENV( #GROUP ), LEVEL )
#define L_BASEFILE ( *__FILE__ == '/' ? strrchr( __FILE__, '/' ) + 1 : __FILE__ )
#ifndef ZYPPNG_ASYNC_ASYNCOP_H_INCLUDED
#define ZYPPNG_ASYNC_ASYNCOP_H_INCLUDED
-
+#include <zypp-core/zyppng/base/Base>
#include <zypp-core/base/Exception.h>
#include <zypp-core/zyppng/base/Signals>
-#include <boost/optional.hpp>
+#include <optional>
#include <memory>
namespace zyppng {
virtual ~AsyncOpNotReadyException();
};
- AsyncOpNotReadyException::~AsyncOpNotReadyException()
+ class CancelNotImplementedException : public zypp::Exception
+ {
+ public:
+ CancelNotImplementedException()
+ : Exception ("AsyncOp does not support cancelling the operation")
+ {}
+ virtual ~CancelNotImplementedException();
+
+
+ };
+
+ inline AsyncOpNotReadyException::~AsyncOpNotReadyException()
{ }
+
+ inline CancelNotImplementedException::~CancelNotImplementedException()
+ { }
+
+
+ struct AsyncOpBase : public Base {
+
+ /*!
+ * Should return true if the async op supports cancelling, otherwise false
+ */
+ virtual bool canCancel () {
+ return false;
+ }
+
+ /*!
+ * Explicitely cancels the async operation if it supports that,
+ * otherwise throws a \ref CancelNotImplementedException
+ *
+ * Generally every AsyncOp HAS TO support cancelling by releasing the last reference
+ * to it, however in some special cases we need to be able to cancel programatically
+ * from the backends and notify the frontends by returning a error from a AsyncOp.
+ */
+ virtual void cancel () {
+ throw CancelNotImplementedException();
+ }
+
+ /*!
+ * Signal that is emitted once the AsyncOp is started
+ */
+ SignalProxy<void()> sigStarted () {
+ return _sigStarted;
+ }
+
+ /*!
+ * Signal that might be emitted during operation of the AsyncOp,
+ * not every AsyncOp will use this
+ */
+ SignalProxy<void( const std::string & /*text*/, int /*current*/, int /*max*/ )> sigProgress () {
+ return _sigProgress;
+ }
+
+ /*!
+ * Signal that is emitted once the AsyncOp is ready and no
+ * callback was registered with \ref onReady
+ */
+ SignalProxy<void()> sigReady () {
+ return _sigReady;
+ }
+
+ protected:
+ Signal<void()> _sigStarted;
+ Signal<void()> _sigReady;
+ Signal<void( const std::string & /*text*/, int /*current*/, int /*max*/ )> _sigProgress;
+ };
+
+
+
/*!
*\class AsyncOp
* The \a AsyncOp template class is the basic building block for the asynchronous pipelines.
* Its the base for all async callbacks as well as the async result type. That basically means
* every pipeline is just a AsyncOp that contains all previous operations that were defined in the pipeline.
*
- * When implementing a async operation it is required to add a operator() to the class taking the
+ * When implementing a async operation that is to be used in a pipeline it is required to add a operator() to the class taking the
* input parameter. After the operation is finished the implementation must call setReady(). Calling
* setReady() must be treated like calling return in a normal function and not execute anymore code on the
* AsyncOp instance afterwards, since the next operation in the pipeline is allowed to free the previous operation
* is cached internally and can be retrieved with \sa get().
*
* A async operation can be cancelled by releasing the result object ( the resulting combinatory object ), this will
- * destroy all previous operations that are either running or pending as well.
+ * destroy all previous operations that are either running or pending as well. The destructors MUST NOT call setReady() but
+ * only release currently running operations.
*/
template <typename Result>
- struct AsyncOp {
+ struct AsyncOp : public AsyncOpBase {
using value_type = Result;
+ using Ptr = std::shared_ptr<AsyncOp<Result>>;
AsyncOp () = default;
* storing it.
*/
void setReady ( value_type && val ) {
- if ( _readyCb )
+ if ( _readyCb ) {
+ // use a weak reference to know if `this` was deleted by the callback()
+ auto weak = weak_from_this();
_readyCb( std::move( val ) );
+ if ( !weak.expired() )
+ _readyCb = {};
+ }
else { //we need to cache the value because no callback is available
_maybeValue = std::move(val);
_sigReady.emit();
* \note This can only be used when no callback is registered.
*/
bool isReady () const {
- return _maybeValue.is_initialized();
+ return _maybeValue.has_value();
}
/*!
* when the object gets into ready state. In case the
* object is in ready state when registering the callback
* it is called right away.
+ * \note this will disable the emitting of sigReady() so it should
+ * be only used by the pipeline implementation
+ *
+ * \note the callback is removed from the AsyncOp after it was executed
+ * when the AsyncOp becomes ready
*/
template< typename Fun >
void onReady ( Fun cb ) {
this->_readyCb = std::forward<Fun>(cb);
if ( isReady() ) {
- _readyCb( std::move( _maybeValue.get()) );
- _maybeValue = boost::optional<value_type>();
+ // use a weak reference to know if `this` was deleted by the callback()
+ auto weak = weak_from_this();
+
+ _readyCb( std::move( _maybeValue.value()) );
+ _maybeValue.reset();
+
+ // reset the callback, it might be a lambda with captured ressources
+ // that need to be released after returning from the func
+ if ( !weak.expired() )
+ _readyCb = {};
}
}
value_type &get (){
if ( !isReady() )
ZYPP_THROW(AsyncOpNotReadyException());
- return _maybeValue.get();
- }
-
- /*!
- * Signal that is emitted once the AsyncOp is ready and no
- * callback was registered with \ref onReady
- */
- SignalProxy<void()> sigReady () {
- return _sigReady;
+ return _maybeValue.value();
}
private:
- Signal<void()> _sigReady;
std::function<void(value_type &&)> _readyCb;
- boost::optional<value_type> _maybeValue;
+ std::optional<value_type> _maybeValue;
};
template <typename T>
- using AsyncOp
Ptr = std::unique_ptr<AsyncOp<T>>;
+ using AsyncOp
Ref = std::shared_ptr<AsyncOp<T>>;
namespace detail {
+ //A async result that is ready right away
+ template <typename T>
+ struct ReadyResult : public zyppng::AsyncOp< T >
+ {
+ ReadyResult( T &&val ) {
+ this->setReady( std::move(val) );
+ }
+ };
+
#if 0
template <typename T>
using has_value_type_t = typename T::value_type;
}
+ template <typename T>
+ AsyncOpRef<T> makeReadyResult ( T && result ) {
+ return std::make_shared<detail::ReadyResult<T>>( std::move(result) );
+ }
+
}
}
/*!
- * Preferred way to connect a signal to a slow, this will automatically take care of tracking the target object in the connection
+ * Preferred way to connect a signal to a slot, this will automatically take care of tracking the target object in the connection
*/
template< typename SenderFunc, typename ReceiverFunc >
static auto connect ( typename internal::MemberFunction<SenderFunc>::ClassType &s, SenderFunc &&sFun, typename internal::MemberFunction<ReceiverFunc>::ClassType &recv, ReceiverFunc &&rFunc ) {
GAbstractEventSource::destruct( src );
});
_runningTimers.clear();
+ _eventSources.clear();
if ( _idleSource ) {
g_source_destroy( _idleSource );
if ( errno == EINTR )
continue;
- break;
+ ERR << "g_poll error: " << strerror(errno) << std::endl;
+ return false;
}
case 1:
eventTriggered = true;
}
}
- int bytesAvailableOnFD(int fd)
+ int64_t bytesAvailableOnFD( int fd )
{
int value;
if ( ioctl( fd, FIONREAD, &value) >= 0 )
- return value;
+ return int64_t(value);
return 0;
}
* Tries to use the FIONREAD ioctl to detect how many bytes are available on a file descriptor,
* this can fail and return 0 so just use it as a indicator on how many bytes are pending
*/
- int bytesAvailableOnFD ( int fd );
+ int64_t bytesAvailableOnFD( int fd );
/*!
* Small helper struct around creating a Unix pipe to ensure RAII with pipes
#define Z_D() auto const d = d_func()
#define Z_Z() auto const z = z_func()
+/*
+ * Helper Macro to forward declare types and ref types
+ */
+#define ZYPP_FWD_DECL_TYPE_WITH_REFS(T) \
+ class T; \
+ using T##Ref = std::shared_ptr<T>; \
+ using T##WeakRef = std::weak_ptr<T>
+
+//@TODO enable for c++20
+#if 0
+#define ZYPP_FWD_DECL_TEMPL_TYPE_WITH_REFS(T, TArg1, ...) \
+ template< typename TArg1 __VA_OPT__(, typename) __VA_ARGS__ > \
+ class T; \
+ template< typename TArg1 __VA_OPT__(, typename) __VA_ARGS__ > \
+ using T##Ref = std::shared_ptr<T<TArg1 __VA_OPT__(,) __VA_ARGS__>>; \
+ template< typename TArg1 __VA_OPT__(, typename) __VA_ARGS__ > \
+ using T##WeakRef = std::weak_ptr<T<TArg1 __VA_OPT__(,) __VA_ARGS__ >>
+#endif
+
#endif
_mapFds.push_back( fd );
}
+ void AbstractSpawnEngine::notifyExited(int status)
+ {
+ _exitStatus = checkStatus( status );
+ _pid = -1;
+ }
+
bool AbstractSpawnEngine::dieWithParent() const
{
return _dieWithParent;
-#include "asyncdatasource.h"
+#include "private/asyncdatasource_p.h"
#include <zypp-core/base/IOTools.h>
#include <zypp-core/zyppng/base/AutoDisconnect>
#include <zypp-core/zyppng/base/EventDispatcher>
-#include <zypp-core/zyppng/io/private/iodevice_p.h>
-#include <zypp-core/zyppng/io/private/iobuffer_p.h>
-#include <zypp-core/zyppng/base/SocketNotifier>
#include <zypp-core/zyppng/base/private/linuxhelpers_p.h>
namespace zyppng {
- class AsyncDataSourcePrivate : public IODevicePrivate {
- ZYPP_DECLARE_PUBLIC(AsyncDataSource);
- public:
- AsyncDataSourcePrivate ( AsyncDataSource &pub ) : IODevicePrivate(pub) {}
- SocketNotifier::Ptr _readNotifier;
- SocketNotifier::Ptr _writeNotifier;
- IOBuffer _writeBuffer;
- int _readFd = -1;
- int _writeFd = -1;
-
- void notifierActivated (const SocketNotifier ¬ify, int evTypes );
- void readyRead ( );
- void readyWrite ( );
-
- void closeWriteChannel ( AsyncDataSource::ChannelCloseReason reason );
- void closeReadChannel ( AsyncDataSource::ChannelCloseReason reason );
-
- Signal<void( AsyncDataSource::ChannelCloseReason )> _sigWriteFdClosed;
- Signal<void( AsyncDataSource::ChannelCloseReason )> _sigReadFdClosed;
- Signal< void (std::size_t)> _sigBytesWritten;
- };
-
void AsyncDataSourcePrivate::notifierActivated( const SocketNotifier ¬ify, int evTypes )
{
- if ( _readNotifier.get() == ¬ify ) {
- readyRead();
- } else if ( _writeNotifier.get() == ¬ify ) {
+ if ( _writeNotifier.get() == ¬ify ) {
if ( evTypes & SocketNotifier::Error ) {
DBG << "Closing due to error when polling" << std::endl;
closeWriteChannel( AsyncDataSource::RemoteClose );
return;
}
readyWrite();
+ } else {
+
+ auto dev = std::find_if( _readFds.begin(), _readFds.end(),
+ [ ¬ify ]( const auto &dev ){ return ( dev._readNotifier.get() == ¬ify ); } );
+
+ if ( dev == _readFds.end() ) {
+ return;
+ }
+
+ readyRead( std::distance( _readFds.begin(), dev ) );
}
}
- void AsyncDataSourcePrivate::readyRead()
+ void AsyncDataSourcePrivate::readyRead( uint channel )
{
- auto bytesToRead = z_func()->rawBytesAvailable();
+ auto bytesToRead = z_func()->rawBytesAvailable( channel );
if ( bytesToRead == 0 ) {
// make sure to check if bytes are available even if the ioctl call returns something different
bytesToRead = 4096;
}
+ auto &_readBuf = _readChannels[channel];
char *buf = _readBuf.reserve( bytesToRead );
- const auto bytesRead = z_func()->readData( buf, bytesToRead );
+ const auto bytesRead = z_func()->readData( channel, buf, bytesToRead );
- if ( bytesRead < 0 ) {
+ if ( bytesRead <= 0 ) {
_readBuf.chop( bytesToRead );
+
+ switch( bytesRead ) {
+ // remote close , close the read channel
+ case 0: {
+ closeReadChannel( channel, AsyncDataSource::RemoteClose );
+ break;
+ }
+ // no data is available , just try again later
+ case -2: break;
+ // anything else
+ default:
+ case -1: {
+ closeReadChannel( channel, AsyncDataSource::InternalError );
+ break;
+ }
+ }
return;
}
- if ( bytesToRead > (size_t)bytesRead )
+ if ( bytesToRead > bytesRead )
_readBuf.chop( bytesToRead-bytesRead );
- if ( bytesRead > 0 ) {
+ if ( channel == _currentReadChannel )
_readyRead.emit();
- return;
- }
- //handle remote close
- else if ( bytesRead == 0 && errno != EAGAIN && errno != EWOULDBLOCK ) {
- closeReadChannel( AsyncDataSource::RemoteClose );
- }
- if ( errno == EAGAIN || errno == EWOULDBLOCK )
- return;
-
- //setError( Socket::InternalError, strerr_cxx() );
- closeReadChannel( AsyncDataSource::InternalError );
+ _channelReadyRead.emit( channel );
+ return;
}
void AsyncDataSourcePrivate::readyWrite()
}
_writeBuffer.discard( written );
_sigBytesWritten.emit( written );
+
+ if ( _writeBuffer.size() == 0 )
+ _sigAllBytesWritten.emit();
}
void AsyncDataSourcePrivate::closeWriteChannel( AsyncDataSource::ChannelCloseReason reason )
_sigWriteFdClosed.emit( reason );
}
- void AsyncDataSourcePrivate::closeReadChannel( AsyncDataSource::ChannelCloseReason reason )
+ void AsyncDataSourcePrivate::closeReadChannel( uint channel, AsyncDataSource::ChannelCloseReason reason )
{
+ auto &readFd = _readFds[channel];
// we do not clear the read buffer so code has the opportunity to read whats left in there
- bool sig = _readFd >= 0;
- _readNotifier.reset();
- _readFd = -1;
+ bool sig = readFd._readFd >= 0;
+ readFd._readNotifier.reset();
+ readFd._readFd = -1;
if ( sig )
- _sigReadFdClosed.emit( reason );
+ _sigReadFdClosed.emit( channel, reason );
}
ZYPP_IMPL_PRIVATE(AsyncDataSource)
AsyncDataSource::AsyncDataSource() : IODevice( *( new AsyncDataSourcePrivate(*this) ) )
{ }
+ AsyncDataSource::AsyncDataSource( AsyncDataSourcePrivate &d )
+ : IODevice(d)
+ {}
+
AsyncDataSource::Ptr AsyncDataSource::create()
{
return std::shared_ptr<AsyncDataSource>( new AsyncDataSource );
}
- bool AsyncDataSource::open( int readFd, int writeFd )
+
+ bool AsyncDataSource::openFds ( std::vector<int> readFds, int writeFd )
{
Z_D();
- close();
+
+ if ( d->_mode != IODevice::Closed )
+ return false;
+
IODevice::OpenMode mode;
- if ( readFd >= 0 ) {
- mode |= IODevice::ReadOnly;
- d->_readFd = readFd;
- zypp::io::setFDBlocking( readFd, false );
- d->_readNotifier = SocketNotifier::create( readFd, SocketNotifier::Read | AbstractEventSource::Error, true );
- d->_readNotifier->connect( &SocketNotifier::sigActivated, *d, &AsyncDataSourcePrivate::notifierActivated );
+
+ bool error = false;
+ for ( const auto readFd : readFds ) {
+ if ( readFd >= 0 ) {
+ mode |= IODevice::ReadOnly;
+ d->_readFds.push_back( {
+ readFd,
+ SocketNotifier::create( readFd, SocketNotifier::Read | AbstractEventSource::Error, true )
+ });
+ if ( zypp::io::setFDBlocking( readFd, false ) == zypp::io::BlockingMode::FailedToSetMode ) {
+ ERR << "Failed to set read FD to non blocking" << std::endl;
+ error = true;
+ break;
+ }
+ d->_readFds.back()._readNotifier->connect( &SocketNotifier::sigActivated, *d, &AsyncDataSourcePrivate::notifierActivated );
+ }
}
- if ( writeFd >= 0 ) {
+
+ if ( writeFd >= 0 && !error ) {
mode |= IODevice::WriteOnly;
- d->_writeFd = writeFd;
- zypp::io::setFDBlocking( writeFd, false );
- d->_writeNotifier = SocketNotifier::create( writeFd, SocketNotifier::Write | AbstractEventSource::Error, false );
- d->_writeNotifier->connect( &SocketNotifier::sigActivated, *d, &AsyncDataSourcePrivate::notifierActivated );
+ if ( zypp::io::setFDBlocking( writeFd, false ) == zypp::io::BlockingMode::FailedToSetMode ) {
+ ERR << "Failed to set write FD to non blocking" << std::endl;
+ error = true;
+ } else {
+ d->_writeFd = writeFd;
+ d->_writeNotifier = SocketNotifier::create( writeFd, SocketNotifier::Write | AbstractEventSource::Error, false );
+ d->_writeNotifier->connect( &SocketNotifier::sigActivated, *d, &AsyncDataSourcePrivate::notifierActivated );
+ }
+ }
+
+ if( error || !IODevice::open( mode ) ) {
+ d->_mode = IODevice::Closed;
+ d->_readFds.clear();
+ d->_writeNotifier.reset();
+ d->_writeFd = -1;
+ return false;
}
- return IODevice::open( mode );
+
+ // make sure we have enough read buffers
+ setReadChannelCount( d->_readFds.size() );
+ return true;
}
- off_t zyppng::AsyncDataSource::writeData( const char *data, off_t count )
+ int64_t zyppng::AsyncDataSource::writeData( const char *data, int64_t count )
{
Z_D();
if ( count > 0 ) {
return count;
}
- off_t zyppng::AsyncDataSource::readData( char *buffer, off_t bufsize )
+ int64_t zyppng::AsyncDataSource::readData( uint channel, char *buffer, int64_t bufsize )
{
Z_D();
- const auto read = eintrSafeCall( ::read, d->_readFd, buffer, bufsize );
-
- // special case for remote close
- if ( read == 0 ) {
- d->closeReadChannel( RemoteClose );
- return -1;
- } else if ( read < 0 ) {
+ if ( channel >= d->_readFds.size() ) {
+ ERR << constants::outOfRangeErrMsg << std::endl;
+ throw std::logic_error( constants::outOfRangeErrMsg.data() );
+ }
+ const auto read = eintrSafeCall( ::read, d->_readFds[channel]._readFd, buffer, bufsize );
+ if ( read < 0 ) {
switch ( errno ) {
-#if EAGAIN != EWOULDBLOCK
- case EWOULDBLOCK:
-#endif
+ #if EAGAIN != EWOULDBLOCK
+ case EWOULDBLOCK:
+ #endif
case EAGAIN: {
- return 0;
- }
- default: {
- d->closeReadChannel( InternalError );
- return -1;
+ return -2;
}
+ default:
+ break;
}
}
return read;
}
- size_t AsyncDataSource::rawBytesAvailable() const
+ int64_t AsyncDataSource::rawBytesAvailable( uint channel ) const
{
+ Z_D();
+
+ if ( channel >= d->_readFds.size() ) {
+ ERR << constants::outOfRangeErrMsg << std::endl;
+ throw std::logic_error( constants::outOfRangeErrMsg.data() );
+ }
+
if ( isOpen() && canRead() )
- return zyppng::bytesAvailableOnFD( d_func()->_readFd );
+ return zyppng::bytesAvailableOnFD( d->_readFds[channel]._readFd );
return 0;
}
+ void AsyncDataSource::readChannelChanged ( uint channel )
+ {
+ Z_D();
+ if ( channel >= d->_readFds.size() ) {
+ ERR << constants::outOfRangeErrMsg << std::endl;
+ throw std::logic_error( constants::outOfRangeErrMsg.data() );
+ }
+ }
+
void zyppng::AsyncDataSource::close()
{
Z_D();
- d->_readNotifier.reset();
+ for( uint i = 0; i < d->_readFds.size(); ++i ) {
+ auto &readChan = d->_readFds[i];
+ readChan._readNotifier.reset();
+ if ( readChan._readFd >= 0)
+ d->_sigReadFdClosed.emit( i, UserRequest );
+ }
+ d->_readFds.clear();
+
d->_writeNotifier.reset();
d->_writeBuffer.clear();
-
- if ( d->_readFd >= 0)
- d->_sigReadFdClosed.emit( UserRequest );
- if ( d->_writeFd >= 0 )
+ if ( d->_writeFd >= 0 ) {
+ d->_writeFd = -1;
d->_sigWriteFdClosed.emit( UserRequest );
+ }
IODevice::close();
}
- bool AsyncDataSource::waitForReadyRead(int timeout)
+ void AsyncDataSource::closeWriteChannel()
+ {
+ Z_D();
+
+ // if we are open writeOnly, simply call close();
+ if ( !canRead() ) {
+ close();
+ return;
+ }
+
+ d->_mode = ReadOnly;
+ d->_writeNotifier.reset();
+ d->_writeBuffer.clear();
+
+ if ( d->_writeFd >= 0 ) {
+ d->_writeFd = -1;
+ d->_sigWriteFdClosed.emit( UserRequest );
+ }
+ }
+
+ bool AsyncDataSource::waitForReadyRead( uint channel, int timeout )
{
Z_D();
if ( !canRead() )
return false;
+ if ( channel >= d->_readFds.size() ) {
+ ERR << constants::outOfRangeErrMsg << std::endl;
+ throw std::logic_error( constants::outOfRangeErrMsg.data() );
+ }
+
bool gotRR = false;
- auto rrConn = AutoDisconnect( d->_readyRead.connect([&](){
- gotRR = true;
+ auto rrConn = AutoDisconnect( d->_channelReadyRead.connect([&]( uint activated ){
+ gotRR = ( channel == activated );
}) );
// we can only wait if we are open for reading and still have a valid fd
- while ( readFdOpen() && canRead() && !gotRR ) {
+ auto &channelRef = d->_readFds[ channel ];
+ while ( readFdOpen(channel) && canRead() && !gotRR ) {
int rEvents = 0;
- if ( EventDispatcher::waitForFdEvent( d->_readFd, AbstractEventSource::Read | AbstractEventSource::Error , rEvents, timeout ) ) {
+ if ( EventDispatcher::waitForFdEvent( channelRef._readFd, AbstractEventSource::Read | AbstractEventSource::Error , rEvents, timeout ) ) {
//simulate signal from read notifier
- d->notifierActivated( *d->_readNotifier, rEvents );
+ d->notifierActivated( *channelRef._readNotifier, rEvents );
} else {
//timeout
return false;
return gotRR;
}
+ void AsyncDataSource::flush ()
+ {
+ Z_D();
+ if ( !canWrite() )
+ return;
+
+ int timeout = -1;
+ while ( canWrite() && d->_writeBuffer.frontSize() ) {
+ int rEvents = 0;
+ if ( EventDispatcher::waitForFdEvent( d->_writeFd, AbstractEventSource::Write | AbstractEventSource::Error , rEvents, timeout ) ) {
+ //simulate signal from write notifier
+ d->readyWrite();
+ } else {
+ //timeout
+ return;
+ }
+ }
+ }
+
SignalProxy<void (AsyncDataSource::ChannelCloseReason)> AsyncDataSource::sigWriteFdClosed()
{
return d_func()->_sigWriteFdClosed;
}
- SignalProxy<void (AsyncDataSource::ChannelCloseReason)> AsyncDataSource::sigReadFdClosed()
+ SignalProxy<void( uint, AsyncDataSource::ChannelCloseReason )> AsyncDataSource::sigReadFdClosed()
{
return d_func()->_sigReadFdClosed;
}
- SignalProxy<void (std::size_t)> AsyncDataSource::sigBytesWritten()
+ bool AsyncDataSource::readFdOpen() const
{
- return d_func()->_sigBytesWritten;
+ Z_D();
+ if ( !d->_readChannels.size() )
+ return false;
+ return readFdOpen( d_func()->_currentReadChannel );
}
- bool AsyncDataSource::readFdOpen() const
+ bool AsyncDataSource::readFdOpen(uint channel) const
{
- return ( d_func()->_readNotifier && d_func()->_readFd >= 0 );
+ Z_D();
+ if ( channel >= d->_readFds.size() ) {
+ ERR << constants::outOfRangeErrMsg << std::endl;
+ throw std::logic_error( constants::outOfRangeErrMsg.data() );
+ }
+ auto &channelRef = d->_readFds[ channel ];
+ return ( channelRef._readNotifier && channelRef._readFd >= 0 );
}
+
}
using WeakPtr = std::weak_ptr<AsyncDataSource>;
static Ptr create ();
- bool open ( int readFd = -1, int writeFd = -1 );
+ bool openFds ( std::vector<int> readFds, int writeFd = -1 );
void close () override;
+ using IODevice::waitForReadyRead;
+
+ virtual void closeWriteChannel ();
+
/*!
- * Blocks the current event loop to wait until there are bytes available to read from the device
+ * Blocks the current event loop to wait until all bytes currently in the buffer have been written to
+ * the write fd.
*
* \note do not use until there is a very good reason, like event processing should not continue until readyRead was sent
*/
- bool waitForReadyRead(int timeout);
+ bool waitForReadyRead(uint channel, int timeout) override;
+
+ /*!
+ * Blocks the current event loop to wait until all bytes currently in the buffer have been written to
+ * the write fd.
+ *
+ * \note do not use until there is a very good reason, like event processing should not continue until bytesWritten was sent
+ */
+ void flush ();
/*!
* Signal is emitted always when the write channel is closed.
* when the write side of a pipe is closed. All data still residing in the read buffer
* can still be read.
*/
- SignalProxy<void( AsyncDataSource::ChannelCloseReason )> sigReadFdClosed();
+ SignalProxy<void( uint, AsyncDataSource::ChannelCloseReason )> sigReadFdClosed();
/*!
- * Signal is emitted every time bytes have been written to the underlying fd.
- * This can be used to track how much data was actually sent.
+ * Returns true as long as the default read channel was not closed ( e.g. sigReadFdClosed was emitted )
*/
- SignalProxy< void (std::size_t)> sigBytesWritten ();
+ bool readFdOpen () const;
/*!
- * Returns true as long as the read channel was not closed ( e.g. sigReadFdClosed was emitted )
+ * Returns true as long as the given read channel was not closed ( e.g. sigReadFdClosed was emitted )
*/
- bool readFdOpen () const;
+ bool readFdOpen ( uint channel ) const;
protected:
AsyncDataSource ( );
- off_t writeData(const char *data, off_t count) override;
- off_t readData(char *buffer, off_t bufsize) override;
- size_t rawBytesAvailable() const override;
+ AsyncDataSource( AsyncDataSourcePrivate &d );
+ int64_t writeData(const char *data, int64_t count) override;
private:
using IODevice::open;
+
+ int64_t readData( uint channel, char *buffer, int64_t bufsize ) override;
+ int64_t rawBytesAvailable( uint channel ) const override;
+ void readChannelChanged ( uint channel ) override;
};
}
#include "private/iobuffer_p.h"
#include <cstring>
+#include <cassert>
namespace zyppng {
DefChunkSize = 4096
};
- IOBuffer::IOBuffer(unsigned chunkSize) : _defaultChunkSize ( chunkSize == 0 ? DefChunkSize : chunkSize )
+ IOBuffer::IOBuffer( int64_t chunkSize ) : _defaultChunkSize ( chunkSize == 0 ? DefChunkSize : chunkSize )
{ }
- char *IOBuffer::reserve( size_t bytes )
+ char *IOBuffer::reserve( int64_t bytes )
{
+ assert( bytes > 0 && size_t(bytes) < ByteArray::maxSize() );
// do we need a new chunk?
if ( _chunks.size() ) {
auto &back = _chunks.back();
// not enough space ready allocate a new one
_chunks.push_back( Chunk{} );
auto &back = _chunks.back();
- back._buffer.insert( back._buffer.end(), std::max<size_t>( _defaultChunkSize, bytes ), '\0' );
+ back._buffer.insert( back._buffer.end(), std::max<int64_t>( _defaultChunkSize, bytes ), '\0' );
back.tail += bytes;
return back.data();
}
return _chunks.front().data();
}
- size_t IOBuffer::frontSize() const
+ int64_t IOBuffer::frontSize() const
{
if ( _chunks.empty() )
return 0;
_chunks.clear();
}
- size_t IOBuffer::discard( size_t bytes )
+ int64_t IOBuffer::discard( int64_t bytes )
{
- const size_t bytesToDiscard = std::min(bytes, size());
+ const int64_t bytesToDiscard = std::min(bytes, size());
if ( bytesToDiscard == size() ) {
clear();
return bytesToDiscard;
}
- size_t discardedSoFar = 0;
+ int64_t discardedSoFar = 0;
// since the chunks might not be used completely we need to iterate over them
// counting how much used bytes we actually discard until we hit the requested amount
/*!
* Removes bytes from the end of the buffer
*/
- void IOBuffer::chop( size_t bytes )
+ void IOBuffer::chop( int64_t bytes )
{
if ( bytes == 0 )
return;
return;
}
- size_t choppedSoFar = 0;
+ int64_t choppedSoFar = 0;
while ( choppedSoFar < bytes && _chunks.size() ) {
auto bytesStillToChop = bytes - choppedSoFar;
auto &chunk = _chunks.back();
}
}
- void IOBuffer::append(const char *data, size_t count)
+ void IOBuffer::append(const char *data, int64_t count)
{
+ if ( count <= 0 )
+ return;
+
+ assert( count > 0 && size_t(count) < ByteArray::maxSize() );
+
char *buf = reserve( count );
if ( count == 1 )
*buf = *data;
append( data.data(), data.size() );
}
- size_t IOBuffer::read( char *buffer, size_t max )
+ int64_t IOBuffer::read( char *buffer, int64_t max )
{
const size_t bytesToRead = std::min( size(), max );
size_t readSoFar = 0;
return readSoFar;
}
- size_t IOBuffer::size() const
+ int64_t IOBuffer::size() const
{
- size_t s = 0;
+ int64_t s = 0;
for ( const auto &c : _chunks )
s+= c.len();
return s;
return _chunks.size();
}
- int64_t IOBuffer::indexOf( const char c, size_t maxCount, size_t pos ) const
+ int64_t IOBuffer::indexOf(const char c, int64_t maxCount, int64_t pos ) const
{
if ( maxCount == 0 )
return -1;
maxCount = std::min<size_t>( maxCount, size() );
- size_t scannedSoFar = 0;
+ int64_t scannedSoFar = 0;
for ( const auto &chunk : _chunks ) {
//as long as pos is still after the current chunk just increase the count
return -1;
}
- ByteArray IOBuffer::readLine(const size_t max)
+ ByteArray IOBuffer::readLine( const int64_t max )
{
+ assert( ( max >= 2 || max == 0 ) && size_t(max) <= ByteArray::maxSize() );
+
const auto idx = indexOf( '\n', max == 0 ? size() : max );
if ( idx == -1 )
return {};
return b;
}
+ int64_t IOBuffer::readLine( char *buffer, int64_t max )
+ {
+ assert( buffer != nullptr && max > 1 );
+ const auto maxRead = max - 1;
+ const auto idx = indexOf( '\n', maxRead );
+ const auto bytesRead = read( buffer, idx == -1 ? maxRead : idx + 1 );
+ buffer[bytesRead] = '\0';
+ return bytesRead;
+ }
+
bool IOBuffer::canReadLine() const
{
return indexOf('\n') >= 0;
{
Z_D();
- d->_readBuf.clear();
d->_mode = mode;
+ d->_readChannels.clear();
+ if ( canRead() ) {
+ d->_readChannels.push_back( IOBuffer( d->_readBufChunkSize ) );
+ setReadChannel( 0 );
+ }
return true;
}
void IODevice::close()
{
- d_func()->_mode = IODevice::Closed;
- d_func()->_readBuf.clear();
+ Z_D();
+ d->_mode = IODevice::Closed;
+ d->_readChannels.clear();
+ }
+
+ void IODevice::setReadChannelCount ( uint channels ) {
+ Z_D();
+ if ( canRead() ) {
+ d->_readChannels.resize( channels );
+ }
+ }
+
+ void IODevice::setReadChannel ( uint channel )
+ {
+ Z_D();
+ if ( !canRead() )
+ return;
+ if ( channel >= d->_readChannels.size() ) {
+ ERR << constants::outOfRangeErrMsg << std::endl;
+ throw std::out_of_range( constants::outOfRangeErrMsg.data() );
+ }
+ d->_currentReadChannel = channel;
+ readChannelChanged( channel );
+ }
+
+ uint IODevice::currentReadChannel () const
+ {
+ Z_D();
+ if ( !canRead() )
+ return 0;
+ return d->_currentReadChannel;
+ }
+
+ int IODevice::readChannelCount () const
+ {
+ Z_D();
+ if ( !canRead() )
+ return 0;
+ return d->_readChannels.size();
}
bool IODevice::canRead() const
if ( !canRead() )
return false;
- return d->_readBuf.canReadLine();
+ return canReadLine( d->_currentReadChannel );
}
- size_t IODevice::bytesAvailable() const
+ int64_t IODevice::bytesAvailable() const
+ {
+ Z_D();
+ return bytesAvailable( d->_currentReadChannel );
+ }
+
+ ByteArray IODevice::readAll()
+ {
+ Z_D();
+ return readAll( d->_currentReadChannel );
+ }
+
+ ByteArray IODevice::read( int64_t maxSize )
+ {
+ if ( !canRead() || maxSize <= 0 )
+ return {};
+ return read( d_func()->_currentReadChannel, maxSize );
+ }
+
+ int64_t IODevice::read(char *buf, int64_t maxSize )
{
Z_D();
if ( !canRead() )
- return 0;
+ return -1;
+ return read( d->_currentReadChannel, buf, maxSize );
+ }
- return d->_readBuf.size() + rawBytesAvailable();
+ ByteArray IODevice::readLine( const int64_t maxSize )
+ {
+ if ( !canRead() )
+ return {};
+
+ return channelReadLine( d_func()->_currentReadChannel, maxSize );
}
- ByteArray IODevice::readAll()
+ ByteArray IODevice::readAll( uint channel )
{
- return read( bytesAvailable() );
+ return read( channel, bytesAvailable( channel ) );
}
- ByteArray IODevice::read( size_t maxSize )
+ ByteArray IODevice::read( uint channel, int64_t maxSize )
{
- if ( !canRead() || maxSize == 0 )
+ if ( !canRead() || maxSize <= 0 )
return {};
ByteArray res( maxSize, '\0' );
- const auto r = read( res.data(), maxSize );
+ const auto r = read( channel, res.data(), maxSize );
res.resize( r );
return res;
}
- size_t IODevice::read( char *buf, size_t maxSize )
+ int64_t IODevice::read( uint channel, char *buf, int64_t maxSize )
{
Z_D();
- if ( !canRead() )
- return 0;
+ if ( !canRead() || maxSize < 0 )
+ return -1;
- size_t readSoFar = d->_readBuf.read( buf, maxSize );
+ if ( channel >= d->_readChannels.size() ) {
+ ERR << constants::outOfRangeErrMsg << std::endl;
+ throw std::out_of_range( constants::outOfRangeErrMsg.data() );
+ }
+
+ int64_t readSoFar = d->_readChannels[ channel ].read( buf, maxSize );
// try to read more from the device
if ( readSoFar < maxSize ) {
- size_t readFromDev = readData( buf+readSoFar, maxSize - readSoFar );
+ int64_t readFromDev = readData( channel, buf+readSoFar, maxSize - readSoFar );
if ( readFromDev > 0 )
return readSoFar + readFromDev;
}
return readSoFar;
}
- ByteArray IODevice::readLine(const size_t maxSize)
+ ByteArray IODevice::channelReadLine( uint channel, int64_t maxSize )
{
- if ( !canRead() )
+ Z_D();
+ if ( !canRead() || maxSize < 0 )
return {};
- return d_func()->_readBuf.readLine( maxSize );
+ if ( channel >= d->_readChannels.size() ) {
+ ERR << constants::outOfRangeErrMsg << std::endl;
+ throw std::out_of_range( constants::outOfRangeErrMsg.data() );
+ }
+
+ ByteArray result;
+ // largest possible ByteArray in int64_t boundaries
+ const auto maxBArrSize = int64_t( std::min( ByteArray::maxSize(), std::size_t(std::numeric_limits<int64_t>::max()) ) );
+ if ( maxSize > maxBArrSize ) {
+ ERR << "Calling channelReadLine with maxSize > int64_t(ByteArray::maxSize) " << std::endl;
+ maxSize = maxBArrSize - 1;
+ }
+
+ // how much did we read?
+ int64_t readSoFar = 0;
+
+ // if we have no size or the size is really big we read incrementally, use the buffer chunk size
+ // to read full chunks from the buffer if possible
+ if ( maxSize == 0 || maxSize >= (maxBArrSize - 1) ) {
+
+ // largest possible ByteArray
+ maxSize = maxBArrSize;
+
+ // we need to read in chunks until we get a \n
+ int64_t lastReadSize = 0;
+ result.resize (1); // leave room for \0
+ do {
+ result.resize( std::min( std::size_t(maxSize), std::size_t(result.size() + d->_readBufChunkSize )) );
+ lastReadSize = channelReadLine( channel, result.data() + readSoFar, result.size() - readSoFar );
+ if ( lastReadSize > 0)
+ readSoFar += lastReadSize;
+
+ // check for equal _readBufSize,
+ // our readData request is always 1 byte bigger than the _readBufChunkSize because of the initial byte we allocated in the result buffer,
+ // so the \0 that is appended by readLine does not make a difference.
+ } while( lastReadSize == d->_readBufChunkSize
+ && result[readSoFar-1] != '\n' );
+
+ } else {
+ result.resize( maxSize );
+ readSoFar = channelReadLine( channel, result.data(), result.size() );
+ }
+
+ if ( readSoFar > 0 ) {
+ // we do not need to keep the \0 in the ByteArray
+ result.resize( readSoFar );
+ } else {
+ result.clear ();
+ }
+
+ // make sure we do not waste memory
+ result.shrink_to_fit();
+
+ return result;
+ }
+
+ int64_t IODevice::channelReadLine( uint channel, char *buf, const int64_t maxSize )
+ {
+ Z_D();
+
+ if ( !canRead() || maxSize < 0 )
+ return -1;
+
+ if ( channel >= d->_readChannels.size() ) {
+ ERR << constants::outOfRangeErrMsg << std::endl;
+ throw std::out_of_range( constants::outOfRangeErrMsg.data() );
+ }
+
+ if ( maxSize < 2 ) {
+ ERR << "channelReadLine needs at least a buffsize of 2" << std::endl;
+ return -1;
+ }
+
+ int64_t toRead = maxSize - 1; // append \0 at the end
+ int64_t readSoFar = 0;
+ if ( d->_readChannels[channel].size () > 0 )
+ readSoFar = d->_readChannels[channel].readLine( buf, toRead + 1 /*IOBuffer appends \0*/ );
+
+ if ( readSoFar == toRead || ( readSoFar > 0 && buf[readSoFar-1] == '\n' ) ) {
+ buf[readSoFar] = '\0';
+ return readSoFar;
+ }
+
+ bool hasError = false;
+ // if we reach here, the buffer was either empty, or does not contain a \n, in both cases we need to
+ // read from the device directly until we hit a ending condition
+ while ( readSoFar < toRead ) {
+ const auto r = readData( channel, buf+readSoFar, 1 );
+ if ( r == 0 ) {
+ // no data available to be read -> EOF, or data stream empty
+ break;
+ }
+ else if ( r < 0 ) {
+ hasError = true;
+ break;
+ }
+ readSoFar+=r;
+
+ if ( buf[readSoFar-1] == '\n' )
+ break;
+ }
+
+ if ( readSoFar == 0 )
+ return hasError ? -1 : 0;
+
+ buf[readSoFar] = '\0';
+ return readSoFar;
}
- off_t zyppng::IODevice::write(const zyppng::ByteArray &data)
+ int64_t IODevice::bytesAvailable ( uint channel ) const
+ {
+ Z_D();
+ if ( !canRead() )
+ return 0;
+ return d->_readChannels[channel].size() + rawBytesAvailable( channel );
+ }
+
+ bool IODevice::canReadLine ( uint channel ) const
+ {
+ Z_D();
+ if ( !canRead() || channel >= d->_readChannels.size() )
+ return false;
+ return d->_readChannels[channel].canReadLine();
+ }
+
+ int64_t zyppng::IODevice::write( const zyppng::ByteArray &data)
{
if ( !canWrite() )
return 0;
return write( data.data(), data.size() );
}
- off_t IODevice::write(const char *data, size_t len)
+ int64_t IODevice::write( const char *data, int64_t len)
{
- if ( !canWrite() )
+ if ( !canWrite() || len <= 0 )
return 0;
return writeData( data, len );
}
+ bool IODevice::waitForReadyRead( int timeout)
+ {
+ Z_D();
+ if ( !canRead() )
+ return false;
+
+ return waitForReadyRead( d->_currentReadChannel, timeout );
+ }
+
SignalProxy<void ()> IODevice::sigReadyRead()
{
return d_func()->_readyRead;
}
+
+ SignalProxy<void (uint)> IODevice::sigChannelReadyRead ()
+ {
+ return d_func()->_channelReadyRead;
+ }
+
+ SignalProxy<void (int64_t)> IODevice::sigBytesWritten()
+ {
+ return d_func()->_sigBytesWritten;
+ }
+
+ SignalProxy< void ()> IODevice::sigAllBytesWritten ()
+ {
+ return d_func()->_sigAllBytesWritten;
+ }
}
namespace zyppng {
class IODevicePrivate;
+
+ /*!
+ * The IODevice class represents a async sequential IO device, like a Socket or Pipe,
+ * to receive and or send data.
+ */
class IODevice : public Base
{
ZYPP_DECLARE_PRIVATE(IODevice);
using WeakPtr = std::weak_ptr<IODevice>;
IODevice();
-
- virtual bool open ( const OpenMode mode );
virtual void close ();
+ void setReadChannel ( uint channel );
+ uint currentReadChannel () const;
+ int readChannelCount () const;
bool canRead () const;
bool canWrite () const;
bool isOpen () const;
- bool canReadLine () const;
ByteArray readAll ();
- ByteArray read ( size_t maxSize );
- size_t read ( char *buf, size_t maxSize );
- virtual ByteArray readLine ( const size_t maxSize = 0 );
- virtual size_t bytesAvailable () const;
+ ByteArray read ( int64_t maxSize );
+ int64_t read ( char *buf, int64_t maxSize );
+ virtual ByteArray readLine (const int64_t maxSize = 0 );
+ virtual int64_t bytesAvailable () const;
+ bool canReadLine () const;
+
+ ByteArray readAll ( uint channel );
+ ByteArray read ( uint channel, int64_t maxSize );
+ int64_t read ( uint channel, char *buf, int64_t maxSize );
+
+ /*!
+ * Convenience function that reads a line from the device into a ByteArray. Since
+ * this function has no way to signal if a error happened, a empty ByteArray is
+ * returned if there was no data or if a error occured.
+ */
+ ByteArray channelReadLine ( uint channel, int64_t maxSize = 0 );
+
+ /*!
+ * Reads data from the device until one of the following conditions are met:
+ * - A \n is encountered
+ * - maxSize nr of bytes have been read
+ * - a error occurs on the device
+ *
+ * If bytes have been read from the device this always returns the number of bytes that
+ * have been read, otherwise if no data was read 0 is returned or if a error occurs -1 is returned.
+ */
+ virtual int64_t channelReadLine ( uint channel, char *buf, const int64_t maxSize );
+ virtual int64_t bytesAvailable( uint channel ) const;
+
+ /*!
+ * Returns true if a line can be read from the currently buffered data in the given channel
+ */
+ bool canReadLine ( uint channel ) const;
+
+ int64_t write ( const ByteArray &data );
+ int64_t write ( const char *data, int64_t len );
+
+ /*!
+ * Blocks the current event loop to wait until there are bytes available to read from the device.
+ * This always operates on the read channel that is selected as the default when the function is first called,
+ * even if the default channel would be changed during the wait.
+ *
+ * \sa zyppng::IODevice::currentReadChannel
+ * \note do not use until there is a very good reason, like event processing should not continue until readyRead was sent
+ */
+ bool waitForReadyRead(int timeout);
+
+ /*!
+ * Blocks the current event loop to wait until there are bytes available to read from the given read channel.
+ *
+ * \note do not use until there is a very good reason, like event processing should not continue until readyRead was sent
+ */
+ virtual bool waitForReadyRead(uint channel, int timeout) = 0;
- off_t write ( const ByteArray &data );
- off_t write ( const char *data, size_t len );
/*!
- * Signal is emitted when there is data available to read
+ * Signal is emitted when there is data available to read on the current default read channel
*/
SignalProxy<void ()> sigReadyRead ();
+ /*!
+ * Signal is emitted when there is data available on the given channel
+ */
+ SignalProxy<void (uint)> sigChannelReadyRead ();
+
+ /*!
+ * Signal is emitted every time bytes have been written to the underlying device.
+ * This can be used to track how much data was actually sent.
+ */
+ SignalProxy< void (int64_t)> sigBytesWritten ();
+
+ /*!
+ * Signal is emitted every time all bytes have been written to the underlying device.
+ */
+ SignalProxy< void ()> sigAllBytesWritten ();
+
protected:
IODevice( IODevicePrivate &d );
- virtual size_t rawBytesAvailable () const = 0;
- virtual off_t writeData ( const char *data, off_t count ) = 0;
- virtual off_t readData ( char *buffer, off_t bufsize ) = 0;
+ virtual bool open ( const OpenMode mode );
+ virtual int64_t rawBytesAvailable ( uint channel ) const = 0;
+ virtual int64_t writeData ( const char *data, int64_t count ) = 0;
+ virtual int64_t readData ( uint channel, char *buffer, int64_t bufsize ) = 0;
+ virtual void readChannelChanged ( uint channel ) = 0;
+ void setReadChannelCount ( uint channels );
};
ZYPP_DECLARE_OPERATORS_FOR_FLAGS( IODevice::OpenMode );
*/
pid_t pid ( );
- /**
+ /*!
* Kickstart the process, if this returns true it is guaranteed that exec() was successful
*/
virtual bool start ( const char *const *argv, int stdin_fd, int stdout_fd, int stderr_fd ) = 0;
virtual bool isRunning ( bool wait = false ) = 0;
+ /*!
+ * Used to notify the backend that the process has ended,
+ * helpful when the process is tracked in another way than calling \ref isRunning,
+ * for example in a eventloop.
+ */
+ virtual void notifyExited ( int status );
+
bool dieWithParent() const;
void setDieWithParent( bool dieWithParent );
--- /dev/null
+#ifndef ZYPP_CORE_ZYPPNG_IO_PRIVATE_ASYNCDATASOURCE_P_H_INCLUDED
+#define ZYPP_CORE_ZYPPNG_IO_PRIVATE_ASYNCDATASOURCE_P_H_INCLUDED
+
+#include <zypp-core/zyppng/io/AsyncDataSource>
+#include <zypp-core/zyppng/base/SocketNotifier>
+#include "iodevice_p.h"
+#include "iobuffer_p.h"
+
+namespace zyppng {
+
+ class AsyncDataSourcePrivate : public IODevicePrivate {
+ ZYPP_DECLARE_PUBLIC(AsyncDataSource);
+ public:
+ AsyncDataSourcePrivate ( AsyncDataSource &pub ) : IODevicePrivate(pub) {}
+ struct ReadChannelDev {
+ int _readFd = -1;
+ SocketNotifier::Ptr _readNotifier;
+ };
+ std::vector<ReadChannelDev> _readFds;
+
+ SocketNotifier::Ptr _writeNotifier;
+ IOBuffer _writeBuffer;
+ int _writeFd = -1;
+
+ void notifierActivated (const SocketNotifier ¬ify, int evTypes );
+ void readyRead ( uint channel );
+ void readyWrite ( );
+
+ void closeWriteChannel ( AsyncDataSource::ChannelCloseReason reason );
+ void closeReadChannel ( uint channel, AsyncDataSource::ChannelCloseReason reason );
+
+ Signal<void( AsyncDataSource::ChannelCloseReason )> _sigWriteFdClosed;
+ Signal<void( uint, AsyncDataSource::ChannelCloseReason )> _sigReadFdClosed;
+ };
+
+}
+
+
+#endif // ZYPP_CORE_ZYPPNG_IO_PRIVATE_ASYNCDATASOURCE_P_H_INCLUDED
struct Chunk {
ByteArray _buffer;
- unsigned head = 0;
- unsigned tail = 0;
+ int64_t head = 0;
+ int64_t tail = 0;
char * data () {
return _buffer.data() + head;
return _buffer.data() + head;
}
- unsigned available() const {
+ int64_t available() const {
return _buffer.size() - tail;
}
- unsigned len () const {
+ int64_t len () const {
return tail - head;
}
};
public:
- IOBuffer( unsigned chunkSize = 0);
+ IOBuffer( int64_t chunkSize = 0 );
- char *reserve( size_t bytes );
+ char *reserve( int64_t bytes );
char *front ();
- size_t frontSize () const;
+ int64_t frontSize () const;
void clear ( );
- size_t discard ( size_t bytes );
- void chop ( size_t bytes );
- void append ( const char *data, size_t count );
+ int64_t discard( int64_t bytes );
+ void chop ( int64_t bytes );
+ void append ( const char *data, int64_t count );
void append ( const ByteArray &data );
- size_t read ( char *buffer, size_t max );
- size_t size ( ) const;
+ int64_t read ( char *buffer, int64_t max );
+ int64_t size ( ) const;
std::vector<Chunk>::size_type chunks () const;
inline int64_t indexOf ( const char c ) const { return indexOf( c, size() ); }
- int64_t indexOf ( const char c, size_t maxCount, size_t pos = 0 ) const;
- ByteArray readLine ( const size_t max = 0 );
+ int64_t indexOf (const char c, int64_t maxCount, int64_t pos = 0 ) const;
+ ByteArray readLine ( const int64_t max = 0 );
+ int64_t readLine( char *buffer, int64_t max );
bool canReadLine () const;
private:
- unsigned _defaultChunkSize;
+ int64_t _defaultChunkSize;
std::vector<Chunk> _chunks;
};
#ifndef ZYPPNG_IO_IODEVICE_P_DEFINED
#define ZYPPNG_IO_IODEVICE_P_DEFINED
+#include <vector>
+#include <functional>
#include <zypp-core/zyppng/io/iodevice.h>
#include <zypp-core/zyppng/base/private/base_p.h>
#include "iobuffer_p.h"
namespace zyppng {
+ namespace constants {
+ constexpr std::string_view outOfRangeErrMsg("Channel index out of range");
+ }
+
+ enum {
+ DefIoDeviceBufChunkSize = 16384
+ };
+
class IODevicePrivate : public BasePrivate {
public:
IODevicePrivate ( IODevice &p );
- IOBuffer _readBuf;
+
+ std::vector<IOBuffer> _readChannels;
+ uint _currentReadChannel = 0;
+ int64_t _readBufChunkSize = DefIoDeviceBufChunkSize;
+
IODevice::OpenMode _mode = IODevice::Closed;
- Signal< void()> _readyRead;
+ Signal<void()> _readyRead;
+ Signal<void(uint)> _channelReadyRead;
+ Signal< void (int64_t)> _sigBytesWritten;
+ Signal< void ()> _sigAllBytesWritten;
};
}
{ }
bool initSocket () ;
- void setError ( Socket::SocketError error, std::string &&err );
+ void setError ( Socket::SocketError error, std::string &&err, bool emit = true );
bool handleConnectError ( int error );
bool transition ( Socket::SocketState newState );
void onSocketActivatedSlot ( const SocketNotifier &, int ev ) {
return onSocketActivated(ev);
}
- int rawBytesAvailable () const;
+ int64_t rawBytesAvailable () const;
bool readRawBytesToBuffer ();
bool writePendingData ();
bool _borrowedSocket = false;
//error handling
+ bool _emittedErr = false;
Socket::SocketError _error = Socket::NoError;
std::string _errorDesc;
//signals
Signal< void(Socket::SocketError)> _sigError;
- Signal< void (std::size_t)> _sigBytesWritten;
Signal< void()> _incomingConnection;
Signal< void()> _connected;
Signal< void()> _disconnected;
#include "process.h"
-#include <zypp-core/zyppng/base/private/base_p.h>
+#include <zypp-core/zyppng/io/private/asyncdatasource_p.h>
#include <zypp-core/zyppng/base/EventDispatcher>
#include <zypp-core/zyppng/io/private/abstractspawnengine_p.h>
#include <zypp-core/zyppng/io/AsyncDataSource>
* signals that the fork has worked out but NOT that the app actually started
*/
- class ProcessPrivate : public BasePrivate
+ class ProcessPrivate : public AsyncDataSourcePrivate
{
public:
- ProcessPrivate( Process &p ) : BasePrivate(p)
+ ProcessPrivate( Process &p ) : AsyncDataSourcePrivate(p)
{ }
+ void cleanup() {
+ _stdinFd = -1;
+ _stderrFd = -1;
+ _stdoutFd = -1;
+ }
+
std::unique_ptr<AbstractSpawnEngine> _spawnEngine = AbstractSpawnEngine::createDefaultEngine();
- AsyncDataSource::Ptr _stdinDevice;
- AsyncDataSource::Ptr _stdoutDevice;
- AsyncDataSource::Ptr _stderrDevice;
zypp::AutoFD _stdinFd = -1;
zypp::AutoFD _stderrFd = -1;
zypp::AutoFD _stdoutFd = -1;
- pid_t _pid = -1;
Signal<void ()> _sigStarted;
Signal<void ( int )> _sigFinished;
Signal<void ()> _sigFailedToStart;
+ Process::OutputChannelMode _channelMode = Process::Seperate;
+ Process::OutputChannel _currentChannel = Process::StdOut;
};
ZYPP_IMPL_PRIVATE(Process)
- Process::Process() : Base( *( new ProcessPrivate(*this) ) )
+ Process::Process() : AsyncDataSource( *( new ProcessPrivate(*this) ) )
{
}
Process::~Process()
{
Z_D();
- if ( d->_pid >= 0 ) {
- EventDispatcher::instance()->untrackChildProcess( d->_pid );
+ if ( d->_spawnEngine->pid() >= 0 ) {
+ EventDispatcher::instance()->untrackChildProcess( d->_spawnEngine->pid() );
DBG << "Process destroyed while still running removing from EventLoop." << std::endl;
}
}
- bool Process::start(const char * const *argv )
+ bool Process::start( const char *const *argv )
{
Z_D();
return false;
}
+ if ( isRunning() )
+ return false;
+
// clean up the previous run
- d->_stdinDevice.reset();
- d->_stdoutDevice.reset();
- d->_stderrDevice.reset();
- d->_stdinFd = -1;
- d->_stderrFd = -1;
- d->_stdoutFd = -1;
+ AsyncDataSource::close();
+ d->cleanup();
// create the pipes we need
auto stdinPipe = Pipe::create( );
return false;
}
- auto stderrPipe = Pipe::create( );
- if ( !stderrPipe ) {
- d->_sigFailedToStart.emit();
- return false;
+ int stderr_fd = -1;
+ std::optional<Pipe> stderrPipe;
+ if ( d->_channelMode == Seperate ) {
+ stderrPipe = Pipe::create( );
+ if ( !stderrPipe ) {
+ d->_sigFailedToStart.emit();
+ return false;
+ }
+ stderr_fd = stderrPipe->writeFd;
+ } else {
+ stderr_fd = stdoutPipe->writeFd;
}
- if ( d->_spawnEngine->start( argv, stdinPipe->readFd, stdoutPipe->writeFd, stderrPipe->writeFd ) ) {
+ if ( d->_spawnEngine->start( argv, stdinPipe->readFd, stdoutPipe->writeFd, stderr_fd ) ) {
// if we reach this point the engine guarantees that exec() was successful
- d->_pid = d->_spawnEngine->pid( );
+ const auto pid = d->_spawnEngine->pid( );
// register to the eventloop right away
- EventDispatcher::instance()->trackChildProcess( d->_pid, [this]( int, int status ){
+ EventDispatcher::instance()->trackChildProcess( pid, [this]( int, int status ){
Z_D();
- d->_spawnEngine->setExitStatus( d->_spawnEngine->checkStatus( status ) );
- d->_pid = -1;
+ d->_spawnEngine->notifyExited( status );
d->_sigFinished.emit( d->_spawnEngine->exitStatus() );
});
// make sure the fds we need are kept open
d->_stdinFd = std::move( stdinPipe->writeFd );
d->_stdoutFd = std::move( stdoutPipe->readFd );
- d->_stderrFd = std::move( stderrPipe->readFd );
-
- d->_stdinDevice = AsyncDataSource::create();
- d->_stdinDevice->open( -1, d->_stdinFd );
- d->_stdoutDevice = AsyncDataSource::create();
- d->_stdoutDevice->open( d->_stdoutFd );
+ std::vector<int> rFds { d->_stdoutFd };
+ if ( stderrPipe ) {
+ d->_stderrFd = std::move( stderrPipe->readFd );
+ rFds.push_back( d->_stderrFd.value() );
+ }
- d->_stderrDevice = AsyncDataSource::create();
- d->_stderrDevice->open( d->_stderrFd );
+ if ( !openFds( rFds, d->_stdinFd ) ) {
+ stop( SIGKILL );
+ return false;
+ }
d->_sigStarted.emit();
return true;
bool Process::isRunning()
{
- return ( d_func()->_pid > -1 );
+ Z_D();
+ return ( d->_spawnEngine->pid() > -1 );
+ }
+
+ void Process::close ()
+ {
+ flush();
+ stop(SIGKILL);
+ d_func()->cleanup();
+ AsyncDataSource::close();
+ }
+
+ void Process::waitForExit()
+ {
+ Z_D();
+ if ( d->_spawnEngine->isRunning() ) {
+ // we will manually track the exit status
+ EventDispatcher::instance()->untrackChildProcess( d->_spawnEngine->pid() );
+ // wait for the process to exit
+ d->_spawnEngine->isRunning( true );
+ }
+ }
+
+ void Process::closeWriteChannel()
+ {
+ Z_D();
+ d->_stdinFd = -1;
+ AsyncDataSource::closeWriteChannel();
}
const std::string &Process::executedCommand() const
pid_t Process::pid()
{
- return d_func()->_pid;
+ return d_func()->_spawnEngine->pid();
}
int Process::exitStatus() const
return d_func()->_spawnEngine->addFd( fd );
}
- std::shared_ptr<IODevice> Process::stdinDevice()
- {
- return d_func()->_stdinDevice;
- }
-
- std::shared_ptr<IODevice> Process::stdoutDevice()
- {
- return d_func()->_stdoutDevice;
- }
-
- std::shared_ptr<IODevice> Process::stderrDevice()
- {
- return d_func()->_stderrDevice;
- }
-
int Process::stdinFd()
{
return d_func()->_stdinFd;
return d_func()->_sigFinished;
}
+ Process::OutputChannelMode Process::outputChannelMode() const { return d_func()->_channelMode; }
+ void Process::setOutputChannelMode(const OutputChannelMode &outputChannelMode) { d_func()->_channelMode = outputChannelMode; }
+
}
#ifndef ZYPPNG_IO_PROCESS_H_DEFINED
#define ZYPPNG_IO_PROCESS_H_DEFINED
-#include <zypp-core/zyppng/base/Base>
+#include <zypp-core/zyppng/io/AsyncDataSource>
#include <zypp-core/zyppng/base/Signals>
#include <memory>
#include <map>
class ProcessPrivate;
class IODevice;
- class Process : public Base
+ class Process : public AsyncDataSource
{
ZYPP_DECLARE_PRIVATE(Process);
public:
* For passing additional environment variables to set
*/
using Environment = std::map<std::string,std::string>;
-
using Ptr = std::shared_ptr<Process>;
using WeakPtr = std::weak_ptr<Process>;
+ enum OutputChannelMode {
+ Seperate,
+ Merged
+ };
+
+ enum OutputChannel {
+ StdOut = 0,
+ StdErr = 1
+ };
+
static Ptr create ();
~Process();
- bool start (const char *const *argv);
+ bool start ( const char *const *argv );
void stop ( int signal = SIGTERM );
bool isRunning ();
+ void close () override;
+
+ /*!
+ * Blocks until the process has exited, during that time readyRead is not
+ * emitted for any read channels. Call \ref readAll to get all remaining data
+ * that was written by the process
+ */
+ void waitForExit ();
+
+ /*!
+ * Close the stdin fd of the subprocess. This is required for processes
+ * that run until their stdin is closed.
+ */
+ void closeWriteChannel () override;
const std::string &executedCommand () const;
const std::string &execError() const;
const std::vector<int> &fdsToMap () const;
void addFd ( int fd );
- std::shared_ptr<IODevice> stdinDevice ();
- std::shared_ptr<IODevice> stdoutDevice ();
- std::shared_ptr<IODevice> stderrDevice ();
-
int stdinFd ();
int stdoutFd ();
int stderrFd ();
SignalProxy<void ()> sigFailedToStart ();
SignalProxy<void ( int )> sigFinished ();
+ OutputChannelMode outputChannelMode() const;
+ void setOutputChannelMode(const OutputChannelMode &outputChannelMode);
+
protected:
Process();
};
if ( _socket >= 0 )
return true;
+ _error = Socket::NoError;
+ _emittedErr = false;
+
// Since Linux 2.6.27 we can pass additional flags with the type argument to avoid fcntl
// if creating sockets fails we might need to change that
_socket = ::socket( _domain, _type | SOCK_NONBLOCK | SOCK_CLOEXEC, _protocol );
return false;
}
- void SocketPrivate::setError(Socket::SocketError error , std::string &&err )
+ void SocketPrivate::setError( Socket::SocketError error , std::string &&err, bool emit )
{
- if ( _error == error )
- return;
- _error = error;
- _errorDesc = std::move(err);
- _sigError.emit( error );
+ // we only remember the first error that happend
+ if ( _error == Socket::NoError && _error != error ) {
+ _emittedErr = emit;
+ _error = error;
+ _errorDesc = std::move(err);
+ }
+ if ( emit && !_emittedErr )
+ _sigError.emit( error );
}
bool SocketPrivate::transition( Socket::SocketState newState )
return true;
}
- int zyppng::SocketPrivate::rawBytesAvailable() const
+ int64_t zyppng::SocketPrivate::rawBytesAvailable() const
{
if ( state() != Socket::ConnectedState )
return 0;
bool SocketPrivate::readRawBytesToBuffer()
{
+ Z_Z();
auto bytesToRead = rawBytesAvailable();
if ( bytesToRead == 0 ) {
// make sure to check if bytes are available even if the ioctl call returns something different
bytesToRead = 4096;
}
- char *buf = _readBuf.reserve( bytesToRead );
- const auto bytesRead = z_func()->readData( buf, bytesToRead );
-
- if ( bytesRead < 0 ) {
- _readBuf.chop( bytesToRead );
- return false;
- }
+ auto &readBuf = _readChannels[0];
+ char *buf = readBuf.reserve( bytesToRead );
+ const auto bytesRead = z_func()->readData( 0, buf, bytesToRead );
- if ( bytesToRead > bytesRead )
- _readBuf.chop( bytesToRead-bytesRead );
+ if ( bytesRead <= 0 ) {
+ readBuf.chop( bytesToRead );
- if ( bytesRead > 0 ) {
- _readyRead.emit();
- return true;
- }
- //handle remote close
- else if ( bytesRead == 0 && errno != EAGAIN && errno != EWOULDBLOCK ) {
- setError( Socket::ConnectionClosedByRemote, "The remote host closed the connection" );
+ switch ( bytesRead ) {
+ case -2:
+ // there is simply no data to read ignore and try again
+ return true;
+ case 0: {
+ // remote close
+ setError( Socket::ConnectionClosedByRemote, "The remote host closed the connection", true );
+ break;
+ }
+ case -1:
+ default: {
+ setError( Socket::InternalError, strerr_cxx(), true );
+ break;
+ }
+ }
+ z->abort();
return false;
}
- if ( errno == EAGAIN || errno == EWOULDBLOCK )
- return true;
+ if ( bytesToRead > bytesRead )
+ readBuf.chop( bytesToRead-bytesRead );
- setError( Socket::InternalError, strerr_cxx() );
- return false;
+ _readyRead.emit();
+ _channelReadyRead.emit(0);
+ return true;
}
bool SocketPrivate::writePendingData()
}
s._writeBuffer.discard( written );
_sigBytesWritten.emit( written );
+ if ( s._writeBuffer.size() == 0 )
+ _sigAllBytesWritten.emit();
}
return true;
}, _state );
: IODevice( *( new SocketPrivate( domain, type, protocol, *this )))
{ }
- size_t Socket::rawBytesAvailable() const
+ int64_t Socket::rawBytesAvailable( uint channel ) const
{
+ if ( channel != 0 ) {
+ constexpr std::string_view msg("Socket does not support multiple read channels");
+ ERR << msg << std::endl;
+ throw std::logic_error( msg.data() );
+ }
return d_func()->rawBytesAvailable();
}
}
- off_t Socket::writeData( const char *data, off_t count )
+ int64_t Socket::writeData( const char *data, int64_t count )
{
Z_D();
if ( d->state() != SocketState::ConnectedState )
if ( written > 0 )
d->_sigBytesWritten.emit( written );
}
+
+ if ( s._writeBuffer.size() == 0 )
+ d->_sigAllBytesWritten.emit();
+
return count;
}
return bufferEmpty;
}
- bool Socket::waitForReadyRead(int timeout)
+ bool Socket::waitForReadyRead( uint channel, int timeout)
{
Z_D();
- if ( d->state() != Socket::ConnectedState )
+ if ( d->state() != Socket::ConnectedState || channel != 0 )
return false;
// we can only wait if we are in connected state
return bytesAvailable() > 0;
}
- off_t Socket::readData( char *buffer, off_t bufsize )
+ int64_t Socket::readData( uint channel, char *buffer, int64_t bufsize )
{
+ if ( channel != 0 ) {
+ constexpr std::string_view msg("Socket does not support multiple read channels");
+ ERR << msg << std::endl;
+ throw std::logic_error( msg.data() );
+ }
Z_D();
if ( d->state() != SocketState::ConnectedState )
// special case for remote close
if ( read == 0 ) {
- d->setError( ConnectionClosedByRemote, "The remote host closed the connection" );
- abort();
- return -1;
+ d->setError( ConnectionClosedByRemote, "The remote host closed the connection", false );
+ return 0;
} else if ( read < 0 ) {
switch ( errno ) {
#if EAGAIN != EWOULDBLOCK
case EWOULDBLOCK:
#endif
case EAGAIN: {
- return 0;
+ return -2;
}
default: {
- d->setError( UnknownSocketError, strerr_cxx( errno ) );
- abort();
+ d->setError( UnknownSocketError, strerr_cxx( errno ), false );
return -1;
}
}
return read;
}
- size_t Socket::bytesAvailable() const
- {;
- return IODevice::bytesAvailable();
+ void Socket::readChannelChanged ( uint channel )
+ {
+ if ( channel != 0 ) {
+ constexpr std::string_view msg("Changing the readChannel on a Socket is not supported");
+ ERR << msg << std::endl;
+ throw std::logic_error( msg.data() );
+ }
}
- size_t Socket::bytesPending() const
+ int64_t Socket::bytesPending() const
{
Z_D();
- return std::visit([&]( const auto &s ) -> size_t {
+ return std::visit([&]( const auto &s ) -> int64_t {
using T = std::decay_t<decltype (s)>;
if constexpr ( std::is_same_v<T, SocketPrivate::ConnectedState> || std::is_same_v<T, SocketPrivate::ClosingState> ) {
return s._writeBuffer.size();
return d_func()->_sigError;
}
- SignalProxy<void (std::size_t)> Socket::sigBytesWritten()
- {
- return d_func()->_sigBytesWritten;
- }
-
}
void close() override;
/*!
- * Returns the current number of bytes that can be read from the socket.
- */
- size_t bytesAvailable() const override;
-
- /*!
* Returns the current number of bytes that are not yet written to the socket.
*/
- size_t bytesPending() const;
+ int64_t bytesPending() const;
/*!
* Returns the current state the socket is in,
*
* \note do not use until there is no other way
*/
- bool waitForReadyRead ( int timeout = -1 );
+ bool waitForReadyRead ( uint channel, int timeout = -1 ) override;
/*!
* Returns the native socket handle.
SignalProxy<void ()> sigDisconnected ();
/*!
- * Signal is emitted every time bytes have been written to the underlying socket.
- * This can be used to track how much data was actually sent.
- */
- SignalProxy<void ( std::size_t )> sigBytesWritten ();
-
- /*!
* Signal is emitted whenever a error happend in the socket. Make sure to check
* the actual error code to determine if the error is fatal.
*/
// IODevice interface
protected:
- size_t rawBytesAvailable() const override;
- off_t writeData(const char *data, off_t count) override;
- off_t readData(char *buffer, off_t bufsize) override;
+ int64_t rawBytesAvailable( uint channel = 0 ) const override;
+ int64_t writeData(const char *data, int64_t count) override;
+ int64_t readData( uint channel, char *buffer, int64_t bufsize ) override;
+ void readChannelChanged ( uint channel ) override;
};
}
#endif
#endif
+/*!
+ * Simple helper template to make a callback that binds the "this" pointer, to be used in
+ * a pipeline.
+ */
+template<typename Obj, typename Ret, typename Arg>
+auto mem_fn_cb( Obj& o, Ret (Obj::*objMemFunc)( Arg&& ) ) {
+ return [tPtr = &o, fun = objMemFunc ]( Arg &&r ){
+ return std::invoke(fun, tPtr, std::move(r) );
+ };
+}
+
+template<typename Obj, typename Ret, typename Arg>
+auto mem_fn_cb( Obj& o, Ret (Obj::*objMemFunc)( const Arg& ) ) {
+ return [tPtr = &o, fun = objMemFunc ]( const Arg &r ){
+ return std::invoke(fun, tPtr, r );
+ };
+}
+
+template<typename Obj, typename Ret, typename Arg>
+auto mem_fn_cb( Obj& o, Ret (Obj::*objMemFunc)( Arg ) ) {
+ return [tPtr = &o, fun = objMemFunc ]( Arg r ){
+ if constexpr ( std::is_invocable_v< decltype (fun), Obj*, Arg&&> ) {
+ return std::invoke(fun, tPtr, std::move(r) );
+ } else {
+ return std::invoke(fun, tPtr, r );
+ }
+ };
+}
+
+
#endif
#endif
-
-#if __cplusplus < 201703L
-
namespace std {
+#if __cplusplus < 202002L
+
//implementation of the detector idiom, used to help with SFINAE
//from https://en.cppreference.com/w/cpp/experimental/is_detected
template <class To, template<class...> class Op, class... Args>
constexpr bool is_detected_convertible_v = is_detected_convertible<To, Op, Args...>::value_t::value;
+#endif
+
+#if __cplusplus < 201703L
+
//https://en.cppreference.com/w/cpp/types/conjunction)
template<class...> struct conjunction : std::true_type { };
template<class B1> struct conjunction<B1> : B1 { };
template<class B>
struct negation : std::bool_constant< !bool(B::value)> { };
-}
#endif
+}
namespace zyppng {
template <typename Prev, typename AOp >
struct AsyncResult<Prev,AOp> : public zyppng::AsyncOp< typename AOp::value_type > {
- AsyncResult ( std::
unique_ptr<Prev> && prevTask, std::unique_ptr<AOp> &&cb )
+ AsyncResult ( std::
shared_ptr<Prev> && prevTask, std::shared_ptr<AOp> &&cb )
: _prevTask( std::move(prevTask) )
, _myTask( std::move(cb) ) {
connect();
if ( _prevTask ) {
//dumpInfo();
- _prevTask.reset(nullptr);
+ _prevTask.reset();
}
_myTask->operator()(std::move(resStore));
}
- std::
unique_ptr<Prev> _prevTask;
- std::
unique_ptr<AOp> _myTask;
+ std::
shared_ptr<Prev> _prevTask;
+ std::
shared_ptr<AOp> _myTask;
};
template<typename AOp, typename In>
struct AsyncResult<void, AOp, In> : public zyppng::AsyncOp< typename AOp::value_type > {
- AsyncResult ( std::
unique_ptr<AOp> &&cb )
+ AsyncResult ( std::
shared_ptr<AOp> &&cb )
: _myTask( std::move(cb) ) {
connect();
}
this->setReady( std::move( in ) );
});
}
- std::unique_ptr<AOp> _myTask;
- };
-
- //A async result that is ready right away
- template <typename T>
- struct ReadyResult : public zyppng::AsyncOp< T >
- {
- ReadyResult( T &&val ) {
- this->setReady( std::move(val) );
- }
+ std::shared_ptr<AOp> _myTask;
};
//need a wrapper to connect a sync callback to a async one
virtual ~SimplifyHelper(){}
- void operator() ( std::
unique_ptr<AOp> &&op ) {
+ void operator() ( std::
shared_ptr<AOp> &&op ) {
assert( !_task );
_task = std::move(op);
_task->onReady( [this]( Ret &&val ){
});
}
private:
- std::
unique_ptr<AOp> _task;
+ std::
shared_ptr<AOp> _task;
};
/*!
* Usually we do not want to wait on the future of a future but get the nested result immediately
*/
template < typename Res >
- inline std::unique_ptr<AsyncOp<Res>> simplify ( std::unique_ptr< AsyncOp<Res> > &&res ) {
+ inline std::shared_ptr<AsyncOp<Res>> simplify ( std::shared_ptr< AsyncOp<Res> > &&res ) {
return std::move(res);
}
template < typename Res,
- typename AOp = AsyncOp< std::unique_ptr< AsyncOp<Res>> > >
- inline std::unique_ptr<AsyncOp<Res>> simplify ( std::unique_ptr< AsyncOp< std::unique_ptr< AsyncOp<Res>> > > &&res ) {
- std::unique_ptr<AsyncOp<Res>> op = std::make_unique< detail::AsyncResult< AOp, SimplifyHelper< AsyncOp<Res>>> >( std::move(res), std::make_unique<SimplifyHelper< AsyncOp<Res>>>() );
+ typename AOp = AsyncOp< std::shared_ptr< AsyncOp<Res>> > >
+ inline std::shared_ptr<AsyncOp<Res>> simplify ( std::shared_ptr< AsyncOp< std::shared_ptr< AsyncOp<Res>> > > &&res ) {
+ std::shared_ptr<AsyncOp<Res>> op = std::make_shared< detail::AsyncResult< AOp, SimplifyHelper< AsyncOp<Res>>> >( std::move(res), std::make_shared<SimplifyHelper< AsyncOp<Res>>>() );
return detail::simplify( std::move(op) );
}
}
- template <typename T>
- AsyncOpPtr<T> makeReadyResult ( T && result ) {
- return std::make_unique<detail::ReadyResult<T>>( std::move(result) );
- }
-
namespace operators {
//case 1 : binding a async message to a async callback
//is the callback signature what we want?
, std::enable_if_t< detail::is_future_monad_cb< Callback, Ret, typename PrevOp::value_type >::value, int> = 0
>
- auto operator| ( std::unique_ptr<PrevOp> &&future, std::unique_ptr<Callback> &&c )
+ auto operator| ( std::shared_ptr<PrevOp> &&future, std::shared_ptr<Callback> &&c )
{
- std::unique_ptr<AsyncOp<Ret>> op = std::make_unique<detail::AsyncResult< PrevOp, Callback>>( std::move(future), std::move(c) );
+ std::shared_ptr<AsyncOp<Ret>> op = std::make_shared<detail::AsyncResult< PrevOp, Callback>>( std::move(future), std::move(c) );
return detail::simplify( std::move(op) );
}
, std::enable_if_t< detail::is_async_op<PrevOp>::value, int> = 0
, std::enable_if_t< detail::is_sync_monad_cb< Callback, typename PrevOp::value_type >::value, int> = 0
>
- auto operator| ( std::
unique_ptr<PrevOp> &&future, Callback &&c )
+ auto operator| ( std::
shared_ptr<PrevOp> &&future, Callback &&c )
{
- std::unique_ptr<AsyncOp<Ret>> op(std::make_unique<detail::AsyncResult< PrevOp, detail::SyncCallbackWrapper<Callback, typename PrevOp::value_type, Ret> >>(
+ std::shared_ptr<AsyncOp<Ret>> op(std::make_shared<detail::AsyncResult< PrevOp, detail::SyncCallbackWrapper<Callback, typename PrevOp::value_type, Ret> >>(
std::move(future)
- , std::make_unique<detail::SyncCallbackWrapper<Callback, typename PrevOp::value_type, Ret>>( std::forward<Callback>(c) ) ));
+ , std::make_shared<detail::SyncCallbackWrapper<Callback, typename PrevOp::value_type, Ret>>( std::forward<Callback>(c) ) ));
return detail::simplify( std::move(op) );
}
, std::enable_if_t< !detail::is_async_op< remove_smart_ptr_t<SyncRes> >::value, int> = 0
, std::enable_if_t< detail::is_future_monad_cb< Callback, Ret, SyncRes >::value, int> = 0
>
- auto operator| ( SyncRes &&in, std::unique_ptr<Callback> &&c )
+ auto operator| ( SyncRes &&in, std::shared_ptr<Callback> &&c )
{
- AsyncOpPtr<Ret> op( std::make_unique<detail::AsyncResult<void, Callback, SyncRes>>( std::move(c) ) );
+ AsyncOpRef<Ret> op( std::make_shared<detail::AsyncResult<void, Callback, SyncRes>>( std::move(c) ) );
static_cast< detail::AsyncResult<void, Callback, SyncRes>* >(op.get())->run( std::move(in) );
return detail::simplify( std::move(op) );
}
typename SignalGetter >
auto await ( SignalGetter &&sigGet )
{
- return std::make_unique<detail::AwaitImpl<T, SignalGetter>>( std::forward<SignalGetter>(sigGet) );
+ return std::make_shared<detail::AwaitImpl<T, SignalGetter>>( std::forward<SignalGetter>(sigGet) );
}
}
return m_value;
}
+ T &operator* ()
+ {
+ return get();
+ }
+ const T &operator* () const
+ {
+ return get();
+ }
T *operator-> ()
{
};
+ template <typename Type, typename Err = std::exception_ptr >
+ static expected<Type,Err> make_expected_success( Type &&t )
+ {
+ return expected<Type,Err>::success( std::forward<Type>(t) );
+ }
+
+ namespace detail {
+
+ // helper to figure out the return type for a mbind callback, if the ArgType is void the callback is considered to take no argument.
+ // Due to how std::conditional works, we cannot pass std::invoke_result_t but instead use the template type std::invoke_result, since
+ // one of the two options have no "::type" because the substitution fails, this breaks the std::conditional_t since it can only work with two well formed
+ // types. Instead we pass in the template types and evaluate the ::type in the end, when the correct invoke_result was chosen.
+ template < typename Function, typename ArgType>
+ using mbind_cb_result_t = typename std::conditional_t< std::is_same_v<ArgType,void>, std::invoke_result<Function>,std::invoke_result<Function, ArgType> >::type;
+ }
+
template < typename T
, typename E
, typename Function
- , typename ResultType = decltype(std::declval<Function>()(std::declval<T>()))
+ , typename ResultType = detail::mbind_cb_result_t<Function, T>
>
std::enable_if_t< !detail::is_async_op< remove_smart_ptr_t<ResultType> >::value, ResultType> mbind( const expected<T, E>& exp, Function f)
{
if (exp) {
- return std::invoke( f, exp.get() );
+ if constexpr ( std::is_same_v<T,void> )
+ return std::invoke( f );
+ else
+ return std::invoke( f, exp.get() );
} else {
return ResultType::error(exp.error());
}
template < typename T
, typename E
, typename Function
- , typename ResultType = decltype(std::declval<Function>()(std::declval<T>()))
+ , typename ResultType = detail::mbind_cb_result_t<Function, T>
>
std::enable_if_t< !detail::is_async_op< remove_smart_ptr_t<ResultType> >::value, ResultType> mbind( expected<T, E>&& exp, Function f)
{
if (exp) {
- return std::invoke( f, std::move(exp.get()) );
+ if constexpr ( std::is_same_v<T,void> )
+ return std::invoke( f );
+ else
+ return std::invoke( f, std::move(exp.get()) );
} else {
return ResultType::error( std::move(exp.error()) );
}
template < typename T
, typename E
, typename Function
- , typename ResultType = decltype(std::declval<Function>()(std::declval<T>()))
+ , typename ResultType = detail::mbind_cb_result_t<Function, T>
>
std::enable_if_t< detail::is_async_op< remove_smart_ptr_t<ResultType> >::value, ResultType> mbind( const expected<T, E>& exp, Function f)
{
if (exp) {
- return std::invoke( f, exp.get() );
+ if constexpr ( std::is_same_v<T,void> )
+ return std::invoke( f );
+ else
+ return std::invoke( f, exp.get() );
} else {
return makeReadyResult( remove_smart_ptr_t<ResultType>::value_type::error(exp.error()) );
}
template < typename T
, typename E
, typename Function
- , typename ResultType = decltype(std::declval<Function>()(std::declval<T>()))
+ , typename ResultType = detail::mbind_cb_result_t<Function, T>
>
std::enable_if_t< detail::is_async_op< remove_smart_ptr_t<ResultType> >::value, ResultType> mbind( expected<T, E>&& exp, Function f)
{
if (exp) {
- return std::invoke( f, std::move(exp.get()) );
+ if constexpr ( std::is_same_v<T,void> )
+ return std::invoke( f );
+ else
+ return std::invoke( f, std::move(exp.get()) );
} else {
return makeReadyResult( remove_smart_ptr_t<ResultType>::value_type::error( std::move(exp.error()) ) );
}
};
template < typename AsyncOp >
- struct lifter< std::
unique_ptr<AsyncOp>, std::void_t< std::enable_if_t< zyppng::detail::is_async_op<AsyncOp>::value > > > {
+ struct lifter< std::
shared_ptr<AsyncOp>, std::void_t< std::enable_if_t< zyppng::detail::is_async_op<AsyncOp>::value > > > {
- using LiftedFun = std::
unique_ptr<AsyncOp>;
+ using LiftedFun = std::
shared_ptr<AsyncOp>;
lifter( LiftedFun &&fun ) : _fun(std::move(fun)) {}
lifter( lifter && ) = default;
};
template< typename MyAsyncOp, typename Pred >
- struct RedoWhileImpl< std::unique_ptr<MyAsyncOp>,Pred, std::enable_if_t< is_async_op< MyAsyncOp >::value > > : public AsyncOp<typename MyAsyncOp::value_type> {
+ struct RedoWhileImpl< std::shared_ptr<MyAsyncOp>,Pred, std::enable_if_t< is_async_op< MyAsyncOp >::value > > : public AsyncOp<typename MyAsyncOp::value_type> {
- using Task = std::unique_ptr<MyAsyncOp>;
+ using Task = std::shared_ptr<MyAsyncOp>;
using OutType = typename MyAsyncOp::value_type;
template <typename T, typename P>
template <typename T, typename P>
static auto create ( T &&t, P &&p ) {
- return std::make_
unique<RedoWhileImpl>( std::forward<T>(t), std::forward<P>(p));
+ return std::make_
shared<RedoWhileImpl>( std::forward<T>(t), std::forward<P>(p));
}
private:
Task _task;
Pred _pred;
- std::
unique_ptr<AsyncOp<OutType>> _pipeline;
+ std::
shared_ptr<AsyncOp<OutType>> _pipeline;
};
//implementation for a function returning a asynchronous result
template<typename InType>
void operator() ( InType &&arg ) {
- _asyncRes.reset(nullptr);
+ _asyncRes.reset();
_asyncRes = _task( InType( arg ) );
_asyncRes->onReady(
[this, inArg = arg ]( OutType &&arg ) mutable {
template <typename T, typename P>
static auto create ( T &&t, P &&p ) {
- return std::make_
unique<RedoWhileImpl>( std::forward<T>(t), std::forward<P>(p));
+ return std::make_
shared<RedoWhileImpl>( std::forward<T>(t), std::forward<P>(p));
}
private:
- std::
unique_ptr<AsyncOp<OutType>> _asyncRes;
+ std::
shared_ptr<AsyncOp<OutType>> _asyncRes;
Task _task;
Pred _pred;
{
Container<Ret> res;
std::transform( std::make_move_iterator(val.begin()), std::make_move_iterator(val.end()), std::back_inserter(res), transformation );
- return std::move(res);
+ return res;
+}
+
+template < template< class, class... > class Container,
+ typename Msg,
+ typename Transformation,
+ typename Ret = std::result_of_t<Transformation(Msg)>,
+ typename ...CArgs >
+Container<Ret> transform( const Container<Msg, CArgs...>& val, Transformation transformation )
+{
+ Container<Ret> res;
+ std::transform( val.begin(), val.end(), std::back_inserter(res), transformation );
+ return res;
}
namespace detail {
struct transform_helper {
Transformation function;
- template<template< class, class... > class Container, typename Msg, typename ...CArgs>
- auto operator()( Container<Msg, CArgs...>&& arg ) {
- return zyppng::transform( std::forward< Container<Msg,CArgs...> >(arg), function );
+ template< class Container >
+ auto operator()( Container&& arg ) {
+ return zyppng::transform( std::forward<Container>(arg), function );
}
};
}
#include <zypp-core/zyppng/pipelines/AsyncResult>
+namespace zyppng {
+
namespace detail {
template < class AsyncOp,
WaitForImpl& operator= ( WaitForImpl &&other ) = default;
WaitForImpl ( WaitForImpl &&other ) = default;
- void operator()( std::vector< std::unique_ptr< AsyncOp > > &&ops ) {
+ void operator()( std::vector< std::shared_ptr< AsyncOp > > &&ops ) {
assert( _allOps.empty() );
+ if ( ops.empty () ) {
+ this->setReady( std::move(_allResults) );
+ }
+
_allOps = std::move( ops );
for ( auto &op : _allOps ) {
op->onReady( [ this ]( typename AsyncOp::value_type &&res ) {
}
}
- std::vector< std::unique_ptr<zyppng::AsyncOp<InnerResult>> > _allOps;
+ std::vector< std::shared_ptr<zyppng::AsyncOp<InnerResult>> > _allOps;
std::vector< InnerResult > _allResults;
};
* Returns a async operation that waits for all async operations that are passed to it and collects their results,
* forwarding them as one
*/
-template < class Res >
-auto waitFor () {
- return std::make_unique<detail::WaitForImpl<zyppng::AsyncOp<Res>>>();
+template < class Res >
+auto waitFor ( ) {
+ return std::make_shared<detail::WaitForImpl<zyppng::AsyncOp<Res>>>();
}
-
+}
#endif
--- /dev/null
+#include "messagestream.h"
\ No newline at end of file
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+----------------------------------------------------------------------*/
+
+#include "messagestream.h"
+
+#include <zypp-core/AutoDispose.h>
+#include <zypp-core/zyppng/base/AutoDisconnect>
+
+namespace zyppng {
+
+ InvalidMessageReceivedException::InvalidMessageReceivedException( const std::string &msg )
+ : zypp::Exception( zypp::str::Str() << "Invalid Message received: (" << msg <<")" )
+ { }
+
+
+ zyppng::RpcMessageStream::RpcMessageStream( IODevice::Ptr iostr ) : _ioDev( std::move(iostr) )
+ {
+ connect( *_nextMessageTimer, &Timer::sigExpired, *this, &RpcMessageStream::timeout );
+ _nextMessageTimer->setSingleShot(false);
+
+ connect( *_ioDev, &IODevice::sigReadyRead, *this, &RpcMessageStream::readAllMessages );
+ if ( _ioDev->isOpen () && _ioDev->canRead () )
+ readAllMessages ();
+ }
+
+ bool RpcMessageStream::readNextMessage( )
+ {
+ if ( _pendingMessageSize == 0 ) {
+ if ( _ioDev->bytesAvailable() >= sizeof( rpc::HeaderSizeType ) ) {
+ _ioDev->read( reinterpret_cast<char *>( &_pendingMessageSize ), sizeof( rpc::HeaderSizeType ) );
+ }
+ }
+
+ if ( _ioDev->bytesAvailable() < _pendingMessageSize ) {
+ return false;
+ }
+
+ auto bytes = _ioDev->read( _pendingMessageSize );
+ _pendingMessageSize = 0;
+
+ zypp::proto::Envelope m;
+ if (! m.ParseFromArray( bytes.data(), bytes.size() ) ) {
+ ERR << "Received malformed message from peer" << std::endl;
+ _sigInvalidMessageReceived.emit();
+ return false;
+ }
+
+ _messages.push_back( std::move(m) );
+ _sigNextMessage.emit ();
+
+ if ( _messages.size() ) {
+ // nag the user code until all messages have been used up
+ _nextMessageTimer->start(0);
+ }
+
+ return true;
+ }
+
+ void RpcMessageStream::timeout(const Timer &)
+ {
+ if ( _messages.size() )
+ _sigNextMessage.emit();
+
+ if ( !_messages.size() )
+ _nextMessageTimer->stop();
+ }
+
+ std::optional<RpcMessage> zyppng::RpcMessageStream::nextMessage( const std::string &msgName )
+ {
+ if ( !_messages.size () ) {
+
+ // try to read the next messages from the fd
+ {
+ _sigNextMessage.block ();
+ zypp::OnScopeExit unblock([&](){
+ _sigNextMessage.unblock();
+ });
+ readAllMessages();
+ }
+
+ if ( !_messages.size () )
+ return {};
+ }
+
+ std::optional<RpcMessage> res;
+
+ if( msgName.empty() ) {
+ res = std::move( _messages.front () );
+ _messages.pop_front();
+
+ } else {
+ const auto i = std::find_if( _messages.begin(), _messages.end(), [&]( const zypp::proto::Envelope &env ) {
+ return env.messagetypename() == msgName;
+ });
+
+ if ( i != _messages.end() ) {
+ res = std::move(*i);
+ _messages.erase(i);
+ }
+ }
+
+ if ( _messages.size() )
+ _nextMessageTimer->start(0);
+ else
+ _nextMessageTimer->stop();
+
+ return res;
+ }
+
+ std::optional<RpcMessage> RpcMessageStream::nextMessageWait( const std::string &msgName )
+ {
+ // make sure the signal is not emitted until we have the next message
+ _sigNextMessage.block ();
+ zypp::OnScopeExit unblock([&](){
+ _sigNextMessage.unblock();
+ });
+
+ bool receivedInvalidMsg = false;
+ AutoDisconnect defered( connectFunc( *this, &RpcMessageStream::sigInvalidMessageReceived, [&](){
+ receivedInvalidMsg = true;
+ }));
+
+ const bool hasMsgName = msgName.size();
+ while ( !receivedInvalidMsg && _ioDev->isOpen() && _ioDev->canRead() ) {
+ if ( _messages.size() ) {
+ if ( hasMsgName ) {
+ std::optional<RpcMessage> msg = nextMessage(msgName);
+ if ( msg ) return msg;
+ }
+ else {
+ break;
+ }
+ }
+
+ if ( !_ioDev->waitForReadyRead ( -1 ) ) {
+ // this can only mean that a error happened, like device was closed
+ return {};
+ }
+ }
+ return nextMessage (msgName);
+ }
+
+ bool zyppng::RpcMessageStream::sendMessage( const RpcMessage &env )
+ {
+ if ( !_ioDev->canWrite () )
+ return false;
+
+ const auto &str = env.SerializeAsString();
+ rpc::HeaderSizeType msgSize = str.length();
+ _ioDev->write( (char *)(&msgSize), sizeof( rpc::HeaderSizeType ) );
+ _ioDev->write( str.data(), str.size() );
+ return true;
+ }
+
+ SignalProxy<void ()> zyppng::RpcMessageStream::sigMessageReceived()
+ {
+ return _sigNextMessage;
+ }
+
+ SignalProxy<void ()> RpcMessageStream::sigInvalidMessageReceived()
+ {
+ return _sigInvalidMessageReceived;
+ }
+
+ void RpcMessageStream::readAllMessages()
+ {
+ bool cont = true;
+ while ( cont && _ioDev->bytesAvailable() ) {
+ cont = readNextMessage ();
+ }
+ }
+}
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+-----------------------------------------------------------------------/
+*
+* This file contains private API, this might break at any time between releases.
+* You have been warned!
+*
+*/
+
+#ifndef ZYPP_CORE_ZYPPNG_RPC_MESSAGESTREAM_H_INCLUDED
+#define ZYPP_CORE_ZYPPNG_RPC_MESSAGESTREAM_H_INCLUDED
+
+#include <zypp-core/zyppng/base/Base>
+#include <zypp-core/zyppng/base/Signals>
+#include <zypp-core/zyppng/base/Timer>
+#include <zypp-core/zyppng/io/IODevice>
+#include <zypp-core/zyppng/pipelines/expected.h>
+#include <zypp-proto/envelope.pb.h>
+#include <zypp-core/zyppng/rpc/rpc.h>
+
+#include <deque>
+#include <optional>
+
+namespace zyppng {
+
+ using RpcMessage = zypp::proto::Envelope;
+
+ namespace rpc {
+ /*!
+ * Helper function to get the type name of a given RPC message type.
+ * Sadly Protobuf does not offer a static function to get the types FQN we
+ * cache it after asking for it the first time. So we need a dummy object just once.
+ */
+ template <typename T>
+ const std::string & messageTypeName() {
+ static std::string name = T().GetTypeName();
+ return name;
+ }
+ }
+
+ class InvalidMessageReceivedException : public zypp::Exception
+ {
+ public:
+ InvalidMessageReceivedException( const std::string &msg = {});
+ };
+
+
+ /*!
+ *
+ * Implements the basic protocol for sending zypp RPC messages over a IODevice
+ *
+ * Communication Format:
+ * ---------------------
+ * Each message is serialized into a zypp.proto.Envelope and sent over the communication medium in binary
+ * format. The binary format looks like:
+ *
+ * +--------------------------------+---------------------------------+
+ * | msglen ( 32 bit unsigned int ) | binary zypp.proto.Envelope data |
+ * +--------------------------------+---------------------------------+
+ *
+ * The header defines the size in bytes of the following data trailer. The header type is a 32 bit uint, endianess is defined by
+ * the underlying CPU arch. The data portion is directly generated by libprotobuf via SerializeToZeroCopyStream() to generate
+ * the binary represenation of the message.
+ *
+ */
+ class RpcMessageStream : public zyppng::Base
+ {
+ public:
+
+ using Ptr = std::shared_ptr<RpcMessageStream>;
+
+ /*!
+ * Uses the given iostream to send and receive messages.
+ * If the device is already open and readable tries to read messages right away.
+ * So make sure to check if messages have already been received via \ref nextMessage
+ */
+ static Ptr create( IODevice::Ptr iostr ) {
+ return Ptr( new RpcMessageStream( std::move(iostr) ) );
+ }
+
+ /*!
+ * Returns the next message in the queue, wait for the \ref sigMessageReceived signal
+ * to know when new messages have arrived.
+ * If \a msgName is specified returns the next message in the queue that matches the msgName
+ */
+ std::optional<RpcMessage> nextMessage ( const std::string &msgName = "" );
+
+ /*!
+ * Waits until at least one message is in the queue and returns it. Will return a empty
+ * optional if a error occurs.
+ *
+ * If \a msgName is set this will block until a message with the given message name arrives and returns it
+ *
+ * \note Make sure to check if there are more than one messages in the queue after this function returns
+ */
+ std::optional<RpcMessage> nextMessageWait ( const std::string &msgName = "" );
+
+ /*!
+ * Send out a RpcMessage to the other side, depending on the underlying device state
+ * this will be buffered and send when the device is writeable again.
+ */
+ bool sendMessage ( const RpcMessage &env );
+
+ /*!
+ * Reads all messages from the underlying IO Device, this is usually called automatically
+ * but when shutting down this can be used to process all remaining messages.
+ */
+ void readAllMessages ();
+
+ /*!
+ * Send a messagee to the server side, it will be enclosed in a Envelope
+ * and immediately sent out.
+ */
+ template <typename T>
+ std::enable_if_t< !std::is_same_v<T, RpcMessage>, bool> sendMessage ( const T &m ) {
+ RpcMessage env;
+
+ env.set_messagetypename( m.GetTypeName() );
+ m.SerializeToString( env.mutable_value() );
+
+ return sendMessage ( env );
+ }
+
+ /*!
+ * Emitted when new messages have arrived. This will continuously be emitted
+ * as long as messages are in the queue.
+ */
+ SignalProxy<void()> sigMessageReceived ();
+
+ /*!
+ * Signal is emitted every time there was data on the line that could not be parsed
+ */
+ SignalProxy<void()> sigInvalidMessageReceived ();
+
+ template<class T>
+ static expected< T > parseMessage ( const RpcMessage &m ) {
+ T p;
+ if ( !p.ParseFromString( m.value() ) ) {
+ const auto &msg = zypp::str::Str() << "Failed to parse " << m.messagetypename() << " message.";
+ ERR << msg << std::endl ;
+ return expected<T>::error( ZYPP_EXCPT_PTR ( InvalidMessageReceivedException(msg) ) );
+ }
+ return expected<T>::success( std::move(p) );
+ }
+
+ template<class T>
+ static expected< void > parseMessageInto ( const RpcMessage &m, T &target ) {
+ if ( !target.ParseFromString( m.value() ) ) {
+ const auto &msg = zypp::str::Str() << "Failed to parse " << m.messagetypename() << " message.";
+ ERR << msg << std::endl ;
+ return expected<void>::error( ZYPP_EXCPT_PTR ( InvalidMessageReceivedException(msg) ) );
+ }
+ return expected<void>::success( );
+ }
+
+ private:
+ RpcMessageStream( IODevice::Ptr iostr );
+ bool readNextMessage ();
+ void timeout( const zyppng::Timer &);
+
+ IODevice::Ptr _ioDev;
+ Timer::Ptr _nextMessageTimer = Timer::create();
+ zyppng::rpc::HeaderSizeType _pendingMessageSize = 0;
+ std::deque<RpcMessage> _messages;
+ Signal<void()> _sigNextMessage;
+ Signal<void()> _sigInvalidMessageReceived;
+
+ };
+}
+
+
+
+#endif // ZYPP_CORE_ZYPPNG_RPC_MESSAGESTREAM_H_INCLUDED
ADD_LIBRARY( zypp-curl STATIC ${zypp_curl_lib_SRCS} ${zypp_curl_lib_HEADERS} )
#we include generated headers, so we need to wait for zypp-protobuf to be ready
add_dependencies( zypp-curl zypp-protobuf )
+
+TARGET_LINK_LIBRARIES( zypp-curl ${CURL_LIBRARIES} )
+TARGET_LINK_LIBRARIES( zypp-curl ${LIBPROXY_LIBRARIES} )
+TARGET_LINK_LIBRARIES( zypp-curl ${LIBXML2_LIBRARIES} )
#include <zypp-curl/auth/CurlAuthData>
#include <zypp-media/MediaException>
#include <list>
+#include <string>
using std::endl;
using namespace zypp;
// bsc#933839: propagate proxy settings passed in the repo URL
zypp::Url propagateQueryParams( zypp::Url url_r, const zypp::Url & template_r )
{
- for ( const std::string ¶m : { "proxy", "proxyport", "proxyuser", "proxypass"} )
+ using namespace std::literals::string_literals;
+ for ( const std::string ¶m : { "proxy"s, "proxyport"s, "proxyuser"s, "proxypass"s} )
{
const std::string & value( template_r.getQueryParam( param ) );
if ( ! value.empty() )
MIL << "Authentication failed for " << req->url() << " trying to recover." << std::endl;
- zypp::media::CredentialManager cm( _parent ? _parent->credManagerOptions() : zypp::media::CredManagerOptions() );
- auto authDataPtr = cm.getCred( req->url() );
-
- // get stored credentials
- NetworkAuthData_Ptr cmcred( authDataPtr ? new NetworkAuthData( *authDataPtr ) : new NetworkAuthData() );
TransferSettings &ts = req->transferSettings();
-
- // We got credentials from store, _triedCredFromStore makes sure we just try the auth from store once
- // if its timestamp does not change between 2 retries
- if ( cmcred && ( !req->_triedCredFromStore || req->_authTimestamp < cmcred->lastDatabaseUpdate() ) ) {
- MIL << "got stored credentials:" << std::endl << *cmcred << std::endl;
- ts.setUsername( cmcred->username() );
- ts.setPassword( cmcred->password() );
- retry = true;
- req->_triedCredFromStore = true;
- _lastTriedAuthTime = req->_authTimestamp = cmcred->lastDatabaseUpdate();
- } else {
-
- //we did not get credentials from the store, emit a signal that allows
- //setting new auth data
-
- NetworkAuthData credFromUser;
- credFromUser.setUrl( req->url() );
-
- //in case we got a auth hint from the server the error object will contain it
- std::string authHint = err.extraInfoValue("authHint", std::string());
-
- //preset from store if we found something
- if ( cmcred && !cmcred->username().empty() )
- credFromUser.setUsername( cmcred->username() );
-
- _sigAuthRequired.emit( *z_func(), credFromUser, authHint );
- if ( credFromUser.valid() ) {
- MIL << "Got user provided credentials" << std::endl;
- ts.setUsername( credFromUser.username() );
- ts.setPassword( credFromUser.password() );
-
+ const auto &applyCredToSettings = [&ts]( AuthData_Ptr auth, const std::string &authHint ) {
+ ts.setUsername( auth->username() );
+ ts.setPassword( auth->password() );
+ auto nwCred = dynamic_cast<NetworkAuthData *>( auth.get() );
+ if ( nwCred ) {
// set available authentication types from the error
- if ( credFromUser.authType() == CURLAUTH_NONE )
- credFromUser.setAuthType( authHint );
+ if ( nwCred->authType() == CURLAUTH_NONE )
+ nwCred->setAuthType( authHint );
// set auth type (seems this must be set _after_ setting the userpwd)
- if ( credFromUser.authType() != CURLAUTH_NONE ) {
+ if ( nwCred->authType() != CURLAUTH_NONE ) {
// FIXME: only overwrite if not empty?
- req->transferSettings().setAuthType(credFromUser.authTypeAsString());
+ ts.setAuthType(nwCred->authTypeAsString());
}
+ }
+ };
+
+ // try to find one in the cache
+ zypp::url::ViewOption vopt;
+ vopt = vopt
+ - zypp::url::ViewOption::WITH_USERNAME
+ - zypp::url::ViewOption::WITH_PASSWORD
+ - zypp::url::ViewOption::WITH_QUERY_STR;
+
+ auto cachedCred = zypp::media::CredentialManager::findIn( _credCache, req->url(), vopt );
+
+ // only consider a cache entry if its newer than what we tried last time
+ if ( cachedCred && cachedCred->lastDatabaseUpdate() > req->_authTimestamp ) {
+ MIL << "Found a credential match in the cache!" << std::endl;
+ applyCredToSettings( cachedCred, "" );
+ _lastTriedAuthTime = req->_authTimestamp = cachedCred->lastDatabaseUpdate();
+ retry = true;
+ } else {
- cm.addCred( credFromUser );
- cm.save();
+ NetworkAuthData_Ptr credFromUser = NetworkAuthData_Ptr( new NetworkAuthData() );
+ credFromUser->setUrl( req->url() );
+ credFromUser->setLastDatabaseUpdate ( req->_authTimestamp );
- // potentially setting this after the file has been touched we might miss a change
- // in a later loop, if another update to the store happens right in the few ms difference
- // between the actual file write and calling time() here. However this is highly unlikely
- _lastTriedAuthTime = req->_authTimestamp = time( nullptr ) ;
+ //in case we got a auth hint from the server the error object will contain it
+ std::string authHint = err.extraInfoValue("authHint", std::string());
+ _sigAuthRequired.emit( *z_func(), *credFromUser, authHint );
+ if ( credFromUser->valid() ) {
+ // remember for next time , we don't want to ask the user again for the same URL set
+ _credCache.insert( credFromUser );
+ applyCredToSettings( credFromUser, authHint );
+ _lastTriedAuthTime = req->_authTimestamp = credFromUser->lastDatabaseUpdate();
retry = true;
}
}
}
//reset state variables
- _emittedSigStart = false;
- _specHasZckInfo = zypp::indeterminate;
+ _specHasZckInfo = zypp::indeterminate;
+ _emittedSigStart = false;
+ _stoppedOnMetalink = false;
+ _lastTriedAuthTime = 0;
// restart the statemachine
if ( cState == Download::Finished )
if ( _spec.settings().proxy().empty() )
::internal::fillSettingsSystemProxy( url, set );
+#if 0
/* Fixes bsc#1174011 "auth=basic ignored in some cases"
* We should proactively add the password to the request if basic auth is configured
* and a password is available in the credentials but not in the URL.
set.setPassword(cred->password());
}
}
+#endif
} catch ( const zypp::media::MediaBadUrlException & e ) {
res = NetworkRequestErrorPrivate::customError( NetworkRequestError::MalformedURL, e.asString(), buildExtraInfo() );
d->forceState ( std::make_unique<FinishedState>( NetworkRequestErrorPrivate::customError( NetworkRequestError::Cancelled, "Download was cancelled explicitly" ), *d_func() ) );
}
+ void Download::setStopOnMetalink(const bool set)
+ {
+ d_func()->_stopOnMetalink = set;
+ }
+
+ bool Download::stoppedOnMetalink() const
+ {
+ return d_func()->_stoppedOnMetalink;
+ }
+
DownloadSpec &Download::spec()
{
return d_func()->_spec;
}
}
- const zypp::media::CredManagerOptions &Downloader::credManagerOptions() const
- {
- return d_func()->_credManagerOptions;
- }
-
- void Downloader::setCredManagerOptions(const zypp::media::CredManagerOptions &options)
- {
- d_func()->_credManagerOptions = options;
- }
-
std::shared_ptr<Download> Downloader::downloadFile(const zyppng::DownloadSpec &spec )
{
Z_D();
#include <zypp-core/zyppng/base/Base>
#include <zypp-core/zyppng/base/signals.h>
#include <zypp-core/zyppng/core/Url>
-#include <zypp-media/auth/CredentialManager>
#include <zypp-curl/ng/network/networkrequesterror.h>
#include <zypp-curl/ng/network/AuthData>
virtual ~Downloader();
/*!
- * Returns the currently used CredentialManager options
- */
- const zypp::media::CredManagerOptions &credManagerOptions () const;
-
- /*!
- * Sets the options for the CredentialManager to retrieve auth data
- */
- void setCredManagerOptions ( const zypp::media::CredManagerOptions & options );
-
- /*!
* Generates a new Download object in waiting state
*/
std::shared_ptr<Download> downloadFile ( const DownloadSpec &spec );
void cancel ();
/*!
+ * This will finalize the download once a metalink file was received. This is a special setting used
+ * if the metalink file should be processed outside of the Downloader.
+ */
+ void setStopOnMetalink ( const bool set = true );
+
+ /*!
+ * Returns true if the download was stopped after receiving metalink data. The target file
+ * will contain the metalink description.
+ */
+ bool stoppedOnMetalink () const;
+
+ /*!
* Returns a reference to the internally used download spec.
* \sa zyppng::DownloadSpec
* \note Changing settings after the download started might result in undefined or weird behaviour
/*!
* Is emitted when a request requires authentication and it was not given or if auth failed.
* A connected slot should fill in the \a auth information in order to provide login credentials.
+ *
+ * \note the URL in the \a auth data is set to the url of the request needing authorization.
+ * This URL can be different than the one in the spec, due to redirect/metalink.
+ * Also the lastDatabaseUpdate field is set to the timestamp of the auth data that was tried already ( if available )
*/
SignalProxy<void ( Download &req, NetworkAuthData &auth, const std::string &availAuth )> sigAuthRequired ( );
----------------------------------------------------------------------*/
#include "downloadspec.h"
#include <string>
-#include <zypp-proto/download.pb.h>
namespace zyppng {
DownloadSpecPrivate() = default;
DownloadSpecPrivate( const DownloadSpecPrivate &other ) = default;
DownloadSpecPrivate( DownloadSpecPrivate &&other ) = default;
- DownloadSpecPrivate( zypp::proto::DownloadSpec spec ) :_protoData( std::move(spec) ) {}
DownloadSpecPrivate *clone () const {
return new DownloadSpecPrivate(*this);
}
- zypp::proto::DownloadSpec _protoData;
+ zypp::Url _url;
+ TransferSettings _settings;
+ zypp::Pathname _delta;
+ zypp::ByteCount _expectedFileSize;
+ zypp::Pathname _targetPath;
+ bool _checkExistanceOnly = false; //< this will NOT download the file, but only query the server if it exists
+ bool _metalink_enabled = true; //< should the download try to use metalinks
+ zypp::ByteCount _headerSize; //< Optional file header size for things like zchunk
+ std::optional<zypp::CheckSum> _headerChecksum; //< Optional file header checksum
+ zypp::ByteCount _preferred_chunk_size = zypp::ByteCount( 4096, zypp::ByteCount::K );
};
ZYPP_IMPL_PRIVATE( DownloadSpec )
DownloadSpec::DownloadSpec( Url file , zypp::filesystem::Pathname targetPath, zypp::ByteCount expectedFileSize ) : d_ptr( new DownloadSpecPrivate() )
{
// default settings
- *d_ptr->_protoData.mutable_settings() = std::move( TransferSettings().protoData() );
-
- setUrl( file );
- setTargetPath( targetPath );
- setExpectedFileSize( expectedFileSize );
- setPreferredChunkSize( zypp::ByteCount( 4096, zypp::ByteCount::K ) );
- d_ptr->_protoData.set_checkexistanceonly( false );
- d_ptr->_protoData.set_metalink_enabled( true );
+ d_ptr->_url = std::move(file);
+ d_ptr->_targetPath = std::move(targetPath);
+ d_ptr->_expectedFileSize = std::move( expectedFileSize );
}
- DownloadSpec::DownloadSpec( const zypp::proto::DownloadSpec &spec ) : d_ptr( new DownloadSpecPrivate( spec ) )
- { }
-
DownloadSpec::DownloadSpec( const DownloadSpec &other ) = default;
DownloadSpec &DownloadSpec::operator=(const DownloadSpec &other) = default;
- Url DownloadSpec::url() const
+ const Url &DownloadSpec::url() const
{
- return d_ptr->_protoData.url();
+ return d_ptr->_url;
}
DownloadSpec &DownloadSpec::setUrl(const Url &url)
{
- d_ptr->_protoData.set_url( url.asCompleteString() );
+ d_ptr->_url = url;
return *this;
}
- zypp::Pathname DownloadSpec::targetPath() const
+ const zypp::Pathname &DownloadSpec::targetPath() const
{
- return zypp::Pathname(d_ptr->_protoData.targetpath());
+ return d_ptr->_targetPath;
}
DownloadSpec &DownloadSpec::setTargetPath(const zypp::filesystem::Pathname &path)
{
- d_ptr->_protoData.set_targetpath( path.asString() );
+ d_ptr->_targetPath = path;
return *this;
}
DownloadSpec &DownloadSpec::setMetalinkEnabled(bool enable)
{
- d_ptr->_protoData.set_metalink_enabled( enable );
+ d_ptr->_metalink_enabled = enable;
return *this;
}
bool DownloadSpec::metalinkEnabled() const
{
- return d_ptr->_protoData.metalink_enabled();
+ return d_ptr->_metalink_enabled;
}
DownloadSpec &DownloadSpec::setCheckExistsOnly(bool set)
{
- d_ptr->_protoData.set_checkexistanceonly( set );
+ d_ptr->_checkExistanceOnly = ( set );
return *this;
}
bool DownloadSpec::checkExistsOnly() const
{
- return d_ptr->_protoData.checkexistanceonly();
+ return d_ptr->_checkExistanceOnly;
}
- DownloadSpec &DownloadSpec::setDeltaFile(const zypp::filesystem::Pathname &file)
+ DownloadSpec &DownloadSpec::setDeltaFile(const zypp::Pathname &file)
{
- if ( !file.empty() )
- d_ptr->_protoData.set_delta( file.asString() );
- else
- d_ptr->_protoData.clear_delta();
+ d_ptr->_delta = file;
return *this;
}
- zypp::filesystem::Pathname DownloadSpec::deltaFile() const
+ zypp::Pathname DownloadSpec::deltaFile() const
{
- return d_ptr->_protoData.delta();
+ return d_ptr->_delta;
}
DownloadSpec &DownloadSpec::setPreferredChunkSize(const zypp::ByteCount &bc)
{
- d_ptr->_protoData.set_preferred_chunk_size( bc.operator long long() );
+ d_ptr->_preferred_chunk_size = bc;
return *this;
}
zypp::ByteCount DownloadSpec::preferredChunkSize() const
{
- return d_ptr->_protoData.preferred_chunk_size();
+ return d_ptr->_preferred_chunk_size;
}
- TransferSettings DownloadSpec::settings() const
+ const TransferSettings &DownloadSpec::settings() const
{
- return TransferSettings( d_ptr->_protoData.settings() );
+ return d_ptr->_settings;
}
DownloadSpec & DownloadSpec::setTransferSettings(TransferSettings &&set)
{
- (*d_ptr->_protoData.mutable_settings()) = std::move( set.protoData() );
+ d_ptr->_settings = std::move( set );
return *this;
}
DownloadSpec & DownloadSpec::setTransferSettings(const TransferSettings &set)
{
- (*d_ptr->_protoData.mutable_settings()) = set.protoData();
+ d_ptr->_settings = set;
return *this;
}
DownloadSpec &DownloadSpec::setExpectedFileSize(const zypp::ByteCount &bc)
{
- d_ptr->_protoData.set_expectedfilesize( bc.operator long long() );
+ d_ptr->_expectedFileSize = bc;
return *this;
}
zypp::ByteCount DownloadSpec::expectedFileSize() const
{
- return d_ptr->_protoData.expectedfilesize();
+ return d_ptr->_expectedFileSize;
}
DownloadSpec &DownloadSpec::setHeaderSize(const zypp::ByteCount &bc)
{
- d_ptr->_protoData.set_headersize( bc.operator long long() );
+ d_ptr->_headerSize = bc;
return *this;
}
zypp::ByteCount DownloadSpec::headerSize() const
{
- return d_ptr->_protoData.headersize();
+ return d_ptr->_headerSize;
}
- std::optional<zypp::CheckSum> DownloadSpec::headerChecksum() const
+ const std::optional<zypp::CheckSum> &DownloadSpec::headerChecksum() const
{
Z_D();
- if ( !d->_protoData.has_headerchecksum() )
- return {};
-
- return zypp::CheckSum( d->_protoData.headerchecksum().type(), d->_protoData.headerchecksum().sum() );
-
+ return d->_headerChecksum;
}
DownloadSpec &DownloadSpec::setHeaderChecksum(const zypp::CheckSum &sum)
{
+ Z_D();
if ( sum.empty() )
- d_func()->_protoData.clear_headerchecksum();
+ d->_headerChecksum.reset();
else {
- auto csum = d_func()->_protoData.mutable_headerchecksum();
- csum->set_type( sum.type() );
- csum->set_sum( sum.checksum() );
+ d->_headerChecksum = sum;
}
return *this;
}
-
- const zypp::proto::DownloadSpec &DownloadSpec::protoData() const
- {
- return d_ptr->_protoData;
- }
-
- zypp::proto::DownloadSpec &DownloadSpec::protoData()
- {
- return d_ptr->_protoData;
- }
}
-
#include <optional>
-namespace zypp::proto {
- class DownloadSpec;
-}
-
namespace zyppng {
public:
DownloadSpec( Url file, zypp::filesystem::Pathname targetPath, zypp::ByteCount expectedFileSize = zypp::ByteCount() );
- DownloadSpec( const zypp::proto::DownloadSpec &spec );
DownloadSpec( const DownloadSpec &other );
DownloadSpec &operator= ( const DownloadSpec &other );
/*!
* Returns the source URL of the download
*/
- Url url () const;
+ const Url &url () const;
DownloadSpec &setUrl ( const Url &url );
/*!
* Returns the target file path, this is where the downloaded data is stored
*/
- zypp::filesystem::Pathname targetPath() const;
+ const zypp::Pathname &targetPath() const;
DownloadSpec &setTargetPath ( const zypp::Pathname &path );
/*!
* possible sub downloads, however authentication data is stripped if the subdownload uses a different host to
* fetch the data from. If there is no auth data known \sa sigAuthRequired is emitted.
*/
- TransferSettings settings () const;
+ const TransferSettings &settings () const;
DownloadSpec &setTransferSettings( TransferSettings &&set );
DownloadSpec &setTransferSettings( const TransferSettings &set );
DownloadSpec &setHeaderSize ( const zypp::ByteCount &bc );
zypp::ByteCount headerSize() const;
- std::optional<zypp::CheckSum> headerChecksum () const;
+ const std::optional<zypp::CheckSum> &headerChecksum () const;
DownloadSpec &setHeaderChecksum ( const zypp::CheckSum &sum );
- const zypp::proto::DownloadSpec &protoData() const;
- zypp::proto::DownloadSpec &protoData();
-
private:
zypp::RWCOW_pointer<DownloadSpecPrivate> d_ptr;
};
}
-
-
-
#endif // ZYPPNG_MEDIA_NETWORK_DOWNLOADSPEC_H
using namespace boost;
-namespace zypp {
- L_ENV_CONSTR_DEFINE_FUNC(ZYPP_MEDIA_CURL_DEBUG)
-}
+L_ENV_CONSTR_DEFINE_FUNC(ZYPP_MEDIA_CURL_DEBUG)
namespace zyppng {
Signal< void ( Downloader &parent, Download& download )> _sigFinished;
Signal< void ( Downloader &parent )> _queueEmpty;
std::shared_ptr<MirrorControl> _mirrors;
- zypp::media::CredManagerOptions _credManagerOptions; //< The credential manager options used to initialize the CredentialManager
};
}
#include <zypp-curl/ng/network/TransferSettings>
#include <zypp-curl/ng/network/private/mirrorcontrol_p.h>
#include <zypp-curl/ng/network/networkrequesterror.h>
+#include <zypp-media/auth/CredentialManager>
namespace zyppng {
}
void disconnectSignals ();
- bool _triedCredFromStore = false; //< already tried to authenticate from credential store?
- time_t _authTimestamp = 0; //< timestamp of the AuthData we tried from the store
+ time_t _authTimestamp = 0; //< timestamp of the AuthData we tried already
Url _originalUrl; //< The unstripped URL as it was passed to Download , before transfer settings are removed
MirrorControl::MirrorHandle _myMirror;
std::shared_ptr<NetworkRequestDispatcher> _requestDispatcher;
std::shared_ptr<MirrorControl> _mirrorControl;
+ zypp::media::CredentialManager::CredentialSet _credCache; //< the credential cache for this download
+
DownloadSpec _spec; // the download settings
mutable zypp::TriBool _specHasZckInfo = zypp::indeterminate;
Downloader *_parent = nullptr;
- time_t _lastTriedAuthTime = 0; //< if initialized this shows the last timestamp that we loaded a cred for the given URL from CredentialManager
+ time_t _lastTriedAuthTime = 0; //< if initialized this shows the last timestamp that got from user code for a auth request
+ bool _stopOnMetalink = false; //< Stop the download if a metalink was received for external parsing
+ bool _stoppedOnMetalink = false; //< Statemachine was stopped after receiving a metalink file
NetworkRequest::Priority _defaultSubRequestPriority = NetworkRequest::High;
Signal< void ( Download &req )> _sigStarted;
bool DlMetaLinkInfoState::initializeRequest(std::shared_ptr<Request> &r )
{
+ MIL << "Requesting Metalink info from server!" << std::endl;
r->transferSettings().addHeader("Accept: */*, application/metalink+xml, application/metalink4+xml");
return BasicDownloaderStateBase::initializeRequest(r);
}
return BasicDownloaderStateBase::gotFinished();
}
+ auto &sm = stateMachine();
+ if ( sm._stopOnMetalink ) {
+ MIL << "Stopping after receiving MetaLink data as requested" << std::endl;
+ sm._stoppedOnMetalink = true;
+ return BasicDownloaderStateBase::gotFinished();
+ }
+
// Move to Prepare Multi state
- MIL << "Downloading on " << stateMachine()._spec.url() << " returned a Metalink " << std::endl;
+ MIL << "Downloading on " << sm._spec.url() << " returned a Metalink " << std::endl;
_sigGotMetalink.emit();
}
#define ZYPP_NG_MEDIADEBUG_H_INCLUDED
#include <zypp-core/base/LogControl.h>
-namespace zypp {
- L_ENV_CONSTR_FWD_DECLARE_FUNC(ZYPP_MEDIA_CURL_DEBUG);
-}
+
+L_ENV_CONSTR_FWD_DECLARE_FUNC(ZYPP_MEDIA_CURL_DEBUG);
#ifdef ZYPP_BASE_LOGGER_LOGGROUP
#undef ZYPP_BASE_LOGGER_LOGGROUP
bool _isInCallback = false;
std::optional<NetworkRequestError> _cachedResult;
+ off_t _lastProgressNow = -1; // last value returned from CURL, lets only send signals if we get actual updates
off_t _downloaded = 0; //downloaded bytes
zypp::ByteCount _contentLenght; // the content length as reported by the server
that->resetActivityTimer();
rmode._isInCallback = true;
- that->_sigProgress.emit( *that->z_func(), dltotal, dlnow, ultotal, ulnow );
+ if ( rmode._lastProgressNow != dlnow ) {
+ rmode._lastProgressNow = dlnow;
+ that->_sigProgress.emit( *that->z_func(), dltotal, dlnow, ultotal, ulnow );
+ }
rmode._isInCallback = false;
return rmode._cachedResult ? CURLE_ABORTED_BY_CALLBACK : CURLE_OK;
void
MetaLinkParser::parse(const Pathname &filename)
{
+ MIL << "Begin parse " << filename << std::endl;
parse(InputStream(filename));
+ MIL << "End parse " << filename << std::endl;
}
void
#include <zypp-core/ExternalProgram.h>
#include <zypp-media/MediaConfig>
-#include <zypp-proto/transfersettings.pb.h>
-
#include <zypp/APIConfig.h>
using std::endl;
class TransferSettings::Impl
{
public:
- Impl() {
- _settingsObj.set_useproxy( false );
- _settingsObj.set_timeout( MediaConfig::instance().download_transfer_timeout() );
- _settingsObj.set_connect_timeout( 60 );
- _settingsObj.set_maxconcurrentconnections( MediaConfig::instance().download_max_concurrent_connections() );
- _settingsObj.set_mindownloadspeed(MediaConfig::instance().download_min_download_speed());
- _settingsObj.set_maxdownloadspeed(MediaConfig::instance().download_max_download_speed());
- _settingsObj.set_maxsilenttries(MediaConfig::instance().download_max_silent_tries() );
- _settingsObj.set_verify_host(false);
- _settingsObj.set_verify_peer(false);
- _settingsObj.set_ca_path("/etc/ssl/certs");
- _settingsObj.set_head_requests_allowed(true);
- }
+ Impl() : _useproxy( false ),
+ _timeout( MediaConfig::instance().download_transfer_timeout() ),
+ _connect_timeout( 60 ),
+ _maxConcurrentConnections( MediaConfig::instance().download_max_concurrent_connections() ),
+ _minDownloadSpeed(MediaConfig::instance().download_min_download_speed()),
+ _maxDownloadSpeed(MediaConfig::instance().download_max_download_speed()),
+ _maxSilentTries(MediaConfig::instance().download_max_silent_tries() ),
+ _verify_host(false),
+ _verify_peer(false),
+ _ca_path("/etc/ssl/certs"),
+ _head_requests_allowed(true)
+ {}
virtual ~Impl()
{}
public:
std::vector<std::string> _headers;
- zypp::proto::TransferSettings _settingsObj;
+ std::string _useragent;
+ std::string _username;
+ std::string _password;
+ bool _useproxy;
+ std::string _proxy;
+ std::string _proxy_username;
+ std::string _proxy_password;
+ std::string _authtype;
+ long _timeout;
+ long _connect_timeout;
+ Url _url;
+ Pathname _targetdir;
+
+ long _maxConcurrentConnections;
+ long _minDownloadSpeed;
+ long _maxDownloadSpeed;
+ long _maxSilentTries;
+
+ bool _verify_host;
+ bool _verify_peer;
+ Pathname _ca_path;
+ Pathname _client_cert_path;
+ Pathname _client_key_path;
+
+ // workarounds
+ bool _head_requests_allowed;
};
TransferSettings::TransferSettings()
: _impl(new TransferSettings::Impl())
{}
- TransferSettings::TransferSettings( const proto::TransferSettings &settings )
- : _impl(new TransferSettings::Impl())
- {
- _impl->_settingsObj = settings;
- }
-
void TransferSettings::reset()
{ _impl.reset(new TransferSettings::Impl()); }
+ void TransferSettings::addHeader( const std::string & val_r )
+ { if ( ! val_r.empty() ) _impl->_headers.push_back(val_r); }
+
void TransferSettings::addHeader( std::string && val_r )
- { if ( ! val_r.empty() ) *_impl->_settingsObj.add_header() = std::move(val_r); }
+ { if ( ! val_r.empty() ) _impl->_headers.push_back(std::move(val_r)); }
- TransferSettings::Headers TransferSettings::headers() const
+ const TransferSettings::Headers &TransferSettings::headers() const
{
//@TODO check if we could use a vector of std::string_view here
- auto vec = Headers();
- for ( const auto &head : _impl->_settingsObj.header() ) {
- vec.push_back( head );
- }
- return vec;
+ return _impl->_headers;
}
+ void TransferSettings::setUserAgentString( const std::string && val_r )
+ { _impl->_useragent = val_r; }
+
void TransferSettings::setUserAgentString( std::string && val_r )
- { _impl->_settingsObj.set_useragent( std::move(val_r) ); }
+ { _impl->_useragent = std::move(val_r); }
- std::string TransferSettings::userAgentString() const
- { return _impl->_settingsObj.useragent(); }
+ const std::string &TransferSettings::userAgentString() const
+ { return _impl->_useragent; }
+ void TransferSettings::setUsername( const std::string &val_r )
+ { _impl->_username = val_r; }
+
void TransferSettings::setUsername( std::string && val_r )
- { _impl->_settingsObj.set_username( std::move(val_r) ); }
+ { _impl->_username = std::move(val_r); }
+
+ const std::string &TransferSettings::username() const
+ { return _impl->_username; }
- std::string TransferSettings::username() const
- { return _impl->_settingsObj.username(); }
+ void TransferSettings::setPassword( const std::string & val_r )
+ { _impl->_password = val_r; }
void TransferSettings::setPassword( std::string && val_r )
- { _impl->_settingsObj.set_password( std::move(val_r) ); }
+ { _impl->_password = std::move(val_r); }
- std::string TransferSettings::password() const
- { return _impl->_settingsObj.password(); }
+ const std::string &TransferSettings::password() const
+ { return _impl->_password; }
std::string TransferSettings::userPassword() const
{
void TransferSettings::setProxyEnabled( bool enabled )
- { _impl->_settingsObj.set_useproxy( enabled ); }
+ { _impl->_useproxy = enabled; }
bool TransferSettings::proxyEnabled() const
- { return _impl->_settingsObj.useproxy(); }
+ { return _impl->_useproxy; }
+ void TransferSettings::setProxy( const std::string &val_r )
+ { _impl->_proxy = val_r; }
+
void TransferSettings::setProxy( std::string && val_r )
- { _impl->_settingsObj.set_proxy( std::move(val_r) ); }
+ { _impl->_proxy = std::move(val_r); }
+
+ const std::string &TransferSettings::proxy() const
+ { return _impl->_proxy; }
- std::string TransferSettings::proxy() const
- { return _impl->_settingsObj.proxy(); }
+ void TransferSettings::setProxyUsername( const std::string &val_r )
+ { _impl->_proxy_username = val_r; }
void TransferSettings::setProxyUsername( std::string && val_r )
- { _impl->_settingsObj.set_proxy_username( std::move(val_r) ); }
+ { _impl->_proxy_username = std::move(val_r); }
+
+ const std::string &TransferSettings::proxyUsername() const
+ { return _impl->_proxy_username; }
- std::string TransferSettings::proxyUsername() const
- { return _impl->_settingsObj.proxy_username(); }
+ void TransferSettings::setProxyPassword( const std::string &val_r )
+ { _impl->_proxy_password = val_r; }
void TransferSettings::setProxyPassword( std::string && val_r )
- { _impl->_settingsObj.set_proxy_password( std::move(val_r) ); }
+ { _impl->_proxy_password = std::move(val_r); }
- std::string TransferSettings::proxyPassword() const
- { return _impl->_settingsObj.proxy_password(); }
+ const std::string &TransferSettings::proxyPassword() const
+ { return _impl->_proxy_password; }
std::string TransferSettings::proxyUserPassword() const
{
void TransferSettings::setTimeout( long t )
- { _impl->_settingsObj.set_timeout(t); }
+ { _impl->_timeout = (t); }
long TransferSettings::timeout() const
- { return _impl->_settingsObj.timeout(); }
+ { return _impl->_timeout; }
void TransferSettings::setConnectTimeout( long t )
- { _impl->_settingsObj.set_connect_timeout(t); }
+ { _impl->_connect_timeout = (t); }
long TransferSettings::connectTimeout() const
- { return _impl->_settingsObj.connect_timeout(); }
+ { return _impl->_connect_timeout; }
void TransferSettings::setMaxConcurrentConnections( long v )
- { _impl->_settingsObj.set_maxconcurrentconnections(v); }
+ { _impl->_maxConcurrentConnections = (v); }
long TransferSettings::maxConcurrentConnections() const
- { return _impl->_settingsObj.maxconcurrentconnections(); }
+ { return _impl->_maxConcurrentConnections; }
void TransferSettings::setMinDownloadSpeed( long v )
- { _impl->_settingsObj.set_mindownloadspeed(v); }
+ { _impl->_minDownloadSpeed = (v); }
long TransferSettings::minDownloadSpeed() const
- { return _impl->_settingsObj.mindownloadspeed(); }
+ { return _impl->_minDownloadSpeed; }
void TransferSettings::setMaxDownloadSpeed( long v )
- { _impl->_settingsObj.set_maxdownloadspeed(v); }
+ { _impl->_maxDownloadSpeed = (v); }
long TransferSettings::maxDownloadSpeed() const
- { return _impl->_settingsObj.maxdownloadspeed(); }
+ { return _impl->_maxDownloadSpeed; }
void TransferSettings::setMaxSilentTries( long v )
- { _impl->_settingsObj.set_maxsilenttries(v); }
+ { _impl->_maxSilentTries = (v); }
long TransferSettings::maxSilentTries() const
- { return _impl->_settingsObj.maxsilenttries(); }
+ { return _impl->_maxSilentTries; }
void TransferSettings::setVerifyHostEnabled( bool enabled )
- { _impl->_settingsObj.set_verify_host(enabled); }
+ { _impl->_verify_host = (enabled); }
bool TransferSettings::verifyHostEnabled() const
- { return _impl->_settingsObj.verify_host(); }
+ { return _impl->_verify_host; }
void TransferSettings::setVerifyPeerEnabled( bool enabled )
- { _impl->_settingsObj.set_verify_peer(enabled); }
+ { _impl->_verify_peer = enabled; }
bool TransferSettings::verifyPeerEnabled() const
- { return _impl->_settingsObj.verify_peer(); }
+ { return _impl->_verify_peer; }
+ void TransferSettings::setClientCertificatePath( const Pathname &val_r )
+ { _impl->_client_cert_path = val_r; }
void TransferSettings::setClientCertificatePath( Pathname && val_r )
- { _impl->_settingsObj.set_client_cert_path( val_r.asString() ); }
+ { _impl->_client_cert_path = std::move( val_r ); }
- Pathname TransferSettings::clientCertificatePath() const
- { return _impl->_settingsObj.client_cert_path(); }
+ const Pathname &TransferSettings::clientCertificatePath() const
+ { return _impl->_client_cert_path; }
+ void TransferSettings::setClientKeyPath( const Pathname &val_r )
+ { _impl->_client_key_path = val_r; }
+
void TransferSettings::setClientKeyPath( Pathname && val_r )
- { _impl->_settingsObj.set_client_key_path( val_r.asString() ); }
+ { _impl->_client_key_path = std::move( val_r ); }
- Pathname TransferSettings::clientKeyPath() const
- { return _impl->_settingsObj.client_key_path(); }
+ const Pathname &TransferSettings::clientKeyPath() const
+ { return _impl->_client_key_path; }
- proto::TransferSettings &TransferSettings::protoData()
- {
- return _impl->_settingsObj;
- }
- const proto::TransferSettings &TransferSettings::protoData() const
- {
- return _impl->_settingsObj;
- }
+ void TransferSettings::setCertificateAuthoritiesPath( const Pathname &val_r )
+ { _impl->_ca_path = val_r; }
void TransferSettings::setCertificateAuthoritiesPath( Pathname && val_r )
- { _impl->_settingsObj.set_ca_path(val_r.asString()); }
+ { _impl->_ca_path = std::move(val_r.asString()); }
- Pathname TransferSettings::certificateAuthoritiesPath() const
- { return _impl->_settingsObj.ca_path(); }
+ const Pathname &TransferSettings::certificateAuthoritiesPath() const
+ { return _impl->_ca_path; }
+ void TransferSettings::setAuthType( const std::string &val_r )
+ { _impl->_authtype = val_r; }
+
void TransferSettings::setAuthType( std::string && val_r )
- { _impl->_settingsObj.set_authtype( std::move(val_r) ); }
+ { _impl->_authtype = std::move(val_r); }
- std::string TransferSettings::authType() const
- { return _impl->_settingsObj.authtype(); }
+ const std::string &TransferSettings::authType() const
+ { return _impl->_authtype; }
void TransferSettings::setHeadRequestsAllowed( bool allowed )
- { _impl->_settingsObj.set_head_requests_allowed(allowed); }
+ { _impl->_head_requests_allowed = allowed; }
bool TransferSettings::headRequestsAllowed() const
- { return _impl->_settingsObj.head_requests_allowed(); }
+ { return _impl->_head_requests_allowed; }
} // namespace media
} // namespace zypp
-
#include <zypp-core/base/PtrTypes.h>
#include <zypp-core/Pathname.h>
#include <zypp-core/Url.h>
-
-namespace zypp::proto {
- class TransferSettings;
-}
-
namespace zypp
{
namespace media
/** Constructs a transfer program cmd line access. */
TransferSettings();
- TransferSettings( const zypp::proto::TransferSettings &settings );
-
typedef std::vector<std::string> Headers;
/** reset the settings to the defaults */
void reset();
-
/** add a header, on the form "Foo: Bar" */
void addHeader( std::string && val_r );
+ void addHeader( const std::string & val_r );
/** returns a list of all added headers */
- Headers headers() const;
+ const Headers &headers() const;
/** sets the user agent ie: "Mozilla v3" */
void setUserAgentString( std::string && val_r );
+ void setUserAgentString( const std::string && val_r );
/** user agent string */
- std::string userAgentString() const;
+ const std::string &userAgentString() const;
/** sets the auth username */
+ void setUsername( const std::string &val_r );
void setUsername( std::string && val_r );
/** auth username */
- std::string username() const;
+ const std::string &username() const;
/** sets the auth password */
+ void setPassword( const std::string & val_r );
void setPassword( std::string && val_r );
/** auth password */
- std::string password() const;
+ const std::string &password() const;
/** returns the user and password as a user:pass string */
std::string userPassword() const;
/** proxy to use if it is enabled */
+ void setProxy( const std::string &val_r );
void setProxy( std::string && val_r );
/** proxy host */
- std::string proxy() const;
+ const std::string &proxy() const;
/** sets the proxy user */
+ void setProxyUsername( const std::string &val_r );
void setProxyUsername( std::string && val_r );
/** proxy auth username */
- std::string proxyUsername() const;
+ const std::string &proxyUsername() const;
/** sets the proxy password */
+ void setProxyPassword( const std::string &val_r );
void setProxyPassword( std::string && val_r );
/** proxy auth password */
- std::string proxyPassword() const;
+ const std::string &proxyPassword() const;
/** returns the proxy user and password as a user:pass string */
std::string proxyUserPassword() const;
/** Sets the SSL certificate authorities path */
+ void setCertificateAuthoritiesPath( const Pathname &val_r );
void setCertificateAuthoritiesPath( Pathname && val_r );
/** SSL certificate authorities path ( default: /etc/ssl/certs ) */
- Pathname certificateAuthoritiesPath() const;
+ const Pathname &certificateAuthoritiesPath() const;
/** set the allowed authentication types */
+ void setAuthType( const std::string &val_r );
void setAuthType( std::string && val_r );
/** get the allowed authentication types */
- std::string authType() const;
+ const std::string &authType() const;
/** set whether HEAD requests are allowed */
/** Sets the SSL client certificate file */
+ void setClientCertificatePath( const Pathname &val_r );
void setClientCertificatePath( Pathname && val_r );
/** SSL client certificate file */
- Pathname clientCertificatePath() const;
+ const Pathname &clientCertificatePath() const;
/** Sets the SSL client key file */
+ void setClientKeyPath( const Pathname &val_r );
void setClientKeyPath( Pathname && val_r );
/** SSL client key file */
- Pathname clientKeyPath() const;
-
- const zypp::proto::TransferSettings &protoData() const;
- zypp::proto::TransferSettings &protoData();
+ const Pathname &clientKeyPath() const;
protected:
class Impl;
--- /dev/null
+#include "cdtools.h"
ADD_DEFINITIONS( -DLOCALEDIR="${CMAKE_INSTALL_PREFIX}/share/locale" -DTEXTDOMAIN="zypp" -DZYPP_DLL )
SET( zypp_media_HEADERS
+ CDTools
+ cdtools.h
+ FileCheckException
+ filecheckexception.h
MediaConfig
mediaconfig.h
mediaexception.h
)
SET( zypp_media_SRCS
+ cdtools.cc
+ filecheckexception.cc
mediaconfig.cc
mediaexception.cc
mount.cc
INSTALL( FILES ${zypp_media_auth_HEADERS} DESTINATION "${INCLUDE_INSTALL_DIR}/zypp-media/auth" )
+SET( zypp_media_ng_HEADERS
+ ng/headervaluemap.h
+ ng/HeaderValueMap
+ ng/provide.h
+ ng/Provide
+ ng/providefwd.h
+ ng/ProvideFwd
+ ng/provide-configvars.h
+ ng/providespec.h
+ ng/ProvideSpec
+ ng/provideres.h
+ ng/provideitem.h
+ ng/ProvideRes
+ ng/mediaverifier.h
+ ng/MediaVerifier
+ ng/worker/devicedriver.h
+ ng/worker/DeviceDriver
+ ng/worker/provideworker.h
+ ng/worker/ProvideWorker
+ ng/worker/mountingworker.h
+ ng/worker/MountingWorker
+)
+
+SET( zypp_media_ng_private_HEADERS
+ ng/private/attachedmediainfo_p.h
+ ng/private/provide_p.h
+ ng/private/providefwd_p.h
+ ng/private/provideitem_p.h
+ ng/private/providemessage_p.h
+ ng/private/providequeue_p.h
+ ng/private/provideres_p.h
+ ng/private/providedbg_p.h
+)
+
+SET( zypp_media_ng_SRCS
+ ng/headervaluemap.cc
+ ng/provide.cc
+ ng/provideres.cc
+ ng/providespec.cc
+ ng/provideitem.cc
+ ng/providemessage.cc
+ ng/providequeue.cc
+ ng/mediaverifier.cc
+ ng/worker/devicedriver.cc
+ ng/worker/provideworker.cc
+ ng/worker/mountingworker.cc
+)
+
+INSTALL( FILES ${zypp_media_ng_HEADERS} DESTINATION "${INCLUDE_INSTALL_DIR}/zypp-media/ng" )
+
SET( zypp_media_lib_SRCS
${zypp_media_SRCS}
${zypp_media_auth_SRCS}
+ ${zypp_media_ng_SRCS}
)
SET( zypp_media_lib_HEADERS
${zypp_media_private_HEADERS} ${zypp_media_HEADERS}
${zypp_media_auth_private_HEADERS} ${zypp_media_auth_HEADERS}
+ ${zypp_media_ng_private_HEADERS} ${zypp_media_ng_HEADERS}
)
# Default loggroup for all files
--- /dev/null
+#include "filecheckexception.h"
_lastChange = time;
}
+const std::map<std::string, std::string> &AuthData::extraValues() const
+{
+ return _extraValues;
+}
+
+std::map<std::string, std::string> &AuthData::extraValues()
+{
+ return _extraValues;
+}
+
std::ostream & AuthData::dumpOn( std::ostream & str ) const
{
if (_url.isValid())
<< "username = " << _username << endl
<< "password = " << _password << endl;
+ for ( const auto &v : _extraValues ) {
+ if ( v.first == "username" || v.first == "password" )
+ continue;
+ str << v.first << " = " << v.second << endl;
+ }
+
return str;
}
#include <zypp-core/Url.h>
#include <zypp-core/base/PtrTypes.h>
+#include <zypp-media/ng/HeaderValueMap>
namespace zypp {
namespace media {
time_t lastDatabaseUpdate () const;
void setLastDatabaseUpdate ( time_t time );
+ const std::map<std::string, std::string> &extraValues() const;
+ std::map<std::string, std::string> &extraValues();
+
virtual std::ostream & dumpOn( std::ostream & str ) const;
virtual std::ostream & dumpAsIniOn( std::ostream & str ) const;
std::string _username;
std::string _password;
time_t _lastChange; //< timestamp of the last change to the database this credential is stored in
+ std::map<std::string, std::string> _extraValues;
};
typedef shared_ptr<AuthData> AuthData_Ptr;
else if ( key_r == "password" )
_secret->setPassword( value_r );
else
- WAR << "Ignore unknown attribute '" << key_r << "=" << value_r << "' in file " << _input << endl;
+ _secret->extraValues()[key_r] = value_r;
}
// else: ignored section due to wrong URL
}
///////////////////////////////////////////////////////////////////
} // namespace zypp
///////////////////////////////////////////////////////////////////
-
}
- static AuthData_Ptr findIn(const CredentialManager::CredentialSet & set,
+ AuthData_Ptr CredentialManager::findIn(const CredentialManager::CredentialSet & set,
const Url & url,
url::ViewOption vopt)
{
return AuthData_Ptr();
}
-
AuthData_Ptr CredentialManager::Impl::getCred(const Url & url) const
{
AuthData_Ptr result;
}
static int save_creds_in_file(
- const CredentialManager::CredentialSet creds,
+ CredentialManager::CredentialSet &creds,
const Pathname & file,
const mode_t mode)
{
int ret = 0;
filesystem::assert_file_mode( file, mode );
+ const auto now = time( nullptr );
+
PathInfo pi { file };
if ( pi.userMayRW() ) try {
// make sure only our thread accesses the file
for_(it, creds.begin(), creds.end())
{
(*it)->dumpAsIniOn(fs);
- (*it)->setLastDatabaseUpdate( time( nullptr ) );
+ (*it)->setLastDatabaseUpdate( now );
fs << endl;
}
if ( !fs ) {
saveInFile(cred, credfile);
}
+ time_t CredentialManager::timestampForCredDatabase ( const zypp::Url &url )
+ {
+ Pathname credfile;
+ if ( url.isValid() ) {
+ credfile = url.getQueryParam("credentials");
+ }
+
+ if (credfile.empty())
+ credfile = _pimpl->_options.userCredFilePath;
+
+ zypp::PathInfo pi(credfile);
+ if ( pi.isExist() && pi.isFile() )
+ return pi.mtime();
+
+ return 0;
+ }
void CredentialManager::addGlobalCred(const AuthData & cred)
{
*/
void clearAll(bool global = false);
+ /*!
+ * Helper function to find a matching AuthData instance in a CredentialSet
+ */
+ static AuthData_Ptr findIn( const CredentialManager::CredentialSet & set, const Url & url, url::ViewOption vopt );
+
+ /*!
+ * Returns the timestamp of the database the given URL creds would be stored
+ */
+ time_t timestampForCredDatabase ( const zypp::Url &url );
CredentialIterator credsGlobalBegin() const;
CredentialIterator credsGlobalEnd() const;
//////////////////////////////////////////////////////////////////////
#endif /* ZYPP_MEDIA_AUTH_CREDENTIALMANAGER_H */
-
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+
+#include "cdtools.h"
+
+extern "C"
+{
+#include <sys/ioctl.h>
+#include <linux/cdrom.h>
+}
+
+#include <fcntl.h>
+#include <unistd.h>
+#include <zypp-core/base/LogControl.h>
+#include <zypp-core/ExternalProgram.h>
+
+
+/*
+** If defined to the full path of the eject utility,
+** it will be used additionally to the eject-ioctl.
+*/
+#define EJECT_TOOL_PATH "/bin/eject"
+
+
+namespace zypp::media {
+
+ bool CDTools::openTray(const std::string &device_r)
+ {
+ int fd = ::open( device_r.c_str(), O_RDONLY|O_NONBLOCK|O_CLOEXEC );
+ int res = -1;
+
+ if ( fd != -1)
+ {
+ res = ::ioctl( fd, CDROMEJECT );
+ ::close( fd );
+ }
+
+ if ( res )
+ {
+ if( fd == -1)
+ {
+ WAR << "Unable to open '" << device_r
+ << "' (" << ::strerror( errno ) << ")" << std::endl;
+ }
+ else
+ {
+ WAR << "Eject " << device_r
+ << " failed (" << ::strerror( errno ) << ")" << std::endl;
+ }
+
+#if defined(EJECT_TOOL_PATH)
+ DBG << "Try to eject " << device_r << " using "
+ << EJECT_TOOL_PATH << " utility" << std::endl;
+
+ const char *cmd[3];
+ cmd[0] = EJECT_TOOL_PATH;
+ cmd[1] = device_r.c_str();
+ cmd[2] = NULL;
+ ExternalProgram eject(cmd, ExternalProgram::Stderr_To_Stdout);
+
+ for(std::string out( eject.receiveLine());
+ out.length(); out = eject.receiveLine())
+ {
+ DBG << " " << out;
+ }
+
+ if(eject.close() != 0)
+ {
+ WAR << "Eject of " << device_r << " failed." << std::endl;
+ return false;
+ }
+#else
+ return false;
+#endif
+ }
+ MIL << "Eject of " << device_r << " successful." << std::endl;
+ return true;
+ }
+
+ bool CDTools::closeTray(const std::string &device_r)
+ {
+ int fd = ::open( device_r.c_str(), O_RDONLY|O_NONBLOCK|O_CLOEXEC );
+ if ( fd == -1 ) {
+ WAR << "Unable to open '" << device_r << "' (" << ::strerror( errno ) << ")" << std::endl;
+ return false;
+ }
+ int res = ::ioctl( fd, CDROMCLOSETRAY );
+ ::close( fd );
+ if ( res ) {
+ WAR << "Close tray " << device_r << " failed (" << ::strerror( errno ) << ")" << std::endl;
+ return false;
+ }
+ DBG << "Close tray " << device_r << std::endl;
+ return true;
+ }
+
+}
| / /__ | | | | | | |
| /_____||_| |_| |_| |
| |
-----------------------------------------------------------------------*/
-#ifndef ZYPP_NG_CORE_CONTEXT_H_INCLUDED
-#define ZYPP_NG_CORE_CONTEXT_H_INCLUDED
+\---------------------------------------------------------------------*/
+/** \file zypp-media/cdtools.h
+ *
+*/
-#include <memory>
+#ifndef ZYPP_MEDIA_CDTOOLS_H
+#define ZYPP_MEDIA_CDTOOLS_H
-namespace zyppng {
+#include <string>
- class EventLoop;
- class MirrorControl;
-
- class Context {
+namespace zypp::media {
+ class CDTools
+ {
public:
-
- using Ptr = std::shared_ptr<Context>;
-
- Context();
- std::shared_ptr<EventLoop> evLoop () const;
- std::shared_ptr<MirrorControl> mirrorControl ();
-
- private:
- std::shared_ptr<EventLoop> _zyppEventLoop;
- std::shared_ptr<MirrorControl> _mirrorControl;
-
+ static bool openTray( const std::string & device_r );
+ static bool closeTray( const std::string & device_r );
};
}
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+/** \file zypp-media/filecheckexception.cc
+ *
+*/
+#include "filecheckexception.h"
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+
+#ifndef ZYPP_MEDIA_FILECHECKEXCEPTION_H
+#define ZYPP_MEDIA_FILECHECKEXCEPTION_H
+
+#include <zypp-core/base/Exception.h>
+
+namespace zypp {
+
+ class FileCheckException : public Exception
+ {
+ public:
+ FileCheckException(const std::string &msg)
+ : Exception(msg)
+ {}
+ };
+
+ class CheckSumCheckException : public FileCheckException
+ {
+ public:
+ CheckSumCheckException(const std::string &msg)
+ : FileCheckException(msg)
+ {}
+ };
+
+ class SignatureCheckException : public FileCheckException
+ {
+ public:
+ SignatureCheckException(const std::string &msg)
+ : FileCheckException(msg)
+ {}
+ };
+
+}
+
+#endif // ZYPP_MEDIA_FILECHECKEXCEPTION_H
return str;
}
+ std::ostream & MediaJammedException::dumpOn( std::ostream & str ) const {
+ str << _("No free ressources available to attach medium.");
+ return str;
+ }
+
/////////////////////////////////////////////////////////////////
} // namespace media
} // namespace zypp
std::string _path;
};
+ class MediaJammedException : public MediaException
+ {
+ public:
+ /** Ctor taking message.
+ * Use \ref ZYPP_THROW to throw exceptions.
+ */
+ MediaJammedException() : MediaException( "Media Jammed Exception" )
+ {}
+
+ /** Dtor. */
+ ~MediaJammedException() noexcept override {};
+ protected:
+ std::ostream & dumpOn( std::ostream & str ) const override;
+
+ private:
+ };
+
class MediaBadFilenameException : public MediaException
{
public:
virtual ~MediaInvalidCredentialsException() noexcept {}
};
+ class MediaRequestCancelledException : public MediaException
+ {
+ public:
+ MediaRequestCancelledException( const std::string & msg = "" )
+ : MediaException(msg)
+ {}
+ virtual ~MediaRequestCancelledException() noexcept {}
+ };
+
/////////////////////////////////////////////////////////////////
} // namespace media
} // namespace zypp
}
+
+bool MountEntry::isBlockDevice() const
+{
+ PathInfo dev_info;
+ return ( str::hasPrefix( Pathname(src).asString(), "/dev/" ) && dev_info(src) && dev_info.isBlk() );
+}
+
Mount::Mount()
{}
return entries;
}
+time_t Mount::getMTime()
+{
+ time_t mtime = zypp::PathInfo("/etc/mtab").mtime();
+ if( mtime <= 0)
+ {
+ WAR << "Failed to retrieve modification time of '/etc/mtab'"
+ << std::endl;
+ }
+ return mtime;
+}
+
} // namespace media
} // namespace zypp
, pass(passnum)
{}
+ /**
+ * Returns true if the src part points to a block device in /dev
+ */
+ bool isBlockDevice () const;
+
std::string src; //!< name of mounted file system
std::string dir; //!< file system path prefix
std::string type; //!< filesystem / mount type
static MountEntries
getEntries(const std::string &mtab = "");
+ /**
+ * Get the modification time of the /etc/mtab file.
+ * \return Modification time of the /etc/mtab file.
+ */
+ static time_t getMTime();
+
private:
#if LEGACY(1722)
ExternalProgram * _bincompat1;
--- /dev/null
+#include "headervaluemap.h"
--- /dev/null
+#include "mediaverifier.h"
--- /dev/null
+#include "provide.h"
--- /dev/null
+#include "zypp-media/ng/providefwd.h"
--- /dev/null
+#include "provideitem.h"
--- /dev/null
+#include "provideres.h"
--- /dev/null
+#include "providespec.h"
--- /dev/null
+#include "headervaluemap.h"
+#include <zypp-core/base/String.h>
+
+namespace zyppng {
+
+ HeaderValueMap::Value HeaderValueMap::InvalidValue;
+
+ HeaderValue::HeaderValue()
+ : _val ( new std::variant<std::monostate, std::string, int32_t, int64_t, double, bool>() )
+ {}
+
+ HeaderValue::HeaderValue( const HeaderValue &other )
+ : _val ( new std::variant<std::monostate, std::string, int32_t, int64_t, double, bool>( *other._val ) )
+ {}
+
+ HeaderValue::HeaderValue( HeaderValue &&other )
+ : _val ( new std::variant<std::monostate, std::string, int32_t, int64_t, double, bool>( std::move(*other._val) ) )
+ {}
+
+ HeaderValue::HeaderValue( const bool val )
+ : _val ( new std::variant<std::monostate, std::string, int32_t, int64_t, double, bool>(val) )
+ {}
+
+ HeaderValue::HeaderValue( const int32_t val )
+ : _val ( new std::variant<std::monostate, std::string, int32_t, int64_t, double, bool>(val) )
+ {}
+
+ HeaderValue::HeaderValue( const int64_t val )
+ : _val ( new std::variant<std::monostate, std::string, int32_t, int64_t, double, bool>(val) )
+ {}
+
+ HeaderValue::HeaderValue( const double val )
+ : _val ( new std::variant<std::monostate, std::string, int32_t, int64_t, double, bool>(val) )
+ {}
+
+ HeaderValue::HeaderValue( const std::string &val )
+ : _val ( new std::variant<std::monostate, std::string, int32_t, int64_t, double, bool>(val) )
+ {}
+
+ HeaderValue::HeaderValue(const char *val)
+ : HeaderValue( zypp::str::asString (val) )
+ {}
+
+ HeaderValue::HeaderValue( std::string &&val )
+ : _val ( new std::variant<std::monostate, std::string, int32_t, int64_t, double, bool>( std::move(val) ) )
+ {}
+
+ bool HeaderValue::valid() const
+ {
+ return ( _val->index () > 0 );
+ }
+
+ bool HeaderValue::isString() const
+ {
+ return std::holds_alternative<std::string>(*_val);
+ }
+
+ bool HeaderValue::isInt() const
+ {
+ return std::holds_alternative<int32_t>(*_val);
+ }
+
+ bool HeaderValue::isInt64() const
+ {
+ return std::holds_alternative<int64_t>(*_val);
+ }
+
+ bool HeaderValue::isDouble() const
+ {
+ return std::holds_alternative<double>(*_val);
+ }
+
+ bool HeaderValue::isBool() const
+ {
+ return std::holds_alternative<bool>(*_val);
+ }
+
+ const std::string &HeaderValue::asString() const
+ {
+ return std::get<std::string>(*_val);
+ }
+
+ int32_t HeaderValue::asInt() const
+ {
+ return std::get<int32_t>(*_val);
+ }
+
+ int64_t HeaderValue::asInt64() const
+ {
+ if ( std::holds_alternative<int32_t>(*_val) )
+ return std::get<int32_t>( *_val );
+ return std::get<int64_t>(*_val);
+ }
+
+ double HeaderValue::asDouble() const
+ {
+ return std::get<double>(*_val);
+ }
+
+ bool HeaderValue::asBool() const
+ {
+ return std::get<bool>(*_val);
+ }
+
+ HeaderValue::value_type &HeaderValue::asVariant()
+ {
+ return *_val;
+ }
+
+ const HeaderValue::value_type &HeaderValue::asVariant() const
+ {
+ return *_val;
+ }
+
+ HeaderValue &HeaderValue::operator=(const HeaderValue &other)
+ {
+ *_val = *other._val;
+ return *this;
+ }
+
+ bool HeaderValue::operator==(const HeaderValue &other) const
+ {
+ return ( *_val == *other._val );
+ }
+
+ HeaderValue &HeaderValue::operator= ( HeaderValue &&other )
+ {
+ *_val = std::move( *other._val );
+ return *this;
+ }
+
+ HeaderValue &HeaderValue::operator= ( const std::string &val )
+ {
+ *_val = val;
+ return *this;
+ }
+
+ HeaderValue &HeaderValue::operator= ( int32_t val )
+ {
+ *_val = val;
+ return *this;
+ }
+
+ HeaderValue &HeaderValue::operator= ( int64_t val )
+ {
+ *_val = val;
+ return *this;
+ }
+
+ HeaderValue &HeaderValue::operator= ( double val )
+ {
+ *_val = val;
+ return *this;
+ }
+
+ HeaderValue &HeaderValue::operator= ( bool val )
+ {
+ *_val = val;
+ return *this;
+ }
+
+
+ HeaderValueMap::HeaderValueMap( std::initializer_list<HeaderValueMap::ValueMap::value_type> init )
+ : _values( std::move(init) )
+ { }
+
+ bool HeaderValueMap::contains(const std::string &key) const
+ {
+ return _values.count (key) > 0 && _values.at(key).size () > 0 ;
+ }
+
+ void HeaderValueMap::set( const std::string &key, const Value &val )
+ {
+ auto i = _values.find (key);
+ if ( i == _values.end() ) {
+ _values.insert ( std::make_pair(key, std::vector<Value>{val}) );
+ } else {
+ i->second = std::vector<Value>{val};
+ }
+ }
+
+ void HeaderValueMap::set(const std::string &key, Value &&val)
+ {
+ auto i = _values.find (key);
+ if ( i == _values.end() ) {
+ _values.insert ( std::make_pair(key, std::vector<Value>{std::move(val)}) );
+ } else {
+ i->second = std::vector<Value>{std::move(val)};
+ }
+ }
+
+ void HeaderValueMap::add(const std::string &key, const Value &val)
+ {
+ auto i = _values.find (key);
+ if ( i == _values.end() ) {
+ _values.insert ( std::make_pair(key, std::vector<Value>{val}) );
+ } else {
+ i->second.push_back(val);
+ }
+ }
+
+ void HeaderValueMap::clear()
+ {
+ _values.clear();
+ }
+
+ HeaderValueMap::ValueMap::size_type HeaderValueMap::size() const noexcept
+ {
+ return _values.size();
+ }
+
+ std::vector<HeaderValueMap::Value> &HeaderValueMap::values(const std::string &key)
+ {
+ return _values[key];
+ }
+
+ const std::vector<HeaderValueMap::Value> &HeaderValueMap::values(const std::string &key) const
+ {
+ return _values.at(key);
+ }
+
+ HeaderValueMap::Value HeaderValueMap::value ( const std::string_view &str, const HeaderValueMap::Value &defaultVal) const
+ { return value( std::string(str), defaultVal ); }
+
+ HeaderValueMap::Value HeaderValueMap::value ( const std::string &str, const HeaderValueMap::Value &defaultVal) const
+ {
+ if ( !contains(str) || !_values.at(str).size() )
+ return defaultVal;
+ return _values.at(str).back();
+ }
+
+ HeaderValueMap::Value &HeaderValueMap::operator[](const std::string &key)
+ {
+ if ( !contains(key) )
+ return InvalidValue;
+ return _values[key].back();
+ }
+
+ HeaderValueMap::Value &HeaderValueMap::operator[]( const std::string_view &key )
+ {
+ return (*this)[std::string(key)];
+ }
+
+ const HeaderValueMap::Value &HeaderValueMap::operator[]( const std::string &key ) const
+ {
+ if ( !contains(key) )
+ return InvalidValue;
+ return _values.at(key).back();
+ }
+
+ const HeaderValueMap::Value &HeaderValueMap::operator[]( const std::string_view &key ) const
+ {
+ return (*this)[std::string(key)];
+ }
+
+ HeaderValueMap::const_iterator HeaderValueMap::erase(const const_iterator &i)
+ {
+ auto yi = _values.erase(i.base());
+ return HeaderValueMap::const_iterator(yi);
+ }
+
+ bool HeaderValueMap::erase(const std::string &key)
+ {
+ return ( _values.erase(key) > 0 );
+ }
+
+}
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+#ifndef ZYPP_MEDIA_NG_HEADERVALUEMAP_H_INCLUDED
+#define ZYPP_MEDIA_NG_HEADERVALUEMAP_H_INCLUDED
+
+#include <variant>
+#include <string>
+#include <map>
+#include <boost/iterator/iterator_adaptor.hpp>
+#include <zypp-core/base/PtrTypes.h>
+
+namespace zyppng {
+
+ class HeaderValue
+ {
+ public:
+ using value_type = std::variant<std::monostate, std::string, int32_t, int64_t, double, bool>;
+
+ HeaderValue();
+
+ HeaderValue( const HeaderValue &other );
+ HeaderValue( HeaderValue &&other );
+
+ HeaderValue( const bool val );
+ HeaderValue( const int32_t val );
+ HeaderValue( const int64_t val );
+ HeaderValue( const double val );
+ HeaderValue( std::string &&val );
+ HeaderValue( const std::string &val );
+ HeaderValue( const char *val );
+
+ bool valid () const;
+
+ bool isString () const;
+ bool isInt () const;
+ bool isInt64 () const;
+ bool isDouble () const;
+ bool isBool () const;
+
+ const std::string &asString () const;
+ int32_t asInt () const;
+ int64_t asInt64 () const;
+ double asDouble() const;
+ bool asBool () const;
+
+ value_type &asVariant ();
+ const value_type &asVariant () const;
+
+ HeaderValue &operator= ( const HeaderValue &other );
+ HeaderValue &operator= ( HeaderValue &&other );
+ HeaderValue &operator= ( const std::string &val );
+ HeaderValue &operator= ( int32_t val );
+ HeaderValue &operator= ( int64_t val );
+ HeaderValue &operator= ( double val );
+ HeaderValue &operator= ( bool val );
+
+ bool operator== ( const HeaderValue &other ) const;
+
+ private:
+ zypp::RWCOW_pointer<value_type> _val;
+ };
+
+ class HeaderValueMap
+ {
+ public:
+ using Value = HeaderValue;
+ using ValueMap = std::map<std::string, std::vector<Value>>;
+
+ static Value InvalidValue;
+
+ class const_iterator
+ : public boost::iterator_adaptor<
+ HeaderValueMap::const_iterator // Derived
+ , ValueMap::const_iterator // Base
+ , const std::pair<std::string, Value> // Value
+ , boost::use_default // CategoryOrTraversal
+ >
+ {
+ public:
+ const_iterator()
+ : const_iterator::iterator_adaptor_() {}
+
+ explicit const_iterator( const ValueMap::const_iterator &val )
+ { this->base_reference() = val; }
+
+ const_iterator( const HeaderValueMap::const_iterator &other )
+ : const_iterator::iterator_adaptor_( other.base() ) {}
+
+ const std::string &key () const {
+ return this->base_reference()->first;
+ }
+
+ const Value &value() const {
+ auto &l = base_reference ()->second;
+ if ( l.empty() ) {
+ return InvalidValue;
+ }
+ return l.back();
+ }
+
+ private:
+ friend class boost::iterator_core_access;
+ void increment() {
+ this->base_reference() = ++this->base_reference();
+ }
+
+ std::pair<std::string, Value> dereference() const
+ {
+ return std::make_pair( key(), value() );
+ }
+ };
+
+ HeaderValueMap() = default;
+ HeaderValueMap( std::initializer_list<ValueMap::value_type> init );
+
+ bool contains( const std::string &key ) const;
+ bool contains( const std::string_view &key ) const {
+ return contains(std::string(key));
+ }
+
+ void set( const std::string &key, const Value &val );
+ void set( const std::string &key, Value &&val );
+ void add( const std::string &key, const Value &val);
+ void clear ();
+ ValueMap::size_type size() const noexcept;
+
+ std::vector<Value> &values ( const std::string &key );
+ const std::vector<Value> &values ( const std::string &key ) const;
+
+ std::vector<Value> &values ( const std::string_view &key ) {
+ return values( std::string(key) );
+ }
+
+ const std::vector<Value> &values ( const std::string_view &key ) const {
+ return values( std::string(key) );
+ }
+
+ /*!
+ * Returns the last entry with key \a str in the list of values
+ * or the default value specified in \a defaultVal
+ */
+ Value value ( const std::string_view &str, const Value &defaultVal = Value() ) const;
+ Value value ( const std::string &str, const Value &defaultVal = Value() ) const;
+
+ Value &operator[]( const std::string &key );
+ Value &operator[]( const std::string_view &key );
+ const Value &operator[]( const std::string &key ) const;
+ const Value &operator[]( const std::string_view &key ) const;
+
+ const_iterator erase( const const_iterator &i );
+ bool erase( const std::string &key );
+
+ const_iterator begin() const {
+ return const_iterator( _values.begin() );
+ }
+ const_iterator end() const {
+ return const_iterator( _values.end() );
+ }
+
+ ValueMap::iterator beginList() {
+ return _values.begin();
+ }
+ ValueMap::iterator endList() {
+ return _values.end();
+ }
+
+ ValueMap::const_iterator beginList() const {
+ return _values.begin();
+ }
+ ValueMap::const_iterator endList() const {
+ return _values.end();
+ }
+
+ ValueMap::const_iterator cbeginList() const {
+ return _values.cbegin();
+ }
+ ValueMap::const_iterator cendList() const {
+ return _values.cend();
+ }
+
+ private:
+ ValueMap _values;
+ };
+}
+
+namespace zypp {
+ template<>
+ inline zyppng::HeaderValue::value_type* rwcowClone<zyppng::HeaderValue::value_type>( const zyppng::HeaderValue::value_type * rhs )
+ { return new zyppng::HeaderValue::value_type(*rhs); }
+}
+
+
+#endif
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+
+#include "mediaverifier.h"
+#include <ostream>
+#include <fstream>
+#include <zypp-core/Pathname.h>
+#include <zypp-core/base/String.h>
+#include <zypp-core/base/Gettext.h>
+#include "private/providedbg_p.h"
+
+namespace zyppng {
+
+ ZYPP_FWD_DECL_TYPE_WITH_REFS (SuseMediaDataVerifier);
+
+ class SuseMediaDataVerifier : public MediaDataVerifier
+ {
+ public:
+ // MediaDataVerifier interface
+ bool valid() const override;
+ bool matches(const MediaDataVerifierRef &rhs) const override;
+ const std::string &mediaVendor() const override;
+ const std::string &mediaIdent() const override;
+ uint totalMedia() const override;
+ std::ostream &toStream(std::ostream &str) const override;
+ bool load( const zypp::Pathname &data ) override;
+ bool loadFromMedium(const zypp::filesystem::Pathname &data, uint expectedMediaNr ) override;
+ zypp::filesystem::Pathname mediaFilePath(uint mediaNr) const override;
+ MediaDataVerifierRef clone () const override;
+ std::string expectedAsUserString( uint mediaNr ) const override;
+
+ private:
+ std::string _mediaVendor;
+ std::string _mediaIdent;
+ uint _totalMedia = 0;
+ };
+
+ zypp::Pathname SuseMediaDataVerifier::mediaFilePath(uint mediaNr) const
+ {
+ zypp::str::Format fmt { "/media.%d/media" };
+ return (fmt % zypp::str::numstring( mediaNr )).asString();
+ }
+
+ bool SuseMediaDataVerifier::loadFromMedium( const zypp::filesystem::Pathname &data, uint expectedMediaNr )
+ {
+ return load ( data / mediaFilePath(expectedMediaNr) );
+ }
+
+ bool SuseMediaDataVerifier::valid() const
+ { return ! (_mediaVendor.empty() || _mediaIdent.empty()); }
+
+ bool SuseMediaDataVerifier::matches(const MediaDataVerifierRef &rhs) const
+ {
+ auto conv = std::dynamic_pointer_cast<SuseMediaDataVerifier>(rhs);
+ return conv && valid() && conv->_mediaVendor == _mediaVendor && conv->_mediaIdent == _mediaIdent;
+ }
+
+ uint SuseMediaDataVerifier::totalMedia() const
+ {
+ return _totalMedia;
+ }
+
+ std::ostream &SuseMediaDataVerifier::toStream( std::ostream &str ) const
+ {
+ return str << "[" << _mediaVendor << "|" << _mediaIdent << "/" << _totalMedia << "]";
+ }
+
+ bool SuseMediaDataVerifier::load( const zypp::Pathname &path_r )
+ {
+ std::ifstream inp( path_r.c_str() );
+ if ( !inp ) {
+ ERR << "Can't setup a SUSEMediaVerifier from file: " << path_r.asString() << std::endl;
+ return false;
+ }
+ getline( inp, _mediaVendor );
+ getline( inp, _mediaIdent );
+ std::string buffer;
+ getline( inp, buffer );
+ zypp::str::strtonum( buffer, _totalMedia );
+ //if ( !_totalMedia ) _totalMedia = 1;
+ // loaded but maybe not valid
+ return true;
+ }
+
+ const std::string &SuseMediaDataVerifier::mediaIdent() const
+ {
+ return _mediaIdent;
+ }
+
+ const std::string &SuseMediaDataVerifier::mediaVendor() const
+ {
+ return _mediaVendor;
+ }
+
+ MediaDataVerifierRef SuseMediaDataVerifier::clone () const
+ {
+ return SuseMediaDataVerifierRef( new SuseMediaDataVerifier( *this ) );
+ }
+
+ std::string SuseMediaDataVerifier::expectedAsUserString( uint mediaNr ) const
+ {
+ // Translator: %1% the expected medium number; %2% the total number of media in the set; %3% the ident file on the medium.
+ zypp::str::Format fmt { _("Expected medium %1%/%2% identified by file '%3%' with content:") };
+ return zypp::str::Str()
+ << ( fmt % mediaNr % _totalMedia % mediaFilePath( mediaNr ) ) << "\n"
+ << " " << _mediaVendor << "\n"
+ << " " << _mediaIdent;
+ }
+
+ MediaDataVerifier::MediaDataVerifier() noexcept
+ { }
+
+ MediaDataVerifier::~MediaDataVerifier()
+ { }
+
+ MediaDataVerifierRef MediaDataVerifier::createVerifier( const std::string &verifierType )
+ {
+ if ( verifierType == "SuseMediaV1" ) {
+ return SuseMediaDataVerifierRef( new SuseMediaDataVerifier() );
+ }
+ return nullptr;
+ }
+
+ std::ostream &operator<<(std::ostream &str, const MediaDataVerifierRef &obj)
+ {
+ if ( obj )
+ return obj->toStream(str);
+ return str << "[MediaVerifier: null]";
+ }
+
+}
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+#ifndef ZYPP_MEDIA_NG_MEDIAVERIFIER_H_INCLUDED
+#define ZYPP_MEDIA_NG_MEDIAVERIFIER_H_INCLUDED
+
+#include <string>
+#include <zypp-core/zyppng/core/ByteArray>
+#include <zypp-media/ng/ProvideFwd>
+
+namespace zypp {
+ namespace filesystem {
+ class Pathname;
+ }
+ using filesystem::Pathname;
+}
+
+namespace zyppng {
+
+ /*!
+ * The MediaDataVerifier is used to verify if a specific medium, e.g. a DVD, is the
+ * medium we actually need. Each verifier is defined by a verifier type and the data
+ * that is used to validate against the medium.
+ */
+ class MediaDataVerifier
+ {
+ public:
+
+ MediaDataVerifier() noexcept;
+ virtual ~MediaDataVerifier();
+
+ static MediaDataVerifierRef createVerifier ( const std::string &verifierType );
+
+ /** Data considered to be valid if we have vendor and ident. */
+ virtual bool valid() const = 0;
+
+ /** Whether \a rhs belongs to the same media set. */
+ virtual bool matches( const MediaDataVerifierRef & rhs ) const = 0;
+
+ /*!
+ * Returns the media vendor string
+ */
+ virtual const std::string &mediaVendor() const = 0;
+
+ /*!
+ * Returns the media ident string
+ */
+ virtual const std::string &mediaIdent() const = 0;
+
+ /*!
+ * Returns the total number of mediums in this set
+ */
+ virtual uint totalMedia() const = 0;
+
+ /*!
+ * Writes the mediaverifier data to stream
+ */
+ virtual std::ostream & toStream ( std::ostream & str ) const = 0;
+
+ /*!
+ * Load verification information from a given file, all media data must
+ * be storeable in a file so that the controller can store a copy of it somewhere.
+ */
+ virtual bool load( const zypp::Pathname &data ) = 0;
+
+ /*!
+ * Generates the file information from a mounted medium, the path given in \a data
+ * is the mountpoint of the device.
+ */
+ virtual bool loadFromMedium( const zypp::Pathname &data, uint mediaNr ) = 0;
+
+ /*!
+ * Returns the path of the media identifier file on the medium
+ */
+ virtual zypp::Pathname mediaFilePath ( uint mediaNr ) const = 0;
+
+ /*!
+ * Clones \a this and returns a reference to the clone
+ */
+ virtual MediaDataVerifierRef clone () const = 0;
+
+ /*!
+ * Returns a error string describing the expected medium.
+ */
+ virtual std::string expectedAsUserString( uint mediaNr = 1 ) const = 0;
+ };
+
+ /** \relates Stream output */
+ std::ostream & operator<<( std::ostream & str, const MediaDataVerifierRef & obj );
+
+
+
+}
+
+#endif
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\----------------------------------------------------------------------/
+*
+* This file contains private API, this might break at any time between releases.
+* You have been warned!
+*
+*/
+#ifndef ZYPP_MEDIA_PRIVATE_ATTACHEDMEDIAINFO_P_H_INCLUDED
+#define ZYPP_MEDIA_PRIVATE_ATTACHEDMEDIAINFO_P_H_INCLUDED
+
+#include "providefwd_p.h"
+#include "providequeue_p.h"
+#include <zypp-media/ng/ProvideSpec>
+#include <string>
+#include <chrono>
+
+namespace zyppng {
+
+ class ProvidePrivate;
+
+ struct AttachedMediaInfo {
+
+ public:
+ void ref() {
+ if ( _refCount == 0 )
+ _idleSince = std::chrono::steady_clock::time_point::max();
+
+ _refCount++;
+ }
+ void unref() {
+ if ( _refCount > 0 ) {
+ _refCount--;
+
+ if ( _refCount == 0 )
+ _idleSince = std::chrono::steady_clock::now();
+ }
+ }
+
+ /*!
+ * Returns true if \a other requests the same medium as this instance
+ */
+ bool isSameMedium ( const std::vector<zypp::Url> &urls, const ProvideMediaSpec &spec ) {
+
+ const auto check = _spec.isSameMedium(spec);
+ if ( !zypp::indeterminate (check) )
+ return (bool)check;
+
+ // let the URL rule
+ return ( std::find( urls.begin(), urls.end(), _attachedUrl ) != urls.end() );
+ }
+
+ std::string _name;
+ ProvideQueueWeakRef _backingQueue; //< if initialized contains a weak reference to the queue that owns this medium
+ ProvideQueue::Config::WorkerType _workerType;
+ zypp::Url _attachedUrl; // the URL that was used for the attach request
+ ProvideMediaSpec _spec;
+ uint _refCount = 0;
+ std::chrono::steady_clock::time_point _idleSince = std::chrono::steady_clock::time_point::max();
+ };
+
+}
+
+#endif
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\----------------------------------------------------------------------/
+*
+* This file contains private API, this might break at any time between releases.
+* You have been warned!
+*
+*/
+#ifndef ZYPP_MEDIA_PRIVATE_PROVIDE_P_H_INCLUDED
+#define ZYPP_MEDIA_PRIVATE_PROVIDE_P_H_INCLUDED
+
+#include "providefwd_p.h"
+#include "providequeue_p.h"
+#include "attachedmediainfo_p.h"
+
+#include <zypp-media/auth/CredentialManager>
+#include <zypp-media/ng/Provide>
+#include <zypp-media/ng/ProvideItem>
+#include <zypp-media/ng/ProvideSpec>
+#include <zypp-proto/envelope.pb.h>
+#include <zypp-proto/provider.pb.h>
+#include <zypp-core/zyppng/base/private/base_p.h>
+#include <zypp-core/zyppng/base/Timer>
+#include <zypp-core/ManagedFile.h>
+
+#include <queue>
+#include <variant>
+
+namespace zyppng {
+
+ namespace constants {
+ constexpr std::string_view DEFAULT_PROVIDE_WORKER_PATH = ZYPP_WORKER_PATH;
+ constexpr std::string_view ATTACHED_MEDIA_SUFFIX = "-media";
+ constexpr auto DEFAULT_ACTIVE_CONN_PER_HOST = 5; //< how many simultanious connections to the same host are allowed
+ constexpr auto DEFAULT_ACTIVE_CONN = 10; //< how many simultanious connections are allowed
+ constexpr auto DEFAULT_MAX_DYNAMIC_WORKERS = 20;
+ constexpr auto DEFAULT_CPU_WORKERS = 4;
+ }
+
+ class ProvideQueue;
+ class RpcMessageStream;
+ using RpcMessageStreamPtr = std::shared_ptr<RpcMessageStream>;
+
+
+ class ProvidePrivate : public BasePrivate
+ {
+ ZYPP_DECLARE_PUBLIC(Provide);
+ public:
+ ProvidePrivate( zypp::Pathname &&workDir, Provide &pub );
+
+ enum ScheduleReason
+ {
+ ProvideStart,
+ QueueIdle,
+ EnqueueItem,
+ EnqueueReq,
+ RestartAttach,
+ FinishReq
+ };
+
+ void schedule( ScheduleReason reason );
+
+ bool queueRequest ( ProvideRequestRef req );
+ bool dequeueRequest( ProvideRequestRef req, std::exception_ptr error );
+ void queueItem ( ProvideItemRef item );
+ void dequeueItem ( ProvideItem *item );
+
+ std::string nextMediaId () const;
+ AttachedMediaInfo &addMedium ( zypp::proto::Capabilities::WorkerType workerType, const zypp::Url &baseUrl, ProvideMediaSpec &spec );
+ AttachedMediaInfo &addMedium ( zypp::proto::Capabilities::WorkerType workerType, ProvideQueueWeakRef backingQueue, const std::string &id, const zypp::Url &baseUrl, ProvideMediaSpec &spec );
+
+ std::string effectiveScheme ( const std::string &scheme ) const;
+
+ void onPulseTimeout ( Timer & );
+ void onQueueIdle ();
+ void onItemStateChanged ( ProvideItem &item );
+ expected<ProvideQueue::Config> schemeConfig(const std::string &scheme);
+
+ std::optional<zypp::ManagedFile> addToFileCache ( const zypp::Pathname &downloadedFile );
+ bool isInCache ( const zypp::Pathname &downloadedFile ) const;
+
+ bool isRunning() const;
+
+ const zypp::Pathname &workerPath() const;
+ const std::string queueName( ProvideQueue &q ) const;
+
+ std::vector<AttachedMediaInfo> &attachedMediaInfos();
+
+ std::list<ProvideItemRef> &items();
+
+ zypp::media::CredManagerOptions &credManagerOptions ();
+
+ ProvideStatusRef log () {
+ return _log;
+ }
+
+ uint32_t nextRequestId();
+
+ Signal< Provide::MediaChangeAction ( const std::string &, const std::string &, const int32_t, const std::vector<std::string> &, const std::optional<std::string> &) > _sigMediaChange;
+ Signal< std::optional<zypp::media::AuthData> ( const zypp::Url &reqUrl, const std::string &triedUsername, const std::map<std::string, std::string> &extraValues ) > _sigAuthRequired;
+
+ protected:
+ void doSchedule (Timer &);
+
+ //@TODO should we make those configurable?
+ std::unordered_map< std::string, std::string > _workerAlias {
+ {"ftp" ,"http"},
+ {"tftp" ,"http"},
+ {"https","http"},
+ {"cifs" ,"smb" },
+ {"nfs4" ,"nfs" },
+ {"cd" ,"disc"},
+ {"dvd" ,"disc"},
+ {"file" ,"dir" },
+ {"hd" ,"disk"}
+ };
+
+ bool _isRunning = false;
+ bool _isScheduling = false;
+ Timer::Ptr _pulseTimer = Timer::create();
+ Timer::Ptr _scheduleTrigger = Timer::create(); //< instead of constantly calling schedule we set a trigger event so it runs as soon as event loop is on again
+ zypp::Pathname _workDir;
+
+ std::list< ProvideItemRef > _items; //< The list of running provide Items, each of them can spawn multiple requests
+ uint32_t _nextRequestId = 0; //< The next request ID , we use controller wide unique IDs instead of worker locals IDs , its easier to track
+
+ struct QueueItem {
+ std::string _schemeName;
+ std::deque<ProvideRequestRef> _requests;
+ };
+ std::deque<QueueItem> _queues; //< List of request queues for the workers, grouped by scheme. We use a deque and not a map because of possible changes to the list of queues during scheduling
+
+
+ std::vector< AttachedMediaInfo > _attachedMediaInfos; //< List of currently attached medias
+
+ std::unordered_map< std::string, ProvideQueueRef > _workerQueues;
+ std::unordered_map< std::string, ProvideQueue::Config > _schemeConfigs;
+
+ struct FileCacheItem {
+ zypp::ManagedFile _file;
+ std::optional<std::chrono::steady_clock::time_point> _deathTimer; // timepoint where this item was seen first without a refcount
+ };
+ std::unordered_map< std::string, FileCacheItem > _fileCache;
+
+ zypp::Pathname _workerPath;
+ zypp::media::CredManagerOptions _credManagerOptions;
+
+ ProvideStatusRef _log;
+ Signal<void()> _sigIdle;
+ };
+}
+
+#endif
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+----------------------------------------------------------------------/
+*
+* This file contains private API, this might break at any time between releases.
+* You have been warned!
+*
+*/
+#ifndef ZYPP_MEDIA_NG_PROVIDEDBG_P_H_INCLUDED
+#define ZYPP_MEDIA_NG_PROVIDEDBG_P_H_INCLUDED
+
+#include <zypp-core/base/LogControl.h>
+
+L_ENV_CONSTR_FWD_DECLARE_FUNC(ZYPP_MEDIA_PROVIDER_DEBUG)
+
+#ifdef ZYPP_BASE_LOGGER_LOGGROUP
+#undef ZYPP_BASE_LOGGER_LOGGROUP
+#endif
+
+#define ZYPP_BASE_LOGGER_LOGGROUP "ZYPP_MEDIA_PROVIDE"
+
+namespace zyppng {
+ inline bool provideDebugEnabled() {
+ return zypp::log::has_env_constr_ZYPP_MEDIA_PROVIDER_DEBUG();
+ }
+}
+
+#define XXX_PRV if( zyppng::provideDebugEnabled() ) XXX
+#define DBG_PRV if( zyppng::provideDebugEnabled() ) DBG
+#define MIL_PRV if( zyppng::provideDebugEnabled() ) MIL
+#define WAR_PRV if( zyppng::provideDebugEnabled() ) WAR
+#define ERR_PRV if( zyppng::provideDebugEnabled() ) ERR
+#define SEC_PRV if( zyppng::provideDebugEnabled() ) SEC
+#define INT_PRV if( zyppng::provideDebugEnabled() ) INT
+#define USR_PRV if( zyppng::provideDebugEnabled() ) USR
+
+
+#endif // ZYPP_MEDIA_NG_PROVIDEDBG_P_H_INCLUDED
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\----------------------------------------------------------------------/
+*
+* This file contains private API, this might break at any time between releases.
+* You have been warned!
+*
+*/
+#ifndef ZYPP_MEDIA_PRIVATE_PROVIDE_FWD_P_H_INCLUDED
+#define ZYPP_MEDIA_PRIVATE_PROVIDE_FWD_P_H_INCLUDED
+
+#include <zypp-media/ng/ProvideFwd>
+namespace zyppng {
+ ZYPP_FWD_DECL_TYPE_WITH_REFS(ProvideQueue);
+ ZYPP_FWD_DECL_TYPE_WITH_REFS(ProvideWorker);
+ ZYPP_FWD_DECL_TYPE_WITH_REFS(ProvideFileItem);
+ ZYPP_FWD_DECL_TYPE_WITH_REFS(AttachMediaItem);
+ ZYPP_FWD_DECL_TYPE_WITH_REFS(DetachMediaItem);
+ ZYPP_FWD_DECL_TYPE_WITH_REFS(ProvideRequest);
+
+ class ProvideMessage;
+
+ template< typename T >
+ class ProvidePromise;
+ template< typename T >
+ using ProvidePromiseRef = std::shared_ptr<ProvidePromise<T>>;
+ template< typename T >
+ using ProvidePromiseWeakRef = std::weak_ptr<ProvidePromise<T>>;
+}
+
+#endif
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\----------------------------------------------------------------------/
+*
+* This file contains private API, this might break at any time between releases.
+* You have been warned!
+*
+*/
+#ifndef ZYPP_MEDIA_PRIVATE_PROVIDE_ITEM_P_H_INCLUDED
+#define ZYPP_MEDIA_PRIVATE_PROVIDE_ITEM_P_H_INCLUDED
+
+#include "providefwd_p.h"
+#include "providequeue_p.h"
+#include "attachedmediainfo_p.h"
+#include "providemessage_p.h"
+#include <zypp-media/ng/Provide>
+#include <zypp-media/ng/ProvideItem>
+#include <zypp-media/ng/ProvideRes>
+#include <zypp-media/ng/ProvideSpec>
+#include <zypp-core/zyppng/base/private/base_p.h>
+#include <set>
+#include <variant>
+
+namespace zyppng {
+
+ /*!
+ * The internal request type, which represents all possible
+ * user requests and exports some convenience functions for the scheduler to
+ * directly access relevant data
+ */
+ class ProvideRequest {
+ public:
+
+ friend class ProvideItem;
+
+ static expected<ProvideRequestRef> create( ProvideItem &owner, const std::vector<zypp::Url> &urls, const std::string &id, ProvideMediaSpec &spec );
+ static expected<ProvideRequestRef> create ( ProvideItem &owner, const std::vector<zypp::Url> &urls, ProvideFileSpec &spec );
+ static expected<ProvideRequestRef> createDetach( const zypp::Url &url );
+
+ ProvideItem * owner() { return _owner; }
+
+ uint code () const { return _message.code(); }
+
+ void setCurrentQueue ( ProvideQueueRef ref );
+ ProvideQueueRef currentQueue ();
+
+ const ProvideMessage &provideMessage () const { return _message; }
+ ProvideMessage &provideMessage () { return _message; }
+
+ const std::optional<zypp::Url> activeUrl() const;
+ void setActiveUrl ( const zypp::Url &urlToUse );
+
+ void setUrls( const std::vector<zypp::Url> & urls ) {
+ _mirrors = urls;
+ }
+
+ const std::vector<zypp::Url> &urls() const {
+ return _mirrors;
+ }
+
+ zypp::Url url() const {
+ return _mirrors.front();
+ }
+
+ void setUrl( const zypp::Url & url ) {
+ _mirrors = {url};
+ }
+
+ void clearForRestart () {
+ _pastRedirects.clear();
+ _activeUrl.reset();
+ _myQueue.reset();
+ }
+
+ private:
+ ProvideRequest( ProvideItem *owner, const std::vector<zypp::Url> &urls, ProvideMessage &&msg ) : _owner(owner), _message(std::move(msg) ), _mirrors(urls) {}
+ ProvideItem *_owner = nullptr; // destructor of ProvideItem will dequeue the item, so no need to do refcount here
+ ProvideMessage _message;
+ std::vector<zypp::Url> _mirrors;
+ std::vector<zypp::Url> _pastRedirects;
+ std::optional<zypp::Url> _activeUrl;
+ ProvideQueueWeakRef _myQueue;
+ };
+
+ class ProvideItemPrivate : public BasePrivate
+ {
+ public:
+ ProvideItemPrivate( ProvidePrivate & parent, ProvideItem &pub ) : BasePrivate(pub), _parent(parent) {}
+ ProvidePrivate &_parent;
+ ProvideItem::State _itemState = ProvideItem::Uninitialized;
+ std::chrono::steady_clock::time_point _itemStarted;
+ std::chrono::steady_clock::time_point _itemFinished;
+ std::optional<ProvideItem::ItemStats> _prevStats;
+ std::optional<ProvideItem::ItemStats> _currStats;
+ Signal<void( ProvideItem &item, ProvideItem::State oldState, ProvideItem::State newState )> _sigStateChanged;
+ };
+
+ /*!
+ * The object returned to the user code to track the internal Item.
+ * Releasing the last reference to it will cancel the operation but the corresponding ProvideItem
+ * will remain in the Queue until the cancel operation was finished.
+ */
+ template< typename T >
+ class ProvidePromise : public AsyncOp<expected<T>>
+ {
+ public:
+ ProvidePromise( ProvideItemRef provideItem )
+ : _myProvide( provideItem )
+ {}
+
+ ~ProvidePromise()
+ {
+ auto prov = _myProvide.lock();
+ if ( prov )
+ prov->released();
+ }
+
+ private:
+ ProvideItemWeakRef _myProvide; //weak reference to the internal item so we can cancel the op on desctruction
+ };
+
+ /*!
+ * Item downloading and providing a file
+ */
+ class ProvideFileItem : public ProvideItem
+ {
+ public:
+
+ static ProvideFileItemRef create ( const std::vector<zypp::Url> &urls,const ProvideFileSpec &request, ProvidePrivate &parent );
+
+ // ProvideItem interface
+ void initialize () override;
+ ProvidePromiseRef<ProvideRes> promise();
+
+ void setMediaRef ( Provide::MediaHandle &&hdl );
+ Provide::MediaHandle & mediaRef ();
+
+ ItemStats makeStats () override;
+ zypp::ByteCount bytesExpected () const override;
+
+ protected:
+ ProvideFileItem ( const std::vector<zypp::Url> &urls,const ProvideFileSpec &request, ProvidePrivate &parent );
+
+ void informalMessage ( ProvideQueue &, ProvideRequestRef req, const ProvideMessage &msg ) override;
+
+ using ProvideItem::finishReq;
+ void finishReq ( ProvideQueue &queue, ProvideRequestRef finishedReq, const ProvideMessage &msg ) override;
+ void cancelWithError ( std::exception_ptr error ) override;
+ expected<zypp::media::AuthData> authenticationRequired ( ProvideQueue &queue, ProvideRequestRef req, const zypp::Url &effectiveUrl, int64_t lastTimestamp, const std::map<std::string, std::string> &extraFields ) override;
+
+ private:
+ Provide::MediaHandle _handleRef; //< If we are using a attached media, this will keep the reference around
+ bool _promiseCreated = false;
+ std::vector<zypp::Url> _mirrorList; //< All available URLs, first one is the primary
+ ProvideFileSpec _initialSpec; //< The initial spec as defined by the user code
+ zypp::Pathname _targetFile; //< The target file as reported by the worker
+ zypp::Pathname _stagingFile; //< The staging file as reported by the worker
+ zypp::ByteCount _expectedBytes; //< The nr of bytes we want to provide
+ ProvidePromiseWeakRef<ProvideRes> _promise;
+ };
+
+
+ /*!
+ * Item attaching and verifying a medium
+ */
+ class AttachMediaItem : public ProvideItem
+ {
+ public:
+ ~AttachMediaItem();
+ static AttachMediaItemRef create ( const std::vector<zypp::Url> &urls, const ProvideMediaSpec &request, ProvidePrivate &parent );
+ SignalProxy< void( const zyppng::expected<AttachedMediaInfo *> & ) > sigReady ();
+
+ ProvidePromiseRef<Provide::MediaHandle> promise();
+
+ protected:
+ AttachMediaItem ( const std::vector<zypp::Url> &urls, const ProvideMediaSpec &request, ProvidePrivate &parent );
+
+ // ProvideItem interface
+ void initialize () override;
+
+ using ProvideItem::finishReq;
+ void finishReq ( ProvideQueue &queue, ProvideRequestRef finishedReq, const ProvideMessage &msg ) override;
+ void cancelWithError( std::exception_ptr error ) override;
+ void finishWithSuccess (AttachedMediaInfo &medium );
+ expected<zypp::media::AuthData> authenticationRequired ( ProvideQueue &queue, ProvideRequestRef req, const zypp::Url &effectiveUrl, int64_t lastTimestamp, const std::map<std::string, std::string> &extraFields ) override;
+
+ void onMasterItemReady ( const zyppng::expected<AttachedMediaInfo *>& result );
+
+ private:
+ Signal< void( const zyppng::expected<AttachedMediaInfo *> & )> _sigReady;
+ bool _promiseCreated = false;
+ connection _masterItemConn;
+ std::vector<zypp::Url> _mirrorList; //< All available URLs, first one is the primary
+ ProvideMediaSpec _initialSpec; //< The initial spec as defined by the user code
+ ProvideQueue::Config::WorkerType _workerType = ProvideQueue::Config::Invalid;
+ ProvidePromiseWeakRef<Provide::MediaHandle> _promise;
+ MediaDataVerifierRef _verifier;
+ };
+}
+
+#endif
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\----------------------------------------------------------------------/
+*
+* This file contains private API, this might break at any time between releases.
+* You have been warned!
+*
+*
+*/
+#ifndef ZYPP_MEDIA_PRIVATE_PROVIDE_MESSAGE_P_H_INCLUDED
+#define ZYPP_MEDIA_PRIVATE_PROVIDE_MESSAGE_P_H_INCLUDED
+
+#include <zypp-core/base/PtrTypes.h>
+#include <zypp-core/zyppng/pipelines/Expected>
+#include <zypp-core/zyppng/rpc/MessageStream>
+#include <zypp-media/ng/ProvideSpec> // for FieldType
+#include <zypp-media/ng/HeaderValueMap>
+#include <variant>
+#include <functional>
+#include <zypp-proto/provider.pb.h>
+
+namespace zypp::proto {
+ class ProvideMessage;
+ class Envelope;
+}
+
+namespace zyppng {
+
+ namespace ProvideStartedMsgFields
+ {
+ constexpr std::string_view Url ("url");
+ constexpr std::string_view LocalFilename ("local_filename");
+ constexpr std::string_view StagingFilename ("staging_filename");
+ }
+
+ namespace ProvideFinishedMsgFields
+ {
+ constexpr std::string_view LocalFilename ("local_filename");
+ constexpr std::string_view CacheHit ("cacheHit");
+ }
+
+ namespace AuthInfoMsgFields
+ {
+ constexpr std::string_view Username ("username");
+ constexpr std::string_view Password ("password");
+ constexpr std::string_view AuthTimestamp ("auth_timestamp");
+ constexpr std::string_view AuthType ("authType");
+ }
+
+ namespace RedirectMsgFields
+ {
+ constexpr std::string_view NewUrl ("new_url");
+ }
+
+ namespace MetalinkRedirectMsgFields
+ {
+ constexpr std::string_view NewUrl ("new_url");
+ }
+
+ namespace ErrMsgFields
+ {
+ constexpr std::string_view Reason ("reason");
+ constexpr std::string_view Transient ("transient");
+ constexpr std::string_view History ("history");
+ }
+
+ namespace ProvideMsgFields
+ {
+ constexpr std::string_view Url ("url");
+ constexpr std::string_view Filename ("filename");
+ constexpr std::string_view DeltaFile ("delta_file");
+ constexpr std::string_view ExpectedFilesize ("expected_filesize");
+ constexpr std::string_view CheckExistOnly ("check_existance_only");
+ constexpr std::string_view MetalinkEnabled ("metalink_enabled");
+ }
+
+ namespace AttachMsgFields
+ {
+ constexpr std::string_view Url ("url");
+ constexpr std::string_view AttachId ("attach_id");
+ constexpr std::string_view VerifyType ("verify_type");
+ constexpr std::string_view VerifyData ("verify_data");
+ constexpr std::string_view MediaNr ("media_nr");
+ constexpr std::string_view Device ("device");
+ constexpr std::string_view Label ("label");
+ }
+
+ namespace DetachMsgFields
+ {
+ constexpr std::string_view Url ("url");
+ }
+
+ namespace AuthDataRequestMsgFields
+ {
+ constexpr std::string_view EffectiveUrl ("effective_url");
+ constexpr std::string_view LastAuthTimestamp ("last_auth_timestamp");
+ constexpr std::string_view LastUser ("username");
+ constexpr std::string_view AuthHint ("authHint");
+ }
+
+ namespace MediaChangeRequestMsgFields
+ {
+ constexpr std::string_view Label ("label");
+ constexpr std::string_view MediaNr ("media_nr");
+ constexpr std::string_view Device ("device");
+ constexpr std::string_view Desc ("desc");
+ }
+
+ namespace EjectMsgFields
+ {
+ constexpr std::string_view device ("device");
+ }
+
+ class ProvideMessage
+ {
+ public:
+ using Code = zypp::proto::MessageCodes;
+ using FieldVal = HeaderValue;
+
+ static expected<ProvideMessage> create ( const zyppng::RpcMessage &message );
+ static expected<ProvideMessage> create ( const zypp::proto::ProvideMessage &message );
+ static ProvideMessage createProvideStarted ( const uint32_t reqId, const zypp::Url &url , const std::optional<std::string> &localFilename = {}, const std::optional<std::string> &stagingFilename = {} );
+ static ProvideMessage createProvideFinished ( const uint32_t reqId, const std::string &localFilename , bool cacheHit );
+ static ProvideMessage createAttachFinished ( const uint32_t reqId );
+ static ProvideMessage createDetachFinished ( const uint32_t reqId );
+ static ProvideMessage createAuthInfo ( const uint32_t reqId, const std::string &user, const std::string &pw, int64_t timestamp, const std::map<std::string, std::string> &extraValues = {} );
+ static ProvideMessage createMediaChanged ( const uint32_t reqId );
+ static ProvideMessage createRedirect ( const uint32_t reqId, const zypp::Url &newUrl );
+ static ProvideMessage createMetalinkRedir ( const uint32_t reqId, const std::vector<zypp::Url> &newUrls );
+ static ProvideMessage createErrorResponse ( const uint32_t reqId, const uint code, const std::string &reason, bool transient = false );
+
+ static ProvideMessage createProvide ( const uint32_t reqId
+ , const zypp::Url &url
+ , const std::optional<std::string> &filename = {}
+ , const std::optional<std::string> &deltaFile = {}
+ , const std::optional<int64_t> &expFilesize = {}
+ , bool checkExistOnly = false );
+
+ static ProvideMessage createCancel ( const uint32_t reqId );
+
+ static ProvideMessage createAttach( const uint32_t reqId
+ , const zypp::Url &url
+ , const std::string attachId
+ , const std::string &label
+ , const std::optional<std::string> &verifyType = {}
+ , const std::optional<std::string> &verifyData = {}
+ , const std::optional<int32_t> &mediaNr = {} );
+
+ static ProvideMessage createDetach ( const uint32_t reqId, const zypp::Url &attachUrl );
+ static ProvideMessage createAuthDataRequest ( const uint32_t reqId, const zypp::Url &effectiveUrl, const std::string &lastTriedUser ="", const std::optional<int64_t> &lastAuthTimestamp = {}, const std::map<std::string, std::string> &extraValues = {} );
+ static ProvideMessage createMediaChangeRequest ( const uint32_t reqId, const std::string &label, int32_t mediaNr, const std::vector<std::string> &devices, const std::optional<std::string> &desc );
+
+ uint requestId () const;
+ void setRequestId ( const uint id );
+
+ uint code () const;
+ void setCode ( const uint newCode );
+
+ std::vector<FieldVal> values ( const std::string_view &str ) const;
+ std::vector<FieldVal> values ( const std::string &str ) const;
+ HeaderValueMap headers() const;
+ /*!
+ * Returns the last entry with key \a str in the list of values
+ * or the default value specified in \a defaultVal
+ */
+ FieldVal value ( const std::string_view &str, const FieldVal &defaultVal = FieldVal() ) const;
+ FieldVal value ( const std::string &str, const FieldVal &defaultVal = FieldVal() ) const;
+ void setValue ( const std::string &name, const FieldVal &value );
+ void setValue ( const std::string_view &name, const FieldVal &value );
+ void addValue ( const std::string &name, const FieldVal &value );
+ void addValue ( const std::string_view &name, const FieldVal &value );
+ void forEachVal( const std::function<bool( const std::string &name, const FieldVal &val)> &cb ) const;
+
+ zypp::proto::ProvideMessage &impl();
+ const zypp::proto::ProvideMessage &impl() const;
+
+ private:
+ ProvideMessage();
+ zypp::RWCOW_pointer<zypp::proto::ProvideMessage> _impl;
+ };
+}
+
+namespace zypp {
+ template<>
+ inline zypp::proto::ProvideMessage* rwcowClone<zypp::proto::ProvideMessage>( const zypp::proto::ProvideMessage * rhs )
+ { return new zypp::proto::ProvideMessage(*rhs); }
+}
+
+
+
+#endif
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\----------------------------------------------------------------------/
+*
+* This file contains private API, this might break at any time between releases.
+* You have been warned!
+*
+*/
+#ifndef ZYPP_MEDIA_PRIVATE_PROVIDE_QUEUE_P_H_INCLUDED
+#define ZYPP_MEDIA_PRIVATE_PROVIDE_QUEUE_P_H_INCLUDED
+
+#include "providefwd_p.h"
+#include <zypp-media/ng/Provide>
+#include <zypp-proto/provider.pb.h>
+#include <zypp-core/zyppng/io/Process>
+#include <zypp-core/ByteCount.h>
+
+#include <deque>
+#include <chrono>
+#include <variant>
+
+namespace zyppng {
+
+ class RpcMessageStream;
+ using RpcMessageStreamPtr = std::shared_ptr<RpcMessageStream>;
+
+ class ProvideQueue : public Base
+ {
+ public:
+ friend struct ProvideResourceData;
+
+ static constexpr uint32_t InvalidId = (uint32_t) -1;
+ using Config = zypp::proto::Capabilities;
+
+ using TimePoint = std::chrono::time_point<std::chrono::steady_clock>;
+
+ struct Item {
+
+ enum State {
+ Pending,
+ Queued,
+ Running,
+ Cancelling,
+ Finished
+ };
+ State _state = Pending;
+ bool isAttachRequest () const;
+ bool isFileRequest () const;
+ bool isDetachRequest() const;
+
+ ProvideRequestRef _request;
+ };
+
+ ProvideQueue( ProvidePrivate &parent );
+ ~ProvideQueue();
+ bool startup ( const std::string &workerScheme, const zypp::Pathname &workDir, const std::string &hostname = "" );
+ void enqueue ( ProvideRequestRef request );
+ void cancel ( ProvideRequest *item, std::exception_ptr error );
+ void detach ( const std::string &id );
+ void scheduleNext ();
+ bool canScheduleMore () const;;
+ bool empty () const;
+
+ /*!
+ * Check if the queue is currently idle
+ */
+ bool isIdle () const;
+
+ /*!
+ * Time point since the queue started to be idle
+ */
+ std::optional<TimePoint> idleSince () const;
+
+ /*!
+ * How many items does this queue currently have
+ */
+ uint requestCount () const;
+
+ /*!
+ * How many active items does this queue currently have
+ */
+ uint activeRequests () const;
+
+ /*!
+ * How much bytes does this queue has to download / process,
+ * for pending requests this is only set if the \ref ProvideSpec
+ * has a expected download size set.
+ */
+ zypp::ByteCount expectedProvideSize() const;
+
+ /*!
+ * Returns the hostname this worker belongs to.
+ * If the worker was not associated with a hostname this will return a empty string.
+ */
+ const std::string &hostname () const;
+
+ const Config &workerConfig () const;
+
+ SignalProxy<void()> sigIdle();
+
+ private:
+ bool doStartup ();
+ void processMessage ( );
+ void readAllStderr ();
+ void forwardToLog ( std::string &&logLine );
+ void processReadyRead( int channel );
+ void procFinished ( int exitCode );
+ uint32_t nextRequestId();
+
+ /*!
+ * Dequeues the request referenced by \a it.
+ * Returns a iterator to the next element in the active list
+ */
+ std::list< ProvideQueue::Item >::iterator dequeueActive ( std::list<Item>::iterator it );
+ void fatalWorkerError ( const std::exception_ptr &reason = nullptr );
+ void immediateShutdown ( const std::exception_ptr &reason );
+
+ /*!
+ * Cancels the item the iterator \a i is pointing to, advancing the iterator to the next element in the list
+ */
+ std::list< ProvideQueue::Item >::iterator cancelActiveItem (std::list<Item>::iterator i, const std::exception_ptr &error );
+
+ private:
+ bool _queueShuttingDown = false;
+ uint8_t _crashCounter = 0;
+ Config _capabilities;
+ zypp::Pathname _currentExe;
+ std::string _myHostname;
+ ProvidePrivate &_parent;
+ std::deque< Item > _waitQueue;
+ std::list< Item > _activeItems;
+ Process::Ptr _workerProc;
+ RpcMessageStreamPtr _messageStream;
+ Signal<void()> _sigIdle;
+ std::optional<TimePoint> _idleSince;
+ };
+
+}
+
+#endif
--- /dev/null
+#ifndef ZYPP_MEDIA_PRIVATE_PROVIDERES_P_H
+#define ZYPP_MEDIA_PRIVATE_PROVIDERES_P_H
+
+#include <zypp-core/ManagedFile.h>
+#include <zypp-core/Url.h>
+#include <zypp-media/ng/Provide>
+#include <zypp-media/ng/HeaderValueMap>
+#include "providefwd_p.h"
+
+namespace zyppng {
+ /*!
+ * \internal
+ * The internal shared data structure for \sa ProvideRes objects.
+ */
+ struct ProvideResourceData {
+ zyppng::Provide::MediaHandle _mediaHandle;
+ zypp::ManagedFile _myFile;
+ zypp::Url _resourceUrl; //< The resource where the file was provided from
+ HeaderValueMap _responseHeaders; //< The response headers
+ };
+}
+
+#endif // ZYPP_MEDIA_PRIVATE_PROVIDERES_P_H
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+#ifndef ZYPP_MEDIA_PROVIDE_CONFIGVARS_H_INCLUDED
+#define ZYPP_MEDIA_PROVIDE_CONFIGVARS_H_INCLUDED
+
+#include <string_view>
+
+namespace zyppng {
+ // special config strings sent to the workers:
+ constexpr std::string_view AGENT_STRING_CONF("zconfig://media/UserAgent");
+ constexpr std::string_view DISTRO_FLAV_CONF("zconfig://media/DistributionFlavor");
+ constexpr std::string_view ANON_ID_CONF("zconfig://media/AnonymousId");
+ constexpr std::string_view ATTACH_POINT("zconfig://media/AttachPoint");
+ constexpr std::string_view PROVIDER_ROOT("zconfig://media/ProviderRoot");
+
+
+ // request related settings:
+ constexpr std::string_view NETWORK_METALINK_ENABLED("zypp-nw-metalink-enabled"); //< Enable or disable metalink for a specific request
+ constexpr std::string_view HANDLER_SPECIFIC_DEVICES("zypp-req-specific-devices"); //< Limit the request to a set of devices. Devices are comma seperated.
+}
+
+#endif
--- /dev/null
+#include "private/provide_p.h"
+#include "private/providedbg_p.h"
+#include "private/providequeue_p.h"
+#include "private/provideitem_p.h"
+#include <zypp-core/zyppng/io/IODevice>
+#include <zypp-core/Url.h>
+#include <zypp-core/base/DtorReset>
+#include <zypp-core/fs/PathInfo.h>
+#include <zypp-media/MediaException>
+#include <zypp-media/FileCheckException>
+#include <zypp-media/CDTools>
+
+// required to generate uuids
+#include <glib.h>
+
+
+L_ENV_CONSTR_DEFINE_FUNC(ZYPP_MEDIA_PROVIDER_DEBUG)
+
+namespace zyppng {
+
+ ProvidePrivate::ProvidePrivate(zypp::filesystem::Pathname &&workDir, Provide &pub)
+ : BasePrivate(pub)
+ , _workDir( std::move(workDir) )
+ , _workerPath( constants::DEFAULT_PROVIDE_WORKER_PATH.data() )
+ {
+ if ( _workDir.empty() ) {
+ _workDir = zypp::Pathname(".").realpath();
+ } else {
+ _workDir = _workDir.realpath();
+ }
+
+ MIL << "Provider workdir is: " << _workDir << std::endl;
+
+ _scheduleTrigger->setSingleShot(true);
+ Base::connect( *_scheduleTrigger, &Timer::sigExpired, *this, &ProvidePrivate::doSchedule );
+ }
+
+ void ProvidePrivate::schedule( ScheduleReason reason )
+ {
+ if ( provideDebugEnabled () ) {
+ std::string_view reasonStr;
+ switch( reason ) {
+ case ProvideStart:
+ reasonStr = "ProvideStart";
+ break;
+ case QueueIdle:
+ reasonStr = "QueueIdle";
+ break;
+ case EnqueueItem:
+ reasonStr = "EnqueueItem";
+ break;
+ case EnqueueReq:
+ reasonStr = "EnqueueReq";
+ break;
+ case FinishReq:
+ reasonStr = "FinishReq";
+ break;
+ case RestartAttach:
+ reasonStr = "RestartAttach";
+ break;
+ }
+ DBG << "Triggering the schedule timer (" << reasonStr << ")" << std::endl;
+ }
+
+ // we use a single shot timer that instantly times out when the event loop is entered the next time
+ // this way we compress many schedule requests that happen during a eventloop run into one
+ _scheduleTrigger->start(0);
+ }
+
+ void ProvidePrivate::doSchedule ( zyppng::Timer & )
+ {
+ if ( !_isRunning )
+ return;
+
+ if ( _isScheduling ) {
+ DBG_PRV << "Scheduling triggered during scheduling, returning immediately." << std::endl;
+ return;
+ }
+
+ const int cpuLimit =
+#ifdef _SC_NPROCESSORS_ONLN
+ sysconf(_SC_NPROCESSORS_ONLN) * 2;
+#else
+ DEFAULT_CPU_WORKERS;
+#endif
+
+ // helper lambda to find the worker that is idle for the longest time
+ constexpr auto findLaziestWorker = []( const auto &workerQueues, const auto &idleNames ) {
+ auto candidate = workerQueues.end();
+ ProvideQueue::TimePoint candidateIdleSince = ProvideQueue::TimePoint::max();
+
+ //find the worker thats idle the longest
+ for ( const auto &name : idleNames ) {
+ auto thisElem = workerQueues.find(name);
+ if ( thisElem == workerQueues.end() )
+ continue;
+
+ const auto idleS = thisElem->second->idleSince();
+ if ( idleS
+ && ( candidate == workerQueues.end() || *idleS < candidateIdleSince ) ) {
+ candidateIdleSince = *idleS;
+ candidate = thisElem;
+ }
+ }
+
+ if ( candidate != workerQueues.end() )
+ MIL_PRV << "Found idle worker:" << candidate->first << " idle since: " << candidateIdleSince.time_since_epoch().count() << std::endl;
+
+ return candidate;
+ };
+
+ // clean up old media
+
+ for ( auto iMedia = _attachedMediaInfos.begin(); iMedia != _attachedMediaInfos.end(); ) {
+ if ( iMedia->_refCount > 0 ) {
+ MIL_PRV << "Not releasing media " << iMedia->_name << " refcount is not zero" << std::endl;
+ ++iMedia;
+ continue;
+ }
+ if ( iMedia->_workerType == ProvideQueue::Config::Downloading ) {
+ // we keep the information around for an hour so we do not constantly download the media files for no reasonDD
+ if ( std::chrono::steady_clock::now() - iMedia->_idleSince >= std::chrono::hours(1) ) {
+ MIL << "Detaching medium " << iMedia->_name << " for baseUrl " << iMedia->_attachedUrl << std::endl;
+ iMedia = _attachedMediaInfos.erase(iMedia);
+ continue;
+ } else {
+ MIL_PRV << "Not releasing media " << iMedia->_name << " downloading worker and not timed out yet." << std::endl;
+ }
+ } else {
+ // mounting handlers, we need to send a request to the workers
+ auto bQueue = iMedia->_backingQueue.lock();
+ if ( bQueue ) {
+ zypp::Url url = iMedia->_attachedUrl;
+ url.setScheme( url.getScheme() + std::string( constants::ATTACHED_MEDIA_SUFFIX) );
+ url.setAuthority( iMedia->_name );
+ const auto &req = ProvideRequest::createDetach( url );
+ if ( req ) {
+ MIL << "Detaching medium " << iMedia->_name << " for baseUrl " << iMedia->_attachedUrl << std::endl;
+ bQueue->enqueue ( *req );
+ iMedia = _attachedMediaInfos.erase(iMedia);
+ continue;
+ } else {
+ ERR << "Could not send detach request, creating the request failed" << std::endl;
+ }
+ } else {
+ ERR << "Could not send detach request since no backing queue was defined" << std::endl;
+ }
+ }
+ ++iMedia;
+ }
+
+ zypp::DtorReset schedFlag( _isScheduling, false );
+ _isScheduling = true;
+
+ const auto schedStart = std::chrono::steady_clock::now();
+ MIL_PRV << "Start scheduling" << std::endl;
+
+ zypp::OnScopeExit deferExitMessage( [&](){
+ const auto dur = std::chrono::steady_clock::now() - schedStart;
+ MIL_PRV << "Exit scheduling after:" << std::chrono::duration_cast<std::chrono::milliseconds>( dur ).count () << std::endl;
+ });
+
+ // bump inactive items
+ for ( auto it = _items.begin (); it != _items.end(); ) {
+ // was maybe released during scheduling
+ if ( !(*it) )
+ it = _items.erase(it);
+ else {
+ auto &item = *it;
+ if ( item->state() == ProvideItem::Uninitialized ) {
+ item->initialize();
+ }
+ it++;
+ }
+ }
+
+ // we are scheduling now, everything that triggered the timer until now we can forget about
+ _scheduleTrigger->stop();
+
+ for( auto queueIter = _queues.begin(); queueIter != _queues.end(); queueIter ++ ) {
+
+ const auto &scheme = queueIter->_schemeName;
+ auto &queue = queueIter->_requests;
+
+ if ( !queue.size() )
+ continue;
+
+ const auto &configOpt = schemeConfig ( scheme );
+
+ MIL_PRV << "Start scheduling for scheme:" << scheme << " queue size is: " << queue.size() << std::endl;
+
+ if ( !configOpt ) {
+ // FAIL all requests in this queue
+ ERR << "Scheme: " << scheme << " failed to return a valid configuration." << std::endl;
+
+ while( queue.size() ) {
+ auto item = std::move( queue.front() );
+ queue.pop_front();
+ if ( item->owner() )
+ item->owner()->finishReq( nullptr, item, ZYPP_EXCPT_PTR(zypp::media::MediaException("Failed to query scheme config.")) );
+ }
+
+ continue;
+ }
+
+ // the scheme config that defines how we schedule requests on this set of queues
+ const auto &config = configOpt.get();
+ const auto isSingleInstance = ( (config.cfg_flags() & ProvideQueue::Config::SingleInstance) == ProvideQueue::Config::SingleInstance );
+ if ( config.worker_type() == ProvideQueue::Config::Downloading && !isSingleInstance ) {
+
+ for( auto i = queue.begin (); i != queue.end(); ) {
+
+ // this is the only place where we remove elements from the queue when the scheduling flag is active
+ // other code just nulls out requests in the queue if during scheduling items need to be removed
+ while ( i != queue.end() && !(*i) ) {
+ i = queue.erase(i);
+ }
+
+ if ( i == queue.end() )
+ break;
+
+ ProvideRequestRef item = *i;
+
+ // Downloading queues do not support attaching via a AttachRequest, this is handled by simply providing the media verification files
+ // If we hit this code path, its a bug
+ if( item->code() == ProvideMessage::Code::Attach || item->code() == ProvideMessage::Code::Detach ) {
+ i = queue.erase(i);
+ if ( item->owner() )
+ item->owner()->finishReq( nullptr, item, ZYPP_EXCPT_PTR( zypp::Exception("Downloading Queues do not support ProvideMessage::Code::Attach requests") ) );
+ continue;
+ }
+
+ MIL_PRV << "Trying to schedule request: " << item->urls().front() << std::endl;
+
+ // how many workers for this type do already exist
+ int existingTypeWorkers = 0;
+
+ // how many currently active connections are there
+ int existingConnections = 0;
+
+ // all currently available possible queues for the request
+ std::vector< std::pair<zypp::Url, ProvideQueue*> > possibleHostWorkers;
+
+ // currently idle workers
+ std::vector<std::string> idleWorkers;
+
+ // all mirrors without a existing worker
+ std::vector<zypp::Url> mirrsWithoutWorker;
+ for ( const auto &url : item->urls() ) {
+
+ if ( effectiveScheme( url.getScheme() ) != scheme ) {
+ MIL << "Mirror URL " << url << " is incompatible with current scheme: " << scheme << ", ignoring." << std::endl;
+ continue;
+ }
+
+ if( item->owner()->canRedirectTo( item, url ) )
+ mirrsWithoutWorker.push_back( url );
+ else {
+ MIL_PRV << "URL was rejected" << url << std::endl;
+ }
+ }
+
+ // at this point the list contains all useable mirrors, if this list is empty the request needs to fail
+ if( mirrsWithoutWorker.size() == 0 ) {
+ MIL << "Request has NO usable URLs" << std::endl;
+ if ( item->owner() )
+ item->owner()->finishReq( nullptr, item, ZYPP_EXCPT_PTR(zypp::media::MediaException("No usable URLs in request spec.")) );
+ i = queue.erase(i);
+ continue;
+ }
+
+
+ for ( auto &[ queueName, workerQueue ] : _workerQueues ) {
+ if ( ProvideQueue::Config::Downloading != workerQueue->workerConfig().worker_type() )
+ continue;
+
+ existingTypeWorkers ++;
+ existingConnections += workerQueue->activeRequests();
+
+ if ( workerQueue->isIdle() )
+ idleWorkers.push_back (queueName);
+
+ if ( !zypp::str::startsWith( queueName, scheme ) )
+ continue;
+
+ for ( auto i = mirrsWithoutWorker.begin (); i != mirrsWithoutWorker.end(); ) {
+ const auto &u = *i;
+ if ( u.getHost() == workerQueue->hostname() ) {
+ if ( workerQueue->requestCount() < constants::DEFAULT_ACTIVE_CONN_PER_HOST )
+ possibleHostWorkers.push_back( {u, workerQueue.get()} );
+ i = mirrsWithoutWorker.erase( i );
+ // we can not stop after removing the first hit, since there could be multiple mirrors with the same hostname
+ } else {
+ ++i;
+ }
+ }
+ }
+
+ if( provideDebugEnabled() ) {
+ MIL << "Current stats: " << std::endl;
+ MIL << "Existing type workers: " << existingTypeWorkers << std::endl;
+ MIL << "Existing active connections: " << existingConnections << std::endl;
+ MIL << "Possible host workers: "<< possibleHostWorkers.size() << std::endl;
+ MIL << "Mirrors without worker: " << mirrsWithoutWorker.size() << std::endl;
+ }
+
+ // need to wait for requests to finish in order to schedule more requests
+ if ( existingConnections >= constants::DEFAULT_ACTIVE_CONN ) {
+ MIL_PRV << "Reached maximum nr of connections, break" << std::endl;
+ break;
+ }
+
+ // if no workers are running, take the first mirror and start a worker for it
+ // if < nr of workers are running, use a mirror we do not have a conn yet to
+ if ( existingTypeWorkers < constants::DEFAULT_MAX_DYNAMIC_WORKERS
+ && mirrsWithoutWorker.size() ) {
+
+ MIL_PRV << "Free worker slots and available mirror URLs, starting a new worker" << std::endl;
+
+ //@TODO out of the available mirrors use the best one based on statistics ( if available )
+ bool found = false;
+ for( const auto &url : mirrsWithoutWorker ) {
+
+ // mark this URL as used now, in case the queue can not be started we won't try it anymore
+ if ( !item->owner()->safeRedirectTo ( item, url ) )
+ continue;
+
+ ProvideQueueRef q = std::make_shared<ProvideQueue>( *this );
+ if ( !q->startup( scheme, _workDir / scheme / url.getHost(), url.getHost() ) ) {
+ break;
+ } else {
+
+ MIL_PRV << "Started worker for " << url.getHost() << " enqueing request" << std::endl;
+
+ item->setActiveUrl(url);
+ found = true;
+
+ std::string str = zypp::str::Format("%1%://%2%") % scheme % url.getHost();
+ _workerQueues[str] = q;
+ q->enqueue( item );
+ break;
+ }
+ }
+
+ if( found ) {
+ i = queue.erase(i);
+ continue;
+ }
+ }
+
+ // if we cannot start a new worker, find the best queue where we can push the item into
+ if ( possibleHostWorkers.size() ) {
+
+ MIL_PRV << "No free worker slots, looking for the best existing worker" << std::endl;
+ bool found = false;
+ while( possibleHostWorkers.size () ) {
+ std::vector< std::pair<zypp::Url, ProvideQueue *> >::iterator candidate = possibleHostWorkers.begin();
+ for ( auto i = candidate+1; i != possibleHostWorkers.end(); i++ ) {
+ if ( i->second->activeRequests () < candidate->second->activeRequests () )
+ candidate = i;
+ }
+
+ if ( !item->owner()->safeRedirectTo( item, candidate->first ) ) {
+ possibleHostWorkers.erase( candidate );
+ continue;
+ }
+
+ MIL_PRV << "Using existing worker " << candidate->first.getHost() << " to download request" << std::endl;
+
+ found = true;
+ item->setActiveUrl( candidate->first );
+ candidate->second->enqueue( item );
+ break;
+ }
+
+ if( found ) {
+ i = queue.erase(i);
+ continue;
+ }
+ }
+
+ // if we reach this place all we can now try is to decomission idle queues and use the new slot to start
+ // a new worker
+ if ( idleWorkers.size() && mirrsWithoutWorker.size() ) {
+
+ MIL_PRV << "No free worker slots, no slots in existing queues, trying to decomission idle queues." << std::endl;
+
+ auto candidate = findLaziestWorker( _workerQueues, idleWorkers );
+ if ( candidate != _workerQueues.end() ) {
+
+ // for now we decomission the worker and start a new one, should we instead introduce a "reset" message
+ // that repurposes the worker to another hostname/workdir config?
+ _workerQueues.erase(candidate);
+
+ //@TODO out of the available mirrors use the best one based on statistics ( if available )
+ bool found = false;
+ for( const auto &url : mirrsWithoutWorker ) {
+
+ if ( !item->owner()->safeRedirectTo ( item, url ) )
+ continue;
+
+ ProvideQueueRef q = std::make_shared<ProvideQueue>( *this );
+ if ( !q->startup( scheme, _workDir / scheme / url.getHost(), url.getHost() ) ) {
+ break;
+ } else {
+
+ MIL_PRV << "Replaced worker for " << url.getHost() << ", enqueing request" << std::endl;
+
+ item->setActiveUrl(url);
+ found = true;
+
+ auto str = zypp::str::Format("%1%://%2%") % scheme % url.getHost();
+ _workerQueues[str] = q;
+ q->enqueue( item );
+ }
+ }
+
+ if( found ) {
+ i = queue.erase(i);
+ continue;
+ }
+ }
+ }
+
+ // if we reach here we skip over the item and try to schedule it again later
+ MIL_PRV << "End of line, deferring request for next try." << std::endl;
+ i++;
+
+ }
+ } else if ( config.worker_type() == ProvideQueue::Config::CPUBound && !isSingleInstance ) {
+
+ for( auto i = queue.begin (); i != queue.end(); ) {
+
+ // this is the only place where we remove elements from the queue when the scheduling flag is active
+ // other code just nulls out requests in the queue if during scheduling items need to be removed
+ while ( i != queue.end() && !(*i) ) {
+ i = queue.erase(i);
+ }
+
+ if ( i == queue.end() )
+ break;
+
+ // make a real reference so it does not dissapear when we remove it from the queue
+ ProvideRequestRef item = *i;
+
+ // CPU bound queues do not support attaching via a AttachRequest, this is handled by simply providing the media verification files
+ // If we hit this code path, its a bug
+ if( item->code() == ProvideMessage::Code::Attach || item->code() == ProvideMessage::Code::Detach ) {
+ i = queue.erase(i);
+ if ( item->owner () )
+ item->owner()->finishReq( nullptr, item, ZYPP_EXCPT_PTR( zypp::Exception("CPU bound Queues do not support ProvideAttachSpecRef requests") ) );
+ continue;
+ }
+
+ MIL_PRV << "Trying to schedule request: " << item->urls().front() << std::endl;
+
+ // how many workers for this type do already exist
+ int existingTypeWorkers = 0;
+ int existingSchemeWorkers = 0;
+
+ // all currently available possible queues for the request
+ std::vector< ProvideQueue* > possibleWorkers;
+
+ // currently idle workers
+ std::vector<std::string> idleWorkers;
+
+ // the URL we are going to use this time
+ zypp::Url url;
+
+ //CPU bound queues do not spawn per mirrors, we use the first compatible URL
+ for ( const auto &tmpurl : item->urls() ) {
+ if ( effectiveScheme( tmpurl.getScheme() ) != scheme ) {
+ MIL << "Mirror URL " << tmpurl << " is incompatible with current scheme: " << scheme << ", ignoring." << std::endl;
+ continue;
+ }
+ url = tmpurl;
+ break;
+ }
+
+ // at this point if the URL is empty the request needs to fail
+ if( !url.isValid() ) {
+ MIL << "Request has NO usable URLs" << std::endl;
+ if ( item->owner() )
+ item->owner()->finishReq( nullptr, item, ZYPP_EXCPT_PTR(zypp::media::MediaException("No usable URLs in request spec.")) );
+ i = queue.erase(i);
+ continue;
+ }
+
+ for ( auto &[ queueName, workerQueue ] : _workerQueues ) {
+
+ if ( ProvideQueue::Config::CPUBound != workerQueue->workerConfig().worker_type() )
+ continue;
+
+ const bool thisScheme = zypp::str::startsWith( queueName, scheme );
+
+ existingTypeWorkers ++;
+ if ( thisScheme ) {
+ existingSchemeWorkers++;
+ if ( workerQueue->canScheduleMore() )
+ possibleWorkers.push_back(workerQueue.get());
+ }
+
+ if ( workerQueue->isIdle() )
+ idleWorkers.push_back(queueName);
+ }
+
+ if( provideDebugEnabled() ) {
+ MIL << "Current stats: " << std::endl;
+ MIL << "Existing type workers: " << existingTypeWorkers << std::endl;
+ MIL << "Possible CPU workers: "<< possibleWorkers.size() << std::endl;
+ }
+
+ // first we use existing idle workers of the current type
+ if ( possibleWorkers.size() ) {
+ bool found = false;
+ for ( auto &w : possibleWorkers ) {
+ if ( w->isIdle() ) {
+ MIL_PRV << "Using existing idle worker to provide request" << std::endl;
+ // this is not really required because we are not doing redirect checks
+ item->owner()->redirectTo ( item, url );
+ item->setActiveUrl( url );
+ w->enqueue( item );
+ i = queue.erase(i);
+ found = true;
+ break;
+ }
+ }
+ if ( found )
+ continue;
+ }
+
+ // we first start as many workers as we need before queueing more request to existing ones
+ if ( existingTypeWorkers < cpuLimit ) {
+
+ MIL_PRV << "Free CPU slots, starting a new worker" << std::endl;
+
+ // this is not really required because we are not doing redirect checks
+ item->owner()->redirectTo ( item, url );
+
+ ProvideQueueRef q = std::make_shared<ProvideQueue>( *this );
+ if ( q->startup( scheme, _workDir / scheme ) ) {
+
+ item->setActiveUrl(url);
+
+ auto str = zypp::str::Format("%1%#%2%") % scheme % existingSchemeWorkers;
+ _workerQueues[str] = q;
+ q->enqueue( item );
+ i = queue.erase(i);
+ continue;
+ } else {
+ // CPU bound requests can not recover from this error
+ i = queue.erase(i);
+ if ( item->owner() )
+ item->owner()->finishReq( nullptr, item, ZYPP_EXCPT_PTR( zypp::Exception("Unable to start worker for request.") ) );
+ continue;
+ }
+ }
+
+ // we can not start more workers, all we can do now is fill up queues of existing ones
+ if ( possibleWorkers.size() ) {
+ MIL_PRV << "No free CPU slots, looking for the best existing worker" << std::endl;
+
+ if( possibleWorkers.size () ) {
+ std::vector<ProvideQueue *>::iterator candidate = possibleWorkers.begin();
+ for ( auto i = candidate+1; i != possibleWorkers.end(); i++ ) {
+ if ( (*i)->activeRequests () < (*candidate)->activeRequests () )
+ candidate = i;
+ }
+
+ // this is not really required because we are not doing redirect checks
+ item->owner()->redirectTo ( item, url );
+
+ MIL_PRV << "Using existing worker to provide request" << std::endl;
+ item->setActiveUrl( url );
+ (*candidate)->enqueue( item );
+ i = queue.erase(i);
+ continue;
+ }
+ }
+
+ // if we reach this place all we can now try is to decomission idle queues and use the new slot to start
+ // a new worker
+ if ( idleWorkers.size() ) {
+
+ MIL_PRV << "No free CPU slots, no slots in existing queues, trying to decomission idle queues." << std::endl;
+
+ auto candidate = findLaziestWorker( _workerQueues, idleWorkers );
+ if ( candidate != _workerQueues.end() ) {
+
+ _workerQueues.erase(candidate);
+
+ // this is not really required because we are not doing redirect checks
+ item->owner()->redirectTo ( item, url );
+
+ ProvideQueueRef q = std::make_shared<ProvideQueue>( *this );
+ if ( q->startup( scheme, _workDir / scheme ) ) {
+
+ MIL_PRV << "Replaced worker, enqueing request" << std::endl;
+
+ item->setActiveUrl(url);
+
+ auto str = zypp::str::Format("%1%#%2%") % scheme % ( existingSchemeWorkers + 1 );
+ _workerQueues[str] = q;
+ q->enqueue( item );
+ i = queue.erase(i);
+ continue;
+ } else {
+ // CPU bound requests can not recover from this error
+ i = queue.erase(i);
+ if ( item->owner() )
+ item->owner()->finishReq( nullptr, item, ZYPP_EXCPT_PTR( zypp::Exception("Unable to start worker for request.") ) );
+ continue;
+ }
+ }
+ } else {
+ MIL_PRV << "No idle workers and no free CPU spots, wait for the next schedule run" << std::endl;
+ break;
+ }
+
+ // if we reach here we skip over the item and try to schedule it again later
+ MIL_PRV << "End of line, deferring request for next try." << std::endl;
+ i++;
+ }
+
+ } else {
+ // either SingleInstance worker or Mounting/VolatileMounting
+
+ for( auto i = queue.begin (); i != queue.end(); ) {
+
+ // this is the only place where we remove elements from the queue when the scheduling flag is active
+ // other code just nulls out requests in the queue if during scheduling items need to be removed
+ while ( i != queue.end() && !(*i) ) {
+ i = queue.erase(i);
+ }
+
+ if ( i == queue.end() )
+ break;
+
+ // make a real reference so it does not dissapear when we remove it from the queue
+ ProvideRequestRef item = *i;
+ MIL_PRV << "Trying to schedule request: " << item->urls().front() << std::endl;
+
+ zypp::Url url;
+
+ //mounting queues do not spawn per mirrors, we use the first compatible URL
+ for ( const auto &tmpurl : item->urls() ) {
+ if ( effectiveScheme( tmpurl.getScheme() ) != scheme ) {
+ MIL << "Mirror URL " << tmpurl << " is incompatible with current scheme: " << scheme << ", ignoring." << std::endl;
+ continue;
+ }
+ url = tmpurl;
+ break;
+ }
+
+ // at this point if the URL is empty the request needs to fail
+ if( !url.isValid() ) {
+ MIL << "Request has NO usable URLs" << std::endl;
+ if ( item->owner() )
+ item->owner()->finishReq( nullptr, item, ZYPP_EXCPT_PTR(zypp::media::MediaException("No usable URLs in request spec.")) );
+ i = queue.erase(i);
+ continue;
+ }
+
+
+ ProvideQueue *qToUse = nullptr;
+ if ( !_workerQueues.count(scheme) ) {
+ ProvideQueueRef q = std::make_shared<ProvideQueue>( *this );
+ if ( !q->startup( scheme, _workDir / scheme ) ) {
+ ERR << "Worker startup failed!" << std::endl;
+ // mounting/single instance requests can not recover from this error
+ i = queue.erase(i);
+
+ if ( item->owner() )
+ item->owner()->finishReq( nullptr, item, ZYPP_EXCPT_PTR( zypp::Exception("Unable to start worker for request.") ) );
+ continue;
+ }
+
+ MIL_PRV << "Started worker, enqueing request" << std::endl;
+ qToUse = q.get();
+ _workerQueues[scheme] = q;
+ } else {
+ MIL_PRV << "Found worker, enqueing request" << std::endl;
+ qToUse = _workerQueues.at(scheme).get();
+ }
+
+ // this is not really required because we are not doing redirect checks
+ item->owner()->redirectTo ( item, url );
+
+ item->setActiveUrl(url);
+ qToUse->enqueue( item );
+ i = queue.erase(i);
+ }
+ }
+ }
+ }
+
+ std::list<ProvideItemRef> &ProvidePrivate::items()
+ {
+ return _items;
+ }
+
+ zypp::media::CredManagerOptions &ProvidePrivate::credManagerOptions ()
+ {
+ return _credManagerOptions;
+ }
+
+ std::vector<AttachedMediaInfo> &ProvidePrivate::attachedMediaInfos()
+ {
+ return _attachedMediaInfos;
+ }
+
+ expected<ProvideQueue::Config> ProvidePrivate::schemeConfig( const std::string &scheme )
+ {
+ if ( auto i = _schemeConfigs.find( scheme ); i != _schemeConfigs.end() ) {
+ return expected<ProvideQueue::Config>::success(i->second);
+ } else {
+ // we do not have the queue config yet, we need to start a worker to get one
+ ProvideQueue q( *this );
+ if ( !q.startup( scheme, _workDir / scheme ) ) {
+ return expected<ProvideQueue::Config>::error(ZYPP_EXCPT_PTR(zypp::media::MediaException("Failed to start worker to read scheme config.")));
+ }
+ auto newItem = _schemeConfigs.insert( std::make_pair( scheme, q.workerConfig() ));
+ return expected<ProvideQueue::Config>::success(newItem.first->second);
+ }
+ }
+
+ std::optional<zypp::ManagedFile> ProvidePrivate::addToFileCache( const zypp::filesystem::Pathname &downloadedFile )
+ {
+ const auto &key = downloadedFile.asString();
+
+ if ( !zypp::PathInfo(downloadedFile).isExist() ) {
+ _fileCache.erase ( key );
+ return {};
+ }
+
+ auto i = _fileCache.insert( { key, FileCacheItem() } );
+ if ( !i.second ) {
+ // file did already exist in the cache, return the shared data
+ i.first->second._deathTimer.reset();
+ return i.first->second._file;
+ }
+
+ i.first->second._file = zypp::ManagedFile( downloadedFile, zypp::filesystem::unlink );
+ return i.first->second._file;
+ }
+
+ bool ProvidePrivate::isInCache ( const zypp::Pathname &downloadedFile ) const
+ {
+ const auto &key = downloadedFile.asString();
+ return (_fileCache.count(key) > 0);
+ }
+
+ void ProvidePrivate::queueItem ( ProvideItemRef item )
+ {
+ _items.push_back( item );
+ schedule( ProvidePrivate::EnqueueItem );
+ }
+
+ void ProvidePrivate::dequeueItem( ProvideItem *item)
+ {
+ auto elem = std::find_if( _items.begin(), _items.end(), [item]( const auto &i){ return i.get() == item; } );
+ if ( elem != _items.end() ) {
+ if ( _isScheduling ) {
+ (*elem).reset();
+ } else {
+ _items.erase(elem);
+ }
+ }
+ }
+
+ std::string ProvidePrivate::nextMediaId() const
+ {
+ zypp::AutoDispose rawStr( g_uuid_string_random (), g_free );
+ return zypp::str::asString ( rawStr.value() );
+ }
+
+ AttachedMediaInfo &ProvidePrivate::addMedium(ProvideQueue::Config::WorkerType workerType, const zypp::Url &baseUrl, ProvideMediaSpec &spec )
+ {
+ auto str = nextMediaId();
+ MIL_PRV << "Generated new ID for media attachment: " << str << std::endl;
+ _attachedMediaInfos.push_back( AttachedMediaInfo{ std::move(str), {}, workerType, baseUrl, spec } );
+ return _attachedMediaInfos.back();
+ }
+
+ AttachedMediaInfo &ProvidePrivate::addMedium(zypp::proto::Capabilities::WorkerType workerType, ProvideQueueWeakRef backingQueue, const std::string &id, const zypp::Url &baseUrl, ProvideMediaSpec &spec)
+ {
+ MIL_PRV << "New media attachment with id: " << id << std::endl;
+ _attachedMediaInfos.push_back( AttachedMediaInfo{ id, backingQueue, workerType, baseUrl, spec } );
+ return _attachedMediaInfos.back();
+ }
+
+ bool ProvidePrivate::queueRequest ( ProvideRequestRef req )
+ {
+ const auto &schemeName = effectiveScheme( req->url().getScheme() );
+ auto existingQ = std::find_if( _queues.begin (), _queues.end(), [&schemeName]( const auto &qItem) {
+ return (qItem._schemeName == schemeName);
+ });
+ if ( existingQ != _queues.end() ) {
+ existingQ->_requests.push_back(req);
+ } else {
+ _queues.push_back( ProvidePrivate::QueueItem{ schemeName, {req} } );
+ }
+
+ schedule( ProvidePrivate::EnqueueReq );
+ return true;
+ }
+
+ bool ProvidePrivate::dequeueRequest(ProvideRequestRef req , std::exception_ptr error)
+ {
+ auto queue = req->currentQueue ();
+ if ( queue ) {
+ queue->cancel( req.get(), error );
+ return true;
+ } else {
+ // Request not started yet, search request queues
+ for ( auto &q : _queues ) {
+ auto elem = std::find( q._requests.begin(), q._requests.end(), req );
+ if ( elem != q._requests.end() ) {
+ q._requests.erase(elem);
+
+ if ( req->owner() )
+ req->owner()->finishReq( nullptr, req, error );
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ const zypp::Pathname &ProvidePrivate::workerPath() const
+ {
+ return _workerPath;
+ }
+
+ const std::string ProvidePrivate::queueName( ProvideQueue &q ) const
+ {
+ for ( const auto &v : _workerQueues ) {
+ if ( v.second.get() == &q )
+ return v.first;
+ }
+ return {};
+ }
+
+ bool ProvidePrivate::isRunning() const
+ {
+ return _isRunning;
+ }
+
+ std::string ProvidePrivate::effectiveScheme(const std::string &scheme) const
+ {
+ const std::string &ss = zypp::str::stripSuffix( scheme, constants::ATTACHED_MEDIA_SUFFIX );
+ if ( auto it = _workerAlias.find ( ss ); it != _workerAlias.end () ) {
+ return it->second;
+ }
+ return ss;
+ }
+
+ void ProvidePrivate::onPulseTimeout( Timer & )
+ {
+ DBG_PRV << "Pulse timeout" << std::endl;
+
+ auto now = std::chrono::steady_clock::now();
+
+ if ( _log ) _log->pulse();
+
+ // release old cache files
+ for ( auto i = _fileCache.begin (); i != _fileCache.end(); ) {
+ auto &cacheItem = i->second;
+ if ( cacheItem._file.unique() ) {
+ if ( cacheItem._deathTimer ) {
+ if ( now - *cacheItem._deathTimer < std::chrono::seconds(20) ) {
+ MIL << "Releasing file " << *i->second._file << " from cache, death timeout." << std::endl;
+ i = _fileCache.erase(i);
+ continue;
+ }
+ } else {
+ // start the death timeout
+ cacheItem._deathTimer = std::chrono::steady_clock::now();
+ }
+ }
+
+ ++i;
+ }
+ }
+
+ void ProvidePrivate::onQueueIdle()
+ {
+ if ( !_items.empty() )
+ return;
+ for ( auto &[k,q] : _workerQueues ) {
+ if ( !q->empty() )
+ return;
+ }
+
+ // all queues are empty
+ _sigIdle.emit();
+ }
+
+ void ProvidePrivate::onItemStateChanged( ProvideItem &item )
+ {
+ if ( item.state() == ProvideItem::Finished ) {
+ auto itemRef = item.shared_this<ProvideItem>();
+ auto i = std::find( _items.begin(), _items.end(), itemRef );
+ if ( i == _items.end() ) {
+ ERR << "State of unknown Item changed, ignoring" << std::endl;
+ return;
+ }
+ if ( _isScheduling )
+ i->reset();
+ else
+ _items.erase(i);
+ }
+ if ( _items.empty() )
+ onQueueIdle();
+ }
+
+ uint32_t ProvidePrivate::nextRequestId()
+ {
+ //@TODO is it required to handle overflow?
+ return ++_nextRequestId;
+ }
+
+ struct ProvideMediaHandle::Data
+ {
+ Data( Provide &parent, const std::string &hdl )
+ : _parent( parent.weak_this<Provide>() )
+ , _hdlName(hdl) { }
+
+ ~Data() {
+ auto p = _parent.lock(); if (p) p->releaseMedia(_hdlName);
+ }
+
+ ProvideWeakRef _parent;
+ std::string _hdlName;
+ };
+
+ ProvideMediaHandle::ProvideMediaHandle( Provide &parent, const std::string &hdl )
+ : _ref( std::make_shared<ProvideMediaHandle::Data>( parent, hdl ) )
+ {}
+
+ std::shared_ptr<Provide> ProvideMediaHandle::parent() const
+ {
+ return _ref->_parent.lock();
+ }
+
+ bool ProvideMediaHandle::isValid() const
+ {
+ return ( _ref.get() != nullptr );
+ }
+
+ std::string ProvideMediaHandle::handle() const
+ {
+ return _ref->_hdlName;
+ }
+
+
+ Provide::Provide( const zypp::Pathname &workDir ) : Base( *new ProvidePrivate( zypp::Pathname(workDir), *this ) )
+ {
+ Z_D();
+ connect( *d->_pulseTimer, &Timer::sigExpired, *d, &ProvidePrivate::onPulseTimeout );
+ }
+
+ ProvideRef Provide::create( const zypp::filesystem::Pathname &workDir )
+ {
+ return ProvideRef( new Provide(workDir) );
+ }
+
+ AsyncOpRef<expected<Provide::MediaHandle>> Provide::attachMedia( const zypp::Url &url, const ProvideMediaSpec &request )
+ {
+ return attachMedia ( std::vector<zypp::Url>{url}, request );
+ }
+
+ AsyncOpRef<expected<Provide::MediaHandle>> Provide::attachMedia( const std::vector<zypp::Url> &urls, const ProvideMediaSpec &request )
+ {
+ Z_D();
+
+ if ( urls.empty() ) {
+ return makeReadyResult( expected<Provide::MediaHandle>::error( ZYPP_EXCPT_PTR( zypp::media::MediaException("No usable mirrors in mirrorlist."))) );
+ }
+
+ // sanitize the mirrors to contain only URLs that have same worker types
+ std::vector<zypp::Url> usableMirrs;
+ std::optional<ProvideQueue::Config> scheme;
+
+ for ( auto mirrIt = urls.begin() ; mirrIt != urls.end(); mirrIt++ ) {
+ const auto &s = d->schemeConfig( d->effectiveScheme( mirrIt->getScheme() ) );
+ if ( !s ) {
+ WAR << "URL: " << *mirrIt << " is not supported, ignoring!" << std::endl;
+ continue;
+ }
+ if ( !scheme ) {
+ scheme = *s;
+ usableMirrs.push_back ( *mirrIt );
+ } else {
+ if ( scheme->worker_type () == s->worker_type () ) {
+ usableMirrs.push_back( *mirrIt );
+ } else {
+ WAR << "URL: " << *mirrIt << " has different worker type than the primary URL: "<< usableMirrs.front() <<", ignoring!" << std::endl;
+ }
+ }
+ }
+
+ if ( !scheme || usableMirrs.empty() ) {
+ return makeReadyResult( expected<Provide::MediaHandle>::error( ZYPP_EXCPT_PTR ( zypp::media::MediaException("No valid mirrors available") )) );
+ }
+
+ // first check if there is a already attached medium we can use as well
+ auto &attachedMedia = d->attachedMediaInfos ();
+ for ( auto &medium : attachedMedia ) {
+ if ( medium.isSameMedium ( usableMirrs, request ) ) {
+ medium.ref();
+ return makeReadyResult( expected<Provide::MediaHandle>::success( Provide::MediaHandle( *this, medium._name) ) );
+ }
+ }
+
+ auto op = AttachMediaItem::create( usableMirrs, request, *d_func() );
+ d->queueItem (op);
+ return op->promise();
+ }
+
+ void Provide::releaseMedia( const std::string &mediaRef )
+ {
+ Z_D();
+
+ if ( mediaRef.empty() )
+ return;
+
+ const auto i = std::find_if( d->_attachedMediaInfos.begin(), d->_attachedMediaInfos.end(), [&]( const auto &info ){ return info._name == mediaRef;} );
+ if ( i == d->_attachedMediaInfos.end() ) {
+ ERR << "Unknown media attach handle" << std::endl;
+ return;
+ }
+
+ // only unref'ing here, the scheduler will generate a message to the queues if needed
+ i->unref();
+ }
+
+ AsyncOpRef< expected<ProvideRes> > Provide::provide( const std::vector<zypp::Url> &urls, const ProvideFileSpec &request )
+ {
+ Z_D();
+ auto op = ProvideFileItem::create( urls, request, *d );
+ d->queueItem (op);
+ return op->promise();
+ }
+
+ AsyncOpRef< expected<ProvideRes> > Provide::provide( const zypp::Url &url, const ProvideFileSpec &request )
+ {
+ return provide( std::vector<zypp::Url>{ url }, request );
+ }
+
+ AsyncOpRef< expected<ProvideRes> > Provide::provide( const MediaHandle &attachHandle, const zypp::Pathname &fileName, const ProvideFileSpec &request )
+ {
+ Z_D();
+ const auto i = std::find_if( d->_attachedMediaInfos.begin(), d->_attachedMediaInfos.end(), [&]( const auto &info ){ return info._name == attachHandle.handle();} );
+ if ( i == d->_attachedMediaInfos.end() ) {
+ return makeReadyResult( expected<ProvideRes>::error( ZYPP_EXCPT_PTR( zypp::media::MediaException("Invalid attach handle")) ) );
+ }
+
+ // for downloading items we need to make the baseUrl part of the request URL
+ zypp::Url url = i->_attachedUrl;
+
+ // real mount devices use a ID to reference a attached medium, for those we do not need to send the baseUrl as well since its already
+ // part of the mount point, so if we mount host:/path/to/repo to the ID 1234 and look for the file /path/to/repo/file1 the request URL will look like: nfs-media://1234/file1
+ if ( i->_workerType == ProvideQueue::Config::SimpleMount || i->_workerType == ProvideQueue::Config::VolatileMount ) {
+ url = zypp::Url();
+ // work around the zypp::Url requirements for certain Url schemes by attaching a suffix, that way we are always able to have a authority
+ url.setScheme( i->_attachedUrl.getScheme() + std::string(constants::ATTACHED_MEDIA_SUFFIX) );
+ url.setAuthority( i->_name );
+ url.setPathName("/");
+ }
+
+ url.appendPathName( fileName );
+ auto op = ProvideFileItem::create( {url}, request, *d );
+
+ i->ref();
+ op->setMediaRef( MediaHandle( *this, i->_name ));
+
+ d->queueItem (op);
+ return op->promise();
+ }
+
+ AsyncOpRef<expected<std::string>> Provide::checksumForFile ( const zypp::Pathname &p, const std::string &algorithm )
+ {
+ using namespace zyppng::operators;
+
+ zypp::Url url("chksum:///");
+ url.setPathName( p );
+ auto fut = provide( url, zyppng::ProvideFileSpec().setCustomHeaderValue( "chksumType", algorithm ) )
+ | mbind( [algorithm]( zyppng::ProvideRes &&chksumRes ) {
+ if ( chksumRes.headers().contains(algorithm) )
+ return expected<std::string>::success( chksumRes.headers().value(algorithm).asString() );
+ return expected<std::string>::error( ZYPP_EXCPT_PTR( zypp::FileCheckException("Invalid/Empty checksum returned from worker") ) );
+ } );
+ return fut;
+ }
+
+ AsyncOpRef<expected<zypp::ManagedFile>> Provide::copyFile ( const zypp::Pathname &source, const zypp::Pathname &target )
+ {
+ using namespace zyppng::operators;
+
+ zypp::Url url("copy:///");
+ url.setPathName( source );
+ auto fut = provide( url, ProvideFileSpec().setDestFilenameHint( target ))
+ | mbind( [&]( ProvideRes &©Res) {
+ return expected<zypp::ManagedFile>::success( copyRes.asManagedFile() );
+ } );
+ return fut;
+ }
+
+ void Provide::start ()
+ {
+ Z_D();
+ d->_isRunning = true;
+ d->_pulseTimer->start( 5000 );
+ d->schedule( ProvidePrivate::ProvideStart );
+ if ( d->_log ) d->_log->provideStart();
+ }
+
+ void Provide::setWorkerPath(const zypp::filesystem::Pathname &path)
+ {
+ d_func()->_workerPath = path;
+ }
+
+ bool Provide::ejectDevice(const std::string &queueRef, const std::string &device)
+ {
+ if ( !queueRef.empty() ) {
+ return zypp::media::CDTools::openTray(device);
+ }
+ return false;
+ }
+
+ void Provide::setStatusTracker( ProvideStatusRef tracker )
+ {
+ d_func()->_log = tracker;
+ }
+
+ const zypp::Pathname &Provide::providerWorkdir () const
+ {
+ return d_func()->_workDir;
+ }
+
+ const zypp::media::CredManagerOptions &Provide::credManangerOptions () const
+ {
+ Z_D();
+ return d->_credManagerOptions;
+ }
+
+ void Provide::setCredManagerOptions( const zypp::media::CredManagerOptions & opt )
+ {
+ d_func()->_credManagerOptions = opt;
+ }
+
+ SignalProxy<void ()> Provide::sigIdle()
+ {
+ return d_func()->_sigIdle;
+ }
+
+ SignalProxy<Provide::MediaChangeAction ( const std::string &queueRef, const std::string &, const int32_t, const std::vector<std::string> &, const std::optional<std::string> &)> Provide::sigMediaChangeRequested()
+ {
+ return d_func()->_sigMediaChange;
+ }
+
+ SignalProxy< std::optional<zypp::media::AuthData> ( const zypp::Url &reqUrl, const std::string &triedUsername, const std::map<std::string, std::string> &extraValues ) > Provide::sigAuthRequired()
+ {
+ return d_func()->_sigAuthRequired;
+ }
+
+ ZYPP_IMPL_PRIVATE(Provide);
+
+ ProvideStatus::ProvideStatus( ProvideRef parent )
+ : _provider( parent )
+ { }
+
+ void ProvideStatus::provideStart ()
+ {
+ _stats = Stats();
+ _stats._startTime = std::chrono::steady_clock::now();
+ _stats._lastPulseTime = std::chrono::steady_clock::now();
+ }
+
+ void ProvideStatus::itemDone ( ProvideItem &item )
+ {
+ const auto &sTime = item.startTime();
+ const auto &fTime = item.finishedTime();
+ if ( sTime > sTime.min() && fTime >= sTime ) {
+ auto duration = std::chrono::duration_cast<std::chrono::seconds>( item.finishedTime() - item.startTime() );
+ if ( duration.count() )
+ MIL << "Item finished after " << duration.count() << " seconds, with " << zypp::ByteCount( item.currentStats()->_bytesProvided.operator zypp::ByteCount::SizeType() / duration.count() ) << "/s" << std::endl;
+ MIL << "Item finished after " << (item.finishedTime() - item.startTime()).count() << " ns" << std::endl;
+ }
+ pulse( );
+ }
+
+ void ProvideStatus::itemFailed ( ProvideItem &item )
+ {
+ MIL << "Item failed" << std::endl;
+ }
+
+ const ProvideStatus::Stats& ProvideStatus::stats() const
+ {
+ return _stats;
+ }
+
+ void ProvideStatus::pulse ( )
+ {
+ auto prov = _provider.lock();
+ if ( !prov )
+ return;
+
+ const auto lastFinishedBytes = _stats._finishedBytes;
+ const auto lastPartialBytes = _stats._partialBytes;
+ _stats._expectedBytes = _stats._finishedBytes; // finished bytes are expected too!
+ zypp::ByteCount tmpPartialBytes (0); // bytes that are finished in staging, but not commited to cache yet
+
+ for ( const auto &i : prov->d_func()->items() ) {
+
+ if ( !i // maybe released during scheduling
+ || i->state() == ProvideItem::Cancelling )
+ continue;
+
+ if ( i->state() == ProvideItem::Uninitialized
+ || i->state() == ProvideItem::Pending ) {
+ _stats._expectedBytes += i->bytesExpected();
+ continue;
+ }
+
+ i->pulse();
+
+ const auto & stats = i->currentStats();
+ const auto & prevStats = i->previousStats();
+ if ( !stats || !prevStats ) {
+ ERR << "Bug! Stats should be initialized by now" << std::endl;
+ continue;
+ }
+
+ if ( i->state() == ProvideItem::Downloading
+ || i->state() == ProvideItem::Processing
+ || i->state() == ProvideItem::Finalizing ) {
+ _stats._expectedBytes += stats->_bytesExpected;
+ tmpPartialBytes += stats->_bytesProvided;
+ } else if ( i->state() == ProvideItem::Finished ) {
+ _stats._finishedBytes += stats->_bytesProvided; // remember those bytes are finished in stats directly
+ _stats._expectedBytes += stats->_bytesProvided;
+ }
+ }
+
+ const auto now = std::chrono::steady_clock::now();
+ const auto sinceLast = std::chrono::duration_cast<std::chrono::seconds>( now - _stats._lastPulseTime );
+ const auto lastFinB = lastPartialBytes + lastFinishedBytes;
+ const auto currFinB = tmpPartialBytes + _stats._finishedBytes;
+
+ const auto diff = currFinB - lastFinB;
+ _stats._lastPulseTime = now;
+ _stats._partialBytes = tmpPartialBytes;
+
+ if ( sinceLast >= std::chrono::seconds(1) )
+ _stats._perSecondSinceLastPulse = ( diff / ( sinceLast.count() ) );
+
+ auto sinceStart = std::chrono::duration_cast<std::chrono::seconds>( _stats._lastPulseTime - _stats._startTime );
+ if ( sinceStart.count() ) {
+ const size_t diff = _stats._finishedBytes + _stats._partialBytes;
+ _stats._perSecond = zypp::ByteCount( diff / sinceStart.count() );
+ }
+ }
+}
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+#ifndef ZYPP_MEDIA_PROVIDE_H_INCLUDED
+#define ZYPP_MEDIA_PROVIDE_H_INCLUDED
+
+#include <zypp-core/ManagedFile.h>
+#include <zypp-core/zyppng/base/zyppglobal.h>
+#include <zypp-core/zyppng/base/Base>
+#include <zypp-core/zyppng/async/AsyncOp>
+#include <zypp-core/zyppng/pipelines/expected.h>
+#include <zypp-core/ByteCount.h>
+#include <zypp-media/ng/ProvideFwd>
+#include <zypp-media/ng/ProvideRes>
+#include <zypp-media/auth/AuthData>
+#include <boost/any.hpp>
+
+namespace zypp {
+ class Url;
+ namespace media {
+ struct CredManagerOptions;
+ }
+}
+
+/*!
+ * @TODO Fix bsc#1174011 "auth=basic ignored in some cases" for provider
+ * We should proactively add the password to the request if basic auth is configured
+ * and a password is available in the credentials but not in the URL.
+ *
+ * We should be a bit paranoid here and require that the URL has a user embedded, otherwise we go the default route
+ * and ask the server first about the auth method
+ */
+namespace zyppng {
+
+ class ProvidePrivate;
+ using AnyMap = std::unordered_map<std::string, boost::any>;
+
+ /*!
+ * RAII helper for media handles
+ */
+ class ProvideMediaHandle
+ {
+ public:
+ ProvideMediaHandle () = default;
+ ProvideMediaHandle ( Provide &parent, const std::string &hdl );
+ std::shared_ptr<Provide> parent() const;
+ bool isValid () const;
+ std::string handle() const;
+ private:
+ struct Data;
+ std::shared_ptr<Data> _ref;
+ };
+
+ /*!
+ * Provide status observer object, this can be used to provide good insight into the status of the provider, its items and
+ * all running requests.
+ */
+ class ProvideStatus
+ {
+ public:
+
+ struct Stats {
+ std::chrono::steady_clock::time_point _startTime;
+ std::chrono::steady_clock::time_point _lastPulseTime;
+ uint _itemsSinceStart = 0; //< How many items have been started since Provide::start() was called
+ uint _runningItems = 0; //< How many items are currently running
+ zypp::ByteCount _finishedBytes; //< The number of bytes that were finished completely
+ zypp::ByteCount _expectedBytes; //< The number of currently expected bytes
+ zypp::ByteCount _partialBytes; //< The number of bytes of items that were already partially downloaded but the item they belong to is not finished
+ zypp::ByteCount _perSecondSinceLastPulse; //< The download speed since the last pulse
+ zypp::ByteCount _perSecond; //< The download speed we are currently operating with
+ };
+
+ ProvideStatus( ProvideRef parent );
+ virtual ~ProvideStatus(){}
+
+ virtual void provideStart ();
+ virtual void provideDone (){}
+ virtual void itemStart ( ProvideItem &item ){}
+ virtual void itemDone ( ProvideItem &item );
+ virtual void itemFailed ( ProvideItem &item );
+ virtual void requestStart ( ProvideItem &item, uint32_t reqId, const zypp::Url &url, const AnyMap &extraData = {} ){}
+ virtual void requestDone ( ProvideItem &item, uint32_t reqId, const AnyMap &extraData = {} ){}
+ virtual void requestRedirect ( ProvideItem &item, uint32_t reqId, const zypp::Url &toUrl, const AnyMap &extraData = {} ){}
+ virtual void requestFailed ( ProvideItem &item, uint32_t reqId, const std::exception_ptr err, const AnyMap &requestData = {} ){}
+ virtual void pulse ( );
+
+ const Stats &stats() const;
+
+ private:
+ Stats _stats;
+ ProvideWeakRef _provider;
+ };
+
+ class Provide : public Base
+ {
+ ZYPP_DECLARE_PRIVATE(Provide);
+ template<class T> friend class ProvidePromise;
+ friend class ProvideItem;
+ friend class ProvideMediaHandle;
+ friend class ProvideStatus;
+ public:
+
+ using MediaHandle = ProvideMediaHandle;
+
+ static ProvideRef create( const zypp::Pathname &workDir = "" );
+
+ AsyncOpRef<expected<MediaHandle>> attachMedia( const std::vector<zypp::Url> &urls, const ProvideMediaSpec &request );
+ AsyncOpRef<expected<MediaHandle>> attachMedia( const zypp::Url &url, const ProvideMediaSpec &request );
+
+ AsyncOpRef<expected<ProvideRes>> provide( const std::vector<zypp::Url> &urls, const ProvideFileSpec &request );
+ AsyncOpRef<expected<ProvideRes>> provide( const zypp::Url &url, const ProvideFileSpec &request );
+ AsyncOpRef<expected<ProvideRes>> provide( const MediaHandle &attachHandle, const zypp::Pathname &fileName, const ProvideFileSpec &request );
+
+ /*!
+ * Schedules a job to calculate the checksum for the given file
+ */
+ AsyncOpRef<expected<std::string>> checksumForFile ( const zypp::Pathname &p, const std::string &algorithm );
+
+ /*!
+ * Schedules a copy job to copy a file from \a source to \a target
+ */
+ AsyncOpRef<expected<zypp::ManagedFile>> copyFile ( const zypp::Pathname &source, const zypp::Pathname &target );
+
+ void start();
+ void setWorkerPath( const zypp::Pathname &path );
+ bool isRunning() const;
+ bool ejectDevice ( const std::string &queueRef, const std::string &device );
+
+ void setStatusTracker( ProvideStatusRef tracker );
+
+ const zypp::Pathname &providerWorkdir () const;
+
+ const zypp::media::CredManagerOptions &credManangerOptions () const;
+ void setCredManagerOptions( const zypp::media::CredManagerOptions & opt );
+
+ SignalProxy<void()> sigIdle();
+
+ enum Action {
+ ABORT, // abort and return error
+ RETRY, // retry
+ SKIP // abort and set skip request
+ };
+ using MediaChangeAction = std::optional<Action>;
+
+ /*!
+ * Connect to this signal to handle media change requests
+ *
+ * \note It is NOT supported to shutdown the provider or cancel items when in this callback
+ * Returning Abort here will effectively cancel the current item anyway.
+ */
+ SignalProxy<MediaChangeAction( const std::string &queueRef, const std::string &label, const int32_t mediaNr, const std::vector<std::string> &devices, const std::optional<std::string> &desc )> sigMediaChangeRequested( );
+
+ /*!
+ * This signal is emitted in case a request signaled a need to get Auth Info and nothing was found
+ * in the \ref zypp::media::CredentialManager.
+ */
+ SignalProxy< std::optional<zypp::media::AuthData> ( const zypp::Url &reqUrl, const std::string &triedUsername, const std::map<std::string, std::string> &extraValues ) > sigAuthRequired();
+
+ private:
+ Provide( const zypp::Pathname &workDir );
+ void releaseMedia( const std::string &mediaRef );
+ };
+
+}
+#endif
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\----------------------------------------------------------------------/
+*
+* This file contains private API, this might break at any time between releases.
+* You have been warned!
+*
+*/
+#ifndef ZYPP_MEDIA_PROVIDE_FWD_H_INCLUDED
+#define ZYPP_MEDIA_PROVIDE_FWD_H_INCLUDED
+
+#include <memory>
+#include <zypp-core/zyppng/base/zyppglobal.h>
+
+namespace zyppng {
+ ZYPP_FWD_DECL_TYPE_WITH_REFS (Provide);
+ ZYPP_FWD_DECL_TYPE_WITH_REFS (ProvideMediaSpec);
+ ZYPP_FWD_DECL_TYPE_WITH_REFS (ProvideFileSpec);
+ ZYPP_FWD_DECL_TYPE_WITH_REFS (ProvideItem);
+ ZYPP_FWD_DECL_TYPE_WITH_REFS (ProvideRequest);
+ ZYPP_FWD_DECL_TYPE_WITH_REFS (MediaDataVerifier);
+ ZYPP_FWD_DECL_TYPE_WITH_REFS (ProvideStatus);
+ class HeaderValueMap;
+ class ProvideMediaHandle;
+}
+
+#endif
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+
+#include "private/providedbg_p.h"
+#include "private/provideitem_p.h"
+#include "private/provide_p.h"
+#include "private/providemessage_p.h"
+#include "private/provideres_p.h"
+#include "provide-configvars.h"
+#include <zypp-media/MediaException>
+#include <zypp-core/base/UserRequestException>
+#include "mediaverifier.h"
+#include <zypp-core/fs/PathInfo.h>
+
+using namespace std::literals;
+
+namespace zyppng {
+
+ static constexpr std::string_view DEFAULT_MEDIA_VERIFIER("SuseMediaV1");
+
+ expected<ProvideRequestRef> ProvideRequest::create(ProvideItem &owner, const std::vector<zypp::Url> &urls, const std::string &id, ProvideMediaSpec &spec )
+ {
+ if ( urls.empty() )
+ return expected<ProvideRequestRef>::error( ZYPP_EXCPT_PTR ( zypp::media::MediaException("List of URLs can not be empty") ) );
+
+ auto m = ProvideMessage::createAttach( ProvideQueue::InvalidId, urls.front(), id, spec.label() );
+ if ( !spec.mediaFile().empty() ) {
+ m.setValue( AttachMsgFields::VerifyType, std::string(DEFAULT_MEDIA_VERIFIER.data()) );
+ m.setValue( AttachMsgFields::VerifyData, spec.mediaFile().asString() );
+ m.setValue( AttachMsgFields::MediaNr, int32_t(spec.medianr()) );
+ }
+
+ const auto &cHeaders = spec.customHeaders();
+ for ( auto i = cHeaders.beginList (); i != cHeaders.endList(); i++) {
+ for ( const auto &val : i->second )
+ m.addValue( i->first, val );
+ }
+
+ return expected<ProvideRequestRef>::success( ProvideRequestRef( new ProvideRequest(&owner, urls, std::move(m))) );
+ }
+
+ expected<ProvideRequestRef> ProvideRequest::create( ProvideItem &owner, const std::vector<zypp::Url> &urls, ProvideFileSpec &spec )
+ {
+ if ( urls.empty() )
+ return expected<ProvideRequestRef>::error( ZYPP_EXCPT_PTR ( zypp::media::MediaException("List of URLs can not be empty") ) );
+
+ auto m = ProvideMessage::createProvide ( ProvideQueue::InvalidId, urls.front() );
+ const auto &destFile = spec.destFilenameHint();
+ const auto &deltaFile = spec.deltafile();
+ const int64_t fSize = spec.downloadSize();;
+
+ if ( !destFile.empty() )
+ m.setValue( ProvideMsgFields::Filename, destFile.asString() );
+ if ( !deltaFile.empty() )
+ m.setValue( ProvideMsgFields::DeltaFile, deltaFile.asString() );
+ if ( fSize )
+ m.setValue( ProvideMsgFields::ExpectedFilesize, fSize );
+ m.setValue( ProvideMsgFields::CheckExistOnly, spec.checkExistsOnly() );
+
+ const auto &cHeaders = spec.customHeaders();
+ for ( auto i = cHeaders.beginList (); i != cHeaders.endList(); i++) {
+ for ( const auto &val : i->second )
+ m.addValue( i->first, val );
+ }
+
+ return expected<ProvideRequestRef>::success( ProvideRequestRef( new ProvideRequest(&owner, urls, std::move(m)) ) );
+ }
+
+ expected<ProvideRequestRef> ProvideRequest::createDetach( const zypp::Url &url )
+ {
+ auto m = ProvideMessage::createDetach ( ProvideQueue::InvalidId , url );
+ return expected<ProvideRequestRef>::success( ProvideRequestRef( new ProvideRequest( nullptr, { url }, std::move(m) ) ) );
+ }
+
+ ZYPP_IMPL_PRIVATE(ProvideItem);
+
+ ProvideItem::ProvideItem( ProvidePrivate &parent )
+ : Base( *new ProvideItemPrivate( parent, *this ) )
+ { }
+
+ ProvideItem::~ProvideItem()
+ { }
+
+ ProvidePrivate &ProvideItem::provider()
+ {
+ return d_func()->_parent;
+ }
+
+ bool ProvideItem::safeRedirectTo( ProvideRequestRef startedReq, const zypp::Url &url )
+ {
+ if ( !canRedirectTo( startedReq, url ) )
+ return false;
+
+ redirectTo( startedReq, url );
+ return true;
+ }
+
+ void ProvideItem::redirectTo( ProvideRequestRef startedReq, const zypp::Url &url )
+ {
+ //@TODO strip irrelevant stuff from URL
+ startedReq->_pastRedirects.push_back ( url );
+ }
+
+ bool ProvideItem::canRedirectTo( ProvideRequestRef startedReq, const zypp::Url &url )
+ {
+ // make sure there is no redirect loop
+ if ( !startedReq->_pastRedirects.size() )
+ return true;
+
+ if ( std::find( startedReq->_pastRedirects.begin(), startedReq->_pastRedirects.end(), url ) != startedReq->_pastRedirects.end() )
+ return false;
+
+ return true;
+ }
+
+ const std::optional<ProvideItem::ItemStats> &ProvideItem::currentStats() const
+ {
+ return d_func()->_currStats;
+ }
+
+ const std::optional<ProvideItem::ItemStats> &ProvideItem::previousStats() const
+ {
+ return d_func()->_prevStats;
+ }
+
+ std::chrono::steady_clock::time_point ProvideItem::startTime() const
+ {
+ return d_func()->_itemStarted;
+ }
+
+ std::chrono::steady_clock::time_point ProvideItem::finishedTime() const {
+ return d_func()->_itemFinished;
+ }
+
+ void ProvideItem::pulse ()
+ {
+ Z_D();
+ if ( d->_currStats )
+ d->_prevStats = d->_currStats;
+
+ d->_currStats = makeStats();
+
+ // once the item is finished the pulse time is always the finish time
+ if ( d->_itemState == Finished )
+ d->_currStats->_pulseTime = d->_itemFinished;
+ }
+
+ zypp::ByteCount ProvideItem::bytesExpected () const
+ {
+ return 0;
+ }
+
+ ProvideItem::ItemStats ProvideItem::makeStats ()
+ {
+ return ItemStats {
+ ._pulseTime = std::chrono::steady_clock::now(),
+ ._runningRequests = _runningReq ? (uint)1 : (uint)0
+ };
+ }
+
+ void ProvideItem::informalMessage ( ProvideQueue &, ProvideRequestRef req, const ProvideMessage &msg )
+ {
+ if ( req != _runningReq ) {
+ WAR << "Received event for unknown request, ignoring" << std::endl;
+ return;
+ }
+
+ if ( msg.code() == ProvideMessage::Code::ProvideStarted ) {
+ MIL << "Request: "<< req->url() << " was started" << std::endl;
+ }
+
+ }
+
+ void ProvideItem::cacheMiss( ProvideRequestRef req )
+ {
+ if ( req != _runningReq ) {
+ WAR << "Received event for unknown request, ignoring" << std::endl;
+ return;
+ }
+
+ MIL << "Request: "<< req->url() << " CACHE MISS, request will be restarted by queue." << std::endl;
+ }
+
+ void ProvideItem::finishReq(ProvideQueue &, ProvideRequestRef finishedReq, const ProvideMessage &msg )
+ {
+ if ( finishedReq != _runningReq ) {
+ WAR << "Received event for unknown request, ignoring" << std::endl;
+ return;
+ }
+
+ auto log = provider().log();
+
+ // explicitely handled codes
+ const auto code = msg.code();
+ if ( code == ProvideMessage::Code::Redirect ) {
+
+ // remove the old request
+ _runningReq.reset();
+
+ try {
+
+ MIL << "Request finished with redirect." << std::endl;
+
+ zypp::Url newUrl( msg.value( RedirectMsgFields::NewUrl ).asString() );
+ if ( !safeRedirectTo( finishedReq, newUrl ) ) {
+ cancelWithError( ZYPP_EXCPT_PTR ( zypp::media::MediaException("Redirect Loop")) );
+ return;
+ }
+
+ MIL << "Request redirected to: " << newUrl << std::endl;
+
+ if ( log ) log->requestRedirect( *this, msg.requestId(), newUrl );
+
+ finishedReq->setUrl( newUrl );
+
+ if ( !enqueueRequest( finishedReq ) ) {
+ cancelWithError( ZYPP_EXCPT_PTR(zypp::media::MediaException("Failed to queue request")) );
+ }
+ } catch ( ... ) {
+ cancelWithError( std::current_exception() );
+ return;
+ }
+ return;
+
+ } else if ( code == ProvideMessage::Code::Metalink ) {
+
+ // remove the old request
+ _runningReq.reset();
+
+ MIL << "Request finished with mirrorlist from server." << std::endl;
+
+ //@TODO do we need to merge this with the mirrorlist we got from the user?
+ // or does a mirrorlist from d.o.o invalidate that?
+
+ std::vector<zypp::Url> urls;
+ const auto &mirrors = msg.values( MetalinkRedirectMsgFields::NewUrl );
+ for( auto i = mirrors.cbegin(); i != mirrors.cend(); i++ ) {
+ try {
+ zypp::Url newUrl( i->asString() );
+ if ( !canRedirectTo( finishedReq, newUrl ) )
+ continue;
+ urls.push_back ( newUrl );
+ } catch ( ... ) {
+ if ( i->isString() )
+ WAR << "Received invalid URL from worker: " << i->asString() << " ignoring!" << std::endl;
+ else
+ WAR << "Received invalid value for newUrl from worker ignoring!" << std::endl;
+ }
+ }
+
+ if ( urls.size () == 0 ) {
+ cancelWithError( ZYPP_EXCPT_PTR ( zypp::media::MediaException("No mirrors left to redirect to.")) );
+ return;
+ }
+
+ MIL << "Found usable nr of mirrors: " << urls.size () << std::endl;
+ finishedReq->setUrls( urls );
+
+ // disable metalink
+ finishedReq->provideMessage().setValue( ProvideMsgFields::MetalinkEnabled, false );
+
+ if ( log ) log->requestDone( *this, msg.requestId() );
+
+ if ( !enqueueRequest( finishedReq ) ) {
+ cancelWithError( ZYPP_EXCPT_PTR(zypp::media::MediaException("Failed to queue request")) );
+ }
+
+ MIL << "End of mirrorlist handling"<< std::endl;
+ return;
+
+ } else if ( code >= ProvideMessage::Code::FirstClientErrCode && code <= ProvideMessage::Code::LastSrvErrCode ) {
+
+ // remove the old request
+ _runningReq.reset();
+
+ std::exception_ptr errPtr;
+ try {
+ const auto reqUrl = finishedReq->activeUrl().value();
+ const auto reason = msg.value( ErrMsgFields::Reason ).asString();
+ switch ( code ) {
+ case ProvideMessage::Code::BadRequest:
+ errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaException (zypp::str::Str() << "Bad request for URL: " << reqUrl << " " << reason ) );
+ break;
+ case ProvideMessage::Code::PeerCertificateInvalid:
+ errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaException(zypp::str::Str() << "PeerCertificateInvalid Error for URL: " << reqUrl << " " << reason) );
+ break;
+ case ProvideMessage::Code::ConnectionFailed:
+ errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaException(zypp::str::Str() << "ConnectionFailed Error for URL: " << reqUrl << " " << reason ) );
+ break;
+ case ProvideMessage::Code::ExpectedSizeExceeded: {
+
+ std::optional<int64_t> filesize;
+ finishedReq->provideMessage ().forEachVal( [&]( const std::string &key, const auto &val ){
+ if ( key == ProvideMsgFields::ExpectedFilesize && val.valid() )
+ filesize = val.asInt64();
+ return true;
+ });
+
+ if ( !filesize ) {
+ errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaException( zypp::str::Str() << "ExceededExpectedSize Error for URL: " << reqUrl << " " << reason ) );
+ } else {
+ errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaFileSizeExceededException(reqUrl, *filesize ) );
+ }
+ break;
+ }
+ case ProvideMessage::Code::Cancelled:
+ errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaException( zypp::str::Str() << "Request was cancelled: " << reqUrl << " " << reason ) );
+ break;
+ case ProvideMessage::Code::InvalidChecksum:
+ errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaException( zypp::str::Str() << "InvalidChecksum Error for URL: " << reqUrl << " " << reason ) );
+ break;
+ case ProvideMessage::Code::Timeout:
+ errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaTimeoutException(reqUrl) );
+ break;
+ case ProvideMessage::Code::NotFound:
+ errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaFileNotFoundException(reqUrl, "") );
+ break;
+ case ProvideMessage::Code::Forbidden:
+ case ProvideMessage::Code::Unauthorized: {
+
+ const auto &hintVal = msg.value( "authHint"sv );
+ std::string hint;
+ if ( hintVal.valid() && hintVal.isString() ) {
+ hint = hintVal.asString();
+ }
+
+ //@TODO retry here with timestamp from cred store check
+ // we let the request fail after it checked the store
+
+ errPtr = ZYPP_EXCPT_PTR ( zypp::media::MediaUnauthorizedException(
+ reqUrl, reason, "", hint
+ ));
+ break;
+
+ }
+ case ProvideMessage::Code::MountFailed:
+ errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaException( zypp::str::Str() << "MountFailed Error for URL: " << reqUrl << " " << reason ) );
+ break;
+ case ProvideMessage::Code::Jammed:
+ errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaJammedException() );
+ break;
+ case ProvideMessage::Code::MediaChangeSkip:
+ errPtr = ZYPP_EXCPT_PTR( zypp::SkipRequestException ( zypp::str::Str() << "User-requested skipping for URL: " << reqUrl << " " << reason ) );
+ break;
+ case ProvideMessage::Code::MediaChangeAbort:
+ errPtr = ZYPP_EXCPT_PTR( zypp::AbortRequestException( zypp::str::Str() <<"Aborting requested by user for URL: " << reqUrl << " " << reason ) );
+ break;
+ case ProvideMessage::Code::InternalError:
+ errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaException( zypp::str::Str() << "WorkerSpecific Error for URL: " << reqUrl << " " << reason ) );
+ break;
+ case ProvideMessage::Code::NotAFile:
+ errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaNotAFileException(reqUrl, "") );
+ break;
+ case ProvideMessage::Code::MediumNotDesired:
+ errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaNotDesiredException(reqUrl) );
+ break;
+ default:
+ errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaException( zypp::str::Str() << "Unknown Error for URL: " << reqUrl << " " << reason ) );
+ break;
+ }
+ } catch (...) {
+ errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaException( zypp::str::Str() << "Invalid error message received for URL: " << *finishedReq->activeUrl() << " code: " << code ) );
+ }
+
+ if ( log ) log->requestFailed( *this, msg.requestId(), errPtr );
+ // finish the request
+ cancelWithError( errPtr );
+ return;
+ }
+
+ // if we reach here we don't know how to handle the message
+ _runningReq.reset();
+ cancelWithError( ZYPP_EXCPT_PTR (zypp::media::MediaException("Unhandled message received for ProvideFileItem")) );
+ }
+
+ void ProvideItem::finishReq(ProvideQueue *, ProvideRequestRef finishedReq , const std::exception_ptr excpt)
+ {
+ if ( finishedReq != _runningReq ) {
+ WAR << "Received event for unknown request, ignoring" << std::endl;
+ return;
+ }
+
+ if ( _runningReq ) {
+ auto log = provider().log();
+ if ( log ) log->requestFailed( *this, finishedReq->provideMessage().requestId(), excpt );
+ }
+
+ _runningReq.reset();
+ cancelWithError(excpt);
+ }
+
+ expected<zypp::media::AuthData> ProvideItem::authenticationRequired ( ProvideQueue &queue, ProvideRequestRef req, const zypp::Url &effectiveUrl, int64_t lastTimestamp, const std::map<std::string, std::string> &extraFields )
+ {
+
+ if ( req != _runningReq ) {
+ WAR << "Received authenticationRequired for unknown request, rejecting" << std::endl;
+ return expected<zypp::media::AuthData>::error( ZYPP_EXCPT_PTR( zypp::media::MediaException("Unknown request in authenticationRequired, this is a bug.") ) );
+ }
+
+ try {
+ zypp::media::CredentialManager mgr ( provider().credManagerOptions() );
+
+ MIL << "Looking for existing auth data for " << effectiveUrl << "more recent then " << lastTimestamp << std::endl;
+
+ auto credPtr = mgr.getCred( effectiveUrl );
+ if ( credPtr && credPtr->lastDatabaseUpdate() > lastTimestamp ) {
+ MIL << "Found existing auth data for " << effectiveUrl << "ts: " << credPtr->lastDatabaseUpdate() << std::endl;
+ return expected<zypp::media::AuthData>::success( *credPtr );
+ }
+
+ if ( credPtr ) MIL << "Found existing auth data for " << effectiveUrl << "but too old ts: " << credPtr->lastDatabaseUpdate() << std::endl;
+
+ std::string username;
+ if ( auto i = extraFields.find( std::string(AuthDataRequestMsgFields::LastUser) ); i != extraFields.end() ) {
+ username = i->second;
+ }
+
+
+ MIL << "NO Auth data found, asking user. Last tried username was: " << username << std::endl;
+
+ auto userAuth = provider()._sigAuthRequired.emit( effectiveUrl, username, extraFields );
+ if ( !userAuth || !userAuth->valid() ) {
+ MIL << "User rejected to give auth" << std::endl;
+ return expected<zypp::media::AuthData>::error( ZYPP_EXCPT_PTR( zypp::media::MediaException("No auth given by user." ) ) );
+ }
+
+ mgr.addCred( *userAuth );
+ mgr.save();
+
+ // rather ugly, but update the timestamp to the last mtime of the cred database our URL belongs to
+ // otherwise we'd need to reload the cred database
+ userAuth->setLastDatabaseUpdate( mgr.timestampForCredDatabase( effectiveUrl ) );
+
+ return expected<zypp::media::AuthData>::success(*userAuth);
+ } catch ( const zypp::Exception &e ) {
+ ZYPP_CAUGHT(e);
+ return expected<zypp::media::AuthData>::error( std::current_exception() );
+ }
+ }
+
+ bool ProvideItem::enqueueRequest( ProvideRequestRef request )
+ {
+ // base item just supports one running request at a time
+ if ( _runningReq )
+ return ( _runningReq == request );
+
+ _runningReq = request;
+ return d_func()->_parent.queueRequest( request );
+ }
+
+ void ProvideItem::updateState( const State newState )
+ {
+ Z_D();
+ if ( d->_itemState != newState ) {
+
+ bool started = ( d->_itemState == Uninitialized && ( newState != Finished ));
+ auto log = provider().log();
+
+ const auto oldState = d->_itemState;
+ d->_itemState = newState;
+ d->_sigStateChanged( *this, oldState, d->_itemState );
+
+ if ( started ) {
+ d->_itemStarted = std::chrono::steady_clock::now();
+ pulse();
+ if ( log ) log->itemStart( *this );
+ }
+
+ if ( newState == Finished ) {
+ d->_itemFinished = std::chrono::steady_clock::now();
+ pulse();
+ if ( log) log->itemDone( *this );
+ d->_parent.dequeueItem(this);
+ }
+ // CAREFUL, 'this' might be invalid from here on
+ }
+ }
+
+ void ProvideItem::released()
+ {
+ if ( state() == Finished || state() == Finalizing )
+ return;
+
+ MIL << "Item Cleanup due to released Promise in state:" << state() << std::endl;
+ cancelWithError( ZYPP_EXCPT_PTR(zypp::media::MediaException("Cancelled by user.")) );
+ }
+
+ ProvideItem::State ProvideItem::state () const
+ {
+ return d_func()->_itemState;
+ }
+
+ void ProvideRequest::setCurrentQueue( ProvideQueueRef ref )
+ {
+ _myQueue = ref;
+ }
+
+ ProvideQueueRef ProvideRequest::currentQueue()
+ {
+ return _myQueue.lock();
+ }
+
+ const std::optional<zypp::Url> ProvideRequest::activeUrl() const
+ {
+ ProvideMessage::FieldVal url;
+ switch ( this->_message.code () ) {
+ case ProvideMessage::Code::Attach:
+ url = _message.value( AttachMsgFields::Url );
+ break;
+ case ProvideMessage::Code::Detach:
+ url = _message.value( DetachMsgFields::Url );
+ break;
+ case ProvideMessage::Code::Provide:
+ url = _message.value( ProvideMsgFields::Url );
+ break;
+ default:
+ // should never happen because we guard the constructor
+ throw std::logic_error("Invalid message type in ProvideRequest");
+ }
+ if ( !url.valid() ) {
+ return {};
+ }
+
+ try {
+ auto u = zypp::Url( url.asString() );
+ return u;
+ } catch ( const zypp::Exception &e ) {
+ ZYPP_CAUGHT(e);
+ }
+
+ return {};
+ }
+
+ void ProvideRequest::setActiveUrl(const zypp::Url &urlToUse) {
+
+ switch ( this->_message.code () ) {
+ case ProvideMessage::Code::Attach:
+ this->_message.setValue( AttachMsgFields::Url, urlToUse.asCompleteString() );
+ break;
+ case ProvideMessage::Code::Detach:
+ this->_message.setValue( DetachMsgFields::Url, urlToUse.asCompleteString() );
+ break;
+ case ProvideMessage::Code::Provide:
+ this->_message.setValue( ProvideMsgFields::Url, urlToUse.asCompleteString() );
+ break;
+ default:
+ // should never happen because we guard the constructor
+ throw std::logic_error("Invalid message type in ProvideRequest");
+ }
+ }
+
+ ProvideFileItem::ProvideFileItem(const std::vector<zypp::Url> &urls, const ProvideFileSpec &request, ProvidePrivate &parent)
+ : ProvideItem( parent )
+ , _mirrorList ( urls )
+ , _initialSpec ( request )
+ { }
+
+ ProvideFileItemRef ProvideFileItem::create(const std::vector<zypp::Url> &urls, const ProvideFileSpec &request, ProvidePrivate &parent )
+ {
+ return ProvideFileItemRef( new ProvideFileItem( urls, request, parent ) );
+ }
+
+ void ProvideFileItem::initialize()
+ {
+ if ( state() != Uninitialized || _runningReq ) {
+ WAR << "Double init of ProvideFileItem!" << std::endl;
+ return;
+ }
+
+ auto req = ProvideRequest::create( *this, _mirrorList, _initialSpec );
+ if ( !req ){
+ cancelWithError( req.error() );
+ return ;
+ }
+
+ if ( enqueueRequest( *req ) ) {
+ _expectedBytes = _initialSpec.downloadSize();
+ updateState( Pending );
+ } else {
+ cancelWithError( ZYPP_EXCPT_PTR(zypp::media::MediaException("Failed to queue request")) );
+ return ;
+ }
+ }
+
+ ProvidePromiseRef<ProvideRes> ProvideFileItem::promise()
+ {
+ if ( !_promiseCreated ) {
+ _promiseCreated = true;
+ auto promiseRef = std::make_shared<ProvidePromise<ProvideRes>>( shared_this<ProvideItem>() );
+ _promise = promiseRef;
+ return promiseRef;
+ }
+ return _promise.lock();
+ }
+
+ void ProvideFileItem::setMediaRef ( Provide::MediaHandle &&hdl )
+ {
+ _handleRef = std::move(hdl);
+ }
+
+ Provide::MediaHandle &ProvideFileItem::mediaRef ()
+ {
+ return _handleRef;
+ }
+
+ void ProvideFileItem::informalMessage ( ProvideQueue &, ProvideRequestRef req, const ProvideMessage &msg )
+ {
+ if ( req != _runningReq ) {
+ WAR << "Received event for unknown request, ignoring" << std::endl;
+ return;
+ }
+
+ if ( msg.code() == ProvideMessage::Code::ProvideStarted ) {
+ MIL << "Provide File Request: "<< req->url() << " was started" << std::endl;
+ auto log = provider().log();
+
+ auto locPath = msg.value( ProvideStartedMsgFields::LocalFilename, std::string() ).asString();
+ if ( !locPath.empty() )
+ _targetFile = zypp::Pathname(locPath);
+
+ locPath = msg.value( ProvideStartedMsgFields::StagingFilename, std::string() ).asString();
+ if ( !locPath.empty() )
+ _stagingFile = zypp::Pathname(locPath);
+
+ if ( log ) {
+ auto effUrl = req->activeUrl().value_or( zypp::Url() );
+ try {
+ effUrl = zypp::Url( msg.value( ProvideStartedMsgFields::Url).asString() );
+ } catch( const zypp::Exception &e ) {
+ ZYPP_CAUGHT(e);
+ }
+
+ AnyMap m;
+ m["spec"] = _initialSpec;
+ if ( log ) log->requestStart( *this, msg.requestId(), effUrl, m );
+ updateState( Downloading );
+ }
+ }
+ }
+
+ void zyppng::ProvideFileItem::ProvideFileItem::finishReq( zyppng::ProvideQueue &queue, ProvideRequestRef finishedReq, const ProvideMessage &msg )
+ {
+ if ( finishedReq != _runningReq ) {
+ WAR << "Received event for unknown request, ignoring" << std::endl;
+ return;
+ }
+
+ if ( msg.code () == ProvideMessage::Code::ProvideFinished ) {
+
+ auto log = provider().log();
+ if ( log ) {
+ AnyMap m;
+ m["spec"] = _initialSpec;
+ if ( log ) log->requestDone( *this, msg.requestId(), m );
+ }
+
+ MIL << "Request was successfully finished!" << std::endl;
+ // request is def done
+ _runningReq.reset();
+
+ try {
+
+ const auto locFilename = msg.value( ProvideFinishedMsgFields::LocalFilename ).asString();
+ const auto cacheHit = msg.value( ProvideFinishedMsgFields::CacheHit ).asBool();
+ const auto &wConf = queue.workerConfig();
+
+ const bool doesDownload = wConf.worker_type() == ProvideQueue::Config::Downloading;
+ const bool fileNeedsCleanup = doesDownload || ( wConf.worker_type() == ProvideQueue::Config::CPUBound && wConf.cfg_flags() & ProvideQueue::Config::FileArtifacts );
+
+ std::optional<zypp::ManagedFile> resFile;
+
+ if ( doesDownload ) {
+
+ resFile = provider().addToFileCache ( locFilename );
+ if ( !resFile ) {
+ if ( cacheHit ) {
+ MIL << "CACHE MISS, file " << locFilename << " was already removed, queueing again" << std::endl;
+ cacheMiss ( finishedReq );
+ finishedReq->clearForRestart();
+ enqueueRequest( finishedReq );
+ return;
+ } else {
+ // if we reach here it seems that a new file, that was not in cache before, vanished between
+ // providing it and receiving the finished message.
+ // unlikely this can happen but better be safe than sorry
+ cancelWithError( ZYPP_EXCPT_PTR( zypp::media::MediaException("File vanished between downloading and adding it to cache.")) );
+ return;
+ }
+ }
+
+ } else {
+ resFile = zypp::ManagedFile( zypp::filesystem::Pathname(locFilename) );
+ if ( fileNeedsCleanup )
+ resFile->setDispose( zypp::filesystem::unlink );
+ else
+ resFile->resetDispose();
+ }
+
+ _targetFile = locFilename;
+
+ // keep the media handle around as long as the file is used by the code
+ auto resObj = std::make_shared<ProvideResourceData>();
+ resObj->_mediaHandle = this->_handleRef;
+ resObj->_myFile = *resFile;
+ resObj->_resourceUrl = *(finishedReq->activeUrl());
+ resObj->_responseHeaders = msg.headers();
+
+ auto p = promise();
+ if ( p ) {
+ try {
+ p->setReady( expected<ProvideRes>::success( ProvideRes( resObj )) );
+ } catch( const zypp::Exception &e ) {
+ ZYPP_CAUGHT(e);
+ }
+ }
+
+ updateState( Finished );
+
+ } catch ( const zypp::Exception &e ) {
+ ZYPP_CAUGHT(e);
+ cancelWithError( std::current_exception() );
+ } catch ( ...) {
+ cancelWithError( std::current_exception() );
+ }
+
+ } else {
+ ProvideItem::finishReq ( queue, finishedReq, msg );
+ }
+ }
+
+
+ void zyppng::ProvideFileItem::cancelWithError( std::exception_ptr error )
+ {
+ if ( _runningReq ) {
+ auto weakThis = weak_from_this ();
+ provider().dequeueRequest ( _runningReq, error );
+ if ( weakThis.expired () )
+ return;
+ }
+
+ // if we reach this place for some reason finishReq was not called, lets clean up manually
+ _runningReq.reset();
+ auto p = promise();
+ if ( p ) {
+ try {
+ p->setReady( expected<ProvideRes>::error( error ) );
+ } catch( const zypp::Exception &e ) {
+ ZYPP_CAUGHT(e);
+ }
+ }
+ updateState( Finished );
+ }
+
+ expected<zypp::media::AuthData> ProvideFileItem::authenticationRequired ( ProvideQueue &queue, ProvideRequestRef req, const zypp::Url &effectiveUrl, int64_t lastTimestamp, const std::map<std::string, std::string> &extraFields )
+ {
+ zypp::Url urlToUse = effectiveUrl;
+ if ( _handleRef.isValid() ) {
+ // if we have a attached medium this overrules the URL we are going to ask the user about... this is how the old media backend did handle this
+ // i guess there were never password protected repositories that have different credentials on the redirection targets
+ auto &attachedMedia = provider().attachedMediaInfos();
+ auto i = std::find_if( attachedMedia.begin(), attachedMedia.end(), [&]( const auto &m ) { return m._name == _handleRef.handle(); } );
+ if ( i == attachedMedia.end() )
+ return expected<zypp::media::AuthData>::error( ZYPP_EXCPT_PTR( zypp::media::MediaException("Attachment handle vanished during request.") ) );
+
+ urlToUse = i->_attachedUrl;
+ }
+ return ProvideItem::authenticationRequired( queue, req, urlToUse, lastTimestamp, extraFields );
+ }
+
+ ProvideFileItem::ItemStats ProvideFileItem::makeStats ()
+ {
+ zypp::ByteCount providedByNow;
+
+ bool checkStaging = false;
+ if ( !_targetFile.empty() ) {
+ zypp::PathInfo inf( _targetFile );
+ if ( inf.isExist() && inf.isFile() )
+ providedByNow = zypp::ByteCount( inf.size() );
+ else
+ checkStaging = true;
+ }
+
+ if ( checkStaging && !_stagingFile.empty() ) {
+ zypp::PathInfo inf( _stagingFile );
+ if ( inf.isExist() && inf.isFile() )
+ providedByNow = zypp::ByteCount( inf.size() );
+ }
+
+ auto baseStats = ProvideItem::makeStats();
+ baseStats._bytesExpected = bytesExpected();
+ baseStats._bytesProvided = providedByNow;
+ return baseStats;
+ }
+
+ zypp::ByteCount ProvideFileItem::bytesExpected () const
+ {
+ return (_initialSpec.checkExistsOnly() ? zypp::ByteCount(0) : _expectedBytes);
+ }
+
+ AttachMediaItem::AttachMediaItem( const std::vector<zypp::Url> &urls, const ProvideMediaSpec &request, ProvidePrivate &parent )
+ : ProvideItem ( parent )
+ , _mirrorList ( urls )
+ , _initialSpec ( request )
+ { }
+
+ AttachMediaItem::~AttachMediaItem()
+ {
+ MIL << "Killing the AttachMediaItem" << std::endl;
+ }
+
+ ProvidePromiseRef<Provide::MediaHandle> AttachMediaItem::promise()
+ {
+ if ( !_promiseCreated ) {
+ _promiseCreated = true;
+ auto promiseRef = std::make_shared<ProvidePromise<Provide::MediaHandle>>( shared_this<ProvideItem>() );
+ _promise = promiseRef;
+ return promiseRef;
+ }
+ return _promise.lock();
+ }
+
+ void AttachMediaItem::initialize()
+ {
+ if ( state() != Uninitialized ) {
+ WAR << "Double init of AttachMediaItem!" << std::endl;
+ return;
+ }
+ updateState(Processing);
+
+ if ( _mirrorList.empty() ) {
+ cancelWithError( ZYPP_EXCPT_PTR( zypp::media::MediaException("No usable mirrors in mirrorlist.")) );
+ return;
+ }
+
+ // shortcut to the provider instance
+ auto &prov= provider();
+
+ // sanitize the mirrors to contain only URLs that have same worker types
+ std::vector<zypp::Url> usableMirrs;
+ std::optional<ProvideQueue::Config> scheme;
+
+ for ( auto mirrIt = _mirrorList.begin() ; mirrIt != _mirrorList.end(); mirrIt++ ) {
+ const auto &s = prov.schemeConfig( prov.effectiveScheme( mirrIt->getScheme() ) );
+ if ( !s ) {
+ WAR << "URL: " << *mirrIt << " is not supported, ignoring!" << std::endl;
+ continue;
+ }
+ if ( !scheme ) {
+ scheme = *s;
+ usableMirrs.push_back ( *mirrIt );
+ } else {
+ if ( scheme->worker_type () == s->worker_type () ) {
+ usableMirrs.push_back( *mirrIt );
+ } else {
+ WAR << "URL: " << *mirrIt << " has different worker type than the primary URL: "<< usableMirrs.front() <<", ignoring!" << std::endl;
+ }
+ }
+ }
+
+ // save the sanitized mirrors
+ _mirrorList = usableMirrs;
+
+ if ( !scheme || _mirrorList.empty() ) {
+ auto prom = promise();
+ if ( prom ) {
+ try {
+ prom->setReady( expected<Provide::MediaHandle>::error( ZYPP_EXCPT_PTR ( zypp::media::MediaException("No valid mirrors available") )) );
+ } catch( const zypp::Exception &e ) {
+ ZYPP_CAUGHT(e);
+ }
+ }
+ updateState( Finished );
+ return;
+ }
+
+ // first check if there is a already attached medium we can use as well
+ auto &attachedMedia = prov.attachedMediaInfos ();
+
+ for ( auto &medium : attachedMedia ) {
+ if ( medium.isSameMedium ( _mirrorList, _initialSpec ) ) {
+ finishWithSuccess ( medium );
+ return;
+ }
+ }
+
+ for ( auto &otherItem : prov.items() ) {
+ auto attachIt = std::dynamic_pointer_cast<AttachMediaItem>(otherItem);
+ if ( !attachIt // not the right type
+ || attachIt.get() == this // do not attach to ourselves
+ || attachIt->state () == Uninitialized // item was not initialized
+ || attachIt->state () == Finalizing // item cleaning up
+ || attachIt->state () == Finished ) // item done
+ continue;
+
+ // does this Item attach the same medium?
+ const auto sameMedium = attachIt->_initialSpec.isSameMedium( _initialSpec);
+ if ( zypp::indeterminate(sameMedium) ) {
+ // check the primary URLs ( should we do a full list compare? )
+ if ( attachIt->_mirrorList.front() != _mirrorList.front() )
+ continue;
+ }
+ else if ( !(bool)sameMedium )
+ continue;
+
+ MIL << "Found item providing the same medium, attaching to finished signal and waiting for it to be finished" << std::endl;
+
+ // it does, connect to its ready signal and just wait
+ _masterItemConn = connect( *attachIt, &AttachMediaItem::sigReady, *this, &AttachMediaItem::onMasterItemReady );
+ return;
+ }
+
+ _workerType = scheme->worker_type();
+
+ switch( _workerType ) {
+ case ProvideQueue::Config::Downloading: {
+
+ // if the media file is empty in the spec we can not do anything
+ // simply pretend attach worked
+ if( _initialSpec.mediaFile().empty() ) {
+ finishWithSuccess( prov.addMedium( _workerType, _mirrorList.front(), _initialSpec ) );
+ return;
+ }
+
+ // prepare the verifier with the data
+ auto smvDataLocal = MediaDataVerifier::createVerifier("SuseMediaV1");
+ if ( !smvDataLocal ) {
+ cancelWithError( ZYPP_EXCPT_PTR( zypp::media::MediaException("Unable to verify the medium, no verifier instance was returned.")) );
+ return;
+ }
+
+ if ( !smvDataLocal->load( _initialSpec.mediaFile() ) ) {
+ cancelWithError( ZYPP_EXCPT_PTR( zypp::media::MediaException("Unable to verify the medium, unable to load local verify data.")) );
+ return;
+ }
+
+ _verifier = smvDataLocal;
+
+ std::vector<zypp::Url> urls;
+ urls.reserve( _mirrorList.size () );
+
+ for ( zypp::Url url : _mirrorList ) {
+ url.appendPathName ( ( zypp::str::Format("/media.%d/media") % _initialSpec.medianr() ).asString() );
+ urls.push_back(url);
+ }
+
+ // for downloading schemes we ask for the /media.x/media file and check the data manually
+ ProvideFileSpec spec;
+ spec.customHeaders() = _initialSpec.customHeaders();
+
+ // disable metalink
+ spec.customHeaders().set( std::string(NETWORK_METALINK_ENABLED), false );
+
+ auto req = ProvideRequest::create( *this, urls, spec );
+ if ( !req ) {
+ cancelWithError( req.error() );
+ return;
+ }
+ if ( !enqueueRequest( *req ) ) {
+ cancelWithError( ZYPP_EXCPT_PTR(zypp::media::MediaException("Failed to queue request")) );
+ return;
+ }
+ updateState ( Downloading );
+ break;
+ }
+ case ProvideQueue::Config::VolatileMount:
+ case ProvideQueue::Config::SimpleMount: {
+
+ const auto &newId = provider().nextMediaId();
+ auto req = ProvideRequest::create( *this, _mirrorList, newId, _initialSpec );
+ if ( !req ) {
+ cancelWithError( req.error() );
+ return;
+ }
+ if ( !enqueueRequest( *req ) ) {
+ ERR << "Failed to queue request" << std::endl;
+ cancelWithError( ZYPP_EXCPT_PTR(zypp::media::MediaException("Failed to queue request")) );
+ return;
+ }
+ break;
+ }
+ default: {
+ auto prom = promise();
+ if ( prom ) {
+ try {
+ prom->setReady( expected<Provide::MediaHandle>::error( ZYPP_EXCPT_PTR ( zypp::media::MediaException("URL scheme does not support attaching.") )) );
+ } catch( const zypp::Exception &e ) {
+ ZYPP_CAUGHT(e);
+ }
+ }
+ updateState( Finished );
+ return;
+ }
+ }
+ }
+
+ void AttachMediaItem::finishWithSuccess( AttachedMediaInfo &medium )
+ {
+
+ updateState(Finalizing);
+
+ // aquire a ref to keep the medium around until we notified all dependant attach operations
+ // currently not really required because only the next schedule run will clean up attached medias
+ // but in case that ever changes the code is safe already
+ medium.ref();
+ zypp::OnScopeExit autoUnref([&]{
+ medium.unref();
+ });
+
+ auto prom = promise();
+ try {
+ if ( prom ) {
+ // the ref for the result we are giving out
+ medium.ref();
+ try {
+ prom->setReady( expected<Provide::MediaHandle>::success( Provide::MediaHandle( *static_cast<Provide*>( provider().z_func() ), medium._name) ) );
+ } catch( const zypp::Exception &e ) {
+ ZYPP_CAUGHT(e);
+ }
+ }
+ } catch ( const std::exception &e ) {
+ ERR << "WTF " << e.what () << std::endl;
+ } catch ( ... ) {
+ ERR << "WTF " << std::endl;
+ }
+
+ // tell others as well
+ _sigReady.emit( zyppng::expected<AttachedMediaInfo *>::success(&medium) );
+
+ prom->isReady ();
+
+ MIL << "Before setFinished" << std::endl;
+ updateState( Finished );
+ return;
+ }
+
+ void AttachMediaItem::cancelWithError( std::exception_ptr error )
+ {
+ MIL << "Cancelling Item with error" << std::endl;
+ updateState(Finalizing);
+
+ // tell children
+ _sigReady.emit( expected<AttachedMediaInfo *>::error(error) );
+
+ if ( _runningReq ) {
+ // we might get deleted when calling dequeueRequest
+ auto weakThis = weak_from_this ();
+ provider().dequeueRequest ( _runningReq, error );
+ if ( weakThis.expired () )
+ return;
+ }
+
+ // if we reach this place we had no runningReq, clean up manually
+ _runningReq.reset();
+ _masterItemConn.disconnect();
+
+ auto p = promise();
+ if ( p ) {
+ try {
+ p->setReady( expected<zyppng::Provide::MediaHandle>::error( error ) );
+ } catch( const zypp::Exception &e ) {
+ ZYPP_CAUGHT(e);
+ }
+ }
+ updateState( Finished );
+ }
+
+ void AttachMediaItem::onMasterItemReady( const zyppng::expected<AttachedMediaInfo *> &result )
+ {
+
+ _masterItemConn.disconnect();
+
+ if ( result ) {
+ AttachedMediaInfo &medium = *result.get();
+ finishWithSuccess(medium);
+ } else {
+ try {
+ std::rethrow_exception ( result.error() );
+ } catch ( const zypp::media::MediaRequestCancelledException & e) {
+ // in case a item was cancelled, we revert to Pending state and trigger the scheduler.
+ // This will make sure that all our sibilings that also depend on the master
+ // can revert to pending state and we only get one new master in the next schedule run
+ MIL_PRV << "Master item was cancelled, reverting to Uninitialized state and waiting for scheduler to run again" << std::endl;
+ updateState (Uninitialized);
+ provider().schedule( ProvidePrivate::RestartAttach );
+
+ } catch ( ... ) {
+ cancelWithError( std::current_exception() );
+ }
+ }
+ }
+
+ AttachMediaItemRef AttachMediaItem::create( const std::vector<zypp::Url> &urls, const ProvideMediaSpec &request, ProvidePrivate &parent )
+ {
+ return AttachMediaItemRef( new AttachMediaItem(urls, request, parent) );
+ }
+
+ SignalProxy<void (const zyppng::expected<AttachedMediaInfo *> &)> AttachMediaItem::sigReady()
+ {
+ return _sigReady;
+ }
+
+ void AttachMediaItem::finishReq ( ProvideQueue &queue, ProvideRequestRef finishedReq, const ProvideMessage &msg )
+ {
+ if ( finishedReq != _runningReq ) {
+ WAR << "Received event for unknown request, ignoring" << std::endl;
+ return;
+ }
+
+ if( _workerType == ProvideQueue::Config::Downloading ) {
+ // success
+ if ( msg.code() == ProvideMessage::Code::ProvideFinished ) {
+
+ updateState(Finalizing);
+
+ zypp::Url baseUrl = *finishedReq->activeUrl();
+ // remove /media.n/media
+ baseUrl.setPathName( zypp::Pathname(baseUrl.getPathName()).dirname().dirname() );
+
+ // we got the file, lets parse it
+ auto smvDataRemote = MediaDataVerifier::createVerifier("SuseMediaV1");
+ if ( !smvDataRemote ) {
+ return cancelWithError( ZYPP_EXCPT_PTR( zypp::media::MediaException("Unable to verify the medium, no verifier instance was returned.")) );
+ }
+
+ if ( !smvDataRemote->load( msg.value( ProvideFinishedMsgFields::LocalFilename ).asString() ) ) {
+ return cancelWithError( ZYPP_EXCPT_PTR( zypp::media::MediaException("Unable to verify the medium, unable to load remote verify data.")) );
+ }
+
+ // check if we got a valid media file
+ if ( !smvDataRemote->valid () ) {
+ return cancelWithError( ZYPP_EXCPT_PTR( zypp::media::MediaException("Unable to verify the medium, unable to load local verify data.")) );
+ }
+
+ // check if the received file matches with the one we have in the spec
+ if (! _verifier->matches( smvDataRemote ) ) {
+ DBG << "expect: " << _verifier << " medium " << _initialSpec.medianr() << std::endl;
+ DBG << "remote: " << smvDataRemote << std::endl;
+ return cancelWithError( ZYPP_EXCPT_PTR( zypp::media::MediaNotDesiredException( *finishedReq->activeUrl() ) ) );
+ }
+
+ // all good, register the medium and tell all child items
+ _runningReq.reset();
+ return finishWithSuccess( provider().addMedium( _workerType, baseUrl, _initialSpec ) );
+
+ } else if ( msg.code() == ProvideMessage::Code::NotFound ) {
+
+ // simple downloading attachment we need to check the media file contents
+ // in case of a error we might tolerate a file not found error in certain situations
+ if ( _verifier->totalMedia () == 1 ) {
+ // relaxed , tolerate a vanished media file
+ _runningReq.reset();
+ return finishWithSuccess( provider().addMedium( _workerType, _mirrorList.front(), _initialSpec) );
+ } else {
+ return ProvideItem::finishReq ( queue, finishedReq, msg );
+ }
+ } else {
+ return ProvideItem::finishReq ( queue, finishedReq, msg );
+ }
+ } else {
+ // real device attach
+ if ( msg.code() == ProvideMessage::Code::AttachFinished ) {
+ _runningReq.reset();
+ return finishWithSuccess( provider().addMedium( _workerType
+ , queue.weak_this<ProvideQueue>()
+ , finishedReq->provideMessage().value( AttachMsgFields::AttachId ).asString()
+ , *finishedReq->activeUrl()
+ , _initialSpec ) );
+ }
+ }
+
+ // unhandled message , let the base impl do it
+ return ProvideItem::finishReq ( queue, finishedReq, msg );
+ }
+
+ expected<zypp::media::AuthData> AttachMediaItem::authenticationRequired ( ProvideQueue &queue, ProvideRequestRef req, const zypp::Url &effectiveUrl, int64_t lastTimestamp, const std::map<std::string, std::string> &extraFields )
+ {
+ zypp::Url baseUrl = effectiveUrl;
+ if( _workerType == ProvideQueue::Config::Downloading ) {
+ // remove /media.n/media
+ baseUrl.setPathName( zypp::Pathname(baseUrl.getPathName()).dirname().dirname() );
+ }
+ return ProvideItem::authenticationRequired( queue, req, baseUrl, lastTimestamp, extraFields );
+ }
+
+}
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+#ifndef ZYPP_MEDIA_PROVIDEITEM_H_INCLUDED
+#define ZYPP_MEDIA_PROVIDEITEM_H_INCLUDED
+
+#include <zypp-core/zyppng/base/zyppglobal.h>
+#include <zypp-core/zyppng/base/Base>
+#include <zypp-media/ng/ProvideFwd>
+#include <zypp-core/ByteCount.h>
+
+namespace zyppng
+{
+ class ProvidePrivate;
+ class ProvideItemPrivate;
+
+ /*!
+ * Represents a operation added to the provide queue by the user code. A "user" operation can have multiple
+ * steps of downloading and processing one or multiple files. Even though this class is in public API space it's
+ * not possible to implement custom Items since more internal API would be required for it. It is only public to
+ * support the \ref ProvideStatus class, so it can query details of a Item.
+ */
+ class ProvideItem : public Base
+ {
+ ZYPP_DECLARE_PRIVATE(ProvideItem);
+ friend class Provide;
+ friend class ProvidePrivate;
+ friend class ProvideQueue;
+ public:
+ enum State {
+ Uninitialized, //< Item was added to the queue but not yet initialized
+ Pending, //< The Item is waiting to be queued and does nothing
+ Downloading, //< The Item is fetching data
+ Processing, //< The Item has fetched all data but is still doing other things
+ Cancelling, //< The Item was cancelled and waits
+ Finalizing, //< The Item has finished all work and is currently cleaning up
+ Finished //< The Item is done and can be dequeued
+ };
+
+ struct ItemStats {
+ std::chrono::steady_clock::time_point _pulseTime;
+ uint _runningRequests;
+ zypp::ByteCount _bytesProvided;
+ zypp::ByteCount _bytesExpected;
+ };
+
+ ProvideItem( ProvidePrivate &parent );
+ ~ProvideItem ();
+
+ /*!
+ * Called by the controller when the item is supposed to start fetching / processing
+ */
+ virtual void initialize () = 0;
+
+ /*!
+ * Called when the promise reference is released by the user process, cancel all running requests if there are any and clean up
+ */
+ virtual void released ();
+
+ State state () const;
+
+ /*!
+ * Signal that is emitted when the state of the Item has changed
+ */
+ SignalProxy<void( ProvideItem &item, State oldState, State newState )> sigStateChanged();
+
+ ProvidePrivate &provider();
+
+ /*!
+ * Returns true if a redirect is allowed and does not conflict with previous redirects.
+ * Otherwise false is returned. This does not remember the passed URL as a redirect
+ */
+ virtual bool canRedirectTo ( ProvideRequestRef startedReq, const zypp::Url &url );
+
+ /*!
+ * Returns the item statistics that were collected the last time pulse() was called on the item.
+ * If the item has not been started yet, this returns a empty optional
+ */
+ const std::optional<ItemStats> ¤tStats() const;
+
+ /*!
+ * Returns the item statistics that were collected the previous time pulse() was called on the item.
+ * \note The item needs to be started and pulse() needs to be called at least once for this func to return something
+ */
+ const std::optional<ItemStats> &previousStats() const;
+
+ /*!
+ * Returns the time point when the item started to process/download.
+ * If the item has not started yet, this returns epoch
+ */
+ virtual std::chrono::steady_clock::time_point startTime() const;
+
+ /*!
+ * Returns the time point when the item was finished.
+ * If the item was not finished yet, this returns epoch
+ */
+ virtual std::chrono::steady_clock::time_point finishedTime() const;
+
+ /*!
+ * Updates the item statistics, this is called automatically by the \ref Provide instance and usually does not
+ * need to be called explicitely by usercode.
+ */
+ void pulse ();
+
+ /*!
+ * Returns the bytes the item expects to provide, the default impl returns 0
+ */
+ virtual zypp::ByteCount bytesExpected () const;
+
+ protected:
+ virtual ItemStats makeStats ();
+
+ /*!
+ * Request received a informal message, e.g. ProvideStarted
+ */
+ virtual void informalMessage ( ProvideQueue &, ProvideRequestRef req, const ProvideMessage &msg );
+
+ /*!
+ * Request had a cache miss and will be queued again, forget all about the request
+ */
+ virtual void cacheMiss ( ProvideRequestRef req );
+
+ /*!
+ * Request was finished by the queue
+ * Base implementation handles redirect, metalink and error messages. If a different message is
+ * received, \ref cancelWithError is called.
+ *
+ * A subclass has to overload this function to handle success messages
+ */
+ virtual void finishReq ( ProvideQueue &queue, ProvideRequestRef finishedReq, const ProvideMessage &msg );
+
+ /*!
+ * Request was finished with a error
+ * The base implementation simply calls \ref cancelWithError
+ *
+ * \note \a queue is allowed to be a nullptr here
+ */
+ virtual void finishReq ( ProvideQueue *queue, ProvideRequestRef finishedReq, const std::exception_ptr excpt );
+
+ /*!
+ * Request needs authentication data, the function is supposed to return the AuthData to use for the response, or an error
+ * The default implementation simply uses the given URL to look for a Auth match in the \ref zypp::media::CredentialManager.
+ */
+ virtual expected<zypp::media::AuthData> authenticationRequired ( ProvideQueue &queue, ProvideRequestRef req, const zypp::Url &effectiveUrl, int64_t lastTimestamp, const std::map<std::string, std::string> &extraFields );
+
+ /*!
+ * Remembers previous redirects and returns false if the URL was encountered before, use this
+ * to prevent the item getting caught in a redirect loop
+ */
+ bool safeRedirectTo ( ProvideRequestRef startedReq, const zypp::Url &url );
+
+ /*!
+ * Similar to \ref safeRedirectTo, but does not check if a URL was already used by this Request before.
+ */
+ void redirectTo ( ProvideRequestRef startedReq, const zypp::Url &url );
+
+ /*!
+ * Enqueue the request in the correct queue, the item implementation is supposed to hold its own
+ * reference to all started requests, the base implementation just keeps track of 1 request at a time.
+ */
+ virtual bool enqueueRequest( ProvideRequestRef request );
+
+ /*!
+ * Cancels all running requests and immediately moves to error state
+ */
+ virtual void cancelWithError ( std::exception_ptr error ) = 0;
+
+ /*!
+ * Dequeue this item and stop all requests in queues running
+ * Call this when the item is cancelled or finished.
+ */
+ bool dequeue ();
+
+ /*!
+ * Call this when the state of the item changes.
+ *
+ * \note calling updateState with state \ref Finished will potentially delete the Item instance
+ */
+ void updateState( const State newState );
+
+ void setFinished ();
+
+ protected:
+ ProvideRequestRef _runningReq;
+ };
+}
+#endif // ZYPP_MEDIA_PROVIDEITEM_H_INCLUDED
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+
+#include "private/providemessage_p.h"
+
+#include <zypp-core/Url.h>
+#include <string_view>
+#include <string>
+
+namespace zyppng {
+
+ static ProvideMessage::FieldVal fieldValFromProto ( const zypp::proto::DataField &field )
+ {
+ ProvideMessage::FieldVal v;
+ switch ( field.field_val_case () ) {
+ case zypp::proto::DataField::FieldValCase::kBoolVal:
+ v = field.bool_val();
+ break;
+ case zypp::proto::DataField::FieldValCase::kDoubleVal:
+ v = field.double_val();
+ break;
+ case zypp::proto::DataField::FieldValCase::kIntVal:
+ v = field.int_val();
+ break;
+ case zypp::proto::DataField::FieldValCase::kLongVal:
+ v = field.long_val();
+ break;
+ case zypp::proto::DataField::FieldValCase::kStrVal:
+ v = field.str_val();
+ break;
+ case zypp::proto::DataField::FieldValCase::FIELD_VAL_NOT_SET:
+ ZYPP_THROW( std::logic_error("Unexpected DataField type"));
+ break;
+ }
+ return v;
+ }
+
+ static void fieldValToProto ( const ProvideMessage::FieldVal &val, zypp::proto::DataField &field )
+ {
+ if ( val.isString() )
+ field.set_str_val( val.asString () );
+ else if ( val.isInt() )
+ field.set_int_val( val.asInt() );
+ else if ( val.isInt64() )
+ field.set_long_val( val.asInt64() );
+ else if ( val.isDouble() )
+ field.set_double_val( val.asDouble() );
+ else if ( val.isBool() )
+ field.set_bool_val( val.asBool() );
+ else
+ ZYPP_THROW( std::logic_error("Unexpected FieldVal type"));
+ }
+
+ static expected<void> validateMessage ( const ProvideMessage &msg )
+ {
+ const auto c = msg.code();
+ const auto validCode = ( c >= ProvideMessage::Code::FirstInformalCode && c <= ProvideMessage::Code::LastInformalCode )
+ || ( c >= ProvideMessage::Code::FirstSuccessCode && c <= ProvideMessage::Code::LastSuccessCode )
+ || ( c >= ProvideMessage::Code::FirstRedirCode && c <= ProvideMessage::Code::LastRedirCode )
+ || ( c >= ProvideMessage::Code::FirstClientErrCode && c <= ProvideMessage::Code::LastClientErrCode )
+ || ( c >= ProvideMessage::Code::FirstSrvErrCode && c <= ProvideMessage::Code::LastSrvErrCode )
+ || ( c >= ProvideMessage::Code::FirstControllerCode && c <= ProvideMessage::Code::LastControllerCode)
+ || ( c >= ProvideMessage::Code::FirstWorkerCode && c <= ProvideMessage::Code::LastWorkerCode );
+ if ( !validCode ) {
+ return zyppng::expected<void>::error( ZYPP_EXCPT_PTR ( InvalidMessageReceivedException("Invalid code in ProvideMessage")) );
+ }
+
+ #define DEF_REQ_FIELD( fname ) bool has_##fname = false
+
+ #define REQ_FIELD_CHECK( msgtype, fname, ftype ) \
+ if ( name == #fname ) { \
+ if ( !std::holds_alternative<ftype>(val.asVariant()) ) { \
+ error = ZYPP_EXCPT_PTR( InvalidMessageReceivedException( zypp::str::Str() << "Parse error " << #msgtype << ", Field " << #fname << " has invalid type" ) ); \
+ return false; \
+ } \
+ has_##fname = true; \
+ }
+
+ #define OR_REQ_FIELD_CHECK( msgtype, fname, ftype ) else REQ_FIELD_CHECK( msgtype, fname, ftype )
+
+ #define OPT_FIELD_CHECK( msgtype, fname, ftype ) \
+ if ( name == #fname ) { \
+ if ( !std::holds_alternative<ftype>(val.asVariant() ) ) { \
+ error = ZYPP_EXCPT_PTR( InvalidMessageReceivedException( zypp::str::Str() << "Parse error " << #msgtype << ", Field " << #fname << " has invalid type" ) ); \
+ return false; \
+ } \
+ }
+
+ #define OR_OPT_FIELD_CHECK( msgtype, fname, ftype ) else OPT_FIELD_CHECK( msgtype, fname, ftype )
+
+ #define FAIL_IF_NOT_SEEN_REQ_FIELD( msgtype, fname ) \
+ if ( !has_##fname ) \
+ return expected<void>::error( ZYPP_EXCPT_PTR( InvalidMessageReceivedException( zypp::str::Str() << #msgtype <<" message does not contain required " << #fname << " field" ) ) )
+
+ #define FAIL_IF_ERROR( ) \
+ if ( error ) return expected<void>::error( error )
+
+ const auto &validateErrorMsg = []( const auto &msg ){
+ std::exception_ptr error;
+ DEF_REQ_FIELD(reason);
+ msg.forEachVal( [&]( const auto &name, const ProvideMessage::FieldVal &val ){
+ REQ_FIELD_CHECK ( Error, reason, std::string )
+ OR_OPT_FIELD_CHECK ( Error, history, std::string )
+ OR_OPT_FIELD_CHECK ( Error, transient, bool )
+ return true;
+ });
+ FAIL_IF_NOT_SEEN_REQ_FIELD( Error, reason );
+ FAIL_IF_ERROR();
+ return expected<void>::success();
+ };
+
+ switch ( c )
+ {
+ case ProvideMessage::Code::ProvideStarted: {
+ std::exception_ptr error;
+ DEF_REQ_FIELD(url);
+ msg.forEachVal( [&]( const auto &name, const ProvideMessage::FieldVal &val ){
+ REQ_FIELD_CHECK ( ProvideStarted, url, std::string )
+ OR_OPT_FIELD_CHECK ( ProvideStarted, local_filename, std::string )
+ OR_OPT_FIELD_CHECK ( ProvideStarted, staging_filename, std::string )
+ return true;
+ });
+ FAIL_IF_NOT_SEEN_REQ_FIELD( ProvideStarted, url );
+ FAIL_IF_ERROR();
+ break;
+ }
+ case ProvideMessage::Code::ProvideFinished: {
+ std::exception_ptr error;
+ DEF_REQ_FIELD(cacheHit);
+ DEF_REQ_FIELD(local_filename);
+ msg.forEachVal( [&]( const auto &name, const ProvideMessage::FieldVal &val ){
+ REQ_FIELD_CHECK ( ProvideFinished, cacheHit, bool )
+ OR_REQ_FIELD_CHECK ( ProvideFinished, local_filename, std::string )
+ return true;
+ });
+ FAIL_IF_NOT_SEEN_REQ_FIELD( ProvideFinished, cacheHit );
+ FAIL_IF_NOT_SEEN_REQ_FIELD( ProvideFinished, local_filename );
+ FAIL_IF_ERROR();
+ break;
+ }
+ case ProvideMessage::Code::AttachFinished: {
+ // no fields
+ break;
+ }
+ case ProvideMessage::Code::DetachFinished: {
+ // no fields
+ break;
+ }
+ case ProvideMessage::Code::AuthInfo: {
+ std::exception_ptr error;
+ DEF_REQ_FIELD(username);
+ DEF_REQ_FIELD(password);
+ DEF_REQ_FIELD(auth_timestamp);
+ msg.forEachVal( [&]( const auto &name, const ProvideMessage::FieldVal &val ){
+ REQ_FIELD_CHECK ( AuthInfo, username, std::string )
+ OR_REQ_FIELD_CHECK ( AuthInfo, password, std::string )
+ OR_REQ_FIELD_CHECK ( AuthInfo, auth_timestamp, int64_t )
+ OR_OPT_FIELD_CHECK ( AuthInfo, authType, std::string )
+ return true;
+ });
+ FAIL_IF_NOT_SEEN_REQ_FIELD( ProvideStarted, username );
+ FAIL_IF_NOT_SEEN_REQ_FIELD( ProvideStarted, password );
+ FAIL_IF_NOT_SEEN_REQ_FIELD( ProvideStarted, auth_timestamp );
+ FAIL_IF_ERROR();
+ break;
+ }
+ case ProvideMessage::Code::MediaChanged:
+ /* No Fields */
+ break;
+ case ProvideMessage::Code::Redirect: {
+ std::exception_ptr error;
+ DEF_REQ_FIELD(new_url);
+ msg.forEachVal( [&]( const auto &name, const ProvideMessage::FieldVal &val ){
+ REQ_FIELD_CHECK ( Redirect, new_url, std::string )
+ return true;
+ });
+ FAIL_IF_NOT_SEEN_REQ_FIELD( Redirect, new_url );
+ FAIL_IF_ERROR();
+ break;
+ }
+ case ProvideMessage::Code::Metalink: {
+ std::exception_ptr error;
+ DEF_REQ_FIELD(new_url);
+ msg.forEachVal( [&]( const auto &name, const ProvideMessage::FieldVal &val ){
+ REQ_FIELD_CHECK ( Metalink, new_url, std::string )
+ return true;
+ });
+ FAIL_IF_NOT_SEEN_REQ_FIELD( Metalink, new_url );
+ FAIL_IF_ERROR();
+ break;
+ }
+ case ProvideMessage::Code::BadRequest:
+ case ProvideMessage::Code::Unauthorized:
+ case ProvideMessage::Code::Forbidden:
+ case ProvideMessage::Code::PeerCertificateInvalid:
+ case ProvideMessage::Code::NotFound:
+ case ProvideMessage::Code::ExpectedSizeExceeded:
+ case ProvideMessage::Code::ConnectionFailed:
+ case ProvideMessage::Code::Timeout:
+ case ProvideMessage::Code::Cancelled:
+ case ProvideMessage::Code::InvalidChecksum:
+ case ProvideMessage::Code::MountFailed:
+ case ProvideMessage::Code::Jammed:
+ case ProvideMessage::Code::NoAuthData:
+ case ProvideMessage::Code::MediaChangeAbort:
+ case ProvideMessage::Code::MediaChangeSkip:
+ case ProvideMessage::Code::InternalError: {
+ const auto &e = validateErrorMsg(msg);
+ if ( !e )
+ return e;
+ break;
+ }
+ case ProvideMessage::Code::Provide: {
+ std::exception_ptr error;
+ DEF_REQ_FIELD(url);
+ msg.forEachVal( [&]( const auto &name, const ProvideMessage::FieldVal &val ){
+ REQ_FIELD_CHECK ( Provide, url, std::string )
+ OR_OPT_FIELD_CHECK ( Provide, filename, std::string )
+ OR_OPT_FIELD_CHECK ( Provide, delta_file, std::string )
+ OR_OPT_FIELD_CHECK ( Provide, expected_filesize, int64_t )
+ OR_OPT_FIELD_CHECK ( Provide, check_existance_only, bool )
+ OR_OPT_FIELD_CHECK ( Provide, metalink_enabled, bool )
+ return true;
+ });
+ FAIL_IF_NOT_SEEN_REQ_FIELD( Provide, url );
+ FAIL_IF_ERROR();
+ break;
+ }
+ case ProvideMessage::Code::Cancel:
+ /* No Fields */
+ break;
+
+ case ProvideMessage::Code::Attach: {
+ std::exception_ptr error;
+
+ DEF_REQ_FIELD(url);
+ DEF_REQ_FIELD(attach_id);
+ DEF_REQ_FIELD(label);
+
+ // not really required, but this way we can check if all false or all true
+ DEF_REQ_FIELD(verify_type);
+ DEF_REQ_FIELD(verify_data);
+ DEF_REQ_FIELD(media_nr);
+
+ msg.forEachVal( [&]( const auto &name, const ProvideMessage::FieldVal &val ){
+ REQ_FIELD_CHECK ( Attach, url , std::string )
+ OR_REQ_FIELD_CHECK ( Attach, attach_id , std::string )
+ OR_REQ_FIELD_CHECK ( Attach, label , std::string )
+ OR_REQ_FIELD_CHECK ( Attach, verify_type, std::string )
+ OR_REQ_FIELD_CHECK ( Attach, verify_data, std::string )
+ OR_REQ_FIELD_CHECK ( Attach, media_nr , int32_t )
+ OR_OPT_FIELD_CHECK ( Attach, device , std::string )
+ return true;
+ });
+ FAIL_IF_NOT_SEEN_REQ_FIELD( Provide, url );
+ FAIL_IF_NOT_SEEN_REQ_FIELD( Provide, label );
+ FAIL_IF_NOT_SEEN_REQ_FIELD( Provide, attach_id );
+ if ( ! ( ( has_verify_data == has_verify_type ) && ( has_verify_type == has_media_nr ) ) )
+ return expected<void>::error( ZYPP_EXCPT_PTR ( InvalidMessageReceivedException("Error in Attach message, one of the following fields is not set or invalid: ( verify_type, verify_data, media_nr ). Either none or all need to be set. ")) );
+ FAIL_IF_ERROR();
+ break;
+ }
+ case ProvideMessage::Code::Detach: {
+ std::exception_ptr error;
+ DEF_REQ_FIELD(url);
+ msg.forEachVal( [&]( const auto &name, const ProvideMessage::FieldVal &val ){
+ REQ_FIELD_CHECK ( Detach, url, std::string )
+ return true;
+ });
+ FAIL_IF_NOT_SEEN_REQ_FIELD( Detach, url );
+ FAIL_IF_ERROR();
+ break;
+ }
+ case ProvideMessage::Code::AuthDataRequest: {
+ std::exception_ptr error;
+ DEF_REQ_FIELD(effective_url);
+ msg.forEachVal( [&]( const auto &name, const ProvideMessage::FieldVal &val ){
+ REQ_FIELD_CHECK ( AuthDataRequest, effective_url, std::string )
+ OR_OPT_FIELD_CHECK ( AuthDataRequest, last_auth_timestamp, int64_t )
+ OR_OPT_FIELD_CHECK ( AuthDataRequest, username, std::string )
+ OR_OPT_FIELD_CHECK ( AuthDataRequest, authHint, std::string )
+ return true;
+ });
+ FAIL_IF_NOT_SEEN_REQ_FIELD( AuthDataRequest, effective_url );
+ FAIL_IF_ERROR();
+ break;
+ }
+ case ProvideMessage::Code::MediaChangeRequest: {
+ std::exception_ptr error;
+ DEF_REQ_FIELD(label);
+ DEF_REQ_FIELD(media_nr);
+ DEF_REQ_FIELD(device);
+ msg.forEachVal( [&]( const auto &name, const ProvideMessage::FieldVal &val ){
+ REQ_FIELD_CHECK ( MediaChangeRequest, label, std::string )
+ OR_REQ_FIELD_CHECK ( MediaChangeRequest, media_nr, int32_t )
+ OR_REQ_FIELD_CHECK ( MediaChangeRequest, device, std::string )
+ OR_OPT_FIELD_CHECK ( MediaChangeRequest, desc, std::string )
+ return true;
+ });
+ FAIL_IF_NOT_SEEN_REQ_FIELD( MediaChangeRequest, label );
+ FAIL_IF_NOT_SEEN_REQ_FIELD( MediaChangeRequest, media_nr );
+ FAIL_IF_NOT_SEEN_REQ_FIELD( MediaChangeRequest, device );
+ FAIL_IF_ERROR();
+ break;
+ }
+ default: {
+ // all error messages have the same format
+ if ( c >= ProvideMessage::Code::FirstClientErrCode && c <= ProvideMessage::Code::LastSrvErrCode ) {
+ const auto &e = validateErrorMsg(msg);
+ if ( !e )
+ return e;
+ }
+ break;
+ }
+ }
+ return expected<void>::success();
+ }
+
+ ProvideMessage::ProvideMessage()
+ : _impl ( new zypp::proto::ProvideMessage )
+ { }
+
+ expected<zyppng::ProvideMessage> ProvideMessage::create(const RpcMessage &message)
+ {
+ ProvideMessage msg;
+ const auto &res = RpcMessageStream::parseMessageInto<zypp::proto::ProvideMessage>( message, *msg._impl );
+ if ( res ) {
+ const auto &valid = validateMessage(msg);
+ if ( !valid ) {
+ ERR << "Invalid message for ID: " << msg._impl->request_id() << std::endl;;
+ return zyppng::expected<zyppng::ProvideMessage>::error( valid.error() );
+ }
+
+ return zyppng::expected<zyppng::ProvideMessage>::success( std::move(msg) );
+ }
+ ERR << "Failed to parse message" << std::endl;;
+ return zyppng::expected<zyppng::ProvideMessage>::error( res.error() );
+ }
+
+ expected<ProvideMessage> ProvideMessage::create( const zypp::proto::ProvideMessage &message )
+ {
+ ProvideMessage msg;
+ *msg._impl = std::move(message);
+ const auto &valid = validateMessage(msg);
+ if ( !valid ) {
+ ERR << "Invalid message for ID: " << msg._impl->request_id() << std::endl;;
+ return zyppng::expected<zyppng::ProvideMessage>::error( valid.error() );
+ }
+
+ return zyppng::expected<zyppng::ProvideMessage>::success( std::move(msg) );
+ }
+
+ ProvideMessage ProvideMessage::createProvideStarted( const uint32_t reqId, const zypp::Url &url, const std::optional<std::string> &localFilename, const std::optional<std::string> &stagingFilename )
+ {
+ ProvideMessage msg;
+ msg.setCode ( ProvideMessage::Code::ProvideStarted );
+ msg.setRequestId ( reqId );
+ msg.setValue ( ProvideStartedMsgFields::Url, url.asCompleteString() );
+ if ( localFilename )
+ msg.setValue ( ProvideStartedMsgFields::LocalFilename, *localFilename );
+ if ( stagingFilename )
+ msg.setValue ( ProvideStartedMsgFields::StagingFilename, *stagingFilename );
+
+ return msg;
+ }
+
+ ProvideMessage ProvideMessage::createProvideFinished( const uint32_t reqId, const std::string &localFilename, bool cacheHit )
+ {
+ ProvideMessage msg;
+ msg.setCode ( ProvideMessage::Code::ProvideFinished );
+ msg.setRequestId ( reqId );
+ msg.setValue ( ProvideFinishedMsgFields::LocalFilename, localFilename );
+ msg.setValue ( ProvideFinishedMsgFields::CacheHit, cacheHit );
+
+ return msg;
+ }
+
+ ProvideMessage ProvideMessage::createAttachFinished( const uint32_t reqId )
+ {
+ ProvideMessage msg;
+ msg.setCode ( ProvideMessage::Code::AttachFinished );
+ msg.setRequestId ( reqId );
+
+ return msg;
+ }
+
+ ProvideMessage ProvideMessage::createDetachFinished(const uint32_t reqId)
+ {
+ ProvideMessage msg;
+ msg.setCode ( ProvideMessage::Code::DetachFinished );
+ msg.setRequestId ( reqId );
+
+ return msg;
+ }
+
+ ProvideMessage ProvideMessage::createAuthInfo( const uint32_t reqId, const std::string &user, const std::string &pw, int64_t timestamp, const std::map<std::string, std::string> &extraValues )
+ {
+ ProvideMessage msg;
+ msg.setCode ( ProvideMessage::Code::AuthInfo );
+ msg.setRequestId ( reqId );
+ msg.setValue ( AuthInfoMsgFields::Username, user );
+ msg.setValue ( AuthInfoMsgFields::Password, pw );
+ msg.setValue ( AuthInfoMsgFields::AuthTimestamp, timestamp );
+ for ( auto i : extraValues ) {
+ msg.setValue( i.first, i.second );
+ }
+ return msg;
+ }
+
+ ProvideMessage ProvideMessage::createMediaChanged( const uint32_t reqId )
+ {
+ ProvideMessage msg;
+ msg.setCode ( ProvideMessage::Code::MediaChanged );
+ msg.setRequestId ( reqId );
+
+ return msg;
+ }
+
+ ProvideMessage ProvideMessage::createRedirect( const uint32_t reqId, const zypp::Url &newUrl )
+ {
+ ProvideMessage msg;
+ msg.setCode ( ProvideMessage::Code::Redirect );
+ msg.setRequestId ( reqId );
+ msg.setValue ( RedirectMsgFields::NewUrl, newUrl.asCompleteString() );
+
+ return msg;
+ }
+
+ ProvideMessage ProvideMessage::createMetalinkRedir( const uint32_t reqId, const std::vector<zypp::Url> &newUrls )
+ {
+ ProvideMessage msg;
+ msg.setCode ( ProvideMessage::Code::Metalink );
+ msg.setRequestId ( reqId );
+ for( const auto &val : newUrls )
+ msg.addValue( MetalinkRedirectMsgFields::NewUrl, val.asCompleteString() );
+
+ return msg;
+ }
+
+ ProvideMessage ProvideMessage::createErrorResponse( const uint32_t reqId, const uint code, const std::string &reason, bool transient )
+ {
+ ProvideMessage msg;
+ if ( code < Code::FirstClientErrCode || code > Code::LastSrvErrCode )
+ ZYPP_THROW(std::out_of_range("code must be between 400 and 599"));
+ msg.setCode ( code );
+ msg.setRequestId ( reqId );
+ msg.setValue ( ErrMsgFields::Reason, reason );
+ msg.setValue ( ErrMsgFields::Transient, transient );
+ return msg;
+ }
+
+ ProvideMessage ProvideMessage::createProvide( const uint32_t reqId, const zypp::Url &url, const std::optional<std::string> &filename, const std::optional<std::string> &deltaFile, const std::optional<int64_t> &expFilesize, bool checkExistOnly )
+ {
+ ProvideMessage msg;
+ msg.setCode ( ProvideMessage::Code::Provide );
+ msg.setRequestId ( reqId );
+ msg.setValue ( ProvideMsgFields::Url, url.asCompleteString() );
+
+ if ( filename )
+ msg.setValue ( ProvideMsgFields::Filename, *filename );
+ if ( deltaFile )
+ msg.setValue ( ProvideMsgFields::DeltaFile, *deltaFile );
+ if ( expFilesize )
+ msg.setValue ( ProvideMsgFields::ExpectedFilesize, *expFilesize );
+ msg.setValue ( ProvideMsgFields::CheckExistOnly, checkExistOnly );
+
+ return msg;
+ }
+
+ ProvideMessage ProvideMessage::createCancel( const uint32_t reqId )
+ {
+ ProvideMessage msg;
+ msg.setCode ( ProvideMessage::Code::Cancel );
+ msg.setRequestId ( reqId );
+
+ return msg;
+ }
+
+ ProvideMessage ProvideMessage::createAttach(const uint32_t reqId, const zypp::Url &url, const std::string attachId, const std::string &label, const std::optional<std::string> &verifyType, const std::optional<std::string> &verifyData, const std::optional<int32_t> &mediaNr )
+ {
+ ProvideMessage msg;
+ msg.setCode ( ProvideMessage::Code::Attach );
+ msg.setRequestId ( reqId );
+ msg.setValue ( AttachMsgFields::Url, url.asCompleteString() );
+ msg.setValue ( AttachMsgFields::AttachId, attachId );
+ msg.setValue ( AttachMsgFields::Label, label );
+
+ if ( verifyType.has_value() && verifyData.has_value() && mediaNr.has_value() ) {
+ msg.setValue ( AttachMsgFields::VerifyType, *verifyType );
+ msg.setValue ( AttachMsgFields::VerifyData, *verifyData );
+ msg.setValue ( AttachMsgFields::MediaNr, *mediaNr );
+ } else {
+ if ( !( ( verifyType.has_value() == verifyData.has_value() ) && ( verifyData.has_value() == mediaNr.has_value() ) ) )
+ WAR << "Attach message requires verifyType, verifyData and mediaNr either set together or not set at all." << std::endl;
+ }
+
+ return msg;
+ }
+
+ ProvideMessage ProvideMessage::createDetach( const uint32_t reqId, const zypp::Url &attachUrl )
+ {
+ ProvideMessage msg;
+ msg.setCode ( ProvideMessage::Code::Detach );
+ msg.setRequestId ( reqId );
+ msg.setValue ( DetachMsgFields::Url, attachUrl.asCompleteString() );
+
+ return msg;
+ }
+
+ ProvideMessage ProvideMessage::createAuthDataRequest( const uint32_t reqId, const zypp::Url &effectiveUrl, const std::string &lastTriedUser, const std::optional<int64_t> &lastAuthTimestamp, const std::map<std::string, std::string> &extraValues )
+ {
+ ProvideMessage msg;
+ msg.setCode ( ProvideMessage::Code::AuthDataRequest );
+ msg.setRequestId ( reqId );
+ msg.setValue ( AuthDataRequestMsgFields::EffectiveUrl, effectiveUrl.asCompleteString() );
+ if ( lastTriedUser.size() )
+ msg.setValue( AuthDataRequestMsgFields::LastUser, lastTriedUser );
+ if ( lastAuthTimestamp )
+ msg.setValue ( AuthDataRequestMsgFields::LastAuthTimestamp, *lastAuthTimestamp );
+
+ return msg;
+ }
+
+ ProvideMessage ProvideMessage::createMediaChangeRequest( const uint32_t reqId, const std::string &label, int32_t mediaNr, const std::vector<std::string> &devices, const std::optional<std::string> &desc )
+ {
+ ProvideMessage msg;
+ msg.setCode ( ProvideMessage::Code::MediaChangeRequest );
+ msg.setRequestId ( reqId );
+ msg.setValue ( MediaChangeRequestMsgFields::Label, label );
+ msg.setValue ( MediaChangeRequestMsgFields::MediaNr, mediaNr );
+ for ( const auto &device : devices )
+ msg.addValue ( MediaChangeRequestMsgFields::Device, device );
+ if ( desc )
+ msg.setValue ( MediaChangeRequestMsgFields::Desc, *desc );
+
+ return msg;
+ }
+
+ uint ProvideMessage::requestId() const
+ {
+ return _impl->request_id();
+ }
+
+ void ProvideMessage::setRequestId(const uint id)
+ {
+ _impl->set_request_id( id );
+ }
+
+ uint ProvideMessage::code() const
+ {
+ return _impl->message_code();
+ }
+
+ void ProvideMessage::setCode( const uint newCode )
+ {
+ _impl->set_message_code ( newCode );
+ }
+
+ std::vector<ProvideMessage::FieldVal> ProvideMessage::values( const std::string_view &str ) const
+ {
+ std::vector<ProvideMessage::FieldVal> values;
+ const auto &fields = _impl->fields();
+ for ( const auto &field : fields ) {
+ if ( field.key() != str )
+ continue;
+ values.push_back( fieldValFromProto(field) );
+ }
+ return values;
+ }
+
+ std::vector<ProvideMessage::FieldVal> ProvideMessage::values( const std::string &str ) const
+ {
+ return values( std::string_view(str));
+ }
+
+ ProvideMessage::FieldVal ProvideMessage::value( const std::string_view &str, const FieldVal &defaultVal ) const
+ {
+ const auto &fields = _impl->fields();
+ auto i = std::find_if( fields.rbegin(), fields.rend(), [&str]( const auto &val ){ return val.key() == str; } );
+ if ( i == fields.rend() )
+ return defaultVal;
+ return fieldValFromProto(*i);
+ }
+
+
+ HeaderValueMap ProvideMessage::headers() const
+ {
+ HeaderValueMap res;
+ auto &fields = _impl->fields();
+ for ( const auto &val : fields ) {
+ res.add( val.key() ,fieldValFromProto(val) );
+ }
+ return res;
+ }
+
+ ProvideMessage::FieldVal ProvideMessage::value( const std::string &str, const FieldVal &defaultVal ) const
+ {
+ return value( std::string_view(str), defaultVal );
+ }
+
+ void ProvideMessage::setValue( const std::string &name, const FieldVal &value )
+ {
+ setValue( std::string_view(name), value );
+ }
+
+ void ProvideMessage::setValue( const std::string_view &name, const FieldVal &value )
+ {
+ auto &fields = *_impl->mutable_fields();
+ auto i = std::find_if( fields.rbegin(), fields.rend(), [&name]( const auto &val ){ return val.key() == name; } );
+ if ( i == fields.rend() ) {
+ auto &newVal = *_impl->add_fields();
+ newVal.set_key( name.data() );
+ fieldValToProto( value, newVal );
+ } else
+ fieldValToProto( value, *i );
+ }
+
+ void ProvideMessage::addValue( const std::string &name, const FieldVal &value )
+ {
+ return addValue( std::string_view(name), value );
+ }
+
+ void ProvideMessage::addValue( const std::string_view &name, const FieldVal &value )
+ {
+ auto &newVal = *_impl->add_fields();
+ newVal.set_key( name.data() );
+ fieldValToProto( value, newVal );
+ }
+
+ void ProvideMessage::forEachVal( const std::function<bool (const std::string &, const FieldVal &)> &cb ) const
+ {
+ auto &fields = _impl->fields();
+ for ( const auto &val : fields ) {
+ if ( !cb( val.key(), fieldValFromProto(val) ) ) {
+ return;
+ }
+ }
+ }
+
+ zypp::proto::ProvideMessage &ProvideMessage::impl()
+ {
+ return *_impl.get();
+ }
+
+ const zypp::proto::ProvideMessage &ProvideMessage::impl() const
+ {
+ return *_impl.get();
+ }
+
+}
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+
+#include "private/providequeue_p.h"
+#include "private/provideitem_p.h"
+#include "private/provide_p.h"
+#include "private/providemessage_p.h"
+#include "private/providedbg_p.h"
+
+#include <zypp-core/fs/PathInfo.h>
+#include <zypp-core/zyppng/rpc/MessageStream>
+#include <zypp-core/base/StringV.h>
+#include <zypp-media/ng/provide-configvars.h>
+#include <zypp-media/MediaException>
+#include <zypp-media/auth/CredentialManager>
+
+#include <zypp/APIConfig.h>
+#include <variant>
+#include <bitset>
+
+namespace zyppng {
+
+ bool ProvideQueue::Item::isAttachRequest() const
+ {
+ return ( _request->code () == ProvideMessage::Code::Attach );
+ }
+
+ bool ProvideQueue::Item::isFileRequest() const
+ {
+ return ( _request->code () == ProvideMessage::Code::Provide );
+ }
+
+ bool ProvideQueue::Item::isDetachRequest() const
+ {
+ return ( _request->code () == ProvideMessage::Code::Detach );
+ }
+
+ ProvideQueue::ProvideQueue(ProvidePrivate &parent) : _parent(parent)
+ { }
+
+ ProvideQueue::~ProvideQueue()
+ {
+ if ( zyppng::provideDebugEnabled() ) {
+ if ( this->_activeItems.size() || this->_waitQueue.size() ) {
+ DBG << "Queue shutdown with Items still running" << std::endl;
+ }
+ }
+ immediateShutdown(std::make_exception_ptr(zypp::media::MediaException("Cancelled by queue shutdown")));
+ }
+
+ bool ProvideQueue::startup(const std::string &workerScheme, const zypp::filesystem::Pathname &workDir, const std::string &hostname ) {
+
+ if ( _workerProc ) {
+ ERR << "Queue Worker was already initialized" << std::endl;
+ return true;
+ }
+
+ _myHostname = hostname;
+
+ const auto &pN = _parent.workerPath() / ( "zypp-media-"+workerScheme ) ;
+ MIL << "Trying to start " << pN << std::endl;
+ const auto &pi = zypp::PathInfo( pN );
+ if ( !pi.isExist() ) {
+ ERR << "Failed to find worker for " << workerScheme << std::endl;
+ return false;
+ }
+
+ if ( !pi.userMayX() ) {
+ ERR << "Failed to start worker for " << workerScheme << " binary " << pi.asString() << " is not executable." << std::endl;
+ return false;
+ }
+
+ if ( zypp::filesystem::assert_dir( workDir ) != 0 ) {
+ ERR << "Failed to assert working directory '" << workDir << "' for worker " << workerScheme << std::endl;
+ return false;
+ }
+
+ _currentExe = pN;
+ _workerProc = Process::create();
+ _workerProc->setWorkingDirectory ( workDir );
+ _messageStream = RpcMessageStream::create( _workerProc );
+ return doStartup();
+ }
+
+
+ void ProvideQueue::enqueue( ProvideRequestRef request )
+ {
+ Item i;
+ i._request = request;
+ i._request->provideMessage().setRequestId( nextRequestId() );
+ request->setCurrentQueue( shared_this<ProvideQueue>() );
+ _waitQueue.push_back( std::move(i) );
+ if ( _parent.isRunning() )
+ scheduleNext();
+ }
+
+ void ProvideQueue::cancel( ProvideRequest *item , std::exception_ptr error )
+ {
+ const auto &isSameItem = [item]( const Item &i ){
+ if ( i.isDetachRequest () )
+ return false;
+ return i._request.get() == item;
+ };
+
+ if ( !item )
+ return;
+
+ if ( item->code() != ProvideMessage::Code::Attach
+ && item->code() != ProvideMessage::Code::Provide ) {
+ ERR << "Can not cancel a " << item->code() << " request!" << std::endl;
+ return;
+ }
+
+ if ( auto i = std::find_if( _waitQueue.begin(), _waitQueue.end(), isSameItem ); i != _waitQueue.end() ) {
+ auto &reqRef = i->_request;
+ reqRef->setCurrentQueue(nullptr);
+ if ( reqRef->owner() )
+ reqRef->owner()->finishReq( this, reqRef, error );
+ _waitQueue.erase(i);
+ _parent.schedule( ProvidePrivate::FinishReq ); // let the parent scheduler run since we have a open spot now
+ } else if ( auto i = std::find_if( _activeItems.begin(), _activeItems.end(), isSameItem ); i != _activeItems.end() ) {
+ cancelActiveItem(i, error);
+ }
+ }
+
+ std::list<ProvideQueue::Item>::iterator ProvideQueue::dequeueActive( std::list<Item>::iterator it )
+ {
+ if ( it == _activeItems.end() )
+ return it;
+
+ if ( it->_request )
+ it->_request->setCurrentQueue( nullptr );
+
+ auto i = _activeItems.erase(it);
+ _parent.schedule ( ProvidePrivate::FinishReq ); // Trigger the scheduler
+ scheduleNext (); // keep the active items full
+ return i;
+ }
+
+ void ProvideQueue::fatalWorkerError( const std::exception_ptr &reason )
+ {
+ immediateShutdown( reason ? reason : std::make_exception_ptr( zypp::media::MediaException("Fatal worker error")) );
+ }
+
+ void ProvideQueue::immediateShutdown( const std::exception_ptr &reason )
+ {
+ _queueShuttingDown = true;
+
+ while ( _waitQueue.size() ) {
+ auto &item = _waitQueue.front();
+ auto &reqRef = item._request;
+ if ( reqRef && reqRef->owner() && !item.isDetachRequest() )
+ reqRef->owner()->finishReq( this, reqRef, reason );
+ _waitQueue.pop_front();
+ }
+
+ for ( auto i = _activeItems.begin(); i != _activeItems.end(); ) {
+ auto &reqRef = i->_request;
+ if ( reqRef && reqRef->owner() && !i->isDetachRequest() ) {
+ i = cancelActiveItem(i, reason );
+ } else {
+ i++;
+ }
+ }
+
+ if ( _workerProc && _workerProc->isRunning() ) {
+ _workerProc->flush();
+ _workerProc->closeWriteChannel();
+ _workerProc->waitForExit();
+ readAllStderr();
+ }
+ }
+
+ std::list< ProvideQueue::Item >::iterator ProvideQueue::cancelActiveItem( std::list< Item >::iterator i , const std::__exception_ptr::exception_ptr &error )
+ {
+ auto &reqRef = i->_request;
+
+ // already in cancelling process or finished
+ if ( i->_state == Item::Cancelling || i->_state == Item::Finished )
+ return (++i);
+
+ // not possible but lets be safe
+ if ( i->_state == Item::Pending ) {
+ reqRef->setCurrentQueue(nullptr);
+ if ( reqRef->owner() )
+ reqRef->owner()->finishReq( this, reqRef, error );
+ return dequeueActive(i);
+ }
+
+ // we first need to cancel the item
+ auto c = ProvideMessage::createCancel ( i->_request->provideMessage().requestId() );
+ if( !_messageStream->sendMessage(c.impl()) )
+ ERR << "Failed to send cancel message to worker" << std::endl;
+
+ i->_state = Item::Cancelling;
+ reqRef->setCurrentQueue(nullptr);
+ if ( reqRef->owner() )
+ reqRef->owner()->finishReq( this, reqRef, error );
+ reqRef.reset();
+ return (++i);
+ }
+
+ void ProvideQueue::scheduleNext()
+ {
+ if ( _queueShuttingDown )
+ return;
+
+ while ( _waitQueue.size() && canScheduleMore() ) {
+ auto item = std::move( _waitQueue.front() );
+ _waitQueue.pop_front();
+
+ auto &reqRef = item._request;
+ if ( !reqRef->activeUrl() ) {
+ ERR << "Item without active URL enqueued, this is a BUG." << std::endl;
+ if ( reqRef->owner() )
+ reqRef->owner()->finishReq( this, reqRef, ZYPP_EXCPT_PTR (zypp::media::MediaException("Item needs a activeURL to be queued.")) );
+ continue;
+ }
+
+ if ( !_messageStream->sendMessage( reqRef->provideMessage().impl() ) ) {
+ ERR << "Failed to send message to worker process." << std::endl;
+ fatalWorkerError( ZYPP_EXCPT_PTR( zypp::media::MediaException("Failed to communicate with worker process.") ) );
+ return;
+ }
+
+ item._state = Item::Queued;
+ _activeItems.push_back( std::move(item) );
+ _idleSince.reset();
+ }
+
+ if ( _waitQueue.empty() && _activeItems.empty() ) {
+ _parent.schedule( ProvidePrivate::QueueIdle );
+ if ( !_idleSince )
+ _idleSince = std::chrono::steady_clock::now();
+ _sigIdle.emit();
+ }
+ }
+
+ bool ProvideQueue::canScheduleMore() const
+ {
+ return ( _activeItems.size() == 0 || ( _capabilities.cfg_flags () & zypp::proto::Capabilities::Pipeline ) == zypp::proto::Capabilities::Pipeline );
+ }
+
+ bool ProvideQueue::isIdle() const
+ {
+ return ( empty() );
+ }
+
+ std::optional<ProvideQueue::TimePoint> ProvideQueue::idleSince() const
+ {
+ return _idleSince;
+ }
+
+ bool ProvideQueue::empty() const
+ {
+ return ( _activeItems.empty() && _waitQueue.empty() );
+ }
+
+ uint ProvideQueue::requestCount() const
+ {
+ return _activeItems.size() + _waitQueue.size();
+ }
+
+ uint ProvideQueue::activeRequests() const
+ {
+ return _activeItems.size();
+ }
+
+ zypp::ByteCount ProvideQueue::expectedProvideSize() const
+ {
+ zypp::ByteCount dlSize;
+ for ( const auto &i : _waitQueue ) {
+ if ( i.isDetachRequest () )
+ continue;
+
+ auto &reqRef = i._request;
+ if ( reqRef->code() != ProvideMessage::Code::Provide )
+ continue;
+ dlSize += reqRef->provideMessage().value( ProvideMsgFields::ExpectedFilesize, int64_t(0) ).asInt64();
+ }
+ for ( const auto &i : _activeItems ) {
+ if ( i.isDetachRequest () )
+ continue;
+ auto &reqRef = i._request;
+ if ( reqRef->code() != ProvideMessage::Code::Provide )
+ continue;
+ dlSize += reqRef->provideMessage().value( ProvideMsgFields::ExpectedFilesize, int64_t(0) ).asInt64();
+ }
+ return dlSize;
+ }
+
+ const std::string &ProvideQueue::hostname() const
+ {
+ return _myHostname;
+ }
+
+ const ProvideQueue::Config &ProvideQueue::workerConfig() const
+ {
+ return _capabilities;
+ }
+
+ SignalProxy<void ()> ProvideQueue::sigIdle()
+ {
+ return _sigIdle;
+ }
+
+ bool ProvideQueue::doStartup()
+ {
+ if ( _currentExe.empty() )
+ return false;
+
+ //const char *argv[] = { "gdbserver", ":10000", _currentExe.c_str(), nullptr };
+ const char *argv[] = { _currentExe.c_str(), nullptr };
+ if ( !_workerProc->start( argv) ) {
+ ERR << "Failed to execute worker" << std::endl;
+
+ _messageStream.reset ();
+ _workerProc.reset ();
+
+ return false;
+ }
+
+ // make sure the default read channel is StdOut so RpcMessageStream gets all the rpc messages
+ _workerProc->setReadChannel ( Process::StdOut );
+
+ // we are ready to send the data
+
+ zypp::proto::Configuration conf;
+ // @TODO actually write real config data :D
+ conf.mutable_values ()->insert ( { AGENT_STRING_CONF.data (), "ZYpp " LIBZYPP_VERSION_STRING } );
+ conf.mutable_values ()->insert ( { ATTACH_POINT.data (), _workerProc->workingDirectory().asString() } );
+ conf.mutable_values ()->insert ( { PROVIDER_ROOT.data (), _parent.z_func()->providerWorkdir().asString() } );
+
+ const auto &cleanupOnErr = [&](){
+ readAllStderr();
+ _messageStream.reset ();
+ _workerProc->close();
+ _workerProc.reset();
+ return false;
+ };
+
+ if ( !_messageStream->sendMessage( conf ) ) {
+ ERR << "Failed to send initial message to queue worker" << std::endl;
+ return cleanupOnErr();
+ }
+
+ // wait for the data to be written
+ _workerProc->flush ();
+
+ // wait until we receive a message
+ const auto &caps = _messageStream->nextMessageWait();
+ if ( !caps || caps->messagetypename() != rpc::messageTypeName<zypp::proto::Capabilities>() ) {
+ ERR << "Worker did not sent a capabilities message, aborting" << std::endl;
+ return cleanupOnErr();
+ }
+
+ {
+ auto p = _messageStream->parseMessage<zypp::proto::Capabilities>( *caps );
+ if ( !p )
+ return cleanupOnErr();
+
+ _capabilities = std::move(*p);
+ }
+
+ DBG << "Received config for worker: " << this->_currentExe.asString() << " Worker Type: " << this->_capabilities.worker_type() << " Flags: " << std::bitset<32>( _capabilities.cfg_flags() ).to_string() << std::endl;
+
+ // now we can set up signals and start processing messages
+ connect( *_messageStream, &RpcMessageStream::sigMessageReceived, *this, &ProvideQueue::processMessage );
+ connect( *_workerProc, &IODevice::sigChannelReadyRead, *this, &ProvideQueue::processReadyRead );
+ connect( *_workerProc, &Process::sigFinished, *this, &ProvideQueue::procFinished );
+
+ // make sure we do not miss messages
+ processMessage();
+ return true;
+ }
+
+ void ProvideQueue::processMessage() {
+
+ const auto &getRequest = [&]( const auto &exp ) -> decltype(_activeItems)::iterator {
+ if ( !exp ) {
+ ERR << "Ignoring invalid request!" << std::endl;
+ return _activeItems.end();
+ }
+
+ auto i = std::find_if( _activeItems.begin(), _activeItems.end(), [&]( const auto &elem ) {
+ return exp->requestId() == elem._request->provideMessage().requestId();
+ });
+
+ if ( i == _activeItems.end() ) {
+ ERR << "Ignoring unknown request ID: " << exp->requestId() << std::endl;
+ return _activeItems.end();
+ }
+
+ return i;
+ };
+
+ const auto &sendErrorToWorker = [&]( const uint32_t reqId, const uint code, const std::string &reason, bool transient = false ) {
+ auto r = ProvideMessage::createErrorResponse ( reqId, code, reason, transient );
+ if ( !_messageStream->sendMessage( r.impl() ) ) {
+ ERR << "Failed to send Error message to worker process." << std::endl;
+ fatalWorkerError( ZYPP_EXCPT_PTR( zypp::media::MediaException("Failed to communicate with worker process.") ) );
+ return false;
+ }
+ return true;
+ };
+
+ const bool doesDownload = this->_capabilities.worker_type() == Config::Downloading;
+ const bool fileNeedsCleanup = doesDownload || ( _capabilities.worker_type() == Config::CPUBound && _capabilities.cfg_flags() & Config::FileArtifacts );
+
+ while ( auto msg = _messageStream->nextMessage () ) {
+
+ if ( msg->messagetypename() == rpc::messageTypeName<zypp::proto::ProvideMessage>() ) {
+
+ const auto &provMsg = ProvideMessage::create(*msg);
+ if ( !provMsg ) {
+ fatalWorkerError( provMsg.error() );
+ return;
+ }
+
+ const auto &reqIter = getRequest( provMsg );
+ if ( reqIter == _activeItems.end() ) {
+ if ( provMsg->code() == ProvideMessage::Code::ProvideFinished && fileNeedsCleanup ) {
+ const auto locFName = provMsg->value( ProvideFinishedMsgFields::LocalFilename ).asString();
+ if ( !_parent.isInCache(locFName) ) {
+ MIL << "Received a ProvideFinished message for a non existant request. Since this worker reported to create file artifacts, the file is cleaned up." << std::endl;
+ zypp::filesystem::unlink( locFName );
+ }
+ }
+ continue;
+ }
+
+ auto &req = *reqIter;
+ auto &reqRef =req._request;
+
+ const auto code = provMsg->code();
+
+ if ( code >= ProvideMessage::Code::FirstInformalCode && code <= ProvideMessage::Code::LastInformalCode ) {
+
+ // send the message to the item but don't dequeue
+ if ( reqRef && reqRef->owner() )
+ reqRef->owner()->informalMessage ( *this, reqRef, *provMsg );
+ continue;
+
+ } else if ( code >= ProvideMessage::Code::FirstSuccessCode && code <= ProvideMessage::Code::LastSuccessCode ) {
+
+ if ( req._state == Item::Cancelling ) {
+ req._state = Item::Finished;
+ dequeueActive( reqIter );
+ continue;
+ }
+
+ if ( code == ProvideMessage::Code::ProvideFinished ) {
+
+ // we are going to register the file to the cache if this is a downloading worker, so it can not leak
+ // no matter if the item does the correct dance or not, this code is duplicated by all ProvideItems that receive ProvideFinished
+ // results that require file cleanups.
+ // we keep the ref around until after sending the result to the item. At that point it should take a reference
+ std::optional<zypp::ManagedFile> dataRef;
+
+ if ( !reqIter->isFileRequest() ) {
+ ERR << "Invalid message for request ID: " << reqIter->_request->provideMessage().requestId() << std::endl;
+ fatalWorkerError();
+ return;
+ }
+
+ // when a worker is downloading we keep a internal book of cache files
+ if ( doesDownload ) {
+ const auto locFName = provMsg->value( ProvideFinishedMsgFields::LocalFilename ).asString();
+ if ( provMsg->value( ProvideFinishedMsgFields::CacheHit, false ).asBool()) {
+ dataRef = _parent.addToFileCache ( locFName );
+ if ( !dataRef ) {
+ MIL << "CACHE MISS, file " << locFName << " was already removed, queueing again" << std::endl;
+ if ( reqRef->owner() )
+ reqRef->owner()->cacheMiss( reqRef );
+ reqRef->provideMessage().setRequestId( InvalidId );
+ req._state = Item::Pending;
+ _waitQueue.push_front( req );
+ dequeueActive( reqIter );
+ continue;
+ }
+ } else {
+ dataRef = _parent.addToFileCache ( locFName );
+
+ // unlikely this can happen but better be safe than sorry
+ if ( !dataRef ) {
+ req._state = Item::Finished;
+ reqRef->setCurrentQueue(nullptr);
+ auto resp = ProvideMessage::createErrorResponse ( provMsg->requestId(), ProvideMessage::Code::InternalError, "File vanished between downloading and adding it to cache." );
+ if ( reqRef->owner() )
+ reqRef->owner()->finishReq( *this, reqRef, resp );
+ dequeueActive( reqIter );
+ continue;
+ }
+ }
+ }
+ }
+
+ // send the message to the item and dequeue
+ reqRef->setCurrentQueue(nullptr);
+ if ( reqRef->owner() )
+ reqRef->owner()->finishReq( *this, reqRef, *provMsg );
+ req._state = Item::Finished;
+ dequeueActive( reqIter );
+ continue;
+
+ } else if ( code >= ProvideMessage::Code::FirstClientErrCode && code <= ProvideMessage::Code::LastSrvErrCode ) {
+
+ if ( req._state == Item::Cancelling ) {
+ req._state = Item::Finished;
+ dequeueActive( reqIter );
+ continue;
+ }
+
+ // send the message to the item and dequeue
+ reqRef->setCurrentQueue(nullptr);
+
+ if ( reqRef->owner() )
+ reqRef->owner()->finishReq( *this, reqRef, *provMsg );
+
+ req._state = Item::Finished;
+ dequeueActive( reqIter );
+ continue;
+
+ } else if ( code >= ProvideMessage::Code::FirstRedirCode && code <= ProvideMessage::Code::LastRedirCode ) {
+
+ // redir is like a finished message, we can simply forgot about a cancelling request
+ if ( req._state == Item::Cancelling ) {
+ req._state = Item::Finished;
+ dequeueActive( reqIter );
+ continue;
+ }
+
+ // send the message to the item and dequeue
+ reqRef->setCurrentQueue(nullptr);
+ if ( reqRef->owner() )
+ reqRef->owner()->finishReq( *this, reqRef, *provMsg );
+ req._state = Item::Finished;
+ dequeueActive( reqIter );
+ continue;
+
+ } else if ( code >= ProvideMessage::Code::FirstControllerCode && code <= ProvideMessage::Code::LastControllerCode ) {
+
+ ERR << "Received Controller message from worker, this is a fatal error. Cancelling all requests!" << std::endl;
+ fatalWorkerError ( ZYPP_EXCPT_PTR( zypp::media::MediaException("Controller message received from worker.") ) );
+ return;
+
+ } else if ( code >= ProvideMessage::Code::FirstWorkerCode && code <= ProvideMessage::Code::LastWorkerCode ) {
+
+ if ( code == ProvideMessage::Code::AuthDataRequest ) {
+ if ( !reqIter->isFileRequest() && !reqIter->isAttachRequest() ) {
+ ERR << "Invalid message for request ID: " << reqRef->provideMessage().requestId() << std::endl;
+ fatalWorkerError();
+ return;
+ }
+
+ // if the file was cancelled we send a failure back
+ if( reqIter->_state == Item::Cancelling ) {
+ if ( !sendErrorToWorker( reqRef->provideMessage().requestId(), ProvideMessage::Code::NoAuthData, "Item was cancelled") )
+ return;
+ continue;
+ }
+
+ // we need a owner item to fetch the auth data for us
+ if ( !reqRef->owner() ) {
+ if ( !sendErrorToWorker( reqRef->provideMessage().requestId(), ProvideMessage::Code::NoAuthData, "Request has no owner" ) )
+ return;
+ continue;
+ }
+
+ if ( !reqRef->activeUrl() ) {
+ if ( !sendErrorToWorker( reqRef->provideMessage().requestId(), ProvideMessage::Code::NoAuthData, "Item has no active URL, this is a bug." ) )
+ return;
+ continue;
+ }
+
+ try {
+ zypp::Url u( provMsg->value( AuthDataRequestMsgFields::EffectiveUrl ).asString() );
+
+ std::map<std::string, std::string> extraVals;
+ provMsg->forEachVal( [&]( const std::string &name, const zyppng::ProvideMessage::FieldVal &val ) {
+
+ if ( name == AuthDataRequestMsgFields::EffectiveUrl
+ || name == AuthDataRequestMsgFields::LastAuthTimestamp )
+ return true;
+
+ if ( !val.isString() ) {
+ WAR << "Ignoring non string value for " << name << std::endl;
+ return true;
+ }
+
+ extraVals[name] = val.asString();
+ return true;
+ });
+
+ const auto &authOpt = reqRef->owner()->authenticationRequired( *this, reqRef, u, provMsg->value( AuthDataRequestMsgFields::LastAuthTimestamp ).asInt64(), extraVals );
+ if ( !authOpt ) {
+ if ( !sendErrorToWorker( reqRef->provideMessage().requestId(), ProvideMessage::Code::NoAuthData, "No auth given by user." ) )
+ return;
+ continue;
+ }
+
+ auto r = ProvideMessage::createAuthInfo ( reqRef->provideMessage().requestId(), authOpt->username(), authOpt->password(), authOpt->lastDatabaseUpdate(), authOpt->extraValues() );
+ if ( !_messageStream->sendMessage( r.impl() ) ) {
+ ERR << "Failed to send AuthorizationInfo to worker process." << std::endl;
+ fatalWorkerError( ZYPP_EXCPT_PTR( zypp::media::MediaException("Failed to communicate with worker process.") ) );
+ return;
+ }
+ continue;
+
+ } catch ( const zypp::Exception &e ) {
+ ZYPP_CAUGHT(e);
+ if ( !sendErrorToWorker( reqRef->provideMessage().requestId(), ProvideMessage::Code::NoAuthData, e.asString() ) )
+ return;
+ continue;
+ }
+
+ } else if ( code == ProvideMessage::Code::MediaChangeRequest ) {
+
+ if ( !reqIter->isAttachRequest() ) {
+ ERR << "Invalid message for request ID: " << reqIter->_request->provideMessage().requestId() << std::endl;
+ fatalWorkerError();
+ return;
+ }
+
+ // if the file was cancelled we send a failure back
+ if( reqIter->_state == Item::Cancelling ) {
+ if ( !sendErrorToWorker( reqRef->provideMessage().requestId(), ProvideMessage::Code::MediaChangeAbort, "Item was cancelled" ) )
+ return;
+ continue;
+ }
+
+ MIL << "Worker sent a MediaChangeRequest, asking the user to insert the correct medium" << std::endl;
+
+ //const std::string &label, const int32_t mediaNr, const std::vector<std::string> &devices, const std::optional<std::string> &desc
+ std::vector<std::string> freeDevs;
+ for ( const auto &val : provMsg->values( MediaChangeRequestMsgFields::Device) ) {
+ freeDevs.push_back( val.asString() );
+ }
+
+ std::optional<std::string> desc;
+ const auto &descVal = provMsg->value( MediaChangeRequestMsgFields::Desc );
+ if ( descVal.valid () && descVal.isString() )
+ desc = descVal.asString();
+
+ auto res = _parent._sigMediaChange.emit(
+ _parent.queueName(*this),
+ provMsg->value( MediaChangeRequestMsgFields::Label ).asString(),
+ provMsg->value( MediaChangeRequestMsgFields::MediaNr ).asInt(),
+ freeDevs,
+ desc
+ );
+
+ auto action = res ? *res : Provide::Action::ABORT;
+ switch ( action ) {
+ case Provide::Action::RETRY: {
+ MIL << "Sending back a MediaChanged message, retrying to find medium " << std::endl;
+ auto r = ProvideMessage::createMediaChanged ( reqIter->_request->provideMessage().requestId() );
+ if ( !_messageStream->sendMessage( r.impl() ) ){
+ ERR << "Failed to send MediaChanged to worker process." << std::endl;
+ fatalWorkerError( ZYPP_EXCPT_PTR( zypp::media::MediaException("Failed to communicate with worker process.") ) );
+ return;
+ }
+ continue;
+ }
+ case Provide::Action::ABORT: {
+ MIL << "Sending back a MediaChangeFailure message, request will fail " << std::endl;
+ if ( !sendErrorToWorker( reqRef->provideMessage().requestId(), ProvideMessage::Code::MediaChangeAbort, "Cancelled by User" ) )
+ return;
+ continue;
+ }
+ case Provide::Action::SKIP: {
+ MIL << "Sending back a MediaChangeFailure message, request will fail " << std::endl;
+ if ( !sendErrorToWorker( reqRef->provideMessage().requestId(), ProvideMessage::Code::MediaChangeSkip, "Skipped by User" ) )
+ return;
+ continue;
+ }
+ }
+ } else {
+ // if there is a unsupported worker request we need to stop immediately because the worker will be blocked until it gets a answer
+ ERR << "Unsupported worker request: "<<code<<", this is a fatal error!" << std::endl;
+ fatalWorkerError();
+ return;
+ }
+
+ } else {
+ // unknown code
+ ERR << "Received unsupported message " << msg->messagetypename() << " with code " << code << " ignoring! " << std::endl;
+ }
+
+ } else {
+ ERR << "Received unsupported message " << msg->messagetypename() << "ignoring" << std::endl;
+ }
+ }
+ }
+
+ /*!
+ * Reads all of the log lines from stderr, call only when shutting down the queue
+ * because this will also read partial lines and forwards them
+ */
+ void ProvideQueue::readAllStderr()
+ {
+ // read all stderr data so we get the full logs
+ auto ba = _workerProc->channelReadLine(Process::StdErr);
+ while ( !ba.empty() ) {
+ forwardToLog(std::string( ba.data(), ba.size() ) );
+ ba = _workerProc->channelReadLine(Process::StdErr);
+ }
+ }
+
+ void ProvideQueue::forwardToLog( std::string &&logLine )
+ {
+ if ( (_capabilities.cfg_flags () & zypp::proto::Capabilities::ZyppLogFormat) == zypp::proto::Capabilities::ZyppLogFormat )
+ zypp::base::LogControl::instance ().logRawLine( std::move(logLine) );
+ else
+ MIL << "Message from worker: " << _capabilities.worker_name() << ":" << logLine << std::endl;
+ }
+
+ void ProvideQueue::processReadyRead(int channel) {
+ // ignore stdout here
+ if ( channel == Process::StdOut )
+ return;
+
+ // forward the stderr output to the log bypassing the formatter
+ // the worker already formatted the line
+ while ( _workerProc->canReadLine(Process::StdErr) ) {
+ const auto &data = _workerProc->channelReadLine( Process::StdErr );
+ if ( data.empty() )
+ return;
+
+ forwardToLog(std::string( data.data(), data.size() ) );
+ }
+ }
+
+ void ProvideQueue::procFinished(int exitCode)
+ {
+ // process all pending messages
+ processMessage();
+
+ // get all of the log lines
+ readAllStderr();
+
+ // shut down
+ // @todo implement worker restart in case of a unexpected exit
+ if ( !_queueShuttingDown )
+ immediateShutdown( ZYPP_EXCPT_PTR( zypp::media::MediaException("Unexpected queue worker exit!") ) );
+
+#if 0
+ if ( !_queueShuttingDown ) {
+
+ _crashCounter++;
+ if ( _crashCounter > 3 ) {
+ immediateShutdown( ZYPP_EXCPT_PTR( zypp::media::MediaException("Unexpected queue worker exit!") ) );
+ return;
+ }
+
+ MIL << "Unexpected queue worker exit with code: " << exitCode << std::endl;
+ // try to spawn the worker again, move active items back to wait list and start over
+
+ if ( !doStartup () ) {
+
+ }
+ }
+#endif
+ }
+
+ uint32_t ProvideQueue::nextRequestId() {
+ return _parent.nextRequestId();
+ }
+}
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+
+#include "provideres.h"
+#include "private/provide_p.h"
+#include "private/providequeue_p.h"
+#include "private/provideres_p.h"
+
+namespace zyppng {
+
+ ProvideRes::ProvideRes( std::shared_ptr<ProvideResourceData> dataPtr ) : _data(dataPtr)
+ { }
+
+ ProvideRes::~ProvideRes()
+ { }
+
+ const zypp::filesystem::Pathname ProvideRes::file() const
+ {
+ return _data->_myFile;
+ }
+
+ const zypp::ManagedFile & ProvideRes::asManagedFile () const
+ {
+ return _data->_myFile;
+ }
+
+ const ProvideMediaHandle &ProvideRes::mediaHandle () const
+ {
+ return _data->_mediaHandle;
+ }
+
+ const zypp::Url &ProvideRes::resourceUrl () const
+ {
+ return _data->_resourceUrl;
+ }
+
+ const HeaderValueMap &ProvideRes::headers () const
+ {
+ return _data->_responseHeaders;
+ }
+
+}
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+/** \file zypp/source/ProvideRes.h
+ */
+#ifndef ZYPP_MEDIA_PROVIDERES_H_INCLUDED
+#define ZYPP_MEDIA_PROVIDERES_H_INCLUDED
+
+#include <zypp-media/ng/ProvideFwd>
+#include <zypp-core/Pathname.h>
+#include <zypp-core/ManagedFile.h>
+#include <memory>
+
+
+namespace zyppng
+{
+
+ struct ProvideResourceData;
+
+ /**
+ * \class ProvideRes
+ * A ProvideRes object is a reference counted ownership of a resource in the cache provided by
+ * a \ref Provide instance.
+ * It is generally advisable to release a ProvideRes instance asap, to make sure resources
+ * can be released and devices are free to be ejected.
+ *
+ * \note all ProvideRes instances will become invalid when the \ref Provide instance is
+ * released.
+ */
+ class ProvideRes
+ {
+ public:
+ ProvideRes( std::shared_ptr<ProvideResourceData> dataPtr );
+ virtual ~ProvideRes();
+
+ /*!
+ * Returns the path to the provided file
+ */
+ const zypp::Pathname file () const;
+
+ /*!
+ * Returns a reference to the internally used managed file instance.
+ * \note If you obtain this for a file that is inside the providers working directory ( e.g. a provide result for a download ),
+ * the continued use after the Provide instance was relased is undefined behaviour and not supported!
+ */
+ const zypp::ManagedFile & asManagedFile () const;
+
+ /*!
+ * Returns a reference to the currently held media handle, this can be a invalid handle
+ */
+ const ProvideMediaHandle &mediaHandle () const;
+
+ /*!
+ * The URL this ressource was provided from, can be empty
+ */
+ const zypp::Url &resourceUrl () const;
+
+ /*!
+ * All headers that were received from the worker when sending the result
+ */
+ const HeaderValueMap &headers () const;
+
+ private:
+ std::shared_ptr<ProvideResourceData> _data;
+ };
+}
+
+
+#endif
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+/** \file zypp-media/providespec.cc
+ *
+*/
+
+#include <iostream>
+#include "providespec.h"
+
+using std::endl;
+
+namespace zyppng
+{
+ class ProvideSpecBasePrivate
+ {
+ public:
+ ProvideSpecBasePrivate() {}
+ virtual ~ProvideSpecBasePrivate() {}
+ HeaderValueMap _customHeaders;
+ };
+
+
+ class ProvideMediaSpec::Impl : public ProvideSpecBasePrivate
+ {
+ public:
+ Impl()
+ {}
+
+ Impl( const std::string &label, const zypp::Pathname &vPath, unsigned medianr )
+ : _label( label )
+ , _medianr( medianr )
+ , _verifyDataPath(vPath)
+ {}
+
+ std::string _label;
+ unsigned _medianr = 0U;
+ zypp::Pathname _verifyDataPath;
+
+ public:
+ /** Offer default Impl. */
+ static zypp::shared_ptr<Impl> nullimpl()
+ { static zypp::shared_ptr<Impl> _nullimpl( new Impl ); return _nullimpl; }
+
+ private:
+ friend ProvideMediaSpec::Impl * zypp::rwcowClone<ProvideMediaSpec::Impl>( const ProvideMediaSpec::Impl * rhs );
+ Impl * clone() const { return new Impl( *this ); }
+ };
+
+ class ProvideFileSpec::Impl : public ProvideSpecBasePrivate
+ {
+ public:
+ Impl()
+ {}
+
+ zypp::Pathname _destFilenameHint;
+ zypp::Pathname _mediaSpecFile;
+ bool _checkExistsOnly = false;
+
+ bool _optional = false;
+ zypp::ByteCount _downloadSize;
+ zypp::CheckSum _checksum;
+
+ zypp::ByteCount _openSize;
+ zypp::CheckSum _openChecksum;
+
+ zypp::ByteCount _headerSize;
+ zypp::CheckSum _headerChecksum;
+
+ zypp::Pathname _deltafile;
+
+
+ public:
+ /** Offer default Impl. */
+ static zypp::shared_ptr<Impl> nullimpl()
+ { static zypp::shared_ptr<Impl> _nullimpl( new Impl ); return _nullimpl; }
+
+ private:
+ friend ProvideFileSpec::Impl * zypp::rwcowClone<ProvideFileSpec::Impl>( const ProvideFileSpec::Impl * rhs );
+ Impl * clone() const { return new Impl( *this ); }
+ };
+
+
+ ProvideMediaSpec::ProvideMediaSpec( const std::string &label, const zypp::filesystem::Pathname &verifyData, unsigned medianr )
+ : _pimpl( new Impl( label, verifyData, medianr ) )
+ {
+
+ }
+
+ const std::string &ProvideMediaSpec::label() const
+ { return _pimpl->_label; }
+
+ ProvideMediaSpec &ProvideMediaSpec::setLabel(const std::string &label)
+ {
+ _pimpl->_label = label;
+ return *this;
+ }
+
+ unsigned ProvideMediaSpec::medianr() const
+ { return _pimpl->_medianr; }
+
+ ProvideMediaSpec &ProvideMediaSpec::setMedianr(unsigned medianr)
+ {
+ _pimpl->_medianr = medianr;
+ return *this;
+ }
+
+ zypp::filesystem::Pathname ProvideMediaSpec::mediaFile() const
+ { return _pimpl->_verifyDataPath; }
+
+ ProvideMediaSpec &ProvideMediaSpec::setMediaFile(const zypp::filesystem::Pathname &pName)
+ {
+ _pimpl->_verifyDataPath = pName;
+ return *this;
+ }
+
+ HeaderValueMap &ProvideMediaSpec::customHeaders()
+ { return _pimpl->_customHeaders; }
+
+ const HeaderValueMap &ProvideMediaSpec::customHeaders() const
+ { return _pimpl->_customHeaders; }
+
+ ProvideMediaSpec &ProvideMediaSpec::setCustomHeaderValue(const std::string &key, const HeaderValueMap::Value &val)
+ {
+ _pimpl->_customHeaders.set( key,val );
+ return *this;
+ }
+
+ ProvideMediaSpec &ProvideMediaSpec::addCustomHeaderValue(const std::string &key, const HeaderValueMap::Value &val)
+ {
+ _pimpl->_customHeaders.add( key,val );
+ return *this;
+ }
+
+ zypp::TriBool ProvideMediaSpec::isSameMedium( const ProvideMediaSpec &other )
+ {
+ // first check if we have the same media data
+ if ( _pimpl->_verifyDataPath != other._pimpl->_verifyDataPath )
+ return false;
+
+ // if the verify file is not empty check the medianr
+ if ( !_pimpl->_verifyDataPath.empty() ) {
+ return _pimpl->_medianr == other._pimpl->_medianr;
+ }
+
+ // can't tell without the URL
+ return zypp::indeterminate;
+ }
+
+ /** \relates ProvideSpec::Impl Stream output */
+ inline std::ostream & operator<<( std::ostream & str, const ProvideFileSpec::Impl & obj )
+ {
+ return str << "{" << obj._destFilenameHint << "{" << obj._downloadSize << "|" << obj._checksum << "|" << obj._deltafile << "}" << "}";
+ }
+
+ /** \relates ProvideSpec::Impl Verbose stream output */
+ inline std::ostream & dumpOn( std::ostream & str, const ProvideFileSpec::Impl & obj )
+ { return str << obj; }
+
+
+ ProvideFileSpec::ProvideFileSpec()
+ : _pimpl( new Impl() )
+ {}
+
+ ProvideFileSpec::ProvideFileSpec( const zypp::OnMediaLocation &loc )
+ : _pimpl( new Impl() )
+ {
+ setDownloadSize( loc.downloadSize() );
+ setOptional( loc.optional() );
+ setChecksum( loc.checksum() );
+ setOpenSize( loc.openSize() );
+ setOpenChecksum( loc.openChecksum() );
+ setHeaderSize( loc.headerSize() );
+ setHeaderChecksum( loc.headerChecksum() );
+ setDeltafile( loc.deltafile() );
+ }
+
+ ProvideFileSpec::~ProvideFileSpec()
+ {}
+
+ const zypp::filesystem::Pathname &ProvideFileSpec::destFilenameHint() const
+ { return _pimpl->_destFilenameHint; }
+
+ ProvideFileSpec &ProvideFileSpec::setDestFilenameHint(const zypp::filesystem::Pathname &filename)
+ { _pimpl->_destFilenameHint = filename; return *this; }
+
+ bool ProvideFileSpec::checkExistsOnly() const
+ { return _pimpl->_checkExistsOnly; }
+
+ ProvideFileSpec &ProvideFileSpec::setCheckExistsOnly(const bool set)
+ { _pimpl->_checkExistsOnly = set; return *this; }
+
+ bool ProvideFileSpec::optional() const
+ { return _pimpl->_optional; }
+
+ ProvideFileSpec & ProvideFileSpec::setOptional( bool val_r )
+ { _pimpl->_optional = (val_r); return *this; }
+
+ const zypp::ByteCount & ProvideFileSpec::downloadSize() const
+ { return _pimpl->_downloadSize; }
+
+ ProvideFileSpec & ProvideFileSpec::setDownloadSize( const zypp::ByteCount &val_r )
+ { _pimpl->_downloadSize = (val_r); return *this; }
+
+ const zypp::CheckSum & ProvideFileSpec::checksum() const
+ { return _pimpl->_checksum; }
+
+ ProvideFileSpec & ProvideFileSpec::setChecksum( const zypp::CheckSum &val_r )
+ { _pimpl->_checksum = (val_r); return *this; }
+
+ const zypp::ByteCount & ProvideFileSpec::openSize() const
+ { return _pimpl->_openSize; }
+
+ ProvideFileSpec & ProvideFileSpec::setOpenSize( const zypp::ByteCount &val_r )
+ { _pimpl->_openSize = (val_r); return *this; }
+
+ const zypp::CheckSum & ProvideFileSpec::openChecksum() const
+ { return _pimpl->_openChecksum; }
+
+ ProvideFileSpec & ProvideFileSpec::setOpenChecksum( const zypp::CheckSum &val_r )
+ { _pimpl->_openChecksum = (val_r); return *this; }
+
+ const zypp::ByteCount & ProvideFileSpec::headerSize() const
+ { return _pimpl->_headerSize; }
+
+ ProvideFileSpec & ProvideFileSpec::setHeaderSize( const zypp::ByteCount &val_r )
+ { _pimpl->_headerSize = (val_r); return *this; }
+
+ const zypp::CheckSum & ProvideFileSpec::headerChecksum() const
+ { return _pimpl->_headerChecksum; }
+
+ ProvideFileSpec & ProvideFileSpec::setHeaderChecksum( const zypp::CheckSum &val_r )
+ { _pimpl->_headerChecksum = (val_r); return *this; }
+
+ const zypp::Pathname &ProvideFileSpec::deltafile() const
+ { return _pimpl->_deltafile; }
+
+ ProvideFileSpec &ProvideFileSpec::setDeltafile( const zypp::Pathname &path )
+ { _pimpl->_deltafile = (path); return *this; }
+
+ HeaderValueMap &ProvideFileSpec::customHeaders()
+ { return _pimpl->_customHeaders; }
+
+ const HeaderValueMap &ProvideFileSpec::customHeaders() const
+ { return _pimpl->_customHeaders; }
+
+ ProvideFileSpec &ProvideFileSpec::setCustomHeaderValue(const std::string &key, const HeaderValueMap::Value &val)
+ {
+ _pimpl->_customHeaders.set( key,val );
+ return *this;
+ }
+
+ ProvideFileSpec &ProvideFileSpec::addCustomHeaderValue(const std::string &key, const HeaderValueMap::Value &val)
+ {
+ _pimpl->_customHeaders.add( key,val );
+ return *this;
+ }
+
+ std::ostream & operator<<( std::ostream & str, const ProvideFileSpec & obj )
+ { return str << *obj._pimpl; }
+
+ std::ostream & dumpOn( std::ostream & str, const ProvideFileSpec & obj )
+ { return dumpOn( str, *obj._pimpl ); }
+
+} // namespace zypp
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+/** \file zypp/source/ProvideSpec.h
+ */
+#ifndef ZYPP_MEDIA_PROVIDESPEC_H_INCLUDED
+#define ZYPP_MEDIA_PROVIDESPEC_H_INCLUDED
+
+#include <iosfwd>
+
+#include <zypp-core/Url.h>
+#include <zypp-core/ByteCount.h>
+#include <zypp-core/CheckSum.h>
+#include <zypp-core/TriBool.h>
+#include <zypp-core/OnMediaLocation>
+#include <zypp-media/ng/ProvideFwd>
+#include <zypp-media/ng/HeaderValueMap>
+#include <boost/iterator/iterator_adaptor.hpp>
+
+namespace zyppng
+{
+
+ class ProvideMediaSpec
+ {
+ public:
+
+ ProvideMediaSpec( const std::string &label, const zypp::Pathname &verifyData = zypp::Pathname(), unsigned medianr = 1 );
+
+ /*!
+ * The label of the medium, this will be shown in case a media change is required
+ */
+ const std::string &label() const;
+
+ /*!
+ * Changes the label of the medium
+ */
+ ProvideMediaSpec &setLabel( const std::string &label );
+
+ /*!
+ * The media number the resource is located on.
+ */
+ unsigned medianr() const;
+
+ /*!
+ * Individual manipulation of \c medianr (prefer \ref setLocation).
+ */
+ ProvideMediaSpec & setMedianr( unsigned medianr );
+
+ /*!
+ * Returns the media validation file path. This can be empty if there
+ * is no file to validate.
+ */
+ zypp::Pathname mediaFile () const;
+
+ /*!
+ * Changes the media file to the name path \a pName.
+ */
+ ProvideMediaSpec &setMediaFile( const zypp::Pathname &pName );
+
+ /*!
+ * Returns a map of custom key->value pairs that can control special aspects
+ * of how all files are provided by the medium after it was attached
+ */
+ HeaderValueMap &customHeaders();
+ const HeaderValueMap &customHeaders() const;
+
+ /*!
+ * Set the custom header value identified by \a key to \a val
+ */
+ ProvideMediaSpec &setCustomHeaderValue ( const std::string &key, const HeaderValueMap::Value &val );
+
+ /*!
+ * Adds the custom header value \a val to the list of values identified by \a key
+ */
+ ProvideMediaSpec &addCustomHeaderValue ( const std::string &key, const HeaderValueMap::Value &val );
+
+ zypp::TriBool isSameMedium ( const ProvideMediaSpec &other );
+
+ public:
+ class Impl; ///< Implementation class.
+ private:
+ zypp::RWCOW_pointer<Impl> _pimpl; ///< Pointer to implementation.
+ };
+
+
+ class ProvideFileSpec
+ {
+ friend std::ostream & operator<<( std::ostream &str, const ProvideFileSpec &obj );
+ friend std::ostream & dumpOn( std::ostream &str, const ProvideFileSpec &obj );
+
+ public:
+ ProvideFileSpec();
+
+ ProvideFileSpec( const zypp::OnMediaLocation &loc );
+
+ /** Dtor */
+ ~ProvideFileSpec();
+
+ template <typename... T>
+ static ProvideFileSpecRef create ( T... args ) {
+ return std::make_shared<ProvideFileSpec>( std::forward<T>(args)... );
+ }
+
+ /*!
+ * The destination file name, this is optional and is only
+ * a hint to the \ref Provide instance where to put the file IF possible. The
+ * file provider does not need to consider this.
+ */
+ const zypp::Pathname &destFilenameHint() const;
+ ProvideFileSpec &setDestFilenameHint ( const zypp::Pathname &filename );
+
+ bool checkExistsOnly () const;
+ ProvideFileSpec & setCheckExistsOnly( const bool set = true );
+
+
+ /** Whether this is an optional resource.
+ * This is a hint to the downloader not to report an error if
+ * the resource is not present on the server.
+ */
+ bool optional() const;
+ /** Set whether the resource is \ref optional. */
+ ProvideFileSpec & setOptional( bool val );
+
+ /** The size of the resource on the server. */
+ const zypp::ByteCount &downloadSize() const;
+ /** Set the \ref downloadSize. */
+ ProvideFileSpec &setDownloadSize( const zypp::ByteCount &val_r );
+
+ /** The checksum of the resource on the server. */
+ const zypp::CheckSum &checksum() const;
+ /** Set the \ref checksum. */
+ ProvideFileSpec &setChecksum( const zypp::CheckSum &val_r );
+
+
+ /** The size of the resource once it has been uncompressed or unpacked. */
+ const zypp::ByteCount &openSize() const;
+ /** Set the \ref openSize. */
+ ProvideFileSpec &setOpenSize( const zypp::ByteCount &val_r );
+
+ /** The checksum of the resource once it has been uncompressed or unpacked. */
+ const zypp::CheckSum &openChecksum() const;
+ /** Set the \ref openChecksum. */
+ ProvideFileSpec &setOpenChecksum( const zypp::CheckSum &val_r );
+
+ /** The size of the header prepending the resource (e.g. for zchunk). */
+ const zypp::ByteCount &headerSize() const;
+ /** Set the \ref headerSize. */
+ ProvideFileSpec &setHeaderSize( const zypp::ByteCount &val_r );
+
+ /** The checksum of the header prepending the resource (e.g. for zchunk). */
+ const zypp::CheckSum &headerChecksum() const;
+ /** Set the \ref headerChecksum. */
+ ProvideFileSpec &setHeaderChecksum( const zypp::CheckSum &val_r );
+
+ /** The existing deltafile that can be used to reduce download size ( zchunk or metalink ) */
+ const zypp::Pathname &deltafile() const;
+ /** Set the \ref deltafile. */
+ ProvideFileSpec &setDeltafile( const zypp::Pathname &path );
+
+ /*!
+ * Returns a map of custom key->value pairs that can control special aspects
+ * of how the provide operation is processed.
+ *
+ * @todo should this actually just be a list of pair(string,string) instead of map? -> easier way to send multiple values to a worker
+ */
+ HeaderValueMap &customHeaders();
+ const HeaderValueMap &customHeaders() const;
+
+ /*!
+ * Set the custom header value identified by \a key to \a val
+ */
+ ProvideFileSpec &setCustomHeaderValue ( const std::string &key, const HeaderValueMap::Value &val );
+
+ /*!
+ * Adds the custom header value \a val to the list of values identified by \a key
+ */
+ ProvideFileSpec &addCustomHeaderValue ( const std::string &key, const HeaderValueMap::Value &val );
+
+ public:
+ class Impl; ///< Implementation class.
+ private:
+ zypp::RWCOW_pointer<Impl> _pimpl; ///< Pointer to implementation.
+ };
+
+ /** \relates ProvideSpec Stream output */
+ std::ostream & operator<<( std::ostream &str, const ProvideFileSpec &obj );
+
+ /** \relates ProvideSpec Verbose stream output */
+ std::ostream & dumpOn( std::ostream &str, const ProvideFileSpec &obj );
+
+} // namespace zypp
+
+#endif // ZYPP_MEDIA_PROVIDESPEC_H_INCLUDED
--- /dev/null
+#include "devicedriver.h"
--- /dev/null
+#include "mountingworker.h"
--- /dev/null
+#include "provideworker.h"
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+
+#include "devicedriver.h"
+#include <zypp-media/ng/private/providedbg_p.h>
+#include <zypp-media/ng/private/providemessage_p.h>
+#include <zypp-media/ng/MediaVerifier>
+#include <zypp-media/MediaException>
+#include <zypp-core/fs/PathInfo.h>
+#include <zypp-core/fs/TmpPath.h>
+#include <zypp-core/Date.h>
+
+#undef ZYPP_BASE_LOGGER_LOGGROUP
+#define ZYPP_BASE_LOGGER_LOGGROUP "zyppng::worker::DeviceDriver"
+
+namespace zyppng::worker
+{
+
+ DeviceDriver::DeviceDriver ( WorkerCaps::WorkerType wType )
+ : _wType( wType )
+ { }
+
+ void DeviceDriver::setProvider ( ProvideWorkerWeakRef workerRef )
+ {
+ _parentWorker = workerRef;
+ }
+
+ zyppng::expected<WorkerCaps> DeviceDriver::initialize(const zyppng::worker::Configuration &conf)
+ {
+ const auto &values = conf.values();
+ if ( const auto &i = values.find( std::string(zyppng::ATTACH_POINT) ); i != values.end() ) {
+ const auto &val = i->second;
+ MIL << "Got attachpoint from controller: " << val << std::endl;
+ _attachRoot = zypp::Pathname(val).realpath();
+ } else {
+ return zyppng::expected<zyppng::worker::WorkerCaps>::error(ZYPP_EXCPT_PTR( zypp::Exception("Attach point required to work.") ));
+ }
+
+ _config = conf;
+
+ zyppng::worker::WorkerCaps caps;
+ caps.set_worker_type ( _wType );
+ caps.set_cfg_flags(
+ zyppng::worker::WorkerCaps::Flags (
+ zyppng::worker::WorkerCaps::Pipeline
+ | zyppng::worker::WorkerCaps::ZyppLogFormat
+ | zyppng::worker::WorkerCaps::SingleInstance
+ )
+ );
+
+ return zyppng::expected<zyppng::worker::WorkerCaps>::success(caps);
+ }
+
+ bool DeviceDriver::detachMedia ( const std::string &attachId )
+ {
+ auto i = _attachedMedia.find( attachId );
+ if ( i == _attachedMedia.end() )
+ return false;
+
+ _attachedMedia.erase(i);
+ return true;
+ }
+
+ void DeviceDriver::releaseIdleDevices ()
+ {
+ for ( auto i = _sysDevs.begin (); i != _sysDevs.end(); ) {
+ if ( i->use_count() == 1 && !(*i)->_mountPoint.empty() ) {
+ MIL << "Unmounting device " << (*i)->_name << " since its not used anymore" << std::endl;
+ unmountDevice(*(*i));
+ if ( (*i)->_ephemeral ) {
+ i = _sysDevs.erase(i);
+ continue;
+ }
+ }
+ ++i;
+ }
+ }
+
+ void DeviceDriver::detectDevices()
+ {
+ return;
+ }
+
+ std::vector<std::shared_ptr<Device>> &DeviceDriver::knownDevices()
+ {
+ return _sysDevs;
+ }
+
+ const std::vector<std::shared_ptr<Device>> &DeviceDriver::knownDevices() const
+ {
+ return _sysDevs;
+ }
+
+ std::unordered_map<std::string, AttachedMedia> &DeviceDriver::attachedMedia()
+ {
+ return _attachedMedia;
+ }
+
+ void DeviceDriver::immediateShutdown()
+ {
+ // here we need to unmount everything
+ for ( auto i = _sysDevs.begin (); i != _sysDevs.end(); ) {
+ unmountDevice(*(*i));
+ if ( (*i)->_ephemeral ) {
+ i = _sysDevs.erase(i);
+ continue;
+ }
+ ++i;
+ }
+ _attachedMedia.clear();
+ }
+
+ ProvideWorkerRef DeviceDriver::parentWorker () const
+ {
+ return _parentWorker.lock();
+ }
+
+ void DeviceDriver::unmountDevice ( Device &dev )
+ {
+ if ( dev._mountPoint.empty () )
+ return;
+ try {
+ zypp::media::Mount mount;
+ mount.umount( dev._mountPoint.asString() );
+ removeAttachPoint( dev._mountPoint );
+ } catch (const zypp::media::MediaException & excpt_r) {
+ ERR << "Failed to unmount device: " << dev._name << std::endl;
+ ZYPP_CAUGHT(excpt_r);
+ }
+ dev._mountPoint = zypp::Pathname();
+ }
+
+ bool DeviceDriver::isVolatile () const
+ {
+ return false;
+ }
+
+ void DeviceDriver::setAttachRoot ( const zypp::Pathname &root )
+ {
+ _attachRoot = root;
+ }
+
+ zypp::Pathname DeviceDriver::attachRoot () const
+ {
+ if ( _attachRoot.empty() ) {
+ MIL << "Attach root is empty" << std::endl;
+ return zypp::Pathname(".").realpath();
+ }
+ return _attachRoot;
+ }
+
+ const zyppng::worker::Configuration &DeviceDriver::config() const
+ {
+ return _config;
+ }
+
+ zyppng::expected<void> DeviceDriver::isDesiredMedium ( const zypp::Url &deviceUrl, const zypp::Pathname &mountPoint, const zyppng::MediaDataVerifierRef &verifier, uint mediaNr )
+ {
+ if ( !verifier ) {
+ // at least the requested path must exist on the medium
+ zypp::PathInfo p( mountPoint );
+ if ( p.isExist() && p.isDir() )
+ return zyppng::expected<void>::success(); // we have no valid data
+ return zyppng::expected<void>::error( ZYPP_EXCPT_PTR( zypp::media::MediaNotDesiredException( deviceUrl ) ) );
+ }
+
+ auto devVerifier = verifier->clone();
+ if ( !devVerifier ) {
+ // unlikely to happen
+ return zyppng::expected<void>::error( ZYPP_EXCPT_PTR( zypp::Exception("Failed to clone verifier") ) );
+ }
+
+ // bsc#1180851: If there is just one not-volatile medium in the set
+ // tolerate a missing (vanished) media identifier and let the URL rule.
+ bool relaxed = verifier->totalMedia() == 1 && !isVolatile();
+
+ const auto &relMediaPath = devVerifier->mediaFilePath( mediaNr );
+ zypp::Pathname mediaFile { mountPoint / relMediaPath };
+ zypp::PathInfo pi( mediaFile );
+ if ( !pi.isExist() ) {
+ if ( relaxed )
+ return zyppng::expected<void>::success();
+ auto excpt = zypp::media::MediaFileNotFoundException( deviceUrl, relMediaPath ) ;
+ excpt.addHistory( verifier->expectedAsUserString( mediaNr ) );
+ return zyppng::expected<void>::error( ZYPP_EXCPT_PTR( std::move(excpt) ) );
+ }
+ if ( !pi.isFile() ) {
+ if ( relaxed )
+ return zyppng::expected<void>::success();
+ auto excpt = zypp::media::MediaNotAFileException( deviceUrl, relMediaPath ) ;
+ excpt.addHistory( verifier->expectedAsUserString( mediaNr ) );
+ return zyppng::expected<void>::error( ZYPP_EXCPT_PTR( std::move(excpt) ) );
+ }
+
+ if ( !devVerifier->load( mediaFile ) ) {
+ return zyppng::expected<void>::error( ZYPP_EXCPT_PTR( zypp::Exception("Failed to load media information from medium") ) );
+ }
+ if ( !verifier->matches( devVerifier ) ) {
+ return zyppng::expected<void>::error( ZYPP_EXCPT_PTR( zypp::media::MediaNotDesiredException( deviceUrl ) ) );
+ }
+ return zyppng::expected<void>::success();
+ }
+
+ zypp::Pathname DeviceDriver::createAttachPoint( const zypp::Pathname &attach_root ) const
+ {
+ zypp::Pathname apoint;
+
+ if( attach_root.empty() || !attach_root.absolute()) {
+ ERR << "Create attach point: invalid attach root: '"
+ << attach_root << "'" << std::endl;
+ return apoint;
+ }
+
+ zypp::PathInfo adir( attach_root );
+ if( !adir.isDir() || (geteuid() != 0 && !adir.userMayRWX())) {
+ DBG << "Create attach point: attach root is not a writable directory: '"
+ << attach_root << "'" << std::endl;
+ return apoint;
+ }
+
+ static bool cleanup_once( true );
+ if ( cleanup_once )
+ {
+ cleanup_once = false;
+ DBG << "Look for orphaned attach points in " << adir << std::endl;
+ std::list<std::string> entries;
+ zypp::filesystem::readdir( entries, attach_root, false );
+ for ( const std::string & entry : entries )
+ {
+ if ( ! zypp::str::hasPrefix( entry, "AP_0x" ) )
+ continue;
+ zypp::PathInfo sdir( attach_root + entry );
+ if ( sdir.isDir()
+ && sdir.dev() == adir.dev()
+ && ( zypp::Date::now()-sdir.mtime() > zypp::Date::month ) )
+ {
+ DBG << "Remove orphaned attach point " << sdir << std::endl;
+ zypp::filesystem::recursive_rmdir( sdir.path() );
+ }
+ }
+ }
+
+ zypp::filesystem::TmpDir tmpdir( attach_root, "AP_0x" );
+ if ( tmpdir )
+ {
+ apoint = tmpdir.path().asString();
+ if ( ! apoint.empty() )
+ {
+ tmpdir.autoCleanup( false ); // Take responsibility for cleanup.
+ }
+ else
+ {
+ ERR << "Unable to resolve real path for attach point " << tmpdir << std::endl;
+ }
+ }
+ else
+ {
+ ERR << "Unable to create attach point below " << attach_root << std::endl;
+ }
+ return apoint;
+ }
+
+ void DeviceDriver::removeAttachPoint( const zypp::filesystem::Pathname &attachRoot ) const
+ {
+ if( !attachRoot.empty() &&
+ zypp::PathInfo(attachRoot).isDir() &&
+ attachRoot != "/" ) {
+ int res = recursive_rmdir( attachRoot );
+ if ( res == 0 ) {
+ MIL << "Deleted default attach point " << attachRoot << std::endl;
+ } else {
+ ERR << "Failed to Delete default attach point " << attachRoot
+ << " errno(" << res << ")" << std::endl;
+ }
+ }
+ }
+
+ bool DeviceDriver::checkAttached ( const zypp::filesystem::Pathname &mountPoint, const std::function<bool (const zypp::media::MountEntry &)> predicate )
+ {
+ bool isAttached = false;
+ time_t old_mtime = _attach_mtime;
+ _attach_mtime = zypp::media::Mount::getMTime();
+ if( !(old_mtime <= 0 || _attach_mtime != old_mtime) ) {
+ // OK, skip the check (we've seen it at least once)
+ isAttached = true;
+ } else {
+ if( old_mtime > 0)
+ DBG << "Mount table changed - rereading it" << std::endl;
+ else
+ DBG << "Forced check of the mount table" << std::endl;
+
+ for( const auto &entry : zypp::media::Mount::getEntries() ) {
+
+ if ( mountPoint != zypp::Pathname(entry.dir) )
+ continue; // at least the mount points must match
+ if ( predicate(entry) ) {
+ isAttached = true;
+ break;
+ }
+ }
+ }
+
+ // force recheck
+ if ( !isAttached )
+ _attach_mtime = 0;
+
+ return isAttached;
+ }
+
+ const std::function<bool (const zypp::media::MountEntry &)> DeviceDriver::devicePredicate( unsigned int majNr, unsigned int minNr )
+ {
+ return [ majNr, minNr ]( const zypp::media::MountEntry &entry ) -> bool {
+ if( entry.isBlockDevice() ) {
+ zypp::PathInfo dev_info( entry.src );
+ if ( dev_info.devMajor () == majNr && dev_info.devMinor () == minNr ) {
+ DBG << "Found device "
+ << majNr << ":" << minNr
+ << " in the mount table as " << entry.src << std::endl;
+ return true;
+ }
+ }
+ return false;
+ };
+ }
+
+ const std::function<bool (const zypp::media::MountEntry &)> DeviceDriver::fstypePredicate( const std::string &src, const std::vector<std::string> &fstypes )
+ {
+ return [ srcdev=src, fst=fstypes ]( const zypp::media::MountEntry &entry ) -> bool {
+ if( !entry.isBlockDevice() ) {
+ if ( std::find( fst.begin(), fst.end(), entry.type ) != fst.end() ) {
+ if ( srcdev == entry.src ) {
+ DBG << "Found media mount"
+ << " in the mount table as " << entry.src << std::endl;
+ return true;
+ }
+ }
+ }
+ return false;
+ };
+ }
+
+ const std::function<bool (const zypp::media::MountEntry &)> DeviceDriver::bindMountPredicate( const std::string &src )
+ {
+ return [ srcdev=src ]( const zypp::media::MountEntry &entry ) -> bool {
+ if( !entry.isBlockDevice() ) {
+ if ( srcdev == entry.src ) {
+ DBG << "Found bound media "
+ << " in the mount table as " << entry.src << std::endl;
+ return true;
+ }
+ }
+ return false;
+ };
+ }
+
+ AttachError::AttachError ( const uint code, const std::string &reason, const bool transient, const HeaderValueMap &extra)
+ : _code( code ),
+ _reason( reason ),
+ _transient( transient ),
+ _extra( extra )
+ {
+
+ }
+
+ AttachError::AttachError ( const uint code, const bool transient, const zypp::Exception &e )
+ : _code( code ),
+ _reason( e.asUserString() ),
+ _transient( transient )
+ {
+ if ( !e.historyEmpty() ) {
+ _extra = { { std::string(zyppng::ErrMsgFields::History), { zyppng::HeaderValueMap::Value(e.historyAsString()) }} };
+ }
+ }
+
+
+}
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+
+#ifndef ZYPP_MEDIA_NG_WORKER_DEVICEDRIVER_H_INCLUDED
+#define ZYPP_MEDIA_NG_WORKER_DEVICEDRIVER_H_INCLUDED
+
+#include <zypp-media/ng/ProvideFwd>
+#include <zypp-media/ng/worker/ProvideWorker>
+#include <zypp-media/ng/HeaderValueMap>
+#include <zypp-media/Mount>
+#include <zypp-core/zyppng/base/Signals>
+#include <zypp-core/zyppng/base/Base>
+#include <zypp-core/zyppng/base/zyppglobal.h>
+#include <zypp-core/zyppng/pipelines/Expected>
+#include <any>
+#include <unordered_map>
+
+namespace zyppng::worker
+{
+
+ ZYPP_FWD_DECL_TYPE_WITH_REFS (DeviceDriver);
+
+ struct Device
+ {
+ std::string _name; //!< Path of the device node or URL for e.g. nfs devices
+ unsigned int _maj_nr = 0; //!< Major number of the device
+ unsigned int _min_nr = 0; //!< Minor number of the device
+ zypp::Pathname _mountPoint = {};//!< Mountpoint of the device, if empty dev is not mounted
+ bool _ephemeral = false; //!< If set to true the device is removed from the internal list after the last attachpoint was released
+ std::unordered_map<std::string, std::any> _properties = {};
+ };
+
+ struct AttachedMedia
+ {
+ std::shared_ptr<Device> _dev;
+ zypp::Pathname _attachRoot;
+ };
+
+ struct AttachError
+ {
+ AttachError ( const uint code, const std::string &reason, const bool transient, const HeaderValueMap &extra = {} );
+ AttachError ( const uint code, const bool transient, const zypp::Exception &e );
+
+ uint _code;
+ std::string _reason;
+ bool _transient;
+ HeaderValueMap _extra;
+ };
+
+ using AttachResult = expected<void, AttachError>;
+
+ /*!
+ * Abstract base class to be used together with the \sa MountingWorker class to control
+ * attaching and detaching of a multitude of different media types via a unified interface.
+ * Reimplement this class to easily support a new backend that utilizes the "mount" command to
+ * attach real filesystems to the system.
+ */
+ class DeviceDriver : public zyppng::Base
+ {
+ public:
+
+ DeviceDriver ( WorkerCaps::WorkerType wType );
+
+ /*!
+ * Tells the driver which provide worker to use when requiring auth data or media changes
+ */
+ void setProvider ( ProvideWorkerWeakRef workerRef );
+
+ /*!
+ * Called by the provide loop whenever a attach request is received.
+ */
+ virtual AttachResult mountDevice ( const uint32_t id, const zypp::Url &mediaUrl, const std::string &attachId, const std::string &label, const HeaderValueMap &extras ) = 0;
+
+
+
+ virtual zyppng::expected<WorkerCaps> initialize(const zyppng::worker::Configuration &conf);
+
+
+ /*!
+ * Detaches the medium referenced by \a attachId.
+ * Returns false if the medium could not be found
+ *
+ * \note this will not unmount the physical devices.
+ * Call \ref releaseIdleDevices to force this.
+ *
+ */
+ bool detachMedia ( const std::string &attachId );
+
+ /*!
+ * Physically detaches all devices that are not referenced by a attachment anymore
+ */
+ void releaseIdleDevices ();
+
+ /*!
+ * Called by the parent to populate the device array every time a provide request arrives, only
+ * relevant for workers that operate on detectable devices.
+ * The base implementation does nothing
+ */
+ virtual void detectDevices();
+
+ /*!
+ * Returns a list of all currently known devices, a subclass should always make sure
+ * that all currently mounted devices are present in that list.
+ */
+ std::vector<std::shared_ptr<Device>> &knownDevices();
+
+ /*!
+ * Returns a list of all currently known devices, a subclass should always make sure
+ * that all currently mounted devices are present in that list.
+ */
+ const std::vector<std::shared_ptr<Device>> &knownDevices() const;
+
+ /*!
+ * Returns the list of currently attached medias
+ */
+ std::unordered_map<std::string, AttachedMedia> &attachedMedia();
+
+ /*!
+ * Returns true if the worker handles volatile devices ( e.g. DVDs ).
+ * The default impl returns false.
+ */
+ virtual bool isVolatile () const;
+
+ /*!
+ * Changes the attach root to a specific path, otherwise realpath(".") is used.
+ */
+ void setAttachRoot ( const zypp::Pathname &root );
+
+ /*!
+ * Returns the \a attachRoot as dictated by the controller
+ */
+ zypp::Pathname attachRoot () const;
+
+ /*!
+ * The system is shutting down, release all ressources
+ */
+ virtual void immediateShutdown();
+
+ /*!
+ * Returns the parent worker if set
+ */
+ ProvideWorkerRef parentWorker () const;
+
+ /*!
+ * Returns the configuration that was sent by the controller
+ */
+ const zyppng::worker::Configuration &config() const;
+
+ protected:
+ /*!
+ * Forcefully unmounts the device, this does not check if there any attached medias still relying on it
+ */
+ virtual void unmountDevice ( Device &dev );
+
+ /*!
+ * Checks if the medium \a deviceUrl mounted on \a path matches the \a verifier and \a mediaNr
+ */
+ zyppng::expected<void> isDesiredMedium ( const zypp::Url &deviceUrl, const zypp::Pathname &mountPoint, const zyppng::MediaDataVerifierRef &verifier, uint mediaNr = 1 );
+
+
+ zypp::Pathname createAttachPoint(const zypp::Pathname &attach_root) const;
+ void removeAttachPoint ( const zypp::Pathname &attach_pt ) const;
+ bool checkAttached ( const zypp::Pathname &mountPoint, const std::function<bool( const zypp::media::MountEntry &)> predicate );
+
+ /*!
+ * Returns a predicate for the \ref checkAttached function that looks for a real device in the mount table
+ */
+ static const std::function<bool( const zypp::media::MountEntry &)> devicePredicate ( unsigned int majNr, unsigned int minNr );
+
+ /*!
+ * Returns a predicate for the \ref checkAttached function that looks for a virtual mount ( like smb or nfs ) in the mount table
+ */
+ static const std::function<bool( const zypp::media::MountEntry &)> fstypePredicate ( const std::string &src, const std::vector<std::string> &fstypes );
+
+ /*!
+ * Returns a predicate for the \ref checkAttached function that looks for a bind mount in the mount table
+ */
+ static const std::function<bool( const zypp::media::MountEntry &)> bindMountPredicate ( const std::string &src );
+
+ private:
+ WorkerCaps::WorkerType _wType;
+ zyppng::worker::Configuration _config;
+ time_t _attach_mtime = 0; //< Timestamp of the mtab we did read
+ zypp::Pathname _attachRoot;
+ std::vector<std::shared_ptr<Device>> _sysDevs;
+ std::unordered_map<std::string, AttachedMedia> _attachedMedia;
+ ProvideWorkerWeakRef _parentWorker;
+ };
+
+
+
+
+}
+
+#endif
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+#include "mountingworker.h"
+#include <zypp-media/ng/private/providedbg_p.h>
+#include <zypp-media/ng/MediaVerifier>
+#include <zypp-core/fs/PathInfo.h>
+
+#undef ZYPP_BASE_LOGGER_LOGGROUP
+#define ZYPP_BASE_LOGGER_LOGGROUP "MountingWorker"
+
+namespace zyppng::worker
+{
+
+ MountingWorker::MountingWorker( std::string_view workerName, DeviceDriverRef driver )
+ : ProvideWorker( workerName )
+ , _driver(driver)
+ { }
+
+ MountingWorker::~MountingWorker()
+ {
+ _driver->immediateShutdown();
+ }
+
+ zyppng::expected<zyppng::worker::WorkerCaps> MountingWorker::initialize( const zyppng::worker::Configuration &conf )
+ {
+ return _driver->initialize(conf);
+ }
+
+ void MountingWorker::provide()
+ {
+ _driver->detectDevices();
+ auto &queue = requestQueue();
+
+ if ( !queue.size() )
+ return;
+
+ auto req = queue.front();
+ queue.pop_front();
+
+ MIL_PRV << "Received provide: " << req->_spec.code() << std::endl;
+
+ try {
+ switch ( req->_spec.code () ) {
+ case zyppng::ProvideMessage::Code::Attach: {
+
+ const auto attachUrl = zypp::Url( req->_spec.value( zyppng::AttachMsgFields::Url ).asString() );
+ const auto label = req->_spec.value( zyppng::AttachMsgFields::Label, "No label" ).asString();
+ const auto attachId = req->_spec.value( zyppng::AttachMsgFields::AttachId ).asString();
+ HeaderValueMap vals;
+ req ->_spec.forEachVal([&]( const std::string &name, const auto &val ) {
+ if ( name == zyppng::AttachMsgFields::Url
+ || name == zyppng::AttachMsgFields::Label
+ || name == zyppng::AttachMsgFields::AttachId )
+ return true;
+ vals.add( name, val );
+ return true;
+ });
+
+ const auto &res = _driver->mountDevice( req->_spec.requestId(), attachUrl, attachId, label, vals );
+ if ( !res ) {
+ const auto &err = res.error();
+ req->_state = zyppng::worker::ProvideWorkerItem::Finished;
+ provideFailed( req->_spec.requestId()
+ , err._code
+ , err._reason
+ , err._transient
+ , err._extra );
+ return;
+ }
+
+ MIL << "Attach of " << attachUrl << " was successfull" << std::endl;
+ attachSuccess( req->_spec.requestId() );
+ return;
+ }
+ case zyppng::ProvideMessage::Code::Detach: {
+
+ const auto url = zypp::Url( req->_spec.value( zyppng::DetachMsgFields::Url ).asString() );
+ const auto &attachId = url.getAuthority();
+
+ if ( _driver->detachMedia( attachId ) ) {
+ detachSuccess ( req->_spec.requestId() );
+ } else {
+ provideFailed( req->_spec.requestId()
+ , zyppng::ProvideMessage::Code::NotFound
+ , "Attach ID not known."
+ , false
+ , {} );
+ return;
+ }
+
+ _driver->releaseIdleDevices();
+ return;
+ }
+
+ case zyppng::ProvideMessage::Code::Provide: {
+
+ const auto url = zypp::Url( req->_spec.value( zyppng::DetachMsgFields::Url ).asString() );
+ const auto &attachId = url.getAuthority();
+ const auto &path = zypp::Pathname(url.getPathName());
+ const auto &availMedia = _driver->attachedMedia();
+
+ auto i = availMedia.find( attachId );
+ if ( i == availMedia.end() ) {
+ ERR << "Unknown Attach ID " << attachId << std::endl;
+ provideFailed( req->_spec.requestId()
+ , zyppng::ProvideMessage::Code::NotFound
+ , "Attach ID not known."
+ , false
+ , {} );
+ return;
+ }
+
+ const auto &locPath = i->second._dev->_mountPoint / i->second._attachRoot / path;
+
+ MIL << "Trying to find file: " << locPath << std::endl;
+
+ zypp::PathInfo info( locPath );
+ if( info.isFile() ) {
+ provideSuccess ( req->_spec.requestId(), false, locPath );
+ return;
+ }
+
+ if (info.isExist())
+ provideFailed( req->_spec.requestId()
+ , zyppng::ProvideMessage::Code::NotAFile
+ , zypp::str::Str() << "Path " << path << " exists, but its not a file"
+ , false
+ , {} );
+ else
+ provideFailed( req->_spec.requestId()
+ , zyppng::ProvideMessage::Code::NotFound
+ , zypp::str::Str() << "File " << path << " not found on medium"
+ , false
+ , {} );
+
+
+ break;
+ }
+ default: {
+ req->_state = zyppng::worker::ProvideWorkerItem::Finished;
+ provideFailed( req->_spec.requestId()
+ , zyppng::ProvideMessage::Code::BadRequest
+ , "Request type not implemented"
+ , false
+ , {} );
+ return;
+ }
+ }
+ } catch ( const zypp::Exception &e ) {
+ req->_state = zyppng::worker::ProvideWorkerItem::Finished;
+ provideFailed( req->_spec.requestId()
+ , zyppng::ProvideMessage::Code::BadRequest
+ , e.asString()
+ , false
+ , {} );
+ return;
+ } catch ( const std::exception &e ) {
+ req->_state = zyppng::worker::ProvideWorkerItem::Finished;
+ provideFailed( req->_spec.requestId()
+ , zyppng::ProvideMessage::Code::BadRequest
+ , e.what()
+ , false
+ , {} );
+ return;
+ } catch ( ... ) {
+ req->_state = zyppng::worker::ProvideWorkerItem::Finished;
+ provideFailed( req->_spec.requestId()
+ , zyppng::ProvideMessage::Code::BadRequest
+ , "Unknown exception"
+ , false
+ , {} );
+ return;
+ }
+ }
+
+ void MountingWorker::cancel( const std::deque<zyppng::worker::ProvideWorkerItemRef>::iterator &i )
+ {
+ ERR << "Bug, cancel should never be called for running items" << std::endl;
+ }
+
+ void MountingWorker::immediateShutdown()
+ {
+ _driver->immediateShutdown();
+ }
+}
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+
+#ifndef ZYPP_MEDIA_NG_WORKER_MOUNTINGWORKER_H_INCLUDED
+#define ZYPP_MEDIA_NG_WORKER_MOUNTINGWORKER_H_INCLUDED
+
+#include <zypp-media/ng/worker/ProvideWorker>
+#include <zypp-media/ng/worker/DeviceDriver>
+#include <zypp-core/zyppng/base/Signals>
+#include <any>
+#include <unordered_map>
+
+namespace zyppng::worker
+{
+ class MountingWorker : public zyppng::worker::ProvideWorker
+ {
+ public:
+ MountingWorker( std::string_view workerName, DeviceDriverRef driver );
+ ~MountingWorker();
+
+ void immediateShutdown() override;
+
+ protected:
+ // ProvideWorker interface
+ zyppng::expected<zyppng::worker::WorkerCaps> initialize(const zyppng::worker::Configuration &conf) override;
+ void provide() override;
+ void cancel( const std::deque<zyppng::worker::ProvideWorkerItemRef>::iterator &i ) override;
+
+ private:
+ DeviceDriverRef _driver;
+ bool _devicesDetected = false; //< We delay device detection to the first attach request, to avoid doing it without needing it
+ };
+}
+
+#endif
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+
+#include "provideworker.h"
+#include <zypp-core/base/DtorReset>
+#include <zypp-core/AutoDispose.h>
+#include <zypp-core/Url.h>
+#include <zypp-core/Date.h>
+#include <zypp-core/zyppng/pipelines/AsyncResult>
+#include <zypp-core/base/LogControl.h>
+#include <zypp-core/fs/PathInfo.h>
+#include <zypp-core/fs/TmpPath.h>
+#include <zypp-core/zyppng/base/private/threaddata_p.h>
+#include <zypp-core/zyppng/base/AutoDisconnect>
+#include <zypp-core/zyppng/base/EventDispatcher>
+#include <zypp-media/MediaConfig>
+#include <ostream>
+#include <fstream>
+
+#include <zypp-media/ng/private/providedbg_p.h>
+
+#undef ZYPP_BASE_LOGGER_LOGGROUP
+#define ZYPP_BASE_LOGGER_LOGGROUP "ProvideWorker"
+
+namespace zyppng::worker {
+
+ using namespace zyppng::operators;
+
+ RequestCancelException::RequestCancelException() : zypp::media::MediaException ("Request was cancelled")
+ { }
+
+ ProvideWorker::ProvideWorker(std::string_view workerName) : _workerName(workerName)
+ {
+ // do not change the order of these calls, otherwise showing the threadname does not work
+ // enableLogForwardingMode will initialize the log which would override the current thread name
+ zypp::base::LogControl::instance().enableLogForwardingMode( true );
+ ThreadData::current().setName( workerName );
+
+ // we use a singleshot timer that triggers message handling
+ connect( *_msgAvail, &Timer::sigExpired, *this, &ProvideWorker::messageLoop );
+ _msgAvail->setSingleShot(true);
+
+ // another timer to trigger a delayed shutdown
+ connectFunc( *_delayedShutdown, &Timer::sigExpired, [this]( zyppng::Timer & ) {
+ maybeDelayedShutdown();
+ }, *this );
+ _delayedShutdown->setSingleShot(true);
+ }
+
+ ProvideWorker::~ProvideWorker()
+ { }
+
+ RpcMessageStream::Ptr ProvideWorker::messageStream() const
+ {
+ return _stream;
+ }
+
+ expected<void> ProvideWorker::run( int recv, int send )
+ {
+ // reentry not supported
+ assert ( !_isRunning );
+
+ zypp::DtorReset res( _isRunning );
+ _isRunning = true;
+
+ initLog();
+
+ zypp::OnScopeExit cleanup([&](){
+ _stream.reset();
+ _controlIO.reset();
+ _loop.reset();
+ });
+
+ _controlIO = AsyncDataSource::create();
+ if ( !_controlIO->openFds( { recv }, send ) ) {
+ return expected<void>::error( ZYPP_EXCPT_PTR(zypp::Exception("Failed to open control FDs")) );
+ }
+
+ connect( *_controlIO, &AsyncDataSource::sigReadFdClosed, *this, &ProvideWorker::readFdClosed );
+ connect( *_controlIO, &AsyncDataSource::sigWriteFdClosed, *this, &ProvideWorker::writeFdClosed );
+
+ _stream = RpcMessageStream::create( _controlIO );
+
+ return executeHandshake () | mbind( [&]() {
+ AutoDisconnect disC[] = {
+ connect( *_stream, &RpcMessageStream::sigMessageReceived, *this, &ProvideWorker::messageReceived ),
+ connect( *_stream, &RpcMessageStream::sigInvalidMessageReceived, *this, &ProvideWorker::onInvalidMessageReceived )
+ };
+ _loop->run();
+ if ( _fatalError )
+ return expected<void>::error( _fatalError );
+ return expected<void>::success();
+ });
+ }
+
+ std::deque<ProvideWorkerItemRef> &ProvideWorker::requestQueue()
+ {
+ return _pendingProvides;
+ }
+
+ ProvideWorker::ProvideNotificatioMode ProvideWorker::provNotificationMode() const {
+ return _provNotificationMode;
+ }
+
+ void ProvideWorker::setProvNotificationMode( const ProvideNotificatioMode &provNotificationMode ) {
+ _provNotificationMode = provNotificationMode;
+ }
+
+ void ProvideWorker::initLog()
+ {
+ // by default we log to strErr, if user code wants to change that it can overload this function
+ zypp::base::LogControl::instance().logToStdErr();
+ }
+
+ ProvideWorkerItemRef ProvideWorker::makeItem( ProvideMessage &&spec )
+ {
+ return std::make_shared<ProvideWorkerItem>( std::move(spec) );
+ }
+
+ void ProvideWorker::provideStart(const uint32_t id, const zypp::Url &url, const zypp::filesystem::Pathname &localFile, const zypp::Pathname &stagingFile )
+ {
+ if ( !_stream->sendMessage( ProvideMessage::createProvideStarted ( id
+ , url
+ , localFile.empty () ? std::optional<std::string>() : localFile.asString ()
+ , stagingFile.empty () ? std::optional<std::string>() : stagingFile.asString ()
+ ).impl() ) ) {
+ ERR << "Failed to send ProvideStart message" << std::endl;
+ }
+ }
+
+ void ProvideWorker::provideSuccess(const uint32_t id, bool cacheHit, const zypp::filesystem::Pathname &localFile, const HeaderValueMap extra )
+ {
+ MIL_PRV << "Sending provideSuccess for id " << id << " file " << localFile << std::endl;
+ auto msg = ProvideMessage::createProvideFinished( id ,localFile.asString() ,cacheHit);
+ for ( auto i = extra.beginList (); i != extra.endList(); i++ ) {
+ for ( const auto &val : i->second )
+ msg.addValue( i->first, val );
+ }
+ if ( !_stream->sendMessage( msg.impl() ) ) {
+ ERR << "Failed to send ProvideSuccess message" << std::endl;
+ }
+ }
+
+ void ProvideWorker::provideFailed(const uint32_t id, const uint code, const std::string &reason, const bool transient, const HeaderValueMap extra )
+ {
+ MIL_PRV << "Sending provideFailed for request " << id << " err: " << reason << std::endl;
+ auto msg = ProvideMessage::createErrorResponse ( id, code, reason, transient );
+ for ( auto i = extra.beginList (); i != extra.endList(); i++ ) {
+ for ( const auto &val : i->second )
+ msg.addValue( i->first, val );
+ }
+ if ( !_stream->sendMessage( msg.impl() ) ) {
+ ERR << "Failed to send ProvideFailed message" << std::endl;
+ }
+ }
+
+
+ void ProvideWorker::provideFailed ( const uint32_t id, const uint code, const bool transient, const zypp::Exception &e )
+ {
+ zyppng::HeaderValueMap extra;
+ if ( !e.historyEmpty() ) {
+ extra = { { std::string(zyppng::ErrMsgFields::History), { zyppng::HeaderValueMap::Value(e.historyAsString()) }} };
+ }
+ provideFailed( id
+ , code
+ , e.asUserString()
+ , transient
+ , extra );
+ }
+
+
+ void ProvideWorker::attachSuccess(const uint32_t id)
+ {
+ MIL_PRV << "Sending attachSuccess for request " << id << std::endl;
+ if ( !_stream->sendMessage( ProvideMessage::createAttachFinished ( id ).impl() ) ) {
+ ERR << "Failed to send AttachFinished message" << std::endl;
+ } else {
+ MIL << "Sent back attach success" << std::endl;
+ }
+ }
+
+ void ProvideWorker::detachSuccess(const uint32_t id)
+ {
+ MIL_PRV << "Sending detachSuccess for request " << id << std::endl;
+ if ( !_stream->sendMessage( ProvideMessage::createDetachFinished ( id ).impl() ) ) {
+ ERR << "Failed to send DetachFinished message" << std::endl;
+ }
+ }
+
+ expected<ProvideMessage> ProvideWorker::sendAndWaitForResponse( const ProvideMessage &request , const std::vector<uint> &responseCodes )
+ {
+ // make sure immediateShutdown is not called while we are blocking here
+ zypp::DtorReset delayedReset( _inControllerRequest );
+ _inControllerRequest = true;
+
+ if ( !_stream->sendMessage( request.impl() ) )
+ return expected<ProvideMessage>::error( ZYPP_EXCPT_PTR(zypp::Exception("Failed to send message")) );
+
+ // flush the io device, this will block until all bytes are written
+ _controlIO->flush();
+
+ while ( !_fatalError ) {
+
+ const auto &msg = _stream->nextMessageWait() | [&]( auto &&nextMessage ) {
+ if ( !nextMessage ) {
+ if ( _fatalError )
+ return expected<RpcMessage>::error( _fatalError );
+ else
+ return expected<RpcMessage>::error( ZYPP_EXCPT_PTR(zypp::Exception("Failed to wait for response")) );
+ }
+ return expected<RpcMessage>::success( std::move(*nextMessage) );
+ } | mbind ( [&]( auto && m) {
+ return parseReceivedMessage(m);
+ } );
+
+ if ( !msg ) {
+ ERR << "Failed to receive message" << std::endl;
+ return msg;
+ }
+
+ if ( std::find( responseCodes.begin (), responseCodes.end(), msg->code() ) != responseCodes.end() ) {
+ return msg;
+ }
+
+ // remember other messages for later
+ MIL << "Remembering message for later: " << msg->code () << std::endl;
+ _pendingMessages.push_back(*msg);
+ _msgAvail->start(0);
+ }
+ return expected<ProvideMessage>::error( _fatalError );
+ }
+
+ ProvideWorker::MediaChangeRes ProvideWorker::requestMediaChange(const uint32_t id, const std::string &label, const int32_t mediaNr, const std::vector<std::string> &devices, const std::optional<std::string> &desc )
+ {
+ return sendAndWaitForResponse( ProvideMessage::createMediaChangeRequest ( id, label, mediaNr, devices, desc ), { ProvideMessage::Code::MediaChanged, ProvideMessage::Code::MediaChangeAbort, ProvideMessage::Code::MediaChangeSkip } )
+ | [&]( expected<ProvideMessage> &&m ) {
+ if ( !m ) {
+ MIL << "Failed to wait for message, aborting the request " << std::endl;
+ return ProvideWorker::MediaChangeRes::ABORT;
+ }
+ MIL << "Wait finished, with messages still pending: " << this->_pendingMessages.size() << " and provs still pending: " << this->_pendingProvides.size() << std::endl;
+ if ( m->code() == ProvideMessage::Code::MediaChanged )
+ return ProvideWorker::MediaChangeRes::SUCCESS;
+ else if ( m->code() == ProvideMessage::Code::MediaChangeSkip )
+ return ProvideWorker::MediaChangeRes::SKIP;
+ else
+ return ProvideWorker::MediaChangeRes::ABORT;
+ };
+ }
+
+ expected<AuthInfo> ProvideWorker::requireAuthorization( const uint32_t id, const zypp::Url &url, const std::string &lastTriedUsername, const int64_t lastTimestamp, const std::map<std::string, std::string> &extraFields )
+ {
+ return sendAndWaitForResponse( ProvideMessage::createAuthDataRequest( id, url, lastTriedUsername, lastTimestamp, extraFields ), { ProvideMessage::Code::AuthInfo, ProvideMessage::Code::NoAuthData } )
+ | mbind( [&]( ProvideMessage &&m ) {
+ if ( m.code() == ProvideMessage::Code::AuthInfo ) {
+
+ AuthInfo inf;
+ m.forEachVal( [&]( const std::string &name, const ProvideMessage::FieldVal &val ) {
+ if ( name == AuthInfoMsgFields::Username ) {
+ inf.username = val.asString();
+ } else if ( name == AuthInfoMsgFields::Password ) {
+ inf.password = val.asString();
+ } else if ( name == AuthInfoMsgFields::AuthTimestamp ) {
+ inf.last_auth_timestamp = val.asInt64();
+ } else {
+ if ( !val.isString() ) {
+ ERR << "Ignoring invalid extra value, " << name << " is not of type string" << std::endl;
+ }
+ inf.extraKeys[name] = val.asString();
+ }
+ return true;
+ });
+ return expected<AuthInfo>::success(inf);
+
+ }
+ return expected<AuthInfo>::error( ZYPP_EXCPT_PTR( zypp::media::MediaException("No Auth data")) );
+ });
+ }
+
+ AsyncDataSource &ProvideWorker::controlIO()
+ {
+ return *_controlIO.get();
+ }
+
+ expected<void> ProvideWorker::executeHandshake()
+ {
+ const auto &helo = _stream->nextMessageWait();
+ if ( !helo ) {
+ ERR << "Could not receive a handshake message, aborting" << std::endl;
+ return expected<void>::error( ZYPP_EXCPT_PTR(zypp::Exception("Failed to receive handshake message")) );;
+ }
+
+ auto exp = _stream->parseMessage<zypp::proto::Configuration>( *helo );
+ if ( !exp ) {
+ invalidMessageReceived( exp.error() );
+ return expected<void>::error(exp.error());
+ }
+
+ return std::move(*exp) | [&]( auto &&conf ) {
+
+ _workerConf = std::move(conf);
+
+ auto &mediaConf = zypp::MediaConfig::instance();
+ for( const auto &[key,value] : _workerConf.values() ) {
+ zypp::Url keyUrl( key );
+ if ( keyUrl.getScheme() == "zconfig" && keyUrl.getAuthority() == "main" ) {
+ mediaConf.setConfigValue( keyUrl.getAuthority(), zypp::Pathname(keyUrl.getPathName()).basename(), value );
+ }
+ }
+
+ return initialize( _workerConf ) | mbind([&]( WorkerCaps &&caps ){
+
+ caps.set_worker_name( _workerName.data() );
+
+ caps.set_cfg_flags ( WorkerCaps::Flags(caps.cfg_flags() | WorkerCaps::ZyppLogFormat) );
+ if ( !_stream->sendMessage ( caps ) ) {
+ return expected<void>::error( ZYPP_EXCPT_PTR(zypp::Exception("Failed to send capabilities")) );
+ }
+ return expected<void>::success ();
+ });
+ };
+ }
+
+ void ProvideWorker::messageLoop( Timer & )
+ {
+ if ( _fatalError )
+ return;
+
+ while ( _pendingMessages.size () ) {
+ auto m = _pendingMessages.front ();
+ _pendingMessages.pop_front ();
+ handleSingleMessage(m);
+ }
+
+ if ( !_fatalError && _pendingProvides.size() ) {
+ provide();
+ }
+
+ // keep poking until there are no provides anymore
+ if ( !_fatalError && ( _pendingMessages.size() || ( _pendingProvides.size () && _provNotificationMode == QUEUE_NOT_EMTPY ) ) ) {
+ _msgAvail->start(0);
+ }
+
+ }
+
+ void ProvideWorker::maybeDelayedShutdown()
+ {
+ if ( _inControllerRequest ) {
+ _delayedShutdown->start(0);
+ return;
+ }
+
+ immediateShutdown();
+ _loop->quit ();
+ }
+
+ void ProvideWorker::readFdClosed( uint, AsyncDataSource::ChannelCloseReason )
+ {
+ MIL << "Read FD closed, exiting." << std::endl;
+ maybeDelayedShutdown();
+ }
+
+ void ProvideWorker::writeFdClosed( AsyncDataSource::ChannelCloseReason )
+ {
+ MIL << "Write FD closed, exiting." << std::endl;
+ maybeDelayedShutdown();
+ }
+
+ void ProvideWorker::messageReceived()
+ {
+ while ( auto message = _stream->nextMessage() ) {
+ if ( _fatalError )
+ break;
+ pushSingleMessage(*message);
+ }
+ }
+
+ void ProvideWorker::onInvalidMessageReceived()
+ {
+ invalidMessageReceived( std::exception_ptr() );
+ }
+
+ void ProvideWorker::invalidMessageReceived( std::exception_ptr p )
+ {
+ ERR << "Received a invalid message on the input stream, aborting" << std::endl;
+ if ( p )
+ _fatalError = p;
+ else
+ _fatalError = ZYPP_EXCPT_PTR( InvalidMessageReceivedException() );
+ immediateShutdown();
+ _loop->quit();
+ }
+
+ void ProvideWorker::handleSingleMessage( const ProvideMessage &provide )
+ {
+ const auto code = provide.code();
+ // we only accept requests here
+ if ( code >= ProvideMessage::Code::FirstControllerCode && code <= ProvideMessage::Code::LastControllerCode ) {
+
+ MIL_PRV << "Received request: " << code << std::endl;
+
+ if ( code == ProvideMessage::Code::Cancel ) {
+ const auto &i = std::find_if( _pendingProvides.begin (), _pendingProvides.end(), [ id = provide.requestId() ]( const auto &it ){ return it->_spec.requestId() == id; } );
+ if ( i != _pendingProvides.end() ) {
+ switch ( (*i)->_state ) {
+ case ProvideWorkerItem::Pending:
+ _stream->sendMessage ( ProvideMessage::createErrorResponse ( provide.requestId (), ProvideMessage::Code::Cancelled, "Cancelled by user." ).impl() );
+ _pendingProvides.erase(i);
+ break;
+ case ProvideWorkerItem::Running:
+ cancel(i);
+ break;
+ case ProvideWorkerItem::Finished:
+ break;
+ }
+ MIL << "Received Cancel for unknown request: " << provide.requestId() << ", ignoring!" << std::endl;
+ }
+ return;
+ }
+
+ _pendingProvides.push_back( makeItem (ProvideMessage(provide)) );
+ return;
+ }
+ ERR << "Unsupported request with code: " << code << " received!" << std::endl;
+ }
+
+ void ProvideWorker::pushSingleMessage( const RpcMessage &message )
+ {
+ const auto &handle = [&]( const RpcMessage &message ){
+ const auto &msgTypeName = message.messagetypename();
+ if ( msgTypeName == rpc::messageTypeName<zypp::proto::ProvideMessage>() ) {
+ return parseReceivedMessage( message )
+ | mbind( [&]( ProvideMessage &&provide ){
+ _pendingMessages.push_back(provide);
+ _msgAvail->start(0);
+ return expected<void>::success();
+ });
+ }
+ return expected<void>::error( ZYPP_EXCPT_PTR( std::invalid_argument(zypp::str::Str()<<"Unknown message received: " << message.messagetypename())) );
+ };
+
+ const auto &exp = handle( message );
+ if ( !exp ) {
+ try {
+ std::rethrow_exception ( exp.error () );
+ } catch ( const zypp::Exception &e ) {
+ ERR << "Catched exception during message handling: " << e << std::endl;
+ } catch ( const std::exception &e ) {
+ ERR << "Catched exception during message handling: " << e.what()<< std::endl;
+ } catch ( ... ) {
+ ERR << "Unknown Exception during message handling" << std::endl;
+ }
+ }
+ }
+
+ expected<ProvideMessage> ProvideWorker::parseReceivedMessage(const RpcMessage &m)
+ {
+ auto exp = ProvideMessage::create(m);
+ if ( !exp )
+ invalidMessageReceived( exp.error() );
+ return exp;
+ }
+}
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+\---------------------------------------------------------------------*/
+
+#ifndef ZYPP_MEDIA_PROVIDE_WORKER_H_INCLUDED
+#define ZYPP_MEDIA_PROVIDE_WORKER_H_INCLUDED
+
+#include <zypp-core/zyppng/base/Base>
+#include <zypp-core/zyppng/base/EventLoop>
+#include <zypp-core/zyppng/base/Timer>
+#include <zypp-core/zyppng/io/AsyncDataSource>
+#include <zypp-core/zyppng/rpc/MessageStream>
+#include <zypp-core/zyppng/pipelines/Expected>
+#include <zypp-proto/provider.pb.h>
+#include <zypp-media/ng/provide-configvars.h>
+#include <zypp-media/ng/private/providemessage_p.h>
+#include <zypp-media/ng/HeaderValueMap>
+#include <zypp-media/MediaException>
+#include <zypp-media/Mount>
+
+#include <string_view>
+#include <deque>
+
+namespace zyppng::worker {
+
+ using WorkerCaps = zypp::proto::Capabilities;
+ using Message = zypp::proto::Envelope;
+ using Configuration = zypp::proto::Configuration;
+
+ struct AuthInfo
+ {
+ std::string username;
+ std::string password;
+ int64_t last_auth_timestamp = 0;
+ std::map<std::string, std::string> extraKeys = {};
+ };
+
+ class RequestCancelException : public zypp::media::MediaException
+ {
+ public:
+ RequestCancelException();
+ };
+
+ ZYPP_FWD_DECL_TYPE_WITH_REFS (ProvideWorker);
+ ZYPP_FWD_DECL_TYPE_WITH_REFS (ProvideWorkerItem);
+
+ class ProvideWorkerItem : public zyppng::Base
+ {
+ public:
+ enum State {
+ Pending,
+ Running,
+ Finished
+ };
+
+ ProvideWorkerItem( ProvideMessage &&spec ) : _spec( std::move(spec) ) { }
+
+ State _state = Pending;
+ ProvideMessage _spec;
+ };
+
+ class ProvideWorker : public Base
+ {
+ public:
+
+ enum ProvideNotificatioMode {
+ ONLY_NEW_PROVIDES, // provide is called only when new provide requests are added to the queue
+ QUEUE_NOT_EMTPY // provide is called continiously until the queue is empty
+ };
+
+ ProvideWorker( std::string_view workerName );
+ virtual ~ProvideWorker();
+
+ RpcMessageStream::Ptr messageStream() const;
+
+ expected<void> run ( int recv = STDIN_FILENO, int send = STDOUT_FILENO );
+
+ std::deque<ProvideWorkerItemRef> &requestQueue();
+ /*!
+ * Called when the worker process exits
+ */
+ virtual void immediateShutdown (){};
+
+ /*!
+ * This will request a media change from the user and BLOCK until it was acknowledged.
+ *
+ */
+ enum MediaChangeRes {
+ SUCCESS,
+ ABORT,
+ SKIP
+ };
+ MediaChangeRes requestMediaChange ( const uint32_t id, const std::string &label, const int32_t mediaNr, const std::vector<std::string> &devices, const std::optional<std::string> &desc = {} );
+
+ /*!
+ * This will send a authorization request message to the controller, asking for credentials for a given \a url.
+ * The \a lastTimstamp should be initialized with the last \ref AuthInfo timestamp that was received from the controller
+ * or -1 if none was received before.
+ *
+ * \note this blocks until a answer is received, all other received messages are delayed
+ */
+ expected<AuthInfo> requireAuthorization ( const uint32_t id, const zypp::Url &url, const std::string &lastTriedUsername = "", const int64_t lastTimestamp = -1, const std::map<std::string, std::string> &extraFields = {} );
+
+ ProvideNotificatioMode provNotificationMode() const;
+ void setProvNotificationMode(const ProvideNotificatioMode &provNotificationMode);
+
+ protected:
+ virtual void initLog();
+ virtual expected<WorkerCaps> initialize ( const Configuration &conf ) = 0;
+
+ /*!
+ * Automatically called whenever a new item is enqueued.
+ */
+ virtual void provide ( ) = 0;
+ virtual void cancel ( const std::deque<ProvideWorkerItemRef>::iterator &request ) = 0;
+
+ /*!
+ * Always called to create new items for the request queue,
+ * override this to populate the queue with instances of custom \ref ProvideItem subclasses.
+ *
+ * Cancel requests are directly handled by calling cancel(), however Attach and Detach requests are enqueued as well
+ */
+ virtual ProvideWorkerItemRef makeItem (ProvideMessage &&spec );
+
+ /*!
+ * Send a \a ProvideStart signal to the controller, this is to notify the controller that we have started providing the file
+ * the argument \a localFile has to refer to the file where the file will be provided into, it will be used to calculate
+ * statistics about download speed on the controller side.
+ * The \a stagingFile argument can be used for cases where the worker uses a staging area to download files into but later moves
+ * the file over to the result filename. In those cases the provider will check both locations for the file when calculating stats
+ *
+ * \note Always call \ref ref before sending this message.
+ */
+ void provideStart ( const uint32_t id, const zypp::Url &url, const zypp::Pathname &localFile, const zypp::Pathname &stagingFile = {} );
+
+ /*!
+ * Send a \a ProvideSuccess message to the controller. This is to signal that we are finished with providing a file
+ * and release the file to be used by the controller side.
+ */
+ void provideSuccess (const uint32_t id, bool cacheHit, const zypp::Pathname &localFile, const HeaderValueMap extra = {} );
+
+ /*!
+ * Send a \a ProvideFailed message to the controller. This is to signal that we are failed providing a resource
+ *
+ * \note If the request referenced a \a ident before make sure to manually release it after sending the message.
+ */
+ void provideFailed ( const uint32_t id, const uint code, const std::string &reason, const bool transient, const HeaderValueMap extra = {} );
+
+ /*!
+ * Overload of provideFailed that takes a \ref zypp::Exception to fill in the error details
+ *
+ * \note If the request referenced a \a ident before make sure to manually release it after sending the message.
+ */
+ void provideFailed ( const uint32_t id, const uint code, const bool transient, const zypp::Exception &e );
+
+ /*!
+ * Send a \a AttachSuccess message to the controller. This is to signal that we are finished with mounting and verifying a medium
+ */
+ void attachSuccess ( const uint32_t id );
+
+ /*!
+ * Send a \a DetachSuccess message to the controller. This is to signal that we are finished unmounting a medium
+ */
+ void detachSuccess ( const uint32_t id );
+
+ /*!
+ * Send a \a Redirect message to the controller for the given request ID. This is similar to sending a \a ProvideSuccess message
+ * and the request will be removed from the queue, which means the worker also has to remove it from its internal queue.
+ */
+ void redirect ( const uint32_t id, const zypp::Url &url, const zypp::Pathname &newPath );
+
+ /*!
+ * Returns the control IO datasource, only valid after \ref run was called
+ */
+ AsyncDataSource &controlIO ();
+
+
+ private:
+ expected<void> executeHandshake ();
+ void maybeDelayedShutdown ();
+ void messageLoop ( Timer & );
+ void readFdClosed ( uint, AsyncDataSource::ChannelCloseReason );
+ void writeFdClosed ( AsyncDataSource::ChannelCloseReason );
+ void messageReceived ();
+ void onInvalidMessageReceived ( );
+ void invalidMessageReceived ( std::exception_ptr p );
+ void handleSingleMessage (const ProvideMessage &provide );
+ void pushSingleMessage ( const RpcMessage &msg );
+ expected<ProvideMessage> sendAndWaitForResponse ( const ProvideMessage &request, const std::vector<uint> &responseCodes );
+ expected<ProvideMessage> parseReceivedMessage( const RpcMessage &m );
+
+ private:
+ ProvideNotificatioMode _provNotificationMode = QUEUE_NOT_EMTPY;
+ bool _inControllerRequest = false; //< Used to signalize that we are currently in a blocking controller callback
+ bool _isRunning = false;
+ std::string_view _workerName;
+ EventLoop::Ptr _loop = EventLoop::create();
+ Timer::Ptr _msgAvail = Timer::create();
+ Timer::Ptr _delayedShutdown = Timer::create();
+ AsyncDataSource::Ptr _controlIO;
+ RpcMessageStream::Ptr _stream;
+ Configuration _workerConf;
+
+ std::exception_ptr _fatalError; //< Error that caused the eventloop to stop
+
+ std::deque<ProvideMessage> _pendingMessages;
+ std::deque<ProvideWorkerItemRef> _pendingProvides;
+ };
+}
+
+
+#endif
)
SET( zypp_media_PROTOBUF_SOURCES
- media/transfersettings.proto
- media/download.proto
- media/messages.proto
- media/networkrequesterror.proto
+ media/provider.proto
)
SET( zypp_target_PROTOBUF_SOURCES
target/commit.proto
)
+SET( zypp_test_PROTOBUF_SOURCES
+ test/tvm.proto
+)
+
protobuf_generate_cpp( ZYPPCORE_PROTO_SRCS ZYPPCORE_PROTO_HDRS ${zypp_core_PROTOBUF_SOURCES} )
protobuf_generate_cpp( ZYPPMEDIA_PROTO_SRCS ZYPPMEDIA_PROTO_HDRS ${zypp_media_PROTOBUF_SOURCES} )
protobuf_generate_cpp( ZYPPTARGET_PROTO_SRCS ZYPPTARGET_PROTO_HDRS ${zypp_target_PROTOBUF_SOURCES} )
+protobuf_generate_cpp( ZYPPTEST_PROTO_SRCS ZYPPTEST_PROTO_HDRS ${zypp_test_PROTOBUF_SOURCES} )
-
-ADD_LIBRARY( zypp-protobuf STATIC ${ZYPPCORE_PROTO_SRCS} ${ZYPPCORE_PROTO_HDRS} ${ZYPPMEDIA_PROTO_SRCS} ${ZYPPMEDIA_PROTO_HDRS} ${ZYPPTARGET_PROTO_HDRS} ${ZYPPTARGET_PROTO_SRCS} )
-
+ADD_LIBRARY( zypp-protobuf STATIC
+ ${ZYPPCORE_PROTO_SRCS} ${ZYPPCORE_PROTO_HDRS}
+ ${ZYPPMEDIA_PROTO_SRCS} ${ZYPPMEDIA_PROTO_HDRS}
+ ${ZYPPTARGET_PROTO_HDRS} ${ZYPPTARGET_PROTO_SRCS}
+ ${ZYPPTEST_PROTO_HDRS} ${ZYPPTEST_PROTO_SRCS}
+)
+++ /dev/null
-syntax = "proto3";
-option optimize_for = LITE_RUNTIME;
-
-import "transfersettings.proto";
-
-package zypp.proto;
-
-message Checksum {
- string type = 1;
- bytes sum = 2;
-}
-
-/*!
- * Specifies a file that needs to be downloaded
- */
-message DownloadSpec {
- string url = 1;
- TransferSettings settings = 2;
- string delta = 3;
- uint64 expectedFileSize = 4;
- string targetPath = 5;
- bool checkExistanceOnly = 6; //< this will NOT download the file, but only query the server if it exists
- bool metalink_enabled = 7; //< should the download try to use metalinks
- uint32 headerSize = 8; //< Optional file header size for things like zchunk
- Checksum headerChecksum = 9; //< Optional file header checksum
- uint32 preferred_chunk_size = 10;
-}
+++ /dev/null
-syntax = "proto3";
-option optimize_for = LITE_RUNTIME;
-
-import "transfersettings.proto";
-import "download.proto";
-import "networkrequesterror.proto";
-
-package zypp.proto;
-
-// the download is silently enqueued and prefetched
-// will return a Status object to the sender
-message Request {
- uint32 requestId = 1;
- DownloadSpec spec = 2;
- bool prioritize = 3; //< this is usually set when a download is required right away, it will be enqueued with highest prio
- bool streamProgress = 4; //< immediately start to stream download updates
-}
-
-// the downloads are silently enqueued and prefetched
-// will return a Status object to the sender
-message Prefetch {
- uint32 requestId = 1;
- repeated Request requests = 2;
-}
-
-// Will reprioritize a download that is not yet started
-message Prioritize {
- uint32 requestId = 1;
-}
-
-// Download has been started in the server, updates are streamed if streamProgress was
-// set to true in the initial request or if SubscribeProgress was issued for this request
-message DownloadStart {
- uint32 requestId = 1;
- string url = 2;
-}
-
-// constant updates about the state of the download
-// if total is NOT set, it is an "Alive" update only
-message DownloadProgress {
- uint32 requestId = 1;
- string url = 2;
- uint64 total = 3;
- uint64 now = 4;
-}
-
-// last message sent about a tracked download
-message DownloadFin {
- uint32 requestId = 1;
- NetworkRequestError error = 2;
- int64 last_auth_timestamp = 3; //< this field is only set if a timestamp from CredentialManager was used
-}
-
-// will start streaming updates about download download progress
-// usually only download start and finished messages are sent
-message SubscribeProgress {
- uint32 requestId = 1;
- bool prioritize = 2; //< setting this flag will also prioritize the download
-}
-
-// will stop streaming updates about download download progress
-// usually only download start and finished messages are sent
-message UnSubscribeProgress {
- uint32 requestId = 1;
-}
-
-// This message can only be sent when a download is tracked.
-// Otherwise, it is silenty ignored without even a return code.
-// The expected answer message for this is a DownloadFin with
-// its status code set to cancelled (if it was not finished before).
-message CancelDownload {
- uint32 requestId = 1;
-}
-
-// message sent to server to indicate that new auth data was stored
-// in the credential manager, this will restart all requests currently
-// sitting in AuthFailed status
-message NewAuthDataAvailable {
- uint32 requestId = 1;
- TransferSettings settings = 2;
-}
-
-// generic status object, showing if a request that has no explicit return message
-// was successful or not. However it can also be returned in place of any other response message
-// which usually means there is a error we can not recover from.
-message Status {
- enum Code {
- Ok = 0;
- InvalidMessage = 1; //< The message could not be parsed from the stream, this will close the stream
- UnknownRequest = 2; //< The Payload in the request was valid but unknown
- MalformedRequest = 3; //< The payload type in the request was known, but it was not possible to deserialize it
- UnknownId = 4; //< The request ID was not found
- }
- uint32 requestId = 1;
- Code code = 2;
- string reason = 3; //< if rejected is false, this will contain the reason why
-}
-
-
-
+++ /dev/null
-syntax = "proto3";
-option optimize_for = LITE_RUNTIME;
-
-package zypp.proto;
-
-message NetworkRequestError {
- /**
- * This will always correspond to the enum value specified in NetworkRequestError::Type
- */
- int32 error = 1;
- string errorDesc = 2;
- string nativeError = 3;
- map<string, string> extra_info = 4;
-}
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+----------------------------------------------------------------------*/
+
+syntax = "proto3";
+option optimize_for = LITE_RUNTIME;
+
+package zypp.proto;
+
+/*!
+ General:
+ -------
+ This file contains the protocol messages used in media worker backends. These backends are
+ used by zypp mainly to acquire files from different types of media via a generalized API. However,
+ it can also be used to implement helpers for special needs, e.g. copying files around or rewriting URLs.
+
+ Every worker type is identified by the schema used in a URL e.g. dvd for getting files from DVDs or https for downloading.
+ This makes it possible to easily extend zypp with support for new schemes by implementing a new worker type without the need
+ to modify zypp itself. A important functionality is the possibility to redirect URLs to different workers, making it possible
+ to implement special schemes that need to remodel a URL in special ways before they are processed by the general backends.
+
+ Communication Format:
+ ---------------------
+ Each message is serialized into a zypp.proto.Envelope and sent over the communication medium in binary
+ format. The binary format looks like:
+
+ +--------------------------------+---------------------------------+
+ | msglen ( 32 bit unsigned int ) | binary zypp.proto.Envelope data |
+ +--------------------------------+---------------------------------+
+
+ The header defines the size in bytes of the following data trailer. The header type is a 32 bit uint, endianess is defined by
+ the underlying CPU arch. The data portion is directly generated by libprotobuf via SerializeToZeroCopyStream() to generate
+ the binary represenation of the message.
+
+
+ Communication channel:
+ ----------------------
+ Communication between the worker processes and the zypp main process will happen via the standard unix file descriptors:
+ stdin, stdout and stderr. stdin and stdout are used to send and receive messages between main and worker processes, while
+ stderr is used to output logging that is generated by the worker processes. These logs are forwarded to the main zypp log.
+
+ The workers are supposed to shut down as soon as their stdin is closed. This is important to not block the controller process
+ that is waiting for this to happen, to fetch all log lines that are emitted by the worker when shutting down.
+
+
+ Worker Requirements:
+ --------------------
+ Async communication:
+ Every worker needs to be able to handle messages asynchronously, the frontend will send requests to the worker as
+ they are given by the frontend code and does not apply any ordering. This means workers need to remember queries they sent to the frontend and manage
+ the code waiting for the result accordingly. The only exception to this is the handshake messages, where the frontend will always
+ send the worker configuration first, directly after spawning the worker process and the worker will answer with its capabilities.
+ Once those messages have been exchanged, zypp will start to send requests as defined below.
+ The frontend will buffer messages sent to the workers, so they do not need to keep the socket empty, however if possible workers should
+ support handling multiple requests.
+
+ Cleaning up:
+ ------------
+ A worker needs to maintain a mountpoint/download location as long as needed. Since only the frontend can decide when a location
+ is not required anymore it will send a "ReleaseResource" message with a worker generated ID that releases said ressource.
+ Examples for this are mountpoints that a worker created or download areas that were used by the worker to < the files to
+
+ Worker Types:
+ -------------
+ Currently zypp knows 4 different types of workers:
+
+ * Downloading: Simplest type of worker that just download the files from a remote location without the need to mount something beforehand.
+ This is the simplest file provider that does not need to implement special functions. It is possible to freely redirect requests
+ between different variations of this worker type, all other types of workers do NOT support redirections.
+ Per remote host a own instance is started except if the SingleInstance flag is set. It is adviceable to set the SingleInstance flag on all
+ workers that do not do any long running requests.
+
+ * Mounting: This type of worker needs to mount filesystems, but is not restricted by local volatile devices. For example smb, nfs, dir or even disk.
+ It has to additionally implement the Attach/Verify workflow.
+ Mounting handlers are always started as SingleInstance workers.
+
+ * VolatileMounting: Most complex type of workers that mounts local devices that are volatile. This worker needs to also implement the device change workflow.
+ This type of worker exists only to implement the dvd worker.
+ VolatileMounting handlers are started as SingleInstance workers.
+
+ * CPU Bound: Simple zypp internal workhorse type that can be implemented to run async processes like extracting a compressed file or creating a checksum.
+ For this type of worker a own instance per CPU is started on demand by default.
+
+ Attach/Mount Workflow ( does not apply to Downloading and CPU bound worker types ) :
+ Each handler that marks itself as Mounting or VolatileMounting needs to implement the attach/mount workflow. This means that the controller needs to rely
+ on the handler to correctly mount and identify a medium. In the Attach message the controller may send media identification data and identification type to the worker
+ to specify the exact medium it is looking for. The worker then needs to mount the filesystem and perform the validation check, if the check succeeds the worker
+ will assign the unique attach point ID it has received from the contoller to the attachpoint and sends back a AttachResult containing the information.
+ At this point the worker needs to keep the mountpoint and attachpoint around until the controller sends a Detach request for it.
+ If the verification fails the worker will unmount the filesystem and send back a AttachResult message with the error bits set.
+ If the controller does not send verification data, every device/remoteFS that is successfully mounted is considered valid.
+
+ After a medium was mounted the controller will send provide requests to the worker that have URLs formed like:
+ <workerscheme>-media://<attachId>/path/to/file. For example: dvd-media://1d6c0810-2bd6-45f3-9890-0268422a6f14/path/to/file , the attachID part is just a example, the controller uses
+ generates IDs that can be expressed as a URL hostname.
+ Another important aspect to keep in mind, is that a attach point is not the same as a mount point. Since a attach point additionally contains the path relative to the
+ filesystem root on the device. For example: the repository drivers is in a subdirectory of a dvd, so we'd get a attach request for the base url: dvd:/drivers, this would
+ result in a attachpoint id of "abcdef123" the device would be mounted in "/mnt/device". All subsequent requests are relative to the attachpoint and would look like:
+ dvd-media://abcdef123/file , which results in a lookup for the file: "/mnt/device/drivers/file1".
+
+ Due to historical reasons a single device/filesystem can contain multiple mediums. So the worker needs to make sure to not unmount the filesystem too early.
+ For example dvd:/repo-1 and dvd:/repo-2 , both mediums are on the same disc, hence the worker will need to maintain two mount IDs and can only release the device itself
+ when both have been detached.
+ In some cases its even possible that multiple mediums reside in the same (sub)directory on a filesystem. For example when a multi DVD set was copied together into
+ one directory shared over NFS.
+
+
+ Message status codes
+ ---------------------
+ The response codes are roughtly modelled like HTTP status codes, they can happen during the full runtime of a certain request.
+
+ Informational messages (100–199)
+ Success messages (200–299)
+ Redirection messages (300–399)
+ Client error messages (400–499)
+ Server error messages (500–599)
+ Controller messages (600–699)
+ Worker messages (700-799)
+
+
+ - Code: 100 - Provide started
+ Desc: Sent to the controller once a worker starts providing a request
+ Note that this is a optional message, its perfectly valid if 200 ( Provide Success ) is sent directly after the file was requested.
+ Fields:
+ required string url
+ string local_filename -> The local filename where the file will be provided to, used by the controller to track progress
+ string staging_filename -> The staging file used while downloading
+
+
+ - Code: 200 - Provide Finished
+ Desc: Sent to the controller once a worker finished a job successfully.
+ Fields:
+ required string local_filename -> The path where the worker has placed the file
+ required bool cacheHit -> Set to true if the file was found in a worker cache
+
+
+ - Code: 201 - Attach Finished
+ Desc: Sent to the controller once a worker mounted a medium successfully.
+ Fields:
+ - None -
+
+ - Code: 202 - Auth Info
+ Desc: Response sent by the controller to a AUTH_REQUIRED request.
+ The last_auth_timestamp contains the last change of the auth database. This timestamp should be included
+ in successive AUTH_REQUIRED requests, so that the controller knows if its outdated or if the Auth was read from the store once
+ and now we need to ask the user to provide new auth info.
+ Fields:
+ required string username
+ required string password
+ required int64 auth_timestamp -> timestamp of the auth data, this should be stored in the worker in case it fails again
+ optional string authType -> comma seperated list of selected auth types, used by the network provider
+
+
+ - Code: 203 - Media changed
+ Desc: Sent to the worker once the user acknowledged a media change request
+ Fields:
+ - None -
+
+ - Code: 204 - Detach Finished
+ Desc: Sent to the controller once a worker detached a medium successfully.
+ Fields:
+ - None -
+
+
+ - Code: 300 - Redirect
+ Desc: A Downloading worker can generate a Redirect message to reroute a request to a different
+ Downloading worker. This can be used to implement special handlers where a URL needs to be rewritten
+ before it can actually be fetched.
+ This is similar to sending a 200 - Provide Finished message since it will take the request out of the workers queue
+
+ Note: Redirecting is ONLY supported between Downloading workers. Sending this message from other worker types
+ or redirecting to non Dowloading workers results in failing the user request with a redirect error.
+ Fields:
+ required string new_url -> the full URL we want to redirect to
+
+
+ - Code: 301 - Metalink Redirect
+ Desc: A worker can generate a list of mirrors, this is similar to a Redirect response but supports
+ giving the Controller a list of possible URLs to try.
+ This is similar to sending a 200 - Provide Finished message since it will take the request out of the workers queue.
+ The mirror list is ordered by priority, meaning the first mirror has the highest.
+ Fields:
+ repeated string new_url -> full URL we want to redirect to
+
+
+
+ 4xx and 5xx messages, all of those messages have the same Response type structure:
+
+ Code: 4xx / 5xx - Client error
+ Desc: Sent to the controller for requests that fail due to a client error
+ Fields:
+ required string reason -> The reason as a string
+ optional string history -> The error history that lead to the error
+ bool transient -> If this is set to true, the controller will try to request the file again at a later time
+
+ Currently reserved error codes:
+ 400: Bad Request -> invalid request, the receiver can not process it
+ 401: Unauthorized -> no auth info avail but auth required
+ extraFields: string authHint
+ 402: Forbidden -> auth was given but failed or no access rights to the requested resource
+ extraFields: string authHint
+ 403: PeerCertificateInvalid -> the peer certificate validation failed
+ 404: Not Found -> the requested resource does not exist on the medium
+ 405: Expected Size Exceeded -> the downloaded data exceeded the requested maximum lenght
+ 406: Connection failed -> connecting to the server failed
+ 407: Timeout -> the request timed out
+ 408: Cancelled -> request was cancelled by the user
+ 409: Invalid checksum -> the downloaded data has a different checksum than expected
+ 410: Mount failed -> mounting the requested resource failed
+ 411: Jammed -> ATTACH request failed due to insufficient resources
+ 412: Media Change Abort -> User decided to abort the request
+ 413: Media Change Skip -> User decided to abort the request with a skip command( request will fail but will report "skipped" as the reason )
+ 414: No Auth data -> Only sent by the controller for a 700 - Auth Data Request
+ 416: Media not desired -> The desired medium was not found on the given URL ( attach request )
+
+ 500: InternalError -> a error in the worker that is not recoverable, check reason string
+
+ Controller -> Worker requests
+ -----------------------------
+
+ - Code: 600 - Provide
+ Desc: Message to tell the worker about a ressource it should provide
+ Fields:
+ required string url -> The URL of the request
+ string filename -> target filename hint, this is not required to be considered by the workers
+ string delta_file -> local path to a file that is supposed to be used for delta downloads
+ int64 expected_filesize -> The expected download filesize, workers should fail if a server does not reports the exact same filesize
+ bool check_existance_only -> this will NOT download the file but only query the server if its existant
+ bool metalink_enabled -> enables/disables metalink handling
+
+ - Code: 601 - Cancel
+ Desc: Sent by the controller if a request should be cancelled. The worker should stop the given request and return a
+ 408 response.
+ Fields:
+ - None -
+
+ - Code: 602 - Attach
+ Desc: If a worker signals the frontend it needs to mount/unmount resources the controller will first send a Attach
+ message for each base_url it encounters that targets the workers scheme. All workers that do not set this flag
+ do not need to care about this special message.
+ In fact receiving this message in a worker that is not a mounting type should result
+ in returning a error to the controller.
+
+ If the verifyType field is NOT set, the verifyData field will be ignored and the worker does not need to
+ verify the device on the given location.
+ Fields:
+ required string url -> the base URL we want to attach to
+ required string attach_id -> controller generated ID string to uniquely identiy a attached medium
+ required string label -> Label of the medium
+ string verify_type -> name of the verification type, currently only suseV1 ( required if verify_data is present )
+ bytes verify_data -> verification data ( required if verify_type is present )
+ int32 media_nr -> the nr of the medium in the set ( required if verify_type is present )
+ repeated string device -> optional field containing a device that can be used for attachment
+
+ - Code: 603 - Detach
+ Desc: Send by the controller process when a medium is no longer required.
+ The worker has to immediately unmount the medium, no answer to the controller is expected.
+ Since there is no connected request, the request ID in this message will not be considered
+ Fields:
+ required string url -> the attachment URL containing the controller generated ID string to uniquely identify a attached medium, e.g. dvd://<attachId>/
+
+
+ Worker -> Controller requests
+ -----------------------------
+
+ - Code: 700 - Auth Data Request
+ Desc: In case a worker needs authorization info it can generate this request.
+ The last_auth_timestamp field is present if the request had received auth info before.
+ Fields:
+ required string effective_url -> the effective URL we want to fetch, this is not necessarily the same as the request URL
+ int64 last_auth_timestamp -> timestamp of previously received auth informationpJuds
+ optional string username -> username tried in the last auth attempt
+ optional string authHint -> comma seperated list of available authentication types
+ optional repeated string <keyname> -> all other key:value pairs are treated as extra keys.
+ Possible Resposes: 414, 202
+
+
+ - Code: 701 - Media Change Request
+ Desc: Generated by a worker to ask the user to insert a different medium. This will BLOCK the controller and the handler
+ until the user answered the request. This is a special request only used by the CDROM handler
+ Fields:
+ required string label -> name of the medium we need
+ required int32 media_nr -> which medium from the media set is required
+ required repeated string device -> free device to be used to insert the media into
+ string desc -> medium desc
+ Possible Resposes: 412, 413, 203
+
+*/
+
+
+enum MessageCodes {
+ option allow_alias = true;
+ NoCode = 0;
+ FirstInformalCode = 100;
+ ProvideStarted = 100;
+ LastInformalCode = 199;
+
+ FirstSuccessCode = 200;
+ ProvideFinished = 200;
+ AttachFinished = 201;
+ AuthInfo = 202;
+ MediaChanged = 203;
+ DetachFinished = 204;
+ LastSuccessCode = 299;
+
+ FirstRedirCode = 300;
+ Redirect = 300;
+ Metalink = 301;
+ LastRedirCode = 399;
+
+ FirstClientErrCode = 400;
+ BadRequest = 400;
+ Unauthorized = 401;
+ Forbidden = 402;
+ PeerCertificateInvalid = 403;
+ NotFound = 404;
+ ExpectedSizeExceeded = 405;
+ ConnectionFailed = 406;
+ Timeout = 407;
+ Cancelled = 408;
+ InvalidChecksum = 409;
+ MountFailed = 410;
+ Jammed = 411;
+ MediaChangeAbort = 412;
+ MediaChangeSkip = 413;
+ NoAuthData = 414;
+ NotAFile = 415;
+ MediumNotDesired = 416;
+ LastClientErrCode = 499;
+
+ FirstSrvErrCode = 500;
+ InternalError = 500;
+ ProtocolError = 501;
+ LastSrvErrCode = 599;
+
+ FirstControllerCode = 600;
+ Provide = 600;
+ Cancel = 601;
+ Attach = 602;
+ Detach = 603;
+ LastControllerCode = 699;
+
+ FirstWorkerCode = 700;
+ AuthDataRequest = 700;
+ MediaChangeRequest = 701;
+ LastWorkerCode = 799;
+}
+
+message DataField {
+ string key = 1;
+ oneof field_val {
+ string str_val = 2;
+ int32 int_val = 3;
+ int64 long_val = 4;
+ double double_val = 5;
+ bool bool_val = 6;
+ }
+}
+
+/*!
+ * This is the general message used to communicate between workers and the controller.
+ * Based on the message_code a message is either a response, signal or request.
+ * Every communication is initiated by a request, the ID given in the request message is used
+ * through the lifetime of the request to link successive requests to the initial request.
+ */
+message ProvideMessage {
+ uint32 request_id = 1; //< the request ID set by the process initiating a request
+ uint32 message_code = 2; //< the response code, see documentation
+ repeated DataField fields = 3;
+}
+
+// --------------------------------------------------------------------------
+// ----------------- Handshake message Controller -> Worker -----------------
+// --------------------------------------------------------------------------
+
+message Configuration {
+ map<string, string> values = 1; // A key value map of all worker related configurations
+}
+
+
+// --------------------------------------------------------------------------
+// ----------------- Handshake Message Worker -> Controller -----------------
+// --------------------------------------------------------------------------
+
+/*!
+ * Message that describes be capabilities of the worker process.
+ * This ALWAYS has to be sent to the controller as the first message immediately after receiving the Configuration message
+ */
+message Capabilities {
+
+ /*! The worker type, see the description in Worker Types above */
+ enum WorkerType {
+ Invalid = 0;
+ Downloading = 1;
+ SimpleMount = 2;
+ VolatileMount = 3;
+ CPUBound = 4;
+ }
+
+ enum Flags {
+ None = 0; // Just to satisfy the protobuf compiler
+ SingleInstance = 1; // If this flag is set a worker can only be started once, this is implicit in some worker types.
+ Pipeline = 2; // The worker can handle multiple requests at the same time
+ ZyppLogFormat = 4; // The worker writes messages to stderr in zypp log format
+ FileArtifacts = 8; // The results of this worker are artifacts, which means they need to be cleaned up. This is implicit for all downloading workers. For all mounting workers this is ignored.
+ // CPU bound workers can use it to signal they leave artifact files behind that need to be cleaned up
+ }
+
+ uint32 protocol_version = 1; // The workers should set this field to the protocol version they implement.
+ WorkerType worker_type = 2;
+ Flags cfg_flags = 3;
+ string worker_name = 4;
+ repeated DataField fields = 5; // customs fields for future extensions
+}
+++ /dev/null
-syntax = "proto3";
-option optimize_for = LITE_RUNTIME;
-
-package zypp.proto;
-
-message TransferSettings {
- repeated string header = 1;
- string useragent = 2;
- string username = 3;
- string password = 4;
- bool useproxy = 5;
- string proxy = 6;
- string proxy_username = 7;
- string proxy_password = 8;
- string authtype = 9;
- int64 timeout = 10;
- int64 connect_timeout = 11;
- string url = 12;
- string targetdir = 13;
-
-
- int64 maxConcurrentConnections = 14;
- int64 minDownloadSpeed = 15;
- int64 maxDownloadSpeed = 16;
- int64 maxSilentTries = 17;
-
- bool verify_host = 18;
- bool verify_peer = 19;
- string ca_path = 20;
- string client_cert_path = 21;
- string client_key_path = 22;
-
- // workarounds
- bool head_requests_allowed = 23;
-}
--- /dev/null
+/*---------------------------------------------------------------------\
+| ____ _ __ __ ___ |
+| |__ / \ / / . \ . \ |
+| / / \ V /| _/ _/ |
+| / /__ | | | | | | |
+| /_____||_| |_| |_| |
+| |
+----------------------------------------------------------------------*/
+
+syntax = "proto3";
+option optimize_for = LITE_RUNTIME;
+
+package zypp.proto.test;
+
+/*!
+ Helper settings for the TVM ( Test Volatile Mouning ) worker. This can be
+ used to define the number of available devices and which of those have inserted media.
+
+ This data is written to a file in the provider dir for the tvm worker to read.
+ The worker only reads the data while the test only writes it. This way we can simulate
+ a user inserting/changing CDs
+ */
+
+message TVMSettings
+{
+ message Device {
+ string name = 1; // the device name
+ string insertedPath = 2; // the path what is currently inserted into the device
+ }
+ repeated Device devices = 1;
+}
DESTINATION ${INCLUDE_INSTALL_DIR}/zypp/media/proxyinfo
)
-add_subdirectory( zyppng )
-
SET( zypp_lib_SRCS
${zypp_misc_SRCS}
${zypp_pool_SRCS}
macro( ADDZYPPLIB LIBNAME )
message( "ADDING lib ${LIBNAME} to project" )
- ADD_LIBRARY( ${LIBNAME} SHARED $<TARGET_OBJECTS:zypp-objlib> $<TARGET_OBJECTS:zyppng-objlib> )
+ ADD_LIBRARY( ${LIBNAME} SHARED $<TARGET_OBJECTS:zypp-objlib> )
SET_TARGET_PROPERTIES( ${LIBNAME} PROPERTIES VERSION "${LIBZYPP_VERSION_INFO}" )
SET_TARGET_PROPERTIES( ${LIBNAME} PROPERTIES SOVERSION "${LIBZYPP_SOVERSION_INFO}" )
# System libraries
TARGET_LINK_LIBRARIES(${LIBNAME} ${UTIL_LIBRARY} )
TARGET_LINK_LIBRARIES(${LIBNAME} ${RPM_LIBRARY} )
TARGET_LINK_LIBRARIES(${LIBNAME} ${GETTEXT_LIBRARIES} )
- TARGET_LINK_LIBRARIES(${LIBNAME} ${CURL_LIBRARIES} )
- TARGET_LINK_LIBRARIES(${LIBNAME} ${LIBXML2_LIBRARIES} )
+ #TARGET_LINK_LIBRARIES(${LIBNAME} ${CURL_LIBRARIES} )
+ #TARGET_LINK_LIBRARIES(${LIBNAME} ${LIBXML2_LIBRARIES} )
TARGET_LINK_LIBRARIES(${LIBNAME} ${ZLIB_LIBRARY} )
TARGET_LINK_LIBRARIES(${LIBNAME} ${LibSolv_LIBRARIES} )
- TARGET_LINK_LIBRARIES(${LIBNAME} ${SIGNALS_LIBRARY})
+ #TARGET_LINK_LIBRARIES(${LIBNAME} ${SIGNALS_LIBRARY})
TARGET_LINK_LIBRARIES(${LIBNAME} ${Boost_THREAD_LIBRARY})
TARGET_LINK_LIBRARIES(${LIBNAME} ${GPGME_PTHREAD_LIBRARIES})
target_link_libraries(${LIBNAME} ${YAML_CPP_LIBRARIES})
target_link_libraries(${LIBNAME} ${PROTOBUF_LITE_LIBRARIES})
TARGET_LINK_LIBRARIES(${LIBNAME} pthread )
- IF (ENABLE_ZSTD_COMPRESSION)
- TARGET_LINK_LIBRARIES(${LIBNAME} ${ZSTD_LIBRARY})
- ENDIF (ENABLE_ZSTD_COMPRESSION)
-
- IF (ENABLE_ZCHUNK_COMPRESSION)
- TARGET_LINK_LIBRARIES(${LIBNAME} ${ZCHUNK_LDFLAGS})
- ENDIF(ENABLE_ZCHUNK_COMPRESSION)
-
IF ( UDEV_FOUND )
TARGET_LINK_LIBRARIES(${LIBNAME} ${UDEV_LIBRARY} )
ELSE ( UDEV_FOUND )
ENDIF ( HAL_FOUND )
ENDIF ( UDEV_FOUND )
- TARGET_LINK_LIBRARIES( ${LIBNAME} ${LIBPROXY_LIBRARIES} )
-
endmacro()
#Release library stripped from most symbols, thats what we release to the packages
#include <iosfwd>
#include <list>
#include <zypp-core/base/DefaultIntegral>
+#include <zypp-media/FileCheckException>
#include <zypp/base/Exception.h>
#include <zypp/base/Function.h>
#include <zypp/PathInfo.h>
*/
typedef function<void ( const Pathname &file )> FileChecker;
- class FileCheckException : public Exception
- {
- public:
- FileCheckException(const std::string &msg)
- : Exception(msg)
- {}
- };
-
- class CheckSumCheckException : public FileCheckException
- {
- public:
- CheckSumCheckException(const std::string &msg)
- : FileCheckException(msg)
- {}
- };
-
- class SignatureCheckException : public FileCheckException
- {
- public:
- SignatureCheckException(const std::string &msg)
- : FileCheckException(msg)
- {}
- };
-
/**
* Built in file checkers
*/
return _status;
}
+ ResStatus & statusReinit() const
+ {
+ _status.setLock( _status.isUserLockQueryMatch(), zypp::ResStatus::USER );
+ _status.resetTransact( zypp::ResStatus::USER );
+ return _status;
+ }
+
public:
bool isUndetermined() const
{
ResStatus & PoolItem::status() const { return _pimpl->status(); }
ResStatus & PoolItem::statusReset() const { return _pimpl->statusReset(); }
+ ResStatus & PoolItem::statusReinit() const { return _pimpl->statusReinit(); }
sat::Solvable PoolItem::buddy() const { return _pimpl->buddy(); }
void PoolItem::setBuddy( const sat::Solvable & solv_r ) { _pimpl->setBuddy( solv_r ); }
bool PoolItem::isUndetermined() const { return _pimpl->isUndetermined(); }
/** Returns the current status. */
ResStatus & status() const;
- /** Reset status. */
+ /** Resets status to the default state (KEEP_STATE bySOLVER; clears any lock!). */
ResStatus & statusReset() const;
+
+ /** Resets status to it's initial state in the ResPool (KEEP_STATE bySOLVER or LOCKED byUSER).
+ * The state depends on whether the PoolItem matched a hard lock defined
+ * in /etc/zypp/locks or not.
+ */
+ ResStatus & statusReinit() const;
//@}
{ return isToBeUninstalled() && fieldValueIs<TransactDetailField>( SOFT_REMOVE ); }
private:
-
- /** \name Internal hard lock maintainance */
+ /** \name Internal hard lock maintenance */
//@{
friend struct resstatus::UserLockQueryManip;
- bool isUserLockQueryMatch() const
- { return fieldValueIs<UserLockQueryField>( USERLOCK_MATCH ); }
-
void setUserLockQueryMatch( bool match_r )
{ fieldValueAssign<UserLockQueryField>( match_r ? USERLOCK_MATCH : USERLOCK_NOMATCH ); }
//@}
+ public:
+ bool isUserLockQueryMatch() const
+ { return fieldValueIs<UserLockQueryField>( USERLOCK_MATCH ); }
public:
//------------------------------------------------------------------------
- // get/set functions, returnig \c true if requested status change
+ // get/set functions, returning \c true if requested status change
// was successful (i.e. leading to the desired transaction).
// If a lower level (e.g.SOLVER) wants to transact, but it's
// already set by a higher level, \c true should be returned.
/** Apply a lock (prevent transaction).
* Currently by USER or APPL_HIGH only, but who knows...
- * Set LOCKED from KEEP_STATE to be shure all transaction
+ * Set LOCKED from KEEP_STATE to be sure all transaction
* details were reset properly.
*/
bool setLock( bool toLock_r, TransactByValue causer_r )
/** Toggle between TRANSACT and KEEP_STATE.
* LOCKED state means KEEP_STATE. But in contrary to KEEP_STATE,
* LOCKED state is immutable for \a causer_r less than TransactByValue.
- * KEEP_STATE may be canged by any \a causer_r.
+ * KEEP_STATE may be changed by any \a causer_r.
*/
bool setTransact( bool toTansact_r, TransactByValue causer_r )
{
\---------------------------------------------------------------------*/
/** \file zypp/base/DrunkenBishop.cc
*/
+#include <cstdint>
#include <iostream>
//#include <zypp/base/LogTools.h>
#include <zypp/base/Flags.h>
#include <zypp/base/Logger.h>
#include <zypp/ExternalProgram.h>
#include <zypp-media/Mount>
+#include <zypp-media/CDTools>
#include <zypp/media/MediaCD.h>
#include <zypp/media/MediaManager.h>
#include <zypp/Url.h>
*/
#define REPORT_EJECT_ERRORS 0
-/*
-** If defined to the full path of the eject utility,
-** it will be used additionally to the eject-ioctl.
-*/
-#define EJECT_TOOL_PATH "/bin/eject"
-
-
//////////////////////////////////////////////////////////////////
namespace zypp
//
bool MediaCD::openTray( const std::string & device_r )
{
- int fd = ::open( device_r.c_str(), O_RDONLY|O_NONBLOCK|O_CLOEXEC );
- int res = -1;
-
- if ( fd != -1)
- {
- res = ::ioctl( fd, CDROMEJECT );
- ::close( fd );
- }
-
- if ( res )
- {
- if( fd == -1)
- {
- WAR << "Unable to open '" << device_r
- << "' (" << ::strerror( errno ) << ")" << endl;
- }
- else
- {
- WAR << "Eject " << device_r
- << " failed (" << ::strerror( errno ) << ")" << endl;
- }
-
-#if defined(EJECT_TOOL_PATH)
- DBG << "Try to eject " << device_r << " using "
- << EJECT_TOOL_PATH << " utility" << std::endl;
-
- const char *cmd[3];
- cmd[0] = EJECT_TOOL_PATH;
- cmd[1] = device_r.c_str();
- cmd[2] = NULL;
- ExternalProgram eject(cmd, ExternalProgram::Stderr_To_Stdout);
-
- for(std::string out( eject.receiveLine());
- out.length(); out = eject.receiveLine())
- {
- DBG << " " << out;
- }
-
- if(eject.close() != 0)
- {
- WAR << "Eject of " << device_r << " failed." << std::endl;
- return false;
- }
-#else
- return false;
-#endif
- }
- MIL << "Eject of " << device_r << " successful." << endl;
- return true;
+ return CDTools::openTray(device_r);
}
///////////////////////////////////////////////////////////////////
//
bool MediaCD::closeTray( const std::string & device_r )
{
- int fd = ::open( device_r.c_str(), O_RDONLY|O_NONBLOCK|O_CLOEXEC );
- if ( fd == -1 ) {
- WAR << "Unable to open '" << device_r << "' (" << ::strerror( errno ) << ")" << endl;
- return false;
- }
- int res = ::ioctl( fd, CDROMCLOSETRAY );
- ::close( fd );
- if ( res ) {
- WAR << "Close tray " << device_r << " failed (" << ::strerror( errno ) << ")" << endl;
- return false;
- }
- DBG << "Close tray " << device_r << endl;
- return true;
+ return CDTools::closeTray(device_r);
}
if ( _curl )
{
- curl_easy_cleanup( _curl );
+ // bsc#1201092: Strange but within global_dtors we may exceptions here.
+ try { curl_easy_cleanup( _curl ); }
+ catch (...) { ; }
_curl = NULL;
}
}
std::string
MediaHandler::getRealPath(const std::string &path)
{
- std::string real;
- if( !path.empty())
- {
-#if __GNUC__ > 2
- /** GNU extension */
- char *ptr = ::realpath(path.c_str(), NULL);
- if( ptr != NULL)
- {
- real = ptr;
- free( ptr);
- }
- else
- /** the SUSv2 way */
- if( EINVAL == errno)
- {
- char buff[PATH_MAX + 2];
- memset(buff, '\0', sizeof(buff));
- if( ::realpath(path.c_str(), buff) != NULL)
- {
- real = buff;
- }
- }
-#else
- char buff[PATH_MAX + 2];
- memset(buff, '\0', sizeof(buff));
- if( ::realpath(path.c_str(), buff) != NULL)
- {
- real = buff;
- }
-#endif
- }
- return real;
+ return getRealPath(zypp::Pathname(path)).asString();
}
zypp::Pathname
MediaHandler::getRealPath(const Pathname &path)
{
- return zypp::Pathname(getRealPath(path.asString()));
+ return path.realpath();
}
if ( ref.attachPoint->path != Pathname(e->dir) )
continue; // at least the mount points must match
- bool is_device = false;
- PathInfo dev_info;
- if( str::hasPrefix( Pathname(e->src).asString(), "/dev/" ) &&
- dev_info(e->src) && dev_info.isBlk() )
- {
- is_device = true;
- }
-
+ bool is_device = e->isBlockDevice();
if( is_device && (ref.mediaSource->maj_nr &&
ref.mediaSource->bdir.empty()))
{
+ PathInfo dev_info(e->src);
std::string mtype(matchMountFs ? e->type : ref.mediaSource->type);
MediaSource media(mtype, e->src, dev_info.devMajor(), dev_info.devMinor());
#include <zypp/media/MediaISO.h>
#include <zypp/media/MediaPlugin.h>
#include <zypp/media/UrlResolverPlugin.h>
-#include <zypp/zyppng/media/MediaNetwork>
namespace zypp::media {
_handler = std::make_unique<MediaCIFS> (url,preferred_attach_point);
else if (scheme == "ftp" || scheme == "tftp" || scheme == "http" || scheme == "https")
{
- enum WhichHandler { choose, curl, multicurl, network };
+ enum WhichHandler { choose, curl, multicurl };
WhichHandler which = choose;
// Leagcy: choose handler in UUrl query
if ( const std::string & queryparam = url.getQueryParam("mediahandler"); ! queryparam.empty() ) {
if ( queryparam == "network" )
- which = network;
+ which = multicurl;
else if ( queryparam == "multicurl" )
which = multicurl;
else if ( queryparam == "curl" )
};
if ( getenvIs( "ZYPP_MEDIANETWORK", "1" ) ) {
- WAR << "network backend manually enabled." << std::endl;
- which = network;
+ WAR << "network backend preview was removed, defaulting to multicurl." << std::endl;
+ which = multicurl;
}
else if ( getenvIs( "ZYPP_MULTICURL", "0" ) ) {
WAR << "multicurl manually disabled." << std::endl;
// Finally use the default
std::unique_ptr<MediaNetworkCommonHandler> handler;
switch ( which ) {
- case network:
- handler = std::make_unique<zyppng::MediaNetwork>( url, preferred_attach_point );
- break;
-
default:
case multicurl:
handler = std::make_unique<MediaMultiCurl>( url, preferred_attach_point );
}
}
-
-
static inline time_t
getMountTableMTime()
{
- time_t mtime = zypp::PathInfo("/etc/mtab").mtime();
- if( mtime <= 0)
- {
- WAR << "Failed to retrieve modification time of '/etc/mtab'"
- << std::endl;
- }
- return mtime;
+ return Mount::getMTime();
}
static inline MountEntries
bool SUSEMediaVerifier::isDesiredMedia( const media::MediaHandler & ref ) const
{
- bool ret = true; // optimistic return unless we definitely now it does not match
+ bool ret = true; // optimistic return unless we definitely know it does not match
const SMVData & smvData = _pimpl->smvData();
if ( ! smvData )
sat::SolvableSpec needrebootSpec;
needrebootSpec.addProvides( Capability("installhint(reboot-needed)") );
needrebootSpec.addProvides( Capability("kernel") );
- needrebootSpec.addIdent( IdString("kernel-firmware") );
Pathname needrebootFile { Pathname::assertprefix( root(), ZConfig::instance().needrebootFile() ) };
if ( PathInfo( needrebootFile ).isFile() )
prog->addFd( scriptPipe->writeFd );
// set up the AsyncDataSource to read script output
- if ( !scriptSource->open( scriptPipe->readFd ) )
+ if ( !scriptSource->openFds( std::vector<int>{ scriptPipe->readFd } ) )
ZYPP_THROW( target::rpm::RpmSubprocessException( "Failed to open scriptFD to subprocess" ) );
prog->sigStarted().connect( [&](){
messagePipe->unrefWrite();
scriptPipe->unrefWrite();
- // read the stdout and forward it to our log
- prog->stdoutDevice()->connectFunc( &zyppng::IODevice::sigReadyRead, [&](){
- while( prog->stdoutDevice()->canReadLine() ) {
- L_DBG("zypp-rpm") << "<stdout> " << prog->stdoutDevice()->readLine().asStringView(); // no endl! - readLine does not trim
- }
- });
-
- // read the stderr and forward it to our log
- prog->stderrDevice()->connectFunc( &zyppng::IODevice::sigReadyRead, [&](){
- while( prog->stderrDevice()->canReadLine() ) {
- L_ERR("zypp-rpm") << "<stderr> " << prog->stderrDevice()->readLine().asStringView(); // no endl! - readLine does not trim
+ // read the stdout and stderr and forward it to our log
+ prog->connectFunc( &zyppng::IODevice::sigChannelReadyRead, [&]( int channel ){
+ while( prog->canReadLine( channel ) ) {
+ L_ERR("zypp-rpm") << ( channel == zyppng::Process::StdOut ? "<stdout> " : "<stderr> " ) << prog->channelReadLine( channel ).asStringView(); // no endl! - readLine does not trim
}
});
// this is the source for control messages from zypp-rpm , we will get structured data information
// in form of protobuf messages
- if ( !msgSource->open( messagePipe->readFd ) )
+ if ( !msgSource->openFds( std::vector<int>{ messagePipe->readFd } ) )
ZYPP_THROW( target::rpm::RpmSubprocessException( "Failed to open read stream to subprocess" ) );
zyppng::rpc::HeaderSizeType pendingMessageSize = 0;
// make sure to read all data from the log source
bool readMsgs = false;
- while( prog->stderrDevice()->canReadLine() ) {
+ while( prog->canReadLine( zyppng::Process::StdErr ) ) {
readMsgs = true;
- MIL << "zypp-rpm: " << prog->stderrDevice()->readLine().asStringView();
+ MIL << "zypp-rpm: " << prog->channelReadLine( zyppng::Process::StdErr ).asStringView();
}
- while( prog->stdoutDevice()->canReadLine() ) {
+ while( prog->canReadLine( zyppng::Process::StdOut ) ) {
readMsgs = true;
- MIL << "zypp-rpm: " << prog->stdoutDevice()->readLine().asStringView();
+ MIL << "zypp-rpm: " << prog->channelReadLine( zyppng::Process::StdOut ).asStringView();
}
while ( scriptSource->canReadLine() ) {
+++ /dev/null
-SET( zyppng_media_SRCS
- media/medianetwork.cc
- media/medianetworkserver.cc
-)
-
-SET( zyppng_media_HEADERS
- media/MediaNetwork
- media/medianetwork.h
-)
-
-SET( zyppng_media_private_HEADERS
- media/private/medianetworkserver_p.h
-)
-
-SET( zyppng_media_private_SOURCES
-)
-
-SET( zyppng_appcode_HEADERS
- Context
- context.h
-)
-
-SET( zyppng_appcode_SRCS
- context.cc
-)
-
-SET ( zyppng_HEADERS
- ${zyppng_appcode_HEADERS}
- ${zyppng_media_HEADERS}
- ${zyppng_media_private_HEADERS}
-)
-
-SET ( zyppng_SRCS
- ${zyppng_appcode_SRCS}
- ${zyppng_media_SRCS}
- ${zyppng_media_private_SOURCES}
-)
-
-ADD_LIBRARY( zyppng-objlib OBJECT ${zyppng_SRCS} ${zyppng_HEADERS} )
-#we include generated headers, so we need to wait for zypp-protobuf to be ready
-add_dependencies( zyppng-objlib zypp-protobuf )
+++ /dev/null
-#include "context.h"
+++ /dev/null
-#include "context.h"
-#include <zypp-core/zyppng/base/EventLoop>
-#include <zypp-curl/ng/network/private/mirrorcontrol_p.h>
-
-namespace zyppng {
-
- Context::Context()
- {
- _zyppEventLoop = EventLoop::create();
- _mirrorControl = MirrorControl::create();
- }
-
- std::shared_ptr<EventLoop> Context::evLoop() const
- {
- return _zyppEventLoop;
- }
-
- std::shared_ptr<MirrorControl> Context::mirrorControl()
- {
- return _mirrorControl;
- }
-
-}
+++ /dev/null
-#include "medianetwork.h"
+++ /dev/null
-/*---------------------------------------------------------------------\
-| ____ _ __ __ ___ |
-| |__ / \ / / . \ . \ |
-| / / \ V /| _/ _/ |
-| / /__ | | | | | | |
-| /_____||_| |_| |_| |
-| |
-----------------------------------------------------------------------*/
-#include "medianetwork.h"
-#include "private/medianetworkserver_p.h"
-
-#include <zypp-media/MediaException>
-#include <zypp-core/zyppng/base/private/linuxhelpers_p.h>
-#include <zypp-core/zyppng/base/EventLoop>
-#include <zypp-core/zyppng/base/AutoDisconnect>
-#include <zypp-core/zyppng/io/SockAddr>
-#include <zypp-curl/ng/network/DownloadSpec>
-
-#include <zypp-curl/ng/network/NetworkRequestDispatcher>
-#include <zypp-curl/ng/network/NetworkRequestError>
-#include <zypp-curl/ng/network/private/mediadebug_p.h>
-#include <zypp-core/zyppng/base/EventDispatcher>
-#include <zypp-core/zyppng/base/AbstractEventSource>
-#include <zypp-core/zyppng/rpc/rpc.h>
-
-#include <zypp-curl/private/curlhelper_p.h>
-#include <zypp-media/MediaException>
-#include <zypp-media/auth/CredentialManager>
-#include <zypp/ZConfig.h>
-#include <zypp/ZYppCallbacks.h>
-#include <zypp/base/String.h>
-#include <zypp/base/Gettext.h>
-#include <zypp/ZYppFactory.h>
-
-#include <zypp/ZYppCallbacks.h>
-
-#undef ZYPP_BASE_LOGGER_LOGGROUP
-#define ZYPP_BASE_LOGGER_LOGGROUP "zypp::MediaNetwork"
-
-namespace zyppng {
-
- using HeaderSizeType = zyppng::rpc::HeaderSizeType;
-
- struct MediaNetwork::ProgressData
- {
- ProgressData( const zyppng::Url & _url = zyppng::Url(),
- zypp::callback::SendReport<zypp::media::DownloadProgressReport> *_report = nullptr )
- : url( _url )
- , report( _report )
- {}
-
- zypp::Url url;
- zypp::callback::SendReport<zypp::media::DownloadProgressReport> *report;
-
- time_t _timeStart = 0; ///< Start total stats
- time_t _timeLast = 0; ///< Start last period(~1sec)
-
- double _dnlTotal = 0.0; ///< Bytes to download or 0 if unknown
- double _dnlLast = 0.0; ///< Bytes downloaded at period start
- double _dnlNow = 0.0; ///< Bytes downloaded now
-
- int _dnlPercent= 0; ///< Percent completed or 0 if _dnlTotal is unknown
-
- double _drateTotal= 0.0; ///< Download rate so far
- double _drateLast = 0.0; ///< Download rate in last period
-
- void updateStats( double dltotal = 0.0, double dlnow = 0.0 )
- {
- time_t now = time(0);
-
- // If called without args (0.0), recompute based on the last values seen
- if ( dltotal && dltotal != _dnlTotal )
- _dnlTotal = dltotal;
-
- if ( dlnow && dlnow != _dnlNow )
- {
- _dnlNow = dlnow;
- }
- else if ( !_dnlNow && !_dnlTotal )
- {
- // Start time counting as soon as first data arrives.
- // Skip the connection / redirection time at begin.
- return;
- }
-
- // init or reset if time jumps back
- if ( !_timeStart || _timeStart > now )
- _timeStart = _timeLast = now;
-
- // percentage:
- if ( _dnlTotal )
- _dnlPercent = int(_dnlNow * 100 / _dnlTotal);
-
- // download rates:
- _drateTotal = _dnlNow / std::max( int(now - _timeStart), 1 );
-
- if ( _timeLast < now )
- {
- _drateLast = (_dnlNow - _dnlLast) / int(now - _timeLast);
- // start new period
- _timeLast = now;
- _dnlLast = _dnlNow;
- }
- else if ( _timeStart == _timeLast )
- _drateLast = _drateTotal;
- }
-
- void reportProgress( bool &cancel ) const
- {
- if ( report && !(*report)->progress( _dnlPercent, url, _drateTotal, _drateLast ) )
- cancel = true; // user requested abort
- }
- };
-
-
- struct MediaNetwork::Request {
-
- Request ( DownloadSpec &&spec, const zypp::Pathname &path, RequestId id ) : _targetFile(path) {
- _targetFile.autoCleanup( true );
- _proto.set_requestid( id );
-
- // download to temporary path instead of the real one
- spec.setTargetPath( _targetFile.path().asString() );
- *_proto.mutable_spec() = std::move( spec.protoData() );
- }
-
- void ref () {
- _requestCount++;
- }
-
- int unref () {
- _requestCount--;
- return _requestCount;
- }
-
- void startReporting ( ProgressData &tracker ) {
- _report = &tracker;
- }
-
- void stopReporting () {
- _report = nullptr;
- }
-
- void setProgress ( double dltotal, double dlnow, bool &cancel ) {
- if ( _report ) {
- _report->updateStats( dltotal, dlnow );
- _report->reportProgress( cancel );
- }
- }
-
- void reset () {
- _result.reset();
- }
-
- const std::optional<zypp::proto::DownloadFin> &result () const {
- return _result;
- }
-
- void setResult ( zypp::proto::DownloadFin &&res ) {
- _result = std::move(res);
- }
-
- zypp::proto::Request &proto () {
- return _proto;
- }
-
- const zypp::proto::Request &proto () const {
- return _proto;
- }
-
- private:
- zypp::proto::Request _proto;
- std::optional<zypp::proto::DownloadFin> _result;
- zypp::filesystem::TmpFile _targetFile;
- ProgressData *_report = nullptr;
- int _requestCount = 1;
- };
-
-
- /*!
- * This object helps us to simulate sync behaviour over async code. We can block waiting
- * for a specific message or response while still handling intermediate message that represent
- * status updates from concurrently running requests.
- *
- * The DispatchContext needs to be released after finishing the operation that was requested from
- * the sync entry points are done. It will tear down the event loop and release the socket fd so we
- * can reuse it at a later point again.
- */
- struct MediaNetwork::DispatchContext {
-
- DispatchContext( MediaNetwork &p ) : parent(&p), ev( zyppng::EventLoop::create() ){
- if ( parent->_socket ) {
- MIL_MEDIA << "Reusing open connection" << std::endl;
- sock = zyppng::Socket::fromSocket( *parent->_socket, zyppng::Socket::ConnectedState );
- } else {
- MIL_MEDIA << "Connecting to backend" << std::endl;
- sock = zyppng::Socket::create( AF_UNIX, SOCK_STREAM, 0 );
- parent->_readBuffer.clear();
-
- while ( true ) {
- sock->connect( std::make_shared<zyppng::UnixSockAddr>( MediaNetworkThread::instance().sockPath(), true ) );
- if ( !sock->waitForConnected() ) {
- if ( sock->lastError() == Socket::ConnectionRefused )
- continue;
- sock.reset();
- } else {
- parent->_socket = sock->nativeSocket();
- MIL_MEDIA << "Connected" << std::endl;
- }
- break;
- }
-
- }
- }
- ~DispatchContext() {
- // if the socket was disconnected we can not keep the fd around anymore
- if ( !sock || sock->state() == Socket::ClosedState ) {
- parent->_socket.reset();
-
- // if the socket is destroyed the read buffer won't make sense anymore
- parent->_readBuffer.clear();
- } else {
- // make sure no data is lost before we release the socket descriptor
- if ( sock->bytesAvailable() )
- parent->_readBuffer.append( sock->readAll() );
- sock->waitForAllBytesWritten();
- sock->releaseSocket();
- }
- }
-
- /*!
- * Send a messagee to the server side, it will be enclosed in a Envelope
- * and immediately sent out.
- */
- template <typename T>
- bool sendEnvelope ( const T &m ) {
- zypp::proto::Envelope env;
-
- env.set_messagetypename( m.GetTypeName() );
- m.SerializeToString( env.mutable_value() );
-
- //DBG << "Preparing to send messagE: " << env.messagetypename() << " " << env.value().size() << std::endl;
-
- const auto &str = env.SerializeAsString();
- HeaderSizeType msgSize = str.length();
- sock->write( (char *)(&msgSize), sizeof( HeaderSizeType ) );
- sock->write( str.data(), str.size() );
- return true;
- }
-
- /*!
- * Executes the EventLoop until a zypp.proto.Status message was received. The
- * Status is returned to the caller. The \a handleOtherMsg callback works just like
- * in \ref waitFor.
- */
- template <typename T>
- zypp::proto::Status waitForStatus ( const RequestId reqId, T &&handleOtherMsg ) {
- zypp::proto::Status res;
- dispatchUntil( [ &reqId, &res, otherMsgCB = std::forward<T>(handleOtherMsg), this ]( const zypp::proto::Envelope &env ){
- if ( env.messagetypename() == "zypp.proto.Status" ) {
- zypp::proto::Status s;
- s.ParseFromString( env.value() );
-
- // we received a out of band response, there is only ONE active request to the
- // server allowed at any time
- if ( s.requestid() != reqId ) {
- ERR_MEDIA << "Out of band status for a request that was not active" << std::endl;
- ZYPP_THROW( zypp::media::MediaException( "Out of band status for a request that was not active" ) );
- }
- res = std::move(s);
- return true;
- }
- // not the message we wanted, process it otherwise
- otherMsgCB( *this, env );
- return false;
- });
- return res;
- }
-
- /*!
- * Executes the EventLoop until a Message of type T was received, the
- * received message is returned to the caller.
- * The \a handleOtherMsg callback is invoked for all messages that are received
- * but are of a different type than the one requested.
- */
- template <typename T, typename Callback>
- T waitFor ( const RequestId reqId, Callback &&handleOtherMsg ) {
- T res;
- dispatchUntil( [ &reqId, &res, otherMsgCB = std::forward<Callback>(handleOtherMsg), this ]( const zypp::proto::Envelope &env ){
- if ( env.messagetypename() == T::default_instance().GetTypeName() ) {
- T s;
- s.ParseFromString( env.value() );
- if ( s.requestid() == reqId ) {
- res = std::move(s);
- return true;
- }
- }
- // not the message we wanted, process it otherwise
- otherMsgCB( *this, env );
- return false;
- });
- return res;
- }
-
- /*!
- * This function executes the EventLoop and keep it running until the predicate
- * returns true. All messages that are received are passed into the predicate,
- * allowing us to implement eventhandling outside of actual async code.
- */
- template <typename Predicate>
- void dispatchUntil( Predicate &&predicate ) const
- {
- if ( !ev || !sock )
- return;
-
- std::optional<int32_t> pendingMessageSize;
- std::exception_ptr excp;
-
- bool stopRequested = false;
-
- auto &readBuf = parent->_readBuffer;
-
- const auto &readMessages = [ &, pred = std::forward<Predicate>(predicate), this ]( ){
-
- // read all data from socket into internal buffer
- readBuf.append( sock->readAll() );
-
- try {
- while ( readBuf.size() ) {
- if ( !pendingMessageSize ) {
- // if we cannot read the message size wait some more
- if ( readBuf.size() < sizeof( HeaderSizeType ) )
- return;
-
- HeaderSizeType msgSize;
- readBuf.read( reinterpret_cast<char *>( &msgSize ), sizeof( decltype (msgSize) ) );
- pendingMessageSize = msgSize;
- }
-
- // wait for the full message size to be available
- if ( readBuf.size() < static_cast<size_t>( *pendingMessageSize ) )
- return;
-
- ByteArray message ( *pendingMessageSize, '\0' );
- readBuf.read( message.data(), *pendingMessageSize );
- pendingMessageSize.reset();
-
- zypp::proto::Envelope m;
- if (! m.ParseFromArray( message.data(), message.size() ) ) {
- //abort, we can not recover from this. Bytes might be mixed up on the socket
- ZYPP_THROW( zypp::media::MediaException("MediaNetwork backend connection broken.") );
- }
-
- //MIL << "Dispatching message " << m.DebugString() << std::endl;
-
- if ( pred(m) ) {
- stopRequested = true;
- ev->quit();
- return;
- }
- }
- } catch ( ... ) {
- // we cannot let the exception travel through the event loop, instead we remember it for later
- stopRequested = true;
- excp = std::current_exception();
- sock->abort();
- ev->quit();
- }
- };
-
- auto rrConn = AutoDisconnect( sock->connectFunc( &zyppng::Socket::sigReadyRead, readMessages ) );
-
- auto disConn = AutoDisconnect ( sock->connectFunc( &zyppng::Socket::sigDisconnected, [&excp, this](){
- try {
- ZYPP_THROW( zypp::media::MediaException("MediaNetwork backend disconnected.") );
- } catch ( ... ) {
- // we cannot let the exception travel through the event loop, instead we remember it for later
- excp = std::current_exception();
- ev->quit();
- }
- }) );
-
- readMessages(); // read all pending messages
- if ( !stopRequested )
- ev->run();
-
- if ( excp )
- std::rethrow_exception( excp );
-
- // if we reach this part, we can not have pending message data
- if ( pendingMessageSize ) ERR << "This is a bug, message was partially read and event loop closed" << std::endl;
- assert( !pendingMessageSize );
- }
-
- MediaNetwork *parent;
- zyppng::EventLoop::Ptr ev;
- zyppng::Socket::Ptr sock;
- };
-
- MediaNetwork::MediaNetwork( const Url & url_r, const zypp::Pathname & attach_point_hint_r )
- : MediaNetworkCommonHandler( url_r, attach_point_hint_r,
- "/", // urlpath at attachpoint
- true ) // does_download
- {
- _workingDir.autoCleanup( true );
-
- MIL << "MediaNetwork::MediaNetwork(" << url_r << ", " << attach_point_hint_r << ")" << std::endl;
-
- if( !attachPoint().empty()){
- zypp::PathInfo ainfo(attachPoint());
- zypp::Pathname apath(attachPoint() + "XXXXXX");
- char *atemp = ::strdup( apath.asString().c_str());
- char *atest = NULL;
- if( !ainfo.isDir() || !ainfo.userMayRWX() ||
- atemp == NULL || (atest=::mkdtemp(atemp)) == NULL) {
- WAR << "attach point " << ainfo.path()
- << " is not useable for " << url_r.getScheme() << std::endl;
- setAttachPoint("", true);
- }
- else if( atest != NULL)
- ::rmdir(atest);
-
- if( atemp != NULL)
- ::free(atemp);
- }
- }
-
- MediaNetwork::~MediaNetwork() {
- DBG << "Releasing MediaNetwork with tmpdir: " << _workingDir.path().asString() << std::endl;
- try { release(); } catch(...) {}
-
- if ( _socket ) {
- DBG_MEDIA << "Explicitely calling socket cleanup, this should've happened during release()" << std::endl;
- zyppng::eintrSafeCall( ::close, *_socket );
- _socket.reset();
- }
- }
-
- std::unique_ptr<MediaNetwork::DispatchContext> MediaNetwork::ensureConnected() const
- {
- auto ctx = std::make_unique<DispatchContext>( *const_cast<MediaNetwork *>(this) );
- if ( !ctx->sock )
- ZYPP_THROW( zypp::media::MediaException("Failed to connect to backend") );
- return ctx;
- }
-
- void MediaNetwork::disconnectFrom()
- {
- if ( _socket ) {
- zyppng::eintrSafeCall( ::close, *_socket );
- _socket.reset();
- }
-
- _readBuffer.clear();
- _requests.clear();
- _nextRequestId = 0;
-
- MediaHandler::disconnectFrom();
- }
-
- void MediaNetwork::attachTo(bool next)
- {
- if ( next )
- ZYPP_THROW( zypp::media::MediaNotSupportedException(_url) );
-
- if ( !_url.isValid() )
- ZYPP_THROW( zypp::media::MediaBadUrlException(_url) );
-
- if( !NetworkRequestDispatcher::supportsProtocol( _url ) ) {
- std::string msg("Unsupported protocol '");
- msg += _url.getScheme();
- msg += "'";
- ZYPP_THROW( zypp::media::MediaBadUrlException(_url, msg) );
- }
-
- if( !isUseableAttachPoint( attachPoint() ) )
- {
- setAttachPoint( createAttachPoint(), true );
- }
-
- disconnectFrom();
-
- // FIXME: need a derived class to propelly compare url's
- zypp::media::MediaSourceRef media( new zypp::media::MediaSource(_url.getScheme(), _url.asString()) );
- setMediaSource(media);
- }
-
- void MediaNetwork::releaseFrom(const std::string &)
- {
- disconnect();
- }
-
- Url MediaNetwork::getFileUrl( const zypp::Pathname & filename_r ) const
- {
- // Simply extend the URLs pathname. An 'absolute' URL path
- // is achieved by encoding the leading '/' in an URL path:
- // URL: ftp://user@server -> ~user
- // URL: ftp://user@server/ -> ~user
- // URL: ftp://user@server// -> ~user
- // URL: ftp://user@server/%2F -> /
- // ^- this '/' is just a separator
- Url newurl( _url );
- newurl.setPathName( ( zypp::Pathname("./"+_url.getPathName()) / filename_r ).asString().substr(1) );
- return newurl;
- }
-
- void MediaNetwork::handleRequestResult(const Request &req, const zypp::filesystem::Pathname &filename ) const
- {
- if ( !req.result() ) {
- ZYPP_THROW( zypp::media::MediaCurlException( req.proto().spec().url(), "Request did not return a result" , "" ) );
- }
-
- if ( !req.result()->has_error() )
- return;
-
- const auto &err = req.result()->error();
- if ( err.error() != NetworkRequestError::NoError ) {
- Url reqUrl = err.extra_info().at("requestUrl");
- switch ( err.error() )
- {
- case NetworkRequestError::Unauthorized: {
- std::string hint = err.extra_info().at("authHint");
- ZYPP_THROW( zypp::media::MediaUnauthorizedException(
- reqUrl, err.errordesc(), err.nativeerror(), hint
- ));
- break;
- }
- case NetworkRequestError::TemporaryProblem:
- ZYPP_THROW( zypp::media::MediaTemporaryProblemException(reqUrl) );
- break;
- case NetworkRequestError::Timeout:
- ZYPP_THROW( zypp::media::MediaTimeoutException(reqUrl) );
- break;
- case NetworkRequestError::Forbidden:
- ZYPP_THROW( zypp::media::MediaForbiddenException(reqUrl, err.errordesc()));
- break;
- case NetworkRequestError::NotFound:
- ZYPP_THROW( zypp::media::MediaFileNotFoundException( reqUrl, filename ) );
- break;
- case NetworkRequestError::ExceededMaxLen:
- ZYPP_THROW( zypp::media::MediaFileSizeExceededException( reqUrl, 0 ) );
- break;
- default:
- break;
- }
- ZYPP_THROW( zypp::media::MediaCurlException( reqUrl, err.errordesc(), err.nativeerror() ) );
- }
- ZYPP_THROW( zypp::media::MediaCurlException( req.proto().spec().url(), err.errordesc() , "" ) );
- }
-
- bool MediaNetwork::retry( DispatchContext &ctx, Request &req ) const
- {
- const auto &res = req.result();
- if ( !res || !res->has_error() )
- return false;
-
- const auto &resErr = res->error();
- if ( resErr.error() != NetworkRequestError::AuthFailed && resErr.error() != NetworkRequestError::Unauthorized )
- return false;
-
- const std::string &availAuth = resErr.extra_info().count("authHint") ? resErr.extra_info().at("authHint") : "";
-
- zypp::callback::SendReport<zypp::media::AuthenticationReport> auth_report;
-
- zypp::media::CredentialManager cm( zypp::media::CredManagerOptions( zypp::ZConfig::instance().repoManagerRoot()) );
- auto authDataPtr = cm.getCred( _url );
-
- NetworkAuthData_Ptr curlcred( new NetworkAuthData() );
- curlcred->setUrl( _url );
-
- // preset the username if present in current url
- if ( !_url.getUsername().empty() )
- curlcred->setUsername(_url.getUsername());
- // if CM has found some credentials, preset the username from there
- else if ( authDataPtr )
- curlcred->setUsername( authDataPtr->username() );
-
- // set available authentication types
- // might be needed in prompt
- curlcred->setAuthType( availAuth );
-
- std::string prompt_msg = zypp::str::Format( _("Authentication required for '%s'") ) % _url.asString();
-
- // the auth data that was used in the request is older than whats in the Credential manager,
- // so instead of asking the user for creds we first try to use the new ones
- if ( res->last_auth_timestamp() > 0 && res->last_auth_timestamp() < authDataPtr->lastDatabaseUpdate() ) {
- _settings.setUsername( authDataPtr->username() );
- _settings.setPassword( authDataPtr->password() );
- retryRequest( ctx, req );
- return true;
- } else {
-
- // ask user
- if ( auth_report->prompt( _url, prompt_msg, *curlcred ) ) {
- DBG << "callback answer: retry" << std::endl
- << "CurlAuthData: " << *curlcred << std::endl;
-
- if ( curlcred->valid() ) {
-
- _settings.setUsername( curlcred->username() );
- _settings.setPassword( curlcred->password() );
-
- // set available authentication types from the exception
- if ( curlcred->authType() == CURLAUTH_NONE )
- curlcred->setAuthType( availAuth );
-
- // set auth type (seems this must be set _after_ setting the userpwd)
- if ( curlcred->authType() != CURLAUTH_NONE) {
- // FIXME: only overwrite if not empty?
- _settings.setAuthType( curlcred->authTypeAsString() );
- }
-
- cm.addCred( *curlcred );
- cm.save();
-
- // we need to restart all already failed requests so we do not double ask
- // for credentials
- for ( auto &r : _requests ) {
- if ( &r == &req )
- continue;
- if ( !r.result() || !r.result()->has_error() )
- continue;
- const auto rErr = r.result()->error();
- if ( rErr.error() != NetworkRequestError::AuthFailed && rErr.error() != NetworkRequestError::Unauthorized ) {
- retryRequest( ctx, r );
- }
- }
-
- retryRequest( ctx, req );
- return true;
- }
- }
- }
- return false;
- }
-
- void MediaNetwork::handleStreamMessage( DispatchContext &ctx, const zypp::proto::Envelope &e ) const
- {
- RequestId dlId = -1;
- bool cancel = false;
-
- const auto &mName = e.messagetypename();
- if ( mName == "zypp.proto.DownloadStart" ) {
- zypp::proto::DownloadStart st;
- st.ParseFromString( e.value() );
-
- auto r = findRequest( st.requestid() );
- if ( !r )
- return;
-
- MIL << "Received the download started event for file "<< zypp::Url( r->proto().spec().url() ) <<" from server." << std::endl;
- dlId = st.requestid();
- r->setProgress( 0.0, 0.0, cancel );
-
- } else if ( mName == "zypp.proto.DownloadProgress" ) {
- zypp::proto::DownloadProgress prog;
- prog.ParseFromString( e.value() );
-
- auto r = findRequest( prog.requestid() );
- if ( !r )
- return;
-
- //MIL << "Received progress for download: " << zypp::Url( r->url() ) << std::endl;
- dlId = prog.requestid();
- r->setProgress( prog.total(), prog.now(), cancel );
-
- } else if ( mName == "zypp.proto.DownloadFin" ) {
- zypp::proto::DownloadFin fin;
- fin.ParseFromString( e.value() );
-
- auto r = findRequest( fin.requestid() );
- if ( !r )
- return;
-
- MIL << "Received FIN for download: " << zypp::Url( r->proto().spec().url() ) << std::endl;
- r->setResult( std::move(fin) );
- } else {
- // error , unknown message
- WAR << "Received unexpected message... ignoring" << std::endl;
- }
-
- if ( dlId >= 0 && cancel ) {
- zypp::proto::CancelDownload cl;
- cl.set_requestid( dlId );
- ctx.sendEnvelope( cl );
- ctx.sock->waitForAllBytesWritten();
- }
- }
-
- MediaNetwork::Request MediaNetwork::makeRequest ( const zypp::OnMediaLocation &loc ) const
- {
- DBG << loc.filename().asString() << std::endl;
-
- if(!_url.isValid())
- ZYPP_THROW(zypp::media::MediaBadUrlException(_url));
-
- if(_url.getHost().empty())
- ZYPP_THROW(zypp::media::MediaBadUrlEmptyHostException(_url));
-
- Url url( getFileUrl(loc.filename()) );
-
- DownloadSpec spec( url, loc.filename(), loc.downloadSize() );
- spec.setTransferSettings( _settings )
- .setHeaderSize( loc.headerSize() )
- .setHeaderChecksum( loc.headerChecksum() )
- .setDeltaFile( loc.deltafile() );
-
- // we are downloading the file into a temporary location first
- Request fileReq( std::move(spec), _workingDir, ++_nextRequestId );
- fileReq.proto().set_streamprogress( false );
- return fileReq;
- }
-
- void MediaNetwork::trackRequest ( DispatchContext &ctx, Request &req ) const
- {
- zypp::proto::DownloadFin result;
-
- bool retry = true;
- while ( retry ) {
-
- retry = false; // try to stop after this iteration
-
- if ( !req.result() ) {
- result = ctx.waitFor<zypp::proto::DownloadFin>( req.proto().requestid(), std::bind( &MediaNetwork::handleStreamMessage, this, std::placeholders::_1, std::placeholders::_2 ) );
- req.setResult( std::move(result) );
- }
-
- retry = this->retry( ctx, req );
- }
- }
-
-
-
- void MediaNetwork::retryRequest ( DispatchContext &ctx, MediaNetwork::Request &req ) const
- {
- req.reset();
- // update request with new settings
- *req.proto().mutable_spec()->mutable_settings() = _settings.protoData();
- ctx.sendEnvelope( req.proto() );
- const auto &s = ctx.waitForStatus( req.proto().requestid(), std::bind( &MediaNetwork::handleStreamMessage, this, std::placeholders::_1, std::placeholders::_2 ) );
- if ( s.code() != zypp::proto::Status::Ok ) {
- ZYPP_THROW( zypp::media::MediaException( s.reason() ) );
- }
- }
-
- MediaNetwork::Request *MediaNetwork::findRequest( const Url url ) const
- {
- const auto &i = std::find_if( _requests.begin(), _requests.end(), [ &url ]( const auto &r ) {
- return ( url == zyppng::Url( r.proto().spec().url() ) );
- });
- if ( i == _requests.end() )
- return nullptr;
- return &(*i);
- }
-
- MediaNetwork::Request *MediaNetwork::findRequest(const RequestId id ) const
- {
- const auto &i = std::find_if( _requests.begin(), _requests.end(), [ &id ]( const auto &r ) {
- return ( id == r.proto().requestid() );
- });
- if ( i == _requests.end() )
- return nullptr;
- return &(*i);
- }
-
- bool MediaNetwork::getDoesFileExist( const zypp::filesystem::Pathname &filename ) const
- {
-
- auto ctx = ensureConnected();
-
- // here we always send a new request without even considering probably already existing ones
- auto req = makeRequest( filename );
- req.proto().mutable_spec()->set_checkexistanceonly( true );
- req.proto().set_prioritize( true );
-
-
- ctx->sendEnvelope( req.proto() );
- const auto &s = ctx->waitForStatus( req.proto().requestid(), std::bind( &MediaNetwork::handleStreamMessage, this, std::placeholders::_1, std::placeholders::_2 ) );
- if ( s.code() != zypp::proto::Status::Ok ) {
- ZYPP_THROW( zypp::media::MediaException( s.reason() ) );
- }
-
- try
- {
- trackRequest( *ctx, req );
-
- //this will throw if the file does not exist
- handleRequestResult( req, filename );
- } catch ( const zypp::media::MediaFileNotFoundException &e ) {
- return false;
- } catch ( const zypp::media::MediaException &e ) {
- // some error, we are not sure about file existence, rethrw
- ZYPP_RETHROW(e);
- }
-
- return true;
- }
-
-
- void MediaNetwork::getFile( const zypp::OnMediaLocation &file ) const
- {
- const auto &filename = file.filename();
- auto ctx = ensureConnected();
-
- const auto url = getFileUrl( filename );
- auto downloadReq = findRequest( url );
-
- zypp::callback::SendReport<zypp::media::DownloadProgressReport> report;
- ProgressData data( url, &report );
-
- report->start( url, filename );
-
- if ( !downloadReq ) {
- auto newReq = makeRequest( file );
-
- newReq.proto().set_streamprogress( true );
- newReq.proto().set_prioritize( true );
-
- ctx->sendEnvelope( newReq.proto() );
-
- const auto &s = ctx->waitForStatus( newReq.proto().requestid(), std::bind( &MediaNetwork::handleStreamMessage, this, std::placeholders::_1, std::placeholders::_2 ) );
- if ( s.code() != zypp::proto::Status::Ok && s.code() ) {
- ZYPP_THROW( zypp::media::MediaException( s.reason() ) );
- }
-
- _requests.push_back( newReq );
- downloadReq = &_requests.back();
-
- } else {
- // if the download is not yet finished we request to track it
- if ( !downloadReq->result() ) {
-
- zypp::proto::SubscribeProgress sub;
- sub.set_requestid( downloadReq->proto().requestid() );
- sub.set_prioritize( true );
- ctx->sendEnvelope( sub );
-
- const auto &s = ctx->waitForStatus( sub.requestid(), std::bind( &MediaNetwork::handleStreamMessage, this, std::placeholders::_1, std::placeholders::_2 ) );
-
- // status could be unknown ID if the request was already in finished state and the message was stuck in our socket!
- if ( s.code() != zypp::proto::Status::Ok && s.code() != zypp::proto::Status::UnknownId ) {
- ZYPP_THROW( zypp::media::MediaException( s.reason() ) );
- }
- }
- }
-
- downloadReq->startReporting( data );
- zypp::OnScopeExit cleanup( [&](){ downloadReq->stopReporting(); });
-
- try {
- trackRequest( *ctx, *downloadReq );
-
- //this will throw if the file does not exist
- handleRequestResult( *downloadReq, filename );
- } catch ( zypp::media::MediaUnauthorizedException & ex_r ) {
- report->finish(url, zypp::media::DownloadProgressReport::ACCESS_DENIED, ex_r.asUserHistory());
- ZYPP_RETHROW(ex_r);
- }
- // unexpected exception
- catch ( zypp::media::MediaException & excpt_r )
- {
- zypp::media::DownloadProgressReport::Error reason = zypp::media::DownloadProgressReport::ERROR;
- if( typeid(excpt_r) == typeid( zypp::media::MediaFileNotFoundException ) ||
- typeid(excpt_r) == typeid( zypp::media::MediaNotAFileException ) )
- {
- reason = zypp::media::DownloadProgressReport::NOT_FOUND;
- }
- report->finish(url, reason, excpt_r.asUserHistory());
- ZYPP_RETHROW(excpt_r);
- }
-
- // seems we were successful
- const auto &targetPath = localPath(filename).absolutename();
-
- const auto errCode = zypp::filesystem::assert_dir( targetPath.dirname() );
- if( errCode ) {
- std::string err = zypp::str::Str() << "assert_dir " << targetPath.dirname() << " failed";
- DBG << err << std::endl;
- ZYPP_THROW( zypp::media::MediaCurlException( url, err, "" ) );
- }
-
- if( zypp::filesystem::hardlinkCopy( downloadReq->proto().spec().targetpath(), targetPath ) != 0 ) {
- std::string err = zypp::str::Str() << "Failed to hardlinkCopy the requested file <<" << downloadReq->proto().spec().targetpath() << " to the targetPath " << targetPath;
- DBG << err << std::endl;
- ZYPP_THROW( zypp::media::MediaCurlException( url, err, "" ) );
- }
-
- if (::chmod( targetPath.c_str(), zypp::filesystem::applyUmaskTo( 0644 ))) {
- ERR << "Failed to chmod file " << targetPath << std::endl;
- }
-
- // check if the file was required the same number of times it was requested
- if ( downloadReq->unref() <= 0 ) {
- _requests.remove_if( [ & ]( const auto &r ){
- return ( r.proto().requestid() == downloadReq->proto().requestid() );
- });
- }
-
- report->finish(url, zypp::media::DownloadProgressReport::NO_ERROR, "");
- }
-
- void MediaNetwork::getDir(const zypp::filesystem::Pathname &dirname, bool recurse_r) const
- {
- //we could make this download concurrently, but it is not used anywhere in the code, so why bother
- zypp::filesystem::DirContent content;
- getDirInfo( content, dirname, /*dots*/false );
-
- for ( zypp::filesystem::DirContent::const_iterator it = content.begin(); it != content.end(); ++it ) {
- zypp::Pathname filename = dirname + it->name;
- int res = 0;
-
- switch ( it->type ) {
- case zypp::filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
- case zypp::filesystem::FT_FILE:
- getFile( zypp::OnMediaLocation( filename ) );
- break;
- case zypp::filesystem::FT_DIR: // newer directory.yast contain at least directory info
- if ( recurse_r ) {
- getDir( filename, recurse_r );
- } else {
- res = assert_dir( localPath( filename ) );
- if ( res ) {
- WAR << "Ignore error (" << res << ") on creating local directory '" << localPath( filename ) << "'" << std::endl;
- }
- }
- break;
- default:
- // don't provide devices, sockets, etc.
- break;
- }
- }
- }
-
- void MediaNetwork::precacheFiles(const std::vector<zypp::OnMediaLocation> &files)
- {
- zypp::proto::Prefetch req;
- req.set_requestid( ++_nextRequestId );
-
- std::vector<Request> sentRequests;
-
- for ( const auto &file : files ) {
- const auto url = getFileUrl( file.filename() );
- auto existingReq = findRequest( url );
- if ( existingReq )
- existingReq->ref();
- else {
- Request fileReq = makeRequest( file );
- fileReq.proto().set_streamprogress( false );
- *req.mutable_requests()->Add() = fileReq.proto();
- sentRequests.push_back( fileReq );
- }
- }
-
- if ( req.requests().size() ) {
-
- auto ctx = ensureConnected();
- ctx->sendEnvelope( req );
-
- const auto &status = ctx->waitForStatus( req.requestid(), std::bind( &MediaNetwork::handleStreamMessage, this, std::placeholders::_1, std::placeholders::_2 ) );
- if ( status.code() == zypp::proto::Status::Ok ) {
- MIL_MEDIA << "Request was acknowledged by server, downloads should start soon" << std::endl;
- _requests.insert( _requests.end(), std::move_iterator(sentRequests.begin()), std::move_iterator(sentRequests.end()) );
- return;
- }
-
- MIL << "Request failed " << status.reason() << std::endl;
- ZYPP_THROW( zypp::media::MediaException( status.reason() ) );
- }
- }
-
- void MediaNetwork::getDirInfo(std::list<std::string> &retlist, const zypp::filesystem::Pathname &dirname, bool dots) const
- {
- getDirectoryYast( retlist, dirname, dots );
- }
-
- void MediaNetwork::getDirInfo(zypp::filesystem::DirContent &retlist, const zypp::filesystem::Pathname &dirname, bool dots) const
- {
- getDirectoryYast( retlist, dirname, dots );
- }
-
- bool MediaNetwork::checkAttachPoint(const zypp::Pathname &apoint) const
- {
- return MediaHandler::checkAttachPoint( apoint, true, true);
- }
-
-
-}
+++ /dev/null
-/*---------------------------------------------------------------------\
-| ____ _ __ __ ___ |
-| |__ / \ / / . \ . \ |
-| / / \ V /| _/ _/ |
-| / /__ | | | | | | |
-| /_____||_| |_| |_| |
-| |
-----------------------------------------------------------------------/
-*
-* This file contains private API, this might break at any time between releases.
-* You have been warned!
-*
-*/
-#ifndef ZYPPNG_MEDIA_MEDIANETWORK_H_INCLUDED
-#define ZYPPNG_MEDIA_MEDIANETWORK_H_INCLUDED
-
-#include <zypp/media/MediaNetworkCommonHandler.h>
-#include <zypp-core/zyppng/core/Url>
-#include <zypp-curl/ng/network/AuthData>
-#include <zypp-proto/envelope.pb.h>
-#include <zypp-proto/messages.pb.h>
-#include <zypp-core/zyppng/io/private/iobuffer_p.h>
-#include <zypp/TmpPath.h>
-#include <optional>
-#include <variant>
-
-namespace zyppng {
-
-class Download;
-class Downloader;
-class Socket;
-class EventDispatcher;
-
-using RequestId = uint32_t;
-
-class MediaNetwork : public zypp::media::MediaNetworkCommonHandler
-{
-public:
- MediaNetwork(const Url &url_r, const zypp::Pathname &attach_point_hint_r);
- virtual ~MediaNetwork() override;
-
- // MediaHandler interface
-protected:
- void attachTo(bool next) override;
- void releaseFrom(const std::string &ejectDev) override;
- void getFile ( const zypp::OnMediaLocation &file ) const override;
- void getDir(const zypp::filesystem::Pathname &dirname, bool recurse_r) const override;
- void getDirInfo(std::list<std::string> &retlist, const zypp::filesystem::Pathname &dirname, bool dots) const override;
- void getDirInfo(zypp::filesystem::DirContent &retlist, const zypp::filesystem::Pathname &dirname, bool dots) const override;
- bool getDoesFileExist(const zypp::filesystem::Pathname &filename) const override;
- bool checkAttachPoint(const zypp::Pathname &apoint) const override;
-
-
- Url getFileUrl(const zypp::Pathname &filename_r) const;
-
-private:
- struct ProgressData;
- struct DispatchContext;
- struct Request;
-
- std::unique_ptr<DispatchContext> ensureConnected () const;
- bool retry ( DispatchContext &ctx, Request &req ) const;
- void retryRequest ( DispatchContext &ctx, Request &req ) const;
- void handleStreamMessage ( DispatchContext &ctx, const zypp::proto::Envelope &e ) const;
- void handleRequestResult ( const Request &req , const zypp::filesystem::Pathname &filename ) const;
-
- Request makeRequest ( const zypp::OnMediaLocation &loc ) const;
- void trackRequest ( DispatchContext &ctx, Request &req ) const ;
-
- Request *findRequest ( const zyppng::Url url ) const;
- Request *findRequest ( const RequestId id ) const;
-
-
-private:
- zypp::filesystem::TmpDir _workingDir;
- mutable RequestId _nextRequestId = 0;
- mutable std::list<Request> _requests;
- mutable std::optional<int> _socket;
- mutable IOBuffer _readBuffer;
-
- // MediaHandler interface
-protected:
- void disconnectFrom() override;
-
- // MediaHandler interface
-public:
- void precacheFiles(const std::vector<zypp::OnMediaLocation> &files) override;
-};
-
-}
-
-
-
-#endif
+++ /dev/null
-/*---------------------------------------------------------------------\
-| ____ _ __ __ ___ |
-| |__ / \ / / . \ . \ |
-| / / \ V /| _/ _/ |
-| / /__ | | | | | | |
-| /_____||_| |_| |_| |
-| |
-----------------------------------------------------------------------*/
-#include "private/medianetworkserver_p.h"
-#include <zypp-curl/ng/network/Downloader>
-#include <zypp-curl/ng/network/DownloadSpec>
-#include <zypp-curl/ng/network/private/mirrorcontrol_p.h>
-#include <zypp-core/zyppng/base/private/threaddata_p.h>
-#include <zypp-core/zyppng/base/private/linuxhelpers_p.h>
-#include <zypp-core/zyppng/base/EventLoop>
-#include <zypp-core/zyppng/base/EventDispatcher>
-#include <zypp-core/zyppng/base/SocketNotifier>
-#include <zypp-core/zyppng/rpc/rpc.h>
-
-#include <zypp-curl/TransferSettings>
-#include <zypp/PathInfo.h>
-
-#include <zypp-proto/envelope.pb.h>
-#include <zypp-proto/messages.pb.h>
-
-#include <zypp-curl/ng/network/private/mediadebug_p.h>
-#include <zypp/Target.h>
-#include <zypp/ZConfig.h>
-
-#include <algorithm>
-#include <csignal>
-
-#undef ZYPP_BASE_LOGGER_LOGGROUP
-#define ZYPP_BASE_LOGGER_LOGGROUP "zypp::MediaNetworkServer"
-
-namespace internal {
- /**
- * initialized only once, this gets the agent string
- * which also includes the curl version
- *
- * Defined in MediaCurl.cc
- */
- const char * agentString();
-}
-
-namespace zyppng {
-
- using HeaderSizeType = zyppng::rpc::HeaderSizeType;
-
- MediaNetworkServer::MediaNetworkServer( )
- : _downloadManager( std::make_shared<Downloader>( MirrorControl::create() ) )
- {
- _downloadManager->setCredManagerOptions( zypp::media::CredManagerOptions( zypp::ZConfig::instance().repoManagerRoot()) );
- auto dispatcher = _downloadManager->requestDispatcher();
- if ( dispatcher ) {
- dispatcher->setAgentString( ::internal::agentString() );
- dispatcher->setHostSpecificHeader("download.opensuse.org", "X-ZYpp-AnonymousId", zypp::Target::anonymousUniqueId( zypp::Pathname()/*guess root*/ ) );
- dispatcher->setHostSpecificHeader("download.opensuse.org", "X-ZYpp-DistributionFlavor", zypp::Target::distributionFlavor( zypp::Pathname()/*guess root*/ ) );
- } else {
- // this branch should never be executed
- ERR << "Unable to initialize user agent string and zypp http headers." << std::endl;
- }
- }
-
- void MediaNetworkServer::listen( const std::string &sockPath )
- {
- if ( _serverSocket )
- return;
-
- _serverSocket = zyppng::Socket::create( AF_UNIX, SOCK_STREAM, 0 );
-
- // bind to a abstract unix domain socket address, which means we do not need to care about cleaning it up
- _serverSocket->bind( std::make_shared<zyppng::UnixSockAddr>( sockPath, true ) );
- _serverSocket->Base::connect( &zyppng::Socket::sigIncomingConnection, *this, &MediaNetworkServer::onIncomingConnection );
- _serverSocket->listen();
-
- MIL << "MediaNetworkServer started to listen for connections " << std::endl;
- }
-
- std::shared_ptr<Downloader> MediaNetworkServer::downloader()
- {
- return _downloadManager;
- }
-
- void MediaNetworkServer::onIncomingConnection()
- {
- auto sock = _serverSocket->accept();
- if ( !sock )
- return;
-
- MIL << "MediaNetworkServer received new connection " << sock.get() << std::endl;
-
- auto client = std::make_shared<MediaNetworkConn>( *this, std::move(sock) );
- client->connectFunc( &MediaNetworkConn::sigDisconnected, [ this, wp = std::weak_ptr(client) ](){
- auto p = wp.lock();
- if (!p) {
- MIL_MEDIA << "MediaNetworkServer client disconnected but the pointer was already deleted" << std::endl;
- return;
- }
-
- MIL << "MediaNetworkServer client disconnected " << p.get() << std::endl;
-
- _clients.erase(
- std::remove_if( _clients.begin(), _clients.end(), [ &p ]( auto listElem ){
- return p.get() == listElem.get();
- }
- ));
- }, *this);
-
- _clients.push_back( client );
- }
-
- struct MediaNetworkConn::Request {
- Request ( ) { }
- ~Request() {
- clearConnections();
- }
-
- void clearConnections () {
- std::for_each( trackingConns.begin(), trackingConns.end(), []( auto &conn){ conn.disconnect(); } );
- trackingConns.clear();
- }
-
- zypp::proto::Request request;
- std::shared_ptr<zyppng::Download> dl;
- std::vector<sigc::connection> trackingConns;
- };
-
-
- MediaNetworkConn::MediaNetworkConn( MediaNetworkServer &server, std::shared_ptr<Socket> &&socket ) : _server(server), _connection ( std::move( socket ) )
- {
- MIL_MEDIA << "Initializing Connection object " << std::endl;
- _socketSigConns.insert( _socketSigConns.end(), {
- _connection->Base::connect( &Socket::sigReadyRead, *this, &MediaNetworkConn::onReadyRead ),
- _connection->Base::connect( &Socket::sigDisconnected, *this, &MediaNetworkConn::onDisconnected ),
- _connection->Base::connect( &Socket::sigError, *this, &MediaNetworkConn::onError )
- } );
-
- //make sure we read possibly available data
- onReadyRead();
- }
-
- MediaNetworkConn::~MediaNetworkConn()
- {
- MIL_MEDIA << "Closing connection " << std::endl;
- while( _requests.size() ) {
- auto req = _requests.front();
- // make sure we do not receive signals while shutting down
- // this would break due to weak_from_this
- req->clearConnections();
- req->dl->cancel();
-
- // still make sure we actually send all signals about finished downloads.
- trackedDlFinished( *req );
- }
- MIL_MEDIA << "Closing connection done!" << std::endl;
-
- // clean up our socket connections, otherwise we receive signals we do not want to handle anymore
- std::for_each( _socketSigConns.begin(), _socketSigConns.end(), []( auto &conn){ conn.disconnect(); } );
- _socketSigConns.clear();
- }
-
- SignalProxy<void ()> MediaNetworkConn::sigDisconnected()
- {
- return _disconnected;
- }
-
- void MediaNetworkConn::onDisconnected()
- {
- MIL_MEDIA << "Socket was closed, requesting cleanup." << std::endl;
- _disconnected.emit();
- }
-
- void MediaNetworkConn::onReadyRead()
- {
- const auto &sendStatus = [this]( RequestId id, auto code, const auto &reason) {
- zypp::proto::Status status;
- status.set_requestid( id );
- status.set_code( code );
- status.set_reason( reason );
- sendMessage( status );
- };
-
- const auto &makeNewRequest = [this] ( zypp::proto::Request &&req ){
- auto newRequest = std::make_shared<Request>( );
- newRequest->request = std::move( req );
- newRequest->dl = _server.downloader()->downloadFile( newRequest->request.spec() );
- newRequest->dl->start();
-
- if ( newRequest->request.prioritize() )
- newRequest->dl->prioritize();
-
- trackRequest( *newRequest );
-
- _requests.push_back( std::move(newRequest) );
- return _requests.back();
- };
-
- // read until buffers are empty
- while ( _connection->bytesAvailable() ) {
-
- // MIL << "Server has bytes" << std::endl;
-
- if ( !_pendingMessageSize ) {
- // if we cannot read the message size wait some more
- if ( _connection->bytesAvailable() < sizeof( HeaderSizeType ) )
- return;
-
- HeaderSizeType msgSize;
- _connection->read( reinterpret_cast<char *>( &msgSize ), sizeof( decltype (msgSize) ) );
- _pendingMessageSize = msgSize;
- }
-
- // wait for the full message size to be available
- if ( _connection->bytesAvailable() < static_cast<size_t>( *_pendingMessageSize ) )
- return;
-
- ByteArray message = _connection->read( *_pendingMessageSize );
- _pendingMessageSize.reset();
-
- zypp::proto::Envelope m;
- if (! m.ParseFromArray( message.data(), message.size() ) ) {
- //send error and close, we can not recover from this. Bytes might be mixed up on the socket
- sendStatus( -1, zypp::proto::Status::InvalidMessage, "This message is misformed." );
- _connection->close();
- return;
- }
-
- //DBG << "Server recv: " << m.messagetypename() << " " << m.value().size() << std::endl;
-
- const auto &mName = m.messagetypename();
- if ( mName == "zypp.proto.Request" ) {
-
- zypp::proto::Request prefetchReq;
- if (! prefetchReq.ParseFromString( m.value() ) ) {
- sendStatus( -1, zypp::proto::Status::MalformedRequest, "Unable to deserialize the request." );
- continue;
- }
- const auto &r = makeNewRequest( std::move(prefetchReq) );
- sendStatus( r->request.requestid(), zypp::proto::Status::Ok, "OK" );
-
- } else if ( mName == "zypp.proto.Prefetch" ) {
-
- zypp::proto::Prefetch prefetchReq;
- if (! prefetchReq.ParseFromString( m.value() ) ) {
- sendStatus( -1, zypp::proto::Status::MalformedRequest, "Unable to deserialize the request." );
- continue;
- }
-
- for ( auto &file : *prefetchReq.mutable_requests() ) {
- makeNewRequest( std::move(file) );
- }
- sendStatus( prefetchReq.requestid(), zypp::proto::Status::Ok, "OK" );
-
- } else if ( mName == "zypp.proto.CancelDownload" ) {
- zypp::proto::CancelDownload cancel;
- if (! cancel.ParseFromString( m.value() ) ) {
- sendStatus( -1, zypp::proto::Status::MalformedRequest, "Unable to deserialize the request." );
- continue;
- }
-
- for ( auto &req : _requests ) {
- if ( req->request.requestid() == cancel.requestid() ) {
- auto lock = req;
- lock->dl->cancel();
- break;
- }
- }
- sendStatus( cancel.requestid(), zypp::proto::Status::Ok, "OK" );
-
- } else if ( mName == "zypp.proto.NewAuthDataAvailable" ) {
- zypp::proto::NewAuthDataAvailable data;
- if (! data.ParseFromString( m.value() ) ) {
- sendStatus( -1, zypp::proto::Status::MalformedRequest, "Unable to deserialize the request." );
- continue;
- }
-
- MIL_MEDIA << "Got new Auth data, restarting failed requests" << std::endl;
-
- // we got new auth data in the CredentialManager, lets restart all our requests that are currently
- // in failed auth state.
- for ( auto &req : _requests ) {
- if ( req->dl->state() == Download::Finished && req->dl->hasError()
- && ( req->dl->lastRequestError().type() == NetworkRequestError::AuthFailed || req->dl->lastRequestError().type() == NetworkRequestError::Unauthorized ) ) {
- MIL_MEDIA << "Found request waiting for Auth, restarting " << req->dl->spec().url() << std::endl;
- req->dl->spec().settings().protoData() = data.settings();
- req->dl->start();
- }
- }
- sendStatus( data.requestid(), zypp::proto::Status::Ok, "OK" );
-
- } else if ( mName == "zypp.proto.SubscribeProgress" ) {
- zypp::proto::SubscribeProgress sub;
- if (! sub.ParseFromString( m.value() ) ) {
- sendStatus( -1, zypp::proto::Status::MalformedRequest, "Unable to deserialize the request." );
- continue;
- }
-
- bool found = false;
- for ( auto &req : _requests ) {
- if ( req->request.requestid() == sub.requestid() ) {
- req->request.set_streamprogress( true );
- if ( sub.prioritize() )
- req->dl->prioritize();
- sendStatus( sub.requestid(), zypp::proto::Status::Ok, "OK" );
- found = true;
- break;
- }
- }
-
- if ( !found )
- sendStatus( sub.requestid(), zypp::proto::Status::UnknownId, "Could not find the request ID in running requests" );
-
- } else if ( mName == "zypp.proto.UnSubscribeProgress" ) {
- zypp::proto::UnSubscribeProgress unSub;
- if (! unSub.ParseFromString( m.value() ) ) {
- sendStatus( -1, zypp::proto::Status::MalformedRequest, "Unable to deserialize the request." );
- return;
- }
-
- bool found = false;
- for ( auto &req : _requests ) {
- if ( req->request.requestid() == unSub.requestid() ) {
- req->request.set_streamprogress( false );
- found = true;
- sendStatus( unSub.requestid(), zypp::proto::Status::Ok, "OK" );
- break;
- }
- }
- if ( !found )
- sendStatus( unSub.requestid(), zypp::proto::Status::UnknownId, "Could not find the request ID in running requests" );
-
- } else {
- sendStatus( -1, zypp::proto::Status::UnknownRequest, "This request type is not known." );
- }
- }
- }
-
- void MediaNetworkConn::onError( Socket::SocketError err )
- {
- MIL << "Socket received error: " << err << std::endl;
- _connection->disconnect();
- }
-
- void MediaNetworkConn::trackRequest( Request &r )
- {
- r.trackingConns = {
- r.dl->connectFunc( &Download::sigStarted, [ this, &r ]( auto & ) {
- this->signalRequestStarted( r );
- }, *this),
-
- r.dl->connectFunc( &Download::sigAlive, [ this, &r ]( auto &, off_t now ) {
-
- if ( !r.request.streamprogress() )
- return;
-
- zypp::proto::DownloadProgress prog;
- prog.set_requestid( r.request.requestid() );
- prog.set_url( r.request.spec().url() );
- prog.set_now( now );
- this->sendMessage( prog );
- }, *this ),
-
- r.dl->connectFunc( &Download::sigProgress, [ this, &r ]( auto &, off_t total, off_t now ) {
-
- if ( !r.request.streamprogress() ) {
- return;
- }
-
- zypp::proto::DownloadProgress prog;
- prog.set_requestid( r.request.requestid() );
- prog.set_url( r.request.spec().url() );
- prog.set_now( now );
- prog.set_total( total );
- this->sendMessage( prog );
- }, *this ),
- r.dl->connectFunc( &Download::sigFinished, [ this, &r ]( auto & ) {
- this->trackedDlFinished( r );
- }, *this )
- };
- }
-
- void MediaNetworkConn::signalRequestStarted(Request &r)
- {
- zypp::proto::DownloadStart start;
- start.set_requestid( r.request.requestid() );
- start.set_url( r.request.spec().url() );
- this->sendMessage( start );
- }
-
- void MediaNetworkConn::trackedDlFinished( MediaNetworkConn::Request &r )
- {
- MIL << "Download finished by MediaNetworkServer: " << r.request.spec().url() << std::endl;
-
- zypp::proto::DownloadFin fin;
- fin.set_requestid( r.request.requestid() );
- if ( r.dl->lastAuthTimestamp() > 0 )
- fin.set_last_auth_timestamp( r.dl->lastAuthTimestamp() );
- r.clearConnections();
-
- std::string err;
- if ( !r.dl->hasError() ) {
- fin.clear_error();
- } else {
- const auto &lerr = r.dl->lastRequestError();
- fin.mutable_error()->set_error( lerr.type() );
- fin.mutable_error()->set_errordesc( err );
- fin.mutable_error()->set_nativeerror( lerr.nativeErrorString() );
-
- // this is rather ugly, we need to convert the extra infos to string
- // we will only handle the ones the client actually cares about
- fin.mutable_error()->mutable_extra_info()->insert( { "requestUrl", lerr.extraInfoValue("requestUrl", r.dl->spec().url()).asString() } );
- if ( lerr.type() == NetworkRequestError::Unauthorized )
- fin.mutable_error()->mutable_extra_info()->insert( { "authHint", lerr.extraInfoValue("authHint", std::string()) } );
- }
-
- this->sendMessage( fin );
-
- MIL_MEDIA << "Download was finished, releasing all ressources." << std::endl;
- _requests.erase (
- std::remove_if( _requests.begin(), _requests.end(), [ &r ]( const auto &reqInList ){
- return reqInList.get() == &r;
- }
- ));
- return;
- // ATTENTION, r will be dangling from here on out
- }
-
- template< typename T >
- void MediaNetworkConn::
- sendMessage( T &m )
- {
- zypp::proto::Envelope env;
- env.set_messagetypename( m.GetTypeName() );
- m.SerializeToString( env.mutable_value() );
-
- //DBG << "Sending message\n" << env.DebugString() << std::endl;
-
- std::string data = env.SerializeAsString();
- const HeaderSizeType size = data.size();
-
- _connection->write( reinterpret_cast<const char *>( &size ), sizeof(decltype(size)) );
- _connection->write( data.data(), size );
- }
-
- MediaNetworkThread::MediaNetworkThread()
- {
- MIL_MEDIA << "Initializing MediaNetworkThread" << std::endl;
- //start up thread
- _t = std::thread( [this](){ threadMain(); });
- }
-
- void MediaNetworkThread::threadMain()
- {
- // force the kernel to pick another thread to handle those signals
- zyppng::blockSignalsForCurrentThread( { SIGTERM, SIGINT, SIGPIPE, } );
-
- zyppng::ThreadData::current().setName("Zypp-MediaNetwork");
-
- auto dispatch = zyppng::EventLoop::create();
-
- auto server = std::make_shared<MediaNetworkServer>();
- server->listen( sockPath() );
-
- // we are using a pipe to wake up from the event loop, the SocketNotifier will throw a signal every
- // time there is data available
- auto sNotify = _shutdownSignal.makeNotifier( false );
- sNotify->connectFunc( &zyppng::SocketNotifier::sigActivated, [&dispatch]( const zyppng::SocketNotifier &, int ) {
- dispatch->quit();
- });
- sNotify->setEnabled( true );
-
- MIL_MEDIA << "Starting event loop " << std::endl;
-
- dispatch->run();
-
- MIL_MEDIA << "Thread exit " << std::endl;
- }
-
- MediaNetworkThread::~MediaNetworkThread()
- {
- _shutdownSignal.notify();
- _t.join();
- }
-
- MediaNetworkThread &MediaNetworkThread::instance()
- {
- static MediaNetworkThread t;
- return t;
- }
-
-}
+++ /dev/null
-/*---------------------------------------------------------------------\
-| ____ _ __ __ ___ |
-| |__ / \ / / . \ . \ |
-| / / \ V /| _/ _/ |
-| / /__ | | | | | | |
-| /_____||_| |_| |_| |
-| |
-----------------------------------------------------------------------/
-*
-* This file contains private API, this might break at any time between releases.
-* You have been warned!
-*
-*/
-#ifndef ZYPPNG_MEDIA_MEDIANETWORKSERVER_H_INCLUDED
-#define ZYPPNG_MEDIA_MEDIANETWORKSERVER_H_INCLUDED
-
-#include <zypp-core/zyppng/base/zyppglobal.h>
-#include <zypp-core/zyppng/base/signals.h>
-#include <zypp/base/String.h>
-#include <zypp-core/zyppng/base/Base>
-#include <zypp-core/zyppng/io/Socket>
-#include <zypp/TmpPath.h>
-#include <zypp-core/zyppng/thread/Wakeup>
-
-#include <sys/types.h>
-#include <unistd.h>
-#include <memory>
-#include <optional>
-#include <list>
-#include <thread>
-
-namespace google::protobuf {
- class Message;
-}
-
-namespace zyppng {
-
- class Downloader;
- class MediaNetworkConn;
-
- using RequestId = uint32_t;
-
- class LIBZYPP_NG_NO_EXPORT MediaNetworkServer : public Base
- {
- public:
- using ConnectionList = std::list< std::shared_ptr<MediaNetworkConn> >;
-
- MediaNetworkServer();
-
- void listen ( const std::string &sockPath );
-
- std::shared_ptr<Downloader> downloader ();
-
- private:
- void onIncomingConnection ();
-
- private:
- std::shared_ptr<Socket> _serverSocket;
- std::shared_ptr<Downloader> _downloadManager;
- ConnectionList _clients;
- };
-
- class LIBZYPP_NG_NO_EXPORT MediaNetworkConn : public Base
- {
- struct Request;
- using ReqPtr = std::shared_ptr<Request>;
- using ReqList = std::list< ReqPtr >;
- public:
- MediaNetworkConn ( MediaNetworkServer &server, std::shared_ptr<Socket> &&socket );
- ~MediaNetworkConn ( );
-
- SignalProxy<void()> sigDisconnected ();
-
- private:
- void onDisconnected ();
- void onReadyRead ();
- void onError ( Socket::SocketError err );
- void trackRequest ( Request &r );
- void signalRequestStarted ( Request &r );
- void trackedDlFinished ( Request &r );
-
- template <typename T>
- void sendMessage ( T &m );
-
- private:
- MediaNetworkServer &_server;
- ReqList _requests;
- std::shared_ptr<Socket> _connection;
- std::optional<int32_t> _pendingMessageSize;
- Signal<void()> _disconnected;
-
- std::vector<sigc::connection> _socketSigConns;
- };
-
- class LIBZYPP_NG_NO_EXPORT MediaNetworkThread : public Base
- {
- public:
- ~MediaNetworkThread();
- static MediaNetworkThread &instance ();
-
- static std::string sockPath () {
- static std::string path = zypp::str::Format("zypp-mediasocket-%1%") % getpid();
- return path;
- }
-
- private:
- MediaNetworkThread();
- void threadMain ();
- std::thread _t;
- zyppng::Wakeup _shutdownSignal;
- };
-
-}
-
-
-
-#endif // MEDIANETWORKSERVER_H
+++ /dev/null
-SET( zyppng_PROTOBUF_SOURCES
-
-)
-
-protobuf_generate_cpp(ZYPPNG_PROTO_SRCS ZYPPNG_PROTO_HDRS ${zyppng_PROTOBUF_SOURCES})
-
-ADD_LIBRARY( zyppng-protobuf OBJECT ${ZYPPNG_PROTO_SRCS} ${ZYPPNG_PROTO_HDRS} )
-
-add_dependencies( zyppng-protobuf zypp-protobuf )
projects / platform / upstream / libzypp.git / commitdiff