From 8ff8f25666a9199347f87c555a7f5eff2fb15494 Mon Sep 17 00:00:00 2001 From: DongHun Kwak Date: Tue, 20 Sep 2022 15:41:02 +0900 Subject: [PATCH] Imported Upstream version 17.31.0 --- .clang-format | 2 +- CMakeLists.txt | 6 +- VERSION.cmake | 6 +- package/libzypp.changes | 20 + po/mk.po | 37 +- tests/lib/WebServer.cc | 29 +- tests/lib/WebServer.h | 20 +- tests/zyppng/IOBuffer_test.cc | 2 +- tests/zyppng/data/downloader/media.1/media | 3 + tests/zyppng/data/provide/cd1/file1 | 1 + tests/zyppng/data/provide/cd1/media.1/media | 3 + tests/zyppng/data/provide/cd2/file2 | 1 + tests/zyppng/data/provide/cd2/media.2/media | 3 + tests/zyppng/data/provide/cd3/file3 | 1 + tests/zyppng/data/provide/cd3/media.3/media | 3 + tests/zyppng/data/provide/cd4/file4 | 1 + tests/zyppng/data/provide/cd4/media.4/media | 3 + tests/zyppng/io/AsyncDataSource_test.cc | 236 +++- tests/zyppng/io/Process_test.cc | 9 +- tests/zyppng/io/UnixSocket_test.cc | 10 +- tests/zyppng/media/CMakeLists.txt | 1 + tests/zyppng/media/EvDownloader_test.cc | 89 +- tests/zyppng/media/Provider_test.cc | 1162 ++++++++++++++++++ tools/CMakeLists.txt | 19 + tools/repomirror/CMakeLists.txt | 16 + tools/repomirror/main.cc | 405 +++++++ tools/repomirror/output.h | 269 +++++ tools/zypp-media-chksum/CMakeLists.txt | 27 + tools/zypp-media-chksum/main.cc | 138 +++ tools/zypp-media-copy/CMakeLists.txt | 27 + tools/zypp-media-copy/main.cc | 147 +++ tools/zypp-media-dir/CMakeLists.txt | 29 + tools/zypp-media-dir/dirprovider.cc | 135 +++ tools/zypp-media-dir/dirprovider.h | 27 + tools/zypp-media-dir/main.cc | 28 + tools/zypp-media-disc/CMakeLists.txt | 37 + tools/zypp-media-disc/discprovider.cc | 407 +++++++ tools/zypp-media-disc/discprovider.h | 31 + tools/zypp-media-disc/main.cc | 27 + tools/zypp-media-disk/CMakeLists.txt | 29 + tools/zypp-media-disk/diskprovider.cc | 363 ++++++ tools/zypp-media-disk/diskprovider.h | 25 + tools/zypp-media-disk/main.cc | 26 + tools/zypp-media-http/CMakeLists.txt | 30 + tools/zypp-media-http/main.cc | 21 + tools/zypp-media-http/networkprovider.cc | 480 ++++++++ tools/zypp-media-http/networkprovider.h | 80 ++ tools/zypp-media-iso/CMakeLists.txt | 44 + tools/zypp-media-iso/isoprovider.cc | 399 ++++++ tools/zypp-media-iso/isoprovider.h | 42 + tools/zypp-media-iso/main.cc | 27 + tools/zypp-media-nfs/CMakeLists.txt | 29 + tools/zypp-media-nfs/main.cc | 23 + tools/zypp-media-nfs/nfsprovider.cc | 262 ++++ tools/zypp-media-nfs/nfsprovider.h | 25 + tools/zypp-media-smb/CMakeLists.txt | 29 + tools/zypp-media-smb/main.cc | 26 + tools/zypp-media-smb/smbprovider.cc | 342 ++++++ tools/zypp-media-smb/smbprovider.h | 24 + tools/zypp-media-tvm/CMakeLists.txt | 25 + tools/zypp-media-tvm/main.cc | 23 + tools/zypp-media-tvm/testvmprovider.cc | 343 ++++++ tools/zypp-media-tvm/testvmprovider.h | 41 + zypp-core/AutoDispose.h | 4 + zypp-core/ByteArray.h | 11 + zypp-core/CMakeLists.txt | 18 + zypp-core/Pathname.cc | 36 + zypp-core/Pathname.h | 3 + zypp-core/base/Exception.h | 4 +- zypp-core/base/LogControl.cc | 47 +- zypp-core/base/LogControl.h | 10 + zypp-core/base/Logger.h | 19 +- zypp-core/zyppng/async/asyncop.h | 137 ++- zypp-core/zyppng/base/base.h | 2 +- zypp-core/zyppng/base/eventdispatcher_glib.cc | 4 +- zypp-core/zyppng/base/linuxhelpers.cc | 4 +- zypp-core/zyppng/base/private/linuxhelpers_p.h | 2 +- zypp-core/zyppng/base/zyppglobal.h | 19 + zypp-core/zyppng/io/abstractspawnengine.cc | 6 + zypp-core/zyppng/io/asyncdatasource.cpp | 303 +++-- zypp-core/zyppng/io/asyncdatasource.h | 39 +- zypp-core/zyppng/io/iobuffer.cc | 51 +- zypp-core/zyppng/io/iodevice.cc | 265 +++- zypp-core/zyppng/io/iodevice.h | 97 +- .../zyppng/io/private/abstractspawnengine_p.h | 9 +- zypp-core/zyppng/io/private/asyncdatasource_p.h | 39 + zypp-core/zyppng/io/private/iobuffer_p.h | 31 +- zypp-core/zyppng/io/private/iodevice_p.h | 21 +- zypp-core/zyppng/io/private/socket_p.h | 6 +- zypp-core/zyppng/io/process.cpp | 127 +- zypp-core/zyppng/io/process.h | 38 +- zypp-core/zyppng/io/socket.cc | 122 +- zypp-core/zyppng/io/socket.h | 22 +- zypp-core/zyppng/meta/functional.h | 30 + zypp-core/zyppng/meta/type_traits.h | 11 +- zypp-core/zyppng/pipelines/asyncresult.h | 52 +- zypp-core/zyppng/pipelines/await.h | 2 +- zypp-core/zyppng/pipelines/expected.h | 52 +- zypp-core/zyppng/pipelines/lift.h | 4 +- zypp-core/zyppng/pipelines/redo.h | 14 +- zypp-core/zyppng/pipelines/transform.h | 20 +- zypp-core/zyppng/pipelines/wait.h | 18 +- zypp-core/zyppng/rpc/MessageStream | 1 + zypp-core/zyppng/rpc/messagestream.cc | 178 +++ zypp-core/zyppng/rpc/messagestream.h | 178 +++ zypp-curl/CMakeLists.txt | 4 + zypp-curl/curlhelper.cc | 4 +- zypp-curl/ng/network/downloader.cc | 113 +- zypp-curl/ng/network/downloader.h | 27 +- zypp-curl/ng/network/downloadspec.cc | 104 +- zypp-curl/ng/network/downloadspec.h | 19 +- zypp-curl/ng/network/networkrequestdispatcher.cc | 4 +- zypp-curl/ng/network/private/downloader_p.h | 1 - .../ng/network/private/downloaderstates/base_p.h | 10 +- .../private/downloaderstates/metalinkinfo_p.cc | 10 +- zypp-curl/ng/network/private/mediadebug_p.h | 5 +- zypp-curl/ng/network/private/request_p.h | 1 + zypp-curl/ng/network/request.cc | 5 +- zypp-curl/parser/metalinkparser.cc | 2 + zypp-curl/transfersettings.cc | 210 ++-- zypp-curl/transfersettings.h | 44 +- zypp-media/CDTools | 1 + zypp-media/CMakeLists.txt | 58 + zypp-media/FileCheckException | 1 + zypp-media/auth/authdata.cc | 16 + zypp-media/auth/authdata.h | 5 + zypp-media/auth/credentialfilereader.cc | 3 +- zypp-media/auth/credentialmanager.cc | 25 +- zypp-media/auth/credentialmanager.h | 10 +- zypp-media/cdtools.cc | 103 ++ zypp/zyppng/context.h => zypp-media/cdtools.h | 32 +- zypp-media/filecheckexception.cc | 12 + zypp-media/filecheckexception.h | 43 + zypp-media/mediaexception.cc | 5 + zypp-media/mediaexception.h | 26 + zypp-media/mount.cc | 18 + zypp-media/mount.h | 11 + zypp-media/ng/HeaderValueMap | 1 + zypp-media/ng/MediaVerifier | 1 + zypp-media/ng/Provide | 1 + zypp-media/ng/ProvideFwd | 1 + zypp-media/ng/ProvideItem | 1 + zypp-media/ng/ProvideRes | 1 + zypp-media/ng/ProvideSpec | 1 + zypp-media/ng/headervaluemap.cc | 267 +++++ zypp-media/ng/headervaluemap.h | 199 +++ zypp-media/ng/mediaverifier.cc | 137 +++ zypp-media/ng/mediaverifier.h | 100 ++ zypp-media/ng/private/attachedmediainfo_p.h | 69 ++ zypp-media/ng/private/provide_p.h | 158 +++ zypp-media/ng/private/providedbg_p.h | 43 + zypp-media/ng/private/providefwd_p.h | 36 + zypp-media/ng/private/provideitem_p.h | 206 ++++ zypp-media/ng/private/providemessage_p.h | 196 +++ zypp-media/ng/private/providequeue_p.h | 145 +++ zypp-media/ng/private/provideres_p.h | 23 + zypp-media/ng/provide-configvars.h | 28 + zypp-media/ng/provide.cc | 1265 ++++++++++++++++++++ zypp-media/ng/provide.h | 171 +++ zypp-media/ng/providefwd.h | 32 + zypp-media/ng/provideitem.cc | 1191 ++++++++++++++++++ zypp-media/ng/provideitem.h | 192 +++ zypp-media/ng/providemessage.cc | 656 ++++++++++ zypp-media/ng/providequeue.cc | 775 ++++++++++++ zypp-media/ng/provideres.cc | 48 + zypp-media/ng/provideres.h | 74 ++ zypp-media/ng/providespec.cc | 270 +++++ zypp-media/ng/providespec.h | 198 +++ zypp-media/ng/worker/DeviceDriver | 1 + zypp-media/ng/worker/MountingWorker | 1 + zypp-media/ng/worker/ProvideWorker | 1 + zypp-media/ng/worker/devicedriver.cc | 382 ++++++ zypp-media/ng/worker/devicedriver.h | 201 ++++ zypp-media/ng/worker/mountingworker.cc | 191 +++ zypp-media/ng/worker/mountingworker.h | 41 + zypp-media/ng/worker/provideworker.cc | 469 ++++++++ zypp-media/ng/worker/provideworker.h | 217 ++++ zypp-proto/CMakeLists.txt | 19 +- zypp-proto/media/download.proto | 27 - zypp-proto/media/messages.proto | 100 -- zypp-proto/media/networkrequesterror.proto | 14 - zypp-proto/media/provider.proto | 419 +++++++ zypp-proto/media/transfersettings.proto | 35 - zypp-proto/test/tvm.proto | 31 + zypp/CMakeLists.txt | 20 +- zypp/FileChecker.h | 25 +- zypp/PoolItem.cc | 8 + zypp/PoolItem.h | 8 +- zypp/ResStatus.h | 15 +- zypp/base/DrunkenBishop.cc | 1 + zypp/media/MediaCD.cc | 72 +- zypp/media/MediaCurl.cc | 4 +- zypp/media/MediaHandler.cc | 45 +- zypp/media/MediaHandlerFactory.cc | 15 +- zypp/media/MediaManager.cc | 8 +- zypp/repo/SUSEMediaVerifier.cc | 2 +- zypp/target/TargetImpl.cc | 28 +- zypp/zyppng/CMakeLists.txt | 41 - zypp/zyppng/Context | 1 - zypp/zyppng/context.cc | 23 - zypp/zyppng/media/MediaNetwork | 1 - zypp/zyppng/media/medianetwork.cc | 987 --------------- zypp/zyppng/media/medianetwork.h | 94 -- zypp/zyppng/media/medianetworkserver.cc | 492 -------- zypp/zyppng/media/private/medianetworkserver_p.h | 117 -- zypp/zyppng/proto/CMakeLists.txt | 9 - 206 files changed, 16812 insertions(+), 3002 deletions(-) create mode 100644 tests/zyppng/data/downloader/media.1/media create mode 100644 tests/zyppng/data/provide/cd1/file1 create mode 100644 tests/zyppng/data/provide/cd1/media.1/media create mode 100644 tests/zyppng/data/provide/cd2/file2 create mode 100644 tests/zyppng/data/provide/cd2/media.2/media create mode 100644 tests/zyppng/data/provide/cd3/file3 create mode 100644 tests/zyppng/data/provide/cd3/media.3/media create mode 100644 tests/zyppng/data/provide/cd4/file4 create mode 100644 tests/zyppng/data/provide/cd4/media.4/media create mode 100644 tests/zyppng/media/Provider_test.cc create mode 100644 tools/repomirror/CMakeLists.txt create mode 100644 tools/repomirror/main.cc create mode 100644 tools/repomirror/output.h create mode 100644 tools/zypp-media-chksum/CMakeLists.txt create mode 100644 tools/zypp-media-chksum/main.cc create mode 100644 tools/zypp-media-copy/CMakeLists.txt create mode 100644 tools/zypp-media-copy/main.cc create mode 100644 tools/zypp-media-dir/CMakeLists.txt create mode 100644 tools/zypp-media-dir/dirprovider.cc create mode 100644 tools/zypp-media-dir/dirprovider.h create mode 100644 tools/zypp-media-dir/main.cc create mode 100644 tools/zypp-media-disc/CMakeLists.txt create mode 100644 tools/zypp-media-disc/discprovider.cc create mode 100644 tools/zypp-media-disc/discprovider.h create mode 100644 tools/zypp-media-disc/main.cc create mode 100644 tools/zypp-media-disk/CMakeLists.txt create mode 100644 tools/zypp-media-disk/diskprovider.cc create mode 100644 tools/zypp-media-disk/diskprovider.h create mode 100644 tools/zypp-media-disk/main.cc create mode 100644 tools/zypp-media-http/CMakeLists.txt create mode 100644 tools/zypp-media-http/main.cc create mode 100644 tools/zypp-media-http/networkprovider.cc create mode 100644 tools/zypp-media-http/networkprovider.h create mode 100644 tools/zypp-media-iso/CMakeLists.txt create mode 100644 tools/zypp-media-iso/isoprovider.cc create mode 100644 tools/zypp-media-iso/isoprovider.h create mode 100644 tools/zypp-media-iso/main.cc create mode 100644 tools/zypp-media-nfs/CMakeLists.txt create mode 100644 tools/zypp-media-nfs/main.cc create mode 100644 tools/zypp-media-nfs/nfsprovider.cc create mode 100644 tools/zypp-media-nfs/nfsprovider.h create mode 100644 tools/zypp-media-smb/CMakeLists.txt create mode 100644 tools/zypp-media-smb/main.cc create mode 100644 tools/zypp-media-smb/smbprovider.cc create mode 100644 tools/zypp-media-smb/smbprovider.h create mode 100644 tools/zypp-media-tvm/CMakeLists.txt create mode 100644 tools/zypp-media-tvm/main.cc create mode 100644 tools/zypp-media-tvm/testvmprovider.cc create mode 100644 tools/zypp-media-tvm/testvmprovider.h create mode 100644 zypp-core/zyppng/io/private/asyncdatasource_p.h create mode 100644 zypp-core/zyppng/rpc/MessageStream create mode 100644 zypp-core/zyppng/rpc/messagestream.cc create mode 100644 zypp-core/zyppng/rpc/messagestream.h create mode 100644 zypp-media/CDTools create mode 100644 zypp-media/FileCheckException create mode 100644 zypp-media/cdtools.cc rename zypp/zyppng/context.h => zypp-media/cdtools.h (51%) create mode 100644 zypp-media/filecheckexception.cc create mode 100644 zypp-media/filecheckexception.h create mode 100644 zypp-media/ng/HeaderValueMap create mode 100644 zypp-media/ng/MediaVerifier create mode 100644 zypp-media/ng/Provide create mode 100644 zypp-media/ng/ProvideFwd create mode 100644 zypp-media/ng/ProvideItem create mode 100644 zypp-media/ng/ProvideRes create mode 100644 zypp-media/ng/ProvideSpec create mode 100644 zypp-media/ng/headervaluemap.cc create mode 100644 zypp-media/ng/headervaluemap.h create mode 100644 zypp-media/ng/mediaverifier.cc create mode 100644 zypp-media/ng/mediaverifier.h create mode 100644 zypp-media/ng/private/attachedmediainfo_p.h create mode 100644 zypp-media/ng/private/provide_p.h create mode 100644 zypp-media/ng/private/providedbg_p.h create mode 100644 zypp-media/ng/private/providefwd_p.h create mode 100644 zypp-media/ng/private/provideitem_p.h create mode 100644 zypp-media/ng/private/providemessage_p.h create mode 100644 zypp-media/ng/private/providequeue_p.h create mode 100644 zypp-media/ng/private/provideres_p.h create mode 100644 zypp-media/ng/provide-configvars.h create mode 100644 zypp-media/ng/provide.cc create mode 100644 zypp-media/ng/provide.h create mode 100644 zypp-media/ng/providefwd.h create mode 100644 zypp-media/ng/provideitem.cc create mode 100644 zypp-media/ng/provideitem.h create mode 100644 zypp-media/ng/providemessage.cc create mode 100644 zypp-media/ng/providequeue.cc create mode 100644 zypp-media/ng/provideres.cc create mode 100644 zypp-media/ng/provideres.h create mode 100644 zypp-media/ng/providespec.cc create mode 100644 zypp-media/ng/providespec.h create mode 100644 zypp-media/ng/worker/DeviceDriver create mode 100644 zypp-media/ng/worker/MountingWorker create mode 100644 zypp-media/ng/worker/ProvideWorker create mode 100644 zypp-media/ng/worker/devicedriver.cc create mode 100644 zypp-media/ng/worker/devicedriver.h create mode 100644 zypp-media/ng/worker/mountingworker.cc create mode 100644 zypp-media/ng/worker/mountingworker.h create mode 100644 zypp-media/ng/worker/provideworker.cc create mode 100644 zypp-media/ng/worker/provideworker.h delete mode 100644 zypp-proto/media/download.proto delete mode 100644 zypp-proto/media/messages.proto delete mode 100644 zypp-proto/media/networkrequesterror.proto create mode 100644 zypp-proto/media/provider.proto delete mode 100644 zypp-proto/media/transfersettings.proto create mode 100644 zypp-proto/test/tvm.proto delete mode 100644 zypp/zyppng/CMakeLists.txt delete mode 100644 zypp/zyppng/Context delete mode 100644 zypp/zyppng/context.cc delete mode 100644 zypp/zyppng/media/MediaNetwork delete mode 100644 zypp/zyppng/media/medianetwork.cc delete mode 100644 zypp/zyppng/media/medianetwork.h delete mode 100644 zypp/zyppng/media/medianetworkserver.cc delete mode 100644 zypp/zyppng/media/private/medianetworkserver_p.h delete mode 100644 zypp/zyppng/proto/CMakeLists.txt diff --git a/.clang-format b/.clang-format index c217262..6e27cb0 100644 --- a/.clang-format +++ b/.clang-format @@ -112,6 +112,6 @@ SpacesInCStyleCastParentheses: false SpacesInParentheses: true SpacesInSquareBrackets: true Standard: Cpp11 -TabWidth: 8 +TabWidth: 4 UseTab: Never ... diff --git a/CMakeLists.txt b/CMakeLists.txt index 8031f85..a0205d4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,6 +19,7 @@ OPTION (DISABLE_AUTODOCS "Do not require doxygen being installed (required to bu OPTION (EXPORT_NG_API "Export experimental libzypp API" OFF) # This option will reroute all tool binaries to the libzypp build dir instead of taking those installed in the default directories. OPTION (ENABLE_DEVEL_BUILD "Developer build, use zypp tools directly from build dir rather than the default locations" OFF) +OPTION (INSTALL_NG_BINARIES "Installs the NG binaries, disabled for now since we are not actively using them." OFF) # Distros using just zypper may want to enable this as default earlier OPTION (ENABLE_PREVIEW_SINGLE_RPMTRANS_AS_DEFAULT_FOR_ZYPPER "[preview] Force zypper into using a single rpm transaction to install" OFF) #-------------------------------------------------------------------------------- @@ -35,12 +36,15 @@ ENDIF (DEBIAN) IF ( ENABLE_DEVEL_BUILD ) MESSAGE( WARNING "Zypp devel build enabled, do not do this in production" ) SET( ZYPP_RPM_BINARY "${CMAKE_BINARY_DIR}/tools/zypp-rpm/zypp-rpm") + SET( ZYPP_WORKER_PATH "${CMAKE_BINARY_DIR}/tools/workers" ) ELSE() SET( ZYPP_RPM_BINARY "${ZYPP_LIBEXEC_INSTALL_DIR}/zypp-rpm") + SET( ZYPP_WORKER_PATH "${ZYPP_LIBEXEC_INSTALL_DIR}/workers" ) ENDIF( ENABLE_DEVEL_BUILD ) message ( "Using zypp-rpm from path: ${ZYPP_RPM_BINARY}" ) -add_definitions( -DZYPP_RPM_BINARY="${ZYPP_RPM_BINARY}") +add_definitions( -DZYPP_RPM_BINARY="${ZYPP_RPM_BINARY}" ) +add_definitions( -DZYPP_WORKER_PATH="${ZYPP_WORKER_PATH}") IF (${have_system} STREQUAL x) MESSAGE (STATUS "Building for SUSE") diff --git a/VERSION.cmake b/VERSION.cmake index ea96e10..de1faf8 100644 --- a/VERSION.cmake +++ b/VERSION.cmake @@ -60,9 +60,9 @@ # SET(LIBZYPP_MAJOR "17") SET(LIBZYPP_COMPATMINOR "22") -SET(LIBZYPP_MINOR "30") -SET(LIBZYPP_PATCH "3") +SET(LIBZYPP_MINOR "31") +SET(LIBZYPP_PATCH "0") # -# LAST RELEASED: 17.30.3 (22) +# LAST RELEASED: 17.31.0 (22) # (The number in parenthesis is LIBZYPP_COMPATMINOR) #======= diff --git a/package/libzypp.changes b/package/libzypp.changes index b91bee2..b85faa8 100644 --- a/package/libzypp.changes +++ b/package/libzypp.changes @@ -1,4 +1,24 @@ ------------------------------------------------------------------- +Tue Jul 19 16:03:01 CEST 2022 - ma@suse.de + +- Add PoolItem::statusReinit to reset the status it's initial + state in the ResPool (might help bsc#1199895) + This may either be 'KEEP_STATE bySOLVER' or 'LOCKED byUSER' if + the PoolItem matched a hard lock defined in /etc/zypp/locks. +- Fix building with GCC 13 on i586 (fixes #407, fixes #396) +- Be prepared to receive exceptions from curl_easy_cleanup + (bsc#1201092) +- Don't auto-flag kernel-firmware as 'reboot-needed' (bsc#1200993) +- Remove Medianetwork and dependend code. + This commit removes the MediaNetwork tech preview and all related + code. First reason for this is that MediaNetwork was just meant + as a way to test the new CURL based downloader and second: since + the Provide API is going to completely replace the current media + backend it would be extra work to ensure that changes on the + Downloader do not break MediaNetwork. +- version 17.31.0 (22) + +------------------------------------------------------------------- Tue Jul 5 10:29:19 CEST 2022 - ma@suse.de - Fix building with GCC 12.x release (#396) diff --git a/po/mk.po b/po/mk.po index ab1e3b0..6433ee4 100644 --- a/po/mk.po +++ b/po/mk.po @@ -7,7 +7,7 @@ msgstr "" "Project-Id-Version: YaST (@memory@)\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2022-03-14 15:21+0100\n" -"PO-Revision-Date: 2022-07-02 23:12+0000\n" +"PO-Revision-Date: 2022-07-12 07:12+0000\n" "Last-Translator: Kristijan Fremen Velkovski \n" "Language-Team: Macedonian \n" @@ -575,7 +575,7 @@ msgstr "Хонгконг" #. :HKG:344: #: zypp/CountryCode.cc:252 msgid "Heard Island and McDonald Islands" -msgstr "" +msgstr "Остров Херд и Острови МекДоналд" # HN #. :HMD:334: @@ -710,17 +710,17 @@ msgstr "Кирибати" #. :KIR:296: #: zypp/CountryCode.cc:275 msgid "Comoros" -msgstr "" +msgstr "Комори" #. :COM:174: #: zypp/CountryCode.cc:276 msgid "Saint Kitts and Nevis" -msgstr "" +msgstr "Свети Кристофер и Невис" #. :KNA:659: #: zypp/CountryCode.cc:277 msgid "North Korea" -msgstr "" +msgstr "Северна Кореја" # ZA #. :PRK:408: @@ -731,15 +731,14 @@ msgstr "Јужна Кореја" #. :KOR:410: #: zypp/CountryCode.cc:279 msgid "Kuwait" -msgstr "" +msgstr "Кувајт" # FO # fuzzy #. :KWT:414: #: zypp/CountryCode.cc:280 -#, fuzzy msgid "Cayman Islands" -msgstr "Ирска" +msgstr "Кајмански Острови" # KZ # fuzzy @@ -751,27 +750,27 @@ msgstr "Тајван" #. :KAZ:398: #: zypp/CountryCode.cc:282 msgid "Lao People's Democratic Republic" -msgstr "" +msgstr "Лаоска Народна Демократска Република" #. :LAO:418: #: zypp/CountryCode.cc:283 msgid "Lebanon" -msgstr "" +msgstr "Либан" #. :LBN:422: #: zypp/CountryCode.cc:284 msgid "Saint Lucia" -msgstr "" +msgstr "Света Луција" #. :LCA:662: #: zypp/CountryCode.cc:285 msgid "Liechtenstein" -msgstr "" +msgstr "Лихтенштајн" #. :LIE:438: #: zypp/CountryCode.cc:286 msgid "Sri Lanka" -msgstr "" +msgstr "Шри Ланка" # LR # fuzzy @@ -950,7 +949,7 @@ msgstr "" #. :MOZ:508: #: zypp/CountryCode.cc:316 msgid "Namibia" -msgstr "" +msgstr "Намибија" # NC # fuzzy @@ -963,17 +962,15 @@ msgstr "Македонија" # fuzzy #. :NCL:540: #: zypp/CountryCode.cc:318 -#, fuzzy msgid "Niger" -msgstr "Србија" +msgstr "Нигер" # FO # fuzzy #. :NER:562: #: zypp/CountryCode.cc:319 -#, fuzzy msgid "Norfolk Island" -msgstr "Ирска" +msgstr "Норфолшки Остров" # NG # fuzzy @@ -1004,13 +1001,13 @@ msgstr "Норвешка" #. :NOR:578: #: zypp/CountryCode.cc:324 msgid "Nepal" -msgstr "" +msgstr "Непал" #. :NPL:524: #. language code: nau na #: zypp/CountryCode.cc:325 zypp/LanguageCode.cc:781 msgid "Nauru" -msgstr "" +msgstr "Науру" #. :NRU:520: #: zypp/CountryCode.cc:326 diff --git a/tests/lib/WebServer.cc b/tests/lib/WebServer.cc index bf18070..8ed5e23 100644 --- a/tests/lib/WebServer.cc +++ b/tests/lib/WebServer.cc @@ -103,7 +103,7 @@ static inline std::string handlerPrefix() return pref; } -WebServer::Request::Request( std::istream::__streambuf_type *rinbuf, std::ostream::__streambuf_type *routbuf, std::ostream::__streambuf_type *rerrbuf ) +WebServer::Request::Request(std::streambuf *rinbuf, std::streambuf *routbuf, std::streambuf *rerrbuf ) : rin ( rinbuf ) , rout ( routbuf ) , rerr ( rerrbuf ) @@ -377,12 +377,12 @@ public: //remove "/handler/" prefix std::string key = req.params.at("SCRIPT_NAME").substr( handlerPrefix().length() ); - auto it = _handlers.find( key ); + const auto &it = _handlers.find( key ); if ( it == _handlers.end() ) { - req.rerr << "Status: 404 Not Found\r\n" - << "Content-Type: text/html\r\n" - << "\r\n" - << "Request handler was not found"; + if ( !_defaultHandler ) { + _defaultHandler = makeDefaultHandler(); + } + _defaultHandler( req ); FCGX_Finish_r(&request); continue; } @@ -451,6 +451,7 @@ public: zypp::Pathname _docroot; zypp::shared_ptr _thrd; std::map _handlers; + WebServer::RequestHandler _defaultHandler; unsigned int _port; int _wakeupPipe[2]; @@ -488,6 +489,12 @@ void WebServer::addRequestHandler( const std::string &path, RequestHandler &&han _pimpl->_handlers[path] = std::move(handler); } +void WebServer::setDefaultHandler ( RequestHandler &&handler ) +{ + std::lock_guard lock( _pimpl->_mut ); + _pimpl->_defaultHandler = std::move(handler); +} + void WebServer::removeRequestHandler(const std::string &path) { std::lock_guard lock( _pimpl->_mut ); @@ -533,6 +540,16 @@ WebServer::RequestHandler WebServer::makeResponse( std::string resp ) { }; }; +WebServer::RequestHandler WebServer::makeDefaultHandler( ) +{ + return []( WebServer::Request & req ){ + req.rerr << "Status: 404 Not Found\r\n" + << "Content-Type: text/html\r\n" + << "\r\n" + << "Request handler was not found"; + }; +} + WebServer::RequestHandler WebServer::makeResponse( std::string status, std::string content ) { return makeResponse( status , std::vector(), content ); }; diff --git a/tests/lib/WebServer.h b/tests/lib/WebServer.h index 7509da9..c5c9959 100644 --- a/tests/lib/WebServer.h +++ b/tests/lib/WebServer.h @@ -38,9 +38,9 @@ class WebServer public: struct Request { - Request ( std::istream::__streambuf_type *rinbuf, - std::ostream::__streambuf_type *routbuf, - std::ostream::__streambuf_type *rerrbuf ); + Request ( std::streambuf *rinbuf, + std::streambuf *routbuf, + std::streambuf *rerrbuf ); std::string path; std::map< std::string, std::string > params; std::istream rin; @@ -100,6 +100,15 @@ class WebServer */ void addRequestHandler ( const std::string &path, RequestHandler &&handler ); + /** + * Sets the default request handler callback as a fallback if no specific handler was registered for + * a given path. The path from the HTTP request can be queried as the "REQUEST_URI" param from the \ref Request structure passed to the callback. + * + * \note a request handler path is always prefixed with /handler so to call a handler named test, the + * URL is "http://localhost/handler/test" + */ + void setDefaultHandler ( RequestHandler &&handler ); + /*! * Removes a registered request hander, can be called at any time */ @@ -111,6 +120,11 @@ class WebServer static RequestHandler makeResponse(std::string resp); /*! + * Creates a request handler that simply returns 404 + */ + static RequestHandler makeDefaultHandler( ); + + /*! * Creates a request handler that sends a HTTP response with \a status and \a content * in the respose */ diff --git a/tests/zyppng/IOBuffer_test.cc b/tests/zyppng/IOBuffer_test.cc index 498c1f6..a97e802 100644 --- a/tests/zyppng/IOBuffer_test.cc +++ b/tests/zyppng/IOBuffer_test.cc @@ -110,7 +110,7 @@ BOOST_AUTO_TEST_CASE(reserveAndRead) // fill buffer with an arithmetic progression for (int i = 1; i < 256; ++i) { zyppng::ByteArray ba(i, char(i)); - char *ringPos = buf.reserve(i); + char *ringPos = buf.reserve( i ); BOOST_REQUIRE(ringPos); memcpy(ringPos, ba.data(), i); } diff --git a/tests/zyppng/data/downloader/media.1/media b/tests/zyppng/data/downloader/media.1/media new file mode 100644 index 0000000..c5c8305 --- /dev/null +++ b/tests/zyppng/data/downloader/media.1/media @@ -0,0 +1,3 @@ +openSUSE - openSUSE-15.0-x86_64-Build267.2-Media +openSUSE-15.0-x86_64-Build267.2 +1 diff --git a/tests/zyppng/data/provide/cd1/file1 b/tests/zyppng/data/provide/cd1/file1 new file mode 100644 index 0000000..4d9a9eb --- /dev/null +++ b/tests/zyppng/data/provide/cd1/file1 @@ -0,0 +1 @@ +Medium File 1 diff --git a/tests/zyppng/data/provide/cd1/media.1/media b/tests/zyppng/data/provide/cd1/media.1/media new file mode 100644 index 0000000..564a9ef --- /dev/null +++ b/tests/zyppng/data/provide/cd1/media.1/media @@ -0,0 +1,3 @@ +openSUSE - Testmedium +openSUSE-15.0-x86_64-Build267.2 +4 diff --git a/tests/zyppng/data/provide/cd2/file2 b/tests/zyppng/data/provide/cd2/file2 new file mode 100644 index 0000000..e0626c3 --- /dev/null +++ b/tests/zyppng/data/provide/cd2/file2 @@ -0,0 +1 @@ +Medium File 2 diff --git a/tests/zyppng/data/provide/cd2/media.2/media b/tests/zyppng/data/provide/cd2/media.2/media new file mode 100644 index 0000000..564a9ef --- /dev/null +++ b/tests/zyppng/data/provide/cd2/media.2/media @@ -0,0 +1,3 @@ +openSUSE - Testmedium +openSUSE-15.0-x86_64-Build267.2 +4 diff --git a/tests/zyppng/data/provide/cd3/file3 b/tests/zyppng/data/provide/cd3/file3 new file mode 100644 index 0000000..06cd67e --- /dev/null +++ b/tests/zyppng/data/provide/cd3/file3 @@ -0,0 +1 @@ +Medium File 3 diff --git a/tests/zyppng/data/provide/cd3/media.3/media b/tests/zyppng/data/provide/cd3/media.3/media new file mode 100644 index 0000000..564a9ef --- /dev/null +++ b/tests/zyppng/data/provide/cd3/media.3/media @@ -0,0 +1,3 @@ +openSUSE - Testmedium +openSUSE-15.0-x86_64-Build267.2 +4 diff --git a/tests/zyppng/data/provide/cd4/file4 b/tests/zyppng/data/provide/cd4/file4 new file mode 100644 index 0000000..33948c1 --- /dev/null +++ b/tests/zyppng/data/provide/cd4/file4 @@ -0,0 +1 @@ +Medium File 4 diff --git a/tests/zyppng/data/provide/cd4/media.4/media b/tests/zyppng/data/provide/cd4/media.4/media new file mode 100644 index 0000000..564a9ef --- /dev/null +++ b/tests/zyppng/data/provide/cd4/media.4/media @@ -0,0 +1,3 @@ +openSUSE - Testmedium +openSUSE-15.0-x86_64-Build267.2 +4 diff --git a/tests/zyppng/io/AsyncDataSource_test.cc b/tests/zyppng/io/AsyncDataSource_test.cc index 570a60e..2608700 100644 --- a/tests/zyppng/io/AsyncDataSource_test.cc +++ b/tests/zyppng/io/AsyncDataSource_test.cc @@ -1,11 +1,16 @@ #include #include +#include #include #include #include #include #include #include +#include + +#include +#include BOOST_AUTO_TEST_CASE ( pipe_read_close ) @@ -27,18 +32,19 @@ BOOST_AUTO_TEST_CASE ( pipe_read_close ) bool gotClosed = false; zypp::ByteArray readData; - BOOST_REQUIRE( dataSource->open( pipeFds[0] ) ); + BOOST_REQUIRE( dataSource->openFds( { pipeFds[0] } ) ); BOOST_REQUIRE( dataSource->canRead() ); BOOST_REQUIRE( dataSource->readFdOpen() ); - dataSource->connectFunc( &zyppng::AsyncDataSource::sigReadyRead, [&](){ - std::cout <<"Got read"<readAll(); readData.insert( readData.end(), d.begin(), d.end() ); - } ); + }; - dataSource->connectFunc( &zyppng::AsyncDataSource::sigReadFdClosed, [&]( auto ){ + dataSource->connectFunc( &zyppng::AsyncDataSource::sigReadyRead, readAllData ); + dataSource->connectFunc( &zyppng::AsyncDataSource::sigReadFdClosed, [&]( auto, auto ){ gotClosed = true; + readAllData(); loop->quit(); }); @@ -57,6 +63,74 @@ BOOST_AUTO_TEST_CASE ( pipe_read_close ) BOOST_REQUIRE ( gotClosed ); } +// basically the same as above, but we break the receiving socket by closing the read end +// during a readAll call. +BOOST_AUTO_TEST_CASE ( pipe_read_close2 ) +{ + std::string_view text ("Hello again"); + int pipeFds[2] { -1, -1 }; + BOOST_REQUIRE( g_unix_open_pipe( pipeFds, FD_CLOEXEC, nullptr ) ); + + std::mutex lock; + std::condition_variable cv; + bool written = false; + + auto loop = zyppng::EventLoop::create(); + auto dataSource = zyppng::AsyncDataSource::create(); + + // make sure we are not stuck + auto timer = zyppng::Timer::create(); + timer->start( 1000 ); + timer->connectFunc( &zyppng::Timer::sigExpired, [&]( auto & ){ + loop->quit(); + }); + + bool gotClosed = false; + zypp::ByteArray readData; + + BOOST_REQUIRE( dataSource->openFds( { pipeFds[0] } ) ); + BOOST_REQUIRE( dataSource->canRead() ); + + const auto &readAllData = [&](){ + zypp::ByteArray d = dataSource->readAll(); + readData.insert( readData.end(), d.begin(), d.end() ); + }; + + dataSource->connectFunc( &zyppng::AsyncDataSource::sigReadyRead, [&]() { + std::unique_lock l(lock); + if ( !written ){ + cv.wait(l, [&](){ return written; }); + } + + // close the pipe here, breaking the receiving socket + ::close( pipeFds[1] ); + readAllData(); + } ); + dataSource->connectFunc( &zyppng::AsyncDataSource::sigReadFdClosed, [&]( auto, auto ){ + gotClosed = true; + readAllData(); + loop->quit(); + + }); + + std::thread writer( [&lock, &written, &cv]( int writeFd, std::string_view text ){ + { + std::unique_lock l(lock); + ::write( writeFd, text.data(), text.length() ); + written = true; + } + cv.notify_all(); + }, pipeFds[1], text ); + + loop->run(); + writer.join(); + + ::close( pipeFds[0] ); + BOOST_REQUIRE_EQUAL( std::string_view( readData.data(), readData.size() ), text ); + BOOST_REQUIRE ( gotClosed ); +} + + BOOST_AUTO_TEST_CASE ( pipe_write_close ) { std::string_view text ("Hello from main"); @@ -73,13 +147,13 @@ BOOST_AUTO_TEST_CASE ( pipe_write_close ) loop->quit(); }); - BOOST_REQUIRE( dataSink->open( -1, pipeFds[1] ) ); + BOOST_REQUIRE( dataSink->openFds( {}, pipeFds[1] ) ); BOOST_REQUIRE( dataSink->canWrite() ); - std::size_t bytesWritten = 0; - dataSink->connectFunc( &zyppng::AsyncDataSource::sigBytesWritten, [&]( std::size_t bytes ){ + int64_t bytesWritten = 0; + dataSink->connectFunc( &zyppng::AsyncDataSource::sigBytesWritten, [&]( int64_t bytes ){ bytesWritten += bytes; - if ( bytesWritten == text.size() ) + if ( std::size_t(bytesWritten) == text.size() ) loop->quit(); }); @@ -90,7 +164,7 @@ BOOST_AUTO_TEST_CASE ( pipe_write_close ) auto loop = zyppng::EventLoop::create(); auto dataSource = zyppng::AsyncDataSource::create(); - if ( !dataSource->open( readFd ) ) + if ( !dataSource->openFds( { readFd } ) ) return; // make sure we are not stuck @@ -108,9 +182,6 @@ BOOST_AUTO_TEST_CASE ( pipe_write_close ) }); loop->run(); - - std::cout << "Thread did read: " << readData.data() << std::endl; - ::close( readFd ); }, pipeFds[0], text ); @@ -131,7 +202,7 @@ BOOST_AUTO_TEST_CASE ( pipe_close ) BOOST_REQUIRE( g_unix_open_pipe( pipeFds, FD_CLOEXEC, nullptr ) ); auto loop = zyppng::EventLoop::create(); - auto dataSink = zyppng::AsyncDataSource::create(); + auto dataSink = zyppng::AsyncDataSource::create(); // make sure we are not stuck auto timer = zyppng::Timer::create(); @@ -142,11 +213,11 @@ BOOST_AUTO_TEST_CASE ( pipe_close ) bool gotClosed = false; - BOOST_REQUIRE( dataSink->open( -1, pipeFds[1] ) ); + BOOST_REQUIRE( dataSink->openFds( {}, pipeFds[1] ) ); BOOST_REQUIRE( dataSink->canWrite() ); - std::size_t bytesWritten = 0; - dataSink->connectFunc( &zyppng::AsyncDataSource::sigBytesWritten, [&]( std::size_t bytes ){ + int64_t bytesWritten = 0; + dataSink->connectFunc( &zyppng::AsyncDataSource::sigBytesWritten, [&]( int64_t bytes ){ bytesWritten += bytes; }); @@ -166,3 +237,134 @@ BOOST_AUTO_TEST_CASE ( pipe_close ) BOOST_REQUIRE ( gotClosed ); } +BOOST_AUTO_TEST_CASE ( multichannel ) +{ + std::string_view text[] = { "Hello to channel1", "Hello to channel2 with more bytes" }; + int pipeFds[2] { -1, -1 }; + BOOST_REQUIRE( g_unix_open_pipe( pipeFds, FD_CLOEXEC, nullptr ) ); + + int pipe2Fds[2] { -1, -1 }; + BOOST_REQUIRE( g_unix_open_pipe( pipe2Fds, FD_CLOEXEC, nullptr ) ); + + auto loop = zyppng::EventLoop::create(); + auto dataSink = zyppng::AsyncDataSource::create(); + auto dataSink2 = zyppng::AsyncDataSource::create(); + + // make sure we are not stuck + auto timer = zyppng::Timer::create(); + timer->start( 10000 ); + + bool timeout = false; + timer->connectFunc( &zyppng::Timer::sigExpired, [&]( auto & ){ + timeout = true; + loop->quit(); + }); + + BOOST_REQUIRE( dataSink->openFds( {}, pipeFds[1] ) ); + BOOST_REQUIRE( dataSink->canWrite() ); + + BOOST_REQUIRE( dataSink2->openFds( {}, pipe2Fds[1] ) ); + BOOST_REQUIRE( dataSink2->canWrite() ); + + int64_t bytesWritten = 0; + dataSink->connectFunc( &zyppng::AsyncDataSource::sigBytesWritten, [&]( int64_t bytes ){ + bytesWritten += bytes; + }); + + int64_t bytesWritten2 = 0; + dataSink2->connectFunc( &zyppng::AsyncDataSource::sigBytesWritten, [&]( int64_t bytes ){ + bytesWritten2 += bytes; + }); + + bool dataSink1ABW = false; + dataSink->connectFunc( &zyppng::AsyncDataSource::sigAllBytesWritten, [&]( ){ + dataSink1ABW = true; + }); + + bool dataSink2ABW = false; + dataSink2->connectFunc( &zyppng::AsyncDataSource::sigAllBytesWritten, [&]( ){ + dataSink2ABW = true; + }); + + auto dataReceiver = zyppng::AsyncDataSource::create(); + BOOST_REQUIRE( dataReceiver->openFds( { pipeFds[0], pipe2Fds[0] }, -1 ) ); + BOOST_REQUIRE( dataReceiver->canRead() ); + BOOST_REQUIRE_EQUAL( dataReceiver->readChannelCount(), 2 ); + BOOST_REQUIRE_EQUAL( dataReceiver->currentReadChannel(), 0 ); + + zypp::ByteArray dataRecv[2]; + + dataReceiver->connectFunc( &zyppng::AsyncDataSource::sigChannelReadyRead, [&]( auto channel ) { + dataReceiver->setReadChannel( channel ); + BOOST_REQUIRE_EQUAL( dataReceiver->currentReadChannel(), channel ); + zypp::ByteArray d = dataReceiver->readAll(); + dataRecv[channel].insert( dataRecv[channel].end(), d.begin(), d.end() ); + if ( dataRecv[0].size() > 0 && dataRecv[0].size() == text[0].size() + && dataRecv[1].size() > 0 && dataRecv[1].size() == text[1].size() ) + loop->quit(); + }); + + bool bytesPushed = false; + loop->eventDispatcher()->invokeOnIdle( [ & ](){ + dataSink->write( text[0].data(), text[0].size() ); + dataSink2->write( text[1].data(), text[1].size() ); + bytesPushed = true; + return false; + }); + + loop->run(); + + BOOST_REQUIRE( !timeout ); + BOOST_REQUIRE( dataSink1ABW ); + BOOST_REQUIRE( dataSink2ABW ); + BOOST_REQUIRE_EQUAL( bytesWritten, text[0].size() ); + BOOST_REQUIRE_EQUAL( bytesWritten2, text[1].size() ); + BOOST_REQUIRE_EQUAL( text[0], std::string_view( dataRecv[0].data(), dataRecv[0].size() ) ); + BOOST_REQUIRE_EQUAL( text[1], std::string_view( dataRecv[1].data(), dataRecv[1].size() ) ); +} + +BOOST_AUTO_TEST_CASE ( readl ) +{ + std::string_view text[] = { "Hello\n", "World\n", "123456" }; + int pipeFds[2] { -1, -1 }; + BOOST_REQUIRE( g_unix_open_pipe( pipeFds, FD_CLOEXEC, nullptr ) ); + + auto loop = zyppng::EventLoop::create(); + auto dataSource = zyppng::AsyncDataSource::create(); + + // make sure we are not stuck + auto timer = zyppng::Timer::create(); + timer->start( 3000 ); + + bool timeout = false; + timer->connectFunc( &zyppng::Timer::sigExpired, [&]( auto & ){ + loop->quit(); + timeout = true; + }); + + BOOST_REQUIRE( dataSource->openFds( { pipeFds[0] } ) ); + BOOST_REQUIRE( dataSource->canRead() ); + + dataSource->setReadChannel( 0 ); + dataSource->sigReadyRead ().connect([&](){ + // now write more data without returning to the ev loop, and check if we get it all + ::write( pipeFds[1], text[1].data(), text[1].size() ); + ::write( pipeFds[1], text[2].data(), text[2].size() ); + + auto ba = dataSource->readLine(); + BOOST_REQUIRE_EQUAL( text[0], std::string_view( ba.data(), ba.size() ) ); + ba = dataSource->readLine( text[1].size () + 1 ); // trigger code path with fixed size + BOOST_REQUIRE_EQUAL( text[1], std::string_view( ba.data(), ba.size() ) ); + ba = dataSource->readLine(); + BOOST_REQUIRE_EQUAL( text[2], std::string_view( ba.data(), ba.size() ) ); + loop->quit(); + }); + + //write the first line + ::write( pipeFds[1], text[0].data(), text[0].size() ); + + loop->run(); + + BOOST_REQUIRE( !timeout ); + +} diff --git a/tests/zyppng/io/Process_test.cc b/tests/zyppng/io/Process_test.cc index 83130c2..1ad5b37 100644 --- a/tests/zyppng/io/Process_test.cc +++ b/tests/zyppng/io/Process_test.cc @@ -62,12 +62,13 @@ BOOST_AUTO_TEST_CASE( Basic ) BOOST_REQUIRE_EQUAL( exitCode, 123 ); BOOST_REQUIRE( !gotFailedToStart ); - BOOST_REQUIRE( proc->stdoutDevice()->bytesAvailable() ); + proc->setReadChannel( zyppng::Process::StdOut ); + BOOST_REQUIRE( proc->bytesAvailable( ) ); - auto data = proc->stdoutDevice()->readLine(); + auto data = proc->readLine(); BOOST_REQUIRE_EQUAL( data.asStringView(), std::string_view("Hello\n") ); - data = proc->stdoutDevice()->readLine(); + data = proc->readLine(); BOOST_REQUIRE_EQUAL( data.asStringView(), std::string_view("I'm the second line\n") ); BOOST_REQUIRE( !proc->isRunning() ); @@ -132,7 +133,7 @@ BOOST_AUTO_TEST_CASE( mapExtraFD ) proc->addFd( pipeB->writeFd ); auto dataSource = zyppng::AsyncDataSource::create(); - BOOST_REQUIRE( dataSource->open( pipeB->readFd, pipeA->writeFd ) ); + BOOST_REQUIRE( dataSource->openFds( { pipeB->readFd }, pipeA->writeFd ) ); BOOST_REQUIRE( dataSource->canRead() ); BOOST_REQUIRE( dataSource->canWrite( ) ); diff --git a/tests/zyppng/io/UnixSocket_test.cc b/tests/zyppng/io/UnixSocket_test.cc index b72b00f..6a0f506 100644 --- a/tests/zyppng/io/UnixSocket_test.cc +++ b/tests/zyppng/io/UnixSocket_test.cc @@ -29,7 +29,7 @@ BOOST_AUTO_TEST_CASE ( lorem ) auto listeningSock = zyppng::Socket::create( AF_UNIX, SOCK_STREAM, 0 ); auto error = zyppng::Socket::NoError; - size_t bytesWritten = 0; + int64_t bytesWritten = 0; zyppng::ByteArray dataReceived; //< This is where the thread will write the data auto addr = std::make_shared( "socktest", true ); @@ -300,10 +300,10 @@ BOOST_AUTO_TEST_CASE ( lotsofdata ) auto error = zyppng::Socket::NoError; const int iterations = 1000000; - size_t bytesShouldBeWritten = 0; - size_t bytesWritten = 0; - size_t bytesWrittenSignaled = 0; - size_t bytesReceived = 0; + int64_t bytesShouldBeWritten = 0; + int64_t bytesWritten = 0; + int64_t bytesWrittenSignaled = 0; + int64_t bytesReceived = 0; auto addr = std::make_shared( "socktest", true ); BOOST_REQUIRE( listeningSock->bind( addr ) ); diff --git a/tests/zyppng/media/CMakeLists.txt b/tests/zyppng/media/CMakeLists.txt index 379a57c..364aff5 100644 --- a/tests/zyppng/media/CMakeLists.txt +++ b/tests/zyppng/media/CMakeLists.txt @@ -2,5 +2,6 @@ IF( NOT DISABLE_MEDIABACKEND_TESTS) ADD_TESTS( NetworkRequestDispatcher EvDownloader + Provider ) ENDIF() diff --git a/tests/zyppng/media/EvDownloader_test.cc b/tests/zyppng/media/EvDownloader_test.cc index c38627b..c89e04b 100644 --- a/tests/zyppng/media/EvDownloader_test.cc +++ b/tests/zyppng/media/EvDownloader_test.cc @@ -43,7 +43,6 @@ BOOST_DATA_TEST_CASE( dltest_basic, bdata::make( withSSL ), withSSL) auto ev = zyppng::EventLoop::create(); zyppng::Downloader::Ptr downloader = std::make_shared(); - downloader->setCredManagerOptions( zypp::media::CredManagerOptions( zypp::ZConfig::instance().repoManagerRoot()) ); //make sure the data here is big enough to cross the threshold of 256 bytes so we get a progress signal emitted and not only the alive signal. std::string dummyContent = "This is just some dummy content,\nto test downloading and signals.\n" @@ -372,7 +371,6 @@ BOOST_DATA_TEST_CASE( test1, bdata::make( generateMirr() ) * bdata::make( withSS zypp::filesystem::TmpFile targetFile; std::shared_ptr downloader = std::make_shared(); - downloader->setCredManagerOptions( zypp::media::CredManagerOptions( zypp::ZConfig::instance().repoManagerRoot()) ); downloader->requestDispatcher()->setMaximumConcurrentConnections( maxDLs ); //first metalink download, generate a fully valid one @@ -539,7 +537,6 @@ BOOST_DATA_TEST_CASE( dltest_auth, bdata::make( withSSL ), withSSL ) auto ev = zyppng::EventLoop::create(); std::shared_ptr downloader = std::make_shared(); - downloader->setCredManagerOptions( zypp::media::CredManagerOptions( zypp::ZConfig::instance().repoManagerRoot()) ); WebServer web((zypp::Pathname(TESTS_SRC_DIR)/"/zyppng/data/downloader").c_str(), 10001, withSSL ); BOOST_REQUIRE( web.start() ); @@ -603,8 +600,10 @@ BOOST_DATA_TEST_CASE( dltest_auth, bdata::make( withSSL ), withSSL ) dl->sigAuthRequired().connect( [&]( zyppng::Download &, zyppng::NetworkAuthData &auth, const std::string &availAuth ){ gotAuthRequest++; + auth.setUrl( web.url() ); auth.setUsername("test"); auth.setPassword("test"); + auth.setLastDatabaseUpdate( time( nullptr ) ); }); dl->sigStateChanged().connect([&]( zyppng::Download &, zyppng::Download::State state ){ @@ -618,88 +617,4 @@ BOOST_DATA_TEST_CASE( dltest_auth, bdata::make( withSSL ), withSSL ) BOOST_REQUIRE_EQUAL( gotAuthRequest, 1 ); BOOST_REQUIRE ( allStates == std::vector({ zyppng::Download::InitialState, zyppng::Download::DlMetaLinkInfo, zyppng::Download::PrepareMulti, zyppng::Download::DlMetalink, zyppng::Download::Finished}) ); } - - { - //the creds should be in the credential manager now, we should not need to specify them again in the slot - bool gotAuthRequest = false; - auto dl = downloader->downloadFile( zyppng::DownloadSpec(weburl, targetFile.path()) ); - dl->spec().setTransferSettings(set); - - dl->sigFinished( ).connect([ &ev ]( zyppng::Download & ){ - ev->quit(); - }); - - dl->sigAuthRequired().connect( [&]( zyppng::Download &, zyppng::NetworkAuthData &auth, const std::string &availAuth ){ - gotAuthRequest = true; - }); - - dl->start(); - ev->run(); - BOOST_TEST_REQ_SUCCESS( dl ); - BOOST_REQUIRE( !gotAuthRequest ); - } -} - -/** - * Test for bsc#1174011 auth=basic ignored in some cases - * - * If the URL specifes ?auth=basic libzypp should proactively send credentials we have available in the cred store - */ -BOOST_DATA_TEST_CASE( dltest_auth_basic, bdata::make( withSSL ), withSSL ) -{ - //don't write or read creds from real settings dir - zypp::filesystem::TmpDir repoManagerRoot; - zypp::ZConfig::instance().setRepoManagerRoot( repoManagerRoot.path() ); - - auto ev = zyppng::EventLoop::create(); - - std::shared_ptr downloader = std::make_shared(); - downloader->setCredManagerOptions( zypp::media::CredManagerOptions( zypp::ZConfig::instance().repoManagerRoot()) ); - - WebServer web((zypp::Pathname(TESTS_SRC_DIR)/"/zyppng/data/downloader").c_str(), 10001, withSSL ); - BOOST_REQUIRE( web.start() ); - - zypp::filesystem::TmpFile targetFile; - zyppng::Url weburl (web.url()); - weburl.setPathName("/handler/test.txt"); - weburl.setQueryParam("auth", "basic"); - weburl.setUsername("test"); - - // make sure the creds are already available - zypp::media::CredentialManager cm( repoManagerRoot.path() ); - zypp::media::AuthData data ("test", "test"); - data.setUrl( weburl ); - cm.addCred( data ); - cm.save(); - - zyppng::TransferSettings set = web.transferSettings(); - - web.addRequestHandler( "test.txt", createAuthHandler() ); - web.addRequestHandler( "quit", [ &ev ]( WebServer::Request & ){ ev->quit();} ); - - { - // simply check by request count if the test was successful: - // if the proactive code adding the credentials to the first request is not executed we will - // have more than 1 request. - int reqCount = 0; - auto dispatcher = downloader->requestDispatcher(); - dispatcher->sigDownloadStarted().connect([&]( zyppng::NetworkRequestDispatcher &, zyppng::NetworkRequest & ){ - reqCount++; - }); - - - auto dl = downloader->downloadFile( zyppng::DownloadSpec(weburl, targetFile.path()) - .setMetalinkEnabled( false ) - .setTransferSettings( set ) - ); - - dl->sigFinished( ).connect([ &ev ]( zyppng::Download & ){ - ev->quit(); - }); - - dl->start(); - ev->run(); - BOOST_TEST_REQ_SUCCESS( dl ); - BOOST_REQUIRE_EQUAL( reqCount, 1 ); - } } diff --git a/tests/zyppng/media/Provider_test.cc b/tests/zyppng/media/Provider_test.cc new file mode 100644 index 0000000..db9966e --- /dev/null +++ b/tests/zyppng/media/Provider_test.cc @@ -0,0 +1,1162 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "WebServer.h" +#include "TestTools.h" + +#include +#include + +#ifndef TESTS_BUILD_DIR +#error "TESTS_BUILD_DIR not defined" +#endif + +#define ZYPP_REQUIRE_THROW( statement, exception ) \ + try { BOOST_REQUIRE_THROW( statement, exception ); } catch( ... ) { BOOST_FAIL( #statement" throws unexpected exception"); } + +namespace bdata = boost::unit_test::data; + +BOOST_AUTO_TEST_CASE( http_prov ) +{ + using namespace zyppng::operators; + + auto ev = zyppng::EventLoop::create (); + + const auto &workerPath = zypp::Pathname ( TESTS_BUILD_DIR ).dirname() / "tools" / "workers"; + const auto &webRoot = zypp::Pathname ( TESTS_SRC_DIR ) / "zyppng" / "data" / "downloader"; + + zypp::filesystem::TmpDir provideRoot; + + auto prov = zyppng::Provide::create ( provideRoot ); + prov->setWorkerPath ( workerPath ); + prov->start(); + + WebServer web( webRoot.c_str(), 10001, false ); + BOOST_REQUIRE( web.start() ); + + auto fileUrl = web.url(); + fileUrl.setPathName( "/media.1/media" ); + auto op = prov->provide( fileUrl, zyppng::ProvideFileSpec() ); + + std::exception_ptr err; + std::optional resOpt; + op->onReady([&]( zyppng::expected &&res ){ + ev->quit(); + if ( !res ) + err = res.error(); + else + resOpt = *res; + }); + + BOOST_REQUIRE( !op->isReady() ); + + if ( !op->isReady() ) + ev->run(); + + BOOST_REQUIRE( !err ); + BOOST_REQUIRE( resOpt.has_value() ); + zypp::PathInfo pi( resOpt->file() ); + BOOST_REQUIRE( pi.isExist() ); + BOOST_REQUIRE( pi.isFile() ); + + std::ifstream in( resOpt->file().asString(), std::ios::binary ); + auto sum = zypp::CheckSum::md5( in ); + BOOST_REQUIRE_EQUAL( sum, std::string("63b4a45ec881d90b83c2e6af7bcbfa78") ); +} + +BOOST_AUTO_TEST_CASE( http_attach ) +{ + using namespace zyppng::operators; + + auto ev = zyppng::EventLoop::create (); + + const auto &workerPath = zypp::Pathname ( TESTS_BUILD_DIR ).dirname() / "tools" / "workers"; + const auto &webRoot = zypp::Pathname (TESTS_SRC_DIR)/"/zyppng/data/downloader"; + zypp::filesystem::TmpDir provideRoot; + + auto prov = zyppng::Provide::create ( provideRoot ); + prov->setWorkerPath ( workerPath ); + prov->start(); + + WebServer web( webRoot.c_str(), 10001, false ); + BOOST_REQUIRE( web.start() ); + + auto op = prov->attachMedia( web.url(), zyppng::ProvideMediaSpec( "OnlineMedia" ) + .setMediaFile( webRoot / "media.1" / "media" ) + .setMedianr(1) ); + + op->onReady([&]( zyppng::expected &&res ){ + ev->quit(); + BOOST_REQUIRE( res.is_valid() ); + BOOST_REQUIRE( !res->handle().empty() ); + }); + + BOOST_REQUIRE( !op->isReady() ); + + if ( !op->isReady() ) + ev->run(); +} + +BOOST_AUTO_TEST_CASE( http_attach_prov ) +{ + using namespace zyppng::operators; + + auto ev = zyppng::EventLoop::create (); + + const auto &workerPath = zypp::Pathname ( TESTS_BUILD_DIR ).dirname() / "tools" / "workers"; + const auto &webRoot = zypp::Pathname ( TESTS_SRC_DIR ) / "zyppng" / "data" / "downloader"; + + zypp::filesystem::TmpDir provideRoot; + + auto prov = zyppng::Provide::create ( provideRoot ); + prov->setWorkerPath ( workerPath ); + prov->start(); + + WebServer web( webRoot.c_str(), 10001, false ); + BOOST_REQUIRE( web.start() ); + + zyppng::Provide::MediaHandle media; + + auto op = prov->attachMedia( web.url(), zyppng::ProvideMediaSpec( "OnlineMedia" ) + .setMediaFile( webRoot / "media.1" / "media" ) + .setMedianr(1) ) + | mbind ( [&]( zyppng::Provide::MediaHandle &&res ){ + media = std::move(res); + return prov->provide( media, "/test.txt", zyppng::ProvideFileSpec() ); + }); + + std::optional fileRes; + op->onReady([&]( zyppng::expected &&res ){ + if ( res ) + fileRes = std::move(*res); + ev->quit(); + }); + + BOOST_REQUIRE( !op->isReady() ); + + if ( !op->isReady() ) + ev->run(); + + BOOST_REQUIRE( fileRes.has_value() ); + zypp::PathInfo pi ( fileRes->file() ); + BOOST_REQUIRE( pi.isExist() && pi.isFile() ); + + std::ifstream in( fileRes->file().asString(), std::ios::binary ); + auto sum = zypp::CheckSum::md5( in ); + BOOST_REQUIRE_EQUAL( sum, std::string("7e562d52c100b68e9d6a561fa8519575") ); +} + +BOOST_AUTO_TEST_CASE( http_attach_prov_404 ) +{ + using namespace zyppng::operators; + + auto ev = zyppng::EventLoop::create (); + + const auto &workerPath = zypp::Pathname ( TESTS_BUILD_DIR ).dirname() / "tools" / "workers"; + const auto &webRoot = zypp::Pathname ( TESTS_SRC_DIR ) / "zyppng" / "data" / "downloader"; + + zypp::filesystem::TmpDir provideRoot; + + auto prov = zyppng::Provide::create ( provideRoot ); + prov->setWorkerPath ( workerPath ); + prov->start(); + + WebServer web( webRoot.c_str(), 10001, false ); + BOOST_REQUIRE( web.start() ); + + zyppng::Provide::MediaHandle media; + + auto op = prov->attachMedia( web.url(), zyppng::ProvideMediaSpec( "OnlineMedia" ) + .setMediaFile( webRoot / "media.1" / "media" ) + .setMedianr(1) ) + | mbind ( [&]( zyppng::Provide::MediaHandle &&res ){ + media = std::move(res); + return prov->provide( media, "/doesnotexist", zyppng::ProvideFileSpec() ); + }); + + std::optional fileRes; + op->onReady([&]( zyppng::expected &&res ){ + ev->quit(); + + BOOST_REQUIRE(!res); + if ( !res ) { + ZYPP_REQUIRE_THROW( std::rethrow_exception( res.error() ), zypp::media::MediaFileNotFoundException ); + } + + }); + + BOOST_REQUIRE( !op->isReady() ); + + if ( !op->isReady() ) + ev->run(); +} + + +// special case where we can detect that something is a directory because we already provided a file from the +// directory that's required as a file. +BOOST_AUTO_TEST_CASE( http_attach_prov_notafile ) +{ + using namespace zyppng::operators; + + auto ev = zyppng::EventLoop::create (); + + const auto &workerPath = zypp::Pathname ( TESTS_BUILD_DIR ).dirname() / "tools" / "workers"; + const auto &webRoot = zypp::Pathname ( TESTS_SRC_DIR ) / "zyppng" / "data" / "downloader"; + + zypp::filesystem::TmpDir provideRoot; + + auto prov = zyppng::Provide::create ( provideRoot ); + prov->setWorkerPath ( workerPath ); + prov->start(); + + WebServer web( webRoot.c_str(), 10001, false ); + BOOST_REQUIRE( web.start() ); + + zyppng::Provide::MediaHandle media; + std::exception_ptr err; + + auto op = prov->attachMedia( web.url(), zyppng::ProvideMediaSpec( "OnlineMedia" ) + .setMediaFile( webRoot / "media.1" / "media" ) + .setMedianr(1) ) + | mbind ( [&]( zyppng::Provide::MediaHandle &&res ){ + media = std::move(res); + return prov->provide( media, "/media.1", zyppng::ProvideFileSpec() ); + }); + + op->onReady([&]( zyppng::expected &&res ){ + ev->quit(); + if ( !res ) + err = res.error(); + }); + + BOOST_REQUIRE( !op->isReady() ); + + if ( !op->isReady() ) + ev->run(); + + BOOST_REQUIRE( err ); + ZYPP_REQUIRE_THROW( std::rethrow_exception( err ), zypp::media::MediaNotAFileException ); +} + +//creates a request handler that requires a authentication to work +WebServer::RequestHandler createAuthHandler ( const zypp::Pathname &webroot = "/" ) +{ + return [ webroot ]( WebServer::Request &req ){ + //Basic dGVzdDp0ZXN0 + auto it = req.params.find( "HTTP_AUTHORIZATION" ); + bool authorized = false; + if ( it != req.params.end() ) + authorized = ( it->second == "Basic dGVzdDp0ZXN0" ); + + if ( !authorized ) { + req.rout << "Status: 401 Unauthorized\r\n" + "Content-Type: text/html; charset=utf-8\r\n" + "WWW-Authenticate: Basic realm=\"User Visible Realm\", charset=\"UTF-8\" \r\n" + "\r\n" + "Sorry you are not authorized."; + return; + } + + //remove "/handler/" prefix + std::string key = req.params.at("SCRIPT_NAME").substr( std::string_view("/handler").length() ); + + req.rout << "Status: 307 Temporary Redirect\r\n" + "Location: "<< webroot / key <<"\r\n\r\n"; + return; + }; +}; + +BOOST_AUTO_TEST_CASE( http_prov_auth ) +{ + using namespace zyppng::operators; + + auto ev = zyppng::EventLoop::create (); + + const auto &workerPath = zypp::Pathname ( TESTS_BUILD_DIR ).dirname() / "tools" / "workers"; + const auto &webRoot = zypp::Pathname ( TESTS_SRC_DIR ) / "zyppng" / "data" / "downloader"; + + zypp::filesystem::TmpDir provideRoot; + + auto prov = zyppng::Provide::create ( provideRoot ); + prov->setWorkerPath ( workerPath ); + prov->start(); + + //don't write or read creds from real settings dir + zypp::filesystem::TmpDir globCredPath; + zypp::filesystem::TmpDir userCredPath; + zypp::media::CredManagerOptions opts; + opts.globalCredFilePath = globCredPath.path() / "credentials.cat"; + opts.userCredFilePath = userCredPath.path() / "credentials.cat"; + prov->setCredManagerOptions( opts ); + + bool gotSigAuthRequired = false; + prov->sigAuthRequired().connect( [&]( const zypp::Url &reqUrl, const std::string &triedUsername, const std::map &extraValues ) -> std::optional { + gotSigAuthRequired = true; + zypp::media::AuthData auth( reqUrl ); + auth.setUsername( "test" ); + auth.setPassword( "test" ); + auth.setLastDatabaseUpdate( time( nullptr ) ); + return auth; + }); + + WebServer web( webRoot.c_str(), 10001, false ); + BOOST_REQUIRE( web.start() ); + + web.setDefaultHandler( createAuthHandler() ); + + auto fileUrl = web.url(); + fileUrl.setPathName( "/handler/test.txt" ); + auto op = prov->provide( fileUrl, zyppng::ProvideFileSpec() ); + + std::exception_ptr err; + std::optional resOpt; + op->onReady([&]( zyppng::expected &&res ){ + ev->quit(); + if ( !res ) + err = res.error(); + else + resOpt = *res; + }); + + BOOST_REQUIRE( !op->isReady() ); + + if ( !op->isReady() ) + ev->run(); + + //BOOST_REQUIRE( gotSigAuthRequired ); + BOOST_REQUIRE( !err ); + BOOST_REQUIRE( resOpt.has_value() ); + zypp::PathInfo pi( resOpt->file() ); + BOOST_REQUIRE( pi.isExist() ); + BOOST_REQUIRE( pi.isFile() ); + + std::ifstream in( resOpt->file().asString(), std::ios::binary ); + auto sum = zypp::CheckSum::md5( in ); + BOOST_REQUIRE_EQUAL( sum, std::string("7e562d52c100b68e9d6a561fa8519575") ); +} + +BOOST_AUTO_TEST_CASE( http_prov_auth_nouserresponse ) +{ + using namespace zyppng::operators; + + //don't write or read creds from real settings dir + zypp::filesystem::TmpDir repoManagerRoot; + zypp::ZConfig::instance().setRepoManagerRoot( repoManagerRoot.path() ); + + auto ev = zyppng::EventLoop::create (); + + const auto &workerPath = zypp::Pathname ( TESTS_BUILD_DIR ).dirname() / "tools" / "workers"; + const auto &webRoot = zypp::Pathname ( TESTS_SRC_DIR ) / "zyppng" / "data" / "downloader"; + + zypp::filesystem::TmpDir provideRoot; + + auto prov = zyppng::Provide::create ( provideRoot ); + prov->setWorkerPath ( workerPath ); + prov->start(); + + //don't write or read creds from real settings dir + zypp::filesystem::TmpDir globCredPath; + zypp::filesystem::TmpDir userCredPath; + zypp::media::CredManagerOptions opts; + opts.globalCredFilePath = globCredPath.path() / "credentials.cat"; + opts.userCredFilePath = userCredPath.path() / "credentials.cat"; + prov->setCredManagerOptions( opts ); + + bool gotSigAuthRequired = false; + prov->sigAuthRequired().connect( [&]( const zypp::Url &, const std::string &, const std::map & ) -> std::optional { + gotSigAuthRequired = true; + return {}; + }); + + WebServer web( webRoot.c_str(), 10001, false ); + BOOST_REQUIRE( web.start() ); + + web.setDefaultHandler( createAuthHandler() ); + + auto fileUrl = web.url(); + fileUrl.setPathName( "/handler/test.txt" ); + auto op = prov->provide( fileUrl, zyppng::ProvideFileSpec() ); + + std::exception_ptr err; + std::optional resOpt; + op->onReady([&]( zyppng::expected &&res ){ + ev->quit(); + if ( !res ) + err = res.error(); + else + resOpt = *res; + }); + + BOOST_REQUIRE( !op->isReady() ); + + if ( !op->isReady() ) + ev->run(); + + BOOST_REQUIRE( gotSigAuthRequired ); + BOOST_REQUIRE( err ); + BOOST_REQUIRE( !resOpt.has_value() ); + ZYPP_REQUIRE_THROW( std::rethrow_exception( err ), zypp::media::MediaUnauthorizedException ); +} + +BOOST_AUTO_TEST_CASE( http_prov_auth_wrongpw ) +{ + using namespace zyppng::operators; + + //don't write or read creds from real settings dir + zypp::filesystem::TmpDir repoManagerRoot; + zypp::ZConfig::instance().setRepoManagerRoot( repoManagerRoot.path() ); + + auto ev = zyppng::EventLoop::create (); + + const auto &workerPath = zypp::Pathname ( TESTS_BUILD_DIR ).dirname() / "tools" / "workers"; + const auto &webRoot = zypp::Pathname ( TESTS_SRC_DIR ) / "zyppng" / "data" / "downloader"; + + zypp::filesystem::TmpDir provideRoot; + + auto prov = zyppng::Provide::create ( provideRoot ); + prov->setWorkerPath ( workerPath ); + prov->start(); + + //don't write or read creds from real settings dir + zypp::filesystem::TmpDir globCredPath; + zypp::filesystem::TmpDir userCredPath; + zypp::media::CredManagerOptions opts; + opts.globalCredFilePath = globCredPath.path() / "credentials.cat"; + opts.userCredFilePath = userCredPath.path() / "credentials.cat"; + prov->setCredManagerOptions( opts ); + + int cntAuthRequired = 0; + prov->sigAuthRequired().connect( [&]( const zypp::Url &reqUrl, const std::string &, const std::map & ) -> std::optional { + cntAuthRequired++; + zypp::media::AuthData auth( reqUrl ); + auth.setLastDatabaseUpdate( time( nullptr ) ); + if ( cntAuthRequired >= 2 ) { + auth.setUsername( "test" ); + auth.setPassword( "test" ); + } else { + auth.setUsername( "test" ); + auth.setPassword( "wrong" ); + } + + return auth; + }); + + WebServer web( webRoot.c_str(), 10001, false ); + BOOST_REQUIRE( web.start() ); + + web.setDefaultHandler( createAuthHandler() ); + + auto fileUrl = web.url(); + fileUrl.setPathName( "/handler/test.txt" ); + auto op = prov->provide( fileUrl, zyppng::ProvideFileSpec() ); + + std::exception_ptr err; + std::optional resOpt; + op->onReady([&]( zyppng::expected &&res ){ + ev->quit(); + if ( !res ) + err = res.error(); + else + resOpt = *res; + }); + + BOOST_REQUIRE( !op->isReady() ); + + if ( !op->isReady() ) + ev->run(); + + BOOST_REQUIRE_EQUAL( cntAuthRequired, 2 ); + BOOST_REQUIRE( !err ); + BOOST_REQUIRE( resOpt.has_value() ); + zypp::PathInfo pi( resOpt->file() ); + BOOST_REQUIRE( pi.isExist() ); + BOOST_REQUIRE( pi.isFile() ); + + std::ifstream in( resOpt->file().asString(), std::ios::binary ); + auto sum = zypp::CheckSum::md5( in ); + BOOST_REQUIRE_EQUAL( sum, std::string("7e562d52c100b68e9d6a561fa8519575") ); +} + +BOOST_AUTO_TEST_CASE( http_attach_prov_auth ) +{ + using namespace zyppng::operators; + + auto ev = zyppng::EventLoop::create (); + + const auto &workerPath = zypp::Pathname ( TESTS_BUILD_DIR ).dirname() / "tools" / "workers"; + const auto &webRoot = zypp::Pathname ( TESTS_SRC_DIR ) / "zyppng" / "data" / "downloader"; + + zypp::filesystem::TmpDir provideRoot; + + auto prov = zyppng::Provide::create ( provideRoot ); + prov->setWorkerPath ( workerPath ); + + //don't write or read creds from real settings dir + zypp::filesystem::TmpDir globCredPath; + zypp::filesystem::TmpDir userCredPath; + zypp::media::CredManagerOptions opts; + opts.globalCredFilePath = globCredPath.path() / "credentials.cat"; + opts.userCredFilePath = userCredPath.path() / "credentials.cat"; + prov->setCredManagerOptions( opts ); + + int cntAuthRequired = 0; + prov->sigAuthRequired().connect( [&]( const zypp::Url &reqUrl, const std::string &, const std::map & ) -> std::optional { + cntAuthRequired++; + zypp::media::AuthData auth( reqUrl ); + auth.setLastDatabaseUpdate( time( nullptr ) ); + auth.setUsername( "test" ); + auth.setPassword( "test" ); + return auth; + }); + + prov->start(); + + WebServer web( webRoot.c_str(), 10001, false ); + web.setDefaultHandler( createAuthHandler() ); + BOOST_REQUIRE( web.start() ); + + zyppng::Provide::MediaHandle media; + + auto baseUrl = web.url(); + baseUrl.setPathName("/handler"); + + auto op = prov->attachMedia( baseUrl, zyppng::ProvideMediaSpec( "OnlineMedia" ) + .setMediaFile( webRoot / "media.1" / "media" ) + .setMedianr(1) ) + | mbind ( [&]( zyppng::Provide::MediaHandle &&res ){ + media = std::move(res); + return prov->provide( media, "/test.txt", zyppng::ProvideFileSpec() ); + }); + + std::optional fileRes; + op->onReady([&]( zyppng::expected &&res ){ + if ( res ) + fileRes = std::move(*res); + ev->quit(); + }); + + BOOST_REQUIRE( !op->isReady() ); + + if ( !op->isReady() ) + ev->run(); + + BOOST_REQUIRE_EQUAL( cntAuthRequired, 1 ); // after the first auth request all others should work + BOOST_REQUIRE( fileRes.has_value() ); + zypp::PathInfo pi ( fileRes->file() ); + BOOST_REQUIRE( pi.isExist() && pi.isFile() ); + + std::ifstream in( fileRes->file().asString(), std::ios::binary ); + auto sum = zypp::CheckSum::md5( in ); + BOOST_REQUIRE_EQUAL( sum, std::string("7e562d52c100b68e9d6a561fa8519575") ); +} + +static void writeTVMConfig ( const zypp::Pathname &file, const zypp::proto::test::TVMSettings &set ) +{ + const auto &fname = file/"tvm.conf"; + int fd = open( fname.asString().data(), O_WRONLY | O_CREAT | O_TRUNC, 0666 ); + if ( fd < 0 ) { + ERR << "Failed to open/create file " << fname << " error: "<attachMedia( zypp::Url("tvm:/"), zyppng::ProvideMediaSpec("CD Test Set") + .setMediaFile( devRoot / (std::string("cd") + zypp::str::numstring(mediaNr)) / (std::string("media.") + zypp::str::numstring(mediaNr)) / "media" ) + .setMedianr(mediaNr)) + | [&prov, mediaNr, fName]( zyppng::expected &&res ){ + if ( res ) { + std::cout << "Attached " << mediaNr << " as: " << res->handle() << std::endl; + return prov->provide ( *res, fName , zyppng::ProvideFileSpec() ) | [ attachId = *res, &prov ]( auto &&res ) { + if ( !res ) { + try { + std::rethrow_exception(res.error()); + } catch ( const zypp::Exception &e ) { + std::cout << "Provide failed with " << e <file () <::error(res.error()) ); + } + } | []( zyppng::expected &&res ) { + if ( !res ) { + try { + std::rethrow_exception(res.error()); + } catch ( const zypp::Exception &e ) { + std::cout << "Provide failed with " << e <::error(res.error()); + } catch ( ... ) { + std::cout << "Provide failed with exception " << std::endl; + return zyppng::expected::error(res.error()); + } + } else { + return zyppng::expected::success(); + } + }; + } + +BOOST_AUTO_TEST_CASE( tvm_basic ) +{ + using namespace zyppng::operators; + + auto ev = zyppng::EventLoop::create (); + + const auto &workerPath = zypp::Pathname ( TESTS_BUILD_DIR ).dirname() / "tools" / "workers"; + const auto &devRoot = zypp::Pathname ( TESTS_SRC_DIR ) / "zyppng" / "data" / "provide"; + + zypp::filesystem::TmpDir provideRoot; + + auto prov = zyppng::Provide::create ( provideRoot ); + prov->setWorkerPath ( workerPath ); + + zypp::proto::test::TVMSettings devSet; + auto dev = devSet.add_devices(); + dev->set_name("/fakedev/tvm/slot1"); + dev->set_insertedpath( (devRoot/"cd1").asString() ); + dev = devSet.add_devices(); + dev->set_name("/fakedev/tvm/slot2"); + dev->set_insertedpath( (devRoot/"cd2").asString() ); + dev = devSet.add_devices(); + dev->set_name("/fakedev/tvm/slot3"); + dev->set_insertedpath( (devRoot/"cd3").asString() ); + writeTVMConfig( provideRoot, devSet ); + + prov->start(); + + std::vector< zyppng::AsyncOpRef>> ops; + + ops.push_back( makeDVDProv( prov, devRoot, 1, "/file1") ); + ops.push_back( makeDVDProv( prov, devRoot, 2, "/file2") ); + ops.push_back( makeDVDProv( prov, devRoot, 3, "/file3") ); + + auto r = std::move(ops) | zyppng::waitFor>(); + r->sigReady().connect([&](){ + ev->quit(); + }); + + BOOST_REQUIRE( !r->isReady() ); + + if ( !r->isReady() ) + ev->run(); + + for ( const auto &res : r->get() ) { + BOOST_REQUIRE(res); + } +} + +BOOST_AUTO_TEST_CASE( tvm_medchange ) +{ + using namespace zyppng::operators; + + auto ev = zyppng::EventLoop::create (); + + const auto &workerPath = zypp::Pathname ( TESTS_BUILD_DIR ).dirname() / "tools" / "workers"; + const auto &devRoot = zypp::Pathname ( TESTS_SRC_DIR ) / "zyppng" / "data" / "provide"; + + zypp::filesystem::TmpDir provideRoot; + + auto prov = zyppng::Provide::create ( provideRoot ); + prov->setWorkerPath ( workerPath ); + + zypp::proto::test::TVMSettings devSet; + auto dev = devSet.add_devices(); + dev->set_name("/fakedev/tvm/slot1"); + dev->set_insertedpath( (devRoot/"cd1").asString() ); + dev = devSet.add_devices(); + dev->set_name("/fakedev/tvm/slot2"); + dev = devSet.add_devices(); + dev->set_name("/fakedev/tvm/slot3"); + writeTVMConfig( provideRoot, devSet ); + + prov->start(); + + bool gotMediaChange = false; + std::vector freeDevices; + int32_t mediaNrAsked = -1; + std::string labelAsked; + prov->sigMediaChangeRequested().connect([&]( const std::string &ref, const std::string &label, const int32_t mediaNr, const std::vector &devices, const std::optional &desc ){ + gotMediaChange = true; + freeDevices = devices; + labelAsked = label; + mediaNrAsked = mediaNr; + devSet.mutable_devices(1)->set_insertedpath( (devRoot/"cd2").asString() ); + writeTVMConfig( provideRoot, devSet ); + return zyppng::Provide::RETRY; + }); + + std::vector< zyppng::AsyncOpRef>> ops; + + ops.push_back( makeDVDProv( prov, devRoot, 1, "/file1") ); + ops.push_back( makeDVDProv( prov, devRoot, 2, "/file2") ); + + auto r = std::move(ops) | zyppng::waitFor>(); + r->sigReady().connect([&](){ + ev->quit(); + }); + + BOOST_REQUIRE( !r->isReady() ); + + if ( !r->isReady() ) + ev->run(); + + BOOST_REQUIRE( gotMediaChange ); + BOOST_REQUIRE_EQUAL( labelAsked, std::string("CD Test Set") ); + BOOST_REQUIRE_EQUAL( mediaNrAsked, 2 ); + BOOST_REQUIRE( std::find( freeDevices.begin(), freeDevices.end(), "/fakedev/tvm/slot2" ) != freeDevices.end() ); + BOOST_REQUIRE( std::find( freeDevices.begin(), freeDevices.end(), "/fakedev/tvm/slot3" ) != freeDevices.end() ); + for ( const auto &res : r->get() ) { + BOOST_REQUIRE(res); + } +} + +zyppng::Provide::Action cancelOps[] = { zyppng::Provide::ABORT, zyppng::Provide::SKIP }; + +BOOST_DATA_TEST_CASE( tvm_medchange_abort, bdata::make( cancelOps ), cancelOp ) +{ + using namespace zyppng::operators; + + auto ev = zyppng::EventLoop::create (); + + const auto &workerPath = zypp::Pathname ( TESTS_BUILD_DIR ).dirname() / "tools" / "workers"; + const auto &devRoot = zypp::Pathname ( TESTS_SRC_DIR ) / "zyppng" / "data" / "provide"; + + zypp::filesystem::TmpDir provideRoot; + + auto prov = zyppng::Provide::create ( provideRoot ); + prov->setWorkerPath ( workerPath ); + + zypp::proto::test::TVMSettings devSet; + auto dev = devSet.add_devices(); + dev->set_name("/fakedev/tvm/slot1"); + dev->set_insertedpath( (devRoot/"cd1").asString() ); + dev = devSet.add_devices(); + dev->set_name("/fakedev/tvm/slot2"); + dev = devSet.add_devices(); + dev->set_name("/fakedev/tvm/slot3"); + writeTVMConfig( provideRoot, devSet ); + + prov->start(); + + bool gotMediaChange = false; + std::vector freeDevices; + int32_t mediaNrAsked = -1; + std::string labelAsked; + prov->sigMediaChangeRequested().connect([&]( const std::string &ref, const std::string &label, const int32_t mediaNr, const std::vector &devices, const std::optional &desc ){ + gotMediaChange = true; + freeDevices = devices; + labelAsked = label; + mediaNrAsked = mediaNr; + return cancelOp; + }); + + auto op1 = makeDVDProv( prov, devRoot, 1, "/file1"); + auto op2 = makeDVDProv( prov, devRoot, 2, "/file2"); + + const auto &readyCB = [&](){ + if ( op1->isReady() && op2->isReady() ) + ev->quit(); + }; + + op1->sigReady().connect( readyCB ); + op2->sigReady().connect( readyCB ); + + BOOST_REQUIRE( !op1->isReady() ); + BOOST_REQUIRE( !op2->isReady() ); + + + if ( !op1->isReady() && !op2->isReady() ) + ev->run(); + + BOOST_REQUIRE( gotMediaChange ); + BOOST_REQUIRE_EQUAL( labelAsked, std::string("CD Test Set") ); + BOOST_REQUIRE_EQUAL( mediaNrAsked, 2 ); + BOOST_REQUIRE( std::find( freeDevices.begin(), freeDevices.end(), "/fakedev/tvm/slot2" ) != freeDevices.end() ); + BOOST_REQUIRE( std::find( freeDevices.begin(), freeDevices.end(), "/fakedev/tvm/slot3" ) != freeDevices.end() ); + BOOST_REQUIRE( op1->get().is_valid() ); + BOOST_REQUIRE( !op2->get().is_valid() ); + if ( cancelOp == zyppng::Provide::ABORT ) { + ZYPP_REQUIRE_THROW( std::rethrow_exception( op2->get().error() ), zypp::AbortRequestException ); + } else { + ZYPP_REQUIRE_THROW( std::rethrow_exception( op2->get().error() ), zypp::SkipRequestException ); + } +} + +BOOST_AUTO_TEST_CASE( tvm_jammed ) +{ + using namespace zyppng::operators; + + auto ev = zyppng::EventLoop::create (); + + const auto &workerPath = zypp::Pathname ( TESTS_BUILD_DIR ).dirname() / "tools" / "workers"; + const auto &devRoot = zypp::Pathname ( TESTS_SRC_DIR ) / "zyppng" / "data" / "provide"; + + zypp::filesystem::TmpDir provideRoot; + + auto prov = zyppng::Provide::create ( provideRoot ); + prov->setWorkerPath ( workerPath ); + + bool gotMediaChange = false; + prov->sigMediaChangeRequested().connect([&]( const std::string &ref, const std::string &label, const int32_t mediaNr, const std::vector &devices, const std::optional &desc ){ + gotMediaChange = true; + return zyppng::Provide::ABORT; + }); + + zypp::proto::test::TVMSettings devSet; + auto dev = devSet.add_devices(); + dev->set_name("/fakedev/tvm/slot1"); + dev->set_insertedpath( (devRoot/"cd1").asString() ); + writeTVMConfig( provideRoot, devSet ); + + prov->start(); + + auto op1 = prov->attachMedia( zypp::Url("tvm:/"), zyppng::ProvideMediaSpec("CD Test Set") + .setMediaFile( devRoot / (std::string("cd") + zypp::str::numstring(1)) / (std::string("media.") + zypp::str::numstring(1)) / "media" ) + .setMedianr(1)); + auto op2 = prov->attachMedia( zypp::Url("tvm:/"), zyppng::ProvideMediaSpec("CD Test Set") + .setMediaFile( devRoot / (std::string("cd") + zypp::str::numstring(2)) / (std::string("media.") + zypp::str::numstring(2)) / "media" ) + .setMedianr(2)); + + + + const auto &readyCB = [&](){ + if ( op1->isReady() && op2->isReady() ) + ev->quit(); + }; + + op1->sigReady().connect( readyCB ); + op2->sigReady().connect( readyCB ); + + + BOOST_REQUIRE( !op1->isReady() ); + BOOST_REQUIRE( !op2->isReady() ); + + + if ( !op1->isReady() && !op2->isReady() ) + ev->run(); + + //we have only one drive which is in use, so we can not be asked for media change + BOOST_REQUIRE( !gotMediaChange ); + BOOST_REQUIRE( op1->get().is_valid() ); + BOOST_REQUIRE( !op2->get().is_valid() ); + ZYPP_REQUIRE_THROW( std::rethrow_exception( op2->get().error() ), zypp::media::MediaJammedException ); +} + +// test if we can release a medium and immediately use the resulting free space for new requests +BOOST_AUTO_TEST_CASE( tvm_jammed_release ) +{ + using namespace zyppng::operators; + + auto ev = zyppng::EventLoop::create (); + + const auto &workerPath = zypp::Pathname ( TESTS_BUILD_DIR ).dirname() / "tools" / "workers"; + const auto &devRoot = zypp::Pathname ( TESTS_SRC_DIR ) / "zyppng" / "data" / "provide"; + + zypp::filesystem::TmpDir provideRoot; + + auto prov = zyppng::Provide::create ( provideRoot ); + prov->setWorkerPath ( workerPath ); + + zypp::proto::test::TVMSettings devSet; + + bool mediaChangeAllowed = false; + bool gotInvalidMediaChange = false; + prov->sigMediaChangeRequested().connect([&]( const std::string &ref, const std::string &label, const int32_t mediaNr, const std::vector &devices, const std::optional &desc ){ + if ( ! mediaChangeAllowed ) { + gotInvalidMediaChange = true; + return zyppng::Provide::ABORT; + } + + devSet.mutable_devices(0)->set_insertedpath( (devRoot/"cd2").asString() ); + writeTVMConfig( provideRoot, devSet ); + return zyppng::Provide::RETRY; + }); + + auto dev = devSet.add_devices(); + dev->set_name("/fakedev/tvm/slot1"); + dev->set_insertedpath( (devRoot/"cd1").asString() ); + writeTVMConfig( provideRoot, devSet ); + + prov->start(); + + auto op1 = prov->attachMedia( zypp::Url("tvm:/"), zyppng::ProvideMediaSpec("CD Test Set") + .setMediaFile( devRoot / (std::string("cd") + zypp::str::numstring(1)) / (std::string("media.") + zypp::str::numstring(1)) / "media" ) + .setMedianr(1)); + + zyppng::AsyncOpRef> op2; + zyppng::AsyncOpRef> op3; + + bool attach1Success = false; + bool attach3Success = false; + std::exception_ptr op2Error; + op1->sigReady().connect( [&]() { + if ( !op1->get() ) { + attach1Success = false; + ev->quit(); + return; + } + + attach1Success = true; + op2 = prov->attachMedia( zypp::Url("tvm:/"), zyppng::ProvideMediaSpec("CD Test Set") + .setMediaFile( devRoot / (std::string("cd") + zypp::str::numstring(2)) / (std::string("media.") + zypp::str::numstring(2)) / "media" ) + .setMedianr(2)); + + op2->sigReady().connect([&](){ + if ( !op2->get().is_valid() ) { + op2Error = op2->get().error(); + + op1.reset(); // kill the first media handle + mediaChangeAllowed = true; + + op3 = prov->attachMedia( zypp::Url("tvm:/"), zyppng::ProvideMediaSpec("CD Test Set") + .setMediaFile( devRoot / (std::string("cd") + zypp::str::numstring(2)) / (std::string("media.") + zypp::str::numstring(2)) / "media" ) + .setMedianr(2)); + op3->sigReady().connect( [&]() { + ev->quit(); + if ( !op3->get() ) + attach3Success = false; + else + attach3Success = true; + }); + return; + } + ev->quit(); + }); + }); + + BOOST_REQUIRE( !op1->isReady() ); + BOOST_REQUIRE( !op2 || !op2->isReady() ); + BOOST_REQUIRE( !op3 || !op3->isReady() ); + + if ( !op1->isReady() ) + ev->run(); + + //we have only one drive which is in use, so we can not be asked for media change + BOOST_REQUIRE( !gotInvalidMediaChange ); + BOOST_REQUIRE( attach1Success ); + BOOST_REQUIRE( op2Error != nullptr ); + ZYPP_REQUIRE_THROW( std::rethrow_exception( op2Error ), zypp::media::MediaJammedException ); + BOOST_REQUIRE( attach3Success ); + BOOST_REQUIRE( !op2->get().is_valid() ); +} + + +#if 0 +BOOST_AUTO_TEST_CASE( dltest_basic ) +{ + using namespace zyppng::operators; + + auto ev = zyppng::EventLoop::create (); + auto prov = zyppng::Provide::create ( "/tmp/provTest" ); + prov->setWorkerPath ( "/home/zbenjamin/build/build-libzypp-Desktop_GCC-Debug/tools/workers" ); + prov->start(); + + std::string baseURL = "http://download.opensuse.org/distribution/leap/15.0/repo/oss"; + std::vector downloads { + "11822f1421ae50fb1a07f72220b79000", "/x86_64/0ad-0.0.22-lp150.2.10.x86_64.rpm", + "b0aaaca4c3763792a495de293c8431c5", "/x86_64/alsa-1.1.5-lp150.4.3.x86_64.rpm", + "a6eb92351c03bcf603a09a2e8eddcead", "/x86_64/amarok-2.9.0-lp150.2.1.x86_64.rpm", + "a2fd84f6d0530abbfe6d5a3da3940d70", "/x86_64/aspell-0.60.6.1-lp150.1.15.x86_64.rpm", + "29b5eab4a9620f158331106df4603866", "/x86_64/atk-devel-2.26.1-lp150.2.4.x86_64.rpm", + "a795874986018674c37af85a62c8f28a", "/x86_64/bing-1.0.5-lp150.2.1.x86_64.rpm", + "ce09cb1af156203c89312f9faffce219", "/x86_64/cdrtools-3.02~a09-lp150.2.11.x86_64.rpm", + "3f57113bd0dea2b5d748b7a343a7fb31", "/x86_64/cfengine-3.11.0-lp150.2.3.x86_64.rpm", + "582cae086f67382e71bf02ff0db554cd", "/x86_64/cgit-1.1-lp150.1.5.x86_64.rpm", + "f10cfc37a20c13d59266241ecc30b152", "/x86_64/ck-devel-0.6.0-lp150.1.3.x86_64.rpm", + "4a19708dc8d58129f8832d934c47888b", "/x86_64/cloud-init-18.2-lp150.1.1.x86_64.rpm", + "7b535587d9bfd8b88edf57b6df5c4d99", "/x86_64/collectd-plugin-pinba-5.7.2-lp150.1.4.x86_64.rpm", + "d86c1d65039645a1895f458f38d9d9e7", "/x86_64/compface-1.5.2-lp150.1.4.x86_64.rpm", + "33a0e878c92b5b298cd6aaec44c0aa46", "/x86_64/compositeproto-devel-0.4.2-lp150.1.6.x86_64.rpm", + "646c6cc180caf27f56bb9ec5b4d50f5b", "/x86_64/corosync-testagents-2.4.4-lp150.3.1.x86_64.rpm", + "10685e733abf77e7439e33471b23612c", "/x86_64/cpupower-bench-4.15-lp150.1.4.x86_64.rpm" + }; + + prov->sigMediaChangeRequested().connect([&]( const std::string &ref, const std::string &label, const int32_t mediaNr, const std::vector &devices, const std::optional &desc ){ + + std::cout << "Please insert medium: " << mediaNr << " of media set " << label << " into one of the free devices: " << std::endl; + if ( desc ) + std::cout << "Media desc: " << *desc << std::endl; + + std::vector choices; + for ( uint i = 0 ; i < devices.size (); i++ ) { + std::cout << "(" << i << ") Free device: " << devices.at(i) << std::endl; + choices.push_back( zypp::str::numstring(i)); + } + + while ( true ) { + std::string choice; + std::cout << "Select the device you want to use, or a for abort" << std::endl; + std::getline( std::cin, choice ); + if ( choice == "a" ) + return zyppng::Provide::ABORT; + + auto i = std::find( choices.begin(), choices.end(), choice ); + if ( i == choices.end() ) { + std::cout << "Invalid answer, please select a device from the list" << std::endl; + continue; + } + + auto sel = std::distance( choices.begin(), i ); + prov->ejectDevice( ref, devices.at(sel) ); + break; + } + + std::string dummy; + std::cout << "Please insert the medium and press a key to continue" << std::endl; + std::getline( std::cin, dummy ); + + return zyppng::Provide::RETRY; + }); + + prov->sigAuthRequired().connect( [&]( const zypp::Url &reqUrl, const std::string &triedUsername, const std::map &extraValues ) -> std::optional { + std::cout << reqUrl << " requires authentication, previously tried username was: " << triedUsername << std::endl; + std::cout << "Please enter the username: " << std::endl; + std::string username; + std::getline ( std::cin, username ); + std::cout << "Please enter the password: " << std::endl; + std::string pass; + std::getline ( std::cin, pass ); + zypp::media::AuthData d( username, pass ); + return d; + }); + + std::vector< zyppng::AsyncOpRef>> ops; +#if 0 + ops.push_back( prov->attachMedia ( baseURL, zyppng::ProvideMediaSpec( "Label1" ).setMediaFile( "/tmp/provTest/media" ).setMedianr(1) ) + | [&]( zyppng::expected &&res ){ + if ( res ) { + std::cout << "Attached as: " << *res << std::endl; + + std::vector>> provs; + for ( std::size_t i = 0; i < downloads.size(); i+=2 ) { + const auto &pName = zypp::Pathname(downloads[i+1]); + std::cout << "Asking to provide file: " << pName << std::endl; + provs.push_back( prov->provide ( *res, pName, zyppng::ProvideFileSpec() ) | []( auto &&res ) { + if ( !res ) { + try { + std::rethrow_exception(res.error()); + } catch ( const zypp::Exception &e ) { + std::cout << "Provide failed with " << e <file () <>(); + } + + std::cout << "Failed to attach media" << std::endl; + return zyppng::makeReadyResult( std::vector> { zyppng::expected::error(res.error()) } ); + } | []( std::vector> &&results ){ + for ( const auto &r : results ) { + if ( !r ) + return zyppng::expected::error( r.error() ); + } + return zyppng::expected::success(); + }); +#endif + + const auto &makeDVDProv = [&]( int mediaNr, const std::string &fName ){ + return prov->attachMedia( zypp::Url("dvd:/"), zyppng::ProvideMediaSpec("CD Test Set") + .setMediaFile("/tmp/provTest/mediacd") + .setMedianr(mediaNr) + .addCustomHeaderValue ("device", "/dev/sr1") + .addCustomHeaderValue ("device", "/dev/sr2")) + | [&]( zyppng::expected &&res ){ + if ( res ) { + std::cout << "Attached as: " << res->handle() << std::endl; + return prov->provide ( *res, fName , zyppng::ProvideFileSpec() ) | [ attachId = *res, &prov ]( auto &&res ) { + if ( !res ) { + try { + std::rethrow_exception(res.error()); + } catch ( const zypp::Exception &e ) { + std::cout << "Provide failed with " << e <file () <::error(res.error()) ); + } + } | []( zyppng::expected &&res ) { + if ( !res ) { + try { + std::rethrow_exception(res.error()); + } catch ( const zypp::Exception &e ) { + std::cout << "Provide failed with " << e <::error(res.error()); + } catch ( ... ) { + std::cout << "Provide failed with exception " << std::endl; + return zyppng::expected::error(res.error()); + } + } else { + return zyppng::expected::success(); + } + }; + }; + + ops.push_back( makeDVDProv(1, "/file1") ); + ops.push_back( makeDVDProv(1, "/file1") ); + ops.push_back( makeDVDProv(1, "/file1") ); + + auto r = std::move(ops) | zyppng::waitFor>(); + r->onReady( [&]( std::vector> &&results ){ + for ( const auto &res : results ) { + if ( res ) { + std::cout << "Request finished successfull" << std::endl; + } else { + try { + std::rethrow_exception (res.error()); + } catch ( const zypp::Exception &e ) { + std::cout << "Request failed with " << e <quit(); + }); + + ev->run (); + return; +} +#endif diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 4c8a8fd..b8d225d 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -17,4 +17,23 @@ ENDFOREACH( loop_var ) INSTALL(TARGETS zypp-CheckAccessDeleted DESTINATION "${CMAKE_INSTALL_PREFIX}/bin") INSTALL(TARGETS zypp-NameReqPrv DESTINATION "${CMAKE_INSTALL_PREFIX}/bin") + +SET( ZYPP_TOOLS_DIR ${CMAKE_CURRENT_SOURCE_DIR} ) + ADD_SUBDIRECTORY( zypp-rpm ) +ADD_SUBDIRECTORY( zypp-media-http ) +ADD_SUBDIRECTORY( zypp-media-disc ) +ADD_SUBDIRECTORY( zypp-media-nfs ) +ADD_SUBDIRECTORY( zypp-media-smb ) +ADD_SUBDIRECTORY( zypp-media-dir ) +ADD_SUBDIRECTORY( zypp-media-disk ) +ADD_SUBDIRECTORY( zypp-media-iso ) +ADD_SUBDIRECTORY( zypp-media-copy ) +ADD_SUBDIRECTORY( zypp-media-tvm ) +ADD_SUBDIRECTORY( zypp-media-chksum ) + +FIND_PACKAGE( Notcurses++ ) +if ( Notcurses++_FOUND) + message("Found Notcurses++, enabling repomirror build") + ADD_SUBDIRECTORY( repomirror ) +endif() diff --git a/tools/repomirror/CMakeLists.txt b/tools/repomirror/CMakeLists.txt new file mode 100644 index 0000000..aa86016 --- /dev/null +++ b/tools/repomirror/CMakeLists.txt @@ -0,0 +1,16 @@ +PROJECT( repomirror C CXX ) + +set (CMAKE_CXX_STANDARD 17) +SET (CMAKE_CXX_EXTENSIONS OFF) + +SET( SOURCES + output.h + main.cc +) + +ADD_DEFINITIONS( -DZYPP_BUILD_DIR="${CMAKE_CURRENT_BINARY_DIR}" ) + +message(${Notcurses++_LIBRARIES}) + +add_executable( ${PROJECT_NAME} ${SOURCES} ) +target_link_libraries( ${PROJECT_NAME} zypp ${Notcurses++_LIBRARIES} -lnotcurses-core ) diff --git a/tools/repomirror/main.cc b/tools/repomirror/main.cc new file mode 100644 index 0000000..b55bdb1 --- /dev/null +++ b/tools/repomirror/main.cc @@ -0,0 +1,405 @@ +#define INCLUDE_TESTSETUP_WITHOUT_BOOST 1 +#include "../../tests/lib/TestSetup.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "output.h" + +class DlSkippedException : public zypp::Exception +{ + public: + DlSkippedException() : zypp::Exception("Download was skipped" ) {} +}; + +class MyDLStatus : public zyppng::ProvideStatus +{ + public: + MyDLStatus ( OutputView &out, zyppng::ProvideRef parent ) : ProvideStatus( parent ), _out(out) {} + + virtual void pulse ( ) { + zyppng::ProvideStatus::pulse(); + + const auto &currStats = stats(); + + if ( currStats._expectedBytes == 0 ) + return; + + auto perc = (double)(currStats._finishedBytes + currStats._partialBytes) / (double)currStats._expectedBytes; + std::string txt = zypp::str::Str() << "Downloading " << zypp::ByteCount((currStats._finishedBytes + currStats._partialBytes)) << "/" << currStats._expectedBytes << " (" << currStats._perSecond<<"/s)"; + _out.updateProgress( txt, perc ); + } + + private: + OutputView &_out; +}; + +/*! + * Downloads a solvable via the given handle. + * Returns the AsyncResult + */ +zyppng::AsyncOpRef> provideSolvable ( std::shared_ptr output, zyppng::ProvideMediaHandle handle, const zypp::sat::Solvable &s ) +{ + using namespace zyppng::operators; + + auto prov = handle.parent(); + if ( !prov ) + return zyppng::makeReadyResult( zyppng::expected::error( std::make_exception_ptr( zypp::Exception("Handle without parent!"))) ); + + zypp::PoolItem pi(s); + if ( !pi->isKind() ) { + //output->putMsgErr( zypp::str::Str() << "Skipping 1: " << pi.asUserString() << "\n" ); + return zyppng::makeReadyResult( zyppng::expected::error( std::make_exception_ptr(DlSkippedException())) ); + } + + auto oml = pi.lookupLocation(); + if ( oml.filename().empty() ) { + output->putMsgErr( zypp::str::Str() << "Skipping: " << pi.asUserString() << "\n" ); + return zyppng::makeReadyResult( zyppng::expected::error( std::make_exception_ptr(DlSkippedException())) ); + } + + // enable for more output, but sloooooows the startup time down significantly + //output->putMsgTxt( zypp::str::Str() << "Downloading " << pi.asUserString() << " size is: " << pi.downloadSize() << "\n" ); + + return prov->provide( handle, oml.filename(), zyppng::ProvideFileSpec( oml ) ) + | [output]( zyppng::expected &&res ) { + if ( res ) { + output->putMsgTxt( zypp::str::Str() << "File downloaded: " << res->file() << "\n" ); + } else { + output->putMsgErr( zypp::str::Str() << "Failed to download file\n" ); + try + { + std::rethrow_exception(res.error()); + } + catch(const zypp::Exception& e) + { + output->putMsgErr( zypp::str::Str() << e.asUserHistory() << '\n' ); + } + catch(const std::exception& e) + { + output->putMsgErr( zypp::str::Str() << e.what() << '\n' ); + } + } + return res; + }; +} + +/*! + * Helper function to copy a ProvideRes underlying file to a target destination, keeping the ProvideRes alive until the copy operation has finished + */ +zyppng::AsyncOpRef> copyProvideResToFile ( std::shared_ptr output, zyppng::ProvideRef prov, zyppng::ProvideRes &&res, const zypp::Pathname &targetDir ) +{ + using namespace zyppng::operators; + + auto fName = res.file(); + return prov->copyFile( fName, targetDir/fName.basename() ) + | [ resSave = std::move(res) ] ( auto &&result ) { + // callback lambda to keep the ProvideRes reference around until the op is finished, + // if the op fails the callback will be cleaned up and so the reference + return result; + }; +} + +/*! + * Helper function to calculate a checksum on a ProvideRes result file, keeping the ProvideRes alive during the operation and returning it again after the test was successful, + * here we could also ask the user if a invalid checksum should be accepted and ignore the result if the user wants to + */ +zyppng::AsyncOpRef> checksumIfRequired ( std::shared_ptr output, zyppng::ProvideRef prov, zypp::sat::Solvable solvable, zyppng::ProvideRes &&res ) +{ + using namespace zyppng::operators; + + auto fName = res.file(); + + const auto oml = solvable.lookupLocation(); + if ( !oml.checksum().empty() ) { + output->putMsgTxt( zypp::str::Str()<<" calculating checksum for file: " << fName << "\n"); + + return prov->checksumForFile( fName, oml.checksum().type() ) + | mbind([ output, expectedSum = oml.checksum(), saveRes = std::move(res) ]( auto &&checksumRes ) { + output->putMsgTxt( zypp::str::Str()<<" Got checksum for file: " << saveRes.file() << " is: " << checksumRes << "\n" ); + try { + zypp::CheckSum s( expectedSum.type(), checksumRes ); + if ( expectedSum == s ) + return zyppng::expected::success(std::move(saveRes)); + else + return zyppng::expected::error( ZYPP_EXCPT_PTR( zypp::CheckSumCheckException( zypp::str::Str() << saveRes.file() << " has wrong checksum" ) ) ); + } catch ( const zypp::Exception &e ) { + ZYPP_CAUGHT(e); + return zyppng::expected::error( ZYPP_EXCPT_PTR(e) ); + } + + }); + } + + return zyppng::makeReadyResult( zyppng::expected::success(std::move(res)) ); +} + + + +int main ( int argc, char *argv[] ) +{ + using namespace zyppng::operators; + + auto ev = zyppng::EventLoop::create(); + + auto output = OutputView::create(); + if ( !output ) { + std::cerr << "Failed to initialize Output view" << std::endl; + return 1; + } + + output->putMsgTxt("Loading System\n"); + output->putMsgErr("Errors go here\n"); + +#if 0 + { + auto prov = zyppng::Provide::create(); + prov->setStatusTracker( std::make_shared( *output, prov) ); + + zypp::Url test("copy://"); + test.setPathName("/tmp/test/test1"); + + auto rootOp = prov->provide( test, zyppng::ProvideFileSpec().setDestFilenameHint("/tmp/test/test2") ) + | [&]( zyppng::expected &&res ) { + if ( !res ) { + output->putMsgErr( zypp::str::Str() << "Failed to copy file: ", false ); + try { + std::rethrow_exception(res.error()); + + } catch(const zypp::Exception& e) { + output->putMsgErr( zypp::str::Str() << e.asUserHistory() << '\n' ); + + } catch(const std::exception& e) { + output->putMsgErr( zypp::str::Str() << e.what() << '\n' ); + + } catch(...) { + output->putMsgErr( zypp::str::Str() << "Unknown exception\n" ); + } + } else { + output->putMsgTxt( zypp::str::Str() << "Yay copy worked\n" ); + } + return 0; + }; + + prov->start(); + if ( !rootOp->isReady() ) + ev->run(); + output->putMsgTxt( zypp::str::Str() << "Waiting, press Return to exit\n" ); + output->waitForKeys( { NCKEY_ENTER } ); + return 0; + } +#endif + + TestSetup t; + try { + t.LoadSystemAt( "/" ); + } catch ( const zypp::Exception &e ) { + output.reset(); //disable curses + std::cerr << "Failed to load repo info from system." << std::endl; + std::cerr << "Got exception: " << e << std::endl; + return 1; + } + + zypp::RepoManager rManager( zypp::RepoManagerOptions("/") ); + auto pool = t.pool(); + auto satPool = t.satpool(); + + output->putMsgTxt("Loading repositories\n"); + + std::vector myRepos; + for ( const auto &r : t.satpool().repos() ) { + if ( r.isSystemRepo() ) + continue; + myRepos.push_back(r); + } + + if ( !myRepos.size() ) { + output->putMsgTxt( zypp::str::Str() << "No repos found, press any key to exit\n" ); + output->waitForKeys(); + return 0; + } + + std::vector reposToDl = output->promptMultiSelect( "Select the repos you want to download", "", myRepos ); + output->renderNow(); + + if( !reposToDl.size() ) { + output->putMsgTxt( zypp::str::Str() << "No repos selected, press any key to exit\n" ); + output->waitForKeys(); + return 0; + } + + const auto &workerPath = zypp::Pathname ( ZYPP_BUILD_DIR ).dirname() / "tools" / "workers"; + auto prov = zyppng::Provide::create(); + prov->setStatusTracker( std::make_shared( *output, prov) ); + prov->sigAuthRequired().connect( [&](const zypp::Url &reqUrl, const std::string &triedUsername, const std::map &extraValues) -> std::optional { + + auto user = output->promptUser( + zypp::str::Str() << "Auth request for URL: " << reqUrl << "\n" + << "Last tried username was: " << triedUsername << "\n", + "Username" ); + if ( !user ) + return {}; + + zypp::media::AuthData d; + d.setUrl( reqUrl ); + d.setUsername( *user ); + + auto pass = output->promptUser("", "Password"); + if ( pass ) + d.setPassword( *pass ); + + return d; + }); + + std::vector< zyppng::AsyncOpRef< std::vector>> > mop; + + // we need a way to remember the data for the full transaction, there is probably a better way for a real application + std::unordered_map>> repoToMediaToSolvables; + + zypp::Pathname workPath = zypp::Pathname(".").realpath() / "allrpms"; + zypp::filesystem::assert_dir( workPath ); + + for ( const auto repoToDl : reposToDl ) { + const auto &r = myRepos[repoToDl]; + output->putMsgTxt( zypp::str::Str() << "Repo selected to download: " << r.asUserString() << "\n" ); + + auto urlSet = r.info().baseUrls(); + if ( urlSet.empty() ) { + output->putMsgTxt( zypp::str::Str() << "Repo has no mirrors, ignoring!\n" ); + continue; + } + + std::vector urlsWithPath; + std::transform( urlSet.begin(), urlSet.end(), std::back_inserter(urlsWithPath), [&]( const zypp::Url &bUrl ){ + zypp::Url withPath(bUrl); + withPath.setPathName( r.info().path() / bUrl.getPathName() ); + return withPath; + }); + + zypp::ByteCount bc; + output->putMsgTxt( zypp::str::Str() << "Adding Solvables for repo: " << r << "\n" ); + std::unordered_map> &mediaToSolvables = repoToMediaToSolvables[repoToDl]; + std::for_each( satPool.solvablesBegin(), satPool.solvablesEnd(), [&]( const zypp::sat::Solvable &s ) { + if ( s.repository() != r ) + return; + bc += s.downloadSize(); + const auto mediaNr = s.lookupLocation().medianr(); + mediaToSolvables[mediaNr].push_back(s); + }); + output->putMsgTxt( zypp::str::Str() << "Download size for repo: " << r << ":" << bc << "\n" ); + + std::transform( mediaToSolvables.begin(), mediaToSolvables.end(), std::back_inserter(mop), [&]( const auto &s ) { + + auto spec = zyppng::ProvideMediaSpec( r.info().name() ); + zypp::Pathname mediafile = r.info().metadataPath(); + if ( !mediafile.empty() ) { + mediafile += "/media.1/media"; + if ( zypp::PathInfo(mediafile).isExist() ) { + spec.setMediaFile( mediafile ); + spec.setMedianr( s.first ); + } + } + + return prov->attachMedia( urlsWithPath, spec ) + | [ &, &solvables = s.second ]( zyppng::expected &&hdl ) { + + if ( !hdl ) { + output->putMsgErr( zypp::str::Str() << "Failed to attach medium: ", false ); + try { + std::rethrow_exception(hdl.error()); + + } catch(const zypp::Exception& e) { + output->putMsgErr( zypp::str::Str() << e.asUserHistory() << '\n' ); + + } catch(const std::exception& e) { + output->putMsgErr( zypp::str::Str() << e.what() << '\n' ); + + } catch(...) { + output->putMsgErr( zypp::str::Str() << "Unknown exception\n" ); + + } + return std::vector>>{ zyppng::makeReadyResult( zyppng::expected::error( hdl.error() ) ) }; + } + + return zyppng::transform( solvables, [ &, hdl=hdl.get() ]( const zypp::sat::Solvable &s ) { + return s | (std::bind( &provideSolvable, output, hdl, std::placeholders::_1 )) + | mbind (std::bind( &checksumIfRequired, output, prov, sat::Solvable(s), std::placeholders::_1 )) + | mbind (std::bind( ©ProvideResToFile, output, prov, std::placeholders::_1, workPath ) ) ; + }); + } + | zyppng::waitFor>() + | [ &r, &output ]( const auto &&res ){ + output->putMsgTxt( zypp::str::Str() << "Finished with rpo: " << r.info().name() << "\n" ); + return res; + }; + }); + } + + std::vector> finalResults; + + auto rootOp = std::move( mop ) + | zyppng::waitFor< std::vector> >() + | [&]( std::vector>> &&allResults ) { + // reduce to one vector of results + for ( auto &innerVec : allResults ) { + std::accumulate( std::make_move_iterator(innerVec.begin()), std::make_move_iterator(innerVec.end()), &finalResults, []( std::vector>* vec, auto &&bla ){ + vec->push_back(std::move(bla)); + return vec; + }); + } + ev->quit(); + return 0; + }; + + prov->start(); + if ( !rootOp->isReady() ) + ev->run(); + + int succ = 0; + int skip = 0; + int err = 0; + for ( const auto &res : finalResults ) { + if ( res ) + succ++; + else { + try { + std::rethrow_exception( res.error() ); + } catch(const DlSkippedException& e) { + skip++; + } catch( const zypp::Exception &e ) { + output->putMsgErr( zypp::str::Str()<<"Request failed with: " << e.asUserString() << "\n"); + err++; + } catch( const std::exception &e ) { + output->putMsgErr( zypp::str::Str()<<"Request failed with: " << e.what() << "\n"); + err++; + } catch( ... ) { + output->putMsgErr( zypp::str::Str()<<"Request failed with an unknown error.\n" ); + err++; + } + } + } + #if 0 + for ( const auto &outer : finalResults ) { + + } + #endif + + output->putMsgTxt( zypp::str::Str() << "All done:\nSuccess:" << succ << "\nErrors: "<< err << "\nSkipped:"<putMsgTxt( zypp::str::Str() << "Waiting, press Return to exit\n" ); + output->waitForKeys( { NCKEY_ENTER } ); + return 0; +} diff --git a/tools/repomirror/output.h b/tools/repomirror/output.h new file mode 100644 index 0000000..8306e34 --- /dev/null +++ b/tools/repomirror/output.h @@ -0,0 +1,269 @@ +#ifndef OUTPUT_H +#define OUTPUT_H + +#include +#include +#include +#include +#include +#include + + +static void zypp_release_ncplane ( struct ncplane *ptr ) { if ( ptr ) ncplane_destroy(ptr); } +static void zypp_release_progbar ( struct ncprogbar *ptr ) { if ( ptr ) ncprogbar_destroy(ptr); } + +class OutputView : public zyppng::Base +{ + public: + ~OutputView() { + } + static std::shared_ptr create( ) { + + auto ptr = std::shared_ptr( new OutputView() ); + ptr->_nc = std::make_unique(); + if ( !ptr->_nc ) + return nullptr; + + ptr->_stdplane = std::unique_ptr( ptr->_nc->get_stdplane() ); + if ( !ptr->_stdplane ) + return nullptr; + + int rows = 0; + int cols = 0; + ptr->_stdplane->get_dim( rows, cols ); + + int top_rows = ( rows - 2 ); + if ( top_rows <= 0 ){ + return nullptr; + } + + int bottom_rows = rows - top_rows; + if ( bottom_rows <= 0 ) { + return nullptr; + } + + // the outer plane this is just used to draw a nice border + ptr->_topOuterPlane = std::make_unique( *ptr->_stdplane, top_rows, cols, 0, 0, nullptr ); + ptr->_topOuterPlane->rounded_box_sized( 0, 0, top_rows, cols, 0 ); + + // the inner plane, here we put the text + int r,c; + ptr->_topOuterPlane->get_dim( r, c ); + ptr->_topPlaneLeft = std::make_unique( *ptr->_topOuterPlane, r-2 , (c-1) / 2, 1, 1 ); + ptr->_topPlaneLeft->set_scrolling( true ); + + ptr->_topPlaneRight = std::make_unique( *ptr->_topOuterPlane, r-2 , c - ptr->_topPlaneLeft->get_dim_x() - 1 , 1, ptr->_topPlaneLeft->get_dim_x() + 1 ); + ptr->_topPlaneRight->set_scrolling( true ); + + // the plane where we show the progressbar text + ptr->_progBarTextPlane = std::make_unique( *ptr->_stdplane, 1 , cols, top_rows, 0 ); + + // to create the progressbar we need to fall back to the C API due to some bugs in the C++ progbar implementation + ncplane_options nopts = { + .y = top_rows+1, + .x = 0, + .rows = bottom_rows-1, + .cols = cols, + .userptr = nullptr, + .name = nullptr, + .resizecb = nullptr, + .flags = 0, + .margin_b = 0, + .margin_r = 0, + }; + + std::unique_ptr bottomPlane( ncplane_create( *ptr->_stdplane, &nopts ), &zypp_release_ncplane ); + if ( !bottomPlane ) { + return nullptr; + } + + struct ncprogbar_options popts = { + .ulchannel = 0, + .urchannel = 0, + .blchannel = 0, + .brchannel = 0, + .flags = 0, + }; + ptr->_progbar = std::unique_ptr ( ncprogbar_create( bottomPlane.release(), &popts ), &zypp_release_progbar ); + if ( !ptr->_progbar ) { + return nullptr; + } + return ptr; + } + + void updateProgress ( const std::string &txt, double prog, bool autoRender = true ) { + if ( !_progbar ) + return; + + _progBarTextPlane->erase(); + _progBarTextPlane->putstr( 0, ncpp::NCAlign::Center, txt.data() ); + ncprogbar_set_progress( _progbar.get(), prog ); + if ( autoRender ) + renderNow(); + } + + void putMsgTxt ( const std::string &txt, bool doAutoRender = true ) { + if ( !_topPlaneLeft ) + return; + + ncplane_puttext( *_topPlaneLeft, -1, NCALIGN_LEFT, txt.data(), nullptr ); + if ( doAutoRender ) + renderNow(); + } + + void putMsgErr ( const std::string &txt, bool doAutoRender = true ) { + if ( !_topPlaneRight ) + return; + + _topPlaneRight->set_fg_rgb( 0xff5349 ); + ncplane_puttext( *_topPlaneRight, -1, NCALIGN_LEFT, txt.data(), nullptr ); + _topPlaneRight->set_fg_default(); + if ( doAutoRender ) + renderNow(); + } + + std::optional promptUser ( const std::string &desc, const std::string &label ) + { + if ( desc.size() ) + putMsgTxt( desc, false ); + putMsgTxt( label + ": " ); + + int curX, curY; + _topPlaneLeft->get_cursor_yx( curY, curX ); + ncplane_options nopts = { + .y = curY, + .x = curX, + .rows = 1, + .cols = _topPlaneLeft->get_dim_x() - curX, + .userptr = nullptr, + .name = nullptr, + .resizecb = nullptr, + .flags = 0, + .margin_b = 0, + .margin_r = 0, + }; + + std::unique_ptr inputPlane( ncplane_create( *_topPlaneLeft, &nopts ), &zypp_release_ncplane ); + if ( !inputPlane ) { + return {}; + } + + auto readOpts = ncreader_options { + .tchannels = 0, + .tattrword = 0, + .flags = NCREADER_OPTION_CURSOR | NCREADER_OPTION_NOCMDKEYS + }; + std::unique_ptr reader( ncreader_create( inputPlane.release(), &readOpts ), []( struct ncreader *relme){ ncreader_destroy(relme, nullptr);} ); + if ( !reader ) + return {}; + + while ( true ) { + renderNow(); + ncinput ni; + _nc->get( true, &ni ); + if ( ni.id == NCKEY_ENTER ) { + break; + } + ncreader_offer_input( reader.get(), &ni ); + } + + std::unique_ptr data( ncreader_contents( reader.get() ), free ); + std::string_view cppData( data.get() ); + if ( cppData.empty() ) + return {}; + + return std::string(cppData); + } + + template + std::vector promptMultiSelect( const std::string &title, const std::string &secondary, const std::vector &data ) { + + // use a boost container here, because the stdlib decided to completely break vector by prematurely optimizing it + boost::container::vector selected( data.size(), false ); + std::vector items; + std::vector> stringsStorage; + items.reserve( data.size() ); + stringsStorage.reserve( data.size() ); + + for ( const T& entry : data ) { + stringsStorage.push_back( std::make_pair( entry.asUserString(), std::string("") ) ); + items.push_back( ncmselector_item{ + .option = stringsStorage.back().first.data(), + .desc = stringsStorage.back().second.data(), + .selected = false + }); + } + items.push_back(ncmselector_item{ + .option = nullptr, + .desc = nullptr, + .selected = false + }); + + std::string_view footer("Press Enter to accept or ESC to cancel."); + + auto selOpts = ncmultiselector_options{ + .title = title.data(), + .secondary = secondary.data(), + .footer = footer.data(), + .items = items.data(), + .maxdisplay = 0, + .opchannels = NCCHANNELS_INITIALIZER(0xe0, 0x80, 0x40, 0, 0, 0), + .descchannels = NCCHANNELS_INITIALIZER(0x80, 0xe0, 0x40, 0, 0, 0), + .titlechannels = NCCHANNELS_INITIALIZER(0x20, 0xff, 0xff, 0, 0, 0x20), + .footchannels = NCCHANNELS_INITIALIZER(0xe0, 0, 0x40, 0x20, 0x20, 0), + .boxchannels = NCCHANNELS_INITIALIZER(0x20, 0xe0, 0xe0, 0x20, 0, 0), + .flags = 0 + }; + + auto tmpPlane = ncpp::Plane( *_stdplane, _stdplane->get_dim_y()-1, _stdplane->get_dim_x()-1, 1, 1 ); + ncpp::MultiSelector selector( tmpPlane, &selOpts); + + while ( true ) { + renderNow(); + ncinput ni; + _nc->get( true, &ni ); + if ( ni.id == NCKEY_ESC ) { + return {}; + } else if ( ni.id == NCKEY_ENTER ) { + break; + } + selector.offer_input( &ni ); + } + + selector.get_selected( selected.data(), selected.size() ); + std::vector selIndices; + for ( int i = 0; i < selected.size(); i++ ) { + if ( selected[i]) + selIndices.push_back(i); + } + + return selIndices; + } + + void renderNow () { + if ( _nc ) _nc->render(); + } + + uint32_t waitForKeys ( std::vector keys = {} ) { + ncinput ni; + while ( true ) { + _nc->get( true, &ni ); + if ( keys.empty() || std::find( keys.begin(), keys.end(), ni.id ) != keys.end() ) + return ni.id; + } + } + + private: + OutputView() : _topOuterPlane( nullptr ), _topPlaneLeft( nullptr ), _topPlaneRight( nullptr ), _progbar( nullptr, &zypp_release_progbar ) {} + + std::unique_ptr _nc; + std::unique_ptr _stdplane; + std::unique_ptr _topOuterPlane; + std::unique_ptr _topPlaneLeft; + std::unique_ptr _topPlaneRight; + std::unique_ptr _progBarTextPlane; + std::unique_ptr _progbar; +}; + + +#endif // OUTPUT_H diff --git a/tools/zypp-media-chksum/CMakeLists.txt b/tools/zypp-media-chksum/CMakeLists.txt new file mode 100644 index 0000000..153a4d5 --- /dev/null +++ b/tools/zypp-media-chksum/CMakeLists.txt @@ -0,0 +1,27 @@ +PROJECT( zypp-media-chksum C CXX ) + +set (CMAKE_CXX_STANDARD 17) +SET (CMAKE_CXX_EXTENSIONS OFF) + +FIND_PACKAGE(Protobuf REQUIRED) + +SET( SOURCES + main.cc +) + +add_executable( ${PROJECT_NAME} ${SOURCES} ) +target_link_libraries( ${PROJECT_NAME} zypp-media ) +target_link_libraries( ${PROJECT_NAME} zypp-core ) +target_link_libraries( ${PROJECT_NAME} zypp-protobuf ) +target_link_libraries( ${PROJECT_NAME} ${PROTOBUF_LITE_LIBRARIES} ) + +IF ( INSTALL_NG_BINARIES ) + INSTALL( TARGETS ${PROJECT_NAME} DESTINATION "${ZYPP_LIBEXEC_INSTALL_DIR}/workers" ) +ENDIF() + +set_target_properties( ${PROJECT_NAME} + PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" +) diff --git a/tools/zypp-media-chksum/main.cc b/tools/zypp-media-chksum/main.cc new file mode 100644 index 0000000..33b6005 --- /dev/null +++ b/tools/zypp-media-chksum/main.cc @@ -0,0 +1,138 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ + +#include +#include +#include +#include +#include +#include +#include + +class ChksumProvider : public zyppng::worker::ProvideWorker +{ + public: + ChksumProvider( std::string_view workerName ) : ProvideWorker( workerName ) { } + + void immediateShutdown() override { } + + protected: + // ProvideWorker interface + zyppng::expected initialize(const zyppng::worker::Configuration &conf) override { + + zyppng::worker::WorkerCaps caps; + caps.set_worker_type ( zyppng::worker::WorkerCaps::CPUBound ); + caps.set_cfg_flags( + zyppng::worker::WorkerCaps::Flags ( + zyppng::worker::WorkerCaps::Pipeline + | zyppng::worker::WorkerCaps::ZyppLogFormat + ) + ); + + return zyppng::expected::success(caps); + } + + void provide() override { + auto &queue = requestQueue(); + + if ( !queue.size() ) + return; + + + auto req = queue.front(); + queue.pop_front(); + + // here we only receive request codes, we only support Provide messages, all others are rejected + // Cancel is never to be received here + if ( req->_spec.code() != zyppng::ProvideMessage::Code::Provide ) { + req->_state = zyppng::worker::ProvideWorkerItem::Finished; + provideFailed( req->_spec.requestId() + , zyppng::ProvideMessage::Code::BadRequest + , "Request type not implemented" + , false + , {} ); + return; + } + + zypp::Url url; + const auto &urlVal = req->_spec.value( zyppng::ProvideMsgFields::Url ); + try { + url = zypp::Url( urlVal.asString() ); + } catch ( const zypp::Exception &excp ) { + ZYPP_CAUGHT(excp); + + std::string err = zypp::str::Str() << "Invalid URL in request: " << urlVal.asString(); + ERR << err << std::endl; + + req->_state = zyppng::worker::ProvideWorkerItem::Finished; + provideFailed( req->_spec.requestId() + , zyppng::ProvideMessage::Code::BadRequest + , err + , false + , {} ); + + return; + } + + std::string chksumType; + try { + chksumType = req->_spec.value( std::string_view("chksumType") ).asString(); + } catch ( const zypp::Exception &excp ) { + ZYPP_CAUGHT(excp); + + std::string err = zypp::str::Str() << "No or invalid chksumType in request"; + ERR << err << std::endl; + + req->_state = zyppng::worker::ProvideWorkerItem::Finished; + provideFailed( req->_spec.requestId() + , zyppng::ProvideMessage::Code::BadRequest + , err + , false + , {} ); + + return; + } + + zypp::Pathname file = url.getPathName(); + if ( ! zypp::PathInfo( file ).isFile() ) { + req->_state = zyppng::worker::ProvideWorkerItem::Finished; + provideFailed( req->_spec.requestId() + , zyppng::ProvideMessage::Code::NotFound + , zypp::str::Str() << "File " << file << " not found." + , false + , {} ); + return; + } + + std::string filesum = zypp::filesystem::checksum( file, chksumType ); + DBG << "Calculated checksum for : " << file << " with type: " << chksumType << " is " << filesum << std::endl; + provideSuccess( req->_spec.requestId(), false, file, {{chksumType, {filesum}}} ); + } + + void cancel(const std::deque::iterator &i ) override { + ERR << "Bug, cancel should never be called for running items" << std::endl; + } + +}; + +int main( int argc, char *argv[] ) +{ + // lets ignore those, SIGPIPE is better handled via the EPIPE error, and we do not want anyone + // to CTRL+C us + zyppng::blockSignalsForCurrentThread( { SIGPIPE, SIGINT } ); + + auto provider = std::make_shared("zypp-media-chksum"); + auto res = provider->run (STDIN_FILENO, STDOUT_FILENO); + provider->immediateShutdown(); + if ( res ) + return 0; + + //@TODO print error + return 1; +} diff --git a/tools/zypp-media-copy/CMakeLists.txt b/tools/zypp-media-copy/CMakeLists.txt new file mode 100644 index 0000000..f0e316e --- /dev/null +++ b/tools/zypp-media-copy/CMakeLists.txt @@ -0,0 +1,27 @@ +PROJECT( zypp-media-copy C CXX ) + +set (CMAKE_CXX_STANDARD 17) +SET (CMAKE_CXX_EXTENSIONS OFF) + +FIND_PACKAGE(Protobuf REQUIRED) + +SET( SOURCES + main.cc +) + +add_executable( ${PROJECT_NAME} ${SOURCES} ) +target_link_libraries( ${PROJECT_NAME} zypp-media ) +target_link_libraries( ${PROJECT_NAME} zypp-core ) +target_link_libraries( ${PROJECT_NAME} zypp-protobuf ) +target_link_libraries( ${PROJECT_NAME} ${PROTOBUF_LITE_LIBRARIES} ) + +IF ( INSTALL_NG_BINARIES ) + INSTALL( TARGETS ${PROJECT_NAME} DESTINATION "${ZYPP_LIBEXEC_INSTALL_DIR}/workers" ) +ENDIF() + +set_target_properties( ${PROJECT_NAME} + PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" +) diff --git a/tools/zypp-media-copy/main.cc b/tools/zypp-media-copy/main.cc new file mode 100644 index 0000000..4970140 --- /dev/null +++ b/tools/zypp-media-copy/main.cc @@ -0,0 +1,147 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ + +#include +#include +#include +#include +#include +#include + +class CopyProvider : public zyppng::worker::ProvideWorker +{ + public: + CopyProvider( std::string_view workerName ) : ProvideWorker( workerName ) { } + + void immediateShutdown() override { } + + protected: + // ProvideWorker interface + zyppng::expected initialize(const zyppng::worker::Configuration &conf) override { + + zyppng::worker::WorkerCaps caps; + caps.set_worker_type ( zyppng::worker::WorkerCaps::CPUBound ); + caps.set_cfg_flags( + zyppng::worker::WorkerCaps::Flags ( + zyppng::worker::WorkerCaps::Pipeline + | zyppng::worker::WorkerCaps::ZyppLogFormat + | zyppng::worker::WorkerCaps::FileArtifacts + ) + ); + + return zyppng::expected::success(caps); + } + + void provide() override { + auto &queue = requestQueue(); + + if ( !queue.size() ) + return; + + + auto req = queue.front(); + queue.pop_front(); + + // here we only receive request codes, we only support Provide messages, all others are rejected + // Cancel is never to be received here + if ( req->_spec.code() != zyppng::ProvideMessage::Code::Provide ) { + req->_state = zyppng::worker::ProvideWorkerItem::Finished; + provideFailed( req->_spec.requestId() + , zyppng::ProvideMessage::Code::BadRequest + , "Request type not implemented" + , false + , {} ); + return; + } + + zypp::Url url; + const auto &urlVal = req->_spec.value( zyppng::ProvideMsgFields::Url ); + try { + url = zypp::Url( urlVal.asString() ); + } catch ( const zypp::Exception &excp ) { + ZYPP_CAUGHT(excp); + + std::string err = zypp::str::Str() << "Invalid URL in request: " << urlVal.asString(); + ERR << err << std::endl; + + req->_state = zyppng::worker::ProvideWorkerItem::Finished; + provideFailed( req->_spec.requestId() + , zyppng::ProvideMessage::Code::BadRequest + , err + , false + , {} ); + + return; + } + + auto targetFileName = req->_spec.value( zyppng::ProvideMsgFields::Filename ); + if ( !targetFileName.valid() ) { + req->_state = zyppng::worker::ProvideWorkerItem::Finished; + provideFailed( req->_spec.requestId() + , zyppng::ProvideMessage::Code::BadRequest + , "Copy worker requires a target filename hint" + , false + , {} ); + return; + } + zypp::Pathname targetFilePath( targetFileName.asString() ); + + zypp::PathInfo pi( url.getPathName() ); + if ( !pi.isExist() ) { + req->_state = zyppng::worker::ProvideWorkerItem::Finished; + provideFailed( req->_spec.requestId() + , zyppng::ProvideMessage::Code::NotFound + , zypp::str::Str() << "File " << pi.path() << " not found." + , false + , {} ); + return; + } + + if ( !pi.isFile() ) { + provideFailed( req->_spec.requestId() + , zyppng::ProvideMessage::Code::NotAFile + , zypp::str::Str() << "Path " << pi.path() << " exists, but its not a file" + , false + , {} ); + return; + } + + auto res = zypp::filesystem::hardlinkCopy( pi.path(), targetFileName.asString() ); + if ( res == 0 ) { + provideSuccess( req->_spec.requestId(), false, targetFilePath ); + } else { + provideFailed( req->_spec.requestId() + , zyppng::ProvideMessage::Code::BadRequest + , zypp::str::Str() << "Failed to create file " << targetFilePath + , false + , {} ); + } + } + + void cancel(const std::deque::iterator &i ) override { + ERR << "Bug, cancel should never be called for running items" << std::endl; + } + +}; + +int main( int argc, char *argv[] ) +{ + // lets ignore those, SIGPIPE is better handled via the EPIPE error, and we do not want anyone + // to CTRL+C us + zyppng::blockSignalsForCurrentThread( { SIGPIPE, SIGINT } ); + + auto provider = std::make_shared("zypp-media-copy"); + auto res = provider->run (STDIN_FILENO, STDOUT_FILENO); + provider->immediateShutdown(); + if ( res ) + return 0; + + //@TODO print error + return 1; +} diff --git a/tools/zypp-media-dir/CMakeLists.txt b/tools/zypp-media-dir/CMakeLists.txt new file mode 100644 index 0000000..098f352 --- /dev/null +++ b/tools/zypp-media-dir/CMakeLists.txt @@ -0,0 +1,29 @@ +PROJECT( zypp-media-dir C CXX ) + +set (CMAKE_CXX_STANDARD 17) +SET (CMAKE_CXX_EXTENSIONS OFF) + +FIND_PACKAGE(Protobuf REQUIRED) + +SET( SOURCES + main.cc + dirprovider.cc + dirprovider.h +) + +add_executable( ${PROJECT_NAME} ${SOURCES} ) +target_link_libraries( ${PROJECT_NAME} zypp-media ) +target_link_libraries( ${PROJECT_NAME} zypp-core ) +target_link_libraries( ${PROJECT_NAME} zypp-protobuf ) +target_link_libraries( ${PROJECT_NAME} ${PROTOBUF_LITE_LIBRARIES} ) + +IF ( INSTALL_NG_BINARIES ) + INSTALL( TARGETS ${PROJECT_NAME} DESTINATION "${ZYPP_LIBEXEC_INSTALL_DIR}/workers" ) +ENDIF() + +set_target_properties( ${PROJECT_NAME} + PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" +) diff --git a/tools/zypp-media-dir/dirprovider.cc b/tools/zypp-media-dir/dirprovider.cc new file mode 100644 index 0000000..cf8d30f --- /dev/null +++ b/tools/zypp-media-dir/dirprovider.cc @@ -0,0 +1,135 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ +#include "dirprovider.h" +#include +#include +#include + +#include +#include +#include + +#include +#include + +#undef ZYPP_BASE_LOGGER_LOGGROUP +#define ZYPP_BASE_LOGGER_LOGGROUP "DirProvider" + + +DirProvider::DirProvider() + : DeviceDriver( zyppng::worker::WorkerCaps::SimpleMount ) +{ } + +DirProvider::~DirProvider() +{ } + +zyppng::worker::AttachResult DirProvider::mountDevice ( const uint32_t id, const zypp::Url &attachUrl, const std::string &attachId, const std::string &label, const zyppng::HeaderValueMap &extras ) +{ + try + { + if ( !attachUrl.getHost().empty() ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , "Host must be empty in dir:// and file:// URLs" + , false ); + } + + // set up the verifier + zyppng::MediaDataVerifierRef verifier; + if ( extras.contains(zyppng::AttachMsgFields::VerifyType) ) { + verifier = zyppng::MediaDataVerifier::createVerifier( extras[zyppng::AttachMsgFields::VerifyType].asString() ); + if ( !verifier ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , "Invalid verifier type" + , false ); + } + + if ( !verifier->load( extras[zyppng::AttachMsgFields::VerifyData].asString() ) ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , "Failed to create verifier from file" + , false ); + } + } + const auto &devs = knownDevices(); + + // we simulate a device by simply using the pathname + zypp::Pathname path = zypp::Pathname( attachUrl.getPathName() ).realpath(); + const auto &pathStr = path.asString(); + + zypp::PathInfo adir( path ); + if( !adir.isDir()) { + // URl did not point to a directory + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , zypp::str::Str()<< "Specified path '" << attachUrl << "' is not a directory" + , false + ); + } + + // lets check if the path is what we want + auto res = isDesiredMedium( attachUrl, path, verifier, extras.value( zyppng::AttachMsgFields::MediaNr, 1 ).asInt() ); + if ( !res ) { + try { + std::rethrow_exception( res.error() ); + } catch( const zypp::Exception& e ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MediumNotDesired + , false + , e ); + } catch ( ... ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MediumNotDesired + , "Checking the medium failed with an uknown error" + , false ); + } + } + + // first check if we have that device already + auto i = std::find_if( devs.begin(), devs.end(), [&]( const auto &d ) { return d->_name == pathStr; } ); + if ( i != devs.end() ) { + attachedMedia().insert( { attachId, zyppng::worker::AttachedMedia{ ._dev = *i, ._attachRoot = "/" } } ); + return zyppng::worker::AttachResult::success(); + } + + // we did not find a existing device, well lets make a new one + MIL << "New device " << path << " mounted on " << path << std::endl; + auto newDev = std::make_shared( zyppng::worker::Device{ + ._name = pathStr, + ._maj_nr = 0, + ._min_nr = 0, + ._mountPoint = path, + ._ephemeral = true, // device should be removed after the last attachment was released + ._properties = {} + }); + attachedMedia().insert( { attachId, zyppng::worker::AttachedMedia{ ._dev = newDev, ._attachRoot = "/" } } ); + return zyppng::worker::AttachResult::success(); + + } catch ( const zypp::Exception &e ) { + return zyppng::worker::AttachResult::error ( + zyppng::ProvideMessage::Code::BadRequest + , false + , e ); + } catch ( const std::exception &e ) { + return zyppng::worker::AttachResult::error ( + zyppng::ProvideMessage::Code::BadRequest + , e.what() + , false ); + } catch ( ... ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::BadRequest + , "Unknown exception" + , false); + } +} + +void DirProvider::unmountDevice ( zyppng::worker::Device &dev ) { + // do nothing , this is just a local dir +} diff --git a/tools/zypp-media-dir/dirprovider.h b/tools/zypp-media-dir/dirprovider.h new file mode 100644 index 0000000..0bd2adc --- /dev/null +++ b/tools/zypp-media-dir/dirprovider.h @@ -0,0 +1,27 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ +#ifndef ZYPP_NG_TOOLS_DIRPROVIDER_H_INCLUDED +#define ZYPP_NG_TOOLS_DIRPROVIDER_H_INCLUDED + +#include + +class DirProvider : public zyppng::worker::DeviceDriver +{ + public: + DirProvider( ); + ~DirProvider(); + + zyppng::worker::AttachResult mountDevice ( const uint32_t id, const zypp::Url &attachUrl, const std::string &attachId, const std::string &label, const zyppng::HeaderValueMap &extras ) override; + + protected: + void unmountDevice ( zyppng::worker::Device &dev ) override; + +}; + +#endif diff --git a/tools/zypp-media-dir/main.cc b/tools/zypp-media-dir/main.cc new file mode 100644 index 0000000..8408bee --- /dev/null +++ b/tools/zypp-media-dir/main.cc @@ -0,0 +1,28 @@ +#include "dirprovider.h" + +#include +#include +#include + + +int main( int , char *[] ) +{ + // lets ignore those, SIGPIPE is better handled via the EPIPE error, and we do not want anyone + // to CTRL+C us + zyppng::blockSignalsForCurrentThread( { SIGPIPE, SIGINT } ); + + auto driver = std::make_shared(); + auto provider = std::make_shared( "zypp-media-dir", driver ); + driver->setProvider( provider ); + + auto res = provider->run (STDIN_FILENO, STDOUT_FILENO); + + MIL << "DIR Worker shutting down" << std::endl; + + provider->immediateShutdown(); + if ( res ) + return 0; + + //@TODO print error + return 1; +} diff --git a/tools/zypp-media-disc/CMakeLists.txt b/tools/zypp-media-disc/CMakeLists.txt new file mode 100644 index 0000000..4712694 --- /dev/null +++ b/tools/zypp-media-disc/CMakeLists.txt @@ -0,0 +1,37 @@ +PROJECT( zypp-media-disc C CXX ) + +set (CMAKE_CXX_STANDARD 17) +SET (CMAKE_CXX_EXTENSIONS OFF) + +FIND_PACKAGE(Protobuf REQUIRED) + +SET( SOURCES + main.cc + discprovider.cc + discprovider.h +) + +add_executable( ${PROJECT_NAME} ${SOURCES} ) +target_link_libraries( ${PROJECT_NAME} zypp-media ) +target_link_libraries( ${PROJECT_NAME} zypp-core ) +target_link_libraries( ${PROJECT_NAME} zypp-protobuf ) +target_link_libraries( ${PROJECT_NAME} ${PROTOBUF_LITE_LIBRARIES} ) + +IF ( INSTALL_NG_BINARIES ) + INSTALL( TARGETS ${PROJECT_NAME} DESTINATION "${ZYPP_LIBEXEC_INSTALL_DIR}/workers" ) +ENDIF() + +set_target_properties( ${PROJECT_NAME} + PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" +) + +IF ( UDEV_FOUND ) + TARGET_LINK_LIBRARIES( ${PROJECT_NAME} ${UDEV_LIBRARY} ) +ELSE ( UDEV_FOUND ) + IF ( HAL_FOUND ) + TARGET_LINK_LIBRARIES( ${PROJECT_NAME} ${HAL_LIBRARY} ${HAL_STORAGE_LIBRARY} ${DBUS_LIBRARY} ) + ENDIF ( HAL_FOUND ) +ENDIF ( UDEV_FOUND ) diff --git a/tools/zypp-media-disc/discprovider.cc b/tools/zypp-media-disc/discprovider.cc new file mode 100644 index 0000000..9cef914 --- /dev/null +++ b/tools/zypp-media-disc/discprovider.cc @@ -0,0 +1,407 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ +#include "discprovider.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#undef ZYPP_BASE_LOGGER_LOGGROUP +#define ZYPP_BASE_LOGGER_LOGGROUP "DiscProvider" + +extern "C" +{ +#include +#include +#if HAVE_UDEV +#include +#endif +} + +using namespace std::literals; + +DiscProvider::DiscProvider( ) : zyppng::worker::DeviceDriver( zyppng::worker::WorkerCaps::VolatileMount ) +{ } + +DiscProvider::~DiscProvider() +{} + +zyppng::worker::AttachResult DiscProvider::mountDevice ( const uint32_t id, const zypp::Url &attachUrl, const std::string &attachId, const std::string &label, const zyppng::HeaderValueMap &extras ) +{ + std::vector devs; + if ( extras.contains( zyppng::AttachMsgFields::Device ) ) { + for ( const auto &d :extras.values( zyppng::AttachMsgFields::Device ) ) { + if ( d.isString() ) { + MIL << "Got device from user we are allowed to use: " << d.asString() << std::endl; + devs.push_back(d.asString()); + } + } + } + + auto &sysDevs = knownDevices(); + + std::vector> possibleDevs; + if ( devs.size () ) { + for ( const auto &d : devs ) { + zypp::PathInfo p(d); + if ( !p.isBlk () ) + continue; + + std::shared_ptr device; + auto i = std::find_if( sysDevs.begin(), sysDevs.end(), [&p]( const auto &sysDev ){ return ( sysDev->_maj_nr == p.devMajor() && sysDev->_min_nr == p.devMinor() ); } ); + if ( i == sysDevs.end() ) { + // this is a device we did not detect before lets see if its a CDROM +#ifdef HAVE_UDEV + zypp::AutoDispose udev( ::udev_new(), ::udev_unref ); + if ( !udev ) { + ERR << "Can't create udev context." << std::endl; + continue; + } + + auto devnum = makedev( p.devMajor(), p.devMinor() ); + zypp::AutoDispose udevDevice( ::udev_device_new_from_devnum ( udev, 'b', devnum ), ::udev_device_unref ); + if ( !udevDevice ) { + ERR << "Can't create udev device from devnum, ignoring." << std::endl; + continue; + } + + if ( zypp::strv::asStringView( ::udev_device_get_property_value( udevDevice, "ID_CDROM" ) ) != "1" ) { + ERR << "Device " << d << " from Attach request is not a CDROM/DVD ignoring!" << std::endl; + continue; + } + + auto dev = std::make_shared( zyppng::worker::Device { + ._name = p.path().asString(), + ._maj_nr = p.devMajor(), + ._min_nr = p.devMinor() } ); + DBG << "Registering device (REQUEST): " << dev->_name << " " << dev->_maj_nr << ":" << dev->_min_nr << std::endl; + sysDevs.push_back ( dev ); + device = dev; +#endif + } else { + device = *i; + } + + if ( device ) + possibleDevs.push_back(device); + } + } else { + possibleDevs = sysDevs; + } + + // if we have no devices at this point controller gets a error + if ( possibleDevs.empty () ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , "No useable device found" + , false + ); + } + + const auto attachRoot = zypp::Pathname( attachUrl.getPathName() ); + const auto attachMediaNr = extras.value( zyppng::AttachMsgFields::MediaNr, 1 ).asInt(); + + // set up the verifier + zyppng::MediaDataVerifierRef verifier; + if ( extras.contains( zyppng::AttachMsgFields::VerifyType ) ) { + verifier = zyppng::MediaDataVerifier::createVerifier( extras.value(zyppng::AttachMsgFields::VerifyType).asString() ); + if ( !verifier ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , "Invalid verifier type" + , false + ); + } + + if ( !verifier->load( extras.value(zyppng::AttachMsgFields::VerifyData).asString() ) ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , "Failed to create verifier from file" + , false + ); + } + } + + //first check if any of the mounted devices are what we want + for( const auto &dev : possibleDevs ) { + if ( dev->_mountPoint.empty() ) + continue; + + auto res = isDesiredMedium( attachUrl, dev->_mountPoint / attachRoot, verifier, extras.value( zyppng::AttachMsgFields::MediaNr, 1 ).asInt() ); + if ( !res ) + continue; + + MIL << "Found requested medium in dev " << dev->_name << std::endl; + + // we found the device we want! + attachedMedia().insert( std::make_pair( attachId, zyppng::worker::AttachedMedia{ dev, attachRoot } ) ); + return zyppng::worker::AttachResult::success(); + } + + std::list filesystems; + + filesystems.push_back("iso9660"); + + // if DVD, try UDF filesystem after iso9660 + if ( attachUrl.getScheme() == "dvd" ) + filesystems.push_back("udf"); + + std::string options; + auto optVal = extras.value( "mountoptions"sv ); + if ( optVal.valid() && optVal.isString() ) + options = optVal.asString(); + + if ( options.empty() ) { + options="ro"; + } + + while ( true ) { + + // remember how many devices we were able to test + uint devicesTested = 0; + + // none of the already mounted devices matched, lets try what we have left + for( const auto &dev : possibleDevs ) { + if ( !dev->_mountPoint.empty() ) + continue; + + MIL << "Trying to mount dev " << dev->_name << std::endl; + + // make sure the tray is closed + zypp::media::CDTools::closeTray( dev->_name ); + + devicesTested++; + zypp::media::Mount mount; + bool mountsucceeded = false; + std::exception_ptr lastErr; + for( auto fsit = filesystems.begin() ; !mountsucceeded && fsit != filesystems.end() ; ++fsit) { + + zypp::Pathname newAp; + try { + newAp = createAttachPoint( this->attachRoot() ); + if ( newAp.empty() ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , "Failed to create mount directory." + , false + ); + } + + mount.mount( dev->_name, newAp.asString(), *fsit, options); + + // wait for /etc/mtab update ... + // (shouldn't be needed) + int limit = 2; + while( !(mountsucceeded=checkAttached( newAp, devicePredicate( dev->_maj_nr, dev->_min_nr ))) && --limit) + { + WAR << "Wait for /proc/mounts update and retry...." << std::endl; + sleep(1); + } + + if( !mountsucceeded) { + try { + mount.umount( newAp.asString() ); + } catch (const zypp::media::MediaException & excpt_r) { + ZYPP_CAUGHT(excpt_r); + } + ZYPP_THROW( zypp::media::MediaMountException( + "Unable to verify that the media was mounted", + dev->_name, newAp.asString() + )); + } else { + dev->_mountPoint = newAp; + break; + } + } + catch (const zypp::media::MediaMountException &e) + { + lastErr = std::current_exception (); + removeAttachPoint(newAp); + ZYPP_CAUGHT(e); + } + catch (const zypp::media::MediaException & excpt_r) + { + removeAttachPoint(newAp); + ZYPP_CAUGHT(excpt_r); + } + } // for filesystems + + if ( mountsucceeded ) { + // lets check if we have the correct medium + bool canUseDevice = false; + + auto res = isDesiredMedium( attachUrl, dev->_mountPoint / attachRoot, verifier, extras.value( zyppng::AttachMsgFields::MediaNr, 1 ).asInt() ); + if ( !res ) { + unmountDevice( *dev ); + continue; + } + + MIL << "Found requested medium in dev " << dev->_name << std::endl; + attachedMedia().insert( std::make_pair( attachId, zyppng::worker::AttachedMedia{ dev, attachRoot } ) ); + return zyppng::worker::AttachResult::success(); + } + } // for each device + + // we did go through all devices, but didn't find any. + if ( devicesTested == 0 ) { + // no devices are free + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::Jammed + , "No free ressources available" + , false + ); + } else { + + const auto &mountedDevs = zypp::media::Mount::getEntries(); + // k, we need to ask the user to give us the medium we need + // first find the devices that are free + std::vector freeDevs; + for( const auto &dev : possibleDevs ) { + if ( !dev->_mountPoint.empty() ) + continue; + + auto i = std::find_if( mountedDevs.begin (), mountedDevs.end(), [&dev]( const zypp::media::MountEntry &e ) { + zypp::PathInfo pi( e.src ); + return ( pi.isBlk() && pi.devMajor() == dev->_maj_nr && pi.devMinor() == dev->_min_nr ); + }); + + if ( i == mountedDevs.end() ) { + MIL << "Adding " << dev->_name << " to list of free devs" << std::endl; + freeDevs.push_back( dev->_name ); + } + } + + // if there are no devices free, we are jammed + if ( freeDevs.empty() ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::Jammed + , "No free ressources available" + , false + ); + } + + auto changeRes = zyppng::worker::ProvideWorker::ABORT; + auto worker = parentWorker(); + if ( worker ) { + changeRes = worker->requestMediaChange ( id, label, attachMediaNr, freeDevs ); + } + + if ( changeRes != zyppng::worker::ProvideWorker::SUCCESS ) { + if ( changeRes == zyppng::worker::ProvideWorker::SKIP) + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MediaChangeSkip + , "User asked to skip the media change." + , false); + else + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MediaChangeAbort + , "User asked to abort the media change." + , false ); + } + } + } +} + +void DiscProvider::detectDevices( ) +{ + + // helper lambda to add a new device to the internal registry, returning it if its either created or already known + const auto registerDevice = [ this ]( const zypp::PathInfo &devnode, std::string_view reason ) { + auto &sysDevs = knownDevices(); + if ( devnode.isBlk() ) { + auto i = std::find_if( sysDevs.begin(), sysDevs.end(), [&]( const auto &dev ){ + return ( dev->_maj_nr == devnode.devMajor() && dev->_min_nr == devnode.devMinor () ); + }); + + // we already know the device + if ( i != sysDevs.end() ) + return i; + + auto dev = std::make_shared( zyppng::worker::Device{ + ._name = devnode.path().asString(), + ._maj_nr = devnode.devMajor(), + ._min_nr = devnode.devMinor() } ); + DBG << "Found (" << reason << "): " << dev->_name << " " << dev->_maj_nr << ":" << dev->_min_nr << std::endl; + + sysDevs.push_back(dev); + return ( --sysDevs.end() ); + } + return sysDevs.end(); + }; + + + auto &sysDevs = knownDevices(); + + // detect devices available in the system if we did not do it before: + if ( !_devicesDetected ) { + _devicesDetected = true; + +#ifdef HAVE_UDEV + // http://www.kernel.org/pub/linux/utils/kernel/hotplug/libudev/index.html + zypp::AutoDispose udev( ::udev_new(), ::udev_unref ); + if ( ! udev ) { + ERR << "Can't create udev context." << std::endl; + } else { + zypp::AutoDispose enumerate( ::udev_enumerate_new(udev), ::udev_enumerate_unref ); + if ( ! enumerate ) { + ERR << "Can't create udev list entry." << std::endl; + } else { + ::udev_enumerate_add_match_subsystem( enumerate, "block" ); + ::udev_enumerate_add_match_property( enumerate, "ID_CDROM", "1" ); + ::udev_enumerate_scan_devices( enumerate ); + + struct udev_list_entry * entry = 0; + udev_list_entry_foreach( entry, ::udev_enumerate_get_list_entry( enumerate ) ) + { + + zypp::AutoDispose device( ::udev_device_new_from_syspath( ::udev_enumerate_get_udev( enumerate ), + ::udev_list_entry_get_name( entry ) ), + ::udev_device_unref ); + if ( ! device ) + { + ERR << "Can't create udev device." << std::endl; + continue; + } + + const char * devnodePtr( ::udev_device_get_devnode( device ) ); + if ( ! devnodePtr ) { + ERR << "Got NULL devicenode." << std::endl; + continue; + } + + // In case we need it someday: + //const char * mountpath = ::udev_device_get_property_value( device, "FSTAB_DIR" ); + auto dev = registerDevice( zypp::PathInfo ( devnodePtr ), "udev" ); + if ( dev != sysDevs.end() ) { + if ( ::udev_device_get_property_value( device, "ID_CDROM_DVD" ) ) { + (*dev)->_properties["DVD"] = true; + } else { + (*dev)->_properties["DVD"] = false; + } + } + } + } + } +#endif + if ( sysDevs.empty() ) + { + WAR << "CD/DVD drive detection with UDEV failed! Guessing..." << std::endl; + auto dev = registerDevice( zypp::PathInfo ( "/dev/dvd" ), "GUESS" ); + if ( dev != sysDevs.end() && (*dev)->_properties.count("DVD") == 0 ) + (*dev)->_properties["DVD"] = true; + dev = registerDevice( zypp::PathInfo ( "/dev/cdrom" ), "GUESS" ); + if ( dev != sysDevs.end() && (*dev)->_properties.count("DVD") == 0 ) + (*dev)->_properties["DVD"] = false; + } + } +} diff --git a/tools/zypp-media-disc/discprovider.h b/tools/zypp-media-disc/discprovider.h new file mode 100644 index 0000000..b3ae672 --- /dev/null +++ b/tools/zypp-media-disc/discprovider.h @@ -0,0 +1,31 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ +#ifndef ZYPP_NG_TOOLS_DISCPROVIDER_H_INCLUDED +#define ZYPP_NG_TOOLS_DISCPROVIDER_H_INCLUDED + +#include +#include +#include +#include + +class DiscProvider : public zyppng::worker::DeviceDriver +{ +public: + DiscProvider(); + ~DiscProvider(); + + // DeviceDriver interface + zyppng::worker::AttachResult mountDevice ( const uint32_t id, const zypp::Url &attachUrl, const std::string &attachId, const std::string &label, const zyppng::HeaderValueMap &extras ) override; + void detectDevices() override; + +private: + bool _devicesDetected = false; //< We delay device detection to the first attach request, to avoid doing it without needing it +}; + +#endif diff --git a/tools/zypp-media-disc/main.cc b/tools/zypp-media-disc/main.cc new file mode 100644 index 0000000..e19533f --- /dev/null +++ b/tools/zypp-media-disc/main.cc @@ -0,0 +1,27 @@ + +#include "discprovider.h" + +#include +#include + +#include + + +int main( int , char *[] ) +{ + // lets ignore those, SIGPIPE is better handled via the EPIPE error, and we do not want anyone + // to CTRL+C us + zyppng::blockSignalsForCurrentThread( { SIGPIPE, SIGINT } ); + + auto driver = std::make_shared(); + auto provider = std::make_shared( "zypp-media-disc", driver ); + driver->setProvider( provider ); + + auto res = provider->run (STDIN_FILENO, STDOUT_FILENO); + provider->immediateShutdown(); + if ( res ) + return 0; + + //@TODO print error + return 1; +} diff --git a/tools/zypp-media-disk/CMakeLists.txt b/tools/zypp-media-disk/CMakeLists.txt new file mode 100644 index 0000000..9ac8304 --- /dev/null +++ b/tools/zypp-media-disk/CMakeLists.txt @@ -0,0 +1,29 @@ +PROJECT( zypp-media-disk C CXX ) + +set (CMAKE_CXX_STANDARD 17) +SET (CMAKE_CXX_EXTENSIONS OFF) + +FIND_PACKAGE(Protobuf REQUIRED) + +SET( SOURCES + main.cc + diskprovider.cc + diskprovider.h +) + +add_executable( ${PROJECT_NAME} ${SOURCES} ) +target_link_libraries( ${PROJECT_NAME} zypp-media ) +target_link_libraries( ${PROJECT_NAME} zypp-core ) +target_link_libraries( ${PROJECT_NAME} zypp-protobuf ) +target_link_libraries( ${PROJECT_NAME} ${PROTOBUF_LITE_LIBRARIES} ) + +IF ( INSTALL_NG_BINARIES ) + INSTALL( TARGETS ${PROJECT_NAME} DESTINATION "${ZYPP_LIBEXEC_INSTALL_DIR}/workers" ) +ENDIF() + +set_target_properties( ${PROJECT_NAME} + PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" +) diff --git a/tools/zypp-media-disk/diskprovider.cc b/tools/zypp-media-disk/diskprovider.cc new file mode 100644 index 0000000..0da023d --- /dev/null +++ b/tools/zypp-media-disk/diskprovider.cc @@ -0,0 +1,363 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ +#include "diskprovider.h" +#include +#include +#include + +#include +#include + +#include +#include + +#undef ZYPP_BASE_LOGGER_LOGGROUP +#define ZYPP_BASE_LOGGER_LOGGROUP "DiskProvider" + + +/*! + * Check if specified device file name is + * a disk volume device or throw an error. + */ +bool verifyIfDiskVolume( const zypp::Pathname &dev_name ) +{ + if( dev_name.empty() || + dev_name.asString().compare(0, sizeof("/dev/")-1, "/dev/")) + { + ERR << "Specified device name " << dev_name + << " is not allowed" << std::endl; + return false; + } + + zypp::PathInfo dev_info(dev_name); + if( !dev_info.isBlk()) + { + ERR << "Specified device name " << dev_name + << " is not a block device" << std::endl; + return false; + } + + // check if a volume using /dev/disk/by-uuid links first + { + zypp::Pathname dpath("/dev/disk/by-uuid"); + std::list dlist; + if( zypp::filesystem::readdir(dlist, dpath) == 0) + { + std::list::const_iterator it; + for(it = dlist.begin(); it != dlist.end(); ++it) + { + zypp::PathInfo vol_info(*it); + if( vol_info.isBlk() && vol_info.devMajor() == dev_info.devMajor() && + vol_info.devMinor() == dev_info.devMinor()) + { + DBG << "Specified device name " << dev_name + << " is a volume (disk/by-uuid link " + << vol_info.path() << ")" + << std::endl; + return true; + } + } + } + } + + // check if a volume using /dev/disk/by-label links + // (e.g. vbd mapped volumes in a XEN vm) + { + zypp::Pathname dpath("/dev/disk/by-label"); + std::list dlist; + if( zypp::filesystem::readdir(dlist, dpath) == 0) + { + std::list::const_iterator it; + for(it = dlist.begin(); it != dlist.end(); ++it) + { + zypp::PathInfo vol_info(*it); + if( vol_info.isBlk() && vol_info.devMajor() == dev_info.devMajor() && + vol_info.devMinor() == dev_info.devMinor()) + { + DBG << "Specified device name " << dev_name + << " is a volume (disk/by-label link " + << vol_info.path() << ")" + << std::endl; + return true; + } + } + } + } + + // check if a filesystem volume using the 'blkid' tool + // (there is no /dev/disk link for some of them) + zypp::ExternalProgram::Arguments args; + args.push_back( "blkid" ); + args.push_back( "-p" ); + args.push_back( dev_name.asString() ); + + zypp::ExternalProgram cmd( args, zypp::ExternalProgram::Stderr_To_Stdout ); + cmd >> DBG; + if ( cmd.close() != 0 ) + { + ERR << cmd.execError() + << "\nSpecified device name " << dev_name + << " is not a usable disk volume" + << std::endl; + return false; + } + return true; +} + + + +DiskProvider::DiskProvider() + : DeviceDriver( zyppng::worker::WorkerCaps::SimpleMount ) +{ } + +DiskProvider::~DiskProvider() +{ } + +zyppng::worker::AttachResult DiskProvider::mountDevice ( const uint32_t id, const zypp::Url &attachUrl, const std::string &attachId, const std::string &label, const zyppng::HeaderValueMap &extras ) +{ + try + { + if ( !attachUrl.getHost().empty() ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , "Host must be empty in dir:// and file:// URLs" + , false + ); + } + + const std::string &device = zypp::Pathname(attachUrl.getQueryParam("device")).asString(); + if ( device.empty() ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , "Media url does not contain a device specification" + , false + ); + } + + std::string filesystem = attachUrl.getQueryParam("filesystem"); + if( filesystem.empty() ) + filesystem="auto"; + + zypp::PathInfo dev_info( device ); + if(!dev_info.isBlk()) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , "Media url does not specify a valid block device" + , false + ); + } + + DBG << "Verifying " << device << " ..." << std::endl; + if( !verifyIfDiskVolume( device)) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , "Could not verify URL points to a disk volume!" + , false + ); + } + + // disks can have a attach root ( path relative to the device root ) + const auto relAttachRoot = zypp::Pathname( attachUrl.getPathName() ); + + // set up the verifier + zyppng::MediaDataVerifierRef verifier; + if ( extras.value( zyppng::AttachMsgFields::VerifyType ).valid() ) { + verifier = zyppng::MediaDataVerifier::createVerifier( extras.value(zyppng::AttachMsgFields::VerifyType).asString() ); + if ( !verifier ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , "Invalid verifier type" + , false + ); + } + + if ( !verifier->load( extras.value(zyppng::AttachMsgFields::VerifyData).asString() ) ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , "Failed to create verifier from file" + , false + ); + } + } + const auto &devs = knownDevices(); + + // first check if we have that device already + auto i = std::find_if( devs.begin(), devs.end(), [&]( const auto &d ) { + return d->_maj_nr == dev_info.devMajor() + && d->_min_nr == dev_info.devMinor(); + }); + if ( i != devs.end() ) { + auto res = isDesiredMedium( attachUrl, (*i)->_mountPoint / relAttachRoot, verifier, extras.value( zyppng::AttachMsgFields::MediaNr, 1 ).asInt() ); + if ( !res ) { + try { + std::rethrow_exception( res.error() ); + } catch( const zypp::Exception& e ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MediumNotDesired + , false + , e + ); + } catch ( ... ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MediumNotDesired + , "Checking the medium failed with an uknown error" + , false + ); + } + } else { + attachedMedia().insert( std::make_pair( attachId, zyppng::worker::AttachedMedia{ *i, relAttachRoot } ) ); + return zyppng::worker::AttachResult::success(); + } + } + + // we did not find a existing mount, well lets make a new one ... + std::optional bindSource; + auto devPtr = std::make_shared( zyppng::worker::Device{ + ._name = dev_info.path().asString(), + ._maj_nr = dev_info.devMajor(), + ._min_nr = dev_info.devMinor(), + ._mountPoint = {}, + ._ephemeral = true, // forget about the device after we are finished with it + ._properties = {} + }); + + // since the kernel will not let us remount a disc ro if it was already mounted in the system as rw we need to + // go over the existing mounts and revert to a bind mount if we find that the device has a mountpoint already (#163486). + zypp::media::MountEntries entries( zypp::media::Mount::getEntries() ); + for( auto e = entries.cbegin(); e != entries.cend(); ++e) + { + bool is_device = false; + std::string dev_path(zypp::Pathname(e->src).asString()); + zypp::PathInfo dev_info; + + if( dev_path.compare(0, sizeof("/dev/")-1, "/dev/") == 0 && + dev_info(e->src) && dev_info.isBlk()) { + is_device = true; + } + + if( is_device && devPtr->_maj_nr == dev_info.devMajor() && + devPtr->_min_nr == dev_info.devMinor()) + { + DBG << "Device " << devPtr->_name << " is already mounted, using bind mount!" << std::endl; + bindSource = e->dir; + break; + } + } + + zypp::media::Mount mount; + zypp::Pathname newAp; + try { + newAp = createAttachPoint( attachRoot() ); + if ( newAp.empty() ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , "Failed to create mount directory." + , false + ); + } + + std::string options = attachUrl.getQueryParam("mountoptions"); + if(options.empty()) { + options = "ro"; + } + + if( bindSource ) { + options += ",bind"; + mount.mount( bindSource->asString(), newAp.asString(), "none", options); + } else { + mount.mount( dev_info.path().asString(), newAp.asString(), filesystem, options); + } + + // wait for /etc/mtab update ... + // (shouldn't be needed) + int limit = 3; + bool mountsucceeded = false; + + const auto &checkAttachedDisk = [&]( const zypp::media::MountEntry &e ) { + if ( bindSource) { + if ( *bindSource == e.src ) { + DBG << "Found bound media " + << devPtr->_name + << " in the mount table as " << e.src << std::endl; + return true; + } + } + return DeviceDriver::devicePredicate( devPtr->_maj_nr, devPtr->_min_nr ) ( e ); + }; + + // mount command came back OK, lets wait for mtab to update + while( !(mountsucceeded=checkAttached( newAp, checkAttachedDisk )) && --limit) { + MIL << "Mount did not appear yet, sleeping for 1s" << std::endl; + sleep(1); + } + + // mount didn't work after all, bail out + if ( !mountsucceeded ) { + try { + mount.umount( newAp.asString() ); + } catch (const zypp::media::MediaException & excpt_r) { + ZYPP_CAUGHT(excpt_r); + } + ZYPP_THROW( zypp::media::MediaMountException( + "Unable to verify that the media was mounted", + devPtr->_name, newAp.asString() + )); + } + + // if we reach this place, mount worked -> YAY, lets see if that is the desired medium! + auto isDesired = isDesiredMedium( attachUrl, newAp / relAttachRoot, verifier, extras.value( zyppng::AttachMsgFields::MediaNr, 1 ).asInt() ); + if ( !isDesired ) { + try { + mount.umount( newAp.asString() ); + } catch (const zypp::media::MediaException & excpt_r) { + ZYPP_CAUGHT(excpt_r); + } + ZYPP_THROW( zypp::media::MediaNotDesiredException( attachUrl ) ); + } + } + catch ( const zypp::Exception &e ) { + removeAttachPoint(newAp); + ZYPP_CAUGHT(e); + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , false + , e + ); + } + catch ( ... ) { + removeAttachPoint(newAp); + std::rethrow_exception( std::current_exception() ); + } + + // mount worked ! YAY + devPtr->_mountPoint = newAp; + knownDevices().push_back( devPtr ); + attachedMedia().insert( std::make_pair( attachId, zyppng::worker::AttachedMedia{ devPtr, relAttachRoot } ) ); + return zyppng::worker::AttachResult::success(); + + } catch ( const zypp::Exception &e ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::BadRequest + , false + , e + ); + } catch ( const std::exception &e ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::BadRequest + , e.what() + , false + ); + } catch ( ... ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::BadRequest + , "Unknown exception" + , false + ); + } +} diff --git a/tools/zypp-media-disk/diskprovider.h b/tools/zypp-media-disk/diskprovider.h new file mode 100644 index 0000000..7d5828b --- /dev/null +++ b/tools/zypp-media-disk/diskprovider.h @@ -0,0 +1,25 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ +#ifndef ZYPP_NG_TOOLS_DISKPROVIDER_H_INCLUDED +#define ZYPP_NG_TOOLS_DISKPROVIDER_H_INCLUDED + +#include + +class DiskProvider : public zyppng::worker::DeviceDriver +{ + public: + DiskProvider(); + ~DiskProvider(); + + // DeviceDriver interface + zyppng::worker::AttachResult mountDevice ( const uint32_t id, const zypp::Url &attachUrl, const std::string &attachId, const std::string &label, const zyppng::HeaderValueMap &extras ) override; + +}; + +#endif diff --git a/tools/zypp-media-disk/main.cc b/tools/zypp-media-disk/main.cc new file mode 100644 index 0000000..208ee56 --- /dev/null +++ b/tools/zypp-media-disk/main.cc @@ -0,0 +1,26 @@ +#include "diskprovider.h" + +#include +#include +#include + +int main( int , char *[] ) +{ + // lets ignore those, SIGPIPE is better handled via the EPIPE error, and we do not want anyone + // to CTRL+C us + zyppng::blockSignalsForCurrentThread( { SIGPIPE, SIGINT } ); + + auto driver = std::make_shared(); + auto worker = std::make_shared( "zypp-media-disk", driver ); + driver->setProvider( worker ); + + auto res = worker->run (STDIN_FILENO, STDOUT_FILENO); + + MIL << "DISK Worker shutting down" << std::endl; + + if ( res ) + return 0; + + //@TODO print error + return 1; +} diff --git a/tools/zypp-media-http/CMakeLists.txt b/tools/zypp-media-http/CMakeLists.txt new file mode 100644 index 0000000..d432f7b --- /dev/null +++ b/tools/zypp-media-http/CMakeLists.txt @@ -0,0 +1,30 @@ +PROJECT( zypp-media-http C CXX ) + +set (CMAKE_CXX_STANDARD 17) +SET (CMAKE_CXX_EXTENSIONS OFF) + +FIND_PACKAGE(Protobuf REQUIRED) + +SET( SOURCES + main.cc + networkprovider.cc + networkprovider.h +) + +add_executable( ${PROJECT_NAME} ${SOURCES} ) +target_link_libraries( ${PROJECT_NAME} zypp-curl ) +target_link_libraries( ${PROJECT_NAME} zypp-media ) +target_link_libraries( ${PROJECT_NAME} zypp-core ) +target_link_libraries( ${PROJECT_NAME} zypp-protobuf ) +target_link_libraries( ${PROJECT_NAME} ${PROTOBUF_LITE_LIBRARIES} ) + +IF ( INSTALL_NG_BINARIES ) + INSTALL( TARGETS ${PROJECT_NAME} DESTINATION "${ZYPP_LIBEXEC_INSTALL_DIR}/workers" ) +ENDIF() + +set_target_properties( ${PROJECT_NAME} + PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" +) diff --git a/tools/zypp-media-http/main.cc b/tools/zypp-media-http/main.cc new file mode 100644 index 0000000..c6fde54 --- /dev/null +++ b/tools/zypp-media-http/main.cc @@ -0,0 +1,21 @@ +#include "networkprovider.h" + +#include +#include + +int main( int , char *[] ) +{ + // lets ignore those, SIGPIPE is better handled via the EPIPE error, and we do not want anyone + // to CTRL+C us + zyppng::blockSignalsForCurrentThread( { SIGPIPE, SIGINT } ); + + auto provider = std::make_shared("zypp-media-http"); + auto res = provider->run (STDIN_FILENO, STDOUT_FILENO); + provider->immediateShutdown(); + if ( res ) + return 0; + + //@TODO print error + return 1; + //@TODO @WARNING @BUG Initialize the requestmanager with user agent and anon headers \sa anonymousIdHeader, distributionFlavorHeader, agentString +} diff --git a/tools/zypp-media-http/networkprovider.cc b/tools/zypp-media-http/networkprovider.cc new file mode 100644 index 0000000..3be04e0 --- /dev/null +++ b/tools/zypp-media-http/networkprovider.cc @@ -0,0 +1,480 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ +#include "networkprovider.h" + +#include +#include +#include +#include +#include +#include +#include + + +#undef ZYPP_BASE_LOGGER_LOGGROUP +#define ZYPP_BASE_LOGGER_LOGGROUP "NetworkProvider" + + +NetworkProvideItem::NetworkProvideItem(NetworkProvider &parent, zyppng::ProvideMessage &&spec) + : zyppng::worker::ProvideWorkerItem( std::move(spec) ) + , _parent(parent) +{} + +NetworkProvideItem::~NetworkProvideItem() +{ + clearConnections(); + if ( _dl ) { + _dl->cancel(); + } +} + +void NetworkProvideItem::startDownload( std::shared_ptr &&dl ) +{ + if ( _state == ProvideWorkerItem::Running || _dl ) + throw std::runtime_error("Can not start a already running Download"); + + MIL_PRV << "Starting download of : " << dl->spec().url() << "with target path: " << dl->spec().targetPath() << std::endl; + + _connections.clear (); + // we set the state to running immediately but only send the message to controller once the item actually started + _state = ProvideWorkerItem::Running; + _dl = std::move(dl); + _connections.emplace_back( connect( *_dl, &zyppng::Download::sigStarted, *this, &NetworkProvideItem::onStarted ) ); + _connections.emplace_back( connect( *_dl, &zyppng::Download::sigFinished, *this, &NetworkProvideItem::onFinished ) ); + _connections.emplace_back( connect( *_dl, &zyppng::Download::sigAuthRequired, *this, &NetworkProvideItem::onAuthRequired ) ); + + _dl->setStopOnMetalink( true ); + _dl->start(); +} + +void NetworkProvideItem::cancelDownload() +{ + _state = zyppng::worker::ProvideWorkerItem::Finished; + if ( _dl ) { + _dl->cancel(); + clearConnections(); + _dl.reset(); + } +} + +void NetworkProvideItem::clearConnections() +{ + for ( auto c : _connections) + c.disconnect(); + _connections.clear(); +} + +void NetworkProvideItem::onStarted( zyppng::Download & ) +{ + _parent.itemStarted( shared_this() ); +} + +void NetworkProvideItem::onFinished( zyppng::Download & ) +{ + _parent.itemFinished( shared_this() ); +} + +void NetworkProvideItem::onAuthRequired( zyppng::Download &, zyppng::NetworkAuthData &auth, const std::string &availAuth ) +{ + _parent.itemAuthRequired( shared_this(), auth, availAuth ); +} + + +NetworkProvider::NetworkProvider( std::string_view workerName ) + : zyppng::worker::ProvideWorker( workerName ) + , _dlManager( std::make_shared() ) +{ + // we only want to hear about new provides + setProvNotificationMode( ProvideWorker::ONLY_NEW_PROVIDES ); +} + +zyppng::expected NetworkProvider::initialize( const zyppng::worker::Configuration &conf ) +{ + const auto &values = conf.values(); + const auto iEnd = values.end(); + if ( const auto &i = values.find( std::string(zyppng::AGENT_STRING_CONF) ); i != iEnd ) { + const auto &val = i->second; + MIL << "Setting agent string to: " << val << std::endl; + _dlManager->requestDispatcher()->setAgentString( val ); + } + if ( const auto &i = values.find( std::string(zyppng::DISTRO_FLAV_CONF) ); i != iEnd ) { + const auto &val = i->second; + MIL << "Setting distro flavor header to: " << val << std::endl; + _dlManager->requestDispatcher()->setHostSpecificHeader("download.opensuse.org", "X-ZYpp-DistributionFlavor", val ); + } + if ( const auto &i = values.find( std::string(zyppng::ANON_ID_CONF) ); i != iEnd ) { + const auto &val = i->second; + MIL << "Got anonymous ID setting from controller" << std::endl; + _dlManager->requestDispatcher()->setHostSpecificHeader("download.opensuse.org", "X-ZYpp-AnonymousId", val ); + } + if ( const auto &i = values.find( std::string(zyppng::ATTACH_POINT) ); i != iEnd ) { + const auto &val = i->second; + MIL << "Got attachpoint from controller: " << val << std::endl; + _attachPoint = val; + } else { + return zyppng::expected::error(ZYPP_EXCPT_PTR( zypp::Exception("Attach point required to work.") )); + } + + zyppng::worker::WorkerCaps caps; + caps.set_worker_type ( zyppng::worker::WorkerCaps::Downloading ); + caps.set_cfg_flags( + zyppng::worker::WorkerCaps::Flags ( + zyppng::worker::WorkerCaps::Pipeline + | zyppng::worker::WorkerCaps::ZyppLogFormat + ) + ); + + return zyppng::expected::success(caps); +} + +void NetworkProvider::provide() +{ + auto &queue = requestQueue(); + + if ( !queue.size() ) + return; + + const auto now = std::chrono::steady_clock::now(); + auto &io = controlIO(); + + while ( io.readFdOpen() ) { + // find next pending item + auto i = std::find_if( queue.begin(), queue.end(), [&now]( const zyppng::worker::ProvideWorkerItemRef &req ){ + NetworkProvideItemRef nReq = std::static_pointer_cast(req); + return nReq && nReq->_state == zyppng::worker::ProvideWorkerItem::Pending && nReq->_scheduleAfter < now; + }); + + // none left, bail out + if ( i == queue.end () ) + break; + + auto req = std::static_pointer_cast( *i ); + if ( req->_state != zyppng::worker::ProvideWorkerItem::Pending ) + return; + + MIL_PRV << "About to schedule: " << req->_spec.code() << " with ID: "<< req->_spec.requestId() << std::endl; + + // here we only receive request codes, we only support Provide messages, all others are rejected + // Cancel is never to be received here + if ( req->_spec.code() != zyppng::ProvideMessage::Code::Provide ) { + req->_state = zyppng::worker::ProvideWorkerItem::Finished; + provideFailed( req->_spec.requestId() + , zyppng::ProvideMessage::Code::BadRequest + , "Request type not implemented" + , false + , {} ); + + queue.erase(i); + continue; + } + + zypp::Url url; + const auto &urlVal = req->_spec.value( zyppng::ProvideMsgFields::Url ); + try { + url = zypp::Url( urlVal.asString() ); + } catch ( const zypp::Exception &excp ) { + ZYPP_CAUGHT(excp); + + std::string err = zypp::str::Str() << "Invalid URL in request: " << urlVal.asString(); + ERR << err << std::endl; + + req->_state = zyppng::worker::ProvideWorkerItem::Finished; + provideFailed( req->_spec.requestId() + , zyppng::ProvideMessage::Code::BadRequest + , err + , false + , {} ); + + queue.erase(i); + continue; + } + + const auto &fPath = zypp::Pathname( url.getPathName() ); + + /* + * We download into the staging directory first, using the md5sum of the full path as filename. + * Once the download is finished its transferred into the cache directory. This way we can clearly + * distinguish between cancelled downloads and downloads we have completely finished. + */ + zypp::Pathname localPath = zypp::Pathname( _attachPoint ) / "cache" / fPath; + zypp::Pathname stagingPath = ( zypp::Pathname( _attachPoint ) / "staging" / zypp::CheckSum::md5FromString( fPath.asString() ).asString() ).extend(".part"); + + MIL_PRV << "Providing " << url << " to local: " << localPath << " staging: " << stagingPath << std::endl; + + // Cache hit? + zypp::PathInfo cacheFile(localPath); + if ( cacheFile.isExist () ) { + + // could be a directory + if ( !cacheFile.isFile() ) { + MIL_PRV << "Path " << url << " exists in Cache but its not a file!" << std::endl; + req->_state = zyppng::worker::ProvideWorkerItem::Finished; + provideFailed( req->_spec.requestId() + , zyppng::ProvideMessage::Code::NotAFile + , "Not a file" + , false + , {} ); + + queue.erase(i); + continue; + } + + MIL_PRV << "Providing " << url << "Cache HIT" << std::endl; + req->_state = zyppng::worker::ProvideWorkerItem::Finished; + provideSuccess ( req->_spec.requestId(), true, localPath ); + queue.erase(i); + continue; + } + + if ( zypp::PathInfo(stagingPath).isExist () ) { + // check if there is another request running with this staging file name + auto runningIt = std::find_if( queue.begin (), queue.end(), [&]( const zyppng::worker::ProvideWorkerItemRef &queueItem ){ + if ( req == queueItem || queueItem->_state != zyppng::worker::ProvideWorkerItem::Running ) + return false; + return ( static_cast( queueItem.get() )->_stagingFileName == stagingPath ); + }); + + // we schedule that later again which should result in a immediate cache hit + if ( runningIt != queue.end() ) { + MIL << "Found a existing request that results in the same staging file, postponing the request for later" << std::endl; + req->_scheduleAfter = now; + continue; + } + + MIL<< "Removing broken staging file" << stagingPath << std::endl; + + // here this is most likely a broken download + zypp::filesystem::unlink( stagingPath ); + } + + auto errCode = zypp::filesystem::assert_dir( localPath.dirname() ); + if( errCode ) { + std::string err = zypp::str::Str() << "assert_dir " << localPath.dirname() << " failed"; + DBG << err << std::endl; + + req->_state = zyppng::worker::ProvideWorkerItem::Finished; + provideFailed( req->_spec.requestId() + , zyppng::ProvideMessage::Code::InternalError + , err + , false + , {} ); + queue.erase(i); + continue; + } + + errCode = zypp::filesystem::assert_dir( stagingPath.dirname() ); + if( errCode ) { + std::string err = zypp::str::Str() << "assert_dir " << stagingPath.dirname() << " failed"; + DBG << err << std::endl; + + req->_state = zyppng::worker::ProvideWorkerItem::Finished; + provideFailed( req->_spec.requestId() + , zyppng::ProvideMessage::Code::InternalError + , err + , false + , {} ); + queue.erase(i); + continue; + } + + bool doMetalink = true; + const auto &metalinkVal = req->_spec.value( zyppng::NETWORK_METALINK_ENABLED ); + if ( metalinkVal.valid() && metalinkVal.isBool () ) { + doMetalink = metalinkVal.asBool(); + MIL << "Explicity setting metalink " << ( doMetalink ? "enabled" : "disabled" ) << std::endl; + } + + req->_targetFileName = localPath; + req->_stagingFileName = stagingPath; + + const auto &expFilesize = req->_spec.value( zyppng::ProvideMsgFields::ExpectedFilesize ); + const auto &checkExistsOnly = req->_spec.value( zyppng::ProvideMsgFields::CheckExistOnly ); + const auto &deltaFile = req->_spec.value( zyppng::ProvideMsgFields::DeltaFile ); + + zyppng::DownloadSpec spec( + url + , stagingPath + , expFilesize.valid() ? zypp::ByteCount( expFilesize.asInt64() ) : zypp::ByteCount() + ); + spec + .setCheckExistsOnly( checkExistsOnly.valid() ? checkExistsOnly.asBool() : false ) + .setDeltaFile ( deltaFile.valid() ? deltaFile.asString() : zypp::Pathname() ) + .setMetalinkEnabled ( doMetalink ); + + req->startDownload( _dlManager->downloadFile ( spec ) ); + } +} + +void NetworkProvider::cancel( const std::deque::iterator &i ) +{ + auto &queue = requestQueue (); + if ( i == queue.end() ) { + ERR << "Unknown request ID, ignoring." << std::endl; + return; + } + + const auto &req = std::static_pointer_cast(*i); + req->cancelDownload(); + queue.erase(i); +} + +void NetworkProvider::immediateShutdown() +{ + for ( const auto &pItem : requestQueue () ) { + auto ref = std::static_pointer_cast( pItem ); + ref->cancelDownload(); + } +} + +zyppng::worker::ProvideWorkerItemRef NetworkProvider::makeItem(zyppng::ProvideMessage &&spec ) +{ + return std::make_shared( *this, std::move(spec) ); +} + +void NetworkProvider::itemStarted( NetworkProvideItemRef item ) +{ + provideStart( item->_spec.requestId(), item->_dl->spec().url().asCompleteString(), item->_targetFileName.asString(), item->_stagingFileName.asString() ); +} + +void NetworkProvider::itemFinished( NetworkProvideItemRef item ) +{ + auto &queue = requestQueue (); + auto i = std::find( queue.begin(), queue.end(), item ); + if ( i == queue.end() ) { + ERR << "Unknown request finished. This is a BUG!" << std::endl; + return; + } + + item->_state = NetworkProvideItem::Finished; + if ( !item->_dl->hasError() ) { + + if ( item->_dl->stoppedOnMetalink () ) { + // need to read the metalink info and send back a redirect + try { + zypp::media::MetaLinkParser pars; + pars.parse( item->_stagingFileName ); + + std::vector urls; + + for ( const auto &mirr : pars.getMirrors () ) { + try { + zypp::Url url( mirr.url.asCompleteString () ); + urls.push_back(url); + + if ( urls.size() == 10 ) + break; + + } catch ( const zypp::Exception &e ) { + ZYPP_CAUGHT (e); + } + } + + // @TODO , restart the download without metalink? + if ( urls.size() == 0 ) + throw zypp::Exception("No usable mirrors in Mirrorlink file"); + + if ( !messageStream()->sendMessage( zyppng::ProvideMessage::createMetalinkRedir( item->_spec.requestId(), urls ).impl() ) ) { + ERR << "Failed to send ProvideSuccess message" << std::endl; + } + + } catch ( const zypp::Exception &exp ) { + provideFailed( item->_spec.requestId(), zyppng::ProvideMessage::Code::InternalError, exp.asUserString (), false ); + } + + zypp::filesystem::unlink( item->_stagingFileName ); + + } else { + const auto errCode = zypp::filesystem::rename( item->_stagingFileName, item->_targetFileName ); + if( errCode ) { + + zypp::filesystem::unlink( item->_stagingFileName ); + + std::string err = zypp::str::Str() << "Renaming " << item->_stagingFileName << " to " << item->_targetFileName << " failed!"; + DBG << err << std::endl; + + provideFailed( item->_spec.requestId() + , zyppng::ProvideMessage::Code::InternalError + , err + , false + , {} ); + + } else { + provideSuccess( item->_spec.requestId(), false, item->_targetFileName ); + } + } + } else { + zyppng::HeaderValueMap extra; + bool isTransient = false; + auto errCode = zyppng::ProvideMessage::Code::InternalError; + + const auto &error = item->_dl->lastRequestError(); + switch( error.type() ) { + case zyppng::NetworkRequestError::NoError: { throw std::runtime_error("DownloadError info broken"); break;} + case zyppng::NetworkRequestError::InternalError: { errCode = zyppng::ProvideMessage::Code::InternalError; break;} + case zyppng::NetworkRequestError::Cancelled: { errCode = zyppng::ProvideMessage::Code::Cancelled; break;} + case zyppng::NetworkRequestError::PeerCertificateInvalid: { errCode = zyppng::ProvideMessage::Code::PeerCertificateInvalid; break;} + case zyppng::NetworkRequestError::ConnectionFailed: { errCode = zyppng::ProvideMessage::Code::ConnectionFailed; break;} + case zyppng::NetworkRequestError::ExceededMaxLen: { errCode = zyppng::ProvideMessage::Code::ExpectedSizeExceeded; break;} + case zyppng::NetworkRequestError::InvalidChecksum: { errCode = zyppng::ProvideMessage::Code::InvalidChecksum; break;} + case zyppng::NetworkRequestError::UnsupportedProtocol: { errCode = zyppng::ProvideMessage::Code::BadRequest; break; } + case zyppng::NetworkRequestError::MalformedURL: { errCode = zyppng::ProvideMessage::Code::BadRequest; break; } + case zyppng::NetworkRequestError::TemporaryProblem: { errCode = zyppng::ProvideMessage::Code::InternalError; isTransient = true; break;} + case zyppng::NetworkRequestError::Timeout: { errCode = zyppng::ProvideMessage::Code::Timeout; break;} + case zyppng::NetworkRequestError::Forbidden: { errCode = zyppng::ProvideMessage::Code::Forbidden; break;} + case zyppng::NetworkRequestError::NotFound: { errCode = zyppng::ProvideMessage::Code::NotFound; break;} + case zyppng::NetworkRequestError::Unauthorized: { + errCode = zyppng::ProvideMessage::Code::Unauthorized; + const auto &authHint = error.extraInfoValue("authHint", std::string()); + if ( authHint.size () ) + extra.set ( "authhint", authHint ); + break; + } + case zyppng::NetworkRequestError::AuthFailed: { + errCode = zyppng::ProvideMessage::Code::NoAuthData; + const auto &authHint = error.extraInfoValue("authHint", std::string()); + if ( authHint.size () ) + extra.set ( "authhint", authHint ); + break; + } + case zyppng::NetworkRequestError::ServerReturnedError: { errCode = zyppng::ProvideMessage::Code::InternalError; break; } + case zyppng::NetworkRequestError::MissingData: { errCode = zyppng::ProvideMessage::Code::BadRequest; break; } + } + + provideFailed( item->_spec.requestId(), errCode, item->_dl->errorString(), isTransient, extra ); + } + queue.erase(i); + + // pick the next request + provide(); +} + +void NetworkProvider::itemAuthRequired( NetworkProvideItemRef item, zyppng::NetworkAuthData &auth, const std::string & availAuth ) +{ + auto res = requireAuthorization( item->_spec.requestId(), auth.url(), auth.username(), auth.lastDatabaseUpdate(), { { std::string( zyppng::AuthDataRequestMsgFields::AuthHint ), availAuth } } ); + if ( !res ) { + auth = zyppng::NetworkAuthData(); + return; + } + + auth.setUrl ( item->_dl->spec().url() ); + auth.setUsername ( res->username ); + auth.setPassword ( res->password ); + auth.setLastDatabaseUpdate ( res->last_auth_timestamp ); + auth.extraValues() = res->extraKeys; + + const auto &keyStr = std::string(zyppng::AuthInfoMsgFields::AuthType); + if ( res->extraKeys.count( keyStr ) ) { + auth.setAuthType( res->extraKeys[keyStr] ); + } + + //@todo migrate away from using NetworkAuthData, AuthData now has a map of extra values that can carry the AuthType + const auto &authTypeKey = std::string( zyppng::AuthInfoMsgFields::AuthType ); + if ( res->extraKeys.count( authTypeKey ) ) + auth.setAuthType( res->extraKeys[ authTypeKey ] ); +} diff --git a/tools/zypp-media-http/networkprovider.h b/tools/zypp-media-http/networkprovider.h new file mode 100644 index 0000000..1fa6807 --- /dev/null +++ b/tools/zypp-media-http/networkprovider.h @@ -0,0 +1,80 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ +#ifndef ZYPP_NG_TOOLS_NETWORKPROVIDER_H_INCLUDED +#define ZYPP_NG_TOOLS_NETWORKPROVIDER_H_INCLUDED + +#include +#include +#include +#include +#include + +namespace zyppng { + class Downloader; + class Download; +} + +class NetworkProvider; + +struct NetworkProvideItem : public zyppng::worker::ProvideWorkerItem +{ +public: + NetworkProvideItem( NetworkProvider &parent, zyppng::ProvideMessage &&spec ); + ~NetworkProvideItem(); + + void startDownload( std::shared_ptr &&dl ); + void cancelDownload (); + + std::shared_ptr _dl; + zypp::Pathname _targetFileName; + zypp::Pathname _stagingFileName; + + std::chrono::steady_clock::time_point _scheduleAfter = std::chrono::steady_clock::time_point::min(); + +private: + void clearConnections (); + void onStarted ( zyppng::Download & ); + void onFinished ( zyppng::Download & ); + void onAuthRequired ( zyppng::Download &, zyppng::NetworkAuthData &auth, const std::string &availAuth ); + +private: + std::vector _connections; + NetworkProvider &_parent; +}; + +using NetworkProvideItemRef = std::shared_ptr; + +class NetworkProvider : public zyppng::worker::ProvideWorker +{ +public: + NetworkProvider( std::string_view workerName ); + void immediateShutdown() override; + +protected: + // ProvideWorker interface + zyppng::expected initialize(const zyppng::worker::Configuration &conf) override; + void provide() override; + void cancel(const std::deque::iterator &i ) override; + zyppng::worker::ProvideWorkerItemRef makeItem(zyppng::ProvideMessage &&spec) override; + + friend struct NetworkProvideItem; + void itemStarted ( NetworkProvideItemRef item ); + void itemFinished ( NetworkProvideItemRef item ); + void itemAuthRequired (NetworkProvideItemRef item, zyppng::NetworkAuthData &auth, const std::string &); + +private: + std::shared_ptr _dlManager; + zypp::Pathname _attachPoint; +}; + + + + + +#endif diff --git a/tools/zypp-media-iso/CMakeLists.txt b/tools/zypp-media-iso/CMakeLists.txt new file mode 100644 index 0000000..b82ce77 --- /dev/null +++ b/tools/zypp-media-iso/CMakeLists.txt @@ -0,0 +1,44 @@ +PROJECT( zypp-media-iso C CXX ) + +set (CMAKE_CXX_STANDARD 17) +SET (CMAKE_CXX_EXTENSIONS OFF) + +FIND_PACKAGE(Protobuf REQUIRED) + +SET( SOURCES + main.cc + isoprovider.cc + isoprovider.h + ${ZYPP_TOOLS_DIR}/zypp-media-dir/dirprovider.h + ${ZYPP_TOOLS_DIR}/zypp-media-dir/dirprovider.cc + ${ZYPP_TOOLS_DIR}/zypp-media-disk/diskprovider.h + ${ZYPP_TOOLS_DIR}/zypp-media-disk/diskprovider.cc + ${ZYPP_TOOLS_DIR}/zypp-media-nfs/nfsprovider.h + ${ZYPP_TOOLS_DIR}/zypp-media-nfs/nfsprovider.cc + ${ZYPP_TOOLS_DIR}/zypp-media-smb/smbprovider.h + ${ZYPP_TOOLS_DIR}/zypp-media-smb/smbprovider.cc +) + +include_directories( + ${ZYPP_TOOLS_DIR}/zypp-media-dir + ${ZYPP_TOOLS_DIR}/zypp-media-disk + ${ZYPP_TOOLS_DIR}/zypp-media-nfs + ${ZYPP_TOOLS_DIR}/zypp-media-smb +) + +add_executable( ${PROJECT_NAME} ${SOURCES} ) +target_link_libraries( ${PROJECT_NAME} zypp-media ) +target_link_libraries( ${PROJECT_NAME} zypp-core ) +target_link_libraries( ${PROJECT_NAME} zypp-protobuf ) +target_link_libraries( ${PROJECT_NAME} ${PROTOBUF_LITE_LIBRARIES} ) + +IF ( INSTALL_NG_BINARIES ) + INSTALL( TARGETS ${PROJECT_NAME} DESTINATION "${ZYPP_LIBEXEC_INSTALL_DIR}/workers" ) +ENDIF() + +set_target_properties( ${PROJECT_NAME} + PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" +) diff --git a/tools/zypp-media-iso/isoprovider.cc b/tools/zypp-media-iso/isoprovider.cc new file mode 100644 index 0000000..c2123a9 --- /dev/null +++ b/tools/zypp-media-iso/isoprovider.cc @@ -0,0 +1,399 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ +#include "isoprovider.h" +#include +#include + +#include +#include + +#include +#include + +#undef ZYPP_BASE_LOGGER_LOGGROUP +#define ZYPP_BASE_LOGGER_LOGGROUP "IsoProvider" + +RaiiHelper::~RaiiHelper() +{ + _backingDriver->detachMedia( id ); + _backingDriver->releaseIdleDevices(); +} + +IsoProvider::IsoProvider( ) + : DeviceDriver( zyppng::worker::WorkerCaps::SimpleMount ) +{ } + +IsoProvider::~IsoProvider() +{ } + +zyppng::worker::AttachResult IsoProvider::mountDevice ( const uint32_t id, const zypp::Url &attachUrl, const std::string &attachId, const std::string &label, const zyppng::HeaderValueMap &extras ) +{ + try + { + zypp::Pathname isofile = attachUrl.getQueryParam("iso"); + if( isofile.empty()) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , zypp::str::Str()<<"Media url: "<load( extras.value(zyppng::AttachMsgFields::VerifyData).asString() ) ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , "Failed to create verifier from file" + , false + ); + } + } + + // first check if we have that device already + const auto &devs = knownDevices(); + auto i = std::find_if( devs.begin(), devs.end(), [&]( const auto &d ) { + return d->_name == fullIsoUrl.asString(); + }); + if ( i != devs.end() ) { + // if we can find the device, isDesiredMedium needs to return true, + // otherwise URL and verifier do not match and its not the desired medium + auto res = isDesiredMedium( attachUrl, (*i)->_mountPoint / relAttachRoot, verifier, extras.value( zyppng::AttachMsgFields::MediaNr, 1 ).asInt() ); + if ( !res ) { + try { + std::rethrow_exception( res.error() ); + } catch( const zypp::Exception& e ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MediumNotDesired + , false + , e + ); + } catch ( ... ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MediumNotDesired + , "Checking the medium failed with an uknown error" + , false + ); + } + } else { + attachedMedia().insert( std::make_pair( attachId, zyppng::worker::AttachedMedia{ *i, relAttachRoot } ) ); + return zyppng::worker::AttachResult::success(); + } + } + + // helper lambda to invoke the deviceDriver + const auto &attachViaHelper = [&]( auto &driver, const std::string &type ) { + + auto mountRes = driver->mountDevice( id, src, attachId, "", {} ); + if ( !mountRes ) { + return mountRes; + } + + // make sure the media is released as soon as its not needed anymore + auto hlpr = std::make_shared(); + hlpr->_backingDriver = driver; + hlpr->id = attachId; + + // k, get the mountpoint from the backing driver + const auto &attMedias = driver->attachedMedia(); + auto i = attMedias.find( attachId ); + if ( i == attMedias.end() ) { + const std::string &err = zypp::str::Str()<< "Failed to query mounted supporting device: " << src.asString(); + ERR << err << std::endl; + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , err + , false ); + } + + // full path to the ISO in the filesystem + auto isopath = zypp::filesystem::expandlink( i->second._dev->_mountPoint / isofile ); + if( isopath.empty() || !zypp::PathInfo(isopath).isFile()) { + const std::string &err = zypp::str::Str()<< "No ISO file found on given source location: " << src.asString(); + ERR << err << std::endl; + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , err + , false ); + } + + // we have everything we need, let's try to mount it + zypp::media::Mount mount; + zypp::Pathname newAp; + try { + newAp = createAttachPoint( attachRoot() ); + if ( newAp.empty() ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , "Failed to create mount directory." + , false + ); + } + + std::string mountpoint( newAp.asString() ); + std::string mountopts("ro,loop"); + + mount.mount( isopath.asString(), mountpoint, + filesystem, mountopts ); + + // wait for /etc/mtab update ... + // (shouldn't be needed) + int limit = 3; + bool mountsucceeded = false; + + // Type ISO: Since 11.1 mtab might contain the name of + // the loop device instead of the iso file: + const auto &checkAttachedISO = [&]( const zypp::media::MountEntry &e ) { + if ( zypp::str::hasPrefix( zypp::Pathname(e.src).asString(), "/dev/loop" ) + && mountpoint == zypp::Pathname(e.dir) ) { + DBG << "Found bound media " + << isopath + << " in the mount table as " << e.src << std::endl; + return true; + } + return false; + }; + + // mount command came back OK, lets wait for mtab to update + while( !(mountsucceeded=checkAttached( newAp, checkAttachedISO )) && --limit) { + MIL << "Mount did not appear yet, sleeping for 1s" << std::endl; + sleep(1); + } + + // mount didn't work after all, bail out + if ( !mountsucceeded ) { + try { + mount.umount( newAp.asString() ); + } catch (const zypp::media::MediaException & excpt_r) { + ZYPP_CAUGHT(excpt_r); + } + ZYPP_THROW( zypp::media::MediaMountException( + "Unable to verify that the media was mounted", + isopath.asString(), newAp.asString() + )); + } + + // if we reach this place, mount worked -> YAY, lets see if that is the desired medium! + auto isDesired = isDesiredMedium( attachUrl, newAp / relAttachRoot, verifier, extras.value( zyppng::AttachMsgFields::MediaNr, 1 ).asInt() ); + if ( !isDesired ) { + try { + mount.umount( newAp.asString() ); + } catch (const zypp::media::MediaException & excpt_r) { + ZYPP_CAUGHT(excpt_r); + } + ZYPP_THROW( zypp::media::MediaNotDesiredException( attachUrl ) ); + } + + } catch ( const zypp::Exception &e ) { + removeAttachPoint(newAp); + ZYPP_CAUGHT(e); + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , false + , e + ); + } + + // here we have our device AND it is verified , lets register it and return it to the user + auto devPtr = std::make_shared( zyppng::worker::Device{ + ._name = fullIsoUrl.asString(), + ._maj_nr = 0, + ._min_nr = 0, + ._mountPoint = newAp, + ._ephemeral = true, // forget about the device after we are finished with it + ._properties = {} + }); + devPtr->_properties["raiiHelper"] = hlpr; + + + knownDevices().push_back( devPtr ); + attachedMedia().insert( std::make_pair( attachId, zyppng::worker::AttachedMedia{ devPtr, relAttachRoot } ) ); + return zyppng::worker::AttachResult::success(); + + }; + + if( !src.isValid()) { + const std::string &err = zypp::str::Str()<<"Invalid iso filename source media url: "<(); + + auto subAttachRoot = ( attachRoot().realpath()/"disk" ); + zypp::filesystem::assert_dir( subAttachRoot ); + + auto confCopy = config(); + (*confCopy.mutable_values())[std::string(zyppng::ATTACH_POINT)] = subAttachRoot.asString(); + const auto &res = _diskWorker->initialize( confCopy ); + if ( !res ) { + const std::string &err = zypp::str::Str()<< "Failed to initialize worker for: " << src.asString(); + ERR << err << std::endl; + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , err + , false ); + } + } + + return attachViaHelper( _diskWorker, "disk" ); + + } else if ( src.getScheme() == "dir" || src.getScheme() == "file" ) { + + if ( !_dirWorker ) { + _dirWorker = std::make_shared(); + + auto subAttachRoot = ( attachRoot().realpath()/"dir" ); + zypp::filesystem::assert_dir( subAttachRoot ); + + auto confCopy = config(); + (*confCopy.mutable_values())[std::string(zyppng::ATTACH_POINT)] = subAttachRoot.asString(); + const auto &res = _dirWorker->initialize( confCopy ); + if ( !res ) { + const std::string &err = zypp::str::Str()<< "Failed to initialize worker for: " << src.asString(); + ERR << err << std::endl; + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , err + , false ); + } + } + + return attachViaHelper( _dirWorker, "dir" ); + + } else if ( src.getScheme() == "nfs" || src.getScheme() == "nfs4" ) { + + if ( !_nfsWorker ) { + _nfsWorker = std::make_shared(); + + auto subAttachRoot = ( attachRoot().realpath()/"nfs" ); + zypp::filesystem::assert_dir( subAttachRoot ); + + auto confCopy = config(); + (*confCopy.mutable_values())[std::string(zyppng::ATTACH_POINT)] = subAttachRoot.asString(); + const auto &res = _nfsWorker->initialize( confCopy ); + MIL << "AFTER INIT " << _nfsWorker->attachRoot() << std::endl; + if ( !res ) { + const std::string &err = zypp::str::Str()<< "Failed to initialize worker for: " << src.asString(); + ERR << err << std::endl; + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , err + , false ); + } + } + + return attachViaHelper( _nfsWorker, "nfs" ); + + } else if ( src.getScheme() == "smb" || src.getScheme() == "cifs" ) { + + if ( !_smbWorker ) { + _smbWorker = std::make_shared(); + + auto subAttachRoot = ( attachRoot().realpath()/"smb" ); + zypp::filesystem::assert_dir( subAttachRoot ); + + auto confCopy = config(); + (*confCopy.mutable_values())[std::string(zyppng::ATTACH_POINT)] = subAttachRoot.asString(); + const auto &res = _smbWorker->initialize( confCopy ); + if ( !res ) { + const std::string &err = zypp::str::Str()<< "Failed to initialize worker for: " << src.asString(); + ERR << err << std::endl; + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , err + , false ); + } + } + + return attachViaHelper( _smbWorker, "smb" ); + + } else { + const std::string &err = zypp::str::Str()<< "ISO filename source media url scheme is not supported: " << src.asString(); + ERR << err << std::endl; + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , err + , false ); + } + } catch ( const zypp::Exception &e ) { + return zyppng::worker::AttachResult::error ( + zyppng::ProvideMessage::Code::BadRequest + , false + , e ); + } catch ( const std::exception &e ) { + return zyppng::worker::AttachResult::error ( + zyppng::ProvideMessage::Code::BadRequest + , e.what() + , false ); + } catch ( ... ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::BadRequest + , "Unknown exception" + , false); + } +} diff --git a/tools/zypp-media-iso/isoprovider.h b/tools/zypp-media-iso/isoprovider.h new file mode 100644 index 0000000..72f39fa --- /dev/null +++ b/tools/zypp-media-iso/isoprovider.h @@ -0,0 +1,42 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ +#ifndef ZYPP_NG_TOOLS_ISOPROVIDER_H_INCLUDED +#define ZYPP_NG_TOOLS_ISOPROVIDER_H_INCLUDED + +#include +#include "dirprovider.h" +#include "smbprovider.h" +#include "diskprovider.h" +#include "nfsprovider.h" + + +struct RaiiHelper +{ + ~RaiiHelper(); + zyppng::worker::DeviceDriverRef _backingDriver; + std::string id; +}; + +class IsoProvider : public zyppng::worker::DeviceDriver +{ + public: + IsoProvider( ); + ~IsoProvider(); + + zyppng::worker::AttachResult mountDevice ( const uint32_t id, const zypp::Url &attachUrl, const std::string &attachId, const std::string &label, const zyppng::HeaderValueMap &extras ) override; + + private: + std::shared_ptr _dirWorker; + std::shared_ptr _diskWorker; + std::shared_ptr _nfsWorker; + std::shared_ptr _smbWorker; + +}; + +#endif diff --git a/tools/zypp-media-iso/main.cc b/tools/zypp-media-iso/main.cc new file mode 100644 index 0000000..a384ce6 --- /dev/null +++ b/tools/zypp-media-iso/main.cc @@ -0,0 +1,27 @@ +#include "isoprovider.h" + +#include +#include +#include + +int main( int , char *[] ) +{ + // lets ignore those, SIGPIPE is better handled via the EPIPE error, and we do not want anyone + // to CTRL+C us + zyppng::blockSignalsForCurrentThread( { SIGPIPE, SIGINT } ); + + auto driver = std::make_shared(); + auto provider = std::make_shared( "zypp-media-iso", driver ); + driver->setProvider( provider ); + + auto res = provider->run (STDIN_FILENO, STDOUT_FILENO); + + MIL << "ISO Worker shutting down" << std::endl; + + provider->immediateShutdown(); + if ( res ) + return 0; + + //@TODO print error + return 1; +} diff --git a/tools/zypp-media-nfs/CMakeLists.txt b/tools/zypp-media-nfs/CMakeLists.txt new file mode 100644 index 0000000..aed5d46 --- /dev/null +++ b/tools/zypp-media-nfs/CMakeLists.txt @@ -0,0 +1,29 @@ +PROJECT( zypp-media-nfs C CXX ) + +set (CMAKE_CXX_STANDARD 17) +SET (CMAKE_CXX_EXTENSIONS OFF) + +FIND_PACKAGE(Protobuf REQUIRED) + +SET( SOURCES + main.cc + nfsprovider.cc + nfsprovider.h +) + +add_executable( ${PROJECT_NAME} ${SOURCES} ) +target_link_libraries( ${PROJECT_NAME} zypp-media ) +target_link_libraries( ${PROJECT_NAME} zypp-core ) +target_link_libraries( ${PROJECT_NAME} zypp-protobuf ) +target_link_libraries( ${PROJECT_NAME} ${PROTOBUF_LITE_LIBRARIES} ) + +IF ( INSTALL_NG_BINARIES ) + INSTALL( TARGETS ${PROJECT_NAME} DESTINATION "${ZYPP_LIBEXEC_INSTALL_DIR}/workers" ) +ENDIF() + +set_target_properties( ${PROJECT_NAME} + PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" +) diff --git a/tools/zypp-media-nfs/main.cc b/tools/zypp-media-nfs/main.cc new file mode 100644 index 0000000..17171e1 --- /dev/null +++ b/tools/zypp-media-nfs/main.cc @@ -0,0 +1,23 @@ +#include "nfsprovider.h" + +#include +#include +#include + +int main( int , char *[] ) +{ + // lets ignore those, SIGPIPE is better handled via the EPIPE error, and we do not want anyone + // to CTRL+C us + zyppng::blockSignalsForCurrentThread( { SIGPIPE, SIGINT } ); + + auto driver = std::make_shared(); + auto worker = std::make_shared( "zypp-media-nfs", driver ); + driver->setProvider( worker ); + + auto res = worker->run (STDIN_FILENO, STDOUT_FILENO); + if ( res ) + return 0; + + //@TODO print error + return 1; +} diff --git a/tools/zypp-media-nfs/nfsprovider.cc b/tools/zypp-media-nfs/nfsprovider.cc new file mode 100644 index 0000000..5a67db9 --- /dev/null +++ b/tools/zypp-media-nfs/nfsprovider.cc @@ -0,0 +1,262 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ +#include "nfsprovider.h" +#include +#include +#include + +#undef ZYPP_BASE_LOGGER_LOGGROUP +#define ZYPP_BASE_LOGGER_LOGGROUP "NfsProvider" + +using namespace std::literals::string_view_literals; + +/** + * Value of NFS mount minor timeout (passed to timeo option + * of the NFS mount) in tenths of a second. + * + * The value of 300 should give a major timeout after 3.5 minutes + * for UDP and 1.5 minutes for TCP. (#350309) + */ +constexpr auto NFS_MOUNT_TIMEOUT = 300; +constexpr std::array NFS_MOUNT_FSTYPES = { "nfs"sv, "nfs4"sv }; + +NfsProvider::NfsProvider( ) + : DeviceDriver( zyppng::worker::WorkerCaps::SimpleMount ) +{ } + +NfsProvider::~NfsProvider() +{ } + +zyppng::worker::AttachResult NfsProvider::mountDevice ( const uint32_t id, const zypp::Url &attachUrl, const std::string &attachId, const std::string &label, const zyppng::HeaderValueMap &extras ) +{ + try + { + if ( attachUrl.getHost().empty() ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , "Host can not be empty" + , false + ); + } + + // set up the verifier + zyppng::MediaDataVerifierRef verifier; + zyppng::MediaDataVerifierRef devVerifier; + if ( extras.value( zyppng::AttachMsgFields::VerifyType ).valid() ) { + verifier = zyppng::MediaDataVerifier::createVerifier( extras.value(zyppng::AttachMsgFields::VerifyType).asString() ); + if ( !verifier ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , "Invalid verifier type" + , false + ); + } + + if ( !verifier->load( extras.value(zyppng::AttachMsgFields::VerifyData).asString() ) ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , "Failed to create verifier from file" + , false + ); + } + } + const auto &devs = knownDevices(); + + // build a device name the same way the old media backend did + std::string path = attachUrl.getHost(); + path += ':'; + path += zypp::Pathname(attachUrl.getPathName()).asString(); + + // first check if we have that device already + auto i = std::find_if( devs.begin(), devs.end(), [&]( const auto &d ) { return d->_name == path; } ); + if ( i != devs.end() ) { + auto res = isDesiredMedium( attachUrl, (*i)->_mountPoint, verifier, extras.value( zyppng::AttachMsgFields::MediaNr, 1 ).asInt() ); + if ( !res ) { + try { + std::rethrow_exception( res.error() ); + } catch( const zypp::Exception& e ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MediumNotDesired + , false + , e ); + } catch ( ... ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MediumNotDesired + , "Checking the medium failed with an uknown error" + , false + ); + } + } else { + attachedMedia().insert( std::make_pair( attachId, zyppng::worker::AttachedMedia{ *i, "/" } ) ); + return zyppng::worker::AttachResult::success(); + } + } + + // we did not find a existing mount, well lets make a new one + // start with some legacy code: + std::string filesystem( attachUrl.getScheme() ); + if ( filesystem != "nfs4" && attachUrl.getQueryParam("type") == "nfs4" ) { + filesystem = "nfs4"; + } + + std::string options = attachUrl.getQueryParam("mountoptions"); + if(options.empty()) { + options="ro"; + } + + std::vector optionList; + zypp::str::split( options, std::back_inserter(optionList), "," ); + std::vector::const_iterator it; + bool contains_lock = false, contains_soft = false, + contains_timeo = false, contains_hard = false; + + for( it = optionList.begin(); it != optionList.end(); ++it ) { + if ( *it == "lock" || *it == "nolock" ) contains_lock = true; + else if ( *it == "soft") contains_soft = true; + else if ( *it == "hard") contains_hard = true; + else if ( it->find("timeo") != std::string::npos ) contains_timeo = true; + } + + if ( !(contains_lock && contains_soft) ) { + // Add option "nolock", unless option "lock" or "unlock" is already set. + // This should prevent the mount command hanging when the portmapper isn't + // running. + if ( !contains_lock ) { + optionList.push_back( "nolock" ); + } + // Add options "soft,timeo=NFS_MOUNT_TIMEOUT", unless they are set + // already or "hard" option is explicitly specified. This prevent + // the mount command from hanging when the nfs server is not responding + // and file transactions from an unresponsive to throw an error after + // a short time instead of hanging forever + if ( !(contains_soft || contains_hard) ) { + optionList.push_back( "soft" ); + if ( !contains_timeo ) { + std::ostringstream s; + s << "timeo=" << NFS_MOUNT_TIMEOUT; + optionList.push_back( s.str() ); + } + } + options = zypp::str::join( optionList, "," ); + } + + zypp::Pathname newAp; + zypp::media::Mount mount; + try + { + newAp = createAttachPoint( attachRoot() ); + if ( newAp.empty() ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , "Failed to create mount directory." + , false + ); + } + + mount.mount( path, newAp.asString(), filesystem, options ); + + // wait for /etc/mtab update ... + // (shouldn't be needed) + int limit = 3; + bool mountsucceeded = false; + + // we only check if our attachpoint is in mounts and of type nfs/nfs4, everything else is not reliable anyways + constexpr auto checkFsTypePredicate = []( const zypp::media::MountEntry &entry ){ + return ( std::find( NFS_MOUNT_FSTYPES.begin(), NFS_MOUNT_FSTYPES.end(), entry.type ) != NFS_MOUNT_FSTYPES.end() ); + }; + while( !(mountsucceeded=checkAttached( newAp, checkFsTypePredicate )) && --limit) { + MIL << "Mount did not appear yet, sleeping for 1s" << std::endl; + sleep(1); + } + + if( !mountsucceeded) { + try { + mount.umount( newAp.asString() ); + } catch (const zypp::media::MediaException & excpt_r) { + ZYPP_CAUGHT(excpt_r); + } + ZYPP_THROW( zypp::media::MediaMountException( + "Unable to verify that the media was mounted", + path, newAp.asString() + )); + } + + // if we reach this place, mount worked -> YAY, lets see if that is the desired medium! + const auto &isDesired = isDesiredMedium( attachUrl, newAp, verifier, extras.value( zyppng::AttachMsgFields::MediaNr, 1 ).asInt() ); + if ( !isDesired ) { + try { + mount.umount( newAp.asString() ); + } catch (const zypp::media::MediaException & excpt_r) { + ZYPP_CAUGHT(excpt_r); + } + ZYPP_THROW( zypp::media::MediaNotDesiredException( attachUrl ) ); + } + + MIL << "New device " << path << " mounted on " << newAp << std::endl; + auto newDev = std::shared_ptr( new zyppng::worker::Device{ + ._name = path, + ._maj_nr = 0, + ._min_nr = 0, + ._mountPoint = newAp, + ._ephemeral = true, // device should be removed after the last attachment was released + ._properties = {} + }); + knownDevices().push_back( newDev ); + attachedMedia().insert( std::make_pair( attachId, zyppng::worker::AttachedMedia{ newDev, "/" } ) ); + return zyppng::worker::AttachResult::success(); + + } + catch (const zypp::media::MediaMountException &e ) + { + removeAttachPoint(newAp); + ZYPP_CAUGHT(e); + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , false + , e + ); + } + catch (const zypp::media::MediaException & e) + { + removeAttachPoint(newAp); + ZYPP_CAUGHT(e); + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , false + , e + ); + } + + // We should never reach this place, but if we do fail the attach request + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , zypp::str::Str()<<"Mounting the medium " << attachUrl << " failed for an unknown reason" + , false + ); + + } catch ( const zypp::Exception &e ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::BadRequest + , false + , e + ); + } catch ( const std::exception &e ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::BadRequest + , e.what() + , false + ); + } catch ( ... ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::BadRequest + , "Unknown exception" + , false + ); + } +} diff --git a/tools/zypp-media-nfs/nfsprovider.h b/tools/zypp-media-nfs/nfsprovider.h new file mode 100644 index 0000000..775eb50 --- /dev/null +++ b/tools/zypp-media-nfs/nfsprovider.h @@ -0,0 +1,25 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ +#ifndef ZYPP_NG_TOOLS_NFSPROVIDER_H_INCLUDED +#define ZYPP_NG_TOOLS_NFSPROVIDER_H_INCLUDED + +#include + +class NfsProvider : public zyppng::worker::DeviceDriver +{ + public: + NfsProvider(); + ~NfsProvider(); + + // DeviceDriver interface + zyppng::worker::AttachResult mountDevice ( const uint32_t id, const zypp::Url &attachUrl, const std::string &attachId, const std::string &label, const zyppng::HeaderValueMap &extras ) override; + +}; + +#endif diff --git a/tools/zypp-media-smb/CMakeLists.txt b/tools/zypp-media-smb/CMakeLists.txt new file mode 100644 index 0000000..24f3aa4 --- /dev/null +++ b/tools/zypp-media-smb/CMakeLists.txt @@ -0,0 +1,29 @@ +PROJECT( zypp-media-smb C CXX ) + +set (CMAKE_CXX_STANDARD 17) +SET (CMAKE_CXX_EXTENSIONS OFF) + +FIND_PACKAGE(Protobuf REQUIRED) + +SET( SOURCES + main.cc + smbprovider.cc + smbprovider.h +) + +add_executable( ${PROJECT_NAME} ${SOURCES} ) +target_link_libraries( ${PROJECT_NAME} zypp-media ) +target_link_libraries( ${PROJECT_NAME} zypp-core ) +target_link_libraries( ${PROJECT_NAME} zypp-protobuf ) +target_link_libraries( ${PROJECT_NAME} ${PROTOBUF_LITE_LIBRARIES} ) + +IF ( INSTALL_NG_BINARIES ) + INSTALL( TARGETS ${PROJECT_NAME} DESTINATION "${ZYPP_LIBEXEC_INSTALL_DIR}/workers" ) +ENDIF() + +set_target_properties( ${PROJECT_NAME} + PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" +) diff --git a/tools/zypp-media-smb/main.cc b/tools/zypp-media-smb/main.cc new file mode 100644 index 0000000..4b26683 --- /dev/null +++ b/tools/zypp-media-smb/main.cc @@ -0,0 +1,26 @@ +#include "smbprovider.h" + +#include +#include +#include + +int main( int , char *[] ) +{ + // lets ignore those, SIGPIPE is better handled via the EPIPE error, and we do not want anyone + // to CTRL+C us + zyppng::blockSignalsForCurrentThread( { SIGPIPE, SIGINT } ); + + auto driver = std::make_shared(); + auto worker = std::make_shared( "zypp-media-smb", driver ); + driver->setProvider( worker ); + + auto res = worker->run (STDIN_FILENO, STDOUT_FILENO); + + MIL << "SMB Worker shutting down" << std::endl; + + if ( res ) + return 0; + + //@TODO print error + return 1; +} diff --git a/tools/zypp-media-smb/smbprovider.cc b/tools/zypp-media-smb/smbprovider.cc new file mode 100644 index 0000000..9288fa3 --- /dev/null +++ b/tools/zypp-media-smb/smbprovider.cc @@ -0,0 +1,342 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ +#include "smbprovider.h" +#include +#include +#include + +#include + +#include +#include + +#undef ZYPP_BASE_LOGGER_LOGGROUP +#define ZYPP_BASE_LOGGER_LOGGROUP "SmbProvider" + +using namespace std::literals::string_view_literals; + + +/*! + * Get the 1st path component (CIFS share name). + */ +inline std::string getShare( zypp::Pathname spath_r ) +{ + if ( spath_r.empty() ) + return std::string(); + + std::string share( spath_r.absolutename().asString() ); + std::string::size_type sep = share.find( "/", 1 ); + if ( sep == std::string::npos ) + share = share.erase( 0, 1 ); // nothing but the share name in spath_r + else + share = share.substr( 1, sep-1 ); + + // deescape %2f in sharename + while ( (sep = share.find( "%2f" )) != std::string::npos ) { + share.replace( sep, 3, "/" ); + } + + return share; +} + +/*! + * Strip off the 1st path component (CIFS share name). + */ +inline zypp::Pathname stripShare( zypp::Pathname spath_r ) +{ + if ( spath_r.empty() ) + return zypp::Pathname(); + + std::string striped( spath_r.absolutename().asString() ); + std::string::size_type sep = striped.find( "/", 1 ); + if ( sep == std::string::npos ) + return "/"; // nothing but the share name in spath_r + + return striped.substr( sep ); +} + + +SmbProvider::SmbProvider( ) + : DeviceDriver( zyppng::worker::WorkerCaps::SimpleMount ) +{ } + +SmbProvider::~SmbProvider() +{ } + +zyppng::worker::AttachResult SmbProvider::mountDevice ( const uint32_t id, const zypp::Url &attachUrl, const std::string &attachId, const std::string &label, const zyppng::HeaderValueMap &extras ) +{ + try + { + if ( attachUrl.getHost().empty() ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , "Host can not be empty" + , false + ); + } + + // set up the verifier + zyppng::MediaDataVerifierRef verifier; + if ( extras.value( zyppng::AttachMsgFields::VerifyType ).valid() ) { + verifier = zyppng::MediaDataVerifier::createVerifier( extras.value(zyppng::AttachMsgFields::VerifyType).asString() ); + if ( !verifier ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , "Invalid verifier type" + , false + ); + } + + if ( !verifier->load( extras.value(zyppng::AttachMsgFields::VerifyData).asString() ) ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , "Failed to create verifier from file" + , false + ); + } + } + const auto &devs = knownDevices(); + + // build a device name the same way the old media backend did + std::string path = "//"; + path += attachUrl.getHost() + "/" + getShare( attachUrl.getPathName() ); + + // first check if we have that device already + auto i = std::find_if( devs.begin(), devs.end(), [&]( const auto &d ) { return d->_name == path; } ); + if ( i != devs.end() ) { + auto res = isDesiredMedium( attachUrl, (*i)->_mountPoint, verifier, extras.value( zyppng::AttachMsgFields::MediaNr, 1 ).asInt() ); + if ( !res ) { + try { + std::rethrow_exception( res.error() ); + } catch( const zypp::Exception& e ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MediumNotDesired + , false + , e + ); + } catch ( ... ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MediumNotDesired + , "Checking the medium failed with an uknown error" + , false + ); + } + } else { + attachedMedia().insert( std::make_pair( attachId, zyppng::worker::AttachedMedia{ *i, "/" } ) ); + return zyppng::worker::AttachResult::success(); + } + } + + // we did not find a existing mount, well lets make a new one + // start with some legacy code: + zypp::media::Mount::Options options( attachUrl.getQueryParam("mountoptions") ); + + int64_t lastTimestamp = -1; + std::string username = attachUrl.getUsername(); + std::string password = attachUrl.getPassword(); + + if ( ! options.has( "rw" ) ) { + options["ro"]; + } + + // look for a workgroup + std::string workgroup = attachUrl.getQueryParam("workgroup"); + if ( workgroup.empty() ) + workgroup = attachUrl.getQueryParam("domain"); + if ( !workgroup.empty() ) + options["domain"] = workgroup; + + // extract 'username', do not overwrite any _url.username + + zypp::media::Mount::Options::iterator toEnv; + toEnv = options.find("username"); + if ( toEnv != options.end() ) { + if ( username.empty() ) + username = toEnv->second; + options.erase( toEnv ); + } + + toEnv = options.find("user"); // actually cifs specific + if ( toEnv != options.end() ) { + if ( username.empty() ) + username = toEnv->second; + options.erase( toEnv ); + } + + // extract 'password', do not overwrite any _url.password + + toEnv = options.find("password"); + if ( toEnv != options.end() ) { + if ( password.empty() ) + password = toEnv->second; + options.erase( toEnv ); + } + + toEnv = options.find("pass"); // actually cifs specific + if ( toEnv != options.end() ) { + if ( password.empty() ) + password = toEnv->second; + options.erase( toEnv ); + } + + bool canContinue = true; + while ( canContinue ) { + + canContinue = false; + + // pass 'username' and 'password' via environment + zypp::media::Mount::Environment environment; + if ( !username.empty() ) + environment["USER"] = username; + if ( !password.empty() ) + environment["PASSWD"] = password; + + // In case we need a tmpfile, credentials will remove + // it in its destructor after the mount call below. + zypp::filesystem::TmpPath credentials; + if ( !username.empty() || !password.empty() ) + { + zypp::filesystem::TmpFile tmp; + std::ofstream outs( tmp.path().asString().c_str() ); + outs << "username=" << username << std::endl; + outs << "password=" << password << std::endl; + outs.close(); + + credentials = tmp; + options["credentials"] = credentials.path().asString(); + + } else { + // Use 'guest' option unless explicitly disabled (bnc #547354) + if ( options.has( "noguest" ) ) + options.erase( "noguest" ); + else + // prevent smbmount from asking for password + // only add this option if 'credentials' is not used (bnc #560496) + options["guest"]; + } + + zypp::Pathname newAp; + zypp::media::Mount mount; + try + { + newAp = createAttachPoint( attachRoot() ); + if ( newAp.empty() ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , "Failed to create mount directory." + , false + ); + } + + mount.mount( path, newAp.asString(), "cifs", options.asString(), environment ); + + // wait for /etc/mtab update ... + // (shouldn't be needed) + int limit = 3; + bool mountsucceeded = false; + + while( !(mountsucceeded=checkAttached( newAp, DeviceDriver::fstypePredicate( path, {"smb", "cifs"} ) )) && --limit) { + MIL << "Mount did not appear yet, sleeping for 1s" << std::endl; + sleep(1); + } + + if ( !mountsucceeded ) { + try { + mount.umount( newAp.asString() ); + } catch (const zypp::media::MediaException & excpt_r) { + ZYPP_CAUGHT(excpt_r); + } + ZYPP_THROW( zypp::media::MediaMountException( + "Unable to verify that the media was mounted", + path, newAp.asString() + )); + } + + // if we reach this place, mount worked -> YAY, lets see if that is the desired medium! + const auto &isDesired = isDesiredMedium( attachUrl, newAp, verifier, extras.value( zyppng::AttachMsgFields::MediaNr, 1 ).asInt() ); + if ( !isDesired ) { + try { + mount.umount( newAp.asString() ); + } catch (const zypp::media::MediaException & excpt_r) { + ZYPP_CAUGHT(excpt_r); + } + ZYPP_THROW( zypp::media::MediaNotDesiredException( attachUrl ) ); + } + + MIL << "New device " << path << " mounted on " << newAp << std::endl; + auto newDev = std::shared_ptr( new zyppng::worker::Device{ + ._name = path, + ._maj_nr = 0, + ._min_nr = 0, + ._mountPoint = newAp, + ._ephemeral = true, // device should be removed after the last attachment was released + ._properties = {} + }); + knownDevices().push_back( newDev ); + attachedMedia().insert( std::make_pair( attachId, zyppng::worker::AttachedMedia{ newDev, "/" } ) ); + return zyppng::worker::AttachResult::success(); + + } catch (const zypp::media::MediaMountException & e) { + ZYPP_CAUGHT( e ); + + if ( e.mountError() == "Permission denied" ) { + auto parent = parentWorker(); + if ( parent ) { + auto res = parent->requireAuthorization( id, attachUrl, username, lastTimestamp ); + if ( res ) { + canContinue = true; + username = res->username; + password = res->password; + lastTimestamp = res->last_auth_timestamp; + } + } + if ( !canContinue ) { + removeAttachPoint(newAp); + return zyppng::worker::AttachResult::error( zyppng::ProvideMessage::Code::Forbidden, "No authentication data", false ); + } + } else { + removeAttachPoint(newAp); + return zyppng::worker::AttachResult::error( zyppng::ProvideMessage::Code::MountFailed, false, e ); + } + } catch (const zypp::media::MediaException & e) { + removeAttachPoint(newAp); + ZYPP_CAUGHT(e); + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , false + , e ); + } + } + + // We should never reach this place, but if we do fail the attach request + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , zypp::str::Str()<<"Mounting the medium " << attachUrl << " failed for an unknown reason" + , false + ); + + } catch ( const zypp::Exception &e ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::BadRequest + , false + , e );; + } catch ( const std::exception &e ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::BadRequest + , e.what() + , false + ); + } catch ( ... ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::BadRequest + , "Unknown exception" + , false + ); + } +} diff --git a/tools/zypp-media-smb/smbprovider.h b/tools/zypp-media-smb/smbprovider.h new file mode 100644 index 0000000..1d83990 --- /dev/null +++ b/tools/zypp-media-smb/smbprovider.h @@ -0,0 +1,24 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ +#ifndef ZYPP_NG_TOOLS_SMBPROVIDER_H_INCLUDED +#define ZYPP_NG_TOOLS_SMBPROVIDER_H_INCLUDED + +#include + +class SmbProvider : public zyppng::worker::DeviceDriver +{ + public: + SmbProvider(); + ~SmbProvider(); + + // DeviceDriver interface + zyppng::worker::AttachResult mountDevice ( const uint32_t id, const zypp::Url &attachUrl, const std::string &attachId, const std::string &label, const zyppng::HeaderValueMap &extras ) override; +}; + +#endif diff --git a/tools/zypp-media-tvm/CMakeLists.txt b/tools/zypp-media-tvm/CMakeLists.txt new file mode 100644 index 0000000..0b394df --- /dev/null +++ b/tools/zypp-media-tvm/CMakeLists.txt @@ -0,0 +1,25 @@ +PROJECT( zypp-media-tvm C CXX ) + +set (CMAKE_CXX_STANDARD 17) +SET (CMAKE_CXX_EXTENSIONS OFF) + +FIND_PACKAGE(Protobuf REQUIRED) + +SET( SOURCES + main.cc + testvmprovider.cc + testvmprovider.h +) + +add_executable( ${PROJECT_NAME} ${SOURCES} ) +target_link_libraries( ${PROJECT_NAME} zypp-media ) +target_link_libraries( ${PROJECT_NAME} zypp-core ) +target_link_libraries( ${PROJECT_NAME} zypp-protobuf ) +target_link_libraries( ${PROJECT_NAME} ${PROTOBUF_LITE_LIBRARIES} ) + +set_target_properties( ${PROJECT_NAME} + PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/workers" +) diff --git a/tools/zypp-media-tvm/main.cc b/tools/zypp-media-tvm/main.cc new file mode 100644 index 0000000..c1ce2c5 --- /dev/null +++ b/tools/zypp-media-tvm/main.cc @@ -0,0 +1,23 @@ +#include "testvmprovider.h" + +#include +#include +#include + +int main( int , char *[] ) +{ + // lets ignore those, SIGPIPE is better handled via the EPIPE error, and we do not want anyone + // to CTRL+C us + zyppng::blockSignalsForCurrentThread( { SIGPIPE, SIGINT } ); + + auto driver = std::make_shared(); + auto worker = std::make_shared( "zypp-media-tvm", driver ); + driver->setProvider(worker); + + auto res = worker->run (STDIN_FILENO, STDOUT_FILENO); + if ( res ) + return 0; + + //@TODO print error + return 1; +} diff --git a/tools/zypp-media-tvm/testvmprovider.cc b/tools/zypp-media-tvm/testvmprovider.cc new file mode 100644 index 0000000..6036c0e --- /dev/null +++ b/tools/zypp-media-tvm/testvmprovider.cc @@ -0,0 +1,343 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ +#include "testvmprovider.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#undef ZYPP_BASE_LOGGER_LOGGROUP +#define ZYPP_BASE_LOGGER_LOGGROUP "TestVMProvider" + +#include +#include + +extern "C" +{ +#include +#include +#if HAVE_UDEV +#include +#endif +} + +using namespace std::literals; + +constexpr std::string_view CONTENTDIR_PROP = "contentDir"; + +TestVMProvider::TestVMProvider( ) : zyppng::worker::DeviceDriver( zyppng::worker::WorkerCaps::VolatileMount ) +{ } + +TestVMProvider::~TestVMProvider() +{ } + +zyppng::expected TestVMProvider::initialize(const zyppng::worker::Configuration &conf) +{ + const auto &values = conf.values(); + + if ( const auto &i = values.find( std::string(zyppng::PROVIDER_ROOT) ); i != values.end() ) { + const auto &val = i->second; + MIL << "Got provider root from controller: " << val << std::endl; + _provRoot = val; + } else { + return zyppng::expected::error(ZYPP_EXCPT_PTR( zypp::Exception("Provider root required to work.") )); + } + + return DeviceDriver::initialize( conf ); +} + +zyppng::worker::AttachResult TestVMProvider::mountDevice ( const uint32_t id, const zypp::Url &attachUrl, const std::string &attachId, const std::string &label, const zyppng::HeaderValueMap &extras ) +{ + std::vector devs; + if ( extras.contains( zyppng::AttachMsgFields::Device ) ) { + for ( const auto &d : extras.values( zyppng::AttachMsgFields::Device ) ) { + if ( d.isString() ) { + MIL << "Got device from user we are allowed to use: " << d.asString() << std::endl; + devs.push_back(d.asString()); + } + } + } + + auto &sysDevs = knownDevices(); + std::vector> possibleDevs; + if ( devs.size () ) { + for ( const auto &d : devs ) { + std::shared_ptr device; + auto i = std::find_if( sysDevs.begin(), sysDevs.end(), [&]( const auto &sysDev ){ return ( sysDev->_name == d ); } ); + if ( i == sysDevs.end() ) { + ERR << "Device " << d << " from Attach request is not known ignoring!" << std::endl; + continue; + } else { + device = *i; + } + + if ( device ) + possibleDevs.push_back(device); + } + } else { + possibleDevs = sysDevs; + } + + // if we have no devices at this point controller gets a error + if ( possibleDevs.empty () ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , "No useable device found" + , false + ); + } + + const auto attachRoot = zypp::Pathname( attachUrl.getPathName() ); + const auto attachMediaNr = extras.value( zyppng::AttachMsgFields::MediaNr, 1 ).asInt(); + + // set up the verifier + zyppng::MediaDataVerifierRef verifier; + if ( extras.contains( zyppng::AttachMsgFields::VerifyType ) ) { + verifier = zyppng::MediaDataVerifier::createVerifier( extras.value(zyppng::AttachMsgFields::VerifyType).asString() ); + if ( !verifier ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , "Invalid verifier type" + , false + ); + } + + if ( !verifier->load( extras.value(zyppng::AttachMsgFields::VerifyData).asString() ) ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , "Failed to create verifier from file" + , false + ); + } + } + + //first check if any of the mounted devices are what we want + for( const auto &dev : possibleDevs ) { + if ( dev->_mountPoint.empty() ) + continue; + + MIL << "Found mounted device: " << dev->_name << " on mountpoint: " << dev->_mountPoint << std::endl; + + auto res = isDesiredMedium( attachUrl, dev->_mountPoint / attachRoot, verifier, attachMediaNr ); + if ( !res ) + continue; + + MIL << "Found requested medium in dev " << dev->_name << std::endl; + + // we found the device we want! + attachedMedia().insert( std::make_pair( attachId, zyppng::worker::AttachedMedia{ dev, attachRoot } ) ); + return zyppng::worker::AttachResult::success(); + } + + while ( true ) { + + // pick up changes from the test + detectDevices(); + + // remember how many devices we were able to test + uint devicesTested = 0; + + // none of the already mounted devices matched, lets try what we have left + for( const auto &dev : possibleDevs ) { + if ( !dev->_mountPoint.empty() ) + continue; + + MIL << "Trying to mount dev " << dev->_name << std::endl; + + devicesTested++; + + bool mountsucceeded = false; + + // we simply create a symbolic link in the attach dir to simulate mounting + zypp::Pathname newAp; + try { + newAp = createAttachPoint( this->attachRoot() ); + if ( newAp.empty() ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MountFailed + , "Failed to create mount directory." + , false + ); + } + + newAp = newAp/"mount"; + + // simulate failed mount + const auto &contDir = std::any_cast(dev->_properties[std::string(CONTENTDIR_PROP)]); + if ( contDir.empty() ) { + ZYPP_THROW( zypp::media::MediaMountException( + "Failed to mount device", + dev->_name, newAp.asString() + )); + } + + const auto res = zypp::filesystem::symlink( contDir, newAp ); + mountsucceeded = ( res == 0 ); + + if( !mountsucceeded) { + ZYPP_THROW( zypp::media::MediaMountException( + "Failed to mount device", + dev->_name, newAp.asString() + )); + } else { + MIL << "Device mounted on " << newAp << std::endl; + dev->_mountPoint = newAp; + } + } catch (const zypp::media::MediaException & excpt_r) { + removeAttachPoint(newAp.dirname()); + ZYPP_CAUGHT(excpt_r); + } + + if ( mountsucceeded ) { + // lets check if we have the correct medium + auto res = isDesiredMedium( attachUrl, dev->_mountPoint / attachRoot, verifier, attachMediaNr ); + if ( !res ) { + unmountDevice( *dev ); + continue; + } + + MIL << "Found requested medium in dev " << dev->_name << std::endl; + attachedMedia().insert( std::make_pair( attachId, zyppng::worker::AttachedMedia{ dev, attachRoot } ) ); + return zyppng::worker::AttachResult::success(); + } + } // for each device + + // we did go through all devices, but didn't find any. + if ( devicesTested == 0 ) { + // no devices are free + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::Jammed + , "No free ressources available" + , false + ); + } else { + + // k, we need to ask the user to give us the medium we need + // first find the devices that are free + std::vector freeDevs; + + for( const auto &dev : possibleDevs ) { + if ( !dev->_mountPoint.empty() ) + continue; + + MIL << "Adding " << dev->_name << " to list of free devs" << std::endl; + freeDevs.push_back( dev->_name ); + } + + // if there are no devices free, we are jammed + if ( freeDevs.empty() ) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::Jammed + , "No free ressources available" + , false + ); + } + + auto changeRes = zyppng::worker::ProvideWorker::ABORT; + auto parent = parentWorker(); + if ( parent ) + changeRes = parent->requestMediaChange ( id, label, attachMediaNr, freeDevs ); + if ( changeRes != zyppng::worker::ProvideWorker::SUCCESS ) { + if ( changeRes == zyppng::worker::ProvideWorker::SKIP) { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MediaChangeSkip + , "User asked to skip the media change." + , false + ); + } else { + return zyppng::worker::AttachResult::error( + zyppng::ProvideMessage::Code::MediaChangeAbort + , "User asked to abort the media change." + , false + ); + } + } + } + } +} + +void TestVMProvider::unmountDevice( zyppng::worker::Device &dev ) +{ + if ( dev._mountPoint.empty () ) + return; + try { + zypp::filesystem::unlink( dev._mountPoint ); + removeAttachPoint( dev._mountPoint.dirname() ); + } catch (const zypp::media::MediaException & excpt_r) { + ERR << "Failed to unmount device: " << dev._name << std::endl; + ZYPP_CAUGHT(excpt_r); + } + dev._mountPoint = zypp::Pathname(); +} + +void TestVMProvider::detectDevices( ) +{ + const auto &fname = _provRoot/"tvm.conf"; + int fd = open( fname.asString().data(), O_RDONLY ); + if ( fd < 0 ) { + ERR << "Failed to open file " << fname << " error: "<<"("<_name == dev.name(); + }); + if ( iDev == sysDevs.end() ) { + MIL << "Previously unknown device detected" << std::endl; + auto d = std::make_shared( zyppng::worker::Device{ + ._name = dev.name() + }); + if ( dev.insertedpath().size() ) { + d->_properties[std::string(CONTENTDIR_PROP)] = zypp::Pathname(dev.insertedpath()); + } + sysDevs.push_back(d); + continue; + } + + // mounted devices are never updated + if ( !(*iDev)->_mountPoint.empty() ) { + continue; + } + + (*iDev)->_properties[std::string(CONTENTDIR_PROP)] = zypp::Pathname(dev.insertedpath()); + } + } else { + for ( int i = 0; i < set.devices_size(); i++ ) { + const auto &dev = set.devices(i); + MIL << "Found device: " << dev.name() << std::endl; + auto d = std::make_shared( zyppng::worker::Device{ + ._name = dev.name() + }); + if ( dev.insertedpath().size() ) { + d->_properties[std::string(CONTENTDIR_PROP)] = zypp::Pathname(dev.insertedpath()); + } + sysDevs.push_back(d); + } + } +} diff --git a/tools/zypp-media-tvm/testvmprovider.h b/tools/zypp-media-tvm/testvmprovider.h new file mode 100644 index 0000000..07bed4b --- /dev/null +++ b/tools/zypp-media-tvm/testvmprovider.h @@ -0,0 +1,41 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ +#ifndef ZYPP_NG_TOOLS_DISCPROVIDER_H_INCLUDED +#define ZYPP_NG_TOOLS_DISCPROVIDER_H_INCLUDED + +#include +#include +#include +#include + +/*! + * \file Contains a testprovider to simulate volatile mounting devices. + * This is only used in the zypp testsuite. + */ + +class TestVMProvider : public zyppng::worker::DeviceDriver +{ +public: + + TestVMProvider(); + ~TestVMProvider(); + + // DeviceDriver interface + zyppng::expected initialize(const zyppng::worker::Configuration &conf) override; + zyppng::worker::AttachResult mountDevice ( const uint32_t id, const zypp::Url &attachUrl, const std::string &attachId, const std::string &label, const zyppng::HeaderValueMap &extras ) override; + void detectDevices(); + +protected: + void unmountDevice ( zyppng::worker::Device &dev ) override; + +private: + zypp::Pathname _provRoot; +}; + +#endif diff --git a/zypp-core/AutoDispose.h b/zypp-core/AutoDispose.h index c682475..ee00934 100644 --- a/zypp-core/AutoDispose.h +++ b/zypp-core/AutoDispose.h @@ -163,6 +163,10 @@ namespace zypp void swap( AutoDispose & rhs ) { _pimpl.swap( rhs._pimpl ); } + /** Returns true if this is the only AutoDispose instance managing the current data object */ + bool unique () const + { return _pimpl.unique(); } + public: /** Return the current dispose function. */ const Dispose & getDispose() const diff --git a/zypp-core/ByteArray.h b/zypp-core/ByteArray.h index d8253ae..a1d372a 100644 --- a/zypp-core/ByteArray.h +++ b/zypp-core/ByteArray.h @@ -33,6 +33,12 @@ namespace zypp { return std::string_view( data(), size() ); } #endif + + static std::size_t maxSize () { + static const auto size = ByteArray().max_size(); + return size; + } + }; class UByteArray : public std::vector @@ -40,6 +46,11 @@ namespace zypp { public: using vector::vector; explicit UByteArray ( const char *data, const int len = -1 ) : UByteArray( data, data + (len == -1 ? strlen(data) : len) ) { } + + static std::size_t maxSize () { + static const auto size = UByteArray().max_size(); + return size; + } }; } diff --git a/zypp-core/CMakeLists.txt b/zypp-core/CMakeLists.txt index 4fae805..c35af08 100644 --- a/zypp-core/CMakeLists.txt +++ b/zypp-core/CMakeLists.txt @@ -243,6 +243,7 @@ SET( zyppng_core_HEADERS ) SET( zyppng_io_private_HEADERS + zyppng/io/private/asyncdatasource_p.h zyppng/io/private/iobuffer_p.h zyppng/io/private/iodevice_p.h zyppng/io/private/socket_p.h @@ -305,11 +306,14 @@ SET( zyppng_pipelines_HEADERS SET( zyppng_rpc_HEADERS zyppng/rpc/rpc.h + zyppng/rpc/MessageStream + zyppng/rpc/messagestream.h zyppng/rpc/zerocopystreams.h ) SET( zyppng_rpc_SRCS zyppng/rpc/rpc.cc + zyppng/rpc/messagestream.cc zyppng/rpc/zerocopystreams.cc ) @@ -367,6 +371,9 @@ SET ( zypp_SRCS # Default loggroup for all files SET_LOGGROUP( "zypp-core" ${zypp_SRCS} ) +# System libraries +SET(UTIL_LIBRARY util) + ADD_LIBRARY( zypp-core STATIC ${zypp_SRCS} ${zypp_HEADERS} ) #we include generated headers, so we need to wait for zypp-protobuf to be ready add_dependencies( zypp-core zypp-protobuf ) @@ -376,3 +383,14 @@ target_link_libraries( zypp-core ${OPENSSL_LIBRARIES} ) target_link_libraries( zypp-core ${CRYPTO_LIBRARIES} ) target_link_libraries( zypp-core pthread ) target_link_libraries( zypp-core ${SIGCPP_LIBRARIES} ) +TARGET_LINK_LIBRARIES( zypp-core ${ZLIB_LIBRARY} ) +message("URIL LIB ${UTIL_LIBRARY}") +TARGET_LINK_LIBRARIES( zypp-core ${UTIL_LIBRARY} ) + +IF (ENABLE_ZSTD_COMPRESSION) + TARGET_LINK_LIBRARIES( zypp-core ${ZSTD_LIBRARY}) +ENDIF (ENABLE_ZSTD_COMPRESSION) + +IF (ENABLE_ZCHUNK_COMPRESSION) + TARGET_LINK_LIBRARIES( zypp-core ${ZCHUNK_LDFLAGS}) +ENDIF(ENABLE_ZCHUNK_COMPRESSION) diff --git a/zypp-core/Pathname.cc b/zypp-core/Pathname.cc index e6aa833..aa582fe 100644 --- a/zypp-core/Pathname.cc +++ b/zypp-core/Pathname.cc @@ -227,6 +227,42 @@ namespace zypp return base.substr( pos ); } + Pathname Pathname::realpath() const + { + std::string real; + if( !empty()) + { + #if __GNUC__ > 2 + /** GNU extension */ + char *ptr = ::realpath(_name.c_str(), NULL); + if( ptr != NULL) + { + real = ptr; + free( ptr); + } + else + /** the SUSv2 way */ + if( EINVAL == errno) + { + char buff[PATH_MAX + 2]; + memset(buff, '\0', sizeof(buff)); + if( ::realpath(_name.c_str(), buff) != NULL) + { + real = buff; + } + } + #else + char buff[PATH_MAX + 2]; + memset(buff, '\0', sizeof(buff)); + if( ::realpath(_name.c_str(), buff) != NULL) + { + real = buff; + } + #endif + } + return zypp::Pathname(real); + } + /////////////////////////////////////////////////////////////////// // // METHOD NAME : Pathname::assertprefix diff --git a/zypp-core/Pathname.h b/zypp-core/Pathname.h index 08924cb..ad3f51f 100644 --- a/zypp-core/Pathname.h +++ b/zypp-core/Pathname.h @@ -145,6 +145,9 @@ namespace zypp static Pathname relativename( const Pathname & name_r ) { return name_r.absolute() ? cat( ".", name_r ) : name_r; } + /** Returns this path as the absolute canonical pathname */ + Pathname realpath() const; + /** Return \c path_r prefixed with \c root_r, unless it is already prefixed. */ static Pathname assertprefix( const Pathname & root_r, const Pathname & path_r ); diff --git a/zypp-core/base/Exception.h b/zypp-core/base/Exception.h index 36faf5d..5d91075 100644 --- a/zypp-core/base/Exception.h +++ b/zypp-core/base/Exception.h @@ -401,7 +401,7 @@ namespace zypp std::exception_ptr do_ZYPP_EXCPT_PTR( const TExcpt & excpt_r, const CodeLocation & where_r ) { excpt_r.relocate( where_r ); - Exception::log( excpt_r, where_r, "THROW: " ); + Exception::log( excpt_r, where_r, "EXCPTR: " ); return std::make_exception_ptr( excpt_r ); } @@ -411,7 +411,7 @@ namespace zypp template> std::exception_ptr do_ZYPP_EXCPT_PTR( const TExcpt & excpt_r, const CodeLocation & where_r ) { - Exception::log( typeid(excpt_r).name(), where_r, "THROW: " ); + Exception::log( typeid(excpt_r).name(), where_r, "EXCPTR: " ); return std::make_exception_ptr( excpt_r ); } diff --git a/zypp-core/base/LogControl.cc b/zypp-core/base/LogControl.cc index e3b2a7d..1623629 100644 --- a/zypp-core/base/LogControl.cc +++ b/zypp-core/base/LogControl.cc @@ -187,9 +187,14 @@ namespace zypp auto writer = getLineWriter(); if ( writer ) { for ( auto &sock : clients ){ - while ( sock->canReadLine() ) { - auto br = sock->readLine(); - writer->writeOut( std::string( br.data(), br.size() - 1 ) ); + auto br = sock->readLine(); + while ( !br.empty() ) { + if ( br.back () == '\n' ) + writer->writeOut( std::string( br.data(), br.size() - 1 ) ); + else + writer->writeOut( std::string( br.data(), br.size() ) ); + + br = sock->readLine(); } } } @@ -555,6 +560,7 @@ namespace zypp /** \overload Setter */ void hideThreadName( bool onOff_r ) { _hideThreadName = onOff_r; } + /** \overload Static getter */ static bool instanceHideThreadName() { @@ -568,6 +574,20 @@ namespace zypp if ( impl ) impl->hideThreadName( onOff_r ); } + /** Hint for formatter wether we forward all logs to a parents log */ + static bool instanceLogToPPID( ) + { + auto impl = LogControlImpl::instance(); + return impl ? impl->_logToPPIDMode : false; + } + + /** \overload Static setter */ + static void instanceSetLogToPPID( bool onOff_r ) + { + auto impl = LogControlImpl::instance(); + if ( impl ) + impl->_logToPPIDMode = onOff_r; + } /** NULL _lineWriter indicates no loggin. */ void setLineWriter( const shared_ptr & writer_r ) @@ -599,6 +619,7 @@ namespace zypp LogClient _logClient; std::ostream _no_stream; bool _excessive; + bool _logToPPIDMode = false; ///< Hint for formatter to use the PPID and always show the thread name mutable TriBool _hideThreadName = indeterminate; ///< Hint for Formater whether to hide the thread name. shared_ptr _lineFormater; @@ -629,6 +650,10 @@ namespace zypp return ret; } + void putRawLine ( std::string &&line ) { + _logClient.pushMessage( std::move(line) ); + } + /** Format and write out a logline from Loglinebuf. */ void putStream( const std::string & group_r, LogLevel level_r, @@ -780,7 +805,9 @@ namespace zypp static char nohostname[] = "unknown"; std::string now( Date::now().form( "%Y-%m-%d %H:%M:%S" ) ); std::string ret; - if ( LogControlImpl::instanceHideThreadName() ) + + const bool logToPPID = LogControlImpl::instanceLogToPPID(); + if ( !logToPPID && LogControlImpl::instanceHideThreadName() ) ret = str::form( "%s <%d> %s(%d) [%s] %s(%s):%d %s", now.c_str(), level_r, ( gethostname( hostname, 1024 ) ? nohostname : hostname ), @@ -792,7 +819,7 @@ namespace zypp ret = str::form( "%s <%d> %s(%d) [%s] %s(%s):%d {T:%s} %s", now.c_str(), level_r, ( gethostname( hostname, 1024 ) ? nohostname : hostname ), - getpid(), + logToPPID ? getppid() : getpid(), group_r.c_str(), file_r, func_r, line_r, zyppng::ThreadData::current().name().c_str(), @@ -851,6 +878,11 @@ namespace zypp impl->setLineFormater( formater_r ); } + void LogControl::enableLogForwardingMode(bool enable) + { + LogControlImpl::instanceSetLogToPPID ( enable ); + } + void LogControl::logNothing() { auto impl = LogControlImpl::instance(); @@ -877,6 +909,11 @@ namespace zypp logger::logControlValidFlag () = 0; } + void LogControl::logRawLine ( std::string &&line ) + { + LogControlImpl::instance ()->putRawLine ( std::move(line) ); + } + /////////////////////////////////////////////////////////////////// // // LogControl::TmpExcessive diff --git a/zypp-core/base/LogControl.h b/zypp-core/base/LogControl.h index 15dd157..87409e1 100644 --- a/zypp-core/base/LogControl.h +++ b/zypp-core/base/LogControl.h @@ -129,6 +129,13 @@ namespace zypp */ void setLineFormater( const shared_ptr & formater_r ); + /*! + * Sets a special log mode that uses the ppid in the log output and always shows the + * thread name (if the default formatter is used). This is only used by internal zypp tools that directly want to log into the + * parents logfile. + */ + void enableLogForwardingMode ( bool enable = true ); + public: /** Set path for the logfile. * Permission for logfiles is set to 0640 unless an explicit mode_t @@ -155,6 +162,9 @@ namespace zypp */ static void notifyFork(); + /** will push a line to the logthread without formatting it */ + void logRawLine ( std::string &&line ); + public: /** Get the current LineWriter */ shared_ptr getLineWriter() const; diff --git a/zypp-core/base/Logger.h b/zypp-core/base/Logger.h index 99991c3..2868be7 100644 --- a/zypp-core/base/Logger.h +++ b/zypp-core/base/Logger.h @@ -109,15 +109,22 @@ namespace zypp #define L_INT(GROUP) ZYPP_BASE_LOGGER_LOG( GROUP, zypp::base::logger::E_INT ) #define L_USR(GROUP) ZYPP_BASE_LOGGER_LOG( GROUP, zypp::base::logger::E_USR ) + #define L_ENV_CONSTR_DEFINE_FUNC(ENV) \ - const char *empty_or_group_if_##ENV ( const char *group ) \ - { \ - static bool has_##ENV = (::getenv(#ENV) != NULL); \ - return has_##ENV ? group : nullptr; \ + namespace zypp::log { \ + bool has_env_constr_##ENV () \ + { \ + static bool has_##ENV = (::getenv(#ENV) != NULL); \ + return has_##ENV; \ + } \ + const char *empty_or_group_if_##ENV ( const char *group ) \ + { \ + return has_env_constr_##ENV() ? group : nullptr; \ + } \ } -#define L_ENV_CONSTR_FWD_DECLARE_FUNC(ENV) const char *empty_or_group_if_##ENV ( const char *group ) -#define L_ENV_CONSTR(ENV,GROUP,LEVEL) ZYPP_BASE_LOGGER_LOG( zypp::empty_or_group_if_##ENV( #GROUP ), LEVEL ) +#define L_ENV_CONSTR_FWD_DECLARE_FUNC(ENV) namespace zypp::log { bool has_env_constr_##ENV (); const char *empty_or_group_if_##ENV ( const char *group ); } +#define L_ENV_CONSTR(ENV,GROUP,LEVEL) ZYPP_BASE_LOGGER_LOG( zypp::log::empty_or_group_if_##ENV( #GROUP ), LEVEL ) #define L_BASEFILE ( *__FILE__ == '/' ? strrchr( __FILE__, '/' ) + 1 : __FILE__ ) diff --git a/zypp-core/zyppng/async/asyncop.h b/zypp-core/zyppng/async/asyncop.h index 0fcbd54..ae192e0 100644 --- a/zypp-core/zyppng/async/asyncop.h +++ b/zypp-core/zyppng/async/asyncop.h @@ -14,10 +14,10 @@ #ifndef ZYPPNG_ASYNC_ASYNCOP_H_INCLUDED #define ZYPPNG_ASYNC_ASYNCOP_H_INCLUDED - +#include #include #include -#include +#include #include namespace zyppng { @@ -35,16 +35,84 @@ namespace zyppng { virtual ~AsyncOpNotReadyException(); }; - AsyncOpNotReadyException::~AsyncOpNotReadyException() + class CancelNotImplementedException : public zypp::Exception + { + public: + CancelNotImplementedException() + : Exception ("AsyncOp does not support cancelling the operation") + {} + virtual ~CancelNotImplementedException(); + + + }; + + inline AsyncOpNotReadyException::~AsyncOpNotReadyException() { } + + inline CancelNotImplementedException::~CancelNotImplementedException() + { } + + + struct AsyncOpBase : public Base { + + /*! + * Should return true if the async op supports cancelling, otherwise false + */ + virtual bool canCancel () { + return false; + } + + /*! + * Explicitely cancels the async operation if it supports that, + * otherwise throws a \ref CancelNotImplementedException + * + * Generally every AsyncOp HAS TO support cancelling by releasing the last reference + * to it, however in some special cases we need to be able to cancel programatically + * from the backends and notify the frontends by returning a error from a AsyncOp. + */ + virtual void cancel () { + throw CancelNotImplementedException(); + } + + /*! + * Signal that is emitted once the AsyncOp is started + */ + SignalProxy sigStarted () { + return _sigStarted; + } + + /*! + * Signal that might be emitted during operation of the AsyncOp, + * not every AsyncOp will use this + */ + SignalProxy sigProgress () { + return _sigProgress; + } + + /*! + * Signal that is emitted once the AsyncOp is ready and no + * callback was registered with \ref onReady + */ + SignalProxy sigReady () { + return _sigReady; + } + + protected: + Signal _sigStarted; + Signal _sigReady; + Signal _sigProgress; + }; + + + /*! *\class AsyncOp * The \a AsyncOp template class is the basic building block for the asynchronous pipelines. * Its the base for all async callbacks as well as the async result type. That basically means * every pipeline is just a AsyncOp that contains all previous operations that were defined in the pipeline. * - * When implementing a async operation it is required to add a operator() to the class taking the + * When implementing a async operation that is to be used in a pipeline it is required to add a operator() to the class taking the * input parameter. After the operation is finished the implementation must call setReady(). Calling * setReady() must be treated like calling return in a normal function and not execute anymore code on the * AsyncOp instance afterwards, since the next operation in the pipeline is allowed to free the previous operation @@ -54,12 +122,14 @@ namespace zyppng { * is cached internally and can be retrieved with \sa get(). * * A async operation can be cancelled by releasing the result object ( the resulting combinatory object ), this will - * destroy all previous operations that are either running or pending as well. + * destroy all previous operations that are either running or pending as well. The destructors MUST NOT call setReady() but + * only release currently running operations. */ template - struct AsyncOp { + struct AsyncOp : public AsyncOpBase { using value_type = Result; + using Ptr = std::shared_ptr>; AsyncOp () = default; @@ -77,8 +147,13 @@ namespace zyppng { * storing it. */ void setReady ( value_type && val ) { - if ( _readyCb ) + if ( _readyCb ) { + // use a weak reference to know if `this` was deleted by the callback() + auto weak = weak_from_this(); _readyCb( std::move( val ) ); + if ( !weak.expired() ) + _readyCb = {}; + } else { //we need to cache the value because no callback is available _maybeValue = std::move(val); _sigReady.emit(); @@ -91,7 +166,7 @@ namespace zyppng { * \note This can only be used when no callback is registered. */ bool isReady () const { - return _maybeValue.is_initialized(); + return _maybeValue.has_value(); } /*! @@ -99,14 +174,27 @@ namespace zyppng { * when the object gets into ready state. In case the * object is in ready state when registering the callback * it is called right away. + * \note this will disable the emitting of sigReady() so it should + * be only used by the pipeline implementation + * + * \note the callback is removed from the AsyncOp after it was executed + * when the AsyncOp becomes ready */ template< typename Fun > void onReady ( Fun cb ) { this->_readyCb = std::forward(cb); if ( isReady() ) { - _readyCb( std::move( _maybeValue.get()) ); - _maybeValue = boost::optional(); + // use a weak reference to know if `this` was deleted by the callback() + auto weak = weak_from_this(); + + _readyCb( std::move( _maybeValue.value()) ); + _maybeValue.reset(); + + // reset the callback, it might be a lambda with captured ressources + // that need to be released after returning from the func + if ( !weak.expired() ) + _readyCb = {}; } } @@ -117,28 +205,28 @@ namespace zyppng { value_type &get (){ if ( !isReady() ) ZYPP_THROW(AsyncOpNotReadyException()); - return _maybeValue.get(); - } - - /*! - * Signal that is emitted once the AsyncOp is ready and no - * callback was registered with \ref onReady - */ - SignalProxy sigReady () { - return _sigReady; + return _maybeValue.value(); } private: - Signal _sigReady; std::function _readyCb; - boost::optional _maybeValue; + std::optional _maybeValue; }; template - using AsyncOpPtr = std::unique_ptr>; + using AsyncOpRef = std::shared_ptr>; namespace detail { + //A async result that is ready right away + template + struct ReadyResult : public zyppng::AsyncOp< T > + { + ReadyResult( T &&val ) { + this->setReady( std::move(val) ); + } + }; + #if 0 template using has_value_type_t = typename T::value_type; @@ -171,6 +259,11 @@ namespace zyppng { } + template + AsyncOpRef makeReadyResult ( T && result ) { + return std::make_shared>( std::move(result) ); + } + } diff --git a/zypp-core/zyppng/base/base.h b/zypp-core/zyppng/base/base.h index 3bdad07..3e8a014 100644 --- a/zypp-core/zyppng/base/base.h +++ b/zypp-core/zyppng/base/base.h @@ -136,7 +136,7 @@ namespace zyppng { } /*! - * Preferred way to connect a signal to a slow, this will automatically take care of tracking the target object in the connection + * Preferred way to connect a signal to a slot, this will automatically take care of tracking the target object in the connection */ template< typename SenderFunc, typename ReceiverFunc > static auto connect ( typename internal::MemberFunction::ClassType &s, SenderFunc &&sFun, typename internal::MemberFunction::ClassType &recv, ReceiverFunc &&rFunc ) { diff --git a/zypp-core/zyppng/base/eventdispatcher_glib.cc b/zypp-core/zyppng/base/eventdispatcher_glib.cc index cf2118c..5de8a8c 100644 --- a/zypp-core/zyppng/base/eventdispatcher_glib.cc +++ b/zypp-core/zyppng/base/eventdispatcher_glib.cc @@ -296,6 +296,7 @@ EventDispatcherPrivate::~EventDispatcherPrivate() GAbstractEventSource::destruct( src ); }); _runningTimers.clear(); + _eventSources.clear(); if ( _idleSource ) { g_source_destroy( _idleSource ); @@ -516,7 +517,8 @@ bool EventDispatcher::waitForFdEvent( const int fd, int events , int &revents , if ( errno == EINTR ) continue; - break; + ERR << "g_poll error: " << strerror(errno) << std::endl; + return false; } case 1: eventTriggered = true; diff --git a/zypp-core/zyppng/base/linuxhelpers.cc b/zypp-core/zyppng/base/linuxhelpers.cc index ad5800a..5d92c3d 100644 --- a/zypp-core/zyppng/base/linuxhelpers.cc +++ b/zypp-core/zyppng/base/linuxhelpers.cc @@ -51,11 +51,11 @@ namespace zyppng { } } - int bytesAvailableOnFD(int fd) + int64_t bytesAvailableOnFD( int fd ) { int value; if ( ioctl( fd, FIONREAD, &value) >= 0 ) - return value; + return int64_t(value); return 0; } diff --git a/zypp-core/zyppng/base/private/linuxhelpers_p.h b/zypp-core/zyppng/base/private/linuxhelpers_p.h index 030e6c6..f1dfc8c 100644 --- a/zypp-core/zyppng/base/private/linuxhelpers_p.h +++ b/zypp-core/zyppng/base/private/linuxhelpers_p.h @@ -37,7 +37,7 @@ namespace zyppng { * Tries to use the FIONREAD ioctl to detect how many bytes are available on a file descriptor, * this can fail and return 0 so just use it as a indicator on how many bytes are pending */ - int bytesAvailableOnFD ( int fd ); + int64_t bytesAvailableOnFD( int fd ); /*! * Small helper struct around creating a Unix pipe to ensure RAII with pipes diff --git a/zypp-core/zyppng/base/zyppglobal.h b/zypp-core/zyppng/base/zyppglobal.h index a6df03d..64b40c2 100644 --- a/zypp-core/zyppng/base/zyppglobal.h +++ b/zypp-core/zyppng/base/zyppglobal.h @@ -104,4 +104,23 @@ template inline auto zyppGetPtrHelper(Ptr &ptr) -> decltype(ptr.o #define Z_D() auto const d = d_func() #define Z_Z() auto const z = z_func() +/* + * Helper Macro to forward declare types and ref types + */ +#define ZYPP_FWD_DECL_TYPE_WITH_REFS(T) \ + class T; \ + using T##Ref = std::shared_ptr; \ + using T##WeakRef = std::weak_ptr + +//@TODO enable for c++20 +#if 0 +#define ZYPP_FWD_DECL_TEMPL_TYPE_WITH_REFS(T, TArg1, ...) \ + template< typename TArg1 __VA_OPT__(, typename) __VA_ARGS__ > \ + class T; \ + template< typename TArg1 __VA_OPT__(, typename) __VA_ARGS__ > \ + using T##Ref = std::shared_ptr>; \ + template< typename TArg1 __VA_OPT__(, typename) __VA_ARGS__ > \ + using T##WeakRef = std::weak_ptr> +#endif + #endif diff --git a/zypp-core/zyppng/io/abstractspawnengine.cc b/zypp-core/zyppng/io/abstractspawnengine.cc index c205c5b..be7d071 100644 --- a/zypp-core/zyppng/io/abstractspawnengine.cc +++ b/zypp-core/zyppng/io/abstractspawnengine.cc @@ -96,6 +96,12 @@ namespace zyppng { _mapFds.push_back( fd ); } + void AbstractSpawnEngine::notifyExited(int status) + { + _exitStatus = checkStatus( status ); + _pid = -1; + } + bool AbstractSpawnEngine::dieWithParent() const { return _dieWithParent; diff --git a/zypp-core/zyppng/io/asyncdatasource.cpp b/zypp-core/zyppng/io/asyncdatasource.cpp index 23de01b..b1c1c1e 100644 --- a/zypp-core/zyppng/io/asyncdatasource.cpp +++ b/zypp-core/zyppng/io/asyncdatasource.cpp @@ -1,84 +1,75 @@ -#include "asyncdatasource.h" +#include "private/asyncdatasource_p.h" #include #include #include -#include -#include -#include #include namespace zyppng { - class AsyncDataSourcePrivate : public IODevicePrivate { - ZYPP_DECLARE_PUBLIC(AsyncDataSource); - public: - AsyncDataSourcePrivate ( AsyncDataSource &pub ) : IODevicePrivate(pub) {} - SocketNotifier::Ptr _readNotifier; - SocketNotifier::Ptr _writeNotifier; - IOBuffer _writeBuffer; - int _readFd = -1; - int _writeFd = -1; - - void notifierActivated (const SocketNotifier ¬ify, int evTypes ); - void readyRead ( ); - void readyWrite ( ); - - void closeWriteChannel ( AsyncDataSource::ChannelCloseReason reason ); - void closeReadChannel ( AsyncDataSource::ChannelCloseReason reason ); - - Signal _sigWriteFdClosed; - Signal _sigReadFdClosed; - Signal< void (std::size_t)> _sigBytesWritten; - }; - void AsyncDataSourcePrivate::notifierActivated( const SocketNotifier ¬ify, int evTypes ) { - if ( _readNotifier.get() == ¬ify ) { - readyRead(); - } else if ( _writeNotifier.get() == ¬ify ) { + if ( _writeNotifier.get() == ¬ify ) { if ( evTypes & SocketNotifier::Error ) { DBG << "Closing due to error when polling" << std::endl; closeWriteChannel( AsyncDataSource::RemoteClose ); return; } readyWrite(); + } else { + + auto dev = std::find_if( _readFds.begin(), _readFds.end(), + [ ¬ify ]( const auto &dev ){ return ( dev._readNotifier.get() == ¬ify ); } ); + + if ( dev == _readFds.end() ) { + return; + } + + readyRead( std::distance( _readFds.begin(), dev ) ); } } - void AsyncDataSourcePrivate::readyRead() + void AsyncDataSourcePrivate::readyRead( uint channel ) { - auto bytesToRead = z_func()->rawBytesAvailable(); + auto bytesToRead = z_func()->rawBytesAvailable( channel ); if ( bytesToRead == 0 ) { // make sure to check if bytes are available even if the ioctl call returns something different bytesToRead = 4096; } + auto &_readBuf = _readChannels[channel]; char *buf = _readBuf.reserve( bytesToRead ); - const auto bytesRead = z_func()->readData( buf, bytesToRead ); + const auto bytesRead = z_func()->readData( channel, buf, bytesToRead ); - if ( bytesRead < 0 ) { + if ( bytesRead <= 0 ) { _readBuf.chop( bytesToRead ); + + switch( bytesRead ) { + // remote close , close the read channel + case 0: { + closeReadChannel( channel, AsyncDataSource::RemoteClose ); + break; + } + // no data is available , just try again later + case -2: break; + // anything else + default: + case -1: { + closeReadChannel( channel, AsyncDataSource::InternalError ); + break; + } + } return; } - if ( bytesToRead > (size_t)bytesRead ) + if ( bytesToRead > bytesRead ) _readBuf.chop( bytesToRead-bytesRead ); - if ( bytesRead > 0 ) { + if ( channel == _currentReadChannel ) _readyRead.emit(); - return; - } - //handle remote close - else if ( bytesRead == 0 && errno != EAGAIN && errno != EWOULDBLOCK ) { - closeReadChannel( AsyncDataSource::RemoteClose ); - } - if ( errno == EAGAIN || errno == EWOULDBLOCK ) - return; - - //setError( Socket::InternalError, strerr_cxx() ); - closeReadChannel( AsyncDataSource::InternalError ); + _channelReadyRead.emit( channel ); + return; } void AsyncDataSourcePrivate::readyWrite() @@ -114,6 +105,9 @@ namespace zyppng { } _writeBuffer.discard( written ); _sigBytesWritten.emit( written ); + + if ( _writeBuffer.size() == 0 ) + _sigAllBytesWritten.emit(); } void AsyncDataSourcePrivate::closeWriteChannel( AsyncDataSource::ChannelCloseReason reason ) @@ -127,14 +121,15 @@ namespace zyppng { _sigWriteFdClosed.emit( reason ); } - void AsyncDataSourcePrivate::closeReadChannel( AsyncDataSource::ChannelCloseReason reason ) + void AsyncDataSourcePrivate::closeReadChannel( uint channel, AsyncDataSource::ChannelCloseReason reason ) { + auto &readFd = _readFds[channel]; // we do not clear the read buffer so code has the opportunity to read whats left in there - bool sig = _readFd >= 0; - _readNotifier.reset(); - _readFd = -1; + bool sig = readFd._readFd >= 0; + readFd._readNotifier.reset(); + readFd._readFd = -1; if ( sig ) - _sigReadFdClosed.emit( reason ); + _sigReadFdClosed.emit( channel, reason ); } ZYPP_IMPL_PRIVATE(AsyncDataSource) @@ -142,34 +137,68 @@ namespace zyppng { AsyncDataSource::AsyncDataSource() : IODevice( *( new AsyncDataSourcePrivate(*this) ) ) { } + AsyncDataSource::AsyncDataSource( AsyncDataSourcePrivate &d ) + : IODevice(d) + {} + AsyncDataSource::Ptr AsyncDataSource::create() { return std::shared_ptr( new AsyncDataSource ); } - bool AsyncDataSource::open( int readFd, int writeFd ) + + bool AsyncDataSource::openFds ( std::vector readFds, int writeFd ) { Z_D(); - close(); + + if ( d->_mode != IODevice::Closed ) + return false; + IODevice::OpenMode mode; - if ( readFd >= 0 ) { - mode |= IODevice::ReadOnly; - d->_readFd = readFd; - zypp::io::setFDBlocking( readFd, false ); - d->_readNotifier = SocketNotifier::create( readFd, SocketNotifier::Read | AbstractEventSource::Error, true ); - d->_readNotifier->connect( &SocketNotifier::sigActivated, *d, &AsyncDataSourcePrivate::notifierActivated ); + + bool error = false; + for ( const auto readFd : readFds ) { + if ( readFd >= 0 ) { + mode |= IODevice::ReadOnly; + d->_readFds.push_back( { + readFd, + SocketNotifier::create( readFd, SocketNotifier::Read | AbstractEventSource::Error, true ) + }); + if ( zypp::io::setFDBlocking( readFd, false ) == zypp::io::BlockingMode::FailedToSetMode ) { + ERR << "Failed to set read FD to non blocking" << std::endl; + error = true; + break; + } + d->_readFds.back()._readNotifier->connect( &SocketNotifier::sigActivated, *d, &AsyncDataSourcePrivate::notifierActivated ); + } } - if ( writeFd >= 0 ) { + + if ( writeFd >= 0 && !error ) { mode |= IODevice::WriteOnly; - d->_writeFd = writeFd; - zypp::io::setFDBlocking( writeFd, false ); - d->_writeNotifier = SocketNotifier::create( writeFd, SocketNotifier::Write | AbstractEventSource::Error, false ); - d->_writeNotifier->connect( &SocketNotifier::sigActivated, *d, &AsyncDataSourcePrivate::notifierActivated ); + if ( zypp::io::setFDBlocking( writeFd, false ) == zypp::io::BlockingMode::FailedToSetMode ) { + ERR << "Failed to set write FD to non blocking" << std::endl; + error = true; + } else { + d->_writeFd = writeFd; + d->_writeNotifier = SocketNotifier::create( writeFd, SocketNotifier::Write | AbstractEventSource::Error, false ); + d->_writeNotifier->connect( &SocketNotifier::sigActivated, *d, &AsyncDataSourcePrivate::notifierActivated ); + } + } + + if( error || !IODevice::open( mode ) ) { + d->_mode = IODevice::Closed; + d->_readFds.clear(); + d->_writeNotifier.reset(); + d->_writeFd = -1; + return false; } - return IODevice::open( mode ); + + // make sure we have enough read buffers + setReadChannelCount( d->_readFds.size() ); + return true; } - off_t zyppng::AsyncDataSource::writeData( const char *data, off_t count ) + int64_t zyppng::AsyncDataSource::writeData( const char *data, int64_t count ) { Z_D(); if ( count > 0 ) { @@ -180,71 +209,116 @@ namespace zyppng { return count; } - off_t zyppng::AsyncDataSource::readData( char *buffer, off_t bufsize ) + int64_t zyppng::AsyncDataSource::readData( uint channel, char *buffer, int64_t bufsize ) { Z_D(); - const auto read = eintrSafeCall( ::read, d->_readFd, buffer, bufsize ); - - // special case for remote close - if ( read == 0 ) { - d->closeReadChannel( RemoteClose ); - return -1; - } else if ( read < 0 ) { + if ( channel >= d->_readFds.size() ) { + ERR << constants::outOfRangeErrMsg << std::endl; + throw std::logic_error( constants::outOfRangeErrMsg.data() ); + } + const auto read = eintrSafeCall( ::read, d->_readFds[channel]._readFd, buffer, bufsize ); + if ( read < 0 ) { switch ( errno ) { -#if EAGAIN != EWOULDBLOCK - case EWOULDBLOCK: -#endif + #if EAGAIN != EWOULDBLOCK + case EWOULDBLOCK: + #endif case EAGAIN: { - return 0; - } - default: { - d->closeReadChannel( InternalError ); - return -1; + return -2; } + default: + break; } } return read; } - size_t AsyncDataSource::rawBytesAvailable() const + int64_t AsyncDataSource::rawBytesAvailable( uint channel ) const { + Z_D(); + + if ( channel >= d->_readFds.size() ) { + ERR << constants::outOfRangeErrMsg << std::endl; + throw std::logic_error( constants::outOfRangeErrMsg.data() ); + } + if ( isOpen() && canRead() ) - return zyppng::bytesAvailableOnFD( d_func()->_readFd ); + return zyppng::bytesAvailableOnFD( d->_readFds[channel]._readFd ); return 0; } + void AsyncDataSource::readChannelChanged ( uint channel ) + { + Z_D(); + if ( channel >= d->_readFds.size() ) { + ERR << constants::outOfRangeErrMsg << std::endl; + throw std::logic_error( constants::outOfRangeErrMsg.data() ); + } + } + void zyppng::AsyncDataSource::close() { Z_D(); - d->_readNotifier.reset(); + for( uint i = 0; i < d->_readFds.size(); ++i ) { + auto &readChan = d->_readFds[i]; + readChan._readNotifier.reset(); + if ( readChan._readFd >= 0) + d->_sigReadFdClosed.emit( i, UserRequest ); + } + d->_readFds.clear(); + d->_writeNotifier.reset(); d->_writeBuffer.clear(); - - if ( d->_readFd >= 0) - d->_sigReadFdClosed.emit( UserRequest ); - if ( d->_writeFd >= 0 ) + if ( d->_writeFd >= 0 ) { + d->_writeFd = -1; d->_sigWriteFdClosed.emit( UserRequest ); + } IODevice::close(); } - bool AsyncDataSource::waitForReadyRead(int timeout) + void AsyncDataSource::closeWriteChannel() + { + Z_D(); + + // if we are open writeOnly, simply call close(); + if ( !canRead() ) { + close(); + return; + } + + d->_mode = ReadOnly; + d->_writeNotifier.reset(); + d->_writeBuffer.clear(); + + if ( d->_writeFd >= 0 ) { + d->_writeFd = -1; + d->_sigWriteFdClosed.emit( UserRequest ); + } + } + + bool AsyncDataSource::waitForReadyRead( uint channel, int timeout ) { Z_D(); if ( !canRead() ) return false; + if ( channel >= d->_readFds.size() ) { + ERR << constants::outOfRangeErrMsg << std::endl; + throw std::logic_error( constants::outOfRangeErrMsg.data() ); + } + bool gotRR = false; - auto rrConn = AutoDisconnect( d->_readyRead.connect([&](){ - gotRR = true; + auto rrConn = AutoDisconnect( d->_channelReadyRead.connect([&]( uint activated ){ + gotRR = ( channel == activated ); }) ); // we can only wait if we are open for reading and still have a valid fd - while ( readFdOpen() && canRead() && !gotRR ) { + auto &channelRef = d->_readFds[ channel ]; + while ( readFdOpen(channel) && canRead() && !gotRR ) { int rEvents = 0; - if ( EventDispatcher::waitForFdEvent( d->_readFd, AbstractEventSource::Read | AbstractEventSource::Error , rEvents, timeout ) ) { + if ( EventDispatcher::waitForFdEvent( channelRef._readFd, AbstractEventSource::Read | AbstractEventSource::Error , rEvents, timeout ) ) { //simulate signal from read notifier - d->notifierActivated( *d->_readNotifier, rEvents ); + d->notifierActivated( *channelRef._readNotifier, rEvents ); } else { //timeout return false; @@ -253,23 +327,52 @@ namespace zyppng { return gotRR; } + void AsyncDataSource::flush () + { + Z_D(); + if ( !canWrite() ) + return; + + int timeout = -1; + while ( canWrite() && d->_writeBuffer.frontSize() ) { + int rEvents = 0; + if ( EventDispatcher::waitForFdEvent( d->_writeFd, AbstractEventSource::Write | AbstractEventSource::Error , rEvents, timeout ) ) { + //simulate signal from write notifier + d->readyWrite(); + } else { + //timeout + return; + } + } + } + SignalProxy AsyncDataSource::sigWriteFdClosed() { return d_func()->_sigWriteFdClosed; } - SignalProxy AsyncDataSource::sigReadFdClosed() + SignalProxy AsyncDataSource::sigReadFdClosed() { return d_func()->_sigReadFdClosed; } - SignalProxy AsyncDataSource::sigBytesWritten() + bool AsyncDataSource::readFdOpen() const { - return d_func()->_sigBytesWritten; + Z_D(); + if ( !d->_readChannels.size() ) + return false; + return readFdOpen( d_func()->_currentReadChannel ); } - bool AsyncDataSource::readFdOpen() const + bool AsyncDataSource::readFdOpen(uint channel) const { - return ( d_func()->_readNotifier && d_func()->_readFd >= 0 ); + Z_D(); + if ( channel >= d->_readFds.size() ) { + ERR << constants::outOfRangeErrMsg << std::endl; + throw std::logic_error( constants::outOfRangeErrMsg.data() ); + } + auto &channelRef = d->_readFds[ channel ]; + return ( channelRef._readNotifier && channelRef._readFd >= 0 ); } + } diff --git a/zypp-core/zyppng/io/asyncdatasource.h b/zypp-core/zyppng/io/asyncdatasource.h index 73bbc53..f1cceca 100644 --- a/zypp-core/zyppng/io/asyncdatasource.h +++ b/zypp-core/zyppng/io/asyncdatasource.h @@ -24,15 +24,28 @@ namespace zyppng { using WeakPtr = std::weak_ptr; static Ptr create (); - bool open ( int readFd = -1, int writeFd = -1 ); + bool openFds ( std::vector readFds, int writeFd = -1 ); void close () override; + using IODevice::waitForReadyRead; + + virtual void closeWriteChannel (); + /*! - * Blocks the current event loop to wait until there are bytes available to read from the device + * Blocks the current event loop to wait until all bytes currently in the buffer have been written to + * the write fd. * * \note do not use until there is a very good reason, like event processing should not continue until readyRead was sent */ - bool waitForReadyRead(int timeout); + bool waitForReadyRead(uint channel, int timeout) override; + + /*! + * Blocks the current event loop to wait until all bytes currently in the buffer have been written to + * the write fd. + * + * \note do not use until there is a very good reason, like event processing should not continue until bytesWritten was sent + */ + void flush (); /*! * Signal is emitted always when the write channel is closed. @@ -46,27 +59,29 @@ namespace zyppng { * when the write side of a pipe is closed. All data still residing in the read buffer * can still be read. */ - SignalProxy sigReadFdClosed(); + SignalProxy sigReadFdClosed(); /*! - * Signal is emitted every time bytes have been written to the underlying fd. - * This can be used to track how much data was actually sent. + * Returns true as long as the default read channel was not closed ( e.g. sigReadFdClosed was emitted ) */ - SignalProxy< void (std::size_t)> sigBytesWritten (); + bool readFdOpen () const; /*! - * Returns true as long as the read channel was not closed ( e.g. sigReadFdClosed was emitted ) + * Returns true as long as the given read channel was not closed ( e.g. sigReadFdClosed was emitted ) */ - bool readFdOpen () const; + bool readFdOpen ( uint channel ) const; protected: AsyncDataSource ( ); - off_t writeData(const char *data, off_t count) override; - off_t readData(char *buffer, off_t bufsize) override; - size_t rawBytesAvailable() const override; + AsyncDataSource( AsyncDataSourcePrivate &d ); + int64_t writeData(const char *data, int64_t count) override; private: using IODevice::open; + + int64_t readData( uint channel, char *buffer, int64_t bufsize ) override; + int64_t rawBytesAvailable( uint channel ) const override; + void readChannelChanged ( uint channel ) override; }; } diff --git a/zypp-core/zyppng/io/iobuffer.cc b/zypp-core/zyppng/io/iobuffer.cc index 5e03bb9..d643b24 100644 --- a/zypp-core/zyppng/io/iobuffer.cc +++ b/zypp-core/zyppng/io/iobuffer.cc @@ -1,5 +1,6 @@ #include "private/iobuffer_p.h" #include +#include namespace zyppng { @@ -7,11 +8,12 @@ namespace zyppng { DefChunkSize = 4096 }; - IOBuffer::IOBuffer(unsigned chunkSize) : _defaultChunkSize ( chunkSize == 0 ? DefChunkSize : chunkSize ) + IOBuffer::IOBuffer( int64_t chunkSize ) : _defaultChunkSize ( chunkSize == 0 ? DefChunkSize : chunkSize ) { } - char *IOBuffer::reserve( size_t bytes ) + char *IOBuffer::reserve( int64_t bytes ) { + assert( bytes > 0 && size_t(bytes) < ByteArray::maxSize() ); // do we need a new chunk? if ( _chunks.size() ) { auto &back = _chunks.back(); @@ -25,7 +27,7 @@ namespace zyppng { // not enough space ready allocate a new one _chunks.push_back( Chunk{} ); auto &back = _chunks.back(); - back._buffer.insert( back._buffer.end(), std::max( _defaultChunkSize, bytes ), '\0' ); + back._buffer.insert( back._buffer.end(), std::max( _defaultChunkSize, bytes ), '\0' ); back.tail += bytes; return back.data(); } @@ -38,7 +40,7 @@ namespace zyppng { return _chunks.front().data(); } - size_t IOBuffer::frontSize() const + int64_t IOBuffer::frontSize() const { if ( _chunks.empty() ) return 0; @@ -50,15 +52,15 @@ namespace zyppng { _chunks.clear(); } - size_t IOBuffer::discard( size_t bytes ) + int64_t IOBuffer::discard( int64_t bytes ) { - const size_t bytesToDiscard = std::min(bytes, size()); + const int64_t bytesToDiscard = std::min(bytes, size()); if ( bytesToDiscard == size() ) { clear(); return bytesToDiscard; } - size_t discardedSoFar = 0; + int64_t discardedSoFar = 0; // since the chunks might not be used completely we need to iterate over them // counting how much used bytes we actually discard until we hit the requested amount @@ -82,7 +84,7 @@ namespace zyppng { /*! * Removes bytes from the end of the buffer */ - void IOBuffer::chop( size_t bytes ) + void IOBuffer::chop( int64_t bytes ) { if ( bytes == 0 ) return; @@ -93,7 +95,7 @@ namespace zyppng { return; } - size_t choppedSoFar = 0; + int64_t choppedSoFar = 0; while ( choppedSoFar < bytes && _chunks.size() ) { auto bytesStillToChop = bytes - choppedSoFar; auto &chunk = _chunks.back(); @@ -108,8 +110,13 @@ namespace zyppng { } } - void IOBuffer::append(const char *data, size_t count) + void IOBuffer::append(const char *data, int64_t count) { + if ( count <= 0 ) + return; + + assert( count > 0 && size_t(count) < ByteArray::maxSize() ); + char *buf = reserve( count ); if ( count == 1 ) *buf = *data; @@ -123,7 +130,7 @@ namespace zyppng { append( data.data(), data.size() ); } - size_t IOBuffer::read( char *buffer, size_t max ) + int64_t IOBuffer::read( char *buffer, int64_t max ) { const size_t bytesToRead = std::min( size(), max ); size_t readSoFar = 0; @@ -144,9 +151,9 @@ namespace zyppng { return readSoFar; } - size_t IOBuffer::size() const + int64_t IOBuffer::size() const { - size_t s = 0; + int64_t s = 0; for ( const auto &c : _chunks ) s+= c.len(); return s; @@ -157,14 +164,14 @@ namespace zyppng { return _chunks.size(); } - int64_t IOBuffer::indexOf( const char c, size_t maxCount, size_t pos ) const + int64_t IOBuffer::indexOf(const char c, int64_t maxCount, int64_t pos ) const { if ( maxCount == 0 ) return -1; maxCount = std::min( maxCount, size() ); - size_t scannedSoFar = 0; + int64_t scannedSoFar = 0; for ( const auto &chunk : _chunks ) { //as long as pos is still after the current chunk just increase the count @@ -195,8 +202,10 @@ namespace zyppng { return -1; } - ByteArray IOBuffer::readLine(const size_t max) + ByteArray IOBuffer::readLine( const int64_t max ) { + assert( ( max >= 2 || max == 0 ) && size_t(max) <= ByteArray::maxSize() ); + const auto idx = indexOf( '\n', max == 0 ? size() : max ); if ( idx == -1 ) return {}; @@ -206,6 +215,16 @@ namespace zyppng { return b; } + int64_t IOBuffer::readLine( char *buffer, int64_t max ) + { + assert( buffer != nullptr && max > 1 ); + const auto maxRead = max - 1; + const auto idx = indexOf( '\n', maxRead ); + const auto bytesRead = read( buffer, idx == -1 ? maxRead : idx + 1 ); + buffer[bytesRead] = '\0'; + return bytesRead; + } + bool IOBuffer::canReadLine() const { return indexOf('\n') >= 0; diff --git a/zypp-core/zyppng/io/iodevice.cc b/zypp-core/zyppng/io/iodevice.cc index 1cbeba5..a49d851 100644 --- a/zypp-core/zyppng/io/iodevice.cc +++ b/zypp-core/zyppng/io/iodevice.cc @@ -17,16 +17,57 @@ namespace zyppng { { Z_D(); - d->_readBuf.clear(); d->_mode = mode; + d->_readChannels.clear(); + if ( canRead() ) { + d->_readChannels.push_back( IOBuffer( d->_readBufChunkSize ) ); + setReadChannel( 0 ); + } return true; } void IODevice::close() { - d_func()->_mode = IODevice::Closed; - d_func()->_readBuf.clear(); + Z_D(); + d->_mode = IODevice::Closed; + d->_readChannels.clear(); + } + + void IODevice::setReadChannelCount ( uint channels ) { + Z_D(); + if ( canRead() ) { + d->_readChannels.resize( channels ); + } + } + + void IODevice::setReadChannel ( uint channel ) + { + Z_D(); + if ( !canRead() ) + return; + if ( channel >= d->_readChannels.size() ) { + ERR << constants::outOfRangeErrMsg << std::endl; + throw std::out_of_range( constants::outOfRangeErrMsg.data() ); + } + d->_currentReadChannel = channel; + readChannelChanged( channel ); + } + + uint IODevice::currentReadChannel () const + { + Z_D(); + if ( !canRead() ) + return 0; + return d->_currentReadChannel; + } + + int IODevice::readChannelCount () const + { + Z_D(); + if ( !canRead() ) + return 0; + return d->_readChannels.size(); } bool IODevice::canRead() const @@ -50,76 +91,254 @@ namespace zyppng { if ( !canRead() ) return false; - return d->_readBuf.canReadLine(); + return canReadLine( d->_currentReadChannel ); } - size_t IODevice::bytesAvailable() const + int64_t IODevice::bytesAvailable() const + { + Z_D(); + return bytesAvailable( d->_currentReadChannel ); + } + + ByteArray IODevice::readAll() + { + Z_D(); + return readAll( d->_currentReadChannel ); + } + + ByteArray IODevice::read( int64_t maxSize ) + { + if ( !canRead() || maxSize <= 0 ) + return {}; + return read( d_func()->_currentReadChannel, maxSize ); + } + + int64_t IODevice::read(char *buf, int64_t maxSize ) { Z_D(); if ( !canRead() ) - return 0; + return -1; + return read( d->_currentReadChannel, buf, maxSize ); + } - return d->_readBuf.size() + rawBytesAvailable(); + ByteArray IODevice::readLine( const int64_t maxSize ) + { + if ( !canRead() ) + return {}; + + return channelReadLine( d_func()->_currentReadChannel, maxSize ); } - ByteArray IODevice::readAll() + ByteArray IODevice::readAll( uint channel ) { - return read( bytesAvailable() ); + return read( channel, bytesAvailable( channel ) ); } - ByteArray IODevice::read( size_t maxSize ) + ByteArray IODevice::read( uint channel, int64_t maxSize ) { - if ( !canRead() || maxSize == 0 ) + if ( !canRead() || maxSize <= 0 ) return {}; ByteArray res( maxSize, '\0' ); - const auto r = read( res.data(), maxSize ); + const auto r = read( channel, res.data(), maxSize ); res.resize( r ); return res; } - size_t IODevice::read( char *buf, size_t maxSize ) + int64_t IODevice::read( uint channel, char *buf, int64_t maxSize ) { Z_D(); - if ( !canRead() ) - return 0; + if ( !canRead() || maxSize < 0 ) + return -1; - size_t readSoFar = d->_readBuf.read( buf, maxSize ); + if ( channel >= d->_readChannels.size() ) { + ERR << constants::outOfRangeErrMsg << std::endl; + throw std::out_of_range( constants::outOfRangeErrMsg.data() ); + } + + int64_t readSoFar = d->_readChannels[ channel ].read( buf, maxSize ); // try to read more from the device if ( readSoFar < maxSize ) { - size_t readFromDev = readData( buf+readSoFar, maxSize - readSoFar ); + int64_t readFromDev = readData( channel, buf+readSoFar, maxSize - readSoFar ); if ( readFromDev > 0 ) return readSoFar + readFromDev; } return readSoFar; } - ByteArray IODevice::readLine(const size_t maxSize) + ByteArray IODevice::channelReadLine( uint channel, int64_t maxSize ) { - if ( !canRead() ) + Z_D(); + if ( !canRead() || maxSize < 0 ) return {}; - return d_func()->_readBuf.readLine( maxSize ); + if ( channel >= d->_readChannels.size() ) { + ERR << constants::outOfRangeErrMsg << std::endl; + throw std::out_of_range( constants::outOfRangeErrMsg.data() ); + } + + ByteArray result; + // largest possible ByteArray in int64_t boundaries + const auto maxBArrSize = int64_t( std::min( ByteArray::maxSize(), std::size_t(std::numeric_limits::max()) ) ); + if ( maxSize > maxBArrSize ) { + ERR << "Calling channelReadLine with maxSize > int64_t(ByteArray::maxSize) " << std::endl; + maxSize = maxBArrSize - 1; + } + + // how much did we read? + int64_t readSoFar = 0; + + // if we have no size or the size is really big we read incrementally, use the buffer chunk size + // to read full chunks from the buffer if possible + if ( maxSize == 0 || maxSize >= (maxBArrSize - 1) ) { + + // largest possible ByteArray + maxSize = maxBArrSize; + + // we need to read in chunks until we get a \n + int64_t lastReadSize = 0; + result.resize (1); // leave room for \0 + do { + result.resize( std::min( std::size_t(maxSize), std::size_t(result.size() + d->_readBufChunkSize )) ); + lastReadSize = channelReadLine( channel, result.data() + readSoFar, result.size() - readSoFar ); + if ( lastReadSize > 0) + readSoFar += lastReadSize; + + // check for equal _readBufSize, + // our readData request is always 1 byte bigger than the _readBufChunkSize because of the initial byte we allocated in the result buffer, + // so the \0 that is appended by readLine does not make a difference. + } while( lastReadSize == d->_readBufChunkSize + && result[readSoFar-1] != '\n' ); + + } else { + result.resize( maxSize ); + readSoFar = channelReadLine( channel, result.data(), result.size() ); + } + + if ( readSoFar > 0 ) { + // we do not need to keep the \0 in the ByteArray + result.resize( readSoFar ); + } else { + result.clear (); + } + + // make sure we do not waste memory + result.shrink_to_fit(); + + return result; + } + + int64_t IODevice::channelReadLine( uint channel, char *buf, const int64_t maxSize ) + { + Z_D(); + + if ( !canRead() || maxSize < 0 ) + return -1; + + if ( channel >= d->_readChannels.size() ) { + ERR << constants::outOfRangeErrMsg << std::endl; + throw std::out_of_range( constants::outOfRangeErrMsg.data() ); + } + + if ( maxSize < 2 ) { + ERR << "channelReadLine needs at least a buffsize of 2" << std::endl; + return -1; + } + + int64_t toRead = maxSize - 1; // append \0 at the end + int64_t readSoFar = 0; + if ( d->_readChannels[channel].size () > 0 ) + readSoFar = d->_readChannels[channel].readLine( buf, toRead + 1 /*IOBuffer appends \0*/ ); + + if ( readSoFar == toRead || ( readSoFar > 0 && buf[readSoFar-1] == '\n' ) ) { + buf[readSoFar] = '\0'; + return readSoFar; + } + + bool hasError = false; + // if we reach here, the buffer was either empty, or does not contain a \n, in both cases we need to + // read from the device directly until we hit a ending condition + while ( readSoFar < toRead ) { + const auto r = readData( channel, buf+readSoFar, 1 ); + if ( r == 0 ) { + // no data available to be read -> EOF, or data stream empty + break; + } + else if ( r < 0 ) { + hasError = true; + break; + } + readSoFar+=r; + + if ( buf[readSoFar-1] == '\n' ) + break; + } + + if ( readSoFar == 0 ) + return hasError ? -1 : 0; + + buf[readSoFar] = '\0'; + return readSoFar; } - off_t zyppng::IODevice::write(const zyppng::ByteArray &data) + int64_t IODevice::bytesAvailable ( uint channel ) const + { + Z_D(); + if ( !canRead() ) + return 0; + return d->_readChannels[channel].size() + rawBytesAvailable( channel ); + } + + bool IODevice::canReadLine ( uint channel ) const + { + Z_D(); + if ( !canRead() || channel >= d->_readChannels.size() ) + return false; + return d->_readChannels[channel].canReadLine(); + } + + int64_t zyppng::IODevice::write( const zyppng::ByteArray &data) { if ( !canWrite() ) return 0; return write( data.data(), data.size() ); } - off_t IODevice::write(const char *data, size_t len) + int64_t IODevice::write( const char *data, int64_t len) { - if ( !canWrite() ) + if ( !canWrite() || len <= 0 ) return 0; return writeData( data, len ); } + bool IODevice::waitForReadyRead( int timeout) + { + Z_D(); + if ( !canRead() ) + return false; + + return waitForReadyRead( d->_currentReadChannel, timeout ); + } + SignalProxy IODevice::sigReadyRead() { return d_func()->_readyRead; } + + SignalProxy IODevice::sigChannelReadyRead () + { + return d_func()->_channelReadyRead; + } + + SignalProxy IODevice::sigBytesWritten() + { + return d_func()->_sigBytesWritten; + } + + SignalProxy< void ()> IODevice::sigAllBytesWritten () + { + return d_func()->_sigAllBytesWritten; + } } diff --git a/zypp-core/zyppng/io/iodevice.h b/zypp-core/zyppng/io/iodevice.h index d4bcc0d..3033984 100644 --- a/zypp-core/zyppng/io/iodevice.h +++ b/zypp-core/zyppng/io/iodevice.h @@ -23,6 +23,11 @@ namespace zyppng { class IODevicePrivate; + + /*! + * The IODevice class represents a async sequential IO device, like a Socket or Pipe, + * to receive and or send data. + */ class IODevice : public Base { ZYPP_DECLARE_PRIVATE(IODevice); @@ -40,35 +45,101 @@ namespace zyppng { using WeakPtr = std::weak_ptr; IODevice(); - - virtual bool open ( const OpenMode mode ); virtual void close (); + void setReadChannel ( uint channel ); + uint currentReadChannel () const; + int readChannelCount () const; bool canRead () const; bool canWrite () const; bool isOpen () const; - bool canReadLine () const; ByteArray readAll (); - ByteArray read ( size_t maxSize ); - size_t read ( char *buf, size_t maxSize ); - virtual ByteArray readLine ( const size_t maxSize = 0 ); - virtual size_t bytesAvailable () const; + ByteArray read ( int64_t maxSize ); + int64_t read ( char *buf, int64_t maxSize ); + virtual ByteArray readLine (const int64_t maxSize = 0 ); + virtual int64_t bytesAvailable () const; + bool canReadLine () const; + + ByteArray readAll ( uint channel ); + ByteArray read ( uint channel, int64_t maxSize ); + int64_t read ( uint channel, char *buf, int64_t maxSize ); + + /*! + * Convenience function that reads a line from the device into a ByteArray. Since + * this function has no way to signal if a error happened, a empty ByteArray is + * returned if there was no data or if a error occured. + */ + ByteArray channelReadLine ( uint channel, int64_t maxSize = 0 ); + + /*! + * Reads data from the device until one of the following conditions are met: + * - A \n is encountered + * - maxSize nr of bytes have been read + * - a error occurs on the device + * + * If bytes have been read from the device this always returns the number of bytes that + * have been read, otherwise if no data was read 0 is returned or if a error occurs -1 is returned. + */ + virtual int64_t channelReadLine ( uint channel, char *buf, const int64_t maxSize ); + virtual int64_t bytesAvailable( uint channel ) const; + + /*! + * Returns true if a line can be read from the currently buffered data in the given channel + */ + bool canReadLine ( uint channel ) const; + + int64_t write ( const ByteArray &data ); + int64_t write ( const char *data, int64_t len ); + + /*! + * Blocks the current event loop to wait until there are bytes available to read from the device. + * This always operates on the read channel that is selected as the default when the function is first called, + * even if the default channel would be changed during the wait. + * + * \sa zyppng::IODevice::currentReadChannel + * \note do not use until there is a very good reason, like event processing should not continue until readyRead was sent + */ + bool waitForReadyRead(int timeout); + + /*! + * Blocks the current event loop to wait until there are bytes available to read from the given read channel. + * + * \note do not use until there is a very good reason, like event processing should not continue until readyRead was sent + */ + virtual bool waitForReadyRead(uint channel, int timeout) = 0; - off_t write ( const ByteArray &data ); - off_t write ( const char *data, size_t len ); /*! - * Signal is emitted when there is data available to read + * Signal is emitted when there is data available to read on the current default read channel */ SignalProxy sigReadyRead (); + /*! + * Signal is emitted when there is data available on the given channel + */ + SignalProxy sigChannelReadyRead (); + + /*! + * Signal is emitted every time bytes have been written to the underlying device. + * This can be used to track how much data was actually sent. + */ + SignalProxy< void (int64_t)> sigBytesWritten (); + + /*! + * Signal is emitted every time all bytes have been written to the underlying device. + */ + SignalProxy< void ()> sigAllBytesWritten (); + protected: IODevice( IODevicePrivate &d ); - virtual size_t rawBytesAvailable () const = 0; - virtual off_t writeData ( const char *data, off_t count ) = 0; - virtual off_t readData ( char *buffer, off_t bufsize ) = 0; + virtual bool open ( const OpenMode mode ); + virtual int64_t rawBytesAvailable ( uint channel ) const = 0; + virtual int64_t writeData ( const char *data, int64_t count ) = 0; + virtual int64_t readData ( uint channel, char *buffer, int64_t bufsize ) = 0; + virtual void readChannelChanged ( uint channel ) = 0; + void setReadChannelCount ( uint channels ); }; ZYPP_DECLARE_OPERATORS_FOR_FLAGS( IODevice::OpenMode ); diff --git a/zypp-core/zyppng/io/private/abstractspawnengine_p.h b/zypp-core/zyppng/io/private/abstractspawnengine_p.h index 850091a..76f4380 100644 --- a/zypp-core/zyppng/io/private/abstractspawnengine_p.h +++ b/zypp-core/zyppng/io/private/abstractspawnengine_p.h @@ -52,13 +52,20 @@ namespace zyppng { */ pid_t pid ( ); - /** + /*! * Kickstart the process, if this returns true it is guaranteed that exec() was successful */ virtual bool start ( const char *const *argv, int stdin_fd, int stdout_fd, int stderr_fd ) = 0; virtual bool isRunning ( bool wait = false ) = 0; + /*! + * Used to notify the backend that the process has ended, + * helpful when the process is tracked in another way than calling \ref isRunning, + * for example in a eventloop. + */ + virtual void notifyExited ( int status ); + bool dieWithParent() const; void setDieWithParent( bool dieWithParent ); diff --git a/zypp-core/zyppng/io/private/asyncdatasource_p.h b/zypp-core/zyppng/io/private/asyncdatasource_p.h new file mode 100644 index 0000000..b1a327e --- /dev/null +++ b/zypp-core/zyppng/io/private/asyncdatasource_p.h @@ -0,0 +1,39 @@ +#ifndef ZYPP_CORE_ZYPPNG_IO_PRIVATE_ASYNCDATASOURCE_P_H_INCLUDED +#define ZYPP_CORE_ZYPPNG_IO_PRIVATE_ASYNCDATASOURCE_P_H_INCLUDED + +#include +#include +#include "iodevice_p.h" +#include "iobuffer_p.h" + +namespace zyppng { + + class AsyncDataSourcePrivate : public IODevicePrivate { + ZYPP_DECLARE_PUBLIC(AsyncDataSource); + public: + AsyncDataSourcePrivate ( AsyncDataSource &pub ) : IODevicePrivate(pub) {} + struct ReadChannelDev { + int _readFd = -1; + SocketNotifier::Ptr _readNotifier; + }; + std::vector _readFds; + + SocketNotifier::Ptr _writeNotifier; + IOBuffer _writeBuffer; + int _writeFd = -1; + + void notifierActivated (const SocketNotifier ¬ify, int evTypes ); + void readyRead ( uint channel ); + void readyWrite ( ); + + void closeWriteChannel ( AsyncDataSource::ChannelCloseReason reason ); + void closeReadChannel ( uint channel, AsyncDataSource::ChannelCloseReason reason ); + + Signal _sigWriteFdClosed; + Signal _sigReadFdClosed; + }; + +} + + +#endif // ZYPP_CORE_ZYPPNG_IO_PRIVATE_ASYNCDATASOURCE_P_H_INCLUDED diff --git a/zypp-core/zyppng/io/private/iobuffer_p.h b/zypp-core/zyppng/io/private/iobuffer_p.h index 0226cdc..5039cc3 100644 --- a/zypp-core/zyppng/io/private/iobuffer_p.h +++ b/zypp-core/zyppng/io/private/iobuffer_p.h @@ -12,8 +12,8 @@ namespace zyppng { struct Chunk { ByteArray _buffer; - unsigned head = 0; - unsigned tail = 0; + int64_t head = 0; + int64_t tail = 0; char * data () { return _buffer.data() + head; @@ -23,35 +23,36 @@ namespace zyppng { return _buffer.data() + head; } - unsigned available() const { + int64_t available() const { return _buffer.size() - tail; } - unsigned len () const { + int64_t len () const { return tail - head; } }; public: - IOBuffer( unsigned chunkSize = 0); + IOBuffer( int64_t chunkSize = 0 ); - char *reserve( size_t bytes ); + char *reserve( int64_t bytes ); char *front (); - size_t frontSize () const; + int64_t frontSize () const; void clear ( ); - size_t discard ( size_t bytes ); - void chop ( size_t bytes ); - void append ( const char *data, size_t count ); + int64_t discard( int64_t bytes ); + void chop ( int64_t bytes ); + void append ( const char *data, int64_t count ); void append ( const ByteArray &data ); - size_t read ( char *buffer, size_t max ); - size_t size ( ) const; + int64_t read ( char *buffer, int64_t max ); + int64_t size ( ) const; std::vector::size_type chunks () const; inline int64_t indexOf ( const char c ) const { return indexOf( c, size() ); } - int64_t indexOf ( const char c, size_t maxCount, size_t pos = 0 ) const; - ByteArray readLine ( const size_t max = 0 ); + int64_t indexOf (const char c, int64_t maxCount, int64_t pos = 0 ) const; + ByteArray readLine ( const int64_t max = 0 ); + int64_t readLine( char *buffer, int64_t max ); bool canReadLine () const; private: - unsigned _defaultChunkSize; + int64_t _defaultChunkSize; std::vector _chunks; }; diff --git a/zypp-core/zyppng/io/private/iodevice_p.h b/zypp-core/zyppng/io/private/iodevice_p.h index bbd9ba1..bca7491 100644 --- a/zypp-core/zyppng/io/private/iodevice_p.h +++ b/zypp-core/zyppng/io/private/iodevice_p.h @@ -15,6 +15,8 @@ #ifndef ZYPPNG_IO_IODEVICE_P_DEFINED #define ZYPPNG_IO_IODEVICE_P_DEFINED +#include +#include #include #include #include "iobuffer_p.h" @@ -22,12 +24,27 @@ namespace zyppng { + namespace constants { + constexpr std::string_view outOfRangeErrMsg("Channel index out of range"); + } + + enum { + DefIoDeviceBufChunkSize = 16384 + }; + class IODevicePrivate : public BasePrivate { public: IODevicePrivate ( IODevice &p ); - IOBuffer _readBuf; + + std::vector _readChannels; + uint _currentReadChannel = 0; + int64_t _readBufChunkSize = DefIoDeviceBufChunkSize; + IODevice::OpenMode _mode = IODevice::Closed; - Signal< void()> _readyRead; + Signal _readyRead; + Signal _channelReadyRead; + Signal< void (int64_t)> _sigBytesWritten; + Signal< void ()> _sigAllBytesWritten; }; } diff --git a/zypp-core/zyppng/io/private/socket_p.h b/zypp-core/zyppng/io/private/socket_p.h index c99148e..0ab54cc 100644 --- a/zypp-core/zyppng/io/private/socket_p.h +++ b/zypp-core/zyppng/io/private/socket_p.h @@ -37,7 +37,7 @@ namespace zyppng { { } bool initSocket () ; - void setError ( Socket::SocketError error, std::string &&err ); + void setError ( Socket::SocketError error, std::string &&err, bool emit = true ); bool handleConnectError ( int error ); bool transition ( Socket::SocketState newState ); @@ -48,7 +48,7 @@ namespace zyppng { void onSocketActivatedSlot ( const SocketNotifier &, int ev ) { return onSocketActivated(ev); } - int rawBytesAvailable () const; + int64_t rawBytesAvailable () const; bool readRawBytesToBuffer (); bool writePendingData (); @@ -61,12 +61,12 @@ namespace zyppng { bool _borrowedSocket = false; //error handling + bool _emittedErr = false; Socket::SocketError _error = Socket::NoError; std::string _errorDesc; //signals Signal< void(Socket::SocketError)> _sigError; - Signal< void (std::size_t)> _sigBytesWritten; Signal< void()> _incomingConnection; Signal< void()> _connected; Signal< void()> _disconnected; diff --git a/zypp-core/zyppng/io/process.cpp b/zypp-core/zyppng/io/process.cpp index 4ef4294..e951a1e 100644 --- a/zypp-core/zyppng/io/process.cpp +++ b/zypp-core/zyppng/io/process.cpp @@ -1,5 +1,5 @@ #include "process.h" -#include +#include #include #include #include @@ -13,29 +13,33 @@ namespace zyppng { * signals that the fork has worked out but NOT that the app actually started */ - class ProcessPrivate : public BasePrivate + class ProcessPrivate : public AsyncDataSourcePrivate { public: - ProcessPrivate( Process &p ) : BasePrivate(p) + ProcessPrivate( Process &p ) : AsyncDataSourcePrivate(p) { } + void cleanup() { + _stdinFd = -1; + _stderrFd = -1; + _stdoutFd = -1; + } + std::unique_ptr _spawnEngine = AbstractSpawnEngine::createDefaultEngine(); - AsyncDataSource::Ptr _stdinDevice; - AsyncDataSource::Ptr _stdoutDevice; - AsyncDataSource::Ptr _stderrDevice; zypp::AutoFD _stdinFd = -1; zypp::AutoFD _stderrFd = -1; zypp::AutoFD _stdoutFd = -1; - pid_t _pid = -1; Signal _sigStarted; Signal _sigFinished; Signal _sigFailedToStart; + Process::OutputChannelMode _channelMode = Process::Seperate; + Process::OutputChannel _currentChannel = Process::StdOut; }; ZYPP_IMPL_PRIVATE(Process) - Process::Process() : Base( *( new ProcessPrivate(*this) ) ) + Process::Process() : AsyncDataSource( *( new ProcessPrivate(*this) ) ) { } @@ -48,13 +52,13 @@ namespace zyppng { Process::~Process() { Z_D(); - if ( d->_pid >= 0 ) { - EventDispatcher::instance()->untrackChildProcess( d->_pid ); + if ( d->_spawnEngine->pid() >= 0 ) { + EventDispatcher::instance()->untrackChildProcess( d->_spawnEngine->pid() ); DBG << "Process destroyed while still running removing from EventLoop." << std::endl; } } - bool Process::start(const char * const *argv ) + bool Process::start( const char *const *argv ) { Z_D(); @@ -63,13 +67,12 @@ namespace zyppng { return false; } + if ( isRunning() ) + return false; + // clean up the previous run - d->_stdinDevice.reset(); - d->_stdoutDevice.reset(); - d->_stderrDevice.reset(); - d->_stdinFd = -1; - d->_stderrFd = -1; - d->_stdoutFd = -1; + AsyncDataSource::close(); + d->cleanup(); // create the pipes we need auto stdinPipe = Pipe::create( ); @@ -84,38 +87,45 @@ namespace zyppng { return false; } - auto stderrPipe = Pipe::create( ); - if ( !stderrPipe ) { - d->_sigFailedToStart.emit(); - return false; + int stderr_fd = -1; + std::optional stderrPipe; + if ( d->_channelMode == Seperate ) { + stderrPipe = Pipe::create( ); + if ( !stderrPipe ) { + d->_sigFailedToStart.emit(); + return false; + } + stderr_fd = stderrPipe->writeFd; + } else { + stderr_fd = stdoutPipe->writeFd; } - if ( d->_spawnEngine->start( argv, stdinPipe->readFd, stdoutPipe->writeFd, stderrPipe->writeFd ) ) { + if ( d->_spawnEngine->start( argv, stdinPipe->readFd, stdoutPipe->writeFd, stderr_fd ) ) { // if we reach this point the engine guarantees that exec() was successful - d->_pid = d->_spawnEngine->pid( ); + const auto pid = d->_spawnEngine->pid( ); // register to the eventloop right away - EventDispatcher::instance()->trackChildProcess( d->_pid, [this]( int, int status ){ + EventDispatcher::instance()->trackChildProcess( pid, [this]( int, int status ){ Z_D(); - d->_spawnEngine->setExitStatus( d->_spawnEngine->checkStatus( status ) ); - d->_pid = -1; + d->_spawnEngine->notifyExited( status ); d->_sigFinished.emit( d->_spawnEngine->exitStatus() ); }); // make sure the fds we need are kept open d->_stdinFd = std::move( stdinPipe->writeFd ); d->_stdoutFd = std::move( stdoutPipe->readFd ); - d->_stderrFd = std::move( stderrPipe->readFd ); - - d->_stdinDevice = AsyncDataSource::create(); - d->_stdinDevice->open( -1, d->_stdinFd ); - d->_stdoutDevice = AsyncDataSource::create(); - d->_stdoutDevice->open( d->_stdoutFd ); + std::vector rFds { d->_stdoutFd }; + if ( stderrPipe ) { + d->_stderrFd = std::move( stderrPipe->readFd ); + rFds.push_back( d->_stderrFd.value() ); + } - d->_stderrDevice = AsyncDataSource::create(); - d->_stderrDevice->open( d->_stderrFd ); + if ( !openFds( rFds, d->_stdinFd ) ) { + stop( SIGKILL ); + return false; + } d->_sigStarted.emit(); return true; @@ -134,7 +144,34 @@ namespace zyppng { bool Process::isRunning() { - return ( d_func()->_pid > -1 ); + Z_D(); + return ( d->_spawnEngine->pid() > -1 ); + } + + void Process::close () + { + flush(); + stop(SIGKILL); + d_func()->cleanup(); + AsyncDataSource::close(); + } + + void Process::waitForExit() + { + Z_D(); + if ( d->_spawnEngine->isRunning() ) { + // we will manually track the exit status + EventDispatcher::instance()->untrackChildProcess( d->_spawnEngine->pid() ); + // wait for the process to exit + d->_spawnEngine->isRunning( true ); + } + } + + void Process::closeWriteChannel() + { + Z_D(); + d->_stdinFd = -1; + AsyncDataSource::closeWriteChannel(); } const std::string &Process::executedCommand() const @@ -179,7 +216,7 @@ namespace zyppng { pid_t Process::pid() { - return d_func()->_pid; + return d_func()->_spawnEngine->pid(); } int Process::exitStatus() const @@ -227,21 +264,6 @@ namespace zyppng { return d_func()->_spawnEngine->addFd( fd ); } - std::shared_ptr Process::stdinDevice() - { - return d_func()->_stdinDevice; - } - - std::shared_ptr Process::stdoutDevice() - { - return d_func()->_stdoutDevice; - } - - std::shared_ptr Process::stderrDevice() - { - return d_func()->_stderrDevice; - } - int Process::stdinFd() { return d_func()->_stdinFd; @@ -272,4 +294,7 @@ namespace zyppng { return d_func()->_sigFinished; } + Process::OutputChannelMode Process::outputChannelMode() const { return d_func()->_channelMode; } + void Process::setOutputChannelMode(const OutputChannelMode &outputChannelMode) { d_func()->_channelMode = outputChannelMode; } + } diff --git a/zypp-core/zyppng/io/process.h b/zypp-core/zyppng/io/process.h index f860c91..ff8237a 100644 --- a/zypp-core/zyppng/io/process.h +++ b/zypp-core/zyppng/io/process.h @@ -15,7 +15,7 @@ #ifndef ZYPPNG_IO_PROCESS_H_DEFINED #define ZYPPNG_IO_PROCESS_H_DEFINED -#include +#include #include #include #include @@ -26,7 +26,7 @@ namespace zyppng { class ProcessPrivate; class IODevice; - class Process : public Base + class Process : public AsyncDataSource { ZYPP_DECLARE_PRIVATE(Process); public: @@ -34,16 +34,39 @@ namespace zyppng { * For passing additional environment variables to set */ using Environment = std::map; - using Ptr = std::shared_ptr; using WeakPtr = std::weak_ptr; + enum OutputChannelMode { + Seperate, + Merged + }; + + enum OutputChannel { + StdOut = 0, + StdErr = 1 + }; + static Ptr create (); ~Process(); - bool start (const char *const *argv); + bool start ( const char *const *argv ); void stop ( int signal = SIGTERM ); bool isRunning (); + void close () override; + + /*! + * Blocks until the process has exited, during that time readyRead is not + * emitted for any read channels. Call \ref readAll to get all remaining data + * that was written by the process + */ + void waitForExit (); + + /*! + * Close the stdin fd of the subprocess. This is required for processes + * that run until their stdin is closed. + */ + void closeWriteChannel () override; const std::string &executedCommand () const; const std::string &execError() const; @@ -72,10 +95,6 @@ namespace zyppng { const std::vector &fdsToMap () const; void addFd ( int fd ); - std::shared_ptr stdinDevice (); - std::shared_ptr stdoutDevice (); - std::shared_ptr stderrDevice (); - int stdinFd (); int stdoutFd (); int stderrFd (); @@ -84,6 +103,9 @@ namespace zyppng { SignalProxy sigFailedToStart (); SignalProxy sigFinished (); + OutputChannelMode outputChannelMode() const; + void setOutputChannelMode(const OutputChannelMode &outputChannelMode); + protected: Process(); }; diff --git a/zypp-core/zyppng/io/socket.cc b/zypp-core/zyppng/io/socket.cc index 6edc821..74ef0ae 100644 --- a/zypp-core/zyppng/io/socket.cc +++ b/zypp-core/zyppng/io/socket.cc @@ -23,6 +23,9 @@ namespace zyppng { if ( _socket >= 0 ) return true; + _error = Socket::NoError; + _emittedErr = false; + // Since Linux 2.6.27 we can pass additional flags with the type argument to avoid fcntl // if creating sockets fails we might need to change that _socket = ::socket( _domain, _type | SOCK_NONBLOCK | SOCK_CLOEXEC, _protocol ); @@ -54,13 +57,16 @@ namespace zyppng { return false; } - void SocketPrivate::setError(Socket::SocketError error , std::string &&err ) + void SocketPrivate::setError( Socket::SocketError error , std::string &&err, bool emit ) { - if ( _error == error ) - return; - _error = error; - _errorDesc = std::move(err); - _sigError.emit( error ); + // we only remember the first error that happend + if ( _error == Socket::NoError && _error != error ) { + _emittedErr = emit; + _error = error; + _errorDesc = std::move(err); + } + if ( emit && !_emittedErr ) + _sigError.emit( error ); } bool SocketPrivate::transition( Socket::SocketState newState ) @@ -191,7 +197,7 @@ namespace zyppng { return true; } - int zyppng::SocketPrivate::rawBytesAvailable() const + int64_t zyppng::SocketPrivate::rawBytesAvailable() const { if ( state() != Socket::ConnectedState ) return 0; @@ -201,38 +207,45 @@ namespace zyppng { bool SocketPrivate::readRawBytesToBuffer() { + Z_Z(); auto bytesToRead = rawBytesAvailable(); if ( bytesToRead == 0 ) { // make sure to check if bytes are available even if the ioctl call returns something different bytesToRead = 4096; } - char *buf = _readBuf.reserve( bytesToRead ); - const auto bytesRead = z_func()->readData( buf, bytesToRead ); - - if ( bytesRead < 0 ) { - _readBuf.chop( bytesToRead ); - return false; - } + auto &readBuf = _readChannels[0]; + char *buf = readBuf.reserve( bytesToRead ); + const auto bytesRead = z_func()->readData( 0, buf, bytesToRead ); - if ( bytesToRead > bytesRead ) - _readBuf.chop( bytesToRead-bytesRead ); + if ( bytesRead <= 0 ) { + readBuf.chop( bytesToRead ); - if ( bytesRead > 0 ) { - _readyRead.emit(); - return true; - } - //handle remote close - else if ( bytesRead == 0 && errno != EAGAIN && errno != EWOULDBLOCK ) { - setError( Socket::ConnectionClosedByRemote, "The remote host closed the connection" ); + switch ( bytesRead ) { + case -2: + // there is simply no data to read ignore and try again + return true; + case 0: { + // remote close + setError( Socket::ConnectionClosedByRemote, "The remote host closed the connection", true ); + break; + } + case -1: + default: { + setError( Socket::InternalError, strerr_cxx(), true ); + break; + } + } + z->abort(); return false; } - if ( errno == EAGAIN || errno == EWOULDBLOCK ) - return true; + if ( bytesToRead > bytesRead ) + readBuf.chop( bytesToRead-bytesRead ); - setError( Socket::InternalError, strerr_cxx() ); - return false; + _readyRead.emit(); + _channelReadyRead.emit(0); + return true; } bool SocketPrivate::writePendingData() @@ -271,6 +284,8 @@ namespace zyppng { } s._writeBuffer.discard( written ); _sigBytesWritten.emit( written ); + if ( s._writeBuffer.size() == 0 ) + _sigAllBytesWritten.emit(); } return true; }, _state ); @@ -404,8 +419,13 @@ namespace zyppng { : IODevice( *( new SocketPrivate( domain, type, protocol, *this ))) { } - size_t Socket::rawBytesAvailable() const + int64_t Socket::rawBytesAvailable( uint channel ) const { + if ( channel != 0 ) { + constexpr std::string_view msg("Socket does not support multiple read channels"); + ERR << msg << std::endl; + throw std::logic_error( msg.data() ); + } return d_func()->rawBytesAvailable(); } @@ -657,7 +677,7 @@ namespace zyppng { } - off_t Socket::writeData( const char *data, off_t count ) + int64_t Socket::writeData( const char *data, int64_t count ) { Z_D(); if ( d->state() != SocketState::ConnectedState ) @@ -704,6 +724,10 @@ namespace zyppng { if ( written > 0 ) d->_sigBytesWritten.emit( written ); } + + if ( s._writeBuffer.size() == 0 ) + d->_sigAllBytesWritten.emit(); + return count; } @@ -758,10 +782,10 @@ namespace zyppng { return bufferEmpty; } - bool Socket::waitForReadyRead(int timeout) + bool Socket::waitForReadyRead( uint channel, int timeout) { Z_D(); - if ( d->state() != Socket::ConnectedState ) + if ( d->state() != Socket::ConnectedState || channel != 0 ) return false; // we can only wait if we are in connected state @@ -777,8 +801,13 @@ namespace zyppng { return bytesAvailable() > 0; } - off_t Socket::readData( char *buffer, off_t bufsize ) + int64_t Socket::readData( uint channel, char *buffer, int64_t bufsize ) { + if ( channel != 0 ) { + constexpr std::string_view msg("Socket does not support multiple read channels"); + ERR << msg << std::endl; + throw std::logic_error( msg.data() ); + } Z_D(); if ( d->state() != SocketState::ConnectedState ) @@ -788,20 +817,18 @@ namespace zyppng { // special case for remote close if ( read == 0 ) { - d->setError( ConnectionClosedByRemote, "The remote host closed the connection" ); - abort(); - return -1; + d->setError( ConnectionClosedByRemote, "The remote host closed the connection", false ); + return 0; } else if ( read < 0 ) { switch ( errno ) { #if EAGAIN != EWOULDBLOCK case EWOULDBLOCK: #endif case EAGAIN: { - return 0; + return -2; } default: { - d->setError( UnknownSocketError, strerr_cxx( errno ) ); - abort(); + d->setError( UnknownSocketError, strerr_cxx( errno ), false ); return -1; } } @@ -809,15 +836,19 @@ namespace zyppng { return read; } - size_t Socket::bytesAvailable() const - {; - return IODevice::bytesAvailable(); + void Socket::readChannelChanged ( uint channel ) + { + if ( channel != 0 ) { + constexpr std::string_view msg("Changing the readChannel on a Socket is not supported"); + ERR << msg << std::endl; + throw std::logic_error( msg.data() ); + } } - size_t Socket::bytesPending() const + int64_t Socket::bytesPending() const { Z_D(); - return std::visit([&]( const auto &s ) -> size_t { + return std::visit([&]( const auto &s ) -> int64_t { using T = std::decay_t; if constexpr ( std::is_same_v || std::is_same_v ) { return s._writeBuffer.size(); @@ -851,9 +882,4 @@ namespace zyppng { return d_func()->_sigError; } - SignalProxy Socket::sigBytesWritten() - { - return d_func()->_sigBytesWritten; - } - } diff --git a/zypp-core/zyppng/io/socket.h b/zypp-core/zyppng/io/socket.h index d2417fe..1803aab 100644 --- a/zypp-core/zyppng/io/socket.h +++ b/zypp-core/zyppng/io/socket.h @@ -87,14 +87,9 @@ namespace zyppng { void close() override; /*! - * Returns the current number of bytes that can be read from the socket. - */ - size_t bytesAvailable() const override; - - /*! * Returns the current number of bytes that are not yet written to the socket. */ - size_t bytesPending() const; + int64_t bytesPending() const; /*! * Returns the current state the socket is in, @@ -166,7 +161,7 @@ namespace zyppng { * * \note do not use until there is no other way */ - bool waitForReadyRead ( int timeout = -1 ); + bool waitForReadyRead ( uint channel, int timeout = -1 ) override; /*! * Returns the native socket handle. @@ -208,12 +203,6 @@ namespace zyppng { SignalProxy sigDisconnected (); /*! - * Signal is emitted every time bytes have been written to the underlying socket. - * This can be used to track how much data was actually sent. - */ - SignalProxy sigBytesWritten (); - - /*! * Signal is emitted whenever a error happend in the socket. Make sure to check * the actual error code to determine if the error is fatal. */ @@ -232,9 +221,10 @@ namespace zyppng { // IODevice interface protected: - size_t rawBytesAvailable() const override; - off_t writeData(const char *data, off_t count) override; - off_t readData(char *buffer, off_t bufsize) override; + int64_t rawBytesAvailable( uint channel = 0 ) const override; + int64_t writeData(const char *data, int64_t count) override; + int64_t readData( uint channel, char *buffer, int64_t bufsize ) override; + void readChannelChanged ( uint channel ) override; }; } #endif diff --git a/zypp-core/zyppng/meta/functional.h b/zypp-core/zyppng/meta/functional.h index 4b76972..e2aa6df 100644 --- a/zypp-core/zyppng/meta/functional.h +++ b/zypp-core/zyppng/meta/functional.h @@ -46,4 +46,34 @@ namespace std { #endif +/*! + * Simple helper template to make a callback that binds the "this" pointer, to be used in + * a pipeline. + */ +template +auto mem_fn_cb( Obj& o, Ret (Obj::*objMemFunc)( Arg&& ) ) { + return [tPtr = &o, fun = objMemFunc ]( Arg &&r ){ + return std::invoke(fun, tPtr, std::move(r) ); + }; +} + +template +auto mem_fn_cb( Obj& o, Ret (Obj::*objMemFunc)( const Arg& ) ) { + return [tPtr = &o, fun = objMemFunc ]( const Arg &r ){ + return std::invoke(fun, tPtr, r ); + }; +} + +template +auto mem_fn_cb( Obj& o, Ret (Obj::*objMemFunc)( Arg ) ) { + return [tPtr = &o, fun = objMemFunc ]( Arg r ){ + if constexpr ( std::is_invocable_v< decltype (fun), Obj*, Arg&&> ) { + return std::invoke(fun, tPtr, std::move(r) ); + } else { + return std::invoke(fun, tPtr, r ); + } + }; +} + + #endif diff --git a/zypp-core/zyppng/meta/type_traits.h b/zypp-core/zyppng/meta/type_traits.h index 6899f98..d3fa14e 100644 --- a/zypp-core/zyppng/meta/type_traits.h +++ b/zypp-core/zyppng/meta/type_traits.h @@ -14,11 +14,10 @@ namespace std { #endif - -#if __cplusplus < 201703L - namespace std { +#if __cplusplus < 202002L + //implementation of the detector idiom, used to help with SFINAE //from https://en.cppreference.com/w/cpp/experimental/is_detected @@ -73,6 +72,10 @@ using is_detected_convertible = std::is_convertible, To> template class Op, class... Args> constexpr bool is_detected_convertible_v = is_detected_convertible::value_t::value; +#endif + +#if __cplusplus < 201703L + //https://en.cppreference.com/w/cpp/types/conjunction) template struct conjunction : std::true_type { }; template struct conjunction : B1 { }; @@ -91,8 +94,8 @@ struct disjunction template struct negation : std::bool_constant< !bool(B::value)> { }; -} #endif +} namespace zyppng { diff --git a/zypp-core/zyppng/pipelines/asyncresult.h b/zypp-core/zyppng/pipelines/asyncresult.h index 15d072f..362c037 100644 --- a/zypp-core/zyppng/pipelines/asyncresult.h +++ b/zypp-core/zyppng/pipelines/asyncresult.h @@ -77,7 +77,7 @@ namespace zyppng { template struct AsyncResult : public zyppng::AsyncOp< typename AOp::value_type > { - AsyncResult ( std::unique_ptr && prevTask, std::unique_ptr &&cb ) + AsyncResult ( std::shared_ptr && prevTask, std::shared_ptr &&cb ) : _prevTask( std::move(prevTask) ) , _myTask( std::move(cb) ) { connect(); @@ -108,20 +108,20 @@ namespace zyppng { if ( _prevTask ) { //dumpInfo(); - _prevTask.reset(nullptr); + _prevTask.reset(); } _myTask->operator()(std::move(resStore)); } - std::unique_ptr _prevTask; - std::unique_ptr _myTask; + std::shared_ptr _prevTask; + std::shared_ptr _myTask; }; template struct AsyncResult : public zyppng::AsyncOp< typename AOp::value_type > { - AsyncResult ( std::unique_ptr &&cb ) + AsyncResult ( std::shared_ptr &&cb ) : _myTask( std::move(cb) ) { connect(); } @@ -144,16 +144,7 @@ namespace zyppng { this->setReady( std::move( in ) ); }); } - std::unique_ptr _myTask; - }; - - //A async result that is ready right away - template - struct ReadyResult : public zyppng::AsyncOp< T > - { - ReadyResult( T &&val ) { - this->setReady( std::move(val) ); - } + std::shared_ptr _myTask; }; //need a wrapper to connect a sync callback to a async one @@ -186,7 +177,7 @@ namespace zyppng { virtual ~SimplifyHelper(){} - void operator() ( std::unique_ptr &&op ) { + void operator() ( std::shared_ptr &&op ) { assert( !_task ); _task = std::move(op); _task->onReady( [this]( Ret &&val ){ @@ -194,7 +185,7 @@ namespace zyppng { }); } private: - std::unique_ptr _task; + std::shared_ptr _task; }; /*! @@ -203,23 +194,18 @@ namespace zyppng { * Usually we do not want to wait on the future of a future but get the nested result immediately */ template < typename Res > - inline std::unique_ptr> simplify ( std::unique_ptr< AsyncOp > &&res ) { + inline std::shared_ptr> simplify ( std::shared_ptr< AsyncOp > &&res ) { return std::move(res); } template < typename Res, - typename AOp = AsyncOp< std::unique_ptr< AsyncOp> > > - inline std::unique_ptr> simplify ( std::unique_ptr< AsyncOp< std::unique_ptr< AsyncOp> > > &&res ) { - std::unique_ptr> op = std::make_unique< detail::AsyncResult< AOp, SimplifyHelper< AsyncOp>> >( std::move(res), std::make_unique>>() ); + typename AOp = AsyncOp< std::shared_ptr< AsyncOp> > > + inline std::shared_ptr> simplify ( std::shared_ptr< AsyncOp< std::shared_ptr< AsyncOp> > > &&res ) { + std::shared_ptr> op = std::make_shared< detail::AsyncResult< AOp, SimplifyHelper< AsyncOp>> >( std::move(res), std::make_shared>>() ); return detail::simplify( std::move(op) ); } } - template - AsyncOpPtr makeReadyResult ( T && result ) { - return std::make_unique>( std::move(result) ); - } - namespace operators { //case 1 : binding a async message to a async callback @@ -230,9 +216,9 @@ namespace zyppng { //is the callback signature what we want? , std::enable_if_t< detail::is_future_monad_cb< Callback, Ret, typename PrevOp::value_type >::value, int> = 0 > - auto operator| ( std::unique_ptr &&future, std::unique_ptr &&c ) + auto operator| ( std::shared_ptr &&future, std::shared_ptr &&c ) { - std::unique_ptr> op = std::make_unique>( std::move(future), std::move(c) ); + std::shared_ptr> op = std::make_shared>( std::move(future), std::move(c) ); return detail::simplify( std::move(op) ); } @@ -243,11 +229,11 @@ namespace zyppng { , std::enable_if_t< detail::is_async_op::value, int> = 0 , std::enable_if_t< detail::is_sync_monad_cb< Callback, typename PrevOp::value_type >::value, int> = 0 > - auto operator| ( std::unique_ptr &&future, Callback &&c ) + auto operator| ( std::shared_ptr &&future, Callback &&c ) { - std::unique_ptr> op(std::make_unique >>( + std::shared_ptr> op(std::make_shared >>( std::move(future) - , std::make_unique>( std::forward(c) ) )); + , std::make_shared>( std::forward(c) ) )); return detail::simplify( std::move(op) ); } @@ -259,9 +245,9 @@ namespace zyppng { , std::enable_if_t< !detail::is_async_op< remove_smart_ptr_t >::value, int> = 0 , std::enable_if_t< detail::is_future_monad_cb< Callback, Ret, SyncRes >::value, int> = 0 > - auto operator| ( SyncRes &&in, std::unique_ptr &&c ) + auto operator| ( SyncRes &&in, std::shared_ptr &&c ) { - AsyncOpPtr op( std::make_unique>( std::move(c) ) ); + AsyncOpRef op( std::make_shared>( std::move(c) ) ); static_cast< detail::AsyncResult* >(op.get())->run( std::move(in) ); return detail::simplify( std::move(op) ); } diff --git a/zypp-core/zyppng/pipelines/await.h b/zypp-core/zyppng/pipelines/await.h index 730c60a..005a16f 100644 --- a/zypp-core/zyppng/pipelines/await.h +++ b/zypp-core/zyppng/pipelines/await.h @@ -63,7 +63,7 @@ namespace zyppng { typename SignalGetter > auto await ( SignalGetter &&sigGet ) { - return std::make_unique>( std::forward(sigGet) ); + return std::make_shared>( std::forward(sigGet) ); } } diff --git a/zypp-core/zyppng/pipelines/expected.h b/zypp-core/zyppng/pipelines/expected.h index 4726dda..e05bf33 100644 --- a/zypp-core/zyppng/pipelines/expected.h +++ b/zypp-core/zyppng/pipelines/expected.h @@ -155,7 +155,15 @@ namespace zyppng { return m_value; } + T &operator* () + { + return get(); + } + const T &operator* () const + { + return get(); + } T *operator-> () { @@ -319,16 +327,35 @@ namespace zyppng { }; + template + static expected make_expected_success( Type &&t ) + { + return expected::success( std::forward(t) ); + } + + namespace detail { + + // helper to figure out the return type for a mbind callback, if the ArgType is void the callback is considered to take no argument. + // Due to how std::conditional works, we cannot pass std::invoke_result_t but instead use the template type std::invoke_result, since + // one of the two options have no "::type" because the substitution fails, this breaks the std::conditional_t since it can only work with two well formed + // types. Instead we pass in the template types and evaluate the ::type in the end, when the correct invoke_result was chosen. + template < typename Function, typename ArgType> + using mbind_cb_result_t = typename std::conditional_t< std::is_same_v, std::invoke_result,std::invoke_result >::type; + } + template < typename T , typename E , typename Function - , typename ResultType = decltype(std::declval()(std::declval())) + , typename ResultType = detail::mbind_cb_result_t > std::enable_if_t< !detail::is_async_op< remove_smart_ptr_t >::value, ResultType> mbind( const expected& exp, Function f) { if (exp) { - return std::invoke( f, exp.get() ); + if constexpr ( std::is_same_v ) + return std::invoke( f ); + else + return std::invoke( f, exp.get() ); } else { return ResultType::error(exp.error()); } @@ -337,12 +364,15 @@ namespace zyppng { template < typename T , typename E , typename Function - , typename ResultType = decltype(std::declval()(std::declval())) + , typename ResultType = detail::mbind_cb_result_t > std::enable_if_t< !detail::is_async_op< remove_smart_ptr_t >::value, ResultType> mbind( expected&& exp, Function f) { if (exp) { - return std::invoke( f, std::move(exp.get()) ); + if constexpr ( std::is_same_v ) + return std::invoke( f ); + else + return std::invoke( f, std::move(exp.get()) ); } else { return ResultType::error( std::move(exp.error()) ); } @@ -351,12 +381,15 @@ namespace zyppng { template < typename T , typename E , typename Function - , typename ResultType = decltype(std::declval()(std::declval())) + , typename ResultType = detail::mbind_cb_result_t > std::enable_if_t< detail::is_async_op< remove_smart_ptr_t >::value, ResultType> mbind( const expected& exp, Function f) { if (exp) { - return std::invoke( f, exp.get() ); + if constexpr ( std::is_same_v ) + return std::invoke( f ); + else + return std::invoke( f, exp.get() ); } else { return makeReadyResult( remove_smart_ptr_t::value_type::error(exp.error()) ); } @@ -365,12 +398,15 @@ namespace zyppng { template < typename T , typename E , typename Function - , typename ResultType = decltype(std::declval()(std::declval())) + , typename ResultType = detail::mbind_cb_result_t > std::enable_if_t< detail::is_async_op< remove_smart_ptr_t >::value, ResultType> mbind( expected&& exp, Function f) { if (exp) { - return std::invoke( f, std::move(exp.get()) ); + if constexpr ( std::is_same_v ) + return std::invoke( f ); + else + return std::invoke( f, std::move(exp.get()) ); } else { return makeReadyResult( remove_smart_ptr_t::value_type::error( std::move(exp.error()) ) ); } diff --git a/zypp-core/zyppng/pipelines/lift.h b/zypp-core/zyppng/pipelines/lift.h index 49a316f..43cf8e1 100644 --- a/zypp-core/zyppng/pipelines/lift.h +++ b/zypp-core/zyppng/pipelines/lift.h @@ -43,9 +43,9 @@ namespace zyppng { }; template < typename AsyncOp > - struct lifter< std::unique_ptr, std::void_t< std::enable_if_t< zyppng::detail::is_async_op::value > > > { + struct lifter< std::shared_ptr, std::void_t< std::enable_if_t< zyppng::detail::is_async_op::value > > > { - using LiftedFun = std::unique_ptr; + using LiftedFun = std::shared_ptr; lifter( LiftedFun &&fun ) : _fun(std::move(fun)) {} lifter( lifter && ) = default; diff --git a/zypp-core/zyppng/pipelines/redo.h b/zypp-core/zyppng/pipelines/redo.h index 913b446..8016695 100644 --- a/zypp-core/zyppng/pipelines/redo.h +++ b/zypp-core/zyppng/pipelines/redo.h @@ -56,9 +56,9 @@ namespace zyppng { }; template< typename MyAsyncOp, typename Pred > - struct RedoWhileImpl< std::unique_ptr,Pred, std::enable_if_t< is_async_op< MyAsyncOp >::value > > : public AsyncOp { + struct RedoWhileImpl< std::shared_ptr,Pred, std::enable_if_t< is_async_op< MyAsyncOp >::value > > : public AsyncOp { - using Task = std::unique_ptr; + using Task = std::shared_ptr; using OutType = typename MyAsyncOp::value_type; template @@ -83,14 +83,14 @@ namespace zyppng { template static auto create ( T &&t, P &&p ) { - return std::make_unique( std::forward(t), std::forward

(p)); + return std::make_shared( std::forward(t), std::forward

(p)); } private: Task _task; Pred _pred; - std::unique_ptr> _pipeline; + std::shared_ptr> _pipeline; }; //implementation for a function returning a asynchronous result @@ -109,7 +109,7 @@ namespace zyppng { template void operator() ( InType &&arg ) { - _asyncRes.reset(nullptr); + _asyncRes.reset(); _asyncRes = _task( InType( arg ) ); _asyncRes->onReady( [this, inArg = arg ]( OutType &&arg ) mutable { @@ -122,11 +122,11 @@ namespace zyppng { template static auto create ( T &&t, P &&p ) { - return std::make_unique( std::forward(t), std::forward

(p)); + return std::make_shared( std::forward(t), std::forward

(p)); } private: - std::unique_ptr> _asyncRes; + std::shared_ptr> _asyncRes; Task _task; Pred _pred; diff --git a/zypp-core/zyppng/pipelines/transform.h b/zypp-core/zyppng/pipelines/transform.h index 841377f..65e0c4b 100644 --- a/zypp-core/zyppng/pipelines/transform.h +++ b/zypp-core/zyppng/pipelines/transform.h @@ -31,7 +31,19 @@ Container transform( Container&& val, Transformation transfo { Container res; std::transform( std::make_move_iterator(val.begin()), std::make_move_iterator(val.end()), std::back_inserter(res), transformation ); - return std::move(res); + return res; +} + +template < template< class, class... > class Container, + typename Msg, + typename Transformation, + typename Ret = std::result_of_t, + typename ...CArgs > +Container transform( const Container& val, Transformation transformation ) +{ + Container res; + std::transform( val.begin(), val.end(), std::back_inserter(res), transformation ); + return res; } namespace detail { @@ -39,9 +51,9 @@ namespace detail { struct transform_helper { Transformation function; - template class Container, typename Msg, typename ...CArgs> - auto operator()( Container&& arg ) { - return zyppng::transform( std::forward< Container >(arg), function ); + template< class Container > + auto operator()( Container&& arg ) { + return zyppng::transform( std::forward(arg), function ); } }; } diff --git a/zypp-core/zyppng/pipelines/wait.h b/zypp-core/zyppng/pipelines/wait.h index 703ab49..741a4b1 100644 --- a/zypp-core/zyppng/pipelines/wait.h +++ b/zypp-core/zyppng/pipelines/wait.h @@ -16,6 +16,8 @@ #include +namespace zyppng { + namespace detail { template < class AsyncOp, @@ -31,9 +33,13 @@ namespace detail { WaitForImpl& operator= ( WaitForImpl &&other ) = default; WaitForImpl ( WaitForImpl &&other ) = default; - void operator()( std::vector< std::unique_ptr< AsyncOp > > &&ops ) { + void operator()( std::vector< std::shared_ptr< AsyncOp > > &&ops ) { assert( _allOps.empty() ); + if ( ops.empty () ) { + this->setReady( std::move(_allResults) ); + } + _allOps = std::move( ops ); for ( auto &op : _allOps ) { op->onReady( [ this ]( typename AsyncOp::value_type &&res ) { @@ -55,7 +61,7 @@ namespace detail { } } - std::vector< std::unique_ptr> > _allOps; + std::vector< std::shared_ptr> > _allOps; std::vector< InnerResult > _allResults; }; @@ -65,10 +71,10 @@ namespace detail { * Returns a async operation that waits for all async operations that are passed to it and collects their results, * forwarding them as one */ -template < class Res > -auto waitFor () { - return std::make_unique>>(); +template < class Res > +auto waitFor ( ) { + return std::make_shared>>(); } - +} #endif diff --git a/zypp-core/zyppng/rpc/MessageStream b/zypp-core/zyppng/rpc/MessageStream new file mode 100644 index 0000000..d0a2716 --- /dev/null +++ b/zypp-core/zyppng/rpc/MessageStream @@ -0,0 +1 @@ +#include "messagestream.h" \ No newline at end of file diff --git a/zypp-core/zyppng/rpc/messagestream.cc b/zypp-core/zyppng/rpc/messagestream.cc new file mode 100644 index 0000000..0c48ad3 --- /dev/null +++ b/zypp-core/zyppng/rpc/messagestream.cc @@ -0,0 +1,178 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +----------------------------------------------------------------------*/ + +#include "messagestream.h" + +#include +#include + +namespace zyppng { + + InvalidMessageReceivedException::InvalidMessageReceivedException( const std::string &msg ) + : zypp::Exception( zypp::str::Str() << "Invalid Message received: (" << msg <<")" ) + { } + + + zyppng::RpcMessageStream::RpcMessageStream( IODevice::Ptr iostr ) : _ioDev( std::move(iostr) ) + { + connect( *_nextMessageTimer, &Timer::sigExpired, *this, &RpcMessageStream::timeout ); + _nextMessageTimer->setSingleShot(false); + + connect( *_ioDev, &IODevice::sigReadyRead, *this, &RpcMessageStream::readAllMessages ); + if ( _ioDev->isOpen () && _ioDev->canRead () ) + readAllMessages (); + } + + bool RpcMessageStream::readNextMessage( ) + { + if ( _pendingMessageSize == 0 ) { + if ( _ioDev->bytesAvailable() >= sizeof( rpc::HeaderSizeType ) ) { + _ioDev->read( reinterpret_cast( &_pendingMessageSize ), sizeof( rpc::HeaderSizeType ) ); + } + } + + if ( _ioDev->bytesAvailable() < _pendingMessageSize ) { + return false; + } + + auto bytes = _ioDev->read( _pendingMessageSize ); + _pendingMessageSize = 0; + + zypp::proto::Envelope m; + if (! m.ParseFromArray( bytes.data(), bytes.size() ) ) { + ERR << "Received malformed message from peer" << std::endl; + _sigInvalidMessageReceived.emit(); + return false; + } + + _messages.push_back( std::move(m) ); + _sigNextMessage.emit (); + + if ( _messages.size() ) { + // nag the user code until all messages have been used up + _nextMessageTimer->start(0); + } + + return true; + } + + void RpcMessageStream::timeout(const Timer &) + { + if ( _messages.size() ) + _sigNextMessage.emit(); + + if ( !_messages.size() ) + _nextMessageTimer->stop(); + } + + std::optional zyppng::RpcMessageStream::nextMessage( const std::string &msgName ) + { + if ( !_messages.size () ) { + + // try to read the next messages from the fd + { + _sigNextMessage.block (); + zypp::OnScopeExit unblock([&](){ + _sigNextMessage.unblock(); + }); + readAllMessages(); + } + + if ( !_messages.size () ) + return {}; + } + + std::optional res; + + if( msgName.empty() ) { + res = std::move( _messages.front () ); + _messages.pop_front(); + + } else { + const auto i = std::find_if( _messages.begin(), _messages.end(), [&]( const zypp::proto::Envelope &env ) { + return env.messagetypename() == msgName; + }); + + if ( i != _messages.end() ) { + res = std::move(*i); + _messages.erase(i); + } + } + + if ( _messages.size() ) + _nextMessageTimer->start(0); + else + _nextMessageTimer->stop(); + + return res; + } + + std::optional RpcMessageStream::nextMessageWait( const std::string &msgName ) + { + // make sure the signal is not emitted until we have the next message + _sigNextMessage.block (); + zypp::OnScopeExit unblock([&](){ + _sigNextMessage.unblock(); + }); + + bool receivedInvalidMsg = false; + AutoDisconnect defered( connectFunc( *this, &RpcMessageStream::sigInvalidMessageReceived, [&](){ + receivedInvalidMsg = true; + })); + + const bool hasMsgName = msgName.size(); + while ( !receivedInvalidMsg && _ioDev->isOpen() && _ioDev->canRead() ) { + if ( _messages.size() ) { + if ( hasMsgName ) { + std::optional msg = nextMessage(msgName); + if ( msg ) return msg; + } + else { + break; + } + } + + if ( !_ioDev->waitForReadyRead ( -1 ) ) { + // this can only mean that a error happened, like device was closed + return {}; + } + } + return nextMessage (msgName); + } + + bool zyppng::RpcMessageStream::sendMessage( const RpcMessage &env ) + { + if ( !_ioDev->canWrite () ) + return false; + + const auto &str = env.SerializeAsString(); + rpc::HeaderSizeType msgSize = str.length(); + _ioDev->write( (char *)(&msgSize), sizeof( rpc::HeaderSizeType ) ); + _ioDev->write( str.data(), str.size() ); + return true; + } + + SignalProxy zyppng::RpcMessageStream::sigMessageReceived() + { + return _sigNextMessage; + } + + SignalProxy RpcMessageStream::sigInvalidMessageReceived() + { + return _sigInvalidMessageReceived; + } + + void RpcMessageStream::readAllMessages() + { + bool cont = true; + while ( cont && _ioDev->bytesAvailable() ) { + cont = readNextMessage (); + } + } +} diff --git a/zypp-core/zyppng/rpc/messagestream.h b/zypp-core/zyppng/rpc/messagestream.h new file mode 100644 index 0000000..6669e3e --- /dev/null +++ b/zypp-core/zyppng/rpc/messagestream.h @@ -0,0 +1,178 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +-----------------------------------------------------------------------/ +* +* This file contains private API, this might break at any time between releases. +* You have been warned! +* +*/ + +#ifndef ZYPP_CORE_ZYPPNG_RPC_MESSAGESTREAM_H_INCLUDED +#define ZYPP_CORE_ZYPPNG_RPC_MESSAGESTREAM_H_INCLUDED + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace zyppng { + + using RpcMessage = zypp::proto::Envelope; + + namespace rpc { + /*! + * Helper function to get the type name of a given RPC message type. + * Sadly Protobuf does not offer a static function to get the types FQN we + * cache it after asking for it the first time. So we need a dummy object just once. + */ + template + const std::string & messageTypeName() { + static std::string name = T().GetTypeName(); + return name; + } + } + + class InvalidMessageReceivedException : public zypp::Exception + { + public: + InvalidMessageReceivedException( const std::string &msg = {}); + }; + + + /*! + * + * Implements the basic protocol for sending zypp RPC messages over a IODevice + * + * Communication Format: + * --------------------- + * Each message is serialized into a zypp.proto.Envelope and sent over the communication medium in binary + * format. The binary format looks like: + * + * +--------------------------------+---------------------------------+ + * | msglen ( 32 bit unsigned int ) | binary zypp.proto.Envelope data | + * +--------------------------------+---------------------------------+ + * + * The header defines the size in bytes of the following data trailer. The header type is a 32 bit uint, endianess is defined by + * the underlying CPU arch. The data portion is directly generated by libprotobuf via SerializeToZeroCopyStream() to generate + * the binary represenation of the message. + * + */ + class RpcMessageStream : public zyppng::Base + { + public: + + using Ptr = std::shared_ptr; + + /*! + * Uses the given iostream to send and receive messages. + * If the device is already open and readable tries to read messages right away. + * So make sure to check if messages have already been received via \ref nextMessage + */ + static Ptr create( IODevice::Ptr iostr ) { + return Ptr( new RpcMessageStream( std::move(iostr) ) ); + } + + /*! + * Returns the next message in the queue, wait for the \ref sigMessageReceived signal + * to know when new messages have arrived. + * If \a msgName is specified returns the next message in the queue that matches the msgName + */ + std::optional nextMessage ( const std::string &msgName = "" ); + + /*! + * Waits until at least one message is in the queue and returns it. Will return a empty + * optional if a error occurs. + * + * If \a msgName is set this will block until a message with the given message name arrives and returns it + * + * \note Make sure to check if there are more than one messages in the queue after this function returns + */ + std::optional nextMessageWait ( const std::string &msgName = "" ); + + /*! + * Send out a RpcMessage to the other side, depending on the underlying device state + * this will be buffered and send when the device is writeable again. + */ + bool sendMessage ( const RpcMessage &env ); + + /*! + * Reads all messages from the underlying IO Device, this is usually called automatically + * but when shutting down this can be used to process all remaining messages. + */ + void readAllMessages (); + + /*! + * Send a messagee to the server side, it will be enclosed in a Envelope + * and immediately sent out. + */ + template + std::enable_if_t< !std::is_same_v, bool> sendMessage ( const T &m ) { + RpcMessage env; + + env.set_messagetypename( m.GetTypeName() ); + m.SerializeToString( env.mutable_value() ); + + return sendMessage ( env ); + } + + /*! + * Emitted when new messages have arrived. This will continuously be emitted + * as long as messages are in the queue. + */ + SignalProxy sigMessageReceived (); + + /*! + * Signal is emitted every time there was data on the line that could not be parsed + */ + SignalProxy sigInvalidMessageReceived (); + + template + static expected< T > parseMessage ( const RpcMessage &m ) { + T p; + if ( !p.ParseFromString( m.value() ) ) { + const auto &msg = zypp::str::Str() << "Failed to parse " << m.messagetypename() << " message."; + ERR << msg << std::endl ; + return expected::error( ZYPP_EXCPT_PTR ( InvalidMessageReceivedException(msg) ) ); + } + return expected::success( std::move(p) ); + } + + template + static expected< void > parseMessageInto ( const RpcMessage &m, T &target ) { + if ( !target.ParseFromString( m.value() ) ) { + const auto &msg = zypp::str::Str() << "Failed to parse " << m.messagetypename() << " message."; + ERR << msg << std::endl ; + return expected::error( ZYPP_EXCPT_PTR ( InvalidMessageReceivedException(msg) ) ); + } + return expected::success( ); + } + + private: + RpcMessageStream( IODevice::Ptr iostr ); + bool readNextMessage (); + void timeout( const zyppng::Timer &); + + IODevice::Ptr _ioDev; + Timer::Ptr _nextMessageTimer = Timer::create(); + zyppng::rpc::HeaderSizeType _pendingMessageSize = 0; + std::deque _messages; + Signal _sigNextMessage; + Signal _sigInvalidMessageReceived; + + }; +} + + + +#endif // ZYPP_CORE_ZYPPNG_RPC_MESSAGESTREAM_H_INCLUDED diff --git a/zypp-curl/CMakeLists.txt b/zypp-curl/CMakeLists.txt index 6ad6307..5c9d872 100644 --- a/zypp-curl/CMakeLists.txt +++ b/zypp-curl/CMakeLists.txt @@ -166,3 +166,7 @@ SET_LOGGROUP( "zypp-curl" $${zypp_curl_lib_SRCS} ) ADD_LIBRARY( zypp-curl STATIC ${zypp_curl_lib_SRCS} ${zypp_curl_lib_HEADERS} ) #we include generated headers, so we need to wait for zypp-protobuf to be ready add_dependencies( zypp-curl zypp-protobuf ) + +TARGET_LINK_LIBRARIES( zypp-curl ${CURL_LIBRARIES} ) +TARGET_LINK_LIBRARIES( zypp-curl ${LIBPROXY_LIBRARIES} ) +TARGET_LINK_LIBRARIES( zypp-curl ${LIBXML2_LIBRARIES} ) diff --git a/zypp-curl/curlhelper.cc b/zypp-curl/curlhelper.cc index e0e3dab..75a7076 100644 --- a/zypp-curl/curlhelper.cc +++ b/zypp-curl/curlhelper.cc @@ -21,6 +21,7 @@ #include #include #include +#include using std::endl; using namespace zypp; @@ -356,7 +357,8 @@ Url clearQueryString(const Url &url) // bsc#933839: propagate proxy settings passed in the repo URL zypp::Url propagateQueryParams( zypp::Url url_r, const zypp::Url & template_r ) { - for ( const std::string ¶m : { "proxy", "proxyport", "proxyuser", "proxypass"} ) + using namespace std::literals::string_literals; + for ( const std::string ¶m : { "proxy"s, "proxyport"s, "proxyuser"s, "proxypass"s} ) { const std::string & value( template_r.getQueryParam( param ) ); if ( ! value.empty() ) diff --git a/zypp-curl/ng/network/downloader.cc b/zypp-curl/ng/network/downloader.cc index b97e4bb..66d2541 100644 --- a/zypp-curl/ng/network/downloader.cc +++ b/zypp-curl/ng/network/downloader.cc @@ -37,61 +37,54 @@ namespace zyppng { MIL << "Authentication failed for " << req->url() << " trying to recover." << std::endl; - zypp::media::CredentialManager cm( _parent ? _parent->credManagerOptions() : zypp::media::CredManagerOptions() ); - auto authDataPtr = cm.getCred( req->url() ); - - // get stored credentials - NetworkAuthData_Ptr cmcred( authDataPtr ? new NetworkAuthData( *authDataPtr ) : new NetworkAuthData() ); TransferSettings &ts = req->transferSettings(); - - // We got credentials from store, _triedCredFromStore makes sure we just try the auth from store once - // if its timestamp does not change between 2 retries - if ( cmcred && ( !req->_triedCredFromStore || req->_authTimestamp < cmcred->lastDatabaseUpdate() ) ) { - MIL << "got stored credentials:" << std::endl << *cmcred << std::endl; - ts.setUsername( cmcred->username() ); - ts.setPassword( cmcred->password() ); - retry = true; - req->_triedCredFromStore = true; - _lastTriedAuthTime = req->_authTimestamp = cmcred->lastDatabaseUpdate(); - } else { - - //we did not get credentials from the store, emit a signal that allows - //setting new auth data - - NetworkAuthData credFromUser; - credFromUser.setUrl( req->url() ); - - //in case we got a auth hint from the server the error object will contain it - std::string authHint = err.extraInfoValue("authHint", std::string()); - - //preset from store if we found something - if ( cmcred && !cmcred->username().empty() ) - credFromUser.setUsername( cmcred->username() ); - - _sigAuthRequired.emit( *z_func(), credFromUser, authHint ); - if ( credFromUser.valid() ) { - MIL << "Got user provided credentials" << std::endl; - ts.setUsername( credFromUser.username() ); - ts.setPassword( credFromUser.password() ); - + const auto &applyCredToSettings = [&ts]( AuthData_Ptr auth, const std::string &authHint ) { + ts.setUsername( auth->username() ); + ts.setPassword( auth->password() ); + auto nwCred = dynamic_cast( auth.get() ); + if ( nwCred ) { // set available authentication types from the error - if ( credFromUser.authType() == CURLAUTH_NONE ) - credFromUser.setAuthType( authHint ); + if ( nwCred->authType() == CURLAUTH_NONE ) + nwCred->setAuthType( authHint ); // set auth type (seems this must be set _after_ setting the userpwd) - if ( credFromUser.authType() != CURLAUTH_NONE ) { + if ( nwCred->authType() != CURLAUTH_NONE ) { // FIXME: only overwrite if not empty? - req->transferSettings().setAuthType(credFromUser.authTypeAsString()); + ts.setAuthType(nwCred->authTypeAsString()); } + } + }; + + // try to find one in the cache + zypp::url::ViewOption vopt; + vopt = vopt + - zypp::url::ViewOption::WITH_USERNAME + - zypp::url::ViewOption::WITH_PASSWORD + - zypp::url::ViewOption::WITH_QUERY_STR; + + auto cachedCred = zypp::media::CredentialManager::findIn( _credCache, req->url(), vopt ); + + // only consider a cache entry if its newer than what we tried last time + if ( cachedCred && cachedCred->lastDatabaseUpdate() > req->_authTimestamp ) { + MIL << "Found a credential match in the cache!" << std::endl; + applyCredToSettings( cachedCred, "" ); + _lastTriedAuthTime = req->_authTimestamp = cachedCred->lastDatabaseUpdate(); + retry = true; + } else { - cm.addCred( credFromUser ); - cm.save(); + NetworkAuthData_Ptr credFromUser = NetworkAuthData_Ptr( new NetworkAuthData() ); + credFromUser->setUrl( req->url() ); + credFromUser->setLastDatabaseUpdate ( req->_authTimestamp ); - // potentially setting this after the file has been touched we might miss a change - // in a later loop, if another update to the store happens right in the few ms difference - // between the actual file write and calling time() here. However this is highly unlikely - _lastTriedAuthTime = req->_authTimestamp = time( nullptr ) ; + //in case we got a auth hint from the server the error object will contain it + std::string authHint = err.extraInfoValue("authHint", std::string()); + _sigAuthRequired.emit( *z_func(), *credFromUser, authHint ); + if ( credFromUser->valid() ) { + // remember for next time , we don't want to ask the user again for the same URL set + _credCache.insert( credFromUser ); + applyCredToSettings( credFromUser, authHint ); + _lastTriedAuthTime = req->_authTimestamp = credFromUser->lastDatabaseUpdate(); retry = true; } } @@ -143,8 +136,10 @@ namespace zyppng { } //reset state variables - _emittedSigStart = false; - _specHasZckInfo = zypp::indeterminate; + _specHasZckInfo = zypp::indeterminate; + _emittedSigStart = false; + _stoppedOnMetalink = false; + _lastTriedAuthTime = 0; // restart the statemachine if ( cState == Download::Finished ) @@ -170,6 +165,7 @@ namespace zyppng { if ( _spec.settings().proxy().empty() ) ::internal::fillSettingsSystemProxy( url, set ); +#if 0 /* Fixes bsc#1174011 "auth=basic ignored in some cases" * We should proactively add the password to the request if basic auth is configured * and a password is available in the credentials but not in the URL. @@ -188,6 +184,7 @@ namespace zyppng { set.setPassword(cred->password()); } } +#endif } catch ( const zypp::media::MediaBadUrlException & e ) { res = NetworkRequestErrorPrivate::customError( NetworkRequestError::MalformedURL, e.asString(), buildExtraInfo() ); @@ -275,6 +272,16 @@ namespace zyppng { d->forceState ( std::make_unique( NetworkRequestErrorPrivate::customError( NetworkRequestError::Cancelled, "Download was cancelled explicitly" ), *d_func() ) ); } + void Download::setStopOnMetalink(const bool set) + { + d_func()->_stopOnMetalink = set; + } + + bool Download::stoppedOnMetalink() const + { + return d_func()->_stoppedOnMetalink; + } + DownloadSpec &Download::spec() { return d_func()->_spec; @@ -378,16 +385,6 @@ namespace zyppng { } } - const zypp::media::CredManagerOptions &Downloader::credManagerOptions() const - { - return d_func()->_credManagerOptions; - } - - void Downloader::setCredManagerOptions(const zypp::media::CredManagerOptions &options) - { - d_func()->_credManagerOptions = options; - } - std::shared_ptr Downloader::downloadFile(const zyppng::DownloadSpec &spec ) { Z_D(); diff --git a/zypp-curl/ng/network/downloader.h b/zypp-curl/ng/network/downloader.h index 60690b5..a98e7aa 100644 --- a/zypp-curl/ng/network/downloader.h +++ b/zypp-curl/ng/network/downloader.h @@ -5,7 +5,6 @@ #include #include #include -#include #include #include @@ -46,16 +45,6 @@ namespace zyppng { virtual ~Downloader(); /*! - * Returns the currently used CredentialManager options - */ - const zypp::media::CredManagerOptions &credManagerOptions () const; - - /*! - * Sets the options for the CredentialManager to retrieve auth data - */ - void setCredManagerOptions ( const zypp::media::CredManagerOptions & options ); - - /*! * Generates a new Download object in waiting state */ std::shared_ptr downloadFile ( const DownloadSpec &spec ); @@ -204,6 +193,18 @@ namespace zyppng { void cancel (); /*! + * This will finalize the download once a metalink file was received. This is a special setting used + * if the metalink file should be processed outside of the Downloader. + */ + void setStopOnMetalink ( const bool set = true ); + + /*! + * Returns true if the download was stopped after receiving metalink data. The target file + * will contain the metalink description. + */ + bool stoppedOnMetalink () const; + + /*! * Returns a reference to the internally used download spec. * \sa zyppng::DownloadSpec * \note Changing settings after the download started might result in undefined or weird behaviour @@ -250,6 +251,10 @@ namespace zyppng { /*! * Is emitted when a request requires authentication and it was not given or if auth failed. * A connected slot should fill in the \a auth information in order to provide login credentials. + * + * \note the URL in the \a auth data is set to the url of the request needing authorization. + * This URL can be different than the one in the spec, due to redirect/metalink. + * Also the lastDatabaseUpdate field is set to the timestamp of the auth data that was tried already ( if available ) */ SignalProxy sigAuthRequired ( ); diff --git a/zypp-curl/ng/network/downloadspec.cc b/zypp-curl/ng/network/downloadspec.cc index 34a3d3d..affae10 100644 --- a/zypp-curl/ng/network/downloadspec.cc +++ b/zypp-curl/ng/network/downloadspec.cc @@ -8,7 +8,6 @@ ----------------------------------------------------------------------*/ #include "downloadspec.h" #include -#include namespace zyppng { @@ -17,13 +16,21 @@ namespace zyppng { DownloadSpecPrivate() = default; DownloadSpecPrivate( const DownloadSpecPrivate &other ) = default; DownloadSpecPrivate( DownloadSpecPrivate &&other ) = default; - DownloadSpecPrivate( zypp::proto::DownloadSpec spec ) :_protoData( std::move(spec) ) {} DownloadSpecPrivate *clone () const { return new DownloadSpecPrivate(*this); } - zypp::proto::DownloadSpec _protoData; + zypp::Url _url; + TransferSettings _settings; + zypp::Pathname _delta; + zypp::ByteCount _expectedFileSize; + zypp::Pathname _targetPath; + bool _checkExistanceOnly = false; //< this will NOT download the file, but only query the server if it exists + bool _metalink_enabled = true; //< should the download try to use metalinks + zypp::ByteCount _headerSize; //< Optional file header size for things like zchunk + std::optional _headerChecksum; //< Optional file header checksum + zypp::ByteCount _preferred_chunk_size = zypp::ByteCount( 4096, zypp::ByteCount::K ); }; ZYPP_IMPL_PRIVATE( DownloadSpec ) @@ -31,161 +38,134 @@ namespace zyppng { DownloadSpec::DownloadSpec( Url file , zypp::filesystem::Pathname targetPath, zypp::ByteCount expectedFileSize ) : d_ptr( new DownloadSpecPrivate() ) { // default settings - *d_ptr->_protoData.mutable_settings() = std::move( TransferSettings().protoData() ); - - setUrl( file ); - setTargetPath( targetPath ); - setExpectedFileSize( expectedFileSize ); - setPreferredChunkSize( zypp::ByteCount( 4096, zypp::ByteCount::K ) ); - d_ptr->_protoData.set_checkexistanceonly( false ); - d_ptr->_protoData.set_metalink_enabled( true ); + d_ptr->_url = std::move(file); + d_ptr->_targetPath = std::move(targetPath); + d_ptr->_expectedFileSize = std::move( expectedFileSize ); } - DownloadSpec::DownloadSpec( const zypp::proto::DownloadSpec &spec ) : d_ptr( new DownloadSpecPrivate( spec ) ) - { } - DownloadSpec::DownloadSpec( const DownloadSpec &other ) = default; DownloadSpec &DownloadSpec::operator=(const DownloadSpec &other) = default; - Url DownloadSpec::url() const + const Url &DownloadSpec::url() const { - return d_ptr->_protoData.url(); + return d_ptr->_url; } DownloadSpec &DownloadSpec::setUrl(const Url &url) { - d_ptr->_protoData.set_url( url.asCompleteString() ); + d_ptr->_url = url; return *this; } - zypp::Pathname DownloadSpec::targetPath() const + const zypp::Pathname &DownloadSpec::targetPath() const { - return zypp::Pathname(d_ptr->_protoData.targetpath()); + return d_ptr->_targetPath; } DownloadSpec &DownloadSpec::setTargetPath(const zypp::filesystem::Pathname &path) { - d_ptr->_protoData.set_targetpath( path.asString() ); + d_ptr->_targetPath = path; return *this; } DownloadSpec &DownloadSpec::setMetalinkEnabled(bool enable) { - d_ptr->_protoData.set_metalink_enabled( enable ); + d_ptr->_metalink_enabled = enable; return *this; } bool DownloadSpec::metalinkEnabled() const { - return d_ptr->_protoData.metalink_enabled(); + return d_ptr->_metalink_enabled; } DownloadSpec &DownloadSpec::setCheckExistsOnly(bool set) { - d_ptr->_protoData.set_checkexistanceonly( set ); + d_ptr->_checkExistanceOnly = ( set ); return *this; } bool DownloadSpec::checkExistsOnly() const { - return d_ptr->_protoData.checkexistanceonly(); + return d_ptr->_checkExistanceOnly; } - DownloadSpec &DownloadSpec::setDeltaFile(const zypp::filesystem::Pathname &file) + DownloadSpec &DownloadSpec::setDeltaFile(const zypp::Pathname &file) { - if ( !file.empty() ) - d_ptr->_protoData.set_delta( file.asString() ); - else - d_ptr->_protoData.clear_delta(); + d_ptr->_delta = file; return *this; } - zypp::filesystem::Pathname DownloadSpec::deltaFile() const + zypp::Pathname DownloadSpec::deltaFile() const { - return d_ptr->_protoData.delta(); + return d_ptr->_delta; } DownloadSpec &DownloadSpec::setPreferredChunkSize(const zypp::ByteCount &bc) { - d_ptr->_protoData.set_preferred_chunk_size( bc.operator long long() ); + d_ptr->_preferred_chunk_size = bc; return *this; } zypp::ByteCount DownloadSpec::preferredChunkSize() const { - return d_ptr->_protoData.preferred_chunk_size(); + return d_ptr->_preferred_chunk_size; } - TransferSettings DownloadSpec::settings() const + const TransferSettings &DownloadSpec::settings() const { - return TransferSettings( d_ptr->_protoData.settings() ); + return d_ptr->_settings; } DownloadSpec & DownloadSpec::setTransferSettings(TransferSettings &&set) { - (*d_ptr->_protoData.mutable_settings()) = std::move( set.protoData() ); + d_ptr->_settings = std::move( set ); return *this; } DownloadSpec & DownloadSpec::setTransferSettings(const TransferSettings &set) { - (*d_ptr->_protoData.mutable_settings()) = set.protoData(); + d_ptr->_settings = set; return *this; } DownloadSpec &DownloadSpec::setExpectedFileSize(const zypp::ByteCount &bc) { - d_ptr->_protoData.set_expectedfilesize( bc.operator long long() ); + d_ptr->_expectedFileSize = bc; return *this; } zypp::ByteCount DownloadSpec::expectedFileSize() const { - return d_ptr->_protoData.expectedfilesize(); + return d_ptr->_expectedFileSize; } DownloadSpec &DownloadSpec::setHeaderSize(const zypp::ByteCount &bc) { - d_ptr->_protoData.set_headersize( bc.operator long long() ); + d_ptr->_headerSize = bc; return *this; } zypp::ByteCount DownloadSpec::headerSize() const { - return d_ptr->_protoData.headersize(); + return d_ptr->_headerSize; } - std::optional DownloadSpec::headerChecksum() const + const std::optional &DownloadSpec::headerChecksum() const { Z_D(); - if ( !d->_protoData.has_headerchecksum() ) - return {}; - - return zypp::CheckSum( d->_protoData.headerchecksum().type(), d->_protoData.headerchecksum().sum() ); - + return d->_headerChecksum; } DownloadSpec &DownloadSpec::setHeaderChecksum(const zypp::CheckSum &sum) { + Z_D(); if ( sum.empty() ) - d_func()->_protoData.clear_headerchecksum(); + d->_headerChecksum.reset(); else { - auto csum = d_func()->_protoData.mutable_headerchecksum(); - csum->set_type( sum.type() ); - csum->set_sum( sum.checksum() ); + d->_headerChecksum = sum; } return *this; } - - const zypp::proto::DownloadSpec &DownloadSpec::protoData() const - { - return d_ptr->_protoData; - } - - zypp::proto::DownloadSpec &DownloadSpec::protoData() - { - return d_ptr->_protoData; - } } - diff --git a/zypp-curl/ng/network/downloadspec.h b/zypp-curl/ng/network/downloadspec.h index bfa0034..25ded18 100644 --- a/zypp-curl/ng/network/downloadspec.h +++ b/zypp-curl/ng/network/downloadspec.h @@ -24,10 +24,6 @@ #include -namespace zypp::proto { - class DownloadSpec; -} - namespace zyppng { @@ -45,7 +41,6 @@ namespace zyppng { public: DownloadSpec( Url file, zypp::filesystem::Pathname targetPath, zypp::ByteCount expectedFileSize = zypp::ByteCount() ); - DownloadSpec( const zypp::proto::DownloadSpec &spec ); DownloadSpec( const DownloadSpec &other ); DownloadSpec &operator= ( const DownloadSpec &other ); @@ -53,13 +48,13 @@ namespace zyppng { /*! * Returns the source URL of the download */ - Url url () const; + const Url &url () const; DownloadSpec &setUrl ( const Url &url ); /*! * Returns the target file path, this is where the downloaded data is stored */ - zypp::filesystem::Pathname targetPath() const; + const zypp::Pathname &targetPath() const; DownloadSpec &setTargetPath ( const zypp::Pathname &path ); /*! @@ -95,7 +90,7 @@ namespace zyppng { * possible sub downloads, however authentication data is stripped if the subdownload uses a different host to * fetch the data from. If there is no auth data known \sa sigAuthRequired is emitted. */ - TransferSettings settings () const; + const TransferSettings &settings () const; DownloadSpec &setTransferSettings( TransferSettings &&set ); DownloadSpec &setTransferSettings( const TransferSettings &set ); @@ -105,19 +100,13 @@ namespace zyppng { DownloadSpec &setHeaderSize ( const zypp::ByteCount &bc ); zypp::ByteCount headerSize() const; - std::optional headerChecksum () const; + const std::optional &headerChecksum () const; DownloadSpec &setHeaderChecksum ( const zypp::CheckSum &sum ); - const zypp::proto::DownloadSpec &protoData() const; - zypp::proto::DownloadSpec &protoData(); - private: zypp::RWCOW_pointer d_ptr; }; } - - - #endif // ZYPPNG_MEDIA_NETWORK_DOWNLOADSPEC_H diff --git a/zypp-curl/ng/network/networkrequestdispatcher.cc b/zypp-curl/ng/network/networkrequestdispatcher.cc index 70c896e..1fbf547 100644 --- a/zypp-curl/ng/network/networkrequestdispatcher.cc +++ b/zypp-curl/ng/network/networkrequestdispatcher.cc @@ -23,9 +23,7 @@ using namespace boost; -namespace zypp { - L_ENV_CONSTR_DEFINE_FUNC(ZYPP_MEDIA_CURL_DEBUG) -} +L_ENV_CONSTR_DEFINE_FUNC(ZYPP_MEDIA_CURL_DEBUG) namespace zyppng { diff --git a/zypp-curl/ng/network/private/downloader_p.h b/zypp-curl/ng/network/private/downloader_p.h index 1f79493..407841c 100644 --- a/zypp-curl/ng/network/private/downloader_p.h +++ b/zypp-curl/ng/network/private/downloader_p.h @@ -110,7 +110,6 @@ namespace zyppng { Signal< void ( Downloader &parent, Download& download )> _sigFinished; Signal< void ( Downloader &parent )> _queueEmpty; std::shared_ptr _mirrors; - zypp::media::CredManagerOptions _credManagerOptions; //< The credential manager options used to initialize the CredentialManager }; } diff --git a/zypp-curl/ng/network/private/downloaderstates/base_p.h b/zypp-curl/ng/network/private/downloaderstates/base_p.h index 99bceec..358e011 100644 --- a/zypp-curl/ng/network/private/downloaderstates/base_p.h +++ b/zypp-curl/ng/network/private/downloaderstates/base_p.h @@ -24,6 +24,7 @@ #include #include #include +#include namespace zyppng { @@ -69,8 +70,7 @@ namespace zyppng { } void disconnectSignals (); - bool _triedCredFromStore = false; //< already tried to authenticate from credential store? - time_t _authTimestamp = 0; //< timestamp of the AuthData we tried from the store + time_t _authTimestamp = 0; //< timestamp of the AuthData we tried already Url _originalUrl; //< The unstripped URL as it was passed to Download , before transfer settings are removed MirrorControl::MirrorHandle _myMirror; @@ -92,12 +92,16 @@ namespace zyppng { std::shared_ptr _requestDispatcher; std::shared_ptr _mirrorControl; + zypp::media::CredentialManager::CredentialSet _credCache; //< the credential cache for this download + DownloadSpec _spec; // the download settings mutable zypp::TriBool _specHasZckInfo = zypp::indeterminate; Downloader *_parent = nullptr; - time_t _lastTriedAuthTime = 0; //< if initialized this shows the last timestamp that we loaded a cred for the given URL from CredentialManager + time_t _lastTriedAuthTime = 0; //< if initialized this shows the last timestamp that got from user code for a auth request + bool _stopOnMetalink = false; //< Stop the download if a metalink was received for external parsing + bool _stoppedOnMetalink = false; //< Statemachine was stopped after receiving a metalink file NetworkRequest::Priority _defaultSubRequestPriority = NetworkRequest::High; Signal< void ( Download &req )> _sigStarted; diff --git a/zypp-curl/ng/network/private/downloaderstates/metalinkinfo_p.cc b/zypp-curl/ng/network/private/downloaderstates/metalinkinfo_p.cc index f2c052b..9112732 100644 --- a/zypp-curl/ng/network/private/downloaderstates/metalinkinfo_p.cc +++ b/zypp-curl/ng/network/private/downloaderstates/metalinkinfo_p.cc @@ -79,6 +79,7 @@ namespace zyppng { bool DlMetaLinkInfoState::initializeRequest(std::shared_ptr &r ) { + MIL << "Requesting Metalink info from server!" << std::endl; r->transferSettings().addHeader("Accept: */*, application/metalink+xml, application/metalink4+xml"); return BasicDownloaderStateBase::initializeRequest(r); } @@ -95,8 +96,15 @@ namespace zyppng { return BasicDownloaderStateBase::gotFinished(); } + auto &sm = stateMachine(); + if ( sm._stopOnMetalink ) { + MIL << "Stopping after receiving MetaLink data as requested" << std::endl; + sm._stoppedOnMetalink = true; + return BasicDownloaderStateBase::gotFinished(); + } + // Move to Prepare Multi state - MIL << "Downloading on " << stateMachine()._spec.url() << " returned a Metalink " << std::endl; + MIL << "Downloading on " << sm._spec.url() << " returned a Metalink " << std::endl; _sigGotMetalink.emit(); } diff --git a/zypp-curl/ng/network/private/mediadebug_p.h b/zypp-curl/ng/network/private/mediadebug_p.h index 2a4cb87..0a626f9 100644 --- a/zypp-curl/ng/network/private/mediadebug_p.h +++ b/zypp-curl/ng/network/private/mediadebug_p.h @@ -15,9 +15,8 @@ #define ZYPP_NG_MEDIADEBUG_H_INCLUDED #include -namespace zypp { - L_ENV_CONSTR_FWD_DECLARE_FUNC(ZYPP_MEDIA_CURL_DEBUG); -} + +L_ENV_CONSTR_FWD_DECLARE_FUNC(ZYPP_MEDIA_CURL_DEBUG); #ifdef ZYPP_BASE_LOGGER_LOGGROUP #undef ZYPP_BASE_LOGGER_LOGGROUP diff --git a/zypp-curl/ng/network/private/request_p.h b/zypp-curl/ng/network/private/request_p.h index 9fd24be..bb88751 100644 --- a/zypp-curl/ng/network/private/request_p.h +++ b/zypp-curl/ng/network/private/request_p.h @@ -123,6 +123,7 @@ namespace zyppng { bool _isInCallback = false; std::optional _cachedResult; + off_t _lastProgressNow = -1; // last value returned from CURL, lets only send signals if we get actual updates off_t _downloaded = 0; //downloaded bytes zypp::ByteCount _contentLenght; // the content length as reported by the server diff --git a/zypp-curl/ng/network/request.cc b/zypp-curl/ng/network/request.cc index 094fea7..805781f 100644 --- a/zypp-curl/ng/network/request.cc +++ b/zypp-curl/ng/network/request.cc @@ -613,7 +613,10 @@ namespace zyppng { that->resetActivityTimer(); rmode._isInCallback = true; - that->_sigProgress.emit( *that->z_func(), dltotal, dlnow, ultotal, ulnow ); + if ( rmode._lastProgressNow != dlnow ) { + rmode._lastProgressNow = dlnow; + that->_sigProgress.emit( *that->z_func(), dltotal, dlnow, ultotal, ulnow ); + } rmode._isInCallback = false; return rmode._cachedResult ? CURLE_ABORTED_BY_CALLBACK : CURLE_OK; diff --git a/zypp-curl/parser/metalinkparser.cc b/zypp-curl/parser/metalinkparser.cc index 56ed8d4..814c541 100644 --- a/zypp-curl/parser/metalinkparser.cc +++ b/zypp-curl/parser/metalinkparser.cc @@ -414,7 +414,9 @@ MetaLinkParser::~MetaLinkParser() void MetaLinkParser::parse(const Pathname &filename) { + MIL << "Begin parse " << filename << std::endl; parse(InputStream(filename)); + MIL << "End parse " << filename << std::endl; } void diff --git a/zypp-curl/transfersettings.cc b/zypp-curl/transfersettings.cc index 631982a..1ef538b 100644 --- a/zypp-curl/transfersettings.cc +++ b/zypp-curl/transfersettings.cc @@ -22,8 +22,6 @@ #include #include -#include - #include using std::endl; @@ -37,19 +35,18 @@ namespace zypp class TransferSettings::Impl { public: - Impl() { - _settingsObj.set_useproxy( false ); - _settingsObj.set_timeout( MediaConfig::instance().download_transfer_timeout() ); - _settingsObj.set_connect_timeout( 60 ); - _settingsObj.set_maxconcurrentconnections( MediaConfig::instance().download_max_concurrent_connections() ); - _settingsObj.set_mindownloadspeed(MediaConfig::instance().download_min_download_speed()); - _settingsObj.set_maxdownloadspeed(MediaConfig::instance().download_max_download_speed()); - _settingsObj.set_maxsilenttries(MediaConfig::instance().download_max_silent_tries() ); - _settingsObj.set_verify_host(false); - _settingsObj.set_verify_peer(false); - _settingsObj.set_ca_path("/etc/ssl/certs"); - _settingsObj.set_head_requests_allowed(true); - } + Impl() : _useproxy( false ), + _timeout( MediaConfig::instance().download_transfer_timeout() ), + _connect_timeout( 60 ), + _maxConcurrentConnections( MediaConfig::instance().download_max_concurrent_connections() ), + _minDownloadSpeed(MediaConfig::instance().download_min_download_speed()), + _maxDownloadSpeed(MediaConfig::instance().download_max_download_speed()), + _maxSilentTries(MediaConfig::instance().download_max_silent_tries() ), + _verify_host(false), + _verify_peer(false), + _ca_path("/etc/ssl/certs"), + _head_requests_allowed(true) + {} virtual ~Impl() {} @@ -69,54 +66,81 @@ namespace zypp public: std::vector _headers; - zypp::proto::TransferSettings _settingsObj; + std::string _useragent; + std::string _username; + std::string _password; + bool _useproxy; + std::string _proxy; + std::string _proxy_username; + std::string _proxy_password; + std::string _authtype; + long _timeout; + long _connect_timeout; + Url _url; + Pathname _targetdir; + + long _maxConcurrentConnections; + long _minDownloadSpeed; + long _maxDownloadSpeed; + long _maxSilentTries; + + bool _verify_host; + bool _verify_peer; + Pathname _ca_path; + Pathname _client_cert_path; + Pathname _client_key_path; + + // workarounds + bool _head_requests_allowed; }; TransferSettings::TransferSettings() : _impl(new TransferSettings::Impl()) {} - TransferSettings::TransferSettings( const proto::TransferSettings &settings ) - : _impl(new TransferSettings::Impl()) - { - _impl->_settingsObj = settings; - } - void TransferSettings::reset() { _impl.reset(new TransferSettings::Impl()); } + void TransferSettings::addHeader( const std::string & val_r ) + { if ( ! val_r.empty() ) _impl->_headers.push_back(val_r); } + void TransferSettings::addHeader( std::string && val_r ) - { if ( ! val_r.empty() ) *_impl->_settingsObj.add_header() = std::move(val_r); } + { if ( ! val_r.empty() ) _impl->_headers.push_back(std::move(val_r)); } - TransferSettings::Headers TransferSettings::headers() const + const TransferSettings::Headers &TransferSettings::headers() const { //@TODO check if we could use a vector of std::string_view here - auto vec = Headers(); - for ( const auto &head : _impl->_settingsObj.header() ) { - vec.push_back( head ); - } - return vec; + return _impl->_headers; } + void TransferSettings::setUserAgentString( const std::string && val_r ) + { _impl->_useragent = val_r; } + void TransferSettings::setUserAgentString( std::string && val_r ) - { _impl->_settingsObj.set_useragent( std::move(val_r) ); } + { _impl->_useragent = std::move(val_r); } - std::string TransferSettings::userAgentString() const - { return _impl->_settingsObj.useragent(); } + const std::string &TransferSettings::userAgentString() const + { return _impl->_useragent; } + void TransferSettings::setUsername( const std::string &val_r ) + { _impl->_username = val_r; } + void TransferSettings::setUsername( std::string && val_r ) - { _impl->_settingsObj.set_username( std::move(val_r) ); } + { _impl->_username = std::move(val_r); } + + const std::string &TransferSettings::username() const + { return _impl->_username; } - std::string TransferSettings::username() const - { return _impl->_settingsObj.username(); } + void TransferSettings::setPassword( const std::string & val_r ) + { _impl->_password = val_r; } void TransferSettings::setPassword( std::string && val_r ) - { _impl->_settingsObj.set_password( std::move(val_r) ); } + { _impl->_password = std::move(val_r); } - std::string TransferSettings::password() const - { return _impl->_settingsObj.password(); } + const std::string &TransferSettings::password() const + { return _impl->_password; } std::string TransferSettings::userPassword() const { @@ -135,30 +159,39 @@ namespace zypp void TransferSettings::setProxyEnabled( bool enabled ) - { _impl->_settingsObj.set_useproxy( enabled ); } + { _impl->_useproxy = enabled; } bool TransferSettings::proxyEnabled() const - { return _impl->_settingsObj.useproxy(); } + { return _impl->_useproxy; } + void TransferSettings::setProxy( const std::string &val_r ) + { _impl->_proxy = val_r; } + void TransferSettings::setProxy( std::string && val_r ) - { _impl->_settingsObj.set_proxy( std::move(val_r) ); } + { _impl->_proxy = std::move(val_r); } + + const std::string &TransferSettings::proxy() const + { return _impl->_proxy; } - std::string TransferSettings::proxy() const - { return _impl->_settingsObj.proxy(); } + void TransferSettings::setProxyUsername( const std::string &val_r ) + { _impl->_proxy_username = val_r; } void TransferSettings::setProxyUsername( std::string && val_r ) - { _impl->_settingsObj.set_proxy_username( std::move(val_r) ); } + { _impl->_proxy_username = std::move(val_r); } + + const std::string &TransferSettings::proxyUsername() const + { return _impl->_proxy_username; } - std::string TransferSettings::proxyUsername() const - { return _impl->_settingsObj.proxy_username(); } + void TransferSettings::setProxyPassword( const std::string &val_r ) + { _impl->_proxy_password = val_r; } void TransferSettings::setProxyPassword( std::string && val_r ) - { _impl->_settingsObj.set_proxy_password( std::move(val_r) ); } + { _impl->_proxy_password = std::move(val_r); } - std::string TransferSettings::proxyPassword() const - { return _impl->_settingsObj.proxy_password(); } + const std::string &TransferSettings::proxyPassword() const + { return _impl->_proxy_password; } std::string TransferSettings::proxyUserPassword() const { @@ -171,104 +204,105 @@ namespace zypp void TransferSettings::setTimeout( long t ) - { _impl->_settingsObj.set_timeout(t); } + { _impl->_timeout = (t); } long TransferSettings::timeout() const - { return _impl->_settingsObj.timeout(); } + { return _impl->_timeout; } void TransferSettings::setConnectTimeout( long t ) - { _impl->_settingsObj.set_connect_timeout(t); } + { _impl->_connect_timeout = (t); } long TransferSettings::connectTimeout() const - { return _impl->_settingsObj.connect_timeout(); } + { return _impl->_connect_timeout; } void TransferSettings::setMaxConcurrentConnections( long v ) - { _impl->_settingsObj.set_maxconcurrentconnections(v); } + { _impl->_maxConcurrentConnections = (v); } long TransferSettings::maxConcurrentConnections() const - { return _impl->_settingsObj.maxconcurrentconnections(); } + { return _impl->_maxConcurrentConnections; } void TransferSettings::setMinDownloadSpeed( long v ) - { _impl->_settingsObj.set_mindownloadspeed(v); } + { _impl->_minDownloadSpeed = (v); } long TransferSettings::minDownloadSpeed() const - { return _impl->_settingsObj.mindownloadspeed(); } + { return _impl->_minDownloadSpeed; } void TransferSettings::setMaxDownloadSpeed( long v ) - { _impl->_settingsObj.set_maxdownloadspeed(v); } + { _impl->_maxDownloadSpeed = (v); } long TransferSettings::maxDownloadSpeed() const - { return _impl->_settingsObj.maxdownloadspeed(); } + { return _impl->_maxDownloadSpeed; } void TransferSettings::setMaxSilentTries( long v ) - { _impl->_settingsObj.set_maxsilenttries(v); } + { _impl->_maxSilentTries = (v); } long TransferSettings::maxSilentTries() const - { return _impl->_settingsObj.maxsilenttries(); } + { return _impl->_maxSilentTries; } void TransferSettings::setVerifyHostEnabled( bool enabled ) - { _impl->_settingsObj.set_verify_host(enabled); } + { _impl->_verify_host = (enabled); } bool TransferSettings::verifyHostEnabled() const - { return _impl->_settingsObj.verify_host(); } + { return _impl->_verify_host; } void TransferSettings::setVerifyPeerEnabled( bool enabled ) - { _impl->_settingsObj.set_verify_peer(enabled); } + { _impl->_verify_peer = enabled; } bool TransferSettings::verifyPeerEnabled() const - { return _impl->_settingsObj.verify_peer(); } + { return _impl->_verify_peer; } + void TransferSettings::setClientCertificatePath( const Pathname &val_r ) + { _impl->_client_cert_path = val_r; } void TransferSettings::setClientCertificatePath( Pathname && val_r ) - { _impl->_settingsObj.set_client_cert_path( val_r.asString() ); } + { _impl->_client_cert_path = std::move( val_r ); } - Pathname TransferSettings::clientCertificatePath() const - { return _impl->_settingsObj.client_cert_path(); } + const Pathname &TransferSettings::clientCertificatePath() const + { return _impl->_client_cert_path; } + void TransferSettings::setClientKeyPath( const Pathname &val_r ) + { _impl->_client_key_path = val_r; } + void TransferSettings::setClientKeyPath( Pathname && val_r ) - { _impl->_settingsObj.set_client_key_path( val_r.asString() ); } + { _impl->_client_key_path = std::move( val_r ); } - Pathname TransferSettings::clientKeyPath() const - { return _impl->_settingsObj.client_key_path(); } + const Pathname &TransferSettings::clientKeyPath() const + { return _impl->_client_key_path; } - proto::TransferSettings &TransferSettings::protoData() - { - return _impl->_settingsObj; - } - const proto::TransferSettings &TransferSettings::protoData() const - { - return _impl->_settingsObj; - } + void TransferSettings::setCertificateAuthoritiesPath( const Pathname &val_r ) + { _impl->_ca_path = val_r; } void TransferSettings::setCertificateAuthoritiesPath( Pathname && val_r ) - { _impl->_settingsObj.set_ca_path(val_r.asString()); } + { _impl->_ca_path = std::move(val_r.asString()); } - Pathname TransferSettings::certificateAuthoritiesPath() const - { return _impl->_settingsObj.ca_path(); } + const Pathname &TransferSettings::certificateAuthoritiesPath() const + { return _impl->_ca_path; } + void TransferSettings::setAuthType( const std::string &val_r ) + { _impl->_authtype = val_r; } + void TransferSettings::setAuthType( std::string && val_r ) - { _impl->_settingsObj.set_authtype( std::move(val_r) ); } + { _impl->_authtype = std::move(val_r); } - std::string TransferSettings::authType() const - { return _impl->_settingsObj.authtype(); } + const std::string &TransferSettings::authType() const + { return _impl->_authtype; } void TransferSettings::setHeadRequestsAllowed( bool allowed ) - { _impl->_settingsObj.set_head_requests_allowed(allowed); } + { _impl->_head_requests_allowed = allowed; } bool TransferSettings::headRequestsAllowed() const - { return _impl->_settingsObj.head_requests_allowed(); } + { return _impl->_head_requests_allowed; } } // namespace media } // namespace zypp - diff --git a/zypp-curl/transfersettings.h b/zypp-curl/transfersettings.h index ca1c80c..be05450 100644 --- a/zypp-curl/transfersettings.h +++ b/zypp-curl/transfersettings.h @@ -18,11 +18,6 @@ #include #include #include - -namespace zypp::proto { - class TransferSettings; -} - namespace zypp { namespace media @@ -37,38 +32,39 @@ namespace zypp /** Constructs a transfer program cmd line access. */ TransferSettings(); - TransferSettings( const zypp::proto::TransferSettings &settings ); - typedef std::vector Headers; /** reset the settings to the defaults */ void reset(); - /** add a header, on the form "Foo: Bar" */ void addHeader( std::string && val_r ); + void addHeader( const std::string & val_r ); /** returns a list of all added headers */ - Headers headers() const; + const Headers &headers() const; /** sets the user agent ie: "Mozilla v3" */ void setUserAgentString( std::string && val_r ); + void setUserAgentString( const std::string && val_r ); /** user agent string */ - std::string userAgentString() const; + const std::string &userAgentString() const; /** sets the auth username */ + void setUsername( const std::string &val_r ); void setUsername( std::string && val_r ); /** auth username */ - std::string username() const; + const std::string &username() const; /** sets the auth password */ + void setPassword( const std::string & val_r ); void setPassword( std::string && val_r ); /** auth password */ - std::string password() const; + const std::string &password() const; /** returns the user and password as a user:pass string */ std::string userPassword() const; @@ -85,23 +81,26 @@ namespace zypp /** proxy to use if it is enabled */ + void setProxy( const std::string &val_r ); void setProxy( std::string && val_r ); /** proxy host */ - std::string proxy() const; + const std::string &proxy() const; /** sets the proxy user */ + void setProxyUsername( const std::string &val_r ); void setProxyUsername( std::string && val_r ); /** proxy auth username */ - std::string proxyUsername() const; + const std::string &proxyUsername() const; /** sets the proxy password */ + void setProxyPassword( const std::string &val_r ); void setProxyPassword( std::string && val_r ); /** proxy auth password */ - std::string proxyPassword() const; + const std::string &proxyPassword() const; /** returns the proxy user and password as a user:pass string */ std::string proxyUserPassword() const; @@ -164,17 +163,19 @@ namespace zypp /** Sets the SSL certificate authorities path */ + void setCertificateAuthoritiesPath( const Pathname &val_r ); void setCertificateAuthoritiesPath( Pathname && val_r ); /** SSL certificate authorities path ( default: /etc/ssl/certs ) */ - Pathname certificateAuthoritiesPath() const; + const Pathname &certificateAuthoritiesPath() const; /** set the allowed authentication types */ + void setAuthType( const std::string &val_r ); void setAuthType( std::string && val_r ); /** get the allowed authentication types */ - std::string authType() const; + const std::string &authType() const; /** set whether HEAD requests are allowed */ @@ -185,20 +186,19 @@ namespace zypp /** Sets the SSL client certificate file */ + void setClientCertificatePath( const Pathname &val_r ); void setClientCertificatePath( Pathname && val_r ); /** SSL client certificate file */ - Pathname clientCertificatePath() const; + const Pathname &clientCertificatePath() const; /** Sets the SSL client key file */ + void setClientKeyPath( const Pathname &val_r ); void setClientKeyPath( Pathname && val_r ); /** SSL client key file */ - Pathname clientKeyPath() const; - - const zypp::proto::TransferSettings &protoData() const; - zypp::proto::TransferSettings &protoData(); + const Pathname &clientKeyPath() const; protected: class Impl; diff --git a/zypp-media/CDTools b/zypp-media/CDTools new file mode 100644 index 0000000..82702a8 --- /dev/null +++ b/zypp-media/CDTools @@ -0,0 +1 @@ +#include "cdtools.h" diff --git a/zypp-media/CMakeLists.txt b/zypp-media/CMakeLists.txt index e36c058..5a27621 100644 --- a/zypp-media/CMakeLists.txt +++ b/zypp-media/CMakeLists.txt @@ -7,6 +7,10 @@ INCLUDE_DIRECTORIES ( ${LIBZYPP_SOURCE_DIR} ) ADD_DEFINITIONS( -DLOCALEDIR="${CMAKE_INSTALL_PREFIX}/share/locale" -DTEXTDOMAIN="zypp" -DZYPP_DLL ) SET( zypp_media_HEADERS + CDTools + cdtools.h + FileCheckException + filecheckexception.h MediaConfig mediaconfig.h mediaexception.h @@ -19,6 +23,8 @@ SET( zypp_media_private_HEADERS ) SET( zypp_media_SRCS + cdtools.cc + filecheckexception.cc mediaconfig.cc mediaexception.cc mount.cc @@ -46,14 +52,66 @@ SET( zypp_media_auth_SRCS INSTALL( FILES ${zypp_media_auth_HEADERS} DESTINATION "${INCLUDE_INSTALL_DIR}/zypp-media/auth" ) +SET( zypp_media_ng_HEADERS + ng/headervaluemap.h + ng/HeaderValueMap + ng/provide.h + ng/Provide + ng/providefwd.h + ng/ProvideFwd + ng/provide-configvars.h + ng/providespec.h + ng/ProvideSpec + ng/provideres.h + ng/provideitem.h + ng/ProvideRes + ng/mediaverifier.h + ng/MediaVerifier + ng/worker/devicedriver.h + ng/worker/DeviceDriver + ng/worker/provideworker.h + ng/worker/ProvideWorker + ng/worker/mountingworker.h + ng/worker/MountingWorker +) + +SET( zypp_media_ng_private_HEADERS + ng/private/attachedmediainfo_p.h + ng/private/provide_p.h + ng/private/providefwd_p.h + ng/private/provideitem_p.h + ng/private/providemessage_p.h + ng/private/providequeue_p.h + ng/private/provideres_p.h + ng/private/providedbg_p.h +) + +SET( zypp_media_ng_SRCS + ng/headervaluemap.cc + ng/provide.cc + ng/provideres.cc + ng/providespec.cc + ng/provideitem.cc + ng/providemessage.cc + ng/providequeue.cc + ng/mediaverifier.cc + ng/worker/devicedriver.cc + ng/worker/provideworker.cc + ng/worker/mountingworker.cc +) + +INSTALL( FILES ${zypp_media_ng_HEADERS} DESTINATION "${INCLUDE_INSTALL_DIR}/zypp-media/ng" ) + SET( zypp_media_lib_SRCS ${zypp_media_SRCS} ${zypp_media_auth_SRCS} + ${zypp_media_ng_SRCS} ) SET( zypp_media_lib_HEADERS ${zypp_media_private_HEADERS} ${zypp_media_HEADERS} ${zypp_media_auth_private_HEADERS} ${zypp_media_auth_HEADERS} + ${zypp_media_ng_private_HEADERS} ${zypp_media_ng_HEADERS} ) # Default loggroup for all files diff --git a/zypp-media/FileCheckException b/zypp-media/FileCheckException new file mode 100644 index 0000000..c67d2c3 --- /dev/null +++ b/zypp-media/FileCheckException @@ -0,0 +1 @@ +#include "filecheckexception.h" diff --git a/zypp-media/auth/authdata.cc b/zypp-media/auth/authdata.cc index 05e401b..8ae4fab 100644 --- a/zypp-media/auth/authdata.cc +++ b/zypp-media/auth/authdata.cc @@ -42,6 +42,16 @@ void AuthData::setLastDatabaseUpdate( time_t time ) _lastChange = time; } +const std::map &AuthData::extraValues() const +{ + return _extraValues; +} + +std::map &AuthData::extraValues() +{ + return _extraValues; +} + std::ostream & AuthData::dumpOn( std::ostream & str ) const { if (_url.isValid()) @@ -67,6 +77,12 @@ std::ostream & AuthData::dumpAsIniOn( std::ostream & str ) const << "username = " << _username << endl << "password = " << _password << endl; + for ( const auto &v : _extraValues ) { + if ( v.first == "username" || v.first == "password" ) + continue; + str << v.first << " = " << v.second << endl; + } + return str; } diff --git a/zypp-media/auth/authdata.h b/zypp-media/auth/authdata.h index fcb67dc..d137a6b 100644 --- a/zypp-media/auth/authdata.h +++ b/zypp-media/auth/authdata.h @@ -14,6 +14,7 @@ #include #include +#include namespace zypp { namespace media { @@ -61,6 +62,9 @@ public: time_t lastDatabaseUpdate () const; void setLastDatabaseUpdate ( time_t time ); + const std::map &extraValues() const; + std::map &extraValues(); + virtual std::ostream & dumpOn( std::ostream & str ) const; virtual std::ostream & dumpAsIniOn( std::ostream & str ) const; @@ -70,6 +74,7 @@ private: std::string _username; std::string _password; time_t _lastChange; //< timestamp of the last change to the database this credential is stored in + std::map _extraValues; }; typedef shared_ptr AuthData_Ptr; diff --git a/zypp-media/auth/credentialfilereader.cc b/zypp-media/auth/credentialfilereader.cc index 0a402e0..5a95577 100644 --- a/zypp-media/auth/credentialfilereader.cc +++ b/zypp-media/auth/credentialfilereader.cc @@ -102,7 +102,7 @@ namespace zypp else if ( key_r == "password" ) _secret->setPassword( value_r ); else - WAR << "Ignore unknown attribute '" << key_r << "=" << value_r << "' in file " << _input << endl; + _secret->extraValues()[key_r] = value_r; } // else: ignored section due to wrong URL } @@ -148,4 +148,3 @@ namespace zypp /////////////////////////////////////////////////////////////////// } // namespace zypp /////////////////////////////////////////////////////////////////// - diff --git a/zypp-media/auth/credentialmanager.cc b/zypp-media/auth/credentialmanager.cc index 56cd3e6..d517d0e 100644 --- a/zypp-media/auth/credentialmanager.cc +++ b/zypp-media/auth/credentialmanager.cc @@ -179,7 +179,7 @@ namespace zypp } - static AuthData_Ptr findIn(const CredentialManager::CredentialSet & set, + AuthData_Ptr CredentialManager::findIn(const CredentialManager::CredentialSet & set, const Url & url, url::ViewOption vopt) { @@ -200,7 +200,6 @@ namespace zypp return AuthData_Ptr(); } - AuthData_Ptr CredentialManager::Impl::getCred(const Url & url) const { AuthData_Ptr result; @@ -267,13 +266,15 @@ namespace zypp } static int save_creds_in_file( - const CredentialManager::CredentialSet creds, + CredentialManager::CredentialSet &creds, const Pathname & file, const mode_t mode) { int ret = 0; filesystem::assert_file_mode( file, mode ); + const auto now = time( nullptr ); + PathInfo pi { file }; if ( pi.userMayRW() ) try { // make sure only our thread accesses the file @@ -284,7 +285,7 @@ namespace zypp for_(it, creds.begin(), creds.end()) { (*it)->dumpAsIniOn(fs); - (*it)->setLastDatabaseUpdate( time( nullptr ) ); + (*it)->setLastDatabaseUpdate( now ); fs << endl; } if ( !fs ) { @@ -349,6 +350,22 @@ namespace zypp saveInFile(cred, credfile); } + time_t CredentialManager::timestampForCredDatabase ( const zypp::Url &url ) + { + Pathname credfile; + if ( url.isValid() ) { + credfile = url.getQueryParam("credentials"); + } + + if (credfile.empty()) + credfile = _pimpl->_options.userCredFilePath; + + zypp::PathInfo pi(credfile); + if ( pi.isExist() && pi.isFile() ) + return pi.mtime(); + + return 0; + } void CredentialManager::addGlobalCred(const AuthData & cred) { diff --git a/zypp-media/auth/credentialmanager.h b/zypp-media/auth/credentialmanager.h index 6992093..2dee88f 100644 --- a/zypp-media/auth/credentialmanager.h +++ b/zypp-media/auth/credentialmanager.h @@ -158,6 +158,15 @@ namespace zypp */ void clearAll(bool global = false); + /*! + * Helper function to find a matching AuthData instance in a CredentialSet + */ + static AuthData_Ptr findIn( const CredentialManager::CredentialSet & set, const Url & url, url::ViewOption vopt ); + + /*! + * Returns the timestamp of the database the given URL creds would be stored + */ + time_t timestampForCredDatabase ( const zypp::Url &url ); CredentialIterator credsGlobalBegin() const; CredentialIterator credsGlobalEnd() const; @@ -184,4 +193,3 @@ namespace zypp ////////////////////////////////////////////////////////////////////// #endif /* ZYPP_MEDIA_AUTH_CREDENTIALMANAGER_H */ - diff --git a/zypp-media/cdtools.cc b/zypp-media/cdtools.cc new file mode 100644 index 0000000..a4c9829 --- /dev/null +++ b/zypp-media/cdtools.cc @@ -0,0 +1,103 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ + +#include "cdtools.h" + +extern "C" +{ +#include +#include +} + +#include +#include +#include +#include + + +/* +** If defined to the full path of the eject utility, +** it will be used additionally to the eject-ioctl. +*/ +#define EJECT_TOOL_PATH "/bin/eject" + + +namespace zypp::media { + + bool CDTools::openTray(const std::string &device_r) + { + int fd = ::open( device_r.c_str(), O_RDONLY|O_NONBLOCK|O_CLOEXEC ); + int res = -1; + + if ( fd != -1) + { + res = ::ioctl( fd, CDROMEJECT ); + ::close( fd ); + } + + if ( res ) + { + if( fd == -1) + { + WAR << "Unable to open '" << device_r + << "' (" << ::strerror( errno ) << ")" << std::endl; + } + else + { + WAR << "Eject " << device_r + << " failed (" << ::strerror( errno ) << ")" << std::endl; + } + +#if defined(EJECT_TOOL_PATH) + DBG << "Try to eject " << device_r << " using " + << EJECT_TOOL_PATH << " utility" << std::endl; + + const char *cmd[3]; + cmd[0] = EJECT_TOOL_PATH; + cmd[1] = device_r.c_str(); + cmd[2] = NULL; + ExternalProgram eject(cmd, ExternalProgram::Stderr_To_Stdout); + + for(std::string out( eject.receiveLine()); + out.length(); out = eject.receiveLine()) + { + DBG << " " << out; + } + + if(eject.close() != 0) + { + WAR << "Eject of " << device_r << " failed." << std::endl; + return false; + } +#else + return false; +#endif + } + MIL << "Eject of " << device_r << " successful." << std::endl; + return true; + } + + bool CDTools::closeTray(const std::string &device_r) + { + int fd = ::open( device_r.c_str(), O_RDONLY|O_NONBLOCK|O_CLOEXEC ); + if ( fd == -1 ) { + WAR << "Unable to open '" << device_r << "' (" << ::strerror( errno ) << ")" << std::endl; + return false; + } + int res = ::ioctl( fd, CDROMCLOSETRAY ); + ::close( fd ); + if ( res ) { + WAR << "Close tray " << device_r << " failed (" << ::strerror( errno ) << ")" << std::endl; + return false; + } + DBG << "Close tray " << device_r << std::endl; + return true; + } + +} diff --git a/zypp/zyppng/context.h b/zypp-media/cdtools.h similarity index 51% rename from zypp/zyppng/context.h rename to zypp-media/cdtools.h index a3c6e94..e5489ce 100644 --- a/zypp/zyppng/context.h +++ b/zypp-media/cdtools.h @@ -5,31 +5,23 @@ | / /__ | | | | | | | | /_____||_| |_| |_| | | | -----------------------------------------------------------------------*/ -#ifndef ZYPP_NG_CORE_CONTEXT_H_INCLUDED -#define ZYPP_NG_CORE_CONTEXT_H_INCLUDED +\---------------------------------------------------------------------*/ +/** \file zypp-media/cdtools.h + * +*/ -#include +#ifndef ZYPP_MEDIA_CDTOOLS_H +#define ZYPP_MEDIA_CDTOOLS_H -namespace zyppng { +#include - class EventLoop; - class MirrorControl; - - class Context { +namespace zypp::media { + class CDTools + { public: - - using Ptr = std::shared_ptr; - - Context(); - std::shared_ptr evLoop () const; - std::shared_ptr mirrorControl (); - - private: - std::shared_ptr _zyppEventLoop; - std::shared_ptr _mirrorControl; - + static bool openTray( const std::string & device_r ); + static bool closeTray( const std::string & device_r ); }; } diff --git a/zypp-media/filecheckexception.cc b/zypp-media/filecheckexception.cc new file mode 100644 index 0000000..2d5384e --- /dev/null +++ b/zypp-media/filecheckexception.cc @@ -0,0 +1,12 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ +/** \file zypp-media/filecheckexception.cc + * +*/ +#include "filecheckexception.h" diff --git a/zypp-media/filecheckexception.h b/zypp-media/filecheckexception.h new file mode 100644 index 0000000..eb1aa80 --- /dev/null +++ b/zypp-media/filecheckexception.h @@ -0,0 +1,43 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ + +#ifndef ZYPP_MEDIA_FILECHECKEXCEPTION_H +#define ZYPP_MEDIA_FILECHECKEXCEPTION_H + +#include + +namespace zypp { + + class FileCheckException : public Exception + { + public: + FileCheckException(const std::string &msg) + : Exception(msg) + {} + }; + + class CheckSumCheckException : public FileCheckException + { + public: + CheckSumCheckException(const std::string &msg) + : FileCheckException(msg) + {} + }; + + class SignatureCheckException : public FileCheckException + { + public: + SignatureCheckException(const std::string &msg) + : FileCheckException(msg) + {} + }; + +} + +#endif // ZYPP_MEDIA_FILECHECKEXCEPTION_H diff --git a/zypp-media/mediaexception.cc b/zypp-media/mediaexception.cc index 9e48abd..a9ceecb 100644 --- a/zypp-media/mediaexception.cc +++ b/zypp-media/mediaexception.cc @@ -216,6 +216,11 @@ namespace zypp return str; } + std::ostream & MediaJammedException::dumpOn( std::ostream & str ) const { + str << _("No free ressources available to attach medium."); + return str; + } + ///////////////////////////////////////////////////////////////// } // namespace media } // namespace zypp diff --git a/zypp-media/mediaexception.h b/zypp-media/mediaexception.h index 522a368..38b8954 100644 --- a/zypp-media/mediaexception.h +++ b/zypp-media/mediaexception.h @@ -113,6 +113,23 @@ namespace zypp std::string _path; }; + class MediaJammedException : public MediaException + { + public: + /** Ctor taking message. + * Use \ref ZYPP_THROW to throw exceptions. + */ + MediaJammedException() : MediaException( "Media Jammed Exception" ) + {} + + /** Dtor. */ + ~MediaJammedException() noexcept override {}; + protected: + std::ostream & dumpOn( std::ostream & str ) const override; + + private: + }; + class MediaBadFilenameException : public MediaException { public: @@ -542,6 +559,15 @@ namespace zypp virtual ~MediaInvalidCredentialsException() noexcept {} }; + class MediaRequestCancelledException : public MediaException + { + public: + MediaRequestCancelledException( const std::string & msg = "" ) + : MediaException(msg) + {} + virtual ~MediaRequestCancelledException() noexcept {} + }; + ///////////////////////////////////////////////////////////////// } // namespace media } // namespace zypp diff --git a/zypp-media/mount.cc b/zypp-media/mount.cc index 6e69d77..74377f8 100644 --- a/zypp-media/mount.cc +++ b/zypp-media/mount.cc @@ -46,6 +46,13 @@ namespace zypp { } + +bool MountEntry::isBlockDevice() const +{ + PathInfo dev_info; + return ( str::hasPrefix( Pathname(src).asString(), "/dev/" ) && dev_info(src) && dev_info.isBlk() ); +} + Mount::Mount() {} @@ -254,5 +261,16 @@ Mount::getEntries(const std::string &mtab) return entries; } +time_t Mount::getMTime() +{ + time_t mtime = zypp::PathInfo("/etc/mtab").mtime(); + if( mtime <= 0) + { + WAR << "Failed to retrieve modification time of '/etc/mtab'" + << std::endl; + } + return mtime; +} + } // namespace media } // namespace zypp diff --git a/zypp-media/mount.h b/zypp-media/mount.h index 60edb8e..944a117 100644 --- a/zypp-media/mount.h +++ b/zypp-media/mount.h @@ -47,6 +47,11 @@ namespace zypp { , pass(passnum) {} + /** + * Returns true if the src part points to a block device in /dev + */ + bool isBlockDevice () const; + std::string src; //!< name of mounted file system std::string dir; //!< file system path prefix std::string type; //!< filesystem / mount type @@ -137,6 +142,12 @@ namespace zypp { static MountEntries getEntries(const std::string &mtab = ""); + /** + * Get the modification time of the /etc/mtab file. + * \return Modification time of the /etc/mtab file. + */ + static time_t getMTime(); + private: #if LEGACY(1722) ExternalProgram * _bincompat1; diff --git a/zypp-media/ng/HeaderValueMap b/zypp-media/ng/HeaderValueMap new file mode 100644 index 0000000..c36a1e7 --- /dev/null +++ b/zypp-media/ng/HeaderValueMap @@ -0,0 +1 @@ +#include "headervaluemap.h" diff --git a/zypp-media/ng/MediaVerifier b/zypp-media/ng/MediaVerifier new file mode 100644 index 0000000..4dce643 --- /dev/null +++ b/zypp-media/ng/MediaVerifier @@ -0,0 +1 @@ +#include "mediaverifier.h" diff --git a/zypp-media/ng/Provide b/zypp-media/ng/Provide new file mode 100644 index 0000000..f0da3a0 --- /dev/null +++ b/zypp-media/ng/Provide @@ -0,0 +1 @@ +#include "provide.h" diff --git a/zypp-media/ng/ProvideFwd b/zypp-media/ng/ProvideFwd new file mode 100644 index 0000000..adb64a2 --- /dev/null +++ b/zypp-media/ng/ProvideFwd @@ -0,0 +1 @@ +#include "zypp-media/ng/providefwd.h" diff --git a/zypp-media/ng/ProvideItem b/zypp-media/ng/ProvideItem new file mode 100644 index 0000000..723ce4d --- /dev/null +++ b/zypp-media/ng/ProvideItem @@ -0,0 +1 @@ +#include "provideitem.h" diff --git a/zypp-media/ng/ProvideRes b/zypp-media/ng/ProvideRes new file mode 100644 index 0000000..f51c6ba --- /dev/null +++ b/zypp-media/ng/ProvideRes @@ -0,0 +1 @@ +#include "provideres.h" diff --git a/zypp-media/ng/ProvideSpec b/zypp-media/ng/ProvideSpec new file mode 100644 index 0000000..3fa0e22 --- /dev/null +++ b/zypp-media/ng/ProvideSpec @@ -0,0 +1 @@ +#include "providespec.h" diff --git a/zypp-media/ng/headervaluemap.cc b/zypp-media/ng/headervaluemap.cc new file mode 100644 index 0000000..7c36923 --- /dev/null +++ b/zypp-media/ng/headervaluemap.cc @@ -0,0 +1,267 @@ +#include "headervaluemap.h" +#include + +namespace zyppng { + + HeaderValueMap::Value HeaderValueMap::InvalidValue; + + HeaderValue::HeaderValue() + : _val ( new std::variant() ) + {} + + HeaderValue::HeaderValue( const HeaderValue &other ) + : _val ( new std::variant( *other._val ) ) + {} + + HeaderValue::HeaderValue( HeaderValue &&other ) + : _val ( new std::variant( std::move(*other._val) ) ) + {} + + HeaderValue::HeaderValue( const bool val ) + : _val ( new std::variant(val) ) + {} + + HeaderValue::HeaderValue( const int32_t val ) + : _val ( new std::variant(val) ) + {} + + HeaderValue::HeaderValue( const int64_t val ) + : _val ( new std::variant(val) ) + {} + + HeaderValue::HeaderValue( const double val ) + : _val ( new std::variant(val) ) + {} + + HeaderValue::HeaderValue( const std::string &val ) + : _val ( new std::variant(val) ) + {} + + HeaderValue::HeaderValue(const char *val) + : HeaderValue( zypp::str::asString (val) ) + {} + + HeaderValue::HeaderValue( std::string &&val ) + : _val ( new std::variant( std::move(val) ) ) + {} + + bool HeaderValue::valid() const + { + return ( _val->index () > 0 ); + } + + bool HeaderValue::isString() const + { + return std::holds_alternative(*_val); + } + + bool HeaderValue::isInt() const + { + return std::holds_alternative(*_val); + } + + bool HeaderValue::isInt64() const + { + return std::holds_alternative(*_val); + } + + bool HeaderValue::isDouble() const + { + return std::holds_alternative(*_val); + } + + bool HeaderValue::isBool() const + { + return std::holds_alternative(*_val); + } + + const std::string &HeaderValue::asString() const + { + return std::get(*_val); + } + + int32_t HeaderValue::asInt() const + { + return std::get(*_val); + } + + int64_t HeaderValue::asInt64() const + { + if ( std::holds_alternative(*_val) ) + return std::get( *_val ); + return std::get(*_val); + } + + double HeaderValue::asDouble() const + { + return std::get(*_val); + } + + bool HeaderValue::asBool() const + { + return std::get(*_val); + } + + HeaderValue::value_type &HeaderValue::asVariant() + { + return *_val; + } + + const HeaderValue::value_type &HeaderValue::asVariant() const + { + return *_val; + } + + HeaderValue &HeaderValue::operator=(const HeaderValue &other) + { + *_val = *other._val; + return *this; + } + + bool HeaderValue::operator==(const HeaderValue &other) const + { + return ( *_val == *other._val ); + } + + HeaderValue &HeaderValue::operator= ( HeaderValue &&other ) + { + *_val = std::move( *other._val ); + return *this; + } + + HeaderValue &HeaderValue::operator= ( const std::string &val ) + { + *_val = val; + return *this; + } + + HeaderValue &HeaderValue::operator= ( int32_t val ) + { + *_val = val; + return *this; + } + + HeaderValue &HeaderValue::operator= ( int64_t val ) + { + *_val = val; + return *this; + } + + HeaderValue &HeaderValue::operator= ( double val ) + { + *_val = val; + return *this; + } + + HeaderValue &HeaderValue::operator= ( bool val ) + { + *_val = val; + return *this; + } + + + HeaderValueMap::HeaderValueMap( std::initializer_list init ) + : _values( std::move(init) ) + { } + + bool HeaderValueMap::contains(const std::string &key) const + { + return _values.count (key) > 0 && _values.at(key).size () > 0 ; + } + + void HeaderValueMap::set( const std::string &key, const Value &val ) + { + auto i = _values.find (key); + if ( i == _values.end() ) { + _values.insert ( std::make_pair(key, std::vector{val}) ); + } else { + i->second = std::vector{val}; + } + } + + void HeaderValueMap::set(const std::string &key, Value &&val) + { + auto i = _values.find (key); + if ( i == _values.end() ) { + _values.insert ( std::make_pair(key, std::vector{std::move(val)}) ); + } else { + i->second = std::vector{std::move(val)}; + } + } + + void HeaderValueMap::add(const std::string &key, const Value &val) + { + auto i = _values.find (key); + if ( i == _values.end() ) { + _values.insert ( std::make_pair(key, std::vector{val}) ); + } else { + i->second.push_back(val); + } + } + + void HeaderValueMap::clear() + { + _values.clear(); + } + + HeaderValueMap::ValueMap::size_type HeaderValueMap::size() const noexcept + { + return _values.size(); + } + + std::vector &HeaderValueMap::values(const std::string &key) + { + return _values[key]; + } + + const std::vector &HeaderValueMap::values(const std::string &key) const + { + return _values.at(key); + } + + HeaderValueMap::Value HeaderValueMap::value ( const std::string_view &str, const HeaderValueMap::Value &defaultVal) const + { return value( std::string(str), defaultVal ); } + + HeaderValueMap::Value HeaderValueMap::value ( const std::string &str, const HeaderValueMap::Value &defaultVal) const + { + if ( !contains(str) || !_values.at(str).size() ) + return defaultVal; + return _values.at(str).back(); + } + + HeaderValueMap::Value &HeaderValueMap::operator[](const std::string &key) + { + if ( !contains(key) ) + return InvalidValue; + return _values[key].back(); + } + + HeaderValueMap::Value &HeaderValueMap::operator[]( const std::string_view &key ) + { + return (*this)[std::string(key)]; + } + + const HeaderValueMap::Value &HeaderValueMap::operator[]( const std::string &key ) const + { + if ( !contains(key) ) + return InvalidValue; + return _values.at(key).back(); + } + + const HeaderValueMap::Value &HeaderValueMap::operator[]( const std::string_view &key ) const + { + return (*this)[std::string(key)]; + } + + HeaderValueMap::const_iterator HeaderValueMap::erase(const const_iterator &i) + { + auto yi = _values.erase(i.base()); + return HeaderValueMap::const_iterator(yi); + } + + bool HeaderValueMap::erase(const std::string &key) + { + return ( _values.erase(key) > 0 ); + } + +} diff --git a/zypp-media/ng/headervaluemap.h b/zypp-media/ng/headervaluemap.h new file mode 100644 index 0000000..c2a643d --- /dev/null +++ b/zypp-media/ng/headervaluemap.h @@ -0,0 +1,199 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ +#ifndef ZYPP_MEDIA_NG_HEADERVALUEMAP_H_INCLUDED +#define ZYPP_MEDIA_NG_HEADERVALUEMAP_H_INCLUDED + +#include +#include +#include +#include +#include + +namespace zyppng { + + class HeaderValue + { + public: + using value_type = std::variant; + + HeaderValue(); + + HeaderValue( const HeaderValue &other ); + HeaderValue( HeaderValue &&other ); + + HeaderValue( const bool val ); + HeaderValue( const int32_t val ); + HeaderValue( const int64_t val ); + HeaderValue( const double val ); + HeaderValue( std::string &&val ); + HeaderValue( const std::string &val ); + HeaderValue( const char *val ); + + bool valid () const; + + bool isString () const; + bool isInt () const; + bool isInt64 () const; + bool isDouble () const; + bool isBool () const; + + const std::string &asString () const; + int32_t asInt () const; + int64_t asInt64 () const; + double asDouble() const; + bool asBool () const; + + value_type &asVariant (); + const value_type &asVariant () const; + + HeaderValue &operator= ( const HeaderValue &other ); + HeaderValue &operator= ( HeaderValue &&other ); + HeaderValue &operator= ( const std::string &val ); + HeaderValue &operator= ( int32_t val ); + HeaderValue &operator= ( int64_t val ); + HeaderValue &operator= ( double val ); + HeaderValue &operator= ( bool val ); + + bool operator== ( const HeaderValue &other ) const; + + private: + zypp::RWCOW_pointer _val; + }; + + class HeaderValueMap + { + public: + using Value = HeaderValue; + using ValueMap = std::map>; + + static Value InvalidValue; + + class const_iterator + : public boost::iterator_adaptor< + HeaderValueMap::const_iterator // Derived + , ValueMap::const_iterator // Base + , const std::pair // Value + , boost::use_default // CategoryOrTraversal + > + { + public: + const_iterator() + : const_iterator::iterator_adaptor_() {} + + explicit const_iterator( const ValueMap::const_iterator &val ) + { this->base_reference() = val; } + + const_iterator( const HeaderValueMap::const_iterator &other ) + : const_iterator::iterator_adaptor_( other.base() ) {} + + const std::string &key () const { + return this->base_reference()->first; + } + + const Value &value() const { + auto &l = base_reference ()->second; + if ( l.empty() ) { + return InvalidValue; + } + return l.back(); + } + + private: + friend class boost::iterator_core_access; + void increment() { + this->base_reference() = ++this->base_reference(); + } + + std::pair dereference() const + { + return std::make_pair( key(), value() ); + } + }; + + HeaderValueMap() = default; + HeaderValueMap( std::initializer_list init ); + + bool contains( const std::string &key ) const; + bool contains( const std::string_view &key ) const { + return contains(std::string(key)); + } + + void set( const std::string &key, const Value &val ); + void set( const std::string &key, Value &&val ); + void add( const std::string &key, const Value &val); + void clear (); + ValueMap::size_type size() const noexcept; + + std::vector &values ( const std::string &key ); + const std::vector &values ( const std::string &key ) const; + + std::vector &values ( const std::string_view &key ) { + return values( std::string(key) ); + } + + const std::vector &values ( const std::string_view &key ) const { + return values( std::string(key) ); + } + + /*! + * Returns the last entry with key \a str in the list of values + * or the default value specified in \a defaultVal + */ + Value value ( const std::string_view &str, const Value &defaultVal = Value() ) const; + Value value ( const std::string &str, const Value &defaultVal = Value() ) const; + + Value &operator[]( const std::string &key ); + Value &operator[]( const std::string_view &key ); + const Value &operator[]( const std::string &key ) const; + const Value &operator[]( const std::string_view &key ) const; + + const_iterator erase( const const_iterator &i ); + bool erase( const std::string &key ); + + const_iterator begin() const { + return const_iterator( _values.begin() ); + } + const_iterator end() const { + return const_iterator( _values.end() ); + } + + ValueMap::iterator beginList() { + return _values.begin(); + } + ValueMap::iterator endList() { + return _values.end(); + } + + ValueMap::const_iterator beginList() const { + return _values.begin(); + } + ValueMap::const_iterator endList() const { + return _values.end(); + } + + ValueMap::const_iterator cbeginList() const { + return _values.cbegin(); + } + ValueMap::const_iterator cendList() const { + return _values.cend(); + } + + private: + ValueMap _values; + }; +} + +namespace zypp { + template<> + inline zyppng::HeaderValue::value_type* rwcowClone( const zyppng::HeaderValue::value_type * rhs ) + { return new zyppng::HeaderValue::value_type(*rhs); } +} + + +#endif diff --git a/zypp-media/ng/mediaverifier.cc b/zypp-media/ng/mediaverifier.cc new file mode 100644 index 0000000..0251dc9 --- /dev/null +++ b/zypp-media/ng/mediaverifier.cc @@ -0,0 +1,137 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ + +#include "mediaverifier.h" +#include +#include +#include +#include +#include +#include "private/providedbg_p.h" + +namespace zyppng { + + ZYPP_FWD_DECL_TYPE_WITH_REFS (SuseMediaDataVerifier); + + class SuseMediaDataVerifier : public MediaDataVerifier + { + public: + // MediaDataVerifier interface + bool valid() const override; + bool matches(const MediaDataVerifierRef &rhs) const override; + const std::string &mediaVendor() const override; + const std::string &mediaIdent() const override; + uint totalMedia() const override; + std::ostream &toStream(std::ostream &str) const override; + bool load( const zypp::Pathname &data ) override; + bool loadFromMedium(const zypp::filesystem::Pathname &data, uint expectedMediaNr ) override; + zypp::filesystem::Pathname mediaFilePath(uint mediaNr) const override; + MediaDataVerifierRef clone () const override; + std::string expectedAsUserString( uint mediaNr ) const override; + + private: + std::string _mediaVendor; + std::string _mediaIdent; + uint _totalMedia = 0; + }; + + zypp::Pathname SuseMediaDataVerifier::mediaFilePath(uint mediaNr) const + { + zypp::str::Format fmt { "/media.%d/media" }; + return (fmt % zypp::str::numstring( mediaNr )).asString(); + } + + bool SuseMediaDataVerifier::loadFromMedium( const zypp::filesystem::Pathname &data, uint expectedMediaNr ) + { + return load ( data / mediaFilePath(expectedMediaNr) ); + } + + bool SuseMediaDataVerifier::valid() const + { return ! (_mediaVendor.empty() || _mediaIdent.empty()); } + + bool SuseMediaDataVerifier::matches(const MediaDataVerifierRef &rhs) const + { + auto conv = std::dynamic_pointer_cast(rhs); + return conv && valid() && conv->_mediaVendor == _mediaVendor && conv->_mediaIdent == _mediaIdent; + } + + uint SuseMediaDataVerifier::totalMedia() const + { + return _totalMedia; + } + + std::ostream &SuseMediaDataVerifier::toStream( std::ostream &str ) const + { + return str << "[" << _mediaVendor << "|" << _mediaIdent << "/" << _totalMedia << "]"; + } + + bool SuseMediaDataVerifier::load( const zypp::Pathname &path_r ) + { + std::ifstream inp( path_r.c_str() ); + if ( !inp ) { + ERR << "Can't setup a SUSEMediaVerifier from file: " << path_r.asString() << std::endl; + return false; + } + getline( inp, _mediaVendor ); + getline( inp, _mediaIdent ); + std::string buffer; + getline( inp, buffer ); + zypp::str::strtonum( buffer, _totalMedia ); + //if ( !_totalMedia ) _totalMedia = 1; + // loaded but maybe not valid + return true; + } + + const std::string &SuseMediaDataVerifier::mediaIdent() const + { + return _mediaIdent; + } + + const std::string &SuseMediaDataVerifier::mediaVendor() const + { + return _mediaVendor; + } + + MediaDataVerifierRef SuseMediaDataVerifier::clone () const + { + return SuseMediaDataVerifierRef( new SuseMediaDataVerifier( *this ) ); + } + + std::string SuseMediaDataVerifier::expectedAsUserString( uint mediaNr ) const + { + // Translator: %1% the expected medium number; %2% the total number of media in the set; %3% the ident file on the medium. + zypp::str::Format fmt { _("Expected medium %1%/%2% identified by file '%3%' with content:") }; + return zypp::str::Str() + << ( fmt % mediaNr % _totalMedia % mediaFilePath( mediaNr ) ) << "\n" + << " " << _mediaVendor << "\n" + << " " << _mediaIdent; + } + + MediaDataVerifier::MediaDataVerifier() noexcept + { } + + MediaDataVerifier::~MediaDataVerifier() + { } + + MediaDataVerifierRef MediaDataVerifier::createVerifier( const std::string &verifierType ) + { + if ( verifierType == "SuseMediaV1" ) { + return SuseMediaDataVerifierRef( new SuseMediaDataVerifier() ); + } + return nullptr; + } + + std::ostream &operator<<(std::ostream &str, const MediaDataVerifierRef &obj) + { + if ( obj ) + return obj->toStream(str); + return str << "[MediaVerifier: null]"; + } + +} diff --git a/zypp-media/ng/mediaverifier.h b/zypp-media/ng/mediaverifier.h new file mode 100644 index 0000000..81ca7b6 --- /dev/null +++ b/zypp-media/ng/mediaverifier.h @@ -0,0 +1,100 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ +#ifndef ZYPP_MEDIA_NG_MEDIAVERIFIER_H_INCLUDED +#define ZYPP_MEDIA_NG_MEDIAVERIFIER_H_INCLUDED + +#include +#include +#include + +namespace zypp { + namespace filesystem { + class Pathname; + } + using filesystem::Pathname; +} + +namespace zyppng { + + /*! + * The MediaDataVerifier is used to verify if a specific medium, e.g. a DVD, is the + * medium we actually need. Each verifier is defined by a verifier type and the data + * that is used to validate against the medium. + */ + class MediaDataVerifier + { + public: + + MediaDataVerifier() noexcept; + virtual ~MediaDataVerifier(); + + static MediaDataVerifierRef createVerifier ( const std::string &verifierType ); + + /** Data considered to be valid if we have vendor and ident. */ + virtual bool valid() const = 0; + + /** Whether \a rhs belongs to the same media set. */ + virtual bool matches( const MediaDataVerifierRef & rhs ) const = 0; + + /*! + * Returns the media vendor string + */ + virtual const std::string &mediaVendor() const = 0; + + /*! + * Returns the media ident string + */ + virtual const std::string &mediaIdent() const = 0; + + /*! + * Returns the total number of mediums in this set + */ + virtual uint totalMedia() const = 0; + + /*! + * Writes the mediaverifier data to stream + */ + virtual std::ostream & toStream ( std::ostream & str ) const = 0; + + /*! + * Load verification information from a given file, all media data must + * be storeable in a file so that the controller can store a copy of it somewhere. + */ + virtual bool load( const zypp::Pathname &data ) = 0; + + /*! + * Generates the file information from a mounted medium, the path given in \a data + * is the mountpoint of the device. + */ + virtual bool loadFromMedium( const zypp::Pathname &data, uint mediaNr ) = 0; + + /*! + * Returns the path of the media identifier file on the medium + */ + virtual zypp::Pathname mediaFilePath ( uint mediaNr ) const = 0; + + /*! + * Clones \a this and returns a reference to the clone + */ + virtual MediaDataVerifierRef clone () const = 0; + + /*! + * Returns a error string describing the expected medium. + */ + virtual std::string expectedAsUserString( uint mediaNr = 1 ) const = 0; + }; + + /** \relates Stream output */ + std::ostream & operator<<( std::ostream & str, const MediaDataVerifierRef & obj ); + + + +} + +#endif diff --git a/zypp-media/ng/private/attachedmediainfo_p.h b/zypp-media/ng/private/attachedmediainfo_p.h new file mode 100644 index 0000000..81240a2 --- /dev/null +++ b/zypp-media/ng/private/attachedmediainfo_p.h @@ -0,0 +1,69 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\----------------------------------------------------------------------/ +* +* This file contains private API, this might break at any time between releases. +* You have been warned! +* +*/ +#ifndef ZYPP_MEDIA_PRIVATE_ATTACHEDMEDIAINFO_P_H_INCLUDED +#define ZYPP_MEDIA_PRIVATE_ATTACHEDMEDIAINFO_P_H_INCLUDED + +#include "providefwd_p.h" +#include "providequeue_p.h" +#include +#include +#include + +namespace zyppng { + + class ProvidePrivate; + + struct AttachedMediaInfo { + + public: + void ref() { + if ( _refCount == 0 ) + _idleSince = std::chrono::steady_clock::time_point::max(); + + _refCount++; + } + void unref() { + if ( _refCount > 0 ) { + _refCount--; + + if ( _refCount == 0 ) + _idleSince = std::chrono::steady_clock::now(); + } + } + + /*! + * Returns true if \a other requests the same medium as this instance + */ + bool isSameMedium ( const std::vector &urls, const ProvideMediaSpec &spec ) { + + const auto check = _spec.isSameMedium(spec); + if ( !zypp::indeterminate (check) ) + return (bool)check; + + // let the URL rule + return ( std::find( urls.begin(), urls.end(), _attachedUrl ) != urls.end() ); + } + + std::string _name; + ProvideQueueWeakRef _backingQueue; //< if initialized contains a weak reference to the queue that owns this medium + ProvideQueue::Config::WorkerType _workerType; + zypp::Url _attachedUrl; // the URL that was used for the attach request + ProvideMediaSpec _spec; + uint _refCount = 0; + std::chrono::steady_clock::time_point _idleSince = std::chrono::steady_clock::time_point::max(); + }; + +} + +#endif diff --git a/zypp-media/ng/private/provide_p.h b/zypp-media/ng/private/provide_p.h new file mode 100644 index 0000000..11f35fd --- /dev/null +++ b/zypp-media/ng/private/provide_p.h @@ -0,0 +1,158 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\----------------------------------------------------------------------/ +* +* This file contains private API, this might break at any time between releases. +* You have been warned! +* +*/ +#ifndef ZYPP_MEDIA_PRIVATE_PROVIDE_P_H_INCLUDED +#define ZYPP_MEDIA_PRIVATE_PROVIDE_P_H_INCLUDED + +#include "providefwd_p.h" +#include "providequeue_p.h" +#include "attachedmediainfo_p.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace zyppng { + + namespace constants { + constexpr std::string_view DEFAULT_PROVIDE_WORKER_PATH = ZYPP_WORKER_PATH; + constexpr std::string_view ATTACHED_MEDIA_SUFFIX = "-media"; + constexpr auto DEFAULT_ACTIVE_CONN_PER_HOST = 5; //< how many simultanious connections to the same host are allowed + constexpr auto DEFAULT_ACTIVE_CONN = 10; //< how many simultanious connections are allowed + constexpr auto DEFAULT_MAX_DYNAMIC_WORKERS = 20; + constexpr auto DEFAULT_CPU_WORKERS = 4; + } + + class ProvideQueue; + class RpcMessageStream; + using RpcMessageStreamPtr = std::shared_ptr; + + + class ProvidePrivate : public BasePrivate + { + ZYPP_DECLARE_PUBLIC(Provide); + public: + ProvidePrivate( zypp::Pathname &&workDir, Provide &pub ); + + enum ScheduleReason + { + ProvideStart, + QueueIdle, + EnqueueItem, + EnqueueReq, + RestartAttach, + FinishReq + }; + + void schedule( ScheduleReason reason ); + + bool queueRequest ( ProvideRequestRef req ); + bool dequeueRequest( ProvideRequestRef req, std::exception_ptr error ); + void queueItem ( ProvideItemRef item ); + void dequeueItem ( ProvideItem *item ); + + std::string nextMediaId () const; + AttachedMediaInfo &addMedium ( zypp::proto::Capabilities::WorkerType workerType, const zypp::Url &baseUrl, ProvideMediaSpec &spec ); + AttachedMediaInfo &addMedium ( zypp::proto::Capabilities::WorkerType workerType, ProvideQueueWeakRef backingQueue, const std::string &id, const zypp::Url &baseUrl, ProvideMediaSpec &spec ); + + std::string effectiveScheme ( const std::string &scheme ) const; + + void onPulseTimeout ( Timer & ); + void onQueueIdle (); + void onItemStateChanged ( ProvideItem &item ); + expected schemeConfig(const std::string &scheme); + + std::optional addToFileCache ( const zypp::Pathname &downloadedFile ); + bool isInCache ( const zypp::Pathname &downloadedFile ) const; + + bool isRunning() const; + + const zypp::Pathname &workerPath() const; + const std::string queueName( ProvideQueue &q ) const; + + std::vector &attachedMediaInfos(); + + std::list &items(); + + zypp::media::CredManagerOptions &credManagerOptions (); + + ProvideStatusRef log () { + return _log; + } + + uint32_t nextRequestId(); + + Signal< Provide::MediaChangeAction ( const std::string &, const std::string &, const int32_t, const std::vector &, const std::optional &) > _sigMediaChange; + Signal< std::optional ( const zypp::Url &reqUrl, const std::string &triedUsername, const std::map &extraValues ) > _sigAuthRequired; + + protected: + void doSchedule (Timer &); + + //@TODO should we make those configurable? + std::unordered_map< std::string, std::string > _workerAlias { + {"ftp" ,"http"}, + {"tftp" ,"http"}, + {"https","http"}, + {"cifs" ,"smb" }, + {"nfs4" ,"nfs" }, + {"cd" ,"disc"}, + {"dvd" ,"disc"}, + {"file" ,"dir" }, + {"hd" ,"disk"} + }; + + bool _isRunning = false; + bool _isScheduling = false; + Timer::Ptr _pulseTimer = Timer::create(); + Timer::Ptr _scheduleTrigger = Timer::create(); //< instead of constantly calling schedule we set a trigger event so it runs as soon as event loop is on again + zypp::Pathname _workDir; + + std::list< ProvideItemRef > _items; //< The list of running provide Items, each of them can spawn multiple requests + uint32_t _nextRequestId = 0; //< The next request ID , we use controller wide unique IDs instead of worker locals IDs , its easier to track + + struct QueueItem { + std::string _schemeName; + std::deque _requests; + }; + std::deque _queues; //< List of request queues for the workers, grouped by scheme. We use a deque and not a map because of possible changes to the list of queues during scheduling + + + std::vector< AttachedMediaInfo > _attachedMediaInfos; //< List of currently attached medias + + std::unordered_map< std::string, ProvideQueueRef > _workerQueues; + std::unordered_map< std::string, ProvideQueue::Config > _schemeConfigs; + + struct FileCacheItem { + zypp::ManagedFile _file; + std::optional _deathTimer; // timepoint where this item was seen first without a refcount + }; + std::unordered_map< std::string, FileCacheItem > _fileCache; + + zypp::Pathname _workerPath; + zypp::media::CredManagerOptions _credManagerOptions; + + ProvideStatusRef _log; + Signal _sigIdle; + }; +} + +#endif diff --git a/zypp-media/ng/private/providedbg_p.h b/zypp-media/ng/private/providedbg_p.h new file mode 100644 index 0000000..b5def83 --- /dev/null +++ b/zypp-media/ng/private/providedbg_p.h @@ -0,0 +1,43 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +----------------------------------------------------------------------/ +* +* This file contains private API, this might break at any time between releases. +* You have been warned! +* +*/ +#ifndef ZYPP_MEDIA_NG_PROVIDEDBG_P_H_INCLUDED +#define ZYPP_MEDIA_NG_PROVIDEDBG_P_H_INCLUDED + +#include + +L_ENV_CONSTR_FWD_DECLARE_FUNC(ZYPP_MEDIA_PROVIDER_DEBUG) + +#ifdef ZYPP_BASE_LOGGER_LOGGROUP +#undef ZYPP_BASE_LOGGER_LOGGROUP +#endif + +#define ZYPP_BASE_LOGGER_LOGGROUP "ZYPP_MEDIA_PROVIDE" + +namespace zyppng { + inline bool provideDebugEnabled() { + return zypp::log::has_env_constr_ZYPP_MEDIA_PROVIDER_DEBUG(); + } +} + +#define XXX_PRV if( zyppng::provideDebugEnabled() ) XXX +#define DBG_PRV if( zyppng::provideDebugEnabled() ) DBG +#define MIL_PRV if( zyppng::provideDebugEnabled() ) MIL +#define WAR_PRV if( zyppng::provideDebugEnabled() ) WAR +#define ERR_PRV if( zyppng::provideDebugEnabled() ) ERR +#define SEC_PRV if( zyppng::provideDebugEnabled() ) SEC +#define INT_PRV if( zyppng::provideDebugEnabled() ) INT +#define USR_PRV if( zyppng::provideDebugEnabled() ) USR + + +#endif // ZYPP_MEDIA_NG_PROVIDEDBG_P_H_INCLUDED diff --git a/zypp-media/ng/private/providefwd_p.h b/zypp-media/ng/private/providefwd_p.h new file mode 100644 index 0000000..7ef0392 --- /dev/null +++ b/zypp-media/ng/private/providefwd_p.h @@ -0,0 +1,36 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\----------------------------------------------------------------------/ +* +* This file contains private API, this might break at any time between releases. +* You have been warned! +* +*/ +#ifndef ZYPP_MEDIA_PRIVATE_PROVIDE_FWD_P_H_INCLUDED +#define ZYPP_MEDIA_PRIVATE_PROVIDE_FWD_P_H_INCLUDED + +#include +namespace zyppng { + ZYPP_FWD_DECL_TYPE_WITH_REFS(ProvideQueue); + ZYPP_FWD_DECL_TYPE_WITH_REFS(ProvideWorker); + ZYPP_FWD_DECL_TYPE_WITH_REFS(ProvideFileItem); + ZYPP_FWD_DECL_TYPE_WITH_REFS(AttachMediaItem); + ZYPP_FWD_DECL_TYPE_WITH_REFS(DetachMediaItem); + ZYPP_FWD_DECL_TYPE_WITH_REFS(ProvideRequest); + + class ProvideMessage; + + template< typename T > + class ProvidePromise; + template< typename T > + using ProvidePromiseRef = std::shared_ptr>; + template< typename T > + using ProvidePromiseWeakRef = std::weak_ptr>; +} + +#endif diff --git a/zypp-media/ng/private/provideitem_p.h b/zypp-media/ng/private/provideitem_p.h new file mode 100644 index 0000000..9bed4a7 --- /dev/null +++ b/zypp-media/ng/private/provideitem_p.h @@ -0,0 +1,206 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\----------------------------------------------------------------------/ +* +* This file contains private API, this might break at any time between releases. +* You have been warned! +* +*/ +#ifndef ZYPP_MEDIA_PRIVATE_PROVIDE_ITEM_P_H_INCLUDED +#define ZYPP_MEDIA_PRIVATE_PROVIDE_ITEM_P_H_INCLUDED + +#include "providefwd_p.h" +#include "providequeue_p.h" +#include "attachedmediainfo_p.h" +#include "providemessage_p.h" +#include +#include +#include +#include +#include +#include +#include + +namespace zyppng { + + /*! + * The internal request type, which represents all possible + * user requests and exports some convenience functions for the scheduler to + * directly access relevant data + */ + class ProvideRequest { + public: + + friend class ProvideItem; + + static expected create( ProvideItem &owner, const std::vector &urls, const std::string &id, ProvideMediaSpec &spec ); + static expected create ( ProvideItem &owner, const std::vector &urls, ProvideFileSpec &spec ); + static expected createDetach( const zypp::Url &url ); + + ProvideItem * owner() { return _owner; } + + uint code () const { return _message.code(); } + + void setCurrentQueue ( ProvideQueueRef ref ); + ProvideQueueRef currentQueue (); + + const ProvideMessage &provideMessage () const { return _message; } + ProvideMessage &provideMessage () { return _message; } + + const std::optional activeUrl() const; + void setActiveUrl ( const zypp::Url &urlToUse ); + + void setUrls( const std::vector & urls ) { + _mirrors = urls; + } + + const std::vector &urls() const { + return _mirrors; + } + + zypp::Url url() const { + return _mirrors.front(); + } + + void setUrl( const zypp::Url & url ) { + _mirrors = {url}; + } + + void clearForRestart () { + _pastRedirects.clear(); + _activeUrl.reset(); + _myQueue.reset(); + } + + private: + ProvideRequest( ProvideItem *owner, const std::vector &urls, ProvideMessage &&msg ) : _owner(owner), _message(std::move(msg) ), _mirrors(urls) {} + ProvideItem *_owner = nullptr; // destructor of ProvideItem will dequeue the item, so no need to do refcount here + ProvideMessage _message; + std::vector _mirrors; + std::vector _pastRedirects; + std::optional _activeUrl; + ProvideQueueWeakRef _myQueue; + }; + + class ProvideItemPrivate : public BasePrivate + { + public: + ProvideItemPrivate( ProvidePrivate & parent, ProvideItem &pub ) : BasePrivate(pub), _parent(parent) {} + ProvidePrivate &_parent; + ProvideItem::State _itemState = ProvideItem::Uninitialized; + std::chrono::steady_clock::time_point _itemStarted; + std::chrono::steady_clock::time_point _itemFinished; + std::optional _prevStats; + std::optional _currStats; + Signal _sigStateChanged; + }; + + /*! + * The object returned to the user code to track the internal Item. + * Releasing the last reference to it will cancel the operation but the corresponding ProvideItem + * will remain in the Queue until the cancel operation was finished. + */ + template< typename T > + class ProvidePromise : public AsyncOp> + { + public: + ProvidePromise( ProvideItemRef provideItem ) + : _myProvide( provideItem ) + {} + + ~ProvidePromise() + { + auto prov = _myProvide.lock(); + if ( prov ) + prov->released(); + } + + private: + ProvideItemWeakRef _myProvide; //weak reference to the internal item so we can cancel the op on desctruction + }; + + /*! + * Item downloading and providing a file + */ + class ProvideFileItem : public ProvideItem + { + public: + + static ProvideFileItemRef create ( const std::vector &urls,const ProvideFileSpec &request, ProvidePrivate &parent ); + + // ProvideItem interface + void initialize () override; + ProvidePromiseRef promise(); + + void setMediaRef ( Provide::MediaHandle &&hdl ); + Provide::MediaHandle & mediaRef (); + + ItemStats makeStats () override; + zypp::ByteCount bytesExpected () const override; + + protected: + ProvideFileItem ( const std::vector &urls,const ProvideFileSpec &request, ProvidePrivate &parent ); + + void informalMessage ( ProvideQueue &, ProvideRequestRef req, const ProvideMessage &msg ) override; + + using ProvideItem::finishReq; + void finishReq ( ProvideQueue &queue, ProvideRequestRef finishedReq, const ProvideMessage &msg ) override; + void cancelWithError ( std::exception_ptr error ) override; + expected authenticationRequired ( ProvideQueue &queue, ProvideRequestRef req, const zypp::Url &effectiveUrl, int64_t lastTimestamp, const std::map &extraFields ) override; + + private: + Provide::MediaHandle _handleRef; //< If we are using a attached media, this will keep the reference around + bool _promiseCreated = false; + std::vector _mirrorList; //< All available URLs, first one is the primary + ProvideFileSpec _initialSpec; //< The initial spec as defined by the user code + zypp::Pathname _targetFile; //< The target file as reported by the worker + zypp::Pathname _stagingFile; //< The staging file as reported by the worker + zypp::ByteCount _expectedBytes; //< The nr of bytes we want to provide + ProvidePromiseWeakRef _promise; + }; + + + /*! + * Item attaching and verifying a medium + */ + class AttachMediaItem : public ProvideItem + { + public: + ~AttachMediaItem(); + static AttachMediaItemRef create ( const std::vector &urls, const ProvideMediaSpec &request, ProvidePrivate &parent ); + SignalProxy< void( const zyppng::expected & ) > sigReady (); + + ProvidePromiseRef promise(); + + protected: + AttachMediaItem ( const std::vector &urls, const ProvideMediaSpec &request, ProvidePrivate &parent ); + + // ProvideItem interface + void initialize () override; + + using ProvideItem::finishReq; + void finishReq ( ProvideQueue &queue, ProvideRequestRef finishedReq, const ProvideMessage &msg ) override; + void cancelWithError( std::exception_ptr error ) override; + void finishWithSuccess (AttachedMediaInfo &medium ); + expected authenticationRequired ( ProvideQueue &queue, ProvideRequestRef req, const zypp::Url &effectiveUrl, int64_t lastTimestamp, const std::map &extraFields ) override; + + void onMasterItemReady ( const zyppng::expected& result ); + + private: + Signal< void( const zyppng::expected & )> _sigReady; + bool _promiseCreated = false; + connection _masterItemConn; + std::vector _mirrorList; //< All available URLs, first one is the primary + ProvideMediaSpec _initialSpec; //< The initial spec as defined by the user code + ProvideQueue::Config::WorkerType _workerType = ProvideQueue::Config::Invalid; + ProvidePromiseWeakRef _promise; + MediaDataVerifierRef _verifier; + }; +} + +#endif diff --git a/zypp-media/ng/private/providemessage_p.h b/zypp-media/ng/private/providemessage_p.h new file mode 100644 index 0000000..0138144 --- /dev/null +++ b/zypp-media/ng/private/providemessage_p.h @@ -0,0 +1,196 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\----------------------------------------------------------------------/ +* +* This file contains private API, this might break at any time between releases. +* You have been warned! +* +* +*/ +#ifndef ZYPP_MEDIA_PRIVATE_PROVIDE_MESSAGE_P_H_INCLUDED +#define ZYPP_MEDIA_PRIVATE_PROVIDE_MESSAGE_P_H_INCLUDED + +#include +#include +#include +#include // for FieldType +#include +#include +#include +#include + +namespace zypp::proto { + class ProvideMessage; + class Envelope; +} + +namespace zyppng { + + namespace ProvideStartedMsgFields + { + constexpr std::string_view Url ("url"); + constexpr std::string_view LocalFilename ("local_filename"); + constexpr std::string_view StagingFilename ("staging_filename"); + } + + namespace ProvideFinishedMsgFields + { + constexpr std::string_view LocalFilename ("local_filename"); + constexpr std::string_view CacheHit ("cacheHit"); + } + + namespace AuthInfoMsgFields + { + constexpr std::string_view Username ("username"); + constexpr std::string_view Password ("password"); + constexpr std::string_view AuthTimestamp ("auth_timestamp"); + constexpr std::string_view AuthType ("authType"); + } + + namespace RedirectMsgFields + { + constexpr std::string_view NewUrl ("new_url"); + } + + namespace MetalinkRedirectMsgFields + { + constexpr std::string_view NewUrl ("new_url"); + } + + namespace ErrMsgFields + { + constexpr std::string_view Reason ("reason"); + constexpr std::string_view Transient ("transient"); + constexpr std::string_view History ("history"); + } + + namespace ProvideMsgFields + { + constexpr std::string_view Url ("url"); + constexpr std::string_view Filename ("filename"); + constexpr std::string_view DeltaFile ("delta_file"); + constexpr std::string_view ExpectedFilesize ("expected_filesize"); + constexpr std::string_view CheckExistOnly ("check_existance_only"); + constexpr std::string_view MetalinkEnabled ("metalink_enabled"); + } + + namespace AttachMsgFields + { + constexpr std::string_view Url ("url"); + constexpr std::string_view AttachId ("attach_id"); + constexpr std::string_view VerifyType ("verify_type"); + constexpr std::string_view VerifyData ("verify_data"); + constexpr std::string_view MediaNr ("media_nr"); + constexpr std::string_view Device ("device"); + constexpr std::string_view Label ("label"); + } + + namespace DetachMsgFields + { + constexpr std::string_view Url ("url"); + } + + namespace AuthDataRequestMsgFields + { + constexpr std::string_view EffectiveUrl ("effective_url"); + constexpr std::string_view LastAuthTimestamp ("last_auth_timestamp"); + constexpr std::string_view LastUser ("username"); + constexpr std::string_view AuthHint ("authHint"); + } + + namespace MediaChangeRequestMsgFields + { + constexpr std::string_view Label ("label"); + constexpr std::string_view MediaNr ("media_nr"); + constexpr std::string_view Device ("device"); + constexpr std::string_view Desc ("desc"); + } + + namespace EjectMsgFields + { + constexpr std::string_view device ("device"); + } + + class ProvideMessage + { + public: + using Code = zypp::proto::MessageCodes; + using FieldVal = HeaderValue; + + static expected create ( const zyppng::RpcMessage &message ); + static expected create ( const zypp::proto::ProvideMessage &message ); + static ProvideMessage createProvideStarted ( const uint32_t reqId, const zypp::Url &url , const std::optional &localFilename = {}, const std::optional &stagingFilename = {} ); + static ProvideMessage createProvideFinished ( const uint32_t reqId, const std::string &localFilename , bool cacheHit ); + static ProvideMessage createAttachFinished ( const uint32_t reqId ); + static ProvideMessage createDetachFinished ( const uint32_t reqId ); + static ProvideMessage createAuthInfo ( const uint32_t reqId, const std::string &user, const std::string &pw, int64_t timestamp, const std::map &extraValues = {} ); + static ProvideMessage createMediaChanged ( const uint32_t reqId ); + static ProvideMessage createRedirect ( const uint32_t reqId, const zypp::Url &newUrl ); + static ProvideMessage createMetalinkRedir ( const uint32_t reqId, const std::vector &newUrls ); + static ProvideMessage createErrorResponse ( const uint32_t reqId, const uint code, const std::string &reason, bool transient = false ); + + static ProvideMessage createProvide ( const uint32_t reqId + , const zypp::Url &url + , const std::optional &filename = {} + , const std::optional &deltaFile = {} + , const std::optional &expFilesize = {} + , bool checkExistOnly = false ); + + static ProvideMessage createCancel ( const uint32_t reqId ); + + static ProvideMessage createAttach( const uint32_t reqId + , const zypp::Url &url + , const std::string attachId + , const std::string &label + , const std::optional &verifyType = {} + , const std::optional &verifyData = {} + , const std::optional &mediaNr = {} ); + + static ProvideMessage createDetach ( const uint32_t reqId, const zypp::Url &attachUrl ); + static ProvideMessage createAuthDataRequest ( const uint32_t reqId, const zypp::Url &effectiveUrl, const std::string &lastTriedUser ="", const std::optional &lastAuthTimestamp = {}, const std::map &extraValues = {} ); + static ProvideMessage createMediaChangeRequest ( const uint32_t reqId, const std::string &label, int32_t mediaNr, const std::vector &devices, const std::optional &desc ); + + uint requestId () const; + void setRequestId ( const uint id ); + + uint code () const; + void setCode ( const uint newCode ); + + std::vector values ( const std::string_view &str ) const; + std::vector values ( const std::string &str ) const; + HeaderValueMap headers() const; + /*! + * Returns the last entry with key \a str in the list of values + * or the default value specified in \a defaultVal + */ + FieldVal value ( const std::string_view &str, const FieldVal &defaultVal = FieldVal() ) const; + FieldVal value ( const std::string &str, const FieldVal &defaultVal = FieldVal() ) const; + void setValue ( const std::string &name, const FieldVal &value ); + void setValue ( const std::string_view &name, const FieldVal &value ); + void addValue ( const std::string &name, const FieldVal &value ); + void addValue ( const std::string_view &name, const FieldVal &value ); + void forEachVal( const std::function &cb ) const; + + zypp::proto::ProvideMessage &impl(); + const zypp::proto::ProvideMessage &impl() const; + + private: + ProvideMessage(); + zypp::RWCOW_pointer _impl; + }; +} + +namespace zypp { + template<> + inline zypp::proto::ProvideMessage* rwcowClone( const zypp::proto::ProvideMessage * rhs ) + { return new zypp::proto::ProvideMessage(*rhs); } +} + + + +#endif diff --git a/zypp-media/ng/private/providequeue_p.h b/zypp-media/ng/private/providequeue_p.h new file mode 100644 index 0000000..907381e --- /dev/null +++ b/zypp-media/ng/private/providequeue_p.h @@ -0,0 +1,145 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\----------------------------------------------------------------------/ +* +* This file contains private API, this might break at any time between releases. +* You have been warned! +* +*/ +#ifndef ZYPP_MEDIA_PRIVATE_PROVIDE_QUEUE_P_H_INCLUDED +#define ZYPP_MEDIA_PRIVATE_PROVIDE_QUEUE_P_H_INCLUDED + +#include "providefwd_p.h" +#include +#include +#include +#include + +#include +#include +#include + +namespace zyppng { + + class RpcMessageStream; + using RpcMessageStreamPtr = std::shared_ptr; + + class ProvideQueue : public Base + { + public: + friend struct ProvideResourceData; + + static constexpr uint32_t InvalidId = (uint32_t) -1; + using Config = zypp::proto::Capabilities; + + using TimePoint = std::chrono::time_point; + + struct Item { + + enum State { + Pending, + Queued, + Running, + Cancelling, + Finished + }; + State _state = Pending; + bool isAttachRequest () const; + bool isFileRequest () const; + bool isDetachRequest() const; + + ProvideRequestRef _request; + }; + + ProvideQueue( ProvidePrivate &parent ); + ~ProvideQueue(); + bool startup ( const std::string &workerScheme, const zypp::Pathname &workDir, const std::string &hostname = "" ); + void enqueue ( ProvideRequestRef request ); + void cancel ( ProvideRequest *item, std::exception_ptr error ); + void detach ( const std::string &id ); + void scheduleNext (); + bool canScheduleMore () const;; + bool empty () const; + + /*! + * Check if the queue is currently idle + */ + bool isIdle () const; + + /*! + * Time point since the queue started to be idle + */ + std::optional idleSince () const; + + /*! + * How many items does this queue currently have + */ + uint requestCount () const; + + /*! + * How many active items does this queue currently have + */ + uint activeRequests () const; + + /*! + * How much bytes does this queue has to download / process, + * for pending requests this is only set if the \ref ProvideSpec + * has a expected download size set. + */ + zypp::ByteCount expectedProvideSize() const; + + /*! + * Returns the hostname this worker belongs to. + * If the worker was not associated with a hostname this will return a empty string. + */ + const std::string &hostname () const; + + const Config &workerConfig () const; + + SignalProxy sigIdle(); + + private: + bool doStartup (); + void processMessage ( ); + void readAllStderr (); + void forwardToLog ( std::string &&logLine ); + void processReadyRead( int channel ); + void procFinished ( int exitCode ); + uint32_t nextRequestId(); + + /*! + * Dequeues the request referenced by \a it. + * Returns a iterator to the next element in the active list + */ + std::list< ProvideQueue::Item >::iterator dequeueActive ( std::list::iterator it ); + void fatalWorkerError ( const std::exception_ptr &reason = nullptr ); + void immediateShutdown ( const std::exception_ptr &reason ); + + /*! + * Cancels the item the iterator \a i is pointing to, advancing the iterator to the next element in the list + */ + std::list< ProvideQueue::Item >::iterator cancelActiveItem (std::list::iterator i, const std::exception_ptr &error ); + + private: + bool _queueShuttingDown = false; + uint8_t _crashCounter = 0; + Config _capabilities; + zypp::Pathname _currentExe; + std::string _myHostname; + ProvidePrivate &_parent; + std::deque< Item > _waitQueue; + std::list< Item > _activeItems; + Process::Ptr _workerProc; + RpcMessageStreamPtr _messageStream; + Signal _sigIdle; + std::optional _idleSince; + }; + +} + +#endif diff --git a/zypp-media/ng/private/provideres_p.h b/zypp-media/ng/private/provideres_p.h new file mode 100644 index 0000000..b129043 --- /dev/null +++ b/zypp-media/ng/private/provideres_p.h @@ -0,0 +1,23 @@ +#ifndef ZYPP_MEDIA_PRIVATE_PROVIDERES_P_H +#define ZYPP_MEDIA_PRIVATE_PROVIDERES_P_H + +#include +#include +#include +#include +#include "providefwd_p.h" + +namespace zyppng { + /*! + * \internal + * The internal shared data structure for \sa ProvideRes objects. + */ + struct ProvideResourceData { + zyppng::Provide::MediaHandle _mediaHandle; + zypp::ManagedFile _myFile; + zypp::Url _resourceUrl; //< The resource where the file was provided from + HeaderValueMap _responseHeaders; //< The response headers + }; +} + +#endif // ZYPP_MEDIA_PRIVATE_PROVIDERES_P_H diff --git a/zypp-media/ng/provide-configvars.h b/zypp-media/ng/provide-configvars.h new file mode 100644 index 0000000..54eccd9 --- /dev/null +++ b/zypp-media/ng/provide-configvars.h @@ -0,0 +1,28 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ +#ifndef ZYPP_MEDIA_PROVIDE_CONFIGVARS_H_INCLUDED +#define ZYPP_MEDIA_PROVIDE_CONFIGVARS_H_INCLUDED + +#include + +namespace zyppng { + // special config strings sent to the workers: + constexpr std::string_view AGENT_STRING_CONF("zconfig://media/UserAgent"); + constexpr std::string_view DISTRO_FLAV_CONF("zconfig://media/DistributionFlavor"); + constexpr std::string_view ANON_ID_CONF("zconfig://media/AnonymousId"); + constexpr std::string_view ATTACH_POINT("zconfig://media/AttachPoint"); + constexpr std::string_view PROVIDER_ROOT("zconfig://media/ProviderRoot"); + + + // request related settings: + constexpr std::string_view NETWORK_METALINK_ENABLED("zypp-nw-metalink-enabled"); //< Enable or disable metalink for a specific request + constexpr std::string_view HANDLER_SPECIFIC_DEVICES("zypp-req-specific-devices"); //< Limit the request to a set of devices. Devices are comma seperated. +} + +#endif diff --git a/zypp-media/ng/provide.cc b/zypp-media/ng/provide.cc new file mode 100644 index 0000000..2afd225 --- /dev/null +++ b/zypp-media/ng/provide.cc @@ -0,0 +1,1265 @@ +#include "private/provide_p.h" +#include "private/providedbg_p.h" +#include "private/providequeue_p.h" +#include "private/provideitem_p.h" +#include +#include +#include +#include +#include +#include +#include + +// required to generate uuids +#include + + +L_ENV_CONSTR_DEFINE_FUNC(ZYPP_MEDIA_PROVIDER_DEBUG) + +namespace zyppng { + + ProvidePrivate::ProvidePrivate(zypp::filesystem::Pathname &&workDir, Provide &pub) + : BasePrivate(pub) + , _workDir( std::move(workDir) ) + , _workerPath( constants::DEFAULT_PROVIDE_WORKER_PATH.data() ) + { + if ( _workDir.empty() ) { + _workDir = zypp::Pathname(".").realpath(); + } else { + _workDir = _workDir.realpath(); + } + + MIL << "Provider workdir is: " << _workDir << std::endl; + + _scheduleTrigger->setSingleShot(true); + Base::connect( *_scheduleTrigger, &Timer::sigExpired, *this, &ProvidePrivate::doSchedule ); + } + + void ProvidePrivate::schedule( ScheduleReason reason ) + { + if ( provideDebugEnabled () ) { + std::string_view reasonStr; + switch( reason ) { + case ProvideStart: + reasonStr = "ProvideStart"; + break; + case QueueIdle: + reasonStr = "QueueIdle"; + break; + case EnqueueItem: + reasonStr = "EnqueueItem"; + break; + case EnqueueReq: + reasonStr = "EnqueueReq"; + break; + case FinishReq: + reasonStr = "FinishReq"; + break; + case RestartAttach: + reasonStr = "RestartAttach"; + break; + } + DBG << "Triggering the schedule timer (" << reasonStr << ")" << std::endl; + } + + // we use a single shot timer that instantly times out when the event loop is entered the next time + // this way we compress many schedule requests that happen during a eventloop run into one + _scheduleTrigger->start(0); + } + + void ProvidePrivate::doSchedule ( zyppng::Timer & ) + { + if ( !_isRunning ) + return; + + if ( _isScheduling ) { + DBG_PRV << "Scheduling triggered during scheduling, returning immediately." << std::endl; + return; + } + + const int cpuLimit = +#ifdef _SC_NPROCESSORS_ONLN + sysconf(_SC_NPROCESSORS_ONLN) * 2; +#else + DEFAULT_CPU_WORKERS; +#endif + + // helper lambda to find the worker that is idle for the longest time + constexpr auto findLaziestWorker = []( const auto &workerQueues, const auto &idleNames ) { + auto candidate = workerQueues.end(); + ProvideQueue::TimePoint candidateIdleSince = ProvideQueue::TimePoint::max(); + + //find the worker thats idle the longest + for ( const auto &name : idleNames ) { + auto thisElem = workerQueues.find(name); + if ( thisElem == workerQueues.end() ) + continue; + + const auto idleS = thisElem->second->idleSince(); + if ( idleS + && ( candidate == workerQueues.end() || *idleS < candidateIdleSince ) ) { + candidateIdleSince = *idleS; + candidate = thisElem; + } + } + + if ( candidate != workerQueues.end() ) + MIL_PRV << "Found idle worker:" << candidate->first << " idle since: " << candidateIdleSince.time_since_epoch().count() << std::endl; + + return candidate; + }; + + // clean up old media + + for ( auto iMedia = _attachedMediaInfos.begin(); iMedia != _attachedMediaInfos.end(); ) { + if ( iMedia->_refCount > 0 ) { + MIL_PRV << "Not releasing media " << iMedia->_name << " refcount is not zero" << std::endl; + ++iMedia; + continue; + } + if ( iMedia->_workerType == ProvideQueue::Config::Downloading ) { + // we keep the information around for an hour so we do not constantly download the media files for no reasonDD + if ( std::chrono::steady_clock::now() - iMedia->_idleSince >= std::chrono::hours(1) ) { + MIL << "Detaching medium " << iMedia->_name << " for baseUrl " << iMedia->_attachedUrl << std::endl; + iMedia = _attachedMediaInfos.erase(iMedia); + continue; + } else { + MIL_PRV << "Not releasing media " << iMedia->_name << " downloading worker and not timed out yet." << std::endl; + } + } else { + // mounting handlers, we need to send a request to the workers + auto bQueue = iMedia->_backingQueue.lock(); + if ( bQueue ) { + zypp::Url url = iMedia->_attachedUrl; + url.setScheme( url.getScheme() + std::string( constants::ATTACHED_MEDIA_SUFFIX) ); + url.setAuthority( iMedia->_name ); + const auto &req = ProvideRequest::createDetach( url ); + if ( req ) { + MIL << "Detaching medium " << iMedia->_name << " for baseUrl " << iMedia->_attachedUrl << std::endl; + bQueue->enqueue ( *req ); + iMedia = _attachedMediaInfos.erase(iMedia); + continue; + } else { + ERR << "Could not send detach request, creating the request failed" << std::endl; + } + } else { + ERR << "Could not send detach request since no backing queue was defined" << std::endl; + } + } + ++iMedia; + } + + zypp::DtorReset schedFlag( _isScheduling, false ); + _isScheduling = true; + + const auto schedStart = std::chrono::steady_clock::now(); + MIL_PRV << "Start scheduling" << std::endl; + + zypp::OnScopeExit deferExitMessage( [&](){ + const auto dur = std::chrono::steady_clock::now() - schedStart; + MIL_PRV << "Exit scheduling after:" << std::chrono::duration_cast( dur ).count () << std::endl; + }); + + // bump inactive items + for ( auto it = _items.begin (); it != _items.end(); ) { + // was maybe released during scheduling + if ( !(*it) ) + it = _items.erase(it); + else { + auto &item = *it; + if ( item->state() == ProvideItem::Uninitialized ) { + item->initialize(); + } + it++; + } + } + + // we are scheduling now, everything that triggered the timer until now we can forget about + _scheduleTrigger->stop(); + + for( auto queueIter = _queues.begin(); queueIter != _queues.end(); queueIter ++ ) { + + const auto &scheme = queueIter->_schemeName; + auto &queue = queueIter->_requests; + + if ( !queue.size() ) + continue; + + const auto &configOpt = schemeConfig ( scheme ); + + MIL_PRV << "Start scheduling for scheme:" << scheme << " queue size is: " << queue.size() << std::endl; + + if ( !configOpt ) { + // FAIL all requests in this queue + ERR << "Scheme: " << scheme << " failed to return a valid configuration." << std::endl; + + while( queue.size() ) { + auto item = std::move( queue.front() ); + queue.pop_front(); + if ( item->owner() ) + item->owner()->finishReq( nullptr, item, ZYPP_EXCPT_PTR(zypp::media::MediaException("Failed to query scheme config.")) ); + } + + continue; + } + + // the scheme config that defines how we schedule requests on this set of queues + const auto &config = configOpt.get(); + const auto isSingleInstance = ( (config.cfg_flags() & ProvideQueue::Config::SingleInstance) == ProvideQueue::Config::SingleInstance ); + if ( config.worker_type() == ProvideQueue::Config::Downloading && !isSingleInstance ) { + + for( auto i = queue.begin (); i != queue.end(); ) { + + // this is the only place where we remove elements from the queue when the scheduling flag is active + // other code just nulls out requests in the queue if during scheduling items need to be removed + while ( i != queue.end() && !(*i) ) { + i = queue.erase(i); + } + + if ( i == queue.end() ) + break; + + ProvideRequestRef item = *i; + + // Downloading queues do not support attaching via a AttachRequest, this is handled by simply providing the media verification files + // If we hit this code path, its a bug + if( item->code() == ProvideMessage::Code::Attach || item->code() == ProvideMessage::Code::Detach ) { + i = queue.erase(i); + if ( item->owner() ) + item->owner()->finishReq( nullptr, item, ZYPP_EXCPT_PTR( zypp::Exception("Downloading Queues do not support ProvideMessage::Code::Attach requests") ) ); + continue; + } + + MIL_PRV << "Trying to schedule request: " << item->urls().front() << std::endl; + + // how many workers for this type do already exist + int existingTypeWorkers = 0; + + // how many currently active connections are there + int existingConnections = 0; + + // all currently available possible queues for the request + std::vector< std::pair > possibleHostWorkers; + + // currently idle workers + std::vector idleWorkers; + + // all mirrors without a existing worker + std::vector mirrsWithoutWorker; + for ( const auto &url : item->urls() ) { + + if ( effectiveScheme( url.getScheme() ) != scheme ) { + MIL << "Mirror URL " << url << " is incompatible with current scheme: " << scheme << ", ignoring." << std::endl; + continue; + } + + if( item->owner()->canRedirectTo( item, url ) ) + mirrsWithoutWorker.push_back( url ); + else { + MIL_PRV << "URL was rejected" << url << std::endl; + } + } + + // at this point the list contains all useable mirrors, if this list is empty the request needs to fail + if( mirrsWithoutWorker.size() == 0 ) { + MIL << "Request has NO usable URLs" << std::endl; + if ( item->owner() ) + item->owner()->finishReq( nullptr, item, ZYPP_EXCPT_PTR(zypp::media::MediaException("No usable URLs in request spec.")) ); + i = queue.erase(i); + continue; + } + + + for ( auto &[ queueName, workerQueue ] : _workerQueues ) { + if ( ProvideQueue::Config::Downloading != workerQueue->workerConfig().worker_type() ) + continue; + + existingTypeWorkers ++; + existingConnections += workerQueue->activeRequests(); + + if ( workerQueue->isIdle() ) + idleWorkers.push_back (queueName); + + if ( !zypp::str::startsWith( queueName, scheme ) ) + continue; + + for ( auto i = mirrsWithoutWorker.begin (); i != mirrsWithoutWorker.end(); ) { + const auto &u = *i; + if ( u.getHost() == workerQueue->hostname() ) { + if ( workerQueue->requestCount() < constants::DEFAULT_ACTIVE_CONN_PER_HOST ) + possibleHostWorkers.push_back( {u, workerQueue.get()} ); + i = mirrsWithoutWorker.erase( i ); + // we can not stop after removing the first hit, since there could be multiple mirrors with the same hostname + } else { + ++i; + } + } + } + + if( provideDebugEnabled() ) { + MIL << "Current stats: " << std::endl; + MIL << "Existing type workers: " << existingTypeWorkers << std::endl; + MIL << "Existing active connections: " << existingConnections << std::endl; + MIL << "Possible host workers: "<< possibleHostWorkers.size() << std::endl; + MIL << "Mirrors without worker: " << mirrsWithoutWorker.size() << std::endl; + } + + // need to wait for requests to finish in order to schedule more requests + if ( existingConnections >= constants::DEFAULT_ACTIVE_CONN ) { + MIL_PRV << "Reached maximum nr of connections, break" << std::endl; + break; + } + + // if no workers are running, take the first mirror and start a worker for it + // if < nr of workers are running, use a mirror we do not have a conn yet to + if ( existingTypeWorkers < constants::DEFAULT_MAX_DYNAMIC_WORKERS + && mirrsWithoutWorker.size() ) { + + MIL_PRV << "Free worker slots and available mirror URLs, starting a new worker" << std::endl; + + //@TODO out of the available mirrors use the best one based on statistics ( if available ) + bool found = false; + for( const auto &url : mirrsWithoutWorker ) { + + // mark this URL as used now, in case the queue can not be started we won't try it anymore + if ( !item->owner()->safeRedirectTo ( item, url ) ) + continue; + + ProvideQueueRef q = std::make_shared( *this ); + if ( !q->startup( scheme, _workDir / scheme / url.getHost(), url.getHost() ) ) { + break; + } else { + + MIL_PRV << "Started worker for " << url.getHost() << " enqueing request" << std::endl; + + item->setActiveUrl(url); + found = true; + + std::string str = zypp::str::Format("%1%://%2%") % scheme % url.getHost(); + _workerQueues[str] = q; + q->enqueue( item ); + break; + } + } + + if( found ) { + i = queue.erase(i); + continue; + } + } + + // if we cannot start a new worker, find the best queue where we can push the item into + if ( possibleHostWorkers.size() ) { + + MIL_PRV << "No free worker slots, looking for the best existing worker" << std::endl; + bool found = false; + while( possibleHostWorkers.size () ) { + std::vector< std::pair >::iterator candidate = possibleHostWorkers.begin(); + for ( auto i = candidate+1; i != possibleHostWorkers.end(); i++ ) { + if ( i->second->activeRequests () < candidate->second->activeRequests () ) + candidate = i; + } + + if ( !item->owner()->safeRedirectTo( item, candidate->first ) ) { + possibleHostWorkers.erase( candidate ); + continue; + } + + MIL_PRV << "Using existing worker " << candidate->first.getHost() << " to download request" << std::endl; + + found = true; + item->setActiveUrl( candidate->first ); + candidate->second->enqueue( item ); + break; + } + + if( found ) { + i = queue.erase(i); + continue; + } + } + + // if we reach this place all we can now try is to decomission idle queues and use the new slot to start + // a new worker + if ( idleWorkers.size() && mirrsWithoutWorker.size() ) { + + MIL_PRV << "No free worker slots, no slots in existing queues, trying to decomission idle queues." << std::endl; + + auto candidate = findLaziestWorker( _workerQueues, idleWorkers ); + if ( candidate != _workerQueues.end() ) { + + // for now we decomission the worker and start a new one, should we instead introduce a "reset" message + // that repurposes the worker to another hostname/workdir config? + _workerQueues.erase(candidate); + + //@TODO out of the available mirrors use the best one based on statistics ( if available ) + bool found = false; + for( const auto &url : mirrsWithoutWorker ) { + + if ( !item->owner()->safeRedirectTo ( item, url ) ) + continue; + + ProvideQueueRef q = std::make_shared( *this ); + if ( !q->startup( scheme, _workDir / scheme / url.getHost(), url.getHost() ) ) { + break; + } else { + + MIL_PRV << "Replaced worker for " << url.getHost() << ", enqueing request" << std::endl; + + item->setActiveUrl(url); + found = true; + + auto str = zypp::str::Format("%1%://%2%") % scheme % url.getHost(); + _workerQueues[str] = q; + q->enqueue( item ); + } + } + + if( found ) { + i = queue.erase(i); + continue; + } + } + } + + // if we reach here we skip over the item and try to schedule it again later + MIL_PRV << "End of line, deferring request for next try." << std::endl; + i++; + + } + } else if ( config.worker_type() == ProvideQueue::Config::CPUBound && !isSingleInstance ) { + + for( auto i = queue.begin (); i != queue.end(); ) { + + // this is the only place where we remove elements from the queue when the scheduling flag is active + // other code just nulls out requests in the queue if during scheduling items need to be removed + while ( i != queue.end() && !(*i) ) { + i = queue.erase(i); + } + + if ( i == queue.end() ) + break; + + // make a real reference so it does not dissapear when we remove it from the queue + ProvideRequestRef item = *i; + + // CPU bound queues do not support attaching via a AttachRequest, this is handled by simply providing the media verification files + // If we hit this code path, its a bug + if( item->code() == ProvideMessage::Code::Attach || item->code() == ProvideMessage::Code::Detach ) { + i = queue.erase(i); + if ( item->owner () ) + item->owner()->finishReq( nullptr, item, ZYPP_EXCPT_PTR( zypp::Exception("CPU bound Queues do not support ProvideAttachSpecRef requests") ) ); + continue; + } + + MIL_PRV << "Trying to schedule request: " << item->urls().front() << std::endl; + + // how many workers for this type do already exist + int existingTypeWorkers = 0; + int existingSchemeWorkers = 0; + + // all currently available possible queues for the request + std::vector< ProvideQueue* > possibleWorkers; + + // currently idle workers + std::vector idleWorkers; + + // the URL we are going to use this time + zypp::Url url; + + //CPU bound queues do not spawn per mirrors, we use the first compatible URL + for ( const auto &tmpurl : item->urls() ) { + if ( effectiveScheme( tmpurl.getScheme() ) != scheme ) { + MIL << "Mirror URL " << tmpurl << " is incompatible with current scheme: " << scheme << ", ignoring." << std::endl; + continue; + } + url = tmpurl; + break; + } + + // at this point if the URL is empty the request needs to fail + if( !url.isValid() ) { + MIL << "Request has NO usable URLs" << std::endl; + if ( item->owner() ) + item->owner()->finishReq( nullptr, item, ZYPP_EXCPT_PTR(zypp::media::MediaException("No usable URLs in request spec.")) ); + i = queue.erase(i); + continue; + } + + for ( auto &[ queueName, workerQueue ] : _workerQueues ) { + + if ( ProvideQueue::Config::CPUBound != workerQueue->workerConfig().worker_type() ) + continue; + + const bool thisScheme = zypp::str::startsWith( queueName, scheme ); + + existingTypeWorkers ++; + if ( thisScheme ) { + existingSchemeWorkers++; + if ( workerQueue->canScheduleMore() ) + possibleWorkers.push_back(workerQueue.get()); + } + + if ( workerQueue->isIdle() ) + idleWorkers.push_back(queueName); + } + + if( provideDebugEnabled() ) { + MIL << "Current stats: " << std::endl; + MIL << "Existing type workers: " << existingTypeWorkers << std::endl; + MIL << "Possible CPU workers: "<< possibleWorkers.size() << std::endl; + } + + // first we use existing idle workers of the current type + if ( possibleWorkers.size() ) { + bool found = false; + for ( auto &w : possibleWorkers ) { + if ( w->isIdle() ) { + MIL_PRV << "Using existing idle worker to provide request" << std::endl; + // this is not really required because we are not doing redirect checks + item->owner()->redirectTo ( item, url ); + item->setActiveUrl( url ); + w->enqueue( item ); + i = queue.erase(i); + found = true; + break; + } + } + if ( found ) + continue; + } + + // we first start as many workers as we need before queueing more request to existing ones + if ( existingTypeWorkers < cpuLimit ) { + + MIL_PRV << "Free CPU slots, starting a new worker" << std::endl; + + // this is not really required because we are not doing redirect checks + item->owner()->redirectTo ( item, url ); + + ProvideQueueRef q = std::make_shared( *this ); + if ( q->startup( scheme, _workDir / scheme ) ) { + + item->setActiveUrl(url); + + auto str = zypp::str::Format("%1%#%2%") % scheme % existingSchemeWorkers; + _workerQueues[str] = q; + q->enqueue( item ); + i = queue.erase(i); + continue; + } else { + // CPU bound requests can not recover from this error + i = queue.erase(i); + if ( item->owner() ) + item->owner()->finishReq( nullptr, item, ZYPP_EXCPT_PTR( zypp::Exception("Unable to start worker for request.") ) ); + continue; + } + } + + // we can not start more workers, all we can do now is fill up queues of existing ones + if ( possibleWorkers.size() ) { + MIL_PRV << "No free CPU slots, looking for the best existing worker" << std::endl; + + if( possibleWorkers.size () ) { + std::vector::iterator candidate = possibleWorkers.begin(); + for ( auto i = candidate+1; i != possibleWorkers.end(); i++ ) { + if ( (*i)->activeRequests () < (*candidate)->activeRequests () ) + candidate = i; + } + + // this is not really required because we are not doing redirect checks + item->owner()->redirectTo ( item, url ); + + MIL_PRV << "Using existing worker to provide request" << std::endl; + item->setActiveUrl( url ); + (*candidate)->enqueue( item ); + i = queue.erase(i); + continue; + } + } + + // if we reach this place all we can now try is to decomission idle queues and use the new slot to start + // a new worker + if ( idleWorkers.size() ) { + + MIL_PRV << "No free CPU slots, no slots in existing queues, trying to decomission idle queues." << std::endl; + + auto candidate = findLaziestWorker( _workerQueues, idleWorkers ); + if ( candidate != _workerQueues.end() ) { + + _workerQueues.erase(candidate); + + // this is not really required because we are not doing redirect checks + item->owner()->redirectTo ( item, url ); + + ProvideQueueRef q = std::make_shared( *this ); + if ( q->startup( scheme, _workDir / scheme ) ) { + + MIL_PRV << "Replaced worker, enqueing request" << std::endl; + + item->setActiveUrl(url); + + auto str = zypp::str::Format("%1%#%2%") % scheme % ( existingSchemeWorkers + 1 ); + _workerQueues[str] = q; + q->enqueue( item ); + i = queue.erase(i); + continue; + } else { + // CPU bound requests can not recover from this error + i = queue.erase(i); + if ( item->owner() ) + item->owner()->finishReq( nullptr, item, ZYPP_EXCPT_PTR( zypp::Exception("Unable to start worker for request.") ) ); + continue; + } + } + } else { + MIL_PRV << "No idle workers and no free CPU spots, wait for the next schedule run" << std::endl; + break; + } + + // if we reach here we skip over the item and try to schedule it again later + MIL_PRV << "End of line, deferring request for next try." << std::endl; + i++; + } + + } else { + // either SingleInstance worker or Mounting/VolatileMounting + + for( auto i = queue.begin (); i != queue.end(); ) { + + // this is the only place where we remove elements from the queue when the scheduling flag is active + // other code just nulls out requests in the queue if during scheduling items need to be removed + while ( i != queue.end() && !(*i) ) { + i = queue.erase(i); + } + + if ( i == queue.end() ) + break; + + // make a real reference so it does not dissapear when we remove it from the queue + ProvideRequestRef item = *i; + MIL_PRV << "Trying to schedule request: " << item->urls().front() << std::endl; + + zypp::Url url; + + //mounting queues do not spawn per mirrors, we use the first compatible URL + for ( const auto &tmpurl : item->urls() ) { + if ( effectiveScheme( tmpurl.getScheme() ) != scheme ) { + MIL << "Mirror URL " << tmpurl << " is incompatible with current scheme: " << scheme << ", ignoring." << std::endl; + continue; + } + url = tmpurl; + break; + } + + // at this point if the URL is empty the request needs to fail + if( !url.isValid() ) { + MIL << "Request has NO usable URLs" << std::endl; + if ( item->owner() ) + item->owner()->finishReq( nullptr, item, ZYPP_EXCPT_PTR(zypp::media::MediaException("No usable URLs in request spec.")) ); + i = queue.erase(i); + continue; + } + + + ProvideQueue *qToUse = nullptr; + if ( !_workerQueues.count(scheme) ) { + ProvideQueueRef q = std::make_shared( *this ); + if ( !q->startup( scheme, _workDir / scheme ) ) { + ERR << "Worker startup failed!" << std::endl; + // mounting/single instance requests can not recover from this error + i = queue.erase(i); + + if ( item->owner() ) + item->owner()->finishReq( nullptr, item, ZYPP_EXCPT_PTR( zypp::Exception("Unable to start worker for request.") ) ); + continue; + } + + MIL_PRV << "Started worker, enqueing request" << std::endl; + qToUse = q.get(); + _workerQueues[scheme] = q; + } else { + MIL_PRV << "Found worker, enqueing request" << std::endl; + qToUse = _workerQueues.at(scheme).get(); + } + + // this is not really required because we are not doing redirect checks + item->owner()->redirectTo ( item, url ); + + item->setActiveUrl(url); + qToUse->enqueue( item ); + i = queue.erase(i); + } + } + } + } + + std::list &ProvidePrivate::items() + { + return _items; + } + + zypp::media::CredManagerOptions &ProvidePrivate::credManagerOptions () + { + return _credManagerOptions; + } + + std::vector &ProvidePrivate::attachedMediaInfos() + { + return _attachedMediaInfos; + } + + expected ProvidePrivate::schemeConfig( const std::string &scheme ) + { + if ( auto i = _schemeConfigs.find( scheme ); i != _schemeConfigs.end() ) { + return expected::success(i->second); + } else { + // we do not have the queue config yet, we need to start a worker to get one + ProvideQueue q( *this ); + if ( !q.startup( scheme, _workDir / scheme ) ) { + return expected::error(ZYPP_EXCPT_PTR(zypp::media::MediaException("Failed to start worker to read scheme config."))); + } + auto newItem = _schemeConfigs.insert( std::make_pair( scheme, q.workerConfig() )); + return expected::success(newItem.first->second); + } + } + + std::optional ProvidePrivate::addToFileCache( const zypp::filesystem::Pathname &downloadedFile ) + { + const auto &key = downloadedFile.asString(); + + if ( !zypp::PathInfo(downloadedFile).isExist() ) { + _fileCache.erase ( key ); + return {}; + } + + auto i = _fileCache.insert( { key, FileCacheItem() } ); + if ( !i.second ) { + // file did already exist in the cache, return the shared data + i.first->second._deathTimer.reset(); + return i.first->second._file; + } + + i.first->second._file = zypp::ManagedFile( downloadedFile, zypp::filesystem::unlink ); + return i.first->second._file; + } + + bool ProvidePrivate::isInCache ( const zypp::Pathname &downloadedFile ) const + { + const auto &key = downloadedFile.asString(); + return (_fileCache.count(key) > 0); + } + + void ProvidePrivate::queueItem ( ProvideItemRef item ) + { + _items.push_back( item ); + schedule( ProvidePrivate::EnqueueItem ); + } + + void ProvidePrivate::dequeueItem( ProvideItem *item) + { + auto elem = std::find_if( _items.begin(), _items.end(), [item]( const auto &i){ return i.get() == item; } ); + if ( elem != _items.end() ) { + if ( _isScheduling ) { + (*elem).reset(); + } else { + _items.erase(elem); + } + } + } + + std::string ProvidePrivate::nextMediaId() const + { + zypp::AutoDispose rawStr( g_uuid_string_random (), g_free ); + return zypp::str::asString ( rawStr.value() ); + } + + AttachedMediaInfo &ProvidePrivate::addMedium(ProvideQueue::Config::WorkerType workerType, const zypp::Url &baseUrl, ProvideMediaSpec &spec ) + { + auto str = nextMediaId(); + MIL_PRV << "Generated new ID for media attachment: " << str << std::endl; + _attachedMediaInfos.push_back( AttachedMediaInfo{ std::move(str), {}, workerType, baseUrl, spec } ); + return _attachedMediaInfos.back(); + } + + AttachedMediaInfo &ProvidePrivate::addMedium(zypp::proto::Capabilities::WorkerType workerType, ProvideQueueWeakRef backingQueue, const std::string &id, const zypp::Url &baseUrl, ProvideMediaSpec &spec) + { + MIL_PRV << "New media attachment with id: " << id << std::endl; + _attachedMediaInfos.push_back( AttachedMediaInfo{ id, backingQueue, workerType, baseUrl, spec } ); + return _attachedMediaInfos.back(); + } + + bool ProvidePrivate::queueRequest ( ProvideRequestRef req ) + { + const auto &schemeName = effectiveScheme( req->url().getScheme() ); + auto existingQ = std::find_if( _queues.begin (), _queues.end(), [&schemeName]( const auto &qItem) { + return (qItem._schemeName == schemeName); + }); + if ( existingQ != _queues.end() ) { + existingQ->_requests.push_back(req); + } else { + _queues.push_back( ProvidePrivate::QueueItem{ schemeName, {req} } ); + } + + schedule( ProvidePrivate::EnqueueReq ); + return true; + } + + bool ProvidePrivate::dequeueRequest(ProvideRequestRef req , std::exception_ptr error) + { + auto queue = req->currentQueue (); + if ( queue ) { + queue->cancel( req.get(), error ); + return true; + } else { + // Request not started yet, search request queues + for ( auto &q : _queues ) { + auto elem = std::find( q._requests.begin(), q._requests.end(), req ); + if ( elem != q._requests.end() ) { + q._requests.erase(elem); + + if ( req->owner() ) + req->owner()->finishReq( nullptr, req, error ); + return true; + } + } + } + return false; + } + + const zypp::Pathname &ProvidePrivate::workerPath() const + { + return _workerPath; + } + + const std::string ProvidePrivate::queueName( ProvideQueue &q ) const + { + for ( const auto &v : _workerQueues ) { + if ( v.second.get() == &q ) + return v.first; + } + return {}; + } + + bool ProvidePrivate::isRunning() const + { + return _isRunning; + } + + std::string ProvidePrivate::effectiveScheme(const std::string &scheme) const + { + const std::string &ss = zypp::str::stripSuffix( scheme, constants::ATTACHED_MEDIA_SUFFIX ); + if ( auto it = _workerAlias.find ( ss ); it != _workerAlias.end () ) { + return it->second; + } + return ss; + } + + void ProvidePrivate::onPulseTimeout( Timer & ) + { + DBG_PRV << "Pulse timeout" << std::endl; + + auto now = std::chrono::steady_clock::now(); + + if ( _log ) _log->pulse(); + + // release old cache files + for ( auto i = _fileCache.begin (); i != _fileCache.end(); ) { + auto &cacheItem = i->second; + if ( cacheItem._file.unique() ) { + if ( cacheItem._deathTimer ) { + if ( now - *cacheItem._deathTimer < std::chrono::seconds(20) ) { + MIL << "Releasing file " << *i->second._file << " from cache, death timeout." << std::endl; + i = _fileCache.erase(i); + continue; + } + } else { + // start the death timeout + cacheItem._deathTimer = std::chrono::steady_clock::now(); + } + } + + ++i; + } + } + + void ProvidePrivate::onQueueIdle() + { + if ( !_items.empty() ) + return; + for ( auto &[k,q] : _workerQueues ) { + if ( !q->empty() ) + return; + } + + // all queues are empty + _sigIdle.emit(); + } + + void ProvidePrivate::onItemStateChanged( ProvideItem &item ) + { + if ( item.state() == ProvideItem::Finished ) { + auto itemRef = item.shared_this(); + auto i = std::find( _items.begin(), _items.end(), itemRef ); + if ( i == _items.end() ) { + ERR << "State of unknown Item changed, ignoring" << std::endl; + return; + } + if ( _isScheduling ) + i->reset(); + else + _items.erase(i); + } + if ( _items.empty() ) + onQueueIdle(); + } + + uint32_t ProvidePrivate::nextRequestId() + { + //@TODO is it required to handle overflow? + return ++_nextRequestId; + } + + struct ProvideMediaHandle::Data + { + Data( Provide &parent, const std::string &hdl ) + : _parent( parent.weak_this() ) + , _hdlName(hdl) { } + + ~Data() { + auto p = _parent.lock(); if (p) p->releaseMedia(_hdlName); + } + + ProvideWeakRef _parent; + std::string _hdlName; + }; + + ProvideMediaHandle::ProvideMediaHandle( Provide &parent, const std::string &hdl ) + : _ref( std::make_shared( parent, hdl ) ) + {} + + std::shared_ptr ProvideMediaHandle::parent() const + { + return _ref->_parent.lock(); + } + + bool ProvideMediaHandle::isValid() const + { + return ( _ref.get() != nullptr ); + } + + std::string ProvideMediaHandle::handle() const + { + return _ref->_hdlName; + } + + + Provide::Provide( const zypp::Pathname &workDir ) : Base( *new ProvidePrivate( zypp::Pathname(workDir), *this ) ) + { + Z_D(); + connect( *d->_pulseTimer, &Timer::sigExpired, *d, &ProvidePrivate::onPulseTimeout ); + } + + ProvideRef Provide::create( const zypp::filesystem::Pathname &workDir ) + { + return ProvideRef( new Provide(workDir) ); + } + + AsyncOpRef> Provide::attachMedia( const zypp::Url &url, const ProvideMediaSpec &request ) + { + return attachMedia ( std::vector{url}, request ); + } + + AsyncOpRef> Provide::attachMedia( const std::vector &urls, const ProvideMediaSpec &request ) + { + Z_D(); + + if ( urls.empty() ) { + return makeReadyResult( expected::error( ZYPP_EXCPT_PTR( zypp::media::MediaException("No usable mirrors in mirrorlist."))) ); + } + + // sanitize the mirrors to contain only URLs that have same worker types + std::vector usableMirrs; + std::optional scheme; + + for ( auto mirrIt = urls.begin() ; mirrIt != urls.end(); mirrIt++ ) { + const auto &s = d->schemeConfig( d->effectiveScheme( mirrIt->getScheme() ) ); + if ( !s ) { + WAR << "URL: " << *mirrIt << " is not supported, ignoring!" << std::endl; + continue; + } + if ( !scheme ) { + scheme = *s; + usableMirrs.push_back ( *mirrIt ); + } else { + if ( scheme->worker_type () == s->worker_type () ) { + usableMirrs.push_back( *mirrIt ); + } else { + WAR << "URL: " << *mirrIt << " has different worker type than the primary URL: "<< usableMirrs.front() <<", ignoring!" << std::endl; + } + } + } + + if ( !scheme || usableMirrs.empty() ) { + return makeReadyResult( expected::error( ZYPP_EXCPT_PTR ( zypp::media::MediaException("No valid mirrors available") )) ); + } + + // first check if there is a already attached medium we can use as well + auto &attachedMedia = d->attachedMediaInfos (); + for ( auto &medium : attachedMedia ) { + if ( medium.isSameMedium ( usableMirrs, request ) ) { + medium.ref(); + return makeReadyResult( expected::success( Provide::MediaHandle( *this, medium._name) ) ); + } + } + + auto op = AttachMediaItem::create( usableMirrs, request, *d_func() ); + d->queueItem (op); + return op->promise(); + } + + void Provide::releaseMedia( const std::string &mediaRef ) + { + Z_D(); + + if ( mediaRef.empty() ) + return; + + const auto i = std::find_if( d->_attachedMediaInfos.begin(), d->_attachedMediaInfos.end(), [&]( const auto &info ){ return info._name == mediaRef;} ); + if ( i == d->_attachedMediaInfos.end() ) { + ERR << "Unknown media attach handle" << std::endl; + return; + } + + // only unref'ing here, the scheduler will generate a message to the queues if needed + i->unref(); + } + + AsyncOpRef< expected > Provide::provide( const std::vector &urls, const ProvideFileSpec &request ) + { + Z_D(); + auto op = ProvideFileItem::create( urls, request, *d ); + d->queueItem (op); + return op->promise(); + } + + AsyncOpRef< expected > Provide::provide( const zypp::Url &url, const ProvideFileSpec &request ) + { + return provide( std::vector{ url }, request ); + } + + AsyncOpRef< expected > Provide::provide( const MediaHandle &attachHandle, const zypp::Pathname &fileName, const ProvideFileSpec &request ) + { + Z_D(); + const auto i = std::find_if( d->_attachedMediaInfos.begin(), d->_attachedMediaInfos.end(), [&]( const auto &info ){ return info._name == attachHandle.handle();} ); + if ( i == d->_attachedMediaInfos.end() ) { + return makeReadyResult( expected::error( ZYPP_EXCPT_PTR( zypp::media::MediaException("Invalid attach handle")) ) ); + } + + // for downloading items we need to make the baseUrl part of the request URL + zypp::Url url = i->_attachedUrl; + + // real mount devices use a ID to reference a attached medium, for those we do not need to send the baseUrl as well since its already + // part of the mount point, so if we mount host:/path/to/repo to the ID 1234 and look for the file /path/to/repo/file1 the request URL will look like: nfs-media://1234/file1 + if ( i->_workerType == ProvideQueue::Config::SimpleMount || i->_workerType == ProvideQueue::Config::VolatileMount ) { + url = zypp::Url(); + // work around the zypp::Url requirements for certain Url schemes by attaching a suffix, that way we are always able to have a authority + url.setScheme( i->_attachedUrl.getScheme() + std::string(constants::ATTACHED_MEDIA_SUFFIX) ); + url.setAuthority( i->_name ); + url.setPathName("/"); + } + + url.appendPathName( fileName ); + auto op = ProvideFileItem::create( {url}, request, *d ); + + i->ref(); + op->setMediaRef( MediaHandle( *this, i->_name )); + + d->queueItem (op); + return op->promise(); + } + + AsyncOpRef> Provide::checksumForFile ( const zypp::Pathname &p, const std::string &algorithm ) + { + using namespace zyppng::operators; + + zypp::Url url("chksum:///"); + url.setPathName( p ); + auto fut = provide( url, zyppng::ProvideFileSpec().setCustomHeaderValue( "chksumType", algorithm ) ) + | mbind( [algorithm]( zyppng::ProvideRes &&chksumRes ) { + if ( chksumRes.headers().contains(algorithm) ) + return expected::success( chksumRes.headers().value(algorithm).asString() ); + return expected::error( ZYPP_EXCPT_PTR( zypp::FileCheckException("Invalid/Empty checksum returned from worker") ) ); + } ); + return fut; + } + + AsyncOpRef> Provide::copyFile ( const zypp::Pathname &source, const zypp::Pathname &target ) + { + using namespace zyppng::operators; + + zypp::Url url("copy:///"); + url.setPathName( source ); + auto fut = provide( url, ProvideFileSpec().setDestFilenameHint( target )) + | mbind( [&]( ProvideRes &©Res) { + return expected::success( copyRes.asManagedFile() ); + } ); + return fut; + } + + void Provide::start () + { + Z_D(); + d->_isRunning = true; + d->_pulseTimer->start( 5000 ); + d->schedule( ProvidePrivate::ProvideStart ); + if ( d->_log ) d->_log->provideStart(); + } + + void Provide::setWorkerPath(const zypp::filesystem::Pathname &path) + { + d_func()->_workerPath = path; + } + + bool Provide::ejectDevice(const std::string &queueRef, const std::string &device) + { + if ( !queueRef.empty() ) { + return zypp::media::CDTools::openTray(device); + } + return false; + } + + void Provide::setStatusTracker( ProvideStatusRef tracker ) + { + d_func()->_log = tracker; + } + + const zypp::Pathname &Provide::providerWorkdir () const + { + return d_func()->_workDir; + } + + const zypp::media::CredManagerOptions &Provide::credManangerOptions () const + { + Z_D(); + return d->_credManagerOptions; + } + + void Provide::setCredManagerOptions( const zypp::media::CredManagerOptions & opt ) + { + d_func()->_credManagerOptions = opt; + } + + SignalProxy Provide::sigIdle() + { + return d_func()->_sigIdle; + } + + SignalProxy &, const std::optional &)> Provide::sigMediaChangeRequested() + { + return d_func()->_sigMediaChange; + } + + SignalProxy< std::optional ( const zypp::Url &reqUrl, const std::string &triedUsername, const std::map &extraValues ) > Provide::sigAuthRequired() + { + return d_func()->_sigAuthRequired; + } + + ZYPP_IMPL_PRIVATE(Provide); + + ProvideStatus::ProvideStatus( ProvideRef parent ) + : _provider( parent ) + { } + + void ProvideStatus::provideStart () + { + _stats = Stats(); + _stats._startTime = std::chrono::steady_clock::now(); + _stats._lastPulseTime = std::chrono::steady_clock::now(); + } + + void ProvideStatus::itemDone ( ProvideItem &item ) + { + const auto &sTime = item.startTime(); + const auto &fTime = item.finishedTime(); + if ( sTime > sTime.min() && fTime >= sTime ) { + auto duration = std::chrono::duration_cast( item.finishedTime() - item.startTime() ); + if ( duration.count() ) + MIL << "Item finished after " << duration.count() << " seconds, with " << zypp::ByteCount( item.currentStats()->_bytesProvided.operator zypp::ByteCount::SizeType() / duration.count() ) << "/s" << std::endl; + MIL << "Item finished after " << (item.finishedTime() - item.startTime()).count() << " ns" << std::endl; + } + pulse( ); + } + + void ProvideStatus::itemFailed ( ProvideItem &item ) + { + MIL << "Item failed" << std::endl; + } + + const ProvideStatus::Stats& ProvideStatus::stats() const + { + return _stats; + } + + void ProvideStatus::pulse ( ) + { + auto prov = _provider.lock(); + if ( !prov ) + return; + + const auto lastFinishedBytes = _stats._finishedBytes; + const auto lastPartialBytes = _stats._partialBytes; + _stats._expectedBytes = _stats._finishedBytes; // finished bytes are expected too! + zypp::ByteCount tmpPartialBytes (0); // bytes that are finished in staging, but not commited to cache yet + + for ( const auto &i : prov->d_func()->items() ) { + + if ( !i // maybe released during scheduling + || i->state() == ProvideItem::Cancelling ) + continue; + + if ( i->state() == ProvideItem::Uninitialized + || i->state() == ProvideItem::Pending ) { + _stats._expectedBytes += i->bytesExpected(); + continue; + } + + i->pulse(); + + const auto & stats = i->currentStats(); + const auto & prevStats = i->previousStats(); + if ( !stats || !prevStats ) { + ERR << "Bug! Stats should be initialized by now" << std::endl; + continue; + } + + if ( i->state() == ProvideItem::Downloading + || i->state() == ProvideItem::Processing + || i->state() == ProvideItem::Finalizing ) { + _stats._expectedBytes += stats->_bytesExpected; + tmpPartialBytes += stats->_bytesProvided; + } else if ( i->state() == ProvideItem::Finished ) { + _stats._finishedBytes += stats->_bytesProvided; // remember those bytes are finished in stats directly + _stats._expectedBytes += stats->_bytesProvided; + } + } + + const auto now = std::chrono::steady_clock::now(); + const auto sinceLast = std::chrono::duration_cast( now - _stats._lastPulseTime ); + const auto lastFinB = lastPartialBytes + lastFinishedBytes; + const auto currFinB = tmpPartialBytes + _stats._finishedBytes; + + const auto diff = currFinB - lastFinB; + _stats._lastPulseTime = now; + _stats._partialBytes = tmpPartialBytes; + + if ( sinceLast >= std::chrono::seconds(1) ) + _stats._perSecondSinceLastPulse = ( diff / ( sinceLast.count() ) ); + + auto sinceStart = std::chrono::duration_cast( _stats._lastPulseTime - _stats._startTime ); + if ( sinceStart.count() ) { + const size_t diff = _stats._finishedBytes + _stats._partialBytes; + _stats._perSecond = zypp::ByteCount( diff / sinceStart.count() ); + } + } +} diff --git a/zypp-media/ng/provide.h b/zypp-media/ng/provide.h new file mode 100644 index 0000000..a5b5ad2 --- /dev/null +++ b/zypp-media/ng/provide.h @@ -0,0 +1,171 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ +#ifndef ZYPP_MEDIA_PROVIDE_H_INCLUDED +#define ZYPP_MEDIA_PROVIDE_H_INCLUDED + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace zypp { + class Url; + namespace media { + struct CredManagerOptions; + } +} + +/*! + * @TODO Fix bsc#1174011 "auth=basic ignored in some cases" for provider + * We should proactively add the password to the request if basic auth is configured + * and a password is available in the credentials but not in the URL. + * + * We should be a bit paranoid here and require that the URL has a user embedded, otherwise we go the default route + * and ask the server first about the auth method + */ +namespace zyppng { + + class ProvidePrivate; + using AnyMap = std::unordered_map; + + /*! + * RAII helper for media handles + */ + class ProvideMediaHandle + { + public: + ProvideMediaHandle () = default; + ProvideMediaHandle ( Provide &parent, const std::string &hdl ); + std::shared_ptr parent() const; + bool isValid () const; + std::string handle() const; + private: + struct Data; + std::shared_ptr _ref; + }; + + /*! + * Provide status observer object, this can be used to provide good insight into the status of the provider, its items and + * all running requests. + */ + class ProvideStatus + { + public: + + struct Stats { + std::chrono::steady_clock::time_point _startTime; + std::chrono::steady_clock::time_point _lastPulseTime; + uint _itemsSinceStart = 0; //< How many items have been started since Provide::start() was called + uint _runningItems = 0; //< How many items are currently running + zypp::ByteCount _finishedBytes; //< The number of bytes that were finished completely + zypp::ByteCount _expectedBytes; //< The number of currently expected bytes + zypp::ByteCount _partialBytes; //< The number of bytes of items that were already partially downloaded but the item they belong to is not finished + zypp::ByteCount _perSecondSinceLastPulse; //< The download speed since the last pulse + zypp::ByteCount _perSecond; //< The download speed we are currently operating with + }; + + ProvideStatus( ProvideRef parent ); + virtual ~ProvideStatus(){} + + virtual void provideStart (); + virtual void provideDone (){} + virtual void itemStart ( ProvideItem &item ){} + virtual void itemDone ( ProvideItem &item ); + virtual void itemFailed ( ProvideItem &item ); + virtual void requestStart ( ProvideItem &item, uint32_t reqId, const zypp::Url &url, const AnyMap &extraData = {} ){} + virtual void requestDone ( ProvideItem &item, uint32_t reqId, const AnyMap &extraData = {} ){} + virtual void requestRedirect ( ProvideItem &item, uint32_t reqId, const zypp::Url &toUrl, const AnyMap &extraData = {} ){} + virtual void requestFailed ( ProvideItem &item, uint32_t reqId, const std::exception_ptr err, const AnyMap &requestData = {} ){} + virtual void pulse ( ); + + const Stats &stats() const; + + private: + Stats _stats; + ProvideWeakRef _provider; + }; + + class Provide : public Base + { + ZYPP_DECLARE_PRIVATE(Provide); + template friend class ProvidePromise; + friend class ProvideItem; + friend class ProvideMediaHandle; + friend class ProvideStatus; + public: + + using MediaHandle = ProvideMediaHandle; + + static ProvideRef create( const zypp::Pathname &workDir = "" ); + + AsyncOpRef> attachMedia( const std::vector &urls, const ProvideMediaSpec &request ); + AsyncOpRef> attachMedia( const zypp::Url &url, const ProvideMediaSpec &request ); + + AsyncOpRef> provide( const std::vector &urls, const ProvideFileSpec &request ); + AsyncOpRef> provide( const zypp::Url &url, const ProvideFileSpec &request ); + AsyncOpRef> provide( const MediaHandle &attachHandle, const zypp::Pathname &fileName, const ProvideFileSpec &request ); + + /*! + * Schedules a job to calculate the checksum for the given file + */ + AsyncOpRef> checksumForFile ( const zypp::Pathname &p, const std::string &algorithm ); + + /*! + * Schedules a copy job to copy a file from \a source to \a target + */ + AsyncOpRef> copyFile ( const zypp::Pathname &source, const zypp::Pathname &target ); + + void start(); + void setWorkerPath( const zypp::Pathname &path ); + bool isRunning() const; + bool ejectDevice ( const std::string &queueRef, const std::string &device ); + + void setStatusTracker( ProvideStatusRef tracker ); + + const zypp::Pathname &providerWorkdir () const; + + const zypp::media::CredManagerOptions &credManangerOptions () const; + void setCredManagerOptions( const zypp::media::CredManagerOptions & opt ); + + SignalProxy sigIdle(); + + enum Action { + ABORT, // abort and return error + RETRY, // retry + SKIP // abort and set skip request + }; + using MediaChangeAction = std::optional; + + /*! + * Connect to this signal to handle media change requests + * + * \note It is NOT supported to shutdown the provider or cancel items when in this callback + * Returning Abort here will effectively cancel the current item anyway. + */ + SignalProxy &devices, const std::optional &desc )> sigMediaChangeRequested( ); + + /*! + * This signal is emitted in case a request signaled a need to get Auth Info and nothing was found + * in the \ref zypp::media::CredentialManager. + */ + SignalProxy< std::optional ( const zypp::Url &reqUrl, const std::string &triedUsername, const std::map &extraValues ) > sigAuthRequired(); + + private: + Provide( const zypp::Pathname &workDir ); + void releaseMedia( const std::string &mediaRef ); + }; + +} +#endif diff --git a/zypp-media/ng/providefwd.h b/zypp-media/ng/providefwd.h new file mode 100644 index 0000000..7400fd3 --- /dev/null +++ b/zypp-media/ng/providefwd.h @@ -0,0 +1,32 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\----------------------------------------------------------------------/ +* +* This file contains private API, this might break at any time between releases. +* You have been warned! +* +*/ +#ifndef ZYPP_MEDIA_PROVIDE_FWD_H_INCLUDED +#define ZYPP_MEDIA_PROVIDE_FWD_H_INCLUDED + +#include +#include + +namespace zyppng { + ZYPP_FWD_DECL_TYPE_WITH_REFS (Provide); + ZYPP_FWD_DECL_TYPE_WITH_REFS (ProvideMediaSpec); + ZYPP_FWD_DECL_TYPE_WITH_REFS (ProvideFileSpec); + ZYPP_FWD_DECL_TYPE_WITH_REFS (ProvideItem); + ZYPP_FWD_DECL_TYPE_WITH_REFS (ProvideRequest); + ZYPP_FWD_DECL_TYPE_WITH_REFS (MediaDataVerifier); + ZYPP_FWD_DECL_TYPE_WITH_REFS (ProvideStatus); + class HeaderValueMap; + class ProvideMediaHandle; +} + +#endif diff --git a/zypp-media/ng/provideitem.cc b/zypp-media/ng/provideitem.cc new file mode 100644 index 0000000..522b854 --- /dev/null +++ b/zypp-media/ng/provideitem.cc @@ -0,0 +1,1191 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ + +#include "private/providedbg_p.h" +#include "private/provideitem_p.h" +#include "private/provide_p.h" +#include "private/providemessage_p.h" +#include "private/provideres_p.h" +#include "provide-configvars.h" +#include +#include +#include "mediaverifier.h" +#include + +using namespace std::literals; + +namespace zyppng { + + static constexpr std::string_view DEFAULT_MEDIA_VERIFIER("SuseMediaV1"); + + expected ProvideRequest::create(ProvideItem &owner, const std::vector &urls, const std::string &id, ProvideMediaSpec &spec ) + { + if ( urls.empty() ) + return expected::error( ZYPP_EXCPT_PTR ( zypp::media::MediaException("List of URLs can not be empty") ) ); + + auto m = ProvideMessage::createAttach( ProvideQueue::InvalidId, urls.front(), id, spec.label() ); + if ( !spec.mediaFile().empty() ) { + m.setValue( AttachMsgFields::VerifyType, std::string(DEFAULT_MEDIA_VERIFIER.data()) ); + m.setValue( AttachMsgFields::VerifyData, spec.mediaFile().asString() ); + m.setValue( AttachMsgFields::MediaNr, int32_t(spec.medianr()) ); + } + + const auto &cHeaders = spec.customHeaders(); + for ( auto i = cHeaders.beginList (); i != cHeaders.endList(); i++) { + for ( const auto &val : i->second ) + m.addValue( i->first, val ); + } + + return expected::success( ProvideRequestRef( new ProvideRequest(&owner, urls, std::move(m))) ); + } + + expected ProvideRequest::create( ProvideItem &owner, const std::vector &urls, ProvideFileSpec &spec ) + { + if ( urls.empty() ) + return expected::error( ZYPP_EXCPT_PTR ( zypp::media::MediaException("List of URLs can not be empty") ) ); + + auto m = ProvideMessage::createProvide ( ProvideQueue::InvalidId, urls.front() ); + const auto &destFile = spec.destFilenameHint(); + const auto &deltaFile = spec.deltafile(); + const int64_t fSize = spec.downloadSize();; + + if ( !destFile.empty() ) + m.setValue( ProvideMsgFields::Filename, destFile.asString() ); + if ( !deltaFile.empty() ) + m.setValue( ProvideMsgFields::DeltaFile, deltaFile.asString() ); + if ( fSize ) + m.setValue( ProvideMsgFields::ExpectedFilesize, fSize ); + m.setValue( ProvideMsgFields::CheckExistOnly, spec.checkExistsOnly() ); + + const auto &cHeaders = spec.customHeaders(); + for ( auto i = cHeaders.beginList (); i != cHeaders.endList(); i++) { + for ( const auto &val : i->second ) + m.addValue( i->first, val ); + } + + return expected::success( ProvideRequestRef( new ProvideRequest(&owner, urls, std::move(m)) ) ); + } + + expected ProvideRequest::createDetach( const zypp::Url &url ) + { + auto m = ProvideMessage::createDetach ( ProvideQueue::InvalidId , url ); + return expected::success( ProvideRequestRef( new ProvideRequest( nullptr, { url }, std::move(m) ) ) ); + } + + ZYPP_IMPL_PRIVATE(ProvideItem); + + ProvideItem::ProvideItem( ProvidePrivate &parent ) + : Base( *new ProvideItemPrivate( parent, *this ) ) + { } + + ProvideItem::~ProvideItem() + { } + + ProvidePrivate &ProvideItem::provider() + { + return d_func()->_parent; + } + + bool ProvideItem::safeRedirectTo( ProvideRequestRef startedReq, const zypp::Url &url ) + { + if ( !canRedirectTo( startedReq, url ) ) + return false; + + redirectTo( startedReq, url ); + return true; + } + + void ProvideItem::redirectTo( ProvideRequestRef startedReq, const zypp::Url &url ) + { + //@TODO strip irrelevant stuff from URL + startedReq->_pastRedirects.push_back ( url ); + } + + bool ProvideItem::canRedirectTo( ProvideRequestRef startedReq, const zypp::Url &url ) + { + // make sure there is no redirect loop + if ( !startedReq->_pastRedirects.size() ) + return true; + + if ( std::find( startedReq->_pastRedirects.begin(), startedReq->_pastRedirects.end(), url ) != startedReq->_pastRedirects.end() ) + return false; + + return true; + } + + const std::optional &ProvideItem::currentStats() const + { + return d_func()->_currStats; + } + + const std::optional &ProvideItem::previousStats() const + { + return d_func()->_prevStats; + } + + std::chrono::steady_clock::time_point ProvideItem::startTime() const + { + return d_func()->_itemStarted; + } + + std::chrono::steady_clock::time_point ProvideItem::finishedTime() const { + return d_func()->_itemFinished; + } + + void ProvideItem::pulse () + { + Z_D(); + if ( d->_currStats ) + d->_prevStats = d->_currStats; + + d->_currStats = makeStats(); + + // once the item is finished the pulse time is always the finish time + if ( d->_itemState == Finished ) + d->_currStats->_pulseTime = d->_itemFinished; + } + + zypp::ByteCount ProvideItem::bytesExpected () const + { + return 0; + } + + ProvideItem::ItemStats ProvideItem::makeStats () + { + return ItemStats { + ._pulseTime = std::chrono::steady_clock::now(), + ._runningRequests = _runningReq ? (uint)1 : (uint)0 + }; + } + + void ProvideItem::informalMessage ( ProvideQueue &, ProvideRequestRef req, const ProvideMessage &msg ) + { + if ( req != _runningReq ) { + WAR << "Received event for unknown request, ignoring" << std::endl; + return; + } + + if ( msg.code() == ProvideMessage::Code::ProvideStarted ) { + MIL << "Request: "<< req->url() << " was started" << std::endl; + } + + } + + void ProvideItem::cacheMiss( ProvideRequestRef req ) + { + if ( req != _runningReq ) { + WAR << "Received event for unknown request, ignoring" << std::endl; + return; + } + + MIL << "Request: "<< req->url() << " CACHE MISS, request will be restarted by queue." << std::endl; + } + + void ProvideItem::finishReq(ProvideQueue &, ProvideRequestRef finishedReq, const ProvideMessage &msg ) + { + if ( finishedReq != _runningReq ) { + WAR << "Received event for unknown request, ignoring" << std::endl; + return; + } + + auto log = provider().log(); + + // explicitely handled codes + const auto code = msg.code(); + if ( code == ProvideMessage::Code::Redirect ) { + + // remove the old request + _runningReq.reset(); + + try { + + MIL << "Request finished with redirect." << std::endl; + + zypp::Url newUrl( msg.value( RedirectMsgFields::NewUrl ).asString() ); + if ( !safeRedirectTo( finishedReq, newUrl ) ) { + cancelWithError( ZYPP_EXCPT_PTR ( zypp::media::MediaException("Redirect Loop")) ); + return; + } + + MIL << "Request redirected to: " << newUrl << std::endl; + + if ( log ) log->requestRedirect( *this, msg.requestId(), newUrl ); + + finishedReq->setUrl( newUrl ); + + if ( !enqueueRequest( finishedReq ) ) { + cancelWithError( ZYPP_EXCPT_PTR(zypp::media::MediaException("Failed to queue request")) ); + } + } catch ( ... ) { + cancelWithError( std::current_exception() ); + return; + } + return; + + } else if ( code == ProvideMessage::Code::Metalink ) { + + // remove the old request + _runningReq.reset(); + + MIL << "Request finished with mirrorlist from server." << std::endl; + + //@TODO do we need to merge this with the mirrorlist we got from the user? + // or does a mirrorlist from d.o.o invalidate that? + + std::vector urls; + const auto &mirrors = msg.values( MetalinkRedirectMsgFields::NewUrl ); + for( auto i = mirrors.cbegin(); i != mirrors.cend(); i++ ) { + try { + zypp::Url newUrl( i->asString() ); + if ( !canRedirectTo( finishedReq, newUrl ) ) + continue; + urls.push_back ( newUrl ); + } catch ( ... ) { + if ( i->isString() ) + WAR << "Received invalid URL from worker: " << i->asString() << " ignoring!" << std::endl; + else + WAR << "Received invalid value for newUrl from worker ignoring!" << std::endl; + } + } + + if ( urls.size () == 0 ) { + cancelWithError( ZYPP_EXCPT_PTR ( zypp::media::MediaException("No mirrors left to redirect to.")) ); + return; + } + + MIL << "Found usable nr of mirrors: " << urls.size () << std::endl; + finishedReq->setUrls( urls ); + + // disable metalink + finishedReq->provideMessage().setValue( ProvideMsgFields::MetalinkEnabled, false ); + + if ( log ) log->requestDone( *this, msg.requestId() ); + + if ( !enqueueRequest( finishedReq ) ) { + cancelWithError( ZYPP_EXCPT_PTR(zypp::media::MediaException("Failed to queue request")) ); + } + + MIL << "End of mirrorlist handling"<< std::endl; + return; + + } else if ( code >= ProvideMessage::Code::FirstClientErrCode && code <= ProvideMessage::Code::LastSrvErrCode ) { + + // remove the old request + _runningReq.reset(); + + std::exception_ptr errPtr; + try { + const auto reqUrl = finishedReq->activeUrl().value(); + const auto reason = msg.value( ErrMsgFields::Reason ).asString(); + switch ( code ) { + case ProvideMessage::Code::BadRequest: + errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaException (zypp::str::Str() << "Bad request for URL: " << reqUrl << " " << reason ) ); + break; + case ProvideMessage::Code::PeerCertificateInvalid: + errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaException(zypp::str::Str() << "PeerCertificateInvalid Error for URL: " << reqUrl << " " << reason) ); + break; + case ProvideMessage::Code::ConnectionFailed: + errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaException(zypp::str::Str() << "ConnectionFailed Error for URL: " << reqUrl << " " << reason ) ); + break; + case ProvideMessage::Code::ExpectedSizeExceeded: { + + std::optional filesize; + finishedReq->provideMessage ().forEachVal( [&]( const std::string &key, const auto &val ){ + if ( key == ProvideMsgFields::ExpectedFilesize && val.valid() ) + filesize = val.asInt64(); + return true; + }); + + if ( !filesize ) { + errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaException( zypp::str::Str() << "ExceededExpectedSize Error for URL: " << reqUrl << " " << reason ) ); + } else { + errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaFileSizeExceededException(reqUrl, *filesize ) ); + } + break; + } + case ProvideMessage::Code::Cancelled: + errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaException( zypp::str::Str() << "Request was cancelled: " << reqUrl << " " << reason ) ); + break; + case ProvideMessage::Code::InvalidChecksum: + errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaException( zypp::str::Str() << "InvalidChecksum Error for URL: " << reqUrl << " " << reason ) ); + break; + case ProvideMessage::Code::Timeout: + errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaTimeoutException(reqUrl) ); + break; + case ProvideMessage::Code::NotFound: + errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaFileNotFoundException(reqUrl, "") ); + break; + case ProvideMessage::Code::Forbidden: + case ProvideMessage::Code::Unauthorized: { + + const auto &hintVal = msg.value( "authHint"sv ); + std::string hint; + if ( hintVal.valid() && hintVal.isString() ) { + hint = hintVal.asString(); + } + + //@TODO retry here with timestamp from cred store check + // we let the request fail after it checked the store + + errPtr = ZYPP_EXCPT_PTR ( zypp::media::MediaUnauthorizedException( + reqUrl, reason, "", hint + )); + break; + + } + case ProvideMessage::Code::MountFailed: + errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaException( zypp::str::Str() << "MountFailed Error for URL: " << reqUrl << " " << reason ) ); + break; + case ProvideMessage::Code::Jammed: + errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaJammedException() ); + break; + case ProvideMessage::Code::MediaChangeSkip: + errPtr = ZYPP_EXCPT_PTR( zypp::SkipRequestException ( zypp::str::Str() << "User-requested skipping for URL: " << reqUrl << " " << reason ) ); + break; + case ProvideMessage::Code::MediaChangeAbort: + errPtr = ZYPP_EXCPT_PTR( zypp::AbortRequestException( zypp::str::Str() <<"Aborting requested by user for URL: " << reqUrl << " " << reason ) ); + break; + case ProvideMessage::Code::InternalError: + errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaException( zypp::str::Str() << "WorkerSpecific Error for URL: " << reqUrl << " " << reason ) ); + break; + case ProvideMessage::Code::NotAFile: + errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaNotAFileException(reqUrl, "") ); + break; + case ProvideMessage::Code::MediumNotDesired: + errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaNotDesiredException(reqUrl) ); + break; + default: + errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaException( zypp::str::Str() << "Unknown Error for URL: " << reqUrl << " " << reason ) ); + break; + } + } catch (...) { + errPtr = ZYPP_EXCPT_PTR( zypp::media::MediaException( zypp::str::Str() << "Invalid error message received for URL: " << *finishedReq->activeUrl() << " code: " << code ) ); + } + + if ( log ) log->requestFailed( *this, msg.requestId(), errPtr ); + // finish the request + cancelWithError( errPtr ); + return; + } + + // if we reach here we don't know how to handle the message + _runningReq.reset(); + cancelWithError( ZYPP_EXCPT_PTR (zypp::media::MediaException("Unhandled message received for ProvideFileItem")) ); + } + + void ProvideItem::finishReq(ProvideQueue *, ProvideRequestRef finishedReq , const std::exception_ptr excpt) + { + if ( finishedReq != _runningReq ) { + WAR << "Received event for unknown request, ignoring" << std::endl; + return; + } + + if ( _runningReq ) { + auto log = provider().log(); + if ( log ) log->requestFailed( *this, finishedReq->provideMessage().requestId(), excpt ); + } + + _runningReq.reset(); + cancelWithError(excpt); + } + + expected ProvideItem::authenticationRequired ( ProvideQueue &queue, ProvideRequestRef req, const zypp::Url &effectiveUrl, int64_t lastTimestamp, const std::map &extraFields ) + { + + if ( req != _runningReq ) { + WAR << "Received authenticationRequired for unknown request, rejecting" << std::endl; + return expected::error( ZYPP_EXCPT_PTR( zypp::media::MediaException("Unknown request in authenticationRequired, this is a bug.") ) ); + } + + try { + zypp::media::CredentialManager mgr ( provider().credManagerOptions() ); + + MIL << "Looking for existing auth data for " << effectiveUrl << "more recent then " << lastTimestamp << std::endl; + + auto credPtr = mgr.getCred( effectiveUrl ); + if ( credPtr && credPtr->lastDatabaseUpdate() > lastTimestamp ) { + MIL << "Found existing auth data for " << effectiveUrl << "ts: " << credPtr->lastDatabaseUpdate() << std::endl; + return expected::success( *credPtr ); + } + + if ( credPtr ) MIL << "Found existing auth data for " << effectiveUrl << "but too old ts: " << credPtr->lastDatabaseUpdate() << std::endl; + + std::string username; + if ( auto i = extraFields.find( std::string(AuthDataRequestMsgFields::LastUser) ); i != extraFields.end() ) { + username = i->second; + } + + + MIL << "NO Auth data found, asking user. Last tried username was: " << username << std::endl; + + auto userAuth = provider()._sigAuthRequired.emit( effectiveUrl, username, extraFields ); + if ( !userAuth || !userAuth->valid() ) { + MIL << "User rejected to give auth" << std::endl; + return expected::error( ZYPP_EXCPT_PTR( zypp::media::MediaException("No auth given by user." ) ) ); + } + + mgr.addCred( *userAuth ); + mgr.save(); + + // rather ugly, but update the timestamp to the last mtime of the cred database our URL belongs to + // otherwise we'd need to reload the cred database + userAuth->setLastDatabaseUpdate( mgr.timestampForCredDatabase( effectiveUrl ) ); + + return expected::success(*userAuth); + } catch ( const zypp::Exception &e ) { + ZYPP_CAUGHT(e); + return expected::error( std::current_exception() ); + } + } + + bool ProvideItem::enqueueRequest( ProvideRequestRef request ) + { + // base item just supports one running request at a time + if ( _runningReq ) + return ( _runningReq == request ); + + _runningReq = request; + return d_func()->_parent.queueRequest( request ); + } + + void ProvideItem::updateState( const State newState ) + { + Z_D(); + if ( d->_itemState != newState ) { + + bool started = ( d->_itemState == Uninitialized && ( newState != Finished )); + auto log = provider().log(); + + const auto oldState = d->_itemState; + d->_itemState = newState; + d->_sigStateChanged( *this, oldState, d->_itemState ); + + if ( started ) { + d->_itemStarted = std::chrono::steady_clock::now(); + pulse(); + if ( log ) log->itemStart( *this ); + } + + if ( newState == Finished ) { + d->_itemFinished = std::chrono::steady_clock::now(); + pulse(); + if ( log) log->itemDone( *this ); + d->_parent.dequeueItem(this); + } + // CAREFUL, 'this' might be invalid from here on + } + } + + void ProvideItem::released() + { + if ( state() == Finished || state() == Finalizing ) + return; + + MIL << "Item Cleanup due to released Promise in state:" << state() << std::endl; + cancelWithError( ZYPP_EXCPT_PTR(zypp::media::MediaException("Cancelled by user.")) ); + } + + ProvideItem::State ProvideItem::state () const + { + return d_func()->_itemState; + } + + void ProvideRequest::setCurrentQueue( ProvideQueueRef ref ) + { + _myQueue = ref; + } + + ProvideQueueRef ProvideRequest::currentQueue() + { + return _myQueue.lock(); + } + + const std::optional ProvideRequest::activeUrl() const + { + ProvideMessage::FieldVal url; + switch ( this->_message.code () ) { + case ProvideMessage::Code::Attach: + url = _message.value( AttachMsgFields::Url ); + break; + case ProvideMessage::Code::Detach: + url = _message.value( DetachMsgFields::Url ); + break; + case ProvideMessage::Code::Provide: + url = _message.value( ProvideMsgFields::Url ); + break; + default: + // should never happen because we guard the constructor + throw std::logic_error("Invalid message type in ProvideRequest"); + } + if ( !url.valid() ) { + return {}; + } + + try { + auto u = zypp::Url( url.asString() ); + return u; + } catch ( const zypp::Exception &e ) { + ZYPP_CAUGHT(e); + } + + return {}; + } + + void ProvideRequest::setActiveUrl(const zypp::Url &urlToUse) { + + switch ( this->_message.code () ) { + case ProvideMessage::Code::Attach: + this->_message.setValue( AttachMsgFields::Url, urlToUse.asCompleteString() ); + break; + case ProvideMessage::Code::Detach: + this->_message.setValue( DetachMsgFields::Url, urlToUse.asCompleteString() ); + break; + case ProvideMessage::Code::Provide: + this->_message.setValue( ProvideMsgFields::Url, urlToUse.asCompleteString() ); + break; + default: + // should never happen because we guard the constructor + throw std::logic_error("Invalid message type in ProvideRequest"); + } + } + + ProvideFileItem::ProvideFileItem(const std::vector &urls, const ProvideFileSpec &request, ProvidePrivate &parent) + : ProvideItem( parent ) + , _mirrorList ( urls ) + , _initialSpec ( request ) + { } + + ProvideFileItemRef ProvideFileItem::create(const std::vector &urls, const ProvideFileSpec &request, ProvidePrivate &parent ) + { + return ProvideFileItemRef( new ProvideFileItem( urls, request, parent ) ); + } + + void ProvideFileItem::initialize() + { + if ( state() != Uninitialized || _runningReq ) { + WAR << "Double init of ProvideFileItem!" << std::endl; + return; + } + + auto req = ProvideRequest::create( *this, _mirrorList, _initialSpec ); + if ( !req ){ + cancelWithError( req.error() ); + return ; + } + + if ( enqueueRequest( *req ) ) { + _expectedBytes = _initialSpec.downloadSize(); + updateState( Pending ); + } else { + cancelWithError( ZYPP_EXCPT_PTR(zypp::media::MediaException("Failed to queue request")) ); + return ; + } + } + + ProvidePromiseRef ProvideFileItem::promise() + { + if ( !_promiseCreated ) { + _promiseCreated = true; + auto promiseRef = std::make_shared>( shared_this() ); + _promise = promiseRef; + return promiseRef; + } + return _promise.lock(); + } + + void ProvideFileItem::setMediaRef ( Provide::MediaHandle &&hdl ) + { + _handleRef = std::move(hdl); + } + + Provide::MediaHandle &ProvideFileItem::mediaRef () + { + return _handleRef; + } + + void ProvideFileItem::informalMessage ( ProvideQueue &, ProvideRequestRef req, const ProvideMessage &msg ) + { + if ( req != _runningReq ) { + WAR << "Received event for unknown request, ignoring" << std::endl; + return; + } + + if ( msg.code() == ProvideMessage::Code::ProvideStarted ) { + MIL << "Provide File Request: "<< req->url() << " was started" << std::endl; + auto log = provider().log(); + + auto locPath = msg.value( ProvideStartedMsgFields::LocalFilename, std::string() ).asString(); + if ( !locPath.empty() ) + _targetFile = zypp::Pathname(locPath); + + locPath = msg.value( ProvideStartedMsgFields::StagingFilename, std::string() ).asString(); + if ( !locPath.empty() ) + _stagingFile = zypp::Pathname(locPath); + + if ( log ) { + auto effUrl = req->activeUrl().value_or( zypp::Url() ); + try { + effUrl = zypp::Url( msg.value( ProvideStartedMsgFields::Url).asString() ); + } catch( const zypp::Exception &e ) { + ZYPP_CAUGHT(e); + } + + AnyMap m; + m["spec"] = _initialSpec; + if ( log ) log->requestStart( *this, msg.requestId(), effUrl, m ); + updateState( Downloading ); + } + } + } + + void zyppng::ProvideFileItem::ProvideFileItem::finishReq( zyppng::ProvideQueue &queue, ProvideRequestRef finishedReq, const ProvideMessage &msg ) + { + if ( finishedReq != _runningReq ) { + WAR << "Received event for unknown request, ignoring" << std::endl; + return; + } + + if ( msg.code () == ProvideMessage::Code::ProvideFinished ) { + + auto log = provider().log(); + if ( log ) { + AnyMap m; + m["spec"] = _initialSpec; + if ( log ) log->requestDone( *this, msg.requestId(), m ); + } + + MIL << "Request was successfully finished!" << std::endl; + // request is def done + _runningReq.reset(); + + try { + + const auto locFilename = msg.value( ProvideFinishedMsgFields::LocalFilename ).asString(); + const auto cacheHit = msg.value( ProvideFinishedMsgFields::CacheHit ).asBool(); + const auto &wConf = queue.workerConfig(); + + const bool doesDownload = wConf.worker_type() == ProvideQueue::Config::Downloading; + const bool fileNeedsCleanup = doesDownload || ( wConf.worker_type() == ProvideQueue::Config::CPUBound && wConf.cfg_flags() & ProvideQueue::Config::FileArtifacts ); + + std::optional resFile; + + if ( doesDownload ) { + + resFile = provider().addToFileCache ( locFilename ); + if ( !resFile ) { + if ( cacheHit ) { + MIL << "CACHE MISS, file " << locFilename << " was already removed, queueing again" << std::endl; + cacheMiss ( finishedReq ); + finishedReq->clearForRestart(); + enqueueRequest( finishedReq ); + return; + } else { + // if we reach here it seems that a new file, that was not in cache before, vanished between + // providing it and receiving the finished message. + // unlikely this can happen but better be safe than sorry + cancelWithError( ZYPP_EXCPT_PTR( zypp::media::MediaException("File vanished between downloading and adding it to cache.")) ); + return; + } + } + + } else { + resFile = zypp::ManagedFile( zypp::filesystem::Pathname(locFilename) ); + if ( fileNeedsCleanup ) + resFile->setDispose( zypp::filesystem::unlink ); + else + resFile->resetDispose(); + } + + _targetFile = locFilename; + + // keep the media handle around as long as the file is used by the code + auto resObj = std::make_shared(); + resObj->_mediaHandle = this->_handleRef; + resObj->_myFile = *resFile; + resObj->_resourceUrl = *(finishedReq->activeUrl()); + resObj->_responseHeaders = msg.headers(); + + auto p = promise(); + if ( p ) { + try { + p->setReady( expected::success( ProvideRes( resObj )) ); + } catch( const zypp::Exception &e ) { + ZYPP_CAUGHT(e); + } + } + + updateState( Finished ); + + } catch ( const zypp::Exception &e ) { + ZYPP_CAUGHT(e); + cancelWithError( std::current_exception() ); + } catch ( ...) { + cancelWithError( std::current_exception() ); + } + + } else { + ProvideItem::finishReq ( queue, finishedReq, msg ); + } + } + + + void zyppng::ProvideFileItem::cancelWithError( std::exception_ptr error ) + { + if ( _runningReq ) { + auto weakThis = weak_from_this (); + provider().dequeueRequest ( _runningReq, error ); + if ( weakThis.expired () ) + return; + } + + // if we reach this place for some reason finishReq was not called, lets clean up manually + _runningReq.reset(); + auto p = promise(); + if ( p ) { + try { + p->setReady( expected::error( error ) ); + } catch( const zypp::Exception &e ) { + ZYPP_CAUGHT(e); + } + } + updateState( Finished ); + } + + expected ProvideFileItem::authenticationRequired ( ProvideQueue &queue, ProvideRequestRef req, const zypp::Url &effectiveUrl, int64_t lastTimestamp, const std::map &extraFields ) + { + zypp::Url urlToUse = effectiveUrl; + if ( _handleRef.isValid() ) { + // if we have a attached medium this overrules the URL we are going to ask the user about... this is how the old media backend did handle this + // i guess there were never password protected repositories that have different credentials on the redirection targets + auto &attachedMedia = provider().attachedMediaInfos(); + auto i = std::find_if( attachedMedia.begin(), attachedMedia.end(), [&]( const auto &m ) { return m._name == _handleRef.handle(); } ); + if ( i == attachedMedia.end() ) + return expected::error( ZYPP_EXCPT_PTR( zypp::media::MediaException("Attachment handle vanished during request.") ) ); + + urlToUse = i->_attachedUrl; + } + return ProvideItem::authenticationRequired( queue, req, urlToUse, lastTimestamp, extraFields ); + } + + ProvideFileItem::ItemStats ProvideFileItem::makeStats () + { + zypp::ByteCount providedByNow; + + bool checkStaging = false; + if ( !_targetFile.empty() ) { + zypp::PathInfo inf( _targetFile ); + if ( inf.isExist() && inf.isFile() ) + providedByNow = zypp::ByteCount( inf.size() ); + else + checkStaging = true; + } + + if ( checkStaging && !_stagingFile.empty() ) { + zypp::PathInfo inf( _stagingFile ); + if ( inf.isExist() && inf.isFile() ) + providedByNow = zypp::ByteCount( inf.size() ); + } + + auto baseStats = ProvideItem::makeStats(); + baseStats._bytesExpected = bytesExpected(); + baseStats._bytesProvided = providedByNow; + return baseStats; + } + + zypp::ByteCount ProvideFileItem::bytesExpected () const + { + return (_initialSpec.checkExistsOnly() ? zypp::ByteCount(0) : _expectedBytes); + } + + AttachMediaItem::AttachMediaItem( const std::vector &urls, const ProvideMediaSpec &request, ProvidePrivate &parent ) + : ProvideItem ( parent ) + , _mirrorList ( urls ) + , _initialSpec ( request ) + { } + + AttachMediaItem::~AttachMediaItem() + { + MIL << "Killing the AttachMediaItem" << std::endl; + } + + ProvidePromiseRef AttachMediaItem::promise() + { + if ( !_promiseCreated ) { + _promiseCreated = true; + auto promiseRef = std::make_shared>( shared_this() ); + _promise = promiseRef; + return promiseRef; + } + return _promise.lock(); + } + + void AttachMediaItem::initialize() + { + if ( state() != Uninitialized ) { + WAR << "Double init of AttachMediaItem!" << std::endl; + return; + } + updateState(Processing); + + if ( _mirrorList.empty() ) { + cancelWithError( ZYPP_EXCPT_PTR( zypp::media::MediaException("No usable mirrors in mirrorlist.")) ); + return; + } + + // shortcut to the provider instance + auto &prov= provider(); + + // sanitize the mirrors to contain only URLs that have same worker types + std::vector usableMirrs; + std::optional scheme; + + for ( auto mirrIt = _mirrorList.begin() ; mirrIt != _mirrorList.end(); mirrIt++ ) { + const auto &s = prov.schemeConfig( prov.effectiveScheme( mirrIt->getScheme() ) ); + if ( !s ) { + WAR << "URL: " << *mirrIt << " is not supported, ignoring!" << std::endl; + continue; + } + if ( !scheme ) { + scheme = *s; + usableMirrs.push_back ( *mirrIt ); + } else { + if ( scheme->worker_type () == s->worker_type () ) { + usableMirrs.push_back( *mirrIt ); + } else { + WAR << "URL: " << *mirrIt << " has different worker type than the primary URL: "<< usableMirrs.front() <<", ignoring!" << std::endl; + } + } + } + + // save the sanitized mirrors + _mirrorList = usableMirrs; + + if ( !scheme || _mirrorList.empty() ) { + auto prom = promise(); + if ( prom ) { + try { + prom->setReady( expected::error( ZYPP_EXCPT_PTR ( zypp::media::MediaException("No valid mirrors available") )) ); + } catch( const zypp::Exception &e ) { + ZYPP_CAUGHT(e); + } + } + updateState( Finished ); + return; + } + + // first check if there is a already attached medium we can use as well + auto &attachedMedia = prov.attachedMediaInfos (); + + for ( auto &medium : attachedMedia ) { + if ( medium.isSameMedium ( _mirrorList, _initialSpec ) ) { + finishWithSuccess ( medium ); + return; + } + } + + for ( auto &otherItem : prov.items() ) { + auto attachIt = std::dynamic_pointer_cast(otherItem); + if ( !attachIt // not the right type + || attachIt.get() == this // do not attach to ourselves + || attachIt->state () == Uninitialized // item was not initialized + || attachIt->state () == Finalizing // item cleaning up + || attachIt->state () == Finished ) // item done + continue; + + // does this Item attach the same medium? + const auto sameMedium = attachIt->_initialSpec.isSameMedium( _initialSpec); + if ( zypp::indeterminate(sameMedium) ) { + // check the primary URLs ( should we do a full list compare? ) + if ( attachIt->_mirrorList.front() != _mirrorList.front() ) + continue; + } + else if ( !(bool)sameMedium ) + continue; + + MIL << "Found item providing the same medium, attaching to finished signal and waiting for it to be finished" << std::endl; + + // it does, connect to its ready signal and just wait + _masterItemConn = connect( *attachIt, &AttachMediaItem::sigReady, *this, &AttachMediaItem::onMasterItemReady ); + return; + } + + _workerType = scheme->worker_type(); + + switch( _workerType ) { + case ProvideQueue::Config::Downloading: { + + // if the media file is empty in the spec we can not do anything + // simply pretend attach worked + if( _initialSpec.mediaFile().empty() ) { + finishWithSuccess( prov.addMedium( _workerType, _mirrorList.front(), _initialSpec ) ); + return; + } + + // prepare the verifier with the data + auto smvDataLocal = MediaDataVerifier::createVerifier("SuseMediaV1"); + if ( !smvDataLocal ) { + cancelWithError( ZYPP_EXCPT_PTR( zypp::media::MediaException("Unable to verify the medium, no verifier instance was returned.")) ); + return; + } + + if ( !smvDataLocal->load( _initialSpec.mediaFile() ) ) { + cancelWithError( ZYPP_EXCPT_PTR( zypp::media::MediaException("Unable to verify the medium, unable to load local verify data.")) ); + return; + } + + _verifier = smvDataLocal; + + std::vector urls; + urls.reserve( _mirrorList.size () ); + + for ( zypp::Url url : _mirrorList ) { + url.appendPathName ( ( zypp::str::Format("/media.%d/media") % _initialSpec.medianr() ).asString() ); + urls.push_back(url); + } + + // for downloading schemes we ask for the /media.x/media file and check the data manually + ProvideFileSpec spec; + spec.customHeaders() = _initialSpec.customHeaders(); + + // disable metalink + spec.customHeaders().set( std::string(NETWORK_METALINK_ENABLED), false ); + + auto req = ProvideRequest::create( *this, urls, spec ); + if ( !req ) { + cancelWithError( req.error() ); + return; + } + if ( !enqueueRequest( *req ) ) { + cancelWithError( ZYPP_EXCPT_PTR(zypp::media::MediaException("Failed to queue request")) ); + return; + } + updateState ( Downloading ); + break; + } + case ProvideQueue::Config::VolatileMount: + case ProvideQueue::Config::SimpleMount: { + + const auto &newId = provider().nextMediaId(); + auto req = ProvideRequest::create( *this, _mirrorList, newId, _initialSpec ); + if ( !req ) { + cancelWithError( req.error() ); + return; + } + if ( !enqueueRequest( *req ) ) { + ERR << "Failed to queue request" << std::endl; + cancelWithError( ZYPP_EXCPT_PTR(zypp::media::MediaException("Failed to queue request")) ); + return; + } + break; + } + default: { + auto prom = promise(); + if ( prom ) { + try { + prom->setReady( expected::error( ZYPP_EXCPT_PTR ( zypp::media::MediaException("URL scheme does not support attaching.") )) ); + } catch( const zypp::Exception &e ) { + ZYPP_CAUGHT(e); + } + } + updateState( Finished ); + return; + } + } + } + + void AttachMediaItem::finishWithSuccess( AttachedMediaInfo &medium ) + { + + updateState(Finalizing); + + // aquire a ref to keep the medium around until we notified all dependant attach operations + // currently not really required because only the next schedule run will clean up attached medias + // but in case that ever changes the code is safe already + medium.ref(); + zypp::OnScopeExit autoUnref([&]{ + medium.unref(); + }); + + auto prom = promise(); + try { + if ( prom ) { + // the ref for the result we are giving out + medium.ref(); + try { + prom->setReady( expected::success( Provide::MediaHandle( *static_cast( provider().z_func() ), medium._name) ) ); + } catch( const zypp::Exception &e ) { + ZYPP_CAUGHT(e); + } + } + } catch ( const std::exception &e ) { + ERR << "WTF " << e.what () << std::endl; + } catch ( ... ) { + ERR << "WTF " << std::endl; + } + + // tell others as well + _sigReady.emit( zyppng::expected::success(&medium) ); + + prom->isReady (); + + MIL << "Before setFinished" << std::endl; + updateState( Finished ); + return; + } + + void AttachMediaItem::cancelWithError( std::exception_ptr error ) + { + MIL << "Cancelling Item with error" << std::endl; + updateState(Finalizing); + + // tell children + _sigReady.emit( expected::error(error) ); + + if ( _runningReq ) { + // we might get deleted when calling dequeueRequest + auto weakThis = weak_from_this (); + provider().dequeueRequest ( _runningReq, error ); + if ( weakThis.expired () ) + return; + } + + // if we reach this place we had no runningReq, clean up manually + _runningReq.reset(); + _masterItemConn.disconnect(); + + auto p = promise(); + if ( p ) { + try { + p->setReady( expected::error( error ) ); + } catch( const zypp::Exception &e ) { + ZYPP_CAUGHT(e); + } + } + updateState( Finished ); + } + + void AttachMediaItem::onMasterItemReady( const zyppng::expected &result ) + { + + _masterItemConn.disconnect(); + + if ( result ) { + AttachedMediaInfo &medium = *result.get(); + finishWithSuccess(medium); + } else { + try { + std::rethrow_exception ( result.error() ); + } catch ( const zypp::media::MediaRequestCancelledException & e) { + // in case a item was cancelled, we revert to Pending state and trigger the scheduler. + // This will make sure that all our sibilings that also depend on the master + // can revert to pending state and we only get one new master in the next schedule run + MIL_PRV << "Master item was cancelled, reverting to Uninitialized state and waiting for scheduler to run again" << std::endl; + updateState (Uninitialized); + provider().schedule( ProvidePrivate::RestartAttach ); + + } catch ( ... ) { + cancelWithError( std::current_exception() ); + } + } + } + + AttachMediaItemRef AttachMediaItem::create( const std::vector &urls, const ProvideMediaSpec &request, ProvidePrivate &parent ) + { + return AttachMediaItemRef( new AttachMediaItem(urls, request, parent) ); + } + + SignalProxy &)> AttachMediaItem::sigReady() + { + return _sigReady; + } + + void AttachMediaItem::finishReq ( ProvideQueue &queue, ProvideRequestRef finishedReq, const ProvideMessage &msg ) + { + if ( finishedReq != _runningReq ) { + WAR << "Received event for unknown request, ignoring" << std::endl; + return; + } + + if( _workerType == ProvideQueue::Config::Downloading ) { + // success + if ( msg.code() == ProvideMessage::Code::ProvideFinished ) { + + updateState(Finalizing); + + zypp::Url baseUrl = *finishedReq->activeUrl(); + // remove /media.n/media + baseUrl.setPathName( zypp::Pathname(baseUrl.getPathName()).dirname().dirname() ); + + // we got the file, lets parse it + auto smvDataRemote = MediaDataVerifier::createVerifier("SuseMediaV1"); + if ( !smvDataRemote ) { + return cancelWithError( ZYPP_EXCPT_PTR( zypp::media::MediaException("Unable to verify the medium, no verifier instance was returned.")) ); + } + + if ( !smvDataRemote->load( msg.value( ProvideFinishedMsgFields::LocalFilename ).asString() ) ) { + return cancelWithError( ZYPP_EXCPT_PTR( zypp::media::MediaException("Unable to verify the medium, unable to load remote verify data.")) ); + } + + // check if we got a valid media file + if ( !smvDataRemote->valid () ) { + return cancelWithError( ZYPP_EXCPT_PTR( zypp::media::MediaException("Unable to verify the medium, unable to load local verify data.")) ); + } + + // check if the received file matches with the one we have in the spec + if (! _verifier->matches( smvDataRemote ) ) { + DBG << "expect: " << _verifier << " medium " << _initialSpec.medianr() << std::endl; + DBG << "remote: " << smvDataRemote << std::endl; + return cancelWithError( ZYPP_EXCPT_PTR( zypp::media::MediaNotDesiredException( *finishedReq->activeUrl() ) ) ); + } + + // all good, register the medium and tell all child items + _runningReq.reset(); + return finishWithSuccess( provider().addMedium( _workerType, baseUrl, _initialSpec ) ); + + } else if ( msg.code() == ProvideMessage::Code::NotFound ) { + + // simple downloading attachment we need to check the media file contents + // in case of a error we might tolerate a file not found error in certain situations + if ( _verifier->totalMedia () == 1 ) { + // relaxed , tolerate a vanished media file + _runningReq.reset(); + return finishWithSuccess( provider().addMedium( _workerType, _mirrorList.front(), _initialSpec) ); + } else { + return ProvideItem::finishReq ( queue, finishedReq, msg ); + } + } else { + return ProvideItem::finishReq ( queue, finishedReq, msg ); + } + } else { + // real device attach + if ( msg.code() == ProvideMessage::Code::AttachFinished ) { + _runningReq.reset(); + return finishWithSuccess( provider().addMedium( _workerType + , queue.weak_this() + , finishedReq->provideMessage().value( AttachMsgFields::AttachId ).asString() + , *finishedReq->activeUrl() + , _initialSpec ) ); + } + } + + // unhandled message , let the base impl do it + return ProvideItem::finishReq ( queue, finishedReq, msg ); + } + + expected AttachMediaItem::authenticationRequired ( ProvideQueue &queue, ProvideRequestRef req, const zypp::Url &effectiveUrl, int64_t lastTimestamp, const std::map &extraFields ) + { + zypp::Url baseUrl = effectiveUrl; + if( _workerType == ProvideQueue::Config::Downloading ) { + // remove /media.n/media + baseUrl.setPathName( zypp::Pathname(baseUrl.getPathName()).dirname().dirname() ); + } + return ProvideItem::authenticationRequired( queue, req, baseUrl, lastTimestamp, extraFields ); + } + +} diff --git a/zypp-media/ng/provideitem.h b/zypp-media/ng/provideitem.h new file mode 100644 index 0000000..b3eddbc --- /dev/null +++ b/zypp-media/ng/provideitem.h @@ -0,0 +1,192 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ +#ifndef ZYPP_MEDIA_PROVIDEITEM_H_INCLUDED +#define ZYPP_MEDIA_PROVIDEITEM_H_INCLUDED + +#include +#include +#include +#include + +namespace zyppng +{ + class ProvidePrivate; + class ProvideItemPrivate; + + /*! + * Represents a operation added to the provide queue by the user code. A "user" operation can have multiple + * steps of downloading and processing one or multiple files. Even though this class is in public API space it's + * not possible to implement custom Items since more internal API would be required for it. It is only public to + * support the \ref ProvideStatus class, so it can query details of a Item. + */ + class ProvideItem : public Base + { + ZYPP_DECLARE_PRIVATE(ProvideItem); + friend class Provide; + friend class ProvidePrivate; + friend class ProvideQueue; + public: + enum State { + Uninitialized, //< Item was added to the queue but not yet initialized + Pending, //< The Item is waiting to be queued and does nothing + Downloading, //< The Item is fetching data + Processing, //< The Item has fetched all data but is still doing other things + Cancelling, //< The Item was cancelled and waits + Finalizing, //< The Item has finished all work and is currently cleaning up + Finished //< The Item is done and can be dequeued + }; + + struct ItemStats { + std::chrono::steady_clock::time_point _pulseTime; + uint _runningRequests; + zypp::ByteCount _bytesProvided; + zypp::ByteCount _bytesExpected; + }; + + ProvideItem( ProvidePrivate &parent ); + ~ProvideItem (); + + /*! + * Called by the controller when the item is supposed to start fetching / processing + */ + virtual void initialize () = 0; + + /*! + * Called when the promise reference is released by the user process, cancel all running requests if there are any and clean up + */ + virtual void released (); + + State state () const; + + /*! + * Signal that is emitted when the state of the Item has changed + */ + SignalProxy sigStateChanged(); + + ProvidePrivate &provider(); + + /*! + * Returns true if a redirect is allowed and does not conflict with previous redirects. + * Otherwise false is returned. This does not remember the passed URL as a redirect + */ + virtual bool canRedirectTo ( ProvideRequestRef startedReq, const zypp::Url &url ); + + /*! + * Returns the item statistics that were collected the last time pulse() was called on the item. + * If the item has not been started yet, this returns a empty optional + */ + const std::optional ¤tStats() const; + + /*! + * Returns the item statistics that were collected the previous time pulse() was called on the item. + * \note The item needs to be started and pulse() needs to be called at least once for this func to return something + */ + const std::optional &previousStats() const; + + /*! + * Returns the time point when the item started to process/download. + * If the item has not started yet, this returns epoch + */ + virtual std::chrono::steady_clock::time_point startTime() const; + + /*! + * Returns the time point when the item was finished. + * If the item was not finished yet, this returns epoch + */ + virtual std::chrono::steady_clock::time_point finishedTime() const; + + /*! + * Updates the item statistics, this is called automatically by the \ref Provide instance and usually does not + * need to be called explicitely by usercode. + */ + void pulse (); + + /*! + * Returns the bytes the item expects to provide, the default impl returns 0 + */ + virtual zypp::ByteCount bytesExpected () const; + + protected: + virtual ItemStats makeStats (); + + /*! + * Request received a informal message, e.g. ProvideStarted + */ + virtual void informalMessage ( ProvideQueue &, ProvideRequestRef req, const ProvideMessage &msg ); + + /*! + * Request had a cache miss and will be queued again, forget all about the request + */ + virtual void cacheMiss ( ProvideRequestRef req ); + + /*! + * Request was finished by the queue + * Base implementation handles redirect, metalink and error messages. If a different message is + * received, \ref cancelWithError is called. + * + * A subclass has to overload this function to handle success messages + */ + virtual void finishReq ( ProvideQueue &queue, ProvideRequestRef finishedReq, const ProvideMessage &msg ); + + /*! + * Request was finished with a error + * The base implementation simply calls \ref cancelWithError + * + * \note \a queue is allowed to be a nullptr here + */ + virtual void finishReq ( ProvideQueue *queue, ProvideRequestRef finishedReq, const std::exception_ptr excpt ); + + /*! + * Request needs authentication data, the function is supposed to return the AuthData to use for the response, or an error + * The default implementation simply uses the given URL to look for a Auth match in the \ref zypp::media::CredentialManager. + */ + virtual expected authenticationRequired ( ProvideQueue &queue, ProvideRequestRef req, const zypp::Url &effectiveUrl, int64_t lastTimestamp, const std::map &extraFields ); + + /*! + * Remembers previous redirects and returns false if the URL was encountered before, use this + * to prevent the item getting caught in a redirect loop + */ + bool safeRedirectTo ( ProvideRequestRef startedReq, const zypp::Url &url ); + + /*! + * Similar to \ref safeRedirectTo, but does not check if a URL was already used by this Request before. + */ + void redirectTo ( ProvideRequestRef startedReq, const zypp::Url &url ); + + /*! + * Enqueue the request in the correct queue, the item implementation is supposed to hold its own + * reference to all started requests, the base implementation just keeps track of 1 request at a time. + */ + virtual bool enqueueRequest( ProvideRequestRef request ); + + /*! + * Cancels all running requests and immediately moves to error state + */ + virtual void cancelWithError ( std::exception_ptr error ) = 0; + + /*! + * Dequeue this item and stop all requests in queues running + * Call this when the item is cancelled or finished. + */ + bool dequeue (); + + /*! + * Call this when the state of the item changes. + * + * \note calling updateState with state \ref Finished will potentially delete the Item instance + */ + void updateState( const State newState ); + + void setFinished (); + + protected: + ProvideRequestRef _runningReq; + }; +} +#endif // ZYPP_MEDIA_PROVIDEITEM_H_INCLUDED diff --git a/zypp-media/ng/providemessage.cc b/zypp-media/ng/providemessage.cc new file mode 100644 index 0000000..a300dea --- /dev/null +++ b/zypp-media/ng/providemessage.cc @@ -0,0 +1,656 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ + +#include "private/providemessage_p.h" + +#include +#include +#include + +namespace zyppng { + + static ProvideMessage::FieldVal fieldValFromProto ( const zypp::proto::DataField &field ) + { + ProvideMessage::FieldVal v; + switch ( field.field_val_case () ) { + case zypp::proto::DataField::FieldValCase::kBoolVal: + v = field.bool_val(); + break; + case zypp::proto::DataField::FieldValCase::kDoubleVal: + v = field.double_val(); + break; + case zypp::proto::DataField::FieldValCase::kIntVal: + v = field.int_val(); + break; + case zypp::proto::DataField::FieldValCase::kLongVal: + v = field.long_val(); + break; + case zypp::proto::DataField::FieldValCase::kStrVal: + v = field.str_val(); + break; + case zypp::proto::DataField::FieldValCase::FIELD_VAL_NOT_SET: + ZYPP_THROW( std::logic_error("Unexpected DataField type")); + break; + } + return v; + } + + static void fieldValToProto ( const ProvideMessage::FieldVal &val, zypp::proto::DataField &field ) + { + if ( val.isString() ) + field.set_str_val( val.asString () ); + else if ( val.isInt() ) + field.set_int_val( val.asInt() ); + else if ( val.isInt64() ) + field.set_long_val( val.asInt64() ); + else if ( val.isDouble() ) + field.set_double_val( val.asDouble() ); + else if ( val.isBool() ) + field.set_bool_val( val.asBool() ); + else + ZYPP_THROW( std::logic_error("Unexpected FieldVal type")); + } + + static expected validateMessage ( const ProvideMessage &msg ) + { + const auto c = msg.code(); + const auto validCode = ( c >= ProvideMessage::Code::FirstInformalCode && c <= ProvideMessage::Code::LastInformalCode ) + || ( c >= ProvideMessage::Code::FirstSuccessCode && c <= ProvideMessage::Code::LastSuccessCode ) + || ( c >= ProvideMessage::Code::FirstRedirCode && c <= ProvideMessage::Code::LastRedirCode ) + || ( c >= ProvideMessage::Code::FirstClientErrCode && c <= ProvideMessage::Code::LastClientErrCode ) + || ( c >= ProvideMessage::Code::FirstSrvErrCode && c <= ProvideMessage::Code::LastSrvErrCode ) + || ( c >= ProvideMessage::Code::FirstControllerCode && c <= ProvideMessage::Code::LastControllerCode) + || ( c >= ProvideMessage::Code::FirstWorkerCode && c <= ProvideMessage::Code::LastWorkerCode ); + if ( !validCode ) { + return zyppng::expected::error( ZYPP_EXCPT_PTR ( InvalidMessageReceivedException("Invalid code in ProvideMessage")) ); + } + + #define DEF_REQ_FIELD( fname ) bool has_##fname = false + + #define REQ_FIELD_CHECK( msgtype, fname, ftype ) \ + if ( name == #fname ) { \ + if ( !std::holds_alternative(val.asVariant()) ) { \ + error = ZYPP_EXCPT_PTR( InvalidMessageReceivedException( zypp::str::Str() << "Parse error " << #msgtype << ", Field " << #fname << " has invalid type" ) ); \ + return false; \ + } \ + has_##fname = true; \ + } + + #define OR_REQ_FIELD_CHECK( msgtype, fname, ftype ) else REQ_FIELD_CHECK( msgtype, fname, ftype ) + + #define OPT_FIELD_CHECK( msgtype, fname, ftype ) \ + if ( name == #fname ) { \ + if ( !std::holds_alternative(val.asVariant() ) ) { \ + error = ZYPP_EXCPT_PTR( InvalidMessageReceivedException( zypp::str::Str() << "Parse error " << #msgtype << ", Field " << #fname << " has invalid type" ) ); \ + return false; \ + } \ + } + + #define OR_OPT_FIELD_CHECK( msgtype, fname, ftype ) else OPT_FIELD_CHECK( msgtype, fname, ftype ) + + #define FAIL_IF_NOT_SEEN_REQ_FIELD( msgtype, fname ) \ + if ( !has_##fname ) \ + return expected::error( ZYPP_EXCPT_PTR( InvalidMessageReceivedException( zypp::str::Str() << #msgtype <<" message does not contain required " << #fname << " field" ) ) ) + + #define FAIL_IF_ERROR( ) \ + if ( error ) return expected::error( error ) + + const auto &validateErrorMsg = []( const auto &msg ){ + std::exception_ptr error; + DEF_REQ_FIELD(reason); + msg.forEachVal( [&]( const auto &name, const ProvideMessage::FieldVal &val ){ + REQ_FIELD_CHECK ( Error, reason, std::string ) + OR_OPT_FIELD_CHECK ( Error, history, std::string ) + OR_OPT_FIELD_CHECK ( Error, transient, bool ) + return true; + }); + FAIL_IF_NOT_SEEN_REQ_FIELD( Error, reason ); + FAIL_IF_ERROR(); + return expected::success(); + }; + + switch ( c ) + { + case ProvideMessage::Code::ProvideStarted: { + std::exception_ptr error; + DEF_REQ_FIELD(url); + msg.forEachVal( [&]( const auto &name, const ProvideMessage::FieldVal &val ){ + REQ_FIELD_CHECK ( ProvideStarted, url, std::string ) + OR_OPT_FIELD_CHECK ( ProvideStarted, local_filename, std::string ) + OR_OPT_FIELD_CHECK ( ProvideStarted, staging_filename, std::string ) + return true; + }); + FAIL_IF_NOT_SEEN_REQ_FIELD( ProvideStarted, url ); + FAIL_IF_ERROR(); + break; + } + case ProvideMessage::Code::ProvideFinished: { + std::exception_ptr error; + DEF_REQ_FIELD(cacheHit); + DEF_REQ_FIELD(local_filename); + msg.forEachVal( [&]( const auto &name, const ProvideMessage::FieldVal &val ){ + REQ_FIELD_CHECK ( ProvideFinished, cacheHit, bool ) + OR_REQ_FIELD_CHECK ( ProvideFinished, local_filename, std::string ) + return true; + }); + FAIL_IF_NOT_SEEN_REQ_FIELD( ProvideFinished, cacheHit ); + FAIL_IF_NOT_SEEN_REQ_FIELD( ProvideFinished, local_filename ); + FAIL_IF_ERROR(); + break; + } + case ProvideMessage::Code::AttachFinished: { + // no fields + break; + } + case ProvideMessage::Code::DetachFinished: { + // no fields + break; + } + case ProvideMessage::Code::AuthInfo: { + std::exception_ptr error; + DEF_REQ_FIELD(username); + DEF_REQ_FIELD(password); + DEF_REQ_FIELD(auth_timestamp); + msg.forEachVal( [&]( const auto &name, const ProvideMessage::FieldVal &val ){ + REQ_FIELD_CHECK ( AuthInfo, username, std::string ) + OR_REQ_FIELD_CHECK ( AuthInfo, password, std::string ) + OR_REQ_FIELD_CHECK ( AuthInfo, auth_timestamp, int64_t ) + OR_OPT_FIELD_CHECK ( AuthInfo, authType, std::string ) + return true; + }); + FAIL_IF_NOT_SEEN_REQ_FIELD( ProvideStarted, username ); + FAIL_IF_NOT_SEEN_REQ_FIELD( ProvideStarted, password ); + FAIL_IF_NOT_SEEN_REQ_FIELD( ProvideStarted, auth_timestamp ); + FAIL_IF_ERROR(); + break; + } + case ProvideMessage::Code::MediaChanged: + /* No Fields */ + break; + case ProvideMessage::Code::Redirect: { + std::exception_ptr error; + DEF_REQ_FIELD(new_url); + msg.forEachVal( [&]( const auto &name, const ProvideMessage::FieldVal &val ){ + REQ_FIELD_CHECK ( Redirect, new_url, std::string ) + return true; + }); + FAIL_IF_NOT_SEEN_REQ_FIELD( Redirect, new_url ); + FAIL_IF_ERROR(); + break; + } + case ProvideMessage::Code::Metalink: { + std::exception_ptr error; + DEF_REQ_FIELD(new_url); + msg.forEachVal( [&]( const auto &name, const ProvideMessage::FieldVal &val ){ + REQ_FIELD_CHECK ( Metalink, new_url, std::string ) + return true; + }); + FAIL_IF_NOT_SEEN_REQ_FIELD( Metalink, new_url ); + FAIL_IF_ERROR(); + break; + } + case ProvideMessage::Code::BadRequest: + case ProvideMessage::Code::Unauthorized: + case ProvideMessage::Code::Forbidden: + case ProvideMessage::Code::PeerCertificateInvalid: + case ProvideMessage::Code::NotFound: + case ProvideMessage::Code::ExpectedSizeExceeded: + case ProvideMessage::Code::ConnectionFailed: + case ProvideMessage::Code::Timeout: + case ProvideMessage::Code::Cancelled: + case ProvideMessage::Code::InvalidChecksum: + case ProvideMessage::Code::MountFailed: + case ProvideMessage::Code::Jammed: + case ProvideMessage::Code::NoAuthData: + case ProvideMessage::Code::MediaChangeAbort: + case ProvideMessage::Code::MediaChangeSkip: + case ProvideMessage::Code::InternalError: { + const auto &e = validateErrorMsg(msg); + if ( !e ) + return e; + break; + } + case ProvideMessage::Code::Provide: { + std::exception_ptr error; + DEF_REQ_FIELD(url); + msg.forEachVal( [&]( const auto &name, const ProvideMessage::FieldVal &val ){ + REQ_FIELD_CHECK ( Provide, url, std::string ) + OR_OPT_FIELD_CHECK ( Provide, filename, std::string ) + OR_OPT_FIELD_CHECK ( Provide, delta_file, std::string ) + OR_OPT_FIELD_CHECK ( Provide, expected_filesize, int64_t ) + OR_OPT_FIELD_CHECK ( Provide, check_existance_only, bool ) + OR_OPT_FIELD_CHECK ( Provide, metalink_enabled, bool ) + return true; + }); + FAIL_IF_NOT_SEEN_REQ_FIELD( Provide, url ); + FAIL_IF_ERROR(); + break; + } + case ProvideMessage::Code::Cancel: + /* No Fields */ + break; + + case ProvideMessage::Code::Attach: { + std::exception_ptr error; + + DEF_REQ_FIELD(url); + DEF_REQ_FIELD(attach_id); + DEF_REQ_FIELD(label); + + // not really required, but this way we can check if all false or all true + DEF_REQ_FIELD(verify_type); + DEF_REQ_FIELD(verify_data); + DEF_REQ_FIELD(media_nr); + + msg.forEachVal( [&]( const auto &name, const ProvideMessage::FieldVal &val ){ + REQ_FIELD_CHECK ( Attach, url , std::string ) + OR_REQ_FIELD_CHECK ( Attach, attach_id , std::string ) + OR_REQ_FIELD_CHECK ( Attach, label , std::string ) + OR_REQ_FIELD_CHECK ( Attach, verify_type, std::string ) + OR_REQ_FIELD_CHECK ( Attach, verify_data, std::string ) + OR_REQ_FIELD_CHECK ( Attach, media_nr , int32_t ) + OR_OPT_FIELD_CHECK ( Attach, device , std::string ) + return true; + }); + FAIL_IF_NOT_SEEN_REQ_FIELD( Provide, url ); + FAIL_IF_NOT_SEEN_REQ_FIELD( Provide, label ); + FAIL_IF_NOT_SEEN_REQ_FIELD( Provide, attach_id ); + if ( ! ( ( has_verify_data == has_verify_type ) && ( has_verify_type == has_media_nr ) ) ) + return expected::error( ZYPP_EXCPT_PTR ( InvalidMessageReceivedException("Error in Attach message, one of the following fields is not set or invalid: ( verify_type, verify_data, media_nr ). Either none or all need to be set. ")) ); + FAIL_IF_ERROR(); + break; + } + case ProvideMessage::Code::Detach: { + std::exception_ptr error; + DEF_REQ_FIELD(url); + msg.forEachVal( [&]( const auto &name, const ProvideMessage::FieldVal &val ){ + REQ_FIELD_CHECK ( Detach, url, std::string ) + return true; + }); + FAIL_IF_NOT_SEEN_REQ_FIELD( Detach, url ); + FAIL_IF_ERROR(); + break; + } + case ProvideMessage::Code::AuthDataRequest: { + std::exception_ptr error; + DEF_REQ_FIELD(effective_url); + msg.forEachVal( [&]( const auto &name, const ProvideMessage::FieldVal &val ){ + REQ_FIELD_CHECK ( AuthDataRequest, effective_url, std::string ) + OR_OPT_FIELD_CHECK ( AuthDataRequest, last_auth_timestamp, int64_t ) + OR_OPT_FIELD_CHECK ( AuthDataRequest, username, std::string ) + OR_OPT_FIELD_CHECK ( AuthDataRequest, authHint, std::string ) + return true; + }); + FAIL_IF_NOT_SEEN_REQ_FIELD( AuthDataRequest, effective_url ); + FAIL_IF_ERROR(); + break; + } + case ProvideMessage::Code::MediaChangeRequest: { + std::exception_ptr error; + DEF_REQ_FIELD(label); + DEF_REQ_FIELD(media_nr); + DEF_REQ_FIELD(device); + msg.forEachVal( [&]( const auto &name, const ProvideMessage::FieldVal &val ){ + REQ_FIELD_CHECK ( MediaChangeRequest, label, std::string ) + OR_REQ_FIELD_CHECK ( MediaChangeRequest, media_nr, int32_t ) + OR_REQ_FIELD_CHECK ( MediaChangeRequest, device, std::string ) + OR_OPT_FIELD_CHECK ( MediaChangeRequest, desc, std::string ) + return true; + }); + FAIL_IF_NOT_SEEN_REQ_FIELD( MediaChangeRequest, label ); + FAIL_IF_NOT_SEEN_REQ_FIELD( MediaChangeRequest, media_nr ); + FAIL_IF_NOT_SEEN_REQ_FIELD( MediaChangeRequest, device ); + FAIL_IF_ERROR(); + break; + } + default: { + // all error messages have the same format + if ( c >= ProvideMessage::Code::FirstClientErrCode && c <= ProvideMessage::Code::LastSrvErrCode ) { + const auto &e = validateErrorMsg(msg); + if ( !e ) + return e; + } + break; + } + } + return expected::success(); + } + + ProvideMessage::ProvideMessage() + : _impl ( new zypp::proto::ProvideMessage ) + { } + + expected ProvideMessage::create(const RpcMessage &message) + { + ProvideMessage msg; + const auto &res = RpcMessageStream::parseMessageInto( message, *msg._impl ); + if ( res ) { + const auto &valid = validateMessage(msg); + if ( !valid ) { + ERR << "Invalid message for ID: " << msg._impl->request_id() << std::endl;; + return zyppng::expected::error( valid.error() ); + } + + return zyppng::expected::success( std::move(msg) ); + } + ERR << "Failed to parse message" << std::endl;; + return zyppng::expected::error( res.error() ); + } + + expected ProvideMessage::create( const zypp::proto::ProvideMessage &message ) + { + ProvideMessage msg; + *msg._impl = std::move(message); + const auto &valid = validateMessage(msg); + if ( !valid ) { + ERR << "Invalid message for ID: " << msg._impl->request_id() << std::endl;; + return zyppng::expected::error( valid.error() ); + } + + return zyppng::expected::success( std::move(msg) ); + } + + ProvideMessage ProvideMessage::createProvideStarted( const uint32_t reqId, const zypp::Url &url, const std::optional &localFilename, const std::optional &stagingFilename ) + { + ProvideMessage msg; + msg.setCode ( ProvideMessage::Code::ProvideStarted ); + msg.setRequestId ( reqId ); + msg.setValue ( ProvideStartedMsgFields::Url, url.asCompleteString() ); + if ( localFilename ) + msg.setValue ( ProvideStartedMsgFields::LocalFilename, *localFilename ); + if ( stagingFilename ) + msg.setValue ( ProvideStartedMsgFields::StagingFilename, *stagingFilename ); + + return msg; + } + + ProvideMessage ProvideMessage::createProvideFinished( const uint32_t reqId, const std::string &localFilename, bool cacheHit ) + { + ProvideMessage msg; + msg.setCode ( ProvideMessage::Code::ProvideFinished ); + msg.setRequestId ( reqId ); + msg.setValue ( ProvideFinishedMsgFields::LocalFilename, localFilename ); + msg.setValue ( ProvideFinishedMsgFields::CacheHit, cacheHit ); + + return msg; + } + + ProvideMessage ProvideMessage::createAttachFinished( const uint32_t reqId ) + { + ProvideMessage msg; + msg.setCode ( ProvideMessage::Code::AttachFinished ); + msg.setRequestId ( reqId ); + + return msg; + } + + ProvideMessage ProvideMessage::createDetachFinished(const uint32_t reqId) + { + ProvideMessage msg; + msg.setCode ( ProvideMessage::Code::DetachFinished ); + msg.setRequestId ( reqId ); + + return msg; + } + + ProvideMessage ProvideMessage::createAuthInfo( const uint32_t reqId, const std::string &user, const std::string &pw, int64_t timestamp, const std::map &extraValues ) + { + ProvideMessage msg; + msg.setCode ( ProvideMessage::Code::AuthInfo ); + msg.setRequestId ( reqId ); + msg.setValue ( AuthInfoMsgFields::Username, user ); + msg.setValue ( AuthInfoMsgFields::Password, pw ); + msg.setValue ( AuthInfoMsgFields::AuthTimestamp, timestamp ); + for ( auto i : extraValues ) { + msg.setValue( i.first, i.second ); + } + return msg; + } + + ProvideMessage ProvideMessage::createMediaChanged( const uint32_t reqId ) + { + ProvideMessage msg; + msg.setCode ( ProvideMessage::Code::MediaChanged ); + msg.setRequestId ( reqId ); + + return msg; + } + + ProvideMessage ProvideMessage::createRedirect( const uint32_t reqId, const zypp::Url &newUrl ) + { + ProvideMessage msg; + msg.setCode ( ProvideMessage::Code::Redirect ); + msg.setRequestId ( reqId ); + msg.setValue ( RedirectMsgFields::NewUrl, newUrl.asCompleteString() ); + + return msg; + } + + ProvideMessage ProvideMessage::createMetalinkRedir( const uint32_t reqId, const std::vector &newUrls ) + { + ProvideMessage msg; + msg.setCode ( ProvideMessage::Code::Metalink ); + msg.setRequestId ( reqId ); + for( const auto &val : newUrls ) + msg.addValue( MetalinkRedirectMsgFields::NewUrl, val.asCompleteString() ); + + return msg; + } + + ProvideMessage ProvideMessage::createErrorResponse( const uint32_t reqId, const uint code, const std::string &reason, bool transient ) + { + ProvideMessage msg; + if ( code < Code::FirstClientErrCode || code > Code::LastSrvErrCode ) + ZYPP_THROW(std::out_of_range("code must be between 400 and 599")); + msg.setCode ( code ); + msg.setRequestId ( reqId ); + msg.setValue ( ErrMsgFields::Reason, reason ); + msg.setValue ( ErrMsgFields::Transient, transient ); + return msg; + } + + ProvideMessage ProvideMessage::createProvide( const uint32_t reqId, const zypp::Url &url, const std::optional &filename, const std::optional &deltaFile, const std::optional &expFilesize, bool checkExistOnly ) + { + ProvideMessage msg; + msg.setCode ( ProvideMessage::Code::Provide ); + msg.setRequestId ( reqId ); + msg.setValue ( ProvideMsgFields::Url, url.asCompleteString() ); + + if ( filename ) + msg.setValue ( ProvideMsgFields::Filename, *filename ); + if ( deltaFile ) + msg.setValue ( ProvideMsgFields::DeltaFile, *deltaFile ); + if ( expFilesize ) + msg.setValue ( ProvideMsgFields::ExpectedFilesize, *expFilesize ); + msg.setValue ( ProvideMsgFields::CheckExistOnly, checkExistOnly ); + + return msg; + } + + ProvideMessage ProvideMessage::createCancel( const uint32_t reqId ) + { + ProvideMessage msg; + msg.setCode ( ProvideMessage::Code::Cancel ); + msg.setRequestId ( reqId ); + + return msg; + } + + ProvideMessage ProvideMessage::createAttach(const uint32_t reqId, const zypp::Url &url, const std::string attachId, const std::string &label, const std::optional &verifyType, const std::optional &verifyData, const std::optional &mediaNr ) + { + ProvideMessage msg; + msg.setCode ( ProvideMessage::Code::Attach ); + msg.setRequestId ( reqId ); + msg.setValue ( AttachMsgFields::Url, url.asCompleteString() ); + msg.setValue ( AttachMsgFields::AttachId, attachId ); + msg.setValue ( AttachMsgFields::Label, label ); + + if ( verifyType.has_value() && verifyData.has_value() && mediaNr.has_value() ) { + msg.setValue ( AttachMsgFields::VerifyType, *verifyType ); + msg.setValue ( AttachMsgFields::VerifyData, *verifyData ); + msg.setValue ( AttachMsgFields::MediaNr, *mediaNr ); + } else { + if ( !( ( verifyType.has_value() == verifyData.has_value() ) && ( verifyData.has_value() == mediaNr.has_value() ) ) ) + WAR << "Attach message requires verifyType, verifyData and mediaNr either set together or not set at all." << std::endl; + } + + return msg; + } + + ProvideMessage ProvideMessage::createDetach( const uint32_t reqId, const zypp::Url &attachUrl ) + { + ProvideMessage msg; + msg.setCode ( ProvideMessage::Code::Detach ); + msg.setRequestId ( reqId ); + msg.setValue ( DetachMsgFields::Url, attachUrl.asCompleteString() ); + + return msg; + } + + ProvideMessage ProvideMessage::createAuthDataRequest( const uint32_t reqId, const zypp::Url &effectiveUrl, const std::string &lastTriedUser, const std::optional &lastAuthTimestamp, const std::map &extraValues ) + { + ProvideMessage msg; + msg.setCode ( ProvideMessage::Code::AuthDataRequest ); + msg.setRequestId ( reqId ); + msg.setValue ( AuthDataRequestMsgFields::EffectiveUrl, effectiveUrl.asCompleteString() ); + if ( lastTriedUser.size() ) + msg.setValue( AuthDataRequestMsgFields::LastUser, lastTriedUser ); + if ( lastAuthTimestamp ) + msg.setValue ( AuthDataRequestMsgFields::LastAuthTimestamp, *lastAuthTimestamp ); + + return msg; + } + + ProvideMessage ProvideMessage::createMediaChangeRequest( const uint32_t reqId, const std::string &label, int32_t mediaNr, const std::vector &devices, const std::optional &desc ) + { + ProvideMessage msg; + msg.setCode ( ProvideMessage::Code::MediaChangeRequest ); + msg.setRequestId ( reqId ); + msg.setValue ( MediaChangeRequestMsgFields::Label, label ); + msg.setValue ( MediaChangeRequestMsgFields::MediaNr, mediaNr ); + for ( const auto &device : devices ) + msg.addValue ( MediaChangeRequestMsgFields::Device, device ); + if ( desc ) + msg.setValue ( MediaChangeRequestMsgFields::Desc, *desc ); + + return msg; + } + + uint ProvideMessage::requestId() const + { + return _impl->request_id(); + } + + void ProvideMessage::setRequestId(const uint id) + { + _impl->set_request_id( id ); + } + + uint ProvideMessage::code() const + { + return _impl->message_code(); + } + + void ProvideMessage::setCode( const uint newCode ) + { + _impl->set_message_code ( newCode ); + } + + std::vector ProvideMessage::values( const std::string_view &str ) const + { + std::vector values; + const auto &fields = _impl->fields(); + for ( const auto &field : fields ) { + if ( field.key() != str ) + continue; + values.push_back( fieldValFromProto(field) ); + } + return values; + } + + std::vector ProvideMessage::values( const std::string &str ) const + { + return values( std::string_view(str)); + } + + ProvideMessage::FieldVal ProvideMessage::value( const std::string_view &str, const FieldVal &defaultVal ) const + { + const auto &fields = _impl->fields(); + auto i = std::find_if( fields.rbegin(), fields.rend(), [&str]( const auto &val ){ return val.key() == str; } ); + if ( i == fields.rend() ) + return defaultVal; + return fieldValFromProto(*i); + } + + + HeaderValueMap ProvideMessage::headers() const + { + HeaderValueMap res; + auto &fields = _impl->fields(); + for ( const auto &val : fields ) { + res.add( val.key() ,fieldValFromProto(val) ); + } + return res; + } + + ProvideMessage::FieldVal ProvideMessage::value( const std::string &str, const FieldVal &defaultVal ) const + { + return value( std::string_view(str), defaultVal ); + } + + void ProvideMessage::setValue( const std::string &name, const FieldVal &value ) + { + setValue( std::string_view(name), value ); + } + + void ProvideMessage::setValue( const std::string_view &name, const FieldVal &value ) + { + auto &fields = *_impl->mutable_fields(); + auto i = std::find_if( fields.rbegin(), fields.rend(), [&name]( const auto &val ){ return val.key() == name; } ); + if ( i == fields.rend() ) { + auto &newVal = *_impl->add_fields(); + newVal.set_key( name.data() ); + fieldValToProto( value, newVal ); + } else + fieldValToProto( value, *i ); + } + + void ProvideMessage::addValue( const std::string &name, const FieldVal &value ) + { + return addValue( std::string_view(name), value ); + } + + void ProvideMessage::addValue( const std::string_view &name, const FieldVal &value ) + { + auto &newVal = *_impl->add_fields(); + newVal.set_key( name.data() ); + fieldValToProto( value, newVal ); + } + + void ProvideMessage::forEachVal( const std::function &cb ) const + { + auto &fields = _impl->fields(); + for ( const auto &val : fields ) { + if ( !cb( val.key(), fieldValFromProto(val) ) ) { + return; + } + } + } + + zypp::proto::ProvideMessage &ProvideMessage::impl() + { + return *_impl.get(); + } + + const zypp::proto::ProvideMessage &ProvideMessage::impl() const + { + return *_impl.get(); + } + +} diff --git a/zypp-media/ng/providequeue.cc b/zypp-media/ng/providequeue.cc new file mode 100644 index 0000000..a06d28a --- /dev/null +++ b/zypp-media/ng/providequeue.cc @@ -0,0 +1,775 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ + +#include "private/providequeue_p.h" +#include "private/provideitem_p.h" +#include "private/provide_p.h" +#include "private/providemessage_p.h" +#include "private/providedbg_p.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace zyppng { + + bool ProvideQueue::Item::isAttachRequest() const + { + return ( _request->code () == ProvideMessage::Code::Attach ); + } + + bool ProvideQueue::Item::isFileRequest() const + { + return ( _request->code () == ProvideMessage::Code::Provide ); + } + + bool ProvideQueue::Item::isDetachRequest() const + { + return ( _request->code () == ProvideMessage::Code::Detach ); + } + + ProvideQueue::ProvideQueue(ProvidePrivate &parent) : _parent(parent) + { } + + ProvideQueue::~ProvideQueue() + { + if ( zyppng::provideDebugEnabled() ) { + if ( this->_activeItems.size() || this->_waitQueue.size() ) { + DBG << "Queue shutdown with Items still running" << std::endl; + } + } + immediateShutdown(std::make_exception_ptr(zypp::media::MediaException("Cancelled by queue shutdown"))); + } + + bool ProvideQueue::startup(const std::string &workerScheme, const zypp::filesystem::Pathname &workDir, const std::string &hostname ) { + + if ( _workerProc ) { + ERR << "Queue Worker was already initialized" << std::endl; + return true; + } + + _myHostname = hostname; + + const auto &pN = _parent.workerPath() / ( "zypp-media-"+workerScheme ) ; + MIL << "Trying to start " << pN << std::endl; + const auto &pi = zypp::PathInfo( pN ); + if ( !pi.isExist() ) { + ERR << "Failed to find worker for " << workerScheme << std::endl; + return false; + } + + if ( !pi.userMayX() ) { + ERR << "Failed to start worker for " << workerScheme << " binary " << pi.asString() << " is not executable." << std::endl; + return false; + } + + if ( zypp::filesystem::assert_dir( workDir ) != 0 ) { + ERR << "Failed to assert working directory '" << workDir << "' for worker " << workerScheme << std::endl; + return false; + } + + _currentExe = pN; + _workerProc = Process::create(); + _workerProc->setWorkingDirectory ( workDir ); + _messageStream = RpcMessageStream::create( _workerProc ); + return doStartup(); + } + + + void ProvideQueue::enqueue( ProvideRequestRef request ) + { + Item i; + i._request = request; + i._request->provideMessage().setRequestId( nextRequestId() ); + request->setCurrentQueue( shared_this() ); + _waitQueue.push_back( std::move(i) ); + if ( _parent.isRunning() ) + scheduleNext(); + } + + void ProvideQueue::cancel( ProvideRequest *item , std::exception_ptr error ) + { + const auto &isSameItem = [item]( const Item &i ){ + if ( i.isDetachRequest () ) + return false; + return i._request.get() == item; + }; + + if ( !item ) + return; + + if ( item->code() != ProvideMessage::Code::Attach + && item->code() != ProvideMessage::Code::Provide ) { + ERR << "Can not cancel a " << item->code() << " request!" << std::endl; + return; + } + + if ( auto i = std::find_if( _waitQueue.begin(), _waitQueue.end(), isSameItem ); i != _waitQueue.end() ) { + auto &reqRef = i->_request; + reqRef->setCurrentQueue(nullptr); + if ( reqRef->owner() ) + reqRef->owner()->finishReq( this, reqRef, error ); + _waitQueue.erase(i); + _parent.schedule( ProvidePrivate::FinishReq ); // let the parent scheduler run since we have a open spot now + } else if ( auto i = std::find_if( _activeItems.begin(), _activeItems.end(), isSameItem ); i != _activeItems.end() ) { + cancelActiveItem(i, error); + } + } + + std::list::iterator ProvideQueue::dequeueActive( std::list::iterator it ) + { + if ( it == _activeItems.end() ) + return it; + + if ( it->_request ) + it->_request->setCurrentQueue( nullptr ); + + auto i = _activeItems.erase(it); + _parent.schedule ( ProvidePrivate::FinishReq ); // Trigger the scheduler + scheduleNext (); // keep the active items full + return i; + } + + void ProvideQueue::fatalWorkerError( const std::exception_ptr &reason ) + { + immediateShutdown( reason ? reason : std::make_exception_ptr( zypp::media::MediaException("Fatal worker error")) ); + } + + void ProvideQueue::immediateShutdown( const std::exception_ptr &reason ) + { + _queueShuttingDown = true; + + while ( _waitQueue.size() ) { + auto &item = _waitQueue.front(); + auto &reqRef = item._request; + if ( reqRef && reqRef->owner() && !item.isDetachRequest() ) + reqRef->owner()->finishReq( this, reqRef, reason ); + _waitQueue.pop_front(); + } + + for ( auto i = _activeItems.begin(); i != _activeItems.end(); ) { + auto &reqRef = i->_request; + if ( reqRef && reqRef->owner() && !i->isDetachRequest() ) { + i = cancelActiveItem(i, reason ); + } else { + i++; + } + } + + if ( _workerProc && _workerProc->isRunning() ) { + _workerProc->flush(); + _workerProc->closeWriteChannel(); + _workerProc->waitForExit(); + readAllStderr(); + } + } + + std::list< ProvideQueue::Item >::iterator ProvideQueue::cancelActiveItem( std::list< Item >::iterator i , const std::__exception_ptr::exception_ptr &error ) + { + auto &reqRef = i->_request; + + // already in cancelling process or finished + if ( i->_state == Item::Cancelling || i->_state == Item::Finished ) + return (++i); + + // not possible but lets be safe + if ( i->_state == Item::Pending ) { + reqRef->setCurrentQueue(nullptr); + if ( reqRef->owner() ) + reqRef->owner()->finishReq( this, reqRef, error ); + return dequeueActive(i); + } + + // we first need to cancel the item + auto c = ProvideMessage::createCancel ( i->_request->provideMessage().requestId() ); + if( !_messageStream->sendMessage(c.impl()) ) + ERR << "Failed to send cancel message to worker" << std::endl; + + i->_state = Item::Cancelling; + reqRef->setCurrentQueue(nullptr); + if ( reqRef->owner() ) + reqRef->owner()->finishReq( this, reqRef, error ); + reqRef.reset(); + return (++i); + } + + void ProvideQueue::scheduleNext() + { + if ( _queueShuttingDown ) + return; + + while ( _waitQueue.size() && canScheduleMore() ) { + auto item = std::move( _waitQueue.front() ); + _waitQueue.pop_front(); + + auto &reqRef = item._request; + if ( !reqRef->activeUrl() ) { + ERR << "Item without active URL enqueued, this is a BUG." << std::endl; + if ( reqRef->owner() ) + reqRef->owner()->finishReq( this, reqRef, ZYPP_EXCPT_PTR (zypp::media::MediaException("Item needs a activeURL to be queued.")) ); + continue; + } + + if ( !_messageStream->sendMessage( reqRef->provideMessage().impl() ) ) { + ERR << "Failed to send message to worker process." << std::endl; + fatalWorkerError( ZYPP_EXCPT_PTR( zypp::media::MediaException("Failed to communicate with worker process.") ) ); + return; + } + + item._state = Item::Queued; + _activeItems.push_back( std::move(item) ); + _idleSince.reset(); + } + + if ( _waitQueue.empty() && _activeItems.empty() ) { + _parent.schedule( ProvidePrivate::QueueIdle ); + if ( !_idleSince ) + _idleSince = std::chrono::steady_clock::now(); + _sigIdle.emit(); + } + } + + bool ProvideQueue::canScheduleMore() const + { + return ( _activeItems.size() == 0 || ( _capabilities.cfg_flags () & zypp::proto::Capabilities::Pipeline ) == zypp::proto::Capabilities::Pipeline ); + } + + bool ProvideQueue::isIdle() const + { + return ( empty() ); + } + + std::optional ProvideQueue::idleSince() const + { + return _idleSince; + } + + bool ProvideQueue::empty() const + { + return ( _activeItems.empty() && _waitQueue.empty() ); + } + + uint ProvideQueue::requestCount() const + { + return _activeItems.size() + _waitQueue.size(); + } + + uint ProvideQueue::activeRequests() const + { + return _activeItems.size(); + } + + zypp::ByteCount ProvideQueue::expectedProvideSize() const + { + zypp::ByteCount dlSize; + for ( const auto &i : _waitQueue ) { + if ( i.isDetachRequest () ) + continue; + + auto &reqRef = i._request; + if ( reqRef->code() != ProvideMessage::Code::Provide ) + continue; + dlSize += reqRef->provideMessage().value( ProvideMsgFields::ExpectedFilesize, int64_t(0) ).asInt64(); + } + for ( const auto &i : _activeItems ) { + if ( i.isDetachRequest () ) + continue; + auto &reqRef = i._request; + if ( reqRef->code() != ProvideMessage::Code::Provide ) + continue; + dlSize += reqRef->provideMessage().value( ProvideMsgFields::ExpectedFilesize, int64_t(0) ).asInt64(); + } + return dlSize; + } + + const std::string &ProvideQueue::hostname() const + { + return _myHostname; + } + + const ProvideQueue::Config &ProvideQueue::workerConfig() const + { + return _capabilities; + } + + SignalProxy ProvideQueue::sigIdle() + { + return _sigIdle; + } + + bool ProvideQueue::doStartup() + { + if ( _currentExe.empty() ) + return false; + + //const char *argv[] = { "gdbserver", ":10000", _currentExe.c_str(), nullptr }; + const char *argv[] = { _currentExe.c_str(), nullptr }; + if ( !_workerProc->start( argv) ) { + ERR << "Failed to execute worker" << std::endl; + + _messageStream.reset (); + _workerProc.reset (); + + return false; + } + + // make sure the default read channel is StdOut so RpcMessageStream gets all the rpc messages + _workerProc->setReadChannel ( Process::StdOut ); + + // we are ready to send the data + + zypp::proto::Configuration conf; + // @TODO actually write real config data :D + conf.mutable_values ()->insert ( { AGENT_STRING_CONF.data (), "ZYpp " LIBZYPP_VERSION_STRING } ); + conf.mutable_values ()->insert ( { ATTACH_POINT.data (), _workerProc->workingDirectory().asString() } ); + conf.mutable_values ()->insert ( { PROVIDER_ROOT.data (), _parent.z_func()->providerWorkdir().asString() } ); + + const auto &cleanupOnErr = [&](){ + readAllStderr(); + _messageStream.reset (); + _workerProc->close(); + _workerProc.reset(); + return false; + }; + + if ( !_messageStream->sendMessage( conf ) ) { + ERR << "Failed to send initial message to queue worker" << std::endl; + return cleanupOnErr(); + } + + // wait for the data to be written + _workerProc->flush (); + + // wait until we receive a message + const auto &caps = _messageStream->nextMessageWait(); + if ( !caps || caps->messagetypename() != rpc::messageTypeName() ) { + ERR << "Worker did not sent a capabilities message, aborting" << std::endl; + return cleanupOnErr(); + } + + { + auto p = _messageStream->parseMessage( *caps ); + if ( !p ) + return cleanupOnErr(); + + _capabilities = std::move(*p); + } + + DBG << "Received config for worker: " << this->_currentExe.asString() << " Worker Type: " << this->_capabilities.worker_type() << " Flags: " << std::bitset<32>( _capabilities.cfg_flags() ).to_string() << std::endl; + + // now we can set up signals and start processing messages + connect( *_messageStream, &RpcMessageStream::sigMessageReceived, *this, &ProvideQueue::processMessage ); + connect( *_workerProc, &IODevice::sigChannelReadyRead, *this, &ProvideQueue::processReadyRead ); + connect( *_workerProc, &Process::sigFinished, *this, &ProvideQueue::procFinished ); + + // make sure we do not miss messages + processMessage(); + return true; + } + + void ProvideQueue::processMessage() { + + const auto &getRequest = [&]( const auto &exp ) -> decltype(_activeItems)::iterator { + if ( !exp ) { + ERR << "Ignoring invalid request!" << std::endl; + return _activeItems.end(); + } + + auto i = std::find_if( _activeItems.begin(), _activeItems.end(), [&]( const auto &elem ) { + return exp->requestId() == elem._request->provideMessage().requestId(); + }); + + if ( i == _activeItems.end() ) { + ERR << "Ignoring unknown request ID: " << exp->requestId() << std::endl; + return _activeItems.end(); + } + + return i; + }; + + const auto &sendErrorToWorker = [&]( const uint32_t reqId, const uint code, const std::string &reason, bool transient = false ) { + auto r = ProvideMessage::createErrorResponse ( reqId, code, reason, transient ); + if ( !_messageStream->sendMessage( r.impl() ) ) { + ERR << "Failed to send Error message to worker process." << std::endl; + fatalWorkerError( ZYPP_EXCPT_PTR( zypp::media::MediaException("Failed to communicate with worker process.") ) ); + return false; + } + return true; + }; + + const bool doesDownload = this->_capabilities.worker_type() == Config::Downloading; + const bool fileNeedsCleanup = doesDownload || ( _capabilities.worker_type() == Config::CPUBound && _capabilities.cfg_flags() & Config::FileArtifacts ); + + while ( auto msg = _messageStream->nextMessage () ) { + + if ( msg->messagetypename() == rpc::messageTypeName() ) { + + const auto &provMsg = ProvideMessage::create(*msg); + if ( !provMsg ) { + fatalWorkerError( provMsg.error() ); + return; + } + + const auto &reqIter = getRequest( provMsg ); + if ( reqIter == _activeItems.end() ) { + if ( provMsg->code() == ProvideMessage::Code::ProvideFinished && fileNeedsCleanup ) { + const auto locFName = provMsg->value( ProvideFinishedMsgFields::LocalFilename ).asString(); + if ( !_parent.isInCache(locFName) ) { + MIL << "Received a ProvideFinished message for a non existant request. Since this worker reported to create file artifacts, the file is cleaned up." << std::endl; + zypp::filesystem::unlink( locFName ); + } + } + continue; + } + + auto &req = *reqIter; + auto &reqRef =req._request; + + const auto code = provMsg->code(); + + if ( code >= ProvideMessage::Code::FirstInformalCode && code <= ProvideMessage::Code::LastInformalCode ) { + + // send the message to the item but don't dequeue + if ( reqRef && reqRef->owner() ) + reqRef->owner()->informalMessage ( *this, reqRef, *provMsg ); + continue; + + } else if ( code >= ProvideMessage::Code::FirstSuccessCode && code <= ProvideMessage::Code::LastSuccessCode ) { + + if ( req._state == Item::Cancelling ) { + req._state = Item::Finished; + dequeueActive( reqIter ); + continue; + } + + if ( code == ProvideMessage::Code::ProvideFinished ) { + + // we are going to register the file to the cache if this is a downloading worker, so it can not leak + // no matter if the item does the correct dance or not, this code is duplicated by all ProvideItems that receive ProvideFinished + // results that require file cleanups. + // we keep the ref around until after sending the result to the item. At that point it should take a reference + std::optional dataRef; + + if ( !reqIter->isFileRequest() ) { + ERR << "Invalid message for request ID: " << reqIter->_request->provideMessage().requestId() << std::endl; + fatalWorkerError(); + return; + } + + // when a worker is downloading we keep a internal book of cache files + if ( doesDownload ) { + const auto locFName = provMsg->value( ProvideFinishedMsgFields::LocalFilename ).asString(); + if ( provMsg->value( ProvideFinishedMsgFields::CacheHit, false ).asBool()) { + dataRef = _parent.addToFileCache ( locFName ); + if ( !dataRef ) { + MIL << "CACHE MISS, file " << locFName << " was already removed, queueing again" << std::endl; + if ( reqRef->owner() ) + reqRef->owner()->cacheMiss( reqRef ); + reqRef->provideMessage().setRequestId( InvalidId ); + req._state = Item::Pending; + _waitQueue.push_front( req ); + dequeueActive( reqIter ); + continue; + } + } else { + dataRef = _parent.addToFileCache ( locFName ); + + // unlikely this can happen but better be safe than sorry + if ( !dataRef ) { + req._state = Item::Finished; + reqRef->setCurrentQueue(nullptr); + auto resp = ProvideMessage::createErrorResponse ( provMsg->requestId(), ProvideMessage::Code::InternalError, "File vanished between downloading and adding it to cache." ); + if ( reqRef->owner() ) + reqRef->owner()->finishReq( *this, reqRef, resp ); + dequeueActive( reqIter ); + continue; + } + } + } + } + + // send the message to the item and dequeue + reqRef->setCurrentQueue(nullptr); + if ( reqRef->owner() ) + reqRef->owner()->finishReq( *this, reqRef, *provMsg ); + req._state = Item::Finished; + dequeueActive( reqIter ); + continue; + + } else if ( code >= ProvideMessage::Code::FirstClientErrCode && code <= ProvideMessage::Code::LastSrvErrCode ) { + + if ( req._state == Item::Cancelling ) { + req._state = Item::Finished; + dequeueActive( reqIter ); + continue; + } + + // send the message to the item and dequeue + reqRef->setCurrentQueue(nullptr); + + if ( reqRef->owner() ) + reqRef->owner()->finishReq( *this, reqRef, *provMsg ); + + req._state = Item::Finished; + dequeueActive( reqIter ); + continue; + + } else if ( code >= ProvideMessage::Code::FirstRedirCode && code <= ProvideMessage::Code::LastRedirCode ) { + + // redir is like a finished message, we can simply forgot about a cancelling request + if ( req._state == Item::Cancelling ) { + req._state = Item::Finished; + dequeueActive( reqIter ); + continue; + } + + // send the message to the item and dequeue + reqRef->setCurrentQueue(nullptr); + if ( reqRef->owner() ) + reqRef->owner()->finishReq( *this, reqRef, *provMsg ); + req._state = Item::Finished; + dequeueActive( reqIter ); + continue; + + } else if ( code >= ProvideMessage::Code::FirstControllerCode && code <= ProvideMessage::Code::LastControllerCode ) { + + ERR << "Received Controller message from worker, this is a fatal error. Cancelling all requests!" << std::endl; + fatalWorkerError ( ZYPP_EXCPT_PTR( zypp::media::MediaException("Controller message received from worker.") ) ); + return; + + } else if ( code >= ProvideMessage::Code::FirstWorkerCode && code <= ProvideMessage::Code::LastWorkerCode ) { + + if ( code == ProvideMessage::Code::AuthDataRequest ) { + if ( !reqIter->isFileRequest() && !reqIter->isAttachRequest() ) { + ERR << "Invalid message for request ID: " << reqRef->provideMessage().requestId() << std::endl; + fatalWorkerError(); + return; + } + + // if the file was cancelled we send a failure back + if( reqIter->_state == Item::Cancelling ) { + if ( !sendErrorToWorker( reqRef->provideMessage().requestId(), ProvideMessage::Code::NoAuthData, "Item was cancelled") ) + return; + continue; + } + + // we need a owner item to fetch the auth data for us + if ( !reqRef->owner() ) { + if ( !sendErrorToWorker( reqRef->provideMessage().requestId(), ProvideMessage::Code::NoAuthData, "Request has no owner" ) ) + return; + continue; + } + + if ( !reqRef->activeUrl() ) { + if ( !sendErrorToWorker( reqRef->provideMessage().requestId(), ProvideMessage::Code::NoAuthData, "Item has no active URL, this is a bug." ) ) + return; + continue; + } + + try { + zypp::Url u( provMsg->value( AuthDataRequestMsgFields::EffectiveUrl ).asString() ); + + std::map extraVals; + provMsg->forEachVal( [&]( const std::string &name, const zyppng::ProvideMessage::FieldVal &val ) { + + if ( name == AuthDataRequestMsgFields::EffectiveUrl + || name == AuthDataRequestMsgFields::LastAuthTimestamp ) + return true; + + if ( !val.isString() ) { + WAR << "Ignoring non string value for " << name << std::endl; + return true; + } + + extraVals[name] = val.asString(); + return true; + }); + + const auto &authOpt = reqRef->owner()->authenticationRequired( *this, reqRef, u, provMsg->value( AuthDataRequestMsgFields::LastAuthTimestamp ).asInt64(), extraVals ); + if ( !authOpt ) { + if ( !sendErrorToWorker( reqRef->provideMessage().requestId(), ProvideMessage::Code::NoAuthData, "No auth given by user." ) ) + return; + continue; + } + + auto r = ProvideMessage::createAuthInfo ( reqRef->provideMessage().requestId(), authOpt->username(), authOpt->password(), authOpt->lastDatabaseUpdate(), authOpt->extraValues() ); + if ( !_messageStream->sendMessage( r.impl() ) ) { + ERR << "Failed to send AuthorizationInfo to worker process." << std::endl; + fatalWorkerError( ZYPP_EXCPT_PTR( zypp::media::MediaException("Failed to communicate with worker process.") ) ); + return; + } + continue; + + } catch ( const zypp::Exception &e ) { + ZYPP_CAUGHT(e); + if ( !sendErrorToWorker( reqRef->provideMessage().requestId(), ProvideMessage::Code::NoAuthData, e.asString() ) ) + return; + continue; + } + + } else if ( code == ProvideMessage::Code::MediaChangeRequest ) { + + if ( !reqIter->isAttachRequest() ) { + ERR << "Invalid message for request ID: " << reqIter->_request->provideMessage().requestId() << std::endl; + fatalWorkerError(); + return; + } + + // if the file was cancelled we send a failure back + if( reqIter->_state == Item::Cancelling ) { + if ( !sendErrorToWorker( reqRef->provideMessage().requestId(), ProvideMessage::Code::MediaChangeAbort, "Item was cancelled" ) ) + return; + continue; + } + + MIL << "Worker sent a MediaChangeRequest, asking the user to insert the correct medium" << std::endl; + + //const std::string &label, const int32_t mediaNr, const std::vector &devices, const std::optional &desc + std::vector freeDevs; + for ( const auto &val : provMsg->values( MediaChangeRequestMsgFields::Device) ) { + freeDevs.push_back( val.asString() ); + } + + std::optional desc; + const auto &descVal = provMsg->value( MediaChangeRequestMsgFields::Desc ); + if ( descVal.valid () && descVal.isString() ) + desc = descVal.asString(); + + auto res = _parent._sigMediaChange.emit( + _parent.queueName(*this), + provMsg->value( MediaChangeRequestMsgFields::Label ).asString(), + provMsg->value( MediaChangeRequestMsgFields::MediaNr ).asInt(), + freeDevs, + desc + ); + + auto action = res ? *res : Provide::Action::ABORT; + switch ( action ) { + case Provide::Action::RETRY: { + MIL << "Sending back a MediaChanged message, retrying to find medium " << std::endl; + auto r = ProvideMessage::createMediaChanged ( reqIter->_request->provideMessage().requestId() ); + if ( !_messageStream->sendMessage( r.impl() ) ){ + ERR << "Failed to send MediaChanged to worker process." << std::endl; + fatalWorkerError( ZYPP_EXCPT_PTR( zypp::media::MediaException("Failed to communicate with worker process.") ) ); + return; + } + continue; + } + case Provide::Action::ABORT: { + MIL << "Sending back a MediaChangeFailure message, request will fail " << std::endl; + if ( !sendErrorToWorker( reqRef->provideMessage().requestId(), ProvideMessage::Code::MediaChangeAbort, "Cancelled by User" ) ) + return; + continue; + } + case Provide::Action::SKIP: { + MIL << "Sending back a MediaChangeFailure message, request will fail " << std::endl; + if ( !sendErrorToWorker( reqRef->provideMessage().requestId(), ProvideMessage::Code::MediaChangeSkip, "Skipped by User" ) ) + return; + continue; + } + } + } else { + // if there is a unsupported worker request we need to stop immediately because the worker will be blocked until it gets a answer + ERR << "Unsupported worker request: "<messagetypename() << " with code " << code << " ignoring! " << std::endl; + } + + } else { + ERR << "Received unsupported message " << msg->messagetypename() << "ignoring" << std::endl; + } + } + } + + /*! + * Reads all of the log lines from stderr, call only when shutting down the queue + * because this will also read partial lines and forwards them + */ + void ProvideQueue::readAllStderr() + { + // read all stderr data so we get the full logs + auto ba = _workerProc->channelReadLine(Process::StdErr); + while ( !ba.empty() ) { + forwardToLog(std::string( ba.data(), ba.size() ) ); + ba = _workerProc->channelReadLine(Process::StdErr); + } + } + + void ProvideQueue::forwardToLog( std::string &&logLine ) + { + if ( (_capabilities.cfg_flags () & zypp::proto::Capabilities::ZyppLogFormat) == zypp::proto::Capabilities::ZyppLogFormat ) + zypp::base::LogControl::instance ().logRawLine( std::move(logLine) ); + else + MIL << "Message from worker: " << _capabilities.worker_name() << ":" << logLine << std::endl; + } + + void ProvideQueue::processReadyRead(int channel) { + // ignore stdout here + if ( channel == Process::StdOut ) + return; + + // forward the stderr output to the log bypassing the formatter + // the worker already formatted the line + while ( _workerProc->canReadLine(Process::StdErr) ) { + const auto &data = _workerProc->channelReadLine( Process::StdErr ); + if ( data.empty() ) + return; + + forwardToLog(std::string( data.data(), data.size() ) ); + } + } + + void ProvideQueue::procFinished(int exitCode) + { + // process all pending messages + processMessage(); + + // get all of the log lines + readAllStderr(); + + // shut down + // @todo implement worker restart in case of a unexpected exit + if ( !_queueShuttingDown ) + immediateShutdown( ZYPP_EXCPT_PTR( zypp::media::MediaException("Unexpected queue worker exit!") ) ); + +#if 0 + if ( !_queueShuttingDown ) { + + _crashCounter++; + if ( _crashCounter > 3 ) { + immediateShutdown( ZYPP_EXCPT_PTR( zypp::media::MediaException("Unexpected queue worker exit!") ) ); + return; + } + + MIL << "Unexpected queue worker exit with code: " << exitCode << std::endl; + // try to spawn the worker again, move active items back to wait list and start over + + if ( !doStartup () ) { + + } + } +#endif + } + + uint32_t ProvideQueue::nextRequestId() { + return _parent.nextRequestId(); + } +} diff --git a/zypp-media/ng/provideres.cc b/zypp-media/ng/provideres.cc new file mode 100644 index 0000000..e4cfae9 --- /dev/null +++ b/zypp-media/ng/provideres.cc @@ -0,0 +1,48 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ + +#include "provideres.h" +#include "private/provide_p.h" +#include "private/providequeue_p.h" +#include "private/provideres_p.h" + +namespace zyppng { + + ProvideRes::ProvideRes( std::shared_ptr dataPtr ) : _data(dataPtr) + { } + + ProvideRes::~ProvideRes() + { } + + const zypp::filesystem::Pathname ProvideRes::file() const + { + return _data->_myFile; + } + + const zypp::ManagedFile & ProvideRes::asManagedFile () const + { + return _data->_myFile; + } + + const ProvideMediaHandle &ProvideRes::mediaHandle () const + { + return _data->_mediaHandle; + } + + const zypp::Url &ProvideRes::resourceUrl () const + { + return _data->_resourceUrl; + } + + const HeaderValueMap &ProvideRes::headers () const + { + return _data->_responseHeaders; + } + +} diff --git a/zypp-media/ng/provideres.h b/zypp-media/ng/provideres.h new file mode 100644 index 0000000..9c43b57 --- /dev/null +++ b/zypp-media/ng/provideres.h @@ -0,0 +1,74 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ +/** \file zypp/source/ProvideRes.h + */ +#ifndef ZYPP_MEDIA_PROVIDERES_H_INCLUDED +#define ZYPP_MEDIA_PROVIDERES_H_INCLUDED + +#include +#include +#include +#include + + +namespace zyppng +{ + + struct ProvideResourceData; + + /** + * \class ProvideRes + * A ProvideRes object is a reference counted ownership of a resource in the cache provided by + * a \ref Provide instance. + * It is generally advisable to release a ProvideRes instance asap, to make sure resources + * can be released and devices are free to be ejected. + * + * \note all ProvideRes instances will become invalid when the \ref Provide instance is + * released. + */ + class ProvideRes + { + public: + ProvideRes( std::shared_ptr dataPtr ); + virtual ~ProvideRes(); + + /*! + * Returns the path to the provided file + */ + const zypp::Pathname file () const; + + /*! + * Returns a reference to the internally used managed file instance. + * \note If you obtain this for a file that is inside the providers working directory ( e.g. a provide result for a download ), + * the continued use after the Provide instance was relased is undefined behaviour and not supported! + */ + const zypp::ManagedFile & asManagedFile () const; + + /*! + * Returns a reference to the currently held media handle, this can be a invalid handle + */ + const ProvideMediaHandle &mediaHandle () const; + + /*! + * The URL this ressource was provided from, can be empty + */ + const zypp::Url &resourceUrl () const; + + /*! + * All headers that were received from the worker when sending the result + */ + const HeaderValueMap &headers () const; + + private: + std::shared_ptr _data; + }; +} + + +#endif diff --git a/zypp-media/ng/providespec.cc b/zypp-media/ng/providespec.cc new file mode 100644 index 0000000..65a66a4 --- /dev/null +++ b/zypp-media/ng/providespec.cc @@ -0,0 +1,270 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ +/** \file zypp-media/providespec.cc + * +*/ + +#include +#include "providespec.h" + +using std::endl; + +namespace zyppng +{ + class ProvideSpecBasePrivate + { + public: + ProvideSpecBasePrivate() {} + virtual ~ProvideSpecBasePrivate() {} + HeaderValueMap _customHeaders; + }; + + + class ProvideMediaSpec::Impl : public ProvideSpecBasePrivate + { + public: + Impl() + {} + + Impl( const std::string &label, const zypp::Pathname &vPath, unsigned medianr ) + : _label( label ) + , _medianr( medianr ) + , _verifyDataPath(vPath) + {} + + std::string _label; + unsigned _medianr = 0U; + zypp::Pathname _verifyDataPath; + + public: + /** Offer default Impl. */ + static zypp::shared_ptr nullimpl() + { static zypp::shared_ptr _nullimpl( new Impl ); return _nullimpl; } + + private: + friend ProvideMediaSpec::Impl * zypp::rwcowClone( const ProvideMediaSpec::Impl * rhs ); + Impl * clone() const { return new Impl( *this ); } + }; + + class ProvideFileSpec::Impl : public ProvideSpecBasePrivate + { + public: + Impl() + {} + + zypp::Pathname _destFilenameHint; + zypp::Pathname _mediaSpecFile; + bool _checkExistsOnly = false; + + bool _optional = false; + zypp::ByteCount _downloadSize; + zypp::CheckSum _checksum; + + zypp::ByteCount _openSize; + zypp::CheckSum _openChecksum; + + zypp::ByteCount _headerSize; + zypp::CheckSum _headerChecksum; + + zypp::Pathname _deltafile; + + + public: + /** Offer default Impl. */ + static zypp::shared_ptr nullimpl() + { static zypp::shared_ptr _nullimpl( new Impl ); return _nullimpl; } + + private: + friend ProvideFileSpec::Impl * zypp::rwcowClone( const ProvideFileSpec::Impl * rhs ); + Impl * clone() const { return new Impl( *this ); } + }; + + + ProvideMediaSpec::ProvideMediaSpec( const std::string &label, const zypp::filesystem::Pathname &verifyData, unsigned medianr ) + : _pimpl( new Impl( label, verifyData, medianr ) ) + { + + } + + const std::string &ProvideMediaSpec::label() const + { return _pimpl->_label; } + + ProvideMediaSpec &ProvideMediaSpec::setLabel(const std::string &label) + { + _pimpl->_label = label; + return *this; + } + + unsigned ProvideMediaSpec::medianr() const + { return _pimpl->_medianr; } + + ProvideMediaSpec &ProvideMediaSpec::setMedianr(unsigned medianr) + { + _pimpl->_medianr = medianr; + return *this; + } + + zypp::filesystem::Pathname ProvideMediaSpec::mediaFile() const + { return _pimpl->_verifyDataPath; } + + ProvideMediaSpec &ProvideMediaSpec::setMediaFile(const zypp::filesystem::Pathname &pName) + { + _pimpl->_verifyDataPath = pName; + return *this; + } + + HeaderValueMap &ProvideMediaSpec::customHeaders() + { return _pimpl->_customHeaders; } + + const HeaderValueMap &ProvideMediaSpec::customHeaders() const + { return _pimpl->_customHeaders; } + + ProvideMediaSpec &ProvideMediaSpec::setCustomHeaderValue(const std::string &key, const HeaderValueMap::Value &val) + { + _pimpl->_customHeaders.set( key,val ); + return *this; + } + + ProvideMediaSpec &ProvideMediaSpec::addCustomHeaderValue(const std::string &key, const HeaderValueMap::Value &val) + { + _pimpl->_customHeaders.add( key,val ); + return *this; + } + + zypp::TriBool ProvideMediaSpec::isSameMedium( const ProvideMediaSpec &other ) + { + // first check if we have the same media data + if ( _pimpl->_verifyDataPath != other._pimpl->_verifyDataPath ) + return false; + + // if the verify file is not empty check the medianr + if ( !_pimpl->_verifyDataPath.empty() ) { + return _pimpl->_medianr == other._pimpl->_medianr; + } + + // can't tell without the URL + return zypp::indeterminate; + } + + /** \relates ProvideSpec::Impl Stream output */ + inline std::ostream & operator<<( std::ostream & str, const ProvideFileSpec::Impl & obj ) + { + return str << "{" << obj._destFilenameHint << "{" << obj._downloadSize << "|" << obj._checksum << "|" << obj._deltafile << "}" << "}"; + } + + /** \relates ProvideSpec::Impl Verbose stream output */ + inline std::ostream & dumpOn( std::ostream & str, const ProvideFileSpec::Impl & obj ) + { return str << obj; } + + + ProvideFileSpec::ProvideFileSpec() + : _pimpl( new Impl() ) + {} + + ProvideFileSpec::ProvideFileSpec( const zypp::OnMediaLocation &loc ) + : _pimpl( new Impl() ) + { + setDownloadSize( loc.downloadSize() ); + setOptional( loc.optional() ); + setChecksum( loc.checksum() ); + setOpenSize( loc.openSize() ); + setOpenChecksum( loc.openChecksum() ); + setHeaderSize( loc.headerSize() ); + setHeaderChecksum( loc.headerChecksum() ); + setDeltafile( loc.deltafile() ); + } + + ProvideFileSpec::~ProvideFileSpec() + {} + + const zypp::filesystem::Pathname &ProvideFileSpec::destFilenameHint() const + { return _pimpl->_destFilenameHint; } + + ProvideFileSpec &ProvideFileSpec::setDestFilenameHint(const zypp::filesystem::Pathname &filename) + { _pimpl->_destFilenameHint = filename; return *this; } + + bool ProvideFileSpec::checkExistsOnly() const + { return _pimpl->_checkExistsOnly; } + + ProvideFileSpec &ProvideFileSpec::setCheckExistsOnly(const bool set) + { _pimpl->_checkExistsOnly = set; return *this; } + + bool ProvideFileSpec::optional() const + { return _pimpl->_optional; } + + ProvideFileSpec & ProvideFileSpec::setOptional( bool val_r ) + { _pimpl->_optional = (val_r); return *this; } + + const zypp::ByteCount & ProvideFileSpec::downloadSize() const + { return _pimpl->_downloadSize; } + + ProvideFileSpec & ProvideFileSpec::setDownloadSize( const zypp::ByteCount &val_r ) + { _pimpl->_downloadSize = (val_r); return *this; } + + const zypp::CheckSum & ProvideFileSpec::checksum() const + { return _pimpl->_checksum; } + + ProvideFileSpec & ProvideFileSpec::setChecksum( const zypp::CheckSum &val_r ) + { _pimpl->_checksum = (val_r); return *this; } + + const zypp::ByteCount & ProvideFileSpec::openSize() const + { return _pimpl->_openSize; } + + ProvideFileSpec & ProvideFileSpec::setOpenSize( const zypp::ByteCount &val_r ) + { _pimpl->_openSize = (val_r); return *this; } + + const zypp::CheckSum & ProvideFileSpec::openChecksum() const + { return _pimpl->_openChecksum; } + + ProvideFileSpec & ProvideFileSpec::setOpenChecksum( const zypp::CheckSum &val_r ) + { _pimpl->_openChecksum = (val_r); return *this; } + + const zypp::ByteCount & ProvideFileSpec::headerSize() const + { return _pimpl->_headerSize; } + + ProvideFileSpec & ProvideFileSpec::setHeaderSize( const zypp::ByteCount &val_r ) + { _pimpl->_headerSize = (val_r); return *this; } + + const zypp::CheckSum & ProvideFileSpec::headerChecksum() const + { return _pimpl->_headerChecksum; } + + ProvideFileSpec & ProvideFileSpec::setHeaderChecksum( const zypp::CheckSum &val_r ) + { _pimpl->_headerChecksum = (val_r); return *this; } + + const zypp::Pathname &ProvideFileSpec::deltafile() const + { return _pimpl->_deltafile; } + + ProvideFileSpec &ProvideFileSpec::setDeltafile( const zypp::Pathname &path ) + { _pimpl->_deltafile = (path); return *this; } + + HeaderValueMap &ProvideFileSpec::customHeaders() + { return _pimpl->_customHeaders; } + + const HeaderValueMap &ProvideFileSpec::customHeaders() const + { return _pimpl->_customHeaders; } + + ProvideFileSpec &ProvideFileSpec::setCustomHeaderValue(const std::string &key, const HeaderValueMap::Value &val) + { + _pimpl->_customHeaders.set( key,val ); + return *this; + } + + ProvideFileSpec &ProvideFileSpec::addCustomHeaderValue(const std::string &key, const HeaderValueMap::Value &val) + { + _pimpl->_customHeaders.add( key,val ); + return *this; + } + + std::ostream & operator<<( std::ostream & str, const ProvideFileSpec & obj ) + { return str << *obj._pimpl; } + + std::ostream & dumpOn( std::ostream & str, const ProvideFileSpec & obj ) + { return dumpOn( str, *obj._pimpl ); } + +} // namespace zypp diff --git a/zypp-media/ng/providespec.h b/zypp-media/ng/providespec.h new file mode 100644 index 0000000..c843f85 --- /dev/null +++ b/zypp-media/ng/providespec.h @@ -0,0 +1,198 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ +/** \file zypp/source/ProvideSpec.h + */ +#ifndef ZYPP_MEDIA_PROVIDESPEC_H_INCLUDED +#define ZYPP_MEDIA_PROVIDESPEC_H_INCLUDED + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace zyppng +{ + + class ProvideMediaSpec + { + public: + + ProvideMediaSpec( const std::string &label, const zypp::Pathname &verifyData = zypp::Pathname(), unsigned medianr = 1 ); + + /*! + * The label of the medium, this will be shown in case a media change is required + */ + const std::string &label() const; + + /*! + * Changes the label of the medium + */ + ProvideMediaSpec &setLabel( const std::string &label ); + + /*! + * The media number the resource is located on. + */ + unsigned medianr() const; + + /*! + * Individual manipulation of \c medianr (prefer \ref setLocation). + */ + ProvideMediaSpec & setMedianr( unsigned medianr ); + + /*! + * Returns the media validation file path. This can be empty if there + * is no file to validate. + */ + zypp::Pathname mediaFile () const; + + /*! + * Changes the media file to the name path \a pName. + */ + ProvideMediaSpec &setMediaFile( const zypp::Pathname &pName ); + + /*! + * Returns a map of custom key->value pairs that can control special aspects + * of how all files are provided by the medium after it was attached + */ + HeaderValueMap &customHeaders(); + const HeaderValueMap &customHeaders() const; + + /*! + * Set the custom header value identified by \a key to \a val + */ + ProvideMediaSpec &setCustomHeaderValue ( const std::string &key, const HeaderValueMap::Value &val ); + + /*! + * Adds the custom header value \a val to the list of values identified by \a key + */ + ProvideMediaSpec &addCustomHeaderValue ( const std::string &key, const HeaderValueMap::Value &val ); + + zypp::TriBool isSameMedium ( const ProvideMediaSpec &other ); + + public: + class Impl; ///< Implementation class. + private: + zypp::RWCOW_pointer _pimpl; ///< Pointer to implementation. + }; + + + class ProvideFileSpec + { + friend std::ostream & operator<<( std::ostream &str, const ProvideFileSpec &obj ); + friend std::ostream & dumpOn( std::ostream &str, const ProvideFileSpec &obj ); + + public: + ProvideFileSpec(); + + ProvideFileSpec( const zypp::OnMediaLocation &loc ); + + /** Dtor */ + ~ProvideFileSpec(); + + template + static ProvideFileSpecRef create ( T... args ) { + return std::make_shared( std::forward(args)... ); + } + + /*! + * The destination file name, this is optional and is only + * a hint to the \ref Provide instance where to put the file IF possible. The + * file provider does not need to consider this. + */ + const zypp::Pathname &destFilenameHint() const; + ProvideFileSpec &setDestFilenameHint ( const zypp::Pathname &filename ); + + bool checkExistsOnly () const; + ProvideFileSpec & setCheckExistsOnly( const bool set = true ); + + + /** Whether this is an optional resource. + * This is a hint to the downloader not to report an error if + * the resource is not present on the server. + */ + bool optional() const; + /** Set whether the resource is \ref optional. */ + ProvideFileSpec & setOptional( bool val ); + + /** The size of the resource on the server. */ + const zypp::ByteCount &downloadSize() const; + /** Set the \ref downloadSize. */ + ProvideFileSpec &setDownloadSize( const zypp::ByteCount &val_r ); + + /** The checksum of the resource on the server. */ + const zypp::CheckSum &checksum() const; + /** Set the \ref checksum. */ + ProvideFileSpec &setChecksum( const zypp::CheckSum &val_r ); + + + /** The size of the resource once it has been uncompressed or unpacked. */ + const zypp::ByteCount &openSize() const; + /** Set the \ref openSize. */ + ProvideFileSpec &setOpenSize( const zypp::ByteCount &val_r ); + + /** The checksum of the resource once it has been uncompressed or unpacked. */ + const zypp::CheckSum &openChecksum() const; + /** Set the \ref openChecksum. */ + ProvideFileSpec &setOpenChecksum( const zypp::CheckSum &val_r ); + + /** The size of the header prepending the resource (e.g. for zchunk). */ + const zypp::ByteCount &headerSize() const; + /** Set the \ref headerSize. */ + ProvideFileSpec &setHeaderSize( const zypp::ByteCount &val_r ); + + /** The checksum of the header prepending the resource (e.g. for zchunk). */ + const zypp::CheckSum &headerChecksum() const; + /** Set the \ref headerChecksum. */ + ProvideFileSpec &setHeaderChecksum( const zypp::CheckSum &val_r ); + + /** The existing deltafile that can be used to reduce download size ( zchunk or metalink ) */ + const zypp::Pathname &deltafile() const; + /** Set the \ref deltafile. */ + ProvideFileSpec &setDeltafile( const zypp::Pathname &path ); + + /*! + * Returns a map of custom key->value pairs that can control special aspects + * of how the provide operation is processed. + * + * @todo should this actually just be a list of pair(string,string) instead of map? -> easier way to send multiple values to a worker + */ + HeaderValueMap &customHeaders(); + const HeaderValueMap &customHeaders() const; + + /*! + * Set the custom header value identified by \a key to \a val + */ + ProvideFileSpec &setCustomHeaderValue ( const std::string &key, const HeaderValueMap::Value &val ); + + /*! + * Adds the custom header value \a val to the list of values identified by \a key + */ + ProvideFileSpec &addCustomHeaderValue ( const std::string &key, const HeaderValueMap::Value &val ); + + public: + class Impl; ///< Implementation class. + private: + zypp::RWCOW_pointer _pimpl; ///< Pointer to implementation. + }; + + /** \relates ProvideSpec Stream output */ + std::ostream & operator<<( std::ostream &str, const ProvideFileSpec &obj ); + + /** \relates ProvideSpec Verbose stream output */ + std::ostream & dumpOn( std::ostream &str, const ProvideFileSpec &obj ); + +} // namespace zypp + +#endif // ZYPP_MEDIA_PROVIDESPEC_H_INCLUDED diff --git a/zypp-media/ng/worker/DeviceDriver b/zypp-media/ng/worker/DeviceDriver new file mode 100644 index 0000000..4942597 --- /dev/null +++ b/zypp-media/ng/worker/DeviceDriver @@ -0,0 +1 @@ +#include "devicedriver.h" diff --git a/zypp-media/ng/worker/MountingWorker b/zypp-media/ng/worker/MountingWorker new file mode 100644 index 0000000..2c46378 --- /dev/null +++ b/zypp-media/ng/worker/MountingWorker @@ -0,0 +1 @@ +#include "mountingworker.h" diff --git a/zypp-media/ng/worker/ProvideWorker b/zypp-media/ng/worker/ProvideWorker new file mode 100644 index 0000000..7d39856 --- /dev/null +++ b/zypp-media/ng/worker/ProvideWorker @@ -0,0 +1 @@ +#include "provideworker.h" diff --git a/zypp-media/ng/worker/devicedriver.cc b/zypp-media/ng/worker/devicedriver.cc new file mode 100644 index 0000000..f926133 --- /dev/null +++ b/zypp-media/ng/worker/devicedriver.cc @@ -0,0 +1,382 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ + +#include "devicedriver.h" +#include +#include +#include +#include +#include +#include +#include + +#undef ZYPP_BASE_LOGGER_LOGGROUP +#define ZYPP_BASE_LOGGER_LOGGROUP "zyppng::worker::DeviceDriver" + +namespace zyppng::worker +{ + + DeviceDriver::DeviceDriver ( WorkerCaps::WorkerType wType ) + : _wType( wType ) + { } + + void DeviceDriver::setProvider ( ProvideWorkerWeakRef workerRef ) + { + _parentWorker = workerRef; + } + + zyppng::expected DeviceDriver::initialize(const zyppng::worker::Configuration &conf) + { + const auto &values = conf.values(); + if ( const auto &i = values.find( std::string(zyppng::ATTACH_POINT) ); i != values.end() ) { + const auto &val = i->second; + MIL << "Got attachpoint from controller: " << val << std::endl; + _attachRoot = zypp::Pathname(val).realpath(); + } else { + return zyppng::expected::error(ZYPP_EXCPT_PTR( zypp::Exception("Attach point required to work.") )); + } + + _config = conf; + + zyppng::worker::WorkerCaps caps; + caps.set_worker_type ( _wType ); + caps.set_cfg_flags( + zyppng::worker::WorkerCaps::Flags ( + zyppng::worker::WorkerCaps::Pipeline + | zyppng::worker::WorkerCaps::ZyppLogFormat + | zyppng::worker::WorkerCaps::SingleInstance + ) + ); + + return zyppng::expected::success(caps); + } + + bool DeviceDriver::detachMedia ( const std::string &attachId ) + { + auto i = _attachedMedia.find( attachId ); + if ( i == _attachedMedia.end() ) + return false; + + _attachedMedia.erase(i); + return true; + } + + void DeviceDriver::releaseIdleDevices () + { + for ( auto i = _sysDevs.begin (); i != _sysDevs.end(); ) { + if ( i->use_count() == 1 && !(*i)->_mountPoint.empty() ) { + MIL << "Unmounting device " << (*i)->_name << " since its not used anymore" << std::endl; + unmountDevice(*(*i)); + if ( (*i)->_ephemeral ) { + i = _sysDevs.erase(i); + continue; + } + } + ++i; + } + } + + void DeviceDriver::detectDevices() + { + return; + } + + std::vector> &DeviceDriver::knownDevices() + { + return _sysDevs; + } + + const std::vector> &DeviceDriver::knownDevices() const + { + return _sysDevs; + } + + std::unordered_map &DeviceDriver::attachedMedia() + { + return _attachedMedia; + } + + void DeviceDriver::immediateShutdown() + { + // here we need to unmount everything + for ( auto i = _sysDevs.begin (); i != _sysDevs.end(); ) { + unmountDevice(*(*i)); + if ( (*i)->_ephemeral ) { + i = _sysDevs.erase(i); + continue; + } + ++i; + } + _attachedMedia.clear(); + } + + ProvideWorkerRef DeviceDriver::parentWorker () const + { + return _parentWorker.lock(); + } + + void DeviceDriver::unmountDevice ( Device &dev ) + { + if ( dev._mountPoint.empty () ) + return; + try { + zypp::media::Mount mount; + mount.umount( dev._mountPoint.asString() ); + removeAttachPoint( dev._mountPoint ); + } catch (const zypp::media::MediaException & excpt_r) { + ERR << "Failed to unmount device: " << dev._name << std::endl; + ZYPP_CAUGHT(excpt_r); + } + dev._mountPoint = zypp::Pathname(); + } + + bool DeviceDriver::isVolatile () const + { + return false; + } + + void DeviceDriver::setAttachRoot ( const zypp::Pathname &root ) + { + _attachRoot = root; + } + + zypp::Pathname DeviceDriver::attachRoot () const + { + if ( _attachRoot.empty() ) { + MIL << "Attach root is empty" << std::endl; + return zypp::Pathname(".").realpath(); + } + return _attachRoot; + } + + const zyppng::worker::Configuration &DeviceDriver::config() const + { + return _config; + } + + zyppng::expected DeviceDriver::isDesiredMedium ( const zypp::Url &deviceUrl, const zypp::Pathname &mountPoint, const zyppng::MediaDataVerifierRef &verifier, uint mediaNr ) + { + if ( !verifier ) { + // at least the requested path must exist on the medium + zypp::PathInfo p( mountPoint ); + if ( p.isExist() && p.isDir() ) + return zyppng::expected::success(); // we have no valid data + return zyppng::expected::error( ZYPP_EXCPT_PTR( zypp::media::MediaNotDesiredException( deviceUrl ) ) ); + } + + auto devVerifier = verifier->clone(); + if ( !devVerifier ) { + // unlikely to happen + return zyppng::expected::error( ZYPP_EXCPT_PTR( zypp::Exception("Failed to clone verifier") ) ); + } + + // bsc#1180851: If there is just one not-volatile medium in the set + // tolerate a missing (vanished) media identifier and let the URL rule. + bool relaxed = verifier->totalMedia() == 1 && !isVolatile(); + + const auto &relMediaPath = devVerifier->mediaFilePath( mediaNr ); + zypp::Pathname mediaFile { mountPoint / relMediaPath }; + zypp::PathInfo pi( mediaFile ); + if ( !pi.isExist() ) { + if ( relaxed ) + return zyppng::expected::success(); + auto excpt = zypp::media::MediaFileNotFoundException( deviceUrl, relMediaPath ) ; + excpt.addHistory( verifier->expectedAsUserString( mediaNr ) ); + return zyppng::expected::error( ZYPP_EXCPT_PTR( std::move(excpt) ) ); + } + if ( !pi.isFile() ) { + if ( relaxed ) + return zyppng::expected::success(); + auto excpt = zypp::media::MediaNotAFileException( deviceUrl, relMediaPath ) ; + excpt.addHistory( verifier->expectedAsUserString( mediaNr ) ); + return zyppng::expected::error( ZYPP_EXCPT_PTR( std::move(excpt) ) ); + } + + if ( !devVerifier->load( mediaFile ) ) { + return zyppng::expected::error( ZYPP_EXCPT_PTR( zypp::Exception("Failed to load media information from medium") ) ); + } + if ( !verifier->matches( devVerifier ) ) { + return zyppng::expected::error( ZYPP_EXCPT_PTR( zypp::media::MediaNotDesiredException( deviceUrl ) ) ); + } + return zyppng::expected::success(); + } + + zypp::Pathname DeviceDriver::createAttachPoint( const zypp::Pathname &attach_root ) const + { + zypp::Pathname apoint; + + if( attach_root.empty() || !attach_root.absolute()) { + ERR << "Create attach point: invalid attach root: '" + << attach_root << "'" << std::endl; + return apoint; + } + + zypp::PathInfo adir( attach_root ); + if( !adir.isDir() || (geteuid() != 0 && !adir.userMayRWX())) { + DBG << "Create attach point: attach root is not a writable directory: '" + << attach_root << "'" << std::endl; + return apoint; + } + + static bool cleanup_once( true ); + if ( cleanup_once ) + { + cleanup_once = false; + DBG << "Look for orphaned attach points in " << adir << std::endl; + std::list entries; + zypp::filesystem::readdir( entries, attach_root, false ); + for ( const std::string & entry : entries ) + { + if ( ! zypp::str::hasPrefix( entry, "AP_0x" ) ) + continue; + zypp::PathInfo sdir( attach_root + entry ); + if ( sdir.isDir() + && sdir.dev() == adir.dev() + && ( zypp::Date::now()-sdir.mtime() > zypp::Date::month ) ) + { + DBG << "Remove orphaned attach point " << sdir << std::endl; + zypp::filesystem::recursive_rmdir( sdir.path() ); + } + } + } + + zypp::filesystem::TmpDir tmpdir( attach_root, "AP_0x" ); + if ( tmpdir ) + { + apoint = tmpdir.path().asString(); + if ( ! apoint.empty() ) + { + tmpdir.autoCleanup( false ); // Take responsibility for cleanup. + } + else + { + ERR << "Unable to resolve real path for attach point " << tmpdir << std::endl; + } + } + else + { + ERR << "Unable to create attach point below " << attach_root << std::endl; + } + return apoint; + } + + void DeviceDriver::removeAttachPoint( const zypp::filesystem::Pathname &attachRoot ) const + { + if( !attachRoot.empty() && + zypp::PathInfo(attachRoot).isDir() && + attachRoot != "/" ) { + int res = recursive_rmdir( attachRoot ); + if ( res == 0 ) { + MIL << "Deleted default attach point " << attachRoot << std::endl; + } else { + ERR << "Failed to Delete default attach point " << attachRoot + << " errno(" << res << ")" << std::endl; + } + } + } + + bool DeviceDriver::checkAttached ( const zypp::filesystem::Pathname &mountPoint, const std::function predicate ) + { + bool isAttached = false; + time_t old_mtime = _attach_mtime; + _attach_mtime = zypp::media::Mount::getMTime(); + if( !(old_mtime <= 0 || _attach_mtime != old_mtime) ) { + // OK, skip the check (we've seen it at least once) + isAttached = true; + } else { + if( old_mtime > 0) + DBG << "Mount table changed - rereading it" << std::endl; + else + DBG << "Forced check of the mount table" << std::endl; + + for( const auto &entry : zypp::media::Mount::getEntries() ) { + + if ( mountPoint != zypp::Pathname(entry.dir) ) + continue; // at least the mount points must match + if ( predicate(entry) ) { + isAttached = true; + break; + } + } + } + + // force recheck + if ( !isAttached ) + _attach_mtime = 0; + + return isAttached; + } + + const std::function DeviceDriver::devicePredicate( unsigned int majNr, unsigned int minNr ) + { + return [ majNr, minNr ]( const zypp::media::MountEntry &entry ) -> bool { + if( entry.isBlockDevice() ) { + zypp::PathInfo dev_info( entry.src ); + if ( dev_info.devMajor () == majNr && dev_info.devMinor () == minNr ) { + DBG << "Found device " + << majNr << ":" << minNr + << " in the mount table as " << entry.src << std::endl; + return true; + } + } + return false; + }; + } + + const std::function DeviceDriver::fstypePredicate( const std::string &src, const std::vector &fstypes ) + { + return [ srcdev=src, fst=fstypes ]( const zypp::media::MountEntry &entry ) -> bool { + if( !entry.isBlockDevice() ) { + if ( std::find( fst.begin(), fst.end(), entry.type ) != fst.end() ) { + if ( srcdev == entry.src ) { + DBG << "Found media mount" + << " in the mount table as " << entry.src << std::endl; + return true; + } + } + } + return false; + }; + } + + const std::function DeviceDriver::bindMountPredicate( const std::string &src ) + { + return [ srcdev=src ]( const zypp::media::MountEntry &entry ) -> bool { + if( !entry.isBlockDevice() ) { + if ( srcdev == entry.src ) { + DBG << "Found bound media " + << " in the mount table as " << entry.src << std::endl; + return true; + } + } + return false; + }; + } + + AttachError::AttachError ( const uint code, const std::string &reason, const bool transient, const HeaderValueMap &extra) + : _code( code ), + _reason( reason ), + _transient( transient ), + _extra( extra ) + { + + } + + AttachError::AttachError ( const uint code, const bool transient, const zypp::Exception &e ) + : _code( code ), + _reason( e.asUserString() ), + _transient( transient ) + { + if ( !e.historyEmpty() ) { + _extra = { { std::string(zyppng::ErrMsgFields::History), { zyppng::HeaderValueMap::Value(e.historyAsString()) }} }; + } + } + + +} diff --git a/zypp-media/ng/worker/devicedriver.h b/zypp-media/ng/worker/devicedriver.h new file mode 100644 index 0000000..abcd0ef --- /dev/null +++ b/zypp-media/ng/worker/devicedriver.h @@ -0,0 +1,201 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ + +#ifndef ZYPP_MEDIA_NG_WORKER_DEVICEDRIVER_H_INCLUDED +#define ZYPP_MEDIA_NG_WORKER_DEVICEDRIVER_H_INCLUDED + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace zyppng::worker +{ + + ZYPP_FWD_DECL_TYPE_WITH_REFS (DeviceDriver); + + struct Device + { + std::string _name; //!< Path of the device node or URL for e.g. nfs devices + unsigned int _maj_nr = 0; //!< Major number of the device + unsigned int _min_nr = 0; //!< Minor number of the device + zypp::Pathname _mountPoint = {};//!< Mountpoint of the device, if empty dev is not mounted + bool _ephemeral = false; //!< If set to true the device is removed from the internal list after the last attachpoint was released + std::unordered_map _properties = {}; + }; + + struct AttachedMedia + { + std::shared_ptr _dev; + zypp::Pathname _attachRoot; + }; + + struct AttachError + { + AttachError ( const uint code, const std::string &reason, const bool transient, const HeaderValueMap &extra = {} ); + AttachError ( const uint code, const bool transient, const zypp::Exception &e ); + + uint _code; + std::string _reason; + bool _transient; + HeaderValueMap _extra; + }; + + using AttachResult = expected; + + /*! + * Abstract base class to be used together with the \sa MountingWorker class to control + * attaching and detaching of a multitude of different media types via a unified interface. + * Reimplement this class to easily support a new backend that utilizes the "mount" command to + * attach real filesystems to the system. + */ + class DeviceDriver : public zyppng::Base + { + public: + + DeviceDriver ( WorkerCaps::WorkerType wType ); + + /*! + * Tells the driver which provide worker to use when requiring auth data or media changes + */ + void setProvider ( ProvideWorkerWeakRef workerRef ); + + /*! + * Called by the provide loop whenever a attach request is received. + */ + virtual AttachResult mountDevice ( const uint32_t id, const zypp::Url &mediaUrl, const std::string &attachId, const std::string &label, const HeaderValueMap &extras ) = 0; + + + + virtual zyppng::expected initialize(const zyppng::worker::Configuration &conf); + + + /*! + * Detaches the medium referenced by \a attachId. + * Returns false if the medium could not be found + * + * \note this will not unmount the physical devices. + * Call \ref releaseIdleDevices to force this. + * + */ + bool detachMedia ( const std::string &attachId ); + + /*! + * Physically detaches all devices that are not referenced by a attachment anymore + */ + void releaseIdleDevices (); + + /*! + * Called by the parent to populate the device array every time a provide request arrives, only + * relevant for workers that operate on detectable devices. + * The base implementation does nothing + */ + virtual void detectDevices(); + + /*! + * Returns a list of all currently known devices, a subclass should always make sure + * that all currently mounted devices are present in that list. + */ + std::vector> &knownDevices(); + + /*! + * Returns a list of all currently known devices, a subclass should always make sure + * that all currently mounted devices are present in that list. + */ + const std::vector> &knownDevices() const; + + /*! + * Returns the list of currently attached medias + */ + std::unordered_map &attachedMedia(); + + /*! + * Returns true if the worker handles volatile devices ( e.g. DVDs ). + * The default impl returns false. + */ + virtual bool isVolatile () const; + + /*! + * Changes the attach root to a specific path, otherwise realpath(".") is used. + */ + void setAttachRoot ( const zypp::Pathname &root ); + + /*! + * Returns the \a attachRoot as dictated by the controller + */ + zypp::Pathname attachRoot () const; + + /*! + * The system is shutting down, release all ressources + */ + virtual void immediateShutdown(); + + /*! + * Returns the parent worker if set + */ + ProvideWorkerRef parentWorker () const; + + /*! + * Returns the configuration that was sent by the controller + */ + const zyppng::worker::Configuration &config() const; + + protected: + /*! + * Forcefully unmounts the device, this does not check if there any attached medias still relying on it + */ + virtual void unmountDevice ( Device &dev ); + + /*! + * Checks if the medium \a deviceUrl mounted on \a path matches the \a verifier and \a mediaNr + */ + zyppng::expected isDesiredMedium ( const zypp::Url &deviceUrl, const zypp::Pathname &mountPoint, const zyppng::MediaDataVerifierRef &verifier, uint mediaNr = 1 ); + + + zypp::Pathname createAttachPoint(const zypp::Pathname &attach_root) const; + void removeAttachPoint ( const zypp::Pathname &attach_pt ) const; + bool checkAttached ( const zypp::Pathname &mountPoint, const std::function predicate ); + + /*! + * Returns a predicate for the \ref checkAttached function that looks for a real device in the mount table + */ + static const std::function devicePredicate ( unsigned int majNr, unsigned int minNr ); + + /*! + * Returns a predicate for the \ref checkAttached function that looks for a virtual mount ( like smb or nfs ) in the mount table + */ + static const std::function fstypePredicate ( const std::string &src, const std::vector &fstypes ); + + /*! + * Returns a predicate for the \ref checkAttached function that looks for a bind mount in the mount table + */ + static const std::function bindMountPredicate ( const std::string &src ); + + private: + WorkerCaps::WorkerType _wType; + zyppng::worker::Configuration _config; + time_t _attach_mtime = 0; //< Timestamp of the mtab we did read + zypp::Pathname _attachRoot; + std::vector> _sysDevs; + std::unordered_map _attachedMedia; + ProvideWorkerWeakRef _parentWorker; + }; + + + + +} + +#endif diff --git a/zypp-media/ng/worker/mountingworker.cc b/zypp-media/ng/worker/mountingworker.cc new file mode 100644 index 0000000..4624573 --- /dev/null +++ b/zypp-media/ng/worker/mountingworker.cc @@ -0,0 +1,191 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ +#include "mountingworker.h" +#include +#include +#include + +#undef ZYPP_BASE_LOGGER_LOGGROUP +#define ZYPP_BASE_LOGGER_LOGGROUP "MountingWorker" + +namespace zyppng::worker +{ + + MountingWorker::MountingWorker( std::string_view workerName, DeviceDriverRef driver ) + : ProvideWorker( workerName ) + , _driver(driver) + { } + + MountingWorker::~MountingWorker() + { + _driver->immediateShutdown(); + } + + zyppng::expected MountingWorker::initialize( const zyppng::worker::Configuration &conf ) + { + return _driver->initialize(conf); + } + + void MountingWorker::provide() + { + _driver->detectDevices(); + auto &queue = requestQueue(); + + if ( !queue.size() ) + return; + + auto req = queue.front(); + queue.pop_front(); + + MIL_PRV << "Received provide: " << req->_spec.code() << std::endl; + + try { + switch ( req->_spec.code () ) { + case zyppng::ProvideMessage::Code::Attach: { + + const auto attachUrl = zypp::Url( req->_spec.value( zyppng::AttachMsgFields::Url ).asString() ); + const auto label = req->_spec.value( zyppng::AttachMsgFields::Label, "No label" ).asString(); + const auto attachId = req->_spec.value( zyppng::AttachMsgFields::AttachId ).asString(); + HeaderValueMap vals; + req ->_spec.forEachVal([&]( const std::string &name, const auto &val ) { + if ( name == zyppng::AttachMsgFields::Url + || name == zyppng::AttachMsgFields::Label + || name == zyppng::AttachMsgFields::AttachId ) + return true; + vals.add( name, val ); + return true; + }); + + const auto &res = _driver->mountDevice( req->_spec.requestId(), attachUrl, attachId, label, vals ); + if ( !res ) { + const auto &err = res.error(); + req->_state = zyppng::worker::ProvideWorkerItem::Finished; + provideFailed( req->_spec.requestId() + , err._code + , err._reason + , err._transient + , err._extra ); + return; + } + + MIL << "Attach of " << attachUrl << " was successfull" << std::endl; + attachSuccess( req->_spec.requestId() ); + return; + } + case zyppng::ProvideMessage::Code::Detach: { + + const auto url = zypp::Url( req->_spec.value( zyppng::DetachMsgFields::Url ).asString() ); + const auto &attachId = url.getAuthority(); + + if ( _driver->detachMedia( attachId ) ) { + detachSuccess ( req->_spec.requestId() ); + } else { + provideFailed( req->_spec.requestId() + , zyppng::ProvideMessage::Code::NotFound + , "Attach ID not known." + , false + , {} ); + return; + } + + _driver->releaseIdleDevices(); + return; + } + + case zyppng::ProvideMessage::Code::Provide: { + + const auto url = zypp::Url( req->_spec.value( zyppng::DetachMsgFields::Url ).asString() ); + const auto &attachId = url.getAuthority(); + const auto &path = zypp::Pathname(url.getPathName()); + const auto &availMedia = _driver->attachedMedia(); + + auto i = availMedia.find( attachId ); + if ( i == availMedia.end() ) { + ERR << "Unknown Attach ID " << attachId << std::endl; + provideFailed( req->_spec.requestId() + , zyppng::ProvideMessage::Code::NotFound + , "Attach ID not known." + , false + , {} ); + return; + } + + const auto &locPath = i->second._dev->_mountPoint / i->second._attachRoot / path; + + MIL << "Trying to find file: " << locPath << std::endl; + + zypp::PathInfo info( locPath ); + if( info.isFile() ) { + provideSuccess ( req->_spec.requestId(), false, locPath ); + return; + } + + if (info.isExist()) + provideFailed( req->_spec.requestId() + , zyppng::ProvideMessage::Code::NotAFile + , zypp::str::Str() << "Path " << path << " exists, but its not a file" + , false + , {} ); + else + provideFailed( req->_spec.requestId() + , zyppng::ProvideMessage::Code::NotFound + , zypp::str::Str() << "File " << path << " not found on medium" + , false + , {} ); + + + break; + } + default: { + req->_state = zyppng::worker::ProvideWorkerItem::Finished; + provideFailed( req->_spec.requestId() + , zyppng::ProvideMessage::Code::BadRequest + , "Request type not implemented" + , false + , {} ); + return; + } + } + } catch ( const zypp::Exception &e ) { + req->_state = zyppng::worker::ProvideWorkerItem::Finished; + provideFailed( req->_spec.requestId() + , zyppng::ProvideMessage::Code::BadRequest + , e.asString() + , false + , {} ); + return; + } catch ( const std::exception &e ) { + req->_state = zyppng::worker::ProvideWorkerItem::Finished; + provideFailed( req->_spec.requestId() + , zyppng::ProvideMessage::Code::BadRequest + , e.what() + , false + , {} ); + return; + } catch ( ... ) { + req->_state = zyppng::worker::ProvideWorkerItem::Finished; + provideFailed( req->_spec.requestId() + , zyppng::ProvideMessage::Code::BadRequest + , "Unknown exception" + , false + , {} ); + return; + } + } + + void MountingWorker::cancel( const std::deque::iterator &i ) + { + ERR << "Bug, cancel should never be called for running items" << std::endl; + } + + void MountingWorker::immediateShutdown() + { + _driver->immediateShutdown(); + } +} diff --git a/zypp-media/ng/worker/mountingworker.h b/zypp-media/ng/worker/mountingworker.h new file mode 100644 index 0000000..f200d13 --- /dev/null +++ b/zypp-media/ng/worker/mountingworker.h @@ -0,0 +1,41 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ + +#ifndef ZYPP_MEDIA_NG_WORKER_MOUNTINGWORKER_H_INCLUDED +#define ZYPP_MEDIA_NG_WORKER_MOUNTINGWORKER_H_INCLUDED + +#include +#include +#include +#include +#include + +namespace zyppng::worker +{ + class MountingWorker : public zyppng::worker::ProvideWorker + { + public: + MountingWorker( std::string_view workerName, DeviceDriverRef driver ); + ~MountingWorker(); + + void immediateShutdown() override; + + protected: + // ProvideWorker interface + zyppng::expected initialize(const zyppng::worker::Configuration &conf) override; + void provide() override; + void cancel( const std::deque::iterator &i ) override; + + private: + DeviceDriverRef _driver; + bool _devicesDetected = false; //< We delay device detection to the first attach request, to avoid doing it without needing it + }; +} + +#endif diff --git a/zypp-media/ng/worker/provideworker.cc b/zypp-media/ng/worker/provideworker.cc new file mode 100644 index 0000000..3052c28 --- /dev/null +++ b/zypp-media/ng/worker/provideworker.cc @@ -0,0 +1,469 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ + +#include "provideworker.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#undef ZYPP_BASE_LOGGER_LOGGROUP +#define ZYPP_BASE_LOGGER_LOGGROUP "ProvideWorker" + +namespace zyppng::worker { + + using namespace zyppng::operators; + + RequestCancelException::RequestCancelException() : zypp::media::MediaException ("Request was cancelled") + { } + + ProvideWorker::ProvideWorker(std::string_view workerName) : _workerName(workerName) + { + // do not change the order of these calls, otherwise showing the threadname does not work + // enableLogForwardingMode will initialize the log which would override the current thread name + zypp::base::LogControl::instance().enableLogForwardingMode( true ); + ThreadData::current().setName( workerName ); + + // we use a singleshot timer that triggers message handling + connect( *_msgAvail, &Timer::sigExpired, *this, &ProvideWorker::messageLoop ); + _msgAvail->setSingleShot(true); + + // another timer to trigger a delayed shutdown + connectFunc( *_delayedShutdown, &Timer::sigExpired, [this]( zyppng::Timer & ) { + maybeDelayedShutdown(); + }, *this ); + _delayedShutdown->setSingleShot(true); + } + + ProvideWorker::~ProvideWorker() + { } + + RpcMessageStream::Ptr ProvideWorker::messageStream() const + { + return _stream; + } + + expected ProvideWorker::run( int recv, int send ) + { + // reentry not supported + assert ( !_isRunning ); + + zypp::DtorReset res( _isRunning ); + _isRunning = true; + + initLog(); + + zypp::OnScopeExit cleanup([&](){ + _stream.reset(); + _controlIO.reset(); + _loop.reset(); + }); + + _controlIO = AsyncDataSource::create(); + if ( !_controlIO->openFds( { recv }, send ) ) { + return expected::error( ZYPP_EXCPT_PTR(zypp::Exception("Failed to open control FDs")) ); + } + + connect( *_controlIO, &AsyncDataSource::sigReadFdClosed, *this, &ProvideWorker::readFdClosed ); + connect( *_controlIO, &AsyncDataSource::sigWriteFdClosed, *this, &ProvideWorker::writeFdClosed ); + + _stream = RpcMessageStream::create( _controlIO ); + + return executeHandshake () | mbind( [&]() { + AutoDisconnect disC[] = { + connect( *_stream, &RpcMessageStream::sigMessageReceived, *this, &ProvideWorker::messageReceived ), + connect( *_stream, &RpcMessageStream::sigInvalidMessageReceived, *this, &ProvideWorker::onInvalidMessageReceived ) + }; + _loop->run(); + if ( _fatalError ) + return expected::error( _fatalError ); + return expected::success(); + }); + } + + std::deque &ProvideWorker::requestQueue() + { + return _pendingProvides; + } + + ProvideWorker::ProvideNotificatioMode ProvideWorker::provNotificationMode() const { + return _provNotificationMode; + } + + void ProvideWorker::setProvNotificationMode( const ProvideNotificatioMode &provNotificationMode ) { + _provNotificationMode = provNotificationMode; + } + + void ProvideWorker::initLog() + { + // by default we log to strErr, if user code wants to change that it can overload this function + zypp::base::LogControl::instance().logToStdErr(); + } + + ProvideWorkerItemRef ProvideWorker::makeItem( ProvideMessage &&spec ) + { + return std::make_shared( std::move(spec) ); + } + + void ProvideWorker::provideStart(const uint32_t id, const zypp::Url &url, const zypp::filesystem::Pathname &localFile, const zypp::Pathname &stagingFile ) + { + if ( !_stream->sendMessage( ProvideMessage::createProvideStarted ( id + , url + , localFile.empty () ? std::optional() : localFile.asString () + , stagingFile.empty () ? std::optional() : stagingFile.asString () + ).impl() ) ) { + ERR << "Failed to send ProvideStart message" << std::endl; + } + } + + void ProvideWorker::provideSuccess(const uint32_t id, bool cacheHit, const zypp::filesystem::Pathname &localFile, const HeaderValueMap extra ) + { + MIL_PRV << "Sending provideSuccess for id " << id << " file " << localFile << std::endl; + auto msg = ProvideMessage::createProvideFinished( id ,localFile.asString() ,cacheHit); + for ( auto i = extra.beginList (); i != extra.endList(); i++ ) { + for ( const auto &val : i->second ) + msg.addValue( i->first, val ); + } + if ( !_stream->sendMessage( msg.impl() ) ) { + ERR << "Failed to send ProvideSuccess message" << std::endl; + } + } + + void ProvideWorker::provideFailed(const uint32_t id, const uint code, const std::string &reason, const bool transient, const HeaderValueMap extra ) + { + MIL_PRV << "Sending provideFailed for request " << id << " err: " << reason << std::endl; + auto msg = ProvideMessage::createErrorResponse ( id, code, reason, transient ); + for ( auto i = extra.beginList (); i != extra.endList(); i++ ) { + for ( const auto &val : i->second ) + msg.addValue( i->first, val ); + } + if ( !_stream->sendMessage( msg.impl() ) ) { + ERR << "Failed to send ProvideFailed message" << std::endl; + } + } + + + void ProvideWorker::provideFailed ( const uint32_t id, const uint code, const bool transient, const zypp::Exception &e ) + { + zyppng::HeaderValueMap extra; + if ( !e.historyEmpty() ) { + extra = { { std::string(zyppng::ErrMsgFields::History), { zyppng::HeaderValueMap::Value(e.historyAsString()) }} }; + } + provideFailed( id + , code + , e.asUserString() + , transient + , extra ); + } + + + void ProvideWorker::attachSuccess(const uint32_t id) + { + MIL_PRV << "Sending attachSuccess for request " << id << std::endl; + if ( !_stream->sendMessage( ProvideMessage::createAttachFinished ( id ).impl() ) ) { + ERR << "Failed to send AttachFinished message" << std::endl; + } else { + MIL << "Sent back attach success" << std::endl; + } + } + + void ProvideWorker::detachSuccess(const uint32_t id) + { + MIL_PRV << "Sending detachSuccess for request " << id << std::endl; + if ( !_stream->sendMessage( ProvideMessage::createDetachFinished ( id ).impl() ) ) { + ERR << "Failed to send DetachFinished message" << std::endl; + } + } + + expected ProvideWorker::sendAndWaitForResponse( const ProvideMessage &request , const std::vector &responseCodes ) + { + // make sure immediateShutdown is not called while we are blocking here + zypp::DtorReset delayedReset( _inControllerRequest ); + _inControllerRequest = true; + + if ( !_stream->sendMessage( request.impl() ) ) + return expected::error( ZYPP_EXCPT_PTR(zypp::Exception("Failed to send message")) ); + + // flush the io device, this will block until all bytes are written + _controlIO->flush(); + + while ( !_fatalError ) { + + const auto &msg = _stream->nextMessageWait() | [&]( auto &&nextMessage ) { + if ( !nextMessage ) { + if ( _fatalError ) + return expected::error( _fatalError ); + else + return expected::error( ZYPP_EXCPT_PTR(zypp::Exception("Failed to wait for response")) ); + } + return expected::success( std::move(*nextMessage) ); + } | mbind ( [&]( auto && m) { + return parseReceivedMessage(m); + } ); + + if ( !msg ) { + ERR << "Failed to receive message" << std::endl; + return msg; + } + + if ( std::find( responseCodes.begin (), responseCodes.end(), msg->code() ) != responseCodes.end() ) { + return msg; + } + + // remember other messages for later + MIL << "Remembering message for later: " << msg->code () << std::endl; + _pendingMessages.push_back(*msg); + _msgAvail->start(0); + } + return expected::error( _fatalError ); + } + + ProvideWorker::MediaChangeRes ProvideWorker::requestMediaChange(const uint32_t id, const std::string &label, const int32_t mediaNr, const std::vector &devices, const std::optional &desc ) + { + return sendAndWaitForResponse( ProvideMessage::createMediaChangeRequest ( id, label, mediaNr, devices, desc ), { ProvideMessage::Code::MediaChanged, ProvideMessage::Code::MediaChangeAbort, ProvideMessage::Code::MediaChangeSkip } ) + | [&]( expected &&m ) { + if ( !m ) { + MIL << "Failed to wait for message, aborting the request " << std::endl; + return ProvideWorker::MediaChangeRes::ABORT; + } + MIL << "Wait finished, with messages still pending: " << this->_pendingMessages.size() << " and provs still pending: " << this->_pendingProvides.size() << std::endl; + if ( m->code() == ProvideMessage::Code::MediaChanged ) + return ProvideWorker::MediaChangeRes::SUCCESS; + else if ( m->code() == ProvideMessage::Code::MediaChangeSkip ) + return ProvideWorker::MediaChangeRes::SKIP; + else + return ProvideWorker::MediaChangeRes::ABORT; + }; + } + + expected ProvideWorker::requireAuthorization( const uint32_t id, const zypp::Url &url, const std::string &lastTriedUsername, const int64_t lastTimestamp, const std::map &extraFields ) + { + return sendAndWaitForResponse( ProvideMessage::createAuthDataRequest( id, url, lastTriedUsername, lastTimestamp, extraFields ), { ProvideMessage::Code::AuthInfo, ProvideMessage::Code::NoAuthData } ) + | mbind( [&]( ProvideMessage &&m ) { + if ( m.code() == ProvideMessage::Code::AuthInfo ) { + + AuthInfo inf; + m.forEachVal( [&]( const std::string &name, const ProvideMessage::FieldVal &val ) { + if ( name == AuthInfoMsgFields::Username ) { + inf.username = val.asString(); + } else if ( name == AuthInfoMsgFields::Password ) { + inf.password = val.asString(); + } else if ( name == AuthInfoMsgFields::AuthTimestamp ) { + inf.last_auth_timestamp = val.asInt64(); + } else { + if ( !val.isString() ) { + ERR << "Ignoring invalid extra value, " << name << " is not of type string" << std::endl; + } + inf.extraKeys[name] = val.asString(); + } + return true; + }); + return expected::success(inf); + + } + return expected::error( ZYPP_EXCPT_PTR( zypp::media::MediaException("No Auth data")) ); + }); + } + + AsyncDataSource &ProvideWorker::controlIO() + { + return *_controlIO.get(); + } + + expected ProvideWorker::executeHandshake() + { + const auto &helo = _stream->nextMessageWait(); + if ( !helo ) { + ERR << "Could not receive a handshake message, aborting" << std::endl; + return expected::error( ZYPP_EXCPT_PTR(zypp::Exception("Failed to receive handshake message")) );; + } + + auto exp = _stream->parseMessage( *helo ); + if ( !exp ) { + invalidMessageReceived( exp.error() ); + return expected::error(exp.error()); + } + + return std::move(*exp) | [&]( auto &&conf ) { + + _workerConf = std::move(conf); + + auto &mediaConf = zypp::MediaConfig::instance(); + for( const auto &[key,value] : _workerConf.values() ) { + zypp::Url keyUrl( key ); + if ( keyUrl.getScheme() == "zconfig" && keyUrl.getAuthority() == "main" ) { + mediaConf.setConfigValue( keyUrl.getAuthority(), zypp::Pathname(keyUrl.getPathName()).basename(), value ); + } + } + + return initialize( _workerConf ) | mbind([&]( WorkerCaps &&caps ){ + + caps.set_worker_name( _workerName.data() ); + + caps.set_cfg_flags ( WorkerCaps::Flags(caps.cfg_flags() | WorkerCaps::ZyppLogFormat) ); + if ( !_stream->sendMessage ( caps ) ) { + return expected::error( ZYPP_EXCPT_PTR(zypp::Exception("Failed to send capabilities")) ); + } + return expected::success (); + }); + }; + } + + void ProvideWorker::messageLoop( Timer & ) + { + if ( _fatalError ) + return; + + while ( _pendingMessages.size () ) { + auto m = _pendingMessages.front (); + _pendingMessages.pop_front (); + handleSingleMessage(m); + } + + if ( !_fatalError && _pendingProvides.size() ) { + provide(); + } + + // keep poking until there are no provides anymore + if ( !_fatalError && ( _pendingMessages.size() || ( _pendingProvides.size () && _provNotificationMode == QUEUE_NOT_EMTPY ) ) ) { + _msgAvail->start(0); + } + + } + + void ProvideWorker::maybeDelayedShutdown() + { + if ( _inControllerRequest ) { + _delayedShutdown->start(0); + return; + } + + immediateShutdown(); + _loop->quit (); + } + + void ProvideWorker::readFdClosed( uint, AsyncDataSource::ChannelCloseReason ) + { + MIL << "Read FD closed, exiting." << std::endl; + maybeDelayedShutdown(); + } + + void ProvideWorker::writeFdClosed( AsyncDataSource::ChannelCloseReason ) + { + MIL << "Write FD closed, exiting." << std::endl; + maybeDelayedShutdown(); + } + + void ProvideWorker::messageReceived() + { + while ( auto message = _stream->nextMessage() ) { + if ( _fatalError ) + break; + pushSingleMessage(*message); + } + } + + void ProvideWorker::onInvalidMessageReceived() + { + invalidMessageReceived( std::exception_ptr() ); + } + + void ProvideWorker::invalidMessageReceived( std::exception_ptr p ) + { + ERR << "Received a invalid message on the input stream, aborting" << std::endl; + if ( p ) + _fatalError = p; + else + _fatalError = ZYPP_EXCPT_PTR( InvalidMessageReceivedException() ); + immediateShutdown(); + _loop->quit(); + } + + void ProvideWorker::handleSingleMessage( const ProvideMessage &provide ) + { + const auto code = provide.code(); + // we only accept requests here + if ( code >= ProvideMessage::Code::FirstControllerCode && code <= ProvideMessage::Code::LastControllerCode ) { + + MIL_PRV << "Received request: " << code << std::endl; + + if ( code == ProvideMessage::Code::Cancel ) { + const auto &i = std::find_if( _pendingProvides.begin (), _pendingProvides.end(), [ id = provide.requestId() ]( const auto &it ){ return it->_spec.requestId() == id; } ); + if ( i != _pendingProvides.end() ) { + switch ( (*i)->_state ) { + case ProvideWorkerItem::Pending: + _stream->sendMessage ( ProvideMessage::createErrorResponse ( provide.requestId (), ProvideMessage::Code::Cancelled, "Cancelled by user." ).impl() ); + _pendingProvides.erase(i); + break; + case ProvideWorkerItem::Running: + cancel(i); + break; + case ProvideWorkerItem::Finished: + break; + } + MIL << "Received Cancel for unknown request: " << provide.requestId() << ", ignoring!" << std::endl; + } + return; + } + + _pendingProvides.push_back( makeItem (ProvideMessage(provide)) ); + return; + } + ERR << "Unsupported request with code: " << code << " received!" << std::endl; + } + + void ProvideWorker::pushSingleMessage( const RpcMessage &message ) + { + const auto &handle = [&]( const RpcMessage &message ){ + const auto &msgTypeName = message.messagetypename(); + if ( msgTypeName == rpc::messageTypeName() ) { + return parseReceivedMessage( message ) + | mbind( [&]( ProvideMessage &&provide ){ + _pendingMessages.push_back(provide); + _msgAvail->start(0); + return expected::success(); + }); + } + return expected::error( ZYPP_EXCPT_PTR( std::invalid_argument(zypp::str::Str()<<"Unknown message received: " << message.messagetypename())) ); + }; + + const auto &exp = handle( message ); + if ( !exp ) { + try { + std::rethrow_exception ( exp.error () ); + } catch ( const zypp::Exception &e ) { + ERR << "Catched exception during message handling: " << e << std::endl; + } catch ( const std::exception &e ) { + ERR << "Catched exception during message handling: " << e.what()<< std::endl; + } catch ( ... ) { + ERR << "Unknown Exception during message handling" << std::endl; + } + } + } + + expected ProvideWorker::parseReceivedMessage(const RpcMessage &m) + { + auto exp = ProvideMessage::create(m); + if ( !exp ) + invalidMessageReceived( exp.error() ); + return exp; + } +} diff --git a/zypp-media/ng/worker/provideworker.h b/zypp-media/ng/worker/provideworker.h new file mode 100644 index 0000000..5227f5d --- /dev/null +++ b/zypp-media/ng/worker/provideworker.h @@ -0,0 +1,217 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ + +#ifndef ZYPP_MEDIA_PROVIDE_WORKER_H_INCLUDED +#define ZYPP_MEDIA_PROVIDE_WORKER_H_INCLUDED + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace zyppng::worker { + + using WorkerCaps = zypp::proto::Capabilities; + using Message = zypp::proto::Envelope; + using Configuration = zypp::proto::Configuration; + + struct AuthInfo + { + std::string username; + std::string password; + int64_t last_auth_timestamp = 0; + std::map extraKeys = {}; + }; + + class RequestCancelException : public zypp::media::MediaException + { + public: + RequestCancelException(); + }; + + ZYPP_FWD_DECL_TYPE_WITH_REFS (ProvideWorker); + ZYPP_FWD_DECL_TYPE_WITH_REFS (ProvideWorkerItem); + + class ProvideWorkerItem : public zyppng::Base + { + public: + enum State { + Pending, + Running, + Finished + }; + + ProvideWorkerItem( ProvideMessage &&spec ) : _spec( std::move(spec) ) { } + + State _state = Pending; + ProvideMessage _spec; + }; + + class ProvideWorker : public Base + { + public: + + enum ProvideNotificatioMode { + ONLY_NEW_PROVIDES, // provide is called only when new provide requests are added to the queue + QUEUE_NOT_EMTPY // provide is called continiously until the queue is empty + }; + + ProvideWorker( std::string_view workerName ); + virtual ~ProvideWorker(); + + RpcMessageStream::Ptr messageStream() const; + + expected run ( int recv = STDIN_FILENO, int send = STDOUT_FILENO ); + + std::deque &requestQueue(); + /*! + * Called when the worker process exits + */ + virtual void immediateShutdown (){}; + + /*! + * This will request a media change from the user and BLOCK until it was acknowledged. + * + */ + enum MediaChangeRes { + SUCCESS, + ABORT, + SKIP + }; + MediaChangeRes requestMediaChange ( const uint32_t id, const std::string &label, const int32_t mediaNr, const std::vector &devices, const std::optional &desc = {} ); + + /*! + * This will send a authorization request message to the controller, asking for credentials for a given \a url. + * The \a lastTimstamp should be initialized with the last \ref AuthInfo timestamp that was received from the controller + * or -1 if none was received before. + * + * \note this blocks until a answer is received, all other received messages are delayed + */ + expected requireAuthorization ( const uint32_t id, const zypp::Url &url, const std::string &lastTriedUsername = "", const int64_t lastTimestamp = -1, const std::map &extraFields = {} ); + + ProvideNotificatioMode provNotificationMode() const; + void setProvNotificationMode(const ProvideNotificatioMode &provNotificationMode); + + protected: + virtual void initLog(); + virtual expected initialize ( const Configuration &conf ) = 0; + + /*! + * Automatically called whenever a new item is enqueued. + */ + virtual void provide ( ) = 0; + virtual void cancel ( const std::deque::iterator &request ) = 0; + + /*! + * Always called to create new items for the request queue, + * override this to populate the queue with instances of custom \ref ProvideItem subclasses. + * + * Cancel requests are directly handled by calling cancel(), however Attach and Detach requests are enqueued as well + */ + virtual ProvideWorkerItemRef makeItem (ProvideMessage &&spec ); + + /*! + * Send a \a ProvideStart signal to the controller, this is to notify the controller that we have started providing the file + * the argument \a localFile has to refer to the file where the file will be provided into, it will be used to calculate + * statistics about download speed on the controller side. + * The \a stagingFile argument can be used for cases where the worker uses a staging area to download files into but later moves + * the file over to the result filename. In those cases the provider will check both locations for the file when calculating stats + * + * \note Always call \ref ref before sending this message. + */ + void provideStart ( const uint32_t id, const zypp::Url &url, const zypp::Pathname &localFile, const zypp::Pathname &stagingFile = {} ); + + /*! + * Send a \a ProvideSuccess message to the controller. This is to signal that we are finished with providing a file + * and release the file to be used by the controller side. + */ + void provideSuccess (const uint32_t id, bool cacheHit, const zypp::Pathname &localFile, const HeaderValueMap extra = {} ); + + /*! + * Send a \a ProvideFailed message to the controller. This is to signal that we are failed providing a resource + * + * \note If the request referenced a \a ident before make sure to manually release it after sending the message. + */ + void provideFailed ( const uint32_t id, const uint code, const std::string &reason, const bool transient, const HeaderValueMap extra = {} ); + + /*! + * Overload of provideFailed that takes a \ref zypp::Exception to fill in the error details + * + * \note If the request referenced a \a ident before make sure to manually release it after sending the message. + */ + void provideFailed ( const uint32_t id, const uint code, const bool transient, const zypp::Exception &e ); + + /*! + * Send a \a AttachSuccess message to the controller. This is to signal that we are finished with mounting and verifying a medium + */ + void attachSuccess ( const uint32_t id ); + + /*! + * Send a \a DetachSuccess message to the controller. This is to signal that we are finished unmounting a medium + */ + void detachSuccess ( const uint32_t id ); + + /*! + * Send a \a Redirect message to the controller for the given request ID. This is similar to sending a \a ProvideSuccess message + * and the request will be removed from the queue, which means the worker also has to remove it from its internal queue. + */ + void redirect ( const uint32_t id, const zypp::Url &url, const zypp::Pathname &newPath ); + + /*! + * Returns the control IO datasource, only valid after \ref run was called + */ + AsyncDataSource &controlIO (); + + + private: + expected executeHandshake (); + void maybeDelayedShutdown (); + void messageLoop ( Timer & ); + void readFdClosed ( uint, AsyncDataSource::ChannelCloseReason ); + void writeFdClosed ( AsyncDataSource::ChannelCloseReason ); + void messageReceived (); + void onInvalidMessageReceived ( ); + void invalidMessageReceived ( std::exception_ptr p ); + void handleSingleMessage (const ProvideMessage &provide ); + void pushSingleMessage ( const RpcMessage &msg ); + expected sendAndWaitForResponse ( const ProvideMessage &request, const std::vector &responseCodes ); + expected parseReceivedMessage( const RpcMessage &m ); + + private: + ProvideNotificatioMode _provNotificationMode = QUEUE_NOT_EMTPY; + bool _inControllerRequest = false; //< Used to signalize that we are currently in a blocking controller callback + bool _isRunning = false; + std::string_view _workerName; + EventLoop::Ptr _loop = EventLoop::create(); + Timer::Ptr _msgAvail = Timer::create(); + Timer::Ptr _delayedShutdown = Timer::create(); + AsyncDataSource::Ptr _controlIO; + RpcMessageStream::Ptr _stream; + Configuration _workerConf; + + std::exception_ptr _fatalError; //< Error that caused the eventloop to stop + + std::deque _pendingMessages; + std::deque _pendingProvides; + }; +} + + +#endif diff --git a/zypp-proto/CMakeLists.txt b/zypp-proto/CMakeLists.txt index 3eb68fb..f7a6a74 100644 --- a/zypp-proto/CMakeLists.txt +++ b/zypp-proto/CMakeLists.txt @@ -3,20 +3,25 @@ SET( zypp_core_PROTOBUF_SOURCES ) SET( zypp_media_PROTOBUF_SOURCES - media/transfersettings.proto - media/download.proto - media/messages.proto - media/networkrequesterror.proto + media/provider.proto ) SET( zypp_target_PROTOBUF_SOURCES target/commit.proto ) +SET( zypp_test_PROTOBUF_SOURCES + test/tvm.proto +) + protobuf_generate_cpp( ZYPPCORE_PROTO_SRCS ZYPPCORE_PROTO_HDRS ${zypp_core_PROTOBUF_SOURCES} ) protobuf_generate_cpp( ZYPPMEDIA_PROTO_SRCS ZYPPMEDIA_PROTO_HDRS ${zypp_media_PROTOBUF_SOURCES} ) protobuf_generate_cpp( ZYPPTARGET_PROTO_SRCS ZYPPTARGET_PROTO_HDRS ${zypp_target_PROTOBUF_SOURCES} ) +protobuf_generate_cpp( ZYPPTEST_PROTO_SRCS ZYPPTEST_PROTO_HDRS ${zypp_test_PROTOBUF_SOURCES} ) - -ADD_LIBRARY( zypp-protobuf STATIC ${ZYPPCORE_PROTO_SRCS} ${ZYPPCORE_PROTO_HDRS} ${ZYPPMEDIA_PROTO_SRCS} ${ZYPPMEDIA_PROTO_HDRS} ${ZYPPTARGET_PROTO_HDRS} ${ZYPPTARGET_PROTO_SRCS} ) - +ADD_LIBRARY( zypp-protobuf STATIC + ${ZYPPCORE_PROTO_SRCS} ${ZYPPCORE_PROTO_HDRS} + ${ZYPPMEDIA_PROTO_SRCS} ${ZYPPMEDIA_PROTO_HDRS} + ${ZYPPTARGET_PROTO_HDRS} ${ZYPPTARGET_PROTO_SRCS} + ${ZYPPTEST_PROTO_HDRS} ${ZYPPTEST_PROTO_SRCS} +) diff --git a/zypp-proto/media/download.proto b/zypp-proto/media/download.proto deleted file mode 100644 index c99272f..0000000 --- a/zypp-proto/media/download.proto +++ /dev/null @@ -1,27 +0,0 @@ -syntax = "proto3"; -option optimize_for = LITE_RUNTIME; - -import "transfersettings.proto"; - -package zypp.proto; - -message Checksum { - string type = 1; - bytes sum = 2; -} - -/*! - * Specifies a file that needs to be downloaded - */ -message DownloadSpec { - string url = 1; - TransferSettings settings = 2; - string delta = 3; - uint64 expectedFileSize = 4; - string targetPath = 5; - bool checkExistanceOnly = 6; //< this will NOT download the file, but only query the server if it exists - bool metalink_enabled = 7; //< should the download try to use metalinks - uint32 headerSize = 8; //< Optional file header size for things like zchunk - Checksum headerChecksum = 9; //< Optional file header checksum - uint32 preferred_chunk_size = 10; -} diff --git a/zypp-proto/media/messages.proto b/zypp-proto/media/messages.proto deleted file mode 100644 index 57d594e..0000000 --- a/zypp-proto/media/messages.proto +++ /dev/null @@ -1,100 +0,0 @@ -syntax = "proto3"; -option optimize_for = LITE_RUNTIME; - -import "transfersettings.proto"; -import "download.proto"; -import "networkrequesterror.proto"; - -package zypp.proto; - -// the download is silently enqueued and prefetched -// will return a Status object to the sender -message Request { - uint32 requestId = 1; - DownloadSpec spec = 2; - bool prioritize = 3; //< this is usually set when a download is required right away, it will be enqueued with highest prio - bool streamProgress = 4; //< immediately start to stream download updates -} - -// the downloads are silently enqueued and prefetched -// will return a Status object to the sender -message Prefetch { - uint32 requestId = 1; - repeated Request requests = 2; -} - -// Will reprioritize a download that is not yet started -message Prioritize { - uint32 requestId = 1; -} - -// Download has been started in the server, updates are streamed if streamProgress was -// set to true in the initial request or if SubscribeProgress was issued for this request -message DownloadStart { - uint32 requestId = 1; - string url = 2; -} - -// constant updates about the state of the download -// if total is NOT set, it is an "Alive" update only -message DownloadProgress { - uint32 requestId = 1; - string url = 2; - uint64 total = 3; - uint64 now = 4; -} - -// last message sent about a tracked download -message DownloadFin { - uint32 requestId = 1; - NetworkRequestError error = 2; - int64 last_auth_timestamp = 3; //< this field is only set if a timestamp from CredentialManager was used -} - -// will start streaming updates about download download progress -// usually only download start and finished messages are sent -message SubscribeProgress { - uint32 requestId = 1; - bool prioritize = 2; //< setting this flag will also prioritize the download -} - -// will stop streaming updates about download download progress -// usually only download start and finished messages are sent -message UnSubscribeProgress { - uint32 requestId = 1; -} - -// This message can only be sent when a download is tracked. -// Otherwise, it is silenty ignored without even a return code. -// The expected answer message for this is a DownloadFin with -// its status code set to cancelled (if it was not finished before). -message CancelDownload { - uint32 requestId = 1; -} - -// message sent to server to indicate that new auth data was stored -// in the credential manager, this will restart all requests currently -// sitting in AuthFailed status -message NewAuthDataAvailable { - uint32 requestId = 1; - TransferSettings settings = 2; -} - -// generic status object, showing if a request that has no explicit return message -// was successful or not. However it can also be returned in place of any other response message -// which usually means there is a error we can not recover from. -message Status { - enum Code { - Ok = 0; - InvalidMessage = 1; //< The message could not be parsed from the stream, this will close the stream - UnknownRequest = 2; //< The Payload in the request was valid but unknown - MalformedRequest = 3; //< The payload type in the request was known, but it was not possible to deserialize it - UnknownId = 4; //< The request ID was not found - } - uint32 requestId = 1; - Code code = 2; - string reason = 3; //< if rejected is false, this will contain the reason why -} - - - diff --git a/zypp-proto/media/networkrequesterror.proto b/zypp-proto/media/networkrequesterror.proto deleted file mode 100644 index 0f12c50..0000000 --- a/zypp-proto/media/networkrequesterror.proto +++ /dev/null @@ -1,14 +0,0 @@ -syntax = "proto3"; -option optimize_for = LITE_RUNTIME; - -package zypp.proto; - -message NetworkRequestError { - /** - * This will always correspond to the enum value specified in NetworkRequestError::Type - */ - int32 error = 1; - string errorDesc = 2; - string nativeError = 3; - map extra_info = 4; -} diff --git a/zypp-proto/media/provider.proto b/zypp-proto/media/provider.proto new file mode 100644 index 0000000..48f93f2 --- /dev/null +++ b/zypp-proto/media/provider.proto @@ -0,0 +1,419 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +----------------------------------------------------------------------*/ + +syntax = "proto3"; +option optimize_for = LITE_RUNTIME; + +package zypp.proto; + +/*! + General: + ------- + This file contains the protocol messages used in media worker backends. These backends are + used by zypp mainly to acquire files from different types of media via a generalized API. However, + it can also be used to implement helpers for special needs, e.g. copying files around or rewriting URLs. + + Every worker type is identified by the schema used in a URL e.g. dvd for getting files from DVDs or https for downloading. + This makes it possible to easily extend zypp with support for new schemes by implementing a new worker type without the need + to modify zypp itself. A important functionality is the possibility to redirect URLs to different workers, making it possible + to implement special schemes that need to remodel a URL in special ways before they are processed by the general backends. + + Communication Format: + --------------------- + Each message is serialized into a zypp.proto.Envelope and sent over the communication medium in binary + format. The binary format looks like: + + +--------------------------------+---------------------------------+ + | msglen ( 32 bit unsigned int ) | binary zypp.proto.Envelope data | + +--------------------------------+---------------------------------+ + + The header defines the size in bytes of the following data trailer. The header type is a 32 bit uint, endianess is defined by + the underlying CPU arch. The data portion is directly generated by libprotobuf via SerializeToZeroCopyStream() to generate + the binary represenation of the message. + + + Communication channel: + ---------------------- + Communication between the worker processes and the zypp main process will happen via the standard unix file descriptors: + stdin, stdout and stderr. stdin and stdout are used to send and receive messages between main and worker processes, while + stderr is used to output logging that is generated by the worker processes. These logs are forwarded to the main zypp log. + + The workers are supposed to shut down as soon as their stdin is closed. This is important to not block the controller process + that is waiting for this to happen, to fetch all log lines that are emitted by the worker when shutting down. + + + Worker Requirements: + -------------------- + Async communication: + Every worker needs to be able to handle messages asynchronously, the frontend will send requests to the worker as + they are given by the frontend code and does not apply any ordering. This means workers need to remember queries they sent to the frontend and manage + the code waiting for the result accordingly. The only exception to this is the handshake messages, where the frontend will always + send the worker configuration first, directly after spawning the worker process and the worker will answer with its capabilities. + Once those messages have been exchanged, zypp will start to send requests as defined below. + The frontend will buffer messages sent to the workers, so they do not need to keep the socket empty, however if possible workers should + support handling multiple requests. + + Cleaning up: + ------------ + A worker needs to maintain a mountpoint/download location as long as needed. Since only the frontend can decide when a location + is not required anymore it will send a "ReleaseResource" message with a worker generated ID that releases said ressource. + Examples for this are mountpoints that a worker created or download areas that were used by the worker to < the files to + + Worker Types: + ------------- + Currently zypp knows 4 different types of workers: + + * Downloading: Simplest type of worker that just download the files from a remote location without the need to mount something beforehand. + This is the simplest file provider that does not need to implement special functions. It is possible to freely redirect requests + between different variations of this worker type, all other types of workers do NOT support redirections. + Per remote host a own instance is started except if the SingleInstance flag is set. It is adviceable to set the SingleInstance flag on all + workers that do not do any long running requests. + + * Mounting: This type of worker needs to mount filesystems, but is not restricted by local volatile devices. For example smb, nfs, dir or even disk. + It has to additionally implement the Attach/Verify workflow. + Mounting handlers are always started as SingleInstance workers. + + * VolatileMounting: Most complex type of workers that mounts local devices that are volatile. This worker needs to also implement the device change workflow. + This type of worker exists only to implement the dvd worker. + VolatileMounting handlers are started as SingleInstance workers. + + * CPU Bound: Simple zypp internal workhorse type that can be implemented to run async processes like extracting a compressed file or creating a checksum. + For this type of worker a own instance per CPU is started on demand by default. + + Attach/Mount Workflow ( does not apply to Downloading and CPU bound worker types ) : + Each handler that marks itself as Mounting or VolatileMounting needs to implement the attach/mount workflow. This means that the controller needs to rely + on the handler to correctly mount and identify a medium. In the Attach message the controller may send media identification data and identification type to the worker + to specify the exact medium it is looking for. The worker then needs to mount the filesystem and perform the validation check, if the check succeeds the worker + will assign the unique attach point ID it has received from the contoller to the attachpoint and sends back a AttachResult containing the information. + At this point the worker needs to keep the mountpoint and attachpoint around until the controller sends a Detach request for it. + If the verification fails the worker will unmount the filesystem and send back a AttachResult message with the error bits set. + If the controller does not send verification data, every device/remoteFS that is successfully mounted is considered valid. + + After a medium was mounted the controller will send provide requests to the worker that have URLs formed like: + -media:///path/to/file. For example: dvd-media://1d6c0810-2bd6-45f3-9890-0268422a6f14/path/to/file , the attachID part is just a example, the controller uses + generates IDs that can be expressed as a URL hostname. + Another important aspect to keep in mind, is that a attach point is not the same as a mount point. Since a attach point additionally contains the path relative to the + filesystem root on the device. For example: the repository drivers is in a subdirectory of a dvd, so we'd get a attach request for the base url: dvd:/drivers, this would + result in a attachpoint id of "abcdef123" the device would be mounted in "/mnt/device". All subsequent requests are relative to the attachpoint and would look like: + dvd-media://abcdef123/file , which results in a lookup for the file: "/mnt/device/drivers/file1". + + Due to historical reasons a single device/filesystem can contain multiple mediums. So the worker needs to make sure to not unmount the filesystem too early. + For example dvd:/repo-1 and dvd:/repo-2 , both mediums are on the same disc, hence the worker will need to maintain two mount IDs and can only release the device itself + when both have been detached. + In some cases its even possible that multiple mediums reside in the same (sub)directory on a filesystem. For example when a multi DVD set was copied together into + one directory shared over NFS. + + + Message status codes + --------------------- + The response codes are roughtly modelled like HTTP status codes, they can happen during the full runtime of a certain request. + + Informational messages (100–199) + Success messages (200–299) + Redirection messages (300–399) + Client error messages (400–499) + Server error messages (500–599) + Controller messages (600–699) + Worker messages (700-799) + + + - Code: 100 - Provide started + Desc: Sent to the controller once a worker starts providing a request + Note that this is a optional message, its perfectly valid if 200 ( Provide Success ) is sent directly after the file was requested. + Fields: + required string url + string local_filename -> The local filename where the file will be provided to, used by the controller to track progress + string staging_filename -> The staging file used while downloading + + + - Code: 200 - Provide Finished + Desc: Sent to the controller once a worker finished a job successfully. + Fields: + required string local_filename -> The path where the worker has placed the file + required bool cacheHit -> Set to true if the file was found in a worker cache + + + - Code: 201 - Attach Finished + Desc: Sent to the controller once a worker mounted a medium successfully. + Fields: + - None - + + - Code: 202 - Auth Info + Desc: Response sent by the controller to a AUTH_REQUIRED request. + The last_auth_timestamp contains the last change of the auth database. This timestamp should be included + in successive AUTH_REQUIRED requests, so that the controller knows if its outdated or if the Auth was read from the store once + and now we need to ask the user to provide new auth info. + Fields: + required string username + required string password + required int64 auth_timestamp -> timestamp of the auth data, this should be stored in the worker in case it fails again + optional string authType -> comma seperated list of selected auth types, used by the network provider + + + - Code: 203 - Media changed + Desc: Sent to the worker once the user acknowledged a media change request + Fields: + - None - + + - Code: 204 - Detach Finished + Desc: Sent to the controller once a worker detached a medium successfully. + Fields: + - None - + + + - Code: 300 - Redirect + Desc: A Downloading worker can generate a Redirect message to reroute a request to a different + Downloading worker. This can be used to implement special handlers where a URL needs to be rewritten + before it can actually be fetched. + This is similar to sending a 200 - Provide Finished message since it will take the request out of the workers queue + + Note: Redirecting is ONLY supported between Downloading workers. Sending this message from other worker types + or redirecting to non Dowloading workers results in failing the user request with a redirect error. + Fields: + required string new_url -> the full URL we want to redirect to + + + - Code: 301 - Metalink Redirect + Desc: A worker can generate a list of mirrors, this is similar to a Redirect response but supports + giving the Controller a list of possible URLs to try. + This is similar to sending a 200 - Provide Finished message since it will take the request out of the workers queue. + The mirror list is ordered by priority, meaning the first mirror has the highest. + Fields: + repeated string new_url -> full URL we want to redirect to + + + + 4xx and 5xx messages, all of those messages have the same Response type structure: + + Code: 4xx / 5xx - Client error + Desc: Sent to the controller for requests that fail due to a client error + Fields: + required string reason -> The reason as a string + optional string history -> The error history that lead to the error + bool transient -> If this is set to true, the controller will try to request the file again at a later time + + Currently reserved error codes: + 400: Bad Request -> invalid request, the receiver can not process it + 401: Unauthorized -> no auth info avail but auth required + extraFields: string authHint + 402: Forbidden -> auth was given but failed or no access rights to the requested resource + extraFields: string authHint + 403: PeerCertificateInvalid -> the peer certificate validation failed + 404: Not Found -> the requested resource does not exist on the medium + 405: Expected Size Exceeded -> the downloaded data exceeded the requested maximum lenght + 406: Connection failed -> connecting to the server failed + 407: Timeout -> the request timed out + 408: Cancelled -> request was cancelled by the user + 409: Invalid checksum -> the downloaded data has a different checksum than expected + 410: Mount failed -> mounting the requested resource failed + 411: Jammed -> ATTACH request failed due to insufficient resources + 412: Media Change Abort -> User decided to abort the request + 413: Media Change Skip -> User decided to abort the request with a skip command( request will fail but will report "skipped" as the reason ) + 414: No Auth data -> Only sent by the controller for a 700 - Auth Data Request + 416: Media not desired -> The desired medium was not found on the given URL ( attach request ) + + 500: InternalError -> a error in the worker that is not recoverable, check reason string + + Controller -> Worker requests + ----------------------------- + + - Code: 600 - Provide + Desc: Message to tell the worker about a ressource it should provide + Fields: + required string url -> The URL of the request + string filename -> target filename hint, this is not required to be considered by the workers + string delta_file -> local path to a file that is supposed to be used for delta downloads + int64 expected_filesize -> The expected download filesize, workers should fail if a server does not reports the exact same filesize + bool check_existance_only -> this will NOT download the file but only query the server if its existant + bool metalink_enabled -> enables/disables metalink handling + + - Code: 601 - Cancel + Desc: Sent by the controller if a request should be cancelled. The worker should stop the given request and return a + 408 response. + Fields: + - None - + + - Code: 602 - Attach + Desc: If a worker signals the frontend it needs to mount/unmount resources the controller will first send a Attach + message for each base_url it encounters that targets the workers scheme. All workers that do not set this flag + do not need to care about this special message. + In fact receiving this message in a worker that is not a mounting type should result + in returning a error to the controller. + + If the verifyType field is NOT set, the verifyData field will be ignored and the worker does not need to + verify the device on the given location. + Fields: + required string url -> the base URL we want to attach to + required string attach_id -> controller generated ID string to uniquely identiy a attached medium + required string label -> Label of the medium + string verify_type -> name of the verification type, currently only suseV1 ( required if verify_data is present ) + bytes verify_data -> verification data ( required if verify_type is present ) + int32 media_nr -> the nr of the medium in the set ( required if verify_type is present ) + repeated string device -> optional field containing a device that can be used for attachment + + - Code: 603 - Detach + Desc: Send by the controller process when a medium is no longer required. + The worker has to immediately unmount the medium, no answer to the controller is expected. + Since there is no connected request, the request ID in this message will not be considered + Fields: + required string url -> the attachment URL containing the controller generated ID string to uniquely identify a attached medium, e.g. dvd:/// + + + Worker -> Controller requests + ----------------------------- + + - Code: 700 - Auth Data Request + Desc: In case a worker needs authorization info it can generate this request. + The last_auth_timestamp field is present if the request had received auth info before. + Fields: + required string effective_url -> the effective URL we want to fetch, this is not necessarily the same as the request URL + int64 last_auth_timestamp -> timestamp of previously received auth informationpJuds + optional string username -> username tried in the last auth attempt + optional string authHint -> comma seperated list of available authentication types + optional repeated string -> all other key:value pairs are treated as extra keys. + Possible Resposes: 414, 202 + + + - Code: 701 - Media Change Request + Desc: Generated by a worker to ask the user to insert a different medium. This will BLOCK the controller and the handler + until the user answered the request. This is a special request only used by the CDROM handler + Fields: + required string label -> name of the medium we need + required int32 media_nr -> which medium from the media set is required + required repeated string device -> free device to be used to insert the media into + string desc -> medium desc + Possible Resposes: 412, 413, 203 + +*/ + + +enum MessageCodes { + option allow_alias = true; + NoCode = 0; + FirstInformalCode = 100; + ProvideStarted = 100; + LastInformalCode = 199; + + FirstSuccessCode = 200; + ProvideFinished = 200; + AttachFinished = 201; + AuthInfo = 202; + MediaChanged = 203; + DetachFinished = 204; + LastSuccessCode = 299; + + FirstRedirCode = 300; + Redirect = 300; + Metalink = 301; + LastRedirCode = 399; + + FirstClientErrCode = 400; + BadRequest = 400; + Unauthorized = 401; + Forbidden = 402; + PeerCertificateInvalid = 403; + NotFound = 404; + ExpectedSizeExceeded = 405; + ConnectionFailed = 406; + Timeout = 407; + Cancelled = 408; + InvalidChecksum = 409; + MountFailed = 410; + Jammed = 411; + MediaChangeAbort = 412; + MediaChangeSkip = 413; + NoAuthData = 414; + NotAFile = 415; + MediumNotDesired = 416; + LastClientErrCode = 499; + + FirstSrvErrCode = 500; + InternalError = 500; + ProtocolError = 501; + LastSrvErrCode = 599; + + FirstControllerCode = 600; + Provide = 600; + Cancel = 601; + Attach = 602; + Detach = 603; + LastControllerCode = 699; + + FirstWorkerCode = 700; + AuthDataRequest = 700; + MediaChangeRequest = 701; + LastWorkerCode = 799; +} + +message DataField { + string key = 1; + oneof field_val { + string str_val = 2; + int32 int_val = 3; + int64 long_val = 4; + double double_val = 5; + bool bool_val = 6; + } +} + +/*! + * This is the general message used to communicate between workers and the controller. + * Based on the message_code a message is either a response, signal or request. + * Every communication is initiated by a request, the ID given in the request message is used + * through the lifetime of the request to link successive requests to the initial request. + */ +message ProvideMessage { + uint32 request_id = 1; //< the request ID set by the process initiating a request + uint32 message_code = 2; //< the response code, see documentation + repeated DataField fields = 3; +} + +// -------------------------------------------------------------------------- +// ----------------- Handshake message Controller -> Worker ----------------- +// -------------------------------------------------------------------------- + +message Configuration { + map values = 1; // A key value map of all worker related configurations +} + + +// -------------------------------------------------------------------------- +// ----------------- Handshake Message Worker -> Controller ----------------- +// -------------------------------------------------------------------------- + +/*! + * Message that describes be capabilities of the worker process. + * This ALWAYS has to be sent to the controller as the first message immediately after receiving the Configuration message + */ +message Capabilities { + + /*! The worker type, see the description in Worker Types above */ + enum WorkerType { + Invalid = 0; + Downloading = 1; + SimpleMount = 2; + VolatileMount = 3; + CPUBound = 4; + } + + enum Flags { + None = 0; // Just to satisfy the protobuf compiler + SingleInstance = 1; // If this flag is set a worker can only be started once, this is implicit in some worker types. + Pipeline = 2; // The worker can handle multiple requests at the same time + ZyppLogFormat = 4; // The worker writes messages to stderr in zypp log format + FileArtifacts = 8; // The results of this worker are artifacts, which means they need to be cleaned up. This is implicit for all downloading workers. For all mounting workers this is ignored. + // CPU bound workers can use it to signal they leave artifact files behind that need to be cleaned up + } + + uint32 protocol_version = 1; // The workers should set this field to the protocol version they implement. + WorkerType worker_type = 2; + Flags cfg_flags = 3; + string worker_name = 4; + repeated DataField fields = 5; // customs fields for future extensions +} diff --git a/zypp-proto/media/transfersettings.proto b/zypp-proto/media/transfersettings.proto deleted file mode 100644 index 7d266d9..0000000 --- a/zypp-proto/media/transfersettings.proto +++ /dev/null @@ -1,35 +0,0 @@ -syntax = "proto3"; -option optimize_for = LITE_RUNTIME; - -package zypp.proto; - -message TransferSettings { - repeated string header = 1; - string useragent = 2; - string username = 3; - string password = 4; - bool useproxy = 5; - string proxy = 6; - string proxy_username = 7; - string proxy_password = 8; - string authtype = 9; - int64 timeout = 10; - int64 connect_timeout = 11; - string url = 12; - string targetdir = 13; - - - int64 maxConcurrentConnections = 14; - int64 minDownloadSpeed = 15; - int64 maxDownloadSpeed = 16; - int64 maxSilentTries = 17; - - bool verify_host = 18; - bool verify_peer = 19; - string ca_path = 20; - string client_cert_path = 21; - string client_key_path = 22; - - // workarounds - bool head_requests_allowed = 23; -} diff --git a/zypp-proto/test/tvm.proto b/zypp-proto/test/tvm.proto new file mode 100644 index 0000000..9e50817 --- /dev/null +++ b/zypp-proto/test/tvm.proto @@ -0,0 +1,31 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +----------------------------------------------------------------------*/ + +syntax = "proto3"; +option optimize_for = LITE_RUNTIME; + +package zypp.proto.test; + +/*! + Helper settings for the TVM ( Test Volatile Mouning ) worker. This can be + used to define the number of available devices and which of those have inserted media. + + This data is written to a file in the provider dir for the tvm worker to read. + The worker only reads the data while the test only writes it. This way we can simulate + a user inserting/changing CDs + */ + +message TVMSettings +{ + message Device { + string name = 1; // the device name + string insertedPath = 2; // the path what is currently inserted into the device + } + repeated Device devices = 1; +} diff --git a/zypp/CMakeLists.txt b/zypp/CMakeLists.txt index 948309e..75bff59 100644 --- a/zypp/CMakeLists.txt +++ b/zypp/CMakeLists.txt @@ -817,8 +817,6 @@ INSTALL( FILES DESTINATION ${INCLUDE_INSTALL_DIR}/zypp/media/proxyinfo ) -add_subdirectory( zyppng ) - SET( zypp_lib_SRCS ${zypp_misc_SRCS} ${zypp_pool_SRCS} @@ -890,7 +888,7 @@ add_dependencies( zypp-objlib zypp-protobuf ) macro( ADDZYPPLIB LIBNAME ) message( "ADDING lib ${LIBNAME} to project" ) - ADD_LIBRARY( ${LIBNAME} SHARED $ $ ) + ADD_LIBRARY( ${LIBNAME} SHARED $ ) SET_TARGET_PROPERTIES( ${LIBNAME} PROPERTIES VERSION "${LIBZYPP_VERSION_INFO}" ) SET_TARGET_PROPERTIES( ${LIBNAME} PROPERTIES SOVERSION "${LIBZYPP_SOVERSION_INFO}" ) # System libraries @@ -906,25 +904,17 @@ macro( ADDZYPPLIB LIBNAME ) TARGET_LINK_LIBRARIES(${LIBNAME} ${UTIL_LIBRARY} ) TARGET_LINK_LIBRARIES(${LIBNAME} ${RPM_LIBRARY} ) TARGET_LINK_LIBRARIES(${LIBNAME} ${GETTEXT_LIBRARIES} ) - TARGET_LINK_LIBRARIES(${LIBNAME} ${CURL_LIBRARIES} ) - TARGET_LINK_LIBRARIES(${LIBNAME} ${LIBXML2_LIBRARIES} ) + #TARGET_LINK_LIBRARIES(${LIBNAME} ${CURL_LIBRARIES} ) + #TARGET_LINK_LIBRARIES(${LIBNAME} ${LIBXML2_LIBRARIES} ) TARGET_LINK_LIBRARIES(${LIBNAME} ${ZLIB_LIBRARY} ) TARGET_LINK_LIBRARIES(${LIBNAME} ${LibSolv_LIBRARIES} ) - TARGET_LINK_LIBRARIES(${LIBNAME} ${SIGNALS_LIBRARY}) + #TARGET_LINK_LIBRARIES(${LIBNAME} ${SIGNALS_LIBRARY}) TARGET_LINK_LIBRARIES(${LIBNAME} ${Boost_THREAD_LIBRARY}) TARGET_LINK_LIBRARIES(${LIBNAME} ${GPGME_PTHREAD_LIBRARIES}) target_link_libraries(${LIBNAME} ${YAML_CPP_LIBRARIES}) target_link_libraries(${LIBNAME} ${PROTOBUF_LITE_LIBRARIES}) TARGET_LINK_LIBRARIES(${LIBNAME} pthread ) - IF (ENABLE_ZSTD_COMPRESSION) - TARGET_LINK_LIBRARIES(${LIBNAME} ${ZSTD_LIBRARY}) - ENDIF (ENABLE_ZSTD_COMPRESSION) - - IF (ENABLE_ZCHUNK_COMPRESSION) - TARGET_LINK_LIBRARIES(${LIBNAME} ${ZCHUNK_LDFLAGS}) - ENDIF(ENABLE_ZCHUNK_COMPRESSION) - IF ( UDEV_FOUND ) TARGET_LINK_LIBRARIES(${LIBNAME} ${UDEV_LIBRARY} ) ELSE ( UDEV_FOUND ) @@ -933,8 +923,6 @@ macro( ADDZYPPLIB LIBNAME ) ENDIF ( HAL_FOUND ) ENDIF ( UDEV_FOUND ) - TARGET_LINK_LIBRARIES( ${LIBNAME} ${LIBPROXY_LIBRARIES} ) - endmacro() #Release library stripped from most symbols, thats what we release to the packages diff --git a/zypp/FileChecker.h b/zypp/FileChecker.h index 4bb716e..2d046c3 100644 --- a/zypp/FileChecker.h +++ b/zypp/FileChecker.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -36,30 +37,6 @@ namespace zypp */ typedef function FileChecker; - class FileCheckException : public Exception - { - public: - FileCheckException(const std::string &msg) - : Exception(msg) - {} - }; - - class CheckSumCheckException : public FileCheckException - { - public: - CheckSumCheckException(const std::string &msg) - : FileCheckException(msg) - {} - }; - - class SignatureCheckException : public FileCheckException - { - public: - SignatureCheckException(const std::string &msg) - : FileCheckException(msg) - {} - }; - /** * Built in file checkers */ diff --git a/zypp/PoolItem.cc b/zypp/PoolItem.cc index b7a55bf..0a492a1 100644 --- a/zypp/PoolItem.cc +++ b/zypp/PoolItem.cc @@ -69,6 +69,13 @@ namespace zypp return _status; } + ResStatus & statusReinit() const + { + _status.setLock( _status.isUserLockQueryMatch(), zypp::ResStatus::USER ); + _status.resetTransact( zypp::ResStatus::USER ); + return _status; + } + public: bool isUndetermined() const { @@ -203,6 +210,7 @@ namespace zypp ResStatus & PoolItem::status() const { return _pimpl->status(); } ResStatus & PoolItem::statusReset() const { return _pimpl->statusReset(); } + ResStatus & PoolItem::statusReinit() const { return _pimpl->statusReinit(); } sat::Solvable PoolItem::buddy() const { return _pimpl->buddy(); } void PoolItem::setBuddy( const sat::Solvable & solv_r ) { _pimpl->setBuddy( solv_r ); } bool PoolItem::isUndetermined() const { return _pimpl->isUndetermined(); } diff --git a/zypp/PoolItem.h b/zypp/PoolItem.h index 8311a93..c86499f 100644 --- a/zypp/PoolItem.h +++ b/zypp/PoolItem.h @@ -75,8 +75,14 @@ namespace zypp /** Returns the current status. */ ResStatus & status() const; - /** Reset status. */ + /** Resets status to the default state (KEEP_STATE bySOLVER; clears any lock!). */ ResStatus & statusReset() const; + + /** Resets status to it's initial state in the ResPool (KEEP_STATE bySOLVER or LOCKED byUSER). + * The state depends on whether the PoolItem matched a hard lock defined + * in /etc/zypp/locks or not. + */ + ResStatus & statusReinit() const; //@} diff --git a/zypp/ResStatus.h b/zypp/ResStatus.h index 894c181..13d04be 100644 --- a/zypp/ResStatus.h +++ b/zypp/ResStatus.h @@ -328,22 +328,21 @@ namespace zypp { return isToBeUninstalled() && fieldValueIs( SOFT_REMOVE ); } private: - - /** \name Internal hard lock maintainance */ + /** \name Internal hard lock maintenance */ //@{ friend struct resstatus::UserLockQueryManip; - bool isUserLockQueryMatch() const - { return fieldValueIs( USERLOCK_MATCH ); } - void setUserLockQueryMatch( bool match_r ) { fieldValueAssign( match_r ? USERLOCK_MATCH : USERLOCK_NOMATCH ); } //@} + public: + bool isUserLockQueryMatch() const + { return fieldValueIs( USERLOCK_MATCH ); } public: //------------------------------------------------------------------------ - // get/set functions, returnig \c true if requested status change + // get/set functions, returning \c true if requested status change // was successful (i.e. leading to the desired transaction). // If a lower level (e.g.SOLVER) wants to transact, but it's // already set by a higher level, \c true should be returned. @@ -382,7 +381,7 @@ namespace zypp /** Apply a lock (prevent transaction). * Currently by USER or APPL_HIGH only, but who knows... - * Set LOCKED from KEEP_STATE to be shure all transaction + * Set LOCKED from KEEP_STATE to be sure all transaction * details were reset properly. */ bool setLock( bool toLock_r, TransactByValue causer_r ) @@ -428,7 +427,7 @@ namespace zypp /** Toggle between TRANSACT and KEEP_STATE. * LOCKED state means KEEP_STATE. But in contrary to KEEP_STATE, * LOCKED state is immutable for \a causer_r less than TransactByValue. - * KEEP_STATE may be canged by any \a causer_r. + * KEEP_STATE may be changed by any \a causer_r. */ bool setTransact( bool toTansact_r, TransactByValue causer_r ) { diff --git a/zypp/base/DrunkenBishop.cc b/zypp/base/DrunkenBishop.cc index aec8d2c..e7c2118 100644 --- a/zypp/base/DrunkenBishop.cc +++ b/zypp/base/DrunkenBishop.cc @@ -8,6 +8,7 @@ \---------------------------------------------------------------------*/ /** \file zypp/base/DrunkenBishop.cc */ +#include #include //#include #include diff --git a/zypp/media/MediaCD.cc b/zypp/media/MediaCD.cc index 4c3dba1..fa49657 100644 --- a/zypp/media/MediaCD.cc +++ b/zypp/media/MediaCD.cc @@ -25,6 +25,7 @@ extern "C" #include #include #include +#include #include #include #include @@ -37,13 +38,6 @@ using std::endl; */ #define REPORT_EJECT_ERRORS 0 -/* -** If defined to the full path of the eject utility, -** it will be used additionally to the eject-ioctl. -*/ -#define EJECT_TOOL_PATH "/bin/eject" - - ////////////////////////////////////////////////////////////////// namespace zypp @@ -182,55 +176,7 @@ namespace zypp // bool MediaCD::openTray( const std::string & device_r ) { - int fd = ::open( device_r.c_str(), O_RDONLY|O_NONBLOCK|O_CLOEXEC ); - int res = -1; - - if ( fd != -1) - { - res = ::ioctl( fd, CDROMEJECT ); - ::close( fd ); - } - - if ( res ) - { - if( fd == -1) - { - WAR << "Unable to open '" << device_r - << "' (" << ::strerror( errno ) << ")" << endl; - } - else - { - WAR << "Eject " << device_r - << " failed (" << ::strerror( errno ) << ")" << endl; - } - -#if defined(EJECT_TOOL_PATH) - DBG << "Try to eject " << device_r << " using " - << EJECT_TOOL_PATH << " utility" << std::endl; - - const char *cmd[3]; - cmd[0] = EJECT_TOOL_PATH; - cmd[1] = device_r.c_str(); - cmd[2] = NULL; - ExternalProgram eject(cmd, ExternalProgram::Stderr_To_Stdout); - - for(std::string out( eject.receiveLine()); - out.length(); out = eject.receiveLine()) - { - DBG << " " << out; - } - - if(eject.close() != 0) - { - WAR << "Eject of " << device_r << " failed." << std::endl; - return false; - } -#else - return false; -#endif - } - MIL << "Eject of " << device_r << " successful." << endl; - return true; + return CDTools::openTray(device_r); } /////////////////////////////////////////////////////////////////// @@ -241,19 +187,7 @@ namespace zypp // bool MediaCD::closeTray( const std::string & device_r ) { - int fd = ::open( device_r.c_str(), O_RDONLY|O_NONBLOCK|O_CLOEXEC ); - if ( fd == -1 ) { - WAR << "Unable to open '" << device_r << "' (" << ::strerror( errno ) << ")" << endl; - return false; - } - int res = ::ioctl( fd, CDROMCLOSETRAY ); - ::close( fd ); - if ( res ) { - WAR << "Close tray " << device_r << " failed (" << ::strerror( errno ) << ")" << endl; - return false; - } - DBG << "Close tray " << device_r << endl; - return true; + return CDTools::closeTray(device_r); } diff --git a/zypp/media/MediaCurl.cc b/zypp/media/MediaCurl.cc index 7a61c79..9693d29 100644 --- a/zypp/media/MediaCurl.cc +++ b/zypp/media/MediaCurl.cc @@ -612,7 +612,9 @@ void MediaCurl::disconnectFrom() if ( _curl ) { - curl_easy_cleanup( _curl ); + // bsc#1201092: Strange but within global_dtors we may exceptions here. + try { curl_easy_cleanup( _curl ); } + catch (...) { ; } _curl = NULL; } } diff --git a/zypp/media/MediaHandler.cc b/zypp/media/MediaHandler.cc index 0f069e4..b3ecbb9 100644 --- a/zypp/media/MediaHandler.cc +++ b/zypp/media/MediaHandler.cc @@ -120,44 +120,13 @@ MediaHandler::resetParentId() std::string MediaHandler::getRealPath(const std::string &path) { - std::string real; - if( !path.empty()) - { -#if __GNUC__ > 2 - /** GNU extension */ - char *ptr = ::realpath(path.c_str(), NULL); - if( ptr != NULL) - { - real = ptr; - free( ptr); - } - else - /** the SUSv2 way */ - if( EINVAL == errno) - { - char buff[PATH_MAX + 2]; - memset(buff, '\0', sizeof(buff)); - if( ::realpath(path.c_str(), buff) != NULL) - { - real = buff; - } - } -#else - char buff[PATH_MAX + 2]; - memset(buff, '\0', sizeof(buff)); - if( ::realpath(path.c_str(), buff) != NULL) - { - real = buff; - } -#endif - } - return real; + return getRealPath(zypp::Pathname(path)).asString(); } zypp::Pathname MediaHandler::getRealPath(const Pathname &path) { - return zypp::Pathname(getRealPath(path.asString())); + return path.realpath(); } @@ -533,17 +502,11 @@ MediaHandler::checkAttached(bool matchMountFs) const if ( ref.attachPoint->path != Pathname(e->dir) ) continue; // at least the mount points must match - bool is_device = false; - PathInfo dev_info; - if( str::hasPrefix( Pathname(e->src).asString(), "/dev/" ) && - dev_info(e->src) && dev_info.isBlk() ) - { - is_device = true; - } - + bool is_device = e->isBlockDevice(); if( is_device && (ref.mediaSource->maj_nr && ref.mediaSource->bdir.empty())) { + PathInfo dev_info(e->src); std::string mtype(matchMountFs ? e->type : ref.mediaSource->type); MediaSource media(mtype, e->src, dev_info.devMajor(), dev_info.devMinor()); diff --git a/zypp/media/MediaHandlerFactory.cc b/zypp/media/MediaHandlerFactory.cc index 072eac9..2b89276 100644 --- a/zypp/media/MediaHandlerFactory.cc +++ b/zypp/media/MediaHandlerFactory.cc @@ -16,7 +16,6 @@ #include #include #include -#include namespace zypp::media { @@ -58,12 +57,12 @@ namespace zypp::media { _handler = std::make_unique (url,preferred_attach_point); else if (scheme == "ftp" || scheme == "tftp" || scheme == "http" || scheme == "https") { - enum WhichHandler { choose, curl, multicurl, network }; + enum WhichHandler { choose, curl, multicurl }; WhichHandler which = choose; // Leagcy: choose handler in UUrl query if ( const std::string & queryparam = url.getQueryParam("mediahandler"); ! queryparam.empty() ) { if ( queryparam == "network" ) - which = network; + which = multicurl; else if ( queryparam == "multicurl" ) which = multicurl; else if ( queryparam == "curl" ) @@ -79,8 +78,8 @@ namespace zypp::media { }; if ( getenvIs( "ZYPP_MEDIANETWORK", "1" ) ) { - WAR << "network backend manually enabled." << std::endl; - which = network; + WAR << "network backend preview was removed, defaulting to multicurl." << std::endl; + which = multicurl; } else if ( getenvIs( "ZYPP_MULTICURL", "0" ) ) { WAR << "multicurl manually disabled." << std::endl; @@ -92,10 +91,6 @@ namespace zypp::media { // Finally use the default std::unique_ptr handler; switch ( which ) { - case network: - handler = std::make_unique( url, preferred_attach_point ); - break; - default: case multicurl: handler = std::make_unique( url, preferred_attach_point ); @@ -134,5 +129,3 @@ namespace zypp::media { } } - - diff --git a/zypp/media/MediaManager.cc b/zypp/media/MediaManager.cc index 7b9c4d6..3e589b9 100644 --- a/zypp/media/MediaManager.cc +++ b/zypp/media/MediaManager.cc @@ -272,13 +272,7 @@ namespace zypp static inline time_t getMountTableMTime() { - time_t mtime = zypp::PathInfo("/etc/mtab").mtime(); - if( mtime <= 0) - { - WAR << "Failed to retrieve modification time of '/etc/mtab'" - << std::endl; - } - return mtime; + return Mount::getMTime(); } static inline MountEntries diff --git a/zypp/repo/SUSEMediaVerifier.cc b/zypp/repo/SUSEMediaVerifier.cc index 1275a70..23e5928 100644 --- a/zypp/repo/SUSEMediaVerifier.cc +++ b/zypp/repo/SUSEMediaVerifier.cc @@ -151,7 +151,7 @@ namespace zypp bool SUSEMediaVerifier::isDesiredMedia( const media::MediaHandler & ref ) const { - bool ret = true; // optimistic return unless we definitely now it does not match + bool ret = true; // optimistic return unless we definitely know it does not match const SMVData & smvData = _pimpl->smvData(); if ( ! smvData ) diff --git a/zypp/target/TargetImpl.cc b/zypp/target/TargetImpl.cc index 4a64a5b..3d6ebbf 100644 --- a/zypp/target/TargetImpl.cc +++ b/zypp/target/TargetImpl.cc @@ -1214,7 +1214,6 @@ namespace zypp sat::SolvableSpec needrebootSpec; needrebootSpec.addProvides( Capability("installhint(reboot-needed)") ); needrebootSpec.addProvides( Capability("kernel") ); - needrebootSpec.addIdent( IdString("kernel-firmware") ); Pathname needrebootFile { Pathname::assertprefix( root(), ZConfig::instance().needrebootFile() ) }; if ( PathInfo( needrebootFile ).isFile() ) @@ -2327,7 +2326,7 @@ namespace zypp prog->addFd( scriptPipe->writeFd ); // set up the AsyncDataSource to read script output - if ( !scriptSource->open( scriptPipe->readFd ) ) + if ( !scriptSource->openFds( std::vector{ scriptPipe->readFd } ) ) ZYPP_THROW( target::rpm::RpmSubprocessException( "Failed to open scriptFD to subprocess" ) ); prog->sigStarted().connect( [&](){ @@ -2336,17 +2335,10 @@ namespace zypp messagePipe->unrefWrite(); scriptPipe->unrefWrite(); - // read the stdout and forward it to our log - prog->stdoutDevice()->connectFunc( &zyppng::IODevice::sigReadyRead, [&](){ - while( prog->stdoutDevice()->canReadLine() ) { - L_DBG("zypp-rpm") << " " << prog->stdoutDevice()->readLine().asStringView(); // no endl! - readLine does not trim - } - }); - - // read the stderr and forward it to our log - prog->stderrDevice()->connectFunc( &zyppng::IODevice::sigReadyRead, [&](){ - while( prog->stderrDevice()->canReadLine() ) { - L_ERR("zypp-rpm") << " " << prog->stderrDevice()->readLine().asStringView(); // no endl! - readLine does not trim + // read the stdout and stderr and forward it to our log + prog->connectFunc( &zyppng::IODevice::sigChannelReadyRead, [&]( int channel ){ + while( prog->canReadLine( channel ) ) { + L_ERR("zypp-rpm") << ( channel == zyppng::Process::StdOut ? " " : " " ) << prog->channelReadLine( channel ).asStringView(); // no endl! - readLine does not trim } }); @@ -2378,7 +2370,7 @@ namespace zypp // this is the source for control messages from zypp-rpm , we will get structured data information // in form of protobuf messages - if ( !msgSource->open( messagePipe->readFd ) ) + if ( !msgSource->openFds( std::vector{ messagePipe->readFd } ) ) ZYPP_THROW( target::rpm::RpmSubprocessException( "Failed to open read stream to subprocess" ) ); zyppng::rpc::HeaderSizeType pendingMessageSize = 0; @@ -2659,13 +2651,13 @@ namespace zypp // make sure to read all data from the log source bool readMsgs = false; - while( prog->stderrDevice()->canReadLine() ) { + while( prog->canReadLine( zyppng::Process::StdErr ) ) { readMsgs = true; - MIL << "zypp-rpm: " << prog->stderrDevice()->readLine().asStringView(); + MIL << "zypp-rpm: " << prog->channelReadLine( zyppng::Process::StdErr ).asStringView(); } - while( prog->stdoutDevice()->canReadLine() ) { + while( prog->canReadLine( zyppng::Process::StdOut ) ) { readMsgs = true; - MIL << "zypp-rpm: " << prog->stdoutDevice()->readLine().asStringView(); + MIL << "zypp-rpm: " << prog->channelReadLine( zyppng::Process::StdOut ).asStringView(); } while ( scriptSource->canReadLine() ) { diff --git a/zypp/zyppng/CMakeLists.txt b/zypp/zyppng/CMakeLists.txt deleted file mode 100644 index 7fbd87a..0000000 --- a/zypp/zyppng/CMakeLists.txt +++ /dev/null @@ -1,41 +0,0 @@ -SET( zyppng_media_SRCS - media/medianetwork.cc - media/medianetworkserver.cc -) - -SET( zyppng_media_HEADERS - media/MediaNetwork - media/medianetwork.h -) - -SET( zyppng_media_private_HEADERS - media/private/medianetworkserver_p.h -) - -SET( zyppng_media_private_SOURCES -) - -SET( zyppng_appcode_HEADERS - Context - context.h -) - -SET( zyppng_appcode_SRCS - context.cc -) - -SET ( zyppng_HEADERS - ${zyppng_appcode_HEADERS} - ${zyppng_media_HEADERS} - ${zyppng_media_private_HEADERS} -) - -SET ( zyppng_SRCS - ${zyppng_appcode_SRCS} - ${zyppng_media_SRCS} - ${zyppng_media_private_SOURCES} -) - -ADD_LIBRARY( zyppng-objlib OBJECT ${zyppng_SRCS} ${zyppng_HEADERS} ) -#we include generated headers, so we need to wait for zypp-protobuf to be ready -add_dependencies( zyppng-objlib zypp-protobuf ) diff --git a/zypp/zyppng/Context b/zypp/zyppng/Context deleted file mode 100644 index 7e33fb6..0000000 --- a/zypp/zyppng/Context +++ /dev/null @@ -1 +0,0 @@ -#include "context.h" diff --git a/zypp/zyppng/context.cc b/zypp/zyppng/context.cc deleted file mode 100644 index 0e7bf2d..0000000 --- a/zypp/zyppng/context.cc +++ /dev/null @@ -1,23 +0,0 @@ -#include "context.h" -#include -#include - -namespace zyppng { - - Context::Context() - { - _zyppEventLoop = EventLoop::create(); - _mirrorControl = MirrorControl::create(); - } - - std::shared_ptr Context::evLoop() const - { - return _zyppEventLoop; - } - - std::shared_ptr Context::mirrorControl() - { - return _mirrorControl; - } - -} diff --git a/zypp/zyppng/media/MediaNetwork b/zypp/zyppng/media/MediaNetwork deleted file mode 100644 index 69ec57c..0000000 --- a/zypp/zyppng/media/MediaNetwork +++ /dev/null @@ -1 +0,0 @@ -#include "medianetwork.h" diff --git a/zypp/zyppng/media/medianetwork.cc b/zypp/zyppng/media/medianetwork.cc deleted file mode 100644 index 345cefe..0000000 --- a/zypp/zyppng/media/medianetwork.cc +++ /dev/null @@ -1,987 +0,0 @@ -/*---------------------------------------------------------------------\ -| ____ _ __ __ ___ | -| |__ / \ / / . \ . \ | -| / / \ V /| _/ _/ | -| / /__ | | | | | | | -| /_____||_| |_| |_| | -| | -----------------------------------------------------------------------*/ -#include "medianetwork.h" -#include "private/medianetworkserver_p.h" - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#undef ZYPP_BASE_LOGGER_LOGGROUP -#define ZYPP_BASE_LOGGER_LOGGROUP "zypp::MediaNetwork" - -namespace zyppng { - - using HeaderSizeType = zyppng::rpc::HeaderSizeType; - - struct MediaNetwork::ProgressData - { - ProgressData( const zyppng::Url & _url = zyppng::Url(), - zypp::callback::SendReport *_report = nullptr ) - : url( _url ) - , report( _report ) - {} - - zypp::Url url; - zypp::callback::SendReport *report; - - time_t _timeStart = 0; ///< Start total stats - time_t _timeLast = 0; ///< Start last period(~1sec) - - double _dnlTotal = 0.0; ///< Bytes to download or 0 if unknown - double _dnlLast = 0.0; ///< Bytes downloaded at period start - double _dnlNow = 0.0; ///< Bytes downloaded now - - int _dnlPercent= 0; ///< Percent completed or 0 if _dnlTotal is unknown - - double _drateTotal= 0.0; ///< Download rate so far - double _drateLast = 0.0; ///< Download rate in last period - - void updateStats( double dltotal = 0.0, double dlnow = 0.0 ) - { - time_t now = time(0); - - // If called without args (0.0), recompute based on the last values seen - if ( dltotal && dltotal != _dnlTotal ) - _dnlTotal = dltotal; - - if ( dlnow && dlnow != _dnlNow ) - { - _dnlNow = dlnow; - } - else if ( !_dnlNow && !_dnlTotal ) - { - // Start time counting as soon as first data arrives. - // Skip the connection / redirection time at begin. - return; - } - - // init or reset if time jumps back - if ( !_timeStart || _timeStart > now ) - _timeStart = _timeLast = now; - - // percentage: - if ( _dnlTotal ) - _dnlPercent = int(_dnlNow * 100 / _dnlTotal); - - // download rates: - _drateTotal = _dnlNow / std::max( int(now - _timeStart), 1 ); - - if ( _timeLast < now ) - { - _drateLast = (_dnlNow - _dnlLast) / int(now - _timeLast); - // start new period - _timeLast = now; - _dnlLast = _dnlNow; - } - else if ( _timeStart == _timeLast ) - _drateLast = _drateTotal; - } - - void reportProgress( bool &cancel ) const - { - if ( report && !(*report)->progress( _dnlPercent, url, _drateTotal, _drateLast ) ) - cancel = true; // user requested abort - } - }; - - - struct MediaNetwork::Request { - - Request ( DownloadSpec &&spec, const zypp::Pathname &path, RequestId id ) : _targetFile(path) { - _targetFile.autoCleanup( true ); - _proto.set_requestid( id ); - - // download to temporary path instead of the real one - spec.setTargetPath( _targetFile.path().asString() ); - *_proto.mutable_spec() = std::move( spec.protoData() ); - } - - void ref () { - _requestCount++; - } - - int unref () { - _requestCount--; - return _requestCount; - } - - void startReporting ( ProgressData &tracker ) { - _report = &tracker; - } - - void stopReporting () { - _report = nullptr; - } - - void setProgress ( double dltotal, double dlnow, bool &cancel ) { - if ( _report ) { - _report->updateStats( dltotal, dlnow ); - _report->reportProgress( cancel ); - } - } - - void reset () { - _result.reset(); - } - - const std::optional &result () const { - return _result; - } - - void setResult ( zypp::proto::DownloadFin &&res ) { - _result = std::move(res); - } - - zypp::proto::Request &proto () { - return _proto; - } - - const zypp::proto::Request &proto () const { - return _proto; - } - - private: - zypp::proto::Request _proto; - std::optional _result; - zypp::filesystem::TmpFile _targetFile; - ProgressData *_report = nullptr; - int _requestCount = 1; - }; - - - /*! - * This object helps us to simulate sync behaviour over async code. We can block waiting - * for a specific message or response while still handling intermediate message that represent - * status updates from concurrently running requests. - * - * The DispatchContext needs to be released after finishing the operation that was requested from - * the sync entry points are done. It will tear down the event loop and release the socket fd so we - * can reuse it at a later point again. - */ - struct MediaNetwork::DispatchContext { - - DispatchContext( MediaNetwork &p ) : parent(&p), ev( zyppng::EventLoop::create() ){ - if ( parent->_socket ) { - MIL_MEDIA << "Reusing open connection" << std::endl; - sock = zyppng::Socket::fromSocket( *parent->_socket, zyppng::Socket::ConnectedState ); - } else { - MIL_MEDIA << "Connecting to backend" << std::endl; - sock = zyppng::Socket::create( AF_UNIX, SOCK_STREAM, 0 ); - parent->_readBuffer.clear(); - - while ( true ) { - sock->connect( std::make_shared( MediaNetworkThread::instance().sockPath(), true ) ); - if ( !sock->waitForConnected() ) { - if ( sock->lastError() == Socket::ConnectionRefused ) - continue; - sock.reset(); - } else { - parent->_socket = sock->nativeSocket(); - MIL_MEDIA << "Connected" << std::endl; - } - break; - } - - } - } - ~DispatchContext() { - // if the socket was disconnected we can not keep the fd around anymore - if ( !sock || sock->state() == Socket::ClosedState ) { - parent->_socket.reset(); - - // if the socket is destroyed the read buffer won't make sense anymore - parent->_readBuffer.clear(); - } else { - // make sure no data is lost before we release the socket descriptor - if ( sock->bytesAvailable() ) - parent->_readBuffer.append( sock->readAll() ); - sock->waitForAllBytesWritten(); - sock->releaseSocket(); - } - } - - /*! - * Send a messagee to the server side, it will be enclosed in a Envelope - * and immediately sent out. - */ - template - bool sendEnvelope ( const T &m ) { - zypp::proto::Envelope env; - - env.set_messagetypename( m.GetTypeName() ); - m.SerializeToString( env.mutable_value() ); - - //DBG << "Preparing to send messagE: " << env.messagetypename() << " " << env.value().size() << std::endl; - - const auto &str = env.SerializeAsString(); - HeaderSizeType msgSize = str.length(); - sock->write( (char *)(&msgSize), sizeof( HeaderSizeType ) ); - sock->write( str.data(), str.size() ); - return true; - } - - /*! - * Executes the EventLoop until a zypp.proto.Status message was received. The - * Status is returned to the caller. The \a handleOtherMsg callback works just like - * in \ref waitFor. - */ - template - zypp::proto::Status waitForStatus ( const RequestId reqId, T &&handleOtherMsg ) { - zypp::proto::Status res; - dispatchUntil( [ &reqId, &res, otherMsgCB = std::forward(handleOtherMsg), this ]( const zypp::proto::Envelope &env ){ - if ( env.messagetypename() == "zypp.proto.Status" ) { - zypp::proto::Status s; - s.ParseFromString( env.value() ); - - // we received a out of band response, there is only ONE active request to the - // server allowed at any time - if ( s.requestid() != reqId ) { - ERR_MEDIA << "Out of band status for a request that was not active" << std::endl; - ZYPP_THROW( zypp::media::MediaException( "Out of band status for a request that was not active" ) ); - } - res = std::move(s); - return true; - } - // not the message we wanted, process it otherwise - otherMsgCB( *this, env ); - return false; - }); - return res; - } - - /*! - * Executes the EventLoop until a Message of type T was received, the - * received message is returned to the caller. - * The \a handleOtherMsg callback is invoked for all messages that are received - * but are of a different type than the one requested. - */ - template - T waitFor ( const RequestId reqId, Callback &&handleOtherMsg ) { - T res; - dispatchUntil( [ &reqId, &res, otherMsgCB = std::forward(handleOtherMsg), this ]( const zypp::proto::Envelope &env ){ - if ( env.messagetypename() == T::default_instance().GetTypeName() ) { - T s; - s.ParseFromString( env.value() ); - if ( s.requestid() == reqId ) { - res = std::move(s); - return true; - } - } - // not the message we wanted, process it otherwise - otherMsgCB( *this, env ); - return false; - }); - return res; - } - - /*! - * This function executes the EventLoop and keep it running until the predicate - * returns true. All messages that are received are passed into the predicate, - * allowing us to implement eventhandling outside of actual async code. - */ - template - void dispatchUntil( Predicate &&predicate ) const - { - if ( !ev || !sock ) - return; - - std::optional pendingMessageSize; - std::exception_ptr excp; - - bool stopRequested = false; - - auto &readBuf = parent->_readBuffer; - - const auto &readMessages = [ &, pred = std::forward(predicate), this ]( ){ - - // read all data from socket into internal buffer - readBuf.append( sock->readAll() ); - - try { - while ( readBuf.size() ) { - if ( !pendingMessageSize ) { - // if we cannot read the message size wait some more - if ( readBuf.size() < sizeof( HeaderSizeType ) ) - return; - - HeaderSizeType msgSize; - readBuf.read( reinterpret_cast( &msgSize ), sizeof( decltype (msgSize) ) ); - pendingMessageSize = msgSize; - } - - // wait for the full message size to be available - if ( readBuf.size() < static_cast( *pendingMessageSize ) ) - return; - - ByteArray message ( *pendingMessageSize, '\0' ); - readBuf.read( message.data(), *pendingMessageSize ); - pendingMessageSize.reset(); - - zypp::proto::Envelope m; - if (! m.ParseFromArray( message.data(), message.size() ) ) { - //abort, we can not recover from this. Bytes might be mixed up on the socket - ZYPP_THROW( zypp::media::MediaException("MediaNetwork backend connection broken.") ); - } - - //MIL << "Dispatching message " << m.DebugString() << std::endl; - - if ( pred(m) ) { - stopRequested = true; - ev->quit(); - return; - } - } - } catch ( ... ) { - // we cannot let the exception travel through the event loop, instead we remember it for later - stopRequested = true; - excp = std::current_exception(); - sock->abort(); - ev->quit(); - } - }; - - auto rrConn = AutoDisconnect( sock->connectFunc( &zyppng::Socket::sigReadyRead, readMessages ) ); - - auto disConn = AutoDisconnect ( sock->connectFunc( &zyppng::Socket::sigDisconnected, [&excp, this](){ - try { - ZYPP_THROW( zypp::media::MediaException("MediaNetwork backend disconnected.") ); - } catch ( ... ) { - // we cannot let the exception travel through the event loop, instead we remember it for later - excp = std::current_exception(); - ev->quit(); - } - }) ); - - readMessages(); // read all pending messages - if ( !stopRequested ) - ev->run(); - - if ( excp ) - std::rethrow_exception( excp ); - - // if we reach this part, we can not have pending message data - if ( pendingMessageSize ) ERR << "This is a bug, message was partially read and event loop closed" << std::endl; - assert( !pendingMessageSize ); - } - - MediaNetwork *parent; - zyppng::EventLoop::Ptr ev; - zyppng::Socket::Ptr sock; - }; - - MediaNetwork::MediaNetwork( const Url & url_r, const zypp::Pathname & attach_point_hint_r ) - : MediaNetworkCommonHandler( url_r, attach_point_hint_r, - "/", // urlpath at attachpoint - true ) // does_download - { - _workingDir.autoCleanup( true ); - - MIL << "MediaNetwork::MediaNetwork(" << url_r << ", " << attach_point_hint_r << ")" << std::endl; - - if( !attachPoint().empty()){ - zypp::PathInfo ainfo(attachPoint()); - zypp::Pathname apath(attachPoint() + "XXXXXX"); - char *atemp = ::strdup( apath.asString().c_str()); - char *atest = NULL; - if( !ainfo.isDir() || !ainfo.userMayRWX() || - atemp == NULL || (atest=::mkdtemp(atemp)) == NULL) { - WAR << "attach point " << ainfo.path() - << " is not useable for " << url_r.getScheme() << std::endl; - setAttachPoint("", true); - } - else if( atest != NULL) - ::rmdir(atest); - - if( atemp != NULL) - ::free(atemp); - } - } - - MediaNetwork::~MediaNetwork() { - DBG << "Releasing MediaNetwork with tmpdir: " << _workingDir.path().asString() << std::endl; - try { release(); } catch(...) {} - - if ( _socket ) { - DBG_MEDIA << "Explicitely calling socket cleanup, this should've happened during release()" << std::endl; - zyppng::eintrSafeCall( ::close, *_socket ); - _socket.reset(); - } - } - - std::unique_ptr MediaNetwork::ensureConnected() const - { - auto ctx = std::make_unique( *const_cast(this) ); - if ( !ctx->sock ) - ZYPP_THROW( zypp::media::MediaException("Failed to connect to backend") ); - return ctx; - } - - void MediaNetwork::disconnectFrom() - { - if ( _socket ) { - zyppng::eintrSafeCall( ::close, *_socket ); - _socket.reset(); - } - - _readBuffer.clear(); - _requests.clear(); - _nextRequestId = 0; - - MediaHandler::disconnectFrom(); - } - - void MediaNetwork::attachTo(bool next) - { - if ( next ) - ZYPP_THROW( zypp::media::MediaNotSupportedException(_url) ); - - if ( !_url.isValid() ) - ZYPP_THROW( zypp::media::MediaBadUrlException(_url) ); - - if( !NetworkRequestDispatcher::supportsProtocol( _url ) ) { - std::string msg("Unsupported protocol '"); - msg += _url.getScheme(); - msg += "'"; - ZYPP_THROW( zypp::media::MediaBadUrlException(_url, msg) ); - } - - if( !isUseableAttachPoint( attachPoint() ) ) - { - setAttachPoint( createAttachPoint(), true ); - } - - disconnectFrom(); - - // FIXME: need a derived class to propelly compare url's - zypp::media::MediaSourceRef media( new zypp::media::MediaSource(_url.getScheme(), _url.asString()) ); - setMediaSource(media); - } - - void MediaNetwork::releaseFrom(const std::string &) - { - disconnect(); - } - - Url MediaNetwork::getFileUrl( const zypp::Pathname & filename_r ) const - { - // Simply extend the URLs pathname. An 'absolute' URL path - // is achieved by encoding the leading '/' in an URL path: - // URL: ftp://user@server -> ~user - // URL: ftp://user@server/ -> ~user - // URL: ftp://user@server// -> ~user - // URL: ftp://user@server/%2F -> / - // ^- this '/' is just a separator - Url newurl( _url ); - newurl.setPathName( ( zypp::Pathname("./"+_url.getPathName()) / filename_r ).asString().substr(1) ); - return newurl; - } - - void MediaNetwork::handleRequestResult(const Request &req, const zypp::filesystem::Pathname &filename ) const - { - if ( !req.result() ) { - ZYPP_THROW( zypp::media::MediaCurlException( req.proto().spec().url(), "Request did not return a result" , "" ) ); - } - - if ( !req.result()->has_error() ) - return; - - const auto &err = req.result()->error(); - if ( err.error() != NetworkRequestError::NoError ) { - Url reqUrl = err.extra_info().at("requestUrl"); - switch ( err.error() ) - { - case NetworkRequestError::Unauthorized: { - std::string hint = err.extra_info().at("authHint"); - ZYPP_THROW( zypp::media::MediaUnauthorizedException( - reqUrl, err.errordesc(), err.nativeerror(), hint - )); - break; - } - case NetworkRequestError::TemporaryProblem: - ZYPP_THROW( zypp::media::MediaTemporaryProblemException(reqUrl) ); - break; - case NetworkRequestError::Timeout: - ZYPP_THROW( zypp::media::MediaTimeoutException(reqUrl) ); - break; - case NetworkRequestError::Forbidden: - ZYPP_THROW( zypp::media::MediaForbiddenException(reqUrl, err.errordesc())); - break; - case NetworkRequestError::NotFound: - ZYPP_THROW( zypp::media::MediaFileNotFoundException( reqUrl, filename ) ); - break; - case NetworkRequestError::ExceededMaxLen: - ZYPP_THROW( zypp::media::MediaFileSizeExceededException( reqUrl, 0 ) ); - break; - default: - break; - } - ZYPP_THROW( zypp::media::MediaCurlException( reqUrl, err.errordesc(), err.nativeerror() ) ); - } - ZYPP_THROW( zypp::media::MediaCurlException( req.proto().spec().url(), err.errordesc() , "" ) ); - } - - bool MediaNetwork::retry( DispatchContext &ctx, Request &req ) const - { - const auto &res = req.result(); - if ( !res || !res->has_error() ) - return false; - - const auto &resErr = res->error(); - if ( resErr.error() != NetworkRequestError::AuthFailed && resErr.error() != NetworkRequestError::Unauthorized ) - return false; - - const std::string &availAuth = resErr.extra_info().count("authHint") ? resErr.extra_info().at("authHint") : ""; - - zypp::callback::SendReport auth_report; - - zypp::media::CredentialManager cm( zypp::media::CredManagerOptions( zypp::ZConfig::instance().repoManagerRoot()) ); - auto authDataPtr = cm.getCred( _url ); - - NetworkAuthData_Ptr curlcred( new NetworkAuthData() ); - curlcred->setUrl( _url ); - - // preset the username if present in current url - if ( !_url.getUsername().empty() ) - curlcred->setUsername(_url.getUsername()); - // if CM has found some credentials, preset the username from there - else if ( authDataPtr ) - curlcred->setUsername( authDataPtr->username() ); - - // set available authentication types - // might be needed in prompt - curlcred->setAuthType( availAuth ); - - std::string prompt_msg = zypp::str::Format( _("Authentication required for '%s'") ) % _url.asString(); - - // the auth data that was used in the request is older than whats in the Credential manager, - // so instead of asking the user for creds we first try to use the new ones - if ( res->last_auth_timestamp() > 0 && res->last_auth_timestamp() < authDataPtr->lastDatabaseUpdate() ) { - _settings.setUsername( authDataPtr->username() ); - _settings.setPassword( authDataPtr->password() ); - retryRequest( ctx, req ); - return true; - } else { - - // ask user - if ( auth_report->prompt( _url, prompt_msg, *curlcred ) ) { - DBG << "callback answer: retry" << std::endl - << "CurlAuthData: " << *curlcred << std::endl; - - if ( curlcred->valid() ) { - - _settings.setUsername( curlcred->username() ); - _settings.setPassword( curlcred->password() ); - - // set available authentication types from the exception - if ( curlcred->authType() == CURLAUTH_NONE ) - curlcred->setAuthType( availAuth ); - - // set auth type (seems this must be set _after_ setting the userpwd) - if ( curlcred->authType() != CURLAUTH_NONE) { - // FIXME: only overwrite if not empty? - _settings.setAuthType( curlcred->authTypeAsString() ); - } - - cm.addCred( *curlcred ); - cm.save(); - - // we need to restart all already failed requests so we do not double ask - // for credentials - for ( auto &r : _requests ) { - if ( &r == &req ) - continue; - if ( !r.result() || !r.result()->has_error() ) - continue; - const auto rErr = r.result()->error(); - if ( rErr.error() != NetworkRequestError::AuthFailed && rErr.error() != NetworkRequestError::Unauthorized ) { - retryRequest( ctx, r ); - } - } - - retryRequest( ctx, req ); - return true; - } - } - } - return false; - } - - void MediaNetwork::handleStreamMessage( DispatchContext &ctx, const zypp::proto::Envelope &e ) const - { - RequestId dlId = -1; - bool cancel = false; - - const auto &mName = e.messagetypename(); - if ( mName == "zypp.proto.DownloadStart" ) { - zypp::proto::DownloadStart st; - st.ParseFromString( e.value() ); - - auto r = findRequest( st.requestid() ); - if ( !r ) - return; - - MIL << "Received the download started event for file "<< zypp::Url( r->proto().spec().url() ) <<" from server." << std::endl; - dlId = st.requestid(); - r->setProgress( 0.0, 0.0, cancel ); - - } else if ( mName == "zypp.proto.DownloadProgress" ) { - zypp::proto::DownloadProgress prog; - prog.ParseFromString( e.value() ); - - auto r = findRequest( prog.requestid() ); - if ( !r ) - return; - - //MIL << "Received progress for download: " << zypp::Url( r->url() ) << std::endl; - dlId = prog.requestid(); - r->setProgress( prog.total(), prog.now(), cancel ); - - } else if ( mName == "zypp.proto.DownloadFin" ) { - zypp::proto::DownloadFin fin; - fin.ParseFromString( e.value() ); - - auto r = findRequest( fin.requestid() ); - if ( !r ) - return; - - MIL << "Received FIN for download: " << zypp::Url( r->proto().spec().url() ) << std::endl; - r->setResult( std::move(fin) ); - } else { - // error , unknown message - WAR << "Received unexpected message... ignoring" << std::endl; - } - - if ( dlId >= 0 && cancel ) { - zypp::proto::CancelDownload cl; - cl.set_requestid( dlId ); - ctx.sendEnvelope( cl ); - ctx.sock->waitForAllBytesWritten(); - } - } - - MediaNetwork::Request MediaNetwork::makeRequest ( const zypp::OnMediaLocation &loc ) const - { - DBG << loc.filename().asString() << std::endl; - - if(!_url.isValid()) - ZYPP_THROW(zypp::media::MediaBadUrlException(_url)); - - if(_url.getHost().empty()) - ZYPP_THROW(zypp::media::MediaBadUrlEmptyHostException(_url)); - - Url url( getFileUrl(loc.filename()) ); - - DownloadSpec spec( url, loc.filename(), loc.downloadSize() ); - spec.setTransferSettings( _settings ) - .setHeaderSize( loc.headerSize() ) - .setHeaderChecksum( loc.headerChecksum() ) - .setDeltaFile( loc.deltafile() ); - - // we are downloading the file into a temporary location first - Request fileReq( std::move(spec), _workingDir, ++_nextRequestId ); - fileReq.proto().set_streamprogress( false ); - return fileReq; - } - - void MediaNetwork::trackRequest ( DispatchContext &ctx, Request &req ) const - { - zypp::proto::DownloadFin result; - - bool retry = true; - while ( retry ) { - - retry = false; // try to stop after this iteration - - if ( !req.result() ) { - result = ctx.waitFor( req.proto().requestid(), std::bind( &MediaNetwork::handleStreamMessage, this, std::placeholders::_1, std::placeholders::_2 ) ); - req.setResult( std::move(result) ); - } - - retry = this->retry( ctx, req ); - } - } - - - - void MediaNetwork::retryRequest ( DispatchContext &ctx, MediaNetwork::Request &req ) const - { - req.reset(); - // update request with new settings - *req.proto().mutable_spec()->mutable_settings() = _settings.protoData(); - ctx.sendEnvelope( req.proto() ); - const auto &s = ctx.waitForStatus( req.proto().requestid(), std::bind( &MediaNetwork::handleStreamMessage, this, std::placeholders::_1, std::placeholders::_2 ) ); - if ( s.code() != zypp::proto::Status::Ok ) { - ZYPP_THROW( zypp::media::MediaException( s.reason() ) ); - } - } - - MediaNetwork::Request *MediaNetwork::findRequest( const Url url ) const - { - const auto &i = std::find_if( _requests.begin(), _requests.end(), [ &url ]( const auto &r ) { - return ( url == zyppng::Url( r.proto().spec().url() ) ); - }); - if ( i == _requests.end() ) - return nullptr; - return &(*i); - } - - MediaNetwork::Request *MediaNetwork::findRequest(const RequestId id ) const - { - const auto &i = std::find_if( _requests.begin(), _requests.end(), [ &id ]( const auto &r ) { - return ( id == r.proto().requestid() ); - }); - if ( i == _requests.end() ) - return nullptr; - return &(*i); - } - - bool MediaNetwork::getDoesFileExist( const zypp::filesystem::Pathname &filename ) const - { - - auto ctx = ensureConnected(); - - // here we always send a new request without even considering probably already existing ones - auto req = makeRequest( filename ); - req.proto().mutable_spec()->set_checkexistanceonly( true ); - req.proto().set_prioritize( true ); - - - ctx->sendEnvelope( req.proto() ); - const auto &s = ctx->waitForStatus( req.proto().requestid(), std::bind( &MediaNetwork::handleStreamMessage, this, std::placeholders::_1, std::placeholders::_2 ) ); - if ( s.code() != zypp::proto::Status::Ok ) { - ZYPP_THROW( zypp::media::MediaException( s.reason() ) ); - } - - try - { - trackRequest( *ctx, req ); - - //this will throw if the file does not exist - handleRequestResult( req, filename ); - } catch ( const zypp::media::MediaFileNotFoundException &e ) { - return false; - } catch ( const zypp::media::MediaException &e ) { - // some error, we are not sure about file existence, rethrw - ZYPP_RETHROW(e); - } - - return true; - } - - - void MediaNetwork::getFile( const zypp::OnMediaLocation &file ) const - { - const auto &filename = file.filename(); - auto ctx = ensureConnected(); - - const auto url = getFileUrl( filename ); - auto downloadReq = findRequest( url ); - - zypp::callback::SendReport report; - ProgressData data( url, &report ); - - report->start( url, filename ); - - if ( !downloadReq ) { - auto newReq = makeRequest( file ); - - newReq.proto().set_streamprogress( true ); - newReq.proto().set_prioritize( true ); - - ctx->sendEnvelope( newReq.proto() ); - - const auto &s = ctx->waitForStatus( newReq.proto().requestid(), std::bind( &MediaNetwork::handleStreamMessage, this, std::placeholders::_1, std::placeholders::_2 ) ); - if ( s.code() != zypp::proto::Status::Ok && s.code() ) { - ZYPP_THROW( zypp::media::MediaException( s.reason() ) ); - } - - _requests.push_back( newReq ); - downloadReq = &_requests.back(); - - } else { - // if the download is not yet finished we request to track it - if ( !downloadReq->result() ) { - - zypp::proto::SubscribeProgress sub; - sub.set_requestid( downloadReq->proto().requestid() ); - sub.set_prioritize( true ); - ctx->sendEnvelope( sub ); - - const auto &s = ctx->waitForStatus( sub.requestid(), std::bind( &MediaNetwork::handleStreamMessage, this, std::placeholders::_1, std::placeholders::_2 ) ); - - // status could be unknown ID if the request was already in finished state and the message was stuck in our socket! - if ( s.code() != zypp::proto::Status::Ok && s.code() != zypp::proto::Status::UnknownId ) { - ZYPP_THROW( zypp::media::MediaException( s.reason() ) ); - } - } - } - - downloadReq->startReporting( data ); - zypp::OnScopeExit cleanup( [&](){ downloadReq->stopReporting(); }); - - try { - trackRequest( *ctx, *downloadReq ); - - //this will throw if the file does not exist - handleRequestResult( *downloadReq, filename ); - } catch ( zypp::media::MediaUnauthorizedException & ex_r ) { - report->finish(url, zypp::media::DownloadProgressReport::ACCESS_DENIED, ex_r.asUserHistory()); - ZYPP_RETHROW(ex_r); - } - // unexpected exception - catch ( zypp::media::MediaException & excpt_r ) - { - zypp::media::DownloadProgressReport::Error reason = zypp::media::DownloadProgressReport::ERROR; - if( typeid(excpt_r) == typeid( zypp::media::MediaFileNotFoundException ) || - typeid(excpt_r) == typeid( zypp::media::MediaNotAFileException ) ) - { - reason = zypp::media::DownloadProgressReport::NOT_FOUND; - } - report->finish(url, reason, excpt_r.asUserHistory()); - ZYPP_RETHROW(excpt_r); - } - - // seems we were successful - const auto &targetPath = localPath(filename).absolutename(); - - const auto errCode = zypp::filesystem::assert_dir( targetPath.dirname() ); - if( errCode ) { - std::string err = zypp::str::Str() << "assert_dir " << targetPath.dirname() << " failed"; - DBG << err << std::endl; - ZYPP_THROW( zypp::media::MediaCurlException( url, err, "" ) ); - } - - if( zypp::filesystem::hardlinkCopy( downloadReq->proto().spec().targetpath(), targetPath ) != 0 ) { - std::string err = zypp::str::Str() << "Failed to hardlinkCopy the requested file <<" << downloadReq->proto().spec().targetpath() << " to the targetPath " << targetPath; - DBG << err << std::endl; - ZYPP_THROW( zypp::media::MediaCurlException( url, err, "" ) ); - } - - if (::chmod( targetPath.c_str(), zypp::filesystem::applyUmaskTo( 0644 ))) { - ERR << "Failed to chmod file " << targetPath << std::endl; - } - - // check if the file was required the same number of times it was requested - if ( downloadReq->unref() <= 0 ) { - _requests.remove_if( [ & ]( const auto &r ){ - return ( r.proto().requestid() == downloadReq->proto().requestid() ); - }); - } - - report->finish(url, zypp::media::DownloadProgressReport::NO_ERROR, ""); - } - - void MediaNetwork::getDir(const zypp::filesystem::Pathname &dirname, bool recurse_r) const - { - //we could make this download concurrently, but it is not used anywhere in the code, so why bother - zypp::filesystem::DirContent content; - getDirInfo( content, dirname, /*dots*/false ); - - for ( zypp::filesystem::DirContent::const_iterator it = content.begin(); it != content.end(); ++it ) { - zypp::Pathname filename = dirname + it->name; - int res = 0; - - switch ( it->type ) { - case zypp::filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all - case zypp::filesystem::FT_FILE: - getFile( zypp::OnMediaLocation( filename ) ); - break; - case zypp::filesystem::FT_DIR: // newer directory.yast contain at least directory info - if ( recurse_r ) { - getDir( filename, recurse_r ); - } else { - res = assert_dir( localPath( filename ) ); - if ( res ) { - WAR << "Ignore error (" << res << ") on creating local directory '" << localPath( filename ) << "'" << std::endl; - } - } - break; - default: - // don't provide devices, sockets, etc. - break; - } - } - } - - void MediaNetwork::precacheFiles(const std::vector &files) - { - zypp::proto::Prefetch req; - req.set_requestid( ++_nextRequestId ); - - std::vector sentRequests; - - for ( const auto &file : files ) { - const auto url = getFileUrl( file.filename() ); - auto existingReq = findRequest( url ); - if ( existingReq ) - existingReq->ref(); - else { - Request fileReq = makeRequest( file ); - fileReq.proto().set_streamprogress( false ); - *req.mutable_requests()->Add() = fileReq.proto(); - sentRequests.push_back( fileReq ); - } - } - - if ( req.requests().size() ) { - - auto ctx = ensureConnected(); - ctx->sendEnvelope( req ); - - const auto &status = ctx->waitForStatus( req.requestid(), std::bind( &MediaNetwork::handleStreamMessage, this, std::placeholders::_1, std::placeholders::_2 ) ); - if ( status.code() == zypp::proto::Status::Ok ) { - MIL_MEDIA << "Request was acknowledged by server, downloads should start soon" << std::endl; - _requests.insert( _requests.end(), std::move_iterator(sentRequests.begin()), std::move_iterator(sentRequests.end()) ); - return; - } - - MIL << "Request failed " << status.reason() << std::endl; - ZYPP_THROW( zypp::media::MediaException( status.reason() ) ); - } - } - - void MediaNetwork::getDirInfo(std::list &retlist, const zypp::filesystem::Pathname &dirname, bool dots) const - { - getDirectoryYast( retlist, dirname, dots ); - } - - void MediaNetwork::getDirInfo(zypp::filesystem::DirContent &retlist, const zypp::filesystem::Pathname &dirname, bool dots) const - { - getDirectoryYast( retlist, dirname, dots ); - } - - bool MediaNetwork::checkAttachPoint(const zypp::Pathname &apoint) const - { - return MediaHandler::checkAttachPoint( apoint, true, true); - } - - -} diff --git a/zypp/zyppng/media/medianetwork.h b/zypp/zyppng/media/medianetwork.h deleted file mode 100644 index d86796a..0000000 --- a/zypp/zyppng/media/medianetwork.h +++ /dev/null @@ -1,94 +0,0 @@ -/*---------------------------------------------------------------------\ -| ____ _ __ __ ___ | -| |__ / \ / / . \ . \ | -| / / \ V /| _/ _/ | -| / /__ | | | | | | | -| /_____||_| |_| |_| | -| | -----------------------------------------------------------------------/ -* -* This file contains private API, this might break at any time between releases. -* You have been warned! -* -*/ -#ifndef ZYPPNG_MEDIA_MEDIANETWORK_H_INCLUDED -#define ZYPPNG_MEDIA_MEDIANETWORK_H_INCLUDED - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace zyppng { - -class Download; -class Downloader; -class Socket; -class EventDispatcher; - -using RequestId = uint32_t; - -class MediaNetwork : public zypp::media::MediaNetworkCommonHandler -{ -public: - MediaNetwork(const Url &url_r, const zypp::Pathname &attach_point_hint_r); - virtual ~MediaNetwork() override; - - // MediaHandler interface -protected: - void attachTo(bool next) override; - void releaseFrom(const std::string &ejectDev) override; - void getFile ( const zypp::OnMediaLocation &file ) const override; - void getDir(const zypp::filesystem::Pathname &dirname, bool recurse_r) const override; - void getDirInfo(std::list &retlist, const zypp::filesystem::Pathname &dirname, bool dots) const override; - void getDirInfo(zypp::filesystem::DirContent &retlist, const zypp::filesystem::Pathname &dirname, bool dots) const override; - bool getDoesFileExist(const zypp::filesystem::Pathname &filename) const override; - bool checkAttachPoint(const zypp::Pathname &apoint) const override; - - - Url getFileUrl(const zypp::Pathname &filename_r) const; - -private: - struct ProgressData; - struct DispatchContext; - struct Request; - - std::unique_ptr ensureConnected () const; - bool retry ( DispatchContext &ctx, Request &req ) const; - void retryRequest ( DispatchContext &ctx, Request &req ) const; - void handleStreamMessage ( DispatchContext &ctx, const zypp::proto::Envelope &e ) const; - void handleRequestResult ( const Request &req , const zypp::filesystem::Pathname &filename ) const; - - Request makeRequest ( const zypp::OnMediaLocation &loc ) const; - void trackRequest ( DispatchContext &ctx, Request &req ) const ; - - Request *findRequest ( const zyppng::Url url ) const; - Request *findRequest ( const RequestId id ) const; - - -private: - zypp::filesystem::TmpDir _workingDir; - mutable RequestId _nextRequestId = 0; - mutable std::list _requests; - mutable std::optional _socket; - mutable IOBuffer _readBuffer; - - // MediaHandler interface -protected: - void disconnectFrom() override; - - // MediaHandler interface -public: - void precacheFiles(const std::vector &files) override; -}; - -} - - - -#endif diff --git a/zypp/zyppng/media/medianetworkserver.cc b/zypp/zyppng/media/medianetworkserver.cc deleted file mode 100644 index 1612f4d..0000000 --- a/zypp/zyppng/media/medianetworkserver.cc +++ /dev/null @@ -1,492 +0,0 @@ -/*---------------------------------------------------------------------\ -| ____ _ __ __ ___ | -| |__ / \ / / . \ . \ | -| / / \ V /| _/ _/ | -| / /__ | | | | | | | -| /_____||_| |_| |_| | -| | -----------------------------------------------------------------------*/ -#include "private/medianetworkserver_p.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include -#include - -#include -#include -#include - -#include -#include - -#undef ZYPP_BASE_LOGGER_LOGGROUP -#define ZYPP_BASE_LOGGER_LOGGROUP "zypp::MediaNetworkServer" - -namespace internal { - /** - * initialized only once, this gets the agent string - * which also includes the curl version - * - * Defined in MediaCurl.cc - */ - const char * agentString(); -} - -namespace zyppng { - - using HeaderSizeType = zyppng::rpc::HeaderSizeType; - - MediaNetworkServer::MediaNetworkServer( ) - : _downloadManager( std::make_shared( MirrorControl::create() ) ) - { - _downloadManager->setCredManagerOptions( zypp::media::CredManagerOptions( zypp::ZConfig::instance().repoManagerRoot()) ); - auto dispatcher = _downloadManager->requestDispatcher(); - if ( dispatcher ) { - dispatcher->setAgentString( ::internal::agentString() ); - dispatcher->setHostSpecificHeader("download.opensuse.org", "X-ZYpp-AnonymousId", zypp::Target::anonymousUniqueId( zypp::Pathname()/*guess root*/ ) ); - dispatcher->setHostSpecificHeader("download.opensuse.org", "X-ZYpp-DistributionFlavor", zypp::Target::distributionFlavor( zypp::Pathname()/*guess root*/ ) ); - } else { - // this branch should never be executed - ERR << "Unable to initialize user agent string and zypp http headers." << std::endl; - } - } - - void MediaNetworkServer::listen( const std::string &sockPath ) - { - if ( _serverSocket ) - return; - - _serverSocket = zyppng::Socket::create( AF_UNIX, SOCK_STREAM, 0 ); - - // bind to a abstract unix domain socket address, which means we do not need to care about cleaning it up - _serverSocket->bind( std::make_shared( sockPath, true ) ); - _serverSocket->Base::connect( &zyppng::Socket::sigIncomingConnection, *this, &MediaNetworkServer::onIncomingConnection ); - _serverSocket->listen(); - - MIL << "MediaNetworkServer started to listen for connections " << std::endl; - } - - std::shared_ptr MediaNetworkServer::downloader() - { - return _downloadManager; - } - - void MediaNetworkServer::onIncomingConnection() - { - auto sock = _serverSocket->accept(); - if ( !sock ) - return; - - MIL << "MediaNetworkServer received new connection " << sock.get() << std::endl; - - auto client = std::make_shared( *this, std::move(sock) ); - client->connectFunc( &MediaNetworkConn::sigDisconnected, [ this, wp = std::weak_ptr(client) ](){ - auto p = wp.lock(); - if (!p) { - MIL_MEDIA << "MediaNetworkServer client disconnected but the pointer was already deleted" << std::endl; - return; - } - - MIL << "MediaNetworkServer client disconnected " << p.get() << std::endl; - - _clients.erase( - std::remove_if( _clients.begin(), _clients.end(), [ &p ]( auto listElem ){ - return p.get() == listElem.get(); - } - )); - }, *this); - - _clients.push_back( client ); - } - - struct MediaNetworkConn::Request { - Request ( ) { } - ~Request() { - clearConnections(); - } - - void clearConnections () { - std::for_each( trackingConns.begin(), trackingConns.end(), []( auto &conn){ conn.disconnect(); } ); - trackingConns.clear(); - } - - zypp::proto::Request request; - std::shared_ptr dl; - std::vector trackingConns; - }; - - - MediaNetworkConn::MediaNetworkConn( MediaNetworkServer &server, std::shared_ptr &&socket ) : _server(server), _connection ( std::move( socket ) ) - { - MIL_MEDIA << "Initializing Connection object " << std::endl; - _socketSigConns.insert( _socketSigConns.end(), { - _connection->Base::connect( &Socket::sigReadyRead, *this, &MediaNetworkConn::onReadyRead ), - _connection->Base::connect( &Socket::sigDisconnected, *this, &MediaNetworkConn::onDisconnected ), - _connection->Base::connect( &Socket::sigError, *this, &MediaNetworkConn::onError ) - } ); - - //make sure we read possibly available data - onReadyRead(); - } - - MediaNetworkConn::~MediaNetworkConn() - { - MIL_MEDIA << "Closing connection " << std::endl; - while( _requests.size() ) { - auto req = _requests.front(); - // make sure we do not receive signals while shutting down - // this would break due to weak_from_this - req->clearConnections(); - req->dl->cancel(); - - // still make sure we actually send all signals about finished downloads. - trackedDlFinished( *req ); - } - MIL_MEDIA << "Closing connection done!" << std::endl; - - // clean up our socket connections, otherwise we receive signals we do not want to handle anymore - std::for_each( _socketSigConns.begin(), _socketSigConns.end(), []( auto &conn){ conn.disconnect(); } ); - _socketSigConns.clear(); - } - - SignalProxy MediaNetworkConn::sigDisconnected() - { - return _disconnected; - } - - void MediaNetworkConn::onDisconnected() - { - MIL_MEDIA << "Socket was closed, requesting cleanup." << std::endl; - _disconnected.emit(); - } - - void MediaNetworkConn::onReadyRead() - { - const auto &sendStatus = [this]( RequestId id, auto code, const auto &reason) { - zypp::proto::Status status; - status.set_requestid( id ); - status.set_code( code ); - status.set_reason( reason ); - sendMessage( status ); - }; - - const auto &makeNewRequest = [this] ( zypp::proto::Request &&req ){ - auto newRequest = std::make_shared( ); - newRequest->request = std::move( req ); - newRequest->dl = _server.downloader()->downloadFile( newRequest->request.spec() ); - newRequest->dl->start(); - - if ( newRequest->request.prioritize() ) - newRequest->dl->prioritize(); - - trackRequest( *newRequest ); - - _requests.push_back( std::move(newRequest) ); - return _requests.back(); - }; - - // read until buffers are empty - while ( _connection->bytesAvailable() ) { - - // MIL << "Server has bytes" << std::endl; - - if ( !_pendingMessageSize ) { - // if we cannot read the message size wait some more - if ( _connection->bytesAvailable() < sizeof( HeaderSizeType ) ) - return; - - HeaderSizeType msgSize; - _connection->read( reinterpret_cast( &msgSize ), sizeof( decltype (msgSize) ) ); - _pendingMessageSize = msgSize; - } - - // wait for the full message size to be available - if ( _connection->bytesAvailable() < static_cast( *_pendingMessageSize ) ) - return; - - ByteArray message = _connection->read( *_pendingMessageSize ); - _pendingMessageSize.reset(); - - zypp::proto::Envelope m; - if (! m.ParseFromArray( message.data(), message.size() ) ) { - //send error and close, we can not recover from this. Bytes might be mixed up on the socket - sendStatus( -1, zypp::proto::Status::InvalidMessage, "This message is misformed." ); - _connection->close(); - return; - } - - //DBG << "Server recv: " << m.messagetypename() << " " << m.value().size() << std::endl; - - const auto &mName = m.messagetypename(); - if ( mName == "zypp.proto.Request" ) { - - zypp::proto::Request prefetchReq; - if (! prefetchReq.ParseFromString( m.value() ) ) { - sendStatus( -1, zypp::proto::Status::MalformedRequest, "Unable to deserialize the request." ); - continue; - } - const auto &r = makeNewRequest( std::move(prefetchReq) ); - sendStatus( r->request.requestid(), zypp::proto::Status::Ok, "OK" ); - - } else if ( mName == "zypp.proto.Prefetch" ) { - - zypp::proto::Prefetch prefetchReq; - if (! prefetchReq.ParseFromString( m.value() ) ) { - sendStatus( -1, zypp::proto::Status::MalformedRequest, "Unable to deserialize the request." ); - continue; - } - - for ( auto &file : *prefetchReq.mutable_requests() ) { - makeNewRequest( std::move(file) ); - } - sendStatus( prefetchReq.requestid(), zypp::proto::Status::Ok, "OK" ); - - } else if ( mName == "zypp.proto.CancelDownload" ) { - zypp::proto::CancelDownload cancel; - if (! cancel.ParseFromString( m.value() ) ) { - sendStatus( -1, zypp::proto::Status::MalformedRequest, "Unable to deserialize the request." ); - continue; - } - - for ( auto &req : _requests ) { - if ( req->request.requestid() == cancel.requestid() ) { - auto lock = req; - lock->dl->cancel(); - break; - } - } - sendStatus( cancel.requestid(), zypp::proto::Status::Ok, "OK" ); - - } else if ( mName == "zypp.proto.NewAuthDataAvailable" ) { - zypp::proto::NewAuthDataAvailable data; - if (! data.ParseFromString( m.value() ) ) { - sendStatus( -1, zypp::proto::Status::MalformedRequest, "Unable to deserialize the request." ); - continue; - } - - MIL_MEDIA << "Got new Auth data, restarting failed requests" << std::endl; - - // we got new auth data in the CredentialManager, lets restart all our requests that are currently - // in failed auth state. - for ( auto &req : _requests ) { - if ( req->dl->state() == Download::Finished && req->dl->hasError() - && ( req->dl->lastRequestError().type() == NetworkRequestError::AuthFailed || req->dl->lastRequestError().type() == NetworkRequestError::Unauthorized ) ) { - MIL_MEDIA << "Found request waiting for Auth, restarting " << req->dl->spec().url() << std::endl; - req->dl->spec().settings().protoData() = data.settings(); - req->dl->start(); - } - } - sendStatus( data.requestid(), zypp::proto::Status::Ok, "OK" ); - - } else if ( mName == "zypp.proto.SubscribeProgress" ) { - zypp::proto::SubscribeProgress sub; - if (! sub.ParseFromString( m.value() ) ) { - sendStatus( -1, zypp::proto::Status::MalformedRequest, "Unable to deserialize the request." ); - continue; - } - - bool found = false; - for ( auto &req : _requests ) { - if ( req->request.requestid() == sub.requestid() ) { - req->request.set_streamprogress( true ); - if ( sub.prioritize() ) - req->dl->prioritize(); - sendStatus( sub.requestid(), zypp::proto::Status::Ok, "OK" ); - found = true; - break; - } - } - - if ( !found ) - sendStatus( sub.requestid(), zypp::proto::Status::UnknownId, "Could not find the request ID in running requests" ); - - } else if ( mName == "zypp.proto.UnSubscribeProgress" ) { - zypp::proto::UnSubscribeProgress unSub; - if (! unSub.ParseFromString( m.value() ) ) { - sendStatus( -1, zypp::proto::Status::MalformedRequest, "Unable to deserialize the request." ); - return; - } - - bool found = false; - for ( auto &req : _requests ) { - if ( req->request.requestid() == unSub.requestid() ) { - req->request.set_streamprogress( false ); - found = true; - sendStatus( unSub.requestid(), zypp::proto::Status::Ok, "OK" ); - break; - } - } - if ( !found ) - sendStatus( unSub.requestid(), zypp::proto::Status::UnknownId, "Could not find the request ID in running requests" ); - - } else { - sendStatus( -1, zypp::proto::Status::UnknownRequest, "This request type is not known." ); - } - } - } - - void MediaNetworkConn::onError( Socket::SocketError err ) - { - MIL << "Socket received error: " << err << std::endl; - _connection->disconnect(); - } - - void MediaNetworkConn::trackRequest( Request &r ) - { - r.trackingConns = { - r.dl->connectFunc( &Download::sigStarted, [ this, &r ]( auto & ) { - this->signalRequestStarted( r ); - }, *this), - - r.dl->connectFunc( &Download::sigAlive, [ this, &r ]( auto &, off_t now ) { - - if ( !r.request.streamprogress() ) - return; - - zypp::proto::DownloadProgress prog; - prog.set_requestid( r.request.requestid() ); - prog.set_url( r.request.spec().url() ); - prog.set_now( now ); - this->sendMessage( prog ); - }, *this ), - - r.dl->connectFunc( &Download::sigProgress, [ this, &r ]( auto &, off_t total, off_t now ) { - - if ( !r.request.streamprogress() ) { - return; - } - - zypp::proto::DownloadProgress prog; - prog.set_requestid( r.request.requestid() ); - prog.set_url( r.request.spec().url() ); - prog.set_now( now ); - prog.set_total( total ); - this->sendMessage( prog ); - }, *this ), - r.dl->connectFunc( &Download::sigFinished, [ this, &r ]( auto & ) { - this->trackedDlFinished( r ); - }, *this ) - }; - } - - void MediaNetworkConn::signalRequestStarted(Request &r) - { - zypp::proto::DownloadStart start; - start.set_requestid( r.request.requestid() ); - start.set_url( r.request.spec().url() ); - this->sendMessage( start ); - } - - void MediaNetworkConn::trackedDlFinished( MediaNetworkConn::Request &r ) - { - MIL << "Download finished by MediaNetworkServer: " << r.request.spec().url() << std::endl; - - zypp::proto::DownloadFin fin; - fin.set_requestid( r.request.requestid() ); - if ( r.dl->lastAuthTimestamp() > 0 ) - fin.set_last_auth_timestamp( r.dl->lastAuthTimestamp() ); - r.clearConnections(); - - std::string err; - if ( !r.dl->hasError() ) { - fin.clear_error(); - } else { - const auto &lerr = r.dl->lastRequestError(); - fin.mutable_error()->set_error( lerr.type() ); - fin.mutable_error()->set_errordesc( err ); - fin.mutable_error()->set_nativeerror( lerr.nativeErrorString() ); - - // this is rather ugly, we need to convert the extra infos to string - // we will only handle the ones the client actually cares about - fin.mutable_error()->mutable_extra_info()->insert( { "requestUrl", lerr.extraInfoValue("requestUrl", r.dl->spec().url()).asString() } ); - if ( lerr.type() == NetworkRequestError::Unauthorized ) - fin.mutable_error()->mutable_extra_info()->insert( { "authHint", lerr.extraInfoValue("authHint", std::string()) } ); - } - - this->sendMessage( fin ); - - MIL_MEDIA << "Download was finished, releasing all ressources." << std::endl; - _requests.erase ( - std::remove_if( _requests.begin(), _requests.end(), [ &r ]( const auto &reqInList ){ - return reqInList.get() == &r; - } - )); - return; - // ATTENTION, r will be dangling from here on out - } - - template< typename T > - void MediaNetworkConn:: - sendMessage( T &m ) - { - zypp::proto::Envelope env; - env.set_messagetypename( m.GetTypeName() ); - m.SerializeToString( env.mutable_value() ); - - //DBG << "Sending message\n" << env.DebugString() << std::endl; - - std::string data = env.SerializeAsString(); - const HeaderSizeType size = data.size(); - - _connection->write( reinterpret_cast( &size ), sizeof(decltype(size)) ); - _connection->write( data.data(), size ); - } - - MediaNetworkThread::MediaNetworkThread() - { - MIL_MEDIA << "Initializing MediaNetworkThread" << std::endl; - //start up thread - _t = std::thread( [this](){ threadMain(); }); - } - - void MediaNetworkThread::threadMain() - { - // force the kernel to pick another thread to handle those signals - zyppng::blockSignalsForCurrentThread( { SIGTERM, SIGINT, SIGPIPE, } ); - - zyppng::ThreadData::current().setName("Zypp-MediaNetwork"); - - auto dispatch = zyppng::EventLoop::create(); - - auto server = std::make_shared(); - server->listen( sockPath() ); - - // we are using a pipe to wake up from the event loop, the SocketNotifier will throw a signal every - // time there is data available - auto sNotify = _shutdownSignal.makeNotifier( false ); - sNotify->connectFunc( &zyppng::SocketNotifier::sigActivated, [&dispatch]( const zyppng::SocketNotifier &, int ) { - dispatch->quit(); - }); - sNotify->setEnabled( true ); - - MIL_MEDIA << "Starting event loop " << std::endl; - - dispatch->run(); - - MIL_MEDIA << "Thread exit " << std::endl; - } - - MediaNetworkThread::~MediaNetworkThread() - { - _shutdownSignal.notify(); - _t.join(); - } - - MediaNetworkThread &MediaNetworkThread::instance() - { - static MediaNetworkThread t; - return t; - } - -} diff --git a/zypp/zyppng/media/private/medianetworkserver_p.h b/zypp/zyppng/media/private/medianetworkserver_p.h deleted file mode 100644 index a18e7c7..0000000 --- a/zypp/zyppng/media/private/medianetworkserver_p.h +++ /dev/null @@ -1,117 +0,0 @@ -/*---------------------------------------------------------------------\ -| ____ _ __ __ ___ | -| |__ / \ / / . \ . \ | -| / / \ V /| _/ _/ | -| / /__ | | | | | | | -| /_____||_| |_| |_| | -| | -----------------------------------------------------------------------/ -* -* This file contains private API, this might break at any time between releases. -* You have been warned! -* -*/ -#ifndef ZYPPNG_MEDIA_MEDIANETWORKSERVER_H_INCLUDED -#define ZYPPNG_MEDIA_MEDIANETWORKSERVER_H_INCLUDED - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace google::protobuf { - class Message; -} - -namespace zyppng { - - class Downloader; - class MediaNetworkConn; - - using RequestId = uint32_t; - - class LIBZYPP_NG_NO_EXPORT MediaNetworkServer : public Base - { - public: - using ConnectionList = std::list< std::shared_ptr >; - - MediaNetworkServer(); - - void listen ( const std::string &sockPath ); - - std::shared_ptr downloader (); - - private: - void onIncomingConnection (); - - private: - std::shared_ptr _serverSocket; - std::shared_ptr _downloadManager; - ConnectionList _clients; - }; - - class LIBZYPP_NG_NO_EXPORT MediaNetworkConn : public Base - { - struct Request; - using ReqPtr = std::shared_ptr; - using ReqList = std::list< ReqPtr >; - public: - MediaNetworkConn ( MediaNetworkServer &server, std::shared_ptr &&socket ); - ~MediaNetworkConn ( ); - - SignalProxy sigDisconnected (); - - private: - void onDisconnected (); - void onReadyRead (); - void onError ( Socket::SocketError err ); - void trackRequest ( Request &r ); - void signalRequestStarted ( Request &r ); - void trackedDlFinished ( Request &r ); - - template - void sendMessage ( T &m ); - - private: - MediaNetworkServer &_server; - ReqList _requests; - std::shared_ptr _connection; - std::optional _pendingMessageSize; - Signal _disconnected; - - std::vector _socketSigConns; - }; - - class LIBZYPP_NG_NO_EXPORT MediaNetworkThread : public Base - { - public: - ~MediaNetworkThread(); - static MediaNetworkThread &instance (); - - static std::string sockPath () { - static std::string path = zypp::str::Format("zypp-mediasocket-%1%") % getpid(); - return path; - } - - private: - MediaNetworkThread(); - void threadMain (); - std::thread _t; - zyppng::Wakeup _shutdownSignal; - }; - -} - - - -#endif // MEDIANETWORKSERVER_H diff --git a/zypp/zyppng/proto/CMakeLists.txt b/zypp/zyppng/proto/CMakeLists.txt deleted file mode 100644 index 8d3957c..0000000 --- a/zypp/zyppng/proto/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -SET( zyppng_PROTOBUF_SOURCES - -) - -protobuf_generate_cpp(ZYPPNG_PROTO_SRCS ZYPPNG_PROTO_HDRS ${zyppng_PROTOBUF_SOURCES}) - -ADD_LIBRARY( zyppng-protobuf OBJECT ${ZYPPNG_PROTO_SRCS} ${ZYPPNG_PROTO_HDRS} ) - -add_dependencies( zyppng-protobuf zypp-protobuf ) -- 2.7.4