From 3f8f0936c3028a3f8008bca472968149b5af7d05 Mon Sep 17 00:00:00 2001 From: Radoslaw Cybulski Date: Tue, 7 Nov 2017 14:38:47 +0100 Subject: [PATCH] Drag functionality Calls e-mod-screen-reader DispatchDragEvent DBUS function. Requires patch https://review.tizen.org/gerrit/#/c/159676 Change-Id: Ibfbcbcee05621c1a0f7558e0acab8136317e08d5 --- src/ActivityChangeRequest.cpp | 15 ++++------- src/ActivityChangeRequest.hpp | 11 ++++---- src/DragActivity.cpp | 59 ++++++++++++++++++++++++++++++++++++++++++ src/MenuBuilder.cpp | 16 +++++++++--- src/ScreenScannerManager.cpp | 24 ++++++++++++++--- src/ScreenScannerManager.hpp | 4 ++- src/SelectActivity.cpp | 60 +++++++++++++++++++++++++++++++------------ src/SwitchManager.cpp | 9 ++++--- 8 files changed, 154 insertions(+), 44 deletions(-) create mode 100644 src/DragActivity.cpp diff --git a/src/ActivityChangeRequest.cpp b/src/ActivityChangeRequest.cpp index 50017da..c1d6faa 100644 --- a/src/ActivityChangeRequest.cpp +++ b/src/ActivityChangeRequest.cpp @@ -16,21 +16,16 @@ #include "ActivityChangeRequest.hpp" -ActivityChangeRequest::ActivityChangeRequest(const std::string &activityType, const std::shared_ptr &uiElement) - : activityType(activityType), uiElement(uiElement) +ActivityChangeRequest::ActivityChangeRequest(std::string activityType, std::vector> uiElements) + : activityType(std::move(activityType)), uiElements(std::move(uiElements)) {} -std::string ActivityChangeRequest::getActivityType() +std::string ActivityChangeRequest::getActivityType() const { return activityType; } -std::shared_ptr ActivityChangeRequest::getUIElement() +std::vector> ActivityChangeRequest::getUIElements() const { - return uiElement; -} - -bool ActivityChangeRequest::hasUIElement() -{ - return uiElement != nullptr; + return uiElements; } diff --git a/src/ActivityChangeRequest.hpp b/src/ActivityChangeRequest.hpp index 4cd7e73..505c1fb 100644 --- a/src/ActivityChangeRequest.hpp +++ b/src/ActivityChangeRequest.hpp @@ -25,14 +25,13 @@ class ActivityChangeRequest { public: - ActivityChangeRequest(const std::string &activityType, const std::shared_ptr &uiElement); - std::string getActivityType(); - std::shared_ptr getUIElement(); - bool hasUIElement(); + ActivityChangeRequest(std::string activityType, std::vector> uiElements); -protected: + std::string getActivityType() const; + std::vector> getUIElements() const; +private: std::string activityType; - std::shared_ptr uiElement; + std::vector> uiElements; }; #endif diff --git a/src/DragActivity.cpp b/src/DragActivity.cpp new file mode 100644 index 0000000..23a8552 --- /dev/null +++ b/src/DragActivity.cpp @@ -0,0 +1,59 @@ +#include "UIActivity.hpp" +#include "ActivityFactory.hpp" +#include "UniversalSwitchLog.hpp" +#include "ScreenScannerManager.hpp" +#include "UIElement.hpp" +#include "dbusLocators.hpp" + +#include + +//DispatchDragEvent will generate 20 middle steps between start point and end point +static constexpr int MIDDLE_STEPS_COUNT = 20; +static constexpr double NO_DELAY_AFTER_TOUCH = 0.0; + +class DragActivity : public UIActivity, private RegisterActivity +{ +public: + constexpr static const char *activityType = "DRAG_DONE"; + DragActivity() : UIActivity(activityType) + {} + + void update(const std::shared_ptr &elem) override + { + uiElements.push_back(elem); + } + void process() override + { + ASSERT(uiElements.size() == 2); + + markAsCompleted(); + auto from = uiElements[0]->getScanningCoordinates(); + auto to = uiElements[1]->getScanningCoordinates(); + + DBus::DBusClient dbus {dbusLocators::accessibilityEMod::BUS, + dbusLocators::accessibilityEMod::OBJ_PATH, + dbusLocators::accessibilityEMod::INTERFACE, + DBus::ConnectionType::SYSTEM}; + dbus.method("DispatchDragEvent"). + call(from.x, from.y, to.x, to.y, MIDDLE_STEPS_COUNT, NO_DELAY_AFTER_TOUCH); + } +private: + std::vector> uiElements; +}; + +class CancelActivity : public UIActivity, private RegisterActivity +{ +public: + constexpr static const char *activityType = "CANCEL"; + CancelActivity() : UIActivity(activityType) + {} + + void update(const std::shared_ptr &elem) override + { + } + void process() override + { + markAsCompleted(); + DEBUG("cancelled"); + } +}; diff --git a/src/MenuBuilder.cpp b/src/MenuBuilder.cpp index 9718145..ce91319 100644 --- a/src/MenuBuilder.cpp +++ b/src/MenuBuilder.cpp @@ -446,13 +446,17 @@ MenuBuilderImplementation::MenuBuilderImplementation() RepetitionPolicy::multiple); auto drag = std::make_shared( std::vector {"IDS_DRAG"}, - defaultImg); - auto done = std::make_shared( + defaultImg, + std::string {}, + "IDS_MENU_DONE_CANCEL_FOR_DRAG"); + auto dragDone = std::make_shared( std::vector {"IDS_DONE"}, - defaultImg); + defaultImg, + "DRAG_DONE"); auto cancel = std::make_shared( std::vector {"IDS_CANCEL"}, - defaultImg); + defaultImg, + "CANCEL"); auto zoomIn = std::make_shared( std::vector {"IDS_ZOOM_IN"}, defaultImg, @@ -751,6 +755,10 @@ MenuBuilderImplementation::MenuBuilderImplementation() addToMap("IDS_MENU_GRANULARITY", std::make_shared(MenuImplementation { {character, word, line, paragraph} })); + addToMap("IDS_MENU_DONE_CANCEL_FOR_DRAG", std::make_shared(MenuImplementation { + { dragDone, cancel }, + Alignment::CENTER, NavigateToSuperMenu::DENY + })); } std::shared_ptr MenuBuilderImplementation::build(std::string menuLabel) diff --git a/src/ScreenScannerManager.cpp b/src/ScreenScannerManager.cpp index 842110e..9f02cd3 100644 --- a/src/ScreenScannerManager.cpp +++ b/src/ScreenScannerManager.cpp @@ -79,7 +79,7 @@ void ScreenScannerManager::onContextChanged(const std::shared_ptr &ob Singleton::instance().getSoundFeedback()->play(SoundFeedback::Sound::NAVIGATION_CONTEXT_CHANGED); mainWindow->bringToFront(); - if (!obj) { + if (!obj || forcedScanningMethod) { rootNavigationElement = std::move(navigationContext); startAutoscanning(); return; @@ -97,6 +97,16 @@ void ScreenScannerManager::onContextChanged(const std::shared_ptr &ob }); } +void ScreenScannerManager::forceScanningMethod(ScanningMethod method) +{ + forcedScanningMethod = method; +} + +void ScreenScannerManager::cancelForcingScanningMethod() +{ + forcedScanningMethod = {}; +} + void ScreenScannerManager::startAutoscanning() { stopAutoscanning(); @@ -126,11 +136,17 @@ void ScreenScannerManager::clearCursor() void ScreenScannerManager::delayedStartAutoScanning() { - if (!scanningMethod) - scanningMethod = properties.getScanningMethod(); + if (!scanningMethod) { + scanningMethod = properties.getScanningMethod(); + } + + if (forcedScanningMethod) { + auto d = Singleton::instance().getMainWindow()->getDimensions(); + properties.setScanningField(d); + } if (rootNavigationElement) { - switch (*scanningMethod) { + switch (forcedScanningMethod ? *forcedScanningMethod : *scanningMethod) { case ScanningMethod::POINT: screenScanner = PointScanner::create(properties, rootNavigationElement); break; diff --git a/src/ScreenScannerManager.hpp b/src/ScreenScannerManager.hpp index 6095480..0945595 100644 --- a/src/ScreenScannerManager.hpp +++ b/src/ScreenScannerManager.hpp @@ -97,6 +97,8 @@ public: void next(); void prev(); + void forceScanningMethod(ScanningMethod method); + void cancelForcingScanningMethod(); private: void startAutoscanning(Optional method); @@ -110,7 +112,7 @@ private: std::unique_ptr screenScanner; NavigationInterface::CallbackHandle contextChangedHandle; ScanningProperties properties; - Optional scanningMethod{}; + Optional scanningMethod{}, forcedScanningMethod{}; std::shared_ptr rootNavigationElement; ecore::Timer delayedStartAutoscanningTimer; diff --git a/src/SelectActivity.cpp b/src/SelectActivity.cpp index 6e29baa..f80329d 100644 --- a/src/SelectActivity.cpp +++ b/src/SelectActivity.cpp @@ -63,6 +63,7 @@ private: void deleteScrollActivitiesData(); void createMenu(); void removeMenu(); + bool isMenuShown() const; void createMenuContent(); void addBackButton(Evas_Object *); void refreshMenu(); @@ -91,7 +92,7 @@ private: VConfInterface::CallbackHandle callbackHandle; std::vector gengridItemsData; std::shared_ptr menu; - std::shared_ptr realUiElement; + std::vector> realUiElements; AtspiComponentPtr scrollableUiElement; std::shared_ptr scrollActivitiesData; Optional realUiElementIsSlider; @@ -117,6 +118,9 @@ void SelectActivity::markAsCompleted() { deleteScrollActivitiesData(); UIActivity::markAsCompleted(); + auto screenScannerManager = Singleton::instance().getScreenScannerManager(); + if (screenScannerManager) + screenScannerManager->cancelForcingScanningMethod(); } void SelectActivity::deleteScrollActivitiesData() @@ -145,14 +149,17 @@ void SelectActivity::process() void SelectActivity::update(const std::shared_ptr &elem) { - DEBUG("Select - updated with new element"); - if (realUiElement) { + DEBUG("Select - updated with new element %d", isMenuShown() ? 1 : 0); + auto screenScannerManager = Singleton::instance().getScreenScannerManager(); + if (screenScannerManager) + screenScannerManager->cancelForcingScanningMethod(); + if (isMenuShown()) { + ASSERT(!realUiElements.empty()); elem->activate(); /*elem is popup menu item*/ return; } - realUiElement = elem; - if (elem->getObject()) { + if (realUiElements.empty() && elem->getObject()) { auto atspi = Singleton::instance().getAtspi(); ASSERT(atspi); auto scrollable = atspi->getScrollableElement(elem->getObject()); @@ -161,6 +168,7 @@ void SelectActivity::update(const std::shared_ptr &elem) scrollableUiElement = std::move(scrollable); } } + realUiElements.push_back(elem); if (Singleton::instance().get(AUTO_TAP_KEY, false)) { autoTapWaitingPeriodCallbackHandle = Singleton::instance().registerAndGet(AUTO_TAP_WAITING_PERIOD_KEY, AUTO_TAP_WAITING_PERIOD_DEFAULT_TIME, @@ -175,8 +183,10 @@ void SelectActivity::update(const std::shared_ptr &elem) return; } - nestedMenusLabels.push_back("IDS_MENU_MAIN"); + if (nestedMenusLabels.empty() || nestedMenusLabels.back() != "IDS_MENU_DONE_CANCEL_FOR_DRAG") + nestedMenusLabels.push_back("IDS_MENU_MAIN"); createMenu(); + ASSERT(isMenuShown()); } void SelectActivity::update(const std::shared_ptr &menuitem) @@ -191,7 +201,7 @@ void SelectActivity::update(const std::shared_ptr &menuitem) void SelectActivity::createMenu() { - ASSERT(realUiElement, "realUiElement is NULL"); + ASSERT(!realUiElements.empty(), "realUiElement is NULL"); if (nestedMenusLabels.empty()) { DEBUG("Empty menu vector"); @@ -229,6 +239,11 @@ void SelectActivity::createMenu() timer.reset(); } +bool SelectActivity::isMenuShown() const +{ + return static_cast(popup); +} + void SelectActivity::removeMenu() { callbackHandle.reset(); @@ -316,12 +331,25 @@ void SelectActivity::setAutoTapWaitingPeriodCb(double value) void SelectActivity::navigateThroughSubMenuOrCreateActivityChangeRequest(MenuItem *menuItem) { + DEBUG("element %s", menuItem->getName().c_str()); + if (menuItem->getName().empty()) return; if (!menuItem->getSubMenuLabel().empty()) { - if (!menuItem->getActivityType().empty()) - notify(std::make_shared(menuItem->getActivityType(), realUiElement)); + if (menuItem->getSubMenuLabel() == "IDS_MENU_DONE_CANCEL_FOR_DRAG") { + removeMenu(); + nestedMenusLabels.clear(); + nestedMenusLabels.push_back(menuItem->getSubMenuLabel()); + auto screenScannerManager = Singleton::instance().getScreenScannerManager(); + screenScannerManager->forceScanningMethod(ScanningMethod::POINT); + screenScannerManager->startAutoscanning(); + return; + } + + if (!menuItem->getActivityType().empty()) { + notify(std::make_shared(menuItem->getActivityType(), realUiElements)); + } navigateThroughSubMenu(menuItem->getSubMenuLabel()); return; @@ -332,7 +360,7 @@ void SelectActivity::navigateThroughSubMenuOrCreateActivityChangeRequest(MenuIte markAsCompleted(); } - notify(std::make_shared(menuItem->getActivityType(), realUiElement)); + notify(std::make_shared(menuItem->getActivityType(), realUiElements)); if (menuItem->isRepeatable()) { Singleton::instance().getScreenScannerManager()->continueAutoscanning(); } @@ -370,9 +398,9 @@ void SelectActivity::navigateBack() void SelectActivity::sendTapActivityChangeRequest() { - ASSERT(realUiElement, "realUiElement is NULL"); + ASSERT(!realUiElements.empty(), "realUiElement is NULL"); markAsCompleted(); - notify(std::make_shared("TAP", realUiElement)); + notify(std::make_shared("TAP", realUiElements)); } std::string SelectActivity::getCompleteLabelOfMenu() @@ -414,10 +442,10 @@ std::string SelectActivity::getCompleteLabelOfMainMenu() if (phoneCallState && *phoneCallState == TELEPHONY_CALL_STATE_CONNECTING) return "IDS_MENU_MAIN_INCOMING_CALL"; - if (realUiElement->getApplicationCategory() == UIElement::ApplicationCategory::HOMESCREEN) + if (realUiElements[0]->getApplicationCategory() == UIElement::ApplicationCategory::HOMESCREEN) return "IDS_MENU_MAIN_HOME_SCREEN"; - if (realUiElement->getApplicationCategory() == UIElement::ApplicationCategory::KEYBOARD) + if (realUiElements[0]->getApplicationCategory() == UIElement::ApplicationCategory::KEYBOARD) return "IDS_MENU_MAIN_SIP"; if (isRealUIElementSlider()) @@ -459,7 +487,7 @@ Optional SelectActivity::getPhoneCallState() bool SelectActivity::isRealUIElementSlider() { if (!realUiElementIsSlider) { - auto role = Singleton::instance().getAtspi()->getRole(realUiElement->getObject()); + auto role = Singleton::instance().getAtspi()->getRole(realUiElements[0]->getObject()); realUiElementIsSlider = role && *role == ATSPI_ROLE_SLIDER; } @@ -469,7 +497,7 @@ bool SelectActivity::isRealUIElementSlider() bool SelectActivity::hasRealUIElementEditableTextIface() { if (!realUiElementHasEditableText) { - auto editableTextIface = Singleton::instance().getAtspi()->getEditableTextInterface(realUiElement->getObject()); + auto editableTextIface = Singleton::instance().getAtspi()->getEditableTextInterface(realUiElements[0]->getObject()); realUiElementHasEditableText = editableTextIface ? true : false; } diff --git a/src/SwitchManager.cpp b/src/SwitchManager.cpp index 8293a42..9bfc921 100644 --- a/src/SwitchManager.cpp +++ b/src/SwitchManager.cpp @@ -108,7 +108,7 @@ void SwitchManager::update(const std::shared_ptr &sw) return; auto activityType = confItem->getActivityType(); - auto request = std::make_shared(activityType, nullptr); + auto request = std::make_shared(activityType, std::vector> {}); changeActivity(request); } @@ -219,8 +219,11 @@ void SwitchManager::changeActivity(const std::shared_ptr activity->attach(shared_from_this(), std::move(cond)); auto uiActivity = std::dynamic_pointer_cast(activity); - if (uiActivity && request->hasUIElement()) - uiActivity->update(request->getUIElement()); + if (uiActivity) { + DEBUG("sending ui element"); + for (auto &p : request->getUIElements()) + uiActivity->update(p); + } DEBUG("Push on stack: %s", activity->getType().c_str()); activities.push(activity); -- 2.7.4