Drag functionality

Calls e-mod-screen-reader DispatchDragEvent DBUS function.

Requires patch https://review.tizen.org/gerrit/#/c/159676

Change-Id: Ibfbcbcee05621c1a0f7558e0acab8136317e08d5

diff --git a/src/ActivityChangeRequest.cpp b/src/ActivityChangeRequest.cpp
index 50017da..c1d6faa 100644 (file)
--- a/src/ActivityChangeRequest.cpp
+++ b/src/ActivityChangeRequest.cpp
@@ -16,21 +16,16 @@
 
 #include "ActivityChangeRequest.hpp"
 
-ActivityChangeRequest::ActivityChangeRequest(const std::string &activityType, const std::shared_ptr<UIElement> &uiElement)
-       : activityType(activityType), uiElement(uiElement)
+ActivityChangeRequest::ActivityChangeRequest(std::string activityType, std::vector<std::shared_ptr<UIElement>> uiElements)
+       : activityType(std::move(activityType)), uiElements(std::move(uiElements))
 {}
 
-std::string ActivityChangeRequest::getActivityType()
+std::string ActivityChangeRequest::getActivityType() const
 {
        return activityType;
 }
 
-std::shared_ptr<UIElement> ActivityChangeRequest::getUIElement()
+std::vector<std::shared_ptr<UIElement>> ActivityChangeRequest::getUIElements() const
 {
-       return uiElement;
-}
-
-bool ActivityChangeRequest::hasUIElement()
-{
-       return uiElement != nullptr;
+       return uiElements;
 }

diff --git a/src/ActivityChangeRequest.hpp b/src/ActivityChangeRequest.hpp
index 4cd7e73..505c1fb 100644 (file)
--- a/src/ActivityChangeRequest.hpp
+++ b/src/ActivityChangeRequest.hpp
@@ -25,14 +25,13 @@
 class ActivityChangeRequest
 {
 public:
-       ActivityChangeRequest(const std::string &activityType, const std::shared_ptr<UIElement> &uiElement);
-       std::string getActivityType();
-       std::shared_ptr<UIElement> getUIElement();
-       bool hasUIElement();
+       ActivityChangeRequest(std::string activityType, std::vector<std::shared_ptr<UIElement>> uiElements);
 
-protected:
+       std::string getActivityType() const;
+       std::vector<std::shared_ptr<UIElement>> getUIElements() const;
+private:
        std::string activityType;
-       std::shared_ptr<UIElement> uiElement;
+       std::vector<std::shared_ptr<UIElement>> uiElements;
 };
 
 #endif

diff --git a/src/DragActivity.cpp b/src/DragActivity.cpp
new file mode 100644 (file)
index 0000000..23a8552
--- /dev/null
+++ b/src/DragActivity.cpp
@@ -0,0 +1,59 @@
+#include "UIActivity.hpp"
+#include "ActivityFactory.hpp"
+#include "UniversalSwitchLog.hpp"
+#include "ScreenScannerManager.hpp"
+#include "UIElement.hpp"
+#include "dbusLocators.hpp"
+
+#include <memory>
+
+//DispatchDragEvent will generate 20 middle steps between start point and end point
+static constexpr int MIDDLE_STEPS_COUNT = 20;
+static constexpr double NO_DELAY_AFTER_TOUCH = 0.0;
+
+class DragActivity : public UIActivity, private RegisterActivity<DragActivity>
+{
+public:
+       constexpr static const char *activityType = "DRAG_DONE";
+       DragActivity() : UIActivity(activityType)
+       {}
+
+       void update(const std::shared_ptr<UIElement> &elem) override
+       {
+               uiElements.push_back(elem);
+       }
+       void process() override
+       {
+               ASSERT(uiElements.size() == 2);
+
+               markAsCompleted();
+               auto from = uiElements[0]->getScanningCoordinates();
+               auto to = uiElements[1]->getScanningCoordinates();
+
+               DBus::DBusClient dbus {dbusLocators::accessibilityEMod::BUS,
+                                                          dbusLocators::accessibilityEMod::OBJ_PATH,
+                                                          dbusLocators::accessibilityEMod::INTERFACE,
+                                                          DBus::ConnectionType::SYSTEM};
+               dbus.method<void(int, int, int, int, int, double)>("DispatchDragEvent").
+               call(from.x, from.y, to.x, to.y, MIDDLE_STEPS_COUNT, NO_DELAY_AFTER_TOUCH);
+       }
+private:
+       std::vector<std::shared_ptr<UIElement>> uiElements;
+};
+
+class CancelActivity : public UIActivity, private RegisterActivity<DragActivity>
+{
+public:
+       constexpr static const char *activityType = "CANCEL";
+       CancelActivity() : UIActivity(activityType)
+       {}
+
+       void update(const std::shared_ptr<UIElement> &elem) override
+       {
+       }
+       void process() override
+       {
+               markAsCompleted();
+               DEBUG("cancelled");
+       }
+};

diff --git a/src/MenuBuilder.cpp b/src/MenuBuilder.cpp
index 9718145..ce91319 100644 (file)
--- a/src/MenuBuilder.cpp
+++ b/src/MenuBuilder.cpp
@@ -446,13 +446,17 @@ MenuBuilderImplementation::MenuBuilderImplementation()
                                                                                RepetitionPolicy::multiple);
        auto drag                                       =       std::make_shared<MenuItemImplementation>(
                                                                                std::vector<std::string> {"IDS_DRAG"},
-                                                                               defaultImg);
-       auto done                                       =       std::make_shared<MenuItemImplementation>(
+                                                                               defaultImg,
+                                                                               std::string {},
+                                                                               "IDS_MENU_DONE_CANCEL_FOR_DRAG");
+       auto dragDone                           =       std::make_shared<MenuItemImplementation>(
                                                                                std::vector<std::string> {"IDS_DONE"},
-                                                                               defaultImg);
+                                                                               defaultImg,
+                                                                               "DRAG_DONE");
        auto cancel                                     =       std::make_shared<MenuItemImplementation>(
                                                                                std::vector<std::string> {"IDS_CANCEL"},
-                                                                               defaultImg);
+                                                                               defaultImg,
+                                                                               "CANCEL");
        auto zoomIn                                     =       std::make_shared<MenuItemImplementation>(
                                                                                std::vector<std::string> {"IDS_ZOOM_IN"},
                                                                                defaultImg,
@@ -751,6 +755,10 @@ MenuBuilderImplementation::MenuBuilderImplementation()
        addToMap("IDS_MENU_GRANULARITY", std::make_shared<MenuImplementation>(MenuImplementation {
                {character, word, line, paragraph}
        }));
+       addToMap("IDS_MENU_DONE_CANCEL_FOR_DRAG", std::make_shared<MenuImplementation>(MenuImplementation {
+               { dragDone, cancel },
+               Alignment::CENTER, NavigateToSuperMenu::DENY
+       }));
 }
 
 std::shared_ptr<Menu> MenuBuilderImplementation::build(std::string menuLabel)

diff --git a/src/ScreenScannerManager.cpp b/src/ScreenScannerManager.cpp
index 842110e..9f02cd3 100644 (file)
--- a/src/ScreenScannerManager.cpp
+++ b/src/ScreenScannerManager.cpp
@@ -79,7 +79,7 @@ void ScreenScannerManager::onContextChanged(const std::shared_ptr<UIElement> &ob
        Singleton<UniversalSwitch>::instance().getSoundFeedback()->play(SoundFeedback::Sound::NAVIGATION_CONTEXT_CHANGED);
        mainWindow->bringToFront();
 
-       if (!obj) {
+       if (!obj || forcedScanningMethod) {
                rootNavigationElement = std::move(navigationContext);
                startAutoscanning();
                return;
@@ -97,6 +97,16 @@ void ScreenScannerManager::onContextChanged(const std::shared_ptr<UIElement> &ob
        });
 }
 
+void ScreenScannerManager::forceScanningMethod(ScanningMethod method)
+{
+       forcedScanningMethod = method;
+}
+
+void ScreenScannerManager::cancelForcingScanningMethod()
+{
+       forcedScanningMethod = {};
+}
+
 void ScreenScannerManager::startAutoscanning()
 {
        stopAutoscanning();
@@ -126,11 +136,17 @@ void ScreenScannerManager::clearCursor()
 
 void ScreenScannerManager::delayedStartAutoScanning()
 {
-       if (!scanningMethod)
-               scanningMethod =  properties.getScanningMethod();
+       if (!scanningMethod) {
+               scanningMethod = properties.getScanningMethod();
+       }
+
+       if (forcedScanningMethod) {
+               auto d = Singleton<UniversalSwitch>::instance().getMainWindow()->getDimensions();
+               properties.setScanningField(d);
+       }
 
        if (rootNavigationElement) {
-               switch (*scanningMethod) {
+               switch (forcedScanningMethod ? *forcedScanningMethod : *scanningMethod) {
                case ScanningMethod::POINT:
                        screenScanner = PointScanner::create(properties, rootNavigationElement);
                        break;

diff --git a/src/ScreenScannerManager.hpp b/src/ScreenScannerManager.hpp
index 6095480..0945595 100644 (file)
--- a/src/ScreenScannerManager.hpp
+++ b/src/ScreenScannerManager.hpp
@@ -97,6 +97,8 @@ public:
        void next();
        void prev();
 
+       void forceScanningMethod(ScanningMethod method);
+       void cancelForcingScanningMethod();
 private:
        void startAutoscanning(Optional<ScanningMethod> method);
 
@@ -110,7 +112,7 @@ private:
        std::unique_ptr<ScreenScanner> screenScanner;
        NavigationInterface::CallbackHandle contextChangedHandle;
        ScanningProperties properties;
-       Optional<ScanningMethod> scanningMethod{};
+       Optional<ScanningMethod> scanningMethod{}, forcedScanningMethod{};
        std::shared_ptr<NavigationElement> rootNavigationElement;
 
        ecore::Timer delayedStartAutoscanningTimer;

diff --git a/src/SelectActivity.cpp b/src/SelectActivity.cpp
index 6e29baa..f80329d 100644 (file)
--- a/src/SelectActivity.cpp
+++ b/src/SelectActivity.cpp
@@ -63,6 +63,7 @@ private:
        void deleteScrollActivitiesData();
        void createMenu();
        void removeMenu();
+       bool isMenuShown() const;
        void createMenuContent();
        void addBackButton(Evas_Object *);
        void refreshMenu();
@@ -91,7 +92,7 @@ private:
        VConfInterface::CallbackHandle callbackHandle;
        std::vector<GengridItemData> gengridItemsData;
        std::shared_ptr<Menu> menu;
-       std::shared_ptr<UIElement> realUiElement;
+       std::vector<std::shared_ptr<UIElement>> realUiElements;
        AtspiComponentPtr scrollableUiElement;
        std::shared_ptr<ScrollActivitiesData> scrollActivitiesData;
        Optional<bool> realUiElementIsSlider;
@@ -117,6 +118,9 @@ void SelectActivity::markAsCompleted()
 {
        deleteScrollActivitiesData();
        UIActivity::markAsCompleted();
+       auto screenScannerManager = Singleton<UniversalSwitch>::instance().getScreenScannerManager();
+       if (screenScannerManager)
+               screenScannerManager->cancelForcingScanningMethod();
 }
 
 void SelectActivity::deleteScrollActivitiesData()
@@ -145,14 +149,17 @@ void SelectActivity::process()
 
 void SelectActivity::update(const std::shared_ptr<UIElement> &elem)
 {
-       DEBUG("Select - updated with new element");
-       if (realUiElement) {
+       DEBUG("Select - updated with new element %d", isMenuShown() ? 1 : 0);
+       auto screenScannerManager = Singleton<UniversalSwitch>::instance().getScreenScannerManager();
+       if (screenScannerManager)
+               screenScannerManager->cancelForcingScanningMethod();
+       if (isMenuShown()) {
+               ASSERT(!realUiElements.empty());
                elem->activate(); /*elem is popup menu item*/
                return;
        }
 
-       realUiElement = elem;
-       if (elem->getObject()) {
+       if (realUiElements.empty() && elem->getObject()) {
                auto atspi = Singleton<UniversalSwitch>::instance().getAtspi();
                ASSERT(atspi);
                auto scrollable = atspi->getScrollableElement(elem->getObject());
@@ -161,6 +168,7 @@ void SelectActivity::update(const std::shared_ptr<UIElement> &elem)
                        scrollableUiElement = std::move(scrollable);
                }
        }
+       realUiElements.push_back(elem);
 
        if (Singleton<VConfInterface>::instance().get(AUTO_TAP_KEY, false)) {
                autoTapWaitingPeriodCallbackHandle = Singleton<VConfInterface>::instance().registerAndGet<double>(AUTO_TAP_WAITING_PERIOD_KEY, AUTO_TAP_WAITING_PERIOD_DEFAULT_TIME,
@@ -175,8 +183,10 @@ void SelectActivity::update(const std::shared_ptr<UIElement> &elem)
                return;
        }
 
-       nestedMenusLabels.push_back("IDS_MENU_MAIN");
+       if (nestedMenusLabels.empty() || nestedMenusLabels.back() != "IDS_MENU_DONE_CANCEL_FOR_DRAG")
+               nestedMenusLabels.push_back("IDS_MENU_MAIN");
        createMenu();
+       ASSERT(isMenuShown());
 }
 
 void SelectActivity::update(const std::shared_ptr<MenuItem> &menuitem)
@@ -191,7 +201,7 @@ void SelectActivity::update(const std::shared_ptr<MenuItem> &menuitem)
 
 void SelectActivity::createMenu()
 {
-       ASSERT(realUiElement, "realUiElement is NULL");
+       ASSERT(!realUiElements.empty(), "realUiElement is NULL");
 
        if (nestedMenusLabels.empty()) {
                DEBUG("Empty menu vector");
@@ -229,6 +239,11 @@ void SelectActivity::createMenu()
        timer.reset();
 }
 
+bool SelectActivity::isMenuShown() const
+{
+       return static_cast<bool>(popup);
+}
+
 void SelectActivity::removeMenu()
 {
        callbackHandle.reset();
@@ -316,12 +331,25 @@ void SelectActivity::setAutoTapWaitingPeriodCb(double value)
 
 void SelectActivity::navigateThroughSubMenuOrCreateActivityChangeRequest(MenuItem *menuItem)
 {
+       DEBUG("element %s", menuItem->getName().c_str());
+
        if (menuItem->getName().empty())
                return;
 
        if (!menuItem->getSubMenuLabel().empty()) {
-               if (!menuItem->getActivityType().empty())
-                       notify(std::make_shared<ActivityChangeRequest>(menuItem->getActivityType(), realUiElement));
+               if (menuItem->getSubMenuLabel() == "IDS_MENU_DONE_CANCEL_FOR_DRAG") {
+                       removeMenu();
+                       nestedMenusLabels.clear();
+                       nestedMenusLabels.push_back(menuItem->getSubMenuLabel());
+                       auto screenScannerManager = Singleton<UniversalSwitch>::instance().getScreenScannerManager();
+                       screenScannerManager->forceScanningMethod(ScanningMethod::POINT);
+                       screenScannerManager->startAutoscanning();
+                       return;
+               }
+
+               if (!menuItem->getActivityType().empty()) {
+                       notify(std::make_shared<ActivityChangeRequest>(menuItem->getActivityType(), realUiElements));
+               }
 
                navigateThroughSubMenu(menuItem->getSubMenuLabel());
                return;
@@ -332,7 +360,7 @@ void SelectActivity::navigateThroughSubMenuOrCreateActivityChangeRequest(MenuIte
                markAsCompleted();
        }
 
-       notify(std::make_shared<ActivityChangeRequest>(menuItem->getActivityType(), realUiElement));
+       notify(std::make_shared<ActivityChangeRequest>(menuItem->getActivityType(), realUiElements));
        if (menuItem->isRepeatable()) {
                Singleton<UniversalSwitch>::instance().getScreenScannerManager()->continueAutoscanning();
        }
@@ -370,9 +398,9 @@ void SelectActivity::navigateBack()
 
 void SelectActivity::sendTapActivityChangeRequest()
 {
-       ASSERT(realUiElement, "realUiElement is NULL");
+       ASSERT(!realUiElements.empty(), "realUiElement is NULL");
        markAsCompleted();
-       notify(std::make_shared<ActivityChangeRequest>("TAP", realUiElement));
+       notify(std::make_shared<ActivityChangeRequest>("TAP", realUiElements));
 }
 
 std::string SelectActivity::getCompleteLabelOfMenu()
@@ -414,10 +442,10 @@ std::string SelectActivity::getCompleteLabelOfMainMenu()
        if (phoneCallState && *phoneCallState == TELEPHONY_CALL_STATE_CONNECTING)
                return "IDS_MENU_MAIN_INCOMING_CALL";
 
-       if (realUiElement->getApplicationCategory() == UIElement::ApplicationCategory::HOMESCREEN)
+       if (realUiElements[0]->getApplicationCategory() == UIElement::ApplicationCategory::HOMESCREEN)
                return "IDS_MENU_MAIN_HOME_SCREEN";
 
-       if (realUiElement->getApplicationCategory() == UIElement::ApplicationCategory::KEYBOARD)
+       if (realUiElements[0]->getApplicationCategory() == UIElement::ApplicationCategory::KEYBOARD)
                return "IDS_MENU_MAIN_SIP";
 
        if (isRealUIElementSlider())
@@ -459,7 +487,7 @@ Optional<telephony_call_state_e>  SelectActivity::getPhoneCallState()
 bool SelectActivity::isRealUIElementSlider()
 {
        if (!realUiElementIsSlider) {
-               auto role = Singleton<UniversalSwitch>::instance().getAtspi()->getRole(realUiElement->getObject());
+               auto role = Singleton<UniversalSwitch>::instance().getAtspi()->getRole(realUiElements[0]->getObject());
                realUiElementIsSlider = role && *role == ATSPI_ROLE_SLIDER;
        }
 
@@ -469,7 +497,7 @@ bool SelectActivity::isRealUIElementSlider()
 bool SelectActivity::hasRealUIElementEditableTextIface()
 {
        if (!realUiElementHasEditableText) {
-               auto editableTextIface = Singleton<UniversalSwitch>::instance().getAtspi()->getEditableTextInterface(realUiElement->getObject());
+               auto editableTextIface = Singleton<UniversalSwitch>::instance().getAtspi()->getEditableTextInterface(realUiElements[0]->getObject());
                realUiElementHasEditableText = editableTextIface ? true : false;
        }
 

diff --git a/src/SwitchManager.cpp b/src/SwitchManager.cpp
index 8293a42..9bfc921 100644 (file)
--- a/src/SwitchManager.cpp
+++ b/src/SwitchManager.cpp
@@ -108,7 +108,7 @@ void SwitchManager::update(const std::shared_ptr<Switch> &sw)
                return;
 
        auto activityType = confItem->getActivityType();
-       auto request = std::make_shared<ActivityChangeRequest>(activityType, nullptr);
+       auto request = std::make_shared<ActivityChangeRequest>(activityType, std::vector<std::shared_ptr<UIElement>> {});
        changeActivity(request);
 }
 
@@ -219,8 +219,11 @@ void SwitchManager::changeActivity(const std::shared_ptr<ActivityChangeRequest>
                activity->attach(shared_from_this(), std::move(cond));
 
                auto uiActivity = std::dynamic_pointer_cast<UIActivity>(activity);
-               if (uiActivity && request->hasUIElement())
-                       uiActivity->update(request->getUIElement());
+               if (uiActivity) {
+                       DEBUG("sending ui element");
+                       for (auto &p : request->getUIElements())
+                               uiActivity->update(p);
+               }
 
                DEBUG("Push on stack: %s", activity->getType().c_str());
                activities.push(activity);