aurum: Add command to get coord relative to the widget's top-level window

There are two diffrent type of coord.
one is screen relative, other one is window relative.

This patch to support window relative coord.

https://github.sec.samsung.net/tizen/aurum/issues/5

Change-Id: I4fbec6d27f2e6a011a55282869a42e8cbaadb8ba

diff --git a/libaurum/inc/Accessibility/AccessibleNode.h b/libaurum/inc/Accessibility/AccessibleNode.h
index 78814fe045725a0d728d1fde660b6577e714b98f..be93c672403e4f807876dbed1736397ab49ac6d7 100644 (file)
--- a/libaurum/inc/Accessibility/AccessibleNode.h
+++ b/libaurum/inc/Accessibility/AccessibleNode.h
@@ -166,7 +166,13 @@ public:
      * @brief TBD
      * @since_tizen 5.5
      */
-    Rect<int> getBoundingBox() const;
+    Rect<int> getScreenBoundingBox() const;
+
+    /**
+     * @brief TBD
+     * @since_tizen 5.5
+     */
+    Rect<int> getWindowBoundingBox() const;
 
     /**
      * @brief TBD
@@ -353,7 +359,12 @@ protected:
     /**
      * @brief TBD
      */
-    Rect<int> mBoundingBox;
+    Rect<int> mScreenBoundingBox;
+
+    /**
+     * @brief TBD
+     */
+    Rect<int> mWindowBoundingBox;
 
     /**
      * @brief TBD

diff --git a/libaurum/inc/Impl/Accessibility/MockAccessibleApplication.h b/libaurum/inc/Impl/Accessibility/MockAccessibleApplication.h
index ab987c2c909c9bf0b994360d857052a2a051c04e..f66a4d579a694d3deb97253a35b789b795cf698e 100644 (file)
--- a/libaurum/inc/Impl/Accessibility/MockAccessibleApplication.h
+++ b/libaurum/inc/Impl/Accessibility/MockAccessibleApplication.h
@@ -18,7 +18,7 @@ public:
      * @brief TBD
      * @since tizen_6.0
      */
-    MockAccessibleApplication(std::shared_ptr<AccessibleNode> parent, std::string text,std::string pkg,std::string role, std::string res,std::string type,std::string style,Rect<int> boundingBox,int supportingIfaces,int featureProperty);
+    MockAccessibleApplication(std::shared_ptr<AccessibleNode> parent, std::string text,std::string pkg,std::string role, std::string res,std::string type,std::string style,Rect<int> screenBoundingBox,int supportingIfaces,int featureProperty);
 
     /**
      * @brief TBD

diff --git a/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h b/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h
index 09130465ae7852e9cbe8caaa98a5294f13ec9430..a102ae2d016cc49e8b2ea77a2fc6d3f1d23e3e75 100644 (file)
--- a/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h
+++ b/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h
@@ -11,7 +11,7 @@ public:
      * @brief TBD
      * @since_tizen 5.5
      */
-    MockAccessibleNode(std::shared_ptr<AccessibleNode> parent, std::string text,std::string pkg,std::string role, std::string id, std::string type,std::string style, std::string automationId, Rect<int> boundingBox,int supportingIfaces,int featureProperty);
+    MockAccessibleNode(std::shared_ptr<AccessibleNode> parent, std::string text,std::string pkg,std::string role, std::string id, std::string type,std::string style, std::string automationId, Rect<int> screenBoundingBox,int supportingIfaces,int featureProperty);
 
     /**
      * @brief TBD
@@ -118,7 +118,7 @@ public:
      * @brief TBD
      * @since_tizen 5.5
      */
-    void setProperties(std::string text,std::string pkg, std::string role, std::string res, std::string type, std::string style,std::string automationId,  Rect<int> boundingBox, int supportingIfaces, int featureProperty);
+    void setProperties(std::string text,std::string pkg, std::string role, std::string res, std::string type, std::string style,std::string automationId,  Rect<int> screenBoundingBox, int supportingIfaces, int featureProperty);
 
 private:
     /**

diff --git a/libaurum/inc/UiObject.h b/libaurum/inc/UiObject.h
index b764fb53a30d7f9f7f1ad9785daf131495eadeac..4fcaf1ae264e36d79965229e2296f7a78b6a293d 100644 (file)
--- a/libaurum/inc/UiObject.h
+++ b/libaurum/inc/UiObject.h
@@ -190,7 +190,13 @@ public:
      * @brief TBD
      * @since_tizen 5.5
      */
-    const Rect<int> getBoundingBox() const;
+    const Rect<int> getScreenBoundingBox() const;
+
+    /**
+     * @brief TBD
+     * @since_tizen 6.5
+     */
+    const Rect<int> getWindowBoundingBox() const;
 
     /**
      * @brief TBD
@@ -331,4 +337,4 @@ private:
      * @brief TBD
      */
     static const unsigned int LOGNCLICK_INTERVAL = 50;
-};
\ No newline at end of file
+};

diff --git a/libaurum/src/Accessibility/AccessibleNode.cc b/libaurum/src/Accessibility/AccessibleNode.cc
index 791913dc764d5ca68bf34b84a6624f568ad2fbe6..740354d634ad0b7b38922f618edeb0d58bc2c711 100644 (file)
--- a/libaurum/src/Accessibility/AccessibleNode.cc
+++ b/libaurum/src/Accessibility/AccessibleNode.cc
@@ -13,7 +13,7 @@ AccessibleNode::~AccessibleNode()
 
 AccessibleNode::AccessibleNode()
 : mText{""}, mPkg{""}, mRole{""}, mId{""}, mType{""}, mStyle{""},
-  mBoundingBox{0,0,0,0}, mSupportingIfaces(0), mFeatureProperty(0), mValid{true}, mLock{}
+  mScreenBoundingBox{0,0,0,0}, mWindowBoundingBox{0,0,0,0}, mSupportingIfaces(0), mFeatureProperty(0), mValid{true}, mLock{}
 {
 }
 
@@ -127,9 +127,14 @@ std::string AccessibleNode::getStyle() const
     return mStyle;
 }
 
-Rect<int> AccessibleNode::getBoundingBox() const
+Rect<int> AccessibleNode::getScreenBoundingBox() const
 {
-    return mBoundingBox;
+    return mScreenBoundingBox;
+}
+
+Rect<int> AccessibleNode::getWindowBoundingBox() const
+{
+    return mWindowBoundingBox;
 }
 
 bool AccessibleNode::isCheckable() const

diff --git a/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc b/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc
index e196ab42816bc422763e310f68f47833e8fb3452..baf262646f7515651e84d4e0efa2eeab3bdc845e 100644 (file)
--- a/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc
+++ b/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc
@@ -151,13 +151,22 @@ void AtspiAccessibleNode::refresh()
         }
         AtspiComponent *component = AtspiWrapper::Atspi_accessible_get_component_iface(mNode);
         if (component) {
-            AtspiRect *extent = AtspiWrapper::Atspi_component_get_extents(
+            AtspiRect *screenExtent = AtspiWrapper::Atspi_component_get_extents(
                 component, ATSPI_COORD_TYPE_SCREEN, NULL);
-            if (extent) {
-                mBoundingBox =
-                    Rect<int>{extent->x, extent->y, extent->x + extent->width,
-                            extent->y + extent->height};
-                g_free(extent);
+            if (screenExtent) {
+                mScreenBoundingBox =
+                    Rect<int>{screenExtent->x, screenExtent->y, screenExtent->x + screenExtent->width,
+                            screenExtent->y + screenExtent->height};\
+                g_free(screenExtent);
+            }
+
+            AtspiRect *windowExtent = AtspiWrapper::Atspi_component_get_extents(
+                component, ATSPI_COORD_TYPE_WINDOW, NULL);
+            if (windowExtent) {
+                mWindowBoundingBox =
+                    Rect<int>{windowExtent->x, windowExtent->y, windowExtent->x + windowExtent->width,
+                            windowExtent->y + windowExtent->height};\
+                g_free(windowExtent);
             }
             g_object_unref(component);
         }

diff --git a/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc b/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc
index 9e3a8e6c9a5d538ea2ab7f5495162f749f432317..39ccbfc74f4acaee8c08d4135c67a3550ba254f9 100644 (file)
--- a/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc
+++ b/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc
@@ -4,13 +4,13 @@
 #include <algorithm>
 #include <iostream>
 
-MockAccessibleNode::MockAccessibleNode(std::shared_ptr<AccessibleNode> parent, std::string text, std::string pkg, std::string role, std::string res, std::string type, std::string style,std::string automationId,  Rect<int> boundingBox, int supportingIfaces,int featureProperty)
+MockAccessibleNode::MockAccessibleNode(std::shared_ptr<AccessibleNode> parent, std::string text, std::string pkg, std::string role, std::string res, std::string type, std::string style,std::string automationId,  Rect<int> screenBoundingBox, int supportingIfaces,int featureProperty)
 : mParentNode(parent), mChildrenList{}, mActionSet{}
 {
     printf("%s:%d / %s\n",__FILE__, __LINE__, __PRETTY_FUNCTION__);
     const auto trickDontRemove = std::shared_ptr<MockAccessibleNode>( this, [](MockAccessibleNode *){} );
 
-    setProperties(text,pkg,role,res,type,style,automationId, boundingBox, supportingIfaces, featureProperty);
+    setProperties(text,pkg,role,res,type,style,automationId, screenBoundingBox, supportingIfaces, featureProperty);
     auto watcher = AccessibleWatcher::getInstance();
     watcher->attach(shared_from_this());
 }
@@ -47,7 +47,7 @@ void* MockAccessibleNode::getRawHandler(void) const
     return (void*)1;
 }
 
-void MockAccessibleNode::setProperties(std::string text,std::string pkg,std::string role,std::string id,std::string type,std::string style,std::string automationId, Rect<int> boundingBox,int supportingIfaces,int featureProperty)
+void MockAccessibleNode::setProperties(std::string text,std::string pkg,std::string role,std::string id,std::string type,std::string style,std::string automationId, Rect<int> screenBoundingBox,int supportingIfaces,int featureProperty)
 {
     mText = text;
     mPkg = pkg;
@@ -56,7 +56,7 @@ void MockAccessibleNode::setProperties(std::string text,std::string pkg,std::str
     mAutomationId = automationId;
     mType = type;
     mStyle = style;
-    mBoundingBox = boundingBox;
+    mScreenBoundingBox = screenBoundingBox;
     mSupportingIfaces = supportingIfaces;
     mFeatureProperty = featureProperty;
 }

diff --git a/libaurum/src/UiObject.cc b/libaurum/src/UiObject.cc
index f802b6ab4f426e216d948ba93df16d15d74f9798..c5b51544f9f966efb5e81421c980ce15a823b857 100644 (file)
--- a/libaurum/src/UiObject.cc
+++ b/libaurum/src/UiObject.cc
@@ -265,16 +265,22 @@ bool UiObject::isValid() const
     return mNode->isValid();
 }
 
-const Rect<int> UiObject::getBoundingBox() const
+const Rect<int> UiObject::getScreenBoundingBox() const
 {
     mNode->refresh();
-    return mNode->getBoundingBox();
+    return mNode->getScreenBoundingBox();
+}
+
+const Rect<int> UiObject::getWindowBoundingBox() const
+{
+    mNode->refresh();
+    return mNode->getWindowBoundingBox();
 }
 
 void UiObject::click() const
 {
     mNode->refresh();
-    const Rect<int> rect = mNode->getBoundingBox();
+    const Rect<int> rect = mNode->getScreenBoundingBox();
     const Point2D<int> midPoint = rect.midPoint();
     mDevice->click(midPoint.x, midPoint.y);
 }
@@ -282,7 +288,7 @@ void UiObject::click() const
 void UiObject::longClick(const unsigned int intv) const
 {
     mNode->refresh();
-    const Rect<int> rect = mNode->getBoundingBox();
+    const Rect<int> rect = mNode->getScreenBoundingBox();
     const Point2D<int> midPoint = rect.midPoint();
     mDevice->click(midPoint.x, midPoint.y, intv);
 }

diff --git a/org.tizen.aurum-bootstrap/src/Commands/DumpObjectTreeCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/DumpObjectTreeCommand.cc
index 84f0002e26178f8705c951fac8369befe5c79fbc..47008f3b6c775d9a712e08772c084b64d58144c8 100644 (file)
--- a/org.tizen.aurum-bootstrap/src/Commands/DumpObjectTreeCommand.cc
+++ b/org.tizen.aurum-bootstrap/src/Commands/DumpObjectTreeCommand.cc
@@ -26,12 +26,19 @@ void DumpObjectTreeCommand::traverse(::aurum::Element *root, std::shared_ptr<Nod
     root->set_elementid(key);
 
     ::aurum::Rect *rect = root->mutable_geometry();
-    const Rect<int> &size = obj->getBoundingBox();
+    const Rect<int> &size = obj->getScreenBoundingBox();
     rect->set_x(size.mTopLeft.x);
     rect->set_y(size.mTopLeft.y);
     rect->set_width(size.width());
     rect->set_height(size.height());
 
+    ::aurum::Rect *windowRect = root->mutable_window_relative_geometry();
+    const Rect<int> &windowSize = obj->getWindowBoundingBox();
+    windowRect->set_x(windowSize.mTopLeft.x);
+    windowRect->set_y(windowSize.mTopLeft.y);
+    windowRect->set_width(windowSize.width());
+    windowRect->set_height(windowSize.height());
+
     root->set_widget_type(obj->getElementType());
     root->set_widget_style(obj->getElementStyle());
 

diff --git a/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc
index ee8f6ff46ef509fcadcbb93ad63cc0a026bd0852..53023836c3cefb2198861dc55e9322750b5e8310 100644 (file)
--- a/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc
+++ b/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc
@@ -76,11 +76,19 @@ std::vector<std::shared_ptr<UiSelector>> FindElementCommand::getSelectors(void)
             elm->set_package(obj->getApplicationPackage());
 
             ::aurum::Rect *rect = elm->mutable_geometry();
-            const Rect<int> &size = obj->getBoundingBox();
+            const Rect<int> &size = obj->getScreenBoundingBox();
             rect->set_x(size.mTopLeft.x);
             rect->set_y(size.mTopLeft.y);
             rect->set_width(size.width());
             rect->set_height(size.height());
+
+            ::aurum::Rect *windowRect = elm->mutable_window_relative_geometry();
+            const Rect<int> &windowRelativeSize = obj->getWindowBoundingBox();
+            windowRect->set_x(windowRelativeSize.mTopLeft.x);
+            windowRect->set_y(windowRelativeSize.mTopLeft.y);
+            windowRect->set_width(windowRelativeSize.width());
+            windowRect->set_height(windowRelativeSize.height());
+
             elm->set_widget_type(obj->getElementType());
             elm->set_widget_style(obj->getElementStyle());
 

diff --git a/org.tizen.aurum-bootstrap/src/Commands/GetSizeCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/GetSizeCommand.cc
index cb7280a7d5146d2f45b21e5d3cfe462b9e987e93..35876f9e09b53c9b1da14853483bcf4c8d1f57f5 100644 (file)
--- a/org.tizen.aurum-bootstrap/src/Commands/GetSizeCommand.cc
+++ b/org.tizen.aurum-bootstrap/src/Commands/GetSizeCommand.cc
@@ -11,15 +11,26 @@ GetSizeCommand::GetSizeCommand(const ::aurum::ReqGetSize *request,
 ::grpc::Status GetSizeCommand::execute()
 {
     LOGI("GetSize --------------- ");
+
+    ::aurum::ReqGetSize_CoordType type = mRequest->type();
     ObjectMapper *mObjMap = ObjectMapper::getInstance();
     std::shared_ptr<UiObject> obj = mObjMap->getElement(mRequest->elementid());
     if (obj) {
-        const Rect<int> &size = obj->getBoundingBox();
         ::aurum::Rect *rect = mResponse->mutable_size();
-        rect->set_x(size.mTopLeft.x);
-        rect->set_y(size.mTopLeft.y);
-        rect->set_width(size.width());
-        rect->set_height(size.height());
+        if (type == ::aurum::ReqGetSize_CoordType::ReqGetSize_CoordType_SCREEN) {
+            const Rect<int> &size = obj->getScreenBoundingBox();
+            rect->set_x(size.mTopLeft.x);
+            rect->set_y(size.mTopLeft.y);
+            rect->set_width(size.width());
+            rect->set_height(size.height());
+        }
+        else {
+            const Rect<int> &windowRelativeSize = obj->getWindowBoundingBox();
+            rect->set_x(windowRelativeSize.mTopLeft.x);
+            rect->set_y(windowRelativeSize.mTopLeft.y);
+            rect->set_width(windowRelativeSize.width());
+            rect->set_height(windowRelativeSize.height());
+        }
     }
 
     return grpc::Status::OK;

diff --git a/protocol/aurum.proto b/protocol/aurum.proto
index a1c1c158173e3b564b6ac0ef23432f3ab58e3033..5d760d9f221e8f367f9ed44ed3b5d69d657a3270 100644 (file)
--- a/protocol/aurum.proto
+++ b/protocol/aurum.proto
@@ -51,28 +51,29 @@ message Element {
    repeated Element child = 2;
 
    Rect geometry = 3;
-
-   string widget_type = 4;
-   string widget_style = 5;
-
-   string text = 6;
-   string id = 7;
-   string automationId = 8;
-   string package = 9;
-   string role = 10;
-
-   bool isChecked = 11;
-   bool isCheckable = 12;
-   bool isClickable = 13;
-   bool isEnabled = 14;
-   bool isFocused = 15;
-   bool isFocusable = 16;
-   bool isScrollable = 17;
-   bool isSelected = 18;
-   bool isShowing = 19;
-   bool isActive = 20;
-   bool isVisible = 21;
-   bool isSelectable = 22;
+   Rect window_relative_geometry = 4;
+
+   string widget_type = 5;
+   string widget_style = 6;
+
+   string text = 7;
+   string id = 8;
+   string automationId = 9;
+   string package = 10;
+   string role = 11;
+
+   bool isChecked = 12;
+   bool isCheckable = 13;
+   bool isClickable = 14;
+   bool isEnabled = 15;
+   bool isFocused = 16;
+   bool isFocusable = 17;
+   bool isScrollable = 18;
+   bool isSelected = 19;
+   bool isShowing = 20;
+   bool isActive = 21;
+   bool isVisible = 22;
+   bool isSelectable = 23;
 }
 
 message Point {
@@ -200,8 +201,14 @@ message RspSetValue {
 }
 
 message ReqGetSize{
-   string elementId = 1;
+   enum CoordType {
+   SCREEN = 0;
+   WINDOW = 1;
+   }
+   CoordType type = 1;
+   string elementId = 2;
 }
+
 message RspGetSize{
    RspStatus status = 1;
    Rect size = 2;

diff --git a/tests/Test_UiObject.cc b/tests/Test_UiObject.cc
index a08a8f97d537d260238306e4409576fe24c3d165..6820f9366ebb16f8ee6c65f7eaecb3a74a37da43 100644 (file)
--- a/tests/Test_UiObject.cc
+++ b/tests/Test_UiObject.cc
@@ -223,11 +223,11 @@ TEST_F(AurumTestUiObject, setText_P1)
     ASSERT_EQ(parent->getText(), "new_test2");
 }
 
-TEST_F(AurumTestUiObject, getBoundingBox_P1)
+TEST_F(AurumTestUiObject, getScreenBoundingBox_P1)
 {
     auto obj = UiDevice::getInstance();
     auto parent = obj->findObject(Sel::text("test2"));
-    auto box = parent->getBoundingBox();
+    auto box = parent->getScreenBoundingBox();
 
     ASSERT_EQ(box.mBottomRight.x, 200 );
     ASSERT_EQ(box.mBottomRight.y, 200 );
@@ -399,7 +399,7 @@ TEST_F(AurumTestUiObject, click_P1)
     ASSERT_NE(obj, nullptr);
     obj->click();
 
-    auto rect = obj->getBoundingBox();
+    auto rect = obj->getScreenBoundingBox();
     const Point2D<int> midPoint = rect.midPoint();
 
     ASSERT_EQ(mDevice->mTouchRelease[1].x, midPoint.x);
@@ -413,7 +413,7 @@ TEST_F(AurumTestUiObject, longClick_P1)
     ASSERT_NE(obj, nullptr);
     obj->longClick(interval);
 
-    auto rect = obj->getBoundingBox();
+    auto rect = obj->getScreenBoundingBox();
     const Point2D<int> midPoint = rect.midPoint();
 
     ASSERT_NEAR(mDevice->mTouchRelease[1].stamp1, mDevice->mTouchRelease[1].stamp2, interval*1000*1.1);