From 04f372b8d3402b90c8a7567bb7827b4f0e4a104d Mon Sep 17 00:00:00 2001 From: Woochanlee Date: Thu, 22 Jul 2021 16:05:53 +0900 Subject: [PATCH] aurum: Add command to get coord relative to the widget's top-level window There are two diffrent type of coord. one is screen relative, other one is window relative. This patch to support window relative coord. https://github.sec.samsung.net/tizen/aurum/issues/5 Change-Id: I4fbec6d27f2e6a011a55282869a42e8cbaadb8ba --- libaurum/inc/Accessibility/AccessibleNode.h | 15 +++++- .../Accessibility/MockAccessibleApplication.h | 2 +- .../Impl/Accessibility/MockAccessibleNode.h | 4 +- libaurum/inc/UiObject.h | 10 +++- libaurum/src/Accessibility/AccessibleNode.cc | 11 ++-- .../Impl/Accessibility/AtspiAccessibleNode.cc | 21 +++++--- .../Impl/Accessibility/MockAccessibleNode.cc | 8 +-- libaurum/src/UiObject.cc | 14 +++-- .../src/Commands/DumpObjectTreeCommand.cc | 9 +++- .../src/Commands/FindElementCommand.cc | 10 +++- .../src/Commands/GetSizeCommand.cc | 21 ++++++-- protocol/aurum.proto | 53 +++++++++++-------- tests/Test_UiObject.cc | 8 +-- 13 files changed, 128 insertions(+), 58 deletions(-) diff --git a/libaurum/inc/Accessibility/AccessibleNode.h b/libaurum/inc/Accessibility/AccessibleNode.h index 78814fe..be93c67 100644 --- a/libaurum/inc/Accessibility/AccessibleNode.h +++ b/libaurum/inc/Accessibility/AccessibleNode.h @@ -166,7 +166,13 @@ public: * @brief TBD * @since_tizen 5.5 */ - Rect getBoundingBox() const; + Rect getScreenBoundingBox() const; + + /** + * @brief TBD + * @since_tizen 5.5 + */ + Rect getWindowBoundingBox() const; /** * @brief TBD @@ -353,7 +359,12 @@ protected: /** * @brief TBD */ - Rect mBoundingBox; + Rect mScreenBoundingBox; + + /** + * @brief TBD + */ + Rect mWindowBoundingBox; /** * @brief TBD diff --git a/libaurum/inc/Impl/Accessibility/MockAccessibleApplication.h b/libaurum/inc/Impl/Accessibility/MockAccessibleApplication.h index ab987c2..f66a4d5 100644 --- a/libaurum/inc/Impl/Accessibility/MockAccessibleApplication.h +++ b/libaurum/inc/Impl/Accessibility/MockAccessibleApplication.h @@ -18,7 +18,7 @@ public: * @brief TBD * @since tizen_6.0 */ - MockAccessibleApplication(std::shared_ptr parent, std::string text,std::string pkg,std::string role, std::string res,std::string type,std::string style,Rect boundingBox,int supportingIfaces,int featureProperty); + MockAccessibleApplication(std::shared_ptr parent, std::string text,std::string pkg,std::string role, std::string res,std::string type,std::string style,Rect screenBoundingBox,int supportingIfaces,int featureProperty); /** * @brief TBD diff --git a/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h b/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h index 0913046..a102ae2 100644 --- a/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h +++ b/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h @@ -11,7 +11,7 @@ public: * @brief TBD * @since_tizen 5.5 */ - MockAccessibleNode(std::shared_ptr parent, std::string text,std::string pkg,std::string role, std::string id, std::string type,std::string style, std::string automationId, Rect boundingBox,int supportingIfaces,int featureProperty); + MockAccessibleNode(std::shared_ptr parent, std::string text,std::string pkg,std::string role, std::string id, std::string type,std::string style, std::string automationId, Rect screenBoundingBox,int supportingIfaces,int featureProperty); /** * @brief TBD @@ -118,7 +118,7 @@ public: * @brief TBD * @since_tizen 5.5 */ - void setProperties(std::string text,std::string pkg, std::string role, std::string res, std::string type, std::string style,std::string automationId, Rect boundingBox, int supportingIfaces, int featureProperty); + void setProperties(std::string text,std::string pkg, std::string role, std::string res, std::string type, std::string style,std::string automationId, Rect screenBoundingBox, int supportingIfaces, int featureProperty); private: /** diff --git a/libaurum/inc/UiObject.h b/libaurum/inc/UiObject.h index b764fb5..4fcaf1a 100644 --- a/libaurum/inc/UiObject.h +++ b/libaurum/inc/UiObject.h @@ -190,7 +190,13 @@ public: * @brief TBD * @since_tizen 5.5 */ - const Rect getBoundingBox() const; + const Rect getScreenBoundingBox() const; + + /** + * @brief TBD + * @since_tizen 6.5 + */ + const Rect getWindowBoundingBox() const; /** * @brief TBD @@ -331,4 +337,4 @@ private: * @brief TBD */ static const unsigned int LOGNCLICK_INTERVAL = 50; -}; \ No newline at end of file +}; diff --git a/libaurum/src/Accessibility/AccessibleNode.cc b/libaurum/src/Accessibility/AccessibleNode.cc index 791913d..740354d 100644 --- a/libaurum/src/Accessibility/AccessibleNode.cc +++ b/libaurum/src/Accessibility/AccessibleNode.cc @@ -13,7 +13,7 @@ AccessibleNode::~AccessibleNode() AccessibleNode::AccessibleNode() : mText{""}, mPkg{""}, mRole{""}, mId{""}, mType{""}, mStyle{""}, - mBoundingBox{0,0,0,0}, mSupportingIfaces(0), mFeatureProperty(0), mValid{true}, mLock{} + mScreenBoundingBox{0,0,0,0}, mWindowBoundingBox{0,0,0,0}, mSupportingIfaces(0), mFeatureProperty(0), mValid{true}, mLock{} { } @@ -127,9 +127,14 @@ std::string AccessibleNode::getStyle() const return mStyle; } -Rect AccessibleNode::getBoundingBox() const +Rect AccessibleNode::getScreenBoundingBox() const { - return mBoundingBox; + return mScreenBoundingBox; +} + +Rect AccessibleNode::getWindowBoundingBox() const +{ + return mWindowBoundingBox; } bool AccessibleNode::isCheckable() const diff --git a/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc b/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc index e196ab4..baf2626 100644 --- a/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc +++ b/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc @@ -151,13 +151,22 @@ void AtspiAccessibleNode::refresh() } AtspiComponent *component = AtspiWrapper::Atspi_accessible_get_component_iface(mNode); if (component) { - AtspiRect *extent = AtspiWrapper::Atspi_component_get_extents( + AtspiRect *screenExtent = AtspiWrapper::Atspi_component_get_extents( component, ATSPI_COORD_TYPE_SCREEN, NULL); - if (extent) { - mBoundingBox = - Rect{extent->x, extent->y, extent->x + extent->width, - extent->y + extent->height}; - g_free(extent); + if (screenExtent) { + mScreenBoundingBox = + Rect{screenExtent->x, screenExtent->y, screenExtent->x + screenExtent->width, + screenExtent->y + screenExtent->height};\ + g_free(screenExtent); + } + + AtspiRect *windowExtent = AtspiWrapper::Atspi_component_get_extents( + component, ATSPI_COORD_TYPE_WINDOW, NULL); + if (windowExtent) { + mWindowBoundingBox = + Rect{windowExtent->x, windowExtent->y, windowExtent->x + windowExtent->width, + windowExtent->y + windowExtent->height};\ + g_free(windowExtent); } g_object_unref(component); } diff --git a/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc b/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc index 9e3a8e6..39ccbfc 100644 --- a/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc +++ b/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc @@ -4,13 +4,13 @@ #include #include -MockAccessibleNode::MockAccessibleNode(std::shared_ptr parent, std::string text, std::string pkg, std::string role, std::string res, std::string type, std::string style,std::string automationId, Rect boundingBox, int supportingIfaces,int featureProperty) +MockAccessibleNode::MockAccessibleNode(std::shared_ptr parent, std::string text, std::string pkg, std::string role, std::string res, std::string type, std::string style,std::string automationId, Rect screenBoundingBox, int supportingIfaces,int featureProperty) : mParentNode(parent), mChildrenList{}, mActionSet{} { printf("%s:%d / %s\n",__FILE__, __LINE__, __PRETTY_FUNCTION__); const auto trickDontRemove = std::shared_ptr( this, [](MockAccessibleNode *){} ); - setProperties(text,pkg,role,res,type,style,automationId, boundingBox, supportingIfaces, featureProperty); + setProperties(text,pkg,role,res,type,style,automationId, screenBoundingBox, supportingIfaces, featureProperty); auto watcher = AccessibleWatcher::getInstance(); watcher->attach(shared_from_this()); } @@ -47,7 +47,7 @@ void* MockAccessibleNode::getRawHandler(void) const return (void*)1; } -void MockAccessibleNode::setProperties(std::string text,std::string pkg,std::string role,std::string id,std::string type,std::string style,std::string automationId, Rect boundingBox,int supportingIfaces,int featureProperty) +void MockAccessibleNode::setProperties(std::string text,std::string pkg,std::string role,std::string id,std::string type,std::string style,std::string automationId, Rect screenBoundingBox,int supportingIfaces,int featureProperty) { mText = text; mPkg = pkg; @@ -56,7 +56,7 @@ void MockAccessibleNode::setProperties(std::string text,std::string pkg,std::str mAutomationId = automationId; mType = type; mStyle = style; - mBoundingBox = boundingBox; + mScreenBoundingBox = screenBoundingBox; mSupportingIfaces = supportingIfaces; mFeatureProperty = featureProperty; } diff --git a/libaurum/src/UiObject.cc b/libaurum/src/UiObject.cc index f802b6a..c5b5154 100644 --- a/libaurum/src/UiObject.cc +++ b/libaurum/src/UiObject.cc @@ -265,16 +265,22 @@ bool UiObject::isValid() const return mNode->isValid(); } -const Rect UiObject::getBoundingBox() const +const Rect UiObject::getScreenBoundingBox() const { mNode->refresh(); - return mNode->getBoundingBox(); + return mNode->getScreenBoundingBox(); +} + +const Rect UiObject::getWindowBoundingBox() const +{ + mNode->refresh(); + return mNode->getWindowBoundingBox(); } void UiObject::click() const { mNode->refresh(); - const Rect rect = mNode->getBoundingBox(); + const Rect rect = mNode->getScreenBoundingBox(); const Point2D midPoint = rect.midPoint(); mDevice->click(midPoint.x, midPoint.y); } @@ -282,7 +288,7 @@ void UiObject::click() const void UiObject::longClick(const unsigned int intv) const { mNode->refresh(); - const Rect rect = mNode->getBoundingBox(); + const Rect rect = mNode->getScreenBoundingBox(); const Point2D midPoint = rect.midPoint(); mDevice->click(midPoint.x, midPoint.y, intv); } diff --git a/org.tizen.aurum-bootstrap/src/Commands/DumpObjectTreeCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/DumpObjectTreeCommand.cc index 84f0002..47008f3 100644 --- a/org.tizen.aurum-bootstrap/src/Commands/DumpObjectTreeCommand.cc +++ b/org.tizen.aurum-bootstrap/src/Commands/DumpObjectTreeCommand.cc @@ -26,12 +26,19 @@ void DumpObjectTreeCommand::traverse(::aurum::Element *root, std::shared_ptrset_elementid(key); ::aurum::Rect *rect = root->mutable_geometry(); - const Rect &size = obj->getBoundingBox(); + const Rect &size = obj->getScreenBoundingBox(); rect->set_x(size.mTopLeft.x); rect->set_y(size.mTopLeft.y); rect->set_width(size.width()); rect->set_height(size.height()); + ::aurum::Rect *windowRect = root->mutable_window_relative_geometry(); + const Rect &windowSize = obj->getWindowBoundingBox(); + windowRect->set_x(windowSize.mTopLeft.x); + windowRect->set_y(windowSize.mTopLeft.y); + windowRect->set_width(windowSize.width()); + windowRect->set_height(windowSize.height()); + root->set_widget_type(obj->getElementType()); root->set_widget_style(obj->getElementStyle()); diff --git a/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc index ee8f6ff..5302383 100644 --- a/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc +++ b/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc @@ -76,11 +76,19 @@ std::vector> FindElementCommand::getSelectors(void) elm->set_package(obj->getApplicationPackage()); ::aurum::Rect *rect = elm->mutable_geometry(); - const Rect &size = obj->getBoundingBox(); + const Rect &size = obj->getScreenBoundingBox(); rect->set_x(size.mTopLeft.x); rect->set_y(size.mTopLeft.y); rect->set_width(size.width()); rect->set_height(size.height()); + + ::aurum::Rect *windowRect = elm->mutable_window_relative_geometry(); + const Rect &windowRelativeSize = obj->getWindowBoundingBox(); + windowRect->set_x(windowRelativeSize.mTopLeft.x); + windowRect->set_y(windowRelativeSize.mTopLeft.y); + windowRect->set_width(windowRelativeSize.width()); + windowRect->set_height(windowRelativeSize.height()); + elm->set_widget_type(obj->getElementType()); elm->set_widget_style(obj->getElementStyle()); diff --git a/org.tizen.aurum-bootstrap/src/Commands/GetSizeCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/GetSizeCommand.cc index cb7280a..35876f9 100644 --- a/org.tizen.aurum-bootstrap/src/Commands/GetSizeCommand.cc +++ b/org.tizen.aurum-bootstrap/src/Commands/GetSizeCommand.cc @@ -11,15 +11,26 @@ GetSizeCommand::GetSizeCommand(const ::aurum::ReqGetSize *request, ::grpc::Status GetSizeCommand::execute() { LOGI("GetSize --------------- "); + + ::aurum::ReqGetSize_CoordType type = mRequest->type(); ObjectMapper *mObjMap = ObjectMapper::getInstance(); std::shared_ptr obj = mObjMap->getElement(mRequest->elementid()); if (obj) { - const Rect &size = obj->getBoundingBox(); ::aurum::Rect *rect = mResponse->mutable_size(); - rect->set_x(size.mTopLeft.x); - rect->set_y(size.mTopLeft.y); - rect->set_width(size.width()); - rect->set_height(size.height()); + if (type == ::aurum::ReqGetSize_CoordType::ReqGetSize_CoordType_SCREEN) { + const Rect &size = obj->getScreenBoundingBox(); + rect->set_x(size.mTopLeft.x); + rect->set_y(size.mTopLeft.y); + rect->set_width(size.width()); + rect->set_height(size.height()); + } + else { + const Rect &windowRelativeSize = obj->getWindowBoundingBox(); + rect->set_x(windowRelativeSize.mTopLeft.x); + rect->set_y(windowRelativeSize.mTopLeft.y); + rect->set_width(windowRelativeSize.width()); + rect->set_height(windowRelativeSize.height()); + } } return grpc::Status::OK; diff --git a/protocol/aurum.proto b/protocol/aurum.proto index a1c1c15..5d760d9 100644 --- a/protocol/aurum.proto +++ b/protocol/aurum.proto @@ -51,28 +51,29 @@ message Element { repeated Element child = 2; Rect geometry = 3; - - string widget_type = 4; - string widget_style = 5; - - string text = 6; - string id = 7; - string automationId = 8; - string package = 9; - string role = 10; - - bool isChecked = 11; - bool isCheckable = 12; - bool isClickable = 13; - bool isEnabled = 14; - bool isFocused = 15; - bool isFocusable = 16; - bool isScrollable = 17; - bool isSelected = 18; - bool isShowing = 19; - bool isActive = 20; - bool isVisible = 21; - bool isSelectable = 22; + Rect window_relative_geometry = 4; + + string widget_type = 5; + string widget_style = 6; + + string text = 7; + string id = 8; + string automationId = 9; + string package = 10; + string role = 11; + + bool isChecked = 12; + bool isCheckable = 13; + bool isClickable = 14; + bool isEnabled = 15; + bool isFocused = 16; + bool isFocusable = 17; + bool isScrollable = 18; + bool isSelected = 19; + bool isShowing = 20; + bool isActive = 21; + bool isVisible = 22; + bool isSelectable = 23; } message Point { @@ -200,8 +201,14 @@ message RspSetValue { } message ReqGetSize{ - string elementId = 1; + enum CoordType { + SCREEN = 0; + WINDOW = 1; + } + CoordType type = 1; + string elementId = 2; } + message RspGetSize{ RspStatus status = 1; Rect size = 2; diff --git a/tests/Test_UiObject.cc b/tests/Test_UiObject.cc index a08a8f9..6820f93 100644 --- a/tests/Test_UiObject.cc +++ b/tests/Test_UiObject.cc @@ -223,11 +223,11 @@ TEST_F(AurumTestUiObject, setText_P1) ASSERT_EQ(parent->getText(), "new_test2"); } -TEST_F(AurumTestUiObject, getBoundingBox_P1) +TEST_F(AurumTestUiObject, getScreenBoundingBox_P1) { auto obj = UiDevice::getInstance(); auto parent = obj->findObject(Sel::text("test2")); - auto box = parent->getBoundingBox(); + auto box = parent->getScreenBoundingBox(); ASSERT_EQ(box.mBottomRight.x, 200 ); ASSERT_EQ(box.mBottomRight.y, 200 ); @@ -399,7 +399,7 @@ TEST_F(AurumTestUiObject, click_P1) ASSERT_NE(obj, nullptr); obj->click(); - auto rect = obj->getBoundingBox(); + auto rect = obj->getScreenBoundingBox(); const Point2D midPoint = rect.midPoint(); ASSERT_EQ(mDevice->mTouchRelease[1].x, midPoint.x); @@ -413,7 +413,7 @@ TEST_F(AurumTestUiObject, longClick_P1) ASSERT_NE(obj, nullptr); obj->longClick(interval); - auto rect = obj->getBoundingBox(); + auto rect = obj->getScreenBoundingBox(); const Point2D midPoint = rect.midPoint(); ASSERT_NEAR(mDevice->mTouchRelease[1].stamp1, mDevice->mTouchRelease[1].stamp2, interval*1000*1.1); -- 2.34.1