From: Hosang Kim Date: Mon, 29 Aug 2022 09:16:02 +0000 (+0900) Subject: aurum: Add interface for searching objects by geometry. X-Git-Tag: submit/tizen/20220831.093959~1 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=56d0edb9443019337639563d2f0f40ed37196235;p=platform%2Fcore%2Fuifw%2Faurum.git aurum: Add interface for searching objects by geometry. Change-Id: I144af89209892cefa640ddb4e85e5c013dcf536b --- diff --git a/libaurum/inc/Misc/Point2D.h b/libaurum/inc/Misc/Point2D.h index 69bc81d..27aa0a7 100644 --- a/libaurum/inc/Misc/Point2D.h +++ b/libaurum/inc/Misc/Point2D.h @@ -73,7 +73,7 @@ public: * * @since_tizen 6.5 */ - inline bool operator==(const Point2D& rhs) + inline bool operator==(const Point2D& rhs) const { return this->x == rhs.x && this->y == rhs.y; } @@ -87,7 +87,7 @@ public: * * @since_tizen 6.5 */ - inline bool operator!=(const Point2D& rhs) + inline bool operator!=(const Point2D& rhs) const { return !(*this == rhs); } diff --git a/libaurum/inc/Misc/Rect.h b/libaurum/inc/Misc/Rect.h index cf9323d..e1aa8a3 100644 --- a/libaurum/inc/Misc/Rect.h +++ b/libaurum/inc/Misc/Rect.h @@ -128,6 +128,15 @@ public: return false; } + bool isInRect(const Rect &rect) const + { + if (rect.mTopLeft.x >= mTopLeft.x && rect.mBottomRight.x <= mBottomRight.x + && rect.mTopLeft.y >= mTopLeft.y && rect.mBottomRight.y <= mBottomRight.y) + return true; + + return false; + } + /** * @brief Checks given Rect is same as this or not. * @@ -137,7 +146,7 @@ public: * * @since_tizen 6.5 */ - inline bool operator==(const Rect& rhs) + inline bool operator==(const Rect& rhs) const { return this->mTopLeft == rhs.mTopLeft && this->mBottomRight == rhs.mBottomRight; } @@ -151,7 +160,8 @@ public: * * @since_tizen 6.5 */ - inline bool operator!=(const Rect& rhs){ + inline bool operator!=(const Rect& rhs) const + { return !(*this == rhs); } diff --git a/libaurum/inc/PartialMatch.h b/libaurum/inc/PartialMatch.h index 4806683..9ef7137 100644 --- a/libaurum/inc/PartialMatch.h +++ b/libaurum/inc/PartialMatch.h @@ -164,6 +164,18 @@ private: */ static bool checkCriteria(const bool boolA, const bool boolB); + /** + * @brief Checks Rectangle matched or not. + * + * @param[in] rectA Rect + * @param[in] rectB Rect + * + * @return ture if matched, else false + * + * @since_tizen 7.0 + */ + static bool checkCriteria(const Rect rectA, const Rect rectB, const bool isEqual); + private: const std::shared_ptr mSelector; const int mDepth; diff --git a/libaurum/inc/UiSelector.h b/libaurum/inc/UiSelector.h index 98f544d..4db056e 100644 --- a/libaurum/inc/UiSelector.h +++ b/libaurum/inc/UiSelector.h @@ -368,6 +368,18 @@ public: */ UiSelector *xpath(std::string xpath); + /** + * @brief Sets the search criteria to match the object's geometry. + * + * @param[in] geometry Rect + * @param[in] isEqual bool + * + * @return UiSelector class instance + * + * @since_tizen 7.0 + */ + UiSelector *geometry(Rect geometry, bool isEqual); + public: std::string mId; std::string mAutomationId; @@ -390,6 +402,7 @@ public: bool mMatchTextPartialMatch; bool mMatchXPath; bool mMatchOcrText; + bool mMatchGeometry; bool mMatchChecked; bool mMatchCheckable; @@ -420,8 +433,12 @@ public: bool mIsvisible; bool mIsselectable; + bool mGeometryIsEqual; + std::vector> mChild; std::shared_ptr mParent; + + Rect mGeometry; }; } diff --git a/libaurum/src/PartialMatch.cc b/libaurum/src/PartialMatch.cc index ba030d1..1e0d018 100644 --- a/libaurum/src/PartialMatch.cc +++ b/libaurum/src/PartialMatch.cc @@ -46,6 +46,16 @@ bool PartialMatch::checkCriteria(const bool boolA, const bool boolB) return boolA != boolB; } +bool PartialMatch::checkCriteria(const Rect rectA, const Rect rectB, bool isEqual) +{ + if (isEqual) { + return rectA != rectB; + } + else { + return !rectA.isInRect(rectB); + } +} + std::string PartialMatch::debugPrint() { return mSelector->description(); @@ -106,6 +116,10 @@ bool PartialMatch::checkCriteria(const std::shared_ptr selector, node->updateRoleName(); if (checkCriteria(selector->mRole, node->getRole(), 0)) return false; } + if (selector->mMatchGeometry) { + node->updateExtents(); + if (checkCriteria(selector->mGeometry, node->getScreenBoundingBox(), selector->mGeometryIsEqual)) return false; + } if (selector->mMatchChecked && checkCriteria(selector->mIschecked, node->isChecked())) return false; if (selector->mMatchCheckable && checkCriteria(selector->mIscheckable, node->isCheckable())) return false; if (selector->mMatchClickable && checkCriteria(selector->mIsclickable, node->isClickable())) return false; diff --git a/libaurum/src/UiSelector.cc b/libaurum/src/UiSelector.cc index 8157b18..74822c9 100644 --- a/libaurum/src/UiSelector.cc +++ b/libaurum/src/UiSelector.cc @@ -28,7 +28,7 @@ UiSelector::UiSelector() mMatchFocused{}, mMatchFocusable{}, mMatchScrollable{}, mMatchSelected{}, mMatchShowing{}, mMatchActive{}, mMatchVisible{}, mMatchSelectable{}, mMinDepth{}, mMaxDepth{}, mIschecked{}, mIscheckable{}, mIsclickable{}, mIsenabled{}, mIsfocused{}, mIsfocusable{}, mIsscrollable{}, mIsselected{}, mIsshowing{}, mIsactive{}, mIsvisible{}, - mIsselectable{}, mChild{}, mParent{} + mIsselectable{}, mChild{}, mParent{}, mGeometry{} { } @@ -46,6 +46,7 @@ std::string UiSelector::description() if(!this->mPkg.empty()) ss << "\"mPkg\":\"" << this->mPkg << "\", "; if(!this->mType.empty()) ss << "\"mType\":\"" << this->mType << "\", "; if(!this->mStyle.empty()) ss << "\"mStyle\":\"" << this->mStyle << "\", "; + if(this->mMatchGeometry) ss << "\"mGeometry\":\"" << this->mGeometry.mTopLeft.x << "//" << this->mGeometry.mTopLeft.y << "//" << this->mGeometry.width() << "//" << this->mGeometry.height() << "\", "; if(this->mMatchId) ss << "\"mMatchId\":\"" << ((this->mMatchId)?"true":"false") << "\", "; if(this->mMatchAutomationId) ss << "\"mMatchAutomationId\":\"" << ((this->mMatchAutomationId)?"true":"false") << "\", "; if(this->mMatchRole) ss << "\"mMatchRole\":\"" << ((this->mMatchRole)?"true":"false") << "\", "; @@ -56,6 +57,7 @@ std::string UiSelector::description() if(this->mMatchPkg) ss << "\"mMatchPkg\":\"" << ((this->mMatchPkg)?"true":"false") << "\", "; if(this->mMatchType) ss << "\"mMatchType\":\"" << ((this->mMatchType)?"true":"false") << "\", "; if(this->mMatchStyle) ss << "\"mMatchStyle\":\"" << ((this->mMatchStyle)?"true":"false" )<< "\", "; + if(this->mMatchGeometry) ss << "\"mMatchGeometry\":\"" << ((this->mMatchGeometry)?"true":"false" )<< "\", "; if(this->mMinDepth) ss << "\"mMinDepth\":\"" << this->mMinDepth << "\", "; if(this->mMaxDepth) ss << "\"mMaxDepth\":\"" << this->mMaxDepth << "\", "; if(this->mMatchChecked) ss << "\"mMatchChecked\":\"" << ((this->mMatchChecked)?"true":"false") << "\", "; @@ -275,3 +277,11 @@ UiSelector *UiSelector::fromParent(std::shared_ptr parent) mParent = parent; return this; } + +UiSelector *UiSelector::geometry(Rect geometry, bool isEqual) +{ + this->mGeometry = geometry; + this->mGeometryIsEqual = isEqual; + this->mMatchGeometry = true; + return this; +} \ No newline at end of file diff --git a/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc index ed92078..ce89249 100644 --- a/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc +++ b/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc @@ -69,6 +69,7 @@ std::shared_ptr FindElementCommand::getSelector(void) if(mRequest->_textpartialmatch_case()) sel->textPartialMatch(mRequest->textpartialmatch()); if(mRequest->_xpath_case()) sel->xpath(mRequest->xpath()); if(mRequest->_ocrtext_case()) sel->ocrText(mRequest->ocrtext()); + if(mRequest->_geometry_case()) sel->geometry(Rect{mRequest->geometry().x(), mRequest->geometry().y(), mRequest->geometry().x() + mRequest->geometry().width(), mRequest->geometry().y() + mRequest->geometry().height()}, true); return sel; } diff --git a/org.tizen.aurum-bootstrap/src/Commands/FindElementsCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/FindElementsCommand.cc index bc2119c..937e31f 100644 --- a/org.tizen.aurum-bootstrap/src/Commands/FindElementsCommand.cc +++ b/org.tizen.aurum-bootstrap/src/Commands/FindElementsCommand.cc @@ -69,6 +69,7 @@ std::vector> FindElementsCommand::getSelectors(void) if(mRequest->_textpartialmatch_case()) sel->textPartialMatch(mRequest->textpartialmatch()); if(mRequest->_xpath_case()) sel->xpath(mRequest->xpath()); if(mRequest->_ocrtext_case()) sel->ocrText(mRequest->ocrtext()); + if(mRequest->_geometry_case()) sel->geometry(Rect{mRequest->geometry().x(), mRequest->geometry().y(), mRequest->geometry().x() + mRequest->geometry().width(), mRequest->geometry().y() + mRequest->geometry().height()}, false); return std::vector>{sel}; } diff --git a/protocol/aurum.proto b/protocol/aurum.proto index 0e076f9..e5e3085 100644 --- a/protocol/aurum.proto +++ b/protocol/aurum.proto @@ -193,7 +193,11 @@ message ReqFindElement { string ocrText = 21; } - repeated ReqFindElement children = 22; + oneof _geometry { + Rect geometry = 22; + } + + repeated ReqFindElement children = 23; } message RspFindElement { @@ -284,7 +288,12 @@ message ReqFindElements { oneof _ocrtext { string ocrText = 21; } - repeated ReqFindElements children = 22; + + oneof _geometry { + Rect geometry = 22; + } + + repeated ReqFindElements children = 23; } message RspFindElements {