From 4deb0db99a89de7825c1316e26f002b29eafb83c Mon Sep 17 00:00:00 2001 From: Woochanlee Date: Wed, 22 Jun 2022 20:05:37 +0900 Subject: [PATCH 01/16] aurum-service: remove unnecessary list tracking to search object. Change-Id: I4db9f82dc8a046196f7c958047b442034f6d1baa --- org.tizen.aurum-bootstrap/src/ObjectMapper.cc | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/org.tizen.aurum-bootstrap/src/ObjectMapper.cc b/org.tizen.aurum-bootstrap/src/ObjectMapper.cc index 4f44032..db129f3 100644 --- a/org.tizen.aurum-bootstrap/src/ObjectMapper.cc +++ b/org.tizen.aurum-bootstrap/src/ObjectMapper.cc @@ -42,13 +42,17 @@ std::string ObjectMapper::addElement(std::shared_ptr object) std::shared_ptr ObjectMapper::getElement(std::string elementId) { LOGI("getElement for elementId(%s)", elementId.c_str()); - if (mObjectMap.count(elementId)) { - std::shared_ptr obj = mObjectMap[elementId]; + + std::shared_ptr obj; + obj = mObjectMap[elementId]; + if (obj) { LOGI("Object find succeeded"); return obj; } - LOGI("Id(%s) is not exist in mObjectMap", elementId.c_str()); - return nullptr; + else { + LOGI("Object find fail"); + return nullptr; + } } bool ObjectMapper::removeElement(const std::string elementId) -- 2.7.4 From abf3e343f04272e05db154f8b8acac3438bec485 Mon Sep 17 00:00:00 2001 From: Hosang Kim Date: Wed, 18 May 2022 17:41:30 +0900 Subject: [PATCH 02/16] pugixml: fix build error Change-Id: Ic512488e2f21aa926d226bb044f7d7d9cfe5382d --- pugixml/meson.build | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pugixml/meson.build b/pugixml/meson.build index 4cd0236..954afa8 100644 --- a/pugixml/meson.build +++ b/pugixml/meson.build @@ -14,7 +14,8 @@ libpugixml_dep = declare_dependency( ) install_headers('pugixml.hpp', - 'pugiconfig.hpp') + 'pugixml.cpp', + 'pugiconfig.hpp') pkgconfig.generate( libpugixml_lib, -- 2.7.4 From d2e6154cdece615f4d985e25919f7d42f1440007 Mon Sep 17 00:00:00 2001 From: Hosang Kim Date: Wed, 18 May 2022 21:03:02 +0900 Subject: [PATCH 03/16] UiSelector: fix build warning Change-Id: I5dec5b361ba7fb6c9ceda2f1cec27cc58e4e8c91 --- libaurum/src/UiSelector.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libaurum/src/UiSelector.cc b/libaurum/src/UiSelector.cc index f97aea3..7d703dc 100644 --- a/libaurum/src/UiSelector.cc +++ b/libaurum/src/UiSelector.cc @@ -22,9 +22,9 @@ using namespace Aurum; UiSelector::UiSelector() -: mId{}, mAutomationId{}, mRole{}, mText{}, mTextPartialMatch{}, mPkg{}, mType{}, mStyle{}, - mMatchId{}, mMatchAutomationId{}, mMatchRole{}, mMatchText{}, mMatchTextPartialMatch{}, mMatchXPath{}, mMatchPkg{}, - mMatchType{}, mMatchStyle{}, mMatchChecked{}, mMatchCheckable{}, mMatchClickable{}, mMatchEnabled{}, mMatchFocused{}, +: mId{}, mAutomationId{}, mRole{}, mText{}, mPkg{}, mType{}, mStyle{}, mTextPartialMatch{}, mXPath{}, + mMatchId{}, mMatchAutomationId{}, mMatchRole{}, mMatchText{}, mMatchPkg{}, mMatchType{}, mMatchStyle{}, + mMatchTextPartialMatch{}, mMatchXPath{}, mMatchChecked{}, mMatchCheckable{}, mMatchClickable{}, mMatchEnabled{}, mMatchFocused{}, mMatchFocusable{}, mMatchScrollable{}, mMatchSelected{}, mMatchShowing{}, mMatchActive{}, mMatchVisible{}, mMatchSelectable{}, mMinDepth{}, mMaxDepth{}, mIschecked{}, mIscheckable{}, mIsclickable{}, mIsenabled{}, mIsfocused{}, mIsfocusable{}, mIsscrollable{}, mIsselected{}, mIsshowing{}, mIsactive{}, mIsvisible{}, @@ -265,4 +265,4 @@ UiSelector *UiSelector::fromParent(std::shared_ptr parent) { mParent = parent; return this; -} \ No newline at end of file +} -- 2.7.4 From 147b42129d0ef8ce114b648092ac2e51293e4f34 Mon Sep 17 00:00:00 2001 From: Hosang Kim Date: Wed, 18 May 2022 20:51:57 +0900 Subject: [PATCH 04/16] libaurum: Introduce new value commands -get minimum value -get maximum value -get current value -get minimum increment -set current value Change-Id: I82fd2b4b9066cdaa7645ee3418db39876aaf5a75 --- libaurum/inc/Accessibility/AccessibleNode.h | 45 ++++++++++++++++++ .../inc/Impl/Accessibility/AtspiAccessibleNode.h | 10 ++++ libaurum/inc/Impl/Accessibility/AtspiWrapper.h | 6 +++ .../inc/Impl/Accessibility/MockAccessibleNode.h | 13 ++++++ libaurum/inc/UiObject.h | 54 ++++++++++++++++++++++ libaurum/src/Accessibility/AccessibleNode.cc | 24 +++++++++- .../src/Impl/Accessibility/AtspiAccessibleNode.cc | 46 +++++++++++++++++- libaurum/src/Impl/Accessibility/AtspiWrapper.cc | 36 +++++++++++++++ .../src/Impl/Accessibility/MockAccessibleNode.cc | 9 ++++ libaurum/src/UiObject.cc | 30 ++++++++++++ .../src/Commands/DumpObjectTreeCommand.cc | 5 ++ .../src/Commands/FindElementCommand.cc | 5 ++ .../src/Commands/FindElementsCommand.cc | 5 ++ .../src/Commands/GetValueCommand.cc | 43 +++++++++++++---- .../src/Commands/SetValueCommand.cc | 31 ++++++++++--- protocol/aurum.proto | 6 +++ 16 files changed, 350 insertions(+), 18 deletions(-) diff --git a/libaurum/inc/Accessibility/AccessibleNode.h b/libaurum/inc/Accessibility/AccessibleNode.h index 2077d23..9a44612 100644 --- a/libaurum/inc/Accessibility/AccessibleNode.h +++ b/libaurum/inc/Accessibility/AccessibleNode.h @@ -183,6 +183,7 @@ public: * @copydoc UiObject::getStyle() */ std::string getStyle() const; + /** * @copydoc UiObject::getXPath() */ @@ -199,6 +200,30 @@ public: Rect getWindowBoundingBox() const; /** + * @copydoc UiObject::getMinValue() + * + */ + double getMinValue() const; + + /** + * @copydoc UiObject::getMaxValue() + * + */ + double getMaxValue() const; + + /** + * @copydoc UiObject::getValue() + * + */ + double getValue() const; + + /** + * @copydoc UiObject::getIncrement() + * + */ + double getIncrement() const; + + /** * @copydoc UiObject::isCheckable() */ bool isCheckable() const; @@ -333,6 +358,11 @@ public: virtual void updateXPath() = 0; /** + * @copydoc UiObject::updateValue() + */ + virtual void updateValue() = 0; + + /** * @copydoc UiObject::setFocus() */ virtual bool setFocus() = 0; @@ -376,6 +406,17 @@ public: virtual bool setValue(std::string text) = 0; /** + * @brief Sets Node's value. + * + * @param[in] value double + * + * @return true if success, else false + * + * @since_tizen 7.0 + */ + virtual bool setValue(double value) = 0; + + /** * @brief Check object valid or not. * * @return true if valid, else false @@ -437,6 +478,10 @@ protected: Rect mWindowBoundingBox; int mSupportingIfaces; int mFeatureProperty; + double mMinValue; + double mMaxValue; + double mValue; + double mIncrement; private: bool mValid; diff --git a/libaurum/inc/Impl/Accessibility/AtspiAccessibleNode.h b/libaurum/inc/Impl/Accessibility/AtspiAccessibleNode.h index dd187f9..2724c93 100644 --- a/libaurum/inc/Impl/Accessibility/AtspiAccessibleNode.h +++ b/libaurum/inc/Impl/Accessibility/AtspiAccessibleNode.h @@ -122,6 +122,11 @@ public: void updateXPath() override; /** + * @copydoc UiObject::updateValue() + */ + void updateValue() override; + + /** * @copydoc UiObject::setFocus() */ bool setFocus() override; @@ -146,6 +151,11 @@ public: */ bool setValue(std::string text) override; + /** + * @copydoc AccessibleNode::setValue() + */ + bool setValue(double value) override; + private: using AccessibleNode::setFeatureProperty; diff --git a/libaurum/inc/Impl/Accessibility/AtspiWrapper.h b/libaurum/inc/Impl/Accessibility/AtspiWrapper.h index e9bcb8e..89a6dae 100644 --- a/libaurum/inc/Impl/Accessibility/AtspiWrapper.h +++ b/libaurum/inc/Impl/Accessibility/AtspiWrapper.h @@ -60,6 +60,12 @@ public: static void Atspi_accessible_clear_cache (AtspiAccessible *node); static gboolean Atspi_component_grab_focus(AtspiComponent *obj, GError **error); static void Atspi_accessible_set_cache_mask(AtspiAccessible *node, AtspiCache mask); + static AtspiValue *Atspi_accessible_get_value(AtspiAccessible *node); + static gdouble Atspi_value_get_minimum_value(AtspiValue *iface, GError **error); + static gdouble Atspi_value_get_current_value(AtspiValue *iface, GError **error); + static gdouble Atspi_value_get_maximum_value(AtspiValue *iface, GError **error); + static gboolean Atspi_value_set_current_value(AtspiValue *iface, gdouble value, GError **error); + static gdouble Atspi_value_get_minimum_increment(AtspiValue *iface, GError **error); private: static std::recursive_mutex mMutex; diff --git a/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h b/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h index 1a2852f..be6ce32 100644 --- a/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h +++ b/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h @@ -116,6 +116,7 @@ public: * @since_tizen 6.5 */ void updateExtents() override; + /** * @brief TBD * @since_tizen 6.5 @@ -126,6 +127,12 @@ public: * @brief TBD * @since_tizen 7.0 */ + void updateValue() override; + + /** + * @brief TBD + * @since_tizen 7.0 + */ bool setFocus() override; /** @@ -152,6 +159,12 @@ public: */ bool setValue(std::string text) override; + /** + * @brief TBD + * @since_tizen 7.0 + */ + bool setValue(double value) override; + public: using AccessibleNode::setFeatureProperty; /** diff --git a/libaurum/inc/UiObject.h b/libaurum/inc/UiObject.h index 4065bda..98c3a6c 100644 --- a/libaurum/inc/UiObject.h +++ b/libaurum/inc/UiObject.h @@ -303,6 +303,53 @@ public: const Rect getWindowBoundingBox() const; /** + * @brief Gets object's minimum value. + * + * @return double + * + * @since_tizen 7.0 + */ + const double getMinValue() const; + + /** + * @brief Gets object's maximum value. + * + * @return double + * + * @since_tizen 7.0 + */ + const double getMaxValue() const; + + /** + * @brief Gets object's current value. + * + * @return double + * + * @since_tizen 7.0 + */ + const double getValue() const; + + /** + * @brief Gets object's current increment. + * + * @return double + * + * @since_tizen 7.0 + */ + const double getIncrement() const; + + /** + * @brief Sets object's value. + * + * @param[in] double value + * + * @return true if success else false + * + * @since_tizen 7.0 + */ + bool setValue(double value); + + /** * @brief Gets object's checkable property. * * @return true if checkable else false @@ -497,6 +544,13 @@ public: void updateXPath() const; /** + * @brief Updates object's value information from atspi server. + * + * @since_tizen 7.0 + */ + void updateValue() const; + + /** * @brief Sets focus to object. * * @since_tizen 7.0 diff --git a/libaurum/src/Accessibility/AccessibleNode.cc b/libaurum/src/Accessibility/AccessibleNode.cc index 60f6baa..4fc7fcf 100644 --- a/libaurum/src/Accessibility/AccessibleNode.cc +++ b/libaurum/src/Accessibility/AccessibleNode.cc @@ -31,8 +31,8 @@ AccessibleNode::~AccessibleNode() } AccessibleNode::AccessibleNode() -: mText{""}, mPkg{""}, mRole{""}, mId{""}, mType{""}, mStyle{""}, - mScreenBoundingBox{0,0,0,0}, mWindowBoundingBox{0,0,0,0}, mSupportingIfaces(0), mFeatureProperty(0), mValid{true}, mLock{} +: mText{""}, mPkg{""}, mRole{""}, mId{""}, mAutomationId{""}, mType{""}, mStyle{""}, mXPath{""}, + mScreenBoundingBox{0,0,0,0}, mWindowBoundingBox{0,0,0,0}, mSupportingIfaces(0), mFeatureProperty(0), mMinValue{0.0}, mMaxValue{0.0}, mValue{0.0}, mIncrement{0.0}, mValid{true}, mLock{} { } @@ -229,3 +229,23 @@ bool AccessibleNode::isActive() const { return hasFeatureProperty(NodeFeatureProperties::ACTIVE); } + +double AccessibleNode::getMinValue() const +{ + return mMinValue; +} + +double AccessibleNode::getMaxValue() const +{ + return mMaxValue; +} + +double AccessibleNode::getValue() const +{ + return mValue; +} + +double AccessibleNode::getIncrement() const +{ + return mIncrement; +} \ No newline at end of file diff --git a/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc b/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc index 3cb67ea..d0b88d1 100644 --- a/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc +++ b/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc @@ -240,6 +240,20 @@ void AtspiAccessibleNode::updateXPath() mXPath = XMLDoc->getXPath(mId); } +void AtspiAccessibleNode::updateValue() +{ + AtspiWrapper::Atspi_accessible_clear_cache(mNode); + + AtspiValue *value = AtspiWrapper::Atspi_accessible_get_value(mNode); + if (value) { + mMinValue= AtspiWrapper::Atspi_value_get_minimum_value(value, NULL); + mMaxValue= AtspiWrapper::Atspi_value_get_maximum_value(value, NULL); + mValue= AtspiWrapper::Atspi_value_get_current_value(value, NULL); + mIncrement= AtspiWrapper::Atspi_value_get_minimum_increment(value, NULL); + g_object_unref(value); + } +} + bool AtspiAccessibleNode::setFocus() { AtspiComponent *component = AtspiWrapper::Atspi_accessible_get_component_iface(mNode); @@ -337,9 +351,19 @@ void AtspiAccessibleNode::refresh(bool updateAll) g_free(windowExtent); } g_object_unref(component); + } - if (updateAll) updateXPath(); + AtspiValue *value = AtspiWrapper::Atspi_accessible_get_value(mNode); + if (value) { + mMinValue= AtspiWrapper::Atspi_value_get_minimum_value(value, NULL); + mMaxValue= AtspiWrapper::Atspi_value_get_maximum_value(value, NULL); + mValue= AtspiWrapper::Atspi_value_get_current_value(value, NULL); + mIncrement= AtspiWrapper::Atspi_value_get_minimum_increment(value, NULL); + g_object_unref(value); } + + if (updateAll) updateXPath(); + } else { setFeatureProperty(ATSPI_STATE_INVALID); } @@ -413,7 +437,7 @@ bool AtspiAccessibleNode::setValue(std::string text) if (!iface) return false; - refresh(); + updateName(); int len = getText().length(); AtspiWrapper::Atspi_editable_text_delete_text(iface, 0, len, NULL); AtspiWrapper::Atspi_editable_text_insert_text(iface, 0, text.c_str(), text.length(), @@ -423,6 +447,24 @@ bool AtspiAccessibleNode::setValue(std::string text) return true; } +bool AtspiAccessibleNode::setValue(double value) +{ + if (!isValid()){ + return false; + } + + AtspiValue *iface = AtspiWrapper::Atspi_accessible_get_value(mNode); + LOGI("set Value iface:%p obj:%p value:%lf",iface, mNode, value); + + if (!iface) return false; + + updateValue(); + AtspiWrapper::Atspi_value_set_current_value(iface, value, NULL); + g_object_unref(iface); + + return true; +} + void AtspiAccessibleNode::setFeatureProperty(AtspiStateType type) { switch(type) { diff --git a/libaurum/src/Impl/Accessibility/AtspiWrapper.cc b/libaurum/src/Impl/Accessibility/AtspiWrapper.cc index 8a81984..34d459f 100644 --- a/libaurum/src/Impl/Accessibility/AtspiWrapper.cc +++ b/libaurum/src/Impl/Accessibility/AtspiWrapper.cc @@ -170,3 +170,39 @@ void AtspiWrapper::Atspi_accessible_set_cache_mask(AtspiAccessible *node, AtspiC std::unique_lock lock(mMutex); return atspi_accessible_set_cache_mask (node, mask); } + +AtspiValue *AtspiWrapper::Atspi_accessible_get_value(AtspiAccessible *node) +{ + std::unique_lock lock(mMutex); + return atspi_accessible_get_value_iface(node); +} + +gdouble AtspiWrapper::Atspi_value_get_minimum_value(AtspiValue *iface, GError **error) +{ + std::unique_lock lock(mMutex); + return atspi_value_get_minimum_value(iface, error); +} + +gdouble AtspiWrapper::Atspi_value_get_current_value(AtspiValue *iface, GError **error) +{ + std::unique_lock lock(mMutex); + return atspi_value_get_current_value(iface, error); +} + +gdouble AtspiWrapper::Atspi_value_get_maximum_value(AtspiValue *iface, GError **error) +{ + std::unique_lock lock(mMutex); + return atspi_value_get_maximum_value(iface, error); +} + +gboolean AtspiWrapper::Atspi_value_set_current_value(AtspiValue *iface, gdouble value, GError **error) +{ + std::unique_lock lock(mMutex); + return atspi_value_set_current_value(iface, value, error); +} + +gdouble AtspiWrapper::Atspi_value_get_minimum_increment(AtspiValue *iface, GError **error) +{ + std::unique_lock lock(mMutex); + return atspi_value_get_minimum_increment(iface, error); +} \ No newline at end of file diff --git a/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc b/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc index cd78ffa..51ba193 100644 --- a/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc +++ b/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc @@ -113,6 +113,10 @@ void MockAccessibleNode::updateXPath() { } +void MockAccessibleNode::updateValue() +{ +} + bool MockAccessibleNode::setFocus() { return false; @@ -143,6 +147,11 @@ bool MockAccessibleNode::setValue(std::string text) return true; } +bool MockAccessibleNode::setValue(double value) +{ + return true; +} + void MockAccessibleNode::setFeatureProperty(int type) { switch(type) { diff --git a/libaurum/src/UiObject.cc b/libaurum/src/UiObject.cc index ccf60ef..706d28f 100644 --- a/libaurum/src/UiObject.cc +++ b/libaurum/src/UiObject.cc @@ -199,6 +199,31 @@ std::string UiObject::getXPath() const return getAccessibleNode()->getXPath(); } +const double UiObject::getMinValue() const +{ + return getAccessibleNode()->getMinValue(); +} + +const double UiObject::getMaxValue() const +{ + return getAccessibleNode()->getMaxValue(); +} + +const double UiObject::getValue() const +{ + return getAccessibleNode()->getValue(); +} + +const double UiObject::getIncrement() const +{ + return getAccessibleNode()->getIncrement(); +} + +bool UiObject::setValue(double value) +{ + return getAccessibleNode()->setValue(value); +} + bool UiObject::setText(std::string text) { return getAccessibleNode()->setValue(text); @@ -314,6 +339,11 @@ void UiObject::updateXPath() const mNode->updateXPath(); } +void UiObject::updateValue() const +{ + mNode->updateValue(); +} + bool UiObject::setFocus() const { return mNode->setFocus(); diff --git a/org.tizen.aurum-bootstrap/src/Commands/DumpObjectTreeCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/DumpObjectTreeCommand.cc index 48e8dee..dfcda80 100644 --- a/org.tizen.aurum-bootstrap/src/Commands/DumpObjectTreeCommand.cc +++ b/org.tizen.aurum-bootstrap/src/Commands/DumpObjectTreeCommand.cc @@ -77,6 +77,11 @@ void DumpObjectTreeCommand::traverse(::aurum::Element *root, std::shared_ptrset_isvisible(obj->isVisible()); root->set_isselectable(obj->isSelectable()); + root->set_minvalue(obj->getMinValue()); + root->set_maxvalue(obj->getMaxValue()); + root->set_value(obj->getValue()); + root->set_increment(obj->getIncrement()); + for( auto && childNode : node->mChildren) { ::aurum::Element *child = root->add_child(); traverse(child, childNode, depth+1); diff --git a/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc index cdcdff2..929119d 100644 --- a/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc +++ b/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc @@ -125,6 +125,11 @@ std::shared_ptr FindElementCommand::getSelector(void) elm->set_isvisible(obj->isVisible()); elm->set_isselectable(obj->isSelectable()); + elm->set_minvalue(obj->getMinValue()); + elm->set_maxvalue(obj->getMaxValue()); + elm->set_value(obj->getValue()); + elm->set_increment(obj->getIncrement()); + mResponse->set_status(::aurum::RspStatus::OK); } else { mResponse->set_status(::aurum::RspStatus::ERROR); diff --git a/org.tizen.aurum-bootstrap/src/Commands/FindElementsCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/FindElementsCommand.cc index 74e281b..2efc5c7 100644 --- a/org.tizen.aurum-bootstrap/src/Commands/FindElementsCommand.cc +++ b/org.tizen.aurum-bootstrap/src/Commands/FindElementsCommand.cc @@ -130,6 +130,11 @@ std::vector> FindElementsCommand::getSelectors(void) elm->set_isactive(obj->isActive()); elm->set_isvisible(obj->isVisible()); elm->set_isselectable(obj->isSelectable()); + + elm->set_minvalue(obj->getMinValue()); + elm->set_maxvalue(obj->getMaxValue()); + elm->set_value(obj->getValue()); + elm->set_increment(obj->getIncrement()); } mResponse->set_status(::aurum::RspStatus::OK); } else { diff --git a/org.tizen.aurum-bootstrap/src/Commands/GetValueCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/GetValueCommand.cc index 0b4f257..e5eaaba 100644 --- a/org.tizen.aurum-bootstrap/src/Commands/GetValueCommand.cc +++ b/org.tizen.aurum-bootstrap/src/Commands/GetValueCommand.cc @@ -20,7 +20,7 @@ #include "UiObject.h" GetValueCommand::GetValueCommand(const ::aurum::ReqGetValue *request, - ::aurum::RspGetValue *response) + ::aurum::RspGetValue *response) : mRequest{request}, mResponse{response} { } @@ -28,14 +28,41 @@ GetValueCommand::GetValueCommand(const ::aurum::ReqGetValue *request, ::grpc::Status GetValueCommand::execute() { LOGI("GetValue --------------- "); - ObjectMapper *mObjMap = ObjectMapper::getInstance(); - std::shared_ptr obj = mObjMap->getElement(mRequest->elementid()); + ::aurum::ParamType param_type = mRequest->type(); + if (param_type == ::aurum::STRING) { + ObjectMapper *mObjMap = ObjectMapper::getInstance(); + std::shared_ptr obj = + mObjMap->getElement(mRequest->elementid()); - if (obj) { - obj->updateName(); - std::string text = obj->getText(); - mResponse->set_stringvalue(text.c_str()); - mResponse->set_status(::aurum::RspStatus::OK); + if (obj) { + obj->updateName(); + std::string text = obj->getText(); + mResponse->set_type(::aurum::STRING); + mResponse->set_stringvalue(text.c_str()); + mResponse->set_status(::aurum::RspStatus::OK); + } else { + mResponse->set_status(::aurum::RspStatus::ERROR); + } + } else if (param_type == ::aurum::INT) { + LOGI("Integer is not supported."); + mResponse->set_status(::aurum::RspStatus::ERROR); + } else if (param_type == ::aurum::DOUBLE) { + ObjectMapper *mObjMap = ObjectMapper::getInstance(); + std::shared_ptr obj = + mObjMap->getElement(mRequest->elementid()); + + if (obj) { + obj->updateValue(); + double value = obj->getValue(); + mResponse->set_type(::aurum::DOUBLE); + mResponse->set_doublevalue(value); + mResponse->set_status(::aurum::RspStatus::OK); + } else { + mResponse->set_status(::aurum::RspStatus::ERROR); + } + } else if (param_type == ::aurum::BOOL) { + LOGI("Boolean is not supported."); + mResponse->set_status(::aurum::RspStatus::ERROR); } return grpc::Status::OK; diff --git a/org.tizen.aurum-bootstrap/src/Commands/SetValueCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/SetValueCommand.cc index ae0f0d7..6f7f1ee 100644 --- a/org.tizen.aurum-bootstrap/src/Commands/SetValueCommand.cc +++ b/org.tizen.aurum-bootstrap/src/Commands/SetValueCommand.cc @@ -28,14 +28,33 @@ SetValueCommand::SetValueCommand(const ::aurum::ReqSetValue* request, { bool ret = false; LOGI("SetValue --------------- "); - LOGI("text:%s", mRequest->stringvalue().c_str()); + ::aurum::ParamType param_type = mRequest->type(); + if (param_type == ::aurum::STRING) { + LOGI("text:%s", mRequest->stringvalue().c_str()); - ObjectMapper *mObjMap = ObjectMapper::getInstance(); - std::shared_ptr obj = mObjMap->getElement(mRequest->elementid()); - if (obj) ret = obj->setText(const_cast(mRequest->stringvalue())); + ObjectMapper* mObjMap = ObjectMapper::getInstance(); + std::shared_ptr obj = + mObjMap->getElement(mRequest->elementid()); + if (obj) + ret = + obj->setText(const_cast(mRequest->stringvalue())); + } else if (param_type == ::aurum::INT) { + LOGI("Integer is not supported."); + } else if (param_type == ::aurum::DOUBLE) { + LOGI("value:%lf", mRequest->doublevalue()); + ObjectMapper* mObjMap = ObjectMapper::getInstance(); + std::shared_ptr obj = + mObjMap->getElement(mRequest->elementid()); + if (obj) ret = obj->setValue(mRequest->doublevalue()); - if (ret) mResponse->set_status(::aurum::RspStatus::OK); - else mResponse->set_status(::aurum::RspStatus::ERROR); + } else if (param_type == ::aurum::BOOL) { + LOGI("Boolean is not supported."); + } + + if (ret) + mResponse->set_status(::aurum::RspStatus::OK); + else + mResponse->set_status(::aurum::RspStatus::ERROR); return grpc::Status::OK; } diff --git a/protocol/aurum.proto b/protocol/aurum.proto index a9ef0f9..6b7d454 100644 --- a/protocol/aurum.proto +++ b/protocol/aurum.proto @@ -78,6 +78,11 @@ message Element { bool isActive = 21; bool isVisible = 22; bool isSelectable = 23; + + double minValue = 24; + double maxValue = 25; + double value = 26; + double increment = 27; } message Point { @@ -279,6 +284,7 @@ message RspFindElements { message ReqGetValue { string elementId = 1; + ParamType type = 2; // TODO : text, widgetstyle, widgettype, automationid } message RspGetValue { -- 2.7.4 From 760f0ca0f18961a1444adfa451be88262da48054 Mon Sep 17 00:00:00 2001 From: Hosang Kim Date: Fri, 10 Jun 2022 14:47:00 +0900 Subject: [PATCH 05/16] AccessibleNode: fix return value for setValue Change-Id: Ib82f74c691fb830617b85ac469df9df251a01c2e --- libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc b/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc index d0b88d1..5d9e922 100644 --- a/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc +++ b/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc @@ -440,11 +440,11 @@ bool AtspiAccessibleNode::setValue(std::string text) updateName(); int len = getText().length(); AtspiWrapper::Atspi_editable_text_delete_text(iface, 0, len, NULL); - AtspiWrapper::Atspi_editable_text_insert_text(iface, 0, text.c_str(), text.length(), - NULL); + bool ret = AtspiWrapper::Atspi_editable_text_insert_text(iface, 0, text.c_str(), text.length(), + NULL); g_object_unref(iface); - return true; + return ret; } bool AtspiAccessibleNode::setValue(double value) -- 2.7.4 From 960e8e042be93fc8d9b0dc937ed4e2501292f686 Mon Sep 17 00:00:00 2001 From: Woochanlee Date: Thu, 26 May 2022 18:07:00 +0900 Subject: [PATCH 06/16] aurum-service: app_context destroy after use Change-Id: Id920bf95e1f0bb6448822ad4fab92865fbb0907d --- org.tizen.aurum-bootstrap/src/Commands/CloseAppCommand.cc | 2 ++ org.tizen.aurum-bootstrap/src/Commands/GetAppInfoCommand.cc | 1 + 2 files changed, 3 insertions(+) diff --git a/org.tizen.aurum-bootstrap/src/Commands/CloseAppCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/CloseAppCommand.cc index 61f096e..7ac889a 100644 --- a/org.tizen.aurum-bootstrap/src/Commands/CloseAppCommand.cc +++ b/org.tizen.aurum-bootstrap/src/Commands/CloseAppCommand.cc @@ -46,9 +46,11 @@ CloseAppCommand::CloseAppCommand(const ::aurum::ReqCloseApp *request, if (ret) { LOGI("Terminate Failed(2/2) Err Code : %d", ret); mResponse->set_status(::aurum::RspStatus::ERROR); + app_context_destroy(app_context); return grpc::Status::OK; } + app_context_destroy(app_context); return grpc::Status::OK; } diff --git a/org.tizen.aurum-bootstrap/src/Commands/GetAppInfoCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/GetAppInfoCommand.cc index 420ff7a..76d9b7c 100644 --- a/org.tizen.aurum-bootstrap/src/Commands/GetAppInfoCommand.cc +++ b/org.tizen.aurum-bootstrap/src/Commands/GetAppInfoCommand.cc @@ -58,6 +58,7 @@ GetAppInfoCommand::GetAppInfoCommand(const ::aurum::ReqGetAppInfo *request, mResponse->set_isfocused(appState & APP_STATE_FOREGROUND); mResponse->set_isrunning(!(appState & APP_STATE_TERMINATED)); } + app_context_destroy(app_context); } return grpc::Status::OK; -- 2.7.4 From fc4584c29b3498ef0f8805ab7eacb8d911cbea8d Mon Sep 17 00:00:00 2001 From: Hosang Kim Date: Thu, 23 Jun 2022 19:03:56 +0900 Subject: [PATCH 07/16] Comparer: fix depth start at 1 Change-Id: Id5013a7d0a6746c5f3fe3939deea026df0b16c1c --- libaurum/src/Comparer.cc | 2 +- libaurum/src/UiObject.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libaurum/src/Comparer.cc b/libaurum/src/Comparer.cc index 16c2242..9bbf841 100644 --- a/libaurum/src/Comparer.cc +++ b/libaurum/src/Comparer.cc @@ -79,7 +79,7 @@ std::vector> Comparer::findObjects(const std::sh std::vector> Comparer::findObjects(const std::shared_ptr root) { std::list> partialList{}; - std::vector> ret = findObjects(root, 0, 0, partialList); + std::vector> ret = findObjects(root, 0, 1, partialList); LOGI("%d object(s) found", (int)ret.size()); return ret; } diff --git a/libaurum/src/UiObject.cc b/libaurum/src/UiObject.cc index 706d28f..51ac790 100644 --- a/libaurum/src/UiObject.cc +++ b/libaurum/src/UiObject.cc @@ -144,7 +144,7 @@ std::shared_ptr UiObject::getChildAt(int index) const { std::vector> UiObject::getChildren() const { - auto sel = Sel::depth(1); + auto sel = Sel::depth(2); return this->findObjects(sel); } -- 2.7.4 From 409595a4fa127499cac742c8cef7bd1a92763a92 Mon Sep 17 00:00:00 2001 From: Hosang Kim Date: Thu, 30 Jun 2022 15:45:55 +0900 Subject: [PATCH 08/16] bootstrap: fix crash when element id is invalid Change-Id: Ie39e166be8ee5fa2ae770d655a2337fc55737494 --- org.tizen.aurum-bootstrap/src/ObjectMapper.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/org.tizen.aurum-bootstrap/src/ObjectMapper.cc b/org.tizen.aurum-bootstrap/src/ObjectMapper.cc index db129f3..f3c7a15 100644 --- a/org.tizen.aurum-bootstrap/src/ObjectMapper.cc +++ b/org.tizen.aurum-bootstrap/src/ObjectMapper.cc @@ -51,6 +51,7 @@ std::shared_ptr ObjectMapper::getElement(std::string elementId) } else { LOGI("Object find fail"); + mObjectMap.erase(elementId); return nullptr; } } -- 2.7.4 From 0028e46376e4e76178cff3805fe6c56630aeb5f5 Mon Sep 17 00:00:00 2001 From: Woochanlee Date: Thu, 30 Jun 2022 15:12:17 +0900 Subject: [PATCH 09/16] Delete requirement packages version Change-Id: Ia1bcfd5b8f6ca7614d60977087d8dba63e948973 --- protocol/resources/python/requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/protocol/resources/python/requirements.txt b/protocol/resources/python/requirements.txt index 3a80b3b..2fef712 100644 --- a/protocol/resources/python/requirements.txt +++ b/protocol/resources/python/requirements.txt @@ -1,4 +1,4 @@ -grpcio==1.23.0 -grpcio-tools==1.23.0 -protobuf==3.9.1 -six==1.12.0 +grpcio +grpcio-tools +protobuf +six -- 2.7.4 From eaf92557f522cb714336c5174ba59f1fe4cb5575 Mon Sep 17 00:00:00 2001 From: Hosang Kim Date: Wed, 6 Jul 2022 20:32:59 +0900 Subject: [PATCH 10/16] AtspiWrapper: add get_process_id API Change-Id: I97d2eda3745aabfa6c941108815d15e28b12fa99 --- libaurum/inc/Impl/Accessibility/AtspiWrapper.h | 1 + libaurum/src/Impl/Accessibility/AtspiWrapper.cc | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/libaurum/inc/Impl/Accessibility/AtspiWrapper.h b/libaurum/inc/Impl/Accessibility/AtspiWrapper.h index 89a6dae..f8f69b1 100644 --- a/libaurum/inc/Impl/Accessibility/AtspiWrapper.h +++ b/libaurum/inc/Impl/Accessibility/AtspiWrapper.h @@ -66,6 +66,7 @@ public: static gdouble Atspi_value_get_maximum_value(AtspiValue *iface, GError **error); static gboolean Atspi_value_set_current_value(AtspiValue *iface, gdouble value, GError **error); static gdouble Atspi_value_get_minimum_increment(AtspiValue *iface, GError **error); + static guint Atspi_accessible_get_process_id(AtspiAccessible *node, GError **error); private: static std::recursive_mutex mMutex; diff --git a/libaurum/src/Impl/Accessibility/AtspiWrapper.cc b/libaurum/src/Impl/Accessibility/AtspiWrapper.cc index 34d459f..93e1c37 100644 --- a/libaurum/src/Impl/Accessibility/AtspiWrapper.cc +++ b/libaurum/src/Impl/Accessibility/AtspiWrapper.cc @@ -205,4 +205,10 @@ gdouble AtspiWrapper::Atspi_value_get_minimum_increment(AtspiValue *iface, GErro { std::unique_lock lock(mMutex); return atspi_value_get_minimum_increment(iface, error); +} + +guint AtspiWrapper::Atspi_accessible_get_process_id(AtspiAccessible *node, GError **error) +{ + std::unique_lock lock(mMutex); + return atspi_accessible_get_process_id(node, error); } \ No newline at end of file -- 2.7.4 From 37d3736ff4771aaf31e72b286b41b161879cc588 Mon Sep 17 00:00:00 2001 From: Hosang Kim Date: Thu, 21 Jul 2022 14:10:21 +0900 Subject: [PATCH 11/16] libaurum: check nullptr with rawHandler Change-Id: I5a64588645bbd0357b6f8fb2269ea8281443eb33 --- libaurum/src/AurumXML.cc | 2 +- libaurum/src/Comparer.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libaurum/src/AurumXML.cc b/libaurum/src/AurumXML.cc index e90790e..fe1e0bf 100644 --- a/libaurum/src/AurumXML.cc +++ b/libaurum/src/AurumXML.cc @@ -79,7 +79,7 @@ void AurumXML::traverse(xml_node element, std::shared_ptr node) int childCnt = node->getChildCount(); for (int i = 0; i < childCnt; i++) { std::shared_ptr childNode = node->getChildAt(i); - if (childNode == nullptr) continue; + if (childNode->getRawHandler() == nullptr) continue; xml_node childElement = element.append_child(""); traverse(childElement, childNode); diff --git a/libaurum/src/Comparer.cc b/libaurum/src/Comparer.cc index 9bbf841..517f05f 100644 --- a/libaurum/src/Comparer.cc +++ b/libaurum/src/Comparer.cc @@ -103,7 +103,7 @@ std::vector> Comparer::findObjects( int childCnt = root->getChildCount(); for (int i = 0; i < childCnt; i++) { std::shared_ptr childNode = root->getChildAt(i); - if (childNode == nullptr) continue; + if (childNode->getRawHandler() == nullptr) continue; std::vector> childret = findObjects(childNode, i, depth + 1, partialMatches); -- 2.7.4 From 9c8a7db7bb28338bea092128e5aa4507356213ee Mon Sep 17 00:00:00 2001 From: Hosang Kim Date: Thu, 21 Jul 2022 15:33:09 +0900 Subject: [PATCH 12/16] libaurum: change delay time between key down and key up Change-Id: I83afa617dd23f917fb57b21726173b33a7a55513 --- libaurum/inc/Impl/TizenDeviceImpl.h | 4 ++-- libaurum/src/UiDevice.cc | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/libaurum/inc/Impl/TizenDeviceImpl.h b/libaurum/inc/Impl/TizenDeviceImpl.h index 242cb60..8852a45 100644 --- a/libaurum/inc/Impl/TizenDeviceImpl.h +++ b/libaurum/inc/Impl/TizenDeviceImpl.h @@ -185,7 +185,7 @@ private: efl_util_inputgen_h mFakeKeyboardHandle; efl_util_inputgen_h mFakeWheelHandle; static const int INTV_CLICK = 5; - static const int INTV_SHORTSTROKE = 100; + static const int INTV_SHORTSTROKE = 10; static const int INTV_LONGSTROKE = 2000; static const int INTV_MINIMUM_DRAG_MS = 25; static const int INTV_MINIMUM_USLEEP = 1000; @@ -204,4 +204,4 @@ private: } -#endif \ No newline at end of file +#endif diff --git a/libaurum/src/UiDevice.cc b/libaurum/src/UiDevice.cc index bafa6d7..15ab27f 100644 --- a/libaurum/src/UiDevice.cc +++ b/libaurum/src/UiDevice.cc @@ -261,7 +261,6 @@ bool UiDevice::pressPower(KeyRequestType type) bool UiDevice::pressKeyCode(std::string keycode, KeyRequestType type) { bool result = mDeviceImpl->pressKeyCode(keycode, type); - waitForIdle(); return result; } -- 2.7.4 From a8e993422925b24c687a71c7f08f91aad47ddafc Mon Sep 17 00:00:00 2001 From: Woochanlee Date: Fri, 22 Jul 2022 13:20:27 +0900 Subject: [PATCH 13/16] libaurum: Get window info from window system We uesed to atspi window event to search target window. It can't support multi window situation and something wrong state has window as well. Change-Id: I0a3f61104402332fb6d8feb74e13d11323cc8401 --- libaurum/inc/Accessibility/AccessibleNode.h | 12 ++ libaurum/inc/Aurum.h | 1 + .../inc/Impl/Accessibility/AtspiAccessibleNode.h | 5 + .../inc/Impl/Accessibility/MockAccessibleNode.h | 6 + libaurum/inc/TizenWindow.h | 59 ++++++++ libaurum/inc/UiDevice.h | 11 ++ libaurum/inc/UiObject.h | 16 ++ libaurum/meson.build | 1 + libaurum/src/Accessibility/AccessibleNode.cc | 7 +- .../src/Impl/Accessibility/AtspiAccessibleNode.cc | 7 + .../src/Impl/Accessibility/MockAccessibleNode.cc | 4 + libaurum/src/TizenWindow.cc | 77 ++++++++++ libaurum/src/UiDevice.cc | 166 +++++++++++++++++++-- libaurum/src/UiObject.cc | 5 + libaurum/src/meson.build | 1 + 15 files changed, 365 insertions(+), 13 deletions(-) create mode 100644 libaurum/inc/TizenWindow.h create mode 100644 libaurum/src/TizenWindow.cc diff --git a/libaurum/inc/Accessibility/AccessibleNode.h b/libaurum/inc/Accessibility/AccessibleNode.h index 9a44612..5b9179a 100644 --- a/libaurum/inc/Accessibility/AccessibleNode.h +++ b/libaurum/inc/Accessibility/AccessibleNode.h @@ -218,6 +218,12 @@ public: double getValue() const; /** + * @copydoc UiObject::getPid() + * + */ + int getPid() const; + + /** * @copydoc UiObject::getIncrement() * */ @@ -363,6 +369,11 @@ public: virtual void updateValue() = 0; /** + * @copydoc UiObject::updatePid() + */ + virtual void updatePid() = 0; + + /** * @copydoc UiObject::setFocus() */ virtual bool setFocus() = 0; @@ -478,6 +489,7 @@ protected: Rect mWindowBoundingBox; int mSupportingIfaces; int mFeatureProperty; + int mPid; double mMinValue; double mMaxValue; double mValue; diff --git a/libaurum/inc/Aurum.h b/libaurum/inc/Aurum.h index 52bb8b2..0003c39 100644 --- a/libaurum/inc/Aurum.h +++ b/libaurum/inc/Aurum.h @@ -47,6 +47,7 @@ #include "UiDevice.h" #include "UiObject.h" #include "UiSelector.h" +#include "TizenWindow.h" #include "Sel.h" #include "Until.h" #include "Waiter.h" diff --git a/libaurum/inc/Impl/Accessibility/AtspiAccessibleNode.h b/libaurum/inc/Impl/Accessibility/AtspiAccessibleNode.h index 2724c93..5b8a3b6 100644 --- a/libaurum/inc/Impl/Accessibility/AtspiAccessibleNode.h +++ b/libaurum/inc/Impl/Accessibility/AtspiAccessibleNode.h @@ -127,6 +127,11 @@ public: void updateValue() override; /** + * @copydoc UiObject::updatePid() + */ + void updatePid() override; + + /** * @copydoc UiObject::setFocus() */ bool setFocus() override; diff --git a/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h b/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h index be6ce32..68ef294 100644 --- a/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h +++ b/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h @@ -133,6 +133,12 @@ public: * @brief TBD * @since_tizen 7.0 */ + void updatePid() override; + + /** + * @brief TBD + * @since_tizen 7.0 + */ bool setFocus() override; /** diff --git a/libaurum/inc/TizenWindow.h b/libaurum/inc/TizenWindow.h new file mode 100644 index 0000000..4c86227 --- /dev/null +++ b/libaurum/inc/TizenWindow.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef _TIZEN_WINDOW_H_ +#define _TIZEN_WINDOW_H_ + +#include + +namespace Aurum { + +class TizenWindow : public std::enable_shared_from_this { +public: + TizenWindow(int pid, Rect geometry, bool transformed, bool alpha, int opaque, int visibility, bool focused, bool mapped, int layer, std::string name); + ~TizenWindow(); + + int getPid(); + Rect getWindowGeometry(); + bool isTransformed(); + bool isAlpah(); + int getOpaque(); + int getVisibility(); + bool isFocused(); + bool isMapped(); + std::string getName(); +private: + int mPid; // window's PID + Rect mWindowGeometry; // window's geometry + bool mIsTransformed; // true : transformed window + bool mIsAlpha; // true : alpha + int mOpaque; // true : opaque state set window + int mVisibility; // 0 : visible + // 1 : partial visible + // 2 : hidden + // -1 : unknown + bool mIsFocused; // true : has focus + bool mIsMapped; // true : mapped + int mLayer; // 200 : normal layer + // 250 : above layer + // 650 ~ 800 : notification layer + std::string mName; // window name +}; + +} + +#endif diff --git a/libaurum/inc/UiDevice.h b/libaurum/inc/UiDevice.h index 669e0b7..ba21d4a 100644 --- a/libaurum/inc/UiDevice.h +++ b/libaurum/inc/UiDevice.h @@ -22,6 +22,7 @@ #include "UiObject.h" #include "UiSelector.h" +#include "TizenWindow.h" #include "IDevice.h" #include "ISearchable.h" @@ -369,6 +370,15 @@ public: * @since_tizen 6.5 */ std::vector> getWindowRoot() const; + + /** + * @brief Gets currently window information from window system. + * + * @return TizenWindow ptr vector + * @since_tizen 7.0 + */ + std::vector> getTizenWindowInfo() const; + private: /** * @brief Waits process idle. @@ -403,6 +413,7 @@ public: private: IDevice *mDeviceImpl; const Waiter *mWaiter; + static std::vector> mTizenWindows; }; } diff --git a/libaurum/inc/UiObject.h b/libaurum/inc/UiObject.h index 98c3a6c..018d1ad 100644 --- a/libaurum/inc/UiObject.h +++ b/libaurum/inc/UiObject.h @@ -330,6 +330,15 @@ public: const double getValue() const; /** + * @brief Gets object's process id. + * + * @return int + * + * @since_tizen 7.0 + */ + const int getPid() const; + + /** * @brief Gets object's current increment. * * @return double @@ -551,6 +560,13 @@ public: void updateValue() const; /** + * @brief Updates object's process id. + * + * @since_tizen 7.0 + */ + void updatePid() const; + + /** * @brief Sets focus to object. * * @since_tizen 7.0 diff --git a/libaurum/meson.build b/libaurum/meson.build index 5440ee7..66c87a0 100644 --- a/libaurum/meson.build +++ b/libaurum/meson.build @@ -4,6 +4,7 @@ libaurum_install_inc = [ './inc/UiDevice.h', './inc/UiObject.h', './inc/UiSelector.h', + './inc/TizenWindow.h', './inc/Sel.h', './inc/Until.h', './inc/Waiter.h', diff --git a/libaurum/src/Accessibility/AccessibleNode.cc b/libaurum/src/Accessibility/AccessibleNode.cc index 4fc7fcf..6e8eb02 100644 --- a/libaurum/src/Accessibility/AccessibleNode.cc +++ b/libaurum/src/Accessibility/AccessibleNode.cc @@ -248,4 +248,9 @@ double AccessibleNode::getValue() const double AccessibleNode::getIncrement() const { return mIncrement; -} \ No newline at end of file +} + +int AccessibleNode::getPid() const +{ + return mPid; +} diff --git a/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc b/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc index 5d9e922..8d04ab3 100644 --- a/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc +++ b/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc @@ -254,6 +254,13 @@ void AtspiAccessibleNode::updateValue() } } +void AtspiAccessibleNode::updatePid() +{ + AtspiWrapper::Atspi_accessible_clear_cache(mNode); + + mPid = AtspiWrapper::Atspi_accessible_get_process_id(mNode, NULL); +} + bool AtspiAccessibleNode::setFocus() { AtspiComponent *component = AtspiWrapper::Atspi_accessible_get_component_iface(mNode); diff --git a/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc b/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc index 51ba193..8900dd8 100644 --- a/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc +++ b/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc @@ -117,6 +117,10 @@ void MockAccessibleNode::updateValue() { } +void MockAccessibleNode::updatePid() +{ +} + bool MockAccessibleNode::setFocus() { return false; diff --git a/libaurum/src/TizenWindow.cc b/libaurum/src/TizenWindow.cc new file mode 100644 index 0000000..ae19068 --- /dev/null +++ b/libaurum/src/TizenWindow.cc @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "Aurum.h" +#include "TizenWindow.h" + +using namespace Aurum; + +TizenWindow::TizenWindow(int pid, Rect geometry, bool transformed, bool alpha, int opaque, int visibility, bool focused, bool mapped, int layer, std::string name) + : mPid(pid), mWindowGeometry(geometry), mIsTransformed(transformed), mIsAlpha(alpha), mOpaque(opaque), mVisibility(visibility), mIsFocused(focused), mIsMapped(mapped), mLayer(layer), mName(name) +{ + +} + +TizenWindow::~TizenWindow() +{ + +} + +int TizenWindow::getPid() +{ + return mPid; +} + +Rect TizenWindow::getWindowGeometry() +{ + return mWindowGeometry; +} + +bool TizenWindow::isTransformed() +{ + return mIsTransformed; +} + +bool TizenWindow::isAlpah() +{ + return mIsAlpha; +} + +int TizenWindow::getOpaque() +{ + return mOpaque; +} + +int TizenWindow::getVisibility() +{ + return mVisibility; +} + +bool TizenWindow::isFocused() +{ + return mIsFocused; +} + +bool TizenWindow::isMapped() +{ + return mIsMapped; +} + +std::string TizenWindow::getName() +{ + return mName; +} \ No newline at end of file diff --git a/libaurum/src/UiDevice.cc b/libaurum/src/UiDevice.cc index 15ab27f..ea76073 100644 --- a/libaurum/src/UiDevice.cc +++ b/libaurum/src/UiDevice.cc @@ -29,10 +29,19 @@ #include #include #include +#include using namespace Aurum; using namespace AurumInternal; +#define WM_BUS_NAME "org.enlightenment.wm" +#define WM_OBJECT_PATH "/org/enlightenment/wm" +#define WM_INTERFACE_NAME "org.enlightenment.wm.proc" +#define WM_METHOD_NAME_INFO "GetVisibleWinInfo_v2" + +std::vector> UiDevice::mTizenWindows; +static GDBusConnection *system_conn; + UiDevice::UiDevice() : UiDevice(nullptr) {} UiDevice::UiDevice(IDevice *impl) @@ -66,21 +75,154 @@ std::shared_ptr UiDevice::getInstance(IDevice *deviceImpl) return device; } +std::vector> UiDevice::getTizenWindowInfo() const +{ + GError *err = NULL; + GDBusMessage *msg; + GDBusMessage *reply; + GDBusConnection *conn; + GVariant *body; + GVariantIter *iter = NULL; + int idx = 0; + int pid; + int x; + int y; + int w; + int h; + gboolean transformed; + gboolean alpha; + int opaque; + int visibility; + gboolean focused; + gboolean mapped; + int layer; + char *name; + + mTizenWindows.clear(); + + if (system_conn == NULL) { + conn = g_bus_get_sync(G_BUS_TYPE_SYSTEM, NULL, &err); + if (conn == NULL) { + LOGE("g_bus_get_sync() is failed. %s", err->message); + g_error_free(err); + return mTizenWindows; + } + system_conn = conn; + } + + msg = g_dbus_message_new_method_call(WM_BUS_NAME, + WM_OBJECT_PATH, + WM_INTERFACE_NAME, + WM_METHOD_NAME_INFO); + if (msg == NULL) { + LOGE("g_dbus_message_new_method_call() is failed."); + return mTizenWindows; + } + + reply = g_dbus_connection_send_message_with_reply_sync(system_conn, msg, + G_DBUS_SEND_MESSAGE_FLAGS_NONE, -1, NULL, NULL, &err); + + if (!reply) { + if (err != NULL) { + LOGE("Failed to get info [%s]", err->message); + g_error_free(err); + } + goto out; + } + + body = g_dbus_message_get_body(reply); + if (!body) { + LOGE("Failed to get body"); + goto out; + } + + g_variant_get(body, "(a(iiiiibbiibbis))", &iter); + + LOGI("%-3s | %-6s | %-4s | %-4s | %-4s | %-4s | %-5s | %-5s | %-6s | %-3s | %-7s | %-6s | %-5s | %-20s", "No" ,"PID", "X", "Y", "W", "H", "Trans", "Alpha", "Opaque", "Vis", "Focused", "Mapped", "Layer", "Name"); + while (g_variant_iter_loop(iter, "(iiiiibbiibbis)", + &pid, + &x, + &y, + &w, + &h, + &transformed, + &alpha, + &opaque, + &visibility, + &focused, + &mapped, + &layer, + &name)) { + LOGI("%-3d | %-6d | %-4d | %-4d | %-4d | %-4d | %-5d | %-5d | %-6d | %-3d | %-7d | %-6d | %-5d | %-20s", idx++, pid, x,y,w,h, transformed, alpha, opaque, visibility, focused, mapped, layer, name); + if (visibility == 0 && pid > 0) + { + Rect geometry = {x, y, w, h}; + std::string winName(name); + mTizenWindows.push_back(std::make_shared(pid, geometry, transformed, alpha, opaque, visibility, focused, mapped, layer, winName)); + } + } + + if (iter) + g_variant_iter_free(iter); +out: + if (msg) + g_object_unref(msg); + if (reply) + g_object_unref(reply); + + return mTizenWindows; +} + std::vector> UiDevice::getWindowRoot() const { + bool dup; + LOGI("Request window info"); + getTizenWindowInfo(); + std::vector> ret{}; - auto appsMap = AccessibleWatcher::getInstance()->getActiveAppMap(); - LOGI("activeAppMap.size: %d" , (int)appsMap.size()); - for (auto itr = appsMap.begin(); itr != appsMap.end(); itr++) + auto apps = AccessibleWatcher::getInstance()->getApplications(); + for (auto app : apps) + { + app->getAccessibleNode()->updateName(); + app->getAccessibleNode()->updatePid(); + LOGI("App(%s) Pid(%d)", app->getPackageName().c_str(), app->getAccessibleNode()->getPid()); + } + + for (auto tWin : mTizenWindows) { - auto activeWindows = itr->second->getActiveWindows(); - std::transform(activeWindows.begin(), activeWindows.end(), std::back_inserter(ret), - [&](std::shared_ptr window){ - LOGI("active pkg: %s, window: %s", window->getAccessibleNode()->getPkg().c_str(), window->getTitle().c_str()); - return window->getAccessibleNode(); + LOGI("Visible win (%d) (%d %d %d %d) (%s)", tWin->getPid(), tWin->getWindowGeometry().mTopLeft.x, tWin->getWindowGeometry().mTopLeft.y, tWin->getWindowGeometry().width(), + tWin->getWindowGeometry().height(), tWin->getName().c_str()); + + for (auto app : apps) + { + dup = false; + if (app->getAccessibleNode()->getPid() == tWin->getPid()) + { + for (const auto &retWin : ret) + { + retWin->getParent()->updatePid(); + LOGI("Pid Dup check in vector (%d) target (%d)", retWin->getParent()->getPid(), tWin->getPid()); + if (retWin->getParent()->getPid() == tWin->getPid()) + { + dup = true; + break; + } + } + + if (!dup) + { + LOGI("Actvie App : (%s) (%d)", tWin->getName().c_str(), tWin->getPid()); + auto wins = app->getWindows(); + std::transform(wins.begin(), wins.end(), std::back_inserter(ret), + [&](std::shared_ptr window){ + LOGI("Target window add pkg: (%s), name (%s)", window->getAccessibleNode()->getPkg().c_str(), window->getTitle().c_str()); + return window->getAccessibleNode(); + } + ); + } } - ); + } } return ret; @@ -143,19 +285,19 @@ bool UiDevice::waitForIdle() const } bool UiDevice::waitForEvents( - const A11yEvent type, const int timeout) const + const A11yEvent type, const int timeout) const { return executeAndWaitForEvents(NULL, type, timeout); } bool UiDevice::executeAndWaitForEvents( - const Runnable *cmd, const A11yEvent type, const int timeout) const + const Runnable *cmd, const A11yEvent type, const int timeout) const { return AccessibleWatcher::getInstance()->executeAndWaitForEvents(cmd, type, timeout); } bool UiDevice::sendKeyAndWaitForEvents( - const std::string keycode, const A11yEvent type, const int timeout) const + const std::string keycode, const A11yEvent type, const int timeout) const { std::unique_ptr cmd = std::make_unique(keycode); return executeAndWaitForEvents(cmd.get(), type, timeout); diff --git a/libaurum/src/UiObject.cc b/libaurum/src/UiObject.cc index 51ac790..fe54f4f 100644 --- a/libaurum/src/UiObject.cc +++ b/libaurum/src/UiObject.cc @@ -344,6 +344,11 @@ void UiObject::updateValue() const mNode->updateValue(); } +void UiObject::updatePid() const +{ + mNode->updatePid(); +} + bool UiObject::setFocus() const { return mNode->setFocus(); diff --git a/libaurum/src/meson.build b/libaurum/src/meson.build index 11af3ea..390e805 100644 --- a/libaurum/src/meson.build +++ b/libaurum/src/meson.build @@ -10,6 +10,7 @@ libaurum_src += [ files('PartialMatch.cc'), files('A11yEvent.cc'), files('AurumXML.cc'), + files('TizenWindow.cc'), ] subdir('Accessibility') -- 2.7.4 From 4ab23188a17cfd73b5a8cf8ae07d212f083d4493 Mon Sep 17 00:00:00 2001 From: Hosang Kim Date: Fri, 29 Jul 2022 16:47:25 +0900 Subject: [PATCH 14/16] AccessibleNode: initialize pid Change-Id: I4fbbbf6c6ef3d47845ccd12d8e8ef3ef14dd0a3c --- libaurum/src/Accessibility/AccessibleNode.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libaurum/src/Accessibility/AccessibleNode.cc b/libaurum/src/Accessibility/AccessibleNode.cc index 6e8eb02..19e81a4 100644 --- a/libaurum/src/Accessibility/AccessibleNode.cc +++ b/libaurum/src/Accessibility/AccessibleNode.cc @@ -32,7 +32,7 @@ AccessibleNode::~AccessibleNode() AccessibleNode::AccessibleNode() : mText{""}, mPkg{""}, mRole{""}, mId{""}, mAutomationId{""}, mType{""}, mStyle{""}, mXPath{""}, - mScreenBoundingBox{0,0,0,0}, mWindowBoundingBox{0,0,0,0}, mSupportingIfaces(0), mFeatureProperty(0), mMinValue{0.0}, mMaxValue{0.0}, mValue{0.0}, mIncrement{0.0}, mValid{true}, mLock{} + mScreenBoundingBox{0,0,0,0}, mWindowBoundingBox{0,0,0,0}, mSupportingIfaces(0), mFeatureProperty(0), mPid(0), mMinValue{0.0}, mMaxValue{0.0}, mValue{0.0}, mIncrement{0.0}, mValid{true}, mLock{} { } -- 2.7.4 From 893e6246123e7f47c927a5bbc027898e72392770 Mon Sep 17 00:00:00 2001 From: Woochanlee Date: Thu, 4 Aug 2022 19:18:35 +0900 Subject: [PATCH 15/16] Introduce Intelligent Ui Automation thorough Screen Analyzer +libaurum: Change name ElementType -> Type +libaurum: efl_util API resource protection with mutex +ScreenAnalyzer: Add mqtt_enabled option for build with or without mosquitto lib ScreenAnalyzer Enable build command : gbs build -A armv7l --define "mqtt 1" --include-all +ScreenAnlayzer: Add SetServerIP for user can choose server ScreenAnalyzer: Code re-arrange and fix warning ScreenAnalyzer: Add commands for GSA with specific features (FR15, FR13, FR14, FR8) ScreenAnalyzer: Add command to get application toolkit info (FR3, FR1) ScreenAnalyzer: Use secvideo_capture API instead of efl_util's for performance (NFR2) ScreenAnalyzer: Add IObject interface then implement in Node, UiObject, SaObject (FR10, FR6, FR11) ScreenAnalyzer: Gets focused app Id and check the atspi window activated or not (FR10) ScreenAnalyzer: Support Ui-Analzyer (FR12) ScreenAnalyzer: Support compatibility Atspi with Screen Analyzer in command (FR9) ScreenAnalyzer: Add functions for request ScreenAnalyzer (FR4) ScreenAnalyzer: Gets UI information from Screen Analyzer and parse it to saobject (FR4, FR5, FR7, FR8) ScreenAnalyzer: Check the extenal application state at runtime (FR3) ScreenAnalyzer: Introduce object class for screen analayzer (FR5) Change-Id: I4f75f61f93985b6ad9a1fe172ddff0882697ad30 --- libaurum/inc/Accessibility/AccessibleNode.h | 111 +++--- libaurum/inc/Aurum.h | 5 + .../inc/Impl/Accessibility/AtspiAccessibleNode.h | 5 + libaurum/inc/Impl/Accessibility/AtspiWrapper.h | 1 + .../inc/Impl/Accessibility/MockAccessibleNode.h | 6 + libaurum/inc/Impl/TizenDeviceImpl.h | 2 + libaurum/inc/Interface/IObject.h | 84 +++++ libaurum/inc/Misc/Rect.h | 16 + libaurum/inc/PartialMatch.h | 6 +- libaurum/inc/SaObject.h | 105 ++++++ .../inc/ScreenAnalyzer/ScreenAnalyzerWatcher.h | 147 ++++++++ libaurum/inc/UiDevice.h | 64 ++++ libaurum/inc/UiObject.h | 171 +++++---- libaurum/inc/UiSelector.h | 13 + libaurum/meson.build | 21 ++ .../src/Accessibility/AccessibleApplication.cc | 4 +- libaurum/src/Accessibility/AccessibleNode.cc | 22 +- .../src/Impl/Accessibility/AtspiAccessibleNode.cc | 17 + libaurum/src/Impl/Accessibility/AtspiWrapper.cc | 8 +- .../src/Impl/Accessibility/MockAccessibleNode.cc | 4 + libaurum/src/Impl/TizenDeviceImpl.cc | 9 +- libaurum/src/PartialMatch.cc | 34 +- libaurum/src/SaObject.cc | 96 +++++ .../src/ScreenAnalyzer/ScreenAnalyzerWatcher.cc | 386 +++++++++++++++++++++ libaurum/src/ScreenAnalyzer/meson.build | 3 + libaurum/src/UiDevice.cc | 45 +++ libaurum/src/UiObject.cc | 23 +- libaurum/src/UiSelector.cc | 15 +- libaurum/src/meson.build | 9 + meson.build | 4 + meson_options.txt | 6 + org.tizen.aurum-bootstrap/inc/AurumServiceImpl.h | 6 + org.tizen.aurum-bootstrap/inc/Commands/Commands.h | 2 + .../inc/Commands/EnableScreenAnalyzerCommand.h | 34 ++ .../inc/Commands/GetActiveAppToolkitNameCommand.h | 34 ++ org.tizen.aurum-bootstrap/meson.build | 2 + org.tizen.aurum-bootstrap/src/AurumServiceImpl.cc | 17 + .../src/Commands/DumpObjectTreeCommand.cc | 74 +++- .../src/Commands/EnableScreenAnalyzerCommand.cc | 41 +++ .../src/Commands/FindElementCommand.cc | 170 +++++---- .../src/Commands/FindElementsCommand.cc | 178 ++++++---- .../src/Commands/GetActiveAppToolkitNameCommand.cc | 33 ++ .../src/Commands/GetSizeCommand.cc | 79 ++++- packaging/aurum.spec | 16 +- protocol/aurum.proto | 75 ++-- tests/Test_UiObject.cc | 4 +- tests/Test_UiSelector.cc | 6 +- 47 files changed, 1911 insertions(+), 302 deletions(-) create mode 100644 libaurum/inc/Interface/IObject.h create mode 100644 libaurum/inc/SaObject.h create mode 100644 libaurum/inc/ScreenAnalyzer/ScreenAnalyzerWatcher.h create mode 100644 libaurum/src/SaObject.cc create mode 100644 libaurum/src/ScreenAnalyzer/ScreenAnalyzerWatcher.cc create mode 100644 libaurum/src/ScreenAnalyzer/meson.build create mode 100644 org.tizen.aurum-bootstrap/inc/Commands/EnableScreenAnalyzerCommand.h create mode 100644 org.tizen.aurum-bootstrap/inc/Commands/GetActiveAppToolkitNameCommand.h create mode 100644 org.tizen.aurum-bootstrap/src/Commands/EnableScreenAnalyzerCommand.cc create mode 100644 org.tizen.aurum-bootstrap/src/Commands/GetActiveAppToolkitNameCommand.cc diff --git a/libaurum/inc/Accessibility/AccessibleNode.h b/libaurum/inc/Accessibility/AccessibleNode.h index 5b9179a..2fa3a41 100644 --- a/libaurum/inc/Accessibility/AccessibleNode.h +++ b/libaurum/inc/Accessibility/AccessibleNode.h @@ -25,6 +25,7 @@ #include #include "IEventConsumer.h" +#include "IObject.h" #include "Rect.h" #include "config.h" @@ -83,7 +84,7 @@ enum class NodeFeatureProperties { * * @since_tizen 6.5 */ -class AccessibleNode : public std::enable_shared_from_this, public IEventConsumer { +class AccessibleNode : public std::enable_shared_from_this, public IEventConsumer, public IObject { public: /** * @brief AccessibleNode constructor. @@ -150,19 +151,70 @@ public: public: /** + * @copydoc UiObject::getId() + */ + std::string getId() const override; + + /** + * @copydoc UiObject::getType() + */ + std::string getType() const override; + + /** + * @copydoc UiObject::getScreenBoundingBox() + */ + const Rect getScreenBoundingBox() const override; + + /** + * @copydoc UiObject::getOcrText() + */ + std::string getOcrText() const override; + + /** + * @copydoc UiObject::isFocusable() + */ + bool isFocusable() const override; + + /** + * @copydoc UiObject::isFocused() + */ + bool isFocused() const override; + + /** + * @copydoc UiObject::isClickable() + */ + bool isClickable() const override; + + /** + * @copydoc UiObject::isActive() + */ + bool isActive() const override; + + /** + * @copydoc UiObject::isShowing() + */ + bool isShowing() const override; + +public: + /** * @copydoc UiObject::getText() */ std::string getText() const; /** - * @copydoc UiObject::getPkg() + * @copydoc UiObject::setText() */ - std::string getPkg() const; + bool setText(std::string text); /** - * @copydoc UiObject::getId() + * @copydoc UiObject::setOcrText() */ - std::string getId() const; + void setOcrText(std::string text); + + /** + * @copydoc UiObject::getPkg() + */ + std::string getPkg() const; /** * @copydoc UiObject::getAutomationId() @@ -175,29 +227,24 @@ public: std::string getRole() const; /** - * @copydoc UiObject::getType() - */ - std::string getType() const; - - /** * @copydoc UiObject::getStyle() */ std::string getStyle() const; /** - * @copydoc UiObject::getXPath() + * @copydoc UiObject::getToolkitName() */ - std::string getXPath() const; + std::string getToolkitName() const; /** - * @copydoc UiObject::getScreenBoundingBox() + * @copydoc UiObject::getWindowBoundingBox() */ - Rect getScreenBoundingBox() const; + const Rect getWindowBoundingBox() const; /** - * @copydoc UiObject::getWindowBoundingBox() + * @copydoc UiObject::getXPath() */ - Rect getWindowBoundingBox() const; + std::string getXPath() const; /** * @copydoc UiObject::getMinValue() @@ -240,26 +287,11 @@ public: bool isChecked() const; /** - * @copydoc UiObject::isClickable() - */ - bool isClickable() const; - - /** * @copydoc UiObject::isEnabled() */ bool isEnabled() const; /** - * @copydoc UiObject::isFocusable() - */ - bool isFocusable() const; - - /** - * @copydoc UiObject::isFocused() - */ - bool isFocused() const; - - /** * @copydoc UiObject::isLongClickable() */ bool isLongClickable() const; @@ -284,16 +316,6 @@ public: */ bool isVisible() const; - /** - * @copydoc UiObject::isShowing() - */ - bool isShowing() const; - - /** - * @copydoc UiObject::isActive() - */ - bool isActive() const; - public: /** * @brief Print Node information. @@ -364,6 +386,11 @@ public: virtual void updateXPath() = 0; /** + * @copydoc UiObject::updateToolkitName() + */ + virtual void updateToolkitName() = 0; + + /** * @copydoc UiObject::updateValue() */ virtual void updateValue() = 0; @@ -478,6 +505,7 @@ public: protected: std::string mText; + std::string mOcrText; std::string mPkg; std::string mRole; std::string mId; @@ -485,6 +513,7 @@ protected: std::string mType; std::string mStyle; std::string mXPath; + std::string mToolkitName; Rect mScreenBoundingBox; Rect mWindowBoundingBox; int mSupportingIfaces; diff --git a/libaurum/inc/Aurum.h b/libaurum/inc/Aurum.h index 0003c39..c916371 100644 --- a/libaurum/inc/Aurum.h +++ b/libaurum/inc/Aurum.h @@ -53,6 +53,7 @@ #include "Waiter.h" #include "ISearchable.h" #include "IDevice.h" +#include "IObject.h" #include "Accessible.h" #include "Point2D.h" #include "Size2D.h" @@ -61,5 +62,9 @@ #include "Comparer.h" #include "A11yEvent.h" #include "Runnables.h" +#ifdef MQTT_ENABLED +#include "ScreenAnalyzerWatcher.h" +#include "SaObject.h" +#endif #endif diff --git a/libaurum/inc/Impl/Accessibility/AtspiAccessibleNode.h b/libaurum/inc/Impl/Accessibility/AtspiAccessibleNode.h index 5b8a3b6..ffd1816 100644 --- a/libaurum/inc/Impl/Accessibility/AtspiAccessibleNode.h +++ b/libaurum/inc/Impl/Accessibility/AtspiAccessibleNode.h @@ -122,6 +122,11 @@ public: void updateXPath() override; /** + * @copydoc UiObject::updateToolkitName() + */ + void updateToolkitName() override; + + /** * @copydoc UiObject::updateValue() */ void updateValue() override; diff --git a/libaurum/inc/Impl/Accessibility/AtspiWrapper.h b/libaurum/inc/Impl/Accessibility/AtspiWrapper.h index f8f69b1..d34ceba 100644 --- a/libaurum/inc/Impl/Accessibility/AtspiWrapper.h +++ b/libaurum/inc/Impl/Accessibility/AtspiWrapper.h @@ -67,6 +67,7 @@ public: static gboolean Atspi_value_set_current_value(AtspiValue *iface, gdouble value, GError **error); static gdouble Atspi_value_get_minimum_increment(AtspiValue *iface, GError **error); static guint Atspi_accessible_get_process_id(AtspiAccessible *node, GError **error); + static gchar *Atspi_accessible_get_toolkit_name(AtspiAccessible *node, GError **error); private: static std::recursive_mutex mMutex; diff --git a/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h b/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h index 68ef294..2a8afa6 100644 --- a/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h +++ b/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h @@ -127,6 +127,12 @@ public: * @brief TBD * @since_tizen 7.0 */ + void updateToolkitName() override; + + /** + * @brief TBD + * @since_tizen 7.0 + */ void updateValue() override; /** diff --git a/libaurum/inc/Impl/TizenDeviceImpl.h b/libaurum/inc/Impl/TizenDeviceImpl.h index 8852a45..253685b 100644 --- a/libaurum/inc/Impl/TizenDeviceImpl.h +++ b/libaurum/inc/Impl/TizenDeviceImpl.h @@ -23,6 +23,7 @@ #include #include +#include using namespace Aurum; @@ -200,6 +201,7 @@ private: * @brief TBD */ Size2D mScreenSize; + static std::mutex CaptureMutex; }; } diff --git a/libaurum/inc/Interface/IObject.h b/libaurum/inc/Interface/IObject.h new file mode 100644 index 0000000..0e6ee9f --- /dev/null +++ b/libaurum/inc/Interface/IObject.h @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef _IOBJECT_H_ +#define _IOBJECT_H_ + +#include "config.h" +#include "Rect.h" +#include + +namespace Aurum { + + +class IObject { +public: + + /** + * @brief IObject destructor + */ + virtual ~IObject() {} + + /** + * @copydoc UiObject::getId() + */ + virtual std::string getId() const = 0; + + /** + * @copydoc UiObject::getType() + */ + virtual std::string getType() const = 0; + + /** + * @copydoc UiObject::getScreenBoundingBox() + */ + virtual const Rect getScreenBoundingBox() const = 0; + + /** + * @copydoc UiObject::getOcrText() + */ + virtual std::string getOcrText() const = 0; + + /** + * @copydoc UiObject::isFocusable() + */ + virtual bool isFocusable() const = 0; + + /** + * @copydoc UiObject::isFocused() + */ + virtual bool isFocused() const = 0; + + /** + * @copydoc UiObject::isClickable() + */ + virtual bool isClickable() const = 0; + + /** + * @copydoc UiObject::isActive() + */ + virtual bool isActive() const = 0; + + /** + * @copydoc UiObject::isShowing() + */ + virtual bool isShowing() const = 0; +}; + +} + +#endif diff --git a/libaurum/inc/Misc/Rect.h b/libaurum/inc/Misc/Rect.h index 70410af..cf9323d 100644 --- a/libaurum/inc/Misc/Rect.h +++ b/libaurum/inc/Misc/Rect.h @@ -113,6 +113,22 @@ public: T height() const { return mBottomRight.y - mTopLeft.y; } /** + * @brief Checks whether the given point in the rect or not. + * + * @return true if point located in rect, otherwise false + * + * @since_tizen 7.0 + */ + bool isInRect(const Point2D &point) const + { + if (point.x >= mTopLeft.x && point.x <= mBottomRight.x + && point.y >= mTopLeft.y && point.y <= mBottomRight.y) + return true; + + return false; + } + + /** * @brief Checks given Rect is same as this or not. * * @param[in] rhs @Rect diff --git a/libaurum/inc/PartialMatch.h b/libaurum/inc/PartialMatch.h index 8413543..4806683 100644 --- a/libaurum/inc/PartialMatch.h +++ b/libaurum/inc/PartialMatch.h @@ -129,13 +129,15 @@ private: * * @param[in] selector @UiSelector * @param[in] node @AccessibleNode target node + * @param[in] device @UiDevice * - * @return ture if satisfied, else false + * @return true if satisfied, else false * * @since_tizen 6.5 */ static bool checkCriteria(const std::shared_ptr selector, - const std::shared_ptr node); + const std::shared_ptr node, + std::shared_ptr device); /** * @brief Checks text matched or not. diff --git a/libaurum/inc/SaObject.h b/libaurum/inc/SaObject.h new file mode 100644 index 0000000..422f264 --- /dev/null +++ b/libaurum/inc/SaObject.h @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef _SA_OBJECT_H_ +#define _SA_OBJECT_H_ + +#include "config.h" +#include + +namespace Aurum { + +/** + * @class SaObject + * + * @ingroup aurum + * + * @brief A SaObject is a result of Screen Analyzer. + */ +class SaObject : public IObject, public std::enable_shared_from_this { +public: + + /** + * @brief SaObject constructor with initial values. + * + * @since_tizen 7.0 + */ + SaObject(std::string id, std::string type, Rect geometry, std::string ocrText, std::vector states); + + /** + * @brief SaObject destructor. + * + * @since_tizen 7.0 + */ + ~SaObject(); + + /** + * @copydoc UiObject::getId() + */ + std::string getId() const override; + + /** + * @copydoc UiObject::getType() + */ + std::string getType() const override; + + /** + * @copydoc UiObject::getScreenBoundingBox() + */ + const Rect getScreenBoundingBox() const override; + + std::string getOcrText() const override; + + /** + * @copydoc UiObject::isFocusable() + */ + bool isFocusable() const override; + + /** + * @copydoc UiObject::isFocused() + */ + bool isFocused() const override; + + /** + * @copydoc UiObject::isClickable() + */ + bool isClickable() const override; + + /** + * @copydoc UiObject::isActive() + */ + bool isActive() const override; + + /** + * @copydoc UiObject::isShowing() + */ + bool isShowing() const override; + +private: + std::string mId; + std::string mElementType; + Rect mScreenBoundingBox; + std::string mOcrText; + bool mIsFocusable; + bool mIsFocused; + bool mIsClickable; + bool mIsActive; +}; + +} + +#endif diff --git a/libaurum/inc/ScreenAnalyzer/ScreenAnalyzerWatcher.h b/libaurum/inc/ScreenAnalyzer/ScreenAnalyzerWatcher.h new file mode 100644 index 0000000..d013c8a --- /dev/null +++ b/libaurum/inc/ScreenAnalyzer/ScreenAnalyzerWatcher.h @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef _SCREEN_ANALYZER_WATCHER_H_ +#define _SCREEN_ANALYZER_WATCHER_H_ + +#include + +namespace Aurum { + +/** + * @class ScreenAnalyzerWatcher + * + * @ingroup aurum + * + * @brief ScreenAnalyzerWatcher communicates with Screen Analyzer Server via MQTT Protocol(mosquitto). + * It requests screen analyzer to server and parses the result that can be used in aurum. + */ +class ScreenAnalyzerWatcher { +public: + + /** + * @brief ScreenAnalyzerWatcher constructor. + * + * @since_tizen 7.0 + */ + ScreenAnalyzerWatcher(); + + /** + * @brief ScreenAnalyzerWatcher destructor. + * + * @since_tizen 7.0 + */ + ~ScreenAnalyzerWatcher(); + + /** + * @brief When message(analyze result) is received from the broker. + * parse the data and create SaObject. + * + * @param[in] mosq mosquitto instance + * @param[in] obj user data provided in mosquitto_new + * @param[in] msg the message data + * + * @since_tizen 7.0 + */ + static void onMessage(struct mosquitto *mosq, void *obj, const struct mosquitto_message *msg); + + /** + * @brief Publish current screen image to the broker. + * capture current screen and publish a message on a given topic. + * + * @since_tizen 7.0 + */ + void PublishData(); + + /** + * @brief Publish current screen image to the broker. + * here capture current screen and publish a message on a given topic. + * + * @since_tizen 7.0 + */ + std::vector> GetSaObjects(); + + /** + * @copydoc PartialMatch::checkCriteria() + */ + bool checkCriteria(const std::string textA, const std::string textB, const bool textPartialMatch); + + /** + * @copydoc PartialMatch::checkCriteria() + */ + bool checkCriteria(const bool boolA, const bool boolB); + + /** + * @copydoc PartialMatch::checkCriteria() + */ + bool checkCriteria(const std::shared_ptr selector, const std::shared_ptr node); + + /** + * @brief find object from screen analyzer result. + * + * @param[in] selector @UiSelector + + * + * @return SaObject vector if found, else nulltpr + * + * @since_tizen 7.0 + */ + std::vector> findSaObjects(const std::shared_ptr selector); + + /** + * @brief find object from screen analyzer result. + * + * @param[in] selector @UiSelector + + * + * @return SaObject if found, else nulltpr + * + * @since_tizen 7.0 + */ + std::shared_ptr findSaObject(const std::shared_ptr selector); + + /** + * @brief Get focused app name. + * + * @param[in] selector @UiSelector + * + * @return SaObject if found, else nulltpr + * + * @since_tizen 7.0 + */ + std::string GetFocusedAppId(); + + /** + * @brief Set server IP address. + * + * @param[in] ip std::string + * + * @since_tizen 7.0 + */ + void SetServerIp(std::string ip); + +private: + struct mosquitto *mMosq; + static std::string mPkgName; + static std::vector> mSaObjects; + static bool mLoadDone; + std::string mIp; +}; + +} + +#endif diff --git a/libaurum/inc/UiDevice.h b/libaurum/inc/UiDevice.h index ba21d4a..05f7c01 100644 --- a/libaurum/inc/UiDevice.h +++ b/libaurum/inc/UiDevice.h @@ -36,6 +36,11 @@ #include #include +#ifdef MQTT_ENABLED +#include "SaObject.h" +#include "ScreenAnalyzerWatcher.h" +#endif + namespace Aurum { /** @@ -367,6 +372,7 @@ public: * @brief Gets currently enabled applications root window. * * @return AccessibleNode ptr vector + * * @since_tizen 6.5 */ std::vector> getWindowRoot() const; @@ -375,10 +381,64 @@ public: * @brief Gets currently window information from window system. * * @return TizenWindow ptr vector + * * @since_tizen 7.0 */ std::vector> getTizenWindowInfo() const; +#ifdef MQTT_ENABLED + /** + * @brief Gets screen analyzer Object vector. + * + * @return SaObject ptr vector + * + * @since_tizen 7.0 + */ + std::vector> getSaObject(); + + /** + * @brief Gets ScreenAnalyzerWatcher instance. + * + * @retrun ScreenAnalyzerWatcher ptr + * + * @since_tizen 7.0 + */ + std::shared_ptr getSAWatcher(); +#endif + /** + * @brief Requests current screen analyze. + * + * @since_tizen 7.0 + */ + void RequestScreenAnalyze(); + + /** + * @brief Gets external application launched or not. + * + * @return true external application launched, otherwise false + * + * @since_tizen 7.0 + */ + bool getExternalAppLaunched(); + + /** + * @brief Sets whether to use screen analyzer for Tizen apps. + * + * @param[in] withScreenAnalyzer boolean value + * + * @since_tizen 7.0 + */ + void setWithScreenAnalyzer(bool withScreenAnalyzer); + + /** + * @brief Gets whether to use screen analyzer for Tizen apps. + * + * @return true screen analyzer use for Tizen apps, otherwise false + * + * @since_tizen 7.0 + */ + bool getWithScreenAnalyzer(); + private: /** * @brief Waits process idle. @@ -414,6 +474,10 @@ private: IDevice *mDeviceImpl; const Waiter *mWaiter; static std::vector> mTizenWindows; + bool mIsWithSA; +#ifdef MQTT_ENABLED + static std::shared_ptr mSAWatcher; +#endif }; } diff --git a/libaurum/inc/UiObject.h b/libaurum/inc/UiObject.h index 018d1ad..1863560 100644 --- a/libaurum/inc/UiObject.h +++ b/libaurum/inc/UiObject.h @@ -22,6 +22,7 @@ #include "Accessible.h" #include "ISearchable.h" +#include "IObject.h" #include "UiSelector.h" #include "Waiter.h" @@ -54,7 +55,7 @@ public: * such as object's properties, states, geometry information. * also user can send and receive event via this class. */ -class UiObject : public ISearchable , public std::enable_shared_from_this { +class UiObject : public ISearchable , public std::enable_shared_from_this, public IObject { public: /** * @brief UiObject constructor with device, selector, node pointer. @@ -157,6 +158,88 @@ public: public: /** + * @brief Gets object's Id. + * + * @return string + * + * @since_tizen 6.5 + */ + std::string getId() const override; + + /** + * @brief Gets object's type. + * + * @return string + * + * @since_tizen 7.0 + */ + std::string getType() const override; + + /** + * @brief Gets object's geometry of the screen. + * + * @return @Rect + * + * @since_tizen 6.5 + */ + const Rect getScreenBoundingBox() const override; + + /** + * @brief Gets object's ocr text. + * + * @return string + * + * @since_tizen 7.0 + */ + std::string getOcrText() const override; + + /** + * @brief Gets object's focusable property. + * + * @return true if focusable else false + * + * @since_tizen 6.5 + */ + bool isFocusable() const override; + + /** + * @brief Gets object's focused property. + * + * @return true if focused else false + * + * @since_tizen 6.5 + */ + bool isFocused() const override; + + /** + * @brief Gets object's clickable property. + * + * @return true if clickable else false + * + * @since_tizen 6.5 + */ + bool isClickable() const override; + + /** + * @brief Gets object's active property. + * + * @return true if active else false + * + * @since_tizen 6.5 + */ + bool isActive() const override; + + /** + * @brief Gets object's showing property. + * + * @return true if showing else false + * + * @since_tizen 6.5 + */ + bool isShowing() const override; + +public: + /** * @brief Gets object's parent. * * @return UiObject pointer @@ -211,15 +294,6 @@ public: std::string getApplicationPackage() const; /** - * @brief Gets object's Id. - * - * @return string - * - * @since_tizen 6.5 - */ - std::string getId() const; - - /** * @brief Gets object's automation Id. * * @return string @@ -229,15 +303,6 @@ public: std::string getAutomationId() const; /** - * @brief Gets object's type. - * - * @return string - * - * @since_tizen 6.5 - */ - std::string getElementType() const; - - /** * @brief Gets object's style. * * @return string @@ -265,6 +330,15 @@ public: std::string getRole() const; /** + * @brief Gets object's Toolkit name. + * + * @return string + * + * @since_tizen 7.0 + */ + std::string getToolkitName() const; + + /** * @brief Gets object's XPath. * * @return string @@ -285,13 +359,13 @@ public: bool setText(std::string text); /** - * @brief Gets object's geometry of the screen. + * @brief Sets object's ocr text. * - * @return @Rect + * @param[in] text string * - * @since_tizen 6.5 + * @since_tizen 7.0 */ - const Rect getScreenBoundingBox() const; + void setOcrText(std::string text); /** * @brief Gets object's geometry of the window. @@ -377,15 +451,6 @@ public: bool isChecked() const; /** - * @brief Gets object's clickable property. - * - * @return true if clickable else false - * - * @since_tizen 6.5 - */ - bool isClickable() const; - - /** * @brief Gets object's enabled property. (to get object enabled. disabled state) * * @return true if enabled else false @@ -395,24 +460,6 @@ public: bool isEnabled() const; /** - * @brief Gets object's focusable property. - * - * @return true if focusable else false - * - * @since_tizen 6.5 - */ - bool isFocusable() const; - - /** - * @brief Gets object's focused property. - * - * @return true if focused else false - * - * @since_tizen 6.5 - */ - bool isFocused() const; - - /** * @brief Gets object's longclickable property. * * @return true if longclickable else false @@ -458,24 +505,6 @@ public: bool isVisible() const; /** - * @brief Gets object's showing property. - * - * @return true if showing else false - * - * @since_tizen 6.5 - */ - bool isShowing() const; - - /** - * @brief Gets object's active property. - * - * @return true if active else false - * - * @since_tizen 6.5 - */ - bool isActive() const; - - /** * @brief Performs a click action on object. * * @since_tizen 6.5 @@ -567,6 +596,12 @@ public: void updatePid() const; /** + * @brief Updates object's toolkit name information from atspi server. + * + * @since_tizen 7.0 + */ + void updateToolkitName() const; + /** * @brief Sets focus to object. * * @since_tizen 7.0 diff --git a/libaurum/inc/UiSelector.h b/libaurum/inc/UiSelector.h index c8f510d..98f544d 100644 --- a/libaurum/inc/UiSelector.h +++ b/libaurum/inc/UiSelector.h @@ -93,6 +93,17 @@ public: UiSelector *text(std::string text); /** + * @brief Sets the search criteria to match the object's ocr text. + * + * @param[in] text object text + * + * @return UiSelector class instance + * + * @since_tizen 7.0 + */ + UiSelector *ocrText(std::string text); + + /** * @brief Sets the search criteria to match the object's text has given text. * * @param[in] text object text @@ -367,6 +378,7 @@ public: std::string mStyle; std::string mTextPartialMatch; std::string mXPath; + std::string mOcrText; bool mMatchId; bool mMatchAutomationId; @@ -377,6 +389,7 @@ public: bool mMatchStyle; bool mMatchTextPartialMatch; bool mMatchXPath; + bool mMatchOcrText; bool mMatchChecked; bool mMatchCheckable; diff --git a/libaurum/meson.build b/libaurum/meson.build index 66c87a0..7ecdcc5 100644 --- a/libaurum/meson.build +++ b/libaurum/meson.build @@ -33,6 +33,13 @@ libaurum_install_inc = [ './inc/Aurum.h', ] +if get_option('mqtt_enabled') == true + libaurum_install_inc += [ + './inc/SaObject.h', + './inc/ScreenAnalyzer/ScreenAnalyzerWatcher.h', + ] +endif + libaurum_inc = [ include_directories('./inc'), include_directories('./inc/Accessibility'), @@ -44,6 +51,12 @@ libaurum_inc = [ root_inc, ] +if get_option('mqtt_enabled') == true + libaurum_inc += [ + include_directories('./inc/ScreenAnalyzer'), + ] +endif + libaurum_src = [] subdir('src') @@ -64,6 +77,14 @@ if get_option('tizen') == true ] endif +if get_option('mqtt_enabled') == true + libaurum_dep += [ + dependency('libmosquitto'), + dependency('jsoncpp'), + dependency('capi-appfw-app-manager'), + ] +endif + libaurum_lib = library('aurum', libaurum_src, dependencies: libaurum_dep, include_directories: libaurum_inc, diff --git a/libaurum/src/Accessibility/AccessibleApplication.cc b/libaurum/src/Accessibility/AccessibleApplication.cc index 21c8fd0..e72c085 100644 --- a/libaurum/src/Accessibility/AccessibleApplication.cc +++ b/libaurum/src/Accessibility/AccessibleApplication.cc @@ -46,8 +46,8 @@ std::vector> AccessibleApplication::getActiveW auto children = getWindows(); children.erase(std::remove_if(children.begin(), children.end(), [](auto child){ - return !(child->isShowing()); // Active, Showing are same meaning currently + return !(child->isActive() && child->isShowing() && (child->getAccessibleNode()->getChildCount() > 0)); }), children.end()); return children; -} \ No newline at end of file +} diff --git a/libaurum/src/Accessibility/AccessibleNode.cc b/libaurum/src/Accessibility/AccessibleNode.cc index 19e81a4..e7bdbd1 100644 --- a/libaurum/src/Accessibility/AccessibleNode.cc +++ b/libaurum/src/Accessibility/AccessibleNode.cc @@ -31,7 +31,7 @@ AccessibleNode::~AccessibleNode() } AccessibleNode::AccessibleNode() -: mText{""}, mPkg{""}, mRole{""}, mId{""}, mAutomationId{""}, mType{""}, mStyle{""}, mXPath{""}, +: mText{""}, mOcrText{""}, mPkg{""}, mRole{""}, mId{""}, mAutomationId{""}, mType{""}, mStyle{""}, mXPath{""}, mToolkitName{""}, mScreenBoundingBox{0,0,0,0}, mWindowBoundingBox{0,0,0,0}, mSupportingIfaces(0), mFeatureProperty(0), mPid(0), mMinValue{0.0}, mMaxValue{0.0}, mValue{0.0}, mIncrement{0.0}, mValid{true}, mLock{} { } @@ -44,6 +44,7 @@ std::string AccessibleNode::description() { ss << "\"mAutomationId\":\"" << this->mAutomationId << "\", "; ss << "\"mRole\":\"" << this->mRole << "\", "; ss << "\"mText\":\"" << this->mText << "\", "; + ss << "\"mOcrText\":\"" << this->mOcrText << "\", "; ss << "\"mPkg\":\"" << this->mPkg << "\", "; ss << "\"mType\":\"" << this->mType << "\", "; ss << "\"mStyle\":\"" << this->mStyle << "\", "; @@ -120,6 +121,16 @@ std::string AccessibleNode::getText() const return mText; } +std::string AccessibleNode::getOcrText() const +{ + return mOcrText; +} + +void AccessibleNode::setOcrText(std::string text) +{ + mOcrText = text; +} + std::string AccessibleNode::getPkg() const { return mPkg; @@ -155,12 +166,17 @@ std::string AccessibleNode::getXPath() const return mXPath; } -Rect AccessibleNode::getScreenBoundingBox() const +std::string AccessibleNode::getToolkitName() const +{ + return mToolkitName; +} + +const Rect AccessibleNode::getScreenBoundingBox() const { return mScreenBoundingBox; } -Rect AccessibleNode::getWindowBoundingBox() const +const Rect AccessibleNode::getWindowBoundingBox() const { return mWindowBoundingBox; } diff --git a/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc b/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc index 8d04ab3..4130cd4 100644 --- a/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc +++ b/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc @@ -144,6 +144,17 @@ void AtspiAccessibleNode::updateName() } } +void AtspiAccessibleNode::updateToolkitName() +{ + AtspiWrapper::Atspi_accessible_clear_cache(mNode); + + gchar *toolkitName = AtspiWrapper::Atspi_accessible_get_toolkit_name(mNode, NULL); + if (toolkitName) { + mToolkitName = toolkitName; + g_free(toolkitName); + } +} + void AtspiAccessibleNode::updateApplication() { AtspiWrapper::Atspi_accessible_clear_cache(mNode); @@ -299,6 +310,12 @@ void AtspiAccessibleNode::refresh(bool updateAll) g_free(name); } + gchar *toolkitName = AtspiWrapper::Atspi_accessible_get_toolkit_name(mNode, NULL); + if (toolkitName) { + mToolkitName = toolkitName; + g_free(toolkitName); + } + AtspiAccessible *app = AtspiWrapper::Atspi_accessible_get_application(mNode, NULL); if (app) { gchar *pkg = AtspiWrapper::Atspi_accessible_get_name(app, NULL); diff --git a/libaurum/src/Impl/Accessibility/AtspiWrapper.cc b/libaurum/src/Impl/Accessibility/AtspiWrapper.cc index 93e1c37..ceef6cf 100644 --- a/libaurum/src/Impl/Accessibility/AtspiWrapper.cc +++ b/libaurum/src/Impl/Accessibility/AtspiWrapper.cc @@ -211,4 +211,10 @@ guint AtspiWrapper::Atspi_accessible_get_process_id(AtspiAccessible *node, GErro { std::unique_lock lock(mMutex); return atspi_accessible_get_process_id(node, error); -} \ No newline at end of file +} + +gchar *AtspiWrapper::Atspi_accessible_get_toolkit_name(AtspiAccessible *node, GError **error) +{ + std::unique_lock lock(mMutex); + return atspi_accessible_get_toolkit_name(node, error); +} diff --git a/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc b/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc index 8900dd8..1650e10 100644 --- a/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc +++ b/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc @@ -121,6 +121,10 @@ void MockAccessibleNode::updatePid() { } +void MockAccessibleNode::updateToolkitName() +{ +} + bool MockAccessibleNode::setFocus() { return false; diff --git a/libaurum/src/Impl/TizenDeviceImpl.cc b/libaurum/src/Impl/TizenDeviceImpl.cc index f8b7d42..ab87c4a 100644 --- a/libaurum/src/Impl/TizenDeviceImpl.cc +++ b/libaurum/src/Impl/TizenDeviceImpl.cc @@ -38,6 +38,9 @@ using namespace AurumInternal; #define NANO_SEC 1000000000.0 #define MICRO_SEC 1000000 + +std::mutex TizenDeviceImpl::CaptureMutex = std::mutex{}; + TizenDeviceImpl::TizenDeviceImpl() : mFakeTouchHandle{0}, mFakeKeyboardHandle{0}, mFakeWheelHandle{0}, tStart{}, isTimerStarted{false}, mTouchSeq{} { @@ -275,6 +278,7 @@ bool TizenDeviceImpl::takeScreenshot(std::string path, float scale, int quality) efl_util_screenshot_h screenshot = NULL; tbm_surface_h tbm_surface = NULL; + CaptureMutex.lock(); screenshot = efl_util_screenshot_initialize(mScreenSize.width, mScreenSize.height); if (screenshot) { @@ -285,13 +289,15 @@ bool TizenDeviceImpl::takeScreenshot(std::string path, float scale, int quality) tbm_surface_destroy(tbm_surface); } else { efl_util_screenshot_deinitialize(screenshot); + CaptureMutex.unlock(); return false; } efl_util_screenshot_deinitialize(screenshot); } else { + CaptureMutex.unlock(); return false; } - + CaptureMutex.unlock(); return true; } @@ -345,6 +351,7 @@ long long TizenDeviceImpl::getSystemTime(TimeRequestType type) const Size2D TizenDeviceImpl::getScreenSize() { TizenDeviceImpl *obj = static_cast(this); + LOGI("getScreenSize : %d %d", obj->mScreenSize.width , obj->mScreenSize.height); return obj->mScreenSize; } diff --git a/libaurum/src/PartialMatch.cc b/libaurum/src/PartialMatch.cc index 54332b4..ba030d1 100644 --- a/libaurum/src/PartialMatch.cc +++ b/libaurum/src/PartialMatch.cc @@ -52,8 +52,37 @@ std::string PartialMatch::debugPrint() } bool PartialMatch::checkCriteria(const std::shared_ptr selector, - const std::shared_ptr node) + const std::shared_ptr node, + std::shared_ptr device) { +#ifdef MQTT_ENABLED + if (selector->mMatchOcrText) + { + if (!device->getWithScreenAnalyzer() || !node->isShowing() || !(node->getText().size() == 0)) return false; + //FIXME: getSaObjectHasText(); + auto saObjs = device->getSAWatcher()->GetSaObjects(); + + node->updateExtents(); + auto objBoundingBox = node->getScreenBoundingBox(); + for (auto saObj : saObjs) { + if (saObj->getOcrText().size() != 0) { + if (objBoundingBox.isInRect(saObj->getScreenBoundingBox().midPoint()) && + ((objBoundingBox.width() * objBoundingBox.height() * 1.5) > (saObj->getScreenBoundingBox().width() * saObj->getScreenBoundingBox().height())) && + ((objBoundingBox.width() * objBoundingBox.height() < (saObj->getScreenBoundingBox().width() * saObj->getScreenBoundingBox().height() * 1.5)))) + { + LOGI("Text Set %s ", saObj->getOcrText().c_str()); + node->setOcrText(saObj->getOcrText()); + } + } + } + + if (node->getOcrText().size() == 0) return false; + if (checkCriteria(selector->mOcrText, node->getOcrText(), 1)) return false; + + LOGI("node ocr = %s, selector ocr = %s",node->getOcrText().c_str(), selector->mOcrText.c_str()); + } +#endif + if (selector->mMatchText || selector->mMatchTextPartialMatch) { node->updateName(); if (selector->mMatchText && checkCriteria(selector->mText, node->getText(), 0)) return false; @@ -115,12 +144,13 @@ std::shared_ptr PartialMatch::accept(const std::shared_ptr mDevice = UiDevice::getInstance(); if ((selector->mMinDepth && relativeDepth < selector->mMinDepth) || (selector->mMaxDepth && relativeDepth > selector->mMaxDepth)) { return std::shared_ptr(nullptr); } - if (PartialMatch::checkCriteria(selector, node)) + if (PartialMatch::checkCriteria(selector, node, mDevice)) match = new PartialMatch(selector, absoluteDepth); return std::shared_ptr(match); } diff --git a/libaurum/src/SaObject.cc b/libaurum/src/SaObject.cc new file mode 100644 index 0000000..ade4f32 --- /dev/null +++ b/libaurum/src/SaObject.cc @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "Aurum.h" + +#include +#include +#include +#include +#include +#include + +#include "ScreenAnalyzerWatcher.h" + +using namespace Aurum; + +SaObject::SaObject(std::string id, std::string type, Rect geometry, std::string ocrText, std::vector states) + : mId(id), mElementType(type), mScreenBoundingBox(geometry), mOcrText(ocrText) +{ + LOGI("SaObject Constructor"); + mIsFocusable = false; + mIsFocused = false; + mIsClickable = false; + mIsActive = false; + for (auto state : states) + { + if (state == "clickable") mIsClickable = true; + else if (state == "focused") mIsFocused = true; + else if (state == "focusable") mIsFocusable = true; + else if (state == "active") mIsActive = true; + } +} + +SaObject::~SaObject() +{ + LOGI("SaObject destructor"); +} + +std::string SaObject::getId() const +{ + return mId; +} + +std::string SaObject::getType() const +{ + return mElementType; +} + +const Rect SaObject::getScreenBoundingBox() const +{ + return mScreenBoundingBox; +} + +std::string SaObject::getOcrText() const +{ + return mOcrText; +} + +bool SaObject::isFocusable() const +{ + return mIsFocusable; +} + +bool SaObject::isFocused() const +{ + return mIsFocused; +} + +bool SaObject::isClickable() const +{ + return mIsClickable; +} + +bool SaObject::isActive() const +{ + return mIsActive; +} + +bool SaObject::isShowing() const +{ + return true; +} \ No newline at end of file diff --git a/libaurum/src/ScreenAnalyzer/ScreenAnalyzerWatcher.cc b/libaurum/src/ScreenAnalyzer/ScreenAnalyzerWatcher.cc new file mode 100644 index 0000000..be0a93c --- /dev/null +++ b/libaurum/src/ScreenAnalyzer/ScreenAnalyzerWatcher.cc @@ -0,0 +1,386 @@ +/* + * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "Aurum.h" + +#include "ScreenAnalyzerWatcher.h" +#include +#include +#include +#include "SaObject.h" +#include +#include +#include +#include +#include +#include +#include + +using namespace Aurum; + +//Initialize static member variables +std::vector> ScreenAnalyzerWatcher::mSaObjects; +bool ScreenAnalyzerWatcher::mLoadDone; +std::string ScreenAnalyzerWatcher::mPkgName; + +void on_connect(struct mosquitto *mosq, void *obj, int reason_code) +{ + LOGI("mosquitto on connect"); + + if(reason_code != 0){ + LOGE("mosquitto on connect error : %d", reason_code); + mosquitto_disconnect(mosq); + } + int rc; + rc = mosquitto_subscribe(mosq, NULL, "screen_analyzer/json_aurum", 1); + if(rc != MOSQ_ERR_SUCCESS){ + LOGE("server on connect subscribe fail"); + mosquitto_disconnect(mosq); + } + +} + +void on_publish(struct mosquitto *mosq, void *obj, int mid) +{ + LOGI("mosquitto on publish"); +} + +void on_subscribe(struct mosquitto *mosq, void *obj, int mid, int qos_count, const int *granted_qos) +{ + LOGI("server on subscribe qos : %d", qos_count); +} + +ScreenAnalyzerWatcher::ScreenAnalyzerWatcher() +{ + LOGI("Screen Analyzer constructor"); + + mosquitto_lib_init(); + LOGI("mosquitto lib init"); + mMosq = mosquitto_new(NULL, true, NULL); + if(mMosq == NULL){ + LOGE("mosquitto new fail"); + return; + } + + mosquitto_connect_callback_set(mMosq, on_connect); + mosquitto_publish_callback_set(mMosq, on_publish); + mosquitto_subscribe_callback_set(mMosq, on_subscribe); + mosquitto_message_callback_set(mMosq, ScreenAnalyzerWatcher::onMessage); + + //FIXME : When the class is created, there is an issue if We do not try to connect the server, + // so I put the AIM team IP to try to connect. + mIp = "10.113.16.21"; //This is AIM default server user can change this using SetServerIp() + + int rc; + rc = mosquitto_connect(mMosq, mIp.c_str(), 1883, 60); + if(rc != MOSQ_ERR_SUCCESS){ + mosquitto_destroy(mMosq); + LOGE("mosquitto connect fail"); + return; + } + rc = mosquitto_loop_start(mMosq); + if(rc != MOSQ_ERR_SUCCESS){ + mosquitto_destroy(mMosq); + LOGE("mosquitto loop fail"); + return; + } +} + +ScreenAnalyzerWatcher::~ScreenAnalyzerWatcher() +{ + LOGI("Screen Analyzer destructor"); + + if (mMosq) + { + mosquitto_disconnect(mMosq); + mosquitto_loop_stop(mMosq, false); + mosquitto_lib_cleanup(); + mMosq = nullptr; + } +} + +void ScreenAnalyzerWatcher::onMessage(struct mosquitto *mosq, void *obj, const struct mosquitto_message *msg) +{ + std::string txt((char *)msg->payload); + + mSaObjects.clear(); + + Json::Reader reader; + Json::Value root; + bool ret = reader.parse(txt, root); + if (!ret) LOGE("mosquitto parse fail"); + + const Json::Value objs = root["objects"]; + LOGI("objects count = %d", root["objectCount"].asInt()); + + std::string id; + std::string type; + Rect geometry; + std::string ocrText; + std::vector states{}; + + std::shared_ptr mDevice = UiDevice::getInstance(); + //Add dummy for window + id = "9999"; + type = "window"; + geometry = {0, 0, mDevice->getScreenSize().width, mDevice->getScreenSize().height}; + states.push_back("active"); + ocrText = mPkgName; + mSaObjects.push_back(std::make_shared(id, type, geometry, ocrText, states)); + + for (unsigned int idx = 0; idx < objs.size(); ++idx) + { + states.clear(); + id = objs[idx]["id"].asString(); + LOGI("objs[%d] ID : %s", idx,id.c_str()); + + type = objs[idx]["type"].asString(); + LOGI("objs[%d] TYPE : %s", idx, objs[idx]["type"].asString().c_str()); + + const Json::Value geo = objs[idx]["geometry"]; + geometry = { geo[0].asInt(), geo[1].asInt(), geo[2].asInt(), geo[3].asInt() }; + LOGI("objs[%d] GEOMETRY : %d %d %d %d", idx, geo[0].asInt(), geo[1].asInt(), geo[2].asInt(), geo[3].asInt()); + + ocrText = objs[idx]["ocrText"].asString(); + LOGI("objs[%d] OCRTEXT : %s", idx, objs[idx]["ocrText"].asString().c_str()); + + const Json::Value sta = objs[idx]["states"]; + for (unsigned int sidx = 0; sidx < sta.size(); ++sidx) { + LOGI("objs[%d] STATE: %s", idx, sta[sidx].asString().c_str()); + states.push_back(sta[sidx].asString()); + } + + mSaObjects.push_back(std::make_shared(id, type, geometry, ocrText, states)); + } + + mLoadDone = true; +} + +void ScreenAnalyzerWatcher::PublishData() +{ + LOGI("Prepare capture data for data publish"); + mLoadDone = false; + void *ptr = NULL; + unsigned char *src = NULL; + unsigned char *dst = NULL; + + efl_util_screenshot_h screenshot = NULL; + tbm_surface_h tbm_surface = NULL; + tbm_surface_info_s info; + std::shared_ptr mDevice = UiDevice::getInstance(); + const int WIDTH = mDevice->getScreenSize().width; + const int HEIGHT = mDevice->getScreenSize().height; + + screenshot = efl_util_screenshot_initialize(WIDTH, HEIGHT); + + if (screenshot) + { + tbm_surface = efl_util_screenshot_take_tbm_surface(screenshot); + if (tbm_surface) + { + tbm_surface_map(tbm_surface, TBM_SURF_OPTION_READ, &info); + + ptr = malloc( WIDTH * HEIGHT * 4 ); + src = (unsigned char *)info.planes[0].ptr; + dst = (unsigned char *)ptr; + + memcpy(dst, src, WIDTH * HEIGHT * 4); + } + else + { + efl_util_screenshot_deinitialize(screenshot); + return; + } + + int payloadlen = WIDTH * HEIGHT * 4; + + LOGI("mosquitto publish start"); + int rc = mosquitto_publish(mMosq, NULL, "screen_analyzer/image_aurum", payloadlen , ptr, 2, false); + if(rc != MOSQ_ERR_SUCCESS) { + LOGE("mosquitto publish fail"); + } + + tbm_surface_destroy(tbm_surface); + efl_util_screenshot_deinitialize(screenshot); + } + else + { + LOGE("Screen shot fail"); + return; + } + + while(!mLoadDone) { + std::this_thread::sleep_for(std::chrono::milliseconds{10}); + } + + if (tbm_surface) { + free(ptr); + tbm_surface_unmap(tbm_surface); + tbm_surface_destroy(tbm_surface); + } +} + +std::vector> ScreenAnalyzerWatcher::GetSaObjects() +{ + return mSaObjects; +} + +bool ScreenAnalyzerWatcher::checkCriteria(const std::string textA, const std::string textB, const bool textPartialMatch) +{ + if (textB.empty()) return true; + + bool rst; + if (textPartialMatch) { + if (textB.find(textA) != std::string::npos) rst = false; + else rst = true; + } + else { + if (!textA.compare(textB)) rst = false; + else rst = true; + } + + return rst; +} + +bool ScreenAnalyzerWatcher::checkCriteria(const bool boolA, const bool boolB) +{ + return boolA != boolB; +} + +bool ScreenAnalyzerWatcher::checkCriteria(const std::shared_ptr selector, + const std::shared_ptr node) +{ + if (selector->mMatchText || selector->mMatchTextPartialMatch) { + if (selector->mMatchText && checkCriteria(selector->mText, node->getOcrText(), 0)) return false; + if (selector->mMatchTextPartialMatch && checkCriteria(selector->mTextPartialMatch, node->getOcrText(), 1)) return false; + } + if (selector->mMatchId) { + if (checkCriteria(selector->mId, node->getId(), 0)) return false; + } + if (selector->mMatchType) { + if (selector->mMatchType && checkCriteria(selector->mType, node->getType(), 0)) return false; + } + if (selector->mMatchClickable && checkCriteria(selector->mIsclickable, node->isClickable())) return false; + if (selector->mMatchFocused && checkCriteria(selector->mIsfocused, node->isFocused())) return false; + if (selector->mMatchFocusable && checkCriteria(selector->mIsfocusable, node->isFocusable())) return false; + if (selector->mMatchActive && checkCriteria(selector->mIsactive, node->isActive())) return false; + if (selector->mMatchShowing && checkCriteria(selector->mIsshowing, node->isShowing())) return false; + + return true; +} + +std::vector> ScreenAnalyzerWatcher::findSaObjects(const std::shared_ptr selector) +{ + std::vector> rets{}; + + LOGI("FindSaObject start"); + for (auto saObj : mSaObjects) { + if (checkCriteria(selector, saObj)) { + LOGI("saObj(%s) pushed", saObj->getId().c_str()); + rets.push_back(saObj); + } + } + + return rets; +} + +std::shared_ptr ScreenAnalyzerWatcher::findSaObject(const std::shared_ptr selector) +{ + LOGI("FindSaObject start"); + for (auto saObj : mSaObjects) { + if (checkCriteria(selector, saObj)) { + LOGI("saObj(%s) pushed", saObj->getId().c_str()); + return saObj; + } + } + + return nullptr; +} + +std::string ScreenAnalyzerWatcher::GetFocusedAppId() +{ + int return_val = APP_MANAGER_ERROR_NONE; + + app_context_h app_context; + char *app_id = NULL; + + return_val = app_manager_get_focused_app_context(&app_context); + if (return_val != APP_MANAGER_ERROR_NONE) { + LOGE("failed to get app-context"); + return NULL; + } + + return_val = app_context_get_app_id(app_context, &app_id); + if (return_val != APP_MANAGER_ERROR_NONE) { + LOGE("failed to get app_id"); + app_context_destroy(app_context); + return NULL; + } + + LOGI("focused app_id = %s", app_id); + + return_val = app_context_destroy(app_context); + if (return_val != APP_MANAGER_ERROR_NONE) { + LOGE("failed to destroy app-context"); + return NULL; + } + + app_context = NULL; + + return app_id; +} + +void ScreenAnalyzerWatcher::SetServerIp(std::string ip) +{ + LOGI("SA Server Initialize"); + mIp = ip; + + if (mMosq) + { + mosquitto_disconnect(mMosq); + mosquitto_loop_stop(mMosq, false); + mosquitto_lib_cleanup(); + mMosq = nullptr; + } + + mosquitto_lib_init(); + LOGI("mosquitto lib init"); + mMosq = mosquitto_new(NULL, true, NULL); + if(mMosq == NULL){ + LOGE("mosquitto new fail"); + return; + } + + mosquitto_connect_callback_set(mMosq, on_connect); + mosquitto_publish_callback_set(mMosq, on_publish); + mosquitto_subscribe_callback_set(mMosq, on_subscribe); + mosquitto_message_callback_set(mMosq, ScreenAnalyzerWatcher::onMessage); + + int rc; + rc = mosquitto_connect(mMosq, mIp.c_str(), 1883, 60); + if (rc != MOSQ_ERR_SUCCESS){ + mosquitto_destroy(mMosq); + LOGE("mosquitto connect fail"); + return; + } + rc = mosquitto_loop_start(mMosq); + if (rc != MOSQ_ERR_SUCCESS){ + mosquitto_destroy(mMosq); + LOGE("mosquitto loop fail"); + return; + } +} diff --git a/libaurum/src/ScreenAnalyzer/meson.build b/libaurum/src/ScreenAnalyzer/meson.build new file mode 100644 index 0000000..5047b43 --- /dev/null +++ b/libaurum/src/ScreenAnalyzer/meson.build @@ -0,0 +1,3 @@ +libaurum_src += [ + files('ScreenAnalyzerWatcher.cc') +] diff --git a/libaurum/src/UiDevice.cc b/libaurum/src/UiDevice.cc index ea76073..6b66377 100644 --- a/libaurum/src/UiDevice.cc +++ b/libaurum/src/UiDevice.cc @@ -42,11 +42,21 @@ using namespace AurumInternal; std::vector> UiDevice::mTizenWindows; static GDBusConnection *system_conn; +#ifdef MQTT_ENABLED +std::shared_ptr UiDevice::mSAWatcher; +#endif + UiDevice::UiDevice() : UiDevice(nullptr) {} UiDevice::UiDevice(IDevice *impl) : mDeviceImpl(impl), mWaiter(new Waiter{this}) { + LOGI("UiDevice constructor"); +#ifdef MQTT_ENABLED + mSAWatcher = std::make_shared(); + mIsWithSA = false; +#endif + LOGI("UiDevice constructor finish"); } UiDevice::~UiDevice() @@ -420,3 +430,38 @@ const Size2D UiDevice::getScreenSize() { return mDeviceImpl->getScreenSize(); } + +#ifdef MQTT_ENABLED +std::vector> UiDevice::getSaObject() +{ + return mSAWatcher->GetSaObjects(); +} + +std::shared_ptr UiDevice::getSAWatcher() +{ + return mSAWatcher; +} +#endif + +void UiDevice::RequestScreenAnalyze() +{ +#ifdef MQTT_ENABLED + mSAWatcher->PublishData(); +#endif +} + +bool UiDevice::getExternalAppLaunched() +{ + auto ret = this->getWindowRoot(); + return (ret.size() > 0) ? false : true; +} + +void UiDevice::setWithScreenAnalyzer(bool withScreenAnalyzer) +{ + mIsWithSA = withScreenAnalyzer; +} + +bool UiDevice::getWithScreenAnalyzer() +{ + return mIsWithSA; +} diff --git a/libaurum/src/UiObject.cc b/libaurum/src/UiObject.cc index fe54f4f..c975859 100644 --- a/libaurum/src/UiObject.cc +++ b/libaurum/src/UiObject.cc @@ -174,7 +174,7 @@ std::string UiObject::getAutomationId() const return getAccessibleNode()->getAutomationId(); } -std::string UiObject::getElementType() const +std::string UiObject::getType() const { return getAccessibleNode()->getType(); } @@ -229,6 +229,22 @@ bool UiObject::setText(std::string text) return getAccessibleNode()->setValue(text); } +std::string UiObject::getOcrText() const +{ + return getAccessibleNode()->getOcrText(); +} + +std::string UiObject::getToolkitName() const +{ + getAccessibleNode()->updateToolkitName(); + return getAccessibleNode()->getToolkitName(); +} + +void UiObject::setOcrText(std::string text) +{ + getAccessibleNode()->setOcrText(text); +} + bool UiObject::isCheckable() const { return getAccessibleNode()->isCheckable(); @@ -349,6 +365,11 @@ void UiObject::updatePid() const mNode->updatePid(); } +void UiObject::updateToolkitName() const +{ + mNode->updateToolkitName(); +} + bool UiObject::setFocus() const { return mNode->setFocus(); diff --git a/libaurum/src/UiSelector.cc b/libaurum/src/UiSelector.cc index 7d703dc..8157b18 100644 --- a/libaurum/src/UiSelector.cc +++ b/libaurum/src/UiSelector.cc @@ -22,10 +22,10 @@ using namespace Aurum; UiSelector::UiSelector() -: mId{}, mAutomationId{}, mRole{}, mText{}, mPkg{}, mType{}, mStyle{}, mTextPartialMatch{}, mXPath{}, +: mId{}, mAutomationId{}, mRole{}, mText{}, mPkg{}, mType{}, mStyle{}, mTextPartialMatch{}, mXPath{}, mOcrText{}, mMatchId{}, mMatchAutomationId{}, mMatchRole{}, mMatchText{}, mMatchPkg{}, mMatchType{}, mMatchStyle{}, - mMatchTextPartialMatch{}, mMatchXPath{}, mMatchChecked{}, mMatchCheckable{}, mMatchClickable{}, mMatchEnabled{}, mMatchFocused{}, - mMatchFocusable{}, mMatchScrollable{}, mMatchSelected{}, mMatchShowing{}, mMatchActive{}, mMatchVisible{}, + mMatchTextPartialMatch{}, mMatchXPath{}, mMatchOcrText{}, mMatchChecked{}, mMatchCheckable{}, mMatchClickable{}, mMatchEnabled{}, + mMatchFocused{}, mMatchFocusable{}, mMatchScrollable{}, mMatchSelected{}, mMatchShowing{}, mMatchActive{}, mMatchVisible{}, mMatchSelectable{}, mMinDepth{}, mMaxDepth{}, mIschecked{}, mIscheckable{}, mIsclickable{}, mIsenabled{}, mIsfocused{}, mIsfocusable{}, mIsscrollable{}, mIsselected{}, mIsshowing{}, mIsactive{}, mIsvisible{}, mIsselectable{}, mChild{}, mParent{} @@ -40,6 +40,7 @@ std::string UiSelector::description() if(!this->mAutomationId.empty()) ss << "\"mAutomationId\":\"" << this->mAutomationId << "\", "; if(!this->mRole.empty()) ss << "\"mRole\":\"" << this->mRole << "\", "; if(!this->mText.empty()) ss << "\"mText\":\"" << this->mText << "\", "; + if(!this->mOcrText.empty()) ss << "\"mOcrText\":\"" << this->mOcrText << "\", "; if(!this->mTextPartialMatch.empty()) ss << "\"mTextPartialMatch\":\"" << this->mTextPartialMatch << "\", "; if(!this->mXPath.empty()) ss << "\"mXPath\":\"" << this->mXPath << "\", "; if(!this->mPkg.empty()) ss << "\"mPkg\":\"" << this->mPkg << "\", "; @@ -49,6 +50,7 @@ std::string UiSelector::description() if(this->mMatchAutomationId) ss << "\"mMatchAutomationId\":\"" << ((this->mMatchAutomationId)?"true":"false") << "\", "; if(this->mMatchRole) ss << "\"mMatchRole\":\"" << ((this->mMatchRole)?"true":"false") << "\", "; if(this->mMatchText) ss << "\"mMatchText\":\"" << ((this->mMatchText)?"true":"false") << "\", "; + if(this->mMatchOcrText) ss << "\"mMatchOcrText\":\"" << ((this->mMatchOcrText)?"true":"false") << "\", "; if(this->mMatchTextPartialMatch) ss << "\"mMatchTextPartialMatch\":\"" << ((this->mMatchTextPartialMatch)?"true":"false") << "\", "; if(this->mMatchXPath) ss << "\"mMatchXPath\":\"" << ((this->mMatchXPath)?"true":"false") << "\", "; if(this->mMatchPkg) ss << "\"mMatchPkg\":\"" << ((this->mMatchPkg)?"true":"false") << "\", "; @@ -89,6 +91,13 @@ UiSelector *UiSelector::text(std::string text) return this; } +UiSelector *UiSelector::ocrText(std::string text) +{ + this->mOcrText = text; + this->mMatchOcrText = true; + return this; +} + UiSelector *UiSelector::textPartialMatch(std::string text) { this->mTextPartialMatch = text; diff --git a/libaurum/src/meson.build b/libaurum/src/meson.build index 390e805..778c7b0 100644 --- a/libaurum/src/meson.build +++ b/libaurum/src/meson.build @@ -13,6 +13,15 @@ libaurum_src += [ files('TizenWindow.cc'), ] +if get_option('mqtt_enabled') == true + libaurum_src += [ + files('SaObject.cc'), + ] +endif + subdir('Accessibility') +if get_option('mqtt_enabled') == true + subdir('ScreenAnalyzer') +endif subdir('Impl') subdir('Runnable') diff --git a/meson.build b/meson.build index 1022f82..e6af72e 100644 --- a/meson.build +++ b/meson.build @@ -17,6 +17,10 @@ if get_option('tizen_gcov') == true config_h.set10('TIZEN_GCOV', true) endif +if get_option('mqtt_enabled') == true + config_h.set10('MQTT_ENABLED', true) +endif + root_inc = include_directories('./') subdir('pugixml') diff --git a/meson_options.txt b/meson_options.txt index 726ed1d..ec44da5 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -26,3 +26,9 @@ option('enable_documentation', value: false, description: 'enable generating documentation by using doxygen' ) + +option('mqtt_enabled', + type: 'boolean', + value: false, + description: 'enable mqtt protocol(mosquitto) for Screen Analyzer' +) diff --git a/org.tizen.aurum-bootstrap/inc/AurumServiceImpl.h b/org.tizen.aurum-bootstrap/inc/AurumServiceImpl.h index 119e36a..deb020b 100644 --- a/org.tizen.aurum-bootstrap/inc/AurumServiceImpl.h +++ b/org.tizen.aurum-bootstrap/inc/AurumServiceImpl.h @@ -120,6 +120,12 @@ public: ::grpc::Status setFocus(::grpc::ServerContext *context, const ::aurum::ReqSetFocus *request, ::aurum::RspSetFocus *response) override; + ::grpc::Status getActiveAppToolkitName(::grpc::ServerContext *context, + const ::aurum::ReqGetActiveAppToolkitName *request, + ::aurum::RspGetActiveAppToolkitName *response) override; + ::grpc::Status enableScreenAnalyzer(::grpc::ServerContext *context, + const ::aurum::ReqEnableScreenAnalyzer *request, + ::aurum::RspEnableScreenAnalyzer *response) override; }; #endif diff --git a/org.tizen.aurum-bootstrap/inc/Commands/Commands.h b/org.tizen.aurum-bootstrap/inc/Commands/Commands.h index e0badc1..6445640 100644 --- a/org.tizen.aurum-bootstrap/inc/Commands/Commands.h +++ b/org.tizen.aurum-bootstrap/inc/Commands/Commands.h @@ -49,3 +49,5 @@ #include "Commands/GetScreenSizeCommand.h" #include "Commands/ActionAndWaitEventCommand.h" #include "Commands/SetFocusCommand.h" +#include "Commands/GetActiveAppToolkitNameCommand.h" +#include "Commands/EnableScreenAnalyzerCommand.h" diff --git a/org.tizen.aurum-bootstrap/inc/Commands/EnableScreenAnalyzerCommand.h b/org.tizen.aurum-bootstrap/inc/Commands/EnableScreenAnalyzerCommand.h new file mode 100644 index 0000000..700611e --- /dev/null +++ b/org.tizen.aurum-bootstrap/inc/Commands/EnableScreenAnalyzerCommand.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include +#include "Commands/Command.h" +#include "ObjectMapper.h" +#include +#include "config.h" + +class EnableScreenAnalyzerCommand : public Command { +private: + const ::aurum::ReqEnableScreenAnalyzer *mRequest; + ::aurum::RspEnableScreenAnalyzer *mResponse; + +public: + EnableScreenAnalyzerCommand(const ::aurum::ReqEnableScreenAnalyzer *request, + ::aurum::RspEnableScreenAnalyzer*response); + ::grpc::Status execute() override; +}; diff --git a/org.tizen.aurum-bootstrap/inc/Commands/GetActiveAppToolkitNameCommand.h b/org.tizen.aurum-bootstrap/inc/Commands/GetActiveAppToolkitNameCommand.h new file mode 100644 index 0000000..e85aba3 --- /dev/null +++ b/org.tizen.aurum-bootstrap/inc/Commands/GetActiveAppToolkitNameCommand.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include +#include "Commands/Command.h" +#include "ObjectMapper.h" +#include +#include "config.h" + +class GetActiveAppToolkitNameCommand : public Command { +private: + const ::aurum::ReqGetActiveAppToolkitName *mRequest; + ::aurum::RspGetActiveAppToolkitName *mResponse; + +public: + GetActiveAppToolkitNameCommand(const ::aurum::ReqGetActiveAppToolkitName *request, + ::aurum::RspGetActiveAppToolkitName *response); + ::grpc::Status execute() override; +}; diff --git a/org.tizen.aurum-bootstrap/meson.build b/org.tizen.aurum-bootstrap/meson.build index 0e90268..fb99f18 100644 --- a/org.tizen.aurum-bootstrap/meson.build +++ b/org.tizen.aurum-bootstrap/meson.build @@ -45,6 +45,8 @@ bootstrap_svr_src += [ files('src/Commands/GetScreenSizeCommand.cc'), files('src/Commands/ActionAndWaitEventCommand.cc'), files('src/Commands/SetFocusCommand.cc'), + files('src/Commands/GetActiveAppToolkitNameCommand.cc'), + files('src/Commands/EnableScreenAnalyzerCommand.cc'), ] bootstrap_svr_dep = [ diff --git a/org.tizen.aurum-bootstrap/src/AurumServiceImpl.cc b/org.tizen.aurum-bootstrap/src/AurumServiceImpl.cc index fe62067..00abf4e 100644 --- a/org.tizen.aurum-bootstrap/src/AurumServiceImpl.cc +++ b/org.tizen.aurum-bootstrap/src/AurumServiceImpl.cc @@ -264,3 +264,20 @@ aurumServiceImpl::~aurumServiceImpl() std::unique_ptr cmd = std::make_unique(request, response); return execute(cmd.get(), true); } + +::grpc::Status aurumServiceImpl::getActiveAppToolkitName(::grpc::ServerContext *context, + const ::aurum::ReqGetActiveAppToolkitName *request, + ::aurum::RspGetActiveAppToolkitName *response) +{ + std::unique_ptr cmd = std::make_unique(request, response); + return execute(cmd.get(), true); +} + +::grpc::Status aurumServiceImpl::enableScreenAnalyzer(::grpc::ServerContext *context, + const ::aurum::ReqEnableScreenAnalyzer *request, + ::aurum::RspEnableScreenAnalyzer *response) +{ + std::unique_ptr cmd = std::make_unique(request, response); + return execute(cmd.get(), true); +} + diff --git a/org.tizen.aurum-bootstrap/src/Commands/DumpObjectTreeCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/DumpObjectTreeCommand.cc index dfcda80..4865ada 100644 --- a/org.tizen.aurum-bootstrap/src/Commands/DumpObjectTreeCommand.cc +++ b/org.tizen.aurum-bootstrap/src/Commands/DumpObjectTreeCommand.cc @@ -55,7 +55,7 @@ void DumpObjectTreeCommand::traverse(::aurum::Element *root, std::shared_ptrset_width(windowSize.width()); windowRect->set_height(windowSize.height()); - root->set_widget_type(obj->getElementType()); + root->set_widget_type(obj->getType()); root->set_widget_style(obj->getElementStyle()); root->set_text(obj->getText()); @@ -91,14 +91,70 @@ void DumpObjectTreeCommand::traverse(::aurum::Element *root, std::shared_ptrelementid().c_str()); - if (mRequest->elementid().length()) { - auto obj = mObjMap->getElement(mRequest->elementid()); - if (!obj) return grpc::Status::OK;; - - auto node = obj->getDescendant(); - ::aurum::Element *root = mResponse->add_roots(); - traverse(root, node, 0); + + std::shared_ptr mDevice = UiDevice::getInstance(); +#ifdef MQTT_ENABLED + if (mDevice->getExternalAppLaunched()) + { + mDevice->RequestScreenAnalyze(); + + auto objs = mDevice->getSAWatcher()->GetSaObjects(); + ::aurum::Element *root; + int idx = 0; + for (auto obj : objs) { + if (!idx) { + root = mResponse->add_roots(); + root->set_elementid(obj->getId()); + root->set_widget_type(obj->getType()); + root->set_text(obj->getOcrText()); + root->set_isclickable(obj->isClickable()); + root->set_isfocused(obj->isFocused()); + root->set_isfocusable(obj->isFocusable()); + root->set_isactive(obj->isActive()); + root->set_isshowing(true); + root->set_isvisible(true); + ::aurum::Rect *rect = root->mutable_geometry(); + const Rect &size = obj->getScreenBoundingBox(); + rect->set_x(size.mTopLeft.x); + rect->set_y(size.mTopLeft.y); + rect->set_width(size.width()); + rect->set_height(size.height()); + } + else { + ::aurum::Element *elm = root->add_child(); + elm->set_elementid(obj->getId()); + elm->set_widget_type(obj->getType()); + elm->set_text(obj->getOcrText()); + elm->set_isclickable(obj->isClickable()); + elm->set_isfocused(obj->isFocused()); + elm->set_isfocusable(obj->isFocusable()); + elm->set_isactive(obj->isActive()); + elm->set_isshowing(true); + elm->set_isvisible(true); + + ::aurum::Rect *rect = elm->mutable_geometry(); + const Rect &size = obj->getScreenBoundingBox(); + rect->set_x(size.mTopLeft.x); + rect->set_y(size.mTopLeft.y); + rect->set_width(size.width()); + rect->set_height(size.height()); + } + + idx++; + } + } + else +#endif + { + LOGI("elementid : %s", mRequest->elementid().c_str()); + if (mRequest->elementid().length()) { + auto obj = mObjMap->getElement(mRequest->elementid()); + if (!obj) return grpc::Status::OK;; + + auto node = obj->getDescendant(); + ::aurum::Element *root = mResponse->add_roots(); + traverse(root, node, 0); + } } return grpc::Status::OK; } diff --git a/org.tizen.aurum-bootstrap/src/Commands/EnableScreenAnalyzerCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/EnableScreenAnalyzerCommand.cc new file mode 100644 index 0000000..5caba46 --- /dev/null +++ b/org.tizen.aurum-bootstrap/src/Commands/EnableScreenAnalyzerCommand.cc @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "bootstrap.h" +#include "EnableScreenAnalyzerCommand.h" +#include "UiDevice.h" + +EnableScreenAnalyzerCommand::EnableScreenAnalyzerCommand(const ::aurum::ReqEnableScreenAnalyzer *request, + ::aurum::RspEnableScreenAnalyzer *response) + : mRequest{request}, mResponse{response} +{ +} + +::grpc::Status EnableScreenAnalyzerCommand::execute() +{ + LOGI("EnableScreenAnalyzer test --------------- "); + + LOGI("Screen Analzyer enabled (%d) Servier IP : (%s)", mRequest->enable(), mRequest->serverip().c_str()); + std::shared_ptr mDevice = UiDevice::getInstance(); + mDevice->setWithScreenAnalyzer(mRequest->enable()); +#ifdef MQTT_ENABLED + if (mRequest->serverip().size() > 0) + mDevice->getSAWatcher()->SetServerIp(mRequest->serverip()); +#endif + + return grpc::Status::OK; +} diff --git a/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc index 929119d..ed92078 100644 --- a/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc +++ b/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc @@ -22,6 +22,9 @@ #include "UiSelector.h" #include "Sel.h" #include "ISearchable.h" +#ifdef MQTT_ENABLED +#include "SaObject.h" +#endif FindElementCommand::FindElementCommand(const ::aurum::ReqFindElement *request, ::aurum::RspFindElement *response) @@ -65,6 +68,7 @@ std::shared_ptr FindElementCommand::getSelector(void) if(mRequest->_packagename_case()) sel->pkg(mRequest->packagename()); if(mRequest->_textpartialmatch_case()) sel->textPartialMatch(mRequest->textpartialmatch()); if(mRequest->_xpath_case()) sel->xpath(mRequest->xpath()); + if(mRequest->_ocrtext_case()) sel->ocrText(mRequest->ocrtext()); return sel; } @@ -74,65 +78,113 @@ std::shared_ptr FindElementCommand::getSelector(void) LOGI("findElement --------------- "); auto searchableObj = getSearchableTop(); auto selector = getSelector(); - - auto found = searchableObj->findObject(selector); - - if (found != nullptr) { - UiObject *obj = found.get(); - obj->refresh(); - if (mObjMap->getElement(obj->getId()) == nullptr) - mObjMap->addElement(std::move(found)); - - LOGI("found object : %p elementId:%s", obj, obj->getId().c_str()); - - ::aurum::Element *elm = mResponse->mutable_element(); - elm->set_elementid(obj->getId()); - elm->set_package(obj->getApplicationPackage()); - - ::aurum::Rect *rect = elm->mutable_geometry(); - const Rect &size = obj->getScreenBoundingBox(); - rect->set_x(size.mTopLeft.x); - rect->set_y(size.mTopLeft.y); - rect->set_width(size.width()); - rect->set_height(size.height()); - - ::aurum::Rect *windowRect = elm->mutable_window_relative_geometry(); - const Rect &windowRelativeSize = obj->getWindowBoundingBox(); - windowRect->set_x(windowRelativeSize.mTopLeft.x); - windowRect->set_y(windowRelativeSize.mTopLeft.y); - windowRect->set_width(windowRelativeSize.width()); - windowRect->set_height(windowRelativeSize.height()); - - elm->set_widget_type(obj->getElementType()); - elm->set_widget_style(obj->getElementStyle()); - - elm->set_text(obj->getText()); - elm->set_xpath(obj->getXPath()); - elm->set_automationid(obj->getAutomationId()); - elm->set_package(obj->getApplicationPackage()); - elm->set_role(obj->getRole()); - - elm->set_ischecked(obj->isChecked()); - elm->set_ischeckable(obj->isCheckable()); - elm->set_isclickable(obj->isClickable()); - elm->set_isenabled(obj->isEnabled()); - elm->set_isfocused(obj->isFocused()); - elm->set_isfocusable(obj->isFocusable()); - elm->set_isscrollable(obj->isScrollable()); - elm->set_isselected(obj->isSelected()); - elm->set_isshowing(obj->isShowing()); - elm->set_isactive(obj->isActive()); - elm->set_isvisible(obj->isVisible()); - elm->set_isselectable(obj->isSelectable()); - - elm->set_minvalue(obj->getMinValue()); - elm->set_maxvalue(obj->getMaxValue()); - elm->set_value(obj->getValue()); - elm->set_increment(obj->getIncrement()); - - mResponse->set_status(::aurum::RspStatus::OK); - } else { - mResponse->set_status(::aurum::RspStatus::ERROR); + std::shared_ptr mDevice = UiDevice::getInstance(); + +#ifdef MQTT_ENABLED + if (mDevice->getExternalAppLaunched()) + { + mDevice->RequestScreenAnalyze(); + + LOGI("Search Object start"); + + auto found = mDevice->getSAWatcher()->findSaObject(selector); + if (found != nullptr) { + SaObject *obj = found.get(); + ::aurum::Element *elm = mResponse->mutable_element(); + elm->set_elementid(obj->getId()); + + ::aurum::Rect *rect = elm->mutable_geometry(); + const Rect &size = obj->getScreenBoundingBox(); + rect->set_x(size.mTopLeft.x); + rect->set_y(size.mTopLeft.y); + rect->set_width(size.width()); + rect->set_height(size.height()); + + elm->set_widget_type(obj->getType()); + + elm->set_text(obj->getOcrText()); + + elm->set_isclickable(obj->isClickable()); + elm->set_isfocused(obj->isFocused()); + elm->set_isfocusable(obj->isFocusable()); + elm->set_isactive(obj->isActive()); + elm->set_isshowing(true); + elm->set_isvisible(true); + + mResponse->set_status(::aurum::RspStatus::OK); + } + else + mResponse->set_status(::aurum::RspStatus::ERROR); + } + else +#endif + { + if (mDevice->getWithScreenAnalyzer()) + { + LOGI("Find object with SA information"); + mDevice->RequestScreenAnalyze(); + } + + auto found = searchableObj->findObject(selector); + + if (found != nullptr) { + UiObject *obj = found.get(); + obj->refresh(); + if (mObjMap->getElement(obj->getId()) == nullptr) + mObjMap->addElement(std::move(found)); + + LOGI("found object : %p elementId:%s", obj, obj->getId().c_str()); + + ::aurum::Element *elm = mResponse->mutable_element(); + elm->set_elementid(obj->getId()); + elm->set_package(obj->getApplicationPackage()); + + ::aurum::Rect *rect = elm->mutable_geometry(); + const Rect &size = obj->getScreenBoundingBox(); + rect->set_x(size.mTopLeft.x); + rect->set_y(size.mTopLeft.y); + rect->set_width(size.width()); + rect->set_height(size.height()); + + ::aurum::Rect *windowRect = elm->mutable_window_relative_geometry(); + const Rect &windowRelativeSize = obj->getWindowBoundingBox(); + windowRect->set_x(windowRelativeSize.mTopLeft.x); + windowRect->set_y(windowRelativeSize.mTopLeft.y); + windowRect->set_width(windowRelativeSize.width()); + windowRect->set_height(windowRelativeSize.height()); + + elm->set_widget_type(obj->getType()); + elm->set_widget_style(obj->getElementStyle()); + + elm->set_text(obj->getText()); + elm->set_xpath(obj->getXPath()); + elm->set_ocrtext(obj->getOcrText()); + elm->set_automationid(obj->getAutomationId()); + elm->set_package(obj->getApplicationPackage()); + elm->set_role(obj->getRole()); + + elm->set_ischecked(obj->isChecked()); + elm->set_ischeckable(obj->isCheckable()); + elm->set_isclickable(obj->isClickable()); + elm->set_isenabled(obj->isEnabled()); + elm->set_isfocused(obj->isFocused()); + elm->set_isfocusable(obj->isFocusable()); + elm->set_isscrollable(obj->isScrollable()); + elm->set_isselected(obj->isSelected()); + elm->set_isshowing(obj->isShowing()); + elm->set_isactive(obj->isActive()); + elm->set_isvisible(obj->isVisible()); + elm->set_isselectable(obj->isSelectable()); + + elm->set_minvalue(obj->getMinValue()); + elm->set_maxvalue(obj->getMaxValue()); + elm->set_value(obj->getValue()); + elm->set_increment(obj->getIncrement()); + + mResponse->set_status(::aurum::RspStatus::OK); + } else { + mResponse->set_status(::aurum::RspStatus::ERROR); + } } return grpc::Status::OK; diff --git a/org.tizen.aurum-bootstrap/src/Commands/FindElementsCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/FindElementsCommand.cc index 2efc5c7..bc2119c 100644 --- a/org.tizen.aurum-bootstrap/src/Commands/FindElementsCommand.cc +++ b/org.tizen.aurum-bootstrap/src/Commands/FindElementsCommand.cc @@ -22,6 +22,9 @@ #include "UiSelector.h" #include "Sel.h" #include "ISearchable.h" +#ifdef MQTT_ENABLED +#include "SaObject.h" +#endif FindElementsCommand::FindElementsCommand(const ::aurum::ReqFindElements *request, ::aurum::RspFindElements *response) @@ -65,6 +68,7 @@ std::vector> FindElementsCommand::getSelectors(void) if(mRequest->_packagename_case()) sel->pkg(mRequest->packagename()); if(mRequest->_textpartialmatch_case()) sel->textPartialMatch(mRequest->textpartialmatch()); if(mRequest->_xpath_case()) sel->xpath(mRequest->xpath()); + if(mRequest->_ocrtext_case()) sel->ocrText(mRequest->ocrtext()); return std::vector>{sel}; } @@ -74,71 +78,125 @@ std::vector> FindElementsCommand::getSelectors(void) LOGI("findElements --------------- "); auto searchableObj = getSearchableTop(); auto selectors = getSelectors(); + std::shared_ptr mDevice = UiDevice::getInstance(); - std::vector> founds = {}; +#ifdef MQTT_ENABLED + if (mDevice->getExternalAppLaunched()) + { + mDevice->RequestScreenAnalyze(); - for ( auto &sel : selectors ) { - auto ret = searchableObj->findObjects(sel); - std::move(std::begin(ret), std::end(ret), std::back_inserter(founds)); + std::vector> founds = {}; + + for ( auto &sel : selectors ) { + auto ret = mDevice->getSAWatcher()->findSaObjects(sel); + std::move(std::begin(ret), std::end(ret), std::back_inserter(founds)); + } + if (founds.size() > 0) { + for (auto& found : founds) { + SaObject *obj = found.get(); + ::aurum::Element *elm = mResponse->add_elements(); + elm->set_elementid(obj->getId()); + + ::aurum::Rect *rect = elm->mutable_geometry(); + const Rect &size = obj->getScreenBoundingBox(); + rect->set_x(size.mTopLeft.x); + rect->set_y(size.mTopLeft.y); + rect->set_width(size.width()); + rect->set_height(size.height()); + + elm->set_widget_type(obj->getType()); + + elm->set_text(obj->getOcrText()); + elm->set_toolkit("External"); + + elm->set_isclickable(obj->isClickable()); + elm->set_isfocused(obj->isFocused()); + elm->set_isfocusable(obj->isFocusable()); + elm->set_isactive(obj->isActive()); + elm->set_isshowing(true); + elm->set_isvisible(true); + } + mResponse->set_status(::aurum::RspStatus::OK); + } + else { + mResponse->set_status(::aurum::RspStatus::ERROR); + } } + else +#endif + { + if (mDevice->getWithScreenAnalyzer()) + { + LOGI("Find object with SA information"); + mDevice->RequestScreenAnalyze(); + } + + std::vector> founds = {}; + + for ( auto &sel : selectors ) { + auto ret = searchableObj->findObjects(sel); + std::move(std::begin(ret), std::end(ret), std::back_inserter(founds)); + } - if (founds.size() > 0) { - for (auto& found : founds) { - UiObject *obj = found.get(); - obj->refresh(); - if (mObjMap->getElement(obj->getId()) == nullptr) - mObjMap->addElement(std::move(found)); - - LOGI("found object : %p elementId:%s", obj, obj->getId().c_str()); - - ::aurum::Element *elm = mResponse->add_elements(); - elm->set_elementid(obj->getId()); - elm->set_package(obj->getApplicationPackage()); - - ::aurum::Rect *rect = elm->mutable_geometry(); - const Rect &size = obj->getScreenBoundingBox(); - rect->set_x(size.mTopLeft.x); - rect->set_y(size.mTopLeft.y); - rect->set_width(size.width()); - rect->set_height(size.height()); - - ::aurum::Rect *windowRect = elm->mutable_window_relative_geometry(); - const Rect &windowRelativeSize = obj->getWindowBoundingBox(); - windowRect->set_x(windowRelativeSize.mTopLeft.x); - windowRect->set_y(windowRelativeSize.mTopLeft.y); - windowRect->set_width(windowRelativeSize.width()); - windowRect->set_height(windowRelativeSize.height()); - - elm->set_widget_type(obj->getElementType()); - elm->set_widget_style(obj->getElementStyle()); - - elm->set_text(obj->getText()); - elm->set_xpath(obj->getXPath()); - elm->set_automationid(obj->getAutomationId()); - elm->set_package(obj->getApplicationPackage()); - elm->set_role(obj->getRole()); - - elm->set_ischecked(obj->isChecked()); - elm->set_ischeckable(obj->isCheckable()); - elm->set_isclickable(obj->isClickable()); - elm->set_isenabled(obj->isEnabled()); - elm->set_isfocused(obj->isFocused()); - elm->set_isfocusable(obj->isFocusable()); - elm->set_isscrollable(obj->isScrollable()); - elm->set_isselected(obj->isSelected()); - elm->set_isshowing(obj->isShowing()); - elm->set_isactive(obj->isActive()); - elm->set_isvisible(obj->isVisible()); - elm->set_isselectable(obj->isSelectable()); - - elm->set_minvalue(obj->getMinValue()); - elm->set_maxvalue(obj->getMaxValue()); - elm->set_value(obj->getValue()); - elm->set_increment(obj->getIncrement()); + if (founds.size() > 0) { + for (auto& found : founds) { + UiObject *obj = found.get(); + obj->refresh(); + if (mObjMap->getElement(obj->getId()) == nullptr) + mObjMap->addElement(std::move(found)); + + LOGI("found object : %p elementId:%s", obj, obj->getId().c_str()); + + ::aurum::Element *elm = mResponse->add_elements(); + elm->set_elementid(obj->getId()); + elm->set_package(obj->getApplicationPackage()); + + ::aurum::Rect *rect = elm->mutable_geometry(); + const Rect &size = obj->getScreenBoundingBox(); + rect->set_x(size.mTopLeft.x); + rect->set_y(size.mTopLeft.y); + rect->set_width(size.width()); + rect->set_height(size.height()); + + ::aurum::Rect *windowRect = elm->mutable_window_relative_geometry(); + const Rect &windowRelativeSize = obj->getWindowBoundingBox(); + windowRect->set_x(windowRelativeSize.mTopLeft.x); + windowRect->set_y(windowRelativeSize.mTopLeft.y); + windowRect->set_width(windowRelativeSize.width()); + windowRect->set_height(windowRelativeSize.height()); + + elm->set_widget_type(obj->getType()); + elm->set_widget_style(obj->getElementStyle()); + + elm->set_text(obj->getText()); + elm->set_xpath(obj->getXPath()); + elm->set_ocrtext(obj->getOcrText()); + elm->set_automationid(obj->getAutomationId()); + elm->set_package(obj->getApplicationPackage()); + elm->set_role(obj->getRole()); + + elm->set_ischecked(obj->isChecked()); + elm->set_ischeckable(obj->isCheckable()); + elm->set_isclickable(obj->isClickable()); + elm->set_isenabled(obj->isEnabled()); + elm->set_isfocused(obj->isFocused()); + elm->set_isfocusable(obj->isFocusable()); + elm->set_isscrollable(obj->isScrollable()); + elm->set_isselected(obj->isSelected()); + elm->set_isshowing(obj->isShowing()); + elm->set_isactive(obj->isActive()); + elm->set_isvisible(obj->isVisible()); + elm->set_isselectable(obj->isSelectable()); + + elm->set_minvalue(obj->getMinValue()); + elm->set_maxvalue(obj->getMaxValue()); + elm->set_value(obj->getValue()); + elm->set_increment(obj->getIncrement()); + } + mResponse->set_status(::aurum::RspStatus::OK); + } else { + mResponse->set_status(::aurum::RspStatus::ERROR); } - mResponse->set_status(::aurum::RspStatus::OK); - } else { - mResponse->set_status(::aurum::RspStatus::ERROR); } return grpc::Status::OK; diff --git a/org.tizen.aurum-bootstrap/src/Commands/GetActiveAppToolkitNameCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/GetActiveAppToolkitNameCommand.cc new file mode 100644 index 0000000..cb9d869 --- /dev/null +++ b/org.tizen.aurum-bootstrap/src/Commands/GetActiveAppToolkitNameCommand.cc @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "bootstrap.h" +#include "GetActiveAppToolkitNameCommand.h" +#include "UiDevice.h" + +GetActiveAppToolkitNameCommand::GetActiveAppToolkitNameCommand(const ::aurum::ReqGetActiveAppToolkitName *request, + ::aurum::RspGetActiveAppToolkitName *response) + : mRequest{request}, mResponse{response} +{ +} + +::grpc::Status GetActiveAppToolkitNameCommand::execute() +{ + LOGI("GetActiveAppToolkitName --------------- "); + + return grpc::Status::OK; +} diff --git a/org.tizen.aurum-bootstrap/src/Commands/GetSizeCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/GetSizeCommand.cc index dfada9c..534e5a1 100644 --- a/org.tizen.aurum-bootstrap/src/Commands/GetSizeCommand.cc +++ b/org.tizen.aurum-bootstrap/src/Commands/GetSizeCommand.cc @@ -18,6 +18,14 @@ #include "bootstrap.h" #include "GetSizeCommand.h" #include "UiObject.h" +#include "UiDevice.h" +#include "UiSelector.h" +#include "Sel.h" +#include "ISearchable.h" +#ifdef MQTT_ENABLED +#include "SaObject.h" +#endif + GetSizeCommand::GetSizeCommand(const ::aurum::ReqGetSize *request, ::aurum::RspGetSize *response) @@ -29,27 +37,60 @@ GetSizeCommand::GetSizeCommand(const ::aurum::ReqGetSize *request, { LOGI("GetSize --------------- "); - ::aurum::ReqGetSize_CoordType type = mRequest->type(); - ObjectMapper *mObjMap = ObjectMapper::getInstance(); - std::shared_ptr obj = mObjMap->getElement(mRequest->elementid()); - if (obj) { - obj->updateExtents(); - ::aurum::Rect *rect = mResponse->mutable_size(); - if (type == ::aurum::ReqGetSize_CoordType::ReqGetSize_CoordType_SCREEN) { - const Rect &size = obj->getScreenBoundingBox(); - rect->set_x(size.mTopLeft.x); - rect->set_y(size.mTopLeft.y); - rect->set_width(size.width()); - rect->set_height(size.height()); + std::shared_ptr mDevice = UiDevice::getInstance(); + +#ifdef MQTT_ENABLED + if (mDevice->getExternalAppLaunched()) + { + mDevice->RequestScreenAnalyze(); + + std::vector> founds = {}; + + auto tempSel = std::make_shared(); + tempSel->id(mRequest->elementid()); + auto selectors = std::vector>{tempSel}; + + for ( auto &sel : selectors ) { + auto ret = mDevice->getSAWatcher()->findSaObjects(sel); + std::move(std::begin(ret), std::end(ret), std::back_inserter(founds)); + } + if (founds.size() > 0) { + for (auto& found : founds) { + SaObject *obj = found.get(); + ::aurum::Rect *rect = mResponse->mutable_size(); + const Rect &size = obj->getScreenBoundingBox(); + rect->set_x(size.mTopLeft.x); + rect->set_y(size.mTopLeft.y); + rect->set_width(size.width()); + rect->set_height(size.height()); + } } - else { - const Rect &windowRelativeSize = obj->getWindowBoundingBox(); - rect->set_x(windowRelativeSize.mTopLeft.x); - rect->set_y(windowRelativeSize.mTopLeft.y); - rect->set_width(windowRelativeSize.width()); - rect->set_height(windowRelativeSize.height()); + } + else +#endif + { + ::aurum::ReqGetSize_CoordType type = mRequest->type(); + ObjectMapper *mObjMap = ObjectMapper::getInstance(); + std::shared_ptr obj = mObjMap->getElement(mRequest->elementid()); + if (obj) { + obj->updateExtents(); + ::aurum::Rect *rect = mResponse->mutable_size(); + if (type == ::aurum::ReqGetSize_CoordType::ReqGetSize_CoordType_SCREEN) { + const Rect &size = obj->getScreenBoundingBox(); + rect->set_x(size.mTopLeft.x); + rect->set_y(size.mTopLeft.y); + rect->set_width(size.width()); + rect->set_height(size.height()); + } + else { + const Rect &windowRelativeSize = obj->getWindowBoundingBox(); + rect->set_x(windowRelativeSize.mTopLeft.x); + rect->set_y(windowRelativeSize.mTopLeft.y); + rect->set_width(windowRelativeSize.width()); + rect->set_height(windowRelativeSize.height()); + } } } return grpc::Status::OK; -} \ No newline at end of file +} diff --git a/packaging/aurum.spec b/packaging/aurum.spec index ce0c060..0928e3a 100644 --- a/packaging/aurum.spec +++ b/packaging/aurum.spec @@ -31,6 +31,11 @@ BuildRequires: pkgconfig(capi-system-system-settings) BuildRequires: pkgconfig(capi-base-utils-i18n) BuildRequires: pkgconfig(vconf) +%if "%{mqtt}" == "1" +BuildRequires: pkgconfig(libmosquitto) +BuildRequires: pkgconfig(jsoncpp) +%endif + %if 0%{?gendoc:1} BuildRequires: doxygen %endif @@ -82,19 +87,16 @@ Group: Graphics & UI Framework/Testing Ui Automation Library Aurum gcov objects %endif - %prep %setup -q cp %{SOURCE1001} . - %if "%{asan}" == "1" %restore_fcommon %else export LDFLAGS+="-Wl,-z,noexecstack" %endif - %if 0%{?gcov:1} export CFLAGS+=" -fprofile-arcs -ftest-coverage " export CXXFLAGS+=" -fprofile-arcs -ftest-coverage " @@ -110,12 +112,20 @@ export LDFLAGS+=" -lgcov" %else %define TIZEN_GEN_DOC false %endif + +%if "%{mqtt}" == "1" +%define MQTT_ENABLED true +%else +%define MQTT_ENABLED false +%endif + meson \ --prefix /usr \ --libdir %{_libdir} \ -Dcpp_std=c++17 \ -Dtizen=true \ -Denable_documentation=%{TIZEN_GEN_DOC} \ + -Dmqtt_enabled=%{MQTT_ENABLED} \ -Dtizen_gcov=%{TIZEN_GCOV} \ -Dtzapp_path=%{TZ_SYS_RO_APP} \ -Dtzpackage_path=%{TZ_SYS_RO_PACKAGES} \ diff --git a/protocol/aurum.proto b/protocol/aurum.proto index 6b7d454..51600f5 100644 --- a/protocol/aurum.proto +++ b/protocol/aurum.proto @@ -33,6 +33,8 @@ service Bootstrap { rpc actionAndWaitEvent(ReqActionAndWaitEvent) returns (RspActionAndWaitEvent) {} rpc setFocus(ReqSetFocus) returns (RspSetFocus) {} rpc findElements(ReqFindElements) returns (RspFindElements) {} + rpc getActiveAppToolkitName(ReqGetActiveAppToolkitName) returns (RspGetActiveAppToolkitName) {} + rpc enableScreenAnalyzer(ReqEnableScreenAnalyzer) returns (RspEnableScreenAnalyzer) {} } // ------------------------------------ // @@ -62,27 +64,29 @@ message Element { string text = 7; string xpath = 8; - string automationId = 9; - string package = 10; - string role = 11; - - bool isChecked = 12; - bool isCheckable = 13; - bool isClickable = 14; - bool isEnabled = 15; - bool isFocused = 16; - bool isFocusable = 17; - bool isScrollable = 18; - bool isSelected = 19; - bool isShowing = 20; - bool isActive = 21; - bool isVisible = 22; - bool isSelectable = 23; - - double minValue = 24; - double maxValue = 25; - double value = 26; - double increment = 27; + string ocrText = 9; + string automationId = 10; + string package = 11; + string role = 12; + string toolkit = 13; + + bool isChecked = 14; + bool isCheckable = 15; + bool isClickable = 16; + bool isEnabled = 17; + bool isFocused = 18; + bool isFocusable = 19; + bool isScrollable = 20; + bool isSelected = 21; + bool isShowing = 22; + bool isActive = 23; + bool isVisible = 24; + bool isSelectable = 25; + + double minValue = 26; + double maxValue = 27; + double value = 28; + double increment = 29; } message Point { @@ -184,7 +188,11 @@ message ReqFindElement { string xpath = 20; } - repeated ReqFindElement children = 21; + oneof _ocrtext { + string ocrText = 21; + } + + repeated ReqFindElement children = 22; } message RspFindElement { @@ -272,7 +280,10 @@ message ReqFindElements { string xpath = 20; } - repeated ReqFindElements children = 21; + oneof _ocrtext { + string ocrText = 21; + } + repeated ReqFindElements children = 22; } message RspFindElements { @@ -564,3 +575,21 @@ message ReqSetFocus { message RspSetFocus { RspStatus status = 1; } + +message ReqGetActiveAppToolkitName { +} + +message RspGetActiveAppToolkitName { + RspStatus status = 1; + string toolkitName = 2; +} + +message ReqEnableScreenAnalyzer { + bool enable = 1; + string serverIp = 2; +} + +message RspEnableScreenAnalyzer { + RspStatus status = 1; +} + diff --git a/tests/Test_UiObject.cc b/tests/Test_UiObject.cc index 9f20ca8..8b9742f 100644 --- a/tests/Test_UiObject.cc +++ b/tests/Test_UiObject.cc @@ -189,11 +189,11 @@ TEST_F(AurumTestUiObject, getAutomationId) ASSERT_EQ(parent->getAutomationId(), ""); } -TEST_F(AurumTestUiObject, getElementType_P1) +TEST_F(AurumTestUiObject, getType_P1) { auto obj = UiDevice::getInstance(); auto parent = obj->findObject(Sel::text("test2")); - ASSERT_EQ(parent->getElementType(), "type"); + ASSERT_EQ(parent->getType(), "type"); } TEST_F(AurumTestUiObject, getElementStyle_P1) diff --git a/tests/Test_UiSelector.cc b/tests/Test_UiSelector.cc index bdf6861..def5a33 100644 --- a/tests/Test_UiSelector.cc +++ b/tests/Test_UiSelector.cc @@ -185,7 +185,7 @@ TEST_F(AurumTestUiSelector, Selector_Advanced_P1) sel->text("win1"); auto found = UiDevice::getInstance()->findObject(sel); - ASSERT_EQ(found->getElementType(), "Elm_Win"); + ASSERT_EQ(found->getType(), "Elm_Win"); auto children = found->getChildren(); ASSERT_EQ(children.size(), 4); @@ -205,7 +205,7 @@ TEST_F(AurumTestUiSelector, Selector_Advanced_P2) sel->text("win1"); auto found = UiDevice::getInstance()->findObject(sel); - ASSERT_EQ(found->getElementType(), "Elm_Win"); + ASSERT_EQ(found->getType(), "Elm_Win"); auto children = found->getChildren(); ASSERT_EQ(children.size(), 4); @@ -282,7 +282,7 @@ TEST_F(AurumTestUiSelector, Selector_Advanced_N1) sel->text("win1"); auto found = UiDevice::getInstance()->findObject(sel); - ASSERT_EQ(found->getElementType(), "Elm_Win"); + ASSERT_EQ(found->getType(), "Elm_Win"); auto children = found->getChildren(); ASSERT_EQ(children.size(), 4); -- 2.7.4 From 0dc5af6a77547e78ac641101825bf7fe7b47666c Mon Sep 17 00:00:00 2001 From: Chihun Jeong Date: Wed, 24 Aug 2022 16:20:26 +0900 Subject: [PATCH 16/16] Introduce getTextMinBoundingRect command Change-Id: I3a7679b6dddd2126e8cfefdad4428700a5f8f5d5 --- libaurum/inc/Accessibility/AccessibleNode.h | 14 +++++- .../inc/Impl/Accessibility/AtspiAccessibleNode.h | 7 ++- libaurum/inc/Impl/Accessibility/AtspiWrapper.h | 1 + .../inc/Impl/Accessibility/MockAccessibleNode.h | 8 +++- libaurum/inc/UiObject.h | 21 ++++++++- libaurum/src/Accessibility/AccessibleNode.cc | 7 ++- .../src/Impl/Accessibility/AtspiAccessibleNode.cc | 23 +++++++++- libaurum/src/Impl/Accessibility/AtspiWrapper.cc | 7 +++ .../src/Impl/Accessibility/MockAccessibleNode.cc | 4 ++ libaurum/src/UiObject.cc | 12 +++++- org.tizen.aurum-bootstrap/inc/AurumServiceImpl.h | 6 ++- org.tizen.aurum-bootstrap/inc/Commands/Commands.h | 4 +- .../inc/Commands/GetTextMinBoundingRectCommand.h | 34 +++++++++++++++ org.tizen.aurum-bootstrap/meson.build | 1 + org.tizen.aurum-bootstrap/src/AurumServiceImpl.cc | 10 ++++- .../src/Commands/GetTextMinBoundingRectCommand.cc | 50 ++++++++++++++++++++++ protocol/aurum.proto | 10 +++++ 17 files changed, 209 insertions(+), 10 deletions(-) create mode 100644 org.tizen.aurum-bootstrap/inc/Commands/GetTextMinBoundingRectCommand.h create mode 100644 org.tizen.aurum-bootstrap/src/Commands/GetTextMinBoundingRectCommand.cc diff --git a/libaurum/inc/Accessibility/AccessibleNode.h b/libaurum/inc/Accessibility/AccessibleNode.h index 2fa3a41..be58257 100644 --- a/libaurum/inc/Accessibility/AccessibleNode.h +++ b/libaurum/inc/Accessibility/AccessibleNode.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -277,6 +277,12 @@ public: double getIncrement() const; /** + * @copydoc UiObject::getTextMinBoundingRect() + * + */ + Rect getTextMinBoundingRect() const; + + /** * @copydoc UiObject::isCheckable() */ bool isCheckable() const; @@ -406,6 +412,11 @@ public: virtual bool setFocus() = 0; /** + * @copydoc UIObject::updateTextMinBoundingRect() + */ + virtual void updateTextMinBoundingRect() = 0; + + /** * @brief Updates Node information from atspi server. * * @since_tizen 6.5 @@ -516,6 +527,7 @@ protected: std::string mToolkitName; Rect mScreenBoundingBox; Rect mWindowBoundingBox; + Rect mTextMinBoundingRect; int mSupportingIfaces; int mFeatureProperty; int mPid; diff --git a/libaurum/inc/Impl/Accessibility/AtspiAccessibleNode.h b/libaurum/inc/Impl/Accessibility/AtspiAccessibleNode.h index ffd1816..6fa1ce3 100644 --- a/libaurum/inc/Impl/Accessibility/AtspiAccessibleNode.h +++ b/libaurum/inc/Impl/Accessibility/AtspiAccessibleNode.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -137,6 +137,11 @@ public: void updatePid() override; /** + * @copydoc UiObject::updateTextMinBoundingRect() + */ + void updateTextMinBoundingRect() override; + + /** * @copydoc UiObject::setFocus() */ bool setFocus() override; diff --git a/libaurum/inc/Impl/Accessibility/AtspiWrapper.h b/libaurum/inc/Impl/Accessibility/AtspiWrapper.h index d34ceba..0389c30 100644 --- a/libaurum/inc/Impl/Accessibility/AtspiWrapper.h +++ b/libaurum/inc/Impl/Accessibility/AtspiWrapper.h @@ -68,6 +68,7 @@ public: static gdouble Atspi_value_get_minimum_increment(AtspiValue *iface, GError **error); static guint Atspi_accessible_get_process_id(AtspiAccessible *node, GError **error); static gchar *Atspi_accessible_get_toolkit_name(AtspiAccessible *node, GError **error); + static AtspiRect *Atspi_text_get_minimum_bounding_rectangles(AtspiText* obj, gint start_offset, gint end_offset, AtspiCoordType type, GError** error); private: static std::recursive_mutex mMutex; diff --git a/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h b/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h index 2a8afa6..abb23c4 100644 --- a/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h +++ b/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -147,6 +147,12 @@ public: */ bool setFocus() override; + /** + * @brief TBD + * @since_tizen 7.0 + */ + void updateTextMinBoundingRect() override; + /** * @brief TBD * @since_tizen 6.5 diff --git a/libaurum/inc/UiObject.h b/libaurum/inc/UiObject.h index 1863560..bb50d19 100644 --- a/libaurum/inc/UiObject.h +++ b/libaurum/inc/UiObject.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -422,6 +422,16 @@ public: const double getIncrement() const; /** + * @brief Gets text object's minimum bounding rectangle(MBR). + * It works only for NUI Text Object. + * + * @return Rect + * + * @since_tizen 7.0 + */ + const Rect getTextMinBoundingRect() const; + + /** * @brief Sets object's value. * * @param[in] double value @@ -601,6 +611,15 @@ public: * @since_tizen 7.0 */ void updateToolkitName() const; + + /* + * @brief Updates text object's minimum bounding rectangle(MBR). + * It works only for NUI Text Object. + * + * @since_tizen 7.0 + */ + void updateTextMinBoundingRect() const; + /** * @brief Sets focus to object. * diff --git a/libaurum/src/Accessibility/AccessibleNode.cc b/libaurum/src/Accessibility/AccessibleNode.cc index e7bdbd1..92e9cb6 100644 --- a/libaurum/src/Accessibility/AccessibleNode.cc +++ b/libaurum/src/Accessibility/AccessibleNode.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -270,3 +270,8 @@ int AccessibleNode::getPid() const { return mPid; } + +Rect AccessibleNode::getTextMinBoundingRect() const +{ + return mTextMinBoundingRect; +} \ No newline at end of file diff --git a/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc b/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc index 4130cd4..00e5f7f 100644 --- a/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc +++ b/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -272,6 +272,27 @@ void AtspiAccessibleNode::updatePid() mPid = AtspiWrapper::Atspi_accessible_get_process_id(mNode, NULL); } +void AtspiAccessibleNode::updateTextMinBoundingRect() +{ + AtspiWrapper::Atspi_accessible_clear_cache(mNode); + + AtspiText *text = atspi_accessible_get_text_iface(mNode); + if (text) + { + gint cc = atspi_text_get_character_count(text, NULL); + AtspiRect *textMinBoundingRectExtent = AtspiWrapper::Atspi_text_get_minimum_bounding_rectangles(text, 0, cc, ATSPI_COORD_TYPE_WINDOW, NULL); + + if (textMinBoundingRectExtent) { + mTextMinBoundingRect = + Rect{textMinBoundingRectExtent->x, textMinBoundingRectExtent->y, textMinBoundingRectExtent->x + textMinBoundingRectExtent->width, + textMinBoundingRectExtent->y + textMinBoundingRectExtent->height}; + g_free(textMinBoundingRectExtent); + } + + g_object_unref(text); + } +} + bool AtspiAccessibleNode::setFocus() { AtspiComponent *component = AtspiWrapper::Atspi_accessible_get_component_iface(mNode); diff --git a/libaurum/src/Impl/Accessibility/AtspiWrapper.cc b/libaurum/src/Impl/Accessibility/AtspiWrapper.cc index ceef6cf..b923969 100644 --- a/libaurum/src/Impl/Accessibility/AtspiWrapper.cc +++ b/libaurum/src/Impl/Accessibility/AtspiWrapper.cc @@ -218,3 +218,10 @@ gchar *AtspiWrapper::Atspi_accessible_get_toolkit_name(AtspiAccessible *node, GE std::unique_lock lock(mMutex); return atspi_accessible_get_toolkit_name(node, error); } + +AtspiRect *AtspiWrapper::Atspi_text_get_minimum_bounding_rectangles(AtspiText* obj, gint start_offset, gint end_offset, AtspiCoordType type, GError** error) +{ + std::unique_lock lock(mMutex); + return atspi_text_get_range_extents(obj, start_offset, end_offset, type, error); +} + diff --git a/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc b/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc index 1650e10..a397f37 100644 --- a/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc +++ b/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc @@ -130,6 +130,10 @@ bool MockAccessibleNode::setFocus() return false; } +void MockAccessibleNode::updateTextMinBoundingRect() +{ +} + void MockAccessibleNode::refresh(bool updateAll) { } diff --git a/libaurum/src/UiObject.cc b/libaurum/src/UiObject.cc index c975859..577ef65 100644 --- a/libaurum/src/UiObject.cc +++ b/libaurum/src/UiObject.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -219,6 +219,11 @@ const double UiObject::getIncrement() const return getAccessibleNode()->getIncrement(); } +const Rect UiObject::getTextMinBoundingRect() const +{ + return getAccessibleNode()->getTextMinBoundingRect(); +} + bool UiObject::setValue(double value) { return getAccessibleNode()->setValue(value); @@ -370,6 +375,11 @@ void UiObject::updateToolkitName() const mNode->updateToolkitName(); } +void UiObject::updateTextMinBoundingRect() const +{ + mNode->updateTextMinBoundingRect(); +} + bool UiObject::setFocus() const { return mNode->setFocus(); diff --git a/org.tizen.aurum-bootstrap/inc/AurumServiceImpl.h b/org.tizen.aurum-bootstrap/inc/AurumServiceImpl.h index deb020b..5857444 100644 --- a/org.tizen.aurum-bootstrap/inc/AurumServiceImpl.h +++ b/org.tizen.aurum-bootstrap/inc/AurumServiceImpl.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -126,6 +126,10 @@ public: ::grpc::Status enableScreenAnalyzer(::grpc::ServerContext *context, const ::aurum::ReqEnableScreenAnalyzer *request, ::aurum::RspEnableScreenAnalyzer *response) override; + ::grpc::Status getTextMinBoundingRect(::grpc::ServerContext *context, + const ::aurum::ReqGetTextMinBoundingRect *request, + ::aurum::RspGetTextMinBoundingRect *response) override; }; #endif + diff --git a/org.tizen.aurum-bootstrap/inc/Commands/Commands.h b/org.tizen.aurum-bootstrap/inc/Commands/Commands.h index 6445640..650b31b 100644 --- a/org.tizen.aurum-bootstrap/inc/Commands/Commands.h +++ b/org.tizen.aurum-bootstrap/inc/Commands/Commands.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -51,3 +51,5 @@ #include "Commands/SetFocusCommand.h" #include "Commands/GetActiveAppToolkitNameCommand.h" #include "Commands/EnableScreenAnalyzerCommand.h" + +#include "Commands/GetTextMinBoundingRectCommand.h" diff --git a/org.tizen.aurum-bootstrap/inc/Commands/GetTextMinBoundingRectCommand.h b/org.tizen.aurum-bootstrap/inc/Commands/GetTextMinBoundingRectCommand.h new file mode 100644 index 0000000..bc04262 --- /dev/null +++ b/org.tizen.aurum-bootstrap/inc/Commands/GetTextMinBoundingRectCommand.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include +#include "Commands/Command.h" +#include "ObjectMapper.h" +#include +#include "config.h" + +class GetTextMinBoundingRectCommand : public Command { +private: + const ::aurum::ReqGetTextMinBoundingRect *mRequest; + ::aurum::RspGetTextMinBoundingRect *mResponse; + +public: + GetTextMinBoundingRectCommand(const ::aurum::ReqGetTextMinBoundingRect *request, + ::aurum::RspGetTextMinBoundingRect *response); + ::grpc::Status execute() override; +}; \ No newline at end of file diff --git a/org.tizen.aurum-bootstrap/meson.build b/org.tizen.aurum-bootstrap/meson.build index fb99f18..3f20d19 100644 --- a/org.tizen.aurum-bootstrap/meson.build +++ b/org.tizen.aurum-bootstrap/meson.build @@ -47,6 +47,7 @@ bootstrap_svr_src += [ files('src/Commands/SetFocusCommand.cc'), files('src/Commands/GetActiveAppToolkitNameCommand.cc'), files('src/Commands/EnableScreenAnalyzerCommand.cc'), + files('src/Commands/GetTextMinBoundingRectCommand.cc'), ] bootstrap_svr_dep = [ diff --git a/org.tizen.aurum-bootstrap/src/AurumServiceImpl.cc b/org.tizen.aurum-bootstrap/src/AurumServiceImpl.cc index 00abf4e..477ab5e 100644 --- a/org.tizen.aurum-bootstrap/src/AurumServiceImpl.cc +++ b/org.tizen.aurum-bootstrap/src/AurumServiceImpl.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -281,3 +281,11 @@ aurumServiceImpl::~aurumServiceImpl() return execute(cmd.get(), true); } +::grpc::Status aurumServiceImpl::getTextMinBoundingRect(::grpc::ServerContext *context, + const ::aurum::ReqGetTextMinBoundingRect *request, + ::aurum::RspGetTextMinBoundingRect *response) +{ + std::unique_ptr cmd = std::make_unique(request, response); + return execute(cmd.get(), true); +} + diff --git a/org.tizen.aurum-bootstrap/src/Commands/GetTextMinBoundingRectCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/GetTextMinBoundingRectCommand.cc new file mode 100644 index 0000000..c512ae5 --- /dev/null +++ b/org.tizen.aurum-bootstrap/src/Commands/GetTextMinBoundingRectCommand.cc @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "bootstrap.h" +#include "GetTextMinBoundingRectCommand.h" +#include "UiObject.h" + +GetTextMinBoundingRectCommand::GetTextMinBoundingRectCommand(const ::aurum::ReqGetTextMinBoundingRect *request, + ::aurum::RspGetTextMinBoundingRect *response) + : mRequest{request}, mResponse{response} +{ +} + +::grpc::Status GetTextMinBoundingRectCommand::execute() +{ + LOGI("GetTextMinBoundingRect --------------- "); + + ObjectMapper *mObjMap = ObjectMapper::getInstance(); + std::shared_ptr obj = mObjMap->getElement(mRequest->elementid()); + if (obj) { + obj->updateTextMinBoundingRect(); + ::aurum::Rect *rect = mResponse->mutable_size(); + const Rect &size = obj->getTextMinBoundingRect(); + + rect->set_x(size.mTopLeft.x); + rect->set_y(size.mTopLeft.y); + rect->set_width(size.width()); + rect->set_height(size.height()); + + mResponse->set_status(::aurum::RspStatus::OK); + } + else + mResponse->set_status(::aurum::RspStatus::ERROR); + + return grpc::Status::OK; +} \ No newline at end of file diff --git a/protocol/aurum.proto b/protocol/aurum.proto index 51600f5..0e076f9 100644 --- a/protocol/aurum.proto +++ b/protocol/aurum.proto @@ -35,6 +35,7 @@ service Bootstrap { rpc findElements(ReqFindElements) returns (RspFindElements) {} rpc getActiveAppToolkitName(ReqGetActiveAppToolkitName) returns (RspGetActiveAppToolkitName) {} rpc enableScreenAnalyzer(ReqEnableScreenAnalyzer) returns (RspEnableScreenAnalyzer) {} + rpc getTextMinBoundingRect(ReqGetTextMinBoundingRect) returns (RspGetTextMinBoundingRect) {} } // ------------------------------------ // @@ -593,3 +594,12 @@ message RspEnableScreenAnalyzer { RspStatus status = 1; } +message ReqGetTextMinBoundingRect { + string elementId = 1; +} + +message RspGetTextMinBoundingRect { + RspStatus status = 1; + Rect size = 2; +} + -- 2.7.4